unicode

Branch: main

Switch branches or tags

Branch: main
Go to file t

413 files indexed

History
Code

tools/ucd/ucd_wbtest.mbt
1///|2let sample_unicode_data : String =3  #|0041;LATIN CAPITAL LETTER A;Lu;0;L;;;;;N;;;;0061;4  #|0061;LATIN SMALL LETTER A;Ll;0;L;;;;;N;;;0041;;00415  #|00A0;NO-BREAK SPACE;Zs;0;CS;<noBreak> 0020;;;;N;;;;;6  #|00C0;LATIN CAPITAL LETTER A WITH GRAVE;Lu;0;L;0041 0300;;;;N;;;;00E0;7  #|0300;COMBINING GRAVE ACCENT;Mn;230;NSM;;;;;N;;;;;8  #|0301;COMBINING ACUTE ACCENT;Mn;230;NSM;;;;;N;;;;;9  #|3400;<CJK Ideograph Extension A, First>;Lo;0;L;;;;;N;;;;;10  #|3402;<CJK Ideograph Extension A, Last>;Lo;0;L;;;;;N;;;;;11  #|AC00;<Hangul Syllable, First>;Lo;0;L;;;;;N;;;;;12  #|AC02;<Hangul Syllable, Last>;Lo;0;L;;;;;N;;;;;1314///|15test "parse UnicodeData fields" {16  let data = parse_unicode_data(sample_unicode_data)17  debug_inspect(18    [0x41, 0x300, 0x301, 0x302].map(cp => data.ccc[cp]),19    content="[0, 230, 230, 0]",20  )21  debug_inspect(22    data.decompositions.map(entry => {23      (entry.code_point, entry.code_points, entry.compatibility)24    }),25    content="[(160, [32], true), (192, [65, 768], false)]",26  )27  inspect(data.simple_uppercase.length(), content="1")28  inspect(data.simple_lowercase.length(), content="2")29  inspect(data.simple_titlecase.length(), content="1")30  // Lu, Ll, Mn, Cn, and Lo inside the First/Last ranges.31  debug_inspect(32    [0x41, 0x61, 0x300, 0x302, 0x3401, 0x3403, 0xAC01].map(cp => {33      data.general_categories[cp]34    }),35    content="[0, 1, 5, 29, 4, 29, 4]",36  )37}3839///|40test "parse CompositionExclusions ranges" {41  let input =42    #|# comment43    #|034044    #|0958..0959 # DEVANAGARI LETTER QA..KHHA45  let exclusions = parse_composition_exclusions(input)46  inspect(exclusions.length(), content="3")47  inspect(exclusions[0], content="832")48  inspect(exclusions[1], content="2392")49  inspect(exclusions[2], content="2393")50}5152///|53test "parse SpecialCasing unconditional mappings only" {54  let input =55    #|00DF; 0073 0073; 0053 0073; 0053 0053; # LATIN SMALL LETTER SHARP S56    #|03A3; 03C3; 03A3; 03A3; Final_Sigma; # conditional, skipped57  let special = parse_special_casing(input)58  inspect(special.lower_mapping.length(), content="1")59  inspect(special.title_mapping[0].targets.length(), content="2")60  inspect(special.title_mapping[0].targets[0], content="83")61  inspect(special.title_mapping[0].targets[1], content="115")62  inspect(special.upper_mapping[0].targets.length(), content="2")63  inspect(special.upper_mapping[0].targets[0], content="83")64  inspect(special.upper_mapping[0].targets[1], content="83")65}6667///|68test "parse binary property ranges" {69  let input =70    #|0030..0039 ; XID_Continue # Nd71    #|0041..005A ; XID_Start # Lu72    #|0041..005A ; XID_Continue # Lu73    #|005F ; XID_Continue # Pc74    #|00B7 ; XID_Continue # Other_ID_Continue75    #|2118 ; XID_Start # Other_ID_Start76    #|2118 ; XID_Continue # Other_ID_Start77    #|212E ; ID_Start # ignored78    #|200C..200F ; Default_Ignorable_Code_Point # Cf79  let xid_start = parse_property_ranges(input, "XID_Start")80  debug_inspect(81    xid_start,82    content="[{ start: 65, end: 90 }, { start: 8472, end: 8472 }]",83  )84  inspect(parse_property_ranges(input, "XID_Continue").length(), content="5")85  debug_inspect(86    parse_property_ranges(input, "Default_Ignorable_Code_Point"),87    content="[{ start: 8204, end: 8207 }]",88  )89}9091///|92test "reject missing White_Space" {93  try94    emit_binary_property(95      "0020 ; Dash",96      property="White_Space",97      source="PropList.txt",98    )99  catch {100    Failure::Failure(message) =>101      assert_true(message.contains("Missing White_Space ranges"))102    error => raise error103  } noraise {104    _ => fail("expected PropList without White_Space to fail")105  }106}107108///|109test "reject XID_Start outside XID_Continue" {110  let input =111    #|0041 ; XID_Start112    #|0042 ; XID_Continue113    #|0041..0042 ; Alphabetic114    #|0061 ; Lowercase115    #|0041 ; Uppercase116  try emit_xid(input) catch {117    Failure::Failure(message) =>118      assert_true(119        message.contains("XID_Start must be a subset of XID_Continue"),120      )121    error => raise error122  } noraise {123    _ => fail("expected invalid XID data to fail")124  }125}126127///|128test "build composition data" {129  let data = parse_unicode_data(sample_unicode_data)130  let compositions = build_composition_table(data, [])131  inspect(compositions[0].first, content="65")132  inspect(compositions[0].second, content="768")133  inspect(compositions[0].result, content="192")134}135136///|137test "build full composition exclusions" {138  let input =139    #|0300;COMBINING GRAVE ACCENT;Mn;230;NSM;;;;;N;;;;;140    #|0340;COMBINING GRAVE TONE MARK;Mn;230;NSM;0300;;;;N;;;;;141    #|0958;DEVANAGARI LETTER QA;Lo;0;L;0915 093C;;;;N;;;;;142  let data = parse_unicode_data(input)143  let exclusions = build_full_composition_exclusions(data, [0x0958])144  inspect(exclusions.length(), content="2")145  inspect(exclusions[0], content="832")146  inspect(exclusions[1], content="2392")147}148149///|150test "emit CCC lookup through shared normalization properties" {151  let source = emit_ccc()152  assert_true(153    source.contains(154      "@normalization_data.lookup_normalization_properties(c) & 255",155    ),156  )157}158159///|160test "emit XID table" {161  let source = emit_xid(162    (163      #|0041..005A ; XID_Start164      #|0041..005A ; XID_Continue165      #|005F ; XID_Continue166      #|E0100..E01EF ; XID_Continue167    ),168  )169  inspect(170    source.contains("let xid_start_trie_leaf : Bytes = ["),171    content="true",172  )173  inspect(174    source.contains("pub fn is_xid_continue(c : Char) -> Bool"),175    content="true",176  )177  inspect(source.contains("    return false\n"), content="true")178  inspect(179    source.contains("    return (cp >= 0xE0100 && cp <= 0xE01EF)\n"),180    content="true",181  )182}183184///|185test "binary property trie shares identical blocks" {186  let out = @emitter.Emitter::new()187  write_binary_property_trie(out, property="Test", [{ start: 0x41, end: 0x5A, }])188  let source = out.finish()189  // One leaf block holds A-Z; every other block is the shared empty block.190  inspect(191    source.contains(192      (193        #|let test_trie_mid : Bytes = [194        #|  0,195        #|  1,196        #|  1,197        #|  1,198        #|  1,199        #|  1,200        #|  1,201        #|  1,202        #|  1,203        #|  1,204        #|  1,205        #|  1,206        #|  1,207        #|  1,208        #|  1,209        #|  1,210        #|]211      ),212    ),213    content="true",214  )215}216217///|218test "emit decomposition arrays" {219  let data = parse_unicode_data(sample_unicode_data)220  let source = emit_decomposition(data)221  for222    expected in [223      "let decomp_data_starts : ReadOnlyArray[Int]", "pub fn get_canonical_decomposition(c : Char) -> Array[Char]?",224      "pub fn append_decomposition(",225      // U+00A0 is the first character with a mapping in the sample data.226       "  if c.to_int() < 0xA0 {",227    ] {228    guard source.contains(expected) else {229      fail("Generated source is missing: \{expected}")230    }231  }232}233234///|235test "emit decomposition without any mapping" {236  let source = emit_decomposition(parse_unicode_data(""))237  // Every character is below the first one with a mapping.238  inspect(source.contains("  if c.to_int() < 0x110000 {"), content="true")239}240241///|242test "emit case mapping records and special mappings" {243  let data = parse_unicode_data(sample_unicode_data)244  let special = parse_special_casing(245    (246      #|00DF; 00DF; 0053 0073; 0053 0053; # LATIN SMALL LETTER SHARP S247      #|0061; 0061; 0041; 0041; # agrees with the simple mappings248    ),249  )250  let source = emit_case_mapping(data, special)251  for252    expected in [253      "let case_trie_top : Bytes", "let case_trie_mid : Bytes", "let case_trie_leaf : Bytes",254      // Records in order of first use: no mapping, U+0041 (shared with255      // U+00C0), U+0061, and U+00DF with special upper and title mappings.256       "let case_upper_deltas : ReadOnlyArray[Int] = [\n  0,\n  0,\n  -32,\n  0,\n]",257      "let case_lower_deltas : ReadOnlyArray[Int] = [\n  0,\n  32,\n  0,\n  0,\n]",258      "let case_title_deltas : ReadOnlyArray[Int] = [\n  0,\n  0,\n  -32,\n  0,\n]",259      "let case_special_flags : Bytes = [\n  0,\n  0,\n  0,\n  5,\n]", "let special_upper_cps : ReadOnlyArray[Int] = [\n  0x00DF,\n]",260      "let special_upper_strings : ReadOnlyArray[String] = [\n  \"\\u{0053}\\u{0053}\",\n]",261      "let special_lower_cps : ReadOnlyArray[Int] = [\n]", "let special_title_strings : ReadOnlyArray[String] = [\n  \"\\u{0053}\\u{0073}\",\n]",262      "let ascii_strings : ReadOnlyArray[String] = [\n  \"\\u{0000}\",\n", "pub fn lookup_simple_uppercase(c : Char) -> Char",263      "pub fn full_lowercase(c : Char) -> String", "pub fn lookup_full_titlecase(c : Char) -> Array[Char]?",264    ] {265    guard source.contains(expected) else {266      fail("Generated source is missing: \{expected}")267    }268  }269}270271///|272test "case mappings above the trie are rejected" {273  let data = parse_unicode_data(274    "20000;TEST CAPITAL LETTER;Lu;0;L;;;;;N;;;;20001;\n",275  )276  try emit_case_mapping(data, parse_special_casing("")) catch {277    Failure::Failure(message) =>278      assert_true(message.contains("needs trie coverage"))279    error => raise error280  } noraise {281    _ => fail("expected a case mapping above the trie to be rejected")282  }283}284285///|286test "emit White_Space table" {287  let source = emit_binary_property(288    (289      #|0009..000D ; White_Space290      #|0020 ; White_Space291      #|0600..0605 ; Prepended_Concatenation_Mark292      #|1F1E6..1F1FF ; Regional_Indicator293    ),294    property="White_Space",295    source="PropList.txt",296  )297  inspect(source.contains("/// White_Space lookup data\n"), content="true")298  inspect(source.contains("/// Generated from PropList.txt\n"), content="true")299  inspect(300    source.contains("let white_space_trie_top : Bytes = ["),301    content="true",302  )303  inspect(304    source.contains("pub fn is_white_space(c : Char) -> Bool"),305    content="true",306  )307}308309///|310test "emit East_Asian_Width table" {311  let source = emit_east_asian_width(312    (313      #|# @missing: 0000..10FFFF; N314      #|0041 ; Na315      #|4E00 ; W316      #|E0100 ; A317      #|318    ),319  )320  for321    expected in [322      "/// East_Asian_Width lookup data\n/// Generated from EastAsianWidth.txt\n",323      "let eaw_trie_top : Bytes = [", "let eaw_trie_mid : ReadOnlyArray[Int] = [",324      "let eaw_trie_leaf : Bytes = [",325      // Planes 4 and up: N from U+40000, A at U+E0100, N again from U+E0101.326       "  0x200003,\n  0x700800,\n  0x70080B,\n]", "pub fn lookup_east_asian_width(c : Char) -> Int {",327    ] {328    guard source.contains(expected) else {329      fail("Generated source is missing: \{expected}")330    }331  }332}333334///|335test "unknown East_Asian_Width is rejected" {336  try emit_east_asian_width("0041 ; X\n") catch {337    error =>338      inspect(339        error.to_string().contains("Unknown East_Asian_Width: X"),340        content="true",341      )342  } noraise {343    _ => fail("expected an unknown East_Asian_Width to be rejected")344  }345}346347///|348test "emit Emoji tables" {349  let source = emit_emoji(350    (351      #|231A..231B ; Emoji_Presentation352      #|1F3FB..1F3FF ; Emoji_Modifier353      #|1F44D ; Emoji_Modifier_Base354      #|00A9 ; Extended_Pictographic355    ),356    (357      #|0023 FE0E  ; text style;  # (1.1) NUMBER SIGN358      #|0023 FE0F  ; emoji style; # (1.1) NUMBER SIGN359    ),360  )361  for362    expected in [363      "/// Generated from emoji-data.txt and emoji-variation-sequences.txt\n", "let emoji_presentation_trie_top : Bytes = [",364      "pub fn is_emoji_presentation(c : Char) -> Bool", "let emoji_modifier_trie_leaf : Bytes = [",365      "pub fn is_emoji_modifier(c : Char) -> Bool", "pub fn is_emoji_modifier_base(c : Char) -> Bool",366      "pub fn is_extended_pictographic(c : Char) -> Bool", "/// Check whether a character starts an emoji variation sequence\npub fn has_emoji_variation_sequence(c : Char) -> Bool",367    ] {368    guard source.contains(expected) else {369      fail("Generated source is missing: \{expected}")370    }371  }372}373374///|375test "emit Joining_Type table" {376  let source = emit_joining_type(377    (378      #|# @missing: 0000..10FFFF; Non_Joining379      #|0620 ; D380      #|0627 ; R381      #|200D ; C382      #|E0100..E01EF ; T383      #|384    ),385  )386  for387    expected in [388      "/// Joining_Type lookup data\n/// Generated from DerivedJoiningType.txt\n",389      "let joining_type_trie_top : Bytes = [", "let joining_type_trie_leaf : Bytes = [",390      // Planes 4 and up: U from U+40000, T at U+E0100..U+E01EF, U again after.391       "  0x200000,\n  0x700805,\n  0x700F80,\n]", "pub fn lookup_joining_type(c : Char) -> Int {",392    ] {393    guard source.contains(expected) else {394      fail("Generated source is missing: \{expected}")395    }396  }397}398399///|400test "unknown Joining_Type is rejected" {401  try emit_joining_type("0041 ; X\n") catch {402    error =>403      inspect(404        error.to_string().contains("Unknown Joining_Type: X"),405        content="true",406      )407  } noraise {408    _ => fail("expected an unknown Joining_Type to be rejected")409  }410}411412///|413test "parse emoji variation sequences into ranges" {414  let ranges = parse_emoji_variation_sequences(415    (416      #|0023 FE0E  ; text style;  # (1.1) NUMBER SIGN417      #|0023 FE0F  ; emoji style; # (1.1) NUMBER SIGN418      #|0030 FE0E  ; text style;  # (1.1) DIGIT ZERO419      #|0030 FE0F  ; emoji style; # (1.1) DIGIT ZERO420      #|0031 FE0E  ; text style;  # (1.1) DIGIT ONE421      #|0031 FE0F  ; emoji style; # (1.1) DIGIT ONE422    ),423  )424  debug_inspect(425    ranges,426    content=(427      #|[{ start: 35, end: 35 }, { start: 48, end: 49 }]428    ),429  )430}431432///|433test "emoji variation sequences must come in pairs" {434  try435    parse_emoji_variation_sequences(436      (437        #|0023 FE0E  ; text style;  # (1.1) NUMBER SIGN438        #|0023 FE0F  ; emoji style; # (1.1) NUMBER SIGN439        #|0030 FE0F  ; emoji style; # (1.1) DIGIT ZERO440      ),441    )442  catch {443    error =>444      inspect(445        error.to_string().contains("have different base characters"),446        content="true",447      )448  } noraise {449    _ => fail("expected an unpaired variation sequence to be rejected")450  }451  try parse_emoji_variation_sequences("0023 FE0F ; text style;\n") catch {452    error =>453      inspect(454        error.to_string().contains("Unexpected emoji variation sequence"),455        content="true",456      )457  } noraise {458    _ => fail("expected a mismatched selector to be rejected")459  }460}461462///|463test "emit Hangul_Syllable_Type table" {464  let source = emit_hangul_syllable_type(465    (466      #|# @missing: 0000..10FFFF; Not_Applicable467      #|1100..115F ; L468      #|AC00 ; LV469      #|AC01..AC1B ; LVT470      #|471    ),472  )473  for474    expected in [475      "/// Hangul_Syllable_Type lookup data\n/// Generated from HangulSyllableType.txt\n",476      "let hangul_syllable_type_trie_leaf : Bytes = [", "pub fn lookup_hangul_syllable_type(c : Char) -> Int {",477    ] {478    guard source.contains(expected) else {479      fail("Generated source is missing: \{expected}")480    }481  }482  try emit_hangul_syllable_type("1100 ; X\n") catch {483    error =>484      inspect(485        error.to_string().contains("Unknown Hangul_Syllable_Type: X"),486        content="true",487      )488  } noraise {489    _ => fail("expected an unknown Hangul_Syllable_Type to be rejected")490  }491}492493///|494test "emit Joining_Group table and enum in dictionary order" {495  let input =496    #|# @missing: 0000..10FFFF; No_Joining_Group497    #|0629 ; Teh_Marbuta498    #|0628 ; Beh499    #|06C3 ; Teh_Marbuta_Goal500    #|0647 ; Heh501    #|062D ; Hah502    #|503  let source = emit_joining_group(input)504  for505    expected in [506      "/// Joining_Group lookup data\n/// Generated from DerivedJoiningGroup.txt\n",507      "let joining_group_trie_leaf : Bytes = [", "/// Returns ordinal value (0-5) corresponding to JoiningGroup enum\n",508      "pub fn lookup_joining_group(c : Char) -> Int {",509    ] {510    guard source.contains(expected) else {511      fail("Generated source is missing: \{expected}")512    }513  }514  let api = emit_joining_group_enum(input)515  for516    expected in [517      // Dictionary order, not the length-first order of `String::compare`.518      "#derive(Debug, Eq)\npub(all) enum JoiningGroup {\n  Beh // Beh\n  Hah // Hah\n  Heh // Heh\n  NoJoiningGroup // No_Joining_Group\n  TehMarbuta // Teh_Marbuta\n  TehMarbutaGoal // Teh_Marbuta_Goal\n}",519      "#deprecated(\"Use `Debug::to_repr` instead\", skip_current_package=true)\n#doc(hidden)\npub extend JoiningGroup with Debug::{to_repr}\n",520      "      TehMarbutaGoal => \"TehMarbutaGoal\"\n", "/// Convert a JoiningGroup ordinal, below 6, to the enum value.\n",521      "fn ordinal_to_joining_group(ordinal : Int) -> JoiningGroup = \"%identity\"\n",522      "  ordinal_to_joining_group(@joining_group.lookup_joining_group(c))\n",523    ] {524    guard api.contains(expected) else {525      fail("Generated source is missing: \{expected}")526    }527  }528}529530///|531test "emit Line_Break table and enum" {532  let lines = StringBuilder()533  lines.write_string("# @missing: 0000..10FFFF; XX\n")534  for cp, entry in line_break_values {535    lines.write_string("\{@emitter.hex_upper(0x100 + cp)} ; \{entry.0}\n")536  }537  let source = emit_line_break(lines.to_string())538  for539    expected in [540      "/// Line_Break lookup data\n/// Generated from LineBreak.txt\n", "let line_break_trie_leaf : Bytes = [",541      "/// Returns ordinal value (0-48) corresponding to LineBreak enum\n", "pub fn lookup_line_break(c : Char) -> Int {",542    ] {543    guard source.contains(expected) else {544      fail("Generated source is missing: \{expected}")545    }546  }547  let api = emit_line_break_enum()548  for549    expected in [550      "#derive(Debug, Eq)\npub(all) enum LineBreak {\n  AI // Ambiguous (Alphabetic or Ideographic)\n",551      "  ZWJ // Zero Width Joiner\n}", "#deprecated(\"Use `Debug::to_repr` instead\", skip_current_package=true)\n#doc(hidden)\npub extend LineBreak with Debug::{to_repr}\n",552      "/// Convert a LineBreak ordinal, below 49, to the enum value.\n", "  ordinal_to_line_break(@line_break.lookup_line_break(c))\n",553    ] {554    guard api.contains(expected) else {555      fail("Generated source is missing: \{expected}")556    }557  }558}559560///|561test "Line_Break values must match the data" {562  try emit_line_break("0041 ; QQ\n") catch {563    error =>564      inspect(565        error.to_string().contains("Unknown Line_Break: QQ"),566        content="true",567      )568  } noraise {569    _ => fail("expected an unknown Line_Break to be rejected")570  }571  try emit_line_break("0041 ; AL\n") catch {572    error =>573      inspect(574        error.to_string().contains("does not occur in LineBreak.txt"),575        content="true",576      )577  } noraise {578    _ =>579      fail("expected a Line_Break value missing from the data to be rejected")580  }581}582583///|584test "punctuation and mark tables come from General_Category" {585  let data = parse_unicode_data(586    (587      #|0021;EXCLAMATION MARK;Po;0;ON;;;;;N;;;;;588      #|002C;COMMA;Po;0;CS;;;;;N;;;;;589      #|002D;HYPHEN-MINUS;Pd;0;ES;;;;;N;;;;;590      #|0028;LEFT PARENTHESIS;Ps;0;ON;;;;;N;;;;;591      #|0029;RIGHT PARENTHESIS;Pe;0;ON;;;;;N;;;;;592      #|0030;DIGIT ZERO;Nd;0;EN;;0;0;0;N;;;;;593      #|0041;LATIN CAPITAL LETTER A;Lu;0;L;;;;;N;;;;0061;594      #|0300;COMBINING GRAVE ACCENT;Mn;230;NSM;;;;;N;;;;;595      #|0301;COMBINING ACUTE ACCENT;Mn;230;NSM;;;;;N;;;;;596    ),597  )598  // Adjacent values of different punctuation categories share a range.599  debug_inspect(600    ranges_where(data.general_categories, gc => gc >= 14 && gc <= 20),601    content=(602      #|[603      #|  { start: 33, end: 33 },604      #|  { start: 40, end: 41 },605      #|  { start: 44, end: 45 },606      #|]607    ),608  )609  let source = emit_punctuation(data)610  assert_true(source.contains("pub fn is_punctuation(c : Char) -> Bool"))611  assert_true(source.contains("punctuation_trie_top"))612  let marks = emit_marks(data)613  assert_true(marks.contains("pub fn is_mark(c : Char) -> Bool"))614  // U+0300 and U+0301 are the two lowest bits of one leaf byte, and no mark615  // lies above the trie.616  assert_true(617    marks.contains(618      "let mark_trie_leaf : Bytes = [\n" + "  0,\n".repeat(16) + "  3,\n",619    ),620  )621  assert_true(marks.contains("    return false\n"))622}623