tools/ucd/ucd_wbtest.mbt1///|2let sample_unicode_data : String =3 #|0041;LATIN CAPITAL LETTER A;Lu;0;L;;;;;N;;;;0061;4 #|0061;LATIN SMALL LETTER A;Ll;0;L;;;;;N;;;0041;;00415 #|00A0;NO-BREAK SPACE;Zs;0;CS;<noBreak> 0020;;;;N;;;;;6 #|00C0;LATIN CAPITAL LETTER A WITH GRAVE;Lu;0;L;0041 0300;;;;N;;;;00E0;7 #|0300;COMBINING GRAVE ACCENT;Mn;230;NSM;;;;;N;;;;;8 #|0301;COMBINING ACUTE ACCENT;Mn;230;NSM;;;;;N;;;;;9 #|3400;<CJK Ideograph Extension A, First>;Lo;0;L;;;;;N;;;;;10 #|3402;<CJK Ideograph Extension A, Last>;Lo;0;L;;;;;N;;;;;11 #|AC00;<Hangul Syllable, First>;Lo;0;L;;;;;N;;;;;12 #|AC02;<Hangul Syllable, Last>;Lo;0;L;;;;;N;;;;;1314///|15test "parse UnicodeData fields" {16 let data = parse_unicode_data(sample_unicode_data)17 debug_inspect(18 [0x41, 0x300, 0x301, 0x302].map(cp => data.ccc[cp]),19 content="[0, 230, 230, 0]",20 )21 debug_inspect(22 data.decompositions.map(entry => {23 (entry.code_point, entry.code_points, entry.compatibility)24 }),25 content="[(160, [32], true), (192, [65, 768], false)]",26 )27 inspect(data.simple_uppercase.length(), content="1")28 inspect(data.simple_lowercase.length(), content="2")29 inspect(data.simple_titlecase.length(), content="1")30 // Lu, Ll, Mn, Cn, and Lo inside the First/Last ranges.31 debug_inspect(32 [0x41, 0x61, 0x300, 0x302, 0x3401, 0x3403, 0xAC01].map(cp => {33 data.general_categories[cp]34 }),35 content="[0, 1, 5, 29, 4, 29, 4]",36 )37}3839///|40test "parse CompositionExclusions ranges" {41 let input =42 #|# comment43 #|034044 #|0958..0959 # DEVANAGARI LETTER QA..KHHA45 let exclusions = parse_composition_exclusions(input)46 inspect(exclusions.length(), content="3")47 inspect(exclusions[0], content="832")48 inspect(exclusions[1], content="2392")49 inspect(exclusions[2], content="2393")50}5152///|53test "parse SpecialCasing unconditional mappings only" {54 let input =55 #|00DF; 0073 0073; 0053 0073; 0053 0053; # LATIN SMALL LETTER SHARP S56 #|03A3; 03C3; 03A3; 03A3; Final_Sigma; # conditional, skipped57 let special = parse_special_casing(input)58 inspect(special.lower_mapping.length(), content="1")59 inspect(special.title_mapping[0].targets.length(), content="2")60 inspect(special.title_mapping[0].targets[0], content="83")61 inspect(special.title_mapping[0].targets[1], content="115")62 inspect(special.upper_mapping[0].targets.length(), content="2")63 inspect(special.upper_mapping[0].targets[0], content="83")64 inspect(special.upper_mapping[0].targets[1], content="83")65}6667///|68test "parse binary property ranges" {69 let input =70 #|0030..0039 ; XID_Continue # Nd71 #|0041..005A ; XID_Start # Lu72 #|0041..005A ; XID_Continue # Lu73 #|005F ; XID_Continue # Pc74 #|00B7 ; XID_Continue # Other_ID_Continue75 #|2118 ; XID_Start # Other_ID_Start76 #|2118 ; XID_Continue # Other_ID_Start77 #|212E ; ID_Start # ignored78 #|200C..200F ; Default_Ignorable_Code_Point # Cf79 let xid_start = parse_property_ranges(input, "XID_Start")80 debug_inspect(81 xid_start,82 content="[{ start: 65, end: 90 }, { start: 8472, end: 8472 }]",83 )84 inspect(parse_property_ranges(input, "XID_Continue").length(), content="5")85 debug_inspect(86 parse_property_ranges(input, "Default_Ignorable_Code_Point"),87 content="[{ start: 8204, end: 8207 }]",88 )89}9091///|92test "reject missing White_Space" {93 try94 emit_binary_property(95 "0020 ; Dash",96 property="White_Space",97 source="PropList.txt",98 )99 catch {100 Failure::Failure(message) =>101 assert_true(message.contains("Missing White_Space ranges"))102 error => raise error103 } noraise {104 _ => fail("expected PropList without White_Space to fail")105 }106}107108///|109test "reject XID_Start outside XID_Continue" {110 let input =111 #|0041 ; XID_Start112 #|0042 ; XID_Continue113 #|0041..0042 ; Alphabetic114 #|0061 ; Lowercase115 #|0041 ; Uppercase116 try emit_xid(input) catch {117 Failure::Failure(message) =>118 assert_true(119 message.contains("XID_Start must be a subset of XID_Continue"),120 )121 error => raise error122 } noraise {123 _ => fail("expected invalid XID data to fail")124 }125}126127///|128test "build composition data" {129 let data = parse_unicode_data(sample_unicode_data)130 let compositions = build_composition_table(data, [])131 inspect(compositions[0].first, content="65")132 inspect(compositions[0].second, content="768")133 inspect(compositions[0].result, content="192")134}135136///|137test "build full composition exclusions" {138 let input =139 #|0300;COMBINING GRAVE ACCENT;Mn;230;NSM;;;;;N;;;;;140 #|0340;COMBINING GRAVE TONE MARK;Mn;230;NSM;0300;;;;N;;;;;141 #|0958;DEVANAGARI LETTER QA;Lo;0;L;0915 093C;;;;N;;;;;142 let data = parse_unicode_data(input)143 let exclusions = build_full_composition_exclusions(data, [0x0958])144 inspect(exclusions.length(), content="2")145 inspect(exclusions[0], content="832")146 inspect(exclusions[1], content="2392")147}148149///|150test "emit CCC lookup through shared normalization properties" {151 let source = emit_ccc()152 assert_true(153 source.contains(154 "@normalization_data.lookup_normalization_properties(c) & 255",155 ),156 )157}158159///|160test "emit XID table" {161 let source = emit_xid(162 (163 #|0041..005A ; XID_Start164 #|0041..005A ; XID_Continue165 #|005F ; XID_Continue166 #|E0100..E01EF ; XID_Continue167 ),168 )169 inspect(170 source.contains("let xid_start_trie_leaf : Bytes = ["),171 content="true",172 )173 inspect(174 source.contains("pub fn is_xid_continue(c : Char) -> Bool"),175 content="true",176 )177 inspect(source.contains(" return false\n"), content="true")178 inspect(179 source.contains(" return (cp >= 0xE0100 && cp <= 0xE01EF)\n"),180 content="true",181 )182}183184///|185test "binary property trie shares identical blocks" {186 let out = @emitter.Emitter::new()187 write_binary_property_trie(out, property="Test", [{ start: 0x41, end: 0x5A, }])188 let source = out.finish()189 // One leaf block holds A-Z; every other block is the shared empty block.190 inspect(191 source.contains(192 (193 #|let test_trie_mid : Bytes = [194 #| 0,195 #| 1,196 #| 1,197 #| 1,198 #| 1,199 #| 1,200 #| 1,201 #| 1,202 #| 1,203 #| 1,204 #| 1,205 #| 1,206 #| 1,207 #| 1,208 #| 1,209 #| 1,210 #|]211 ),212 ),213 content="true",214 )215}216217///|218test "emit decomposition arrays" {219 let data = parse_unicode_data(sample_unicode_data)220 let source = emit_decomposition(data)221 for222 expected in [223 "let decomp_data_starts : ReadOnlyArray[Int]", "pub fn get_canonical_decomposition(c : Char) -> Array[Char]?",224 "pub fn append_decomposition(",225 // U+00A0 is the first character with a mapping in the sample data.226 " if c.to_int() < 0xA0 {",227 ] {228 guard source.contains(expected) else {229 fail("Generated source is missing: \{expected}")230 }231 }232}233234///|235test "emit decomposition without any mapping" {236 let source = emit_decomposition(parse_unicode_data(""))237 // Every character is below the first one with a mapping.238 inspect(source.contains(" if c.to_int() < 0x110000 {"), content="true")239}240241///|242test "emit case mapping records and special mappings" {243 let data = parse_unicode_data(sample_unicode_data)244 let special = parse_special_casing(245 (246 #|00DF; 00DF; 0053 0073; 0053 0053; # LATIN SMALL LETTER SHARP S247 #|0061; 0061; 0041; 0041; # agrees with the simple mappings248 ),249 )250 let source = emit_case_mapping(data, special)251 for252 expected in [253 "let case_trie_top : Bytes", "let case_trie_mid : Bytes", "let case_trie_leaf : Bytes",254 // Records in order of first use: no mapping, U+0041 (shared with255 // U+00C0), U+0061, and U+00DF with special upper and title mappings.256 "let case_upper_deltas : ReadOnlyArray[Int] = [\n 0,\n 0,\n -32,\n 0,\n]",257 "let case_lower_deltas : ReadOnlyArray[Int] = [\n 0,\n 32,\n 0,\n 0,\n]",258 "let case_title_deltas : ReadOnlyArray[Int] = [\n 0,\n 0,\n -32,\n 0,\n]",259 "let case_special_flags : Bytes = [\n 0,\n 0,\n 0,\n 5,\n]", "let special_upper_cps : ReadOnlyArray[Int] = [\n 0x00DF,\n]",260 "let special_upper_strings : ReadOnlyArray[String] = [\n \"\\u{0053}\\u{0053}\",\n]",261 "let special_lower_cps : ReadOnlyArray[Int] = [\n]", "let special_title_strings : ReadOnlyArray[String] = [\n \"\\u{0053}\\u{0073}\",\n]",262 "let ascii_strings : ReadOnlyArray[String] = [\n \"\\u{0000}\",\n", "pub fn lookup_simple_uppercase(c : Char) -> Char",263 "pub fn full_lowercase(c : Char) -> String", "pub fn lookup_full_titlecase(c : Char) -> Array[Char]?",264 ] {265 guard source.contains(expected) else {266 fail("Generated source is missing: \{expected}")267 }268 }269}270271///|272test "case mappings above the trie are rejected" {273 let data = parse_unicode_data(274 "20000;TEST CAPITAL LETTER;Lu;0;L;;;;;N;;;;20001;\n",275 )276 try emit_case_mapping(data, parse_special_casing("")) catch {277 Failure::Failure(message) =>278 assert_true(message.contains("needs trie coverage"))279 error => raise error280 } noraise {281 _ => fail("expected a case mapping above the trie to be rejected")282 }283}284285///|286test "emit White_Space table" {287 let source = emit_binary_property(288 (289 #|0009..000D ; White_Space290 #|0020 ; White_Space291 #|0600..0605 ; Prepended_Concatenation_Mark292 #|1F1E6..1F1FF ; Regional_Indicator293 ),294 property="White_Space",295 source="PropList.txt",296 )297 inspect(source.contains("/// White_Space lookup data\n"), content="true")298 inspect(source.contains("/// Generated from PropList.txt\n"), content="true")299 inspect(300 source.contains("let white_space_trie_top : Bytes = ["),301 content="true",302 )303 inspect(304 source.contains("pub fn is_white_space(c : Char) -> Bool"),305 content="true",306 )307}308309///|310test "emit East_Asian_Width table" {311 let source = emit_east_asian_width(312 (313 #|# @missing: 0000..10FFFF; N314 #|0041 ; Na315 #|4E00 ; W316 #|E0100 ; A317 #|318 ),319 )320 for321 expected in [322 "/// East_Asian_Width lookup data\n/// Generated from EastAsianWidth.txt\n",323 "let eaw_trie_top : Bytes = [", "let eaw_trie_mid : ReadOnlyArray[Int] = [",324 "let eaw_trie_leaf : Bytes = [",325 // Planes 4 and up: N from U+40000, A at U+E0100, N again from U+E0101.326 " 0x200003,\n 0x700800,\n 0x70080B,\n]", "pub fn lookup_east_asian_width(c : Char) -> Int {",327 ] {328 guard source.contains(expected) else {329 fail("Generated source is missing: \{expected}")330 }331 }332}333334///|335test "unknown East_Asian_Width is rejected" {336 try emit_east_asian_width("0041 ; X\n") catch {337 error =>338 inspect(339 error.to_string().contains("Unknown East_Asian_Width: X"),340 content="true",341 )342 } noraise {343 _ => fail("expected an unknown East_Asian_Width to be rejected")344 }345}346347///|348test "emit Emoji tables" {349 let source = emit_emoji(350 (351 #|231A..231B ; Emoji_Presentation352 #|1F3FB..1F3FF ; Emoji_Modifier353 #|1F44D ; Emoji_Modifier_Base354 #|00A9 ; Extended_Pictographic355 ),356 (357 #|0023 FE0E ; text style; # (1.1) NUMBER SIGN358 #|0023 FE0F ; emoji style; # (1.1) NUMBER SIGN359 ),360 )361 for362 expected in [363 "/// Generated from emoji-data.txt and emoji-variation-sequences.txt\n", "let emoji_presentation_trie_top : Bytes = [",364 "pub fn is_emoji_presentation(c : Char) -> Bool", "let emoji_modifier_trie_leaf : Bytes = [",365 "pub fn is_emoji_modifier(c : Char) -> Bool", "pub fn is_emoji_modifier_base(c : Char) -> Bool",366 "pub fn is_extended_pictographic(c : Char) -> Bool", "/// Check whether a character starts an emoji variation sequence\npub fn has_emoji_variation_sequence(c : Char) -> Bool",367 ] {368 guard source.contains(expected) else {369 fail("Generated source is missing: \{expected}")370 }371 }372}373374///|375test "emit Joining_Type table" {376 let source = emit_joining_type(377 (378 #|# @missing: 0000..10FFFF; Non_Joining379 #|0620 ; D380 #|0627 ; R381 #|200D ; C382 #|E0100..E01EF ; T383 #|384 ),385 )386 for387 expected in [388 "/// Joining_Type lookup data\n/// Generated from DerivedJoiningType.txt\n",389 "let joining_type_trie_top : Bytes = [", "let joining_type_trie_leaf : Bytes = [",390 // Planes 4 and up: U from U+40000, T at U+E0100..U+E01EF, U again after.391 " 0x200000,\n 0x700805,\n 0x700F80,\n]", "pub fn lookup_joining_type(c : Char) -> Int {",392 ] {393 guard source.contains(expected) else {394 fail("Generated source is missing: \{expected}")395 }396 }397}398399///|400test "unknown Joining_Type is rejected" {401 try emit_joining_type("0041 ; X\n") catch {402 error =>403 inspect(404 error.to_string().contains("Unknown Joining_Type: X"),405 content="true",406 )407 } noraise {408 _ => fail("expected an unknown Joining_Type to be rejected")409 }410}411412///|413test "parse emoji variation sequences into ranges" {414 let ranges = parse_emoji_variation_sequences(415 (416 #|0023 FE0E ; text style; # (1.1) NUMBER SIGN417 #|0023 FE0F ; emoji style; # (1.1) NUMBER SIGN418 #|0030 FE0E ; text style; # (1.1) DIGIT ZERO419 #|0030 FE0F ; emoji style; # (1.1) DIGIT ZERO420 #|0031 FE0E ; text style; # (1.1) DIGIT ONE421 #|0031 FE0F ; emoji style; # (1.1) DIGIT ONE422 ),423 )424 debug_inspect(425 ranges,426 content=(427 #|[{ start: 35, end: 35 }, { start: 48, end: 49 }]428 ),429 )430}431432///|433test "emoji variation sequences must come in pairs" {434 try435 parse_emoji_variation_sequences(436 (437 #|0023 FE0E ; text style; # (1.1) NUMBER SIGN438 #|0023 FE0F ; emoji style; # (1.1) NUMBER SIGN439 #|0030 FE0F ; emoji style; # (1.1) DIGIT ZERO440 ),441 )442 catch {443 error =>444 inspect(445 error.to_string().contains("have different base characters"),446 content="true",447 )448 } noraise {449 _ => fail("expected an unpaired variation sequence to be rejected")450 }451 try parse_emoji_variation_sequences("0023 FE0F ; text style;\n") catch {452 error =>453 inspect(454 error.to_string().contains("Unexpected emoji variation sequence"),455 content="true",456 )457 } noraise {458 _ => fail("expected a mismatched selector to be rejected")459 }460}461462///|463test "emit Hangul_Syllable_Type table" {464 let source = emit_hangul_syllable_type(465 (466 #|# @missing: 0000..10FFFF; Not_Applicable467 #|1100..115F ; L468 #|AC00 ; LV469 #|AC01..AC1B ; LVT470 #|471 ),472 )473 for474 expected in [475 "/// Hangul_Syllable_Type lookup data\n/// Generated from HangulSyllableType.txt\n",476 "let hangul_syllable_type_trie_leaf : Bytes = [", "pub fn lookup_hangul_syllable_type(c : Char) -> Int {",477 ] {478 guard source.contains(expected) else {479 fail("Generated source is missing: \{expected}")480 }481 }482 try emit_hangul_syllable_type("1100 ; X\n") catch {483 error =>484 inspect(485 error.to_string().contains("Unknown Hangul_Syllable_Type: X"),486 content="true",487 )488 } noraise {489 _ => fail("expected an unknown Hangul_Syllable_Type to be rejected")490 }491}492493///|494test "emit Joining_Group table and enum in dictionary order" {495 let input =496 #|# @missing: 0000..10FFFF; No_Joining_Group497 #|0629 ; Teh_Marbuta498 #|0628 ; Beh499 #|06C3 ; Teh_Marbuta_Goal500 #|0647 ; Heh501 #|062D ; Hah502 #|503 let source = emit_joining_group(input)504 for505 expected in [506 "/// Joining_Group lookup data\n/// Generated from DerivedJoiningGroup.txt\n",507 "let joining_group_trie_leaf : Bytes = [", "/// Returns ordinal value (0-5) corresponding to JoiningGroup enum\n",508 "pub fn lookup_joining_group(c : Char) -> Int {",509 ] {510 guard source.contains(expected) else {511 fail("Generated source is missing: \{expected}")512 }513 }514 let api = emit_joining_group_enum(input)515 for516 expected in [517 // Dictionary order, not the length-first order of `String::compare`.518 "#derive(Debug, Eq)\npub(all) enum JoiningGroup {\n Beh // Beh\n Hah // Hah\n Heh // Heh\n NoJoiningGroup // No_Joining_Group\n TehMarbuta // Teh_Marbuta\n TehMarbutaGoal // Teh_Marbuta_Goal\n}",519 "#deprecated(\"Use `Debug::to_repr` instead\", skip_current_package=true)\n#doc(hidden)\npub extend JoiningGroup with Debug::{to_repr}\n",520 " TehMarbutaGoal => \"TehMarbutaGoal\"\n", "/// Convert a JoiningGroup ordinal, below 6, to the enum value.\n",521 "fn ordinal_to_joining_group(ordinal : Int) -> JoiningGroup = \"%identity\"\n",522 " ordinal_to_joining_group(@joining_group.lookup_joining_group(c))\n",523 ] {524 guard api.contains(expected) else {525 fail("Generated source is missing: \{expected}")526 }527 }528}529530///|531test "emit Line_Break table and enum" {532 let lines = StringBuilder()533 lines.write_string("# @missing: 0000..10FFFF; XX\n")534 for cp, entry in line_break_values {535 lines.write_string("\{@emitter.hex_upper(0x100 + cp)} ; \{entry.0}\n")536 }537 let source = emit_line_break(lines.to_string())538 for539 expected in [540 "/// Line_Break lookup data\n/// Generated from LineBreak.txt\n", "let line_break_trie_leaf : Bytes = [",541 "/// Returns ordinal value (0-48) corresponding to LineBreak enum\n", "pub fn lookup_line_break(c : Char) -> Int {",542 ] {543 guard source.contains(expected) else {544 fail("Generated source is missing: \{expected}")545 }546 }547 let api = emit_line_break_enum()548 for549 expected in [550 "#derive(Debug, Eq)\npub(all) enum LineBreak {\n AI // Ambiguous (Alphabetic or Ideographic)\n",551 " ZWJ // Zero Width Joiner\n}", "#deprecated(\"Use `Debug::to_repr` instead\", skip_current_package=true)\n#doc(hidden)\npub extend LineBreak with Debug::{to_repr}\n",552 "/// Convert a LineBreak ordinal, below 49, to the enum value.\n", " ordinal_to_line_break(@line_break.lookup_line_break(c))\n",553 ] {554 guard api.contains(expected) else {555 fail("Generated source is missing: \{expected}")556 }557 }558}559560///|561test "Line_Break values must match the data" {562 try emit_line_break("0041 ; QQ\n") catch {563 error =>564 inspect(565 error.to_string().contains("Unknown Line_Break: QQ"),566 content="true",567 )568 } noraise {569 _ => fail("expected an unknown Line_Break to be rejected")570 }571 try emit_line_break("0041 ; AL\n") catch {572 error =>573 inspect(574 error.to_string().contains("does not occur in LineBreak.txt"),575 content="true",576 )577 } noraise {578 _ =>579 fail("expected a Line_Break value missing from the data to be rejected")580 }581}582583///|584test "punctuation and mark tables come from General_Category" {585 let data = parse_unicode_data(586 (587 #|0021;EXCLAMATION MARK;Po;0;ON;;;;;N;;;;;588 #|002C;COMMA;Po;0;CS;;;;;N;;;;;589 #|002D;HYPHEN-MINUS;Pd;0;ES;;;;;N;;;;;590 #|0028;LEFT PARENTHESIS;Ps;0;ON;;;;;N;;;;;591 #|0029;RIGHT PARENTHESIS;Pe;0;ON;;;;;N;;;;;592 #|0030;DIGIT ZERO;Nd;0;EN;;0;0;0;N;;;;;593 #|0041;LATIN CAPITAL LETTER A;Lu;0;L;;;;;N;;;;0061;594 #|0300;COMBINING GRAVE ACCENT;Mn;230;NSM;;;;;N;;;;;595 #|0301;COMBINING ACUTE ACCENT;Mn;230;NSM;;;;;N;;;;;596 ),597 )598 // Adjacent values of different punctuation categories share a range.599 debug_inspect(600 ranges_where(data.general_categories, gc => gc >= 14 && gc <= 20),601 content=(602 #|[603 #| { start: 33, end: 33 },604 #| { start: 40, end: 41 },605 #| { start: 44, end: 45 },606 #|]607 ),608 )609 let source = emit_punctuation(data)610 assert_true(source.contains("pub fn is_punctuation(c : Char) -> Bool"))611 assert_true(source.contains("punctuation_trie_top"))612 let marks = emit_marks(data)613 assert_true(marks.contains("pub fn is_mark(c : Char) -> Bool"))614 // U+0300 and U+0301 are the two lowest bits of one leaf byte, and no mark615 // lies above the trie.616 assert_true(617 marks.contains(618 "let mark_trie_leaf : Bytes = [\n" + " 0,\n".repeat(16) + " 3,\n",619 ),620 )621 assert_true(marks.contains(" return false\n"))622}623
///|
let sample_unicode_data : String =
#|0041;LATIN CAPITAL LETTER A;Lu;0;L;;;;;N;;;;0061;
#|0061;LATIN SMALL LETTER A;Ll;0;L;;;;;N;;;0041;;0041
#|00A0;NO-BREAK SPACE;Zs;0;CS;<noBreak> 0020;;;;N;;;;;
#|00C0;LATIN CAPITAL LETTER A WITH GRAVE;Lu;0;L;0041 0300;;;;N;;;;00E0;
#|0300;COMBINING GRAVE ACCENT;Mn;230;NSM;;;;;N;;;;;
#|0301;COMBINING ACUTE ACCENT;Mn;230;NSM;;;;;N;;;;;
#|3400;<CJK Ideograph Extension A, First>;Lo;0;L;;;;;N;;;;;
#|3402;<CJK Ideograph Extension A, Last>;Lo;0;L;;;;;N;;;;;
#|AC00;<Hangul Syllable, First>;Lo;0;L;;;;;N;;;;;
#|AC02;<Hangul Syllable, Last>;Lo;0;L;;;;;N;;;;;
///|
test "parse UnicodeData fields" {
let data = parse_unicode_data(sample_unicode_data)
debug_inspect(
[0x41, 0x300, 0x301, 0x302].map(cp => data.ccc[cp]),
content="[0, 230, 230, 0]",
)
debug_inspect(
data.decompositions.map(entry => {
(entry.code_point, entry.code_points, entry.compatibility)
}),
content="[(160, [32], true), (192, [65, 768], false)]",
)
inspect(data.simple_uppercase.length(), content="1")
inspect(data.simple_lowercase.length(), content="2")
inspect(data.simple_titlecase.length(), content="1")
// Lu, Ll, Mn, Cn, and Lo inside the First/Last ranges.
debug_inspect(
[0x41, 0x61, 0x300, 0x302, 0x3401, 0x3403, 0xAC01].map(cp => {
data.general_categories[cp]
}),
content="[0, 1, 5, 29, 4, 29, 4]",
)
}
///|
test "parse CompositionExclusions ranges" {
let input =
#|# comment
#|0340
#|0958..0959 # DEVANAGARI LETTER QA..KHHA
let exclusions = parse_composition_exclusions(input)
inspect(exclusions.length(), content="3")
inspect(exclusions[0], content="832")
inspect(exclusions[1], content="2392")
inspect(exclusions[2], content="2393")
}
///|
test "parse SpecialCasing unconditional mappings only" {
let input =
#|00DF; 0073 0073; 0053 0073; 0053 0053; # LATIN SMALL LETTER SHARP S
#|03A3; 03C3; 03A3; 03A3; Final_Sigma; # conditional, skipped
let special = parse_special_casing(input)
inspect(special.lower_mapping.length(), content="1")
inspect(special.title_mapping[0].targets.length(), content="2")
inspect(special.title_mapping[0].targets[0], content="83")
inspect(special.title_mapping[0].targets[1], content="115")
inspect(special.upper_mapping[0].targets.length(), content="2")
inspect(special.upper_mapping[0].targets[0], content="83")
inspect(special.upper_mapping[0].targets[1], content="83")
}
///|
test "parse binary property ranges" {
let input =
#|0030..0039 ; XID_Continue # Nd
#|0041..005A ; XID_Start # Lu
#|0041..005A ; XID_Continue # Lu
#|005F ; XID_Continue # Pc
#|00B7 ; XID_Continue # Other_ID_Continue
#|2118 ; XID_Start # Other_ID_Start
#|2118 ; XID_Continue # Other_ID_Start
#|212E ; ID_Start # ignored
#|200C..200F ; Default_Ignorable_Code_Point # Cf
let xid_start = parse_property_ranges(input, "XID_Start")
debug_inspect(
xid_start,
content="[{ start: 65, end: 90 }, { start: 8472, end: 8472 }]",
)
inspect(parse_property_ranges(input, "XID_Continue").length(), content="5")
debug_inspect(
parse_property_ranges(input, "Default_Ignorable_Code_Point"),
content="[{ start: 8204, end: 8207 }]",
)
}
///|
test "reject missing White_Space" {
try
emit_binary_property(
"0020 ; Dash",
property="White_Space",
source="PropList.txt",
)
catch {
Failure::Failure(message) =>
assert_true(message.contains("Missing White_Space ranges"))
error => raise error
} noraise {
_ => fail("expected PropList without White_Space to fail")
}
}
///|
test "reject XID_Start outside XID_Continue" {
let input =
#|0041 ; XID_Start
#|0042 ; XID_Continue
#|0041..0042 ; Alphabetic
#|0061 ; Lowercase
#|0041 ; Uppercase
try emit_xid(input) catch {
Failure::Failure(message) =>
assert_true(
message.contains("XID_Start must be a subset of XID_Continue"),
)
error => raise error
} noraise {
_ => fail("expected invalid XID data to fail")
}
}
///|
test "build composition data" {
let data = parse_unicode_data(sample_unicode_data)
let compositions = build_composition_table(data, [])
inspect(compositions[0].first, content="65")
inspect(compositions[0].second, content="768")
inspect(compositions[0].result, content="192")
}
///|
test "build full composition exclusions" {
let input =
#|0300;COMBINING GRAVE ACCENT;Mn;230;NSM;;;;;N;;;;;
#|0340;COMBINING GRAVE TONE MARK;Mn;230;NSM;0300;;;;N;;;;;
#|0958;DEVANAGARI LETTER QA;Lo;0;L;0915 093C;;;;N;;;;;
let data = parse_unicode_data(input)
let exclusions = build_full_composition_exclusions(data, [0x0958])
inspect(exclusions.length(), content="2")
inspect(exclusions[0], content="832")
inspect(exclusions[1], content="2392")
}
///|
test "emit CCC lookup through shared normalization properties" {
let source = emit_ccc()
assert_true(
source.contains(
"@normalization_data.lookup_normalization_properties(c) & 255",
),
)
}
///|
test "emit XID table" {
let source = emit_xid(
(
#|0041..005A ; XID_Start
#|0041..005A ; XID_Continue
#|005F ; XID_Continue
#|E0100..E01EF ; XID_Continue
),
)
inspect(
source.contains("let xid_start_trie_leaf : Bytes = ["),
content="true",
)
inspect(
source.contains("pub fn is_xid_continue(c : Char) -> Bool"),
content="true",
)
inspect(source.contains(" return false\n"), content="true")
inspect(
source.contains(" return (cp >= 0xE0100 && cp <= 0xE01EF)\n"),
content="true",
)
}
///|
test "binary property trie shares identical blocks" {
let out = @emitter.Emitter::new()
write_binary_property_trie(out, property="Test", [{ start: 0x41, end: 0x5A, }])
let source = out.finish()
// One leaf block holds A-Z; every other block is the shared empty block.
inspect(
source.contains(
(
#|let test_trie_mid : Bytes = [
#| 0,
#| 1,
#| 1,
#| 1,
#| 1,
#| 1,
#| 1,
#| 1,
#| 1,
#| 1,
#| 1,
#| 1,
#| 1,
#| 1,
#| 1,
#| 1,
#|]
),
),
content="true",
)
}
///|
test "emit decomposition arrays" {
let data = parse_unicode_data(sample_unicode_data)
let source = emit_decomposition(data)
for
expected in [
"let decomp_data_starts : ReadOnlyArray[Int]", "pub fn get_canonical_decomposition(c : Char) -> Array[Char]?",
"pub fn append_decomposition(",
// U+00A0 is the first character with a mapping in the sample data.
" if c.to_int() < 0xA0 {",
] {
guard source.contains(expected) else {
fail("Generated source is missing: \{expected}")
}
}
}
///|
test "emit decomposition without any mapping" {
let source = emit_decomposition(parse_unicode_data(""))
// Every character is below the first one with a mapping.
inspect(source.contains(" if c.to_int() < 0x110000 {"), content="true")
}
///|
test "emit case mapping records and special mappings" {
let data = parse_unicode_data(sample_unicode_data)
let special = parse_special_casing(
(
#|00DF; 00DF; 0053 0073; 0053 0053; # LATIN SMALL LETTER SHARP S
#|0061; 0061; 0041; 0041; # agrees with the simple mappings
),
)
let source = emit_case_mapping(data, special)
for
expected in [
"let case_trie_top : Bytes", "let case_trie_mid : Bytes", "let case_trie_leaf : Bytes",
// Records in order of first use: no mapping, U+0041 (shared with
// U+00C0), U+0061, and U+00DF with special upper and title mappings.
"let case_upper_deltas : ReadOnlyArray[Int] = [\n 0,\n 0,\n -32,\n 0,\n]",
"let case_lower_deltas : ReadOnlyArray[Int] = [\n 0,\n 32,\n 0,\n 0,\n]",
"let case_title_deltas : ReadOnlyArray[Int] = [\n 0,\n 0,\n -32,\n 0,\n]",
"let case_special_flags : Bytes = [\n 0,\n 0,\n 0,\n 5,\n]", "let special_upper_cps : ReadOnlyArray[Int] = [\n 0x00DF,\n]",
"let special_upper_strings : ReadOnlyArray[String] = [\n \"\\u{0053}\\u{0053}\",\n]",
"let special_lower_cps : ReadOnlyArray[Int] = [\n]", "let special_title_strings : ReadOnlyArray[String] = [\n \"\\u{0053}\\u{0073}\",\n]",
"let ascii_strings : ReadOnlyArray[String] = [\n \"\\u{0000}\",\n", "pub fn lookup_simple_uppercase(c : Char) -> Char",
"pub fn full_lowercase(c : Char) -> String", "pub fn lookup_full_titlecase(c : Char) -> Array[Char]?",
] {
guard source.contains(expected) else {
fail("Generated source is missing: \{expected}")
}
}
}
///|
test "case mappings above the trie are rejected" {
let data = parse_unicode_data(
"20000;TEST CAPITAL LETTER;Lu;0;L;;;;;N;;;;20001;\n",
)
try emit_case_mapping(data, parse_special_casing("")) catch {
Failure::Failure(message) =>
assert_true(message.contains("needs trie coverage"))
error => raise error
} noraise {
_ => fail("expected a case mapping above the trie to be rejected")
}
}
///|
test "emit White_Space table" {
let source = emit_binary_property(
(
#|0009..000D ; White_Space
#|0020 ; White_Space
#|0600..0605 ; Prepended_Concatenation_Mark
#|1F1E6..1F1FF ; Regional_Indicator
),
property="White_Space",
source="PropList.txt",
)
inspect(source.contains("/// White_Space lookup data\n"), content="true")
inspect(source.contains("/// Generated from PropList.txt\n"), content="true")
inspect(
source.contains("let white_space_trie_top : Bytes = ["),
content="true",
)
inspect(
source.contains("pub fn is_white_space(c : Char) -> Bool"),
content="true",
)
}
///|
test "emit East_Asian_Width table" {
let source = emit_east_asian_width(
(
#|# @missing: 0000..10FFFF; N
#|0041 ; Na
#|4E00 ; W
#|E0100 ; A
#|
),
)
for
expected in [
"/// East_Asian_Width lookup data\n/// Generated from EastAsianWidth.txt\n",
"let eaw_trie_top : Bytes = [", "let eaw_trie_mid : ReadOnlyArray[Int] = [",
"let eaw_trie_leaf : Bytes = [",
// Planes 4 and up: N from U+40000, A at U+E0100, N again from U+E0101.
" 0x200003,\n 0x700800,\n 0x70080B,\n]", "pub fn lookup_east_asian_width(c : Char) -> Int {",
] {
guard source.contains(expected) else {
fail("Generated source is missing: \{expected}")
}
}
}
///|
test "unknown East_Asian_Width is rejected" {
try emit_east_asian_width("0041 ; X\n") catch {
error =>
inspect(
error.to_string().contains("Unknown East_Asian_Width: X"),
content="true",
)
} noraise {
_ => fail("expected an unknown East_Asian_Width to be rejected")
}
}
///|
test "emit Emoji tables" {
let source = emit_emoji(
(
#|231A..231B ; Emoji_Presentation
#|1F3FB..1F3FF ; Emoji_Modifier
#|1F44D ; Emoji_Modifier_Base
#|00A9 ; Extended_Pictographic
),
(
#|0023 FE0E ; text style; # (1.1) NUMBER SIGN
#|0023 FE0F ; emoji style; # (1.1) NUMBER SIGN
),
)
for
expected in [
"/// Generated from emoji-data.txt and emoji-variation-sequences.txt\n", "let emoji_presentation_trie_top : Bytes = [",
"pub fn is_emoji_presentation(c : Char) -> Bool", "let emoji_modifier_trie_leaf : Bytes = [",
"pub fn is_emoji_modifier(c : Char) -> Bool", "pub fn is_emoji_modifier_base(c : Char) -> Bool",
"pub fn is_extended_pictographic(c : Char) -> Bool", "/// Check whether a character starts an emoji variation sequence\npub fn has_emoji_variation_sequence(c : Char) -> Bool",
] {
guard source.contains(expected) else {
fail("Generated source is missing: \{expected}")
}
}
}
///|
test "emit Joining_Type table" {
let source = emit_joining_type(
(
#|# @missing: 0000..10FFFF; Non_Joining
#|0620 ; D
#|0627 ; R
#|200D ; C
#|E0100..E01EF ; T
#|
),
)
for
expected in [
"/// Joining_Type lookup data\n/// Generated from DerivedJoiningType.txt\n",
"let joining_type_trie_top : Bytes = [", "let joining_type_trie_leaf : Bytes = [",
// Planes 4 and up: U from U+40000, T at U+E0100..U+E01EF, U again after.
" 0x200000,\n 0x700805,\n 0x700F80,\n]", "pub fn lookup_joining_type(c : Char) -> Int {",
] {
guard source.contains(expected) else {
fail("Generated source is missing: \{expected}")
}
}
}
///|
test "unknown Joining_Type is rejected" {
try emit_joining_type("0041 ; X\n") catch {
error =>
inspect(
error.to_string().contains("Unknown Joining_Type: X"),
content="true",
)
} noraise {
_ => fail("expected an unknown Joining_Type to be rejected")
}
}
///|
test "parse emoji variation sequences into ranges" {
let ranges = parse_emoji_variation_sequences(
(
#|0023 FE0E ; text style; # (1.1) NUMBER SIGN
#|0023 FE0F ; emoji style; # (1.1) NUMBER SIGN
#|0030 FE0E ; text style; # (1.1) DIGIT ZERO
#|0030 FE0F ; emoji style; # (1.1) DIGIT ZERO
#|0031 FE0E ; text style; # (1.1) DIGIT ONE
#|0031 FE0F ; emoji style; # (1.1) DIGIT ONE
),
)
debug_inspect(
ranges,
content=(
#|[{ start: 35, end: 35 }, { start: 48, end: 49 }]
),
)
}
///|
test "emoji variation sequences must come in pairs" {
try
parse_emoji_variation_sequences(
(
#|0023 FE0E ; text style; # (1.1) NUMBER SIGN
#|0023 FE0F ; emoji style; # (1.1) NUMBER SIGN
#|0030 FE0F ; emoji style; # (1.1) DIGIT ZERO
),
)
catch {
error =>
inspect(
error.to_string().contains("have different base characters"),
content="true",
)
} noraise {
_ => fail("expected an unpaired variation sequence to be rejected")
}
try parse_emoji_variation_sequences("0023 FE0F ; text style;\n") catch {
error =>
inspect(
error.to_string().contains("Unexpected emoji variation sequence"),
content="true",
)
} noraise {
_ => fail("expected a mismatched selector to be rejected")
}
}
///|
test "emit Hangul_Syllable_Type table" {
let source = emit_hangul_syllable_type(
(
#|# @missing: 0000..10FFFF; Not_Applicable
#|1100..115F ; L
#|AC00 ; LV
#|AC01..AC1B ; LVT
#|
),
)
for
expected in [
"/// Hangul_Syllable_Type lookup data\n/// Generated from HangulSyllableType.txt\n",
"let hangul_syllable_type_trie_leaf : Bytes = [", "pub fn lookup_hangul_syllable_type(c : Char) -> Int {",
] {
guard source.contains(expected) else {
fail("Generated source is missing: \{expected}")
}
}
try emit_hangul_syllable_type("1100 ; X\n") catch {
error =>
inspect(
error.to_string().contains("Unknown Hangul_Syllable_Type: X"),
content="true",
)
} noraise {
_ => fail("expected an unknown Hangul_Syllable_Type to be rejected")
}
}
///|
test "emit Joining_Group table and enum in dictionary order" {
let input =
#|# @missing: 0000..10FFFF; No_Joining_Group
#|0629 ; Teh_Marbuta
#|0628 ; Beh
#|06C3 ; Teh_Marbuta_Goal
#|0647 ; Heh
#|062D ; Hah
#|
let source = emit_joining_group(input)
for
expected in [
"/// Joining_Group lookup data\n/// Generated from DerivedJoiningGroup.txt\n",
"let joining_group_trie_leaf : Bytes = [", "/// Returns ordinal value (0-5) corresponding to JoiningGroup enum\n",
"pub fn lookup_joining_group(c : Char) -> Int {",
] {
guard source.contains(expected) else {
fail("Generated source is missing: \{expected}")
}
}
let api = emit_joining_group_enum(input)
for
expected in [
// Dictionary order, not the length-first order of `String::compare`.
"#derive(Debug, Eq)\npub(all) enum JoiningGroup {\n Beh // Beh\n Hah // Hah\n Heh // Heh\n NoJoiningGroup // No_Joining_Group\n TehMarbuta // Teh_Marbuta\n TehMarbutaGoal // Teh_Marbuta_Goal\n}",
"#deprecated(\"Use `Debug::to_repr` instead\", skip_current_package=true)\n#doc(hidden)\npub extend JoiningGroup with Debug::{to_repr}\n",
" TehMarbutaGoal => \"TehMarbutaGoal\"\n", "/// Convert a JoiningGroup ordinal, below 6, to the enum value.\n",
"fn ordinal_to_joining_group(ordinal : Int) -> JoiningGroup = \"%identity\"\n",
" ordinal_to_joining_group(@joining_group.lookup_joining_group(c))\n",
] {
guard api.contains(expected) else {
fail("Generated source is missing: \{expected}")
}
}
}
///|
test "emit Line_Break table and enum" {
let lines = StringBuilder()
lines.write_string("# @missing: 0000..10FFFF; XX\n")
for cp, entry in line_break_values {
lines.write_string("\{@emitter.hex_upper(0x100 + cp)} ; \{entry.0}\n")
}
let source = emit_line_break(lines.to_string())
for
expected in [
"/// Line_Break lookup data\n/// Generated from LineBreak.txt\n", "let line_break_trie_leaf : Bytes = [",
"/// Returns ordinal value (0-48) corresponding to LineBreak enum\n", "pub fn lookup_line_break(c : Char) -> Int {",
] {
guard source.contains(expected) else {
fail("Generated source is missing: \{expected}")
}
}
let api = emit_line_break_enum()
for
expected in [
"#derive(Debug, Eq)\npub(all) enum LineBreak {\n AI // Ambiguous (Alphabetic or Ideographic)\n",
" ZWJ // Zero Width Joiner\n}", "#deprecated(\"Use `Debug::to_repr` instead\", skip_current_package=true)\n#doc(hidden)\npub extend LineBreak with Debug::{to_repr}\n",
"/// Convert a LineBreak ordinal, below 49, to the enum value.\n", " ordinal_to_line_break(@line_break.lookup_line_break(c))\n",
] {
guard api.contains(expected) else {
fail("Generated source is missing: \{expected}")
}
}
}
///|
test "Line_Break values must match the data" {
try emit_line_break("0041 ; QQ\n") catch {
error =>
inspect(
error.to_string().contains("Unknown Line_Break: QQ"),
content="true",
)
} noraise {
_ => fail("expected an unknown Line_Break to be rejected")
}
try emit_line_break("0041 ; AL\n") catch {
error =>
inspect(
error.to_string().contains("does not occur in LineBreak.txt"),
content="true",
)
} noraise {
_ =>
fail("expected a Line_Break value missing from the data to be rejected")
}
}
///|
test "punctuation and mark tables come from General_Category" {
let data = parse_unicode_data(
(
#|0021;EXCLAMATION MARK;Po;0;ON;;;;;N;;;;;
#|002C;COMMA;Po;0;CS;;;;;N;;;;;
#|002D;HYPHEN-MINUS;Pd;0;ES;;;;;N;;;;;
#|0028;LEFT PARENTHESIS;Ps;0;ON;;;;;N;;;;;
#|0029;RIGHT PARENTHESIS;Pe;0;ON;;;;;N;;;;;
#|0030;DIGIT ZERO;Nd;0;EN;;0;0;0;N;;;;;
#|0041;LATIN CAPITAL LETTER A;Lu;0;L;;;;;N;;;;0061;
#|0300;COMBINING GRAVE ACCENT;Mn;230;NSM;;;;;N;;;;;
#|0301;COMBINING ACUTE ACCENT;Mn;230;NSM;;;;;N;;;;;
),
)
// Adjacent values of different punctuation categories share a range.
debug_inspect(
ranges_where(data.general_categories, gc => gc >= 14 && gc <= 20),
content=(
#|[
#| { start: 33, end: 33 },
#| { start: 40, end: 41 },
#| { start: 44, end: 45 },
#|]
),
)
let source = emit_punctuation(data)
assert_true(source.contains("pub fn is_punctuation(c : Char) -> Bool"))
assert_true(source.contains("punctuation_trie_top"))
let marks = emit_marks(data)
assert_true(marks.contains("pub fn is_mark(c : Char) -> Bool"))
// U+0300 and U+0301 are the two lowest bits of one leaf byte, and no mark
// lies above the trie.
assert_true(
marks.contains(
"let mark_trie_leaf : Bytes = [\n" + " 0,\n".repeat(16) + " 3,\n",
),
)
assert_true(marks.contains(" return false\n"))
}