ucd/ccc/normalization_test.mbt1///|2test "packed normalization properties keep all four Quick_Check values" {3 // Bits 8..11 hold NFC/NFKC (Yes=0, No=1, Maybe=2), then NFD/NFKD.4 inspect(@ccc.lookup_normalization_properties('A'), content="0")5 inspect(@ccc.lookup_normalization_properties('é'), content="12288")6 inspect(@ccc.lookup_normalization_properties('ffi'), content="9216")7 inspect(@ccc.lookup_normalization_properties('Å'), content="13568")8 inspect(@ccc.lookup_normalization_properties('\u{0301}'), content="2790")9 inspect(@ccc.lookup_normalization_properties('\u{0323}'), content="2780")10}1112///|13test "CCC lookup masks out Quick_Check flags" {14 inspect(@ccc.lookup_ccc('é'), content="0")15 inspect(@ccc.lookup_ccc('\u{0301}'), content="230")16 inspect(@ccc.lookup_ccc('\u{0323}'), content="220")17 inspect(@ccc.lookup_ccc('\u{1D165}'), content="216")18}1920///|21test "normalization trie covers supplementary characters and default tail" {22 inspect(@ccc.lookup_normalization_properties('\u{1D15E}'), content="13568")23 inspect(@ccc.lookup_normalization_properties('\u{2FA1D}'), content="13568")24 inspect(@ccc.lookup_normalization_properties('\u{2FA1E}'), content="0")25 inspect(@ccc.lookup_normalization_properties('\u{2FA7F}'), content="0")26 inspect(@ccc.lookup_normalization_properties('\u{2FA80}'), content="0")27 inspect(@ccc.lookup_normalization_properties('\u{10FFFF}'), content="0")28}2930///|31test "Hangul Quick_Check is independent of decomposition table mappings" {32 // UnicodeData represents syllables as a range with no explicit mappings.33 for cp in 0xAC00..<=0xD7A3 {34 assert_true(35 @ccc.lookup_normalization_properties(cp.unsafe_to_char()) == 0x3000,36 )37 }38 // A vowel/trailing jamo may compose with preceding Hangul; a leading jamo cannot.39 inspect(@ccc.lookup_normalization_properties('\u{1100}'), content="0")40 inspect(@ccc.lookup_normalization_properties('\u{1161}'), content="2560")41 inspect(@ccc.lookup_normalization_properties('\u{11A8}'), content="2560")42}4344///|45test "norm16 distinguishes direct properties from composition-only records" {46 for c in ['!', 'A', '\u{0301}', '\u{10FFFF}'] {47 assert_true(@decomposition.get_canonical_decomposition(c) == None)48 assert_true(@decomposition.get_compat_decomposition(c) == None)49 }50 assert_true(@composition.lookup_composition('A', '\u{030A}') == Some('Å'))51 assert_true(@composition.lookup_composition('!', '\u{030A}') == None)52}5354///|55test "norm16 supports decomposition and composition on the same character" {56 assert_true(57 @decomposition.get_canonical_decomposition('Å') == Some(['A', '\u{030A}']),58 )59 assert_true(@decomposition.get_compat_decomposition('Å') == None)60 assert_true(@composition.lookup_composition('Å', '\u{0301}') == Some('Ǻ'))61 inspect(@ccc.lookup_normalization_properties('Å'), content="12288")62 assert_true(@decomposition.get_canonical_decomposition('ffi') == None)63 assert_true(64 @decomposition.get_compat_decomposition('ffi') == Some(['f', 'f', 'i']),65 )66}6768///|69test "composition searches stay within each starter group" {70 // A's first and last candidates, and values outside its candidate range.71 assert_true(@composition.lookup_composition('A', '\u{0300}') == Some('À'))72 assert_true(@composition.lookup_composition('A', '\u{0328}') == Some('Ą'))73 assert_true(@composition.lookup_composition('A', '\u{0000}') == None)74 assert_true(@composition.lookup_composition('A', '\u{10FFFF}') == None)75 assert_true(@composition.lookup_composition('B', '\u{0300}') == None)76 // Devanagari QA is excluded even though its decomposition has two characters.77 assert_true(@composition.lookup_composition('\u{0915}', '\u{093C}') == None)78 assert_true(79 @composition.lookup_composition('\u{11099}', '\u{110BA}') ==80 Some('\u{1109A}'),81 )82}8384///|85test "normalization BMP blocks and plane boundaries" {86 for87 (cp, expected) in [88 (0x001F, 0),89 (0x0020, 0),90 (0x00BF, 0),91 (0x00C0, 12288),92 (0x031F, 220),93 (0x0320, 220),94 (0xD7FF, 0),95 (0xE000, 0),96 (0xFFFF, 0),97 (0x10000, 0),98 ] {99 assert_eq(100 @ccc.lookup_normalization_properties(cp.unsafe_to_char()),101 expected,102 )103 }104}105
///|
test "packed normalization properties keep all four Quick_Check values" {
// Bits 8..11 hold NFC/NFKC (Yes=0, No=1, Maybe=2), then NFD/NFKD.
inspect(@ccc.lookup_normalization_properties('A'), content="0")
inspect(@ccc.lookup_normalization_properties('é'), content="12288")
inspect(@ccc.lookup_normalization_properties('ffi'), content="9216")
inspect(@ccc.lookup_normalization_properties('Å'), content="13568")
inspect(@ccc.lookup_normalization_properties('\u{0301}'), content="2790")
inspect(@ccc.lookup_normalization_properties('\u{0323}'), content="2780")
}
///|
test "CCC lookup masks out Quick_Check flags" {
inspect(@ccc.lookup_ccc('é'), content="0")
inspect(@ccc.lookup_ccc('\u{0301}'), content="230")
inspect(@ccc.lookup_ccc('\u{0323}'), content="220")
inspect(@ccc.lookup_ccc('\u{1D165}'), content="216")
}
///|
test "normalization trie covers supplementary characters and default tail" {
inspect(@ccc.lookup_normalization_properties('\u{1D15E}'), content="13568")
inspect(@ccc.lookup_normalization_properties('\u{2FA1D}'), content="13568")
inspect(@ccc.lookup_normalization_properties('\u{2FA1E}'), content="0")
inspect(@ccc.lookup_normalization_properties('\u{2FA7F}'), content="0")
inspect(@ccc.lookup_normalization_properties('\u{2FA80}'), content="0")
inspect(@ccc.lookup_normalization_properties('\u{10FFFF}'), content="0")
}
///|
test "Hangul Quick_Check is independent of decomposition table mappings" {
// UnicodeData represents syllables as a range with no explicit mappings.
for cp in 0xAC00..<=0xD7A3 {
assert_true(
@ccc.lookup_normalization_properties(cp.unsafe_to_char()) == 0x3000,
)
}
// A vowel/trailing jamo may compose with preceding Hangul; a leading jamo cannot.
inspect(@ccc.lookup_normalization_properties('\u{1100}'), content="0")
inspect(@ccc.lookup_normalization_properties('\u{1161}'), content="2560")
inspect(@ccc.lookup_normalization_properties('\u{11A8}'), content="2560")
}
///|
test "norm16 distinguishes direct properties from composition-only records" {
for c in ['!', 'A', '\u{0301}', '\u{10FFFF}'] {
assert_true(@decomposition.get_canonical_decomposition(c) == None)
assert_true(@decomposition.get_compat_decomposition(c) == None)
}
assert_true(@composition.lookup_composition('A', '\u{030A}') == Some('Å'))
assert_true(@composition.lookup_composition('!', '\u{030A}') == None)
}
///|
test "norm16 supports decomposition and composition on the same character" {
assert_true(
@decomposition.get_canonical_decomposition('Å') == Some(['A', '\u{030A}']),
)
assert_true(@decomposition.get_compat_decomposition('Å') == None)
assert_true(@composition.lookup_composition('Å', '\u{0301}') == Some('Ǻ'))
inspect(@ccc.lookup_normalization_properties('Å'), content="12288")
assert_true(@decomposition.get_canonical_decomposition('ffi') == None)
assert_true(
@decomposition.get_compat_decomposition('ffi') == Some(['f', 'f', 'i']),
)
}
///|
test "composition searches stay within each starter group" {
// A's first and last candidates, and values outside its candidate range.
assert_true(@composition.lookup_composition('A', '\u{0300}') == Some('À'))
assert_true(@composition.lookup_composition('A', '\u{0328}') == Some('Ą'))
assert_true(@composition.lookup_composition('A', '\u{0000}') == None)
assert_true(@composition.lookup_composition('A', '\u{10FFFF}') == None)
assert_true(@composition.lookup_composition('B', '\u{0300}') == None)
// Devanagari QA is excluded even though its decomposition has two characters.
assert_true(@composition.lookup_composition('\u{0915}', '\u{093C}') == None)
assert_true(
@composition.lookup_composition('\u{11099}', '\u{110BA}') ==
Some('\u{1109A}'),
)
}
///|
test "normalization BMP blocks and plane boundaries" {
for
(cp, expected) in [
(0x001F, 0),
(0x0020, 0),
(0x00BF, 0),
(0x00C0, 12288),
(0x031F, 220),
(0x0320, 220),
(0xD7FF, 0),
(0xE000, 0),
(0xFFFF, 0),
(0x10000, 0),
] {
assert_eq(
@ccc.lookup_normalization_properties(cp.unsafe_to_char()),
expected,
)
}
}