unicode

Branch: main

Switch branches or tags

Branch: main
Go to file t

413 files indexed

History
Code

ucd/ccc/normalization_test.mbt
1///|2test "packed normalization properties keep all four Quick_Check values" {3  // Bits 8..11 hold NFC/NFKC (Yes=0, No=1, Maybe=2), then NFD/NFKD.4  inspect(@ccc.lookup_normalization_properties('A'), content="0")5  inspect(@ccc.lookup_normalization_properties('é'), content="12288")6  inspect(@ccc.lookup_normalization_properties('ffi'), content="9216")7  inspect(@ccc.lookup_normalization_properties('Å'), content="13568")8  inspect(@ccc.lookup_normalization_properties('\u{0301}'), content="2790")9  inspect(@ccc.lookup_normalization_properties('\u{0323}'), content="2780")10}1112///|13test "CCC lookup masks out Quick_Check flags" {14  inspect(@ccc.lookup_ccc('é'), content="0")15  inspect(@ccc.lookup_ccc('\u{0301}'), content="230")16  inspect(@ccc.lookup_ccc('\u{0323}'), content="220")17  inspect(@ccc.lookup_ccc('\u{1D165}'), content="216")18}1920///|21test "normalization trie covers supplementary characters and default tail" {22  inspect(@ccc.lookup_normalization_properties('\u{1D15E}'), content="13568")23  inspect(@ccc.lookup_normalization_properties('\u{2FA1D}'), content="13568")24  inspect(@ccc.lookup_normalization_properties('\u{2FA1E}'), content="0")25  inspect(@ccc.lookup_normalization_properties('\u{2FA7F}'), content="0")26  inspect(@ccc.lookup_normalization_properties('\u{2FA80}'), content="0")27  inspect(@ccc.lookup_normalization_properties('\u{10FFFF}'), content="0")28}2930///|31test "Hangul Quick_Check is independent of decomposition table mappings" {32  // UnicodeData represents syllables as a range with no explicit mappings.33  for cp in 0xAC00..<=0xD7A3 {34    assert_true(35      @ccc.lookup_normalization_properties(cp.unsafe_to_char()) == 0x3000,36    )37  }38  // A vowel/trailing jamo may compose with preceding Hangul; a leading jamo cannot.39  inspect(@ccc.lookup_normalization_properties('\u{1100}'), content="0")40  inspect(@ccc.lookup_normalization_properties('\u{1161}'), content="2560")41  inspect(@ccc.lookup_normalization_properties('\u{11A8}'), content="2560")42}4344///|45test "norm16 distinguishes direct properties from composition-only records" {46  for c in ['!', 'A', '\u{0301}', '\u{10FFFF}'] {47    assert_true(@decomposition.get_canonical_decomposition(c) == None)48    assert_true(@decomposition.get_compat_decomposition(c) == None)49  }50  assert_true(@composition.lookup_composition('A', '\u{030A}') == Some('Å'))51  assert_true(@composition.lookup_composition('!', '\u{030A}') == None)52}5354///|55test "norm16 supports decomposition and composition on the same character" {56  assert_true(57    @decomposition.get_canonical_decomposition('Å') == Some(['A', '\u{030A}']),58  )59  assert_true(@decomposition.get_compat_decomposition('Å') == None)60  assert_true(@composition.lookup_composition('Å', '\u{0301}') == Some('Ǻ'))61  inspect(@ccc.lookup_normalization_properties('Å'), content="12288")62  assert_true(@decomposition.get_canonical_decomposition('ffi') == None)63  assert_true(64    @decomposition.get_compat_decomposition('ffi') == Some(['f', 'f', 'i']),65  )66}6768///|69test "composition searches stay within each starter group" {70  // A's first and last candidates, and values outside its candidate range.71  assert_true(@composition.lookup_composition('A', '\u{0300}') == Some('À'))72  assert_true(@composition.lookup_composition('A', '\u{0328}') == Some('Ą'))73  assert_true(@composition.lookup_composition('A', '\u{0000}') == None)74  assert_true(@composition.lookup_composition('A', '\u{10FFFF}') == None)75  assert_true(@composition.lookup_composition('B', '\u{0300}') == None)76  // Devanagari QA is excluded even though its decomposition has two characters.77  assert_true(@composition.lookup_composition('\u{0915}', '\u{093C}') == None)78  assert_true(79    @composition.lookup_composition('\u{11099}', '\u{110BA}') ==80    Some('\u{1109A}'),81  )82}8384///|85test "normalization BMP blocks and plane boundaries" {86  for87    (cp, expected) in [88      (0x001F, 0),89      (0x0020, 0),90      (0x00BF, 0),91      (0x00C0, 12288),92      (0x031F, 220),93      (0x0320, 220),94      (0xD7FF, 0),95      (0xE000, 0),96      (0xFFFF, 0),97      (0x10000, 0),98    ] {99    assert_eq(100      @ccc.lookup_normalization_properties(cp.unsafe_to_char()),101      expected,102    )103  }104}105