unicode

Branch: main

Switch branches or tags

Branch: main
Go to file t

413 files indexed

History
Code

normalization/normalization_test.mbt
1///|2/// Tests for Unicode Normalization34// ==================== NFD Tests ====================56///|7test "nfd: ASCII unchanged" {8  // ASCII text should remain unchanged9  inspect(@normalization.nfd("Hello, World!"), content="Hello, World!")10  inspect(@normalization.nfd("abc123"), content="abc123")11}1213///|14test "nfd: already decomposed" {15  // Already decomposed text should remain unchanged16  let decomposed = "A\u{030A}" // A + combining ring above17  inspect(@normalization.nfd(decomposed), content="A\u{030A}")18}1920///|21test "nfd: precomposed to decomposed" {22  // Å (U+00C5) should decompose to A (U+0041) + combining ring above (U+030A)23  inspect(@normalization.nfd("\u{00C5}"), content="A\u{030A}")24  // é (U+00E9) should decompose to e (U+0065) + combining acute (U+0301)25  inspect(@normalization.nfd("\u{00E9}"), content="e\u{0301}")26  // ñ (U+00F1) should decompose to n (U+006E) + combining tilde (U+0303)27  inspect(@normalization.nfd("\u{00F1}"), content="n\u{0303}")28}2930///|31test "nfd: multi-level decomposition" {32  // Some characters decompose in multiple steps33  // Ṩ (U+1E68) -> S + dot below + dot above34  let result = @normalization.nfd("\u{1E68}")35  // Should be S (0x53) + combining dot below (0x323) + combining dot above (0x307)36  inspect(result, content="S\u{0323}\u{0307}")37}3839///|40test "nfd: canonical ordering" {41  // Combining marks should be reordered by CCC42  // Input: a + combining dot above (CCC=230) + combining dot below (CCC=220)43  // Output: a + combining dot below (CCC=220) + combining dot above (CCC=230)44  let input = "a\u{0307}\u{0323}"45  inspect(@normalization.nfd(input), content="a\u{0323}\u{0307}")46}4748///|49test "nfd: hangul decomposition" {50  // 가 (U+AC00) -> ᄀ (U+1100) + ᅡ (U+1161)51  inspect(@normalization.nfd("\u{AC00}"), content="\u{1100}\u{1161}")52  // 각 (U+AC01) -> ᄀ (U+1100) + ᅡ (U+1161) + ᆨ (U+11A8)53  inspect(@normalization.nfd("\u{AC01}"), content="\u{1100}\u{1161}\u{11A8}")54  // 한 (U+D55C) -> ᄒ (U+1112) + ᅡ (U+1161) + ᆫ (U+11AB)55  inspect(@normalization.nfd("\u{D55C}"), content="\u{1112}\u{1161}\u{11AB}")56}5758// ==================== NFC Tests ====================5960///|61test "nfc: ASCII unchanged" {62  inspect(@normalization.nfc("Hello, World!"), content="Hello, World!")63}6465///|66test "nfc: already composed" {67  // Already composed text should remain unchanged68  inspect(@normalization.nfc("\u{00C5}"), content="\u{00C5}") // Å69  inspect(@normalization.nfc("\u{00E9}"), content="\u{00E9}") // é70}7172///|73test "nfc: decomposed to composed" {74  // A + combining ring above -> Å75  inspect(@normalization.nfc("A\u{030A}"), content="\u{00C5}")76  // e + combining acute -> é77  inspect(@normalization.nfc("e\u{0301}"), content="\u{00E9}")78  // n + combining tilde -> ñ79  inspect(@normalization.nfc("n\u{0303}"), content="\u{00F1}")80}8182///|83test "nfc: hangul composition" {84  // ᄀ + ᅡ -> 가85  inspect(@normalization.nfc("\u{1100}\u{1161}"), content="\u{AC00}")86  // ᄀ + ᅡ + ᆨ -> 각87  inspect(@normalization.nfc("\u{1100}\u{1161}\u{11A8}"), content="\u{AC01}")88  // 가 + ᆨ -> 각89  inspect(@normalization.nfc("\u{AC00}\u{11A8}"), content="\u{AC01}")90}9192///|93test "nfc: canonical ordering before composition" {94  // Input has combining marks in wrong order95  // a + dot above (CCC=230) + dot below (CCC=220) + acute (CCC=230)96  // After ordering: a + dot below (220) + dot above (230) + acute (230)97  // Composition: ạ (a + dot below) cannot compose with dot above98  let input = "a\u{0307}\u{0323}\u{0301}"99  let result = @normalization.nfc(input)100  // The result should have proper ordering101  assert_true(result.length() > 0)102}103104///|105test "nfc: blocked composition" {106  // When a combining mark blocks composition107  // a + cedilla (CCC=202) + acute (CCC=230)108  // The cedilla doesn't block acute from composing with 'a' because CCC(cedilla) < CCC(acute)109  // But a + ogonek (CCC=202) + dot below (CCC=220) - ogonek blocks dot below110  let input = "a\u{0328}\u{0323}" // a + ogonek + dot below111  let result = @normalization.nfc(input)112  // Should NOT compose because ogonek blocks113  inspect(result, content="ą̣")114}115116// ==================== NFKD Tests ====================117118///|119test "nfkd: compatibility decomposition" {120  // fi (U+FB01) -> fi121  inspect(@normalization.nfkd("\u{FB01}"), content="fi")122  // fl (U+FB02) -> fl123  inspect(@normalization.nfkd("\u{FB02}"), content="fl")124}125126///|127test "nfkd: superscript decomposition" {128  // ² (U+00B2) -> 2129  inspect(@normalization.nfkd("\u{00B2}"), content="2")130  // ³ (U+00B3) -> 3131  inspect(@normalization.nfkd("\u{00B3}"), content="3")132}133134///|135test "nfkd: fraction decomposition" {136  // ½ (U+00BD) -> 1⁄2 (1 + fraction slash + 2)137  let result = @normalization.nfkd("\u{00BD}")138  assert_true(result.contains("1"))139  assert_true(result.contains("2"))140}141142///|143test "nfkd: width variants" {144  // a (U+FF41 fullwidth a) -> a145  inspect(@normalization.nfkd("\u{FF41}"), content="a")146  // A (U+FF21 fullwidth A) -> A147  inspect(@normalization.nfkd("\u{FF21}"), content="A")148}149150// ==================== NFKC Tests ====================151152///|153test "nfkc: compatibility then compose" {154  // fi (U+FB01) -> fi (no further composition)155  inspect(@normalization.nfkc("\u{FB01}"), content="fi")156}157158///|159test "nfkc: hangul in fullwidth" {160  // Fullwidth followed by composable should work161  let result = @normalization.nfkc("\u{1100}\u{1161}")162  inspect(result, content="\u{AC00}")163}164165// ==================== Property Tests ====================166167///|168test "idempotence: nfd" {169  let test_strings = [170    "Hello", "\u{00C5}", "\u{AC00}\u{AC01}", "a\u{0307}\u{0323}",171  ]172  for s in test_strings {173    let once = @normalization.nfd(s)174    let twice = @normalization.nfd(once)175    assert_eq(once, twice)176  }177}178179///|180test "idempotence: nfc" {181  let test_strings = ["Hello", "\u{00C5}", "\u{AC00}\u{AC01}", "A\u{030A}"]182  for s in test_strings {183    let once = @normalization.nfc(s)184    let twice = @normalization.nfc(once)185    assert_eq(once, twice)186  }187}188189///|190test "idempotence: nfkd" {191  let test_strings = ["Hello", "\u{FB01}", "\u{00B2}"]192  for s in test_strings {193    let once = @normalization.nfkd(s)194    let twice = @normalization.nfkd(once)195    assert_eq(once, twice)196  }197}198199///|200test "idempotence: nfkc" {201  let test_strings = ["Hello", "\u{FB01}", "\u{00B2}"]202  for s in test_strings {203    let once = @normalization.nfkc(s)204    let twice = @normalization.nfkc(once)205    assert_eq(once, twice)206  }207}208209///|210test "is_normalized" {211  // ASCII is always normalized212  assert_true(@normalization.is_normalized("Hello", @normalization.NFC))213  assert_true(@normalization.is_normalized("Hello", @normalization.NFD))214  // Precomposed Å is NFC but not NFD215  assert_true(@normalization.is_normalized("\u{00C5}", @normalization.NFC))216  assert_false(@normalization.is_normalized("\u{00C5}", @normalization.NFD))217  // Decomposed A + ring is NFD but not NFC218  assert_true(@normalization.is_normalized("A\u{030A}", @normalization.NFD))219  assert_false(@normalization.is_normalized("A\u{030A}", @normalization.NFC))220}221222///|223test "empty string" {224  inspect(@normalization.nfd(""), content="")225  inspect(@normalization.nfc(""), content="")226  inspect(@normalization.nfkd(""), content="")227  inspect(@normalization.nfkc(""), content="")228}229230///|231test "normalize function" {232  let s = "\u{00C5}" // Å233  assert_eq(234    @normalization.normalize(s, @normalization.NFC),235    @normalization.nfc(s),236  )237  assert_eq(238    @normalization.normalize(s, @normalization.NFD),239    @normalization.nfd(s),240  )241  assert_eq(242    @normalization.normalize(s, @normalization.NFKC),243    @normalization.nfkc(s),244  )245  assert_eq(246    @normalization.normalize(s, @normalization.NFKD),247    @normalization.nfkd(s),248  )249}250251// ==================== Conformance Tests (selected from NormalizationTest.txt) ====================252253///|254test "conformance: angstrom sign" {255  // U+212B ANGSTROM SIGN256  // NFD: A (0041) + ring above (030A)257  // NFC: Å (00C5)258  let angstrom = "\u{212B}"259  inspect(@normalization.nfd(angstrom), content="A\u{030A}")260  inspect(@normalization.nfc(angstrom), content="\u{00C5}")261}262263///|264test "conformance: ohm sign" {265  // U+2126 OHM SIGN -> U+03A9 GREEK CAPITAL LETTER OMEGA266  // This is a singleton decomposition267  inspect(@normalization.nfd("\u{2126}"), content="\u{03A9}")268  inspect(@normalization.nfc("\u{2126}"), content="\u{03A9}")269}270271///|272test "conformance: combining sequence" {273  // Test from UAX #15 examples274  // D (0044) + combining dot above (0307) -> Ḋ (1E0A)275  inspect(@normalization.nfc("D\u{0307}"), content="\u{1E0A}")276  inspect(@normalization.nfd("\u{1E0A}"), content="D\u{0307}")277}278279///|280test "NFD: marks around a Hangul syllable keep their ordering rules" {281  // U+0315 (class 232) before U+0300 (class 230) is reordered after the jamo.282  assert_eq(283    @normalization.nfd("\u{D55C}\u{0315}\u{0300}"),284    "\u{1112}\u{1161}\u{11AB}\u{0300}\u{0315}",285  )286  // The jamo are starters, so a mark before the syllable stays where it is.287  assert_eq(288    @normalization.nfd("a\u{0315}\u{AC00}\u{0300}"),289    "a\u{0315}\u{1100}\u{1161}\u{0300}",290  )291  assert_eq(@normalization.nfkd("\u{AC01}"), "\u{1100}\u{1161}\u{11A8}")292}293294///|295test "NFD and NFKD: recursive and compatibility decompositions" {296  // U+1EC7 decomposes in two steps.297  assert_eq(@normalization.nfd("\u{1EC7}"), "e\u{0323}\u{0302}")298  // U+01C4 has only a compatibility mapping, whose second character299  // decomposes canonically.300  assert_eq(@normalization.nfd("\u{01C4}"), "\u{01C4}")301  assert_eq(@normalization.nfkd("\u{01C4}"), "DZ\u{030C}")302  assert_eq(@normalization.nfkd("\u{FB03}\u{00E9}"), "ffie\u{0301}")303}304305///|306test "NFC: the last character of a clean run composes with what follows" {307  // U+11099 is a supplementary starter after a clean run; the nukta308  // (Quick_Check=Maybe) composes with it.309  assert_eq(@normalization.nfc("ab\u{11099}\u{110BA}"), "ab\u{1109A}")310  // The marks are reordered before composing: U+0323 (class 220) composes311  // with `e`, and U+0301 (class 230) then has no composite.312  assert_eq(@normalization.nfc("e\u{0301}\u{0323}"), "\u{1EB9}\u{0301}")313}314315///|316test "NFC and NFKC: conjoining jamo compose only when adjacent" {317  assert_eq(@normalization.nfc("x\u{1100}"), "x\u{1100}")318  assert_eq(@normalization.nfc("\u{1100}a\u{1161}"), "\u{1100}a\u{1161}")319  // A mark between the syllable and a trailing jamo blocks it.320  assert_eq(321    @normalization.nfc("\u{1100}\u{1161}\u{0301}\u{11A8}"),322    "\u{AC00}\u{0301}\u{11A8}",323  )324  assert_eq(325    @normalization.nfc("\u{AC00}\u{0323}\u{11A8}"),326    "\u{AC00}\u{0323}\u{11A8}",327  )328  // The trailing jamo past the view's end is not read.329  let text = "x\u{1100}\u{1161}\u{11A8}"330  assert_eq(@normalization.nfc(text[1:3]), "\u{AC00}")331  assert_eq(@normalization.nfc(text[1:4]), "\u{AC01}")332  // U+3131 decomposes to a leading jamo, which composes with the vowel.333  assert_eq(@normalization.nfkc("\u{3131}\u{1161}"), "\u{AC00}")334}335336///|337test "NFKC: an LV syllable composes with a compatibility trailing jamo" {338  // U+3133 and U+FFA3 decompose to the trailing jamo U+11AA, which composes339  // with the syllable that the leading jamo and vowel before it form.340  assert_eq(@normalization.nfkc("\u{1100}\u{1161}\u{3133}"), "\u{AC03}")341  assert_eq(@normalization.nfkc("\u{1100}\u{1161}\u{FFA3}"), "\u{AC03}")342  assert_eq(@normalization.nfkc("\u{AC00}\u{3133}"), "\u{AC03}")343  // Normalizing again changes nothing.344  assert_eq(@normalization.nfkc("\u{AC03}"), "\u{AC03}")345  assert_true(@normalization.is_normalized("\u{AC03}", NFKC))346  assert_true(!@normalization.is_normalized("\u{1100}\u{1161}\u{3133}", NFKC))347  // NFC leaves the compatibility jamo, so the syllable stays LV.348  assert_eq(@normalization.nfc("\u{1100}\u{1161}\u{3133}"), "\u{AC00}\u{3133}")349  // The LV syllable still ends its segment before what follows.350  assert_eq(@normalization.nfkc("\u{1100}\u{1161}a"), "\u{AC00}a")351  assert_eq(352    @normalization.nfkc("\u{1100}\u{1161}\u{1100}\u{1161}"),353    "\u{AC00}\u{AC00}",354  )355}356