normalization/normalization_test.mbt1///|2/// Tests for Unicode Normalization34// ==================== NFD Tests ====================56///|7test "nfd: ASCII unchanged" {8 // ASCII text should remain unchanged9 inspect(@normalization.nfd("Hello, World!"), content="Hello, World!")10 inspect(@normalization.nfd("abc123"), content="abc123")11}1213///|14test "nfd: already decomposed" {15 // Already decomposed text should remain unchanged16 let decomposed = "A\u{030A}" // A + combining ring above17 inspect(@normalization.nfd(decomposed), content="A\u{030A}")18}1920///|21test "nfd: precomposed to decomposed" {22 // Å (U+00C5) should decompose to A (U+0041) + combining ring above (U+030A)23 inspect(@normalization.nfd("\u{00C5}"), content="A\u{030A}")24 // é (U+00E9) should decompose to e (U+0065) + combining acute (U+0301)25 inspect(@normalization.nfd("\u{00E9}"), content="e\u{0301}")26 // ñ (U+00F1) should decompose to n (U+006E) + combining tilde (U+0303)27 inspect(@normalization.nfd("\u{00F1}"), content="n\u{0303}")28}2930///|31test "nfd: multi-level decomposition" {32 // Some characters decompose in multiple steps33 // Ṩ (U+1E68) -> S + dot below + dot above34 let result = @normalization.nfd("\u{1E68}")35 // Should be S (0x53) + combining dot below (0x323) + combining dot above (0x307)36 inspect(result, content="S\u{0323}\u{0307}")37}3839///|40test "nfd: canonical ordering" {41 // Combining marks should be reordered by CCC42 // Input: a + combining dot above (CCC=230) + combining dot below (CCC=220)43 // Output: a + combining dot below (CCC=220) + combining dot above (CCC=230)44 let input = "a\u{0307}\u{0323}"45 inspect(@normalization.nfd(input), content="a\u{0323}\u{0307}")46}4748///|49test "nfd: hangul decomposition" {50 // 가 (U+AC00) -> ᄀ (U+1100) + ᅡ (U+1161)51 inspect(@normalization.nfd("\u{AC00}"), content="\u{1100}\u{1161}")52 // 각 (U+AC01) -> ᄀ (U+1100) + ᅡ (U+1161) + ᆨ (U+11A8)53 inspect(@normalization.nfd("\u{AC01}"), content="\u{1100}\u{1161}\u{11A8}")54 // 한 (U+D55C) -> ᄒ (U+1112) + ᅡ (U+1161) + ᆫ (U+11AB)55 inspect(@normalization.nfd("\u{D55C}"), content="\u{1112}\u{1161}\u{11AB}")56}5758// ==================== NFC Tests ====================5960///|61test "nfc: ASCII unchanged" {62 inspect(@normalization.nfc("Hello, World!"), content="Hello, World!")63}6465///|66test "nfc: already composed" {67 // Already composed text should remain unchanged68 inspect(@normalization.nfc("\u{00C5}"), content="\u{00C5}") // Å69 inspect(@normalization.nfc("\u{00E9}"), content="\u{00E9}") // é70}7172///|73test "nfc: decomposed to composed" {74 // A + combining ring above -> Å75 inspect(@normalization.nfc("A\u{030A}"), content="\u{00C5}")76 // e + combining acute -> é77 inspect(@normalization.nfc("e\u{0301}"), content="\u{00E9}")78 // n + combining tilde -> ñ79 inspect(@normalization.nfc("n\u{0303}"), content="\u{00F1}")80}8182///|83test "nfc: hangul composition" {84 // ᄀ + ᅡ -> 가85 inspect(@normalization.nfc("\u{1100}\u{1161}"), content="\u{AC00}")86 // ᄀ + ᅡ + ᆨ -> 각87 inspect(@normalization.nfc("\u{1100}\u{1161}\u{11A8}"), content="\u{AC01}")88 // 가 + ᆨ -> 각89 inspect(@normalization.nfc("\u{AC00}\u{11A8}"), content="\u{AC01}")90}9192///|93test "nfc: canonical ordering before composition" {94 // Input has combining marks in wrong order95 // a + dot above (CCC=230) + dot below (CCC=220) + acute (CCC=230)96 // After ordering: a + dot below (220) + dot above (230) + acute (230)97 // Composition: ạ (a + dot below) cannot compose with dot above98 let input = "a\u{0307}\u{0323}\u{0301}"99 let result = @normalization.nfc(input)100 // The result should have proper ordering101 assert_true(result.length() > 0)102}103104///|105test "nfc: blocked composition" {106 // When a combining mark blocks composition107 // a + cedilla (CCC=202) + acute (CCC=230)108 // The cedilla doesn't block acute from composing with 'a' because CCC(cedilla) < CCC(acute)109 // But a + ogonek (CCC=202) + dot below (CCC=220) - ogonek blocks dot below110 let input = "a\u{0328}\u{0323}" // a + ogonek + dot below111 let result = @normalization.nfc(input)112 // Should NOT compose because ogonek blocks113 inspect(result, content="ą̣")114}115116// ==================== NFKD Tests ====================117118///|119test "nfkd: compatibility decomposition" {120 // fi (U+FB01) -> fi121 inspect(@normalization.nfkd("\u{FB01}"), content="fi")122 // fl (U+FB02) -> fl123 inspect(@normalization.nfkd("\u{FB02}"), content="fl")124}125126///|127test "nfkd: superscript decomposition" {128 // ² (U+00B2) -> 2129 inspect(@normalization.nfkd("\u{00B2}"), content="2")130 // ³ (U+00B3) -> 3131 inspect(@normalization.nfkd("\u{00B3}"), content="3")132}133134///|135test "nfkd: fraction decomposition" {136 // ½ (U+00BD) -> 1⁄2 (1 + fraction slash + 2)137 let result = @normalization.nfkd("\u{00BD}")138 assert_true(result.contains("1"))139 assert_true(result.contains("2"))140}141142///|143test "nfkd: width variants" {144 // a (U+FF41 fullwidth a) -> a145 inspect(@normalization.nfkd("\u{FF41}"), content="a")146 // A (U+FF21 fullwidth A) -> A147 inspect(@normalization.nfkd("\u{FF21}"), content="A")148}149150// ==================== NFKC Tests ====================151152///|153test "nfkc: compatibility then compose" {154 // fi (U+FB01) -> fi (no further composition)155 inspect(@normalization.nfkc("\u{FB01}"), content="fi")156}157158///|159test "nfkc: hangul in fullwidth" {160 // Fullwidth followed by composable should work161 let result = @normalization.nfkc("\u{1100}\u{1161}")162 inspect(result, content="\u{AC00}")163}164165// ==================== Property Tests ====================166167///|168test "idempotence: nfd" {169 let test_strings = [170 "Hello", "\u{00C5}", "\u{AC00}\u{AC01}", "a\u{0307}\u{0323}",171 ]172 for s in test_strings {173 let once = @normalization.nfd(s)174 let twice = @normalization.nfd(once)175 assert_eq(once, twice)176 }177}178179///|180test "idempotence: nfc" {181 let test_strings = ["Hello", "\u{00C5}", "\u{AC00}\u{AC01}", "A\u{030A}"]182 for s in test_strings {183 let once = @normalization.nfc(s)184 let twice = @normalization.nfc(once)185 assert_eq(once, twice)186 }187}188189///|190test "idempotence: nfkd" {191 let test_strings = ["Hello", "\u{FB01}", "\u{00B2}"]192 for s in test_strings {193 let once = @normalization.nfkd(s)194 let twice = @normalization.nfkd(once)195 assert_eq(once, twice)196 }197}198199///|200test "idempotence: nfkc" {201 let test_strings = ["Hello", "\u{FB01}", "\u{00B2}"]202 for s in test_strings {203 let once = @normalization.nfkc(s)204 let twice = @normalization.nfkc(once)205 assert_eq(once, twice)206 }207}208209///|210test "is_normalized" {211 // ASCII is always normalized212 assert_true(@normalization.is_normalized("Hello", @normalization.NFC))213 assert_true(@normalization.is_normalized("Hello", @normalization.NFD))214 // Precomposed Å is NFC but not NFD215 assert_true(@normalization.is_normalized("\u{00C5}", @normalization.NFC))216 assert_false(@normalization.is_normalized("\u{00C5}", @normalization.NFD))217 // Decomposed A + ring is NFD but not NFC218 assert_true(@normalization.is_normalized("A\u{030A}", @normalization.NFD))219 assert_false(@normalization.is_normalized("A\u{030A}", @normalization.NFC))220}221222///|223test "empty string" {224 inspect(@normalization.nfd(""), content="")225 inspect(@normalization.nfc(""), content="")226 inspect(@normalization.nfkd(""), content="")227 inspect(@normalization.nfkc(""), content="")228}229230///|231test "normalize function" {232 let s = "\u{00C5}" // Å233 assert_eq(234 @normalization.normalize(s, @normalization.NFC),235 @normalization.nfc(s),236 )237 assert_eq(238 @normalization.normalize(s, @normalization.NFD),239 @normalization.nfd(s),240 )241 assert_eq(242 @normalization.normalize(s, @normalization.NFKC),243 @normalization.nfkc(s),244 )245 assert_eq(246 @normalization.normalize(s, @normalization.NFKD),247 @normalization.nfkd(s),248 )249}250251// ==================== Conformance Tests (selected from NormalizationTest.txt) ====================252253///|254test "conformance: angstrom sign" {255 // U+212B ANGSTROM SIGN256 // NFD: A (0041) + ring above (030A)257 // NFC: Å (00C5)258 let angstrom = "\u{212B}"259 inspect(@normalization.nfd(angstrom), content="A\u{030A}")260 inspect(@normalization.nfc(angstrom), content="\u{00C5}")261}262263///|264test "conformance: ohm sign" {265 // U+2126 OHM SIGN -> U+03A9 GREEK CAPITAL LETTER OMEGA266 // This is a singleton decomposition267 inspect(@normalization.nfd("\u{2126}"), content="\u{03A9}")268 inspect(@normalization.nfc("\u{2126}"), content="\u{03A9}")269}270271///|272test "conformance: combining sequence" {273 // Test from UAX #15 examples274 // D (0044) + combining dot above (0307) -> Ḋ (1E0A)275 inspect(@normalization.nfc("D\u{0307}"), content="\u{1E0A}")276 inspect(@normalization.nfd("\u{1E0A}"), content="D\u{0307}")277}278279///|280test "NFD: marks around a Hangul syllable keep their ordering rules" {281 // U+0315 (class 232) before U+0300 (class 230) is reordered after the jamo.282 assert_eq(283 @normalization.nfd("\u{D55C}\u{0315}\u{0300}"),284 "\u{1112}\u{1161}\u{11AB}\u{0300}\u{0315}",285 )286 // The jamo are starters, so a mark before the syllable stays where it is.287 assert_eq(288 @normalization.nfd("a\u{0315}\u{AC00}\u{0300}"),289 "a\u{0315}\u{1100}\u{1161}\u{0300}",290 )291 assert_eq(@normalization.nfkd("\u{AC01}"), "\u{1100}\u{1161}\u{11A8}")292}293294///|295test "NFD and NFKD: recursive and compatibility decompositions" {296 // U+1EC7 decomposes in two steps.297 assert_eq(@normalization.nfd("\u{1EC7}"), "e\u{0323}\u{0302}")298 // U+01C4 has only a compatibility mapping, whose second character299 // decomposes canonically.300 assert_eq(@normalization.nfd("\u{01C4}"), "\u{01C4}")301 assert_eq(@normalization.nfkd("\u{01C4}"), "DZ\u{030C}")302 assert_eq(@normalization.nfkd("\u{FB03}\u{00E9}"), "ffie\u{0301}")303}304305///|306test "NFC: the last character of a clean run composes with what follows" {307 // U+11099 is a supplementary starter after a clean run; the nukta308 // (Quick_Check=Maybe) composes with it.309 assert_eq(@normalization.nfc("ab\u{11099}\u{110BA}"), "ab\u{1109A}")310 // The marks are reordered before composing: U+0323 (class 220) composes311 // with `e`, and U+0301 (class 230) then has no composite.312 assert_eq(@normalization.nfc("e\u{0301}\u{0323}"), "\u{1EB9}\u{0301}")313}314315///|316test "NFC and NFKC: conjoining jamo compose only when adjacent" {317 assert_eq(@normalization.nfc("x\u{1100}"), "x\u{1100}")318 assert_eq(@normalization.nfc("\u{1100}a\u{1161}"), "\u{1100}a\u{1161}")319 // A mark between the syllable and a trailing jamo blocks it.320 assert_eq(321 @normalization.nfc("\u{1100}\u{1161}\u{0301}\u{11A8}"),322 "\u{AC00}\u{0301}\u{11A8}",323 )324 assert_eq(325 @normalization.nfc("\u{AC00}\u{0323}\u{11A8}"),326 "\u{AC00}\u{0323}\u{11A8}",327 )328 // The trailing jamo past the view's end is not read.329 let text = "x\u{1100}\u{1161}\u{11A8}"330 assert_eq(@normalization.nfc(text[1:3]), "\u{AC00}")331 assert_eq(@normalization.nfc(text[1:4]), "\u{AC01}")332 // U+3131 decomposes to a leading jamo, which composes with the vowel.333 assert_eq(@normalization.nfkc("\u{3131}\u{1161}"), "\u{AC00}")334}335336///|337test "NFKC: an LV syllable composes with a compatibility trailing jamo" {338 // U+3133 and U+FFA3 decompose to the trailing jamo U+11AA, which composes339 // with the syllable that the leading jamo and vowel before it form.340 assert_eq(@normalization.nfkc("\u{1100}\u{1161}\u{3133}"), "\u{AC03}")341 assert_eq(@normalization.nfkc("\u{1100}\u{1161}\u{FFA3}"), "\u{AC03}")342 assert_eq(@normalization.nfkc("\u{AC00}\u{3133}"), "\u{AC03}")343 // Normalizing again changes nothing.344 assert_eq(@normalization.nfkc("\u{AC03}"), "\u{AC03}")345 assert_true(@normalization.is_normalized("\u{AC03}", NFKC))346 assert_true(!@normalization.is_normalized("\u{1100}\u{1161}\u{3133}", NFKC))347 // NFC leaves the compatibility jamo, so the syllable stays LV.348 assert_eq(@normalization.nfc("\u{1100}\u{1161}\u{3133}"), "\u{AC00}\u{3133}")349 // The LV syllable still ends its segment before what follows.350 assert_eq(@normalization.nfkc("\u{1100}\u{1161}a"), "\u{AC00}a")351 assert_eq(352 @normalization.nfkc("\u{1100}\u{1161}\u{1100}\u{1161}"),353 "\u{AC00}\u{AC00}",354 )355}356
///|
/// Tests for Unicode Normalization
// ==================== NFD Tests ====================
///|
test "nfd: ASCII unchanged" {
// ASCII text should remain unchanged
inspect(@normalization.nfd("Hello, World!"), content="Hello, World!")
inspect(@normalization.nfd("abc123"), content="abc123")
}
///|
test "nfd: already decomposed" {
// Already decomposed text should remain unchanged
let decomposed = "A\u{030A}" // A + combining ring above
inspect(@normalization.nfd(decomposed), content="A\u{030A}")
}
///|
test "nfd: precomposed to decomposed" {
// Å (U+00C5) should decompose to A (U+0041) + combining ring above (U+030A)
inspect(@normalization.nfd("\u{00C5}"), content="A\u{030A}")
// é (U+00E9) should decompose to e (U+0065) + combining acute (U+0301)
inspect(@normalization.nfd("\u{00E9}"), content="e\u{0301}")
// ñ (U+00F1) should decompose to n (U+006E) + combining tilde (U+0303)
inspect(@normalization.nfd("\u{00F1}"), content="n\u{0303}")
}
///|
test "nfd: multi-level decomposition" {
// Some characters decompose in multiple steps
// Ṩ (U+1E68) -> S + dot below + dot above
let result = @normalization.nfd("\u{1E68}")
// Should be S (0x53) + combining dot below (0x323) + combining dot above (0x307)
inspect(result, content="S\u{0323}\u{0307}")
}
///|
test "nfd: canonical ordering" {
// Combining marks should be reordered by CCC
// Input: a + combining dot above (CCC=230) + combining dot below (CCC=220)
// Output: a + combining dot below (CCC=220) + combining dot above (CCC=230)
let input = "a\u{0307}\u{0323}"
inspect(@normalization.nfd(input), content="a\u{0323}\u{0307}")
}
///|
test "nfd: hangul decomposition" {
// 가 (U+AC00) -> ᄀ (U+1100) + ᅡ (U+1161)
inspect(@normalization.nfd("\u{AC00}"), content="\u{1100}\u{1161}")
// 각 (U+AC01) -> ᄀ (U+1100) + ᅡ (U+1161) + ᆨ (U+11A8)
inspect(@normalization.nfd("\u{AC01}"), content="\u{1100}\u{1161}\u{11A8}")
// 한 (U+D55C) -> ᄒ (U+1112) + ᅡ (U+1161) + ᆫ (U+11AB)
inspect(@normalization.nfd("\u{D55C}"), content="\u{1112}\u{1161}\u{11AB}")
}
// ==================== NFC Tests ====================
///|
test "nfc: ASCII unchanged" {
inspect(@normalization.nfc("Hello, World!"), content="Hello, World!")
}
///|
test "nfc: already composed" {
// Already composed text should remain unchanged
inspect(@normalization.nfc("\u{00C5}"), content="\u{00C5}") // Å
inspect(@normalization.nfc("\u{00E9}"), content="\u{00E9}") // é
}
///|
test "nfc: decomposed to composed" {
// A + combining ring above -> Å
inspect(@normalization.nfc("A\u{030A}"), content="\u{00C5}")
// e + combining acute -> é
inspect(@normalization.nfc("e\u{0301}"), content="\u{00E9}")
// n + combining tilde -> ñ
inspect(@normalization.nfc("n\u{0303}"), content="\u{00F1}")
}
///|
test "nfc: hangul composition" {
// ᄀ + ᅡ -> 가
inspect(@normalization.nfc("\u{1100}\u{1161}"), content="\u{AC00}")
// ᄀ + ᅡ + ᆨ -> 각
inspect(@normalization.nfc("\u{1100}\u{1161}\u{11A8}"), content="\u{AC01}")
// 가 + ᆨ -> 각
inspect(@normalization.nfc("\u{AC00}\u{11A8}"), content="\u{AC01}")
}
///|
test "nfc: canonical ordering before composition" {
// Input has combining marks in wrong order
// a + dot above (CCC=230) + dot below (CCC=220) + acute (CCC=230)
// After ordering: a + dot below (220) + dot above (230) + acute (230)
// Composition: ạ (a + dot below) cannot compose with dot above
let input = "a\u{0307}\u{0323}\u{0301}"
let result = @normalization.nfc(input)
// The result should have proper ordering
assert_true(result.length() > 0)
}
///|
test "nfc: blocked composition" {
// When a combining mark blocks composition
// a + cedilla (CCC=202) + acute (CCC=230)
// The cedilla doesn't block acute from composing with 'a' because CCC(cedilla) < CCC(acute)
// But a + ogonek (CCC=202) + dot below (CCC=220) - ogonek blocks dot below
let input = "a\u{0328}\u{0323}" // a + ogonek + dot below
let result = @normalization.nfc(input)
// Should NOT compose because ogonek blocks
inspect(result, content="ą̣")
}
// ==================== NFKD Tests ====================
///|
test "nfkd: compatibility decomposition" {
// fi (U+FB01) -> fi
inspect(@normalization.nfkd("\u{FB01}"), content="fi")
// fl (U+FB02) -> fl
inspect(@normalization.nfkd("\u{FB02}"), content="fl")
}
///|
test "nfkd: superscript decomposition" {
// ² (U+00B2) -> 2
inspect(@normalization.nfkd("\u{00B2}"), content="2")
// ³ (U+00B3) -> 3
inspect(@normalization.nfkd("\u{00B3}"), content="3")
}
///|
test "nfkd: fraction decomposition" {
// ½ (U+00BD) -> 1⁄2 (1 + fraction slash + 2)
let result = @normalization.nfkd("\u{00BD}")
assert_true(result.contains("1"))
assert_true(result.contains("2"))
}
///|
test "nfkd: width variants" {
// a (U+FF41 fullwidth a) -> a
inspect(@normalization.nfkd("\u{FF41}"), content="a")
// A (U+FF21 fullwidth A) -> A
inspect(@normalization.nfkd("\u{FF21}"), content="A")
}
// ==================== NFKC Tests ====================
///|
test "nfkc: compatibility then compose" {
// fi (U+FB01) -> fi (no further composition)
inspect(@normalization.nfkc("\u{FB01}"), content="fi")
}
///|
test "nfkc: hangul in fullwidth" {
// Fullwidth followed by composable should work
let result = @normalization.nfkc("\u{1100}\u{1161}")
inspect(result, content="\u{AC00}")
}
// ==================== Property Tests ====================
///|
test "idempotence: nfd" {
let test_strings = [
"Hello", "\u{00C5}", "\u{AC00}\u{AC01}", "a\u{0307}\u{0323}",
]
for s in test_strings {
let once = @normalization.nfd(s)
let twice = @normalization.nfd(once)
assert_eq(once, twice)
}
}
///|
test "idempotence: nfc" {
let test_strings = ["Hello", "\u{00C5}", "\u{AC00}\u{AC01}", "A\u{030A}"]
for s in test_strings {
let once = @normalization.nfc(s)
let twice = @normalization.nfc(once)
assert_eq(once, twice)
}
}
///|
test "idempotence: nfkd" {
let test_strings = ["Hello", "\u{FB01}", "\u{00B2}"]
for s in test_strings {
let once = @normalization.nfkd(s)
let twice = @normalization.nfkd(once)
assert_eq(once, twice)
}
}
///|
test "idempotence: nfkc" {
let test_strings = ["Hello", "\u{FB01}", "\u{00B2}"]
for s in test_strings {
let once = @normalization.nfkc(s)
let twice = @normalization.nfkc(once)
assert_eq(once, twice)
}
}
///|
test "is_normalized" {
// ASCII is always normalized
assert_true(@normalization.is_normalized("Hello", @normalization.NFC))
assert_true(@normalization.is_normalized("Hello", @normalization.NFD))
// Precomposed Å is NFC but not NFD
assert_true(@normalization.is_normalized("\u{00C5}", @normalization.NFC))
assert_false(@normalization.is_normalized("\u{00C5}", @normalization.NFD))
// Decomposed A + ring is NFD but not NFC
assert_true(@normalization.is_normalized("A\u{030A}", @normalization.NFD))
assert_false(@normalization.is_normalized("A\u{030A}", @normalization.NFC))
}
///|
test "empty string" {
inspect(@normalization.nfd(""), content="")
inspect(@normalization.nfc(""), content="")
inspect(@normalization.nfkd(""), content="")
inspect(@normalization.nfkc(""), content="")
}
///|
test "normalize function" {
let s = "\u{00C5}" // Å
assert_eq(
@normalization.normalize(s, @normalization.NFC),
@normalization.nfc(s),
)
assert_eq(
@normalization.normalize(s, @normalization.NFD),
@normalization.nfd(s),
)
assert_eq(
@normalization.normalize(s, @normalization.NFKC),
@normalization.nfkc(s),
)
assert_eq(
@normalization.normalize(s, @normalization.NFKD),
@normalization.nfkd(s),
)
}
// ==================== Conformance Tests (selected from NormalizationTest.txt) ====================
///|
test "conformance: angstrom sign" {
// U+212B ANGSTROM SIGN
// NFD: A (0041) + ring above (030A)
// NFC: Å (00C5)
let angstrom = "\u{212B}"
inspect(@normalization.nfd(angstrom), content="A\u{030A}")
inspect(@normalization.nfc(angstrom), content="\u{00C5}")
}
///|
test "conformance: ohm sign" {
// U+2126 OHM SIGN -> U+03A9 GREEK CAPITAL LETTER OMEGA
// This is a singleton decomposition
inspect(@normalization.nfd("\u{2126}"), content="\u{03A9}")
inspect(@normalization.nfc("\u{2126}"), content="\u{03A9}")
}
///|
test "conformance: combining sequence" {
// Test from UAX #15 examples
// D (0044) + combining dot above (0307) -> Ḋ (1E0A)
inspect(@normalization.nfc("D\u{0307}"), content="\u{1E0A}")
inspect(@normalization.nfd("\u{1E0A}"), content="D\u{0307}")
}
///|
test "NFD: marks around a Hangul syllable keep their ordering rules" {
// U+0315 (class 232) before U+0300 (class 230) is reordered after the jamo.
assert_eq(
@normalization.nfd("\u{D55C}\u{0315}\u{0300}"),
"\u{1112}\u{1161}\u{11AB}\u{0300}\u{0315}",
)
// The jamo are starters, so a mark before the syllable stays where it is.
assert_eq(
@normalization.nfd("a\u{0315}\u{AC00}\u{0300}"),
"a\u{0315}\u{1100}\u{1161}\u{0300}",
)
assert_eq(@normalization.nfkd("\u{AC01}"), "\u{1100}\u{1161}\u{11A8}")
}
///|
test "NFD and NFKD: recursive and compatibility decompositions" {
// U+1EC7 decomposes in two steps.
assert_eq(@normalization.nfd("\u{1EC7}"), "e\u{0323}\u{0302}")
// U+01C4 has only a compatibility mapping, whose second character
// decomposes canonically.
assert_eq(@normalization.nfd("\u{01C4}"), "\u{01C4}")
assert_eq(@normalization.nfkd("\u{01C4}"), "DZ\u{030C}")
assert_eq(@normalization.nfkd("\u{FB03}\u{00E9}"), "ffie\u{0301}")
}
///|
test "NFC: the last character of a clean run composes with what follows" {
// U+11099 is a supplementary starter after a clean run; the nukta
// (Quick_Check=Maybe) composes with it.
assert_eq(@normalization.nfc("ab\u{11099}\u{110BA}"), "ab\u{1109A}")
// The marks are reordered before composing: U+0323 (class 220) composes
// with `e`, and U+0301 (class 230) then has no composite.
assert_eq(@normalization.nfc("e\u{0301}\u{0323}"), "\u{1EB9}\u{0301}")
}
///|
test "NFC and NFKC: conjoining jamo compose only when adjacent" {
assert_eq(@normalization.nfc("x\u{1100}"), "x\u{1100}")
assert_eq(@normalization.nfc("\u{1100}a\u{1161}"), "\u{1100}a\u{1161}")
// A mark between the syllable and a trailing jamo blocks it.
assert_eq(
@normalization.nfc("\u{1100}\u{1161}\u{0301}\u{11A8}"),
"\u{AC00}\u{0301}\u{11A8}",
)
assert_eq(
@normalization.nfc("\u{AC00}\u{0323}\u{11A8}"),
"\u{AC00}\u{0323}\u{11A8}",
)
// The trailing jamo past the view's end is not read.
let text = "x\u{1100}\u{1161}\u{11A8}"
assert_eq(@normalization.nfc(text[1:3]), "\u{AC00}")
assert_eq(@normalization.nfc(text[1:4]), "\u{AC01}")
// U+3131 decomposes to a leading jamo, which composes with the vowel.
assert_eq(@normalization.nfkc("\u{3131}\u{1161}"), "\u{AC00}")
}
///|
test "NFKC: an LV syllable composes with a compatibility trailing jamo" {
// U+3133 and U+FFA3 decompose to the trailing jamo U+11AA, which composes
// with the syllable that the leading jamo and vowel before it form.
assert_eq(@normalization.nfkc("\u{1100}\u{1161}\u{3133}"), "\u{AC03}")
assert_eq(@normalization.nfkc("\u{1100}\u{1161}\u{FFA3}"), "\u{AC03}")
assert_eq(@normalization.nfkc("\u{AC00}\u{3133}"), "\u{AC03}")
// Normalizing again changes nothing.
assert_eq(@normalization.nfkc("\u{AC03}"), "\u{AC03}")
assert_true(@normalization.is_normalized("\u{AC03}", NFKC))
assert_true(!@normalization.is_normalized("\u{1100}\u{1161}\u{3133}", NFKC))
// NFC leaves the compatibility jamo, so the syllable stays LV.
assert_eq(@normalization.nfc("\u{1100}\u{1161}\u{3133}"), "\u{AC00}\u{3133}")
// The LV syllable still ends its segment before what follows.
assert_eq(@normalization.nfkc("\u{1100}\u{1161}a"), "\u{AC00}a")
assert_eq(
@normalization.nfkc("\u{1100}\u{1161}\u{1100}\u{1161}"),
"\u{AC00}\u{AC00}",
)
}