normalization/quick_check_test.mbt1///|2test "quick check handles Yes No and contextual Maybe" {3 for4 form in [5 @normalization.NFC,6 @normalization.NFD,7 @normalization.NFKC,8 @normalization.NFKD,9 ] {10 assert_true(@normalization.is_normalized("", form))11 assert_true(@normalization.is_normalized("Hello 世界 😀", form))12 assert_true(@normalization.is_normalized("\u{0301}", form))13 }14 assert_true(@normalization.is_normalized("é", @normalization.NFC))15 assert_false(@normalization.is_normalized("é", @normalization.NFD))16 assert_false(@normalization.is_normalized("e\u{0301}", @normalization.NFC))17 // Maybe does not imply that a composition is possible in this context.18 assert_true(@normalization.is_normalized("x\u{0301}", @normalization.NFC))19 assert_true(20 @normalization.is_normalized("a\u{0305}\u{0301}", @normalization.NFC),21 )22 assert_false(23 @normalization.is_normalized("x\u{0301}ffi", @normalization.NFKC),24 )25}2627///|28test "quick check verifies order and resets at starters" {29 for30 form in [31 @normalization.NFC,32 @normalization.NFD,33 @normalization.NFKC,34 @normalization.NFKD,35 ] {36 // These non-composing marks individually have Quick_Check=Yes.37 assert_false(@normalization.is_normalized("\u{0315}\u{0316}", form))38 assert_eq(39 @normalization.normalize("\u{0315}\u{0316}", form),40 "\u{0316}\u{0315}",41 )42 assert_true(@normalization.is_normalized("\u{0315}x\u{0316}", form))43 }44}4546///|47test "spans retain precomposed starters and supplementary offsets" {48 assert_eq(49 @normalization.nfc("😀abcă\u{0328}世界😀e\u{0301}!"),50 "😀abcą\u{0306}世界😀é!",51 )52 assert_eq(@normalization.nfd("😀é|Å😀"), "😀e\u{0301}|A\u{030A}😀")53 assert_eq(@normalization.nfkc("😀A\u{030A}|ffi😀"), "😀Å|ffi😀")54 assert_eq(55 @normalization.nfkd("😀A\u{030A}|ffi😀"),56 "😀A\u{030A}|ffi😀",57 )58}5960///|61test "CCC zero Maybe characters do not split Hangul composition" {62 for form in [@normalization.NFC, @normalization.NFKC] {63 assert_eq(64 @normalization.normalize("😀각|각😀", form),65 "😀각|각😀",66 )67 assert_true(@normalization.is_normalized("ᅡ", form))68 assert_false(@normalization.is_normalized("가", form))69 assert_eq(70 @normalization.normalize("ᄀ\u{0301}ᅡ", form),71 "ᄀ\u{0301}ᅡ",72 )73 }74}7576///|77test "Unicode 16 composite Maybe characters keep their preceding context" {78 // Kirat Rai: AA + AI must decompose AI before composing with AA.79 for form in [@normalization.NFC, @normalization.NFKC] {80 assert_true(@normalization.is_normalized("\u{16D68}", form))81 assert_false(@normalization.is_normalized("\u{16D63}\u{16D68}", form))82 assert_eq(83 @normalization.normalize("😀\u{16D63}\u{16D68}|", form),84 "😀\u{16D6A}|",85 )86 }87}8889///|90test "local segments preserve short and long copied runs" {91 for gap_length in [0, 1, 15, 16, 17, 40] {92 let gap = "-".repeat(gap_length)93 let prefix = "😀prefix".repeat(8)94 let suffix = "suffix😀".repeat(8)95 let decomposed = "\{prefix}e\u{0301}\{gap}a\u{030A}\{suffix}"96 let composed = "\{prefix}é\{gap}å\{suffix}"97 for form in [@normalization.NFC, @normalization.NFKC] {98 assert_eq(@normalization.normalize(decomposed, form), composed)99 }100 for form in [@normalization.NFD, @normalization.NFKD] {101 assert_eq(@normalization.normalize(composed, form), decomposed)102 }103 }104}105106///|107test "local checking continues after unchanged Maybe segments" {108 let prefix = "x\u{0301}😀".repeat(32)109 for form in [@normalization.NFC, @normalization.NFKC] {110 assert_true(@normalization.is_normalized(prefix, form))111 for112 (tail, expected) in [113 ("e\u{0301}", "é"),114 ("ă\u{0328}", "ą\u{0306}"),115 ("\u{0315}\u{0316}", "\u{0316}\u{0315}"),116 ("각", "각"),117 ("\u{16D63}\u{16D68}", "\u{16D6A}"),118 ] {119 let input = prefix + tail120 assert_false(@normalization.is_normalized(input, form))121 assert_eq(@normalization.normalize(input, form), prefix + expected)122 // Also exercise a changed segment followed by unchanged Maybe segments.123 assert_eq(124 @normalization.normalize(tail + prefix, form),125 expected + prefix,126 )127 }128 }129}130131///|132test "scratch reuse after expansion composition and long combining runs" {133 let marks = "\u{0315}".repeat(64) + "\u{0316}".repeat(64)134 let ordered = "\u{0316}".repeat(64) + "\u{0315}".repeat(64)135 let input = "ffi|\{marks}|e\u{0301}|x\u{0301}|😀A\u{030A}"136 for137 (form, expected) in [138 (@normalization.NFC, "ffi|\{ordered}|é|x\u{0301}|😀A\u{030A}"),139 (@normalization.NFKC, "ffi|\{ordered}|é|x\u{0301}|😀Å"),140 (@normalization.NFD, "ffi|\{ordered}|e\u{0301}|x\u{0301}|😀A\u{030A}"),141 (@normalization.NFKD, "ffi|\{ordered}|e\u{0301}|x\u{0301}|😀A\u{030A}"),142 ] {143 assert_eq(@normalization.normalize(input, form), expected)144 assert_false(@normalization.is_normalized(input, form))145 assert_true(@normalization.is_normalized(expected, form))146 }147}148149///|150test "fast runs retain boundaries across ordered marks and surrogate pairs" {151 for152 prefix in [153 "",154 "ASCII",155 "世界",156 "😀",157 "\u{10000}",158 "\u{10FFFF}",159 "世界😀".repeat(16),160 ] {161 for162 (tail, expected) in [163 ("A\u{0316}\u{0301}", "Á\u{0316}"),164 ("\u{0316}\u{0315}", "\u{0316}\u{0315}"),165 ("\u{0315}\u{0316}", "\u{0316}\u{0315}"),166 ("\u{0315}😀\u{0316}", "\u{0315}😀\u{0316}"),167 ("\u{0315}A\u{0301}", "\u{0315}Á"),168 ("\u{16D63}\u{16D68}", "\u{16D6A}"),169 ] {170 let input = prefix + tail171 let normalized = prefix + expected172 assert_eq(@normalization.nfc(input), normalized)173 assert_eq(174 @normalization.is_normalized(input, @normalization.NFC),175 input == normalized,176 )177 }178 }179}180181///|182test "resume fast runs between suspect segments and at the end" {183 for184 gap in [185 "",186 "A",187 "AA",188 "AAA",189 "Hello world! ".repeat(256),190 "世界😀".repeat(64),191 ] {192 for193 (form, input_part, normalized_part) in [194 (@normalization.NFC, "e\u{0301}", "é"),195 (@normalization.NFC, "x\u{0301}", "x\u{0301}"),196 (@normalization.NFC, "각", "각"),197 (@normalization.NFC, "\u{16D63}\u{16D68}", "\u{16D6A}"),198 (@normalization.NFD, "é", "e\u{0301}"),199 (@normalization.NFKC, "ffiA\u{030A}", "ffiÅ"),200 (@normalization.NFKD, "ffiÅ", "ffiA\u{030A}"),201 ] {202 for tail in ["", "\u{0315}\u{0316}", "😀\u{0315}\u{0316}"] {203 let ordered_tail = if tail == "" {204 ""205 } else if tail == "\u{0315}\u{0316}" {206 "\u{0316}\u{0315}"207 } else {208 "😀\u{0316}\u{0315}"209 }210 // Separators prevent the expected pieces from interacting; the final211 // gap + marks also checks recovery of the last skipped starter.212 let input = input_part + "|" + gap + "|" + input_part + "|" + gap + tail213 let expected = normalized_part +214 "|" +215 gap +216 "|" +217 normalized_part +218 "|" +219 gap +220 ordered_tail221 assert_eq(@normalization.normalize(input, form), expected)222 assert_eq(@normalization.is_normalized(input, form), input == expected)223 assert_true(@normalization.is_normalized(expected, form))224 }225 }226 }227 // After an unchanged Maybe segment, recover A as the next composing starter.228 assert_eq(@normalization.nfc("x\u{0301}AAA\u{0301}"), "x\u{0301}AAÁ")229 assert_false(230 @normalization.is_normalized("x\u{0301}AAA\u{0301}", @normalization.NFC),231 )232}233234///|235test "surrogate summaries respect form masks and recover complete boundaries" {236 for gap in ["😀", "\u{10FFFD}", "\u{10000}", "\u{1D400}"] {237 for238 (form, changed, fixed, normalized_gap) in [239 (@normalization.NFC, "e\u{0301}", "é", gap),240 (@normalization.NFD, "é", "e\u{0301}", gap),241 (242 @normalization.NFKC,243 "e\u{0301}",244 "é",245 if gap == "\u{1D400}" {246 "A"247 } else {248 gap249 },250 ),251 (252 @normalization.NFKD,253 "é",254 "e\u{0301}",255 if gap == "\u{1D400}" {256 "A"257 } else {258 gap259 },260 ),261 ] {262 // No separator before the marks: recovery must retain the full pair,263 // including when a changed prefix has already allocated scratch storage.264 for prefix in ["", changed + "|"] {265 let fixed_prefix = if prefix == "" { "" } else { fixed + "|" }266 let input = prefix + gap.repeat(8) + "\u{0315}\u{0316}|" + changed267 let expected = fixed_prefix +268 normalized_gap.repeat(8) +269 "\u{0316}\u{0315}|" +270 fixed271 assert_eq(@normalization.normalize(input, form), expected)272 assert_false(@normalization.is_normalized(input, form))273 assert_true(@normalization.is_normalized(expected, form))274 }275 }276 }277 // Active groups still decode mappings, combining marks and backward composition.278 assert_eq(@normalization.nfd("\u{1D15E}"), "\u{1D157}\u{1D165}")279 assert_eq(@normalization.nfc("\u{11099}\u{110BA}"), "\u{1109A}")280 assert_eq(281 @normalization.nfc("😀\u{1D165}\u{1D167}"),282 "😀\u{1D167}\u{1D165}",283 )284}285286///|287test "surrogate summaries keep isolated code units unchanged" {288 for cp in [0xD800, 0xD835, 0xDBFF, 0xDC00, 0xDFFF] {289 let lone = String::from_array([cp.unsafe_to_char()])290 for input in [lone, lone + "A", "A" + lone] {291 for292 form in [293 @normalization.NFC,294 @normalization.NFD,295 @normalization.NFKC,296 @normalization.NFKD,297 ] {298 assert_eq(@normalization.normalize(input, form), input)299 assert_true(@normalization.is_normalized(input, form))300 }301 }302 }303}304305///|306test "unpaired surrogates stay clean next to suspect characters" {307 // U+D835 leads a group of 1024 code points that has flagged characters.308 let lead = String::from_array([(0xD835).unsafe_to_char()])309 let trail = String::from_array([(0xDC00).unsafe_to_char()])310 for311 (input, expected) in [312 (lead + "e\u{0301}", lead + "é"),313 ("e\u{0301}" + lead, "é" + lead),314 (trail + lead + "e\u{0301}", trail + lead + "é"),315 (lead + "\u{0301}", lead + "\u{0301}"),316 (lead + "\u{0315}\u{0316}", lead + "\u{0316}\u{0315}"),317 // The second unit pairs with the trailing one; the first stays alone.318 (319 lead + lead + trail + "\u{0315}\u{0316}",320 lead + lead + trail + "\u{0316}\u{0315}",321 ),322 (323 "e\u{0301}|" + lead + lead + trail + lead,324 "é|" + lead + lead + trail + lead,325 ),326 ] {327 assert_eq(@normalization.nfc(input), expected)328 assert_eq(329 @normalization.is_normalized(input, @normalization.NFC),330 input == expected,331 )332 assert_true(@normalization.is_normalized(expected, @normalization.NFC))333 }334}335336///|337test "clean pairs in flagged groups become the safe boundary" {338 // U+1D400 is clean for NFC and NFD, but shares its group with U+1D15E.339 let pair = "\u{1D400}"340 for prefix in ["", "世界", "e\u{0301}|"] {341 let fixed = if prefix == "e\u{0301}|" { "é|" } else { prefix }342 for count in [1, 2, 64] {343 let run = pair.repeat(count)344 // Marks attach to the last pair, so only they are reordered.345 assert_eq(346 @normalization.nfc(prefix + run + "\u{0315}\u{0316}"),347 fixed + run + "\u{0316}\u{0315}",348 )349 // A starter after the run composes without disturbing it.350 assert_eq(351 @normalization.nfc(prefix + run + "e\u{0301}"),352 fixed + run + "é",353 )354 assert_eq(355 @normalization.is_normalized(prefix + run, @normalization.NFC),356 prefix == fixed,357 )358 }359 }360}361362///|363test "block scans stop at every offset within and between blocks" {364 // Each case is a character that only some forms leave alone, with its365 // normalization in NFC, NFD, NFKC and NFKD.366 for367 (suspect, nfc, nfd, nfkc, nfkd) in [368 ("\u{A0}", "\u{A0}", "\u{A0}", " ", " "),369 ("é", "é", "e\u{0301}", "é", "e\u{0301}"),370 ("e\u{0301}", "é", "e\u{0301}", "é", "e\u{0301}"),371 ("😀", "😀", "😀", "😀", "😀"),372 (373 "\u{1D15E}", "\u{1D157}\u{1D165}", "\u{1D157}\u{1D165}", "\u{1D157}\u{1D165}",374 "\u{1D157}\u{1D165}",375 ),376 ] {377 for before in 0..<48 {378 for after in [0, 1, 7, 8, 9, 17] {379 let prefix = "a".repeat(before)380 let suffix = "z".repeat(after)381 let input = prefix + suspect + suffix382 for383 (form, expected) in [384 (@normalization.NFC, nfc),385 (@normalization.NFD, nfd),386 (@normalization.NFKC, nfkc),387 (@normalization.NFKD, nfkd),388 ] {389 let normalized = prefix + expected + suffix390 assert_eq(@normalization.normalize(input, form), normalized)391 assert_eq(392 @normalization.is_normalized(input, form),393 input == normalized,394 )395 }396 }397 }398 }399}400401///|402test "long ASCII runs at every length" {403 for404 form in [405 @normalization.NFC,406 @normalization.NFD,407 @normalization.NFKC,408 @normalization.NFKD,409 ] {410 for ascii in 0..<=40 {411 let run = "a".repeat(ascii)412 // These prefixes decompose under no form, so prefix + run is unchanged.413 for prefix in ["", "世界", "😀"] {414 assert_true(@normalization.is_normalized(prefix + run, form))415 assert_eq(@normalization.normalize(prefix + run, form), prefix + run)416 }417 }418 }419 for ascii in 0..<=40 {420 let run = "a".repeat(ascii)421 // A change before the run, after it, and on both sides: the run itself is422 // copied as it is either way.423 assert_eq(@normalization.nfc("e\u{0301}" + run), "é" + run)424 assert_false(425 @normalization.is_normalized("e\u{0301}" + run, @normalization.NFC),426 )427 assert_eq(428 @normalization.nfc("世界" + run + "e\u{0301}"),429 "世界" + run + "é",430 )431 assert_eq(@normalization.nfkc("ffi" + run), "ffi" + run)432 assert_eq(@normalization.nfc(run + "e\u{0301}"), run + "é")433 assert_false(434 @normalization.is_normalized(run + "e\u{0301}", @normalization.NFC),435 )436 }437}438
///|
test "quick check handles Yes No and contextual Maybe" {
for
form in [
@normalization.NFC,
@normalization.NFD,
@normalization.NFKC,
@normalization.NFKD,
] {
assert_true(@normalization.is_normalized("", form))
assert_true(@normalization.is_normalized("Hello 世界 😀", form))
assert_true(@normalization.is_normalized("\u{0301}", form))
}
assert_true(@normalization.is_normalized("é", @normalization.NFC))
assert_false(@normalization.is_normalized("é", @normalization.NFD))
assert_false(@normalization.is_normalized("e\u{0301}", @normalization.NFC))
// Maybe does not imply that a composition is possible in this context.
assert_true(@normalization.is_normalized("x\u{0301}", @normalization.NFC))
assert_true(
@normalization.is_normalized("a\u{0305}\u{0301}", @normalization.NFC),
)
assert_false(
@normalization.is_normalized("x\u{0301}ffi", @normalization.NFKC),
)
}
///|
test "quick check verifies order and resets at starters" {
for
form in [
@normalization.NFC,
@normalization.NFD,
@normalization.NFKC,
@normalization.NFKD,
] {
// These non-composing marks individually have Quick_Check=Yes.
assert_false(@normalization.is_normalized("\u{0315}\u{0316}", form))
assert_eq(
@normalization.normalize("\u{0315}\u{0316}", form),
"\u{0316}\u{0315}",
)
assert_true(@normalization.is_normalized("\u{0315}x\u{0316}", form))
}
}
///|
test "spans retain precomposed starters and supplementary offsets" {
assert_eq(
@normalization.nfc("😀abcă\u{0328}世界😀e\u{0301}!"),
"😀abcą\u{0306}世界😀é!",
)
assert_eq(@normalization.nfd("😀é|Å😀"), "😀e\u{0301}|A\u{030A}😀")
assert_eq(@normalization.nfkc("😀A\u{030A}|ffi😀"), "😀Å|ffi😀")
assert_eq(
@normalization.nfkd("😀A\u{030A}|ffi😀"),
"😀A\u{030A}|ffi😀",
)
}
///|
test "CCC zero Maybe characters do not split Hangul composition" {
for form in [@normalization.NFC, @normalization.NFKC] {
assert_eq(
@normalization.normalize("😀각|각😀", form),
"😀각|각😀",
)
assert_true(@normalization.is_normalized("ᅡ", form))
assert_false(@normalization.is_normalized("가", form))
assert_eq(
@normalization.normalize("ᄀ\u{0301}ᅡ", form),
"ᄀ\u{0301}ᅡ",
)
}
}
///|
test "Unicode 16 composite Maybe characters keep their preceding context" {
// Kirat Rai: AA + AI must decompose AI before composing with AA.
for form in [@normalization.NFC, @normalization.NFKC] {
assert_true(@normalization.is_normalized("\u{16D68}", form))
assert_false(@normalization.is_normalized("\u{16D63}\u{16D68}", form))
assert_eq(
@normalization.normalize("😀\u{16D63}\u{16D68}|", form),
"😀\u{16D6A}|",
)
}
}
///|
test "local segments preserve short and long copied runs" {
for gap_length in [0, 1, 15, 16, 17, 40] {
let gap = "-".repeat(gap_length)
let prefix = "😀prefix".repeat(8)
let suffix = "suffix😀".repeat(8)
let decomposed = "\{prefix}e\u{0301}\{gap}a\u{030A}\{suffix}"
let composed = "\{prefix}é\{gap}å\{suffix}"
for form in [@normalization.NFC, @normalization.NFKC] {
assert_eq(@normalization.normalize(decomposed, form), composed)
}
for form in [@normalization.NFD, @normalization.NFKD] {
assert_eq(@normalization.normalize(composed, form), decomposed)
}
}
}
///|
test "local checking continues after unchanged Maybe segments" {
let prefix = "x\u{0301}😀".repeat(32)
for form in [@normalization.NFC, @normalization.NFKC] {
assert_true(@normalization.is_normalized(prefix, form))
for
(tail, expected) in [
("e\u{0301}", "é"),
("ă\u{0328}", "ą\u{0306}"),
("\u{0315}\u{0316}", "\u{0316}\u{0315}"),
("각", "각"),
("\u{16D63}\u{16D68}", "\u{16D6A}"),
] {
let input = prefix + tail
assert_false(@normalization.is_normalized(input, form))
assert_eq(@normalization.normalize(input, form), prefix + expected)
// Also exercise a changed segment followed by unchanged Maybe segments.
assert_eq(
@normalization.normalize(tail + prefix, form),
expected + prefix,
)
}
}
}
///|
test "scratch reuse after expansion composition and long combining runs" {
let marks = "\u{0315}".repeat(64) + "\u{0316}".repeat(64)
let ordered = "\u{0316}".repeat(64) + "\u{0315}".repeat(64)
let input = "ffi|\{marks}|e\u{0301}|x\u{0301}|😀A\u{030A}"
for
(form, expected) in [
(@normalization.NFC, "ffi|\{ordered}|é|x\u{0301}|😀A\u{030A}"),
(@normalization.NFKC, "ffi|\{ordered}|é|x\u{0301}|😀Å"),
(@normalization.NFD, "ffi|\{ordered}|e\u{0301}|x\u{0301}|😀A\u{030A}"),
(@normalization.NFKD, "ffi|\{ordered}|e\u{0301}|x\u{0301}|😀A\u{030A}"),
] {
assert_eq(@normalization.normalize(input, form), expected)
assert_false(@normalization.is_normalized(input, form))
assert_true(@normalization.is_normalized(expected, form))
}
}
///|
test "fast runs retain boundaries across ordered marks and surrogate pairs" {
for
prefix in [
"",
"ASCII",
"世界",
"😀",
"\u{10000}",
"\u{10FFFF}",
"世界😀".repeat(16),
] {
for
(tail, expected) in [
("A\u{0316}\u{0301}", "Á\u{0316}"),
("\u{0316}\u{0315}", "\u{0316}\u{0315}"),
("\u{0315}\u{0316}", "\u{0316}\u{0315}"),
("\u{0315}😀\u{0316}", "\u{0315}😀\u{0316}"),
("\u{0315}A\u{0301}", "\u{0315}Á"),
("\u{16D63}\u{16D68}", "\u{16D6A}"),
] {
let input = prefix + tail
let normalized = prefix + expected
assert_eq(@normalization.nfc(input), normalized)
assert_eq(
@normalization.is_normalized(input, @normalization.NFC),
input == normalized,
)
}
}
}
///|
test "resume fast runs between suspect segments and at the end" {
for
gap in [
"",
"A",
"AA",
"AAA",
"Hello world! ".repeat(256),
"世界😀".repeat(64),
] {
for
(form, input_part, normalized_part) in [
(@normalization.NFC, "e\u{0301}", "é"),
(@normalization.NFC, "x\u{0301}", "x\u{0301}"),
(@normalization.NFC, "각", "각"),
(@normalization.NFC, "\u{16D63}\u{16D68}", "\u{16D6A}"),
(@normalization.NFD, "é", "e\u{0301}"),
(@normalization.NFKC, "ffiA\u{030A}", "ffiÅ"),
(@normalization.NFKD, "ffiÅ", "ffiA\u{030A}"),
] {
for tail in ["", "\u{0315}\u{0316}", "😀\u{0315}\u{0316}"] {
let ordered_tail = if tail == "" {
""
} else if tail == "\u{0315}\u{0316}" {
"\u{0316}\u{0315}"
} else {
"😀\u{0316}\u{0315}"
}
// Separators prevent the expected pieces from interacting; the final
// gap + marks also checks recovery of the last skipped starter.
let input = input_part + "|" + gap + "|" + input_part + "|" + gap + tail
let expected = normalized_part +
"|" +
gap +
"|" +
normalized_part +
"|" +
gap +
ordered_tail
assert_eq(@normalization.normalize(input, form), expected)
assert_eq(@normalization.is_normalized(input, form), input == expected)
assert_true(@normalization.is_normalized(expected, form))
}
}
}
// After an unchanged Maybe segment, recover A as the next composing starter.
assert_eq(@normalization.nfc("x\u{0301}AAA\u{0301}"), "x\u{0301}AAÁ")
assert_false(
@normalization.is_normalized("x\u{0301}AAA\u{0301}", @normalization.NFC),
)
}
///|
test "surrogate summaries respect form masks and recover complete boundaries" {
for gap in ["😀", "\u{10FFFD}", "\u{10000}", "\u{1D400}"] {
for
(form, changed, fixed, normalized_gap) in [
(@normalization.NFC, "e\u{0301}", "é", gap),
(@normalization.NFD, "é", "e\u{0301}", gap),
(
@normalization.NFKC,
"e\u{0301}",
"é",
if gap == "\u{1D400}" {
"A"
} else {
gap
},
),
(
@normalization.NFKD,
"é",
"e\u{0301}",
if gap == "\u{1D400}" {
"A"
} else {
gap
},
),
] {
// No separator before the marks: recovery must retain the full pair,
// including when a changed prefix has already allocated scratch storage.
for prefix in ["", changed + "|"] {
let fixed_prefix = if prefix == "" { "" } else { fixed + "|" }
let input = prefix + gap.repeat(8) + "\u{0315}\u{0316}|" + changed
let expected = fixed_prefix +
normalized_gap.repeat(8) +
"\u{0316}\u{0315}|" +
fixed
assert_eq(@normalization.normalize(input, form), expected)
assert_false(@normalization.is_normalized(input, form))
assert_true(@normalization.is_normalized(expected, form))
}
}
}
// Active groups still decode mappings, combining marks and backward composition.
assert_eq(@normalization.nfd("\u{1D15E}"), "\u{1D157}\u{1D165}")
assert_eq(@normalization.nfc("\u{11099}\u{110BA}"), "\u{1109A}")
assert_eq(
@normalization.nfc("😀\u{1D165}\u{1D167}"),
"😀\u{1D167}\u{1D165}",
)
}
///|
test "surrogate summaries keep isolated code units unchanged" {
for cp in [0xD800, 0xD835, 0xDBFF, 0xDC00, 0xDFFF] {
let lone = String::from_array([cp.unsafe_to_char()])
for input in [lone, lone + "A", "A" + lone] {
for
form in [
@normalization.NFC,
@normalization.NFD,
@normalization.NFKC,
@normalization.NFKD,
] {
assert_eq(@normalization.normalize(input, form), input)
assert_true(@normalization.is_normalized(input, form))
}
}
}
}
///|
test "unpaired surrogates stay clean next to suspect characters" {
// U+D835 leads a group of 1024 code points that has flagged characters.
let lead = String::from_array([(0xD835).unsafe_to_char()])
let trail = String::from_array([(0xDC00).unsafe_to_char()])
for
(input, expected) in [
(lead + "e\u{0301}", lead + "é"),
("e\u{0301}" + lead, "é" + lead),
(trail + lead + "e\u{0301}", trail + lead + "é"),
(lead + "\u{0301}", lead + "\u{0301}"),
(lead + "\u{0315}\u{0316}", lead + "\u{0316}\u{0315}"),
// The second unit pairs with the trailing one; the first stays alone.
(
lead + lead + trail + "\u{0315}\u{0316}",
lead + lead + trail + "\u{0316}\u{0315}",
),
(
"e\u{0301}|" + lead + lead + trail + lead,
"é|" + lead + lead + trail + lead,
),
] {
assert_eq(@normalization.nfc(input), expected)
assert_eq(
@normalization.is_normalized(input, @normalization.NFC),
input == expected,
)
assert_true(@normalization.is_normalized(expected, @normalization.NFC))
}
}
///|
test "clean pairs in flagged groups become the safe boundary" {
// U+1D400 is clean for NFC and NFD, but shares its group with U+1D15E.
let pair = "\u{1D400}"
for prefix in ["", "世界", "e\u{0301}|"] {
let fixed = if prefix == "e\u{0301}|" { "é|" } else { prefix }
for count in [1, 2, 64] {
let run = pair.repeat(count)
// Marks attach to the last pair, so only they are reordered.
assert_eq(
@normalization.nfc(prefix + run + "\u{0315}\u{0316}"),
fixed + run + "\u{0316}\u{0315}",
)
// A starter after the run composes without disturbing it.
assert_eq(
@normalization.nfc(prefix + run + "e\u{0301}"),
fixed + run + "é",
)
assert_eq(
@normalization.is_normalized(prefix + run, @normalization.NFC),
prefix == fixed,
)
}
}
}
///|
test "block scans stop at every offset within and between blocks" {
// Each case is a character that only some forms leave alone, with its
// normalization in NFC, NFD, NFKC and NFKD.
for
(suspect, nfc, nfd, nfkc, nfkd) in [
("\u{A0}", "\u{A0}", "\u{A0}", " ", " "),
("é", "é", "e\u{0301}", "é", "e\u{0301}"),
("e\u{0301}", "é", "e\u{0301}", "é", "e\u{0301}"),
("😀", "😀", "😀", "😀", "😀"),
(
"\u{1D15E}", "\u{1D157}\u{1D165}", "\u{1D157}\u{1D165}", "\u{1D157}\u{1D165}",
"\u{1D157}\u{1D165}",
),
] {
for before in 0..<48 {
for after in [0, 1, 7, 8, 9, 17] {
let prefix = "a".repeat(before)
let suffix = "z".repeat(after)
let input = prefix + suspect + suffix
for
(form, expected) in [
(@normalization.NFC, nfc),
(@normalization.NFD, nfd),
(@normalization.NFKC, nfkc),
(@normalization.NFKD, nfkd),
] {
let normalized = prefix + expected + suffix
assert_eq(@normalization.normalize(input, form), normalized)
assert_eq(
@normalization.is_normalized(input, form),
input == normalized,
)
}
}
}
}
}
///|
test "long ASCII runs at every length" {
for
form in [
@normalization.NFC,
@normalization.NFD,
@normalization.NFKC,
@normalization.NFKD,
] {
for ascii in 0..<=40 {
let run = "a".repeat(ascii)
// These prefixes decompose under no form, so prefix + run is unchanged.
for prefix in ["", "世界", "😀"] {
assert_true(@normalization.is_normalized(prefix + run, form))
assert_eq(@normalization.normalize(prefix + run, form), prefix + run)
}
}
}
for ascii in 0..<=40 {
let run = "a".repeat(ascii)
// A change before the run, after it, and on both sides: the run itself is
// copied as it is either way.
assert_eq(@normalization.nfc("e\u{0301}" + run), "é" + run)
assert_false(
@normalization.is_normalized("e\u{0301}" + run, @normalization.NFC),
)
assert_eq(
@normalization.nfc("世界" + run + "e\u{0301}"),
"世界" + run + "é",
)
assert_eq(@normalization.nfkc("ffi" + run), "ffi" + run)
assert_eq(@normalization.nfc(run + "e\u{0301}"), run + "é")
assert_false(
@normalization.is_normalized(run + "e\u{0301}", @normalization.NFC),
)
}
}