normalization/fuzz_test.mbt1///|2/// Property-based fuzz tests for Unicode normalization3/// Tests invariants that should hold for any Unicode string45///|6/// Number of iterations for fuzz tests7let fuzz_iterations : Int32 = 100089///|10/// Seed for reproducibility (must be exactly 32 bytes for chacha8)11let fuzz_seed : Bytes = b"unicode-fuzz-seed-for-testing!32"1213// =============================================================================14// Property 1: Idempotence - applying normalization twice yields same result15// =============================================================================1617///|18test "fuzz: nfc idempotence" {19 let rng = @random.Rand::chacha8(seed=fuzz_seed)20 for _ in 0..<fuzz_iterations {21 let s = @fuzz.random_unicode_string(rng, max_len=50)22 let once = @normalization.nfc(s)23 let twice = @normalization.nfc(once)24 assert_eq(once, twice)25 }26}2728///|29test "fuzz: nfd idempotence" {30 let rng = @random.Rand::chacha8(seed=fuzz_seed)31 for _ in 0..<fuzz_iterations {32 let s = @fuzz.random_unicode_string(rng, max_len=50)33 let once = @normalization.nfd(s)34 let twice = @normalization.nfd(once)35 assert_eq(once, twice)36 }37}3839///|40test "fuzz: nfkc idempotence" {41 let rng = @random.Rand::chacha8(seed=fuzz_seed)42 for _ in 0..<fuzz_iterations {43 let s = @fuzz.random_unicode_string(rng, max_len=50)44 let once = @normalization.nfkc(s)45 let twice = @normalization.nfkc(once)46 assert_eq(once, twice)47 }48}4950///|51test "fuzz: nfkd idempotence" {52 let rng = @random.Rand::chacha8(seed=fuzz_seed)53 for _ in 0..<fuzz_iterations {54 let s = @fuzz.random_unicode_string(rng, max_len=50)55 let once = @normalization.nfkd(s)56 let twice = @normalization.nfkd(once)57 assert_eq(once, twice)58 }59}6061// =============================================================================62// Property 2: Canonical equivalence - nfc(nfd(s)) == nfc(s)63// =============================================================================6465///|66test "fuzz: canonical equivalence via nfd" {67 let rng = @random.Rand::chacha8(seed=fuzz_seed)68 for _ in 0..<fuzz_iterations {69 let s = @fuzz.random_unicode_string(rng, max_len=50)70 let via_nfd = @normalization.nfc(@normalization.nfd(s))71 let direct = @normalization.nfc(s)72 assert_eq(via_nfd, direct)73 }74}7576///|77test "fuzz: nfc then nfd equals nfd" {78 let rng = @random.Rand::chacha8(seed=fuzz_seed)79 for _ in 0..<fuzz_iterations {80 let s = @fuzz.random_unicode_string(rng, max_len=50)81 let nfc_then_nfd = @normalization.nfd(@normalization.nfc(s))82 let just_nfd = @normalization.nfd(s)83 assert_eq(nfc_then_nfd, just_nfd)84 }85}8687// =============================================================================88// Property 3: Compatibility implies canonical89// =============================================================================9091///|92test "fuzz: nfkc of nfc equals nfkc" {93 let rng = @random.Rand::chacha8(seed=fuzz_seed)94 for _ in 0..<fuzz_iterations {95 let s = @fuzz.random_unicode_string(rng, max_len=50)96 let nfkc_of_nfc = @normalization.nfkc(@normalization.nfc(s))97 let just_nfkc = @normalization.nfkc(s)98 assert_eq(nfkc_of_nfc, just_nfkc)99 }100}101102///|103test "fuzz: nfkd of nfd equals nfkd" {104 let rng = @random.Rand::chacha8(seed=fuzz_seed)105 for _ in 0..<fuzz_iterations {106 let s = @fuzz.random_unicode_string(rng, max_len=50)107 let nfkd_of_nfd = @normalization.nfkd(@normalization.nfd(s))108 let just_nfkd = @normalization.nfkd(s)109 assert_eq(nfkd_of_nfd, just_nfkd)110 }111}112113// =============================================================================114// Property 4: is_normalized consistency115// =============================================================================116117///|118test "fuzz: is_normalized NFC after nfc" {119 let rng = @random.Rand::chacha8(seed=fuzz_seed)120 for _ in 0..<fuzz_iterations {121 let s = @fuzz.random_unicode_string(rng, max_len=50)122 let normalized = @normalization.nfc(s)123 assert_true(@normalization.is_normalized(normalized, NFC))124 }125}126127///|128test "fuzz: is_normalized NFD after nfd" {129 let rng = @random.Rand::chacha8(seed=fuzz_seed)130 for _ in 0..<fuzz_iterations {131 let s = @fuzz.random_unicode_string(rng, max_len=50)132 let normalized = @normalization.nfd(s)133 assert_true(@normalization.is_normalized(normalized, NFD))134 }135}136137///|138test "fuzz: is_normalized NFKC after nfkc" {139 let rng = @random.Rand::chacha8(seed=fuzz_seed)140 for _ in 0..<fuzz_iterations {141 let s = @fuzz.random_unicode_string(rng, max_len=50)142 let normalized = @normalization.nfkc(s)143 assert_true(@normalization.is_normalized(normalized, NFKC))144 }145}146147///|148test "fuzz: is_normalized NFKD after nfkd" {149 let rng = @random.Rand::chacha8(seed=fuzz_seed)150 for _ in 0..<fuzz_iterations {151 let s = @fuzz.random_unicode_string(rng, max_len=50)152 let normalized = @normalization.nfkd(s)153 assert_true(@normalization.is_normalized(normalized, NFKD))154 }155}156157// =============================================================================158// Property 5: CCC ordering after decomposition159// =============================================================================160161///|162test "fuzz: combining marks sorted by CCC after nfd" {163 let rng = @random.Rand::chacha8(seed=fuzz_seed)164 for _ in 0..<500 {165 let s = @fuzz.random_combining_sequence(rng, base_count=3, mark_count=5)166 let nfd_result = @normalization.nfd(s)167 assert_true(@normalization_fuzz.ccc_is_sorted(nfd_result))168 }169}170171///|172test "fuzz: combining marks sorted by CCC after nfkd" {173 let rng = @random.Rand::chacha8(seed=fuzz_seed)174 for _ in 0..<500 {175 let s = @fuzz.random_combining_sequence(rng, base_count=3, mark_count=5)176 let nfkd_result = @normalization.nfkd(s)177 assert_true(@normalization_fuzz.ccc_is_sorted(nfkd_result))178 }179}180181// =============================================================================182// Targeted stress tests183// =============================================================================184185///|186test "fuzz: hangul roundtrip" {187 let rng = @random.Rand::chacha8(seed=fuzz_seed)188 for _ in 0..<500 {189 let s = @fuzz.random_hangul_string(rng, max_len=20)190 // NFD decomposes Hangul, NFC recomposes it191 let roundtrip = @normalization.nfc(@normalization.nfd(s))192 // The roundtrip should be equivalent (but may differ in jamo representation)193 let nfc_original = @normalization.nfc(s)194 assert_eq(roundtrip, nfc_original)195 }196}197198///|199test "fuzz: deep decomposition idempotence" {200 let rng = @random.Rand::chacha8(seed=fuzz_seed)201 for _ in 0..<500 {202 let s = @fuzz.random_deep_decomposition_string(rng, max_len=20)203 let once = @normalization.nfd(s)204 let twice = @normalization.nfd(once)205 assert_eq(once, twice)206 }207}208209///|210test "fuzz: mixed content stability" {211 let rng = @random.Rand::chacha8(seed=fuzz_seed)212 for _ in 0..<fuzz_iterations {213 let s = @fuzz.random_unicode_string(rng, max_len=100)214 // All four forms should be stable215 let nfc = @normalization.nfc(s)216 let nfd = @normalization.nfd(s)217 let nfkc = @normalization.nfkc(s)218 let nfkd = @normalization.nfkd(s)219 // Each should be idempotent220 assert_eq(@normalization.nfc(nfc), nfc)221 assert_eq(@normalization.nfd(nfd), nfd)222 assert_eq(@normalization.nfkc(nfkc), nfkc)223 assert_eq(@normalization.nfkd(nfkd), nfkd)224 }225}226227// =============================================================================228// Empty and boundary cases229// =============================================================================230231///|232test "fuzz: empty string" {233 let empty = ""234 assert_eq(@normalization.nfc(empty), "")235 assert_eq(@normalization.nfd(empty), "")236 assert_eq(@normalization.nfkc(empty), "")237 assert_eq(@normalization.nfkd(empty), "")238}239240///|241test "fuzz: single combining mark" {242 // A combining mark without a base should still work243 let combining_acute = "\u{0301}" // Combining acute accent244 let nfc = @normalization.nfc(combining_acute)245 let nfd = @normalization.nfd(combining_acute)246 // Should be stable247 assert_eq(@normalization.nfc(nfc), nfc)248 assert_eq(@normalization.nfd(nfd), nfd)249}250
///|
/// Property-based fuzz tests for Unicode normalization
/// Tests invariants that should hold for any Unicode string
///|
/// Number of iterations for fuzz tests
let fuzz_iterations : Int32 = 1000
///|
/// Seed for reproducibility (must be exactly 32 bytes for chacha8)
let fuzz_seed : Bytes = b"unicode-fuzz-seed-for-testing!32"
// =============================================================================
// Property 1: Idempotence - applying normalization twice yields same result
// =============================================================================
///|
test "fuzz: nfc idempotence" {
let rng = @random.Rand::chacha8(seed=fuzz_seed)
for _ in 0..<fuzz_iterations {
let s = @fuzz.random_unicode_string(rng, max_len=50)
let once = @normalization.nfc(s)
let twice = @normalization.nfc(once)
assert_eq(once, twice)
}
}
///|
test "fuzz: nfd idempotence" {
let rng = @random.Rand::chacha8(seed=fuzz_seed)
for _ in 0..<fuzz_iterations {
let s = @fuzz.random_unicode_string(rng, max_len=50)
let once = @normalization.nfd(s)
let twice = @normalization.nfd(once)
assert_eq(once, twice)
}
}
///|
test "fuzz: nfkc idempotence" {
let rng = @random.Rand::chacha8(seed=fuzz_seed)
for _ in 0..<fuzz_iterations {
let s = @fuzz.random_unicode_string(rng, max_len=50)
let once = @normalization.nfkc(s)
let twice = @normalization.nfkc(once)
assert_eq(once, twice)
}
}
///|
test "fuzz: nfkd idempotence" {
let rng = @random.Rand::chacha8(seed=fuzz_seed)
for _ in 0..<fuzz_iterations {
let s = @fuzz.random_unicode_string(rng, max_len=50)
let once = @normalization.nfkd(s)
let twice = @normalization.nfkd(once)
assert_eq(once, twice)
}
}
// =============================================================================
// Property 2: Canonical equivalence - nfc(nfd(s)) == nfc(s)
// =============================================================================
///|
test "fuzz: canonical equivalence via nfd" {
let rng = @random.Rand::chacha8(seed=fuzz_seed)
for _ in 0..<fuzz_iterations {
let s = @fuzz.random_unicode_string(rng, max_len=50)
let via_nfd = @normalization.nfc(@normalization.nfd(s))
let direct = @normalization.nfc(s)
assert_eq(via_nfd, direct)
}
}
///|
test "fuzz: nfc then nfd equals nfd" {
let rng = @random.Rand::chacha8(seed=fuzz_seed)
for _ in 0..<fuzz_iterations {
let s = @fuzz.random_unicode_string(rng, max_len=50)
let nfc_then_nfd = @normalization.nfd(@normalization.nfc(s))
let just_nfd = @normalization.nfd(s)
assert_eq(nfc_then_nfd, just_nfd)
}
}
// =============================================================================
// Property 3: Compatibility implies canonical
// =============================================================================
///|
test "fuzz: nfkc of nfc equals nfkc" {
let rng = @random.Rand::chacha8(seed=fuzz_seed)
for _ in 0..<fuzz_iterations {
let s = @fuzz.random_unicode_string(rng, max_len=50)
let nfkc_of_nfc = @normalization.nfkc(@normalization.nfc(s))
let just_nfkc = @normalization.nfkc(s)
assert_eq(nfkc_of_nfc, just_nfkc)
}
}
///|
test "fuzz: nfkd of nfd equals nfkd" {
let rng = @random.Rand::chacha8(seed=fuzz_seed)
for _ in 0..<fuzz_iterations {
let s = @fuzz.random_unicode_string(rng, max_len=50)
let nfkd_of_nfd = @normalization.nfkd(@normalization.nfd(s))
let just_nfkd = @normalization.nfkd(s)
assert_eq(nfkd_of_nfd, just_nfkd)
}
}
// =============================================================================
// Property 4: is_normalized consistency
// =============================================================================
///|
test "fuzz: is_normalized NFC after nfc" {
let rng = @random.Rand::chacha8(seed=fuzz_seed)
for _ in 0..<fuzz_iterations {
let s = @fuzz.random_unicode_string(rng, max_len=50)
let normalized = @normalization.nfc(s)
assert_true(@normalization.is_normalized(normalized, NFC))
}
}
///|
test "fuzz: is_normalized NFD after nfd" {
let rng = @random.Rand::chacha8(seed=fuzz_seed)
for _ in 0..<fuzz_iterations {
let s = @fuzz.random_unicode_string(rng, max_len=50)
let normalized = @normalization.nfd(s)
assert_true(@normalization.is_normalized(normalized, NFD))
}
}
///|
test "fuzz: is_normalized NFKC after nfkc" {
let rng = @random.Rand::chacha8(seed=fuzz_seed)
for _ in 0..<fuzz_iterations {
let s = @fuzz.random_unicode_string(rng, max_len=50)
let normalized = @normalization.nfkc(s)
assert_true(@normalization.is_normalized(normalized, NFKC))
}
}
///|
test "fuzz: is_normalized NFKD after nfkd" {
let rng = @random.Rand::chacha8(seed=fuzz_seed)
for _ in 0..<fuzz_iterations {
let s = @fuzz.random_unicode_string(rng, max_len=50)
let normalized = @normalization.nfkd(s)
assert_true(@normalization.is_normalized(normalized, NFKD))
}
}
// =============================================================================
// Property 5: CCC ordering after decomposition
// =============================================================================
///|
test "fuzz: combining marks sorted by CCC after nfd" {
let rng = @random.Rand::chacha8(seed=fuzz_seed)
for _ in 0..<500 {
let s = @fuzz.random_combining_sequence(rng, base_count=3, mark_count=5)
let nfd_result = @normalization.nfd(s)
assert_true(@normalization_fuzz.ccc_is_sorted(nfd_result))
}
}
///|
test "fuzz: combining marks sorted by CCC after nfkd" {
let rng = @random.Rand::chacha8(seed=fuzz_seed)
for _ in 0..<500 {
let s = @fuzz.random_combining_sequence(rng, base_count=3, mark_count=5)
let nfkd_result = @normalization.nfkd(s)
assert_true(@normalization_fuzz.ccc_is_sorted(nfkd_result))
}
}
// =============================================================================
// Targeted stress tests
// =============================================================================
///|
test "fuzz: hangul roundtrip" {
let rng = @random.Rand::chacha8(seed=fuzz_seed)
for _ in 0..<500 {
let s = @fuzz.random_hangul_string(rng, max_len=20)
// NFD decomposes Hangul, NFC recomposes it
let roundtrip = @normalization.nfc(@normalization.nfd(s))
// The roundtrip should be equivalent (but may differ in jamo representation)
let nfc_original = @normalization.nfc(s)
assert_eq(roundtrip, nfc_original)
}
}
///|
test "fuzz: deep decomposition idempotence" {
let rng = @random.Rand::chacha8(seed=fuzz_seed)
for _ in 0..<500 {
let s = @fuzz.random_deep_decomposition_string(rng, max_len=20)
let once = @normalization.nfd(s)
let twice = @normalization.nfd(once)
assert_eq(once, twice)
}
}
///|
test "fuzz: mixed content stability" {
let rng = @random.Rand::chacha8(seed=fuzz_seed)
for _ in 0..<fuzz_iterations {
let s = @fuzz.random_unicode_string(rng, max_len=100)
// All four forms should be stable
let nfc = @normalization.nfc(s)
let nfd = @normalization.nfd(s)
let nfkc = @normalization.nfkc(s)
let nfkd = @normalization.nfkd(s)
// Each should be idempotent
assert_eq(@normalization.nfc(nfc), nfc)
assert_eq(@normalization.nfd(nfd), nfd)
assert_eq(@normalization.nfkc(nfkc), nfkc)
assert_eq(@normalization.nfkd(nfkd), nfkd)
}
}
// =============================================================================
// Empty and boundary cases
// =============================================================================
///|
test "fuzz: empty string" {
let empty = ""
assert_eq(@normalization.nfc(empty), "")
assert_eq(@normalization.nfd(empty), "")
assert_eq(@normalization.nfkc(empty), "")
assert_eq(@normalization.nfkd(empty), "")
}
///|
test "fuzz: single combining mark" {
// A combining mark without a base should still work
let combining_acute = "\u{0301}" // Combining acute accent
let nfc = @normalization.nfc(combining_acute)
let nfd = @normalization.nfd(combining_acute)
// Should be stable
assert_eq(@normalization.nfc(nfc), nfc)
assert_eq(@normalization.nfd(nfd), nfd)
}