unicode

Branch: main

Switch branches or tags

Branch: main
Go to file t

413 files indexed

History
Code

normalization/fuzz_test.mbt
1///|2/// Property-based fuzz tests for Unicode normalization3/// Tests invariants that should hold for any Unicode string45///|6/// Number of iterations for fuzz tests7let fuzz_iterations : Int32 = 100089///|10/// Seed for reproducibility (must be exactly 32 bytes for chacha8)11let fuzz_seed : Bytes = b"unicode-fuzz-seed-for-testing!32"1213// =============================================================================14// Property 1: Idempotence - applying normalization twice yields same result15// =============================================================================1617///|18test "fuzz: nfc idempotence" {19  let rng = @random.Rand::chacha8(seed=fuzz_seed)20  for _ in 0..<fuzz_iterations {21    let s = @fuzz.random_unicode_string(rng, max_len=50)22    let once = @normalization.nfc(s)23    let twice = @normalization.nfc(once)24    assert_eq(once, twice)25  }26}2728///|29test "fuzz: nfd idempotence" {30  let rng = @random.Rand::chacha8(seed=fuzz_seed)31  for _ in 0..<fuzz_iterations {32    let s = @fuzz.random_unicode_string(rng, max_len=50)33    let once = @normalization.nfd(s)34    let twice = @normalization.nfd(once)35    assert_eq(once, twice)36  }37}3839///|40test "fuzz: nfkc idempotence" {41  let rng = @random.Rand::chacha8(seed=fuzz_seed)42  for _ in 0..<fuzz_iterations {43    let s = @fuzz.random_unicode_string(rng, max_len=50)44    let once = @normalization.nfkc(s)45    let twice = @normalization.nfkc(once)46    assert_eq(once, twice)47  }48}4950///|51test "fuzz: nfkd idempotence" {52  let rng = @random.Rand::chacha8(seed=fuzz_seed)53  for _ in 0..<fuzz_iterations {54    let s = @fuzz.random_unicode_string(rng, max_len=50)55    let once = @normalization.nfkd(s)56    let twice = @normalization.nfkd(once)57    assert_eq(once, twice)58  }59}6061// =============================================================================62// Property 2: Canonical equivalence - nfc(nfd(s)) == nfc(s)63// =============================================================================6465///|66test "fuzz: canonical equivalence via nfd" {67  let rng = @random.Rand::chacha8(seed=fuzz_seed)68  for _ in 0..<fuzz_iterations {69    let s = @fuzz.random_unicode_string(rng, max_len=50)70    let via_nfd = @normalization.nfc(@normalization.nfd(s))71    let direct = @normalization.nfc(s)72    assert_eq(via_nfd, direct)73  }74}7576///|77test "fuzz: nfc then nfd equals nfd" {78  let rng = @random.Rand::chacha8(seed=fuzz_seed)79  for _ in 0..<fuzz_iterations {80    let s = @fuzz.random_unicode_string(rng, max_len=50)81    let nfc_then_nfd = @normalization.nfd(@normalization.nfc(s))82    let just_nfd = @normalization.nfd(s)83    assert_eq(nfc_then_nfd, just_nfd)84  }85}8687// =============================================================================88// Property 3: Compatibility implies canonical89// =============================================================================9091///|92test "fuzz: nfkc of nfc equals nfkc" {93  let rng = @random.Rand::chacha8(seed=fuzz_seed)94  for _ in 0..<fuzz_iterations {95    let s = @fuzz.random_unicode_string(rng, max_len=50)96    let nfkc_of_nfc = @normalization.nfkc(@normalization.nfc(s))97    let just_nfkc = @normalization.nfkc(s)98    assert_eq(nfkc_of_nfc, just_nfkc)99  }100}101102///|103test "fuzz: nfkd of nfd equals nfkd" {104  let rng = @random.Rand::chacha8(seed=fuzz_seed)105  for _ in 0..<fuzz_iterations {106    let s = @fuzz.random_unicode_string(rng, max_len=50)107    let nfkd_of_nfd = @normalization.nfkd(@normalization.nfd(s))108    let just_nfkd = @normalization.nfkd(s)109    assert_eq(nfkd_of_nfd, just_nfkd)110  }111}112113// =============================================================================114// Property 4: is_normalized consistency115// =============================================================================116117///|118test "fuzz: is_normalized NFC after nfc" {119  let rng = @random.Rand::chacha8(seed=fuzz_seed)120  for _ in 0..<fuzz_iterations {121    let s = @fuzz.random_unicode_string(rng, max_len=50)122    let normalized = @normalization.nfc(s)123    assert_true(@normalization.is_normalized(normalized, NFC))124  }125}126127///|128test "fuzz: is_normalized NFD after nfd" {129  let rng = @random.Rand::chacha8(seed=fuzz_seed)130  for _ in 0..<fuzz_iterations {131    let s = @fuzz.random_unicode_string(rng, max_len=50)132    let normalized = @normalization.nfd(s)133    assert_true(@normalization.is_normalized(normalized, NFD))134  }135}136137///|138test "fuzz: is_normalized NFKC after nfkc" {139  let rng = @random.Rand::chacha8(seed=fuzz_seed)140  for _ in 0..<fuzz_iterations {141    let s = @fuzz.random_unicode_string(rng, max_len=50)142    let normalized = @normalization.nfkc(s)143    assert_true(@normalization.is_normalized(normalized, NFKC))144  }145}146147///|148test "fuzz: is_normalized NFKD after nfkd" {149  let rng = @random.Rand::chacha8(seed=fuzz_seed)150  for _ in 0..<fuzz_iterations {151    let s = @fuzz.random_unicode_string(rng, max_len=50)152    let normalized = @normalization.nfkd(s)153    assert_true(@normalization.is_normalized(normalized, NFKD))154  }155}156157// =============================================================================158// Property 5: CCC ordering after decomposition159// =============================================================================160161///|162test "fuzz: combining marks sorted by CCC after nfd" {163  let rng = @random.Rand::chacha8(seed=fuzz_seed)164  for _ in 0..<500 {165    let s = @fuzz.random_combining_sequence(rng, base_count=3, mark_count=5)166    let nfd_result = @normalization.nfd(s)167    assert_true(@normalization_fuzz.ccc_is_sorted(nfd_result))168  }169}170171///|172test "fuzz: combining marks sorted by CCC after nfkd" {173  let rng = @random.Rand::chacha8(seed=fuzz_seed)174  for _ in 0..<500 {175    let s = @fuzz.random_combining_sequence(rng, base_count=3, mark_count=5)176    let nfkd_result = @normalization.nfkd(s)177    assert_true(@normalization_fuzz.ccc_is_sorted(nfkd_result))178  }179}180181// =============================================================================182// Targeted stress tests183// =============================================================================184185///|186test "fuzz: hangul roundtrip" {187  let rng = @random.Rand::chacha8(seed=fuzz_seed)188  for _ in 0..<500 {189    let s = @fuzz.random_hangul_string(rng, max_len=20)190    // NFD decomposes Hangul, NFC recomposes it191    let roundtrip = @normalization.nfc(@normalization.nfd(s))192    // The roundtrip should be equivalent (but may differ in jamo representation)193    let nfc_original = @normalization.nfc(s)194    assert_eq(roundtrip, nfc_original)195  }196}197198///|199test "fuzz: deep decomposition idempotence" {200  let rng = @random.Rand::chacha8(seed=fuzz_seed)201  for _ in 0..<500 {202    let s = @fuzz.random_deep_decomposition_string(rng, max_len=20)203    let once = @normalization.nfd(s)204    let twice = @normalization.nfd(once)205    assert_eq(once, twice)206  }207}208209///|210test "fuzz: mixed content stability" {211  let rng = @random.Rand::chacha8(seed=fuzz_seed)212  for _ in 0..<fuzz_iterations {213    let s = @fuzz.random_unicode_string(rng, max_len=100)214    // All four forms should be stable215    let nfc = @normalization.nfc(s)216    let nfd = @normalization.nfd(s)217    let nfkc = @normalization.nfkc(s)218    let nfkd = @normalization.nfkd(s)219    // Each should be idempotent220    assert_eq(@normalization.nfc(nfc), nfc)221    assert_eq(@normalization.nfd(nfd), nfd)222    assert_eq(@normalization.nfkc(nfkc), nfkc)223    assert_eq(@normalization.nfkd(nfkd), nfkd)224  }225}226227// =============================================================================228// Empty and boundary cases229// =============================================================================230231///|232test "fuzz: empty string" {233  let empty = ""234  assert_eq(@normalization.nfc(empty), "")235  assert_eq(@normalization.nfd(empty), "")236  assert_eq(@normalization.nfkc(empty), "")237  assert_eq(@normalization.nfkd(empty), "")238}239240///|241test "fuzz: single combining mark" {242  // A combining mark without a base should still work243  let combining_acute = "\u{0301}" // Combining acute accent244  let nfc = @normalization.nfc(combining_acute)245  let nfd = @normalization.nfd(combining_acute)246  // Should be stable247  assert_eq(@normalization.nfc(nfc), nfc)248  assert_eq(@normalization.nfd(nfd), nfd)249}250