unicode

Branch: main

Switch branches or tags

Branch: main
Go to file t

413 files indexed

History
Code

idna/internal/base_fuzz/fuzz.mbt
1///|2/// Generic random Unicode string generator for fuzz testing3/// Generates strings with various Unicode categories45///|6/// Code point ranges for generating interesting test cases7let ascii_range : (Int32, Int32) = (0x20, 0x7E) // Printable ASCII89///|10let latin_extended_range : (Int32, Int32) = (0x00C0, 0x024F) // Many decomposable chars1112///|13let combining_marks_range : (Int32, Int32) = (0x0300, 0x036F) // Combining diacritical marks1415///|16let hangul_jamo_l_range : (Int32, Int32) = (0x1100, 0x1112) // Leading consonants1718///|19let hangul_jamo_v_range : (Int32, Int32) = (0x1161, 0x1175) // Vowels2021///|22let hangul_jamo_t_range : (Int32, Int32) = (0x11A8, 0x11C2) // Trailing consonants2324///|25let hangul_syllable_range : (Int32, Int32) = (0xAC00, 0xD7A3) // Precomposed syllables2627///|28let supplementary_range : (Int32, Int32) = (0x10000, 0x10FFFF) // Supplementary planes2930///|31let arabic_range : (Int32, Int32) = (0x0600, 0x06FF) // Arabic script (RTL)3233///|34let hebrew_range : (Int32, Int32) = (0x0590, 0x05FF) // Hebrew script (RTL)3536///|37let arabic_number_range : (Int32, Int32) = (0x0660, 0x0669) // Arabic-Indic digits3839///|40/// Characters that have interesting decompositions (singletons, multi-char)41let decomposable_chars : Array[Char] = [42  '\u{00C5}', // Å - decomposes to A + combining ring43   '\u{00D1}', // Ñ - decomposes to N + combining tilde44   '\u{00E9}', // é - decomposes to e + combining acute45   '\u{01FA}', // Ǻ - multi-level decomposition46   '\u{2126}', // Ω (Ohm) - singleton decomposition to Greek Omega47   '\u{212A}', // K (Kelvin) - singleton decomposition to K48   '\u{212B}', // Å (Angstrom) - singleton decomposition to Å then to A + ring49   '\u{1E69}', // ṩ - multi-char decomposition50   '\u{0132}', // IJ ligature (compatibility)51   '\u{00BC}', // ¼ (compatibility)52   '\u{2160}', // Ⅰ Roman numeral (compatibility)53   '\u{FF21}', // A Fullwidth A (compatibility)54]5556///|57/// Characters that are IGNORED in IDNA processing58let idna_ignored_chars : Array[Char] = [59  '\u{00AD}', // Soft hyphen60   '\u{034F}', // Combining grapheme joiner61   '\u{180B}', // Mongolian free variation selector one62   '\u{FE00}', // Variation selector-163]6465///|66/// Characters that are DISALLOWED in IDNA processing67let idna_disallowed_chars : Array[Char] = [68  '\u{0000}', // NULL69   '\u{0020}', // SPACE70   '\u{007F}', // DELETE71   '\u{00A0}', // NO-BREAK SPACE72   '\u{2028}', // LINE SEPARATOR73   '\u{2029}', // PARAGRAPH SEPARATOR74   '\u{FFFD}', // REPLACEMENT CHARACTER75]7677///|78/// Deviation characters - treated as valid in nontransitional mode79let idna_deviation_chars : Array[Char] = [80  '\u{00DF}', // ß (German sharp s)81   '\u{03C2}', // ς (Greek final sigma)82   '\u{200C}', // ZWNJ83   '\u{200D}', // ZWJ84]8586///|87/// Characters that are mapped to other characters in IDNA88let idna_mapped_chars : Array[Char] = [89  '\u{FF21}', // A Fullwidth A -> a90   '\u{FF22}', // B Fullwidth B -> b91   '\u{FF23}', // C Fullwidth C -> c92   '\u{0041}', // A -> a (case fold)93   '\u{0042}', // B -> b (case fold)94   '\u{0043}', // C -> c (case fold)95   '\u{2126}', // Ω (Ohm) -> ω96   '\u{212A}', // K (Kelvin) -> k97   '\u{212B}', // Å (Angstrom) -> å98]99100///|101/// Generate a random code point within a range (inclusive)102fn random_codepoint_in_range(103  rng : @random.Rand,104  range : (Int32, Int32),105) -> Char {106  let (lo, hi) = range107  (lo + rng.int(limit=hi - lo + 1)).unsafe_to_char()108}109110///|111/// Category of code points to generate112pub(all) enum CodePointCategory {113  Ascii114  LatinExtended115  CombiningMark116  HangulJamoL117  HangulJamoV118  HangulJamoT119  HangulSyllable120  Supplementary121  Decomposable122  ArabicRTL123  HebrewRTL124  ArabicNumber125  IdnaIgnored126  IdnaDisallowed127  IdnaDeviation128  IdnaMapped129}130131///|132/// Generate a random code point from a specific category133pub fn random_codepoint(134  rng : @random.Rand,135  category : CodePointCategory,136) -> Char {137  match category {138    Ascii => random_codepoint_in_range(rng, ascii_range)139    LatinExtended => random_codepoint_in_range(rng, latin_extended_range)140    CombiningMark => random_codepoint_in_range(rng, combining_marks_range)141    HangulJamoL => random_codepoint_in_range(rng, hangul_jamo_l_range)142    HangulJamoV => random_codepoint_in_range(rng, hangul_jamo_v_range)143    HangulJamoT => random_codepoint_in_range(rng, hangul_jamo_t_range)144    HangulSyllable => random_codepoint_in_range(rng, hangul_syllable_range)145    Supplementary => random_codepoint_in_range(rng, supplementary_range)146    Decomposable =>147      decomposable_chars[rng.int(limit=decomposable_chars.length())]148    ArabicRTL => random_codepoint_in_range(rng, arabic_range)149    HebrewRTL => random_codepoint_in_range(rng, hebrew_range)150    ArabicNumber => random_codepoint_in_range(rng, arabic_number_range)151    IdnaIgnored =>152      idna_ignored_chars[rng.int(limit=idna_ignored_chars.length())]153    IdnaDisallowed =>154      idna_disallowed_chars[rng.int(limit=idna_disallowed_chars.length())]155    IdnaDeviation =>156      idna_deviation_chars[rng.int(limit=idna_deviation_chars.length())]157    IdnaMapped => idna_mapped_chars[rng.int(limit=idna_mapped_chars.length())]158  }159}160161// Total valid Unicode code points (excluding surrogates U+D800-U+DFFF)162// U+0000 to U+10FFFF = 1114112 total, minus 2048 surrogates = 1112064 valid163164///|165let unicode_valid_count : Int32 = 0x110000 - 0x800 // 1112064166167// Non-ASCII valid code points (U+0080 to U+10FFFF, excluding surrogates)168// 1112064 total minus 128 ASCII = 1111936169170///|171let non_ascii_valid_count : Int32 = unicode_valid_count - 0x80 // 1111936172173///|174/// Map an index to a valid Unicode code point (skipping surrogates)175fn index_to_codepoint(index : Int32) -> Int32 {176  // If index < 0xD800, return as-is (before surrogate range)177  // Otherwise, add 0x800 to skip the surrogate range178  if index < 0xD800 {179    index180  } else {181    index + 0x800182  }183}184185///|186/// Generate a random non-ASCII code point (U+0080 to U+10FFFF, excluding surrogates)187pub fn random_non_ascii_codepoint(rng : @random.Rand) -> Char {188  // Generate index in range [0, non_ascii_valid_count)189  // Then offset by 0x80 to skip ASCII, and apply surrogate skip190  let index = rng.int(limit=non_ascii_valid_count)191  index_to_codepoint(index + 0x80).unsafe_to_char()192}193194///|195/// Generate a random Unicode code point (U+0000 to U+10FFFF, excluding surrogates)196pub fn random_any_codepoint(rng : @random.Rand) -> Char {197  // Generate index in range [0, unicode_valid_count)198  let index = rng.int(limit=unicode_valid_count)199  index_to_codepoint(index).unsafe_to_char()200}201202///|203/// Generate a random Unicode string with mixed categories204pub fn random_unicode_string(205  rng : @random.Rand,206  max_len~ : Int32 = 50,207) -> String {208  let len = rng.int(limit=max_len) + 1209  let sb = StringBuilder()210  let categories : Array[CodePointCategory] = [211    Ascii,212    Ascii,213    Ascii, // Weight ASCII more heavily214    LatinExtended,215    LatinExtended,216    CombiningMark,217    HangulSyllable,218    Decomposable,219  ]220  for _ in 0..<len {221    let cat = categories[rng.int(limit=categories.length())]222    let cp = random_codepoint(rng, cat)223    sb.write_char(cp)224  }225  sb.to_string()226}227228///|229/// Generate a string with a base character followed by combining marks230/// This tests CCC ordering specifically231pub fn random_combining_sequence(232  rng : @random.Rand,233  base_count~ : Int32 = 3,234  mark_count~ : Int32 = 5,235) -> String {236  let sb = StringBuilder()237  for _ in 0..<base_count {238    // Add a base character (Latin Extended has many decomposable base chars)239    let base = random_codepoint(rng, LatinExtended)240    sb.write_char(base)241    // Add random combining marks242    let marks = rng.int(limit=mark_count) + 1243    for _ in 0..<marks {244      let mark = random_codepoint(rng, CombiningMark)245      sb.write_char(mark)246    }247  }248  sb.to_string()249}250251///|252/// Generate a random Hangul string (jamo and syllables mixed)253pub fn random_hangul_string(254  rng : @random.Rand,255  max_len~ : Int32 = 20,256) -> String {257  let len = rng.int(limit=max_len) + 1258  let sb = StringBuilder()259  for _ in 0..<len {260    let choice = rng.int(limit=5)261    let cp = if choice < 2 {262      // Precomposed syllable (most common)263      random_codepoint(rng, HangulSyllable)264    } else if choice == 2 {265      random_codepoint(rng, HangulJamoL)266    } else if choice == 3 {267      random_codepoint(rng, HangulJamoV)268    } else {269      random_codepoint(rng, HangulJamoT)270    }271    sb.write_char(cp)272  }273  sb.to_string()274}275276///|277/// Generate a string that should stress deep decomposition278pub fn random_deep_decomposition_string(279  rng : @random.Rand,280  max_len~ : Int32 = 20,281) -> String {282  let len = rng.int(limit=max_len) + 1283  let sb = StringBuilder()284  for _ in 0..<len {285    let cp = random_codepoint(rng, Decomposable)286    sb.write_char(cp)287  }288  sb.to_string()289}290