idna/internal/base_fuzz/fuzz.mbt1///|2/// Generic random Unicode string generator for fuzz testing3/// Generates strings with various Unicode categories45///|6/// Code point ranges for generating interesting test cases7let ascii_range : (Int32, Int32) = (0x20, 0x7E) // Printable ASCII89///|10let latin_extended_range : (Int32, Int32) = (0x00C0, 0x024F) // Many decomposable chars1112///|13let combining_marks_range : (Int32, Int32) = (0x0300, 0x036F) // Combining diacritical marks1415///|16let hangul_jamo_l_range : (Int32, Int32) = (0x1100, 0x1112) // Leading consonants1718///|19let hangul_jamo_v_range : (Int32, Int32) = (0x1161, 0x1175) // Vowels2021///|22let hangul_jamo_t_range : (Int32, Int32) = (0x11A8, 0x11C2) // Trailing consonants2324///|25let hangul_syllable_range : (Int32, Int32) = (0xAC00, 0xD7A3) // Precomposed syllables2627///|28let supplementary_range : (Int32, Int32) = (0x10000, 0x10FFFF) // Supplementary planes2930///|31let arabic_range : (Int32, Int32) = (0x0600, 0x06FF) // Arabic script (RTL)3233///|34let hebrew_range : (Int32, Int32) = (0x0590, 0x05FF) // Hebrew script (RTL)3536///|37let arabic_number_range : (Int32, Int32) = (0x0660, 0x0669) // Arabic-Indic digits3839///|40/// Characters that have interesting decompositions (singletons, multi-char)41let decomposable_chars : Array[Char] = [42 '\u{00C5}', // Å - decomposes to A + combining ring43 '\u{00D1}', // Ñ - decomposes to N + combining tilde44 '\u{00E9}', // é - decomposes to e + combining acute45 '\u{01FA}', // Ǻ - multi-level decomposition46 '\u{2126}', // Ω (Ohm) - singleton decomposition to Greek Omega47 '\u{212A}', // K (Kelvin) - singleton decomposition to K48 '\u{212B}', // Å (Angstrom) - singleton decomposition to Å then to A + ring49 '\u{1E69}', // ṩ - multi-char decomposition50 '\u{0132}', // IJ ligature (compatibility)51 '\u{00BC}', // ¼ (compatibility)52 '\u{2160}', // Ⅰ Roman numeral (compatibility)53 '\u{FF21}', // A Fullwidth A (compatibility)54]5556///|57/// Characters that are IGNORED in IDNA processing58let idna_ignored_chars : Array[Char] = [59 '\u{00AD}', // Soft hyphen60 '\u{034F}', // Combining grapheme joiner61 '\u{180B}', // Mongolian free variation selector one62 '\u{FE00}', // Variation selector-163]6465///|66/// Characters that are DISALLOWED in IDNA processing67let idna_disallowed_chars : Array[Char] = [68 '\u{0000}', // NULL69 '\u{0020}', // SPACE70 '\u{007F}', // DELETE71 '\u{00A0}', // NO-BREAK SPACE72 '\u{2028}', // LINE SEPARATOR73 '\u{2029}', // PARAGRAPH SEPARATOR74 '\u{FFFD}', // REPLACEMENT CHARACTER75]7677///|78/// Deviation characters - treated as valid in nontransitional mode79let idna_deviation_chars : Array[Char] = [80 '\u{00DF}', // ß (German sharp s)81 '\u{03C2}', // ς (Greek final sigma)82 '\u{200C}', // ZWNJ83 '\u{200D}', // ZWJ84]8586///|87/// Characters that are mapped to other characters in IDNA88let idna_mapped_chars : Array[Char] = [89 '\u{FF21}', // A Fullwidth A -> a90 '\u{FF22}', // B Fullwidth B -> b91 '\u{FF23}', // C Fullwidth C -> c92 '\u{0041}', // A -> a (case fold)93 '\u{0042}', // B -> b (case fold)94 '\u{0043}', // C -> c (case fold)95 '\u{2126}', // Ω (Ohm) -> ω96 '\u{212A}', // K (Kelvin) -> k97 '\u{212B}', // Å (Angstrom) -> å98]99100///|101/// Generate a random code point within a range (inclusive)102fn random_codepoint_in_range(103 rng : @random.Rand,104 range : (Int32, Int32),105) -> Char {106 let (lo, hi) = range107 (lo + rng.int(limit=hi - lo + 1)).unsafe_to_char()108}109110///|111/// Category of code points to generate112pub(all) enum CodePointCategory {113 Ascii114 LatinExtended115 CombiningMark116 HangulJamoL117 HangulJamoV118 HangulJamoT119 HangulSyllable120 Supplementary121 Decomposable122 ArabicRTL123 HebrewRTL124 ArabicNumber125 IdnaIgnored126 IdnaDisallowed127 IdnaDeviation128 IdnaMapped129}130131///|132/// Generate a random code point from a specific category133pub fn random_codepoint(134 rng : @random.Rand,135 category : CodePointCategory,136) -> Char {137 match category {138 Ascii => random_codepoint_in_range(rng, ascii_range)139 LatinExtended => random_codepoint_in_range(rng, latin_extended_range)140 CombiningMark => random_codepoint_in_range(rng, combining_marks_range)141 HangulJamoL => random_codepoint_in_range(rng, hangul_jamo_l_range)142 HangulJamoV => random_codepoint_in_range(rng, hangul_jamo_v_range)143 HangulJamoT => random_codepoint_in_range(rng, hangul_jamo_t_range)144 HangulSyllable => random_codepoint_in_range(rng, hangul_syllable_range)145 Supplementary => random_codepoint_in_range(rng, supplementary_range)146 Decomposable =>147 decomposable_chars[rng.int(limit=decomposable_chars.length())]148 ArabicRTL => random_codepoint_in_range(rng, arabic_range)149 HebrewRTL => random_codepoint_in_range(rng, hebrew_range)150 ArabicNumber => random_codepoint_in_range(rng, arabic_number_range)151 IdnaIgnored =>152 idna_ignored_chars[rng.int(limit=idna_ignored_chars.length())]153 IdnaDisallowed =>154 idna_disallowed_chars[rng.int(limit=idna_disallowed_chars.length())]155 IdnaDeviation =>156 idna_deviation_chars[rng.int(limit=idna_deviation_chars.length())]157 IdnaMapped => idna_mapped_chars[rng.int(limit=idna_mapped_chars.length())]158 }159}160161// Total valid Unicode code points (excluding surrogates U+D800-U+DFFF)162// U+0000 to U+10FFFF = 1114112 total, minus 2048 surrogates = 1112064 valid163164///|165let unicode_valid_count : Int32 = 0x110000 - 0x800 // 1112064166167// Non-ASCII valid code points (U+0080 to U+10FFFF, excluding surrogates)168// 1112064 total minus 128 ASCII = 1111936169170///|171let non_ascii_valid_count : Int32 = unicode_valid_count - 0x80 // 1111936172173///|174/// Map an index to a valid Unicode code point (skipping surrogates)175fn index_to_codepoint(index : Int32) -> Int32 {176 // If index < 0xD800, return as-is (before surrogate range)177 // Otherwise, add 0x800 to skip the surrogate range178 if index < 0xD800 {179 index180 } else {181 index + 0x800182 }183}184185///|186/// Generate a random non-ASCII code point (U+0080 to U+10FFFF, excluding surrogates)187pub fn random_non_ascii_codepoint(rng : @random.Rand) -> Char {188 // Generate index in range [0, non_ascii_valid_count)189 // Then offset by 0x80 to skip ASCII, and apply surrogate skip190 let index = rng.int(limit=non_ascii_valid_count)191 index_to_codepoint(index + 0x80).unsafe_to_char()192}193194///|195/// Generate a random Unicode code point (U+0000 to U+10FFFF, excluding surrogates)196pub fn random_any_codepoint(rng : @random.Rand) -> Char {197 // Generate index in range [0, unicode_valid_count)198 let index = rng.int(limit=unicode_valid_count)199 index_to_codepoint(index).unsafe_to_char()200}201202///|203/// Generate a random Unicode string with mixed categories204pub fn random_unicode_string(205 rng : @random.Rand,206 max_len~ : Int32 = 50,207) -> String {208 let len = rng.int(limit=max_len) + 1209 let sb = StringBuilder()210 let categories : Array[CodePointCategory] = [211 Ascii,212 Ascii,213 Ascii, // Weight ASCII more heavily214 LatinExtended,215 LatinExtended,216 CombiningMark,217 HangulSyllable,218 Decomposable,219 ]220 for _ in 0..<len {221 let cat = categories[rng.int(limit=categories.length())]222 let cp = random_codepoint(rng, cat)223 sb.write_char(cp)224 }225 sb.to_string()226}227228///|229/// Generate a string with a base character followed by combining marks230/// This tests CCC ordering specifically231pub fn random_combining_sequence(232 rng : @random.Rand,233 base_count~ : Int32 = 3,234 mark_count~ : Int32 = 5,235) -> String {236 let sb = StringBuilder()237 for _ in 0..<base_count {238 // Add a base character (Latin Extended has many decomposable base chars)239 let base = random_codepoint(rng, LatinExtended)240 sb.write_char(base)241 // Add random combining marks242 let marks = rng.int(limit=mark_count) + 1243 for _ in 0..<marks {244 let mark = random_codepoint(rng, CombiningMark)245 sb.write_char(mark)246 }247 }248 sb.to_string()249}250251///|252/// Generate a random Hangul string (jamo and syllables mixed)253pub fn random_hangul_string(254 rng : @random.Rand,255 max_len~ : Int32 = 20,256) -> String {257 let len = rng.int(limit=max_len) + 1258 let sb = StringBuilder()259 for _ in 0..<len {260 let choice = rng.int(limit=5)261 let cp = if choice < 2 {262 // Precomposed syllable (most common)263 random_codepoint(rng, HangulSyllable)264 } else if choice == 2 {265 random_codepoint(rng, HangulJamoL)266 } else if choice == 3 {267 random_codepoint(rng, HangulJamoV)268 } else {269 random_codepoint(rng, HangulJamoT)270 }271 sb.write_char(cp)272 }273 sb.to_string()274}275276///|277/// Generate a string that should stress deep decomposition278pub fn random_deep_decomposition_string(279 rng : @random.Rand,280 max_len~ : Int32 = 20,281) -> String {282 let len = rng.int(limit=max_len) + 1283 let sb = StringBuilder()284 for _ in 0..<len {285 let cp = random_codepoint(rng, Decomposable)286 sb.write_char(cp)287 }288 sb.to_string()289}290
///|
/// Generic random Unicode string generator for fuzz testing
/// Generates strings with various Unicode categories
///|
/// Code point ranges for generating interesting test cases
let ascii_range : (Int32, Int32) = (0x20, 0x7E) // Printable ASCII
///|
let latin_extended_range : (Int32, Int32) = (0x00C0, 0x024F) // Many decomposable chars
///|
let combining_marks_range : (Int32, Int32) = (0x0300, 0x036F) // Combining diacritical marks
///|
let hangul_jamo_l_range : (Int32, Int32) = (0x1100, 0x1112) // Leading consonants
///|
let hangul_jamo_v_range : (Int32, Int32) = (0x1161, 0x1175) // Vowels
///|
let hangul_jamo_t_range : (Int32, Int32) = (0x11A8, 0x11C2) // Trailing consonants
///|
let hangul_syllable_range : (Int32, Int32) = (0xAC00, 0xD7A3) // Precomposed syllables
///|
let supplementary_range : (Int32, Int32) = (0x10000, 0x10FFFF) // Supplementary planes
///|
let arabic_range : (Int32, Int32) = (0x0600, 0x06FF) // Arabic script (RTL)
///|
let hebrew_range : (Int32, Int32) = (0x0590, 0x05FF) // Hebrew script (RTL)
///|
let arabic_number_range : (Int32, Int32) = (0x0660, 0x0669) // Arabic-Indic digits
///|
/// Characters that have interesting decompositions (singletons, multi-char)
let decomposable_chars : Array[Char] = [
'\u{00C5}', // Å - decomposes to A + combining ring
'\u{00D1}', // Ñ - decomposes to N + combining tilde
'\u{00E9}', // é - decomposes to e + combining acute
'\u{01FA}', // Ǻ - multi-level decomposition
'\u{2126}', // Ω (Ohm) - singleton decomposition to Greek Omega
'\u{212A}', // K (Kelvin) - singleton decomposition to K
'\u{212B}', // Å (Angstrom) - singleton decomposition to Å then to A + ring
'\u{1E69}', // ṩ - multi-char decomposition
'\u{0132}', // IJ ligature (compatibility)
'\u{00BC}', // ¼ (compatibility)
'\u{2160}', // Ⅰ Roman numeral (compatibility)
'\u{FF21}', // A Fullwidth A (compatibility)
]
///|
/// Characters that are IGNORED in IDNA processing
let idna_ignored_chars : Array[Char] = [
'\u{00AD}', // Soft hyphen
'\u{034F}', // Combining grapheme joiner
'\u{180B}', // Mongolian free variation selector one
'\u{FE00}', // Variation selector-1
]
///|
/// Characters that are DISALLOWED in IDNA processing
let idna_disallowed_chars : Array[Char] = [
'\u{0000}', // NULL
'\u{0020}', // SPACE
'\u{007F}', // DELETE
'\u{00A0}', // NO-BREAK SPACE
'\u{2028}', // LINE SEPARATOR
'\u{2029}', // PARAGRAPH SEPARATOR
'\u{FFFD}', // REPLACEMENT CHARACTER
]
///|
/// Deviation characters - treated as valid in nontransitional mode
let idna_deviation_chars : Array[Char] = [
'\u{00DF}', // ß (German sharp s)
'\u{03C2}', // ς (Greek final sigma)
'\u{200C}', // ZWNJ
'\u{200D}', // ZWJ
]
///|
/// Characters that are mapped to other characters in IDNA
let idna_mapped_chars : Array[Char] = [
'\u{FF21}', // A Fullwidth A -> a
'\u{FF22}', // B Fullwidth B -> b
'\u{FF23}', // C Fullwidth C -> c
'\u{0041}', // A -> a (case fold)
'\u{0042}', // B -> b (case fold)
'\u{0043}', // C -> c (case fold)
'\u{2126}', // Ω (Ohm) -> ω
'\u{212A}', // K (Kelvin) -> k
'\u{212B}', // Å (Angstrom) -> å
]
///|
/// Generate a random code point within a range (inclusive)
fn random_codepoint_in_range(
rng : @random.Rand,
range : (Int32, Int32),
) -> Char {
let (lo, hi) = range
(lo + rng.int(limit=hi - lo + 1)).unsafe_to_char()
}
///|
/// Category of code points to generate
pub(all) enum CodePointCategory {
Ascii
LatinExtended
CombiningMark
HangulJamoL
HangulJamoV
HangulJamoT
HangulSyllable
Supplementary
Decomposable
ArabicRTL
HebrewRTL
ArabicNumber
IdnaIgnored
IdnaDisallowed
IdnaDeviation
IdnaMapped
}
///|
/// Generate a random code point from a specific category
pub fn random_codepoint(
rng : @random.Rand,
category : CodePointCategory,
) -> Char {
match category {
Ascii => random_codepoint_in_range(rng, ascii_range)
LatinExtended => random_codepoint_in_range(rng, latin_extended_range)
CombiningMark => random_codepoint_in_range(rng, combining_marks_range)
HangulJamoL => random_codepoint_in_range(rng, hangul_jamo_l_range)
HangulJamoV => random_codepoint_in_range(rng, hangul_jamo_v_range)
HangulJamoT => random_codepoint_in_range(rng, hangul_jamo_t_range)
HangulSyllable => random_codepoint_in_range(rng, hangul_syllable_range)
Supplementary => random_codepoint_in_range(rng, supplementary_range)
Decomposable =>
decomposable_chars[rng.int(limit=decomposable_chars.length())]
ArabicRTL => random_codepoint_in_range(rng, arabic_range)
HebrewRTL => random_codepoint_in_range(rng, hebrew_range)
ArabicNumber => random_codepoint_in_range(rng, arabic_number_range)
IdnaIgnored =>
idna_ignored_chars[rng.int(limit=idna_ignored_chars.length())]
IdnaDisallowed =>
idna_disallowed_chars[rng.int(limit=idna_disallowed_chars.length())]
IdnaDeviation =>
idna_deviation_chars[rng.int(limit=idna_deviation_chars.length())]
IdnaMapped => idna_mapped_chars[rng.int(limit=idna_mapped_chars.length())]
}
}
// Total valid Unicode code points (excluding surrogates U+D800-U+DFFF)
// U+0000 to U+10FFFF = 1114112 total, minus 2048 surrogates = 1112064 valid
///|
let unicode_valid_count : Int32 = 0x110000 - 0x800 // 1112064
// Non-ASCII valid code points (U+0080 to U+10FFFF, excluding surrogates)
// 1112064 total minus 128 ASCII = 1111936
///|
let non_ascii_valid_count : Int32 = unicode_valid_count - 0x80 // 1111936
///|
/// Map an index to a valid Unicode code point (skipping surrogates)
fn index_to_codepoint(index : Int32) -> Int32 {
// If index < 0xD800, return as-is (before surrogate range)
// Otherwise, add 0x800 to skip the surrogate range
if index < 0xD800 {
index
} else {
index + 0x800
}
}
///|
/// Generate a random non-ASCII code point (U+0080 to U+10FFFF, excluding surrogates)
pub fn random_non_ascii_codepoint(rng : @random.Rand) -> Char {
// Generate index in range [0, non_ascii_valid_count)
// Then offset by 0x80 to skip ASCII, and apply surrogate skip
let index = rng.int(limit=non_ascii_valid_count)
index_to_codepoint(index + 0x80).unsafe_to_char()
}
///|
/// Generate a random Unicode code point (U+0000 to U+10FFFF, excluding surrogates)
pub fn random_any_codepoint(rng : @random.Rand) -> Char {
// Generate index in range [0, unicode_valid_count)
let index = rng.int(limit=unicode_valid_count)
index_to_codepoint(index).unsafe_to_char()
}
///|
/// Generate a random Unicode string with mixed categories
pub fn random_unicode_string(
rng : @random.Rand,
max_len~ : Int32 = 50,
) -> String {
let len = rng.int(limit=max_len) + 1
let sb = StringBuilder()
let categories : Array[CodePointCategory] = [
Ascii,
Ascii,
Ascii, // Weight ASCII more heavily
LatinExtended,
LatinExtended,
CombiningMark,
HangulSyllable,
Decomposable,
]
for _ in 0..<len {
let cat = categories[rng.int(limit=categories.length())]
let cp = random_codepoint(rng, cat)
sb.write_char(cp)
}
sb.to_string()
}
///|
/// Generate a string with a base character followed by combining marks
/// This tests CCC ordering specifically
pub fn random_combining_sequence(
rng : @random.Rand,
base_count~ : Int32 = 3,
mark_count~ : Int32 = 5,
) -> String {
let sb = StringBuilder()
for _ in 0..<base_count {
// Add a base character (Latin Extended has many decomposable base chars)
let base = random_codepoint(rng, LatinExtended)
sb.write_char(base)
// Add random combining marks
let marks = rng.int(limit=mark_count) + 1
for _ in 0..<marks {
let mark = random_codepoint(rng, CombiningMark)
sb.write_char(mark)
}
}
sb.to_string()
}
///|
/// Generate a random Hangul string (jamo and syllables mixed)
pub fn random_hangul_string(
rng : @random.Rand,
max_len~ : Int32 = 20,
) -> String {
let len = rng.int(limit=max_len) + 1
let sb = StringBuilder()
for _ in 0..<len {
let choice = rng.int(limit=5)
let cp = if choice < 2 {
// Precomposed syllable (most common)
random_codepoint(rng, HangulSyllable)
} else if choice == 2 {
random_codepoint(rng, HangulJamoL)
} else if choice == 3 {
random_codepoint(rng, HangulJamoV)
} else {
random_codepoint(rng, HangulJamoT)
}
sb.write_char(cp)
}
sb.to_string()
}
///|
/// Generate a string that should stress deep decomposition
pub fn random_deep_decomposition_string(
rng : @random.Rand,
max_len~ : Int32 = 20,
) -> String {
let len = rng.int(limit=max_len) + 1
let sb = StringBuilder()
for _ in 0..<len {
let cp = random_codepoint(rng, Decomposable)
sb.write_char(cp)
}
sb.to_string()
}