bench/normperf/corpus_test.mbt1///|2#derive(@json.FromJson)3priv struct CorpusFile {4 name : String5 bytes : Int326 sha256 : String7 encoding : String8}910///|11#derive(@json.FromJson)12priv struct Corpus {13 files : Array[CorpusFile]14}1516///|17priv struct Fixture {18 name : String19 original : String20 nfc : String21 nfd : String22}2324///|25fn corpus_directory() -> String raise {26 for path in [".", "normperf", "bench/normperf"] {27 if @fs.path_exists("\{path}/corpus.json") {28 return path29 }30 }31 fail("Cannot find corpus.json; run moon bench ./bench/normperf")32}3334///|35/// Read and validate original bytes before decoding; preserve all line endings.36/// NFC/NFD variants are benchmark inputs, not independent correctness oracles.37fn fixtures() -> Array[Fixture] raise {38 let directory = corpus_directory()39 let corpus : Corpus = @json.from_json(40 @json.parse(@fs.read_file_to_string("\{directory}/corpus.json")),41 )42 corpus.files.map(file => {43 let raw = @fs.read_file_to_bytes("\{directory}/data/\{file.name}.txt")44 guard raw.length() == file.bytes &&45 @crypto.bytes_to_hex_string(@crypto.sha256(raw)) == file.sha256 else {46 fail("Corpus checksum mismatch: \{file.name}")47 }48 guard file.encoding == "utf-8-sig" else {49 fail("Unsupported corpus encoding: \{file.encoding}")50 }51 let original = @utf8.decode(raw, ignore_bom=true)52 {53 name: file.name,54 original,55 nfc: @normalization.normalize(original, NFC),56 nfd: @normalization.normalize(original, NFD),57 }58 })59}6061///|62/// Exercise file IO and the complete input matrix without running timers.63test "corpus integrity and normalization consistency" {64 let inputs = fixtures()65 inspect(inputs.length(), content="14")66 for fixture in inputs {67 for input in [fixture.original, fixture.nfc, fixture.nfd] {68 assert_eq(@normalization.normalize(input, NFC), fixture.nfc)69 assert_eq(@normalization.normalize(input, NFD), fixture.nfd)70 assert_eq(@normalization.is_normalized(input, NFC), input == fixture.nfc)71 assert_eq(@normalization.is_normalized(input, NFD), input == fixture.nfd)72 }73 }74}75
///|
#derive(@json.FromJson)
priv struct CorpusFile {
name : String
bytes : Int32
sha256 : String
encoding : String
}
///|
#derive(@json.FromJson)
priv struct Corpus {
files : Array[CorpusFile]
}
///|
priv struct Fixture {
name : String
original : String
nfc : String
nfd : String
}
///|
fn corpus_directory() -> String raise {
for path in [".", "normperf", "bench/normperf"] {
if @fs.path_exists("\{path}/corpus.json") {
return path
}
}
fail("Cannot find corpus.json; run moon bench ./bench/normperf")
}
///|
/// Read and validate original bytes before decoding; preserve all line endings.
/// NFC/NFD variants are benchmark inputs, not independent correctness oracles.
fn fixtures() -> Array[Fixture] raise {
let directory = corpus_directory()
let corpus : Corpus = @json.from_json(
@json.parse(@fs.read_file_to_string("\{directory}/corpus.json")),
)
corpus.files.map(file => {
let raw = @fs.read_file_to_bytes("\{directory}/data/\{file.name}.txt")
guard raw.length() == file.bytes &&
@crypto.bytes_to_hex_string(@crypto.sha256(raw)) == file.sha256 else {
fail("Corpus checksum mismatch: \{file.name}")
}
guard file.encoding == "utf-8-sig" else {
fail("Unsupported corpus encoding: \{file.encoding}")
}
let original = @utf8.decode(raw, ignore_bom=true)
{
name: file.name,
original,
nfc: @normalization.normalize(original, NFC),
nfd: @normalization.normalize(original, NFD),
}
})
}
///|
/// Exercise file IO and the complete input matrix without running timers.
test "corpus integrity and normalization consistency" {
let inputs = fixtures()
inspect(inputs.length(), content="14")
for fixture in inputs {
for input in [fixture.original, fixture.nfc, fixture.nfd] {
assert_eq(@normalization.normalize(input, NFC), fixture.nfc)
assert_eq(@normalization.normalize(input, NFD), fixture.nfd)
assert_eq(@normalization.is_normalized(input, NFC), input == fixture.nfc)
assert_eq(@normalization.is_normalized(input, NFD), input == fixture.nfd)
}
}
}