unicode

Branch: main

Switch branches or tags

Branch: main
Go to file t

413 files indexed

History
Code

bench/normperf/corpus_test.mbt
1///|2#derive(@json.FromJson)3priv struct CorpusFile {4  name : String5  bytes : Int326  sha256 : String7  encoding : String8}910///|11#derive(@json.FromJson)12priv struct Corpus {13  files : Array[CorpusFile]14}1516///|17priv struct Fixture {18  name : String19  original : String20  nfc : String21  nfd : String22}2324///|25fn corpus_directory() -> String raise {26  for path in [".", "normperf", "bench/normperf"] {27    if @fs.path_exists("\{path}/corpus.json") {28      return path29    }30  }31  fail("Cannot find corpus.json; run moon bench ./bench/normperf")32}3334///|35/// Read and validate original bytes before decoding; preserve all line endings.36/// NFC/NFD variants are benchmark inputs, not independent correctness oracles.37fn fixtures() -> Array[Fixture] raise {38  let directory = corpus_directory()39  let corpus : Corpus = @json.from_json(40    @json.parse(@fs.read_file_to_string("\{directory}/corpus.json")),41  )42  corpus.files.map(file => {43    let raw = @fs.read_file_to_bytes("\{directory}/data/\{file.name}.txt")44    guard raw.length() == file.bytes &&45      @crypto.bytes_to_hex_string(@crypto.sha256(raw)) == file.sha256 else {46      fail("Corpus checksum mismatch: \{file.name}")47    }48    guard file.encoding == "utf-8-sig" else {49      fail("Unsupported corpus encoding: \{file.encoding}")50    }51    let original = @utf8.decode(raw, ignore_bom=true)52    {53      name: file.name,54      original,55      nfc: @normalization.normalize(original, NFC),56      nfd: @normalization.normalize(original, NFD),57    }58  })59}6061///|62/// Exercise file IO and the complete input matrix without running timers.63test "corpus integrity and normalization consistency" {64  let inputs = fixtures()65  inspect(inputs.length(), content="14")66  for fixture in inputs {67    for input in [fixture.original, fixture.nfc, fixture.nfd] {68      assert_eq(@normalization.normalize(input, NFC), fixture.nfc)69      assert_eq(@normalization.normalize(input, NFD), fixture.nfd)70      assert_eq(@normalization.is_normalized(input, NFC), input == fixture.nfc)71      assert_eq(@normalization.is_normalized(input, NFD), input == fixture.nfd)72    }73  }74}75