unicode

Branch: main

Switch branches or tags

Branch: main
Go to file t

413 files indexed

History
Code

bench/xlang/moonbit/main.mbt
1///|2fn read(path : String) -> String raise {3  @utf8.decode(@fs.read_file_to_bytes(path))4}56///|7/// Input names in `directory`: the `.txt` files without their extension.8fn names(directory : String) -> Array[String] raise {9  let result = []10  for file in @fs.read_dir(directory) {11    if file.strip_suffix(".txt") is Some(name) {12      result.push(name.to_owned())13    }14  }15  result.sort()16  result17}1819///|20/// The `name<TAB>value` lines of a `.tsv` input.21fn pairs(path : String) -> Array[(String, String)] raise {22  let result = []23  for line in read(path).split("\n") {24    if line.split_once("\t") is Some((name, value)) {25      result.push((name.to_owned(), value.to_owned()))26    }27  }28  result29}3031///|32/// Elapsed microseconds for `iterations` calls. The results are folded into33/// the returned sink so that the calls cannot be optimized away.34fn run(iterations : Int32, f : () -> Int32) -> (Double, Int32) {35  let mut sink = 036  let start = @bench.monotonic_clock_start()37  for _ in 0..<iterations {38    sink = sink ^ f()39  }40  (@bench.monotonic_clock_end(start), sink)41}4243///|44/// The latest string a case built. Every result is stored here, where it can45/// be read later, so that the optimizer cannot drop the work that builds it:46/// V8 drops a string that only has its length read, as it can compute the47/// length without the string.48let kept_string : Ref[String] = Ref("")4950///|51/// Keep `result` and return its length.52fn keep_string(result : String) -> Int32 {53  kept_string.val = result54  result.length()55}5657///|58/// The latest bidi levels a case built, kept as `kept_string` keeps strings.59let kept_levels : Ref[Array[Int32]] = Ref([])6061///|62/// Keep `levels` and return their number.63fn keep_levels(levels : Array[Int32]) -> Int32 {64  kept_levels.val = levels65  levels.length()66}6768///|69/// Print one result row: `library<TAB>case<TAB>ns<TAB>units<TAB>check`.70///71/// Every harness in `bench/xlang` follows the same protocol: double the72/// iteration count until one sample takes at least 5 ms, then report the73/// fastest of 5 samples, the first of which is that one. `units` is the74/// number of code points processed per call (1 for per-call cases) and75/// `check` is a value every implementation must agree on. The library column76/// is empty: this harness has one library.77fn measure(78  name : String,79  units : Int32,80  check : Int32,81  f : () -> Int32,82) -> Unit {83  let mut iterations = 184  let mut best = run(iterations, f).085  while best < 5000.0 {86    iterations = iterations * 287    best = run(iterations, f).088  }89  for _ in 1..<5 {90    let elapsed = run(iterations, f).091    if elapsed < best {92      best = elapsed93    }94  }95  let ns = best * 1000.0 / iterations.to_double()96  println("\t\{name}\t\{ns}\t\{units}\t\{check}")97}9899///|100fn bench_normalization(inputs : String) -> Unit raise {101  for name in names("\{inputs}/norm") {102    let original = read("\{inputs}/norm/\{name}.txt")103    let decomposed = read("\{inputs}/normd/\{name}.txt")104    let units = original.char_length()105    for106      (label, form) in [107        ("nfc", @normalization.NFC),108        ("nfd", @normalization.NFD),109        ("nfkc", @normalization.NFKC),110        ("nfkd", @normalization.NFKD),111      ] {112      measure(113        "norm/\{label}/\{name}",114        units,115        @normalization.normalize(original, form).char_length(),116        () => keep_string(@normalization.normalize(original, form)),117      )118    }119    measure(120      "norm/nfd>nfc/\{name}",121      decomposed.char_length(),122      @normalization.normalize(decomposed, NFC).char_length(),123      () => keep_string(@normalization.normalize(decomposed, NFC)),124    )125    let is_nfc = () => {126      if @normalization.is_normalized(original, NFC) {127        1128      } else {129        0130      }131    }132    measure("norm/is_nfc/\{name}", units, is_nfc(), is_nfc)133  }134}135136///|137fn bench_segmentation(inputs : String) -> Unit raise {138  for name in names("\{inputs}/seg") {139    let text = read("\{inputs}/seg/\{name}.txt")140    let count = () => {141      let cursor = @segmentation.graphemes(text)142      let mut clusters = 0143      while cursor.next().length() > 0 {144        clusters += 1145      }146      clusters147    }148    measure("seg/graphemes/\{name}", text.char_length(), count(), count)149  }150}151152///|153fn bench_bidi(inputs : String) -> Unit raise {154  for name in names("\{inputs}/bidi") {155    let text = read("\{inputs}/bidi/\{name}.txt")156    let mut level_sum = 0157    for level in @bidi.process(text).levels {158      level_sum += level159    }160    measure("bidi/process/\{name}", text.char_length(), level_sum, () => {161      keep_levels(@bidi.process(text).levels)162    })163  }164}165166///|167fn bench_idna(inputs : String) -> Unit raise {168  for (name, domain) in pairs("\{inputs}/idna.tsv") {169    // Every input converts successfully without STD3 rules, so the closures170    // time successful conversions only.171    let ascii = @idna.to_ascii(domain, use_std3_ascii_rules=false)172    let unicode = @idna.to_unicode(domain, use_std3_ascii_rules=false)173    measure("idna/to_ascii/\{name}", 1, ascii.char_length(), () => {174      keep_string(175        @idna.to_ascii(domain, use_std3_ascii_rules=false) catch {176          _ => domain177        },178      )179    })180    measure("idna/to_unicode/\{name}", 1, unicode.char_length(), () => {181      keep_string(182        @idna.to_unicode(domain, use_std3_ascii_rules=false) catch {183          _ => domain184        },185      )186    })187  }188}189190///|191fn bench_punycode(inputs : String) -> Unit raise {192  for (name, input) in pairs("\{inputs}/puny.tsv") {193    let encoded = @punycode.encode(input)194    measure("puny/encode/\{name}", 1, encoded.length(), () => {195      keep_string(@punycode.encode(input) catch { _ => input })196    })197    measure("puny/decode/\{name}", 1, input.char_length(), () => {198      keep_string(@punycode.decode(encoded) catch { _ => encoded })199    })200  }201}202203///|204fn bench_properties(inputs : String) -> Unit raise {205  for name in names("\{inputs}/case") {206    let text = read("\{inputs}/case/\{name}.txt")207    let units = text.char_length()208    let simple_upper = () => {209      let mut sum = 0210      for c in text {211        sum = sum ^ @ucd.to_simple_uppercase(c).to_int()212      }213      sum214    }215    let lowercase_letters = () => {216      let mut count = 0217      for c in text {218        if @ucd.general_category(c) is Ll {219          count += 1220        }221      }222      count223    }224    measure("case/simple_upper/\{name}", units, simple_upper(), simple_upper)225    measure(226      "case/full_upper/\{name}",227      units,228      @ucd.uppercase(text).char_length(),229      () => keep_string(@ucd.uppercase(text)),230    )231    for232      (label, fold) in [233        ("fold", @casefold.case_fold),234        ("nfkc_casefold", @casefold.nfkc_casefold),235      ] {236      measure("case/\{label}/\{name}", units, fold(text).char_length(), () => {237        keep_string(fold(text))238      })239    }240    measure(241      "prop/general_category_Ll/\{name}",242      units,243      lowercase_letters(),244      lowercase_letters,245    )246  }247}248249///|250fn main raise {251  let command = @argparse.Command(252    "xlang-moonbit",253    about="Time this repository's Unicode modules on the shared xlang inputs",254    positionals=[255      @argparse.PositionArg("inputs", about="Directory of prepared inputs"),256      @argparse.PositionArg(257        "group",258        about="Only run this group: norm, seg, bidi, idna, puny or case",259        num_args=@argparse.ValueRange(lower=0, upper=1),260      ),261    ],262  )263  let matches = command.parse()264  guard matches.values.get("inputs") is Some([inputs, ..]) else {265    fail(command.render_help())266  }267  let group = matches.values.get("group").map(values => values[0])268  let groups : Array[(String, (String) -> Unit raise)] = [269    ("norm", bench_normalization),270    ("seg", bench_segmentation),271    ("bidi", bench_bidi),272    ("idna", bench_idna),273    ("puny", bench_punycode),274    ("case", bench_properties),275  ]276  for (name, bench) in groups {277    if group is Some(selected) && selected != name {278      continue279    }280    bench(inputs)281  }282}283