unicode

Branch: main

Switch branches or tags

Branch: main
Go to file t

413 files indexed

History
Code

bench/xlang/report.mbt
1///|2/// The column every other column is compared with.3const Baseline : String = "moonbit-native"45///|6/// One measurement printed by a harness.7priv struct Row {8  /// The peer, followed by the library when the peer measures several.9  column : String10  /// `group/operation/input`.11  name : String12  ns : Double13  /// Code points processed per call; 1 for per-call cases.14  units : Int3215  /// A value every implementation must agree on, or `-` when none applies.16  check : String17}1819///|20/// Parse the `library<TAB>case<TAB>ns<TAB>units<TAB>check` lines of a harness.21/// Anything else it printed is ignored.22fn parse_rows(peer : String, output : String) -> Array[Row] {23  let rows = []24  for line in output.split("\n") {25    guard line.split("\t").collect() is [library, name, ns, units, check] else {26      continue27    }28    let ns = @string.parse_double(ns) catch { _ => continue }29    let units = @string.parse_int(units) catch { _ => continue }30    let column = if library.is_empty() { peer } else { "\{peer} \{library}" }31    rows.push({32      column,33      name: name.to_owned(),34      ns,35      units,36      check: check.trim().to_owned(),37    })38  }39  rows40}4142///|43/// The rounds of one case in one column.44priv struct Cell {45  mut fastest : Double46  mut slowest : Double47  units : Int3248  check : String49}5051///|52/// Time per unit of the fastest round.53fn Cell::cost(self : Cell) -> Double {54  self.fastest / self.units.to_double()55}5657///|58/// Every measurement, as column -> case -> cell, in first-seen order.59priv struct Results(Map[String, Map[String, Cell]])6061///|62fn Results::add(self : Results, row : Row) -> Unit {63  let cases = self.0.get_or_init(row.column, () => Map([]))64  match cases.get(row.name) {65    Some(cell) => {66      if row.ns < cell.fastest {67        cell.fastest = row.ns68      }69      if row.ns > cell.slowest {70        cell.slowest = row.ns71      }72    }73    None =>74      cases[row.name] = {75        fastest: row.ns,76        slowest: row.ns,77        units: row.units,78        check: row.check,79      }80  }81}8283///|84fn Results::cell(self : Results, column : String, name : String) -> Cell? {85  self.0.get(column).bind(cases => cases.get(name))86}8788///|89/// The cases grouped as group -> operation -> case names, in first-seen order.90fn Results::groups(self : Results) -> Map[String, Map[String, Array[String]]] {91  let groups : Map[String, Map[String, Array[String]]] = Map([])92  for _, cases in self.0 {93    for name, _ in cases {94      guard name.split("/").collect() is [group, operation, ..] else {95        continue96      }97      let names = groups98        .get_or_init(group.to_owned(), () => Map([]))99        .get_or_init(operation.to_owned(), () => [])100      if !names.contains(name) {101        names.push(name)102      }103    }104  }105  groups106}107108///|109fn geometric_mean(values : Array[Double]) -> Double? {110  guard values.length() > 0 else { return None }111  let mut sum = 0.0112  for value in values {113    sum += @math.ln(value)114  }115  Some(@math.exp(sum / values.length().to_double()))116}117118///|119/// Two decimals below 100, none from there on.120fn format_number(value : Double) -> String {121  if value >= 100.0 {122    return value.round().to_int().to_string()123  }124  let hundredths = (value * 100.0).round().to_int()125  let fraction = (hundredths % 100).to_string().pad_start(2, '0')126  "\{hundredths / 100}.\{fraction}"127}128129///|130/// One Markdown table: a row per `(label, cases)`, a column per harness that131/// measured any of the cases. A cell is the geometric mean over the row's132/// cases; other columns also show that mean relative to the baseline's, taken133/// over the cases both measured.134fn Results::write_table(135  self : Results,136  out : StringBuilder,137  title : String,138  label_header : String,139  rows : Array[(String, Array[String])],140) -> Unit {141  let columns = []142  let mut per_call = true143  for column, cases in self.0 {144    let mut used = false145    for row in rows {146      for name in row.1 {147        if cases.get(name) is Some(cell) {148          used = true149          per_call = per_call && cell.units == 1150        }151      }152    }153    if used {154      columns.push(column)155    }156  }157  guard columns.length() > 0 else { return }158  let unit = if per_call { "ns per call" } else { "ns per code point" }159  out.write_string("\n### \{title} (\{unit})\n\n| \{label_header} |")160  for column in columns {161    out.write_string(" \{column} |")162  }163  out.write_string("\n|---|")164  for _ in columns {165    out.write_string("---|")166  }167  out.write_string("\n")168  for row in rows {169    let (label, names) = row170    out.write_string("| \{label} |")171    for column in columns {172      let costs = []173      let baseline_costs = []174      for name in names {175        guard self.cell(column, name) is Some(cell) else { continue }176        costs.push(cell.cost())177        if self.cell(Baseline, name) is Some(baseline) {178          baseline_costs.push(baseline.cost())179        }180      }181      guard geometric_mean(costs) is Some(cost) else {182        out.write_string(" – |")183        continue184      }185      out.write_string(" \{format_number(cost)}")186      if column != Baseline &&187        baseline_costs.length() == costs.length() &&188        geometric_mean(baseline_costs) is Some(baseline) {189        out.write_string(" (\{format_number(cost / baseline)}x)")190      }191      out.write_string(" |")192    }193    out.write_string("\n")194  }195}196197///|198/// The cases whose check values differ between columns, one line each.199fn Results::check_mismatches(self : Results) -> Array[String] {200  let by_case : Map[String, Map[String, Array[String]]] = Map([])201  for column, cases in self.0 {202    for name, cell in cases {203      guard cell.check != "-" else { continue }204      by_case205      .get_or_init(name, () => Map([]))206      .get_or_init(cell.check, () => [])207      .push(column)208    }209  }210  let mismatches = []211  for name, values in by_case {212    guard values.length() > 1 else { continue }213    let parts = [214      for value, columns in values => "\{value} from \{columns.join(", ")}"215    ]216    mismatches.push("\{name}: \{parts.join("; ")}")217  }218  mismatches219}220221///|222/// The median over a column's cases of slowest round / fastest round.223fn Results::spread(self : Results, column : String) -> Double? {224  guard self.0.get(column) is Some(cases) else { return None }225  let ratios = [ for _, cell in cases => cell.slowest / cell.fastest ]226  guard ratios.length() > 0 else { return None }227  ratios.sort_by((a, b) => a.compare(b))228  Some(ratios[ratios.length() / 2])229}230231///|232/// The Markdown report: per group, the geometric mean of every operation over233/// its inputs, then (with `detail`) every case; the check mismatches; and how234/// much the rounds disagreed.235fn Results::report(self : Results, rounds~ : Int32, detail~ : Bool) -> String {236  let out = StringBuilder()237  out.write_string(238    (239      #|# Cross-language Unicode benchmark240      #|241      #|Each cell is the fastest of $rounds round(s). In parentheses: the time242      #|relative to `moonbit-native` on the same cases, so above 1 means slower243      #|than MoonBit native and below 1 means faster.244      #|245    ).replace(old="$rounds", new=rounds.to_string()),246  )247  for group, operations in self.groups() {248    let summary = operations.to_array()249    self.write_table(250      out,251      "\{group}: geometric mean over inputs",252      "operation",253      summary,254    )255    if detail {256      for operation, names in operations {257        self.write_table(258          out,259          "\{group}/\{operation}",260          "input",261          names.map(name => (name.split("/").collect()[2:].join("/"), [name])),262        )263      }264    }265  }266  out.write_string("\n### Checks\n\n")267  let mismatches = self.check_mismatches()268  if mismatches.is_empty() {269    out.write_string("Every implementation agrees on every case.\n")270  } else {271    out.write_string("Implementations disagree on these cases:\n\n")272    for mismatch in mismatches {273      out.write_string("- \{mismatch}\n")274    }275  }276  if rounds > 1 {277    out.write_string(278      "\n### Noise (median of slowest round / fastest round)\n\n| column | spread |\n|---|---|\n",279    )280    for column, _ in self.0 {281      if self.spread(column) is Some(spread) {282        out.write_string("| \{column} | \{format_number(spread)}x |\n")283      }284    }285  }286  out.to_string()287}288