bench/xlang/report.mbt1///|2/// The column every other column is compared with.3const Baseline : String = "moonbit-native"45///|6/// One measurement printed by a harness.7priv struct Row {8 /// The peer, followed by the library when the peer measures several.9 column : String10 /// `group/operation/input`.11 name : String12 ns : Double13 /// Code points processed per call; 1 for per-call cases.14 units : Int3215 /// A value every implementation must agree on, or `-` when none applies.16 check : String17}1819///|20/// Parse the `library<TAB>case<TAB>ns<TAB>units<TAB>check` lines of a harness.21/// Anything else it printed is ignored.22fn parse_rows(peer : String, output : String) -> Array[Row] {23 let rows = []24 for line in output.split("\n") {25 guard line.split("\t").collect() is [library, name, ns, units, check] else {26 continue27 }28 let ns = @string.parse_double(ns) catch { _ => continue }29 let units = @string.parse_int(units) catch { _ => continue }30 let column = if library.is_empty() { peer } else { "\{peer} \{library}" }31 rows.push({32 column,33 name: name.to_owned(),34 ns,35 units,36 check: check.trim().to_owned(),37 })38 }39 rows40}4142///|43/// The rounds of one case in one column.44priv struct Cell {45 mut fastest : Double46 mut slowest : Double47 units : Int3248 check : String49}5051///|52/// Time per unit of the fastest round.53fn Cell::cost(self : Cell) -> Double {54 self.fastest / self.units.to_double()55}5657///|58/// Every measurement, as column -> case -> cell, in first-seen order.59priv struct Results(Map[String, Map[String, Cell]])6061///|62fn Results::add(self : Results, row : Row) -> Unit {63 let cases = self.0.get_or_init(row.column, () => Map([]))64 match cases.get(row.name) {65 Some(cell) => {66 if row.ns < cell.fastest {67 cell.fastest = row.ns68 }69 if row.ns > cell.slowest {70 cell.slowest = row.ns71 }72 }73 None =>74 cases[row.name] = {75 fastest: row.ns,76 slowest: row.ns,77 units: row.units,78 check: row.check,79 }80 }81}8283///|84fn Results::cell(self : Results, column : String, name : String) -> Cell? {85 self.0.get(column).bind(cases => cases.get(name))86}8788///|89/// The cases grouped as group -> operation -> case names, in first-seen order.90fn Results::groups(self : Results) -> Map[String, Map[String, Array[String]]] {91 let groups : Map[String, Map[String, Array[String]]] = Map([])92 for _, cases in self.0 {93 for name, _ in cases {94 guard name.split("/").collect() is [group, operation, ..] else {95 continue96 }97 let names = groups98 .get_or_init(group.to_owned(), () => Map([]))99 .get_or_init(operation.to_owned(), () => [])100 if !names.contains(name) {101 names.push(name)102 }103 }104 }105 groups106}107108///|109fn geometric_mean(values : Array[Double]) -> Double? {110 guard values.length() > 0 else { return None }111 let mut sum = 0.0112 for value in values {113 sum += @math.ln(value)114 }115 Some(@math.exp(sum / values.length().to_double()))116}117118///|119/// Two decimals below 100, none from there on.120fn format_number(value : Double) -> String {121 if value >= 100.0 {122 return value.round().to_int().to_string()123 }124 let hundredths = (value * 100.0).round().to_int()125 let fraction = (hundredths % 100).to_string().pad_start(2, '0')126 "\{hundredths / 100}.\{fraction}"127}128129///|130/// One Markdown table: a row per `(label, cases)`, a column per harness that131/// measured any of the cases. A cell is the geometric mean over the row's132/// cases; other columns also show that mean relative to the baseline's, taken133/// over the cases both measured.134fn Results::write_table(135 self : Results,136 out : StringBuilder,137 title : String,138 label_header : String,139 rows : Array[(String, Array[String])],140) -> Unit {141 let columns = []142 let mut per_call = true143 for column, cases in self.0 {144 let mut used = false145 for row in rows {146 for name in row.1 {147 if cases.get(name) is Some(cell) {148 used = true149 per_call = per_call && cell.units == 1150 }151 }152 }153 if used {154 columns.push(column)155 }156 }157 guard columns.length() > 0 else { return }158 let unit = if per_call { "ns per call" } else { "ns per code point" }159 out.write_string("\n### \{title} (\{unit})\n\n| \{label_header} |")160 for column in columns {161 out.write_string(" \{column} |")162 }163 out.write_string("\n|---|")164 for _ in columns {165 out.write_string("---|")166 }167 out.write_string("\n")168 for row in rows {169 let (label, names) = row170 out.write_string("| \{label} |")171 for column in columns {172 let costs = []173 let baseline_costs = []174 for name in names {175 guard self.cell(column, name) is Some(cell) else { continue }176 costs.push(cell.cost())177 if self.cell(Baseline, name) is Some(baseline) {178 baseline_costs.push(baseline.cost())179 }180 }181 guard geometric_mean(costs) is Some(cost) else {182 out.write_string(" – |")183 continue184 }185 out.write_string(" \{format_number(cost)}")186 if column != Baseline &&187 baseline_costs.length() == costs.length() &&188 geometric_mean(baseline_costs) is Some(baseline) {189 out.write_string(" (\{format_number(cost / baseline)}x)")190 }191 out.write_string(" |")192 }193 out.write_string("\n")194 }195}196197///|198/// The cases whose check values differ between columns, one line each.199fn Results::check_mismatches(self : Results) -> Array[String] {200 let by_case : Map[String, Map[String, Array[String]]] = Map([])201 for column, cases in self.0 {202 for name, cell in cases {203 guard cell.check != "-" else { continue }204 by_case205 .get_or_init(name, () => Map([]))206 .get_or_init(cell.check, () => [])207 .push(column)208 }209 }210 let mismatches = []211 for name, values in by_case {212 guard values.length() > 1 else { continue }213 let parts = [214 for value, columns in values => "\{value} from \{columns.join(", ")}"215 ]216 mismatches.push("\{name}: \{parts.join("; ")}")217 }218 mismatches219}220221///|222/// The median over a column's cases of slowest round / fastest round.223fn Results::spread(self : Results, column : String) -> Double? {224 guard self.0.get(column) is Some(cases) else { return None }225 let ratios = [ for _, cell in cases => cell.slowest / cell.fastest ]226 guard ratios.length() > 0 else { return None }227 ratios.sort_by((a, b) => a.compare(b))228 Some(ratios[ratios.length() / 2])229}230231///|232/// The Markdown report: per group, the geometric mean of every operation over233/// its inputs, then (with `detail`) every case; the check mismatches; and how234/// much the rounds disagreed.235fn Results::report(self : Results, rounds~ : Int32, detail~ : Bool) -> String {236 let out = StringBuilder()237 out.write_string(238 (239 #|# Cross-language Unicode benchmark240 #|241 #|Each cell is the fastest of $rounds round(s). In parentheses: the time242 #|relative to `moonbit-native` on the same cases, so above 1 means slower243 #|than MoonBit native and below 1 means faster.244 #|245 ).replace(old="$rounds", new=rounds.to_string()),246 )247 for group, operations in self.groups() {248 let summary = operations.to_array()249 self.write_table(250 out,251 "\{group}: geometric mean over inputs",252 "operation",253 summary,254 )255 if detail {256 for operation, names in operations {257 self.write_table(258 out,259 "\{group}/\{operation}",260 "input",261 names.map(name => (name.split("/").collect()[2:].join("/"), [name])),262 )263 }264 }265 }266 out.write_string("\n### Checks\n\n")267 let mismatches = self.check_mismatches()268 if mismatches.is_empty() {269 out.write_string("Every implementation agrees on every case.\n")270 } else {271 out.write_string("Implementations disagree on these cases:\n\n")272 for mismatch in mismatches {273 out.write_string("- \{mismatch}\n")274 }275 }276 if rounds > 1 {277 out.write_string(278 "\n### Noise (median of slowest round / fastest round)\n\n| column | spread |\n|---|---|\n",279 )280 for column, _ in self.0 {281 if self.spread(column) is Some(spread) {282 out.write_string("| \{column} | \{format_number(spread)}x |\n")283 }284 }285 }286 out.to_string()287}288
///|
/// The column every other column is compared with.
const Baseline : String = "moonbit-native"
///|
/// One measurement printed by a harness.
priv struct Row {
/// The peer, followed by the library when the peer measures several.
column : String
/// `group/operation/input`.
name : String
ns : Double
/// Code points processed per call; 1 for per-call cases.
units : Int32
/// A value every implementation must agree on, or `-` when none applies.
check : String
}
///|
/// Parse the `library<TAB>case<TAB>ns<TAB>units<TAB>check` lines of a harness.
/// Anything else it printed is ignored.
fn parse_rows(peer : String, output : String) -> Array[Row] {
let rows = []
for line in output.split("\n") {
guard line.split("\t").collect() is [library, name, ns, units, check] else {
continue
}
let ns = @string.parse_double(ns) catch { _ => continue }
let units = @string.parse_int(units) catch { _ => continue }
let column = if library.is_empty() { peer } else { "\{peer} \{library}" }
rows.push({
column,
name: name.to_owned(),
ns,
units,
check: check.trim().to_owned(),
})
}
rows
}
///|
/// The rounds of one case in one column.
priv struct Cell {
mut fastest : Double
mut slowest : Double
units : Int32
check : String
}
///|
/// Time per unit of the fastest round.
fn Cell::cost(self : Cell) -> Double {
self.fastest / self.units.to_double()
}
///|
/// Every measurement, as column -> case -> cell, in first-seen order.
priv struct Results(Map[String, Map[String, Cell]])
///|
fn Results::add(self : Results, row : Row) -> Unit {
let cases = self.0.get_or_init(row.column, () => Map([]))
match cases.get(row.name) {
Some(cell) => {
if row.ns < cell.fastest {
cell.fastest = row.ns
}
if row.ns > cell.slowest {
cell.slowest = row.ns
}
}
None =>
cases[row.name] = {
fastest: row.ns,
slowest: row.ns,
units: row.units,
check: row.check,
}
}
}
///|
fn Results::cell(self : Results, column : String, name : String) -> Cell? {
self.0.get(column).bind(cases => cases.get(name))
}
///|
/// The cases grouped as group -> operation -> case names, in first-seen order.
fn Results::groups(self : Results) -> Map[String, Map[String, Array[String]]] {
let groups : Map[String, Map[String, Array[String]]] = Map([])
for _, cases in self.0 {
for name, _ in cases {
guard name.split("/").collect() is [group, operation, ..] else {
continue
}
let names = groups
.get_or_init(group.to_owned(), () => Map([]))
.get_or_init(operation.to_owned(), () => [])
if !names.contains(name) {
names.push(name)
}
}
}
groups
}
///|
fn geometric_mean(values : Array[Double]) -> Double? {
guard values.length() > 0 else { return None }
let mut sum = 0.0
for value in values {
sum += @math.ln(value)
}
Some(@math.exp(sum / values.length().to_double()))
}
///|
/// Two decimals below 100, none from there on.
fn format_number(value : Double) -> String {
if value >= 100.0 {
return value.round().to_int().to_string()
}
let hundredths = (value * 100.0).round().to_int()
let fraction = (hundredths % 100).to_string().pad_start(2, '0')
"\{hundredths / 100}.\{fraction}"
}
///|
/// One Markdown table: a row per `(label, cases)`, a column per harness that
/// measured any of the cases. A cell is the geometric mean over the row's
/// cases; other columns also show that mean relative to the baseline's, taken
/// over the cases both measured.
fn Results::write_table(
self : Results,
out : StringBuilder,
title : String,
label_header : String,
rows : Array[(String, Array[String])],
) -> Unit {
let columns = []
let mut per_call = true
for column, cases in self.0 {
let mut used = false
for row in rows {
for name in row.1 {
if cases.get(name) is Some(cell) {
used = true
per_call = per_call && cell.units == 1
}
}
}
if used {
columns.push(column)
}
}
guard columns.length() > 0 else { return }
let unit = if per_call { "ns per call" } else { "ns per code point" }
out.write_string("\n### \{title} (\{unit})\n\n| \{label_header} |")
for column in columns {
out.write_string(" \{column} |")
}
out.write_string("\n|---|")
for _ in columns {
out.write_string("---|")
}
out.write_string("\n")
for row in rows {
let (label, names) = row
out.write_string("| \{label} |")
for column in columns {
let costs = []
let baseline_costs = []
for name in names {
guard self.cell(column, name) is Some(cell) else { continue }
costs.push(cell.cost())
if self.cell(Baseline, name) is Some(baseline) {
baseline_costs.push(baseline.cost())
}
}
guard geometric_mean(costs) is Some(cost) else {
out.write_string(" – |")
continue
}
out.write_string(" \{format_number(cost)}")
if column != Baseline &&
baseline_costs.length() == costs.length() &&
geometric_mean(baseline_costs) is Some(baseline) {
out.write_string(" (\{format_number(cost / baseline)}x)")
}
out.write_string(" |")
}
out.write_string("\n")
}
}
///|
/// The cases whose check values differ between columns, one line each.
fn Results::check_mismatches(self : Results) -> Array[String] {
let by_case : Map[String, Map[String, Array[String]]] = Map([])
for column, cases in self.0 {
for name, cell in cases {
guard cell.check != "-" else { continue }
by_case
.get_or_init(name, () => Map([]))
.get_or_init(cell.check, () => [])
.push(column)
}
}
let mismatches = []
for name, values in by_case {
guard values.length() > 1 else { continue }
let parts = [
for value, columns in values => "\{value} from \{columns.join(", ")}"
]
mismatches.push("\{name}: \{parts.join("; ")}")
}
mismatches
}
///|
/// The median over a column's cases of slowest round / fastest round.
fn Results::spread(self : Results, column : String) -> Double? {
guard self.0.get(column) is Some(cases) else { return None }
let ratios = [ for _, cell in cases => cell.slowest / cell.fastest ]
guard ratios.length() > 0 else { return None }
ratios.sort_by((a, b) => a.compare(b))
Some(ratios[ratios.length() / 2])
}
///|
/// The Markdown report: per group, the geometric mean of every operation over
/// its inputs, then (with `detail`) every case; the check mismatches; and how
/// much the rounds disagreed.
fn Results::report(self : Results, rounds~ : Int32, detail~ : Bool) -> String {
let out = StringBuilder()
out.write_string(
(
#|# Cross-language Unicode benchmark
#|
#|Each cell is the fastest of $rounds round(s). In parentheses: the time
#|relative to `moonbit-native` on the same cases, so above 1 means slower
#|than MoonBit native and below 1 means faster.
#|
).replace(old="$rounds", new=rounds.to_string()),
)
for group, operations in self.groups() {
let summary = operations.to_array()
self.write_table(
out,
"\{group}: geometric mean over inputs",
"operation",
summary,
)
if detail {
for operation, names in operations {
self.write_table(
out,
"\{group}/\{operation}",
"input",
names.map(name => (name.split("/").collect()[2:].join("/"), [name])),
)
}
}
}
out.write_string("\n### Checks\n\n")
let mismatches = self.check_mismatches()
if mismatches.is_empty() {
out.write_string("Every implementation agrees on every case.\n")
} else {
out.write_string("Implementations disagree on these cases:\n\n")
for mismatch in mismatches {
out.write_string("- \{mismatch}\n")
}
}
if rounds > 1 {
out.write_string(
"\n### Noise (median of slowest round / fastest round)\n\n| column | spread |\n|---|---|\n",
)
for column, _ in self.0 {
if self.spread(column) is Some(spread) {
out.write_string("| \{column} | \{format_number(spread)}x |\n")
}
}
}
out.to_string()
}