bench/xlang/compare.mbt1///|2/// `value` rounded to `digits` significant digits. More digits than the noise3/// allows only make the JSON larger.4fn round_significant(value : Double, digits : Int32) -> Double {5 guard value > 0.0 else { return value }6 let exponent = digits - 1 - @math.log10(value).floor().to_int()7 let scale = @math.pow(10.0, exponent.to_double())8 (value * scale).round() / scale9}1011///|12fn number(value : Double) -> Json {13 Json::number(round_significant(value, 5))14}1516///|17/// The `group/operation` part of a case name.18fn operation_of(name : String) -> String? {19 guard name.split("/").collect() is [group, operation, ..] else { return None }20 Some("\{group}/\{operation}")21}2223///|24/// One run as JSON: `about` describes it, and `cases` maps every case to its25/// cost in each column, in ns per unit of the fastest round.26fn Results::snapshot_json(self : Results, about : Map[String, Json]) -> Json {27 let cases : Map[String, Map[String, Json]] = Map([])28 for column, by_name in self.0 {29 for name, cell in by_name {30 cases.get_or_init(name, () => Map([]))[column] = number(cell.cost())31 }32 }33 let cases_json : Map[String, Json] = Map([])34 for name, columns in cases {35 cases_json[name] = Json::object(columns)36 }37 Json::object({38 "about": Json::object(about),39 "cases": Json::object(cases_json),40 })41}4243///|44/// For every operation and column measured by both runs, the geometric mean45/// over the operation's cases of head cost / base cost. Below 1, head is46/// faster. Cases only one of the runs measured are left out.47fn compare_operations(48 base : Results,49 head : Results,50) -> Map[String, Map[String, Double]] {51 let ratios : Map[String, Map[String, Array[Double]]] = Map([])52 for column, by_name in head.0 {53 for name, head_cell in by_name {54 guard base.cell(column, name) is Some(base_cell) else { continue }55 guard operation_of(name) is Some(operation) else { continue }56 ratios57 .get_or_init(operation, () => Map([]))58 .get_or_init(column, () => [])59 .push(head_cell.cost() / base_cell.cost())60 }61 }62 let result : Map[String, Map[String, Double]] = Map([])63 for operation, columns in ratios {64 let means : Map[String, Double] = Map([])65 for column, values in columns {66 if geometric_mean(values) is Some(mean) {67 means[column] = mean68 }69 }70 result[operation] = means71 }72 result73}7475///|76/// The larger of the two runs' spreads in each column: how much rounds of the77/// same code disagreed, so how large a change has to be to mean anything.78fn compare_spreads(base : Results, head : Results) -> Map[String, Double] {79 let spreads : Map[String, Double] = Map([])80 for column, _ in head.0 {81 match (base.spread(column), head.spread(column)) {82 (Some(a), Some(b)) => spreads[column] = if a > b { a } else { b }83 (Some(spread), None) | (None, Some(spread)) => spreads[column] = spread84 (None, None) => ()85 }86 }87 spreads88}8990///|91/// An A/B comparison as JSON: `about` describes it; `operations` holds the92/// ratios of `compare_operations`, `spread` those of `compare_spreads`, and93/// `cases` both costs of every case in each column, base first.94fn compare_json(95 base : Results,96 head : Results,97 about : Map[String, Json],98) -> Json {99 let operations : Map[String, Json] = Map([])100 for operation, columns in compare_operations(base, head) {101 let ratios : Map[String, Json] = Map([])102 for column, ratio in columns {103 ratios[column] = number(ratio)104 }105 operations[operation] = Json::object(ratios)106 }107 let spreads : Map[String, Json] = Map([])108 for column, spread in compare_spreads(base, head) {109 spreads[column] = number(spread)110 }111 let cases : Map[String, Map[String, Json]] = Map([])112 for column, by_name in head.0 {113 for name, head_cell in by_name {114 guard base.cell(column, name) is Some(base_cell) else { continue }115 cases.get_or_init(name, () => Map([]))[column] = Json::array([116 number(base_cell.cost()),117 number(head_cell.cost()),118 ])119 }120 }121 let cases_json : Map[String, Json] = Map([])122 for name, columns in cases {123 cases_json[name] = Json::object(columns)124 }125 Json::object({126 "about": Json::object(about),127 "operations": Json::object(operations),128 "spread": Json::object(spreads),129 "cases": Json::object(cases_json),130 })131}132133///|134/// A ratio as a signed percentage with one decimal: 1.034 is `+3.4%`.135fn format_change(ratio : Double) -> String {136 let tenths = ((ratio - 1.0) * 1000.0).round().to_int()137 let sign = if tenths > 0 { "+" } else if tenths < 0 { "-" } else { "" }138 let magnitude = tenths.abs()139 "\{sign}\{magnitude / 10}.\{magnitude % 10}%"140}141142///|143/// Changes at least this large are shown in bold in the Markdown report.144const Notable : Double = 0.05145146///|147/// The A/B comparison as Markdown: a table with a row per operation and a148/// column per harness, each cell the change from base to head.149fn compare_report(150 base : Results,151 head : Results,152 base_label~ : String,153 head_label~ : String,154 rounds~ : Int32,155) -> String {156 let out = StringBuilder()157 out.write_string(158 "# MoonBit benchmark: \{head_label} against \{base_label}\n\n",159 )160 out.write_string(161 (162 #|Each cell is the change in time from the base to the head commit,163 #|the geometric mean over the operation's inputs of the fastest of164 #|$rounds round(s) on each side. Negative is faster. Changes of 5% or165 #|more are in bold; compare them with the noise below.166 #|167 ).replace(old="$rounds", new=rounds.to_string()),168 )169 let operations = compare_operations(base, head)170 let columns = []171 for column, _ in head.0 {172 if operations.values().any(ratios => ratios.contains(column)) {173 columns.push(column)174 }175 }176 if columns.is_empty() {177 out.write_string("\nThe two runs have no case in common.\n")178 return out.to_string()179 }180 out.write_string("\n| operation |")181 for column in columns {182 out.write_string(" \{column} |")183 }184 out.write_string("\n|---|")185 for _ in columns {186 out.write_string("---|")187 }188 out.write_string("\n")189 for operation, ratios in operations {190 out.write_string("| \{operation} |")191 for column in columns {192 match ratios.get(column) {193 Some(ratio) => {194 let change = format_change(ratio)195 let notable = ratio >= 1.0 + Notable || ratio <= 1.0 - Notable196 out.write_string(197 if notable {198 " **\{change}** |"199 } else {200 " \{change} |"201 },202 )203 }204 None => out.write_string(" – |")205 }206 }207 out.write_string("\n")208 }209 let spreads = compare_spreads(base, head)210 // A single round cannot tell how noisy the machine is.211 if rounds > 1 && !spreads.is_empty() {212 out.write_string(213 "\nNoise, the median over cases of slowest round / fastest round:",214 )215 let parts = [216 for column in columns => {217 if spreads.get(column) is Some(spread) {218 "\{column} \{format_change(spread)}"219 } else {220 "\{column} –"221 }222 }223 ]224 out.write_string(" \{parts.join(", ")}.\n")225 }226 out.to_string()227}228
///|
/// `value` rounded to `digits` significant digits. More digits than the noise
/// allows only make the JSON larger.
fn round_significant(value : Double, digits : Int32) -> Double {
guard value > 0.0 else { return value }
let exponent = digits - 1 - @math.log10(value).floor().to_int()
let scale = @math.pow(10.0, exponent.to_double())
(value * scale).round() / scale
}
///|
fn number(value : Double) -> Json {
Json::number(round_significant(value, 5))
}
///|
/// The `group/operation` part of a case name.
fn operation_of(name : String) -> String? {
guard name.split("/").collect() is [group, operation, ..] else { return None }
Some("\{group}/\{operation}")
}
///|
/// One run as JSON: `about` describes it, and `cases` maps every case to its
/// cost in each column, in ns per unit of the fastest round.
fn Results::snapshot_json(self : Results, about : Map[String, Json]) -> Json {
let cases : Map[String, Map[String, Json]] = Map([])
for column, by_name in self.0 {
for name, cell in by_name {
cases.get_or_init(name, () => Map([]))[column] = number(cell.cost())
}
}
let cases_json : Map[String, Json] = Map([])
for name, columns in cases {
cases_json[name] = Json::object(columns)
}
Json::object({
"about": Json::object(about),
"cases": Json::object(cases_json),
})
}
///|
/// For every operation and column measured by both runs, the geometric mean
/// over the operation's cases of head cost / base cost. Below 1, head is
/// faster. Cases only one of the runs measured are left out.
fn compare_operations(
base : Results,
head : Results,
) -> Map[String, Map[String, Double]] {
let ratios : Map[String, Map[String, Array[Double]]] = Map([])
for column, by_name in head.0 {
for name, head_cell in by_name {
guard base.cell(column, name) is Some(base_cell) else { continue }
guard operation_of(name) is Some(operation) else { continue }
ratios
.get_or_init(operation, () => Map([]))
.get_or_init(column, () => [])
.push(head_cell.cost() / base_cell.cost())
}
}
let result : Map[String, Map[String, Double]] = Map([])
for operation, columns in ratios {
let means : Map[String, Double] = Map([])
for column, values in columns {
if geometric_mean(values) is Some(mean) {
means[column] = mean
}
}
result[operation] = means
}
result
}
///|
/// The larger of the two runs' spreads in each column: how much rounds of the
/// same code disagreed, so how large a change has to be to mean anything.
fn compare_spreads(base : Results, head : Results) -> Map[String, Double] {
let spreads : Map[String, Double] = Map([])
for column, _ in head.0 {
match (base.spread(column), head.spread(column)) {
(Some(a), Some(b)) => spreads[column] = if a > b { a } else { b }
(Some(spread), None) | (None, Some(spread)) => spreads[column] = spread
(None, None) => ()
}
}
spreads
}
///|
/// An A/B comparison as JSON: `about` describes it; `operations` holds the
/// ratios of `compare_operations`, `spread` those of `compare_spreads`, and
/// `cases` both costs of every case in each column, base first.
fn compare_json(
base : Results,
head : Results,
about : Map[String, Json],
) -> Json {
let operations : Map[String, Json] = Map([])
for operation, columns in compare_operations(base, head) {
let ratios : Map[String, Json] = Map([])
for column, ratio in columns {
ratios[column] = number(ratio)
}
operations[operation] = Json::object(ratios)
}
let spreads : Map[String, Json] = Map([])
for column, spread in compare_spreads(base, head) {
spreads[column] = number(spread)
}
let cases : Map[String, Map[String, Json]] = Map([])
for column, by_name in head.0 {
for name, head_cell in by_name {
guard base.cell(column, name) is Some(base_cell) else { continue }
cases.get_or_init(name, () => Map([]))[column] = Json::array([
number(base_cell.cost()),
number(head_cell.cost()),
])
}
}
let cases_json : Map[String, Json] = Map([])
for name, columns in cases {
cases_json[name] = Json::object(columns)
}
Json::object({
"about": Json::object(about),
"operations": Json::object(operations),
"spread": Json::object(spreads),
"cases": Json::object(cases_json),
})
}
///|
/// A ratio as a signed percentage with one decimal: 1.034 is `+3.4%`.
fn format_change(ratio : Double) -> String {
let tenths = ((ratio - 1.0) * 1000.0).round().to_int()
let sign = if tenths > 0 { "+" } else if tenths < 0 { "-" } else { "" }
let magnitude = tenths.abs()
"\{sign}\{magnitude / 10}.\{magnitude % 10}%"
}
///|
/// Changes at least this large are shown in bold in the Markdown report.
const Notable : Double = 0.05
///|
/// The A/B comparison as Markdown: a table with a row per operation and a
/// column per harness, each cell the change from base to head.
fn compare_report(
base : Results,
head : Results,
base_label~ : String,
head_label~ : String,
rounds~ : Int32,
) -> String {
let out = StringBuilder()
out.write_string(
"# MoonBit benchmark: \{head_label} against \{base_label}\n\n",
)
out.write_string(
(
#|Each cell is the change in time from the base to the head commit,
#|the geometric mean over the operation's inputs of the fastest of
#|$rounds round(s) on each side. Negative is faster. Changes of 5% or
#|more are in bold; compare them with the noise below.
#|
).replace(old="$rounds", new=rounds.to_string()),
)
let operations = compare_operations(base, head)
let columns = []
for column, _ in head.0 {
if operations.values().any(ratios => ratios.contains(column)) {
columns.push(column)
}
}
if columns.is_empty() {
out.write_string("\nThe two runs have no case in common.\n")
return out.to_string()
}
out.write_string("\n| operation |")
for column in columns {
out.write_string(" \{column} |")
}
out.write_string("\n|---|")
for _ in columns {
out.write_string("---|")
}
out.write_string("\n")
for operation, ratios in operations {
out.write_string("| \{operation} |")
for column in columns {
match ratios.get(column) {
Some(ratio) => {
let change = format_change(ratio)
let notable = ratio >= 1.0 + Notable || ratio <= 1.0 - Notable
out.write_string(
if notable {
" **\{change}** |"
} else {
" \{change} |"
},
)
}
None => out.write_string(" – |")
}
}
out.write_string("\n")
}
let spreads = compare_spreads(base, head)
// A single round cannot tell how noisy the machine is.
if rounds > 1 && !spreads.is_empty() {
out.write_string(
"\nNoise, the median over cases of slowest round / fastest round:",
)
let parts = [
for column in columns => {
if spreads.get(column) is Some(spread) {
"\{column} \{format_change(spread)}"
} else {
"\{column} –"
}
}
]
out.write_string(" \{parts.join(", ")}.\n")
}
out.to_string()
}