bench/xlang/compare_wbtest.mbt1///|2/// Results of one harness from the given outputs, one per round.3fn results_of(column : String, outputs : Array[String]) -> Results {4 let results = Results(Map([]))5 for output in outputs {6 for row in parse_rows(column, output) {7 results.add(row)8 }9 }10 results11}1213///|14/// Two runs of the same harness: head halves the cost of `norm/nfc/a`, keeps15/// `norm/nfc/b`, and measures `norm/nfd/new` only.16fn sample_pair() -> (Results, Results) {17 let base = results_of("moonbit-native", [18 (19 #| norm/nfc/a 200 100 720 #| norm/nfc/b 800 100 921 #| idna/to_ascii/x 50 1 1522 ),23 (24 #| norm/nfc/a 220 100 725 #| norm/nfc/b 840 100 926 #| idna/to_ascii/x 55 1 1527 ),28 ])29 let head = results_of("moonbit-native", [30 (31 #| norm/nfc/a 100 100 732 #| norm/nfc/b 800 100 933 #| norm/nfd/new 300 100 934 #| idna/to_ascii/x 52.5 1 1535 ),36 ])37 (base, head)38}3940///|41test "numbers keep five significant digits" {42 inspect(round_significant(0.296612345, 5), content="0.29661")43 inspect(round_significant(1002.34567, 5), content="1002.3")44 inspect(round_significant(123456.7, 5), content="123460")45 inspect(round_significant(0.0, 5), content="0")46}4748///|49test "changes are signed percentages" {50 inspect(format_change(1.034), content="+3.4%")51 inspect(format_change(0.5), content="-50.0%")52 inspect(format_change(1.0), content="0.0%")53 inspect(format_change(0.99996), content="0.0%")54}5556///|57test "operations compare the cases both runs measured" {58 let (base, head) = sample_pair()59 let operations = compare_operations(base, head)60 let ratio = (operation : String) => {61 operations62 .get(operation)63 .bind(columns => columns.get("moonbit-native"))64 .unwrap_or(-1.0)65 }66 // norm/nfc: sqrt(0.5 * 1.0); norm/nfd has no base case.67 inspect(ratio("norm/nfc"), content="0.7071067811865475")68 inspect(operations.get("norm/nfd") is None, content="true")69 inspect(ratio("idna/to_ascii"), content="1.05")70}7172///|73test "comparison as JSON" {74 let (base, head) = sample_pair()75 let json = compare_json(base, head, { "rounds": Json::number(2.0) })76 inspect(77 json.stringify(),78 content=(79 #|{"about":{"rounds":2},"operations":{"norm/nfc":{"moonbit-native":0.70711},"idna/to_ascii":{"moonbit-native":1.05}},"spread":{"moonbit-native":1.1},"cases":{"norm/nfc/a":{"moonbit-native":[2,1]},"norm/nfc/b":{"moonbit-native":[8,8]},"idna/to_ascii/x":{"moonbit-native":[50,52.5]}}}80 ),81 )82}8384///|85test "comparison report" {86 let (base, head) = sample_pair()87 inspect(88 compare_report(base, head, base_label="aaa", head_label="bbb", rounds=2),89 content=(90 #|# MoonBit benchmark: bbb against aaa91 #|92 #|Each cell is the change in time from the base to the head commit,93 #|the geometric mean over the operation's inputs of the fastest of94 #|2 round(s) on each side. Negative is faster. Changes of 5% or95 #|more are in bold; compare them with the noise below.96 #|97 #|| operation | moonbit-native |98 #||---|---|99 #|| norm/nfc | **-29.3%** |100 #|| idna/to_ascii | **+5.0%** |101 #|102 #|Noise, the median over cases of slowest round / fastest round: moonbit-native +10.0%.103 #|104 ),105 )106}107108///|109test "snapshot as JSON" {110 let (_, head) = sample_pair()111 inspect(112 head.snapshot_json({ "rounds": Json::number(1.0) }).stringify(),113 content=(114 #|{"about":{"rounds":1},"cases":{"norm/nfc/a":{"moonbit-native":1},"norm/nfc/b":{"moonbit-native":8},"norm/nfd/new":{"moonbit-native":3},"idna/to_ascii/x":{"moonbit-native":52.5}}}115 ),116 )117}118
///|
/// Results of one harness from the given outputs, one per round.
fn results_of(column : String, outputs : Array[String]) -> Results {
let results = Results(Map([]))
for output in outputs {
for row in parse_rows(column, output) {
results.add(row)
}
}
results
}
///|
/// Two runs of the same harness: head halves the cost of `norm/nfc/a`, keeps
/// `norm/nfc/b`, and measures `norm/nfd/new` only.
fn sample_pair() -> (Results, Results) {
let base = results_of("moonbit-native", [
(
#| norm/nfc/a 200 100 7
#| norm/nfc/b 800 100 9
#| idna/to_ascii/x 50 1 15
),
(
#| norm/nfc/a 220 100 7
#| norm/nfc/b 840 100 9
#| idna/to_ascii/x 55 1 15
),
])
let head = results_of("moonbit-native", [
(
#| norm/nfc/a 100 100 7
#| norm/nfc/b 800 100 9
#| norm/nfd/new 300 100 9
#| idna/to_ascii/x 52.5 1 15
),
])
(base, head)
}
///|
test "numbers keep five significant digits" {
inspect(round_significant(0.296612345, 5), content="0.29661")
inspect(round_significant(1002.34567, 5), content="1002.3")
inspect(round_significant(123456.7, 5), content="123460")
inspect(round_significant(0.0, 5), content="0")
}
///|
test "changes are signed percentages" {
inspect(format_change(1.034), content="+3.4%")
inspect(format_change(0.5), content="-50.0%")
inspect(format_change(1.0), content="0.0%")
inspect(format_change(0.99996), content="0.0%")
}
///|
test "operations compare the cases both runs measured" {
let (base, head) = sample_pair()
let operations = compare_operations(base, head)
let ratio = (operation : String) => {
operations
.get(operation)
.bind(columns => columns.get("moonbit-native"))
.unwrap_or(-1.0)
}
// norm/nfc: sqrt(0.5 * 1.0); norm/nfd has no base case.
inspect(ratio("norm/nfc"), content="0.7071067811865475")
inspect(operations.get("norm/nfd") is None, content="true")
inspect(ratio("idna/to_ascii"), content="1.05")
}
///|
test "comparison as JSON" {
let (base, head) = sample_pair()
let json = compare_json(base, head, { "rounds": Json::number(2.0) })
inspect(
json.stringify(),
content=(
#|{"about":{"rounds":2},"operations":{"norm/nfc":{"moonbit-native":0.70711},"idna/to_ascii":{"moonbit-native":1.05}},"spread":{"moonbit-native":1.1},"cases":{"norm/nfc/a":{"moonbit-native":[2,1]},"norm/nfc/b":{"moonbit-native":[8,8]},"idna/to_ascii/x":{"moonbit-native":[50,52.5]}}}
),
)
}
///|
test "comparison report" {
let (base, head) = sample_pair()
inspect(
compare_report(base, head, base_label="aaa", head_label="bbb", rounds=2),
content=(
#|# MoonBit benchmark: bbb against aaa
#|
#|Each cell is the change in time from the base to the head commit,
#|the geometric mean over the operation's inputs of the fastest of
#|2 round(s) on each side. Negative is faster. Changes of 5% or
#|more are in bold; compare them with the noise below.
#|
#|| operation | moonbit-native |
#||---|---|
#|| norm/nfc | **-29.3%** |
#|| idna/to_ascii | **+5.0%** |
#|
#|Noise, the median over cases of slowest round / fastest round: moonbit-native +10.0%.
#|
),
)
}
///|
test "snapshot as JSON" {
let (_, head) = sample_pair()
inspect(
head.snapshot_json({ "rounds": Json::number(1.0) }).stringify(),
content=(
#|{"about":{"rounds":1},"cases":{"norm/nfc/a":{"moonbit-native":1},"norm/nfc/b":{"moonbit-native":8},"norm/nfd/new":{"moonbit-native":3},"idna/to_ascii/x":{"moonbit-native":52.5}}}
),
)
}