bench/xlang/main.mbt1///|2/// Run `step` with extra arguments, returning its exit status and its output3/// (stdout only, or stdout and stderr merged for builds). `@shell` cannot4/// merge the two streams at the descriptor level, so a build's captured5/// streams are concatenated for the failure diagnostics below, and the6/// capture limit is raised above its 8 MiB default because build logs are7/// chatty.8async fn run_step(9 step : Step,10 extra~ : Array[String] = [],11 merge_stderr~ : Bool = false,12) -> (Int32, String) {13 let arguments = [..step.arguments, ..extra]14 let output = @shell.Cmd(15 step.program,16 arguments,17 cwd=step.cwd,18 stderr=if merge_stderr { Capture } else { Inherit },19 ).output(max_output_bytes=64 * 1024 * 1024)20 let text = if merge_stderr {21 "\{output.stdout()}\{output.stderr()}"22 } else {23 output.stdout()24 }25 (output.exit_code(), text)26}2728///|29/// Set `peer` up; `Some(reason)` when it cannot take part in the comparison,30/// typically because its toolchain, runtime or library is not installed.31async fn setup_failure(peer : Peer) -> String? {32 for step in peer.setup {33 let (status, output) = run_step(step, merge_stderr=true) catch {34 error if @async.is_being_cancelled() => raise error35 error => return Some("cannot run \{step.program}: \{error}")36 }37 if status != 0 {38 return Some("\{step.program} exited with \{status}\n\{output.trim()}")39 }40 }41 None42}4344///|45/// The first line that `command` prints through `sh` in `cwd`, or `None`46/// when it fails or prints nothing. Its errors are discarded.47async fn first_line(command : String, cwd : String) -> String? {48 let step = {49 program: "sh",50 arguments: ["-c", "\{command} 2>/dev/null"],51 cwd,52 }53 let (status, output) = run_step(step) catch {54 error if @async.is_being_cancelled() => raise error55 _ => return None56 }57 guard status == 0 else { return None }58 let line = match output.trim().split_once("\n") {59 Some((first, _)) => first.trim()60 None => output.trim()61 }62 if line.is_empty() {63 None64 } else {65 Some(line.to_owned())66 }67}6869///|70/// The processor and the operating system, for the record of a run.71async fn machine(root : String) -> String {72 let cpu = match first_line("sysctl -n machdep.cpu.brand_string", root) {73 Some(cpu) => cpu74 None =>75 first_line("sed -n 's/^model name[[:space:]]*: //p' /proc/cpuinfo", root).unwrap_or(76 "unknown processor",77 )78 }79 match first_line("uname -sm", root) {80 Some(system) => "\{cpu}, \{system}"81 None => cpu82 }83}8485///|86/// The commit checked out in `root`: its hash, subject, first line of the87/// body and commit date, and whether the checkout has uncommitted changes.88async fn commit_json(root : String) -> Json {89 let fields : Map[String, Json] = Map([])90 for91 (key, command) in [92 ("commit", "git log -1 --format=%H"),93 ("subject", "git log -1 --format=%s"),94 // For a merge commit of a pull request, its title.95 ("title", "git log -1 --format=%b"),96 ("date", "git log -1 --format=%cI"),97 ] {98 if first_line(command, root) is Some(value) {99 fields[key] = Json::string(value)100 }101 }102 fields["dirty"] = Json::boolean(103 first_line("git status --porcelain --untracked-files=no", root) is Some(_),104 )105 Json::object(fields)106}107108///|109/// A commit as `abc1234 (subject)` for the Markdown report.110fn commit_label(commit : Json) -> String {111 guard commit is Object(fields) else { return "unknown commit" }112 let hash = match fields.get("commit") {113 Some(String(hash)) if hash.length() > 7 => hash[:7].to_owned()114 Some(String(hash)) => hash115 _ => "unknown commit"116 }117 match fields.get("subject") {118 Some(String(subject)) => "\{hash} (\{subject})"119 _ => hash120 }121}122123///|124/// The versions of what the peers measure, one per label.125async fn versions(peers : Array[Peer], root : String) -> Json {126 let result : Map[String, Json] = Map([])127 for peer in peers {128 for version in peer.versions {129 let (label, command) = version130 if !result.contains(label) && first_line(command, root) is Some(text) {131 result[label] = Json::string(text)132 }133 }134 }135 Json::object(result)136}137138///|139/// The rows `peer` prints, or `None` when it fails; the failure is reported.140async fn run_peer(peer : Peer, extra : Array[String]) -> Array[Row]? {141 let (status, output) = run_step(peer.run, extra~) catch {142 error if @async.is_being_cancelled() => raise error143 error => {144 println(" Failed: \{error}")145 return None146 }147 }148 if status != 0 {149 println(" Failed: exit status \{status}; its results are dropped")150 return None151 }152 Some(parse_rows(peer.name, output))153}154155///|156/// Check `revision` out in `directory` as a detached worktree of the checkout157/// in `root`, replacing any worktree left there by an earlier run. The MoonBit158/// harness is then copied from `root`, so that both commits run the same159/// harness and a change to it does not show up as a change to the library.160/// The bench module's `moon.mod` goes with it, so that a harness importing a161/// module the base did not import yet still builds there.162async fn check_out(163 root : String,164 revision : String,165 directory : String,166) -> Unit {167 let git = (arguments : Array[String]) => {168 ({ program: "git", arguments, cwd: root, } : Step)169 }170 ignore(171 run_step(172 git(["worktree", "remove", "--force", directory]),173 merge_stderr=true,174 ),175 )176 ignore(run_step(git(["worktree", "prune"]), merge_stderr=true))177 let (status, output) = run_step(178 git(["worktree", "add", "--detach", directory, revision]),179 merge_stderr=true,180 )181 guard status == 0 else {182 fail("Cannot check out \{revision}: \{output.trim()}")183 }184 let harness = "bench/xlang/moonbit"185 let (status, output) = run_step(186 {187 program: "sh",188 arguments: [189 "-c", "rm -rf \"$2/$3\" && cp -R \"$1/$3\" \"$2/$3\" && cp \"$1/$4\" \"$2/$4\"",190 "sh", root, directory, harness, "bench/moon.mod",191 ],192 cwd: root,193 },194 merge_stderr=true,195 )196 guard status == 0 else {197 fail("Cannot copy the harness into \{directory}: \{output.trim()}")198 }199}200201///|202async fn main {203 let command = @argparse.Command(204 "xlang",205 about="Compare this repository's Unicode modules with other languages' libraries",206 flags=[207 @argparse.FlagArg(208 "detail",209 long="detail",210 about="Also report every case, not only the mean per operation",211 ),212 ],213 options=[214 @argparse.OptionArg(215 "rounds",216 long="rounds",217 about="Run every harness this many times and keep the fastest result",218 default_values=["3"],219 ),220 @argparse.OptionArg(221 "impl",222 long="impl",223 about="Comma-separated harnesses to run (default: all that can be set up)",224 ),225 @argparse.OptionArg(226 "group",227 long="group",228 about="Only run this group: \{groups.join(", ")}",229 ),230 @argparse.OptionArg(231 "base",232 long="base",233 about="Instead, compare the MoonBit harnesses with those of this commit, alternating the two",234 ),235 ],236 )237 let matches = command.parse()238 let detail = matches.flags.get("detail").unwrap_or(false)239 let rounds = match matches.values.get("rounds") {240 Some([value, ..]) => @string.parse_int(value)241 _ => 3242 }243 guard rounds >= 1 else { fail("--rounds must be at least 1, not \{rounds}") }244 let group = matches.values.get("group").map(values => values[0])245 if group is Some(group) && !groups.contains(group) {246 fail("Unknown group \{group}; choose from \{groups.join(", ")}")247 }248 let base = matches.values.get("base").map(values => values[0])249 guard @fs.exists("moon.work") && @fs.exists("bench/xlang/moon.pkg") else {250 fail("Run this from the repository root")251 }252 let root = @fs.realpath(".")253 let work = "\{root}/_build/xlang"254 let peers = all_peers(root, work)255 let selected = match matches.values.get("impl") {256 Some([names, ..]) => {257 let names = names.split(",").map(name => name.trim().to_owned()).collect()258 for name in names {259 guard peers.any(peer => peer.name == name) else {260 let known = peers.map(peer => peer.name).join(", ")261 fail("Unknown harness \{name}; choose from \{known}")262 }263 }264 peers.filter(peer => names.contains(peer.name))265 }266 // A comparison of two commits only concerns this repository's code.267 _ if base is Some(_) =>268 peers.filter(peer => peer.name.has_prefix("moonbit-"))269 _ => peers270 }271 if base is Some(_) {272 for peer in selected {273 guard peer.name.has_prefix("moonbit-") else {274 fail("--base compares MoonBit harnesses only, not \{peer.name}")275 }276 }277 }278 let inputs = "\{work}/inputs"279 println("==> Preparing inputs in \{inputs}")280 prepare_inputs(root, inputs)281 let base_root = "\{work}/base"282 if base is Some(revision) {283 println("==> Checking out \{revision} in \{base_root}")284 check_out(root, revision, base_root)285 }286 // Pairs of the harness for this checkout and, with --base, the same harness287 // for the base commit, measuring the same cases.288 let ready : Array[(Peer, Peer?)] = []289 for peer in selected {290 println("==> Setting up \{peer.name}")291 if setup_failure(peer) is Some(reason) {292 println(" Skipped: \{reason}")293 continue294 }295 guard base is Some(_) else {296 ready.push((peer, None))297 continue298 }299 let target = peer.name.strip_prefix("moonbit-").unwrap().to_owned()300 let base_peer = moonbit_peer(base_root, target)301 println("==> Setting up \{peer.name} at the base commit")302 match setup_failure(base_peer) {303 Some(reason) => println(" Skipped: \{reason}")304 None => ready.push((peer, Some(base_peer)))305 }306 }307 guard !ready.is_empty() else { fail("None of the harnesses could be set up") }308 let extra = if group is Some(group) { [inputs, group] } else { [inputs] }309 let results = Results(Map([]))310 let base_results = Results(Map([]))311 let raw = StringBuilder()312 raw.write_string("round\tcommit\tcolumn\tcase\tns\tunits\tcheck\n")313 // One harness at a time, so that they do not compete for the machine, and314 // round by round, so that a noisy period does not hit one harness only.315 // The two commits of a comparison take turns going first.316 for round in 1..<=rounds {317 for pair in ready {318 let (peer, base_peer) = pair319 let sides = match base_peer {320 None => [("head", peer, results)]321 Some(base_peer) if round % 2 == 1 =>322 [("base", base_peer, base_results), ("head", peer, results)]323 Some(base_peer) =>324 [("head", peer, results), ("base", base_peer, base_results)]325 }326 for side in sides {327 let (commit, peer, into) = side328 let label = if base_peer is Some(_) { " (\{commit})" } else { "" }329 println("==> Round \{round}/\{rounds}: \{peer.name}\{label}")330 guard run_peer(peer, extra) is Some(rows) else { continue }331 for row in rows {332 into.add(row)333 raw.write_string(334 "\{round}\t\{commit}\t\{row.column}\t\{row.name}\t\{row.ns}\t\{row.units}\t\{row.check}\n",335 )336 }337 }338 }339 }340 // A report without measurements would claim that every implementation agrees.341 guard !results.0.is_empty() else { fail("No harness produced any result") }342 let about : Map[String, Json] = {343 "date": Json::string(344 first_line("date -u +%Y-%m-%dT%H:%M:%SZ", root).unwrap_or(""),345 ),346 "rounds": Json::number(rounds.to_double()),347 "machine": Json::string(machine(root)),348 "versions": versions(ready.map(pair => pair.0), root),349 }350 write_text("\{work}/results.tsv", raw.to_string())351 match base {352 None => {353 about["commit"] = commit_json(root)354 let report = results.report(rounds~, detail~)355 write_text("\{work}/report.md", report)356 write_text(357 "\{work}/results.json",358 results.snapshot_json(about).stringify() + "\n",359 )360 println("")361 println(report)362 println(363 "Raw results: \{work}/results.tsv; this report: \{work}/report.md; for the dashboard: \{work}/results.json",364 )365 }366 Some(_) => {367 guard !base_results.0.is_empty() else {368 fail("The base commit produced no result")369 }370 let base_commit = commit_json(base_root)371 let head_commit = commit_json(root)372 about["base"] = base_commit373 about["head"] = head_commit374 let report = compare_report(375 base_results,376 results,377 base_label=commit_label(base_commit),378 head_label=commit_label(head_commit),379 rounds~,380 )381 write_text("\{work}/compare.md", report)382 write_text(383 "\{work}/compare.json",384 compare_json(base_results, results, about).stringify() + "\n",385 )386 println("")387 println(report)388 println(389 "Raw results: \{work}/results.tsv; this report: \{work}/compare.md; for the dashboard: \{work}/compare.json",390 )391 }392 }393}394
///|
/// Run `step` with extra arguments, returning its exit status and its output
/// (stdout only, or stdout and stderr merged for builds). `@shell` cannot
/// merge the two streams at the descriptor level, so a build's captured
/// streams are concatenated for the failure diagnostics below, and the
/// capture limit is raised above its 8 MiB default because build logs are
/// chatty.
async fn run_step(
step : Step,
extra~ : Array[String] = [],
merge_stderr~ : Bool = false,
) -> (Int32, String) {
let arguments = [..step.arguments, ..extra]
let output = @shell.Cmd(
step.program,
arguments,
cwd=step.cwd,
stderr=if merge_stderr { Capture } else { Inherit },
).output(max_output_bytes=64 * 1024 * 1024)
let text = if merge_stderr {
"\{output.stdout()}\{output.stderr()}"
} else {
output.stdout()
}
(output.exit_code(), text)
}
///|
/// Set `peer` up; `Some(reason)` when it cannot take part in the comparison,
/// typically because its toolchain, runtime or library is not installed.
async fn setup_failure(peer : Peer) -> String? {
for step in peer.setup {
let (status, output) = run_step(step, merge_stderr=true) catch {
error if @async.is_being_cancelled() => raise error
error => return Some("cannot run \{step.program}: \{error}")
}
if status != 0 {
return Some("\{step.program} exited with \{status}\n\{output.trim()}")
}
}
None
}
///|
/// The first line that `command` prints through `sh` in `cwd`, or `None`
/// when it fails or prints nothing. Its errors are discarded.
async fn first_line(command : String, cwd : String) -> String? {
let step = {
program: "sh",
arguments: ["-c", "\{command} 2>/dev/null"],
cwd,
}
let (status, output) = run_step(step) catch {
error if @async.is_being_cancelled() => raise error
_ => return None
}
guard status == 0 else { return None }
let line = match output.trim().split_once("\n") {
Some((first, _)) => first.trim()
None => output.trim()
}
if line.is_empty() {
None
} else {
Some(line.to_owned())
}
}
///|
/// The processor and the operating system, for the record of a run.
async fn machine(root : String) -> String {
let cpu = match first_line("sysctl -n machdep.cpu.brand_string", root) {
Some(cpu) => cpu
None =>
first_line("sed -n 's/^model name[[:space:]]*: //p' /proc/cpuinfo", root).unwrap_or(
"unknown processor",
)
}
match first_line("uname -sm", root) {
Some(system) => "\{cpu}, \{system}"
None => cpu
}
}
///|
/// The commit checked out in `root`: its hash, subject, first line of the
/// body and commit date, and whether the checkout has uncommitted changes.
async fn commit_json(root : String) -> Json {
let fields : Map[String, Json] = Map([])
for
(key, command) in [
("commit", "git log -1 --format=%H"),
("subject", "git log -1 --format=%s"),
// For a merge commit of a pull request, its title.
("title", "git log -1 --format=%b"),
("date", "git log -1 --format=%cI"),
] {
if first_line(command, root) is Some(value) {
fields[key] = Json::string(value)
}
}
fields["dirty"] = Json::boolean(
first_line("git status --porcelain --untracked-files=no", root) is Some(_),
)
Json::object(fields)
}
///|
/// A commit as `abc1234 (subject)` for the Markdown report.
fn commit_label(commit : Json) -> String {
guard commit is Object(fields) else { return "unknown commit" }
let hash = match fields.get("commit") {
Some(String(hash)) if hash.length() > 7 => hash[:7].to_owned()
Some(String(hash)) => hash
_ => "unknown commit"
}
match fields.get("subject") {
Some(String(subject)) => "\{hash} (\{subject})"
_ => hash
}
}
///|
/// The versions of what the peers measure, one per label.
async fn versions(peers : Array[Peer], root : String) -> Json {
let result : Map[String, Json] = Map([])
for peer in peers {
for version in peer.versions {
let (label, command) = version
if !result.contains(label) && first_line(command, root) is Some(text) {
result[label] = Json::string(text)
}
}
}
Json::object(result)
}
///|
/// The rows `peer` prints, or `None` when it fails; the failure is reported.
async fn run_peer(peer : Peer, extra : Array[String]) -> Array[Row]? {
let (status, output) = run_step(peer.run, extra~) catch {
error if @async.is_being_cancelled() => raise error
error => {
println(" Failed: \{error}")
return None
}
}
if status != 0 {
println(" Failed: exit status \{status}; its results are dropped")
return None
}
Some(parse_rows(peer.name, output))
}
///|
/// Check `revision` out in `directory` as a detached worktree of the checkout
/// in `root`, replacing any worktree left there by an earlier run. The MoonBit
/// harness is then copied from `root`, so that both commits run the same
/// harness and a change to it does not show up as a change to the library.
/// The bench module's `moon.mod` goes with it, so that a harness importing a
/// module the base did not import yet still builds there.
async fn check_out(
root : String,
revision : String,
directory : String,
) -> Unit {
let git = (arguments : Array[String]) => {
({ program: "git", arguments, cwd: root, } : Step)
}
ignore(
run_step(
git(["worktree", "remove", "--force", directory]),
merge_stderr=true,
),
)
ignore(run_step(git(["worktree", "prune"]), merge_stderr=true))
let (status, output) = run_step(
git(["worktree", "add", "--detach", directory, revision]),
merge_stderr=true,
)
guard status == 0 else {
fail("Cannot check out \{revision}: \{output.trim()}")
}
let harness = "bench/xlang/moonbit"
let (status, output) = run_step(
{
program: "sh",
arguments: [
"-c", "rm -rf \"$2/$3\" && cp -R \"$1/$3\" \"$2/$3\" && cp \"$1/$4\" \"$2/$4\"",
"sh", root, directory, harness, "bench/moon.mod",
],
cwd: root,
},
merge_stderr=true,
)
guard status == 0 else {
fail("Cannot copy the harness into \{directory}: \{output.trim()}")
}
}
///|
async fn main {
let command = @argparse.Command(
"xlang",
about="Compare this repository's Unicode modules with other languages' libraries",
flags=[
@argparse.FlagArg(
"detail",
long="detail",
about="Also report every case, not only the mean per operation",
),
],
options=[
@argparse.OptionArg(
"rounds",
long="rounds",
about="Run every harness this many times and keep the fastest result",
default_values=["3"],
),
@argparse.OptionArg(
"impl",
long="impl",
about="Comma-separated harnesses to run (default: all that can be set up)",
),
@argparse.OptionArg(
"group",
long="group",
about="Only run this group: \{groups.join(", ")}",
),
@argparse.OptionArg(
"base",
long="base",
about="Instead, compare the MoonBit harnesses with those of this commit, alternating the two",
),
],
)
let matches = command.parse()
let detail = matches.flags.get("detail").unwrap_or(false)
let rounds = match matches.values.get("rounds") {
Some([value, ..]) => @string.parse_int(value)
_ => 3
}
guard rounds >= 1 else { fail("--rounds must be at least 1, not \{rounds}") }
let group = matches.values.get("group").map(values => values[0])
if group is Some(group) && !groups.contains(group) {
fail("Unknown group \{group}; choose from \{groups.join(", ")}")
}
let base = matches.values.get("base").map(values => values[0])
guard @fs.exists("moon.work") && @fs.exists("bench/xlang/moon.pkg") else {
fail("Run this from the repository root")
}
let root = @fs.realpath(".")
let work = "\{root}/_build/xlang"
let peers = all_peers(root, work)
let selected = match matches.values.get("impl") {
Some([names, ..]) => {
let names = names.split(",").map(name => name.trim().to_owned()).collect()
for name in names {
guard peers.any(peer => peer.name == name) else {
let known = peers.map(peer => peer.name).join(", ")
fail("Unknown harness \{name}; choose from \{known}")
}
}
peers.filter(peer => names.contains(peer.name))
}
// A comparison of two commits only concerns this repository's code.
_ if base is Some(_) =>
peers.filter(peer => peer.name.has_prefix("moonbit-"))
_ => peers
}
if base is Some(_) {
for peer in selected {
guard peer.name.has_prefix("moonbit-") else {
fail("--base compares MoonBit harnesses only, not \{peer.name}")
}
}
}
let inputs = "\{work}/inputs"
println("==> Preparing inputs in \{inputs}")
prepare_inputs(root, inputs)
let base_root = "\{work}/base"
if base is Some(revision) {
println("==> Checking out \{revision} in \{base_root}")
check_out(root, revision, base_root)
}
// Pairs of the harness for this checkout and, with --base, the same harness
// for the base commit, measuring the same cases.
let ready : Array[(Peer, Peer?)] = []
for peer in selected {
println("==> Setting up \{peer.name}")
if setup_failure(peer) is Some(reason) {
println(" Skipped: \{reason}")
continue
}
guard base is Some(_) else {
ready.push((peer, None))
continue
}
let target = peer.name.strip_prefix("moonbit-").unwrap().to_owned()
let base_peer = moonbit_peer(base_root, target)
println("==> Setting up \{peer.name} at the base commit")
match setup_failure(base_peer) {
Some(reason) => println(" Skipped: \{reason}")
None => ready.push((peer, Some(base_peer)))
}
}
guard !ready.is_empty() else { fail("None of the harnesses could be set up") }
let extra = if group is Some(group) { [inputs, group] } else { [inputs] }
let results = Results(Map([]))
let base_results = Results(Map([]))
let raw = StringBuilder()
raw.write_string("round\tcommit\tcolumn\tcase\tns\tunits\tcheck\n")
// One harness at a time, so that they do not compete for the machine, and
// round by round, so that a noisy period does not hit one harness only.
// The two commits of a comparison take turns going first.
for round in 1..<=rounds {
for pair in ready {
let (peer, base_peer) = pair
let sides = match base_peer {
None => [("head", peer, results)]
Some(base_peer) if round % 2 == 1 =>
[("base", base_peer, base_results), ("head", peer, results)]
Some(base_peer) =>
[("head", peer, results), ("base", base_peer, base_results)]
}
for side in sides {
let (commit, peer, into) = side
let label = if base_peer is Some(_) { " (\{commit})" } else { "" }
println("==> Round \{round}/\{rounds}: \{peer.name}\{label}")
guard run_peer(peer, extra) is Some(rows) else { continue }
for row in rows {
into.add(row)
raw.write_string(
"\{round}\t\{commit}\t\{row.column}\t\{row.name}\t\{row.ns}\t\{row.units}\t\{row.check}\n",
)
}
}
}
}
// A report without measurements would claim that every implementation agrees.
guard !results.0.is_empty() else { fail("No harness produced any result") }
let about : Map[String, Json] = {
"date": Json::string(
first_line("date -u +%Y-%m-%dT%H:%M:%SZ", root).unwrap_or(""),
),
"rounds": Json::number(rounds.to_double()),
"machine": Json::string(machine(root)),
"versions": versions(ready.map(pair => pair.0), root),
}
write_text("\{work}/results.tsv", raw.to_string())
match base {
None => {
about["commit"] = commit_json(root)
let report = results.report(rounds~, detail~)
write_text("\{work}/report.md", report)
write_text(
"\{work}/results.json",
results.snapshot_json(about).stringify() + "\n",
)
println("")
println(report)
println(
"Raw results: \{work}/results.tsv; this report: \{work}/report.md; for the dashboard: \{work}/results.json",
)
}
Some(_) => {
guard !base_results.0.is_empty() else {
fail("The base commit produced no result")
}
let base_commit = commit_json(base_root)
let head_commit = commit_json(root)
about["base"] = base_commit
about["head"] = head_commit
let report = compare_report(
base_results,
results,
base_label=commit_label(base_commit),
head_label=commit_label(head_commit),
rounds~,
)
write_text("\{work}/compare.md", report)
write_text(
"\{work}/compare.json",
compare_json(base_results, results, about).stringify() + "\n",
)
println("")
println(report)
println(
"Raw results: \{work}/results.tsv; this report: \{work}/compare.md; for the dashboard: \{work}/compare.json",
)
}
}
}