unicode

Branch: main

Switch branches or tags

Branch: main
Go to file t

413 files indexed

History
Code

bench/xlang/main.mbt
1///|2/// Run `step` with extra arguments, returning its exit status and its output3/// (stdout only, or stdout and stderr merged for builds). `@shell` cannot4/// merge the two streams at the descriptor level, so a build's captured5/// streams are concatenated for the failure diagnostics below, and the6/// capture limit is raised above its 8 MiB default because build logs are7/// chatty.8async fn run_step(9  step : Step,10  extra~ : Array[String] = [],11  merge_stderr~ : Bool = false,12) -> (Int32, String) {13  let arguments = [..step.arguments, ..extra]14  let output = @shell.Cmd(15    step.program,16    arguments,17    cwd=step.cwd,18    stderr=if merge_stderr { Capture } else { Inherit },19  ).output(max_output_bytes=64 * 1024 * 1024)20  let text = if merge_stderr {21    "\{output.stdout()}\{output.stderr()}"22  } else {23    output.stdout()24  }25  (output.exit_code(), text)26}2728///|29/// Set `peer` up; `Some(reason)` when it cannot take part in the comparison,30/// typically because its toolchain, runtime or library is not installed.31async fn setup_failure(peer : Peer) -> String? {32  for step in peer.setup {33    let (status, output) = run_step(step, merge_stderr=true) catch {34      error if @async.is_being_cancelled() => raise error35      error => return Some("cannot run \{step.program}: \{error}")36    }37    if status != 0 {38      return Some("\{step.program} exited with \{status}\n\{output.trim()}")39    }40  }41  None42}4344///|45/// The first line that `command` prints through `sh` in `cwd`, or `None`46/// when it fails or prints nothing. Its errors are discarded.47async fn first_line(command : String, cwd : String) -> String? {48  let step = {49    program: "sh",50    arguments: ["-c", "\{command} 2>/dev/null"],51    cwd,52  }53  let (status, output) = run_step(step) catch {54    error if @async.is_being_cancelled() => raise error55    _ => return None56  }57  guard status == 0 else { return None }58  let line = match output.trim().split_once("\n") {59    Some((first, _)) => first.trim()60    None => output.trim()61  }62  if line.is_empty() {63    None64  } else {65    Some(line.to_owned())66  }67}6869///|70/// The processor and the operating system, for the record of a run.71async fn machine(root : String) -> String {72  let cpu = match first_line("sysctl -n machdep.cpu.brand_string", root) {73    Some(cpu) => cpu74    None =>75      first_line("sed -n 's/^model name[[:space:]]*: //p' /proc/cpuinfo", root).unwrap_or(76        "unknown processor",77      )78  }79  match first_line("uname -sm", root) {80    Some(system) => "\{cpu}, \{system}"81    None => cpu82  }83}8485///|86/// The commit checked out in `root`: its hash, subject, first line of the87/// body and commit date, and whether the checkout has uncommitted changes.88async fn commit_json(root : String) -> Json {89  let fields : Map[String, Json] = Map([])90  for91    (key, command) in [92      ("commit", "git log -1 --format=%H"),93      ("subject", "git log -1 --format=%s"),94      // For a merge commit of a pull request, its title.95      ("title", "git log -1 --format=%b"),96      ("date", "git log -1 --format=%cI"),97    ] {98    if first_line(command, root) is Some(value) {99      fields[key] = Json::string(value)100    }101  }102  fields["dirty"] = Json::boolean(103    first_line("git status --porcelain --untracked-files=no", root) is Some(_),104  )105  Json::object(fields)106}107108///|109/// A commit as `abc1234 (subject)` for the Markdown report.110fn commit_label(commit : Json) -> String {111  guard commit is Object(fields) else { return "unknown commit" }112  let hash = match fields.get("commit") {113    Some(String(hash)) if hash.length() > 7 => hash[:7].to_owned()114    Some(String(hash)) => hash115    _ => "unknown commit"116  }117  match fields.get("subject") {118    Some(String(subject)) => "\{hash} (\{subject})"119    _ => hash120  }121}122123///|124/// The versions of what the peers measure, one per label.125async fn versions(peers : Array[Peer], root : String) -> Json {126  let result : Map[String, Json] = Map([])127  for peer in peers {128    for version in peer.versions {129      let (label, command) = version130      if !result.contains(label) && first_line(command, root) is Some(text) {131        result[label] = Json::string(text)132      }133    }134  }135  Json::object(result)136}137138///|139/// The rows `peer` prints, or `None` when it fails; the failure is reported.140async fn run_peer(peer : Peer, extra : Array[String]) -> Array[Row]? {141  let (status, output) = run_step(peer.run, extra~) catch {142    error if @async.is_being_cancelled() => raise error143    error => {144      println("    Failed: \{error}")145      return None146    }147  }148  if status != 0 {149    println("    Failed: exit status \{status}; its results are dropped")150    return None151  }152  Some(parse_rows(peer.name, output))153}154155///|156/// Check `revision` out in `directory` as a detached worktree of the checkout157/// in `root`, replacing any worktree left there by an earlier run. The MoonBit158/// harness is then copied from `root`, so that both commits run the same159/// harness and a change to it does not show up as a change to the library.160/// The bench module's `moon.mod` goes with it, so that a harness importing a161/// module the base did not import yet still builds there.162async fn check_out(163  root : String,164  revision : String,165  directory : String,166) -> Unit {167  let git = (arguments : Array[String]) => {168    ({ program: "git", arguments, cwd: root, } : Step)169  }170  ignore(171    run_step(172      git(["worktree", "remove", "--force", directory]),173      merge_stderr=true,174    ),175  )176  ignore(run_step(git(["worktree", "prune"]), merge_stderr=true))177  let (status, output) = run_step(178    git(["worktree", "add", "--detach", directory, revision]),179    merge_stderr=true,180  )181  guard status == 0 else {182    fail("Cannot check out \{revision}: \{output.trim()}")183  }184  let harness = "bench/xlang/moonbit"185  let (status, output) = run_step(186    {187      program: "sh",188      arguments: [189        "-c", "rm -rf \"$2/$3\" && cp -R \"$1/$3\" \"$2/$3\" && cp \"$1/$4\" \"$2/$4\"",190        "sh", root, directory, harness, "bench/moon.mod",191      ],192      cwd: root,193    },194    merge_stderr=true,195  )196  guard status == 0 else {197    fail("Cannot copy the harness into \{directory}: \{output.trim()}")198  }199}200201///|202async fn main {203  let command = @argparse.Command(204    "xlang",205    about="Compare this repository's Unicode modules with other languages' libraries",206    flags=[207      @argparse.FlagArg(208        "detail",209        long="detail",210        about="Also report every case, not only the mean per operation",211      ),212    ],213    options=[214      @argparse.OptionArg(215        "rounds",216        long="rounds",217        about="Run every harness this many times and keep the fastest result",218        default_values=["3"],219      ),220      @argparse.OptionArg(221        "impl",222        long="impl",223        about="Comma-separated harnesses to run (default: all that can be set up)",224      ),225      @argparse.OptionArg(226        "group",227        long="group",228        about="Only run this group: \{groups.join(", ")}",229      ),230      @argparse.OptionArg(231        "base",232        long="base",233        about="Instead, compare the MoonBit harnesses with those of this commit, alternating the two",234      ),235    ],236  )237  let matches = command.parse()238  let detail = matches.flags.get("detail").unwrap_or(false)239  let rounds = match matches.values.get("rounds") {240    Some([value, ..]) => @string.parse_int(value)241    _ => 3242  }243  guard rounds >= 1 else { fail("--rounds must be at least 1, not \{rounds}") }244  let group = matches.values.get("group").map(values => values[0])245  if group is Some(group) && !groups.contains(group) {246    fail("Unknown group \{group}; choose from \{groups.join(", ")}")247  }248  let base = matches.values.get("base").map(values => values[0])249  guard @fs.exists("moon.work") && @fs.exists("bench/xlang/moon.pkg") else {250    fail("Run this from the repository root")251  }252  let root = @fs.realpath(".")253  let work = "\{root}/_build/xlang"254  let peers = all_peers(root, work)255  let selected = match matches.values.get("impl") {256    Some([names, ..]) => {257      let names = names.split(",").map(name => name.trim().to_owned()).collect()258      for name in names {259        guard peers.any(peer => peer.name == name) else {260          let known = peers.map(peer => peer.name).join(", ")261          fail("Unknown harness \{name}; choose from \{known}")262        }263      }264      peers.filter(peer => names.contains(peer.name))265    }266    // A comparison of two commits only concerns this repository's code.267    _ if base is Some(_) =>268      peers.filter(peer => peer.name.has_prefix("moonbit-"))269    _ => peers270  }271  if base is Some(_) {272    for peer in selected {273      guard peer.name.has_prefix("moonbit-") else {274        fail("--base compares MoonBit harnesses only, not \{peer.name}")275      }276    }277  }278  let inputs = "\{work}/inputs"279  println("==> Preparing inputs in \{inputs}")280  prepare_inputs(root, inputs)281  let base_root = "\{work}/base"282  if base is Some(revision) {283    println("==> Checking out \{revision} in \{base_root}")284    check_out(root, revision, base_root)285  }286  // Pairs of the harness for this checkout and, with --base, the same harness287  // for the base commit, measuring the same cases.288  let ready : Array[(Peer, Peer?)] = []289  for peer in selected {290    println("==> Setting up \{peer.name}")291    if setup_failure(peer) is Some(reason) {292      println("    Skipped: \{reason}")293      continue294    }295    guard base is Some(_) else {296      ready.push((peer, None))297      continue298    }299    let target = peer.name.strip_prefix("moonbit-").unwrap().to_owned()300    let base_peer = moonbit_peer(base_root, target)301    println("==> Setting up \{peer.name} at the base commit")302    match setup_failure(base_peer) {303      Some(reason) => println("    Skipped: \{reason}")304      None => ready.push((peer, Some(base_peer)))305    }306  }307  guard !ready.is_empty() else { fail("None of the harnesses could be set up") }308  let extra = if group is Some(group) { [inputs, group] } else { [inputs] }309  let results = Results(Map([]))310  let base_results = Results(Map([]))311  let raw = StringBuilder()312  raw.write_string("round\tcommit\tcolumn\tcase\tns\tunits\tcheck\n")313  // One harness at a time, so that they do not compete for the machine, and314  // round by round, so that a noisy period does not hit one harness only.315  // The two commits of a comparison take turns going first.316  for round in 1..<=rounds {317    for pair in ready {318      let (peer, base_peer) = pair319      let sides = match base_peer {320        None => [("head", peer, results)]321        Some(base_peer) if round % 2 == 1 =>322          [("base", base_peer, base_results), ("head", peer, results)]323        Some(base_peer) =>324          [("head", peer, results), ("base", base_peer, base_results)]325      }326      for side in sides {327        let (commit, peer, into) = side328        let label = if base_peer is Some(_) { " (\{commit})" } else { "" }329        println("==> Round \{round}/\{rounds}: \{peer.name}\{label}")330        guard run_peer(peer, extra) is Some(rows) else { continue }331        for row in rows {332          into.add(row)333          raw.write_string(334            "\{round}\t\{commit}\t\{row.column}\t\{row.name}\t\{row.ns}\t\{row.units}\t\{row.check}\n",335          )336        }337      }338    }339  }340  // A report without measurements would claim that every implementation agrees.341  guard !results.0.is_empty() else { fail("No harness produced any result") }342  let about : Map[String, Json] = {343    "date": Json::string(344      first_line("date -u +%Y-%m-%dT%H:%M:%SZ", root).unwrap_or(""),345    ),346    "rounds": Json::number(rounds.to_double()),347    "machine": Json::string(machine(root)),348    "versions": versions(ready.map(pair => pair.0), root),349  }350  write_text("\{work}/results.tsv", raw.to_string())351  match base {352    None => {353      about["commit"] = commit_json(root)354      let report = results.report(rounds~, detail~)355      write_text("\{work}/report.md", report)356      write_text(357        "\{work}/results.json",358        results.snapshot_json(about).stringify() + "\n",359      )360      println("")361      println(report)362      println(363        "Raw results: \{work}/results.tsv; this report: \{work}/report.md; for the dashboard: \{work}/results.json",364      )365    }366    Some(_) => {367      guard !base_results.0.is_empty() else {368        fail("The base commit produced no result")369      }370      let base_commit = commit_json(base_root)371      let head_commit = commit_json(root)372      about["base"] = base_commit373      about["head"] = head_commit374      let report = compare_report(375        base_results,376        results,377        base_label=commit_label(base_commit),378        head_label=commit_label(head_commit),379        rounds~,380      )381      write_text("\{work}/compare.md", report)382      write_text(383        "\{work}/compare.json",384        compare_json(base_results, results, about).stringify() + "\n",385      )386      println("")387      println(report)388      println(389        "Raw results: \{work}/results.tsv; this report: \{work}/compare.md; for the dashboard: \{work}/compare.json",390      )391    }392  }393}394