unicode

Branch: main

Switch branches or tags

Branch: main
Go to file t

413 files indexed

History
Code

bench/xlang/compare.mbt
1///|2/// `value` rounded to `digits` significant digits. More digits than the noise3/// allows only make the JSON larger.4fn round_significant(value : Double, digits : Int32) -> Double {5  guard value > 0.0 else { return value }6  let exponent = digits - 1 - @math.log10(value).floor().to_int()7  let scale = @math.pow(10.0, exponent.to_double())8  (value * scale).round() / scale9}1011///|12fn number(value : Double) -> Json {13  Json::number(round_significant(value, 5))14}1516///|17/// The `group/operation` part of a case name.18fn operation_of(name : String) -> String? {19  guard name.split("/").collect() is [group, operation, ..] else { return None }20  Some("\{group}/\{operation}")21}2223///|24/// One run as JSON: `about` describes it, and `cases` maps every case to its25/// cost in each column, in ns per unit of the fastest round.26fn Results::snapshot_json(self : Results, about : Map[String, Json]) -> Json {27  let cases : Map[String, Map[String, Json]] = Map([])28  for column, by_name in self.0 {29    for name, cell in by_name {30      cases.get_or_init(name, () => Map([]))[column] = number(cell.cost())31    }32  }33  let cases_json : Map[String, Json] = Map([])34  for name, columns in cases {35    cases_json[name] = Json::object(columns)36  }37  Json::object({38    "about": Json::object(about),39    "cases": Json::object(cases_json),40  })41}4243///|44/// For every operation and column measured by both runs, the geometric mean45/// over the operation's cases of head cost / base cost. Below 1, head is46/// faster. Cases only one of the runs measured are left out.47fn compare_operations(48  base : Results,49  head : Results,50) -> Map[String, Map[String, Double]] {51  let ratios : Map[String, Map[String, Array[Double]]] = Map([])52  for column, by_name in head.0 {53    for name, head_cell in by_name {54      guard base.cell(column, name) is Some(base_cell) else { continue }55      guard operation_of(name) is Some(operation) else { continue }56      ratios57      .get_or_init(operation, () => Map([]))58      .get_or_init(column, () => [])59      .push(head_cell.cost() / base_cell.cost())60    }61  }62  let result : Map[String, Map[String, Double]] = Map([])63  for operation, columns in ratios {64    let means : Map[String, Double] = Map([])65    for column, values in columns {66      if geometric_mean(values) is Some(mean) {67        means[column] = mean68      }69    }70    result[operation] = means71  }72  result73}7475///|76/// The larger of the two runs' spreads in each column: how much rounds of the77/// same code disagreed, so how large a change has to be to mean anything.78fn compare_spreads(base : Results, head : Results) -> Map[String, Double] {79  let spreads : Map[String, Double] = Map([])80  for column, _ in head.0 {81    match (base.spread(column), head.spread(column)) {82      (Some(a), Some(b)) => spreads[column] = if a > b { a } else { b }83      (Some(spread), None) | (None, Some(spread)) => spreads[column] = spread84      (None, None) => ()85    }86  }87  spreads88}8990///|91/// An A/B comparison as JSON: `about` describes it; `operations` holds the92/// ratios of `compare_operations`, `spread` those of `compare_spreads`, and93/// `cases` both costs of every case in each column, base first.94fn compare_json(95  base : Results,96  head : Results,97  about : Map[String, Json],98) -> Json {99  let operations : Map[String, Json] = Map([])100  for operation, columns in compare_operations(base, head) {101    let ratios : Map[String, Json] = Map([])102    for column, ratio in columns {103      ratios[column] = number(ratio)104    }105    operations[operation] = Json::object(ratios)106  }107  let spreads : Map[String, Json] = Map([])108  for column, spread in compare_spreads(base, head) {109    spreads[column] = number(spread)110  }111  let cases : Map[String, Map[String, Json]] = Map([])112  for column, by_name in head.0 {113    for name, head_cell in by_name {114      guard base.cell(column, name) is Some(base_cell) else { continue }115      cases.get_or_init(name, () => Map([]))[column] = Json::array([116        number(base_cell.cost()),117        number(head_cell.cost()),118      ])119    }120  }121  let cases_json : Map[String, Json] = Map([])122  for name, columns in cases {123    cases_json[name] = Json::object(columns)124  }125  Json::object({126    "about": Json::object(about),127    "operations": Json::object(operations),128    "spread": Json::object(spreads),129    "cases": Json::object(cases_json),130  })131}132133///|134/// A ratio as a signed percentage with one decimal: 1.034 is `+3.4%`.135fn format_change(ratio : Double) -> String {136  let tenths = ((ratio - 1.0) * 1000.0).round().to_int()137  let sign = if tenths > 0 { "+" } else if tenths < 0 { "-" } else { "" }138  let magnitude = tenths.abs()139  "\{sign}\{magnitude / 10}.\{magnitude % 10}%"140}141142///|143/// Changes at least this large are shown in bold in the Markdown report.144const Notable : Double = 0.05145146///|147/// The A/B comparison as Markdown: a table with a row per operation and a148/// column per harness, each cell the change from base to head.149fn compare_report(150  base : Results,151  head : Results,152  base_label~ : String,153  head_label~ : String,154  rounds~ : Int32,155) -> String {156  let out = StringBuilder()157  out.write_string(158    "# MoonBit benchmark: \{head_label} against \{base_label}\n\n",159  )160  out.write_string(161    (162      #|Each cell is the change in time from the base to the head commit,163      #|the geometric mean over the operation's inputs of the fastest of164      #|$rounds round(s) on each side. Negative is faster. Changes of 5% or165      #|more are in bold; compare them with the noise below.166      #|167    ).replace(old="$rounds", new=rounds.to_string()),168  )169  let operations = compare_operations(base, head)170  let columns = []171  for column, _ in head.0 {172    if operations.values().any(ratios => ratios.contains(column)) {173      columns.push(column)174    }175  }176  if columns.is_empty() {177    out.write_string("\nThe two runs have no case in common.\n")178    return out.to_string()179  }180  out.write_string("\n| operation |")181  for column in columns {182    out.write_string(" \{column} |")183  }184  out.write_string("\n|---|")185  for _ in columns {186    out.write_string("---|")187  }188  out.write_string("\n")189  for operation, ratios in operations {190    out.write_string("| \{operation} |")191    for column in columns {192      match ratios.get(column) {193        Some(ratio) => {194          let change = format_change(ratio)195          let notable = ratio >= 1.0 + Notable || ratio <= 1.0 - Notable196          out.write_string(197            if notable {198              " **\{change}** |"199            } else {200              " \{change} |"201            },202          )203        }204        None => out.write_string(" – |")205      }206    }207    out.write_string("\n")208  }209  let spreads = compare_spreads(base, head)210  // A single round cannot tell how noisy the machine is.211  if rounds > 1 && !spreads.is_empty() {212    out.write_string(213      "\nNoise, the median over cases of slowest round / fastest round:",214    )215    let parts = [216      for column in columns => {217        if spreads.get(column) is Some(spread) {218          "\{column} \{format_change(spread)}"219        } else {220          "\{column} –"221        }222      }223    ]224    out.write_string(" \{parts.join(", ")}.\n")225  }226  out.to_string()227}228