bench/xlang/moonbit/main.mbt1///|2fn read(path : String) -> String raise {3 @utf8.decode(@fs.read_file_to_bytes(path))4}56///|7/// Input names in `directory`: the `.txt` files without their extension.8fn names(directory : String) -> Array[String] raise {9 let result = []10 for file in @fs.read_dir(directory) {11 if file.strip_suffix(".txt") is Some(name) {12 result.push(name.to_owned())13 }14 }15 result.sort()16 result17}1819///|20/// The `name<TAB>value` lines of a `.tsv` input.21fn pairs(path : String) -> Array[(String, String)] raise {22 let result = []23 for line in read(path).split("\n") {24 if line.split_once("\t") is Some((name, value)) {25 result.push((name.to_owned(), value.to_owned()))26 }27 }28 result29}3031///|32/// Elapsed microseconds for `iterations` calls. The results are folded into33/// the returned sink so that the calls cannot be optimized away.34fn run(iterations : Int32, f : () -> Int32) -> (Double, Int32) {35 let mut sink = 036 let start = @bench.monotonic_clock_start()37 for _ in 0..<iterations {38 sink = sink ^ f()39 }40 (@bench.monotonic_clock_end(start), sink)41}4243///|44/// The latest string a case built. Every result is stored here, where it can45/// be read later, so that the optimizer cannot drop the work that builds it:46/// V8 drops a string that only has its length read, as it can compute the47/// length without the string.48let kept_string : Ref[String] = Ref("")4950///|51/// Keep `result` and return its length.52fn keep_string(result : String) -> Int32 {53 kept_string.val = result54 result.length()55}5657///|58/// The latest bidi levels a case built, kept as `kept_string` keeps strings.59let kept_levels : Ref[Array[Int32]] = Ref([])6061///|62/// Keep `levels` and return their number.63fn keep_levels(levels : Array[Int32]) -> Int32 {64 kept_levels.val = levels65 levels.length()66}6768///|69/// Print one result row: `library<TAB>case<TAB>ns<TAB>units<TAB>check`.70///71/// Every harness in `bench/xlang` follows the same protocol: double the72/// iteration count until one sample takes at least 5 ms, then report the73/// fastest of 5 samples, the first of which is that one. `units` is the74/// number of code points processed per call (1 for per-call cases) and75/// `check` is a value every implementation must agree on. The library column76/// is empty: this harness has one library.77fn measure(78 name : String,79 units : Int32,80 check : Int32,81 f : () -> Int32,82) -> Unit {83 let mut iterations = 184 let mut best = run(iterations, f).085 while best < 5000.0 {86 iterations = iterations * 287 best = run(iterations, f).088 }89 for _ in 1..<5 {90 let elapsed = run(iterations, f).091 if elapsed < best {92 best = elapsed93 }94 }95 let ns = best * 1000.0 / iterations.to_double()96 println("\t\{name}\t\{ns}\t\{units}\t\{check}")97}9899///|100fn bench_normalization(inputs : String) -> Unit raise {101 for name in names("\{inputs}/norm") {102 let original = read("\{inputs}/norm/\{name}.txt")103 let decomposed = read("\{inputs}/normd/\{name}.txt")104 let units = original.char_length()105 for106 (label, form) in [107 ("nfc", @normalization.NFC),108 ("nfd", @normalization.NFD),109 ("nfkc", @normalization.NFKC),110 ("nfkd", @normalization.NFKD),111 ] {112 measure(113 "norm/\{label}/\{name}",114 units,115 @normalization.normalize(original, form).char_length(),116 () => keep_string(@normalization.normalize(original, form)),117 )118 }119 measure(120 "norm/nfd>nfc/\{name}",121 decomposed.char_length(),122 @normalization.normalize(decomposed, NFC).char_length(),123 () => keep_string(@normalization.normalize(decomposed, NFC)),124 )125 let is_nfc = () => {126 if @normalization.is_normalized(original, NFC) {127 1128 } else {129 0130 }131 }132 measure("norm/is_nfc/\{name}", units, is_nfc(), is_nfc)133 }134}135136///|137fn bench_segmentation(inputs : String) -> Unit raise {138 for name in names("\{inputs}/seg") {139 let text = read("\{inputs}/seg/\{name}.txt")140 let count = () => {141 let cursor = @segmentation.graphemes(text)142 let mut clusters = 0143 while cursor.next().length() > 0 {144 clusters += 1145 }146 clusters147 }148 measure("seg/graphemes/\{name}", text.char_length(), count(), count)149 }150}151152///|153fn bench_bidi(inputs : String) -> Unit raise {154 for name in names("\{inputs}/bidi") {155 let text = read("\{inputs}/bidi/\{name}.txt")156 let mut level_sum = 0157 for level in @bidi.process(text).levels {158 level_sum += level159 }160 measure("bidi/process/\{name}", text.char_length(), level_sum, () => {161 keep_levels(@bidi.process(text).levels)162 })163 }164}165166///|167fn bench_idna(inputs : String) -> Unit raise {168 for (name, domain) in pairs("\{inputs}/idna.tsv") {169 // Every input converts successfully without STD3 rules, so the closures170 // time successful conversions only.171 let ascii = @idna.to_ascii(domain, use_std3_ascii_rules=false)172 let unicode = @idna.to_unicode(domain, use_std3_ascii_rules=false)173 measure("idna/to_ascii/\{name}", 1, ascii.char_length(), () => {174 keep_string(175 @idna.to_ascii(domain, use_std3_ascii_rules=false) catch {176 _ => domain177 },178 )179 })180 measure("idna/to_unicode/\{name}", 1, unicode.char_length(), () => {181 keep_string(182 @idna.to_unicode(domain, use_std3_ascii_rules=false) catch {183 _ => domain184 },185 )186 })187 }188}189190///|191fn bench_punycode(inputs : String) -> Unit raise {192 for (name, input) in pairs("\{inputs}/puny.tsv") {193 let encoded = @punycode.encode(input)194 measure("puny/encode/\{name}", 1, encoded.length(), () => {195 keep_string(@punycode.encode(input) catch { _ => input })196 })197 measure("puny/decode/\{name}", 1, input.char_length(), () => {198 keep_string(@punycode.decode(encoded) catch { _ => encoded })199 })200 }201}202203///|204fn bench_properties(inputs : String) -> Unit raise {205 for name in names("\{inputs}/case") {206 let text = read("\{inputs}/case/\{name}.txt")207 let units = text.char_length()208 let simple_upper = () => {209 let mut sum = 0210 for c in text {211 sum = sum ^ @ucd.to_simple_uppercase(c).to_int()212 }213 sum214 }215 let lowercase_letters = () => {216 let mut count = 0217 for c in text {218 if @ucd.general_category(c) is Ll {219 count += 1220 }221 }222 count223 }224 measure("case/simple_upper/\{name}", units, simple_upper(), simple_upper)225 measure(226 "case/full_upper/\{name}",227 units,228 @ucd.uppercase(text).char_length(),229 () => keep_string(@ucd.uppercase(text)),230 )231 for232 (label, fold) in [233 ("fold", @casefold.case_fold),234 ("nfkc_casefold", @casefold.nfkc_casefold),235 ] {236 measure("case/\{label}/\{name}", units, fold(text).char_length(), () => {237 keep_string(fold(text))238 })239 }240 measure(241 "prop/general_category_Ll/\{name}",242 units,243 lowercase_letters(),244 lowercase_letters,245 )246 }247}248249///|250fn main raise {251 let command = @argparse.Command(252 "xlang-moonbit",253 about="Time this repository's Unicode modules on the shared xlang inputs",254 positionals=[255 @argparse.PositionArg("inputs", about="Directory of prepared inputs"),256 @argparse.PositionArg(257 "group",258 about="Only run this group: norm, seg, bidi, idna, puny or case",259 num_args=@argparse.ValueRange(lower=0, upper=1),260 ),261 ],262 )263 let matches = command.parse()264 guard matches.values.get("inputs") is Some([inputs, ..]) else {265 fail(command.render_help())266 }267 let group = matches.values.get("group").map(values => values[0])268 let groups : Array[(String, (String) -> Unit raise)] = [269 ("norm", bench_normalization),270 ("seg", bench_segmentation),271 ("bidi", bench_bidi),272 ("idna", bench_idna),273 ("puny", bench_punycode),274 ("case", bench_properties),275 ]276 for (name, bench) in groups {277 if group is Some(selected) && selected != name {278 continue279 }280 bench(inputs)281 }282}283
///|
fn read(path : String) -> String raise {
@utf8.decode(@fs.read_file_to_bytes(path))
}
///|
/// Input names in `directory`: the `.txt` files without their extension.
fn names(directory : String) -> Array[String] raise {
let result = []
for file in @fs.read_dir(directory) {
if file.strip_suffix(".txt") is Some(name) {
result.push(name.to_owned())
}
}
result.sort()
result
}
///|
/// The `name<TAB>value` lines of a `.tsv` input.
fn pairs(path : String) -> Array[(String, String)] raise {
let result = []
for line in read(path).split("\n") {
if line.split_once("\t") is Some((name, value)) {
result.push((name.to_owned(), value.to_owned()))
}
}
result
}
///|
/// Elapsed microseconds for `iterations` calls. The results are folded into
/// the returned sink so that the calls cannot be optimized away.
fn run(iterations : Int32, f : () -> Int32) -> (Double, Int32) {
let mut sink = 0
let start = @bench.monotonic_clock_start()
for _ in 0..<iterations {
sink = sink ^ f()
}
(@bench.monotonic_clock_end(start), sink)
}
///|
/// The latest string a case built. Every result is stored here, where it can
/// be read later, so that the optimizer cannot drop the work that builds it:
/// V8 drops a string that only has its length read, as it can compute the
/// length without the string.
let kept_string : Ref[String] = Ref("")
///|
/// Keep `result` and return its length.
fn keep_string(result : String) -> Int32 {
kept_string.val = result
result.length()
}
///|
/// The latest bidi levels a case built, kept as `kept_string` keeps strings.
let kept_levels : Ref[Array[Int32]] = Ref([])
///|
/// Keep `levels` and return their number.
fn keep_levels(levels : Array[Int32]) -> Int32 {
kept_levels.val = levels
levels.length()
}
///|
/// Print one result row: `library<TAB>case<TAB>ns<TAB>units<TAB>check`.
///
/// Every harness in `bench/xlang` follows the same protocol: double the
/// iteration count until one sample takes at least 5 ms, then report the
/// fastest of 5 samples, the first of which is that one. `units` is the
/// number of code points processed per call (1 for per-call cases) and
/// `check` is a value every implementation must agree on. The library column
/// is empty: this harness has one library.
fn measure(
name : String,
units : Int32,
check : Int32,
f : () -> Int32,
) -> Unit {
let mut iterations = 1
let mut best = run(iterations, f).0
while best < 5000.0 {
iterations = iterations * 2
best = run(iterations, f).0
}
for _ in 1..<5 {
let elapsed = run(iterations, f).0
if elapsed < best {
best = elapsed
}
}
let ns = best * 1000.0 / iterations.to_double()
println("\t\{name}\t\{ns}\t\{units}\t\{check}")
}
///|
fn bench_normalization(inputs : String) -> Unit raise {
for name in names("\{inputs}/norm") {
let original = read("\{inputs}/norm/\{name}.txt")
let decomposed = read("\{inputs}/normd/\{name}.txt")
let units = original.char_length()
for
(label, form) in [
("nfc", @normalization.NFC),
("nfd", @normalization.NFD),
("nfkc", @normalization.NFKC),
("nfkd", @normalization.NFKD),
] {
measure(
"norm/\{label}/\{name}",
units,
@normalization.normalize(original, form).char_length(),
() => keep_string(@normalization.normalize(original, form)),
)
}
measure(
"norm/nfd>nfc/\{name}",
decomposed.char_length(),
@normalization.normalize(decomposed, NFC).char_length(),
() => keep_string(@normalization.normalize(decomposed, NFC)),
)
let is_nfc = () => {
if @normalization.is_normalized(original, NFC) {
1
} else {
0
}
}
measure("norm/is_nfc/\{name}", units, is_nfc(), is_nfc)
}
}
///|
fn bench_segmentation(inputs : String) -> Unit raise {
for name in names("\{inputs}/seg") {
let text = read("\{inputs}/seg/\{name}.txt")
let count = () => {
let cursor = @segmentation.graphemes(text)
let mut clusters = 0
while cursor.next().length() > 0 {
clusters += 1
}
clusters
}
measure("seg/graphemes/\{name}", text.char_length(), count(), count)
}
}
///|
fn bench_bidi(inputs : String) -> Unit raise {
for name in names("\{inputs}/bidi") {
let text = read("\{inputs}/bidi/\{name}.txt")
let mut level_sum = 0
for level in @bidi.process(text).levels {
level_sum += level
}
measure("bidi/process/\{name}", text.char_length(), level_sum, () => {
keep_levels(@bidi.process(text).levels)
})
}
}
///|
fn bench_idna(inputs : String) -> Unit raise {
for (name, domain) in pairs("\{inputs}/idna.tsv") {
// Every input converts successfully without STD3 rules, so the closures
// time successful conversions only.
let ascii = @idna.to_ascii(domain, use_std3_ascii_rules=false)
let unicode = @idna.to_unicode(domain, use_std3_ascii_rules=false)
measure("idna/to_ascii/\{name}", 1, ascii.char_length(), () => {
keep_string(
@idna.to_ascii(domain, use_std3_ascii_rules=false) catch {
_ => domain
},
)
})
measure("idna/to_unicode/\{name}", 1, unicode.char_length(), () => {
keep_string(
@idna.to_unicode(domain, use_std3_ascii_rules=false) catch {
_ => domain
},
)
})
}
}
///|
fn bench_punycode(inputs : String) -> Unit raise {
for (name, input) in pairs("\{inputs}/puny.tsv") {
let encoded = @punycode.encode(input)
measure("puny/encode/\{name}", 1, encoded.length(), () => {
keep_string(@punycode.encode(input) catch { _ => input })
})
measure("puny/decode/\{name}", 1, input.char_length(), () => {
keep_string(@punycode.decode(encoded) catch { _ => encoded })
})
}
}
///|
fn bench_properties(inputs : String) -> Unit raise {
for name in names("\{inputs}/case") {
let text = read("\{inputs}/case/\{name}.txt")
let units = text.char_length()
let simple_upper = () => {
let mut sum = 0
for c in text {
sum = sum ^ @ucd.to_simple_uppercase(c).to_int()
}
sum
}
let lowercase_letters = () => {
let mut count = 0
for c in text {
if @ucd.general_category(c) is Ll {
count += 1
}
}
count
}
measure("case/simple_upper/\{name}", units, simple_upper(), simple_upper)
measure(
"case/full_upper/\{name}",
units,
@ucd.uppercase(text).char_length(),
() => keep_string(@ucd.uppercase(text)),
)
for
(label, fold) in [
("fold", @casefold.case_fold),
("nfkc_casefold", @casefold.nfkc_casefold),
] {
measure("case/\{label}/\{name}", units, fold(text).char_length(), () => {
keep_string(fold(text))
})
}
measure(
"prop/general_category_Ll/\{name}",
units,
lowercase_letters(),
lowercase_letters,
)
}
}
///|
fn main raise {
let command = @argparse.Command(
"xlang-moonbit",
about="Time this repository's Unicode modules on the shared xlang inputs",
positionals=[
@argparse.PositionArg("inputs", about="Directory of prepared inputs"),
@argparse.PositionArg(
"group",
about="Only run this group: norm, seg, bidi, idna, puny or case",
num_args=@argparse.ValueRange(lower=0, upper=1),
),
],
)
let matches = command.parse()
guard matches.values.get("inputs") is Some([inputs, ..]) else {
fail(command.render_help())
}
let group = matches.values.get("group").map(values => values[0])
let groups : Array[(String, (String) -> Unit raise)] = [
("norm", bench_normalization),
("seg", bench_segmentation),
("bidi", bench_bidi),
("idna", bench_idna),
("puny", bench_punycode),
("case", bench_properties),
]
for (name, bench) in groups {
if group is Some(selected) && selected != name {
continue
}
bench(inputs)
}
}