bench/xlang/rust/src/main.rs1//! Rust peer of `bench/xlang/moonbit`: the same cases on the same inputs,2//! timed with the same protocol and printed in the same row format.34use std::hint::black_box;5use std::time::Instant;67use icu_casemap::CaseMapper;8use icu_normalizer::{ComposingNormalizerBorrowed, DecomposingNormalizerBorrowed};9use icu_properties::{props::GeneralCategory, CodePointMapData};10use icu_segmenter::GraphemeClusterSegmenter;11use unicode_normalization::UnicodeNormalization;12use unicode_segmentation::UnicodeSegmentation;1314fn run(iterations: u64, f: &mut dyn FnMut() -> usize) -> f64 {15 let mut sink = 0usize;16 let start = Instant::now();17 for _ in 0..iterations {18 sink ^= black_box(f());19 }20 black_box(sink);21 start.elapsed().as_secs_f64() * 1e922}2324/// Double the iteration count until one sample takes at least 5 ms, then25/// print the fastest of 5 samples, the first of which is that one, as26/// `library<TAB>case<TAB>ns<TAB>units<TAB>check`.27fn measure(library: &str, name: &str, units: usize, check: usize, mut f: impl FnMut() -> usize) {28 let mut iterations = 1u64;29 let mut first = run(iterations, &mut f);30 while first < 5e6 {31 iterations *= 2;32 first = run(iterations, &mut f);33 }34 let best = (1..5).map(|_| run(iterations, &mut f)).fold(first, f64::min);35 println!("{library}\t{name}\t{}\t{units}\t{check}", best / iterations as f64);36}3738fn read(path: String) -> String {39 std::fs::read_to_string(&path).unwrap_or_else(|e| panic!("{path}: {e}"))40}4142/// Input names in `directory`: the `.txt` files without their extension.43fn names(directory: String) -> Vec<String> {44 let mut result: Vec<String> = std::fs::read_dir(&directory)45 .unwrap_or_else(|e| panic!("{directory}: {e}"))46 .filter_map(|entry| entry.ok()?.file_name().to_str()?.strip_suffix(".txt").map(String::from))47 .collect();48 result.sort();49 result50}5152/// The `name<TAB>value` lines of a `.tsv` input.53fn pairs(path: String) -> Vec<(String, String)> {54 read(path)55 .lines()56 .filter_map(|line| line.split_once('\t'))57 .map(|(name, value)| (name.to_string(), value.to_string()))58 .collect()59}6061fn chars(text: &str) -> usize {62 text.chars().count()63}6465fn chars16(text: &[u16]) -> usize {66 char::decode_utf16(text.iter().copied()).count()67}6869fn bench_normalization(inputs: &str) {70 let nfc = ComposingNormalizerBorrowed::new_nfc();71 let nfkc = ComposingNormalizerBorrowed::new_nfkc();72 let nfd = DecomposingNormalizerBorrowed::new_nfd();73 let nfkd = DecomposingNormalizerBorrowed::new_nfkd();74 for name in names(format!("{inputs}/norm")) {75 let original = read(format!("{inputs}/norm/{name}.txt"));76 let decomposed = read(format!("{inputs}/normd/{name}.txt"));77 let original16: Vec<u16> = original.encode_utf16().collect();78 let decomposed16: Vec<u16> = decomposed.encode_utf16().collect();79 let units = chars(&original);80 let decomposed_units = chars(&decomposed);81 let case = |operation: &str| format!("norm/{operation}/{name}");8283 // UTF-8 in, UTF-8 out; always builds a new string.84 let library = "unicode-normalization";85 measure(library, &case("nfc"), units, chars(&original.nfc().collect::<String>()),86 || black_box(black_box(&original).nfc().collect::<String>()).len());87 measure(library, &case("nfd"), units, chars(&original.nfd().collect::<String>()),88 || black_box(black_box(&original).nfd().collect::<String>()).len());89 measure(library, &case("nfkc"), units, chars(&original.nfkc().collect::<String>()),90 || black_box(black_box(&original).nfkc().collect::<String>()).len());91 measure(library, &case("nfkd"), units, chars(&original.nfkd().collect::<String>()),92 || black_box(black_box(&original).nfkd().collect::<String>()).len());93 measure(library, &case("nfd>nfc"), decomposed_units,94 chars(&decomposed.nfc().collect::<String>()),95 || black_box(black_box(&decomposed).nfc().collect::<String>()).len());96 measure(library, &case("is_nfc"), units, unicode_normalization::is_nfc(&original) as usize,97 || unicode_normalization::is_nfc(black_box(&original)) as usize);9899 let library = "icu4x-utf8";100 measure(library, &case("nfc"), units, chars(&nfc.normalize(&original)),101 || black_box(nfc.normalize(black_box(&original))).len());102 measure(library, &case("nfd"), units, chars(&nfd.normalize(&original)),103 || black_box(nfd.normalize(black_box(&original))).len());104 measure(library, &case("nfkc"), units, chars(&nfkc.normalize(&original)),105 || black_box(nfkc.normalize(black_box(&original))).len());106 measure(library, &case("nfkd"), units, chars(&nfkd.normalize(&original)),107 || black_box(nfkd.normalize(black_box(&original))).len());108 measure(library, &case("nfd>nfc"), decomposed_units, chars(&nfc.normalize(&decomposed)),109 || black_box(nfc.normalize(black_box(&decomposed))).len());110 measure(library, &case("is_nfc"), units, nfc.is_normalized(&original) as usize,111 || nfc.is_normalized(black_box(&original)) as usize);112113 // The same code unit as MoonBit strings.114 let library = "icu4x-utf16";115 measure(library, &case("nfc"), units, chars16(&nfc.normalize_utf16(&original16)),116 || black_box(nfc.normalize_utf16(black_box(&original16))).len());117 measure(library, &case("nfd"), units, chars16(&nfd.normalize_utf16(&original16)),118 || black_box(nfd.normalize_utf16(black_box(&original16))).len());119 measure(library, &case("nfkc"), units, chars16(&nfkc.normalize_utf16(&original16)),120 || black_box(nfkc.normalize_utf16(black_box(&original16))).len());121 measure(library, &case("nfkd"), units, chars16(&nfkd.normalize_utf16(&original16)),122 || black_box(nfkd.normalize_utf16(black_box(&original16))).len());123 measure(library, &case("nfd>nfc"), decomposed_units,124 chars16(&nfc.normalize_utf16(&decomposed16)),125 || black_box(nfc.normalize_utf16(black_box(&decomposed16))).len());126 measure(library, &case("is_nfc"), units, nfc.is_normalized_utf16(&original16) as usize,127 || nfc.is_normalized_utf16(black_box(&original16)) as usize);128 }129}130131fn bench_segmentation(inputs: &str) {132 let segmenter = GraphemeClusterSegmenter::new();133 for name in names(format!("{inputs}/seg")) {134 let text = read(format!("{inputs}/seg/{name}.txt"));135 let text16: Vec<u16> = text.encode_utf16().collect();136 let units = chars(&text);137 let case = format!("seg/graphemes/{name}");138 measure("unicode-segmentation", &case, units, text.graphemes(true).count(),139 || black_box(&text).graphemes(true).count());140 // ICU4X yields boundaries, including the one at offset 0.141 measure("icu4x-utf8", &case, units, segmenter.segment_str(&text).count() - 1,142 || segmenter.segment_str(black_box(&text)).count() - 1);143 measure("icu4x-utf16", &case, units, segmenter.segment_utf16(&text16).count() - 1,144 || segmenter.segment_utf16(black_box(&text16)).count() - 1);145 }146}147148fn bench_bidi(inputs: &str) {149 for name in names(format!("{inputs}/bidi")) {150 let text = read(format!("{inputs}/bidi/{name}.txt"));151 let text16: Vec<u16> = text.encode_utf16().collect();152 let units = chars(&text);153 let info = unicode_bidi::BidiInfo::new(&text, None);154 let level_sum: usize =155 text.char_indices().map(|(i, _)| info.levels[i].number() as usize).sum();156 let case = format!("bidi/process/{name}");157 measure("unicode-bidi", &case, units, level_sum,158 || black_box(unicode_bidi::BidiInfo::new(black_box(&text), None)).levels.len());159 measure("unicode-bidi-utf16", &case, units, level_sum,160 || black_box(unicode_bidi::utf16::BidiInfo::new(black_box(&text16), None)).levels.len());161 }162}163164fn bench_idna(inputs: &str) {165 for (name, domain) in pairs(format!("{inputs}/idna.tsv")) {166 let ascii = idna::domain_to_ascii(&domain).unwrap();167 let (unicode, result) = idna::domain_to_unicode(&domain);168 result.unwrap();169 measure("idna", &format!("idna/to_ascii/{name}"), 1, chars(&ascii),170 || black_box(idna::domain_to_ascii(black_box(&domain))).map_or(0, |s| s.len()));171 measure("idna", &format!("idna/to_unicode/{name}"), 1, chars(&unicode),172 || black_box(idna::domain_to_unicode(black_box(&domain))).0.len());173 }174}175176fn bench_punycode(inputs: &str) {177 for (name, input) in pairs(format!("{inputs}/puny.tsv")) {178 let encoded = idna::punycode::encode_str(&input).unwrap();179 measure("idna", &format!("puny/encode/{name}"), 1, encoded.len(),180 || black_box(idna::punycode::encode_str(black_box(&input))).map_or(0, |s| s.len()));181 measure("idna", &format!("puny/decode/{name}"), 1, chars(&input),182 || black_box(idna::punycode::decode_to_string(black_box(&encoded))).map_or(0, |s| s.len()));183 }184}185186fn bench_properties(inputs: &str) {187 let mapper = CaseMapper::new();188 let categories = CodePointMapData::<GeneralCategory>::new();189 for name in names(format!("{inputs}/case")) {190 let text = read(format!("{inputs}/case/{name}.txt"));191 let units = chars(&text);192 let simple_upper =193 |s: &str| s.chars().fold(0usize, |sum, c| sum ^ mapper.simple_uppercase(c) as usize);194 // An iterator per character, counted without building a string; MoonBit's195 // `uppercase` builds one string for the whole text.196 let full_upper = |s: &str| s.chars().map(|c| c.to_uppercase().count()).sum::<usize>();197 let lowercase_icu4x = |s: &str| {198 s.chars().filter(|&c| categories.get(c) == GeneralCategory::LowercaseLetter).count()199 };200 let lowercase_ugc = |s: &str| {201 s.chars()202 .filter(|&c| {203 unicode_general_category::get_general_category(c)204 == unicode_general_category::GeneralCategory::LowercaseLetter205 })206 .count()207 };208 measure("icu4x-utf8", &format!("case/simple_upper/{name}"), units, simple_upper(&text),209 || simple_upper(black_box(&text)));210 measure("std", &format!("case/full_upper/{name}"), units, full_upper(&text),211 || full_upper(black_box(&text)));212 measure("icu4x-utf8", &format!("case/fold/{name}"), units, chars(&mapper.fold_string(&text)),213 || black_box(mapper.fold_string(black_box(&text))).len());214 let case = format!("prop/general_category_Ll/{name}");215 measure("icu4x-utf8", &case, units, lowercase_icu4x(&text),216 || lowercase_icu4x(black_box(&text)));217 measure("unicode-general-category", &case, units, lowercase_ugc(&text),218 || lowercase_ugc(black_box(&text)));219 }220}221222fn main() {223 let arguments: Vec<String> = std::env::args().collect();224 let Some(inputs) = arguments.get(1) else {225 eprintln!("usage: xlang <inputs> [group]");226 std::process::exit(2);227 };228 let group = arguments.get(2).map(String::as_str);229 let groups: [(&str, fn(&str)); 6] = [230 ("norm", bench_normalization),231 ("seg", bench_segmentation),232 ("bidi", bench_bidi),233 ("idna", bench_idna),234 ("puny", bench_punycode),235 ("case", bench_properties),236 ];237 for (name, bench) in groups {238 if group.is_none_or(|selected| selected == name) {239 bench(inputs);240 }241 }242}243
//! Rust peer of `bench/xlang/moonbit`: the same cases on the same inputs,
//! timed with the same protocol and printed in the same row format.
use std::hint::black_box;
use std::time::Instant;
use icu_casemap::CaseMapper;
use icu_normalizer::{ComposingNormalizerBorrowed, DecomposingNormalizerBorrowed};
use icu_properties::{props::GeneralCategory, CodePointMapData};
use icu_segmenter::GraphemeClusterSegmenter;
use unicode_normalization::UnicodeNormalization;
use unicode_segmentation::UnicodeSegmentation;
fn run(iterations: u64, f: &mut dyn FnMut() -> usize) -> f64 {
let mut sink = 0usize;
let start = Instant::now();
for _ in 0..iterations {
sink ^= black_box(f());
}
black_box(sink);
start.elapsed().as_secs_f64() * 1e9
}
/// Double the iteration count until one sample takes at least 5 ms, then
/// print the fastest of 5 samples, the first of which is that one, as
/// `library<TAB>case<TAB>ns<TAB>units<TAB>check`.
fn measure(library: &str, name: &str, units: usize, check: usize, mut f: impl FnMut() -> usize) {
let mut iterations = 1u64;
let mut first = run(iterations, &mut f);
while first < 5e6 {
iterations *= 2;
first = run(iterations, &mut f);
}
let best = (1..5).map(|_| run(iterations, &mut f)).fold(first, f64::min);
println!("{library}\t{name}\t{}\t{units}\t{check}", best / iterations as f64);
}
fn read(path: String) -> String {
std::fs::read_to_string(&path).unwrap_or_else(|e| panic!("{path}: {e}"))
}
/// Input names in `directory`: the `.txt` files without their extension.
fn names(directory: String) -> Vec<String> {
let mut result: Vec<String> = std::fs::read_dir(&directory)
.unwrap_or_else(|e| panic!("{directory}: {e}"))
.filter_map(|entry| entry.ok()?.file_name().to_str()?.strip_suffix(".txt").map(String::from))
.collect();
result.sort();
result
}
/// The `name<TAB>value` lines of a `.tsv` input.
fn pairs(path: String) -> Vec<(String, String)> {
read(path)
.lines()
.filter_map(|line| line.split_once('\t'))
.map(|(name, value)| (name.to_string(), value.to_string()))
.collect()
}
fn chars(text: &str) -> usize {
text.chars().count()
}
fn chars16(text: &[u16]) -> usize {
char::decode_utf16(text.iter().copied()).count()
}
fn bench_normalization(inputs: &str) {
let nfc = ComposingNormalizerBorrowed::new_nfc();
let nfkc = ComposingNormalizerBorrowed::new_nfkc();
let nfd = DecomposingNormalizerBorrowed::new_nfd();
let nfkd = DecomposingNormalizerBorrowed::new_nfkd();
for name in names(format!("{inputs}/norm")) {
let original = read(format!("{inputs}/norm/{name}.txt"));
let decomposed = read(format!("{inputs}/normd/{name}.txt"));
let original16: Vec<u16> = original.encode_utf16().collect();
let decomposed16: Vec<u16> = decomposed.encode_utf16().collect();
let units = chars(&original);
let decomposed_units = chars(&decomposed);
let case = |operation: &str| format!("norm/{operation}/{name}");
// UTF-8 in, UTF-8 out; always builds a new string.
let library = "unicode-normalization";
measure(library, &case("nfc"), units, chars(&original.nfc().collect::<String>()),
|| black_box(black_box(&original).nfc().collect::<String>()).len());
measure(library, &case("nfd"), units, chars(&original.nfd().collect::<String>()),
|| black_box(black_box(&original).nfd().collect::<String>()).len());
measure(library, &case("nfkc"), units, chars(&original.nfkc().collect::<String>()),
|| black_box(black_box(&original).nfkc().collect::<String>()).len());
measure(library, &case("nfkd"), units, chars(&original.nfkd().collect::<String>()),
|| black_box(black_box(&original).nfkd().collect::<String>()).len());
measure(library, &case("nfd>nfc"), decomposed_units,
chars(&decomposed.nfc().collect::<String>()),
|| black_box(black_box(&decomposed).nfc().collect::<String>()).len());
measure(library, &case("is_nfc"), units, unicode_normalization::is_nfc(&original) as usize,
|| unicode_normalization::is_nfc(black_box(&original)) as usize);
let library = "icu4x-utf8";
measure(library, &case("nfc"), units, chars(&nfc.normalize(&original)),
|| black_box(nfc.normalize(black_box(&original))).len());
measure(library, &case("nfd"), units, chars(&nfd.normalize(&original)),
|| black_box(nfd.normalize(black_box(&original))).len());
measure(library, &case("nfkc"), units, chars(&nfkc.normalize(&original)),
|| black_box(nfkc.normalize(black_box(&original))).len());
measure(library, &case("nfkd"), units, chars(&nfkd.normalize(&original)),
|| black_box(nfkd.normalize(black_box(&original))).len());
measure(library, &case("nfd>nfc"), decomposed_units, chars(&nfc.normalize(&decomposed)),
|| black_box(nfc.normalize(black_box(&decomposed))).len());
measure(library, &case("is_nfc"), units, nfc.is_normalized(&original) as usize,
|| nfc.is_normalized(black_box(&original)) as usize);
// The same code unit as MoonBit strings.
let library = "icu4x-utf16";
measure(library, &case("nfc"), units, chars16(&nfc.normalize_utf16(&original16)),
|| black_box(nfc.normalize_utf16(black_box(&original16))).len());
measure(library, &case("nfd"), units, chars16(&nfd.normalize_utf16(&original16)),
|| black_box(nfd.normalize_utf16(black_box(&original16))).len());
measure(library, &case("nfkc"), units, chars16(&nfkc.normalize_utf16(&original16)),
|| black_box(nfkc.normalize_utf16(black_box(&original16))).len());
measure(library, &case("nfkd"), units, chars16(&nfkd.normalize_utf16(&original16)),
|| black_box(nfkd.normalize_utf16(black_box(&original16))).len());
measure(library, &case("nfd>nfc"), decomposed_units,
chars16(&nfc.normalize_utf16(&decomposed16)),
|| black_box(nfc.normalize_utf16(black_box(&decomposed16))).len());
measure(library, &case("is_nfc"), units, nfc.is_normalized_utf16(&original16) as usize,
|| nfc.is_normalized_utf16(black_box(&original16)) as usize);
}
}
fn bench_segmentation(inputs: &str) {
let segmenter = GraphemeClusterSegmenter::new();
for name in names(format!("{inputs}/seg")) {
let text = read(format!("{inputs}/seg/{name}.txt"));
let text16: Vec<u16> = text.encode_utf16().collect();
let units = chars(&text);
let case = format!("seg/graphemes/{name}");
measure("unicode-segmentation", &case, units, text.graphemes(true).count(),
|| black_box(&text).graphemes(true).count());
// ICU4X yields boundaries, including the one at offset 0.
measure("icu4x-utf8", &case, units, segmenter.segment_str(&text).count() - 1,
|| segmenter.segment_str(black_box(&text)).count() - 1);
measure("icu4x-utf16", &case, units, segmenter.segment_utf16(&text16).count() - 1,
|| segmenter.segment_utf16(black_box(&text16)).count() - 1);
}
}
fn bench_bidi(inputs: &str) {
for name in names(format!("{inputs}/bidi")) {
let text = read(format!("{inputs}/bidi/{name}.txt"));
let text16: Vec<u16> = text.encode_utf16().collect();
let units = chars(&text);
let info = unicode_bidi::BidiInfo::new(&text, None);
let level_sum: usize =
text.char_indices().map(|(i, _)| info.levels[i].number() as usize).sum();
let case = format!("bidi/process/{name}");
measure("unicode-bidi", &case, units, level_sum,
|| black_box(unicode_bidi::BidiInfo::new(black_box(&text), None)).levels.len());
measure("unicode-bidi-utf16", &case, units, level_sum,
|| black_box(unicode_bidi::utf16::BidiInfo::new(black_box(&text16), None)).levels.len());
}
}
fn bench_idna(inputs: &str) {
for (name, domain) in pairs(format!("{inputs}/idna.tsv")) {
let ascii = idna::domain_to_ascii(&domain).unwrap();
let (unicode, result) = idna::domain_to_unicode(&domain);
result.unwrap();
measure("idna", &format!("idna/to_ascii/{name}"), 1, chars(&ascii),
|| black_box(idna::domain_to_ascii(black_box(&domain))).map_or(0, |s| s.len()));
measure("idna", &format!("idna/to_unicode/{name}"), 1, chars(&unicode),
|| black_box(idna::domain_to_unicode(black_box(&domain))).0.len());
}
}
fn bench_punycode(inputs: &str) {
for (name, input) in pairs(format!("{inputs}/puny.tsv")) {
let encoded = idna::punycode::encode_str(&input).unwrap();
measure("idna", &format!("puny/encode/{name}"), 1, encoded.len(),
|| black_box(idna::punycode::encode_str(black_box(&input))).map_or(0, |s| s.len()));
measure("idna", &format!("puny/decode/{name}"), 1, chars(&input),
|| black_box(idna::punycode::decode_to_string(black_box(&encoded))).map_or(0, |s| s.len()));
}
}
fn bench_properties(inputs: &str) {
let mapper = CaseMapper::new();
let categories = CodePointMapData::<GeneralCategory>::new();
for name in names(format!("{inputs}/case")) {
let text = read(format!("{inputs}/case/{name}.txt"));
let units = chars(&text);
let simple_upper =
|s: &str| s.chars().fold(0usize, |sum, c| sum ^ mapper.simple_uppercase(c) as usize);
// An iterator per character, counted without building a string; MoonBit's
// `uppercase` builds one string for the whole text.
let full_upper = |s: &str| s.chars().map(|c| c.to_uppercase().count()).sum::<usize>();
let lowercase_icu4x = |s: &str| {
s.chars().filter(|&c| categories.get(c) == GeneralCategory::LowercaseLetter).count()
};
let lowercase_ugc = |s: &str| {
s.chars()
.filter(|&c| {
unicode_general_category::get_general_category(c)
== unicode_general_category::GeneralCategory::LowercaseLetter
})
.count()
};
measure("icu4x-utf8", &format!("case/simple_upper/{name}"), units, simple_upper(&text),
|| simple_upper(black_box(&text)));
measure("std", &format!("case/full_upper/{name}"), units, full_upper(&text),
|| full_upper(black_box(&text)));
measure("icu4x-utf8", &format!("case/fold/{name}"), units, chars(&mapper.fold_string(&text)),
|| black_box(mapper.fold_string(black_box(&text))).len());
let case = format!("prop/general_category_Ll/{name}");
measure("icu4x-utf8", &case, units, lowercase_icu4x(&text),
|| lowercase_icu4x(black_box(&text)));
measure("unicode-general-category", &case, units, lowercase_ugc(&text),
|| lowercase_ugc(black_box(&text)));
}
}
fn main() {
let arguments: Vec<String> = std::env::args().collect();
let Some(inputs) = arguments.get(1) else {
eprintln!("usage: xlang <inputs> [group]");
std::process::exit(2);
};
let group = arguments.get(2).map(String::as_str);
let groups: [(&str, fn(&str)); 6] = [
("norm", bench_normalization),
("seg", bench_segmentation),
("bidi", bench_bidi),
("idna", bench_idna),
("puny", bench_punycode),
("case", bench_properties),
];
for (name, bench) in groups {
if group.is_none_or(|selected| selected == name) {
bench(inputs);
}
}
}