unicode

Branch: main

Switch branches or tags

Branch: main
Go to file t

413 files indexed

History
Code

bench/xlang/rust/src/main.rs
1//! Rust peer of `bench/xlang/moonbit`: the same cases on the same inputs,2//! timed with the same protocol and printed in the same row format.34use std::hint::black_box;5use std::time::Instant;67use icu_casemap::CaseMapper;8use icu_normalizer::{ComposingNormalizerBorrowed, DecomposingNormalizerBorrowed};9use icu_properties::{props::GeneralCategory, CodePointMapData};10use icu_segmenter::GraphemeClusterSegmenter;11use unicode_normalization::UnicodeNormalization;12use unicode_segmentation::UnicodeSegmentation;1314fn run(iterations: u64, f: &mut dyn FnMut() -> usize) -> f64 {15    let mut sink = 0usize;16    let start = Instant::now();17    for _ in 0..iterations {18        sink ^= black_box(f());19    }20    black_box(sink);21    start.elapsed().as_secs_f64() * 1e922}2324/// Double the iteration count until one sample takes at least 5 ms, then25/// print the fastest of 5 samples, the first of which is that one, as26/// `library<TAB>case<TAB>ns<TAB>units<TAB>check`.27fn measure(library: &str, name: &str, units: usize, check: usize, mut f: impl FnMut() -> usize) {28    let mut iterations = 1u64;29    let mut first = run(iterations, &mut f);30    while first < 5e6 {31        iterations *= 2;32        first = run(iterations, &mut f);33    }34    let best = (1..5).map(|_| run(iterations, &mut f)).fold(first, f64::min);35    println!("{library}\t{name}\t{}\t{units}\t{check}", best / iterations as f64);36}3738fn read(path: String) -> String {39    std::fs::read_to_string(&path).unwrap_or_else(|e| panic!("{path}: {e}"))40}4142/// Input names in `directory`: the `.txt` files without their extension.43fn names(directory: String) -> Vec<String> {44    let mut result: Vec<String> = std::fs::read_dir(&directory)45        .unwrap_or_else(|e| panic!("{directory}: {e}"))46        .filter_map(|entry| entry.ok()?.file_name().to_str()?.strip_suffix(".txt").map(String::from))47        .collect();48    result.sort();49    result50}5152/// The `name<TAB>value` lines of a `.tsv` input.53fn pairs(path: String) -> Vec<(String, String)> {54    read(path)55        .lines()56        .filter_map(|line| line.split_once('\t'))57        .map(|(name, value)| (name.to_string(), value.to_string()))58        .collect()59}6061fn chars(text: &str) -> usize {62    text.chars().count()63}6465fn chars16(text: &[u16]) -> usize {66    char::decode_utf16(text.iter().copied()).count()67}6869fn bench_normalization(inputs: &str) {70    let nfc = ComposingNormalizerBorrowed::new_nfc();71    let nfkc = ComposingNormalizerBorrowed::new_nfkc();72    let nfd = DecomposingNormalizerBorrowed::new_nfd();73    let nfkd = DecomposingNormalizerBorrowed::new_nfkd();74    for name in names(format!("{inputs}/norm")) {75        let original = read(format!("{inputs}/norm/{name}.txt"));76        let decomposed = read(format!("{inputs}/normd/{name}.txt"));77        let original16: Vec<u16> = original.encode_utf16().collect();78        let decomposed16: Vec<u16> = decomposed.encode_utf16().collect();79        let units = chars(&original);80        let decomposed_units = chars(&decomposed);81        let case = |operation: &str| format!("norm/{operation}/{name}");8283        // UTF-8 in, UTF-8 out; always builds a new string.84        let library = "unicode-normalization";85        measure(library, &case("nfc"), units, chars(&original.nfc().collect::<String>()),86            || black_box(black_box(&original).nfc().collect::<String>()).len());87        measure(library, &case("nfd"), units, chars(&original.nfd().collect::<String>()),88            || black_box(black_box(&original).nfd().collect::<String>()).len());89        measure(library, &case("nfkc"), units, chars(&original.nfkc().collect::<String>()),90            || black_box(black_box(&original).nfkc().collect::<String>()).len());91        measure(library, &case("nfkd"), units, chars(&original.nfkd().collect::<String>()),92            || black_box(black_box(&original).nfkd().collect::<String>()).len());93        measure(library, &case("nfd>nfc"), decomposed_units,94            chars(&decomposed.nfc().collect::<String>()),95            || black_box(black_box(&decomposed).nfc().collect::<String>()).len());96        measure(library, &case("is_nfc"), units, unicode_normalization::is_nfc(&original) as usize,97            || unicode_normalization::is_nfc(black_box(&original)) as usize);9899        let library = "icu4x-utf8";100        measure(library, &case("nfc"), units, chars(&nfc.normalize(&original)),101            || black_box(nfc.normalize(black_box(&original))).len());102        measure(library, &case("nfd"), units, chars(&nfd.normalize(&original)),103            || black_box(nfd.normalize(black_box(&original))).len());104        measure(library, &case("nfkc"), units, chars(&nfkc.normalize(&original)),105            || black_box(nfkc.normalize(black_box(&original))).len());106        measure(library, &case("nfkd"), units, chars(&nfkd.normalize(&original)),107            || black_box(nfkd.normalize(black_box(&original))).len());108        measure(library, &case("nfd>nfc"), decomposed_units, chars(&nfc.normalize(&decomposed)),109            || black_box(nfc.normalize(black_box(&decomposed))).len());110        measure(library, &case("is_nfc"), units, nfc.is_normalized(&original) as usize,111            || nfc.is_normalized(black_box(&original)) as usize);112113        // The same code unit as MoonBit strings.114        let library = "icu4x-utf16";115        measure(library, &case("nfc"), units, chars16(&nfc.normalize_utf16(&original16)),116            || black_box(nfc.normalize_utf16(black_box(&original16))).len());117        measure(library, &case("nfd"), units, chars16(&nfd.normalize_utf16(&original16)),118            || black_box(nfd.normalize_utf16(black_box(&original16))).len());119        measure(library, &case("nfkc"), units, chars16(&nfkc.normalize_utf16(&original16)),120            || black_box(nfkc.normalize_utf16(black_box(&original16))).len());121        measure(library, &case("nfkd"), units, chars16(&nfkd.normalize_utf16(&original16)),122            || black_box(nfkd.normalize_utf16(black_box(&original16))).len());123        measure(library, &case("nfd>nfc"), decomposed_units,124            chars16(&nfc.normalize_utf16(&decomposed16)),125            || black_box(nfc.normalize_utf16(black_box(&decomposed16))).len());126        measure(library, &case("is_nfc"), units, nfc.is_normalized_utf16(&original16) as usize,127            || nfc.is_normalized_utf16(black_box(&original16)) as usize);128    }129}130131fn bench_segmentation(inputs: &str) {132    let segmenter = GraphemeClusterSegmenter::new();133    for name in names(format!("{inputs}/seg")) {134        let text = read(format!("{inputs}/seg/{name}.txt"));135        let text16: Vec<u16> = text.encode_utf16().collect();136        let units = chars(&text);137        let case = format!("seg/graphemes/{name}");138        measure("unicode-segmentation", &case, units, text.graphemes(true).count(),139            || black_box(&text).graphemes(true).count());140        // ICU4X yields boundaries, including the one at offset 0.141        measure("icu4x-utf8", &case, units, segmenter.segment_str(&text).count() - 1,142            || segmenter.segment_str(black_box(&text)).count() - 1);143        measure("icu4x-utf16", &case, units, segmenter.segment_utf16(&text16).count() - 1,144            || segmenter.segment_utf16(black_box(&text16)).count() - 1);145    }146}147148fn bench_bidi(inputs: &str) {149    for name in names(format!("{inputs}/bidi")) {150        let text = read(format!("{inputs}/bidi/{name}.txt"));151        let text16: Vec<u16> = text.encode_utf16().collect();152        let units = chars(&text);153        let info = unicode_bidi::BidiInfo::new(&text, None);154        let level_sum: usize =155            text.char_indices().map(|(i, _)| info.levels[i].number() as usize).sum();156        let case = format!("bidi/process/{name}");157        measure("unicode-bidi", &case, units, level_sum,158            || black_box(unicode_bidi::BidiInfo::new(black_box(&text), None)).levels.len());159        measure("unicode-bidi-utf16", &case, units, level_sum,160            || black_box(unicode_bidi::utf16::BidiInfo::new(black_box(&text16), None)).levels.len());161    }162}163164fn bench_idna(inputs: &str) {165    for (name, domain) in pairs(format!("{inputs}/idna.tsv")) {166        let ascii = idna::domain_to_ascii(&domain).unwrap();167        let (unicode, result) = idna::domain_to_unicode(&domain);168        result.unwrap();169        measure("idna", &format!("idna/to_ascii/{name}"), 1, chars(&ascii),170            || black_box(idna::domain_to_ascii(black_box(&domain))).map_or(0, |s| s.len()));171        measure("idna", &format!("idna/to_unicode/{name}"), 1, chars(&unicode),172            || black_box(idna::domain_to_unicode(black_box(&domain))).0.len());173    }174}175176fn bench_punycode(inputs: &str) {177    for (name, input) in pairs(format!("{inputs}/puny.tsv")) {178        let encoded = idna::punycode::encode_str(&input).unwrap();179        measure("idna", &format!("puny/encode/{name}"), 1, encoded.len(),180            || black_box(idna::punycode::encode_str(black_box(&input))).map_or(0, |s| s.len()));181        measure("idna", &format!("puny/decode/{name}"), 1, chars(&input),182            || black_box(idna::punycode::decode_to_string(black_box(&encoded))).map_or(0, |s| s.len()));183    }184}185186fn bench_properties(inputs: &str) {187    let mapper = CaseMapper::new();188    let categories = CodePointMapData::<GeneralCategory>::new();189    for name in names(format!("{inputs}/case")) {190        let text = read(format!("{inputs}/case/{name}.txt"));191        let units = chars(&text);192        let simple_upper =193            |s: &str| s.chars().fold(0usize, |sum, c| sum ^ mapper.simple_uppercase(c) as usize);194        // An iterator per character, counted without building a string; MoonBit's195        // `uppercase` builds one string for the whole text.196        let full_upper = |s: &str| s.chars().map(|c| c.to_uppercase().count()).sum::<usize>();197        let lowercase_icu4x = |s: &str| {198            s.chars().filter(|&c| categories.get(c) == GeneralCategory::LowercaseLetter).count()199        };200        let lowercase_ugc = |s: &str| {201            s.chars()202                .filter(|&c| {203                    unicode_general_category::get_general_category(c)204                        == unicode_general_category::GeneralCategory::LowercaseLetter205                })206                .count()207        };208        measure("icu4x-utf8", &format!("case/simple_upper/{name}"), units, simple_upper(&text),209            || simple_upper(black_box(&text)));210        measure("std", &format!("case/full_upper/{name}"), units, full_upper(&text),211            || full_upper(black_box(&text)));212        measure("icu4x-utf8", &format!("case/fold/{name}"), units, chars(&mapper.fold_string(&text)),213            || black_box(mapper.fold_string(black_box(&text))).len());214        let case = format!("prop/general_category_Ll/{name}");215        measure("icu4x-utf8", &case, units, lowercase_icu4x(&text),216            || lowercase_icu4x(black_box(&text)));217        measure("unicode-general-category", &case, units, lowercase_ugc(&text),218            || lowercase_ugc(black_box(&text)));219    }220}221222fn main() {223    let arguments: Vec<String> = std::env::args().collect();224    let Some(inputs) = arguments.get(1) else {225        eprintln!("usage: xlang <inputs> [group]");226        std::process::exit(2);227    };228    let group = arguments.get(2).map(String::as_str);229    let groups: [(&str, fn(&str)); 6] = [230        ("norm", bench_normalization),231        ("seg", bench_segmentation),232        ("bidi", bench_bidi),233        ("idna", bench_idna),234        ("puny", bench_punycode),235        ("case", bench_properties),236    ];237    for (name, bench) in groups {238        if group.is_none_or(|selected| selected == name) {239            bench(inputs);240        }241    }242}243