unicode

Branch: main

Switch branches or tags

Branch: main
Go to file t

413 files indexed

History
Code

bench/xlang/icu4c/main.cpp
1// ICU4C peer of bench/xlang/moonbit: the same cases on the same inputs, timed2// with the same protocol and printed in the same row format. ICU4C works on3// UTF-16, the same code unit as MoonBit strings.45#include <algorithm>6#include <chrono>7#include <cstdio>8#include <cstdlib>9#include <filesystem>10#include <fstream>11#include <functional>12#include <sstream>13#include <string>14#include <utility>15#include <vector>1617#include <unicode/brkiter.h>18#include <unicode/normalizer2.h>19#include <unicode/ubidi.h>20#include <unicode/uchar.h>21#include <unicode/uidna.h>22#include <unicode/unistr.h>23#include <unicode/ustring.h>2425using icu::BreakIterator;26using icu::Locale;27using icu::Normalizer2;28using icu::UnicodeString;2930static volatile long g_sink;3132static double run(long iterations, const std::function<long()> &f) {33  long sink = 0;34  auto start = std::chrono::steady_clock::now();35  for (long i = 0; i < iterations; i++) sink ^= f();36  auto elapsed = std::chrono::steady_clock::now() - start;37  g_sink = sink;38  return std::chrono::duration<double, std::nano>(elapsed).count();39}4041// Double the iteration count until one sample takes at least 5 ms, then print42// the fastest of 5 samples, the first of which is that one, as43// library<TAB>case<TAB>ns<TAB>units<TAB>check.44// The library column is empty: this harness has one library.45static void measure(const std::string &name, long units, long check,46                    const std::function<long()> &f) {47  long iterations = 1;48  double best = run(iterations, f);49  while (best < 5e6) {50    iterations *= 2;51    best = run(iterations, f);52  }53  for (int i = 1; i < 5; i++) best = std::min(best, run(iterations, f));54  printf("\t%s\t%f\t%ld\t%ld\n", name.c_str(), best / iterations, units, check);55  fflush(stdout);56}5758static UnicodeString read(const std::string &path) {59  std::ifstream in(path, std::ios::binary);60  if (!in) {61    fprintf(stderr, "cannot read %s\n", path.c_str());62    exit(1);63  }64  std::stringstream contents;65  contents << in.rdbuf();66  return UnicodeString::fromUTF8(contents.str());67}6869// Input names in `directory`: the .txt files without their extension.70static std::vector<std::string> names(const std::string &directory) {71  std::vector<std::string> result;72  for (const auto &entry : std::filesystem::directory_iterator(directory))73    if (entry.path().extension() == ".txt") result.push_back(entry.path().stem().string());74  std::sort(result.begin(), result.end());75  return result;76}7778// The name<TAB>value lines of a .tsv input.79static std::vector<std::pair<std::string, std::string>> pairs(const std::string &path) {80  std::ifstream in(path);81  if (!in) {82    fprintf(stderr, "cannot read %s\n", path.c_str());83    exit(1);84  }85  std::vector<std::pair<std::string, std::string>> result;86  std::string line;87  while (std::getline(in, line)) {88    auto tab = line.find('\t');89    if (tab != std::string::npos) result.emplace_back(line.substr(0, tab), line.substr(tab + 1));90  }91  return result;92}9394static void check_status(UErrorCode status, const char *what) {95  if (U_FAILURE(status)) {96    fprintf(stderr, "%s: %s\n", what, u_errorName(status));97    exit(1);98  }99}100101// A fresh destination per call, like the allocating APIs of the other harnesses.102static UnicodeString normalized(const Normalizer2 *form, const UnicodeString &source) {103  UnicodeString destination;104  UErrorCode status = U_ZERO_ERROR;105  form->normalize(source, destination, status);106  return destination;107}108109static void bench_normalization(const std::string &inputs) {110  UErrorCode status = U_ZERO_ERROR;111  const std::pair<const char *, const Normalizer2 *> forms[] = {112      {"nfc", Normalizer2::getNFCInstance(status)},113      {"nfd", Normalizer2::getNFDInstance(status)},114      {"nfkc", Normalizer2::getNFKCInstance(status)},115      {"nfkd", Normalizer2::getNFKDInstance(status)}};116  check_status(status, "Normalizer2");117  const Normalizer2 *nfc = forms[0].second;118  for (const std::string &name : names(inputs + "/norm")) {119    UnicodeString original = read(inputs + "/norm/" + name + ".txt");120    UnicodeString decomposed = read(inputs + "/normd/" + name + ".txt");121    long units = original.countChar32();122    for (const auto &[label, form] : forms) {123      const Normalizer2 *normalizer = form;124      measure(std::string("norm/") + label + "/" + name, units,125              normalized(normalizer, original).countChar32(),126              [&]() { return (long)normalized(normalizer, original).length(); });127    }128    measure("norm/nfd>nfc/" + name, decomposed.countChar32(),129            normalized(nfc, decomposed).countChar32(),130            [&]() { return (long)normalized(nfc, decomposed).length(); });131    measure("norm/is_nfc/" + name, units, nfc->isNormalized(original, status), [&]() {132      UErrorCode status = U_ZERO_ERROR;133      return (long)nfc->isNormalized(original, status);134    });135  }136}137138static void bench_segmentation(const std::string &inputs) {139  UErrorCode status = U_ZERO_ERROR;140  BreakIterator *iterator = BreakIterator::createCharacterInstance(Locale::getRoot(), status);141  check_status(status, "BreakIterator");142  for (const std::string &name : names(inputs + "/seg")) {143    UnicodeString text = read(inputs + "/seg/" + name + ".txt");144    auto count = [&]() {145      iterator->setText(text);146      long clusters = 0;147      while (iterator->next() != BreakIterator::DONE) clusters++;148      return clusters;149    };150    measure("seg/graphemes/" + name, text.countChar32(), count(), count);151  }152  delete iterator;153}154155static void bench_bidi(const std::string &inputs) {156  UBiDi *bidi = ubidi_open();157  for (const std::string &name : names(inputs + "/bidi")) {158    UnicodeString text = read(inputs + "/bidi/" + name + ".txt");159    UErrorCode status = U_ZERO_ERROR;160    ubidi_setPara(bidi, text.getBuffer(), text.length(), UBIDI_DEFAULT_LTR, nullptr, &status);161    const UBiDiLevel *levels = ubidi_getLevels(bidi, &status);162    check_status(status, "ubidi");163    long level_sum = 0;164    for (int32_t i = 0; i < text.length(); i++)165      if (!U16_IS_TRAIL(text[i])) level_sum += levels[i];166    measure("bidi/process/" + name, text.countChar32(), level_sum, [&]() {167      UErrorCode status = U_ZERO_ERROR;168      ubidi_setPara(bidi, text.getBuffer(), text.length(), UBIDI_DEFAULT_LTR, nullptr, &status);169      // Levels are computed lazily; ask for them as the other harnesses do.170      const UBiDiLevel *levels = ubidi_getLevels(bidi, &status);171      return (long)levels[text.length() - 1] + ubidi_getProcessedLength(bidi);172    });173  }174  ubidi_close(bidi);175}176177static void bench_idna(const std::string &inputs) {178  // The closest match to the other harnesses: UTS #46 non-transitional179  // processing with Bidi and joiner checks but without STD3 rules.180  UErrorCode status = U_ZERO_ERROR;181  UIDNA *idna = uidna_openUTS46(UIDNA_CHECK_BIDI | UIDNA_CHECK_CONTEXTJ |182                                    UIDNA_NONTRANSITIONAL_TO_ASCII |183                                    UIDNA_NONTRANSITIONAL_TO_UNICODE,184                                &status);185  check_status(status, "uidna_openUTS46");186  const int32_t capacity = 512;187  for (const auto &[name, domain_utf8] : pairs(inputs + "/idna.tsv")) {188    UnicodeString domain = UnicodeString::fromUTF8(domain_utf8);189    auto to_ascii = [&](char16_t *destination) {190      UIDNAInfo info = UIDNA_INFO_INITIALIZER;191      UErrorCode status = U_ZERO_ERROR;192      return (long)uidna_nameToASCII(idna, domain.getBuffer(), domain.length(), destination,193                                     capacity, &info, &status);194    };195    auto to_unicode = [&](char16_t *destination) {196      UIDNAInfo info = UIDNA_INFO_INITIALIZER;197      UErrorCode status = U_ZERO_ERROR;198      return (long)uidna_nameToUnicode(idna, domain.getBuffer(), domain.length(), destination,199                                       capacity, &info, &status);200    };201    char16_t buffer[capacity];202    long ascii_length = to_ascii(buffer);203    long unicode_length = u_countChar32(buffer, (int32_t)to_unicode(buffer));204    measure("idna/to_ascii/" + name, 1, ascii_length, [&]() {205      char16_t destination[capacity];206      return to_ascii(destination);207    });208    measure("idna/to_unicode/" + name, 1, unicode_length, [&]() {209      char16_t destination[capacity];210      return to_unicode(destination);211    });212  }213  uidna_close(idna);214}215216static void bench_properties(const std::string &inputs) {217  UErrorCode status = U_ZERO_ERROR;218  const Normalizer2 *nfkc_casefold = Normalizer2::getNFKCCasefoldInstance(status);219  check_status(status, "Normalizer2");220  // Full default case folding into a fresh destination per call. Folding221  // expands a UTF-16 code unit to at most three. UnicodeString::foldCase222  // copies the source first and takes twice as long.223  auto fold = [](const UnicodeString &source) {224    UnicodeString destination;225    int32_t capacity = source.length() * 3;226    UErrorCode status = U_ZERO_ERROR;227    int32_t length = u_strFoldCase(destination.getBuffer(capacity), capacity, source.getBuffer(),228                                   source.length(), U_FOLD_CASE_DEFAULT, &status);229    destination.releaseBuffer(U_SUCCESS(status) ? length : 0);230    return destination;231  };232  // These are calls into the ICU shared library: nothing is inlined into the loop.233  for (const std::string &name : names(inputs + "/case")) {234    UnicodeString text = read(inputs + "/case/" + name + ".txt");235    long units = text.countChar32();236    auto simple_upper = [&]() {237      long sum = 0;238      for (int32_t i = 0; i < text.length();) {239        UChar32 c = text.char32At(i);240        sum ^= u_toupper(c);241        i += U16_LENGTH(c);242      }243      return sum;244    };245    auto lowercase_letters = [&]() {246      long count = 0;247      for (int32_t i = 0; i < text.length();) {248        UChar32 c = text.char32At(i);249        count += u_charType(c) == U_LOWERCASE_LETTER;250        i += U16_LENGTH(c);251      }252      return count;253    };254    measure("case/simple_upper/" + name, units, simple_upper(), simple_upper);255    measure("case/fold/" + name, units, fold(text).countChar32(),256            [&]() { return (long)fold(text).length(); });257    measure("case/nfkc_casefold/" + name, units, normalized(nfkc_casefold, text).countChar32(),258            [&]() { return (long)normalized(nfkc_casefold, text).length(); });259    measure("prop/general_category_Ll/" + name, units, lowercase_letters(), lowercase_letters);260  }261}262263int main(int argc, char **argv) {264  if (argc < 2) {265    fprintf(stderr, "usage: xlang <inputs> [group]\n");266    return 2;267  }268  std::string inputs = argv[1];269  std::string group = argc > 2 ? argv[2] : "";270  const std::pair<const char *, void (*)(const std::string &)> groups[] = {271      {"norm", bench_normalization}, {"seg", bench_segmentation}, {"bidi", bench_bidi},272      {"idna", bench_idna},          {"case", bench_properties}};273  for (const auto &[name, bench] : groups)274    if (group.empty() || group == name) bench(inputs);275  return 0;276}277