bench/xlang/icu4c/main.cpp1// ICU4C peer of bench/xlang/moonbit: the same cases on the same inputs, timed2// with the same protocol and printed in the same row format. ICU4C works on3// UTF-16, the same code unit as MoonBit strings.45#include <algorithm>6#include <chrono>7#include <cstdio>8#include <cstdlib>9#include <filesystem>10#include <fstream>11#include <functional>12#include <sstream>13#include <string>14#include <utility>15#include <vector>1617#include <unicode/brkiter.h>18#include <unicode/normalizer2.h>19#include <unicode/ubidi.h>20#include <unicode/uchar.h>21#include <unicode/uidna.h>22#include <unicode/unistr.h>23#include <unicode/ustring.h>2425using icu::BreakIterator;26using icu::Locale;27using icu::Normalizer2;28using icu::UnicodeString;2930static volatile long g_sink;3132static double run(long iterations, const std::function<long()> &f) {33 long sink = 0;34 auto start = std::chrono::steady_clock::now();35 for (long i = 0; i < iterations; i++) sink ^= f();36 auto elapsed = std::chrono::steady_clock::now() - start;37 g_sink = sink;38 return std::chrono::duration<double, std::nano>(elapsed).count();39}4041// Double the iteration count until one sample takes at least 5 ms, then print42// the fastest of 5 samples, the first of which is that one, as43// library<TAB>case<TAB>ns<TAB>units<TAB>check.44// The library column is empty: this harness has one library.45static void measure(const std::string &name, long units, long check,46 const std::function<long()> &f) {47 long iterations = 1;48 double best = run(iterations, f);49 while (best < 5e6) {50 iterations *= 2;51 best = run(iterations, f);52 }53 for (int i = 1; i < 5; i++) best = std::min(best, run(iterations, f));54 printf("\t%s\t%f\t%ld\t%ld\n", name.c_str(), best / iterations, units, check);55 fflush(stdout);56}5758static UnicodeString read(const std::string &path) {59 std::ifstream in(path, std::ios::binary);60 if (!in) {61 fprintf(stderr, "cannot read %s\n", path.c_str());62 exit(1);63 }64 std::stringstream contents;65 contents << in.rdbuf();66 return UnicodeString::fromUTF8(contents.str());67}6869// Input names in `directory`: the .txt files without their extension.70static std::vector<std::string> names(const std::string &directory) {71 std::vector<std::string> result;72 for (const auto &entry : std::filesystem::directory_iterator(directory))73 if (entry.path().extension() == ".txt") result.push_back(entry.path().stem().string());74 std::sort(result.begin(), result.end());75 return result;76}7778// The name<TAB>value lines of a .tsv input.79static std::vector<std::pair<std::string, std::string>> pairs(const std::string &path) {80 std::ifstream in(path);81 if (!in) {82 fprintf(stderr, "cannot read %s\n", path.c_str());83 exit(1);84 }85 std::vector<std::pair<std::string, std::string>> result;86 std::string line;87 while (std::getline(in, line)) {88 auto tab = line.find('\t');89 if (tab != std::string::npos) result.emplace_back(line.substr(0, tab), line.substr(tab + 1));90 }91 return result;92}9394static void check_status(UErrorCode status, const char *what) {95 if (U_FAILURE(status)) {96 fprintf(stderr, "%s: %s\n", what, u_errorName(status));97 exit(1);98 }99}100101// A fresh destination per call, like the allocating APIs of the other harnesses.102static UnicodeString normalized(const Normalizer2 *form, const UnicodeString &source) {103 UnicodeString destination;104 UErrorCode status = U_ZERO_ERROR;105 form->normalize(source, destination, status);106 return destination;107}108109static void bench_normalization(const std::string &inputs) {110 UErrorCode status = U_ZERO_ERROR;111 const std::pair<const char *, const Normalizer2 *> forms[] = {112 {"nfc", Normalizer2::getNFCInstance(status)},113 {"nfd", Normalizer2::getNFDInstance(status)},114 {"nfkc", Normalizer2::getNFKCInstance(status)},115 {"nfkd", Normalizer2::getNFKDInstance(status)}};116 check_status(status, "Normalizer2");117 const Normalizer2 *nfc = forms[0].second;118 for (const std::string &name : names(inputs + "/norm")) {119 UnicodeString original = read(inputs + "/norm/" + name + ".txt");120 UnicodeString decomposed = read(inputs + "/normd/" + name + ".txt");121 long units = original.countChar32();122 for (const auto &[label, form] : forms) {123 const Normalizer2 *normalizer = form;124 measure(std::string("norm/") + label + "/" + name, units,125 normalized(normalizer, original).countChar32(),126 [&]() { return (long)normalized(normalizer, original).length(); });127 }128 measure("norm/nfd>nfc/" + name, decomposed.countChar32(),129 normalized(nfc, decomposed).countChar32(),130 [&]() { return (long)normalized(nfc, decomposed).length(); });131 measure("norm/is_nfc/" + name, units, nfc->isNormalized(original, status), [&]() {132 UErrorCode status = U_ZERO_ERROR;133 return (long)nfc->isNormalized(original, status);134 });135 }136}137138static void bench_segmentation(const std::string &inputs) {139 UErrorCode status = U_ZERO_ERROR;140 BreakIterator *iterator = BreakIterator::createCharacterInstance(Locale::getRoot(), status);141 check_status(status, "BreakIterator");142 for (const std::string &name : names(inputs + "/seg")) {143 UnicodeString text = read(inputs + "/seg/" + name + ".txt");144 auto count = [&]() {145 iterator->setText(text);146 long clusters = 0;147 while (iterator->next() != BreakIterator::DONE) clusters++;148 return clusters;149 };150 measure("seg/graphemes/" + name, text.countChar32(), count(), count);151 }152 delete iterator;153}154155static void bench_bidi(const std::string &inputs) {156 UBiDi *bidi = ubidi_open();157 for (const std::string &name : names(inputs + "/bidi")) {158 UnicodeString text = read(inputs + "/bidi/" + name + ".txt");159 UErrorCode status = U_ZERO_ERROR;160 ubidi_setPara(bidi, text.getBuffer(), text.length(), UBIDI_DEFAULT_LTR, nullptr, &status);161 const UBiDiLevel *levels = ubidi_getLevels(bidi, &status);162 check_status(status, "ubidi");163 long level_sum = 0;164 for (int32_t i = 0; i < text.length(); i++)165 if (!U16_IS_TRAIL(text[i])) level_sum += levels[i];166 measure("bidi/process/" + name, text.countChar32(), level_sum, [&]() {167 UErrorCode status = U_ZERO_ERROR;168 ubidi_setPara(bidi, text.getBuffer(), text.length(), UBIDI_DEFAULT_LTR, nullptr, &status);169 // Levels are computed lazily; ask for them as the other harnesses do.170 const UBiDiLevel *levels = ubidi_getLevels(bidi, &status);171 return (long)levels[text.length() - 1] + ubidi_getProcessedLength(bidi);172 });173 }174 ubidi_close(bidi);175}176177static void bench_idna(const std::string &inputs) {178 // The closest match to the other harnesses: UTS #46 non-transitional179 // processing with Bidi and joiner checks but without STD3 rules.180 UErrorCode status = U_ZERO_ERROR;181 UIDNA *idna = uidna_openUTS46(UIDNA_CHECK_BIDI | UIDNA_CHECK_CONTEXTJ |182 UIDNA_NONTRANSITIONAL_TO_ASCII |183 UIDNA_NONTRANSITIONAL_TO_UNICODE,184 &status);185 check_status(status, "uidna_openUTS46");186 const int32_t capacity = 512;187 for (const auto &[name, domain_utf8] : pairs(inputs + "/idna.tsv")) {188 UnicodeString domain = UnicodeString::fromUTF8(domain_utf8);189 auto to_ascii = [&](char16_t *destination) {190 UIDNAInfo info = UIDNA_INFO_INITIALIZER;191 UErrorCode status = U_ZERO_ERROR;192 return (long)uidna_nameToASCII(idna, domain.getBuffer(), domain.length(), destination,193 capacity, &info, &status);194 };195 auto to_unicode = [&](char16_t *destination) {196 UIDNAInfo info = UIDNA_INFO_INITIALIZER;197 UErrorCode status = U_ZERO_ERROR;198 return (long)uidna_nameToUnicode(idna, domain.getBuffer(), domain.length(), destination,199 capacity, &info, &status);200 };201 char16_t buffer[capacity];202 long ascii_length = to_ascii(buffer);203 long unicode_length = u_countChar32(buffer, (int32_t)to_unicode(buffer));204 measure("idna/to_ascii/" + name, 1, ascii_length, [&]() {205 char16_t destination[capacity];206 return to_ascii(destination);207 });208 measure("idna/to_unicode/" + name, 1, unicode_length, [&]() {209 char16_t destination[capacity];210 return to_unicode(destination);211 });212 }213 uidna_close(idna);214}215216static void bench_properties(const std::string &inputs) {217 UErrorCode status = U_ZERO_ERROR;218 const Normalizer2 *nfkc_casefold = Normalizer2::getNFKCCasefoldInstance(status);219 check_status(status, "Normalizer2");220 // Full default case folding into a fresh destination per call. Folding221 // expands a UTF-16 code unit to at most three. UnicodeString::foldCase222 // copies the source first and takes twice as long.223 auto fold = [](const UnicodeString &source) {224 UnicodeString destination;225 int32_t capacity = source.length() * 3;226 UErrorCode status = U_ZERO_ERROR;227 int32_t length = u_strFoldCase(destination.getBuffer(capacity), capacity, source.getBuffer(),228 source.length(), U_FOLD_CASE_DEFAULT, &status);229 destination.releaseBuffer(U_SUCCESS(status) ? length : 0);230 return destination;231 };232 // These are calls into the ICU shared library: nothing is inlined into the loop.233 for (const std::string &name : names(inputs + "/case")) {234 UnicodeString text = read(inputs + "/case/" + name + ".txt");235 long units = text.countChar32();236 auto simple_upper = [&]() {237 long sum = 0;238 for (int32_t i = 0; i < text.length();) {239 UChar32 c = text.char32At(i);240 sum ^= u_toupper(c);241 i += U16_LENGTH(c);242 }243 return sum;244 };245 auto lowercase_letters = [&]() {246 long count = 0;247 for (int32_t i = 0; i < text.length();) {248 UChar32 c = text.char32At(i);249 count += u_charType(c) == U_LOWERCASE_LETTER;250 i += U16_LENGTH(c);251 }252 return count;253 };254 measure("case/simple_upper/" + name, units, simple_upper(), simple_upper);255 measure("case/fold/" + name, units, fold(text).countChar32(),256 [&]() { return (long)fold(text).length(); });257 measure("case/nfkc_casefold/" + name, units, normalized(nfkc_casefold, text).countChar32(),258 [&]() { return (long)normalized(nfkc_casefold, text).length(); });259 measure("prop/general_category_Ll/" + name, units, lowercase_letters(), lowercase_letters);260 }261}262263int main(int argc, char **argv) {264 if (argc < 2) {265 fprintf(stderr, "usage: xlang <inputs> [group]\n");266 return 2;267 }268 std::string inputs = argv[1];269 std::string group = argc > 2 ? argv[2] : "";270 const std::pair<const char *, void (*)(const std::string &)> groups[] = {271 {"norm", bench_normalization}, {"seg", bench_segmentation}, {"bidi", bench_bidi},272 {"idna", bench_idna}, {"case", bench_properties}};273 for (const auto &[name, bench] : groups)274 if (group.empty() || group == name) bench(inputs);275 return 0;276}277
// ICU4C peer of bench/xlang/moonbit: the same cases on the same inputs, timed
// with the same protocol and printed in the same row format. ICU4C works on
// UTF-16, the same code unit as MoonBit strings.
#include <algorithm>
#include <chrono>
#include <cstdio>
#include <cstdlib>
#include <filesystem>
#include <fstream>
#include <functional>
#include <sstream>
#include <string>
#include <utility>
#include <vector>
#include <unicode/brkiter.h>
#include <unicode/normalizer2.h>
#include <unicode/ubidi.h>
#include <unicode/uchar.h>
#include <unicode/uidna.h>
#include <unicode/unistr.h>
#include <unicode/ustring.h>
using icu::BreakIterator;
using icu::Locale;
using icu::Normalizer2;
using icu::UnicodeString;
static volatile long g_sink;
static double run(long iterations, const std::function<long()> &f) {
long sink = 0;
auto start = std::chrono::steady_clock::now();
for (long i = 0; i < iterations; i++) sink ^= f();
auto elapsed = std::chrono::steady_clock::now() - start;
g_sink = sink;
return std::chrono::duration<double, std::nano>(elapsed).count();
}
// Double the iteration count until one sample takes at least 5 ms, then print
// the fastest of 5 samples, the first of which is that one, as
// library<TAB>case<TAB>ns<TAB>units<TAB>check.
// The library column is empty: this harness has one library.
static void measure(const std::string &name, long units, long check,
const std::function<long()> &f) {
long iterations = 1;
double best = run(iterations, f);
while (best < 5e6) {
iterations *= 2;
best = run(iterations, f);
}
for (int i = 1; i < 5; i++) best = std::min(best, run(iterations, f));
printf("\t%s\t%f\t%ld\t%ld\n", name.c_str(), best / iterations, units, check);
fflush(stdout);
}
static UnicodeString read(const std::string &path) {
std::ifstream in(path, std::ios::binary);
if (!in) {
fprintf(stderr, "cannot read %s\n", path.c_str());
exit(1);
}
std::stringstream contents;
contents << in.rdbuf();
return UnicodeString::fromUTF8(contents.str());
}
// Input names in `directory`: the .txt files without their extension.
static std::vector<std::string> names(const std::string &directory) {
std::vector<std::string> result;
for (const auto &entry : std::filesystem::directory_iterator(directory))
if (entry.path().extension() == ".txt") result.push_back(entry.path().stem().string());
std::sort(result.begin(), result.end());
return result;
}
// The name<TAB>value lines of a .tsv input.
static std::vector<std::pair<std::string, std::string>> pairs(const std::string &path) {
std::ifstream in(path);
if (!in) {
fprintf(stderr, "cannot read %s\n", path.c_str());
exit(1);
}
std::vector<std::pair<std::string, std::string>> result;
std::string line;
while (std::getline(in, line)) {
auto tab = line.find('\t');
if (tab != std::string::npos) result.emplace_back(line.substr(0, tab), line.substr(tab + 1));
}
return result;
}
static void check_status(UErrorCode status, const char *what) {
if (U_FAILURE(status)) {
fprintf(stderr, "%s: %s\n", what, u_errorName(status));
exit(1);
}
}
// A fresh destination per call, like the allocating APIs of the other harnesses.
static UnicodeString normalized(const Normalizer2 *form, const UnicodeString &source) {
UnicodeString destination;
UErrorCode status = U_ZERO_ERROR;
form->normalize(source, destination, status);
return destination;
}
static void bench_normalization(const std::string &inputs) {
UErrorCode status = U_ZERO_ERROR;
const std::pair<const char *, const Normalizer2 *> forms[] = {
{"nfc", Normalizer2::getNFCInstance(status)},
{"nfd", Normalizer2::getNFDInstance(status)},
{"nfkc", Normalizer2::getNFKCInstance(status)},
{"nfkd", Normalizer2::getNFKDInstance(status)}};
check_status(status, "Normalizer2");
const Normalizer2 *nfc = forms[0].second;
for (const std::string &name : names(inputs + "/norm")) {
UnicodeString original = read(inputs + "/norm/" + name + ".txt");
UnicodeString decomposed = read(inputs + "/normd/" + name + ".txt");
long units = original.countChar32();
for (const auto &[label, form] : forms) {
const Normalizer2 *normalizer = form;
measure(std::string("norm/") + label + "/" + name, units,
normalized(normalizer, original).countChar32(),
[&]() { return (long)normalized(normalizer, original).length(); });
}
measure("norm/nfd>nfc/" + name, decomposed.countChar32(),
normalized(nfc, decomposed).countChar32(),
[&]() { return (long)normalized(nfc, decomposed).length(); });
measure("norm/is_nfc/" + name, units, nfc->isNormalized(original, status), [&]() {
UErrorCode status = U_ZERO_ERROR;
return (long)nfc->isNormalized(original, status);
});
}
}
static void bench_segmentation(const std::string &inputs) {
UErrorCode status = U_ZERO_ERROR;
BreakIterator *iterator = BreakIterator::createCharacterInstance(Locale::getRoot(), status);
check_status(status, "BreakIterator");
for (const std::string &name : names(inputs + "/seg")) {
UnicodeString text = read(inputs + "/seg/" + name + ".txt");
auto count = [&]() {
iterator->setText(text);
long clusters = 0;
while (iterator->next() != BreakIterator::DONE) clusters++;
return clusters;
};
measure("seg/graphemes/" + name, text.countChar32(), count(), count);
}
delete iterator;
}
static void bench_bidi(const std::string &inputs) {
UBiDi *bidi = ubidi_open();
for (const std::string &name : names(inputs + "/bidi")) {
UnicodeString text = read(inputs + "/bidi/" + name + ".txt");
UErrorCode status = U_ZERO_ERROR;
ubidi_setPara(bidi, text.getBuffer(), text.length(), UBIDI_DEFAULT_LTR, nullptr, &status);
const UBiDiLevel *levels = ubidi_getLevels(bidi, &status);
check_status(status, "ubidi");
long level_sum = 0;
for (int32_t i = 0; i < text.length(); i++)
if (!U16_IS_TRAIL(text[i])) level_sum += levels[i];
measure("bidi/process/" + name, text.countChar32(), level_sum, [&]() {
UErrorCode status = U_ZERO_ERROR;
ubidi_setPara(bidi, text.getBuffer(), text.length(), UBIDI_DEFAULT_LTR, nullptr, &status);
// Levels are computed lazily; ask for them as the other harnesses do.
const UBiDiLevel *levels = ubidi_getLevels(bidi, &status);
return (long)levels[text.length() - 1] + ubidi_getProcessedLength(bidi);
});
}
ubidi_close(bidi);
}
static void bench_idna(const std::string &inputs) {
// The closest match to the other harnesses: UTS #46 non-transitional
// processing with Bidi and joiner checks but without STD3 rules.
UErrorCode status = U_ZERO_ERROR;
UIDNA *idna = uidna_openUTS46(UIDNA_CHECK_BIDI | UIDNA_CHECK_CONTEXTJ |
UIDNA_NONTRANSITIONAL_TO_ASCII |
UIDNA_NONTRANSITIONAL_TO_UNICODE,
&status);
check_status(status, "uidna_openUTS46");
const int32_t capacity = 512;
for (const auto &[name, domain_utf8] : pairs(inputs + "/idna.tsv")) {
UnicodeString domain = UnicodeString::fromUTF8(domain_utf8);
auto to_ascii = [&](char16_t *destination) {
UIDNAInfo info = UIDNA_INFO_INITIALIZER;
UErrorCode status = U_ZERO_ERROR;
return (long)uidna_nameToASCII(idna, domain.getBuffer(), domain.length(), destination,
capacity, &info, &status);
};
auto to_unicode = [&](char16_t *destination) {
UIDNAInfo info = UIDNA_INFO_INITIALIZER;
UErrorCode status = U_ZERO_ERROR;
return (long)uidna_nameToUnicode(idna, domain.getBuffer(), domain.length(), destination,
capacity, &info, &status);
};
char16_t buffer[capacity];
long ascii_length = to_ascii(buffer);
long unicode_length = u_countChar32(buffer, (int32_t)to_unicode(buffer));
measure("idna/to_ascii/" + name, 1, ascii_length, [&]() {
char16_t destination[capacity];
return to_ascii(destination);
});
measure("idna/to_unicode/" + name, 1, unicode_length, [&]() {
char16_t destination[capacity];
return to_unicode(destination);
});
}
uidna_close(idna);
}
static void bench_properties(const std::string &inputs) {
UErrorCode status = U_ZERO_ERROR;
const Normalizer2 *nfkc_casefold = Normalizer2::getNFKCCasefoldInstance(status);
check_status(status, "Normalizer2");
// Full default case folding into a fresh destination per call. Folding
// expands a UTF-16 code unit to at most three. UnicodeString::foldCase
// copies the source first and takes twice as long.
auto fold = [](const UnicodeString &source) {
UnicodeString destination;
int32_t capacity = source.length() * 3;
UErrorCode status = U_ZERO_ERROR;
int32_t length = u_strFoldCase(destination.getBuffer(capacity), capacity, source.getBuffer(),
source.length(), U_FOLD_CASE_DEFAULT, &status);
destination.releaseBuffer(U_SUCCESS(status) ? length : 0);
return destination;
};
// These are calls into the ICU shared library: nothing is inlined into the loop.
for (const std::string &name : names(inputs + "/case")) {
UnicodeString text = read(inputs + "/case/" + name + ".txt");
long units = text.countChar32();
auto simple_upper = [&]() {
long sum = 0;
for (int32_t i = 0; i < text.length();) {
UChar32 c = text.char32At(i);
sum ^= u_toupper(c);
i += U16_LENGTH(c);
}
return sum;
};
auto lowercase_letters = [&]() {
long count = 0;
for (int32_t i = 0; i < text.length();) {
UChar32 c = text.char32At(i);
count += u_charType(c) == U_LOWERCASE_LETTER;
i += U16_LENGTH(c);
}
return count;
};
measure("case/simple_upper/" + name, units, simple_upper(), simple_upper);
measure("case/fold/" + name, units, fold(text).countChar32(),
[&]() { return (long)fold(text).length(); });
measure("case/nfkc_casefold/" + name, units, normalized(nfkc_casefold, text).countChar32(),
[&]() { return (long)normalized(nfkc_casefold, text).length(); });
measure("prop/general_category_Ll/" + name, units, lowercase_letters(), lowercase_letters);
}
}
int main(int argc, char **argv) {
if (argc < 2) {
fprintf(stderr, "usage: xlang <inputs> [group]\n");
return 2;
}
std::string inputs = argv[1];
std::string group = argc > 2 ? argv[2] : "";
const std::pair<const char *, void (*)(const std::string &)> groups[] = {
{"norm", bench_normalization}, {"seg", bench_segmentation}, {"bidi", bench_bidi},
{"idna", bench_idna}, {"case", bench_properties}};
for (const auto &[name, bench] : groups)
if (group.empty() || group == name) bench(inputs);
return 0;
}