bench/xlang/inputs.mbt1///|2/// Write `text` as UTF-8, without a BOM and without touching line endings.3async fn write_text(path : String, text : String) -> Unit {4 @fs.write_text(path, text, permission=0o644)5}67///|8/// Write the inputs every harness reads into `directory`:9///10/// - `norm/<corpus>.txt`: the ICU normperf corpora of `bench/normperf`, without11/// their BOM, and `normd/<corpus>.txt`: the same text in NFD12/// - `seg/`, `bidi/`, `case/`: one text per file13/// - `idna.tsv`, `puny.tsv`: `name<TAB>value` lines14///15/// The harnesses list these directories, so the inputs are defined only here.16async fn prepare_inputs(root : String, directory : String) -> Unit {17 if @fs.exists(directory) {18 @fs.remove(directory, recursive=true)19 }20 for group in ["norm", "normd", "seg", "bidi", "case"] {21 @fs.mkdir("\{directory}/\{group}", permission=0o755, recursive=true)22 }23 let corpora : Map[String, String] = Map([])24 let data = "\{root}/bench/normperf/data"25 for file in @fs.readdir(data, sort=true) {26 guard file.strip_suffix(".txt") is Some(name) else { continue }27 let text = @utf8.decode(@fs.read_binary("\{data}/\{file}"), ignore_bom=true)28 corpora[name.to_owned()] = text29 write_text("\{directory}/norm/\{file}", text)30 write_text(31 "\{directory}/normd/\{file}",32 @normalization.normalize(text, NFD),33 )34 }35 let corpus = (name : String) => {36 guard corpora.get(name) is Some(text) else {37 fail("Missing corpus \{name} in \{data}")38 }39 text40 }41 let texts : Array[(String, String)] = [42 ("seg/Latin", corpus("TestNames_Latin")),43 ("seg/Russian", corpus("TestNames_Russian")),44 ("seg/Chinese", corpus("TestNames_Chinese")),45 ("seg/Korean", corpus("TestNames_Korean")),46 (47 "seg/KoreanJamo",48 @normalization.normalize(corpus("TestNames_Korean"), NFD),49 ),50 ("seg/Thai", corpus("TestNames_Thai")),51 ("seg/English", corpus("vfear11a")),52 (53 "seg/emoji",54 "👨👩👧👦 🇯🇵🇺🇸 👍🏽 🏳️🌈 ❤️ 😀 text 🧑🏿🚀 ".repeat(55 400,56 ),57 ),58 (59 "seg/Hindi",60 "हिन्दी भाषा में क्षत्रिय और विद्यार्थी शब्द संयुक्ताक्षर के उदाहरण हैं। ".repeat(61 300,62 ),63 ),64 (65 "bidi/ASCII",66 "Hello world, this is a test (of brackets) 12345. ".repeat(32),67 ),68 ("bidi/Arabic", "مرحبا بالعالم (اختبار) 123 ".repeat(32)),69 (70 "bidi/mixed",71 "Hello مرحبا world بالعالم [test] 12,345.67 ".repeat(32),72 ),73 ("bidi/CJK", "世界こんにちは、テスト。".repeat(64)),74 ("bidi/supplementary", "𐤀𐤁𐤂 😀 𝒜𝒷 ".repeat(64)),75 (76 "bidi/explicit",77 "abc \u{2067}\u{05E9}\u{05DC}\u{05D5}\u{05DD} (x) 12\u{2069} def \u{202B}\u{05E2}\u{05D1}\u{05E8}\u{05D9}\u{05EA}\u{202C} ".repeat(78 32,79 ),80 ),81 ("bidi/brackets", "a (b [c {d} \u{05D0}] 1) \u{05D1} ".repeat(48)),82 ("case/ASCII", "Hello world, this is a Test. ".repeat(32)),83 ("case/Latin", "Ünïcödé Çàfé Straße ".repeat(32)),84 ("case/Greek", "Ὀδυσσεύς καὶ ᾈχιλλεύς ".repeat(32)),85 ("case/CJK", "世界こんにちは、テスト。".repeat(64)),86 // Real text with its own mix of cases: capitalized names and English prose.87 ("case/names_Latin", corpus("TestNames_Latin")),88 ("case/names_Russian", corpus("TestNames_Russian")),89 ("case/English", corpus("vfear11a")),90 // Mostly uppercase, so that nearly every letter has a mapping to apply.91 ("case/names_Latin_upper", uppercase(corpus("TestNames_Latin"))),92 ]93 for (name, text) in texts {94 write_text("\{directory}/\{name}.txt", text)95 }96 let tables : Array[(String, Array[(String, String)])] = [97 (98 "idna",99 [100 ("ASCII", "www.example.com"),101 ("longASCII", "the-quick-brown-fox.jumps-over-the-lazy-dog.example.org"),102 ("mixedcaseASCII", "WWW.Example.COM"),103 ("Latin", "bücher.münchen.example"),104 ("mixedcaseLatin", "Bücher.München.Example"),105 ("CJK", "例え.テスト.日本語"),106 ("Arabic", "مثال.إختبار"),107 ("Alabels", "xn--bcher-kva.xn--mnchen-3ya.example"),108 ],109 ),110 (111 "puny",112 [113 ("ASCII", "example"),114 ("Latin", "bücher"),115 ("Arabic", "مثال"),116 ("CJK", "日本語テスト例え"),117 ("supplementary", "😀😃😄𝒜"),118 (119 "long", "ünïcödé-ünïcödé-ünïcödé-ünïcödé-ünïcödé-ünïcödé",120 ),121 ],122 ),123 ]124 for (name, pairs) in tables {125 let lines = StringBuilder()126 for (key, value) in pairs {127 lines.write_string("\{key}\t\{value}\n")128 }129 write_text("\{directory}/\{name}.tsv", lines.to_string())130 }131}132133///|134/// `text` with every character replaced by its full uppercase mapping.135fn uppercase(text : String) -> String {136 let result = StringBuilder(capacity=text.length())137 for c in text {138 result.write_string(@ucd.to_uppercase(c))139 }140 result.to_string()141}142
///|
/// Write `text` as UTF-8, without a BOM and without touching line endings.
async fn write_text(path : String, text : String) -> Unit {
@fs.write_text(path, text, permission=0o644)
}
///|
/// Write the inputs every harness reads into `directory`:
///
/// - `norm/<corpus>.txt`: the ICU normperf corpora of `bench/normperf`, without
/// their BOM, and `normd/<corpus>.txt`: the same text in NFD
/// - `seg/`, `bidi/`, `case/`: one text per file
/// - `idna.tsv`, `puny.tsv`: `name<TAB>value` lines
///
/// The harnesses list these directories, so the inputs are defined only here.
async fn prepare_inputs(root : String, directory : String) -> Unit {
if @fs.exists(directory) {
@fs.remove(directory, recursive=true)
}
for group in ["norm", "normd", "seg", "bidi", "case"] {
@fs.mkdir("\{directory}/\{group}", permission=0o755, recursive=true)
}
let corpora : Map[String, String] = Map([])
let data = "\{root}/bench/normperf/data"
for file in @fs.readdir(data, sort=true) {
guard file.strip_suffix(".txt") is Some(name) else { continue }
let text = @utf8.decode(@fs.read_binary("\{data}/\{file}"), ignore_bom=true)
corpora[name.to_owned()] = text
write_text("\{directory}/norm/\{file}", text)
write_text(
"\{directory}/normd/\{file}",
@normalization.normalize(text, NFD),
)
}
let corpus = (name : String) => {
guard corpora.get(name) is Some(text) else {
fail("Missing corpus \{name} in \{data}")
}
text
}
let texts : Array[(String, String)] = [
("seg/Latin", corpus("TestNames_Latin")),
("seg/Russian", corpus("TestNames_Russian")),
("seg/Chinese", corpus("TestNames_Chinese")),
("seg/Korean", corpus("TestNames_Korean")),
(
"seg/KoreanJamo",
@normalization.normalize(corpus("TestNames_Korean"), NFD),
),
("seg/Thai", corpus("TestNames_Thai")),
("seg/English", corpus("vfear11a")),
(
"seg/emoji",
"👨👩👧👦 🇯🇵🇺🇸 👍🏽 🏳️🌈 ❤️ 😀 text 🧑🏿🚀 ".repeat(
400,
),
),
(
"seg/Hindi",
"हिन्दी भाषा में क्षत्रिय और विद्यार्थी शब्द संयुक्ताक्षर के उदाहरण हैं। ".repeat(
300,
),
),
(
"bidi/ASCII",
"Hello world, this is a test (of brackets) 12345. ".repeat(32),
),
("bidi/Arabic", "مرحبا بالعالم (اختبار) 123 ".repeat(32)),
(
"bidi/mixed",
"Hello مرحبا world بالعالم [test] 12,345.67 ".repeat(32),
),
("bidi/CJK", "世界こんにちは、テスト。".repeat(64)),
("bidi/supplementary", "𐤀𐤁𐤂 😀 𝒜𝒷 ".repeat(64)),
(
"bidi/explicit",
"abc \u{2067}\u{05E9}\u{05DC}\u{05D5}\u{05DD} (x) 12\u{2069} def \u{202B}\u{05E2}\u{05D1}\u{05E8}\u{05D9}\u{05EA}\u{202C} ".repeat(
32,
),
),
("bidi/brackets", "a (b [c {d} \u{05D0}] 1) \u{05D1} ".repeat(48)),
("case/ASCII", "Hello world, this is a Test. ".repeat(32)),
("case/Latin", "Ünïcödé Çàfé Straße ".repeat(32)),
("case/Greek", "Ὀδυσσεύς καὶ ᾈχιλλεύς ".repeat(32)),
("case/CJK", "世界こんにちは、テスト。".repeat(64)),
// Real text with its own mix of cases: capitalized names and English prose.
("case/names_Latin", corpus("TestNames_Latin")),
("case/names_Russian", corpus("TestNames_Russian")),
("case/English", corpus("vfear11a")),
// Mostly uppercase, so that nearly every letter has a mapping to apply.
("case/names_Latin_upper", uppercase(corpus("TestNames_Latin"))),
]
for (name, text) in texts {
write_text("\{directory}/\{name}.txt", text)
}
let tables : Array[(String, Array[(String, String)])] = [
(
"idna",
[
("ASCII", "www.example.com"),
("longASCII", "the-quick-brown-fox.jumps-over-the-lazy-dog.example.org"),
("mixedcaseASCII", "WWW.Example.COM"),
("Latin", "bücher.münchen.example"),
("mixedcaseLatin", "Bücher.München.Example"),
("CJK", "例え.テスト.日本語"),
("Arabic", "مثال.إختبار"),
("Alabels", "xn--bcher-kva.xn--mnchen-3ya.example"),
],
),
(
"puny",
[
("ASCII", "example"),
("Latin", "bücher"),
("Arabic", "مثال"),
("CJK", "日本語テスト例え"),
("supplementary", "😀😃😄𝒜"),
(
"long", "ünïcödé-ünïcödé-ünïcödé-ünïcödé-ünïcödé-ünïcödé",
),
],
),
]
for (name, pairs) in tables {
let lines = StringBuilder()
for (key, value) in pairs {
lines.write_string("\{key}\t\{value}\n")
}
write_text("\{directory}/\{name}.tsv", lines.to_string())
}
}
///|
/// `text` with every character replaced by its full uppercase mapping.
fn uppercase(text : String) -> String {
let result = StringBuilder(capacity=text.length())
for c in text {
result.write_string(@ucd.to_uppercase(c))
}
result.to_string()
}