unicode

Branch: main

Switch branches or tags

Branch: main
Go to file t

413 files indexed

History
Code

bench/xlang/inputs.mbt
1///|2/// Write `text` as UTF-8, without a BOM and without touching line endings.3async fn write_text(path : String, text : String) -> Unit {4  @fs.write_text(path, text, permission=0o644)5}67///|8/// Write the inputs every harness reads into `directory`:9///10/// - `norm/<corpus>.txt`: the ICU normperf corpora of `bench/normperf`, without11///   their BOM, and `normd/<corpus>.txt`: the same text in NFD12/// - `seg/`, `bidi/`, `case/`: one text per file13/// - `idna.tsv`, `puny.tsv`: `name<TAB>value` lines14///15/// The harnesses list these directories, so the inputs are defined only here.16async fn prepare_inputs(root : String, directory : String) -> Unit {17  if @fs.exists(directory) {18    @fs.remove(directory, recursive=true)19  }20  for group in ["norm", "normd", "seg", "bidi", "case"] {21    @fs.mkdir("\{directory}/\{group}", permission=0o755, recursive=true)22  }23  let corpora : Map[String, String] = Map([])24  let data = "\{root}/bench/normperf/data"25  for file in @fs.readdir(data, sort=true) {26    guard file.strip_suffix(".txt") is Some(name) else { continue }27    let text = @utf8.decode(@fs.read_binary("\{data}/\{file}"), ignore_bom=true)28    corpora[name.to_owned()] = text29    write_text("\{directory}/norm/\{file}", text)30    write_text(31      "\{directory}/normd/\{file}",32      @normalization.normalize(text, NFD),33    )34  }35  let corpus = (name : String) => {36    guard corpora.get(name) is Some(text) else {37      fail("Missing corpus \{name} in \{data}")38    }39    text40  }41  let texts : Array[(String, String)] = [42    ("seg/Latin", corpus("TestNames_Latin")),43    ("seg/Russian", corpus("TestNames_Russian")),44    ("seg/Chinese", corpus("TestNames_Chinese")),45    ("seg/Korean", corpus("TestNames_Korean")),46    (47      "seg/KoreanJamo",48      @normalization.normalize(corpus("TestNames_Korean"), NFD),49    ),50    ("seg/Thai", corpus("TestNames_Thai")),51    ("seg/English", corpus("vfear11a")),52    (53      "seg/emoji",54      "👨‍👩‍👧‍👦 🇯🇵🇺🇸 👍🏽 🏳️‍🌈 ❤️ 😀 text 🧑🏿‍🚀 ".repeat(55        400,56      ),57    ),58    (59      "seg/Hindi",60      "हिन्दी भाषा में क्षत्रिय और विद्यार्थी शब्द संयुक्ताक्षर के उदाहरण हैं। ".repeat(61        300,62      ),63    ),64    (65      "bidi/ASCII",66      "Hello world, this is a test (of brackets) 12345. ".repeat(32),67    ),68    ("bidi/Arabic", "مرحبا بالعالم (اختبار) 123 ".repeat(32)),69    (70      "bidi/mixed",71      "Hello مرحبا world بالعالم [test] 12,345.67 ".repeat(32),72    ),73    ("bidi/CJK", "世界こんにちは、テスト。".repeat(64)),74    ("bidi/supplementary", "𐤀𐤁𐤂 😀 𝒜𝒷 ".repeat(64)),75    (76      "bidi/explicit",77      "abc \u{2067}\u{05E9}\u{05DC}\u{05D5}\u{05DD} (x) 12\u{2069} def \u{202B}\u{05E2}\u{05D1}\u{05E8}\u{05D9}\u{05EA}\u{202C} ".repeat(78        32,79      ),80    ),81    ("bidi/brackets", "a (b [c {d} \u{05D0}] 1) \u{05D1} ".repeat(48)),82    ("case/ASCII", "Hello world, this is a Test. ".repeat(32)),83    ("case/Latin", "Ünïcödé Çàfé Straße ".repeat(32)),84    ("case/Greek", "Ὀδυσσεύς καὶ ᾈχιλλεύς ".repeat(32)),85    ("case/CJK", "世界こんにちは、テスト。".repeat(64)),86    // Real text with its own mix of cases: capitalized names and English prose.87    ("case/names_Latin", corpus("TestNames_Latin")),88    ("case/names_Russian", corpus("TestNames_Russian")),89    ("case/English", corpus("vfear11a")),90    // Mostly uppercase, so that nearly every letter has a mapping to apply.91    ("case/names_Latin_upper", uppercase(corpus("TestNames_Latin"))),92  ]93  for (name, text) in texts {94    write_text("\{directory}/\{name}.txt", text)95  }96  let tables : Array[(String, Array[(String, String)])] = [97    (98      "idna",99      [100        ("ASCII", "www.example.com"),101        ("longASCII", "the-quick-brown-fox.jumps-over-the-lazy-dog.example.org"),102        ("mixedcaseASCII", "WWW.Example.COM"),103        ("Latin", "bücher.münchen.example"),104        ("mixedcaseLatin", "Bücher.München.Example"),105        ("CJK", "例え.テスト.日本語"),106        ("Arabic", "مثال.إختبار"),107        ("Alabels", "xn--bcher-kva.xn--mnchen-3ya.example"),108      ],109    ),110    (111      "puny",112      [113        ("ASCII", "example"),114        ("Latin", "bücher"),115        ("Arabic", "مثال"),116        ("CJK", "日本語テスト例え"),117        ("supplementary", "😀😃😄𝒜"),118        (119          "long", "ünïcödé-ünïcödé-ünïcödé-ünïcödé-ünïcödé-ünïcödé",120        ),121      ],122    ),123  ]124  for (name, pairs) in tables {125    let lines = StringBuilder()126    for (key, value) in pairs {127      lines.write_string("\{key}\t\{value}\n")128    }129    write_text("\{directory}/\{name}.tsv", lines.to_string())130  }131}132133///|134/// `text` with every character replaced by its full uppercase mapping.135fn uppercase(text : String) -> String {136  let result = StringBuilder(capacity=text.length())137  for c in text {138    result.write_string(@ucd.to_uppercase(c))139  }140  result.to_string()141}142