unicode

Branch: main

Switch branches or tags

Branch: main
Go to file t

413 files indexed

History
Code

bench/xlang/python/main.py
1"""Python peer of bench/xlang/moonbit: normalization with the standard2library's unicodedata and full case folding with str.casefold, on the same3inputs, timed with the same protocol and printed in the same row format."""45import os6import sys7import time8import unicodedata91011def run(iterations, f):12    start = time.perf_counter_ns()13    for _ in range(iterations):14        f()15    return time.perf_counter_ns() - start161718def measure(name, units, check, f):19    """Double the iteration count until one sample takes at least 5 ms, then20    print the fastest of 5 samples, the first of which is that one, as21    library<TAB>case<TAB>ns<TAB>units<TAB>check.22    The library column is empty: this harness has one library."""23    iterations = 124    first = run(iterations, f)25    while first < 5e6:26        iterations *= 227        first = run(iterations, f)28    best = min([first] + [run(iterations, f) for _ in range(4)])29    print(f"\t{name}\t{best / iterations}\t{units}\t{check}", flush=True)303132def read(path):33    with open(path, encoding="utf-8", newline="") as file:34        return file.read()353637def names(directory):38    """Input names in `directory`: the .txt files without their extension."""39    return sorted(file[:-4] for file in os.listdir(directory) if file.endswith(".txt"))404142def bench_normalization(inputs):43    normalize = unicodedata.normalize44    for name in names(f"{inputs}/norm"):45        original = read(f"{inputs}/norm/{name}.txt")46        decomposed = read(f"{inputs}/normd/{name}.txt")47        for form in ["NFC", "NFD", "NFKC", "NFKD"]:48            measure(f"norm/{form.lower()}/{name}", len(original), len(normalize(form, original)),49                    lambda: normalize(form, original))50        measure(f"norm/nfd>nfc/{name}", len(decomposed), len(normalize("NFC", decomposed)),51                lambda: normalize("NFC", decomposed))52        measure(f"norm/is_nfc/{name}", len(original),53                int(unicodedata.is_normalized("NFC", original)),54                lambda: unicodedata.is_normalized("NFC", original))555657def bench_case(inputs):58    for name in names(f"{inputs}/case"):59        text = read(f"{inputs}/case/{name}.txt")60        measure(f"case/fold/{name}", len(text), len(text.casefold()), text.casefold)616263if __name__ == "__main__":64    if len(sys.argv) < 2:65        sys.exit("usage: main.py <inputs> [group]")66    group = sys.argv[2] if len(sys.argv) > 2 else None67    for selected, bench in [("norm", bench_normalization), ("case", bench_case)]:68        if group in (None, selected):69            bench(sys.argv[1])70