unicode

Branch: main

Switch branches or tags

Branch: main
Go to file t

413 files indexed

History
Code

bench/xlang/go/main.go
1// Go peer of bench/xlang/moonbit: the same cases on the same inputs, timed2// with the same protocol and printed in the same row format.3package main45import (6	"fmt"7	"math"8	"os"9	"sort"10	"strings"11	"time"12	"unicode"13	"unicode/utf8"1415	"github.com/rivo/uniseg"16	"golang.org/x/net/idna"17	"golang.org/x/text/cases"18	"golang.org/x/text/unicode/bidi"19	"golang.org/x/text/unicode/norm"20)2122var sink int2324func run(iterations int, f func() int) float64 {25	start := time.Now()26	for i := 0; i < iterations; i++ {27		sink ^= f()28	}29	return float64(time.Since(start).Nanoseconds())30}3132// measure doubles the iteration count until one sample takes at least 5 ms,33// then prints the fastest of 5 samples, the first of which is that one, as34// library<TAB>case<TAB>ns<TAB>units<TAB>check.35func measure(library, name string, units int, check any, f func() int) {36	iterations := 137	best := run(iterations, f)38	for best < 5e6 {39		iterations *= 240		best = run(iterations, f)41	}42	for i := 1; i < 5; i++ {43		best = math.Min(best, run(iterations, f))44	}45	fmt.Printf("%s\t%s\t%v\t%d\t%v\n", library, name, best/float64(iterations), units, check)46}4748func read(path string) string {49	data, err := os.ReadFile(path)50	if err != nil {51		panic(err)52	}53	return string(data)54}5556// names lists the inputs in directory: the .txt files without their extension.57func names(directory string) []string {58	entries, err := os.ReadDir(directory)59	if err != nil {60		panic(err)61	}62	var result []string63	for _, entry := range entries {64		if name, ok := strings.CutSuffix(entry.Name(), ".txt"); ok {65			result = append(result, name)66		}67	}68	sort.Strings(result)69	return result70}7172// pairs returns the name<TAB>value lines of a .tsv input.73func pairs(path string) [][2]string {74	var result [][2]string75	for _, line := range strings.Split(read(path), "\n") {76		if name, value, ok := strings.Cut(line, "\t"); ok {77			result = append(result, [2]string{name, value})78		}79	}80	return result81}8283func toInt(b bool) int {84	if b {85		return 186	}87	return 088}8990func benchNormalization(inputs string) {91	const library = "x/text"92	forms := []struct {93		name string94		form norm.Form95	}{{"nfc", norm.NFC}, {"nfd", norm.NFD}, {"nfkc", norm.NFKC}, {"nfkd", norm.NFKD}}96	for _, name := range names(inputs + "/norm") {97		original := read(inputs + "/norm/" + name + ".txt")98		decomposed := read(inputs + "/normd/" + name + ".txt")99		units := utf8.RuneCountInString(original)100		for _, f := range forms {101			form := f.form102			measure(library, "norm/"+f.name+"/"+name, units,103				utf8.RuneCountInString(form.String(original)),104				func() int { return len(form.String(original)) })105		}106		measure(library, "norm/nfd>nfc/"+name, utf8.RuneCountInString(decomposed),107			utf8.RuneCountInString(norm.NFC.String(decomposed)),108			func() int { return len(norm.NFC.String(decomposed)) })109		measure(library, "norm/is_nfc/"+name, units, toInt(norm.NFC.IsNormalString(original)),110			func() int { return toInt(norm.NFC.IsNormalString(original)) })111	}112}113114func benchSegmentation(inputs string) {115	for _, name := range names(inputs + "/seg") {116		text := read(inputs + "/seg/" + name + ".txt")117		measure("uniseg", "seg/graphemes/"+name, utf8.RuneCountInString(text),118			uniseg.GraphemeClusterCount(text),119			func() int { return uniseg.GraphemeClusterCount(text) })120	}121}122123func benchBidi(inputs string) {124	for _, name := range names(inputs + "/bidi") {125		text := read(inputs + "/bidi/" + name + ".txt")126		order := func() int {127			var paragraph bidi.Paragraph128			if _, err := paragraph.SetString(text); err != nil {129				panic(err)130			}131			ordering, err := paragraph.Order()132			if err != nil {133				panic(err)134			}135			return ordering.NumRuns()136		}137		// x/text exposes runs rather than levels, so there is no comparable check.138		measure("x/text", "bidi/process/"+name, utf8.RuneCountInString(text), "-", order)139	}140}141142func benchIdna(inputs string) {143	// The closest match to the other harnesses: UTS #46 non-transitional144	// processing with Bidi, joiner and hyphen checks but without STD3 rules.145	profile := idna.New(idna.MapForLookup(), idna.BidiRule(), idna.CheckJoiners(true),146		idna.CheckHyphens(true), idna.StrictDomainName(false))147	for _, pair := range pairs(inputs + "/idna.tsv") {148		name, domain := pair[0], pair[1]149		ascii, err := profile.ToASCII(domain)150		if err != nil {151			panic(err)152		}153		unicode, err := profile.ToUnicode(domain)154		if err != nil {155			panic(err)156		}157		measure("x/net/idna", "idna/to_ascii/"+name, 1, utf8.RuneCountInString(ascii),158			func() int { result, _ := profile.ToASCII(domain); return len(result) })159		measure("x/net/idna", "idna/to_unicode/"+name, 1, utf8.RuneCountInString(unicode),160			func() int { result, _ := profile.ToUnicode(domain); return len(result) })161	}162}163164func benchProperties(inputs string) {165	for _, name := range names(inputs + "/case") {166		text := read(inputs + "/case/" + name + ".txt")167		units := utf8.RuneCountInString(text)168		simpleUpper := func() int {169			sum := 0170			for _, r := range text {171				sum ^= int(unicode.ToUpper(r))172			}173			return sum174		}175		lowercaseLetters := func() int {176			count := 0177			for _, r := range text {178				if unicode.Is(unicode.Ll, r) {179					count++180				}181			}182			return count183		}184		fold := cases.Fold()185		measure("std", "case/simple_upper/"+name, units, simpleUpper(), simpleUpper)186		measure("x/text", "case/fold/"+name, units, utf8.RuneCountInString(fold.String(text)),187			func() int { return len(fold.String(text)) })188		measure("std", "prop/general_category_Ll/"+name, units, lowercaseLetters(), lowercaseLetters)189	}190}191192func main() {193	if len(os.Args) < 2 {194		fmt.Fprintln(os.Stderr, "usage: xlang <inputs> [group]")195		os.Exit(2)196	}197	inputs, group := os.Args[1], ""198	if len(os.Args) > 2 {199		group = os.Args[2]200	}201	groups := []struct {202		name  string203		bench func(string)204	}{205		{"norm", benchNormalization}, {"seg", benchSegmentation}, {"bidi", benchBidi},206		{"idna", benchIdna}, {"case", benchProperties},207	}208	for _, g := range groups {209		if group == "" || group == g.name {210			g.bench(inputs)211		}212	}213}214