bench/xlang/go/main.go1// Go peer of bench/xlang/moonbit: the same cases on the same inputs, timed2// with the same protocol and printed in the same row format.3package main45import (6 "fmt"7 "math"8 "os"9 "sort"10 "strings"11 "time"12 "unicode"13 "unicode/utf8"1415 "github.com/rivo/uniseg"16 "golang.org/x/net/idna"17 "golang.org/x/text/cases"18 "golang.org/x/text/unicode/bidi"19 "golang.org/x/text/unicode/norm"20)2122var sink int2324func run(iterations int, f func() int) float64 {25 start := time.Now()26 for i := 0; i < iterations; i++ {27 sink ^= f()28 }29 return float64(time.Since(start).Nanoseconds())30}3132// measure doubles the iteration count until one sample takes at least 5 ms,33// then prints the fastest of 5 samples, the first of which is that one, as34// library<TAB>case<TAB>ns<TAB>units<TAB>check.35func measure(library, name string, units int, check any, f func() int) {36 iterations := 137 best := run(iterations, f)38 for best < 5e6 {39 iterations *= 240 best = run(iterations, f)41 }42 for i := 1; i < 5; i++ {43 best = math.Min(best, run(iterations, f))44 }45 fmt.Printf("%s\t%s\t%v\t%d\t%v\n", library, name, best/float64(iterations), units, check)46}4748func read(path string) string {49 data, err := os.ReadFile(path)50 if err != nil {51 panic(err)52 }53 return string(data)54}5556// names lists the inputs in directory: the .txt files without their extension.57func names(directory string) []string {58 entries, err := os.ReadDir(directory)59 if err != nil {60 panic(err)61 }62 var result []string63 for _, entry := range entries {64 if name, ok := strings.CutSuffix(entry.Name(), ".txt"); ok {65 result = append(result, name)66 }67 }68 sort.Strings(result)69 return result70}7172// pairs returns the name<TAB>value lines of a .tsv input.73func pairs(path string) [][2]string {74 var result [][2]string75 for _, line := range strings.Split(read(path), "\n") {76 if name, value, ok := strings.Cut(line, "\t"); ok {77 result = append(result, [2]string{name, value})78 }79 }80 return result81}8283func toInt(b bool) int {84 if b {85 return 186 }87 return 088}8990func benchNormalization(inputs string) {91 const library = "x/text"92 forms := []struct {93 name string94 form norm.Form95 }{{"nfc", norm.NFC}, {"nfd", norm.NFD}, {"nfkc", norm.NFKC}, {"nfkd", norm.NFKD}}96 for _, name := range names(inputs + "/norm") {97 original := read(inputs + "/norm/" + name + ".txt")98 decomposed := read(inputs + "/normd/" + name + ".txt")99 units := utf8.RuneCountInString(original)100 for _, f := range forms {101 form := f.form102 measure(library, "norm/"+f.name+"/"+name, units,103 utf8.RuneCountInString(form.String(original)),104 func() int { return len(form.String(original)) })105 }106 measure(library, "norm/nfd>nfc/"+name, utf8.RuneCountInString(decomposed),107 utf8.RuneCountInString(norm.NFC.String(decomposed)),108 func() int { return len(norm.NFC.String(decomposed)) })109 measure(library, "norm/is_nfc/"+name, units, toInt(norm.NFC.IsNormalString(original)),110 func() int { return toInt(norm.NFC.IsNormalString(original)) })111 }112}113114func benchSegmentation(inputs string) {115 for _, name := range names(inputs + "/seg") {116 text := read(inputs + "/seg/" + name + ".txt")117 measure("uniseg", "seg/graphemes/"+name, utf8.RuneCountInString(text),118 uniseg.GraphemeClusterCount(text),119 func() int { return uniseg.GraphemeClusterCount(text) })120 }121}122123func benchBidi(inputs string) {124 for _, name := range names(inputs + "/bidi") {125 text := read(inputs + "/bidi/" + name + ".txt")126 order := func() int {127 var paragraph bidi.Paragraph128 if _, err := paragraph.SetString(text); err != nil {129 panic(err)130 }131 ordering, err := paragraph.Order()132 if err != nil {133 panic(err)134 }135 return ordering.NumRuns()136 }137 // x/text exposes runs rather than levels, so there is no comparable check.138 measure("x/text", "bidi/process/"+name, utf8.RuneCountInString(text), "-", order)139 }140}141142func benchIdna(inputs string) {143 // The closest match to the other harnesses: UTS #46 non-transitional144 // processing with Bidi, joiner and hyphen checks but without STD3 rules.145 profile := idna.New(idna.MapForLookup(), idna.BidiRule(), idna.CheckJoiners(true),146 idna.CheckHyphens(true), idna.StrictDomainName(false))147 for _, pair := range pairs(inputs + "/idna.tsv") {148 name, domain := pair[0], pair[1]149 ascii, err := profile.ToASCII(domain)150 if err != nil {151 panic(err)152 }153 unicode, err := profile.ToUnicode(domain)154 if err != nil {155 panic(err)156 }157 measure("x/net/idna", "idna/to_ascii/"+name, 1, utf8.RuneCountInString(ascii),158 func() int { result, _ := profile.ToASCII(domain); return len(result) })159 measure("x/net/idna", "idna/to_unicode/"+name, 1, utf8.RuneCountInString(unicode),160 func() int { result, _ := profile.ToUnicode(domain); return len(result) })161 }162}163164func benchProperties(inputs string) {165 for _, name := range names(inputs + "/case") {166 text := read(inputs + "/case/" + name + ".txt")167 units := utf8.RuneCountInString(text)168 simpleUpper := func() int {169 sum := 0170 for _, r := range text {171 sum ^= int(unicode.ToUpper(r))172 }173 return sum174 }175 lowercaseLetters := func() int {176 count := 0177 for _, r := range text {178 if unicode.Is(unicode.Ll, r) {179 count++180 }181 }182 return count183 }184 fold := cases.Fold()185 measure("std", "case/simple_upper/"+name, units, simpleUpper(), simpleUpper)186 measure("x/text", "case/fold/"+name, units, utf8.RuneCountInString(fold.String(text)),187 func() int { return len(fold.String(text)) })188 measure("std", "prop/general_category_Ll/"+name, units, lowercaseLetters(), lowercaseLetters)189 }190}191192func main() {193 if len(os.Args) < 2 {194 fmt.Fprintln(os.Stderr, "usage: xlang <inputs> [group]")195 os.Exit(2)196 }197 inputs, group := os.Args[1], ""198 if len(os.Args) > 2 {199 group = os.Args[2]200 }201 groups := []struct {202 name string203 bench func(string)204 }{205 {"norm", benchNormalization}, {"seg", benchSegmentation}, {"bidi", benchBidi},206 {"idna", benchIdna}, {"case", benchProperties},207 }208 for _, g := range groups {209 if group == "" || group == g.name {210 g.bench(inputs)211 }212 }213}214
// Go peer of bench/xlang/moonbit: the same cases on the same inputs, timed
// with the same protocol and printed in the same row format.
package main
import (
"fmt"
"math"
"os"
"sort"
"strings"
"time"
"unicode"
"unicode/utf8"
"github.com/rivo/uniseg"
"golang.org/x/net/idna"
"golang.org/x/text/cases"
"golang.org/x/text/unicode/bidi"
"golang.org/x/text/unicode/norm"
)
var sink int
func run(iterations int, f func() int) float64 {
start := time.Now()
for i := 0; i < iterations; i++ {
sink ^= f()
}
return float64(time.Since(start).Nanoseconds())
}
// measure doubles the iteration count until one sample takes at least 5 ms,
// then prints the fastest of 5 samples, the first of which is that one, as
// library<TAB>case<TAB>ns<TAB>units<TAB>check.
func measure(library, name string, units int, check any, f func() int) {
iterations := 1
best := run(iterations, f)
for best < 5e6 {
iterations *= 2
best = run(iterations, f)
}
for i := 1; i < 5; i++ {
best = math.Min(best, run(iterations, f))
}
fmt.Printf("%s\t%s\t%v\t%d\t%v\n", library, name, best/float64(iterations), units, check)
}
func read(path string) string {
data, err := os.ReadFile(path)
if err != nil {
panic(err)
}
return string(data)
}
// names lists the inputs in directory: the .txt files without their extension.
func names(directory string) []string {
entries, err := os.ReadDir(directory)
if err != nil {
panic(err)
}
var result []string
for _, entry := range entries {
if name, ok := strings.CutSuffix(entry.Name(), ".txt"); ok {
result = append(result, name)
}
}
sort.Strings(result)
return result
}
// pairs returns the name<TAB>value lines of a .tsv input.
func pairs(path string) [][2]string {
var result [][2]string
for _, line := range strings.Split(read(path), "\n") {
if name, value, ok := strings.Cut(line, "\t"); ok {
result = append(result, [2]string{name, value})
}
}
return result
}
func toInt(b bool) int {
if b {
return 1
}
return 0
}
func benchNormalization(inputs string) {
const library = "x/text"
forms := []struct {
name string
form norm.Form
}{{"nfc", norm.NFC}, {"nfd", norm.NFD}, {"nfkc", norm.NFKC}, {"nfkd", norm.NFKD}}
for _, name := range names(inputs + "/norm") {
original := read(inputs + "/norm/" + name + ".txt")
decomposed := read(inputs + "/normd/" + name + ".txt")
units := utf8.RuneCountInString(original)
for _, f := range forms {
form := f.form
measure(library, "norm/"+f.name+"/"+name, units,
utf8.RuneCountInString(form.String(original)),
func() int { return len(form.String(original)) })
}
measure(library, "norm/nfd>nfc/"+name, utf8.RuneCountInString(decomposed),
utf8.RuneCountInString(norm.NFC.String(decomposed)),
func() int { return len(norm.NFC.String(decomposed)) })
measure(library, "norm/is_nfc/"+name, units, toInt(norm.NFC.IsNormalString(original)),
func() int { return toInt(norm.NFC.IsNormalString(original)) })
}
}
func benchSegmentation(inputs string) {
for _, name := range names(inputs + "/seg") {
text := read(inputs + "/seg/" + name + ".txt")
measure("uniseg", "seg/graphemes/"+name, utf8.RuneCountInString(text),
uniseg.GraphemeClusterCount(text),
func() int { return uniseg.GraphemeClusterCount(text) })
}
}
func benchBidi(inputs string) {
for _, name := range names(inputs + "/bidi") {
text := read(inputs + "/bidi/" + name + ".txt")
order := func() int {
var paragraph bidi.Paragraph
if _, err := paragraph.SetString(text); err != nil {
panic(err)
}
ordering, err := paragraph.Order()
if err != nil {
panic(err)
}
return ordering.NumRuns()
}
// x/text exposes runs rather than levels, so there is no comparable check.
measure("x/text", "bidi/process/"+name, utf8.RuneCountInString(text), "-", order)
}
}
func benchIdna(inputs string) {
// The closest match to the other harnesses: UTS #46 non-transitional
// processing with Bidi, joiner and hyphen checks but without STD3 rules.
profile := idna.New(idna.MapForLookup(), idna.BidiRule(), idna.CheckJoiners(true),
idna.CheckHyphens(true), idna.StrictDomainName(false))
for _, pair := range pairs(inputs + "/idna.tsv") {
name, domain := pair[0], pair[1]
ascii, err := profile.ToASCII(domain)
if err != nil {
panic(err)
}
unicode, err := profile.ToUnicode(domain)
if err != nil {
panic(err)
}
measure("x/net/idna", "idna/to_ascii/"+name, 1, utf8.RuneCountInString(ascii),
func() int { result, _ := profile.ToASCII(domain); return len(result) })
measure("x/net/idna", "idna/to_unicode/"+name, 1, utf8.RuneCountInString(unicode),
func() int { result, _ := profile.ToUnicode(domain); return len(result) })
}
}
func benchProperties(inputs string) {
for _, name := range names(inputs + "/case") {
text := read(inputs + "/case/" + name + ".txt")
units := utf8.RuneCountInString(text)
simpleUpper := func() int {
sum := 0
for _, r := range text {
sum ^= int(unicode.ToUpper(r))
}
return sum
}
lowercaseLetters := func() int {
count := 0
for _, r := range text {
if unicode.Is(unicode.Ll, r) {
count++
}
}
return count
}
fold := cases.Fold()
measure("std", "case/simple_upper/"+name, units, simpleUpper(), simpleUpper)
measure("x/text", "case/fold/"+name, units, utf8.RuneCountInString(fold.String(text)),
func() int { return len(fold.String(text)) })
measure("std", "prop/general_category_Ll/"+name, units, lowercaseLetters(), lowercaseLetters)
}
}
func main() {
if len(os.Args) < 2 {
fmt.Fprintln(os.Stderr, "usage: xlang <inputs> [group]")
os.Exit(2)
}
inputs, group := os.Args[1], ""
if len(os.Args) > 2 {
group = os.Args[2]
}
groups := []struct {
name string
bench func(string)
}{
{"norm", benchNormalization}, {"seg", benchSegmentation}, {"bidi", benchBidi},
{"idna", benchIdna}, {"case", benchProperties},
}
for _, g := range groups {
if group == "" || group == g.name {
g.bench(inputs)
}
}
}