Skip to main content

cmd/gobench/benchstat.go

1// Note(alexwlchan, 2026-11-04): This file was retrieved from
2// https://cs.opensource.google/go/x/perf/+/be2c69fb:cmd/benchstat/main.go
3// so I can "embed" the benchstat CLI and users of gobench don't need to
4// install it separately.
5//
6// The only modification is commenting out the benchstat main() function
7// and updating the imports for the internal packages.
9// Copyright 2021 The Go Authors. All rights reserved.
10// Use of this source code is governed by a BSD-style
11// license that can be found in the LICENSE file.
13// Note: Blocks that begin with "$ benchstat" below will be tested by
14// doc_test.go.
16// Benchstat computes statistical summaries and A/B comparisons of Go
17// benchmarks.
18//
19// Usage:
20//
21// benchstat [flags] inputs...
22//
23// Each input file should be in the Go benchmark format
24// (https://golang.org/design/14313-benchmark-format), such as the
25// output of “go test -bench .”. Typically, there should be two (or
26// more) inputs files for before and after some change (or series of
27// changes) to be measured. Each benchmark should be run at least 10
28// times to gather a statistically significant sample of results. For
29// each benchmark, benchstat computes the median and the confidence
30// interval for the median. By default, if there are two or more
31// inputs files, it compares each benchmark in the first file to the
32// same benchmark in each subsequent file and reports whether there
33// was a statistically significant difference, though it can be
34// configured to compare on other dimensions.
35//
36// # Example
37//
38// Suppose we collect results from running a set of benchmarks 10 times
39// before a particular change:
40//
41// go test -run='^$' -bench=. -count=10 > old.txt
42//
43// And the same benchmarks 10 times after:
44//
45// go test -run='^$' -bench=. -count=10 > new.txt
46//
47// The file old.txt contains:
48//
49// goos: linux
50// goarch: amd64
51// pkg: golang.org/x/perf/cmd/benchstat/testdata
52// BenchmarkEncode/format=json-48 690848 1726 ns/op
53// BenchmarkEncode/format=json-48 684861 1723 ns/op
54// BenchmarkEncode/format=json-48 693285 1707 ns/op
55// BenchmarkEncode/format=json-48 677692 1707 ns/op
56// BenchmarkEncode/format=json-48 692130 1713 ns/op
57// BenchmarkEncode/format=json-48 684164 1729 ns/op
58// BenchmarkEncode/format=json-48 682500 1736 ns/op
59// BenchmarkEncode/format=json-48 677509 1707 ns/op
60// BenchmarkEncode/format=json-48 687295 1705 ns/op
61// BenchmarkEncode/format=json-48 695533 1774 ns/op
62// BenchmarkEncode/format=gob-48 372699 3069 ns/op
63// BenchmarkEncode/format=gob-48 394740 3075 ns/op
64// BenchmarkEncode/format=gob-48 391335 3069 ns/op
65// BenchmarkEncode/format=gob-48 383588 3067 ns/op
66// BenchmarkEncode/format=gob-48 385885 3207 ns/op
67// BenchmarkEncode/format=gob-48 389970 3064 ns/op
68// BenchmarkEncode/format=gob-48 393361 3064 ns/op
69// BenchmarkEncode/format=gob-48 393882 3058 ns/op
70// BenchmarkEncode/format=gob-48 396171 3059 ns/op
71// BenchmarkEncode/format=gob-48 397812 3062 ns/op
72//
73// The file new.txt contains:
74//
75// goos: linux
76// goarch: amd64
77// pkg: golang.org/x/perf/cmd/benchstat/testdata
78// BenchmarkEncode/format=json-48 714387 1423 ns/op
79// BenchmarkEncode/format=json-48 845445 1416 ns/op
80// BenchmarkEncode/format=json-48 815714 1411 ns/op
81// BenchmarkEncode/format=json-48 828824 1413 ns/op
82// BenchmarkEncode/format=json-48 834070 1412 ns/op
83// BenchmarkEncode/format=json-48 828123 1426 ns/op
84// BenchmarkEncode/format=json-48 834493 1422 ns/op
85// BenchmarkEncode/format=json-48 838406 1424 ns/op
86// BenchmarkEncode/format=json-48 836227 1447 ns/op
87// BenchmarkEncode/format=json-48 830835 1425 ns/op
88// BenchmarkEncode/format=gob-48 394441 3075 ns/op
89// BenchmarkEncode/format=gob-48 393207 3065 ns/op
90// BenchmarkEncode/format=gob-48 392374 3059 ns/op
91// BenchmarkEncode/format=gob-48 396037 3065 ns/op
92// BenchmarkEncode/format=gob-48 393255 3060 ns/op
93// BenchmarkEncode/format=gob-48 382629 3081 ns/op
94// BenchmarkEncode/format=gob-48 389558 3186 ns/op
95// BenchmarkEncode/format=gob-48 392668 3135 ns/op
96// BenchmarkEncode/format=gob-48 392313 3087 ns/op
97// BenchmarkEncode/format=gob-48 394274 3062 ns/op
98//
99// The order of the lines in the file does not matter, except that the
100// output lists benchmarks in order of appearance.
101//
102// If we run “benchstat old.txt new.txt”, it will summarize the
103// benchmarks and compare the before and after results:
104//
105// $ benchstat old.txt new.txt
106// goos: linux
107// goarch: amd64
108// pkg: golang.org/x/perf/cmd/benchstat/testdata
109// │ old.txt │ new.txt │
110// │ sec/op │ sec/op vs base │
111// Encode/format=json-48 1.718µ ± 1% 1.423µ ± 1% -17.20% (p=0.000 n=10)
112// Encode/format=gob-48 3.066µ ± 0% 3.070µ ± 2% ~ (p=0.446 n=10)
113// geomean 2.295µ 2.090µ -8.94%
114//
115// Before the comparison table, we see common file-level
116// configuration. If there are benchmarks with different configuration
117// (for example, from different packages), benchstat will print
118// separate tables for each configuration.
119//
120// The table then compares the two input files for each benchmark. It
121// shows the median and 95% confidence interval summaries for each
122// benchmark before and after the change, and an A/B comparison under
123// "vs base". The comparison shows that Encode/format=json got 17.20%
124// faster with a p-value of 0.000 and 10 samples from each input file.
125// The p-value measures how likely it is that any differences were due
126// to random chance (i.e., noise). In this case, it's extremely
127// unlikely the difference between the medians was due to chance. For
128// Encode/format=gob, the "~" means benchstat did not detect a
129// statistically significant difference between the two inputs. In
130// this case, we see a p-value of 0.446, meaning it's very likely the
131// differences for this benchmark are simply due to random chance.
132//
133// Note that "statistically significant" is not the same as "large":
134// with enough low-noise data, even very small changes can be
135// distinguished from noise and considered statistically significant.
136// It is, of course, generally easier to distinguish large changes
137// from noise.
138//
139// Finally, the last row of the table shows the geometric mean of each
140// column, giving an overall picture of how the benchmarks changed.
141// Proportional changes in the geomean reflect proportional changes in
142// the benchmarks. For example, given n benchmarks, if sec/op for one
143// of them increases by a factor of 2, then the sec/op geomean will
144// increase by a factor of ⁿ√2.
145//
146// # Filtering
147//
148// benchstat has a very flexible system of configuring exactly which
149// benchmarks are summarized and compared. First, all inputs are
150// filtered according to an expression provided as the -filter flag.
151//
152// Filters are built from key-value terms:
153//
154// key:value - Match if key equals value.
155// key:"value" - Same, but value is a double-quoted Go string that
156// may contain spaces or other special characters.
157// "key":value - Keys may also be double-quoted.
158// key:/regexp/ - Match if key matches a regular expression.
159// key:(val1 OR val2 OR ...)
160// - Short-hand for key:val1 OR key:val2. Values may be
161// double-quoted strings or regexps.
162// * - Match everything.
163//
164// These terms can be combined into larger expressions as follows:
165//
166// x y ... - Match if x, y, etc. all match.
167// x AND y - Same as x y.
168// x OR y - Match if x or y match.
169// -x - Match if x does not match.
170// (...) - Subexpression.
171//
172// Each key is one of the following:
173//
174// .name - The base name of a benchmark
175// .fullname - The full name of a benchmark (including configuration)
176// .file - The name of the input file or user-provided file label
177// /{name-key} - Per-benchmark sub-name configuration key
178// {file-key} - File-level configuration key
179// .unit - The name of a unit for a particular metric
180//
181// For example, the following matches benchmarks with "/format=json"
182// in the sub-name keys with file-level configuration "goos" equal to
183// "linux" and extracts the "ns/op" and "B/op" measurements:
184//
185// $ benchstat -filter "/format:json goos:linux .unit:(ns/op OR B/op)" old.txt new.txt
186// goos: linux
187// goarch: amd64
188// pkg: golang.org/x/perf/cmd/benchstat/testdata
189// │ old.txt │ new.txt │
190// │ sec/op │ sec/op vs base │
191// Encode/format=json-48 1.718µ ± 1% 1.423µ ± 1% -17.20% (p=0.000 n=10)
192//
193// # Configuring comparisons
194//
195// The way benchstat groups and compares results is configurable using
196// a similar set of keys as used for filtering. By default, benchstat
197// groups results into tables using all file-level configuration keys,
198// then within each table, it groups results into rows by .fullname
199// (the benchmark's full name) and compares across columns by .file
200// (the name of each input file). This can be changed via the
201// following flags:
202//
203// -table KEYS - Group results into tables by KEYS
204// -row KEYS - Group results into table rows by KEYS
205// -col KEYS - Compare across results with different values of KEYS
206//
207// Using these flags, benchstat "projects" each result into a
208// particular table cell. Each KEYS argument is a comma- or
209// space-separated list of keys, each of which can optionally also
210// specify a sort order (described below).
211//
212// Each key is one of the following:
213//
214// .name - The base name of a benchmark
215// .fullname - The full name of a benchmark (including configuration)
216// .file - The name of the input file or user-provided file label
217// /{name-key} - Per-benchmark sub-name configuration key
218// {file-key} - File-level configuration key
219// .config - All file-level configuration keys
220//
221// Some of these keys can overlap. For example, ".config" includes the
222// file-level key "goos", and ".fullname" includes the sub-name key
223// "/format". When keys overlap like this, benchstat omits the more
224// specific key from the general key. For example, if -table is the
225// full file-level configuration ".config", and -col is the specific
226// file key "goos", benchstat will omit "goos" from ".config".
227//
228// Finally, the -ignore flag can list keys that benchstat should
229// ignore when grouping results. Continuing the previous example, if
230// -table is ".config" and -ignore is "goos", benchstat will omit
231// "goos" from ".config", but also not use it for any grouping.
232//
233// For precise details of the filter syntax and supported keys, see
234// https://pkg.go.dev/golang.org/x/perf/benchproc/syntax.
235//
236// # Projection examples
237//
238// Returning to our first example, we can now see how the default
239// projection flags produce this output:
240//
241// $ benchstat -table .config -row .fullname -col .file old.txt new.txt
242// goos: linux
243// goarch: amd64
244// pkg: golang.org/x/perf/cmd/benchstat/testdata
245// │ old.txt │ new.txt │
246// │ sec/op │ sec/op vs base │
247// Encode/format=json-48 1.718µ ± 1% 1.423µ ± 1% -17.20% (p=0.000 n=10)
248// Encode/format=gob-48 3.066µ ± 0% 3.070µ ± 2% ~ (p=0.446 n=10)
249// geomean 2.295µ 2.090µ -8.94%
250//
251// In this example, all benchmarks have the same file-level
252// configuration, consisting of "goos", "goarch", and "pkg", so
253// ".config" groups them into just one table. Within this table,
254// results are grouped into rows by their full name, including
255// configuration, and grouped into columns by the name of each input
256// file.
257//
258// Suppose we instead want to compare json encoding to gob encoding
259// from new.txt.
260//
261// $ benchstat -col /format new.txt
262// goos: linux
263// goarch: amd64
264// pkg: golang.org/x/perf/cmd/benchstat/testdata
265// │ json │ gob │
266// │ sec/op │ sec/op vs base │
267// Encode-48 1.423µ ± 1% 3.070µ ± 2% +115.82% (p=0.000 n=10)
268//
269// The columns are now labeled by the "/format" configuration from the
270// benchmark name. benchstat still compares columns even though we've
271// only provided a single input file. We also see that /format has
272// been removed from the benchmark name to make a single row.
273//
274// We can simplify the output by grouping rows by just the benchmark name,
275// rather than the full name:
276//
277// $ benchstat -col /format -row .name new.txt
278// goos: linux
279// goarch: amd64
280// pkg: golang.org/x/perf/cmd/benchstat/testdata
281// │ json │ gob │
282// │ sec/op │ sec/op vs base │
283// Encode 1.423µ ± 1% 3.070µ ± 2% +115.82% (p=0.000 n=10)
284//
285// benchstat will attempt to detect and warn if projections strip away
286// too much information. For example, here we group together json and
287// gob results into a single row:
288//
289// $ benchstat -row .name new.txt
290// goos: linux
291// goarch: amd64
292// pkg: golang.org/x/perf/cmd/benchstat/testdata
293// │ new.txt │
294// │ sec/op │
295// Encode 2.253µ ± 37% ¹
296// ¹ benchmarks vary in .fullname
297//
298// Since this is probably not a meaningful comparison, benchstat warns
299// that the benchmarks it grouped together vary in a hidden dimension.
300// If this really were our intent, we could -ignore .fullname.
301//
302// # Sorting
303//
304// By default, benchstat sorts each dimension according to the order
305// in which it first observes each value of that dimension. This can
306// be overridden in each projection using the following syntax:
307//
308// {key}@{order} - specifies one of the built-in named sort orders.
309// This can be "alpha" or "num" for alphabetic or numeric sorting.
310// "num" understands basic use of metric and IEC prefixes like "2k"
311// and "1Mi".
312//
313// {key}@({value} {value} ...) - specifies a fixed value order for
314// key. It also specifies a filter: if key has a value that isn't any
315// of the specified values, the result is filtered out.
316//
317// For example, we can use a fixed order to compare the improvement of
318// json over gob rather than the other way around:
319//
320// $ benchstat -col "/format@(gob json)" -row .name -ignore .file new.txt
321// goos: linux
322// goarch: amd64
323// pkg: golang.org/x/perf/cmd/benchstat/testdata
324// │ gob │ json │
325// │ sec/op │ sec/op vs base │
326// Encode 3.070µ ± 2% 1.423µ ± 1% -53.66% (p=0.000 n=10)
327//
328// # Overriding .file
329//
330// Often, you want to compare results from different files, but want
331// to provide more meaningful (or perhaps shorter) column labels than
332// raw file names. File name labels can be overridden by specifying an
333// input argument of the form "label=path" instead of just "path".
334// This provides a custom value for the .file key.
335//
336// For example, the following will perform the default comparison, but
337// label the columns O and N instead of old.txt and new.txt:
338//
339// $ benchstat O=old.txt N=new.txt
340// goos: linux
341// goarch: amd64
342// pkg: golang.org/x/perf/cmd/benchstat/testdata
343// │ O │ N │
344// │ sec/op │ sec/op vs base │
345// Encode/format=json-48 1.718µ ± 1% 1.423µ ± 1% -17.20% (p=0.000 n=10)
346// Encode/format=gob-48 3.066µ ± 0% 3.070µ ± 2% ~ (p=0.446 n=10)
347// geomean 2.295µ 2.090µ -8.94%
348//
349// # Units
350//
351// benchstat normalizes the units "ns" to "sec" and "MB" to "B" to
352// avoid creating nonsense units like "µns/op". These appear in the
353// testing package's default metrics and are also common in custom
354// metrics.
355//
356// benchstat supports custom unit metadata (see
357// https://golang.org/design/14313-benchmark-format). In particular,
358// "assume" metadata is useful for controlling the statistics used by
359// benchstat. By default, units use "assume=nothing", so benchstat
360// uses non-parametric statistics: median for summaries, and the
361// Mann-Whitney U-test for A/B comparisons.
362//
363// Some benchmarks measure things that have no noise, such as the size
364// of a binary produced by a compiler. These do not benefit from
365// repeated measurements or non-parametric statistics. For these
366// units, it's useful to set "assume=exact". This will cause benchstat
367// to warn if there's any variation in the measured values, and to
368// show A/B comparisons even if there's only one before and after
369// measurement.
370//
371// # Tips
372//
373// Reducing noise and/or increasing the number of benchmark runs will
374// enable benchstat to discern smaller changes as "statistically
375// significant". To reduce noise, make sure you run benchmarks on an
376// otherwise idle machine, ideally one that isn't running on battery
377// and isn't likely to be affected by thermal throttling.
378// https://llvm.org/docs/Benchmarking.html has many good tips on
379// reducing noise in benchmarks.
380//
381// It's also important that noise is evenly distributed across
382// benchmark runs. The best way to do this is to interleave before and
383// after runs, rather than running, say, 10 iterations of the before
384// benchmark, and then 10 iterations of the after benchmark. For Go
385// benchmarks, you can often speed up this process by using "go test
386// -c" to pre-compile the benchmark binary.
387//
388// Pick a number of benchmark runs (at least 10, ideally 20) and stick
389// to it. If benchstat reports no statistically significant change,
390// avoid simply rerunning your benchmarks until it reports a
391// significant change. This is known as "multiple testing" and is a
392// common statistical error. By default, benchstat uses an ɑ threshold
393// of 0.05, which means it is *expected* to show a difference 5% of
394// the time even if there is no difference. Hence, if you rerun
395// benchmarks looking for a change, benchstat will probably eventually
396// say there is a change, even if there isn't, which creates a
397// statistical bias.
398//
399// As an extension of this, if you compare a large number of
400// benchmarks, you should expect that about 5% of them will report a
401// statistically significant change even if there is no difference
402// between the before and after.
403package main
405import (
406 "flag"
407 "fmt"
408 "io"
409 "os"
411 "alexwlchan.net/gotools/internal/benchtab"
412 "golang.org/x/perf/benchfmt"
413 "golang.org/x/perf/benchmath"
414 "golang.org/x/perf/benchproc"
417// TODO: Add a flag to perform Holm–Bonferroni correction for
418// family-wise error rates. This can be done after-the-fact on a
419// collection of benchstat.Comparison values.
421// TODO: -unit flag.
423// TODO: Support sorting by commit order.
425// TODO: Add some quick usage examples to the -h output?
427// TODO: If the projection results in a very sparse table, that's
428// usually the result of correlated keys. Can we detect that and
429// suggest fixes?
431// func main() {
432// if err := benchstat(os.Stdout, os.Stderr, os.Args[1:]); err != nil {
433// fmt.Fprintf(os.Stderr, "benchstat: %s\n", err)
434// }
435// }
437func benchstat(w, wErr io.Writer, args []string) error {
438 flags := flag.NewFlagSet("", flag.ExitOnError)
439 flags.Usage = func() {
440 fmt.Fprintf(flags.Output(), `Usage: benchstat [flags] inputs...
442benchstat computes statistical summaries and A/B comparisons of Go
443benchmarks. It shows benchmark medians in a table with a row for each
444benchmark and a column for each input file. If there is more than one
445input file, it also shows A/B comparisons between the files. If a
446difference is likely to be noise, it shows "~".
448For details, see https://pkg.go.dev/golang.org/x/perf/cmd/benchstat.
449`)
450 flags.PrintDefaults()
451 }
453 thresholds := benchmath.DefaultThresholds
454 flagTable := flags.String("table", ".config", "split results into tables by distinct values of `projection`")
455 flagRow := flags.String("row", ".fullname", "split results into rows by distinct values of `projection`")
456 flagCol := flags.String("col", ".file", "split results into columns by distinct values of `projection`")
457 flagIgnore := flags.String("ignore", "", "ignore variations in `keys`")
458 flagFilter := flags.String("filter", "*", "use only benchmarks matching benchfilter `query`")
459 flags.Float64Var(&thresholds.CompareAlpha, "alpha", thresholds.CompareAlpha, "consider change significant if p < `α`")
460 // TODO: Support -confidence none to disable CI column? This
461 // would be equivalent to benchstat v1's -norange for CSV.
462 flagConfidence := flags.Float64("confidence", 0.95, "confidence `level` for ranges")
463 flagFormat := flags.String("format", "text", "print results in `format`:\n text - plain text\n csv - comma-separated values (warnings will be written to stderr)\n")
464 flags.Parse(args)
466 if flags.NArg() == 0 {
467 flags.Usage()
468 os.Exit(2)
469 }
471 filter, err := benchproc.NewFilter(*flagFilter)
472 if err != nil {
473 return fmt.Errorf("parsing -filter: %s", err)
474 }
476 var parser benchproc.ProjectionParser
477 var parseErr error
478 mustParse := func(name, val string, unit bool) *benchproc.Projection {
479 var proj *benchproc.Projection
480 var err error
481 if unit {
482 proj, _, err = parser.ParseWithUnit(val, filter)
483 } else {
484 proj, err = parser.Parse(val, filter)
485 }
486 if err != nil && parseErr == nil {
487 parseErr = fmt.Errorf("parsing %s: %s", name, err)
488 }
489 return proj
490 }
491 tableBy := mustParse("-table", *flagTable, true)
492 rowBy := mustParse("-row", *flagRow, false)
493 colBy := mustParse("-col", *flagCol, false)
494 mustParse("-ignore", *flagIgnore, false)
495 residue := parser.Residue()
496 if parseErr != nil {
497 return parseErr
498 }
500 if thresholds.CompareAlpha < 0 || thresholds.CompareAlpha > 1 {
501 return fmt.Errorf("-alpha must be in range [0, 1]")
502 }
503 if *flagConfidence < 0 || *flagConfidence > 1 {
504 return fmt.Errorf("-confidence must be in range [0, 1]")
505 }
506 var format func(t *benchtab.Tables) error
507 switch *flagFormat {
508 default:
509 return fmt.Errorf("-format must be text or csv")
510 case "text":
511 format = func(t *benchtab.Tables) error { return t.ToText(w, false) }
512 case "csv":
513 format = func(t *benchtab.Tables) error { return t.ToCSV(w, wErr) }
514 }
516 stat := benchtab.NewBuilder(tableBy, rowBy, colBy, residue)
517 files := benchfmt.Files{Paths: flags.Args(), AllowStdin: true, AllowLabels: true}
518 for files.Scan() {
519 switch rec := files.Result(); rec := rec.(type) {
520 case *benchfmt.SyntaxError:
521 // Non-fatal result parse error. Warn
522 // but keep going.
523 fmt.Fprintln(wErr, rec)
524 case *benchfmt.Result:
525 if ok, err := filter.Apply(rec); !ok {
526 if err != nil {
527 // Print the reason we rejected this result.
528 fmt.Fprintln(wErr, err)
529 }
530 continue
531 }
533 stat.Add(rec)
534 }
535 }
536 if err := files.Err(); err != nil {
537 return err
538 }
540 tables := stat.ToTables(benchtab.TableOpts{
541 Confidence: *flagConfidence,
542 Thresholds: &thresholds,
543 Units: files.Units(),
544 })
545 return format(tables)