1 // Note(alexwlchan, 2026-11-04): This file was retrieved from
2 // https://cs.opensource.google/go/x/perf/+/be2c69fb:cmd/benchstat/main.go
3 // so I can "embed" the benchstat CLI and users of gobench don't need to
4 // install it separately.
6 // The only modification is commenting out the benchstat main() function
7 // and updating the imports for the internal packages.
9 // Copyright 2021 The Go Authors. All rights reserved.
10 // Use of this source code is governed by a BSD-style
11 // license that can be found in the LICENSE file.
13 // Note: Blocks that begin with "$ benchstat" below will be tested by
16 // Benchstat computes statistical summaries and A/B comparisons of Go
21 // benchstat [flags] inputs...
23 // Each input file should be in the Go benchmark format
24 // (https://golang.org/design/14313-benchmark-format), such as the
25 // output of “go test -bench .”. Typically, there should be two (or
26 // more) inputs files for before and after some change (or series of
27 // changes) to be measured. Each benchmark should be run at least 10
28 // times to gather a statistically significant sample of results. For
29 // each benchmark, benchstat computes the median and the confidence
30 // interval for the median. By default, if there are two or more
31 // inputs files, it compares each benchmark in the first file to the
32 // same benchmark in each subsequent file and reports whether there
33 // was a statistically significant difference, though it can be
34 // configured to compare on other dimensions.
38 // Suppose we collect results from running a set of benchmarks 10 times
39 // before a particular change:
41 // go test -run='^$' -bench=. -count=10 > old.txt
43 // And the same benchmarks 10 times after:
45 // go test -run='^$' -bench=. -count=10 > new.txt
47 // The file old.txt contains:
51 // pkg: golang.org/x/perf/cmd/benchstat/testdata
52 // BenchmarkEncode/format=json-48 690848 1726 ns/op
53 // BenchmarkEncode/format=json-48 684861 1723 ns/op
54 // BenchmarkEncode/format=json-48 693285 1707 ns/op
55 // BenchmarkEncode/format=json-48 677692 1707 ns/op
56 // BenchmarkEncode/format=json-48 692130 1713 ns/op
57 // BenchmarkEncode/format=json-48 684164 1729 ns/op
58 // BenchmarkEncode/format=json-48 682500 1736 ns/op
59 // BenchmarkEncode/format=json-48 677509 1707 ns/op
60 // BenchmarkEncode/format=json-48 687295 1705 ns/op
61 // BenchmarkEncode/format=json-48 695533 1774 ns/op
62 // BenchmarkEncode/format=gob-48 372699 3069 ns/op
63 // BenchmarkEncode/format=gob-48 394740 3075 ns/op
64 // BenchmarkEncode/format=gob-48 391335 3069 ns/op
65 // BenchmarkEncode/format=gob-48 383588 3067 ns/op
66 // BenchmarkEncode/format=gob-48 385885 3207 ns/op
67 // BenchmarkEncode/format=gob-48 389970 3064 ns/op
68 // BenchmarkEncode/format=gob-48 393361 3064 ns/op
69 // BenchmarkEncode/format=gob-48 393882 3058 ns/op
70 // BenchmarkEncode/format=gob-48 396171 3059 ns/op
71 // BenchmarkEncode/format=gob-48 397812 3062 ns/op
73 // The file new.txt contains:
77 // pkg: golang.org/x/perf/cmd/benchstat/testdata
78 // BenchmarkEncode/format=json-48 714387 1423 ns/op
79 // BenchmarkEncode/format=json-48 845445 1416 ns/op
80 // BenchmarkEncode/format=json-48 815714 1411 ns/op
81 // BenchmarkEncode/format=json-48 828824 1413 ns/op
82 // BenchmarkEncode/format=json-48 834070 1412 ns/op
83 // BenchmarkEncode/format=json-48 828123 1426 ns/op
84 // BenchmarkEncode/format=json-48 834493 1422 ns/op
85 // BenchmarkEncode/format=json-48 838406 1424 ns/op
86 // BenchmarkEncode/format=json-48 836227 1447 ns/op
87 // BenchmarkEncode/format=json-48 830835 1425 ns/op
88 // BenchmarkEncode/format=gob-48 394441 3075 ns/op
89 // BenchmarkEncode/format=gob-48 393207 3065 ns/op
90 // BenchmarkEncode/format=gob-48 392374 3059 ns/op
91 // BenchmarkEncode/format=gob-48 396037 3065 ns/op
92 // BenchmarkEncode/format=gob-48 393255 3060 ns/op
93 // BenchmarkEncode/format=gob-48 382629 3081 ns/op
94 // BenchmarkEncode/format=gob-48 389558 3186 ns/op
95 // BenchmarkEncode/format=gob-48 392668 3135 ns/op
96 // BenchmarkEncode/format=gob-48 392313 3087 ns/op
97 // BenchmarkEncode/format=gob-48 394274 3062 ns/op
99 // The order of the lines in the file does not matter, except that the
100 // output lists benchmarks in order of appearance.
102 // If we run “benchstat old.txt new.txt”, it will summarize the
103 // benchmarks and compare the before and after results:
105 // $ benchstat old.txt new.txt
108 // pkg: golang.org/x/perf/cmd/benchstat/testdata
109 // │ old.txt │ new.txt │
110 // │ sec/op │ sec/op vs base │
111 // Encode/format=json-48 1.718µ ± 1% 1.423µ ± 1% -17.20% (p=0.000 n=10)
112 // Encode/format=gob-48 3.066µ ± 0% 3.070µ ± 2% ~ (p=0.446 n=10)
113 // geomean 2.295µ 2.090µ -8.94%
115 // Before the comparison table, we see common file-level
116 // configuration. If there are benchmarks with different configuration
117 // (for example, from different packages), benchstat will print
118 // separate tables for each configuration.
120 // The table then compares the two input files for each benchmark. It
121 // shows the median and 95% confidence interval summaries for each
122 // benchmark before and after the change, and an A/B comparison under
123 // "vs base". The comparison shows that Encode/format=json got 17.20%
124 // faster with a p-value of 0.000 and 10 samples from each input file.
125 // The p-value measures how likely it is that any differences were due
126 // to random chance (i.e., noise). In this case, it's extremely
127 // unlikely the difference between the medians was due to chance. For
128 // Encode/format=gob, the "~" means benchstat did not detect a
129 // statistically significant difference between the two inputs. In
130 // this case, we see a p-value of 0.446, meaning it's very likely the
131 // differences for this benchmark are simply due to random chance.
133 // Note that "statistically significant" is not the same as "large":
134 // with enough low-noise data, even very small changes can be
135 // distinguished from noise and considered statistically significant.
136 // It is, of course, generally easier to distinguish large changes
139 // Finally, the last row of the table shows the geometric mean of each
140 // column, giving an overall picture of how the benchmarks changed.
141 // Proportional changes in the geomean reflect proportional changes in
142 // the benchmarks. For example, given n benchmarks, if sec/op for one
143 // of them increases by a factor of 2, then the sec/op geomean will
144 // increase by a factor of ⁿ√2.
148 // benchstat has a very flexible system of configuring exactly which
149 // benchmarks are summarized and compared. First, all inputs are
150 // filtered according to an expression provided as the -filter flag.
152 // Filters are built from key-value terms:
154 // key:value - Match if key equals value.
155 // key:"value" - Same, but value is a double-quoted Go string that
156 // may contain spaces or other special characters.
157 // "key":value - Keys may also be double-quoted.
158 // key:/regexp/ - Match if key matches a regular expression.
159 // key:(val1 OR val2 OR ...)
160 // - Short-hand for key:val1 OR key:val2. Values may be
161 // double-quoted strings or regexps.
162 // * - Match everything.
164 // These terms can be combined into larger expressions as follows:
166 // x y ... - Match if x, y, etc. all match.
167 // x AND y - Same as x y.
168 // x OR y - Match if x or y match.
169 // -x - Match if x does not match.
170 // (...) - Subexpression.
172 // Each key is one of the following:
174 // .name - The base name of a benchmark
175 // .fullname - The full name of a benchmark (including configuration)
176 // .file - The name of the input file or user-provided file label
177 // /{name-key} - Per-benchmark sub-name configuration key
178 // {file-key} - File-level configuration key
179 // .unit - The name of a unit for a particular metric
181 // For example, the following matches benchmarks with "/format=json"
182 // in the sub-name keys with file-level configuration "goos" equal to
183 // "linux" and extracts the "ns/op" and "B/op" measurements:
185 // $ benchstat -filter "/format:json goos:linux .unit:(ns/op OR B/op)" old.txt new.txt
188 // pkg: golang.org/x/perf/cmd/benchstat/testdata
189 // │ old.txt │ new.txt │
190 // │ sec/op │ sec/op vs base │
191 // Encode/format=json-48 1.718µ ± 1% 1.423µ ± 1% -17.20% (p=0.000 n=10)
193 // # Configuring comparisons
195 // The way benchstat groups and compares results is configurable using
196 // a similar set of keys as used for filtering. By default, benchstat
197 // groups results into tables using all file-level configuration keys,
198 // then within each table, it groups results into rows by .fullname
199 // (the benchmark's full name) and compares across columns by .file
200 // (the name of each input file). This can be changed via the
203 // -table KEYS - Group results into tables by KEYS
204 // -row KEYS - Group results into table rows by KEYS
205 // -col KEYS - Compare across results with different values of KEYS
207 // Using these flags, benchstat "projects" each result into a
208 // particular table cell. Each KEYS argument is a comma- or
209 // space-separated list of keys, each of which can optionally also
210 // specify a sort order (described below).
212 // Each key is one of the following:
214 // .name - The base name of a benchmark
215 // .fullname - The full name of a benchmark (including configuration)
216 // .file - The name of the input file or user-provided file label
217 // /{name-key} - Per-benchmark sub-name configuration key
218 // {file-key} - File-level configuration key
219 // .config - All file-level configuration keys
221 // Some of these keys can overlap. For example, ".config" includes the
222 // file-level key "goos", and ".fullname" includes the sub-name key
223 // "/format". When keys overlap like this, benchstat omits the more
224 // specific key from the general key. For example, if -table is the
225 // full file-level configuration ".config", and -col is the specific
226 // file key "goos", benchstat will omit "goos" from ".config".
228 // Finally, the -ignore flag can list keys that benchstat should
229 // ignore when grouping results. Continuing the previous example, if
230 // -table is ".config" and -ignore is "goos", benchstat will omit
231 // "goos" from ".config", but also not use it for any grouping.
233 // For precise details of the filter syntax and supported keys, see
234 // https://pkg.go.dev/golang.org/x/perf/benchproc/syntax.
236 // # Projection examples
238 // Returning to our first example, we can now see how the default
239 // projection flags produce this output:
241 // $ benchstat -table .config -row .fullname -col .file old.txt new.txt
244 // pkg: golang.org/x/perf/cmd/benchstat/testdata
245 // │ old.txt │ new.txt │
246 // │ sec/op │ sec/op vs base │
247 // Encode/format=json-48 1.718µ ± 1% 1.423µ ± 1% -17.20% (p=0.000 n=10)
248 // Encode/format=gob-48 3.066µ ± 0% 3.070µ ± 2% ~ (p=0.446 n=10)
249 // geomean 2.295µ 2.090µ -8.94%
251 // In this example, all benchmarks have the same file-level
252 // configuration, consisting of "goos", "goarch", and "pkg", so
253 // ".config" groups them into just one table. Within this table,
254 // results are grouped into rows by their full name, including
255 // configuration, and grouped into columns by the name of each input
258 // Suppose we instead want to compare json encoding to gob encoding
261 // $ benchstat -col /format new.txt
264 // pkg: golang.org/x/perf/cmd/benchstat/testdata
266 // │ sec/op │ sec/op vs base │
267 // Encode-48 1.423µ ± 1% 3.070µ ± 2% +115.82% (p=0.000 n=10)
269 // The columns are now labeled by the "/format" configuration from the
270 // benchmark name. benchstat still compares columns even though we've
271 // only provided a single input file. We also see that /format has
272 // been removed from the benchmark name to make a single row.
274 // We can simplify the output by grouping rows by just the benchmark name,
275 // rather than the full name:
277 // $ benchstat -col /format -row .name new.txt
280 // pkg: golang.org/x/perf/cmd/benchstat/testdata
282 // │ sec/op │ sec/op vs base │
283 // Encode 1.423µ ± 1% 3.070µ ± 2% +115.82% (p=0.000 n=10)
285 // benchstat will attempt to detect and warn if projections strip away
286 // too much information. For example, here we group together json and
287 // gob results into a single row:
289 // $ benchstat -row .name new.txt
292 // pkg: golang.org/x/perf/cmd/benchstat/testdata
295 // Encode 2.253µ ± 37% ¹
296 // ¹ benchmarks vary in .fullname
298 // Since this is probably not a meaningful comparison, benchstat warns
299 // that the benchmarks it grouped together vary in a hidden dimension.
300 // If this really were our intent, we could -ignore .fullname.
304 // By default, benchstat sorts each dimension according to the order
305 // in which it first observes each value of that dimension. This can
306 // be overridden in each projection using the following syntax:
308 // {key}@{order} - specifies one of the built-in named sort orders.
309 // This can be "alpha" or "num" for alphabetic or numeric sorting.
310 // "num" understands basic use of metric and IEC prefixes like "2k"
313 // {key}@({value} {value} ...) - specifies a fixed value order for
314 // key. It also specifies a filter: if key has a value that isn't any
315 // of the specified values, the result is filtered out.
317 // For example, we can use a fixed order to compare the improvement of
318 // json over gob rather than the other way around:
320 // $ benchstat -col "/format@(gob json)" -row .name -ignore .file new.txt
323 // pkg: golang.org/x/perf/cmd/benchstat/testdata
325 // │ sec/op │ sec/op vs base │
326 // Encode 3.070µ ± 2% 1.423µ ± 1% -53.66% (p=0.000 n=10)
330 // Often, you want to compare results from different files, but want
331 // to provide more meaningful (or perhaps shorter) column labels than
332 // raw file names. File name labels can be overridden by specifying an
333 // input argument of the form "label=path" instead of just "path".
334 // This provides a custom value for the .file key.
336 // For example, the following will perform the default comparison, but
337 // label the columns O and N instead of old.txt and new.txt:
339 // $ benchstat O=old.txt N=new.txt
342 // pkg: golang.org/x/perf/cmd/benchstat/testdata
344 // │ sec/op │ sec/op vs base │
345 // Encode/format=json-48 1.718µ ± 1% 1.423µ ± 1% -17.20% (p=0.000 n=10)
346 // Encode/format=gob-48 3.066µ ± 0% 3.070µ ± 2% ~ (p=0.446 n=10)
347 // geomean 2.295µ 2.090µ -8.94%
351 // benchstat normalizes the units "ns" to "sec" and "MB" to "B" to
352 // avoid creating nonsense units like "µns/op". These appear in the
353 // testing package's default metrics and are also common in custom
356 // benchstat supports custom unit metadata (see
357 // https://golang.org/design/14313-benchmark-format). In particular,
358 // "assume" metadata is useful for controlling the statistics used by
359 // benchstat. By default, units use "assume=nothing", so benchstat
360 // uses non-parametric statistics: median for summaries, and the
361 // Mann-Whitney U-test for A/B comparisons.
363 // Some benchmarks measure things that have no noise, such as the size
364 // of a binary produced by a compiler. These do not benefit from
365 // repeated measurements or non-parametric statistics. For these
366 // units, it's useful to set "assume=exact". This will cause benchstat
367 // to warn if there's any variation in the measured values, and to
368 // show A/B comparisons even if there's only one before and after
373 // Reducing noise and/or increasing the number of benchmark runs will
374 // enable benchstat to discern smaller changes as "statistically
375 // significant". To reduce noise, make sure you run benchmarks on an
376 // otherwise idle machine, ideally one that isn't running on battery
377 // and isn't likely to be affected by thermal throttling.
378 // https://llvm.org/docs/Benchmarking.html has many good tips on
379 // reducing noise in benchmarks.
381 // It's also important that noise is evenly distributed across
382 // benchmark runs. The best way to do this is to interleave before and
383 // after runs, rather than running, say, 10 iterations of the before
384 // benchmark, and then 10 iterations of the after benchmark. For Go
385 // benchmarks, you can often speed up this process by using "go test
386 // -c" to pre-compile the benchmark binary.
388 // Pick a number of benchmark runs (at least 10, ideally 20) and stick
389 // to it. If benchstat reports no statistically significant change,
390 // avoid simply rerunning your benchmarks until it reports a
391 // significant change. This is known as "multiple testing" and is a
392 // common statistical error. By default, benchstat uses an ɑ threshold
393 // of 0.05, which means it is *expected* to show a difference 5% of
394 // the time even if there is no difference. Hence, if you rerun
395 // benchmarks looking for a change, benchstat will probably eventually
396 // say there is a change, even if there isn't, which creates a
399 // As an extension of this, if you compare a large number of
400 // benchmarks, you should expect that about 5% of them will report a
401 // statistically significant change even if there is no difference
402 // between the before and after.
411 "alexwlchan.net/gotools/internal/benchtab"
412 "golang.org/x/perf/benchfmt"
413 "golang.org/x/perf/benchmath"
414 "golang.org/x/perf/benchproc"
417 // TODO: Add a flag to perform Holm–Bonferroni correction for
418 // family-wise error rates. This can be done after-the-fact on a
419 // collection of benchstat.Comparison values.
423 // TODO: Support sorting by commit order.
425 // TODO: Add some quick usage examples to the -h output?
427 // TODO: If the projection results in a very sparse table, that's
428 // usually the result of correlated keys. Can we detect that and
432 // if err := benchstat(os.Stdout, os.Stderr, os.Args[1:]); err != nil {
433 // fmt.Fprintf(os.Stderr, "benchstat: %s\n", err)
437 func benchstat ( w , wErr io . Writer , args [] string ) error {
438 flags := flag . NewFlagSet ( "" , flag . ExitOnError )
439 flags . Usage = func () {
440 fmt . Fprintf ( flags . Output (), `Usage: benchstat [flags] inputs...
442 benchstat computes statistical summaries and A/B comparisons of Go
443 benchmarks. It shows benchmark medians in a table with a row for each
444 benchmark and a column for each input file. If there is more than one
445 input file, it also shows A/B comparisons between the files. If a
446 difference is likely to be noise, it shows "~".
448 For details, see https://pkg.go.dev/golang.org/x/perf/cmd/benchstat.
450 flags . PrintDefaults ()
453 thresholds := benchmath . DefaultThresholds
454 flagTable := flags . String ( "table" , ".config" , "split results into tables by distinct values of `projection`" )
455 flagRow := flags . String ( "row" , ".fullname" , "split results into rows by distinct values of `projection`" )
456 flagCol := flags . String ( "col" , ".file" , "split results into columns by distinct values of `projection`" )
457 flagIgnore := flags . String ( "ignore" , "" , "ignore variations in `keys`" )
458 flagFilter := flags . String ( "filter" , "*" , "use only benchmarks matching benchfilter `query`" )
459 flags . Float64Var ( & thresholds . CompareAlpha , "alpha" , thresholds . CompareAlpha , "consider change significant if p < `α`" )
460 // TODO: Support -confidence none to disable CI column? This
461 // would be equivalent to benchstat v1's -norange for CSV.
462 flagConfidence := flags . Float64 ( "confidence" , 0.95 , "confidence `level` for ranges" )
463 flagFormat := flags . String ( "format" , "text" , "print results in `format`:\n text - plain text\n csv - comma-separated values (warnings will be written to stderr)\n" )
466 if flags . NArg () == 0 {
471 filter , err := benchproc . NewFilter ( * flagFilter )
473 return fmt . Errorf ( "parsing -filter: %s" , err )
476 var parser benchproc . ProjectionParser
478 mustParse := func ( name , val string , unit bool ) * benchproc . Projection {
479 var proj * benchproc . Projection
482 proj , _ , err = parser . ParseWithUnit ( val , filter )
484 proj , err = parser . Parse ( val , filter )
486 if err != nil && parseErr == nil {
487 parseErr = fmt . Errorf ( "parsing %s: %s" , name , err )
491 tableBy := mustParse ( "-table" , * flagTable , true )
492 rowBy := mustParse ( "-row" , * flagRow , false )
493 colBy := mustParse ( "-col" , * flagCol , false )
494 mustParse ( "-ignore" , * flagIgnore , false )
495 residue := parser . Residue ()
500 if thresholds . CompareAlpha < 0 || thresholds . CompareAlpha > 1 {
501 return fmt . Errorf ( "-alpha must be in range [0, 1]" )
503 if * flagConfidence < 0 || * flagConfidence > 1 {
504 return fmt . Errorf ( "-confidence must be in range [0, 1]" )
506 var format func ( t * benchtab . Tables ) error
509 return fmt . Errorf ( "-format must be text or csv" )
511 format = func ( t * benchtab . Tables ) error { return t . ToText ( w , false ) }
513 format = func ( t * benchtab . Tables ) error { return t . ToCSV ( w , wErr ) }
516 stat := benchtab . NewBuilder ( tableBy , rowBy , colBy , residue )
517 files := benchfmt . Files { Paths : flags . Args (), AllowStdin : true , AllowLabels : true }
519 switch rec := files . Result (); rec := rec .( type ) {
520 case * benchfmt . SyntaxError :
521 // Non-fatal result parse error. Warn
523 fmt . Fprintln ( wErr , rec )
524 case * benchfmt . Result :
525 if ok , err := filter . Apply ( rec ); ! ok {
527 // Print the reason we rejected this result.
528 fmt . Fprintln ( wErr , err )
536 if err := files . Err (); err != nil {
540 tables := stat . ToTables ( benchtab . TableOpts {
541 Confidence : * flagConfidence ,
542 Thresholds : & thresholds ,
543 Units : files . Units (),
545 return format ( tables )