1// Note(alexwlchan, 2026-11-04): This file was retrieved from
2// https://cs.opensource.google/go/x/perf/+/be2c69fb:cmd/benchstat/internal/benchtab/builder.go
4// Copyright 2020 The Go Authors. All rights reserved.
5// Use of this source code is governed by a BSD-style
6// license that can be found in the LICENSE file.
8// Package benchtab presents benchmark results as comparison tables.
21 "github.com/aclements/go-moremath/stats"
22 "golang.org/x/perf/benchfmt"
23 "golang.org/x/perf/benchmath"
24 "golang.org/x/perf/benchproc"
27// TODO: Color by good/bad (or nothing for unknown units)
29// A Builder collects benchmark results into a Tables set.
31 tableBy, rowBy, colBy *benchproc.Projection
32 residue *benchproc.Projection
34 unitField *benchproc.Field
36 // tables maps from tableBy to table.
37 tables map[benchproc.Key]*builderTable
40type builderTable struct {
41 // Observed row and col Keys within this group. Within the
42 // group, we show only the row and col labels for the data in
43 // the group, but we sort them according to the global
44 // observation order for consistency across groups.
45 rows map[benchproc.Key]struct{}
46 cols map[benchproc.Key]struct{}
48 // cells maps from (row, col) to each cell.
49 cells map[TableKey]*builderCell
52type builderCell struct {
53 // values are the observed values in this cell.
55 // residue is the set of residue keys mapped to this cell.
56 // It is used to check for non-unique keys.
57 residue map[benchproc.Key]struct{}
60// NewBuilder creates a new Builder for collecting benchmark results
61// into tables. Each result will be mapped to a Table by tableBy.
62// Within each table, the results are mapped to cells by rowBy and
63// colBy. Any results within a single cell that vary by residue will
64// be reported as warnings. tableBy must have a ".unit" field.
65func NewBuilder(tableBy, rowBy, colBy, residue *benchproc.Projection) *Builder {
66 tableFields := tableBy.Fields()
67 unitField := tableFields[len(tableFields)-1]
68 if unitField.Name != ".unit" {
69 panic("tableBy projection missing .unit field")
72 tableBy: tableBy, rowBy: rowBy, colBy: colBy, residue: residue,
74 tables: make(map[benchproc.Key]*builderTable),
78// Add adds all of the values in result to the tables in the Builder.
79func (b *Builder) Add(result *benchfmt.Result) {
80 // Project the result.
81 tableKeys := b.tableBy.ProjectValues(result)
82 rowKey := b.rowBy.Project(result)
83 colKey := b.colBy.Project(result)
84 residueKey := b.residue.Project(result)
85 cellKey := TableKey{rowKey, colKey}
88 for unitI, tableKey := range tableKeys {
89 table := b.tables[tableKey]
92 b.tables[tableKey] = table
96 c := table.cells[cellKey]
99 c.residue = make(map[benchproc.Key]struct{})
100 table.cells[cellKey] = c
101 table.rows[rowKey] = struct{}{}
102 table.cols[colKey] = struct{}{}
106 c.values = append(c.values, result.Values[unitI].Value)
107 c.residue[residueKey] = struct{}{}
111func (b *Builder) newTable() *builderTable {
112 return &builderTable{
113 rows: make(map[benchproc.Key]struct{}),
114 cols: make(map[benchproc.Key]struct{}),
115 cells: make(map[TableKey]*builderCell),
119// TableOpts provides options for constructing the final analysis
120// tables from a Builder.
121type TableOpts struct {
122 // Confidence is the desired confidence level in summary
123 // intervals; e.g., 0.95 for 95%.
126 // Thresholds are the thresholds to use for statistical tests.
127 Thresholds *benchmath.Thresholds
129 // Units is the unit metadata. This gives distributional
130 // assumptions for units, among other properties.
131 Units benchfmt.UnitMetadataMap
134// Tables is a sequence of benchmark statistic tables.
136 // Tables is a slice of statistic tables. Within a Table, all
137 // results have the same table Key (including unit).
139 // Keys is a slice of table keys, corresponding 1:1 to
140 // the Tables slice. These always end with a ".unit"
141 // field giving the unit.
145// ToTables finalizes a Builder into a sequence of statistic tables.
146func (b *Builder) ToTables(opts TableOpts) *Tables {
148 var keys []benchproc.Key
149 for k := range b.tables {
150 keys = append(keys, k)
152 benchproc.SortKeys(keys)
154 // We're going to compute table cells in parallel because the
155 // statistics are somewhat expensive. This is entirely
156 // CPU-bound, so we put a simple concurrency limit on it.
157 limit := make(chan struct{}, 2*runtime.GOMAXPROCS(-1))
158 var wg sync.WaitGroup
160 // Process each table.
162 for _, k := range keys {
163 cTable := b.tables[k]
165 // Get the configured assumption for this unit.
166 unit := k.Get(b.unitField)
167 assumption := opts.Units.GetAssumption(unit)
169 // Sort the rows and columns.
170 rowKeys, colKeys := mapKeys(cTable.rows), mapKeys(cTable.cols)
174 Assumption: assumption,
177 Cells: make(map[TableKey]*TableCell),
179 tables = append(tables, table)
181 // Create all TableCells and fill their Samples. This
182 // is fast enough it's not worth parallelizing. This
183 // enables the second pass to look up baselines and
185 for k, cCell := range cTable.cells {
186 table.Cells[k] = &TableCell{
187 Sample: benchmath.NewSample(cCell.values, opts.Thresholds),
192 baselineCfg := colKeys[0]
193 wg.Add(len(cTable.cells))
194 for k, cCell := range cTable.cells {
195 cell := table.Cells[k]
197 // Look up the baseline.
198 if k.Col != baselineCfg {
199 base, ok := table.Cells[TableKey{k.Row, baselineCfg}]
208 summarizeCell(cCell, cell, assumption, opts.Confidence)
216 // Add summary rows to each table.
217 for _, table := range tables {
218 table.SummaryLabel = "geomean"
219 table.Summary = make(map[benchproc.Key]*TableSummary)
221 // Count the number of baseline benchmarks. If later
222 // columns don't have the same number of baseline
223 // pairings, we know the benchmark sets don't match.
225 baseCol := table.Cols[0]
226 for _, row := range table.Rows {
227 if _, ok := table.Cells[TableKey{row, baseCol}]; ok {
232 for i, col := range table.Cols {
234 table.Summary[col] = &s
238 table, col := table, col
241 summarizeCol(table, col, &s, nBase, isBase)
249 return &Tables{tables, keys}
252func mapKeys(m map[benchproc.Key]struct{}) []benchproc.Key {
253 var keys []benchproc.Key
255 keys = append(keys, k)
257 benchproc.SortKeys(keys)
261func summarizeCell(cCell *builderCell, cell *TableCell, assumption benchmath.Assumption, confidence float64) {
262 cell.Summary = assumption.Summary(cell.Sample, confidence)
264 // If there's a baseline, compute comparison.
265 if cell.Baseline != nil {
266 cell.Comparison = assumption.Compare(cell.Baseline.Sample, cell.Sample)
269 // Warn for non-singular keys in this cell.
270 nsk := benchproc.NonSingularFields(mapKeys(cCell.residue))
273 var warn strings.Builder
274 warn.WriteString("benchmarks vary in ")
275 for i, field := range nsk {
277 warn.WriteString(", ")
279 warn.WriteString(field.Name)
282 cell.Sample.Warnings = append(cell.Sample.Warnings, errors.New(warn.String()))
286func summarizeCol(table *Table, col benchproc.Key, s *TableSummary, nBase int, isBase bool) {
289 // This computes the geomean of the summary ratios rather than
290 // ratio of the summary geomeans. These are identical *if* the
291 // benchmark sets are the same. But if the benchmark sets
292 // differ, this leads to more sensible ratios because it's
293 // still the geomean of the column, rather than being a
294 // comparison of two incomparable numbers. It's still easy to
295 // misinterpret, but at least it's not meaningless.
296 var summaries, ratios []float64
298 for _, row := range table.Rows {
299 cell, ok := table.Cells[TableKey{row, col}]
303 summaries = append(summaries, cell.Summary.Center)
304 if cell.Baseline != nil {
306 a, b := cell.Summary.Center, cell.Baseline.Summary.Center
312 // Keep nBase check working.
313 ratios = append(ratios, 0)
318 ratios = append(ratios, ratio)
322 // If the number of cells in this column that had a baseline
323 // is the same as the total number of baselines, then we know
324 // the benchmark sets match. Otherwise, they don't and these
325 // numbers are probably misleading.
326 if !isBase && nBase != len(ratios) {
327 s.Warnings = append(s.Warnings, fmt.Errorf("benchmark set differs from baseline; geomeans may not be comparable"))
330 // Summarize centers.
331 gm := stats.GeoMean(summaries)
333 s.Warnings = append(s.Warnings, fmt.Errorf("summaries must be >0 to compute geomean"))
340 if !isBase && !badRatio {
341 gm := stats.GeoMean(ratios)
343 s.Warnings = append(s.Warnings, fmt.Errorf("ratios must be >0 to compute geomean"))
351// ToText renders t to a textual representation, assuming a
353func (t *Tables) ToText(w io.Writer, color bool) error {
354 return t.printTables(func(hdr string) error {
355 _, err := fmt.Fprintf(w, "%s\n", hdr)
357 }, func(table *Table) error {
358 return table.ToText(w, color)
362// ToCSV renders t in CSV (comma-separated values) format.
364// Warnings are written to a separate stream so as not to interrupt
365// the regular format of the CSV table.
366func (t *Tables) ToCSV(w, warnings io.Writer) error {
367 o := csv.NewWriter(w)
370 err := t.printTables(func(hdr string) error {
371 o.Write([]string{hdr})
374 }, func(table *Table) error {
375 nRows := table.ToCSV(o, row, warnings)
386func (t *Tables) printTables(hdr func(string) error, cb func(*Table) error) error {
387 if len(t.Tables) == 0 {
391 var prevKey benchproc.Key
392 fields := t.Keys[0].Projection().FlattenedFields()
394 for i, table := range t.Tables {
396 // Blank line between tables.
397 if err := hdr(""); err != nil {
402 // Print table key changes.
404 for _, f := range fields {
405 if f.Name == ".unit" {
406 // Skip .unit because it's shown in the table itself.
410 if prevKey.IsZero() || val != prevKey.Get(f) {
411 if err := hdr(fmt.Sprintf("%s: %s", f.Name, val)); err != nil {
419 if err := cb(table); err != nil {