Skip to content

Commit 74b7d89

Browse files
rafaelfrancarafaelfranca
committed
Extract class to build the results table from benchmark data
1 parent d049075 commit 74b7d89

3 files changed

Lines changed: 305 additions & 56 deletions

File tree

‎lib/results_table_builder.rb‎

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
require_relative '../misc/stats'
2+
3+
class ResultsTableBuilder
4+
def initialize(executable_names:, bench_data:, bench_names:, include_rss: false)
5+
@executable_names = executable_names
6+
@bench_data = bench_data
7+
@bench_names = bench_names
8+
@include_rss = include_rss
9+
@base_name = executable_names.first
10+
@other_names = executable_names[1..]
11+
end
12+
13+
def build
14+
table = [build_header]
15+
format = build_format
16+
17+
@bench_names.each do |bench_name|
18+
# Skip this bench_name if we failed to get data for any of the executables.
19+
next unless @bench_data.all? { |(_k, v)| v[bench_name] }
20+
21+
row = build_row(bench_name)
22+
table << row
23+
end
24+
25+
[table, format]
26+
end
27+
28+
private
29+
30+
def build_header
31+
header = ["bench"]
32+
33+
@executable_names.each do |name|
34+
header += ["#{name} (ms)", "stddev (%)"]
35+
header += ["RSS (MiB)"] if @include_rss
36+
end
37+
38+
@other_names.each do |name|
39+
header += ["#{name} 1st itr"]
40+
end
41+
42+
@other_names.each do |name|
43+
header += ["#{@base_name}/#{name}"]
44+
end
45+
46+
header
47+
end
48+
49+
def build_format
50+
format = ["%s"]
51+
52+
@executable_names.each do |_name|
53+
format += ["%.1f", "%.1f"]
54+
format += ["%.1f"] if @include_rss
55+
end
56+
57+
@other_names.each do |_name|
58+
format += ["%.3f"]
59+
end
60+
61+
@other_names.each do |_name|
62+
format += ["%.3f"]
63+
end
64+
65+
format
66+
end
67+
68+
def build_row(bench_name)
69+
t0s = @executable_names.map { |name| (bench_data_for(name, bench_name)['warmup'][0] || bench_data_for(name, bench_name)['bench'][0]) * 1000.0 }
70+
times_no_warmup = @executable_names.map { |name| bench_data_for(name, bench_name)['bench'].map { |v| v * 1000.0 } }
71+
rsss = @executable_names.map { |name| bench_data_for(name, bench_name)['rss'] / 1024.0 / 1024.0 }
72+
73+
base_t0, *other_t0s = t0s
74+
base_t, *other_ts = times_no_warmup
75+
base_rss, *other_rsss = rsss
76+
77+
ratio_1sts = other_t0s.map { |other_t0| base_t0 / other_t0 }
78+
ratios = other_ts.map { |other_t| mean(base_t) / mean(other_t) }
79+
80+
row = [bench_name, mean(base_t), 100 * stddev(base_t) / mean(base_t)]
81+
row << base_rss if @include_rss
82+
83+
other_ts.zip(other_rsss).each do |other_t, other_rss|
84+
row += [mean(other_t), 100 * stddev(other_t) / mean(other_t)]
85+
row << other_rss if @include_rss
86+
end
87+
88+
row += ratio_1sts + ratios
89+
90+
row
91+
end
92+
93+
def bench_data_for(name, bench_name)
94+
@bench_data[name][bench_name]
95+
end
96+
97+
def mean(values)
98+
Stats.new(values).mean
99+
end
100+
101+
def stddev(values)
102+
Stats.new(values).stddev
103+
end
104+
end

‎run_benchmarks.rb‎

Lines changed: 9 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -8,20 +8,12 @@
88
require 'rbconfig'
99
require 'etc'
1010
require 'yaml'
11-
require_relative 'misc/stats'
1211
require_relative 'lib/cpu_config'
1312
require_relative 'lib/benchmark_runner'
1413
require_relative 'lib/benchmark_suite'
1514
require_relative 'lib/table_formatter'
1615
require_relative 'lib/argument_parser'
17-
18-
def mean(values)
19-
Stats.new(values).mean
20-
end
21-
22-
def stddev(values)
23-
Stats.new(values).stddev
24-
end
16+
require_relative 'lib/results_table_builder'
2517

2618
def sort_benchmarks(bench_names)
2719
benchmarks_metadata = YAML.load_file('benchmarks.yml')
@@ -72,55 +64,16 @@ def sort_benchmarks(bench_names)
7264

7365
puts
7466

75-
# Table for the data we've gathered
67+
# Build results table
7668
all_names = args.executables.keys
7769
base_name, *other_names = all_names
78-
table = [["bench"]]
79-
format = ["%s"]
80-
all_names.each do |name|
81-
table[0] += ["#{name} (ms)", "stddev (%)"]
82-
format += ["%.1f", "%.1f"]
83-
if args.rss
84-
table[0] += ["RSS (MiB)"]
85-
format += ["%.1f"]
86-
end
87-
end
88-
other_names.each do |name|
89-
table[0] += ["#{name} 1st itr"]
90-
format += ["%.3f"]
91-
end
92-
other_names.each do |name|
93-
table[0] += ["#{base_name}/#{name}"]
94-
format += ["%.3f"]
95-
end
96-
97-
# Format the results table
98-
bench_names.each do |bench_name|
99-
# Skip this bench_name if we failed to get data for any of the executables.
100-
next unless bench_data.all? { |(_k, v)| v[bench_name] }
101-
102-
t0s = all_names.map { |name| (bench_data[name][bench_name]['warmup'][0] || bench_data[name][bench_name]['bench'][0]) * 1000.0 }
103-
times_no_warmup = all_names.map { |name| bench_data[name][bench_name]['bench'].map { |v| v * 1000.0 } }
104-
rsss = all_names.map { |name| bench_data[name][bench_name]['rss'] / 1024.0 / 1024.0 }
105-
106-
base_t0, *other_t0s = t0s
107-
base_t, *other_ts = times_no_warmup
108-
base_rss, *other_rsss = rsss
109-
110-
ratio_1sts = other_t0s.map { |other_t0| base_t0 / other_t0 }
111-
ratios = other_ts.map { |other_t| mean(base_t) / mean(other_t) }
112-
113-
row = [bench_name, mean(base_t), 100 * stddev(base_t) / mean(base_t)]
114-
row << base_rss if args.rss
115-
other_ts.zip(other_rsss).each do |other_t, other_rss|
116-
row += [mean(other_t), 100 * stddev(other_t) / mean(other_t)]
117-
row << other_rss if args.rss
118-
end
119-
120-
row += ratio_1sts + ratios
121-
122-
table << row
123-
end
70+
builder = ResultsTableBuilder.new(
71+
executable_names: all_names,
72+
bench_data: bench_data,
73+
bench_names: bench_names,
74+
include_rss: args.rss
75+
)
76+
table, format = builder.build
12477

12578
output_path = nil
12679
if args.out_override

‎test/results_table_builder_test.rb‎

Lines changed: 192 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,192 @@
1+
require_relative 'test_helper'
2+
require_relative '../lib/results_table_builder'
3+
4+
describe ResultsTableBuilder do
5+
describe '#build' do
6+
it 'builds a table with header and data rows' do
7+
executable_names = ['ruby', 'ruby-yjit']
8+
bench_data = {
9+
'ruby' => {
10+
'fib' => {
11+
'warmup' => [0.1],
12+
'bench' => [0.1, 0.11, 0.09],
13+
'rss' => 1024 * 1024 * 10
14+
}
15+
},
16+
'ruby-yjit' => {
17+
'fib' => {
18+
'warmup' => [0.05],
19+
'bench' => [0.05, 0.06, 0.04],
20+
'rss' => 1024 * 1024 * 12
21+
}
22+
}
23+
}
24+
bench_names = ['fib']
25+
26+
builder = ResultsTableBuilder.new(
27+
executable_names: executable_names,
28+
bench_data: bench_data,
29+
bench_names: bench_names,
30+
include_rss: false
31+
)
32+
33+
table, format = builder.build
34+
35+
assert_equal ['bench', 'ruby (ms)', 'stddev (%)', 'ruby-yjit (ms)', 'stddev (%)', 'ruby-yjit 1st itr', 'ruby/ruby-yjit'], table[0]
36+
37+
assert_equal ['%s', '%.1f', '%.1f', '%.1f', '%.1f', '%.3f', '%.3f'], format
38+
39+
assert_equal 'fib', table[1][0]
40+
assert_in_delta 100.0, table[1][1], 1.0
41+
assert_in_delta 50.0, table[1][3], 1.0
42+
assert_in_delta 2.0, table[1][5], 0.1
43+
assert_in_delta 2.0, table[1][6], 0.1
44+
end
45+
46+
it 'includes RSS columns when include_rss is true' do
47+
executable_names = ['ruby']
48+
bench_data = {
49+
'ruby' => {
50+
'fib' => {
51+
'warmup' => [0.1],
52+
'bench' => [0.1],
53+
'rss' => 1024 * 1024 * 10
54+
}
55+
}
56+
}
57+
bench_names = ['fib']
58+
59+
builder = ResultsTableBuilder.new(
60+
executable_names: executable_names,
61+
bench_data: bench_data,
62+
bench_names: bench_names,
63+
include_rss: true
64+
)
65+
66+
table, format = builder.build
67+
68+
assert_equal ['bench', 'ruby (ms)', 'stddev (%)', 'RSS (MiB)'], table[0]
69+
70+
assert_equal ['%s', '%.1f', '%.1f', '%.1f'], format
71+
72+
assert_in_delta 10.0, table[1][3], 0.1
73+
end
74+
75+
it 'skips benchmarks with missing data' do
76+
executable_names = ['ruby', 'ruby-yjit']
77+
bench_data = {
78+
'ruby' => {
79+
'fib' => {
80+
'warmup' => [0.1],
81+
'bench' => [0.1],
82+
'rss' => 1024 * 1024 * 10
83+
},
84+
'loop' => {
85+
'warmup' => [0.2],
86+
'bench' => [0.2],
87+
'rss' => 1024 * 1024 * 10
88+
}
89+
},
90+
'ruby-yjit' => {
91+
'fib' => {
92+
'warmup' => [0.05],
93+
'bench' => [0.05],
94+
'rss' => 1024 * 1024 * 12
95+
}
96+
}
97+
}
98+
bench_names = ['fib', 'loop']
99+
100+
builder = ResultsTableBuilder.new(
101+
executable_names: executable_names,
102+
bench_data: bench_data,
103+
bench_names: bench_names,
104+
include_rss: false
105+
)
106+
107+
table, _format = builder.build
108+
109+
assert_equal 2, table.length
110+
assert_equal 'fib', table[1][0]
111+
end
112+
113+
it 'handles multiple executables correctly' do
114+
executable_names = ['ruby', 'ruby-yjit', 'ruby-rjit']
115+
bench_data = {
116+
'ruby' => {
117+
'fib' => {
118+
'warmup' => [0.1],
119+
'bench' => [0.1],
120+
'rss' => 1024 * 1024 * 10
121+
}
122+
},
123+
'ruby-yjit' => {
124+
'fib' => {
125+
'warmup' => [0.05],
126+
'bench' => [0.05],
127+
'rss' => 1024 * 1024 * 12
128+
}
129+
},
130+
'ruby-rjit' => {
131+
'fib' => {
132+
'warmup' => [0.07],
133+
'bench' => [0.07],
134+
'rss' => 1024 * 1024 * 11
135+
}
136+
}
137+
}
138+
bench_names = ['fib']
139+
140+
builder = ResultsTableBuilder.new(
141+
executable_names: executable_names,
142+
bench_data: bench_data,
143+
bench_names: bench_names,
144+
include_rss: false
145+
)
146+
147+
table, format = builder.build
148+
149+
expected_header = [
150+
'bench',
151+
'ruby (ms)', 'stddev (%)',
152+
'ruby-yjit (ms)', 'stddev (%)',
153+
'ruby-rjit (ms)', 'stddev (%)',
154+
'ruby-yjit 1st itr',
155+
'ruby-rjit 1st itr',
156+
'ruby/ruby-yjit',
157+
'ruby/ruby-rjit'
158+
]
159+
assert_equal expected_header, table[0]
160+
161+
expected_format = ['%s', '%.1f', '%.1f', '%.1f', '%.1f', '%.1f', '%.1f', '%.3f', '%.3f', '%.3f', '%.3f']
162+
assert_equal expected_format, format
163+
end
164+
165+
it 'uses bench data when warmup is missing' do
166+
executable_names = ['ruby']
167+
bench_data = {
168+
'ruby' => {
169+
'fib' => {
170+
'warmup' => [],
171+
'bench' => [0.1, 0.11],
172+
'rss' => 1024 * 1024 * 10
173+
}
174+
}
175+
}
176+
bench_names = ['fib']
177+
178+
builder = ResultsTableBuilder.new(
179+
executable_names: executable_names,
180+
bench_data: bench_data,
181+
bench_names: bench_names,
182+
include_rss: false
183+
)
184+
185+
table, _format = builder.build
186+
187+
assert_equal 2, table.length
188+
assert_equal 'fib', table[1][0]
189+
assert_in_delta 100.0, table[1][1], 5.0
190+
end
191+
end
192+
end

0 commit comments

Comments
 (0)