Skip to content

Commit 5e14a1d

Browse files
rafaelfrancarafaelfranca
committed
Add tests for run_benchmarks.rb helper methods
1 parent c34aeca commit 5e14a1d

5 files changed

Lines changed: 745 additions & 123 deletions

File tree

‎Rakefile‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
# frozen_string_literal: true
2+
3+
require 'rake/testtask'
4+
5+
desc 'Run all tests'
6+
Rake::TestTask.new(:test) do |t|
7+
t.libs << 'test'
8+
t.libs << 'lib'
9+
t.test_files = FileList['test/**/*_test.rb']
10+
t.verbose = true
11+
t.warning = true
12+
end
13+
14+
task default: :test

‎lib/benchmark_runner.rb‎

Lines changed: 158 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,158 @@
1+
# frozen_string_literal: true
2+
3+
require 'csv'
4+
require 'json'
5+
require 'rbconfig'
6+
7+
# Extracted helper methods from run_benchmarks.rb for testing
8+
module BenchmarkRunner
9+
module_function
10+
11+
# Format benchmark data as a string table
12+
def table_to_str(table_data, format, failures)
13+
# Trim numbers to one decimal for console display
14+
# Keep two decimals for the speedup ratios
15+
16+
failure_rows = failures.map { |_exe, data| data.keys }.flatten.uniq
17+
.map { |name| [name] + (['N/A'] * (table_data.first.size - 1)) }
18+
19+
table_data = table_data.first(1) + failure_rows + table_data.drop(1).map { |row|
20+
format.zip(row).map { |fmt, data| fmt % data }
21+
}
22+
23+
num_rows = table_data.length
24+
num_cols = table_data[0].length
25+
26+
# Pad each column to the maximum width in the column
27+
(0...num_cols).each do |c|
28+
cell_lens = (0...num_rows).map { |r| table_data[r][c].length }
29+
max_width = cell_lens.max
30+
(0...num_rows).each { |r| table_data[r][c] = table_data[r][c].ljust(max_width) }
31+
end
32+
33+
# Row of separator dashes
34+
sep_row = (0...num_cols).map { |i| '-' * table_data[0][i].length }.join(' ').rstrip
35+
36+
out = sep_row + "\n"
37+
38+
table_data.each do |row|
39+
out += row.join(' ').rstrip + "\n"
40+
end
41+
42+
out += sep_row + "\n"
43+
44+
out
45+
end
46+
47+
# Find the first available file number for output files
48+
def free_file_no(prefix)
49+
(1..).each do |file_no|
50+
out_path = File.join(prefix, "output_%03d.csv" % file_no)
51+
return file_no unless File.exist?(out_path)
52+
end
53+
end
54+
55+
# Get benchmark categories from metadata
56+
def benchmark_categories(name, metadata)
57+
benchmark_metadata = metadata.find { |benchmark, _metadata| benchmark == name }&.last || {}
58+
categories = [benchmark_metadata.fetch('category', 'other')]
59+
categories << 'ractor' if benchmark_metadata['ractor']
60+
categories
61+
end
62+
63+
# Check if the name matches any of the names in a list of filters
64+
def match_filter(entry, categories:, name_filters:, metadata:)
65+
name_filters = process_name_filters(name_filters)
66+
name = entry.sub(/\.rb\z/, '')
67+
(categories.empty? || benchmark_categories(name, metadata).any? { |cat| categories.include?(cat) }) &&
68+
(name_filters.empty? || name_filters.any? { |filter| filter === name })
69+
end
70+
71+
# Process "/my_benchmark/i" into /my_benchmark/i
72+
def process_name_filters(name_filters)
73+
name_filters.map do |name_filter|
74+
if name_filter[0] == "/"
75+
regexp_str = name_filter[1..-1].reverse.sub(/\A(\w*)\//, "")
76+
regexp_opts = ::Regexp.last_match(1).to_s
77+
regexp_str.reverse!
78+
r = /#{regexp_str}/
79+
if !regexp_opts.empty?
80+
# Convert option string to Regexp option flags
81+
flags = 0
82+
flags |= Regexp::IGNORECASE if regexp_opts.include?('i')
83+
flags |= Regexp::MULTILINE if regexp_opts.include?('m')
84+
flags |= Regexp::EXTENDED if regexp_opts.include?('x')
85+
r = Regexp.new(regexp_str, flags)
86+
end
87+
r
88+
else
89+
name_filter
90+
end
91+
end
92+
end
93+
94+
# Resolve the pre_init file path into a form that can be required
95+
def expand_pre_init(path)
96+
require 'pathname'
97+
98+
path = Pathname.new(path)
99+
100+
unless path.exist?
101+
puts "--with-pre-init called with non-existent file!"
102+
exit(-1)
103+
end
104+
105+
if path.directory?
106+
puts "--with-pre-init called with a directory, please pass a .rb file"
107+
exit(-1)
108+
end
109+
110+
library_name = path.basename(path.extname)
111+
load_path = path.parent.expand_path
112+
113+
[
114+
"-I", load_path,
115+
"-r", library_name
116+
]
117+
end
118+
119+
# Sort benchmarks with headlines first, then others, then micro
120+
def sort_benchmarks(bench_names, metadata)
121+
headline_benchmarks = metadata.select { |_, meta| meta['category'] == 'headline' }.keys
122+
micro_benchmarks = metadata.select { |_, meta| meta['category'] == 'micro' }.keys
123+
124+
headline_names, bench_names = bench_names.partition { |name| headline_benchmarks.include?(name) }
125+
micro_names, other_names = bench_names.partition { |name| micro_benchmarks.include?(name) }
126+
headline_names.sort + other_names.sort + micro_names.sort
127+
end
128+
129+
# Check which OS we are running
130+
def os
131+
@os ||= (
132+
host_os = RbConfig::CONFIG['host_os']
133+
case host_os
134+
when /mswin|msys|mingw|cygwin|bccwin|wince|emc/
135+
:windows
136+
when /darwin|mac os/
137+
:macosx
138+
when /linux/
139+
:linux
140+
when /solaris|bsd/
141+
:unix
142+
else
143+
raise "unknown os: #{host_os.inspect}"
144+
end
145+
)
146+
end
147+
148+
# Generate setarch prefix for Linux
149+
def setarch_prefix
150+
# Disable address space randomization (for determinism)
151+
prefix = ["setarch", `uname -m`.strip, "-R"]
152+
153+
# Abort if we don't have permission (perhaps in a docker container).
154+
return [] unless system(*prefix, "true", out: File::NULL, err: File::NULL)
155+
156+
prefix
157+
end
158+
end

‎run_benchmarks.rb‎

Lines changed: 8 additions & 123 deletions
Original file line numberDiff line numberDiff line change
@@ -11,24 +11,11 @@
1111
require 'etc'
1212
require 'yaml'
1313
require_relative 'misc/stats'
14+
require_relative 'lib/benchmark_runner'
1415

1516
# Check which OS we are running
1617
def os
17-
@os ||= (
18-
host_os = RbConfig::CONFIG['host_os']
19-
case host_os
20-
when /mswin|msys|mingw|cygwin|bccwin|wince|emc/
21-
:windows
22-
when /darwin|mac os/
23-
:macosx
24-
when /linux/
25-
:linux
26-
when /solaris|bsd/
27-
:unix
28-
else
29-
raise "unknown os: #{host_os.inspect}"
30-
end
31-
)
18+
BenchmarkRunner.os
3219
end
3320

3421
# Checked system - error or return info if the command fails
@@ -123,38 +110,7 @@ def performance_governor?
123110
end
124111

125112
def table_to_str(table_data, format, failures)
126-
# Trim numbers to one decimal for console display
127-
# Keep two decimals for the speedup ratios
128-
129-
failure_rows = failures.map { |_exe, data| data.keys }.flatten.uniq
130-
.map { |name| [name] + (['N/A'] * (table_data.first.size - 1)) }
131-
132-
table_data = table_data.first(1) + failure_rows + table_data.drop(1).map { |row|
133-
format.zip(row).map { |fmt, data| fmt % data }
134-
}
135-
136-
num_rows = table_data.length
137-
num_cols = table_data[0].length
138-
139-
# Pad each column to the maximum width in the column
140-
(0...num_cols).each do |c|
141-
cell_lens = (0...num_rows).map { |r| table_data[r][c].length }
142-
max_width = cell_lens.max
143-
(0...num_rows).each { |r| table_data[r][c] = table_data[r][c].ljust(max_width) }
144-
end
145-
146-
# Row of separator dashes
147-
sep_row = (0...num_cols).map { |i| '-' * table_data[0][i].length }.join(' ')
148-
149-
out = sep_row + "\n"
150-
151-
table_data.each do |row|
152-
out += row.join(' ') + "\n"
153-
end
154-
155-
out += sep_row
156-
157-
return out
113+
BenchmarkRunner.table_to_str(table_data, format, failures)
158114
end
159115

160116
def mean(values)
@@ -165,92 +121,21 @@ def stddev(values)
165121
Stats.new(values).stddev
166122
end
167123

168-
def free_file_no(prefix)
169-
(1..).each do |file_no|
170-
out_path = File.join(prefix, "output_%03d.csv" % file_no)
171-
if !File.exist?(out_path)
172-
return file_no
173-
end
174-
end
175-
end
176-
177-
def benchmark_categories(name)
178-
metadata = benchmarks_metadata.find { |benchmark, _metadata| benchmark == name }&.last || {}
179-
categories = [metadata.fetch('category', 'other')]
180-
categories << 'ractor' if metadata['ractor']
181-
categories
182-
end
183-
184124
# Check if the name matches any of the names in a list of filters
185125
def match_filter(entry, categories:, name_filters:)
186-
name_filters = process_name_filters(name_filters)
187-
name = entry.sub(/\.rb\z/, '')
188-
(categories.empty? || benchmark_categories(name).any? { |cat| categories.include?(cat) }) &&
189-
(name_filters.empty? || name_filters.any? { |filter| filter === name })
190-
end
191-
192-
# process "/my_benchmark/i" into /my_benchmark/i
193-
def process_name_filters(name_filters)
194-
name_filters.map do |name_filter|
195-
if name_filter[0] == "/"
196-
regexp_str = name_filter[1..-1].reverse.sub(/\A(\w*)\//, "")
197-
regexp_opts = $1.to_s
198-
regexp_str.reverse!
199-
r = /#{regexp_str}/
200-
if !regexp_opts.empty?
201-
r = Regexp.compile(r.to_s, regexp_opts)
202-
end
203-
r
204-
else
205-
name_filter
206-
end
207-
end
208-
end
209-
210-
# Resolve the pre_init file path into a form that can be required
211-
def expand_pre_init(path)
212-
path = Pathname.new(path)
213-
214-
unless path.exist?
215-
puts "--with-pre-init called with non-existent file!"
216-
exit(-1)
217-
end
218-
219-
if path.directory?
220-
puts "--with-pre-init called with a directory, please pass a .rb file"
221-
exit(-1)
222-
end
223-
224-
library_name = path.basename(path.extname)
225-
load_path = path.parent.expand_path
226-
227-
[
228-
"-I", load_path,
229-
"-r", library_name
230-
]
126+
BenchmarkRunner.match_filter(entry, categories: categories, name_filters: name_filters, metadata: benchmarks_metadata)
231127
end
232128

233129
def benchmarks_metadata
234130
@benchmarks_metadata ||= YAML.load_file('benchmarks.yml')
235131
end
236132

237133
def sort_benchmarks(bench_names)
238-
headline_benchmarks = benchmarks_metadata.select { |_, metadata| metadata['category'] == 'headline' }.keys
239-
micro_benchmarks = benchmarks_metadata.select { |_, metadata| metadata['category'] == 'micro' }.keys
240-
241-
headline_names, bench_names = bench_names.partition { |name| headline_benchmarks.include?(name) }
242-
micro_names, other_names = bench_names.partition { |name| micro_benchmarks.include?(name) }
243-
headline_names.sort + other_names.sort + micro_names.sort
134+
BenchmarkRunner.sort_benchmarks(bench_names, benchmarks_metadata)
244135
end
245136

246137
def setarch_prefix
247-
# Disable address space randomization (for determinism)
248-
prefix = ["setarch", `uname -m`.strip, "-R"]
249-
250-
# Abort if we don't have permission (perhaps in a docker container).
251-
return [] unless system(*prefix, "true")
252-
253-
prefix
138+
BenchmarkRunner.setarch_prefix
254139
end
255140

256141
# Run all the benchmarks and record execution times
@@ -284,7 +169,7 @@ def run_benchmarks(ruby:, ruby_description:, categories:, name_filters:, out_pat
284169
end
285170

286171
if pre_init
287-
pre_init = expand_pre_init(pre_init)
172+
pre_init = BenchmarkRunner.expand_pre_init(pre_init)
288173
end
289174

290175

@@ -603,7 +488,7 @@ def run_benchmarks(ruby:, ruby_description:, categories:, name_filters:, out_pat
603488
output_path = args.out_override
604489
else
605490
# If no out path is specified, find a free file index for the output files
606-
file_no = free_file_no(args.out_path)
491+
file_no = BenchmarkRunner.free_file_no(args.out_path)
607492
output_path = File.join(args.out_path, "output_%03d" % file_no)
608493
end
609494

0 commit comments

Comments
 (0)