diff --git a/lib/benchmark/compare.rb b/lib/benchmark/compare.rb index 57e2e77..51f9952 100644 --- a/lib/benchmark/compare.rb +++ b/lib/benchmark/compare.rb @@ -26,31 +26,67 @@ module Benchmark # Reduce using to_proc: 247295.4 i/s - 1.13x slower # # Besides regular Calculating report, this will also indicates which one is slower. + # + # +x.compare!+ also takes an +order: :baseline+ option. + # + # Example: + # > Benchmark.ips do |x| + # x.report('Reduce using block') { [*1..10].reduce { |sum, n| sum + n } } + # x.report('Reduce using tag') { [*1..10].reduce(:+) } + # x.report('Reduce using to_proc') { [*1..10].reduce(&:+) } + # x.compare!(order: :baseline) + # end + # + # Calculating ------------------------------------- + # Reduce using block 886.202k (± 2.2%) i/s - 4.521M in 5.103774s + # Reduce using tag 1.821M (± 1.6%) i/s - 9.111M in 5.004183s + # Reduce using to_proc 895.948k (± 1.6%) i/s - 4.528M in 5.055368s + # + # Comparison: + # Reduce using block: 886202.5 i/s + # Reduce using tag: 1821055.0 i/s - 2.05x (± 0.00) faster + # Reduce using to_proc: 895948.1 i/s - same-ish: difference falls within error + # + # The first report is considered the baseline against which other reports are compared. module Compare # Compare between reports, prints out facts of each report: # runtime, comparative speed difference. # @param entries [Array] Reports to compare. - def compare(*entries) + def compare(*entries, order: :fastest) return if entries.size < 2 - sorted = entries.sort_by{ |e| e.stats.central_tendency }.reverse - - best = sorted.shift + case order + when :baseline + baseline = entries.shift + sorted = entries.sort_by{ |e| e.stats.central_tendency }.reverse + when :fastest + sorted = entries.sort_by{ |e| e.stats.central_tendency }.reverse + baseline = sorted.shift + else + raise ArgumentError, "Unknwon order: #{order.inspect}" + end $stdout.puts "\nComparison:" - $stdout.printf "%20s: %10.1f i/s\n", best.label.to_s, best.stats.central_tendency + $stdout.printf "%20s: %10.1f i/s\n", baseline.label.to_s, baseline.stats.central_tendency sorted.each do |report| name = report.label.to_s $stdout.printf "%20s: %10.1f i/s - ", name, report.stats.central_tendency - if report.stats.overlaps?(best.stats) + if report.stats.overlaps?(baseline.stats) $stdout.print "same-ish: difference falls within error" + elsif report.stats.central_tendency > baseline.stats.central_tendency + speedup, error = report.stats.speedup(baseline.stats) + $stdout.printf "%.2fx ", speedup + if error + $stdout.printf " (± %.2f)", error + end + $stdout.print " faster" else - slowdown, error = report.stats.slowdown(best.stats) + slowdown, error = report.stats.slowdown(baseline.stats) $stdout.printf "%.2fx ", slowdown if error $stdout.printf " (± %.2f)", error @@ -61,7 +97,7 @@ def compare(*entries) $stdout.puts end - footer = best.stats.footer + footer = baseline.stats.footer $stdout.puts footer.rjust(40) if footer $stdout.puts diff --git a/lib/benchmark/ips/job.rb b/lib/benchmark/ips/job.rb index 1a7799f..d815838 100644 --- a/lib/benchmark/ips/job.rb +++ b/lib/benchmark/ips/job.rb @@ -65,6 +65,7 @@ def initialize opts={} @run_single = false @json_path = false @compare = false + @compare_order = :fastest @held_path = nil @held_results = nil @@ -117,8 +118,9 @@ def compare? end # Run comparison utility. - def compare! + def compare!(order: :fastest) @compare = true + @compare_order = order end # Return true if results are held while multiple Ruby invocations @@ -373,7 +375,7 @@ def create_stats(samples) # Run comparison of entries in +@full_report+. def run_comparison - @full_report.run_comparison if compare? + @full_report.run_comparison(@compare_order) if compare? end # Generate json from +@full_report+. diff --git a/lib/benchmark/ips/report.rb b/lib/benchmark/ips/report.rb index 461713a..5404ef2 100644 --- a/lib/benchmark/ips/report.rb +++ b/lib/benchmark/ips/report.rb @@ -176,8 +176,8 @@ def data end # Run comparison of entries. - def run_comparison - Benchmark.compare(*@entries) + def run_comparison(order) + Benchmark.compare(*@entries, order: order) end # Generate json from Report#data to given path. diff --git a/lib/benchmark/ips/stats/bootstrap.rb b/lib/benchmark/ips/stats/bootstrap.rb index beee145..79ed30a 100644 --- a/lib/benchmark/ips/stats/bootstrap.rb +++ b/lib/benchmark/ips/stats/bootstrap.rb @@ -33,6 +33,10 @@ def slowdown(baseline) [slowdown, error] end + def speedup(baseline) + baseline.slowdown(self) + end + def footer "with #{(@confidence.to_f * 100).round(1)}% confidence" end diff --git a/lib/benchmark/ips/stats/sd.rb b/lib/benchmark/ips/stats/sd.rb index 8ac4bd3..bc1bb97 100644 --- a/lib/benchmark/ips/stats/sd.rb +++ b/lib/benchmark/ips/stats/sd.rb @@ -30,6 +30,10 @@ def slowdown(baseline) end end + def speedup(baseline) + baseline.slowdown(self) + end + def footer nil end