compare!(order: :baseline) option (#123)

When enabled the first report is considered the baseline against
which other reports are compared.

```
Comparison:
  Reduce using block:   886202.5 i/s
    Reduce using tag:  1821055.0 i/s - 2.05x  (± 0.00) faster
Reduce using to_proc:   895948.1 i/s - same-ish: difference falls within error
```

Instead of:

```
Comparison:
    Reduce using tag:  1826997.4 i/s
Reduce using to_proc:   881011.9 i/s - 2.07x  (± 0.00) slower
  Reduce using block:   879261.1 i/s - 2.08x  (± 0.00) slower
```

Co-authored-by: Jean Boussier <[email protected]>

Author: casperisfine

lib/benchmark/compare.rb

@@ -26,31 +26,67 @@ module Benchmark
   #   Reduce using to_proc:   247295.4 i/s - 1.13x slower
   #
   # Besides regular Calculating report, this will also indicates which one is slower.
+  #
+  # +x.compare!+ also takes an +order: :baseline+ option.
+  #
+  # Example:
+  #  > Benchmark.ips do |x|
+  #   x.report('Reduce using block')   { [*1..10].reduce { |sum, n| sum + n } }
+  #   x.report('Reduce using tag')     { [*1..10].reduce(:+) }
+  #   x.report('Reduce using to_proc') { [*1..10].reduce(&:+) }
+  #   x.compare!(order: :baseline)
+  # end
+  #
+  # Calculating -------------------------------------
+  #   Reduce using block    886.202k (± 2.2%) i/s -      4.521M in   5.103774s
+  #     Reduce using tag      1.821M (± 1.6%) i/s -      9.111M in   5.004183s
+  # Reduce using to_proc    895.948k (± 1.6%) i/s -      4.528M in   5.055368s
+  #
+  # Comparison:
+  #   Reduce using block:   886202.5 i/s
+  #     Reduce using tag:  1821055.0 i/s - 2.05x  (± 0.00) faster
+  # Reduce using to_proc:   895948.1 i/s - same-ish: difference falls within error
+  #
+  # The first report is considered the baseline against which other reports are compared.
   module Compare
 
     # Compare between reports, prints out facts of each report:
     # runtime, comparative speed difference.
     # @param entries [Array<Report::Entry>] Reports to compare.
-    def compare(*entries)
+    def compare(*entries, order: :fastest)
       return if entries.size < 2
 
-      sorted = entries.sort_by{ |e| e.stats.central_tendency }.reverse
-
-      best = sorted.shift
+      case order
+      when :baseline
+        baseline = entries.shift
+        sorted = entries.sort_by{ |e| e.stats.central_tendency }.reverse
+      when :fastest
+        sorted = entries.sort_by{ |e| e.stats.central_tendency }.reverse
+        baseline = sorted.shift
+      else
+        raise ArgumentError, "Unknwon order: #{order.inspect}"
+      end
 
       $stdout.puts "\nComparison:"
 
-      $stdout.printf "%20s: %10.1f i/s\n", best.label.to_s, best.stats.central_tendency
+      $stdout.printf "%20s: %10.1f i/s\n", baseline.label.to_s, baseline.stats.central_tendency
 
       sorted.each do |report|
         name = report.label.to_s
 
         $stdout.printf "%20s: %10.1f i/s - ", name, report.stats.central_tendency
 
-        if report.stats.overlaps?(best.stats)
+        if report.stats.overlaps?(baseline.stats)
           $stdout.print "same-ish: difference falls within error"
+        elsif report.stats.central_tendency > baseline.stats.central_tendency
+          speedup, error = report.stats.speedup(baseline.stats)
+          $stdout.printf "%.2fx ", speedup
+          if error
+            $stdout.printf " (± %.2f)", error
+          end
+          $stdout.print " faster"
         else
-          slowdown, error = report.stats.slowdown(best.stats)
+          slowdown, error = report.stats.slowdown(baseline.stats)
           $stdout.printf "%.2fx ", slowdown
           if error
             $stdout.printf " (± %.2f)", error
@@ -61,7 +97,7 @@ def compare(*entries)
         $stdout.puts
       end
 
-      footer = best.stats.footer
+      footer = baseline.stats.footer
       $stdout.puts footer.rjust(40) if footer
 
       $stdout.puts

lib/benchmark/ips/job.rb

@@ -65,6 +65,7 @@ def initialize opts={}
         @run_single = false
         @json_path = false
         @compare = false
+        @compare_order = :fastest
         @held_path = nil
         @held_results = nil
 
@@ -117,8 +118,9 @@ def compare?
       end
 
       # Run comparison utility.
-      def compare!
+      def compare!(order: :fastest)
         @compare = true
+        @compare_order = order
       end
 
       # Return true if results are held while multiple Ruby invocations
@@ -373,7 +375,7 @@ def create_stats(samples)
 
       # Run comparison of entries in +@full_report+.
       def run_comparison
-        @full_report.run_comparison if compare?
+        @full_report.run_comparison(@compare_order) if compare?
       end
 
       # Generate json from +@full_report+.

lib/benchmark/ips/report.rb

@@ -176,8 +176,8 @@ def data
       end
 
       # Run comparison of entries.
-      def run_comparison
-        Benchmark.compare(*@entries)
+      def run_comparison(order)
+        Benchmark.compare(*@entries, order: order)
       end
 
       # Generate json from Report#data to given path.

lib/benchmark/ips/stats/bootstrap.rb

@@ -33,6 +33,10 @@ def slowdown(baseline)
           [slowdown, error]
         end
 
+        def speedup(baseline)
+          baseline.slowdown(self)
+        end
+
         def footer
           "with #{(@confidence.to_f * 100).round(1)}% confidence"
         end

lib/benchmark/ips/stats/sd.rb

@@ -30,6 +30,10 @@ def slowdown(baseline)
           end
         end
 
+        def speedup(baseline)
+          baseline.slowdown(self)
+        end
+
         def footer
           nil
         end