runplotlytests: report perf timings inline and in a sample summary table

Author: robertoffmoura

plotly/testing/PlotlyPerfTestCase.m

@@ -77,6 +77,31 @@
             end
             ok = true;
         end
+        function s = sampleStats(tc)
+            % statistics of the collected samples; NumSamples is 0 when
+            % nothing was measured
+            if tc.p_nSamples == 0
+                s = struct('NumSamples', 0, 'Mean', [], 'Std', [], ...
+                    'Min', [], 'Max', [], 'RelMoE', [], ...
+                    'NumWarmups', tc.p_warmupsDone);
+                return;
+            end
+            x = tc.p_times(1:tc.p_nSamples);
+            m = mean(x);
+            sd = std(x);
+            if m > 0
+                relMoE = tc.tQuantile(tc.p_nSamples - 1) * sd / (m * sqrt(tc.p_nSamples));
+            else
+                relMoE = [];
+            end
+            s = struct('NumSamples', tc.p_nSamples, ...
+                'Mean', m, ...
+                'Std', sd, ...
+                'Min', min(x), ...
+                'Max', max(x), ...
+                'RelMoE', relMoE, ...
+                'NumWarmups', tc.p_warmupsDone);
+        end
     end
 
     methods (Access = private)

plotly/testing/runplotlytests.m

@@ -102,6 +102,8 @@ function runplotlytests(varargin)
         % including each parameterized case
         hasSetup = ismember('setUp', classMeths);
         hasTearDown = ismember('tearDown', classMeths);
+        isPerfClass = isa(tc, 'PlotlyPerfTestCase');
+        perfRows = {};
         nTests = 0;
         verdicts = struct('Name', {}, 'Passed', {}, 'VerificationFailures', {}, ...
             'Diagnostics', {}, 'ErrorTrace', {}, 'Duration', {}, 'Errored', {});
@@ -230,6 +232,16 @@ function runplotlytests(varargin)
                 else
                     fprintf('.');
                 end
+
+                if isPerfClass
+                    st = tc.sampleStats();
+                    if st.NumSamples > 0
+                        fprintf('\n  %s: %s mean, %s MoE, n=%d (%d warmups)\n', ...
+                            caseName, fmtTime(st.Mean), fmtMoE(st.RelMoE), ...
+                            st.NumSamples, st.NumWarmups);
+                        perfRows{end+1} = struct('Name', caseName, 'Stats', st); %#ok<AGROW>
+                    end
+                end
             end
         end
 
@@ -242,6 +254,25 @@ function runplotlytests(varargin)
                 'ErrorTrace', '', 'Duration', 0, 'Errored', true);
         end
 
+        if ~isempty(perfRows)
+            maxName = 0;
+            for r = 1:numel(perfRows)
+                maxName = max(maxName, length(perfRows{r}.Name));
+            end
+            fprintf('\nSample summary:\n');
+            hdr = sprintf('     %-*s  %4s  %10s  %10s  %10s  %10s  %7s\n', ...
+                maxName, 'Name', 'n', 'Mean', 'Std', 'Min', 'Max', 'MoE');
+            fprintf('%s', hdr);
+            fprintf('     %s\n', repmat('-', 1, numel(hdr) - 1));
+            for r = 1:numel(perfRows)
+                st = perfRows{r}.Stats;
+                fprintf('     %-*s  %4d  %10s  %10s  %10s  %10s  %7s\n', ...
+                    maxName, perfRows{r}.Name, st.NumSamples, ...
+                    fmtSec(st.Mean), fmtSec(st.Std), fmtSec(st.Min), ...
+                    fmtSec(st.Max), fmtMoE(st.RelMoE));
+            end
+        end
+
         fprintf('\nDone %s\n', className);
         fprintf('__________\n\n');
 
@@ -542,6 +573,29 @@ function runplotlytests(varargin)
     end
 end
 
+function s = fmtTime(t)
+    % auto-scaled time for the inline per-case line
+    if t < 1
+        s = sprintf('%.2f ms', t * 1000);
+    else
+        s = sprintf('%.3f s', t);
+    end
+end
+
+function s = fmtSec(t)
+    % fixed seconds for the sample summary columns
+    s = sprintf('%.6g', t);
+end
+
+function s = fmtMoE(r)
+    % relative margin of error as a percentage
+    if isempty(r) || ~isfinite(r)
+        s = 'n/a';
+    else
+        s = sprintf('%.1f%%', r * 100);
+    end
+end
+
 function trace = errorTrace(e)
     trace = sprintf('''%s''\n%s', e.identifier, e.message);
     for si = 1:numel(e.stack)