Allow display of secondary metrics from profilers

retronym · retronym · commit 5a6d23c666de · 2020-05-27T16:47:09.000+10:00
diff --git a/README.md b/README.md
@@ -83,6 +83,8 @@ benchdb takes a JMH result file plus some captured environment data (platform, J
   \----------------------------------------------------------+--------+------+-----+------------+----------+-------/
   ```
 
+Secondary metrics can be used in place of the primary metric with `-metric`, (for instance, `-metric ·gc.alloc.rate.norm` shows the the per-operation allocations recorded by JMH's `-prof gc`.
+
 - Extractor patterns can be used to extract additional parameters from benchmark names. They are glob patterns with regular expression-like capture groups. Unnamed groups are discarded, named groups are extracted into parameters. For example:
 
   ```
diff --git a/core/src/main/scala/com/lightbend/benchdb/GenerateCharts.scala b/core/src/main/scala/com/lightbend/benchdb/GenerateCharts.scala
@@ -3,7 +3,7 @@ package com.lightbend.benchdb
 import scala.collection.JavaConverters._
 import com.typesafe.config.{Config, ConfigFactory, ConfigObject, ConfigRenderOptions, ConfigValueFactory}
 
-class GenerateCharts(go: GlobalOptions, scorePrecision: Int) extends Logging {
+class GenerateCharts(go: GlobalOptions, scorePrecision: Int, metric: Option[String]) extends Logging {
 
   def generatePivoted(pivoted: Iterable[(RunResult, IndexedSeq[Option[RunResult]])],
                       pivotSets: Seq[Seq[String]], pivotParams: Seq[String], otherParams: Seq[String]): String = {
@@ -38,7 +38,7 @@ class GenerateCharts(go: GlobalOptions, scorePrecision: Int) extends Logging {
               }
             }).asJava
           }.asJava
-          renderChartData(n, pn, rs.head._1.primaryMetric.scoreUnit, lineChartOptionsBase, columns, rows)
+          renderChartData(n, pn, vAxisTitle(rs.head._1.primaryMetricOr(metric).scoreUnit), lineChartOptionsBase, columns, rows)
         }
       }
     }.mkString("[", ",", "]")
@@ -63,12 +63,17 @@ class GenerateCharts(go: GlobalOptions, scorePrecision: Int) extends Logging {
           val rows = rs.zip(paramValues.map(_.get)).sortBy(_._2).map { case (r, pv) =>
             (pv +: seriesRowData(s"$pn = $pv", r)).asJava
           }.asJava
-          renderChartData(n, pn, rs.head.primaryMetric.scoreUnit, lineChartOptionsBase, columns, rows)
+          renderChartData(n, pn, vAxisTitle(rs.head.primaryMetricOr(metric).scoreUnit), lineChartOptionsBase, columns, rows)
         }
       }
     }.mkString("[", ",", "]")
   }
 
+  private def vAxisTitle(unit: String) = metric match {
+    case Some(x) => x + " " + unit
+    case None => unit
+  }
+
   private def renderChartData(title: String, hAxis: String, vAxis: String, baseConfig: Config, columns: Any, rows: Any): String = {
     val options = optionsConfig(title, hAxis, vAxis, baseConfig)
     val c = ConfigValueFactory.fromMap(Map("options" -> options, "columns" -> columns, "rows" -> rows).asJava)
@@ -91,10 +96,11 @@ class GenerateCharts(go: GlobalOptions, scorePrecision: Int) extends Logging {
   )
 
   private def seriesRowData(name: String, r: RunResult): Seq[Any] = {
-    val score = r.primaryMetric.score
-    val err = r.primaryMetric.scoreError
+    val metric1 = r.primaryMetricOr(metric)
+    val score = metric1.score
+    val err = metric1.scoreError
     val scoreS = ScoreFormatter(score, scorePrecision)
     val errS = ScoreFormatter(err, scorePrecision)
-    Seq(score, score-err, score+err, s"$name<br/>Score: <b>$scoreS</b> ± $errS ${r.primaryMetric.scoreUnit}")
+    Seq(score, score-err, score+err, s"$name<br/>Score: <b>$scoreS</b> ± $errS ${metric1.scoreUnit}")
   }
 }
diff --git a/core/src/main/scala/com/lightbend/benchdb/Main.scala b/core/src/main/scala/com/lightbend/benchdb/Main.scala
@@ -68,10 +68,11 @@ object Main extends Logging {
       "Query the database for test results and print them.") {
       val pivot = Opts.options[String]("pivot", help = "Parameter names to pivot in table output.").map(_.toList).orElse(Opts(Nil))
       val raw = Opts.flag("raw", "Print raw JSON data instead of a table.").orFalse
-      (runs, benchs, extract, regex, scorePrecision, pivot, raw).mapN { case (runs, benchs, extract, regex, sp, pivot, raw) =>
+      val metric = Opts.option[String]("metric", "Secondary metric to report rather than the primary, e.g. ·gc.alloc.rate.norm").orNone
+      (runs, benchs, extract, regex, scorePrecision, pivot, raw, metric).mapN { case (runs, benchs, extract, regex, sp, pivot, raw, metric) =>
       { go =>
         if(raw && pivot.nonEmpty) logger.error("Cannot pivot in raw output mode.")
-        else queryResults(go, runs, benchs, extract, regex, sp, pivot, raw)
+        else queryResults(go, runs, benchs, extract, regex, sp, pivot, raw, metric)
       }
       }
     }
@@ -80,8 +81,9 @@ object Main extends Logging {
       val template = Opts.option[Path]("template", "HTML template containing file.").orNone
       val out = Opts.option[Path]("out", short = "o", help = "Output file to generate, or '-' for stdout.").orNone
       val pivot = Opts.options[String]("pivot", help = "Parameter names to combine in a chart.").map(_.toList).orElse(Opts(Nil))
-      (runs, benchs, extract, regex, scorePrecision, pivot, template, out).mapN { case (runs, benchs, extract, regex, sp, pivot, template, out) =>
-        createChart(_, runs, benchs, extract, regex, sp, pivot, template, out, args)
+      val metric = Opts.option[String]("metric", "Secondary metric to report rather than the primary, e.g. ·gc.alloc.rate.norm").orNone
+      (runs, benchs, extract, regex, scorePrecision, pivot, metric, template, out).mapN { case (runs, benchs, extract, regex, sp, pivot, metric, template, out) =>
+        createChart(_, runs, benchs, extract, regex, sp, pivot, metric, template, out, args)
       }
     }
 
@@ -169,12 +171,16 @@ object Main extends Logging {
     }
   }
 
-  def queryResults(go: GlobalOptions, runs: Seq[String], benchs: Seq[String], extract: Seq[String], regex: Boolean, scorePrecision: Int, pivot: Seq[String], raw: Boolean): Unit = try {
+  def queryResults(go: GlobalOptions, runs: Seq[String], benchs: Seq[String], extract: Seq[String], regex: Boolean, scorePrecision: Int, pivot: Seq[String], raw: Boolean, metric: Option[String]): Unit = try {
     new Global(go).use { g =>
       val multi = runs.size > 1
       val allRs = g.dao.run(g.dao.checkVersion andThen g.dao.queryResults(runs))
         .map { case (rr, runId) => RunResult.fromDb(rr, runId, multi) }
       val rs = RunResult.extract(extract, regex, RunResult.filterByName(benchs, allRs)).toSeq
+      val benchmarkLabel = metric match {
+        case Some(id) => s"Benchmark [$id]"
+        case None => "Benchmark"
+      }
       if(raw) {
         print("[")
         rs.zipWithIndex.foreach { case (r, idx) =>
@@ -200,22 +206,23 @@ object Main extends Logging {
             paramNames.map(s => Format(Align.Right)),
             Seq(Format(), Format(Align.Right), Format(Align.Right), Format(Align.Right), Format())
           ).flatten
-          val header = (("Benchmark" +: paramNames.map(s => s"($s)")) ++ Vector("Mode", "Cnt", "Score", "Error", "Units")).map(formatHeader)
+          val header = ((benchmarkLabel +: paramNames.map(s => s"($s)")) ++ Vector("Mode", "Cnt", "Score", "Error", "Units")).map(formatHeader)
           val data = rs.iterator.map { r =>
+            val score = r.primaryMetricOr(metric)
             Vector(
               Seq(r.name),
               paramNames.map(k => r.params.getOrElse(k, "")),
               Seq(r.db.mode, r.cnt,
-                ScoreFormatter(r.primaryMetric.score, scorePrecision), ScoreFormatter(r.primaryMetric.scoreError, scorePrecision),
-                r.primaryMetric.scoreUnit)
+                ScoreFormatter(score.score, scorePrecision), ScoreFormatter(score.scoreError, scorePrecision),
+                score.scoreUnit)
             ).flatten
           }.toVector
           val table = new TableFormatter(go).apply(columns, Vector(header, null) ++ data)
           table.foreach(println)
         } else if(paramNames.length + pivotSet.size != allParamNames.length) {
             logger.error(s"Illegal pivot parameters.")
         } else {
-          val (pivoted, pivotSets) = RunResult.pivot(rs, pivot, paramNames)
+          val (pivoted, pivotSets) = RunResult.pivot(rs, pivot, paramNames, metric)
           val columns = Vector(
             Seq(Format()),
             paramNames.map(s => Format(Align.Right)),
@@ -230,23 +237,25 @@ object Main extends Logging {
             Seq(""),
           ).flatten
           val header2 = Vector(
-            Seq("Benchmark"),
+            Seq(benchmarkLabel),
             paramNames.map(s => s"($s)"),
             Seq("Mode", "Cnt"),
             pivotSets.flatMap(_ => Seq("Score", "Error")),
             Seq("Units"),
           ).flatten.map(formatHeader)
           val data = pivoted.iterator.map { case (r, pivotData) =>
             val scoreData = pivotData.flatMap {
-              case Some(rr) => Seq(ScoreFormatter(rr.primaryMetric.score, scorePrecision), ScoreFormatter(rr.primaryMetric.scoreError, scorePrecision))
+              case Some(rr) =>
+                val score = rr.primaryMetricOr(metric)
+                Seq(ScoreFormatter(score.score, scorePrecision), ScoreFormatter(score.scoreError, scorePrecision))
               case None => Seq(null, null)
             }
             Vector(
               Seq(r.name),
               paramNames.map(k => r.params.getOrElse(k, "")),
               Seq(r.db.mode, r.cnt),
               scoreData,
-              Seq(r.primaryMetric.scoreUnit)
+              Seq(r.primaryMetricOr(metric).scoreUnit)
             ).flatten
           }.toVector
           val table = new TableFormatter(go).apply(columns, Vector(header1, header2, null) ++ data)
@@ -259,7 +268,7 @@ object Main extends Logging {
     case ex: PatternSyntaxException => logger.error(ex.toString)
   }
 
-  def createChart(go: GlobalOptions, runs: Seq[String], benchs: Seq[String], extract: Seq[String], regex: Boolean, scorePrecision: Int, pivot: Seq[String], template: Option[Path], out: Option[Path], cmdLine: Array[String]): Unit = {
+  def createChart(go: GlobalOptions, runs: Seq[String], benchs: Seq[String], extract: Seq[String], regex: Boolean, scorePrecision: Int, pivot: Seq[String], metric: Option[String], template: Option[Path], out: Option[Path], cmdLine: Array[String]): Unit = {
     new Global(go).use { g =>
       val multi = runs.size > 1
       val allRs = g.dao.run(g.dao.checkVersion andThen g.dao.queryResults(runs))
@@ -268,14 +277,14 @@ object Main extends Logging {
       val allParamNames = rs.flatMap(_.params.keys).distinct.toVector
       val pivotSet = pivot.toSet
       val paramNames = allParamNames.filterNot(pivotSet.contains)
-      val gen = new GenerateCharts(go, scorePrecision)
+      val gen = new GenerateCharts(go, scorePrecision, metric)
       val data =
         if(pivot.isEmpty) gen.generate(rs)
         else if(paramNames.length + pivotSet.size != allParamNames.length) {
           logger.error(s"Illegal pivot parameters.")
           ""
         } else {
-          val (pivoted, pivotSets) = RunResult.pivot(rs, pivot, paramNames)
+          val (pivoted, pivotSets) = RunResult.pivot(rs, pivot, paramNames, metric)
           gen.generatePivoted(pivoted, pivotSets, pivot, paramNames)
         }
       val templateHtml = template match {
diff --git a/core/src/main/scala/com/lightbend/benchdb/RunResult.scala b/core/src/main/scala/com/lightbend/benchdb/RunResult.scala
@@ -26,13 +26,22 @@ final class RunResult(val db: DbRunResult, val rc: Config, val runId: Long, val
   lazy val dbJvmArgs: Seq[DbJvmArg] = jvmArgs.zipWithIndex.map { case (s, idx) => new DbJvmArg(db.uuid, idx, s) }
   lazy val dbParams: Seq[DbRunResultParam] = params.iterator.map { case (k, v) => new DbRunResultParam(None, db.uuid, k, v) }.toSeq
 
-  lazy val primaryMetric: RunResult.Metric =
+  final def primaryMetricOr(secondary: Option[String]): RunResult.Metric = {
+    secondary match {
+      case Some(m) => secondaryMetrics(m)
+      case None => primaryMetric
+    }
+  }
+
+  private lazy val primaryMetric: RunResult.Metric =
     parseMetric(rc.getConfig("primaryMetric"))
 
-  lazy val secondaryMetrics: Map[String, RunResult.Metric] = {
-    val sc = rc.getConfig("secondaryMetrics")
-    sc.entrySet().asScala.iterator.map { me =>
-      (me.getKey, parseMetric(sc.getConfig(me.getKey)))
+  private lazy val secondaryMetrics: Map[String, RunResult.Metric] = {
+    val path = "secondaryMetrics"
+    val sc = rc.getConfig(path)
+    val keys = rc.getAnyRef(path).asInstanceOf[java.util.Map[String, _]].keySet()
+    keys.asScala.iterator.map { k =>
+      (k, parseMetric(sc.getConfig("\"" + k + "\"")))
     }.toMap
   }
 
@@ -172,7 +181,7 @@ object RunResult extends Logging {
     }
   }
 
-  def pivot(rs: Iterable[RunResult], pivot: Seq[String], other: Seq[String]): (Iterable[(RunResult, IndexedSeq[Option[RunResult]])], Seq[Seq[String]]) = {
+  def pivot(rs: Iterable[RunResult], pivot: Seq[String], other: Seq[String], metric: Option[String]): (Iterable[(RunResult, IndexedSeq[Option[RunResult]])], Seq[Seq[String]]) = {
     val pivotValues = pivot.map { p => rs.iterator.map(r => r.params.getOrElse(p, null)).toVector.distinct }
     val otherValues = other.map { p => rs.iterator.map(r => r.params.getOrElse(p, null)).toVector.distinct }
     def types(values: Seq[Seq[String]]) = values.map { ss =>
@@ -200,7 +209,7 @@ object RunResult extends Logging {
     val pivotColumns = valueCombinations(0, pivot.length).sorted(new ParamOrdering(pivotTypes))
 
     val fixedTypes = Seq((false, false), (false, false), (true, true), (false, false))
-    val groupsMap = rs.groupBy(r => Seq(r.name, r.db.mode, r.cnt.toString, r.primaryMetric.scoreUnit) ++ other.map(p => r.params.getOrElse(p, null)))
+    val groupsMap = rs.groupBy(r => Seq(r.name, r.db.mode, r.cnt.toString, r.primaryMetricOr(metric).scoreUnit) ++ other.map(p => r.params.getOrElse(p, null)))
     val groups = groupsMap.toSeq.sortBy(_._1)(new ParamOrdering(fixedTypes ++ otherTypes))
     //groups.foreach { case (groupParams, groupData) => println(s"group: $groupParams -> ${groupData.size}") }
     val grouped = groups.map { case (gr, data) =>

Original file line number	Diff line number	Diff line change
`@@ -3,7 +3,7 @@ package com.lightbend.benchdb`
`3`	`3`	`import scala.collection.JavaConverters._`
`4`	`4`	`import com.typesafe.config.{Config, ConfigFactory, ConfigObject, ConfigRenderOptions, ConfigValueFactory}`
`5`	`5`
`6`		`-class GenerateCharts(go: GlobalOptions, scorePrecision: Int) extends Logging {`
	`6`	`+class GenerateCharts(go: GlobalOptions, scorePrecision: Int, metric: Option[String]) extends Logging {`
`7`	`7`
`8`	`8`	`def generatePivoted(pivoted: Iterable[(RunResult, IndexedSeq[Option[RunResult]])],`
`9`	`9`	`pivotSets: Seq[Seq[String]], pivotParams: Seq[String], otherParams: Seq[String]): String = {`
`@@ -38,7 +38,7 @@ class GenerateCharts(go: GlobalOptions, scorePrecision: Int) extends Logging {`
`38`	`38`	`}`
`39`	`39`	`}).asJava`
`40`	`40`	`}.asJava`
`41`		`- renderChartData(n, pn, rs.head._1.primaryMetric.scoreUnit, lineChartOptionsBase, columns, rows)`
	`41`	`+ renderChartData(n, pn, vAxisTitle(rs.head._1.primaryMetricOr(metric).scoreUnit), lineChartOptionsBase, columns, rows)`
`42`	`42`	`}`
`43`	`43`	`}`
`44`	`44`	`}.mkString("[", ",", "]")`
`@@ -63,12 +63,17 @@ class GenerateCharts(go: GlobalOptions, scorePrecision: Int) extends Logging {`
`63`	`63`	`val rows = rs.zip(paramValues.map(_.get)).sortBy(_._2).map { case (r, pv) =>`
`64`	`64`	`(pv +: seriesRowData(s"$pn = $pv", r)).asJava`
`65`	`65`	`}.asJava`
`66`		`- renderChartData(n, pn, rs.head.primaryMetric.scoreUnit, lineChartOptionsBase, columns, rows)`
	`66`	`+ renderChartData(n, pn, vAxisTitle(rs.head.primaryMetricOr(metric).scoreUnit), lineChartOptionsBase, columns, rows)`
`67`	`67`	`}`
`68`	`68`	`}`
`69`	`69`	`}.mkString("[", ",", "]")`
`70`	`70`	`}`
`71`	`71`
	`72`	`+ private def vAxisTitle(unit: String) = metric match {`
	`73`	`+ case Some(x) => x + " " + unit`
	`74`	`+ case None => unit`
	`75`	`+ }`
	`76`	`+`
`72`	`77`	`private def renderChartData(title: String, hAxis: String, vAxis: String, baseConfig: Config, columns: Any, rows: Any): String = {`
`73`	`78`	`val options = optionsConfig(title, hAxis, vAxis, baseConfig)`
`74`	`79`	`val c = ConfigValueFactory.fromMap(Map("options" -> options, "columns" -> columns, "rows" -> rows).asJava)`
`@@ -91,10 +96,11 @@ class GenerateCharts(go: GlobalOptions, scorePrecision: Int) extends Logging {`
`91`	`96`	`)`
`92`	`97`
`93`	`98`	`private def seriesRowData(name: String, r: RunResult): Seq[Any] = {`
`94`		`- val score = r.primaryMetric.score`
`95`		`- val err = r.primaryMetric.scoreError`
	`99`	`+ val metric1 = r.primaryMetricOr(metric)`
	`100`	`+ val score = metric1.score`
	`101`	`+ val err = metric1.scoreError`
`96`	`102`	`val scoreS = ScoreFormatter(score, scorePrecision)`
`97`	`103`	`val errS = ScoreFormatter(err, scorePrecision)`
`98`		`- Seq(score, score-err, score+err, s"$name<br/>Score: <b>$scoreS</b> ± $errS ${r.primaryMetric.scoreUnit}")`
	`104`	`+ Seq(score, score-err, score+err, s"$name<br/>Score: <b>$scoreS</b> ± $errS ${metric1.scoreUnit}")`
`99`	`105`	`}`
`100`	`106`	`}`