Added Improvements to the Features and Messaging to the GC Infra re: ASP.NET Benchmarks (#3293)

* Added significant improvements to the ASP.NET Benchmarks

* Fixed up header for the analysis

* Fix for the downloadOutputOutput field

* Cosmetic improvement and an example of the new params

* Improved messaging from the analysis from

* The create suite command now should reference the entire folder rather than a single corerun.exe

* Fixed linting issues

* One more Lint hydra fix

* Fixed internal references in the markdown output

* Added example of how to use the new config

* Addressed feedback and added contextually named log files if the variables are specified

* Added the name of the file and the log file extension

* Added the analysis notebook

* Adjusted the csv files to be without spaces and modified the Linux and Generic runs

* Updated README to reflect changes

* Updated documentation

* Fixed md liniting issues

* Fixed up the notebook to work for both windows and linux

* Continued work on improving the usability of the analysis

* Added fix for certain cases where the log wasn't being downloaded

* implemented the volatility and difference comparisons

* Incorporated feedback, detection if the process timed out and notebook improvements

* Cleared output

* Updated plot for vol

* Added benchmark filtering and  fix path in the notebook

* Ordering in terms of filters and matches on ends + regexes

Author: mrsharm

src/benchmarks/gc/GC.Infrastructure/Configurations/ASPNetBenchmarks/ASPNetBenchmarks - Linux.csv

@@ -41,23 +41,14 @@ PlaintextPlatformInline_Windows, --config https://raw.githubusercontent.com/aspn
 JsonPlatform_Windows, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/platform.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/azure.profile.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario json --profile intel-win-app --profile intel-load2-load  --application.collectDependencies true --application.options.collectCounters true 
 PlaintextPlatform_Windows, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/platform.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/azure.profile.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario plaintext --profile intel-win-app --profile intel-load2-load --application.framework net8.0 --application.collectDependencies true --application.options.collectCounters true
 Stage1_Windows, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/goldilocks.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario basicminimalapivanilla --profile intel-win-app --profile intel-lin-load --profile amd-lin2-db --application.framework net8.0 --application.options.collectCounters true
-Stage1Aot_Windows, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/goldilocks.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario basicminimalapipublishaot --profile intel-win-app --profile intel-lin-load --profile amd-lin2-db --application.packageReferences "Microsoft.Dotnet.ILCompiler=$(MicrosoftNETCoreAppPackageVersion)" --application.framework net8.0 --application.options.collectCounters true
-Stage1GrpcAotServerGC_Windows, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/goldilocks.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario basicgrpcpublishaot --profile intel-win-app --profile intel-lin-load --application.packageReferences Microsoft.Dotnet.ILCompiler=$(MicrosoftNETCoreAppPackageVersion) --application.environmentVariables DOTNET_gcServer=1  --application.options.collectCounters true 
-Stage1GrpcAot_Windows, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/goldilocks.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario basicgrpcpublishaot --profile intel-win-app --profile intel-lin-load --application.packageReferences Microsoft.Dotnet.ILCompiler=$(MicrosoftNETCoreAppPackageVersion)  --application.options.collectCounters true 
 Stage1GrpcTrimR2RSingleFile_Windows, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/goldilocks.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario basicgrpcpublishtrimr2rsinglefile --profile intel-win-app --profile intel-lin-load  --application.options.collectCounters true 
 Stage1GrpcServerGC_Windows, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/goldilocks.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario basicgrpcvanilla --profile intel-win-app --profile intel-lin-load  --application.options.collectCounters true 
 Stage1Grpc_Windows, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/goldilocks.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario basicgrpcvanilla --profile intel-win-app --profile intel-lin-load  --application.options.collectCounters true 
-Stage2AotServerGC_Windows, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/goldilocks.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario todosapipublishaot --profile intel-win-app --profile intel-lin-load --profile amd-lin2-db --application.packageReferences Microsoft.Dotnet.ILCompiler=$(MicrosoftNETCoreAppPackageVersion) --application.buildArguments /p:ServerGarbageCollection=true  --application.options.collectCounters true 
-Stage2Aot_Windows, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/goldilocks.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario todosapipublishaot --profile intel-win-app --profile intel-lin-load --profile amd-lin2-db --application.packageReferences Microsoft.Dotnet.ILCompiler=$(MicrosoftNETCoreAppPackageVersion)  --application.options.collectCounters true 
 Stage2TrimR2RSingleFile_Windows, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/goldilocks.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario todosapipublishtrimr2rsinglefile --profile intel-win-app --profile intel-lin-load --profile amd-lin2-db  --application.options.collectCounters true 
 Stage2ServerGC_Windows, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/goldilocks.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario todosapivanilla --profile intel-win-app --profile intel-lin-load --profile amd-lin2-db --application.buildArguments /p:ServerGarbageCollection=true  --application.options.collectCounters true 
 Stage2_Windows, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/goldilocks.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario todosapivanilla --profile intel-win-app --profile intel-lin-load --profile amd-lin2-db  --application.options.collectCounters true 
-Stage1AotSpeedOpt_Windows, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/goldilocks.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario basicminimalapipublishaot --profile intel-win-app --profile intel-lin-load --profile amd-lin2-db --application.packageReferences Microsoft.Dotnet.ILCompiler=$(MicrosoftNETCoreAppPackageVersion) --application.buildArguments /p:OptimizationPreference=Speed  --application.options.collectCounters true 
-Stage1AotServerGC_Windows, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/goldilocks.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario basicminimalapipublishaot --profile intel-win-app --profile intel-lin-load --application.packageReferences Microsoft.Dotnet.ILCompiler=$(MicrosoftNETCoreAppPackageVersion) --application.environmentVariables DOTNET_gcServer=1  --application.options.collectCounters true 
 Stage1TrimR2RSingleFile_Windows, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/goldilocks.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario basicminimalapipublishtrimr2rsinglefile --profile intel-win-app --profile intel-lin-load  --application.options.collectCounters true 
 Stage1ServerGC_Windows, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/goldilocks.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario basicminimalapivanilla --profile intel-win-app --profile intel-lin-load --application.environmentVariables DOTNET_gcServer=1  --application.options.collectCounters true 
 Stage1GrpcPgo_Windows, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/goldilocks.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario basicgrpcvanilla --profile intel-win-app --profile intel-lin-load --application.environmentVariables DOTNET_TieredPGO=1  --application.options.collectCounters true 
 Stage2Pgo_Windows, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/goldilocks.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario todosapivanilla --profile intel-win-app --profile intel-lin-load --profile amd-lin2-db --application.environmentVariables DOTNET_TieredPGO=1  --application.options.collectCounters true 
-Stage1Pgo_Windows, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/goldilocks.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario basicminimalapivanilla --profile intel-win-app --profile intel-lin-load --application.environmentVariables DOTNET_TieredPGO=1  --application.options.collectCounters true 
-Stage1GrpcAotSpeedOpt_Windows, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/goldilocks.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario basicgrpcpublishaot --profile intel-win-app --profile intel-lin-load --profile amd-lin2-db --application.packageReferences Microsoft.Dotnet.ILCompiler=$(MicrosoftNETCoreAppPackageVersion) --application.buildArguments /p:OptimizationPreference=Speed  --application.options.collectCounters true 
-Stage2AotSpeedOpt_Windows, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/goldilocks.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario todosapipublishaot --profile intel-win-app --profile intel-lin-load --profile amd-lin2-db --application.packageReferences Microsoft.Dotnet.ILCompiler=$(MicrosoftNETCoreAppPackageVersion) --application.buildArguments /p:OptimizationPreference=Speed  --application.options.collectCounters true 
+Stage1Pgo_Windows, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/goldilocks.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario basicminimalapivanilla --profile intel-win-app --profile intel-lin-load --application.environmentVariables DOTNET_TieredPGO=1  --application.options.collectCounters true 

src/benchmarks/gc/GC.Infrastructure/Configurations/ASPNetBenchmarks/ASPNetBenchmarks - All.csv renamed to src/benchmarks/gc/GC.Infrastructure/Configurations/ASPNetBenchmarks/ASPNetBenchmarks-All.csv

@@ -0,0 +1,7 @@
+Legend,Base CommandLine
+Stage1Aot_Windows, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/goldilocks.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario basicminimalapipublishaot --profile intel-win-app --profile intel-lin-load --profile amd-lin2-db --application.packageReferences "Microsoft.Dotnet.ILCompiler=$(MicrosoftNETCoreAppPackageVersion)" --application.framework net8.0 --application.options.collectCounters true
+Stage1GrpcAot_Windows, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/goldilocks.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario basicgrpcpublishaot --profile intel-win-app --profile intel-lin-load --application.packageReferences Microsoft.Dotnet.ILCompiler=$(MicrosoftNETCoreAppPackageVersion)  --application.options.collectCounters true 
+Stage1AotSpeedOpt_Windows, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/goldilocks.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario basicminimalapipublishaot --profile intel-win-app --profile intel-lin-load --profile amd-lin2-db --application.packageReferences Microsoft.Dotnet.ILCompiler=$(MicrosoftNETCoreAppPackageVersion) --application.buildArguments /p:OptimizationPreference=Speed  --application.options.collectCounters true 
+Stage1GrpcAotSpeedOpt_Windows, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/goldilocks.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario basicgrpcpublishaot --profile intel-win-app --profile intel-lin-load --profile amd-lin2-db --application.packageReferences Microsoft.Dotnet.ILCompiler=$(MicrosoftNETCoreAppPackageVersion) --application.buildArguments /p:OptimizationPreference=Speed  --application.options.collectCounters true 
+Stage2Aot_Windows, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/goldilocks.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario todosapipublishaot --profile intel-win-app --profile intel-lin-load --profile amd-lin2-db --application.packageReferences Microsoft.Dotnet.ILCompiler=$(MicrosoftNETCoreAppPackageVersion)  --application.options.collectCounters true 
+Stage2AotSpeedOpt_Windows, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/goldilocks.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario todosapipublishaot --profile intel-win-app --profile intel-lin-load --profile amd-lin2-db --application.packageReferences Microsoft.Dotnet.ILCompiler=$(MicrosoftNETCoreAppPackageVersion) --application.buildArguments /p:OptimizationPreference=Speed  --application.options.collectCounters true 

src/benchmarks/gc/GC.Infrastructure/Configurations/ASPNetBenchmarks/ASPNetBenchmarks-Aot-Windows.csv

@@ -0,0 +1,4 @@
+Legend,Base CommandLine
+FortunesEf_Linux, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/database.benchmarks.yml --scenario fortunes_ef --application.framework net8.0 --application.options.collectCounters true --profile aspnet-citrine-lin --property os=linux --property arch=x64
+JsonMin_Linux, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/json.benchmarks.yml --scenario mapaction --application.framework net8.0 --application.options.collectCounters true --profile aspnet-citrine-lin --property os=linux --property arch=x64
+Stage1Grpc_Linux, crank --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/goldilocks.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario basicgrpcvanilla --profile intel-load2-app --profile amd-lin2-load --profile amd-lin2-db --application.options.collectCounters true

src/benchmarks/gc/GC.Infrastructure/Configurations/ASPNetBenchmarks/ASPNetBenchmarks-Linux.csv

@@ -1,2 +1,3 @@
 Legend,Base CommandLine
-Stage1Grpc_Linux, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/goldilocks.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario basicgrpcvanilla --profile intel-load2-app --profile amd-lin2-load --profile amd-lin2-db --application.options.collectCounters true
+JsonMvc_Windows, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/json.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/azure.profile.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario mvc --profile intel-win-app --profile intel-load2-load  --application.collectDependencies true --application.options.collectCounters true 
+FortunesEf_Windows, --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/database.benchmarks.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/azure.profile.yml --config https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml --scenario fortunes_ef --profile intel-win-app --profile intel-load-load --profile intel-db-db  --application.options.collectCounters true --application.collectDependencies true 

src/benchmarks/gc/GC.Infrastructure/Configurations/ASPNetBenchmarks/ASPNetBenchmarks.csv

@@ -3,15 +3,23 @@ runs:
     corerun: C:\CoreRuns\Empty\
     environment_variables:
       COMPlus_GCServer: 1
+    framework_version: net6.0
   run:
     corerun: C:\CoreRuns\Empty\
     environment_variables:
       COMPlus_GCServer: 1
 environment:
   environment_variables: {}
   default_max_seconds: 300
+  framework_version: net8.0
 benchmark_settings:
   benchmark_file: C:\InfraRuns\RunNew_All\Suites\ASPNETBenchmarks\ASPNetBenchmarks.csv
+  # To take a dump: --application.options.dumpType full --application.options.dumpOutput <filename>
+  # To fetch the build artifacts: --application.options.fetch true
+  additional_arguments: --chart --chart-type hex 
+  # Can optionally filter for specific benchmarks with a list of regexes.
+  # benchmark_filters:
+  # - Platform*
 output:
   path: C:\InfraRuns\RunNew_All\ASPNetBenchmarks
   columns: