Skip to content

Commit 0d68b65

Browse files
jsingletjsinglet
authored
Merge pull request #292 from github/jsinglet/include-perf-tools
import performance suite
2 parents 015fc83 + de71869 commit 0d68b65

8 files changed

+739
-0
lines changed

‎scripts/performance_testing/Config.ps1

+4
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
Import-Module -Name "$PSScriptRoot/../PSCodingStandards/CodingStandards"
2+
3+
$REQUIRED_CODEQL_VERSION = (Get-Content (Join-Path (Get-RepositoryRoot) "supported_codeql_configs.json") | ConvertFrom-Json).supported_environment.codeql_cli
4+

‎scripts/performance_testing/Convert-DurationStringToMs.ps1

+77
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
function Convert-DurationStringToMs {
2+
param(
3+
[Parameter(Mandatory)]
4+
[string]
5+
$DurationString
6+
)
7+
8+
$durationStack = @()
9+
$unitStack = @()
10+
11+
12+
$durationBuff = $false
13+
$unitBuff = $false
14+
15+
for($i=0; $i -le $DurationString.Length; $i++){
16+
$s = $DurationString[$i]
17+
#Write-Host $s
18+
if($s -match "\d|\."){ # consume if it is a number or a decimal
19+
20+
# init buffer
21+
if($durationBuff -eq $false){
22+
$durationBuff = ""
23+
}
24+
25+
# accept last unit
26+
if(-Not $unitBuff -eq $false){
27+
$unitStack += $unitBuff
28+
$unitBuff = $false
29+
}
30+
31+
$durationBuff += $s
32+
}else{ # otherwise it is a unit -- multiply by it to get the ms.
33+
34+
# init buffer
35+
if($unitBuff -eq $false){
36+
$unitBuff = ""
37+
}
38+
39+
# accept last digit buffer
40+
if(-Not $durationBuff -eq $false){
41+
$durationStack += $durationBuff
42+
$durationBuff = $false
43+
}
44+
45+
$unitBuff += $s
46+
}
47+
}
48+
49+
# should always end with accepting the last one (because it will be a
50+
# unit)
51+
$unitStack += $unitBuff
52+
53+
$totalMs = 0
54+
55+
for($i=0; $i -le $unitStack.Length; $i++){
56+
57+
$time = [System.Convert]::ToDecimal($durationStack[$i])
58+
$unit = $unitStack[$i]
59+
60+
if($unit -eq 'h'){
61+
$time = $time * (60*60*1000)
62+
}
63+
if($unit -eq 'm'){
64+
$time = $time * (60*1000)
65+
}
66+
if($unit -eq 's'){
67+
$time = $time * (1000)
68+
}
69+
if($unit -eq 'ms'){
70+
$time = $time
71+
}
72+
73+
$totalMs += $time
74+
}
75+
76+
return $totalMs
77+
}

‎scripts/performance_testing/Get-DurationString.ps1

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
function Get-DurationString {
2+
param(
3+
[Parameter(Mandatory)]
4+
[string]
5+
$LogLine
6+
)
7+
$In = $LogLine.IndexOf('eval')+5
8+
$Out = $LogLine.indexof(']')
9+
10+
return $LogLine.substring($In, $Out - $In)
11+
}
12+

‎scripts/performance_testing/Get-QueryString.ps1

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
function Get-QueryString {
2+
param(
3+
[Parameter(Mandatory)]
4+
[string]
5+
$LogLine
6+
)
7+
$In = $LogLine.IndexOf('Evaluation done; writing results to ')+36
8+
$Out = $LogLine.IndexOf('.bqrs')
9+
10+
return $LogLine.SubString($In, $Out - $In)
11+
}
12+

‎scripts/performance_testing/Get-TestTmpDirectory.ps1

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
function Get-TestTmpDirectory {
2+
$Dir = [System.IO.Path]::GetTempPath()
3+
return Join-Path $Dir "$([System.Guid]::NewGuid())"
4+
}
5+

‎scripts/performance_testing/README.md

+213
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,213 @@
1+
# Performance Testing
2+
3+
Performance testing may be accomplished by using the performance testing tool found in this directory, `Test-ReleasePerformance.ps1`. These results may be further processed to provide predicate level performance details by using the script `profile_predicates.py`, which is documented in the [Profiling Predicates section.](#profiling-predicates), below.
4+
5+
Note that this script depends on other files from this repository. It may be run on external builds of Coding Standards through the `-CodingStandardsPath` flag, but it should be run from a fresh checkout of this repository.
6+
7+
This script requires `pwsh` to be installed. Note that the Windows native Powershell is not sufficient and you should download PowerShell Core.
8+
9+
- Installing on Windows: https://learn.microsoft.com/en-us/powershell/scripting/install/installing-powershell-on-windows?view=powershell-7.3
10+
- Installing on Linux: https://learn.microsoft.com/en-us/powershell/scripting/install/installing-powershell-on-linux?view=powershell-7.3
11+
- Installing on MacOS: https://learn.microsoft.com/en-us/powershell/scripting/install/installing-powershell-on-macos?view=powershell-7.3
12+
13+
Before invoking this script you should start a powershell session by typing `pwsh` at a command prompt.
14+
15+
## Usage
16+
17+
```
18+
NAME
19+
.\scripts\performance_testing\Test-ReleasePerformance.ps1
20+
21+
SYNOPSIS
22+
Test release performance. Generates outputs 2 csv files containing the slowest predicates as well as the queries
23+
causing work. Note that the method of computing query execution time is inaccurate due to the shared nature of
24+
predicates.
25+
26+
27+
SYNTAX
28+
C:\Projects\codeql-coding-standards\scripts\performance_testing\Test-ReleasePerformance.ps1 -RunTests [-Threads <String>] -DatabaseArchive <String>
29+
[-TestTimestamp <String>] [-CodingStandardsPath <String>] [-ResultsDirectory <String>] [-ReleaseTag <String>] -Suite <String> [-Platform <String>] -Language
30+
<String> [<CommonParameters>]
31+
32+
C:\Projects\codeql-coding-standards\scripts\performance_testing\Test-ReleasePerformance.ps1 -ProcessResults -ResultsFile <String> [-ResultsDirectory <String>]
33+
[-ReleaseTag <String>] -Suite <String> [-Platform <String>] -Language <String> [<CommonParameters>]
34+
35+
36+
DESCRIPTION
37+
Test release performance. Generates outputs 2 csv files containing the slowest predicates as well as the queries
38+
causing work. Note that the method of computing query execution time is inaccurate due to the shared nature of
39+
predicates.
40+
41+
42+
PARAMETERS
43+
-RunTests [<SwitchParameter>]
44+
Configures tool to run tests.
45+
46+
Required? true
47+
Position? named
48+
Default value False
49+
Accept pipeline input? false
50+
Accept wildcard characters? false
51+
52+
-Threads <String>
53+
Specifies the number of threads to use.
54+
55+
Required? false
56+
Position? named
57+
Default value 5
58+
Accept pipeline input? false
59+
Accept wildcard characters? false
60+
61+
-DatabaseArchive <String>
62+
Specifies the database to use for testing. Should be a zipped database
63+
directory.
64+
65+
Required? true
66+
Position? named
67+
Default value
68+
Accept pipeline input? false
69+
Accept wildcard characters? false
70+
71+
-TestTimestamp <String>
72+
The timestamp to use for the test.
73+
74+
Required? false
75+
Position? named
76+
Default value (Get-Date -Format "yyyy-MM-dd_HH-mm-ss")
77+
Accept pipeline input? false
78+
Accept wildcard characters? false
79+
80+
-CodingStandardsPath <String>
81+
The path to the coding standards root directory. This can be either the
82+
root of the repository or the root of the coding standards directory.
83+
84+
Required? false
85+
Position? named
86+
Default value "$PSScriptRoot../../"
87+
Accept pipeline input? false
88+
Accept wildcard characters? false
89+
90+
-ProcessResults [<SwitchParameter>]
91+
92+
Required? true
93+
Position? named
94+
Default value False
95+
Accept pipeline input? false
96+
Accept wildcard characters? false
97+
98+
-ResultsFile <String>
99+
Configures tool to process results.
100+
101+
Required? true
102+
Position? named
103+
Default value
104+
Accept pipeline input? false
105+
Accept wildcard characters? false
106+
107+
-ResultsDirectory <String>
108+
Where results should be written to.
109+
110+
Required? false
111+
Position? named
112+
Default value (Get-Location)
113+
Accept pipeline input? false
114+
Accept wildcard characters? false
115+
116+
-ReleaseTag <String>
117+
The release tag to use for the test.
118+
119+
Required? false
120+
Position? named
121+
Default value current
122+
Accept pipeline input? false
123+
Accept wildcard characters? false
124+
125+
-Suite <String>
126+
Which suite to run.
127+
128+
Required? true
129+
Position? named
130+
Default value
131+
Accept pipeline input? false
132+
Accept wildcard characters? false
133+
134+
-Platform <String>
135+
The platform to run on. This is just a descriptive string.
136+
137+
Required? false
138+
Position? named
139+
Default value $PSVersionTable.Platform
140+
Accept pipeline input? false
141+
Accept wildcard characters? false
142+
143+
-Language <String>
144+
The language to run on.
145+
146+
Required? true
147+
Position? named
148+
Default value
149+
Accept pipeline input? false
150+
Accept wildcard characters? false
151+
152+
<CommonParameters>
153+
This cmdlet supports the common parameters: Verbose, Debug,
154+
ErrorAction, ErrorVariable, WarningAction, WarningVariable,
155+
OutBuffer, PipelineVariable, and OutVariable. For more information, see
156+
about_CommonParameters (https://go.microsoft.com/fwlink/?LinkID=113216).
157+
158+
INPUTS
159+
160+
OUTPUTS
161+
162+
163+
RELATED LINKS
164+
165+
```
166+
## Example Usage
167+
168+
Run the `cert` suite for `c` from within the Coding Standards repository.
169+
170+
```
171+
.\scripts\performance_testing\Test-ReleasePerformance.ps1 -RunTests -DatabaseArchive ..\codeql-coding-standards-release-engineering\data\commaai-openpilot-72d1744d830bc249d8761a1d843a98fb0ced49fe-cpp.zip -Suite cert -Language c
172+
```
173+
174+
Run the `cert` suite for `c` on an external release, specifying a `-ReleaseTag` as well. The `-ReleaseTag` parameter is used for configuring performance tool to generate files within subdirectories with the `-ReleaseTag` as a prefix. For example, specifying `-ReleaseTag "2.16.0"` will cause files to be generated in the `release=2.16.0` directory.
175+
176+
```
177+
.\scripts\performance_testing\Test-ReleasePerformance.ps1 -RunTests -DatabaseArchive ..\codeql-coding-standards-release-engineering\data\commaai-openpilot-72d1744d830bc249d8761a1d843a98fb0ced49fe-cpp.zip -Suite cert -Language c -ReleaseTag "2.16.0" -CodingStandardsPath "Downloads\code-scanning-cpp-query-pack-2.16.0\codeql-coding-standards\"
178+
```
179+
180+
181+
182+
## Outputs
183+
184+
The `Test-ReleasePerformance.ps1` produces three files in the `ResultsDirectory` location, which defaults `performance_tests` within the current working directory.
185+
186+
- `suite=$Suite,datum=queries.csv` - Which contains the run time for each query.
187+
- `suite=$Suite,datum=evaluator-log.json` - Which contains the evaluator log.
188+
- `suite=$Suite,datum=sarif.sarif` - The sarif log file for the run.
189+
190+
## Profiling Predicates
191+
192+
If you wish to extract predicate-level profiling information, you may use the script `profile_predicates.py` located in this directory. It requires Python3 with `pandas` and `numpy` to work. If you wish to use a virtual environment you may create one as follows on a Unix-based platform:
193+
194+
```
195+
python -mvenv venv
196+
source venv/bin/activate
197+
pip install pandas numpy
198+
```
199+
200+
The script works by summarizing ALL of the csv and json files within a given directory. Thus, if you want to profile multiple suites or multiple releases you may place the files within that directory by repeatedly invoking `Test-ReleasePerformance.ps1.` Make sure to supply the same output directory each time so that the results accumulate in the correct location.
201+
202+
To invoke the script run:
203+
204+
```
205+
python scripts/performance_testing/profile_predicates.py <path to output directory>
206+
```
207+
208+
For example:
209+
```
210+
python .\scripts\performance_testing\profile_predicates.py .\performance_tests\
211+
```
212+
213+
This will produce an additional CSV file per release, platform, and language within that directory called: `slow-log,datum=predicates,release={release},platform={platform},language={language}.csv` which will contain the execution times of all of the predicates used during execution.

0 commit comments

Comments
 (0)