-
Notifications
You must be signed in to change notification settings - Fork 470
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
adding scripts to easily run and compare results
- Loading branch information
Showing
23 changed files
with
1,182 additions
and
7 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -16,6 +16,7 @@ obj/ | |
| .dotnet/ | ||
| .tools/ | ||
| .packages/ | ||
| temp/ | ||
|
|
||
| # Per-user project properties | ||
| launchSettings.json | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,3 @@ | ||
| @echo off | ||
| powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0eng\perf\PerfCore.ps1""" %*" | ||
| exit /b %ErrorLevel% |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,3 @@ | ||
| @echo off | ||
| powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0PerfCore.ps1""" -v diag -etl -diff %*" | ||
| exit /b %ErrorLevel% |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,38 @@ | ||
| [CmdletBinding(PositionalBinding=$false)] | ||
| Param( | ||
| [String] $baseline, # folder that contains the baseline results | ||
| [String] $results # folder that contains the performance results | ||
| ) | ||
|
|
||
| function EnsureFolder { | ||
| param ( | ||
| [String] $path | ||
| ) | ||
| If(!(test-path $path)) | ||
| { | ||
| New-Item -ItemType Directory -Force -Path $path | ||
| } | ||
| } | ||
|
|
||
| $currentLocation = Get-Location | ||
| try { | ||
|
|
||
| #Get result files | ||
| $baselineFolder = Join-Path $baseline "results" | ||
|
|
||
| $resultsFolder = Join-Path $results "results" | ||
|
|
||
| $RepoRoot = Resolve-Path (Join-Path $PSScriptRoot '..\..') | ||
| $perfDiff = Join-Path $RepoRoot "src\Tools\PerfDiff\PerfDiff.csproj" | ||
| Invoke-Expression "dotnet run -c Release --project $perfDiff -- --baseline $baselineFolder --results $resultsFolder --failOnRegression --verbosity diag" | ||
| } | ||
| catch { | ||
| Write-Host $_ | ||
| Write-Host $_.Exception | ||
| Write-Host $_.ScriptStackTrace | ||
| $host.SetShouldExit(1) | ||
| exit | ||
| } | ||
| finally { | ||
| Set-Location $currentLocation | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,86 @@ | ||
| [CmdletBinding(PositionalBinding=$false)] | ||
| Param( | ||
| [String] $baselineSHA, # git SHA to use as the baseline for performance | ||
| [String] $testSHA, # git SHA to use as the test for performance | ||
| [String] $output, # common folder to write the benchmark results to | ||
| [string] $projects, # semicolon separated list of relative paths to benchmark projects to run | ||
| [string] $filter, # filter for tests to run (supports wildcards) | ||
| [switch] $etl, # capture etl traces for performance tests | ||
| [switch] $ci # run in ci mode (fail fast an keep all partial artifacts) | ||
| ) | ||
|
|
||
| function EnsureFolder { | ||
| param ( | ||
| [String] $path # path to create if it does not exist | ||
| ) | ||
| If(!(test-path $path)) | ||
| { | ||
| New-Item -ItemType Directory -Force -Path $path | ||
| } | ||
| } | ||
|
|
||
| function RunTest { | ||
| param ( | ||
| [String] $RunPerfTests, # path to the RunPerfTests.ps1 script | ||
| [String] $output, # common folder to write the benchmark results to | ||
| [String] $resultsFolder, # folder name to write the benchmark results to | ||
| [String] $temp, # root folder to check repos into | ||
| [String] $repoName, # folder name to place checked-out repo in | ||
| [String] $repoSHA, # git SHA run performance tests against | ||
| [String] $projects, # semicolon separated list of relative paths to benchmark projects to run | ||
| [String] $filter, # filter for tests to run (supports wildcards) | ||
| [bool] $etl, # capture etl traces for performance tests | ||
| [bool] $ci # run in ci mode (fail fast an keep all partial artifacts) | ||
| ) | ||
|
|
||
| # Ensure output directory has been created | ||
| $resultsOutput = Join-Path $output $resultsFolder | ||
| EnsureFolder $resultsOutput | ||
|
|
||
| # Checkout SHA | ||
| $repoFolder = Join-Path $Temp $repoName | ||
| Invoke-Expression "git worktree add $repoFolder $repoSHA" | ||
|
|
||
| $commandArgs = "-root $repoFolder -output $resultsOutput -projects $projects -filter $filter" | ||
| if ($etl) { | ||
| $commandArgs = "$commandArgs -etl" | ||
| } | ||
|
|
||
| if ($ci) { | ||
| $commandArgs = "$commandArgs -ci" | ||
| } | ||
|
|
||
| Invoke-Expression "$RunPerfTests $commandArgs" | ||
| } | ||
|
|
||
| $currentLocation = Get-Location | ||
|
|
||
| # Setup paths | ||
| $RepoRoot = Resolve-Path (Join-Path $PSScriptRoot '..\..') | ||
| $RunPerfTests = Join-Path $PSScriptRoot "RunPerfTests.ps1" | ||
| $ComparePerfResults = Join-Path $PSScriptRoot "ComparePerfResults.ps1" | ||
| $Temp = Join-Path $RepoRoot "temp" | ||
|
|
||
| try { | ||
| # Get baseline results | ||
| RunTest -RunPerfTests $RunPerfTests -output $output -resultsFolder "baseline" -temp $Temp -repoName "perfBaseline" -repoSHA $baselineSHA -projects $projects -filter $filter -etl $etl.IsPresent -ci $ci.IsPresent | ||
|
|
||
| # Get perf results | ||
| RunTest -RunPerfTests $RunPerfTests -output $output -resultsFolder "perfTest" -temp $Temp -repoName "perfTest" -repoSHA $testSHA -projects $projects -filter $filter -etl $etl.IsPresent -ci $ci.IsPresent | ||
|
|
||
| # Diff perf results | ||
| Invoke-Expression "$ComparePerfResults -baseline $baselineOutput -results $testOutput" | ||
| } | ||
| catch { | ||
| Write-Host $_ | ||
| Write-Host $_.Exception | ||
| Write-Host $_.ScriptStackTrace | ||
| $host.SetShouldExit(1) | ||
| exit | ||
| } | ||
| finally { | ||
| Invoke-Expression 'git worktree remove perfBaseline' | ||
| Invoke-Expression 'git worktree remove perfTest' | ||
| Invoke-Expression 'git worktree prune' | ||
| Set-Location $currentLocation | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,86 @@ | ||
| [CmdletBinding(PositionalBinding=$false)] | ||
| Param( | ||
| [string] $projects, | ||
| [string][Alias('v')]$verbosity = "minimal", | ||
| [string] $filter, | ||
| [switch] $etl, | ||
| [switch] $diff, | ||
| [switch] $ci, | ||
| [switch] $help, | ||
| [Parameter(ValueFromRemainingArguments=$true)][String[]]$properties | ||
| ) | ||
|
|
||
| function Print-Usage() { | ||
| Write-Host "Common settings:" | ||
| Write-Host " -verbosity <value> Msbuild verbosity: q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic] (short: -v)" | ||
| Write-Host " -help Print help and exit" | ||
| Write-Host "" | ||
|
|
||
| Write-Host "Actions:" | ||
| Write-Host " -diff Compare HEAD to baseline perf results" | ||
| Write-Host "" | ||
|
|
||
| Write-Host "Advanced settings:" | ||
| Write-Host " -etl Capture ETL traces of performance tests (requires admin permissions, default value is 'false')" | ||
| Write-Host " -filter Filter for tests to run (supports wildcards)" | ||
| Write-Host " -projects <value> Semi-colon delimited list of relative paths to benchmark projects." | ||
| Write-Host "" | ||
|
|
||
| Write-Host "Command line arguments not listed above are passed thru to msbuild." | ||
| Write-Host "The above arguments can be shortened as much as to be unambiguous (e.g. -co for configuration, -t for test, etc.)." | ||
| } | ||
|
|
||
| try { | ||
| if ($help -or (($null -ne $properties) -and ($properties.Contains('/help') -or $properties.Contains('/?')))) { | ||
| Print-Usage | ||
| exit 0 | ||
| } | ||
|
|
||
| if ([string]::IsNullOrWhiteSpace($projects)) { | ||
| $projects = "src\PerformanceTests\Tests\PerformanceTests.csproj" | ||
| } | ||
|
|
||
| if ([string]::IsNullOrWhiteSpace($filter)) { | ||
| $filter = "*" | ||
| } | ||
|
|
||
| $RepoRoot = Resolve-Path (Join-Path $PSScriptRoot '..\..') | ||
| $output = Join-Path $RepoRoot "artifacts\performance\perfResults" | ||
|
|
||
| # Diff two different SHAs | ||
| if ($diff) { | ||
| $DiffPerfToBaseLine = Join-Path $RepoRoot "eng\perf\DiffPerfToBaseLine.ps1" | ||
| $baselinejson = Get-Content -Raw -Path (Join-Path $RepoRoot "eng\perf\baseline.json") | ConvertFrom-Json | ||
| $baselineSHA = $baselinejson.sha | ||
| $commandArguments = "-baselineSHA $baselineSHA -testSHA HEAD -projects '$projects' -output '$output' -filter '$filter'" | ||
| if ($etl) { | ||
| $commandArguments = "$commandArguments -etl" | ||
| } | ||
| if ($ci) { | ||
| $commandArguments = "$commandArguments -ci" | ||
| } | ||
| Invoke-Expression "$DiffPerfToBaseLine $commandArguments" | ||
| exit | ||
| } | ||
|
|
||
| $commandArguments = "-projects '$projects' -root '$RepoRoot' -output '$output\perfTest' -filter '$filter'" | ||
| if ($etl) { | ||
| $commandArguments = "$commandArguments -etl" | ||
| } | ||
| if ($ci) { | ||
| $commandArguments = "$commandArguments -ci" | ||
| } | ||
|
|
||
| $RunPerfTests = Join-Path $RepoRoot "eng\perf\RunPerfTests.ps1" | ||
| Invoke-Expression "$RunPerfTests $commandArguments" | ||
| exit | ||
| } | ||
| catch { | ||
| Write-Host $_ | ||
| Write-Host $_.Exception | ||
| Write-Host $_.ScriptStackTrace | ||
| $host.SetShouldExit(1) | ||
| exit | ||
| } | ||
| finally { | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,35 @@ | ||
| [CmdletBinding(PositionalBinding=$false)] | ||
| Param( | ||
| [string] $projects, # semicolon separated list of relative paths to benchmark projects to run | ||
| [string] $filter, # filter for tests to run (supports wildcards) | ||
| [String] $root, # root folder all of the benchmark projects share | ||
| [String] $output, # folder to write the benchmark results to | ||
| [switch] $etl, # capture etl traces for performance tests | ||
| [switch] $ci # run in ci mode (fail fast an keep all partial artifacts) | ||
| ) | ||
|
|
||
| try { | ||
| $projectsList = $projects -split ";" | ||
| foreach ($project in $projectsList){ | ||
| $projectFullPath = Join-Path $root $project | ||
| $comandArguments = "run -c Release --project $projectFullPath -- --memory --exporters JSON --artifacts $output" | ||
| if ($ci) { | ||
| $comandArguments = "$comandArguments --stopOnFirstError --keepFiles" | ||
| } | ||
| if ($etl) { | ||
| Start-Process -Wait -FilePath "dotnet" -Verb RunAs -ArgumentList "$comandArguments --profiler ETW --filter $filter" | ||
| } | ||
| else { | ||
| Invoke-Expression "dotnet $comandArguments --filter $filter" | ||
| } | ||
| } | ||
| } | ||
| catch { | ||
| Write-Host $_ | ||
| Write-Host $_.Exception | ||
| Write-Host $_.ScriptStackTrace | ||
| $host.SetShouldExit(1) | ||
| exit | ||
| } | ||
| finally { | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,5 @@ | ||
| { | ||
| "release": "6.0", | ||
| "label": "roslyn analyzers baseline for main", | ||
| "sha": "f986c7b7aff265feb7536e1c7f98efcb9f14e077" | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,8 @@ | ||
| ### Configuration for PublicAPI analyzers executed on this folder ### | ||
| [*.{cs,vb}] | ||
|
|
||
| dotnet_diagnostic.IDE1030.severity = none | ||
| dotnet_diagnostic.CA1052.severity = none | ||
| dotnet_diagnostic.CA1063.severity = none | ||
| dotnet_diagnostic.CA1724.severity = none | ||
| dotnet_diagnostic.CA1819.severity = none |
Oops, something went wrong.