diff --git a/.gitignore b/.gitignore
index 2a0a37eae9..8c87aa3f57 100644
--- a/.gitignore
+++ b/.gitignore
@@ -16,6 +16,7 @@ obj/
.dotnet/
.tools/
.packages/
+temp/
# Per-user project properties
launchSettings.json
diff --git a/Perf.cmd b/Perf.cmd
new file mode 100644
index 0000000000..76a5e3b5f0
--- /dev/null
+++ b/Perf.cmd
@@ -0,0 +1,3 @@
+@echo off
+powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0eng\perf\PerfCore.ps1""" %*"
+exit /b %ErrorLevel%
\ No newline at end of file
diff --git a/RoslynAnalyzers.sln b/RoslynAnalyzers.sln
index 550d09e20e..7e72ce605d 100644
--- a/RoslynAnalyzers.sln
+++ b/RoslynAnalyzers.sln
@@ -1,4 +1,5 @@
-Microsoft Visual Studio Solution File, Format Version 12.00
+
+Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio Version 16
VisualStudioVersion = 16.0.28531.58
MinimumVisualStudioVersion = 10.0.40219.1
@@ -188,11 +189,11 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CommonPerfUtilities", "src\
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "VisualBasicPerfUtilities", "src\PerformanceTests\Utilities\VisualBasic\VisualBasicPerfUtilities.csproj", "{8D764DDE-3762-4218-8BC4-CC1048095D5F}"
EndProject
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{E1A6F359-DF41-4E51-9D4F-2E6DDF135BA7}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "PerformanceTests", "src\PerformanceTests\Tests\PerformanceTests.csproj", "{4FE82B4D-4AF1-4076-9E85-529205866A80}"
EndProject
-Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "PerformanceTests", "PerformanceTests", "{9CD62A3C-A138-4607-9231-5E2266AB59BC}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "PerfDiff", "src\Tools\PerfDiff\PerfDiff.csproj", "{AE549290-1702-4ECE-AE57-5FF4E85492C6}"
EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PerformanceTests", "src\PerformanceTests\Tests\PerformanceTests.csproj", "{4FE82B4D-4AF1-4076-9E85-529205866A80}"
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "PerfDiff", "PerfDiff", "{BEE4B62A-86BD-4613-8F94-F40364350643}"
EndProject
Global
GlobalSection(SharedMSBuildProjectFiles) = preSolution
@@ -498,6 +499,10 @@ Global
{4FE82B4D-4AF1-4076-9E85-529205866A80}.Debug|Any CPU.Build.0 = Debug|Any CPU
{4FE82B4D-4AF1-4076-9E85-529205866A80}.Release|Any CPU.ActiveCfg = Release|Any CPU
{4FE82B4D-4AF1-4076-9E85-529205866A80}.Release|Any CPU.Build.0 = Release|Any CPU
+ {AE549290-1702-4ECE-AE57-5FF4E85492C6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {AE549290-1702-4ECE-AE57-5FF4E85492C6}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {AE549290-1702-4ECE-AE57-5FF4E85492C6}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {AE549290-1702-4ECE-AE57-5FF4E85492C6}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
@@ -579,8 +584,9 @@ Global
{6A3B1816-97D3-4D30-8C6F-2F2DC288410B} = {43852421-59D7-4D63-977E-F862E1CF467D}
{8EC7A06A-01F2-49CF-9C45-4357AAD82546} = {43852421-59D7-4D63-977E-F862E1CF467D}
{8D764DDE-3762-4218-8BC4-CC1048095D5F} = {43852421-59D7-4D63-977E-F862E1CF467D}
- {9CD62A3C-A138-4607-9231-5E2266AB59BC} = {E1A6F359-DF41-4E51-9D4F-2E6DDF135BA7}
- {4FE82B4D-4AF1-4076-9E85-529205866A80} = {9CD62A3C-A138-4607-9231-5E2266AB59BC}
+ {4FE82B4D-4AF1-4076-9E85-529205866A80} = {CE87A0B7-E19B-4230-A73C-60F6D65EB7DC}
+ {AE549290-1702-4ECE-AE57-5FF4E85492C6} = {BEE4B62A-86BD-4613-8F94-F40364350643}
+ {BEE4B62A-86BD-4613-8F94-F40364350643} = {C0B86774-8307-444F-9EE4-98D62C3424F9}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {FC44ACA9-AEA3-4EE6-881C-2E08ED281B5F}
diff --git a/eng/Versions.props b/eng/Versions.props
index 595f268343..22db0f3b36 100644
--- a/eng/Versions.props
+++ b/eng/Versions.props
@@ -42,14 +42,20 @@
1.0.1-beta1.21202.2
- 0.12.1
+ 0.13.0
2.2.0
1.1.2
+ 2.0.69
+ 6.0.0-preview.5.21301.5
6.0.0-preview.4.21253.7
10.1.0
16.1.8
+ 12.0.1
+ 0.2.1
1.1.2
1.3.1
+ 2.0.0-beta1.20074.1
+ 2.0.0-beta1.21216.1
4.7.0
4.7.0
1.4.2
diff --git a/eng/perf/CIPerf.cmd b/eng/perf/CIPerf.cmd
new file mode 100644
index 0000000000..88ca610edc
--- /dev/null
+++ b/eng/perf/CIPerf.cmd
@@ -0,0 +1,3 @@
+@echo off
+powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0PerfCore.ps1""" -v diag -etl -diff %*"
+exit /b %ErrorLevel%
\ No newline at end of file
diff --git a/eng/perf/ComparePerfResults.ps1 b/eng/perf/ComparePerfResults.ps1
new file mode 100644
index 0000000000..79363b11c3
--- /dev/null
+++ b/eng/perf/ComparePerfResults.ps1
@@ -0,0 +1,38 @@
+[CmdletBinding(PositionalBinding=$false)]
+Param(
+ [String] $baseline, # folder that contains the baseline results
+ [String] $results # folder that contains the performance results
+ )
+
+function EnsureFolder {
+param (
+ [String] $path
+)
+ If(!(test-path $path))
+ {
+ New-Item -ItemType Directory -Force -Path $path
+ }
+}
+
+$currentLocation = Get-Location
+try {
+
+ #Get result files
+ $baselineFolder = Join-Path $baseline "results"
+
+ $resultsFolder = Join-Path $results "results"
+
+ $RepoRoot = Resolve-Path (Join-Path $PSScriptRoot '..\..')
+ $perfDiff = Join-Path $RepoRoot "src\Tools\PerfDiff\PerfDiff.csproj"
+ Invoke-Expression "dotnet run -c Release --project $perfDiff -- --baseline $baselineFolder --results $resultsFolder --failOnRegression --verbosity diag"
+}
+catch {
+ Write-Host $_
+ Write-Host $_.Exception
+ Write-Host $_.ScriptStackTrace
+ $host.SetShouldExit(1)
+ exit
+}
+finally {
+ Set-Location $currentLocation
+}
\ No newline at end of file
diff --git a/eng/perf/DiffPerfToBaseLine.ps1 b/eng/perf/DiffPerfToBaseLine.ps1
new file mode 100644
index 0000000000..20f58f32b5
--- /dev/null
+++ b/eng/perf/DiffPerfToBaseLine.ps1
@@ -0,0 +1,86 @@
+[CmdletBinding(PositionalBinding=$false)]
+Param(
+ [String] $baselineSHA, # git SHA to use as the baseline for performance
+ [String] $testSHA, # git SHA to use as the test for performance
+ [String] $output, # common folder to write the benchmark results to
+ [string] $projects, # semicolon separated list of relative paths to benchmark projects to run
+ [string] $filter, # filter for tests to run (supports wildcards)
+ [switch] $etl, # capture etl traces for performance tests
+ [switch] $ci # run in ci mode (fail fast an keep all partial artifacts)
+ )
+
+function EnsureFolder {
+ param (
+ [String] $path # path to create if it does not exist
+ )
+ If(!(test-path $path))
+ {
+ New-Item -ItemType Directory -Force -Path $path
+ }
+}
+
+function RunTest {
+ param (
+ [String] $RunPerfTests, # path to the RunPerfTests.ps1 script
+ [String] $output, # common folder to write the benchmark results to
+ [String] $resultsFolder, # folder name to write the benchmark results to
+ [String] $temp, # root folder to check repos into
+ [String] $repoName, # folder name to place checked-out repo in
+ [String] $repoSHA, # git SHA run performance tests against
+ [String] $projects, # semicolon separated list of relative paths to benchmark projects to run
+ [String] $filter, # filter for tests to run (supports wildcards)
+ [bool] $etl, # capture etl traces for performance tests
+ [bool] $ci # run in ci mode (fail fast an keep all partial artifacts)
+ )
+
+ # Ensure output directory has been created
+ $resultsOutput = Join-Path $output $resultsFolder
+ EnsureFolder $resultsOutput
+
+ # Checkout SHA
+ $repoFolder = Join-Path $Temp $repoName
+ Invoke-Expression "git worktree add $repoFolder $repoSHA"
+
+ $commandArgs = "-root $repoFolder -output $resultsOutput -projects $projects -filter $filter"
+ if ($etl) {
+ $commandArgs = "$commandArgs -etl"
+ }
+
+ if ($ci) {
+ $commandArgs = "$commandArgs -ci"
+ }
+
+ Invoke-Expression "$RunPerfTests $commandArgs"
+}
+
+$currentLocation = Get-Location
+
+# Setup paths
+$RepoRoot = Resolve-Path (Join-Path $PSScriptRoot '..\..')
+$RunPerfTests = Join-Path $PSScriptRoot "RunPerfTests.ps1"
+$ComparePerfResults = Join-Path $PSScriptRoot "ComparePerfResults.ps1"
+$Temp = Join-Path $RepoRoot "temp"
+
+try {
+ # Get baseline results
+ RunTest -RunPerfTests $RunPerfTests -output $output -resultsFolder "baseline" -temp $Temp -repoName "perfBaseline" -repoSHA $baselineSHA -projects $projects -filter $filter -etl $etl.IsPresent -ci $ci.IsPresent
+
+ # Get perf results
+ RunTest -RunPerfTests $RunPerfTests -output $output -resultsFolder "perfTest" -temp $Temp -repoName "perfTest" -repoSHA $testSHA -projects $projects -filter $filter -etl $etl.IsPresent -ci $ci.IsPresent
+
+ # Diff perf results
+ Invoke-Expression "$ComparePerfResults -baseline $baselineOutput -results $testOutput"
+}
+catch {
+ Write-Host $_
+ Write-Host $_.Exception
+ Write-Host $_.ScriptStackTrace
+ $host.SetShouldExit(1)
+ exit
+}
+finally {
+ Invoke-Expression 'git worktree remove perfBaseline'
+ Invoke-Expression 'git worktree remove perfTest'
+ Invoke-Expression 'git worktree prune'
+ Set-Location $currentLocation
+}
\ No newline at end of file
diff --git a/eng/perf/PerfCore.ps1 b/eng/perf/PerfCore.ps1
new file mode 100644
index 0000000000..5cc38e3ba8
--- /dev/null
+++ b/eng/perf/PerfCore.ps1
@@ -0,0 +1,86 @@
+[CmdletBinding(PositionalBinding=$false)]
+Param(
+ [string] $projects,
+ [string][Alias('v')]$verbosity = "minimal",
+ [string] $filter,
+ [switch] $etl,
+ [switch] $diff,
+ [switch] $ci,
+ [switch] $help,
+ [Parameter(ValueFromRemainingArguments=$true)][String[]]$properties
+ )
+
+ function Print-Usage() {
+ Write-Host "Common settings:"
+ Write-Host " -verbosity Msbuild verbosity: q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic] (short: -v)"
+ Write-Host " -help Print help and exit"
+ Write-Host ""
+
+ Write-Host "Actions:"
+ Write-Host " -diff Compare HEAD to baseline perf results"
+ Write-Host ""
+
+ Write-Host "Advanced settings:"
+ Write-Host " -etl Capture ETL traces of performance tests (requires admin permissions, default value is 'false')"
+ Write-Host " -filter Filter for tests to run (supports wildcards)"
+ Write-Host " -projects Semi-colon delimited list of relative paths to benchmark projects."
+ Write-Host ""
+
+ Write-Host "Command line arguments not listed above are passed thru to msbuild."
+ Write-Host "The above arguments can be shortened as much as to be unambiguous (e.g. -co for configuration, -t for test, etc.)."
+ }
+
+try {
+ if ($help -or (($null -ne $properties) -and ($properties.Contains('/help') -or $properties.Contains('/?')))) {
+ Print-Usage
+ exit 0
+ }
+
+ if ([string]::IsNullOrWhiteSpace($projects)) {
+ $projects = "src\PerformanceTests\Tests\PerformanceTests.csproj"
+ }
+
+ if ([string]::IsNullOrWhiteSpace($filter)) {
+ $filter = "*"
+ }
+
+ $RepoRoot = Resolve-Path (Join-Path $PSScriptRoot '..\..')
+ $output = Join-Path $RepoRoot "artifacts\performance\perfResults"
+
+ # Diff two different SHAs
+ if ($diff) {
+ $DiffPerfToBaseLine = Join-Path $RepoRoot "eng\perf\DiffPerfToBaseLine.ps1"
+ $baselinejson = Get-Content -Raw -Path (Join-Path $RepoRoot "eng\perf\baseline.json") | ConvertFrom-Json
+ $baselineSHA = $baselinejson.sha
+ $commandArguments = "-baselineSHA $baselineSHA -testSHA HEAD -projects '$projects' -output '$output' -filter '$filter'"
+ if ($etl) {
+ $commandArguments = "$commandArguments -etl"
+ }
+ if ($ci) {
+ $commandArguments = "$commandArguments -ci"
+ }
+ Invoke-Expression "$DiffPerfToBaseLine $commandArguments"
+ exit
+ }
+
+ $commandArguments = "-projects '$projects' -root '$RepoRoot' -output '$output\perfTest' -filter '$filter'"
+ if ($etl) {
+ $commandArguments = "$commandArguments -etl"
+ }
+ if ($ci) {
+ $commandArguments = "$commandArguments -ci"
+ }
+
+ $RunPerfTests = Join-Path $RepoRoot "eng\perf\RunPerfTests.ps1"
+ Invoke-Expression "$RunPerfTests $commandArguments"
+ exit
+}
+catch {
+ Write-Host $_
+ Write-Host $_.Exception
+ Write-Host $_.ScriptStackTrace
+ $host.SetShouldExit(1)
+ exit
+}
+finally {
+}
\ No newline at end of file
diff --git a/eng/perf/RunPerfTests.ps1 b/eng/perf/RunPerfTests.ps1
new file mode 100644
index 0000000000..1943b2da33
--- /dev/null
+++ b/eng/perf/RunPerfTests.ps1
@@ -0,0 +1,35 @@
+[CmdletBinding(PositionalBinding=$false)]
+Param(
+ [string] $projects, # semicolon separated list of relative paths to benchmark projects to run
+ [string] $filter, # filter for tests to run (supports wildcards)
+ [String] $root, # root folder all of the benchmark projects share
+ [String] $output, # folder to write the benchmark results to
+ [switch] $etl, # capture etl traces for performance tests
+ [switch] $ci # run in ci mode (fail fast an keep all partial artifacts)
+ )
+
+try {
+ $projectsList = $projects -split ";"
+ foreach ($project in $projectsList){
+ $projectFullPath = Join-Path $root $project
+ $comandArguments = "run -c Release --project $projectFullPath -- --memory --exporters JSON --artifacts $output"
+ if ($ci) {
+ $comandArguments = "$comandArguments --stopOnFirstError --keepFiles"
+ }
+ if ($etl) {
+ Start-Process -Wait -FilePath "dotnet" -Verb RunAs -ArgumentList "$comandArguments --profiler ETW --filter $filter"
+ }
+ else {
+ Invoke-Expression "dotnet $comandArguments --filter $filter"
+ }
+ }
+}
+catch {
+ Write-Host $_
+ Write-Host $_.Exception
+ Write-Host $_.ScriptStackTrace
+ $host.SetShouldExit(1)
+ exit
+}
+finally {
+}
\ No newline at end of file
diff --git a/eng/perf/baseline.json b/eng/perf/baseline.json
new file mode 100644
index 0000000000..289e28bc3c
--- /dev/null
+++ b/eng/perf/baseline.json
@@ -0,0 +1,5 @@
+{
+ "release": "6.0",
+ "label": "roslyn analyzers baseline for main",
+ "sha": "f986c7b7aff265feb7536e1c7f98efcb9f14e077"
+}
\ No newline at end of file
diff --git a/src/Tools/PerfDiff/.editorconfig b/src/Tools/PerfDiff/.editorconfig
new file mode 100644
index 0000000000..4032e86375
--- /dev/null
+++ b/src/Tools/PerfDiff/.editorconfig
@@ -0,0 +1,8 @@
+### Configuration for PublicAPI analyzers executed on this folder ###
+[*.{cs,vb}]
+
+dotnet_diagnostic.IDE1030.severity = none
+dotnet_diagnostic.CA1052.severity = none
+dotnet_diagnostic.CA1063.severity = none
+dotnet_diagnostic.CA1724.severity = none
+dotnet_diagnostic.CA1819.severity = none
diff --git a/src/Tools/PerfDiff/BDN/BenchmarkDotNetDiffer.cs b/src/Tools/PerfDiff/BDN/BenchmarkDotNetDiffer.cs
new file mode 100644
index 0000000000..7e57c4410a
--- /dev/null
+++ b/src/Tools/PerfDiff/BDN/BenchmarkDotNetDiffer.cs
@@ -0,0 +1,188 @@
+// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+using System.IO;
+using System.Linq;
+using System.Threading.Tasks;
+using DataTransferContracts;
+using Microsoft.Extensions.Logging;
+using Newtonsoft.Json;
+using Perfolizer.Mathematics.SignificanceTesting;
+using Perfolizer.Mathematics.Thresholds;
+
+namespace PerfDiff
+{
+ public static class BenchmarkDotNetDiffer
+ {
+ public static async Task<(bool success, bool shouldCheckETL)> TryCompareBenchmarkDotNetResultsAsync(string baselineFolder, string resultsFolder, ILogger logger)
+ {
+ bool shouldCheckETL = false;
+
+ // search folder for benchmark dotnet results
+ var comparison = await TryGetBdnResultsAsync(baselineFolder, resultsFolder, logger).ConfigureAwait(false);
+ if (comparison is null)
+ {
+ return (false, shouldCheckETL);
+ }
+
+ // compare bdn results
+ // TODO: let these be optional parameters
+ _ = Threshold.TryParse("15%", out var testThreshold);
+ _ = Threshold.TryParse("0.3ns", out var noiseThreshold);
+
+ var notSame = FindRegressions(comparison, testThreshold, noiseThreshold);
+
+ if (!notSame.Any())
+ {
+ logger.LogInformation($"No differences found between the benchmark results with threshold {testThreshold}.");
+ return (false, shouldCheckETL);
+ }
+
+ var better = notSame.Where(result => result.conclusion == EquivalenceTestConclusion.Faster);
+ var worse = notSame.Where(result => result.conclusion == EquivalenceTestConclusion.Slower);
+ var betterCount = better.Count();
+ int worseCount = worse.Count();
+
+ // If the baseline doesn't have the same set of tests, you wind up with Infinity in the list of diffs.
+ // Exclude them for purposes of geomean.
+ worse = worse.Where(x => GetRatio(x) != double.PositiveInfinity);
+ better = better.Where(x => GetRatio(x) != double.PositiveInfinity);
+
+ if (betterCount > 0)
+ {
+ var betterGeoMean = Math.Pow(10, better.Skip(1).Aggregate(Math.Log10(GetRatio(better.First())), (x, y) => x + Math.Log10(GetRatio(y))) / better.Count());
+ logger.LogInformation($"better: {betterCount}, geomean: {betterGeoMean:F3}");
+ }
+
+ if (worseCount > 0)
+ {
+ var worseGeoMean = Math.Pow(10, worse.Skip(1).Aggregate(Math.Log10(GetRatio(worse.First())), (x, y) => x + Math.Log10(GetRatio(y))) / worse.Count());
+ logger.LogInformation($"worse: {worseCount}, geomean: {worseGeoMean:F3}");
+ }
+
+ if (worseCount > 0)
+ {
+ shouldCheckETL = true;
+ }
+
+ return (true, shouldCheckETL);
+ }
+
+ private static double GetRatio((string id, Benchmark baseResult, Benchmark diffResult, EquivalenceTestConclusion conclusion) item) => GetRatio(item.conclusion, item.baseResult, item.diffResult);
+
+ private static double GetRatio(EquivalenceTestConclusion conclusion, Benchmark baseResult, Benchmark diffResult)
+ => conclusion == EquivalenceTestConclusion.Faster
+ ? baseResult.Statistics.Median / diffResult.Statistics.Median
+ : diffResult.Statistics.Median / baseResult.Statistics.Median;
+
+ private static async Task<(string id, Benchmark baseResult, Benchmark diffResult)[]?> TryGetBdnResultsAsync(
+ string baselineFolder,
+ string resultsFolder,
+ ILogger logger)
+ {
+ if (!TryGetFilesToParse(baselineFolder, out var baseFiles))
+ {
+ logger.LogError($"Provided path does NOT exist or does not contain perf results '{baselineFolder}'");
+ return null;
+ }
+
+ if (!TryGetFilesToParse(resultsFolder, out var resultsFiles))
+ {
+ logger.LogError($"Provided path does NOT exist or does not contain perf results '{resultsFolder}'");
+ return null;
+ }
+
+ if (!baseFiles.Any() || !resultsFiles.Any())
+ {
+ logger.LogError($"Provided paths contained no '{FullBdnJsonFileExtension}' files.");
+ return null;
+ }
+
+ var (baseResultsSuccess, baseResults) = await TryGetBdnResultAsync(baseFiles, logger).ConfigureAwait(false);
+ if (!baseResultsSuccess)
+ {
+ return null;
+ }
+
+ var (resultsSuccess, diffResults) = await TryGetBdnResultAsync(resultsFiles, logger).ConfigureAwait(false);
+ if (!resultsSuccess)
+ {
+ return null;
+ }
+
+ var benchmarkIdToDiffResults = diffResults
+ .SelectMany(result => result.Benchmarks)
+ .ToDictionary(benchmarkResult => benchmarkResult.DisplayInfo, benchmarkResult => benchmarkResult);
+
+ return baseResults
+ .SelectMany(result => result.Benchmarks)
+ .ToDictionary(benchmarkResult => benchmarkResult.DisplayInfo, benchmarkResult => benchmarkResult) // we use ToDictionary to make sure the results have unique IDs
+ .Where(baseResult => benchmarkIdToDiffResults.ContainsKey(baseResult.Key))
+ .Select(baseResult => (id: baseResult.Key, baseResult: baseResult.Value, diffResult: benchmarkIdToDiffResults[baseResult.Key]))
+ .ToArray();
+ }
+
+ private const string FullBdnJsonFileExtension = "full-compressed.json";
+
+ private static bool TryGetFilesToParse(string path, [NotNullWhen(true)] out string[]? files)
+ {
+ if (Directory.Exists(path))
+ {
+ files = Directory.GetFiles(path, $"*{FullBdnJsonFileExtension}", SearchOption.AllDirectories);
+ return true;
+ }
+ else if (File.Exists(path) || !path.EndsWith(FullBdnJsonFileExtension, StringComparison.OrdinalIgnoreCase))
+ {
+ files = new[] { path };
+ return true;
+ }
+
+ files = null;
+ return false;
+ }
+
+ private static async Task<(bool success, BdnResult[] results)> TryGetBdnResultAsync(string[] paths, ILogger logger)
+ {
+ var results = await Task.WhenAll(paths.Select(path => ReadFromFileAsync(path, logger))).ConfigureAwait(false);
+ return (!results.Any(x => x is null), results)!;
+ }
+
+ private static async Task ReadFromFileAsync(string resultFilePath, ILogger logger)
+ {
+ try
+ {
+ return JsonConvert.DeserializeObject(await File.ReadAllTextAsync(resultFilePath).ConfigureAwait(false));
+ }
+ catch (JsonSerializationException)
+ {
+ logger.LogError($"Exception while reading the {resultFilePath} file.");
+ return null;
+ }
+ }
+
+ private static (string id, Benchmark baseResult, Benchmark diffResult, EquivalenceTestConclusion conclusion)[] FindRegressions((string id, Benchmark baseResult, Benchmark diffResult)[] comparison, Threshold testThreshold, Threshold noiseThreshold)
+ {
+ var results = new List<(string id, Benchmark baseResult, Benchmark diffResult, EquivalenceTestConclusion conclusion)>();
+ foreach ((string id, Benchmark baseResult, Benchmark diffResult) in comparison
+ .Where(result => result.baseResult.Statistics != null && result.diffResult.Statistics != null)) // failures
+ {
+ var baseValues = baseResult.GetOriginalValues();
+ var diffValues = diffResult.GetOriginalValues();
+
+ var userTresholdResult = StatisticalTestHelper.CalculateTost(MannWhitneyTest.Instance, baseValues, diffValues, testThreshold);
+ if (userTresholdResult.Conclusion == EquivalenceTestConclusion.Same)
+ continue;
+
+ var noiseResult = StatisticalTestHelper.CalculateTost(MannWhitneyTest.Instance, baseValues, diffValues, noiseThreshold);
+ if (noiseResult.Conclusion == EquivalenceTestConclusion.Same)
+ continue;
+
+ results.Add((id, baseResult, diffResult, conclusion: userTresholdResult.Conclusion));
+ }
+
+ return results.ToArray();
+ }
+ }
+}
diff --git a/src/Tools/PerfDiff/BDN/DataContracts/ChronometerFrequency.cs b/src/Tools/PerfDiff/BDN/DataContracts/ChronometerFrequency.cs
new file mode 100644
index 0000000000..b1c5050c64
--- /dev/null
+++ b/src/Tools/PerfDiff/BDN/DataContracts/ChronometerFrequency.cs
@@ -0,0 +1,133 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+//
+
+using System.Collections.Generic;
+using System.Linq;
+
+namespace DataTransferContracts // generated with http://json2csharp.com/#
+{
+ public class ChronometerFrequency
+ {
+ public int Hertz { get; set; }
+ }
+
+ public class HostEnvironmentInfo
+ {
+ public string BenchmarkDotNetCaption { get; set; }
+ public string BenchmarkDotNetVersion { get; set; }
+ public string OsVersion { get; set; }
+ public string ProcessorName { get; set; }
+ public int? PhysicalProcessorCount { get; set; }
+ public int? PhysicalCoreCount { get; set; }
+ public int? LogicalCoreCount { get; set; }
+ public string RuntimeVersion { get; set; }
+ public string Architecture { get; set; }
+ public bool? HasAttachedDebugger { get; set; }
+ public bool? HasRyuJit { get; set; }
+ public string Configuration { get; set; }
+ public string JitModules { get; set; }
+ public string DotNetCliVersion { get; set; }
+ public ChronometerFrequency ChronometerFrequency { get; set; }
+ public string HardwareTimerKind { get; set; }
+ }
+
+ public class ConfidenceInterval
+ {
+ public int N { get; set; }
+ public double Mean { get; set; }
+ public double StandardError { get; set; }
+ public int Level { get; set; }
+ public double Margin { get; set; }
+ public double Lower { get; set; }
+ public double Upper { get; set; }
+ }
+
+ public class Percentiles
+ {
+ public double P0 { get; set; }
+ public double P25 { get; set; }
+ public double P50 { get; set; }
+ public double P67 { get; set; }
+ public double P80 { get; set; }
+ public double P85 { get; set; }
+ public double P90 { get; set; }
+ public double P95 { get; set; }
+ public double P100 { get; set; }
+ }
+
+ public class Statistics
+ {
+ public int N { get; set; }
+ public double Min { get; set; }
+ public double LowerFence { get; set; }
+ public double Q1 { get; set; }
+ public double Median { get; set; }
+ public double Mean { get; set; }
+ public double Q3 { get; set; }
+ public double UpperFence { get; set; }
+ public double Max { get; set; }
+ public double InterquartileRange { get; set; }
+ public List LowerOutliers { get; set; }
+ public List UpperOutliers { get; set; }
+ public List AllOutliers { get; set; }
+ public double StandardError { get; set; }
+ public double Variance { get; set; }
+ public double StandardDeviation { get; set; }
+ public double? Skewness { get; set; }
+ public double? Kurtosis { get; set; }
+ public ConfidenceInterval ConfidenceInterval { get; set; }
+ public Percentiles Percentiles { get; set; }
+ }
+
+ public class Memory
+ {
+ public int Gen0Collections { get; set; }
+ public int Gen1Collections { get; set; }
+ public int Gen2Collections { get; set; }
+ public long TotalOperations { get; set; }
+ public long BytesAllocatedPerOperation { get; set; }
+ }
+
+ public class Measurement
+ {
+ public string IterationStage { get; set; }
+ public int LaunchIndex { get; set; }
+ public int IterationIndex { get; set; }
+ public long Operations { get; set; }
+ public double Nanoseconds { get; set; }
+ }
+
+ public class Benchmark
+ {
+ public string DisplayInfo { get; set; }
+ public object Namespace { get; set; }
+ public string Type { get; set; }
+ public string Method { get; set; }
+ public string MethodTitle { get; set; }
+ public string Parameters { get; set; }
+ public string FullName { get; set; }
+ public Statistics Statistics { get; set; }
+ public Memory Memory { get; set; }
+ public List Measurements { get; set; }
+
+ ///
+ /// this method was not auto-generated by a tool, it was added manually
+ ///
+ /// an array of the actual workload results (not warmup, not pilot)
+ internal double[] GetOriginalValues()
+ => Measurements
+ .Where(measurement => measurement.IterationStage == "Result")
+ .Select(measurement => measurement.Nanoseconds / measurement.Operations)
+ .ToArray();
+ }
+
+ public class BdnResult
+ {
+ public string Title { get; set; }
+ public HostEnvironmentInfo HostEnvironmentInfo { get; set; }
+ public List Benchmarks { get; set; }
+ }
+}
diff --git a/src/Tools/PerfDiff/DiffCommand.cs b/src/Tools/PerfDiff/DiffCommand.cs
new file mode 100644
index 0000000000..2fb4de5008
--- /dev/null
+++ b/src/Tools/PerfDiff/DiffCommand.cs
@@ -0,0 +1,37 @@
+// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System.CommandLine;
+using System.Threading.Tasks;
+
+namespace PerfDiff
+{
+ internal static class DiffCommand
+ {
+ // This delegate should be kept in Sync with the FormatCommand options and argument names
+ // so that values bind correctly.
+ internal delegate Task Handler(
+ string baseline,
+ string results,
+ string? verbosity,
+ bool failOnRegression,
+ IConsole console);
+
+ internal static string[] VerbosityLevels => new[] { "q", "quiet", "m", "minimal", "n", "normal", "d", "detailed", "diag", "diagnostic" };
+
+ internal static RootCommand CreateCommandLineOptions()
+ {
+ // Sync changes to option and argument names with the FormatCommant.Handler above.
+ var rootCommand = new RootCommand
+ {
+ new Option("--baseline", () => null, "folder that contains the baseline performance run data").LegalFilePathsOnly(),
+ new Option("--results", () => null, "folder that contains the performance restults").LegalFilePathsOnly(),
+ new Option(new[] { "--verbosity", "-v" }, "Set the verbosity level. Allowed values are q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic]").FromAmong(VerbosityLevels),
+ new Option(new[] { "--failOnRegression" }, "Should return non-zero exit code if regression detected"),
+ };
+
+ rootCommand.Description = "diff two sets of performance results";
+
+ return rootCommand;
+ }
+ }
+}
diff --git a/src/Tools/PerfDiff/ETL/EtlDiffer.cs b/src/Tools/PerfDiff/ETL/EtlDiffer.cs
new file mode 100644
index 0000000000..62d2b74303
--- /dev/null
+++ b/src/Tools/PerfDiff/ETL/EtlDiffer.cs
@@ -0,0 +1,169 @@
+// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using System.Collections.Immutable;
+using System.IO;
+using System.Linq;
+
+using Microsoft.Diagnostics.Symbols;
+using Microsoft.Diagnostics.Tracing.Etlx;
+using Microsoft.Diagnostics.Tracing.Parsers.Kernel;
+using Microsoft.Diagnostics.Tracing.Stacks;
+
+namespace PerfDiff
+{
+ internal static class EtlDiffer
+ {
+ public static bool TryCompareETL(string sourceEtlPath, string baselineEtlPath, out bool regression)
+ {
+ regression = false;
+ CallTree sourceCallTree = GetCallTree(sourceEtlPath);
+ CallTree baselineCallTree = GetCallTree(baselineEtlPath);
+ var report = GenerateOverweightReport(sourceCallTree, baselineCallTree);
+
+ // print results
+ Console.WriteLine(string.Join(Environment.NewLine, report.Take(10)));
+ return true;
+ }
+
+ private static CallTree GetCallTree(string eltPath)
+ {
+ var traceProcess = GetTraceProcessFromETLFile(eltPath);
+ var stackSource = CreateStackSourceFromTraceProcess(traceProcess);
+ return CreateCallTreeFromStackSource(stackSource);
+ }
+
+ public static TraceProcess GetTraceProcessFromETLFile(string eltPath)
+ {
+ var traceLog = TraceLog.OpenOrConvert(eltPath);
+ return traceLog.Processes
+ .First(p => p.Name.Equals("dotnet", StringComparison.OrdinalIgnoreCase));
+ }
+
+ public static StackSource CreateStackSourceFromTraceProcess(TraceProcess process)
+ {
+ var events = process.EventsInProcess;
+ var start = Math.Max(events.StartTimeRelativeMSec, process.StartTimeRelativeMsec);
+ var end = Math.Min(events.EndTimeRelativeMSec, process.EndTimeRelativeMsec);
+ events = events.FilterByTime(start, end);
+ events = events.Filter(x => x is SampledProfileTraceData && x.ProcessID == process.ProcessID);
+
+ using var symbolReader = new SymbolReader(new StringWriter(), @"SRV*https://msdl.microsoft.com/download/symbols");
+ symbolReader.SecurityCheck = path => true;
+
+ var traceLog = process.Log;
+ foreach (var module in process.LoadedModules)
+ {
+ traceLog.CodeAddresses.LookupSymbolsForModule(symbolReader, module.ModuleFile);
+ }
+
+ return new TraceEventStackSource(events);
+ }
+
+ public static CallTree CreateCallTreeFromStackSource(StackSource stackSource)
+ {
+ var calltree = new CallTree(ScalingPolicyKind.ScaleToData);
+ calltree.StackSource = stackSource;
+ return calltree;
+ }
+
+ public static ImmutableArray GenerateOverweightReport(CallTree source, CallTree baseline)
+ {
+ var sourceTotal = LoadTrace(source, out var sourceData);
+ var baselineTotal = LoadTrace(baseline, out var baselineData);
+
+ if (sourceTotal != baselineTotal)
+ {
+ return ComputeOverweights(sourceTotal, sourceData, baselineTotal, baselineData);
+ }
+
+ return ImmutableArray.Empty;
+
+ static float LoadTrace(CallTree callTree, out Dictionary data)
+ {
+ data = new Dictionary();
+ float total = 0;
+ foreach (var node in callTree.ByID)
+ {
+ if (node.InclusiveMetric == 0)
+ {
+ continue;
+ }
+
+ string key = node.Name;
+ data.TryGetValue(key, out var weight);
+ data[key] = weight + node.InclusiveMetric;
+
+ total += node.ExclusiveMetric;
+ }
+ return total;
+ }
+
+ static ImmutableArray ComputeOverweights(float sourceTotal, Dictionary sourceData, float baselineTotal, Dictionary baselineData)
+ {
+ var totalDelta = sourceTotal - baselineTotal;
+ var growth = sourceTotal / baselineTotal;
+ var results = ImmutableArray.CreateBuilder();
+ foreach (var key in baselineData.Keys)
+ {
+ // skip symbols that are not in both traces
+ if (!sourceData.ContainsKey(key))
+ {
+ continue;
+ }
+
+ var baselineValue = baselineData[key];
+ var sourceValue = sourceData[key];
+ var expectedDelta = baselineValue * (growth - 1);
+ var delta = sourceValue - baselineValue;
+ var overweight = delta / expectedDelta * 100;
+ var percent = delta / totalDelta;
+ // Calculate interest level
+ var interest = Math.Abs(overweight) > 110 ? 1 : 0;
+ interest += Math.Abs(percent) > 5 ? 1 : 0;
+ interest += Math.Abs(percent) > 20 ? 1 : 0;
+ interest += Math.Abs(percent) > 100 ? 1 : 0;
+ interest += sourceValue / sourceTotal < 0.95 ? 1 : 0; // Ignore top of the stack frames
+ interest += sourceValue / sourceTotal < 0.75 ? 1 : 0; // Bonus point for being further down the stack.
+
+ results.Add(new OverWeightResult
+ (
+ Name: key,
+ Before: baselineValue,
+ After: sourceValue,
+ Delta: delta,
+ Overweight: overweight,
+ Percent: percent,
+ Interest: interest
+ ));
+ }
+
+ results.Sort((left, right) =>
+ {
+ if (left.Interest < right.Interest)
+ return 1;
+
+ if (left.Interest > right.Interest)
+ return -1;
+
+ if (left.Overweight < right.Overweight)
+ return 1;
+
+ if (left.Overweight > right.Overweight)
+ return -1;
+
+ if (left.Delta < right.Delta)
+ return -1;
+
+ if (left.Delta > right.Delta)
+ return 1;
+
+ return 0;
+ });
+
+ return results.ToImmutable();
+ }
+ }
+ }
+}
diff --git a/src/Tools/PerfDiff/ETL/OverWeightResult.cs b/src/Tools/PerfDiff/ETL/OverWeightResult.cs
new file mode 100644
index 0000000000..4c0bfd55be
--- /dev/null
+++ b/src/Tools/PerfDiff/ETL/OverWeightResult.cs
@@ -0,0 +1,10 @@
+// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+namespace PerfDiff
+{
+ internal record OverWeightResult(string Name, float Before, float After, float Delta, float Overweight, float Percent, int Interest)
+ {
+ public override string ToString()
+ => $"'{Name}':, Overweight: '{Overweight}%', Before: '{Before}ms', After: '{After}ms', Interest :'{Interest}'";
+ }
+}
diff --git a/src/Tools/PerfDiff/Logging/NullScope.cs b/src/Tools/PerfDiff/Logging/NullScope.cs
new file mode 100644
index 0000000000..849b4923a0
--- /dev/null
+++ b/src/Tools/PerfDiff/Logging/NullScope.cs
@@ -0,0 +1,19 @@
+// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+
+namespace PerfDiff.Logging
+{
+ internal class NullScope : IDisposable
+ {
+ public static NullScope Instance { get; } = new NullScope();
+
+ private NullScope()
+ {
+ }
+
+ public void Dispose()
+ {
+ }
+ }
+}
diff --git a/src/Tools/PerfDiff/Logging/SimpleConsoleLogger.cs b/src/Tools/PerfDiff/Logging/SimpleConsoleLogger.cs
new file mode 100644
index 0000000000..5924734f38
--- /dev/null
+++ b/src/Tools/PerfDiff/Logging/SimpleConsoleLogger.cs
@@ -0,0 +1,95 @@
+// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using Microsoft.Extensions.Logging;
+
+using System;
+using System.Collections.Generic;
+using System.Collections.Immutable;
+using System.CommandLine;
+using System.CommandLine.Rendering;
+
+namespace PerfDiff.Logging
+{
+ internal class SimpleConsoleLogger : ILogger
+ {
+ private readonly object _gate = new();
+
+ private readonly IConsole _console;
+ private readonly ITerminal _terminal;
+ private readonly LogLevel _minimalLogLevel;
+ private readonly LogLevel _minimalErrorLevel;
+
+ private static ImmutableDictionary LogLevelColorMap => new Dictionary
+ {
+ [LogLevel.Critical] = ConsoleColor.Red,
+ [LogLevel.Error] = ConsoleColor.Red,
+ [LogLevel.Warning] = ConsoleColor.Yellow,
+ [LogLevel.Information] = ConsoleColor.White,
+ [LogLevel.Debug] = ConsoleColor.Gray,
+ [LogLevel.Trace] = ConsoleColor.Gray,
+ [LogLevel.None] = ConsoleColor.White,
+ }.ToImmutableDictionary();
+
+ public SimpleConsoleLogger(IConsole console, LogLevel minimalLogLevel, LogLevel minimalErrorLevel)
+ {
+ _terminal = console.GetTerminal();
+ _console = console;
+ _minimalLogLevel = minimalLogLevel;
+ _minimalErrorLevel = minimalErrorLevel;
+ }
+
+ public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter)
+ {
+ if (!IsEnabled(logLevel))
+ {
+ return;
+ }
+
+ lock (_gate)
+ {
+ var message = formatter(state, exception);
+ var logToErrorStream = logLevel >= _minimalErrorLevel;
+ if (_terminal is null)
+ {
+ LogToConsole(_console, message, logToErrorStream);
+ }
+ else
+ {
+ LogToTerminal(message, logLevel, logToErrorStream);
+ }
+ }
+ }
+
+ public bool IsEnabled(LogLevel logLevel)
+ {
+ return (int)logLevel >= (int)_minimalLogLevel;
+ }
+
+ public IDisposable BeginScope(TState state)
+ {
+ return NullScope.Instance;
+ }
+
+ private void LogToTerminal(string message, LogLevel logLevel, bool logToErrorStream)
+ {
+ var messageColor = LogLevelColorMap[logLevel];
+ _terminal.ForegroundColor = messageColor;
+
+ LogToConsole(_terminal, message, logToErrorStream);
+
+ _terminal.ResetColor();
+ }
+
+ private static void LogToConsole(IConsole console, string message, bool logToErrorStream)
+ {
+ if (logToErrorStream)
+ {
+ console.Error.Write($"{message}{Environment.NewLine}");
+ }
+ else
+ {
+ console.Out.Write($" {message}{Environment.NewLine}");
+ }
+ }
+ }
+}
diff --git a/src/Tools/PerfDiff/Logging/SimpleConsoleLoggerFactoryExtensions.cs b/src/Tools/PerfDiff/Logging/SimpleConsoleLoggerFactoryExtensions.cs
new file mode 100644
index 0000000000..e90891919c
--- /dev/null
+++ b/src/Tools/PerfDiff/Logging/SimpleConsoleLoggerFactoryExtensions.cs
@@ -0,0 +1,17 @@
+// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System.CommandLine;
+
+using Microsoft.Extensions.Logging;
+
+namespace PerfDiff.Logging
+{
+ internal static class SimpleConsoleLoggerFactoryExtensions
+ {
+ public static ILoggerFactory AddSimpleConsole(this ILoggerFactory factory, IConsole console, LogLevel minimalLogLevel, LogLevel minimalErrorLevel)
+ {
+ factory.AddProvider(new SimpleConsoleLoggerProvider(console, minimalLogLevel, minimalErrorLevel));
+ return factory;
+ }
+ }
+}
diff --git a/src/Tools/PerfDiff/Logging/SimpleConsoleLoggerProvider.cs b/src/Tools/PerfDiff/Logging/SimpleConsoleLoggerProvider.cs
new file mode 100644
index 0000000000..e153765e5b
--- /dev/null
+++ b/src/Tools/PerfDiff/Logging/SimpleConsoleLoggerProvider.cs
@@ -0,0 +1,31 @@
+// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using Microsoft.Extensions.Logging;
+
+using System.CommandLine;
+
+namespace PerfDiff.Logging
+{
+ internal class SimpleConsoleLoggerProvider : ILoggerProvider
+ {
+ private readonly IConsole _console;
+ private readonly LogLevel _minimalLogLevel;
+ private readonly LogLevel _minimalErrorLevel;
+
+ public SimpleConsoleLoggerProvider(IConsole console, LogLevel minimalLogLevel, LogLevel minimalErrorLevel)
+ {
+ _console = console;
+ _minimalLogLevel = minimalLogLevel;
+ _minimalErrorLevel = minimalErrorLevel;
+ }
+
+ public ILogger CreateLogger(string categoryName)
+ {
+ return new SimpleConsoleLogger(_console, _minimalLogLevel, _minimalErrorLevel);
+ }
+
+ public void Dispose()
+ {
+ }
+ }
+}
diff --git a/src/Tools/PerfDiff/PerfDiff.cs b/src/Tools/PerfDiff/PerfDiff.cs
new file mode 100644
index 0000000000..40abfb7a7a
--- /dev/null
+++ b/src/Tools/PerfDiff/PerfDiff.cs
@@ -0,0 +1,75 @@
+// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Diagnostics.CodeAnalysis;
+using System.IO;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+
+using Microsoft.Extensions.Logging;
+
+namespace PerfDiff
+{
+ public static partial class PerfDiff
+ {
+ public static async Task CompareAsync(
+ string baselineFolder, string resultsFolder, bool failOnRegression, ILogger logger, CancellationToken token)
+ {
+ token.ThrowIfCancellationRequested();
+
+ var (success, shouldCheckETL) = await BenchmarkDotNetDiffer.TryCompareBenchmarkDotNetResultsAsync(baselineFolder, resultsFolder, logger).ConfigureAwait(false);
+ if (!success)
+ {
+ return 1;
+ }
+
+ if (shouldCheckETL)
+ {
+ // get file paths
+ if (!TryGetETLPaths(baselineFolder, out var baselineEtlPath))
+ {
+ return 1;
+ }
+
+ if (!TryGetETLPaths(resultsFolder, out var resultsEtlPath))
+ {
+ return 1;
+ }
+
+ // Compare ETL
+ if (!EtlDiffer.TryCompareETL(resultsEtlPath, baselineEtlPath, out var regression))
+ {
+ return 1;
+ }
+
+ if (regression && failOnRegression)
+ {
+ return 1;
+ }
+ }
+
+ return 0;
+ }
+
+ private const string ETLFileExtension = "etl.zip";
+
+ private static bool TryGetETLPaths(string path, [NotNullWhen(true)] out string? etlPath)
+ {
+ if (Directory.Exists(path))
+ {
+ var files = Directory.GetFiles(path, $"*{ETLFileExtension}", SearchOption.AllDirectories);
+ etlPath = files.Single();
+ return true;
+ }
+ else if (File.Exists(path) || !path.EndsWith(ETLFileExtension, StringComparison.OrdinalIgnoreCase))
+ {
+ etlPath = path;
+ return true;
+ }
+
+ etlPath = null;
+ return false;
+ }
+ }
+}
diff --git a/src/Tools/PerfDiff/PerfDiff.csproj b/src/Tools/PerfDiff/PerfDiff.csproj
new file mode 100644
index 0000000000..7a829e12ea
--- /dev/null
+++ b/src/Tools/PerfDiff/PerfDiff.csproj
@@ -0,0 +1,19 @@
+
+
+
+ Exe
+ net6.0
+ true
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/src/Tools/PerfDiff/Program.cs b/src/Tools/PerfDiff/Program.cs
new file mode 100644
index 0000000000..953a816e35
--- /dev/null
+++ b/src/Tools/PerfDiff/Program.cs
@@ -0,0 +1,105 @@
+// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging;
+
+using PerfDiff.Logging;
+
+using System;
+using System.CommandLine;
+using System.CommandLine.Invocation;
+using System.CommandLine.Parsing;
+using System.IO;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace PerfDiff
+{
+ internal class Program
+ {
+ internal const int UnhandledExceptionExitCode = 1;
+ private static ParseResult? s_parseResult;
+
+ private static async Task Main(string[] args)
+ {
+ var rootCommand = DiffCommand.CreateCommandLineOptions();
+ rootCommand.Handler = CommandHandler.Create(new DiffCommand.Handler(Run));
+
+ // Parse the incoming args so we can give warnings when deprecated options are used.
+ s_parseResult = rootCommand.Parse(args);
+
+ return await rootCommand.InvokeAsync(args).ConfigureAwait(false);
+ }
+
+ public static async Task Run(
+ string baseline,
+ string results,
+ string? verbosity,
+ bool failOnRegression,
+ IConsole console)
+ {
+ if (s_parseResult == null)
+ {
+ return 1;
+ }
+
+ // Setup logging.
+ var logLevel = GetLogLevel(verbosity);
+ var logger = SetupLogging(console, minimalLogLevel: logLevel, minimalErrorLevel: LogLevel.Warning);
+
+ // Hook so we can cancel and exit when ctrl+c is pressed.
+ var cancellationTokenSource = new CancellationTokenSource();
+ Console.CancelKeyPress += (sender, e) =>
+ {
+ e.Cancel = true;
+ cancellationTokenSource.Cancel();
+ };
+
+ var currentDirectory = string.Empty;
+
+ try
+ {
+ return await PerfDiff.CompareAsync(baseline, results, failOnRegression, logger, cancellationTokenSource.Token).ConfigureAwait(false);
+ }
+ catch (FileNotFoundException fex)
+ {
+ logger.LogError(fex.Message);
+ return UnhandledExceptionExitCode;
+ }
+ catch (OperationCanceledException)
+ {
+ return UnhandledExceptionExitCode;
+ }
+ finally
+ {
+ if (!string.IsNullOrEmpty(currentDirectory))
+ {
+ Environment.CurrentDirectory = currentDirectory;
+ }
+ }
+
+ static ILogger SetupLogging(IConsole console, LogLevel minimalLogLevel, LogLevel minimalErrorLevel)
+ {
+ var serviceCollection = new ServiceCollection();
+ serviceCollection.AddSingleton(new LoggerFactory().AddSimpleConsole(console, minimalLogLevel, minimalErrorLevel));
+ serviceCollection.AddLogging();
+
+ var serviceProvider = serviceCollection.BuildServiceProvider();
+ var logger = serviceProvider.GetService>();
+
+ return logger!;
+ }
+
+ static LogLevel GetLogLevel(string? verbosity)
+ => verbosity switch
+ {
+ "q" or "quiet" => LogLevel.Error,
+ "m" or "minimal" => LogLevel.Warning,
+ "n" or "normal" => LogLevel.Information,
+ "d" or "detailed" => LogLevel.Debug,
+ "diag" or "diagnostic" => LogLevel.Trace,
+ _ => LogLevel.Information,
+ };
+ }
+ }
+}