diff --git a/.gitignore b/.gitignore index 2a0a37eae9..8c87aa3f57 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,7 @@ obj/ .dotnet/ .tools/ .packages/ +temp/ # Per-user project properties launchSettings.json diff --git a/Perf.cmd b/Perf.cmd new file mode 100644 index 0000000000..76a5e3b5f0 --- /dev/null +++ b/Perf.cmd @@ -0,0 +1,3 @@ +@echo off +powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0eng\perf\PerfCore.ps1""" %*" +exit /b %ErrorLevel% \ No newline at end of file diff --git a/RoslynAnalyzers.sln b/RoslynAnalyzers.sln index 550d09e20e..7e72ce605d 100644 --- a/RoslynAnalyzers.sln +++ b/RoslynAnalyzers.sln @@ -1,4 +1,5 @@ -Microsoft Visual Studio Solution File, Format Version 12.00 + +Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio Version 16 VisualStudioVersion = 16.0.28531.58 MinimumVisualStudioVersion = 10.0.40219.1 @@ -188,11 +189,11 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CommonPerfUtilities", "src\ EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "VisualBasicPerfUtilities", "src\PerformanceTests\Utilities\VisualBasic\VisualBasicPerfUtilities.csproj", "{8D764DDE-3762-4218-8BC4-CC1048095D5F}" EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{E1A6F359-DF41-4E51-9D4F-2E6DDF135BA7}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "PerformanceTests", "src\PerformanceTests\Tests\PerformanceTests.csproj", "{4FE82B4D-4AF1-4076-9E85-529205866A80}" EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "PerformanceTests", "PerformanceTests", "{9CD62A3C-A138-4607-9231-5E2266AB59BC}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "PerfDiff", "src\Tools\PerfDiff\PerfDiff.csproj", "{AE549290-1702-4ECE-AE57-5FF4E85492C6}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PerformanceTests", "src\PerformanceTests\Tests\PerformanceTests.csproj", "{4FE82B4D-4AF1-4076-9E85-529205866A80}" +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "PerfDiff", "PerfDiff", "{BEE4B62A-86BD-4613-8F94-F40364350643}" EndProject Global GlobalSection(SharedMSBuildProjectFiles) = preSolution @@ -498,6 +499,10 @@ Global {4FE82B4D-4AF1-4076-9E85-529205866A80}.Debug|Any CPU.Build.0 = Debug|Any CPU {4FE82B4D-4AF1-4076-9E85-529205866A80}.Release|Any CPU.ActiveCfg = Release|Any CPU {4FE82B4D-4AF1-4076-9E85-529205866A80}.Release|Any CPU.Build.0 = Release|Any CPU + {AE549290-1702-4ECE-AE57-5FF4E85492C6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {AE549290-1702-4ECE-AE57-5FF4E85492C6}.Debug|Any CPU.Build.0 = Debug|Any CPU + {AE549290-1702-4ECE-AE57-5FF4E85492C6}.Release|Any CPU.ActiveCfg = Release|Any CPU + {AE549290-1702-4ECE-AE57-5FF4E85492C6}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -579,8 +584,9 @@ Global {6A3B1816-97D3-4D30-8C6F-2F2DC288410B} = {43852421-59D7-4D63-977E-F862E1CF467D} {8EC7A06A-01F2-49CF-9C45-4357AAD82546} = {43852421-59D7-4D63-977E-F862E1CF467D} {8D764DDE-3762-4218-8BC4-CC1048095D5F} = {43852421-59D7-4D63-977E-F862E1CF467D} - {9CD62A3C-A138-4607-9231-5E2266AB59BC} = {E1A6F359-DF41-4E51-9D4F-2E6DDF135BA7} - {4FE82B4D-4AF1-4076-9E85-529205866A80} = {9CD62A3C-A138-4607-9231-5E2266AB59BC} + {4FE82B4D-4AF1-4076-9E85-529205866A80} = {CE87A0B7-E19B-4230-A73C-60F6D65EB7DC} + {AE549290-1702-4ECE-AE57-5FF4E85492C6} = {BEE4B62A-86BD-4613-8F94-F40364350643} + {BEE4B62A-86BD-4613-8F94-F40364350643} = {C0B86774-8307-444F-9EE4-98D62C3424F9} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {FC44ACA9-AEA3-4EE6-881C-2E08ED281B5F} diff --git a/eng/Versions.props b/eng/Versions.props index 595f268343..22db0f3b36 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -42,14 +42,20 @@ 1.0.1-beta1.21202.2 - 0.12.1 + 0.13.0 2.2.0 1.1.2 + 2.0.69 + 6.0.0-preview.5.21301.5 6.0.0-preview.4.21253.7 10.1.0 16.1.8 + 12.0.1 + 0.2.1 1.1.2 1.3.1 + 2.0.0-beta1.20074.1 + 2.0.0-beta1.21216.1 4.7.0 4.7.0 1.4.2 diff --git a/eng/perf/CIPerf.cmd b/eng/perf/CIPerf.cmd new file mode 100644 index 0000000000..88ca610edc --- /dev/null +++ b/eng/perf/CIPerf.cmd @@ -0,0 +1,3 @@ +@echo off +powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0PerfCore.ps1""" -v diag -etl -diff %*" +exit /b %ErrorLevel% \ No newline at end of file diff --git a/eng/perf/ComparePerfResults.ps1 b/eng/perf/ComparePerfResults.ps1 new file mode 100644 index 0000000000..79363b11c3 --- /dev/null +++ b/eng/perf/ComparePerfResults.ps1 @@ -0,0 +1,38 @@ +[CmdletBinding(PositionalBinding=$false)] +Param( + [String] $baseline, # folder that contains the baseline results + [String] $results # folder that contains the performance results + ) + +function EnsureFolder { +param ( + [String] $path +) + If(!(test-path $path)) + { + New-Item -ItemType Directory -Force -Path $path + } +} + +$currentLocation = Get-Location +try { + + #Get result files + $baselineFolder = Join-Path $baseline "results" + + $resultsFolder = Join-Path $results "results" + + $RepoRoot = Resolve-Path (Join-Path $PSScriptRoot '..\..') + $perfDiff = Join-Path $RepoRoot "src\Tools\PerfDiff\PerfDiff.csproj" + Invoke-Expression "dotnet run -c Release --project $perfDiff -- --baseline $baselineFolder --results $resultsFolder --failOnRegression --verbosity diag" +} +catch { + Write-Host $_ + Write-Host $_.Exception + Write-Host $_.ScriptStackTrace + $host.SetShouldExit(1) + exit +} +finally { + Set-Location $currentLocation +} \ No newline at end of file diff --git a/eng/perf/DiffPerfToBaseLine.ps1 b/eng/perf/DiffPerfToBaseLine.ps1 new file mode 100644 index 0000000000..20f58f32b5 --- /dev/null +++ b/eng/perf/DiffPerfToBaseLine.ps1 @@ -0,0 +1,86 @@ +[CmdletBinding(PositionalBinding=$false)] +Param( + [String] $baselineSHA, # git SHA to use as the baseline for performance + [String] $testSHA, # git SHA to use as the test for performance + [String] $output, # common folder to write the benchmark results to + [string] $projects, # semicolon separated list of relative paths to benchmark projects to run + [string] $filter, # filter for tests to run (supports wildcards) + [switch] $etl, # capture etl traces for performance tests + [switch] $ci # run in ci mode (fail fast an keep all partial artifacts) + ) + +function EnsureFolder { + param ( + [String] $path # path to create if it does not exist + ) + If(!(test-path $path)) + { + New-Item -ItemType Directory -Force -Path $path + } +} + +function RunTest { + param ( + [String] $RunPerfTests, # path to the RunPerfTests.ps1 script + [String] $output, # common folder to write the benchmark results to + [String] $resultsFolder, # folder name to write the benchmark results to + [String] $temp, # root folder to check repos into + [String] $repoName, # folder name to place checked-out repo in + [String] $repoSHA, # git SHA run performance tests against + [String] $projects, # semicolon separated list of relative paths to benchmark projects to run + [String] $filter, # filter for tests to run (supports wildcards) + [bool] $etl, # capture etl traces for performance tests + [bool] $ci # run in ci mode (fail fast an keep all partial artifacts) + ) + + # Ensure output directory has been created + $resultsOutput = Join-Path $output $resultsFolder + EnsureFolder $resultsOutput + + # Checkout SHA + $repoFolder = Join-Path $Temp $repoName + Invoke-Expression "git worktree add $repoFolder $repoSHA" + + $commandArgs = "-root $repoFolder -output $resultsOutput -projects $projects -filter $filter" + if ($etl) { + $commandArgs = "$commandArgs -etl" + } + + if ($ci) { + $commandArgs = "$commandArgs -ci" + } + + Invoke-Expression "$RunPerfTests $commandArgs" +} + +$currentLocation = Get-Location + +# Setup paths +$RepoRoot = Resolve-Path (Join-Path $PSScriptRoot '..\..') +$RunPerfTests = Join-Path $PSScriptRoot "RunPerfTests.ps1" +$ComparePerfResults = Join-Path $PSScriptRoot "ComparePerfResults.ps1" +$Temp = Join-Path $RepoRoot "temp" + +try { + # Get baseline results + RunTest -RunPerfTests $RunPerfTests -output $output -resultsFolder "baseline" -temp $Temp -repoName "perfBaseline" -repoSHA $baselineSHA -projects $projects -filter $filter -etl $etl.IsPresent -ci $ci.IsPresent + + # Get perf results + RunTest -RunPerfTests $RunPerfTests -output $output -resultsFolder "perfTest" -temp $Temp -repoName "perfTest" -repoSHA $testSHA -projects $projects -filter $filter -etl $etl.IsPresent -ci $ci.IsPresent + + # Diff perf results + Invoke-Expression "$ComparePerfResults -baseline $baselineOutput -results $testOutput" +} +catch { + Write-Host $_ + Write-Host $_.Exception + Write-Host $_.ScriptStackTrace + $host.SetShouldExit(1) + exit +} +finally { + Invoke-Expression 'git worktree remove perfBaseline' + Invoke-Expression 'git worktree remove perfTest' + Invoke-Expression 'git worktree prune' + Set-Location $currentLocation +} \ No newline at end of file diff --git a/eng/perf/PerfCore.ps1 b/eng/perf/PerfCore.ps1 new file mode 100644 index 0000000000..5cc38e3ba8 --- /dev/null +++ b/eng/perf/PerfCore.ps1 @@ -0,0 +1,86 @@ +[CmdletBinding(PositionalBinding=$false)] +Param( + [string] $projects, + [string][Alias('v')]$verbosity = "minimal", + [string] $filter, + [switch] $etl, + [switch] $diff, + [switch] $ci, + [switch] $help, + [Parameter(ValueFromRemainingArguments=$true)][String[]]$properties + ) + + function Print-Usage() { + Write-Host "Common settings:" + Write-Host " -verbosity Msbuild verbosity: q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic] (short: -v)" + Write-Host " -help Print help and exit" + Write-Host "" + + Write-Host "Actions:" + Write-Host " -diff Compare HEAD to baseline perf results" + Write-Host "" + + Write-Host "Advanced settings:" + Write-Host " -etl Capture ETL traces of performance tests (requires admin permissions, default value is 'false')" + Write-Host " -filter Filter for tests to run (supports wildcards)" + Write-Host " -projects Semi-colon delimited list of relative paths to benchmark projects." + Write-Host "" + + Write-Host "Command line arguments not listed above are passed thru to msbuild." + Write-Host "The above arguments can be shortened as much as to be unambiguous (e.g. -co for configuration, -t for test, etc.)." + } + +try { + if ($help -or (($null -ne $properties) -and ($properties.Contains('/help') -or $properties.Contains('/?')))) { + Print-Usage + exit 0 + } + + if ([string]::IsNullOrWhiteSpace($projects)) { + $projects = "src\PerformanceTests\Tests\PerformanceTests.csproj" + } + + if ([string]::IsNullOrWhiteSpace($filter)) { + $filter = "*" + } + + $RepoRoot = Resolve-Path (Join-Path $PSScriptRoot '..\..') + $output = Join-Path $RepoRoot "artifacts\performance\perfResults" + + # Diff two different SHAs + if ($diff) { + $DiffPerfToBaseLine = Join-Path $RepoRoot "eng\perf\DiffPerfToBaseLine.ps1" + $baselinejson = Get-Content -Raw -Path (Join-Path $RepoRoot "eng\perf\baseline.json") | ConvertFrom-Json + $baselineSHA = $baselinejson.sha + $commandArguments = "-baselineSHA $baselineSHA -testSHA HEAD -projects '$projects' -output '$output' -filter '$filter'" + if ($etl) { + $commandArguments = "$commandArguments -etl" + } + if ($ci) { + $commandArguments = "$commandArguments -ci" + } + Invoke-Expression "$DiffPerfToBaseLine $commandArguments" + exit + } + + $commandArguments = "-projects '$projects' -root '$RepoRoot' -output '$output\perfTest' -filter '$filter'" + if ($etl) { + $commandArguments = "$commandArguments -etl" + } + if ($ci) { + $commandArguments = "$commandArguments -ci" + } + + $RunPerfTests = Join-Path $RepoRoot "eng\perf\RunPerfTests.ps1" + Invoke-Expression "$RunPerfTests $commandArguments" + exit +} +catch { + Write-Host $_ + Write-Host $_.Exception + Write-Host $_.ScriptStackTrace + $host.SetShouldExit(1) + exit +} +finally { +} \ No newline at end of file diff --git a/eng/perf/RunPerfTests.ps1 b/eng/perf/RunPerfTests.ps1 new file mode 100644 index 0000000000..1943b2da33 --- /dev/null +++ b/eng/perf/RunPerfTests.ps1 @@ -0,0 +1,35 @@ +[CmdletBinding(PositionalBinding=$false)] +Param( + [string] $projects, # semicolon separated list of relative paths to benchmark projects to run + [string] $filter, # filter for tests to run (supports wildcards) + [String] $root, # root folder all of the benchmark projects share + [String] $output, # folder to write the benchmark results to + [switch] $etl, # capture etl traces for performance tests + [switch] $ci # run in ci mode (fail fast an keep all partial artifacts) + ) + +try { + $projectsList = $projects -split ";" + foreach ($project in $projectsList){ + $projectFullPath = Join-Path $root $project + $comandArguments = "run -c Release --project $projectFullPath -- --memory --exporters JSON --artifacts $output" + if ($ci) { + $comandArguments = "$comandArguments --stopOnFirstError --keepFiles" + } + if ($etl) { + Start-Process -Wait -FilePath "dotnet" -Verb RunAs -ArgumentList "$comandArguments --profiler ETW --filter $filter" + } + else { + Invoke-Expression "dotnet $comandArguments --filter $filter" + } + } +} +catch { + Write-Host $_ + Write-Host $_.Exception + Write-Host $_.ScriptStackTrace + $host.SetShouldExit(1) + exit +} +finally { +} \ No newline at end of file diff --git a/eng/perf/baseline.json b/eng/perf/baseline.json new file mode 100644 index 0000000000..289e28bc3c --- /dev/null +++ b/eng/perf/baseline.json @@ -0,0 +1,5 @@ +{ + "release": "6.0", + "label": "roslyn analyzers baseline for main", + "sha": "f986c7b7aff265feb7536e1c7f98efcb9f14e077" +} \ No newline at end of file diff --git a/src/Tools/PerfDiff/.editorconfig b/src/Tools/PerfDiff/.editorconfig new file mode 100644 index 0000000000..4032e86375 --- /dev/null +++ b/src/Tools/PerfDiff/.editorconfig @@ -0,0 +1,8 @@ +### Configuration for PublicAPI analyzers executed on this folder ### +[*.{cs,vb}] + +dotnet_diagnostic.IDE1030.severity = none +dotnet_diagnostic.CA1052.severity = none +dotnet_diagnostic.CA1063.severity = none +dotnet_diagnostic.CA1724.severity = none +dotnet_diagnostic.CA1819.severity = none diff --git a/src/Tools/PerfDiff/BDN/BenchmarkDotNetDiffer.cs b/src/Tools/PerfDiff/BDN/BenchmarkDotNetDiffer.cs new file mode 100644 index 0000000000..7e57c4410a --- /dev/null +++ b/src/Tools/PerfDiff/BDN/BenchmarkDotNetDiffer.cs @@ -0,0 +1,188 @@ +// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using DataTransferContracts; +using Microsoft.Extensions.Logging; +using Newtonsoft.Json; +using Perfolizer.Mathematics.SignificanceTesting; +using Perfolizer.Mathematics.Thresholds; + +namespace PerfDiff +{ + public static class BenchmarkDotNetDiffer + { + public static async Task<(bool success, bool shouldCheckETL)> TryCompareBenchmarkDotNetResultsAsync(string baselineFolder, string resultsFolder, ILogger logger) + { + bool shouldCheckETL = false; + + // search folder for benchmark dotnet results + var comparison = await TryGetBdnResultsAsync(baselineFolder, resultsFolder, logger).ConfigureAwait(false); + if (comparison is null) + { + return (false, shouldCheckETL); + } + + // compare bdn results + // TODO: let these be optional parameters + _ = Threshold.TryParse("15%", out var testThreshold); + _ = Threshold.TryParse("0.3ns", out var noiseThreshold); + + var notSame = FindRegressions(comparison, testThreshold, noiseThreshold); + + if (!notSame.Any()) + { + logger.LogInformation($"No differences found between the benchmark results with threshold {testThreshold}."); + return (false, shouldCheckETL); + } + + var better = notSame.Where(result => result.conclusion == EquivalenceTestConclusion.Faster); + var worse = notSame.Where(result => result.conclusion == EquivalenceTestConclusion.Slower); + var betterCount = better.Count(); + int worseCount = worse.Count(); + + // If the baseline doesn't have the same set of tests, you wind up with Infinity in the list of diffs. + // Exclude them for purposes of geomean. + worse = worse.Where(x => GetRatio(x) != double.PositiveInfinity); + better = better.Where(x => GetRatio(x) != double.PositiveInfinity); + + if (betterCount > 0) + { + var betterGeoMean = Math.Pow(10, better.Skip(1).Aggregate(Math.Log10(GetRatio(better.First())), (x, y) => x + Math.Log10(GetRatio(y))) / better.Count()); + logger.LogInformation($"better: {betterCount}, geomean: {betterGeoMean:F3}"); + } + + if (worseCount > 0) + { + var worseGeoMean = Math.Pow(10, worse.Skip(1).Aggregate(Math.Log10(GetRatio(worse.First())), (x, y) => x + Math.Log10(GetRatio(y))) / worse.Count()); + logger.LogInformation($"worse: {worseCount}, geomean: {worseGeoMean:F3}"); + } + + if (worseCount > 0) + { + shouldCheckETL = true; + } + + return (true, shouldCheckETL); + } + + private static double GetRatio((string id, Benchmark baseResult, Benchmark diffResult, EquivalenceTestConclusion conclusion) item) => GetRatio(item.conclusion, item.baseResult, item.diffResult); + + private static double GetRatio(EquivalenceTestConclusion conclusion, Benchmark baseResult, Benchmark diffResult) + => conclusion == EquivalenceTestConclusion.Faster + ? baseResult.Statistics.Median / diffResult.Statistics.Median + : diffResult.Statistics.Median / baseResult.Statistics.Median; + + private static async Task<(string id, Benchmark baseResult, Benchmark diffResult)[]?> TryGetBdnResultsAsync( + string baselineFolder, + string resultsFolder, + ILogger logger) + { + if (!TryGetFilesToParse(baselineFolder, out var baseFiles)) + { + logger.LogError($"Provided path does NOT exist or does not contain perf results '{baselineFolder}'"); + return null; + } + + if (!TryGetFilesToParse(resultsFolder, out var resultsFiles)) + { + logger.LogError($"Provided path does NOT exist or does not contain perf results '{resultsFolder}'"); + return null; + } + + if (!baseFiles.Any() || !resultsFiles.Any()) + { + logger.LogError($"Provided paths contained no '{FullBdnJsonFileExtension}' files."); + return null; + } + + var (baseResultsSuccess, baseResults) = await TryGetBdnResultAsync(baseFiles, logger).ConfigureAwait(false); + if (!baseResultsSuccess) + { + return null; + } + + var (resultsSuccess, diffResults) = await TryGetBdnResultAsync(resultsFiles, logger).ConfigureAwait(false); + if (!resultsSuccess) + { + return null; + } + + var benchmarkIdToDiffResults = diffResults + .SelectMany(result => result.Benchmarks) + .ToDictionary(benchmarkResult => benchmarkResult.DisplayInfo, benchmarkResult => benchmarkResult); + + return baseResults + .SelectMany(result => result.Benchmarks) + .ToDictionary(benchmarkResult => benchmarkResult.DisplayInfo, benchmarkResult => benchmarkResult) // we use ToDictionary to make sure the results have unique IDs + .Where(baseResult => benchmarkIdToDiffResults.ContainsKey(baseResult.Key)) + .Select(baseResult => (id: baseResult.Key, baseResult: baseResult.Value, diffResult: benchmarkIdToDiffResults[baseResult.Key])) + .ToArray(); + } + + private const string FullBdnJsonFileExtension = "full-compressed.json"; + + private static bool TryGetFilesToParse(string path, [NotNullWhen(true)] out string[]? files) + { + if (Directory.Exists(path)) + { + files = Directory.GetFiles(path, $"*{FullBdnJsonFileExtension}", SearchOption.AllDirectories); + return true; + } + else if (File.Exists(path) || !path.EndsWith(FullBdnJsonFileExtension, StringComparison.OrdinalIgnoreCase)) + { + files = new[] { path }; + return true; + } + + files = null; + return false; + } + + private static async Task<(bool success, BdnResult[] results)> TryGetBdnResultAsync(string[] paths, ILogger logger) + { + var results = await Task.WhenAll(paths.Select(path => ReadFromFileAsync(path, logger))).ConfigureAwait(false); + return (!results.Any(x => x is null), results)!; + } + + private static async Task ReadFromFileAsync(string resultFilePath, ILogger logger) + { + try + { + return JsonConvert.DeserializeObject(await File.ReadAllTextAsync(resultFilePath).ConfigureAwait(false)); + } + catch (JsonSerializationException) + { + logger.LogError($"Exception while reading the {resultFilePath} file."); + return null; + } + } + + private static (string id, Benchmark baseResult, Benchmark diffResult, EquivalenceTestConclusion conclusion)[] FindRegressions((string id, Benchmark baseResult, Benchmark diffResult)[] comparison, Threshold testThreshold, Threshold noiseThreshold) + { + var results = new List<(string id, Benchmark baseResult, Benchmark diffResult, EquivalenceTestConclusion conclusion)>(); + foreach ((string id, Benchmark baseResult, Benchmark diffResult) in comparison + .Where(result => result.baseResult.Statistics != null && result.diffResult.Statistics != null)) // failures + { + var baseValues = baseResult.GetOriginalValues(); + var diffValues = diffResult.GetOriginalValues(); + + var userTresholdResult = StatisticalTestHelper.CalculateTost(MannWhitneyTest.Instance, baseValues, diffValues, testThreshold); + if (userTresholdResult.Conclusion == EquivalenceTestConclusion.Same) + continue; + + var noiseResult = StatisticalTestHelper.CalculateTost(MannWhitneyTest.Instance, baseValues, diffValues, noiseThreshold); + if (noiseResult.Conclusion == EquivalenceTestConclusion.Same) + continue; + + results.Add((id, baseResult, diffResult, conclusion: userTresholdResult.Conclusion)); + } + + return results.ToArray(); + } + } +} diff --git a/src/Tools/PerfDiff/BDN/DataContracts/ChronometerFrequency.cs b/src/Tools/PerfDiff/BDN/DataContracts/ChronometerFrequency.cs new file mode 100644 index 0000000000..b1c5050c64 --- /dev/null +++ b/src/Tools/PerfDiff/BDN/DataContracts/ChronometerFrequency.cs @@ -0,0 +1,133 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +// + +using System.Collections.Generic; +using System.Linq; + +namespace DataTransferContracts // generated with http://json2csharp.com/# +{ + public class ChronometerFrequency + { + public int Hertz { get; set; } + } + + public class HostEnvironmentInfo + { + public string BenchmarkDotNetCaption { get; set; } + public string BenchmarkDotNetVersion { get; set; } + public string OsVersion { get; set; } + public string ProcessorName { get; set; } + public int? PhysicalProcessorCount { get; set; } + public int? PhysicalCoreCount { get; set; } + public int? LogicalCoreCount { get; set; } + public string RuntimeVersion { get; set; } + public string Architecture { get; set; } + public bool? HasAttachedDebugger { get; set; } + public bool? HasRyuJit { get; set; } + public string Configuration { get; set; } + public string JitModules { get; set; } + public string DotNetCliVersion { get; set; } + public ChronometerFrequency ChronometerFrequency { get; set; } + public string HardwareTimerKind { get; set; } + } + + public class ConfidenceInterval + { + public int N { get; set; } + public double Mean { get; set; } + public double StandardError { get; set; } + public int Level { get; set; } + public double Margin { get; set; } + public double Lower { get; set; } + public double Upper { get; set; } + } + + public class Percentiles + { + public double P0 { get; set; } + public double P25 { get; set; } + public double P50 { get; set; } + public double P67 { get; set; } + public double P80 { get; set; } + public double P85 { get; set; } + public double P90 { get; set; } + public double P95 { get; set; } + public double P100 { get; set; } + } + + public class Statistics + { + public int N { get; set; } + public double Min { get; set; } + public double LowerFence { get; set; } + public double Q1 { get; set; } + public double Median { get; set; } + public double Mean { get; set; } + public double Q3 { get; set; } + public double UpperFence { get; set; } + public double Max { get; set; } + public double InterquartileRange { get; set; } + public List LowerOutliers { get; set; } + public List UpperOutliers { get; set; } + public List AllOutliers { get; set; } + public double StandardError { get; set; } + public double Variance { get; set; } + public double StandardDeviation { get; set; } + public double? Skewness { get; set; } + public double? Kurtosis { get; set; } + public ConfidenceInterval ConfidenceInterval { get; set; } + public Percentiles Percentiles { get; set; } + } + + public class Memory + { + public int Gen0Collections { get; set; } + public int Gen1Collections { get; set; } + public int Gen2Collections { get; set; } + public long TotalOperations { get; set; } + public long BytesAllocatedPerOperation { get; set; } + } + + public class Measurement + { + public string IterationStage { get; set; } + public int LaunchIndex { get; set; } + public int IterationIndex { get; set; } + public long Operations { get; set; } + public double Nanoseconds { get; set; } + } + + public class Benchmark + { + public string DisplayInfo { get; set; } + public object Namespace { get; set; } + public string Type { get; set; } + public string Method { get; set; } + public string MethodTitle { get; set; } + public string Parameters { get; set; } + public string FullName { get; set; } + public Statistics Statistics { get; set; } + public Memory Memory { get; set; } + public List Measurements { get; set; } + + /// + /// this method was not auto-generated by a tool, it was added manually + /// + /// an array of the actual workload results (not warmup, not pilot) + internal double[] GetOriginalValues() + => Measurements + .Where(measurement => measurement.IterationStage == "Result") + .Select(measurement => measurement.Nanoseconds / measurement.Operations) + .ToArray(); + } + + public class BdnResult + { + public string Title { get; set; } + public HostEnvironmentInfo HostEnvironmentInfo { get; set; } + public List Benchmarks { get; set; } + } +} diff --git a/src/Tools/PerfDiff/DiffCommand.cs b/src/Tools/PerfDiff/DiffCommand.cs new file mode 100644 index 0000000000..2fb4de5008 --- /dev/null +++ b/src/Tools/PerfDiff/DiffCommand.cs @@ -0,0 +1,37 @@ +// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.CommandLine; +using System.Threading.Tasks; + +namespace PerfDiff +{ + internal static class DiffCommand + { + // This delegate should be kept in Sync with the FormatCommand options and argument names + // so that values bind correctly. + internal delegate Task Handler( + string baseline, + string results, + string? verbosity, + bool failOnRegression, + IConsole console); + + internal static string[] VerbosityLevels => new[] { "q", "quiet", "m", "minimal", "n", "normal", "d", "detailed", "diag", "diagnostic" }; + + internal static RootCommand CreateCommandLineOptions() + { + // Sync changes to option and argument names with the FormatCommant.Handler above. + var rootCommand = new RootCommand + { + new Option("--baseline", () => null, "folder that contains the baseline performance run data").LegalFilePathsOnly(), + new Option("--results", () => null, "folder that contains the performance restults").LegalFilePathsOnly(), + new Option(new[] { "--verbosity", "-v" }, "Set the verbosity level. Allowed values are q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic]").FromAmong(VerbosityLevels), + new Option(new[] { "--failOnRegression" }, "Should return non-zero exit code if regression detected"), + }; + + rootCommand.Description = "diff two sets of performance results"; + + return rootCommand; + } + } +} diff --git a/src/Tools/PerfDiff/ETL/EtlDiffer.cs b/src/Tools/PerfDiff/ETL/EtlDiffer.cs new file mode 100644 index 0000000000..62d2b74303 --- /dev/null +++ b/src/Tools/PerfDiff/ETL/EtlDiffer.cs @@ -0,0 +1,169 @@ +// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.IO; +using System.Linq; + +using Microsoft.Diagnostics.Symbols; +using Microsoft.Diagnostics.Tracing.Etlx; +using Microsoft.Diagnostics.Tracing.Parsers.Kernel; +using Microsoft.Diagnostics.Tracing.Stacks; + +namespace PerfDiff +{ + internal static class EtlDiffer + { + public static bool TryCompareETL(string sourceEtlPath, string baselineEtlPath, out bool regression) + { + regression = false; + CallTree sourceCallTree = GetCallTree(sourceEtlPath); + CallTree baselineCallTree = GetCallTree(baselineEtlPath); + var report = GenerateOverweightReport(sourceCallTree, baselineCallTree); + + // print results + Console.WriteLine(string.Join(Environment.NewLine, report.Take(10))); + return true; + } + + private static CallTree GetCallTree(string eltPath) + { + var traceProcess = GetTraceProcessFromETLFile(eltPath); + var stackSource = CreateStackSourceFromTraceProcess(traceProcess); + return CreateCallTreeFromStackSource(stackSource); + } + + public static TraceProcess GetTraceProcessFromETLFile(string eltPath) + { + var traceLog = TraceLog.OpenOrConvert(eltPath); + return traceLog.Processes + .First(p => p.Name.Equals("dotnet", StringComparison.OrdinalIgnoreCase)); + } + + public static StackSource CreateStackSourceFromTraceProcess(TraceProcess process) + { + var events = process.EventsInProcess; + var start = Math.Max(events.StartTimeRelativeMSec, process.StartTimeRelativeMsec); + var end = Math.Min(events.EndTimeRelativeMSec, process.EndTimeRelativeMsec); + events = events.FilterByTime(start, end); + events = events.Filter(x => x is SampledProfileTraceData && x.ProcessID == process.ProcessID); + + using var symbolReader = new SymbolReader(new StringWriter(), @"SRV*https://msdl.microsoft.com/download/symbols"); + symbolReader.SecurityCheck = path => true; + + var traceLog = process.Log; + foreach (var module in process.LoadedModules) + { + traceLog.CodeAddresses.LookupSymbolsForModule(symbolReader, module.ModuleFile); + } + + return new TraceEventStackSource(events); + } + + public static CallTree CreateCallTreeFromStackSource(StackSource stackSource) + { + var calltree = new CallTree(ScalingPolicyKind.ScaleToData); + calltree.StackSource = stackSource; + return calltree; + } + + public static ImmutableArray GenerateOverweightReport(CallTree source, CallTree baseline) + { + var sourceTotal = LoadTrace(source, out var sourceData); + var baselineTotal = LoadTrace(baseline, out var baselineData); + + if (sourceTotal != baselineTotal) + { + return ComputeOverweights(sourceTotal, sourceData, baselineTotal, baselineData); + } + + return ImmutableArray.Empty; + + static float LoadTrace(CallTree callTree, out Dictionary data) + { + data = new Dictionary(); + float total = 0; + foreach (var node in callTree.ByID) + { + if (node.InclusiveMetric == 0) + { + continue; + } + + string key = node.Name; + data.TryGetValue(key, out var weight); + data[key] = weight + node.InclusiveMetric; + + total += node.ExclusiveMetric; + } + return total; + } + + static ImmutableArray ComputeOverweights(float sourceTotal, Dictionary sourceData, float baselineTotal, Dictionary baselineData) + { + var totalDelta = sourceTotal - baselineTotal; + var growth = sourceTotal / baselineTotal; + var results = ImmutableArray.CreateBuilder(); + foreach (var key in baselineData.Keys) + { + // skip symbols that are not in both traces + if (!sourceData.ContainsKey(key)) + { + continue; + } + + var baselineValue = baselineData[key]; + var sourceValue = sourceData[key]; + var expectedDelta = baselineValue * (growth - 1); + var delta = sourceValue - baselineValue; + var overweight = delta / expectedDelta * 100; + var percent = delta / totalDelta; + // Calculate interest level + var interest = Math.Abs(overweight) > 110 ? 1 : 0; + interest += Math.Abs(percent) > 5 ? 1 : 0; + interest += Math.Abs(percent) > 20 ? 1 : 0; + interest += Math.Abs(percent) > 100 ? 1 : 0; + interest += sourceValue / sourceTotal < 0.95 ? 1 : 0; // Ignore top of the stack frames + interest += sourceValue / sourceTotal < 0.75 ? 1 : 0; // Bonus point for being further down the stack. + + results.Add(new OverWeightResult + ( + Name: key, + Before: baselineValue, + After: sourceValue, + Delta: delta, + Overweight: overweight, + Percent: percent, + Interest: interest + )); + } + + results.Sort((left, right) => + { + if (left.Interest < right.Interest) + return 1; + + if (left.Interest > right.Interest) + return -1; + + if (left.Overweight < right.Overweight) + return 1; + + if (left.Overweight > right.Overweight) + return -1; + + if (left.Delta < right.Delta) + return -1; + + if (left.Delta > right.Delta) + return 1; + + return 0; + }); + + return results.ToImmutable(); + } + } + } +} diff --git a/src/Tools/PerfDiff/ETL/OverWeightResult.cs b/src/Tools/PerfDiff/ETL/OverWeightResult.cs new file mode 100644 index 0000000000..4c0bfd55be --- /dev/null +++ b/src/Tools/PerfDiff/ETL/OverWeightResult.cs @@ -0,0 +1,10 @@ +// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace PerfDiff +{ + internal record OverWeightResult(string Name, float Before, float After, float Delta, float Overweight, float Percent, int Interest) + { + public override string ToString() + => $"'{Name}':, Overweight: '{Overweight}%', Before: '{Before}ms', After: '{After}ms', Interest :'{Interest}'"; + } +} diff --git a/src/Tools/PerfDiff/Logging/NullScope.cs b/src/Tools/PerfDiff/Logging/NullScope.cs new file mode 100644 index 0000000000..849b4923a0 --- /dev/null +++ b/src/Tools/PerfDiff/Logging/NullScope.cs @@ -0,0 +1,19 @@ +// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; + +namespace PerfDiff.Logging +{ + internal class NullScope : IDisposable + { + public static NullScope Instance { get; } = new NullScope(); + + private NullScope() + { + } + + public void Dispose() + { + } + } +} diff --git a/src/Tools/PerfDiff/Logging/SimpleConsoleLogger.cs b/src/Tools/PerfDiff/Logging/SimpleConsoleLogger.cs new file mode 100644 index 0000000000..5924734f38 --- /dev/null +++ b/src/Tools/PerfDiff/Logging/SimpleConsoleLogger.cs @@ -0,0 +1,95 @@ +// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Microsoft.Extensions.Logging; + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.CommandLine; +using System.CommandLine.Rendering; + +namespace PerfDiff.Logging +{ + internal class SimpleConsoleLogger : ILogger + { + private readonly object _gate = new(); + + private readonly IConsole _console; + private readonly ITerminal _terminal; + private readonly LogLevel _minimalLogLevel; + private readonly LogLevel _minimalErrorLevel; + + private static ImmutableDictionary LogLevelColorMap => new Dictionary + { + [LogLevel.Critical] = ConsoleColor.Red, + [LogLevel.Error] = ConsoleColor.Red, + [LogLevel.Warning] = ConsoleColor.Yellow, + [LogLevel.Information] = ConsoleColor.White, + [LogLevel.Debug] = ConsoleColor.Gray, + [LogLevel.Trace] = ConsoleColor.Gray, + [LogLevel.None] = ConsoleColor.White, + }.ToImmutableDictionary(); + + public SimpleConsoleLogger(IConsole console, LogLevel minimalLogLevel, LogLevel minimalErrorLevel) + { + _terminal = console.GetTerminal(); + _console = console; + _minimalLogLevel = minimalLogLevel; + _minimalErrorLevel = minimalErrorLevel; + } + + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) + { + if (!IsEnabled(logLevel)) + { + return; + } + + lock (_gate) + { + var message = formatter(state, exception); + var logToErrorStream = logLevel >= _minimalErrorLevel; + if (_terminal is null) + { + LogToConsole(_console, message, logToErrorStream); + } + else + { + LogToTerminal(message, logLevel, logToErrorStream); + } + } + } + + public bool IsEnabled(LogLevel logLevel) + { + return (int)logLevel >= (int)_minimalLogLevel; + } + + public IDisposable BeginScope(TState state) + { + return NullScope.Instance; + } + + private void LogToTerminal(string message, LogLevel logLevel, bool logToErrorStream) + { + var messageColor = LogLevelColorMap[logLevel]; + _terminal.ForegroundColor = messageColor; + + LogToConsole(_terminal, message, logToErrorStream); + + _terminal.ResetColor(); + } + + private static void LogToConsole(IConsole console, string message, bool logToErrorStream) + { + if (logToErrorStream) + { + console.Error.Write($"{message}{Environment.NewLine}"); + } + else + { + console.Out.Write($" {message}{Environment.NewLine}"); + } + } + } +} diff --git a/src/Tools/PerfDiff/Logging/SimpleConsoleLoggerFactoryExtensions.cs b/src/Tools/PerfDiff/Logging/SimpleConsoleLoggerFactoryExtensions.cs new file mode 100644 index 0000000000..e90891919c --- /dev/null +++ b/src/Tools/PerfDiff/Logging/SimpleConsoleLoggerFactoryExtensions.cs @@ -0,0 +1,17 @@ +// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.CommandLine; + +using Microsoft.Extensions.Logging; + +namespace PerfDiff.Logging +{ + internal static class SimpleConsoleLoggerFactoryExtensions + { + public static ILoggerFactory AddSimpleConsole(this ILoggerFactory factory, IConsole console, LogLevel minimalLogLevel, LogLevel minimalErrorLevel) + { + factory.AddProvider(new SimpleConsoleLoggerProvider(console, minimalLogLevel, minimalErrorLevel)); + return factory; + } + } +} diff --git a/src/Tools/PerfDiff/Logging/SimpleConsoleLoggerProvider.cs b/src/Tools/PerfDiff/Logging/SimpleConsoleLoggerProvider.cs new file mode 100644 index 0000000000..e153765e5b --- /dev/null +++ b/src/Tools/PerfDiff/Logging/SimpleConsoleLoggerProvider.cs @@ -0,0 +1,31 @@ +// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Microsoft.Extensions.Logging; + +using System.CommandLine; + +namespace PerfDiff.Logging +{ + internal class SimpleConsoleLoggerProvider : ILoggerProvider + { + private readonly IConsole _console; + private readonly LogLevel _minimalLogLevel; + private readonly LogLevel _minimalErrorLevel; + + public SimpleConsoleLoggerProvider(IConsole console, LogLevel minimalLogLevel, LogLevel minimalErrorLevel) + { + _console = console; + _minimalLogLevel = minimalLogLevel; + _minimalErrorLevel = minimalErrorLevel; + } + + public ILogger CreateLogger(string categoryName) + { + return new SimpleConsoleLogger(_console, _minimalLogLevel, _minimalErrorLevel); + } + + public void Dispose() + { + } + } +} diff --git a/src/Tools/PerfDiff/PerfDiff.cs b/src/Tools/PerfDiff/PerfDiff.cs new file mode 100644 index 0000000000..40abfb7a7a --- /dev/null +++ b/src/Tools/PerfDiff/PerfDiff.cs @@ -0,0 +1,75 @@ +// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Diagnostics.CodeAnalysis; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +using Microsoft.Extensions.Logging; + +namespace PerfDiff +{ + public static partial class PerfDiff + { + public static async Task CompareAsync( + string baselineFolder, string resultsFolder, bool failOnRegression, ILogger logger, CancellationToken token) + { + token.ThrowIfCancellationRequested(); + + var (success, shouldCheckETL) = await BenchmarkDotNetDiffer.TryCompareBenchmarkDotNetResultsAsync(baselineFolder, resultsFolder, logger).ConfigureAwait(false); + if (!success) + { + return 1; + } + + if (shouldCheckETL) + { + // get file paths + if (!TryGetETLPaths(baselineFolder, out var baselineEtlPath)) + { + return 1; + } + + if (!TryGetETLPaths(resultsFolder, out var resultsEtlPath)) + { + return 1; + } + + // Compare ETL + if (!EtlDiffer.TryCompareETL(resultsEtlPath, baselineEtlPath, out var regression)) + { + return 1; + } + + if (regression && failOnRegression) + { + return 1; + } + } + + return 0; + } + + private const string ETLFileExtension = "etl.zip"; + + private static bool TryGetETLPaths(string path, [NotNullWhen(true)] out string? etlPath) + { + if (Directory.Exists(path)) + { + var files = Directory.GetFiles(path, $"*{ETLFileExtension}", SearchOption.AllDirectories); + etlPath = files.Single(); + return true; + } + else if (File.Exists(path) || !path.EndsWith(ETLFileExtension, StringComparison.OrdinalIgnoreCase)) + { + etlPath = path; + return true; + } + + etlPath = null; + return false; + } + } +} diff --git a/src/Tools/PerfDiff/PerfDiff.csproj b/src/Tools/PerfDiff/PerfDiff.csproj new file mode 100644 index 0000000000..7a829e12ea --- /dev/null +++ b/src/Tools/PerfDiff/PerfDiff.csproj @@ -0,0 +1,19 @@ + + + + Exe + net6.0 + true + + + + + + + + + + + + + diff --git a/src/Tools/PerfDiff/Program.cs b/src/Tools/PerfDiff/Program.cs new file mode 100644 index 0000000000..953a816e35 --- /dev/null +++ b/src/Tools/PerfDiff/Program.cs @@ -0,0 +1,105 @@ +// Copyright (c) Microsoft. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; + +using PerfDiff.Logging; + +using System; +using System.CommandLine; +using System.CommandLine.Invocation; +using System.CommandLine.Parsing; +using System.IO; +using System.Threading; +using System.Threading.Tasks; + +namespace PerfDiff +{ + internal class Program + { + internal const int UnhandledExceptionExitCode = 1; + private static ParseResult? s_parseResult; + + private static async Task Main(string[] args) + { + var rootCommand = DiffCommand.CreateCommandLineOptions(); + rootCommand.Handler = CommandHandler.Create(new DiffCommand.Handler(Run)); + + // Parse the incoming args so we can give warnings when deprecated options are used. + s_parseResult = rootCommand.Parse(args); + + return await rootCommand.InvokeAsync(args).ConfigureAwait(false); + } + + public static async Task Run( + string baseline, + string results, + string? verbosity, + bool failOnRegression, + IConsole console) + { + if (s_parseResult == null) + { + return 1; + } + + // Setup logging. + var logLevel = GetLogLevel(verbosity); + var logger = SetupLogging(console, minimalLogLevel: logLevel, minimalErrorLevel: LogLevel.Warning); + + // Hook so we can cancel and exit when ctrl+c is pressed. + var cancellationTokenSource = new CancellationTokenSource(); + Console.CancelKeyPress += (sender, e) => + { + e.Cancel = true; + cancellationTokenSource.Cancel(); + }; + + var currentDirectory = string.Empty; + + try + { + return await PerfDiff.CompareAsync(baseline, results, failOnRegression, logger, cancellationTokenSource.Token).ConfigureAwait(false); + } + catch (FileNotFoundException fex) + { + logger.LogError(fex.Message); + return UnhandledExceptionExitCode; + } + catch (OperationCanceledException) + { + return UnhandledExceptionExitCode; + } + finally + { + if (!string.IsNullOrEmpty(currentDirectory)) + { + Environment.CurrentDirectory = currentDirectory; + } + } + + static ILogger SetupLogging(IConsole console, LogLevel minimalLogLevel, LogLevel minimalErrorLevel) + { + var serviceCollection = new ServiceCollection(); + serviceCollection.AddSingleton(new LoggerFactory().AddSimpleConsole(console, minimalLogLevel, minimalErrorLevel)); + serviceCollection.AddLogging(); + + var serviceProvider = serviceCollection.BuildServiceProvider(); + var logger = serviceProvider.GetService>(); + + return logger!; + } + + static LogLevel GetLogLevel(string? verbosity) + => verbosity switch + { + "q" or "quiet" => LogLevel.Error, + "m" or "minimal" => LogLevel.Warning, + "n" or "normal" => LogLevel.Information, + "d" or "detailed" => LogLevel.Debug, + "diag" or "diagnostic" => LogLevel.Trace, + _ => LogLevel.Information, + }; + } + } +}