From 26d642a0454a4f22203f81a14e42097565d59d84 Mon Sep 17 00:00:00 2001 From: "Brett V. Forsgren" Date: Fri, 13 Dec 2024 10:57:03 -0700 Subject: [PATCH 01/20] add .NET SDK package correlation projects --- .../lib/NuGetUpdater/Directory.Packages.props | 1 + .../CorrelatorTests.cs | 92 +++++++++ .../DotNetPackageCorrelation.Test.csproj | 18 ++ .../DotNetPackageCorrelation/Correlator.cs | 192 ++++++++++++++++++ .../DotNetPackageCorrelation.csproj | 14 ++ .../Model/PackageSet.cs | 8 + .../DotNetPackageCorrelation/Model/Release.cs | 9 + .../Model/ReleasesFile.cs | 9 + .../DotNetPackageCorrelation/Model/Sdk.cs | 12 ++ .../Model/SdkPackages.cs | 8 + .../Model/SemVerComparer.cs | 14 ++ .../Model/SemVersionConverter.cs | 31 +++ .../DotNetPackageCorrelation/Program.cs | 30 +++ .../helpers/lib/NuGetUpdater/NuGetUpdater.sln | 13 +- nuget/script/ci-test | 1 + 15 files changed, 451 insertions(+), 1 deletion(-) create mode 100644 nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/CorrelatorTests.cs create mode 100644 nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/DotNetPackageCorrelation.Test.csproj create mode 100644 nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Correlator.cs create mode 100644 nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/DotNetPackageCorrelation.csproj create mode 100644 nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/PackageSet.cs create mode 100644 nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/Release.cs create mode 100644 nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/ReleasesFile.cs create mode 100644 nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/Sdk.cs create mode 100644 nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/SdkPackages.cs create mode 100644 nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/SemVerComparer.cs create mode 100644 nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/SemVersionConverter.cs create mode 100644 nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Program.cs diff --git a/nuget/helpers/lib/NuGetUpdater/Directory.Packages.props b/nuget/helpers/lib/NuGetUpdater/Directory.Packages.props index e1367ab390..3ea838da0a 100644 --- a/nuget/helpers/lib/NuGetUpdater/Directory.Packages.props +++ b/nuget/helpers/lib/NuGetUpdater/Directory.Packages.props @@ -26,6 +26,7 @@ + diff --git a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/CorrelatorTests.cs b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/CorrelatorTests.cs new file mode 100644 index 0000000000..b24dfe7c4b --- /dev/null +++ b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/CorrelatorTests.cs @@ -0,0 +1,92 @@ +using Semver; + +using Xunit; + +namespace DotNetPackageCorrelation.Tests; + +public class CorrelatorTests +{ + [Fact] + public async Task AllFilesShapedAppropriately() + { + // the JSON and markdown are shaped as expected + var (packages, warnings) = await RunFromFilesAsync( + ("8.0/releases.json", """ + { + "releases": [ + { + "sdk": { + "version": "8.0.100", + "runtime-version": "8.0.0" + } + } + ] + } + """), + ("8.0/8.0.0/8.0.0.md", """ + Package name | Version + :-- | :-- + Package.A | 8.0.0 + Package.B | 1.2.3 + """) + ); + Assert.Empty(warnings); + AssertPackageVersion(packages, "8.0.100", "Package.A", "8.0.0"); + AssertPackageVersion(packages, "8.0.100", "Package.B", "1.2.3"); + } + + [Theory] + [InlineData("Some.Package | 1.2.3", "Some.Package", "1.2.3")] // happy path + [InlineData("Some.Package.1.2.3", "Some.Package", "1.2.3")] // looks like a restore directory + [InlineData("Some.Package | 1.2 | 1.2.3.nupkg", "Some.Package", "1.2.3")] // extra columns from a bad filename split + [InlineData("Some.Package | 1.2.3.nupkg", "Some.Package", "1.2.3")] // version contains package extension + [InlineData("Some.Package | 1.2.3.symbols.nupkg", "Some.Package", "1.2.3")] // version contains symbols package extension + [InlineData("some.package.1.2.3.nupkg", "some.package", "1.2.3")] // first column is a filename, second column is missing + [InlineData("some.package.1.2.3.nupkg |", "some.package", "1.2.3")] // first column is a filename, second column is empty + public void PackagesParsedFromMarkdown(string markdownLine, string expectedPackageName, string expectedPackageVersion) + { + var markdownContent = $""" + Package name | Version + :-- | :-- + {markdownLine} + """; + var warnings = new List(); + var packages = Correlator.GetPackagesFromMarkdown("test.md", markdownContent, warnings); + Assert.Empty(warnings); + var actualpackage = Assert.Single(packages); + Assert.Equal(expectedPackageName, actualpackage.Name); + Assert.Equal(expectedPackageVersion, actualpackage.Version.ToString()); + } + + private static void AssertPackageVersion(SdkPackages packages, string sdkVersion, string packageName, string expectedPackageVersion) + { + Assert.True(packages.Packages.TryGetValue(SemVersion.Parse(sdkVersion), out var packageSet), $"Unable to find SDK verison [{sdkVersion}]"); + Assert.True(packageSet.Packages.TryGetValue(packageName, out var packageVersion), $"Unable to find package [{packageName}] under SDK version [{sdkVersion}]"); + var actualPackageVersion = packageVersion.ToString(); + Assert.Equal(expectedPackageVersion, actualPackageVersion); + } + + private static async Task<(SdkPackages Packages, IEnumerable Warnings)> RunFromFilesAsync(params (string Path, string Content)[] files) + { + var testDirectory = Path.Combine(Path.GetDirectoryName(typeof(CorrelatorTests).Assembly.Location)!, "test-data", Guid.NewGuid().ToString("D")); + Directory.CreateDirectory(testDirectory); + + try + { + foreach (var (path, content) in files) + { + var fullPath = Path.Combine(testDirectory, path); + Directory.CreateDirectory(Path.GetDirectoryName(fullPath)!); + await File.WriteAllTextAsync(fullPath, content); + } + + var correlator = new Correlator(new DirectoryInfo(testDirectory)); + var result = await correlator.RunAsync(); + return result; + } + finally + { + Directory.Delete(testDirectory, recursive: true); + } + } +} diff --git a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/DotNetPackageCorrelation.Test.csproj b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/DotNetPackageCorrelation.Test.csproj new file mode 100644 index 0000000000..f7791a7c55 --- /dev/null +++ b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/DotNetPackageCorrelation.Test.csproj @@ -0,0 +1,18 @@ + + + + $(CommonTargetFramework) + Exe + + + + + + + + + + + + + diff --git a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Correlator.cs b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Correlator.cs new file mode 100644 index 0000000000..01c0d131b6 --- /dev/null +++ b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Correlator.cs @@ -0,0 +1,192 @@ +using System.Collections.Immutable; +using System.Text.Json; +using System.Text.RegularExpressions; + +using Semver; + +namespace DotNetPackageCorrelation; + +public partial class Correlator +{ + internal static readonly JsonSerializerOptions SerializerOptions = new() + { + WriteIndented = true, + Converters = { new SemVersionConverter() }, + }; + + private readonly DirectoryInfo _releaseNotesDirectory; + + public Correlator(DirectoryInfo releaseNotesDirectory) + { + _releaseNotesDirectory = releaseNotesDirectory; + } + + public async Task<(SdkPackages Packages, IEnumerable Warnings)> RunAsync() + { + var runtimeVersions = new List(); + foreach (var directory in Directory.EnumerateDirectories(_releaseNotesDirectory.FullName)) + { + var directoryName = Path.GetFileName(directory); + if (Version.TryParse(directoryName, out var version)) + { + runtimeVersions.Add(version); + } + } + + var sdkPackages = new SdkPackages(); + var warnings = new List(); + foreach (var version in runtimeVersions) + { + var releasesJsonPath = Path.Combine(_releaseNotesDirectory.FullName, version.ToString(), "releases.json"); + if (!File.Exists(releasesJsonPath)) + { + warnings.Add($"Unable to find releases.json file for version {version}"); + continue; + } + + var releasesJson = await File.ReadAllTextAsync(releasesJsonPath); + var releasesFile = JsonSerializer.Deserialize(releasesJson, SerializerOptions)!; // TODO + + foreach (var release in releasesFile.Releases) + { + if (release.Sdk.Version is null) + { + warnings.Add($"Skipping release with missing version information from {releasesJson}"); + continue; + } + + if (release.Sdk.RuntimeVersion is null) + { + warnings.Add($"Skipping release with missing runtime version information from {releasesJson}"); + continue; + } + + if (!sdkPackages.Packages.TryGetValue(release.Sdk.Version, out var packagesAndVersions)) + { + packagesAndVersions = new PackageSet(); + sdkPackages.Packages[release.Sdk.Version] = packagesAndVersions; + } + + var runtimeDirectory = new DirectoryInfo(Path.Combine(_releaseNotesDirectory.FullName, version.ToString(), release.Sdk.RuntimeVersion.ToString())); + var runtimeMarkdownPath = Path.Combine(runtimeDirectory.FullName, $"{release.Sdk.RuntimeVersion}.md"); + if (!File.Exists(runtimeMarkdownPath)) + { + warnings.Add($"Unable to find expected markdown file {runtimeMarkdownPath}"); + continue; + } + + var markdownContent = await File.ReadAllTextAsync(runtimeMarkdownPath); + var packages = GetPackagesFromMarkdown(runtimeMarkdownPath, markdownContent, warnings); + foreach (var (packageName, packageVersion) in packages) + { + packagesAndVersions.Packages[packageName] = packageVersion; + } + } + } + + return (sdkPackages, warnings); + } + + public static ImmutableArray<(string Name, SemVersion Version)> GetPackagesFromMarkdown(string markdownPath, string markdownContent, List warnings) + { + var lines = markdownContent.Split("\n").Select(l => l.Trim()).ToArray(); + + // the markdown file contains a table that looks like this: + // Package name | Version + // :----------- | :------------------ + // Some.Package | 1.2.3 + // ... + // however there are some formatting issues with some elements that prevent markdown parsers from + // discovering it, so we fall back to manual parsing + + var tableStartLine = -1; + for (int i = 0; i < lines.Length; i++) + { + if (Regex.IsMatch(lines[i], "Package name.*Version")) + { + tableStartLine = i; + break; + } + } + + if (tableStartLine == -1) + { + warnings.Add($"Unable to find table start in file {markdownPath}"); + return []; + } + + // skip the column names and separator line + tableStartLine += 2; + + var tableEndLine = lines.Length; // assume the end of the file unless we find a blank line + for (int i = tableStartLine; i < lines.Length; i++) + { + if (string.IsNullOrEmpty(lines[i])) + { + tableEndLine = i; + break; + } + } + + var packages = new List<(string Name, SemVersion Version)>(); + for (int i = tableStartLine; i < tableEndLine; i++) + { + var line = lines[i].Trim(); + var foundMatch = false; + foreach (var pattern in SpecialCasePatterns) + { + var match = pattern.Match(line); + if (match.Success) + { + var packageName = match.Groups["PackageName"].Value; + var packageVersionString = match.Groups["PackageVersion"].Value; + if (SemVersion.TryParse(packageVersionString, out var packageVersion)) + { + packages.Add((packageName, packageVersion)); + foundMatch = true; + break; ; + } + } + } + + if (!foundMatch) + { + warnings.Add($"Unable to parse package and version from string [{line}] in file [{markdownPath}]:{i}"); + } + } + + return packages.ToImmutableArray(); + } + + // The different patterns the lines in the markdown might take. Due to issues with regular expressions, this list + // is in a very specific order. + private static ImmutableArray SpecialCasePatterns { get; } = [ + StandardLineWithFileExtensions(), + StandardLine(), + PackageNameDotVersion(), + PackageFileNameWithOptionalTrailingPipe(), + MultiColumnWithOptionalFileSuffix(), + ]; + + [GeneratedRegex(@"^(?[^|\s]+)\s*\|\s*(?[^|\s]+?)(\.symbols)?\.nupkg$", RegexOptions.Compiled)] + // Some.Package | 1.2.3.nupkg + // Some.Package | 1.2.3.symbols.nupkg + private static partial Regex StandardLineWithFileExtensions(); + + [GeneratedRegex(@"^(?[^|\s]+)\s*\|\s*(?[^|\s]+)$", RegexOptions.Compiled)] + // Some.Package | 1.2.3 + private static partial Regex StandardLine(); + + [GeneratedRegex(@"^(?[^\d]+)\.(?[\d].+)$", RegexOptions.Compiled)] + // Some.Package.1.2.3 + private static partial Regex PackageNameDotVersion(); + + [GeneratedRegex(@"^(?[^\d]+)\.(?\d.+?)\.nupkg(\s+\|)?$", RegexOptions.Compiled)] + // some.package.1.2.3.nupkg + // some.package.1.2.3.nupkg | + private static partial Regex PackageFileNameWithOptionalTrailingPipe(); + + [GeneratedRegex(@"^(?[^|\s]+)\s*\|[^|]*\|\s*(?.*?)(\.nupkg)?$", RegexOptions.Compiled)] + // Some.Package | 1.2 | 1.2.3.nupkg + private static partial Regex MultiColumnWithOptionalFileSuffix(); +} diff --git a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/DotNetPackageCorrelation.csproj b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/DotNetPackageCorrelation.csproj new file mode 100644 index 0000000000..3672a3eaee --- /dev/null +++ b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/DotNetPackageCorrelation.csproj @@ -0,0 +1,14 @@ + + + + $(CommonTargetFramework) + Exe + + + + + + + + + diff --git a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/PackageSet.cs b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/PackageSet.cs new file mode 100644 index 0000000000..9c017ced8e --- /dev/null +++ b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/PackageSet.cs @@ -0,0 +1,8 @@ +using Semver; + +namespace DotNetPackageCorrelation; + +public record PackageSet +{ + public SortedDictionary Packages { get; init; } = new(StringComparer.OrdinalIgnoreCase); +} diff --git a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/Release.cs b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/Release.cs new file mode 100644 index 0000000000..13788e698f --- /dev/null +++ b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/Release.cs @@ -0,0 +1,9 @@ +using System.Text.Json.Serialization; + +namespace DotNetPackageCorrelation; + +public record Release +{ + [JsonPropertyName("sdk")] + public required Sdk Sdk { get; init; } +} diff --git a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/ReleasesFile.cs b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/ReleasesFile.cs new file mode 100644 index 0000000000..a1c6192182 --- /dev/null +++ b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/ReleasesFile.cs @@ -0,0 +1,9 @@ +using System.Text.Json.Serialization; + +namespace DotNetPackageCorrelation; + +public record ReleasesFile +{ + [JsonPropertyName("releases")] + public required Release[] Releases { get; init; } +} diff --git a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/Sdk.cs b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/Sdk.cs new file mode 100644 index 0000000000..500a3746db --- /dev/null +++ b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/Sdk.cs @@ -0,0 +1,12 @@ +using Semver; +using System.Text.Json.Serialization; + +namespace DotNetPackageCorrelation; + +public record Sdk +{ + [JsonPropertyName("version")] + public required SemVersion? Version { get; init; } + [JsonPropertyName("runtime-version")] + public SemVersion? RuntimeVersion { get; init; } +} diff --git a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/SdkPackages.cs b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/SdkPackages.cs new file mode 100644 index 0000000000..b931e4bbdf --- /dev/null +++ b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/SdkPackages.cs @@ -0,0 +1,8 @@ +using Semver; + +namespace DotNetPackageCorrelation; + +public record SdkPackages +{ + public SortedDictionary Packages { get; init; } = new(new SemVerComparer()); +} diff --git a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/SemVerComparer.cs b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/SemVerComparer.cs new file mode 100644 index 0000000000..31a258922e --- /dev/null +++ b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/SemVerComparer.cs @@ -0,0 +1,14 @@ +using Semver; + +namespace DotNetPackageCorrelation; + +public class SemVerComparer : IComparer +{ + public int Compare(SemVersion? x, SemVersion? y) + { + ArgumentNullException.ThrowIfNull(x); + ArgumentNullException.ThrowIfNull(y); + + return x.CompareSortOrderTo(y); + } +} diff --git a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/SemVersionConverter.cs b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/SemVersionConverter.cs new file mode 100644 index 0000000000..95829ebc98 --- /dev/null +++ b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/SemVersionConverter.cs @@ -0,0 +1,31 @@ +using System.Diagnostics.CodeAnalysis; +using System.Text.Json; +using System.Text.Json.Serialization; + +using Semver; + +namespace DotNetPackageCorrelation; + +public class SemVersionConverter : JsonConverter +{ + public override SemVersion? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + var value = reader.GetString(); + if (SemVersion.TryParse(value, out var result)) + { + return result; + } + + return null; + } + + public override void Write(Utf8JsonWriter writer, SemVersion? value, JsonSerializerOptions options) + { + writer.WriteStringValue(value?.ToString()); + } + + public override void WriteAsPropertyName(Utf8JsonWriter writer, [DisallowNull] SemVersion value, JsonSerializerOptions options) + { + writer.WritePropertyName(value.ToString()); + } +} diff --git a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Program.cs b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Program.cs new file mode 100644 index 0000000000..65e8ca248a --- /dev/null +++ b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Program.cs @@ -0,0 +1,30 @@ +using System.CommandLine; +using System.Text.Json; + +namespace DotNetPackageCorrelation; + +public class Program +{ + public static async Task Main(string[] args) + { + var coreLocationOption = new Option("--core-location", "The location of the .NET Core source code.") { IsRequired = true }; + var outputOption = new Option("--output", "The location to write the result.") { IsRequired = true }; + var command = new Command("build") + { + coreLocationOption, + outputOption, + }; + command.TreatUnmatchedTokensAsErrors = true; + command.SetHandler(async (coreLocationDirectory, output) => + { + // the tool is expected to be given the path to the .NET Core repository, but the correlator only needs a specific subdirectory + var releaseNotesDirectory = new DirectoryInfo(Path.Combine(coreLocationDirectory.FullName, "release-notes")); + var correlator = new Correlator(releaseNotesDirectory); + var result = await correlator.RunAsync(); + var json = JsonSerializer.Serialize(result, Correlator.SerializerOptions); + await File.WriteAllTextAsync(output.FullName, json); + }, coreLocationOption, outputOption); + var exitCode = await command.InvokeAsync(args); + return exitCode; + } +} diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.sln b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.sln index 819cbc2597..f4889927c0 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.sln +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.sln @@ -1,4 +1,3 @@ - Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio Version 17 VisualStudioVersion = 17.5.33516.290 @@ -43,6 +42,10 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "NuGet.Versioning", "NuGetPr EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "NuGetUpdater.Cli.Test", "NuGetUpdater.Cli.Test\NuGetUpdater.Cli.Test.csproj", "{BDBEBF91-F5FD-4589-B4FB-B3DE3103B04B}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DotNetPackageCorrelation", "DotNetPackageCorrelation\DotNetPackageCorrelation.csproj", "{52A6437B-7E72-4CCF-8E1E-355000F5DC10}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DotNetPackageCorrelation.Test", "DotNetPackageCorrelation.Test\DotNetPackageCorrelation.Test.csproj", "{0945703C-C8DC-44F0-B1D8-0EFE011411AE}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -125,6 +128,14 @@ Global {BDBEBF91-F5FD-4589-B4FB-B3DE3103B04B}.Debug|Any CPU.Build.0 = Debug|Any CPU {BDBEBF91-F5FD-4589-B4FB-B3DE3103B04B}.Release|Any CPU.ActiveCfg = Release|Any CPU {BDBEBF91-F5FD-4589-B4FB-B3DE3103B04B}.Release|Any CPU.Build.0 = Release|Any CPU + {52A6437B-7E72-4CCF-8E1E-355000F5DC10}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {52A6437B-7E72-4CCF-8E1E-355000F5DC10}.Debug|Any CPU.Build.0 = Debug|Any CPU + {52A6437B-7E72-4CCF-8E1E-355000F5DC10}.Release|Any CPU.ActiveCfg = Release|Any CPU + {52A6437B-7E72-4CCF-8E1E-355000F5DC10}.Release|Any CPU.Build.0 = Release|Any CPU + {0945703C-C8DC-44F0-B1D8-0EFE011411AE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {0945703C-C8DC-44F0-B1D8-0EFE011411AE}.Debug|Any CPU.Build.0 = Debug|Any CPU + {0945703C-C8DC-44F0-B1D8-0EFE011411AE}.Release|Any CPU.ActiveCfg = Release|Any CPU + {0945703C-C8DC-44F0-B1D8-0EFE011411AE}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/nuget/script/ci-test b/nuget/script/ci-test index d82e523e2d..157692290f 100755 --- a/nuget/script/ci-test +++ b/nuget/script/ci-test @@ -11,6 +11,7 @@ popd pushd ./helpers/lib/NuGetUpdater dotnet restore dotnet build --configuration Release +dotnet test --configuration Release --no-restore --no-build --logger "console;verbosity=normal" --blame-hang-timeout 5m ./DotNetPackageCorrelation.Test/DotNetPackageCorrelation.Test.csproj dotnet test --configuration Release --no-restore --no-build --logger "console;verbosity=normal" --blame-hang-timeout 5m ./NuGetUpdater.Cli.Test/NuGetUpdater.Cli.Test.csproj dotnet test --configuration Release --no-restore --no-build --logger "console;verbosity=normal" --blame-hang-timeout 5m ./NuGetUpdater.Core.Test/NuGetUpdater.Core.Test.csproj popd From 10e5576230447397ae2eb9b13241c7ddf617be6b Mon Sep 17 00:00:00 2001 From: "Brett V. Forsgren" Date: Fri, 20 Dec 2024 12:02:54 -0700 Subject: [PATCH 02/20] build package correlation and include in updater --- .gitmodules | 3 +++ nuget/helpers/lib/NuGetUpdater/.gitignore | 1 + .../DotNetPackageCorrelation/Program.cs | 4 ++-- .../EnsureDotNetPackageCorrelation.targets | 24 +++++++++++++++++++ .../NuGetUpdater.Core.csproj | 2 ++ nuget/helpers/lib/dotnet-core | 1 + 6 files changed, 33 insertions(+), 2 deletions(-) create mode 100644 nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/EnsureDotNetPackageCorrelation.targets create mode 160000 nuget/helpers/lib/dotnet-core diff --git a/.gitmodules b/.gitmodules index d30b825732..3e20502e0f 100644 --- a/.gitmodules +++ b/.gitmodules @@ -2,3 +2,6 @@ path = nuget/helpers/lib/NuGet.Client url = https://github.com/NuGet/NuGet.Client branch = release-6.12.x +[submodule "nuget/helpers/lib/dotnet-core"] + path = nuget/helpers/lib/dotnet-core + url = https://github.com/dotnet/core diff --git a/nuget/helpers/lib/NuGetUpdater/.gitignore b/nuget/helpers/lib/NuGetUpdater/.gitignore index 6b39588b66..3a80b6dc44 100644 --- a/nuget/helpers/lib/NuGetUpdater/.gitignore +++ b/nuget/helpers/lib/NuGetUpdater/.gitignore @@ -4,4 +4,5 @@ bin/ obj/ Properties/launchSettings.json NuGetUpdater.sln.DotSettings.user +NuGetUpdater.Core/dotnet-package-correlation.json *.binlog diff --git a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Program.cs b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Program.cs index 65e8ca248a..db0c576f7c 100644 --- a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Program.cs +++ b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Program.cs @@ -20,8 +20,8 @@ public static async Task Main(string[] args) // the tool is expected to be given the path to the .NET Core repository, but the correlator only needs a specific subdirectory var releaseNotesDirectory = new DirectoryInfo(Path.Combine(coreLocationDirectory.FullName, "release-notes")); var correlator = new Correlator(releaseNotesDirectory); - var result = await correlator.RunAsync(); - var json = JsonSerializer.Serialize(result, Correlator.SerializerOptions); + var (packages, _warnings) = await correlator.RunAsync(); + var json = JsonSerializer.Serialize(packages, Correlator.SerializerOptions); await File.WriteAllTextAsync(output.FullName, json); }, coreLocationOption, outputOption); var exitCode = await command.InvokeAsync(args); diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/EnsureDotNetPackageCorrelation.targets b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/EnsureDotNetPackageCorrelation.targets new file mode 100644 index 0000000000..42e20bebcb --- /dev/null +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/EnsureDotNetPackageCorrelation.targets @@ -0,0 +1,24 @@ + + + + $(MSBuildThisFileDirectory)..\DotNetPackageCorrelation + $(MSBuildThisFileDirectory)..\..\dotnet-core + $(MSBuildThisFileDirectory)dotnet-package-correlation.json + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/NuGetUpdater.Core.csproj b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/NuGetUpdater.Core.csproj index 7bb1102c92..e01f1bbebd 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/NuGetUpdater.Core.csproj +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/NuGetUpdater.Core.csproj @@ -31,4 +31,6 @@ + + diff --git a/nuget/helpers/lib/dotnet-core b/nuget/helpers/lib/dotnet-core new file mode 160000 index 0000000000..649080cc3a --- /dev/null +++ b/nuget/helpers/lib/dotnet-core @@ -0,0 +1 @@ +Subproject commit 649080cc3a0e927abb88d05591256e23c29e2170 From aae00c83c4b444b23f338386f390f4a84c05871f Mon Sep 17 00:00:00 2001 From: "Brett V. Forsgren" Date: Tue, 7 Jan 2025 15:57:06 -0700 Subject: [PATCH 03/20] incorporate sdk package correlation --- .../CorrelatorTests.cs | 2 +- .../EndToEndTests.cs | 31 +++ .../SdkPackagesTests.cs | 205 ++++++++++++++++++ .../DotNetPackageCorrelation/Correlator.cs | 59 ++--- .../Model/PackageSet.cs | 3 + .../DotNetPackageCorrelation/Model/Release.cs | 16 ++ .../Model/SdkPackages.cs | 5 +- .../Model/SemVerComparer.cs | 4 +- .../Model/SemVersionConverter.cs | 11 + .../SdkPackagesExtensions.cs | 28 +++ .../EntryPointTests.Discover.cs | 70 ++++++ .../Discover/SdkProjectDiscovery.cs | 44 +++- .../EnsureDotNetPackageCorrelation.targets | 3 +- .../NuGetUpdater.Core.csproj | 1 + 14 files changed, 447 insertions(+), 35 deletions(-) create mode 100644 nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/EndToEndTests.cs create mode 100644 nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/SdkPackagesTests.cs create mode 100644 nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/SdkPackagesExtensions.cs diff --git a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/CorrelatorTests.cs b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/CorrelatorTests.cs index b24dfe7c4b..5400bbf2fe 100644 --- a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/CorrelatorTests.cs +++ b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/CorrelatorTests.cs @@ -7,7 +7,7 @@ namespace DotNetPackageCorrelation.Tests; public class CorrelatorTests { [Fact] - public async Task AllFilesShapedAppropriately() + public async Task FileHandling_AllFilesShapedAppropriately() { // the JSON and markdown are shaped as expected var (packages, warnings) = await RunFromFilesAsync( diff --git a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/EndToEndTests.cs b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/EndToEndTests.cs new file mode 100644 index 0000000000..b04f882a81 --- /dev/null +++ b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/EndToEndTests.cs @@ -0,0 +1,31 @@ +using System.Runtime.CompilerServices; + +using Semver; + +using Xunit; + +namespace DotNetPackageCorrelation.Tests; + +public class EndToEndTests +{ + [Fact] + public async Task IntegrationTest() + { + // arrange + var thisFileDirectory = Path.GetDirectoryName(GetThisFilePath())!; + var dotnetCoreDirectory = Path.Combine(thisFileDirectory, "..", "..", "dotnet-core"); + var correlator = new Correlator(new DirectoryInfo(Path.Combine(dotnetCoreDirectory, "release-notes"))); + + // act + var (packages, _warnings) = await correlator.RunAsync(); + var sdkVersion = SemVersion.Parse("8.0.307"); + + // SDK 8.0.307 has no System.Text.Json, but 8.0.306 provides System.Text.Json 8.0.5 + var systemTextJsonPackageVersion = packages.GetReplacementPackageVersion(sdkVersion, "system.TEXT.json"); + + // assert + Assert.Equal("8.0.5", systemTextJsonPackageVersion?.ToString()); + } + + private static string GetThisFilePath([CallerFilePath] string? path = null) => path ?? throw new ArgumentNullException(nameof(path)); +} diff --git a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/SdkPackagesTests.cs b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/SdkPackagesTests.cs new file mode 100644 index 0000000000..9e1a0b136e --- /dev/null +++ b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/SdkPackagesTests.cs @@ -0,0 +1,205 @@ +using Semver; + +using Xunit; + +namespace DotNetPackageCorrelation; + +public class SdkPackagesTests +{ + [Theory] + [MemberData(nameof(CorrelatedPackageCanBeFoundData))] + public void CorrelatedPackageCanBeFound(SdkPackages packages, string sdkVersionString, string packageName, string? expectedPackageVersionString) + { + var sdkVersion = SemVersion.Parse(sdkVersionString); + var actualReplacementPackageVersion = packages.GetReplacementPackageVersion(sdkVersion, packageName); + var expectedPackageVersion = expectedPackageVersionString is not null + ? SemVersion.Parse(expectedPackageVersionString) + : null; + Assert.Equal(expectedPackageVersion, actualReplacementPackageVersion); + } + + public static IEnumerable CorrelatedPackageCanBeFoundData() + { + // package not found in current sdk, but is in parent; more recent sdk has package, but that's not returned + yield return + [ + // packages + new SdkPackages() + { + Packages = new SortedDictionary(SemVerComparer.Instance) + { + { + SemVersion.Parse("1.0.100"), + new PackageSet() + { + Packages = new SortedDictionary(StringComparer.OrdinalIgnoreCase) + { + { "Some.Package", SemVersion.Parse("1.0.1") } + } + } + }, + { + SemVersion.Parse("1.0.101"), + new PackageSet() + { + Packages = new SortedDictionary(StringComparer.OrdinalIgnoreCase) + { + // empty + } + } + }, + { + SemVersion.Parse("1.0.102"), + new PackageSet() + { + Packages = new SortedDictionary(StringComparer.OrdinalIgnoreCase) + { + { "Some.Package", SemVersion.Parse("1.0.2") } + } + } + }, + } + }, + // sdkVersionString + "1.0.101", + // packageName + "Some.Package", + // expectedPackageVersionString + "1.0.1" + ]; + + // package differing in case is found + yield return + [ + // packages + new SdkPackages() + { + Packages = new SortedDictionary(SemVerComparer.Instance) + { + { + SemVersion.Parse("1.0.100"), + new PackageSet() + { + Packages = new SortedDictionary(StringComparer.OrdinalIgnoreCase) + { + { "some.package", SemVersion.Parse("1.0.1") } + } + } + }, + { + SemVersion.Parse("1.0.101"), + new PackageSet() + { + Packages = new SortedDictionary(StringComparer.OrdinalIgnoreCase) + { + // empty + } + } + }, + } + }, + // sdkVersionString + "1.0.101", + // packageName + "Some.Package", + // expectedPackageVersionString + "1.0.1" + ]; + + // package not found results in null version + yield return + [ + // packages + new SdkPackages() + { + Packages = new SortedDictionary(SemVerComparer.Instance) + { + { + SemVersion.Parse("1.0.100"), + new PackageSet() + { + Packages = new SortedDictionary(StringComparer.OrdinalIgnoreCase) + { + { "Some.Package", SemVersion.Parse("1.0.1") } + } + } + }, + { + SemVersion.Parse("1.0.101"), + new PackageSet() + { + Packages = new SortedDictionary(StringComparer.OrdinalIgnoreCase) + { + // empty + } + } + }, + } + }, + // sdkVersionString + "1.0.101", + // packageName + "UnrelatedPackage", + // expectedPackageVersionString + null + ]; + + // only SDKs with matching major version are considered + yield return + [ + // packages + new SdkPackages() + { + Packages = new SortedDictionary(SemVerComparer.Instance) + { + { + SemVersion.Parse("1.0.100"), + new PackageSet() + { + Packages = new SortedDictionary(StringComparer.OrdinalIgnoreCase) + { + { "Some.Package", SemVersion.Parse("1.0.0") } + } + } + }, + { + SemVersion.Parse("2.0.100"), + new PackageSet() + { + Packages = new SortedDictionary(StringComparer.OrdinalIgnoreCase) + { + { "Some.Package", SemVersion.Parse("2.0.1") } + } + } + }, + { + SemVersion.Parse("2.0.200"), + new PackageSet() + { + Packages = new SortedDictionary(StringComparer.OrdinalIgnoreCase) + { + // empty + } + } + }, + { + SemVersion.Parse("3.0.100"), + new PackageSet() + { + Packages = new SortedDictionary(StringComparer.OrdinalIgnoreCase) + { + { "Some.Package", SemVersion.Parse("3.0.1") } + } + } + }, + } + }, + // sdkVersionString + "2.0.200", + // packageName + "Some.Package", + // expectedPackageVersionString + "2.0.1" + ]; + } +} diff --git a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Correlator.cs b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Correlator.cs index 01c0d131b6..cf6e89bcd5 100644 --- a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Correlator.cs +++ b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Correlator.cs @@ -8,7 +8,7 @@ namespace DotNetPackageCorrelation; public partial class Correlator { - internal static readonly JsonSerializerOptions SerializerOptions = new() + public static readonly JsonSerializerOptions SerializerOptions = new() { WriteIndented = true, Converters = { new SemVersionConverter() }, @@ -45,41 +45,44 @@ public Correlator(DirectoryInfo releaseNotesDirectory) } var releasesJson = await File.ReadAllTextAsync(releasesJsonPath); - var releasesFile = JsonSerializer.Deserialize(releasesJson, SerializerOptions)!; // TODO + var releasesFile = JsonSerializer.Deserialize(releasesJson, SerializerOptions)!; foreach (var release in releasesFile.Releases) { - if (release.Sdk.Version is null) + foreach (var sdk in release.GetSdks()) { - warnings.Add($"Skipping release with missing version information from {releasesJson}"); - continue; - } + if (sdk.Version is null) + { + warnings.Add($"Skipping release with missing version information from {releasesJson}"); + continue; + } - if (release.Sdk.RuntimeVersion is null) - { - warnings.Add($"Skipping release with missing runtime version information from {releasesJson}"); - continue; - } + if (sdk.RuntimeVersion is null) + { + warnings.Add($"Skipping release with missing runtime version information from {releasesJson}"); + continue; + } - if (!sdkPackages.Packages.TryGetValue(release.Sdk.Version, out var packagesAndVersions)) - { - packagesAndVersions = new PackageSet(); - sdkPackages.Packages[release.Sdk.Version] = packagesAndVersions; - } + if (!sdkPackages.Packages.TryGetValue(sdk.Version, out var packagesAndVersions)) + { + packagesAndVersions = new PackageSet(); + sdkPackages.Packages[sdk.Version] = packagesAndVersions; + } - var runtimeDirectory = new DirectoryInfo(Path.Combine(_releaseNotesDirectory.FullName, version.ToString(), release.Sdk.RuntimeVersion.ToString())); - var runtimeMarkdownPath = Path.Combine(runtimeDirectory.FullName, $"{release.Sdk.RuntimeVersion}.md"); - if (!File.Exists(runtimeMarkdownPath)) - { - warnings.Add($"Unable to find expected markdown file {runtimeMarkdownPath}"); - continue; - } + var runtimeDirectory = new DirectoryInfo(Path.Combine(_releaseNotesDirectory.FullName, version.ToString(), sdk.RuntimeVersion.ToString())); + var runtimeMarkdownPath = Path.Combine(runtimeDirectory.FullName, $"{sdk.RuntimeVersion}.md"); + if (!File.Exists(runtimeMarkdownPath)) + { + warnings.Add($"Unable to find expected markdown file {runtimeMarkdownPath}"); + continue; + } - var markdownContent = await File.ReadAllTextAsync(runtimeMarkdownPath); - var packages = GetPackagesFromMarkdown(runtimeMarkdownPath, markdownContent, warnings); - foreach (var (packageName, packageVersion) in packages) - { - packagesAndVersions.Packages[packageName] = packageVersion; + var markdownContent = await File.ReadAllTextAsync(runtimeMarkdownPath); + var packages = GetPackagesFromMarkdown(runtimeMarkdownPath, markdownContent, warnings); + foreach (var (packageName, packageVersion) in packages) + { + packagesAndVersions.Packages[packageName] = packageVersion; + } } } } diff --git a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/PackageSet.cs b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/PackageSet.cs index 9c017ced8e..90461f6b39 100644 --- a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/PackageSet.cs +++ b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/PackageSet.cs @@ -1,8 +1,11 @@ +using System.Text.Json.Serialization; + using Semver; namespace DotNetPackageCorrelation; public record PackageSet { + [JsonObjectCreationHandling(JsonObjectCreationHandling.Populate)] public SortedDictionary Packages { get; init; } = new(StringComparer.OrdinalIgnoreCase); } diff --git a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/Release.cs b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/Release.cs index 13788e698f..010c3dc4b7 100644 --- a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/Release.cs +++ b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/Release.cs @@ -1,3 +1,4 @@ +using System.Collections.Immutable; using System.Text.Json.Serialization; namespace DotNetPackageCorrelation; @@ -6,4 +7,19 @@ public record Release { [JsonPropertyName("sdk")] public required Sdk Sdk { get; init; } + + [JsonPropertyName("sdks")] + public ImmutableArray? Sdks { get; init; } = []; +} + +public static class ReleaseExtensions +{ + public static IEnumerable GetSdks(this Release release) + { + yield return release.Sdk; + foreach (var sdk in release.Sdks ?? []) + { + yield return sdk; + } + } } diff --git a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/SdkPackages.cs b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/SdkPackages.cs index b931e4bbdf..ba51a349f9 100644 --- a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/SdkPackages.cs +++ b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/SdkPackages.cs @@ -1,8 +1,11 @@ +using System.Text.Json.Serialization; + using Semver; namespace DotNetPackageCorrelation; public record SdkPackages { - public SortedDictionary Packages { get; init; } = new(new SemVerComparer()); + [JsonObjectCreationHandling(JsonObjectCreationHandling.Populate)] + public SortedDictionary Packages { get; init; } = new(SemVerComparer.Instance); } diff --git a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/SemVerComparer.cs b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/SemVerComparer.cs index 31a258922e..bd130d9628 100644 --- a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/SemVerComparer.cs +++ b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/SemVerComparer.cs @@ -4,11 +4,13 @@ namespace DotNetPackageCorrelation; public class SemVerComparer : IComparer { + public static SemVerComparer Instance = new(); + public int Compare(SemVersion? x, SemVersion? y) { ArgumentNullException.ThrowIfNull(x); ArgumentNullException.ThrowIfNull(y); - return x.CompareSortOrderTo(y); + return x.ComparePrecedenceTo(y); } } diff --git a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/SemVersionConverter.cs b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/SemVersionConverter.cs index 95829ebc98..43b43c7dc5 100644 --- a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/SemVersionConverter.cs +++ b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/SemVersionConverter.cs @@ -19,6 +19,17 @@ public class SemVersionConverter : JsonConverter return null; } + public override SemVersion ReadAsPropertyName(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + var value = Read(ref reader, typeToConvert, options); + if (value is null) + { + throw new JsonException($"Unable to read {nameof(SemVersion)} as property name."); + } + + return value; + } + public override void Write(Utf8JsonWriter writer, SemVersion? value, JsonSerializerOptions options) { writer.WriteStringValue(value?.ToString()); diff --git a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/SdkPackagesExtensions.cs b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/SdkPackagesExtensions.cs new file mode 100644 index 0000000000..308be1dd35 --- /dev/null +++ b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/SdkPackagesExtensions.cs @@ -0,0 +1,28 @@ +using System.Collections.Immutable; + +using Semver; + +namespace DotNetPackageCorrelation; + +public static class SdkPackagesExtensions +{ + public static SemVersion? GetReplacementPackageVersion(this SdkPackages packages, SemVersion sdkVersion, string packageName) + { + var sdkVersionsToCheck = packages.Packages.Keys + .Where(v => v.Major == sdkVersion.Major) + .Where(v => v.ComparePrecedenceTo(sdkVersion) <= 0) + .OrderBy(v => v, SemVerComparer.Instance) + .Reverse() + .ToImmutableArray(); + foreach (var sdkVersionToCheck in sdkVersionsToCheck) + { + var sdkPackages = packages.Packages[sdkVersionToCheck]; + if (sdkPackages.Packages.TryGetValue(packageName, out var packageVersion)) + { + return packageVersion; + } + } + + return null; + } +} diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli.Test/EntryPointTests.Discover.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli.Test/EntryPointTests.Discover.cs index 2622c56e85..2913aae7fb 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli.Test/EntryPointTests.Discover.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli.Test/EntryPointTests.Discover.cs @@ -465,6 +465,76 @@ await RunAsync(path => ); } + [Fact] + public async Task SdkManagedPackagesAreAppropriatelyReturned() + { + // this test uses live packages + + // .NET SDK 8.0.306 ships with System.Text.Json/8.0.5 + await RunAsync(path => + [ + "discover", + "--job-path", + Path.Combine(path, "job.json"), + "--repo-root", + path, + "--workspace", + "src", + "--output", + Path.Combine(path, DiscoveryWorker.DiscoveryResultFileName) + ], + experimentsManager: new ExperimentsManager() { UseDirectDiscovery = true, InstallDotnetSdks = true }, + initialFiles: [ + ("src/global.json", """ + { + "sdk": { + "version": "8.0.307", + "rollForward": "latestMinor" + } + } + """), + ("src/project.csproj", """ + + + net8.0 + + + + + + """) + ], + expectedResult: new() + { + Path = "src", + Projects = [ + new() + { + FilePath = "project.csproj", + Dependencies = [ + new("System.Text.Encodings.Web", "8.0.0", DependencyType.Unknown, TargetFrameworks: ["net8.0"], IsTransitive: true), + new("System.Text.Json", "8.0.5", DependencyType.PackageReference, TargetFrameworks: ["net8.0"], IsDirect: true), + ], + Properties = [ + new("TargetFramework", "net8.0", "src/project.csproj"), + ], + TargetFrameworks = ["net8.0"], + ReferencedProjectPaths = [], + ImportedFiles = [], + AdditionalFiles = [], + } + ], + GlobalJson = new() + { + FilePath = "global.json", + Dependencies = [ + new("Microsoft.NET.Sdk", "8.0.307", DependencyType.MSBuildSdk), + ] + } + } + ); + } + private static async Task RunAsync( Func getArgs, TestFile[] initialFiles, diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Discover/SdkProjectDiscovery.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Discover/SdkProjectDiscovery.cs index 266d746fa3..b20c085df9 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Discover/SdkProjectDiscovery.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Discover/SdkProjectDiscovery.cs @@ -1,20 +1,34 @@ using System.Collections.Immutable; using System.Reflection; +using System.Text.Json; using System.Xml.Linq; using System.Xml.XPath; +using DotNetPackageCorrelation; + using Microsoft.Build.Logging.StructuredLogger; using NuGet.Versioning; using NuGetUpdater.Core.Utilities; +using Semver; + using LoggerProperty = Microsoft.Build.Logging.StructuredLogger.Property; namespace NuGetUpdater.Core.Discover; internal static class SdkProjectDiscovery { + private static readonly SdkPackages _sdkPackages; + + static SdkProjectDiscovery() + { + var packageCorrelationPath = Path.Combine(Path.GetDirectoryName(typeof(SdkProjectDiscovery).Assembly.Location)!, "dotnet-package-correlation.json"); + var packageCorrelationJson = File.ReadAllText(packageCorrelationPath); + _sdkPackages = JsonSerializer.Deserialize(packageCorrelationJson, Correlator.SerializerOptions)!; + } + private static readonly HashSet TopLevelPackageItemNames = new HashSet(StringComparer.OrdinalIgnoreCase) { "PackageReference" @@ -164,7 +178,7 @@ public static async Task> DiscoverWithBin } break; case NamedNode namedNode when namedNode is AddItem or RemoveItem: - ProcessResolvedPackageReference(namedNode, packagesPerProject, topLevelPackagesPerProject); + ProcessResolvedPackageReference(namedNode, packagesPerProject, topLevelPackagesPerProject, experimentsManager); if (namedNode is AddItem addItem) { @@ -285,10 +299,18 @@ public static async Task> DiscoverWithBin return projectDiscoveryResults; } + private static string? GetCorrespondingSdkManagedPackageVersion(string packageName, string sdkVersionString) + { + var sdkVersion = SemVersion.Parse(sdkVersionString); + var replacementPackageVersion = _sdkPackages.GetReplacementPackageVersion(sdkVersion, packageName); + return replacementPackageVersion?.ToString(); + } + private static void ProcessResolvedPackageReference( NamedNode node, Dictionary>> packagesPerProject, // projectPath -> tfm -> (packageName, packageVersion) - Dictionary> topLevelPackagesPerProject + Dictionary> topLevelPackagesPerProject, + ExperimentsManager experimentsManager ) { var doRemoveOperation = node is RemoveItem; @@ -348,7 +370,23 @@ Dictionary> topLevelPackagesPerProject if (doRemoveOperation) { - packagesPerTfm.Remove(packageName); + var wasRemoved = packagesPerTfm.Remove(packageName); + if (wasRemoved) + { + if (experimentsManager.InstallDotnetSdks) + { + // dotnet package correlation correction requires specific dotnet sdk handling + var sdkVersionString = GetPropertyValueFromProjectEvaluation(projectEvaluation, "NETCoreSdkVersion"); + if (sdkVersionString is not null) + { + var replacementVersion = GetCorrespondingSdkManagedPackageVersion(packageName, sdkVersionString); + if (replacementVersion is not null) + { + packagesPerTfm[packageName] = replacementVersion.ToString(); + } + } + } + } } if (doAddOperation) diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/EnsureDotNetPackageCorrelation.targets b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/EnsureDotNetPackageCorrelation.targets index 42e20bebcb..031c1fd050 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/EnsureDotNetPackageCorrelation.targets +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/EnsureDotNetPackageCorrelation.targets @@ -8,9 +8,10 @@ + <_DotNetCoreFiles Include="$(DotNetCoreDirectory)\**\*" /> - + diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/NuGetUpdater.Core.csproj b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/NuGetUpdater.Core.csproj index e01f1bbebd..fdee5408ec 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/NuGetUpdater.Core.csproj +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/NuGetUpdater.Core.csproj @@ -14,6 +14,7 @@ + From 27c9877926e3423ec93b9f717bc7dd552e4b5d17 Mon Sep 17 00:00:00 2001 From: "Brett V. Forsgren" Date: Fri, 13 Dec 2024 14:01:31 -0700 Subject: [PATCH 04/20] add update tests --- .../UpdateWorkerTests.PackageReference.cs | 79 +++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Update/UpdateWorkerTests.PackageReference.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Update/UpdateWorkerTests.PackageReference.cs index d0239e43c1..13b5ca1d64 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Update/UpdateWorkerTests.PackageReference.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Update/UpdateWorkerTests.PackageReference.cs @@ -3487,5 +3487,84 @@ await TestUpdateForProject("Some.Package", "1.0.0", "1.1.0", } ); } + + [Fact] + public async Task UpdateSdkManagedPackage_DirectDependency() + { + // this test uses live packages + await TestUpdateForProject("System.Text.Json", "8.0.0", "8.0.5", + experimentsManager: new ExperimentsManager() { UseDirectDiscovery = true, InstallDotnetSdks = true }, + projectContents: """ + + + net8.0 + + + + + + """, + additionalFiles: [ + ("global.json", """ + { + "sdk": { + "version": "8.0.307", + "rollForward": "latestMinor" + } + } + """) + ], + expectedProjectContents: """ + + + net8.0 + + + + + + """ + ); + } + + [Fact] + public async Task UpdateSdkManagedPackage_TransitiveDependency() + { + // this test uses live packages + await TestUpdateForProject("System.Text.Json", "6.0.9", "6.0.10", isTransitive: true, + experimentsManager: new ExperimentsManager() { UseDirectDiscovery = true, InstallDotnetSdks = true }, + projectContents: """ + + + net8.0 + + + + + + """, + additionalFiles: [ + ("global.json", """ + { + "sdk": { + "version": "8.0.307", + "rollForward": "latestMinor" + } + } + """) + ], + expectedProjectContents: """ + + + net8.0 + + + + + + + """ + ); + } } } From 332c6dedcb44522966608eb8c10b9ff71fb7e48a Mon Sep 17 00:00:00 2001 From: "Brett V. Forsgren" Date: Fri, 13 Dec 2024 14:06:10 -0700 Subject: [PATCH 05/20] add comment --- .../NuGetUpdater.Cli.Test/EntryPointTests.Discover.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli.Test/EntryPointTests.Discover.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli.Test/EntryPointTests.Discover.cs index 2913aae7fb..1d81436c89 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli.Test/EntryPointTests.Discover.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli.Test/EntryPointTests.Discover.cs @@ -470,6 +470,7 @@ public async Task SdkManagedPackagesAreAppropriatelyReturned() { // this test uses live packages + // .NET SDK 8.0.303 ships with System.Text.Json/8.0.4 // .NET SDK 8.0.306 ships with System.Text.Json/8.0.5 await RunAsync(path => [ From 162b046215ee041d6a057bbeeaed23679c51b5d8 Mon Sep 17 00:00:00 2001 From: "Brett V. Forsgren" Date: Fri, 13 Dec 2024 14:24:34 -0700 Subject: [PATCH 06/20] fix typo --- .../DotNetPackageCorrelation.Test/CorrelatorTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/CorrelatorTests.cs b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/CorrelatorTests.cs index 5400bbf2fe..82beda5a15 100644 --- a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/CorrelatorTests.cs +++ b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/CorrelatorTests.cs @@ -60,7 +60,7 @@ Package name | Version private static void AssertPackageVersion(SdkPackages packages, string sdkVersion, string packageName, string expectedPackageVersion) { - Assert.True(packages.Packages.TryGetValue(SemVersion.Parse(sdkVersion), out var packageSet), $"Unable to find SDK verison [{sdkVersion}]"); + Assert.True(packages.Packages.TryGetValue(SemVersion.Parse(sdkVersion), out var packageSet), $"Unable to find SDK version [{sdkVersion}]"); Assert.True(packageSet.Packages.TryGetValue(packageName, out var packageVersion), $"Unable to find package [{packageName}] under SDK version [{sdkVersion}]"); var actualPackageVersion = packageVersion.ToString(); Assert.Equal(expectedPackageVersion, actualPackageVersion); From 8f150cf4d4ff72e5f4933a1a141f7fdc57828a49 Mon Sep 17 00:00:00 2001 From: "Brett V. Forsgren" Date: Mon, 16 Dec 2024 10:42:07 -0700 Subject: [PATCH 07/20] create separate cli for package correlation --- .../DotNetPackageCorrelation.Cli.csproj | 16 ++++++++++++++++ .../Program.cs | 4 +++- .../DotNetPackageCorrelation.csproj | 2 -- .../EnsureDotNetPackageCorrelation.targets | 4 ++-- nuget/helpers/lib/NuGetUpdater/NuGetUpdater.sln | 6 ++++++ 5 files changed, 27 insertions(+), 5 deletions(-) create mode 100644 nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Cli/DotNetPackageCorrelation.Cli.csproj rename nuget/helpers/lib/NuGetUpdater/{DotNetPackageCorrelation => DotNetPackageCorrelation.Cli}/Program.cs (94%) diff --git a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Cli/DotNetPackageCorrelation.Cli.csproj b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Cli/DotNetPackageCorrelation.Cli.csproj new file mode 100644 index 0000000000..8e99811c84 --- /dev/null +++ b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Cli/DotNetPackageCorrelation.Cli.csproj @@ -0,0 +1,16 @@ + + + + $(CommonTargetFramework) + Exe + + + + + + + + + + + diff --git a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Program.cs b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Cli/Program.cs similarity index 94% rename from nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Program.cs rename to nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Cli/Program.cs index db0c576f7c..3caca06364 100644 --- a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Program.cs +++ b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Cli/Program.cs @@ -1,7 +1,9 @@ using System.CommandLine; using System.Text.Json; -namespace DotNetPackageCorrelation; +using DotNetPackageCorrelation; + +namespace DotNetPackageCorrelation.Cli; public class Program { diff --git a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/DotNetPackageCorrelation.csproj b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/DotNetPackageCorrelation.csproj index 3672a3eaee..8d631d5aeb 100644 --- a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/DotNetPackageCorrelation.csproj +++ b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/DotNetPackageCorrelation.csproj @@ -2,12 +2,10 @@ $(CommonTargetFramework) - Exe - diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/EnsureDotNetPackageCorrelation.targets b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/EnsureDotNetPackageCorrelation.targets index 031c1fd050..b24ce2967f 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/EnsureDotNetPackageCorrelation.targets +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/EnsureDotNetPackageCorrelation.targets @@ -1,7 +1,7 @@ - $(MSBuildThisFileDirectory)..\DotNetPackageCorrelation + $(MSBuildThisFileDirectory)..\DotNetPackageCorrelation.Cli $(MSBuildThisFileDirectory)..\..\dotnet-core $(MSBuildThisFileDirectory)dotnet-package-correlation.json @@ -12,7 +12,7 @@ - + diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.sln b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.sln index f4889927c0..6fa3f0a93d 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.sln +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.sln @@ -46,6 +46,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DotNetPackageCorrelation", EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DotNetPackageCorrelation.Test", "DotNetPackageCorrelation.Test\DotNetPackageCorrelation.Test.csproj", "{0945703C-C8DC-44F0-B1D8-0EFE011411AE}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DotNetPackageCorrelation.Cli", "DotNetPackageCorrelation.Cli\DotNetPackageCorrelation.Cli.csproj", "{509454EE-629F-4767-B1D4-7F2DF86C11B5}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -136,6 +138,10 @@ Global {0945703C-C8DC-44F0-B1D8-0EFE011411AE}.Debug|Any CPU.Build.0 = Debug|Any CPU {0945703C-C8DC-44F0-B1D8-0EFE011411AE}.Release|Any CPU.ActiveCfg = Release|Any CPU {0945703C-C8DC-44F0-B1D8-0EFE011411AE}.Release|Any CPU.Build.0 = Release|Any CPU + {509454EE-629F-4767-B1D4-7F2DF86C11B5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {509454EE-629F-4767-B1D4-7F2DF86C11B5}.Debug|Any CPU.Build.0 = Debug|Any CPU + {509454EE-629F-4767-B1D4-7F2DF86C11B5}.Release|Any CPU.ActiveCfg = Release|Any CPU + {509454EE-629F-4767-B1D4-7F2DF86C11B5}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE From 7b14b7159087f8044e517dfefc6862588b99909e Mon Sep 17 00:00:00 2001 From: "Brett V. Forsgren" Date: Mon, 16 Dec 2024 10:58:29 -0700 Subject: [PATCH 08/20] fix publishing --- .../DotNetPackageCorrelation/Model/Sdk.cs | 3 ++- .../Discover/SdkProjectDiscovery.cs | 18 ++++++++++++------ 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/Sdk.cs b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/Sdk.cs index 500a3746db..a2098ec883 100644 --- a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/Sdk.cs +++ b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/Sdk.cs @@ -1,6 +1,7 @@ -using Semver; using System.Text.Json.Serialization; +using Semver; + namespace DotNetPackageCorrelation; public record Sdk diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Discover/SdkProjectDiscovery.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Discover/SdkProjectDiscovery.cs index b20c085df9..4d7718162b 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Discover/SdkProjectDiscovery.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Discover/SdkProjectDiscovery.cs @@ -373,16 +373,22 @@ ExperimentsManager experimentsManager var wasRemoved = packagesPerTfm.Remove(packageName); if (wasRemoved) { - if (experimentsManager.InstallDotnetSdks) + // we only want to track packages that were explicitly part of the SDK's conflict resolution + if (node.Name == "RuntimeCopyLocalItems" && + node.Parent is Target target && + target.Name == "GenerateBuildDependencyFile") { // dotnet package correlation correction requires specific dotnet sdk handling - var sdkVersionString = GetPropertyValueFromProjectEvaluation(projectEvaluation, "NETCoreSdkVersion"); - if (sdkVersionString is not null) + if (experimentsManager.InstallDotnetSdks) { - var replacementVersion = GetCorrespondingSdkManagedPackageVersion(packageName, sdkVersionString); - if (replacementVersion is not null) + var sdkVersionString = GetPropertyValueFromProjectEvaluation(projectEvaluation, "NETCoreSdkVersion"); + if (sdkVersionString is not null) { - packagesPerTfm[packageName] = replacementVersion.ToString(); + var replacementVersion = GetCorrespondingSdkManagedPackageVersion(packageName, sdkVersionString); + if (replacementVersion is not null) + { + packagesPerTfm[packageName] = replacementVersion.ToString(); + } } } } From 0b7f20937071319166824ed642d02ad04d552f54 Mon Sep 17 00:00:00 2001 From: "Brett V. Forsgren" Date: Tue, 7 Jan 2025 15:58:33 -0700 Subject: [PATCH 09/20] make unit test SDK agnostic --- .../EntryPointTests.Discover.cs | 71 --------------- .../Discover/DiscoveryWorkerTests.Project.cs | 88 +++++++++++++++++++ .../Discover/SdkProjectDiscovery.cs | 31 ++++++- 3 files changed, 116 insertions(+), 74 deletions(-) diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli.Test/EntryPointTests.Discover.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli.Test/EntryPointTests.Discover.cs index 1d81436c89..2622c56e85 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli.Test/EntryPointTests.Discover.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli.Test/EntryPointTests.Discover.cs @@ -465,77 +465,6 @@ await RunAsync(path => ); } - [Fact] - public async Task SdkManagedPackagesAreAppropriatelyReturned() - { - // this test uses live packages - - // .NET SDK 8.0.303 ships with System.Text.Json/8.0.4 - // .NET SDK 8.0.306 ships with System.Text.Json/8.0.5 - await RunAsync(path => - [ - "discover", - "--job-path", - Path.Combine(path, "job.json"), - "--repo-root", - path, - "--workspace", - "src", - "--output", - Path.Combine(path, DiscoveryWorker.DiscoveryResultFileName) - ], - experimentsManager: new ExperimentsManager() { UseDirectDiscovery = true, InstallDotnetSdks = true }, - initialFiles: [ - ("src/global.json", """ - { - "sdk": { - "version": "8.0.307", - "rollForward": "latestMinor" - } - } - """), - ("src/project.csproj", """ - - - net8.0 - - - - - - """) - ], - expectedResult: new() - { - Path = "src", - Projects = [ - new() - { - FilePath = "project.csproj", - Dependencies = [ - new("System.Text.Encodings.Web", "8.0.0", DependencyType.Unknown, TargetFrameworks: ["net8.0"], IsTransitive: true), - new("System.Text.Json", "8.0.5", DependencyType.PackageReference, TargetFrameworks: ["net8.0"], IsDirect: true), - ], - Properties = [ - new("TargetFramework", "net8.0", "src/project.csproj"), - ], - TargetFrameworks = ["net8.0"], - ReferencedProjectPaths = [], - ImportedFiles = [], - AdditionalFiles = [], - } - ], - GlobalJson = new() - { - FilePath = "global.json", - Dependencies = [ - new("Microsoft.NET.Sdk", "8.0.307", DependencyType.MSBuildSdk), - ] - } - } - ); - } - private static async Task RunAsync( Func getArgs, TestFile[] initialFiles, diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Discover/DiscoveryWorkerTests.Project.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Discover/DiscoveryWorkerTests.Project.cs index fc0defa228..9d3cb0b0d1 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Discover/DiscoveryWorkerTests.Project.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Discover/DiscoveryWorkerTests.Project.cs @@ -1263,5 +1263,93 @@ await TestDiscoveryAsync( } ); } + + [Fact] + public async Task PackagesManagedAndRemovedByTheSdkAreReported() + { + // To avoid a unit test that's tightly coupled to the installed SDK, some files are faked. + // First up, the `dotnet-package-correlation.json` is faked to have the appropriate shape to report a + // package replacement. Doing this requires a temporary file and environment variable override. + using var tempDirectory = new TemporaryDirectory(); + var packageCorrelationFile = Path.Combine(tempDirectory.DirectoryPath, "dotnet-package-correlation.json"); + await File.WriteAllTextAsync(packageCorrelationFile, """ + { + "Packages": { + "8.0.100": { + "Packages": { + "Test.Only.Package": "1.0.99" + } + } + } + } + """); + using var tempEnvironment = new TemporaryEnvironment([("DOTNET_PACKAGE_CORRELATION_FILE_PATH", packageCorrelationFile)]); + + // The SDK package handling is detected in a very specific circumstance; an assembly being removed from the + // `@(RuntimeCopyLocalItems)` item group in the `GenerateBuildDependencyFile` target. Since we don't want + // to involve the real SDK, we fake some required targets. + await TestDiscoveryAsync( + experimentsManager: new ExperimentsManager() { InstallDotnetSdks = true, UseDirectDiscovery = true }, + packages: [], + workspacePath: "", + files: + [ + ("project.csproj", """ + + + + + + + + + 8.0.100 + net8.0 + + + + + + + + + + + + + + + + + + + + + + """) + ], + expectedResult: new() + { + Path = "", + Projects = [ + new() + { + FilePath = "project.csproj", + Dependencies = [ + new("Test.Only.Package", "1.0.99", DependencyType.Unknown, TargetFrameworks: ["net8.0"], IsTransitive: true) + ], + Properties = [ + new("NETCoreSdkVersion", "8.0.100", "project.csproj"), + new("TargetFramework", "net8.0", "project.csproj"), + ], + TargetFrameworks = ["net8.0"], + ReferencedProjectPaths = [], + ImportedFiles = [], + AdditionalFiles = [], + } + ] + } + ); + } } } diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Discover/SdkProjectDiscovery.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Discover/SdkProjectDiscovery.cs index 4d7718162b..7131743238 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Discover/SdkProjectDiscovery.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Discover/SdkProjectDiscovery.cs @@ -21,12 +21,12 @@ namespace NuGetUpdater.Core.Discover; internal static class SdkProjectDiscovery { private static readonly SdkPackages _sdkPackages; + private static readonly Dictionary _sdkPackagesByOverrideFile = new(); static SdkProjectDiscovery() { var packageCorrelationPath = Path.Combine(Path.GetDirectoryName(typeof(SdkProjectDiscovery).Assembly.Location)!, "dotnet-package-correlation.json"); - var packageCorrelationJson = File.ReadAllText(packageCorrelationPath); - _sdkPackages = JsonSerializer.Deserialize(packageCorrelationJson, Correlator.SerializerOptions)!; + _sdkPackages = LoadPackageCorrelationsFromFile(packageCorrelationPath); } private static readonly HashSet TopLevelPackageItemNames = new HashSet(StringComparer.OrdinalIgnoreCase) @@ -302,10 +302,35 @@ public static async Task> DiscoverWithBin private static string? GetCorrespondingSdkManagedPackageVersion(string packageName, string sdkVersionString) { var sdkVersion = SemVersion.Parse(sdkVersionString); - var replacementPackageVersion = _sdkPackages.GetReplacementPackageVersion(sdkVersion, packageName); + var replacementPackageVersion = GetSdkPackageCorrelations().GetReplacementPackageVersion(sdkVersion, packageName); return replacementPackageVersion?.ToString(); } + private static SdkPackages GetSdkPackageCorrelations() + { + var packageCorrelationFileOverride = Environment.GetEnvironmentVariable("DOTNET_PACKAGE_CORRELATION_FILE_PATH"); + if (packageCorrelationFileOverride is not null) + { + // this is used as a test hook to allow unit tests to be SDK agnostic + if (_sdkPackagesByOverrideFile.TryGetValue(packageCorrelationFileOverride, out var sdkPackages)) + { + return sdkPackages; + } + + sdkPackages = LoadPackageCorrelationsFromFile(packageCorrelationFileOverride); + _sdkPackagesByOverrideFile[packageCorrelationFileOverride] = sdkPackages; + return sdkPackages; + } + + return _sdkPackages; + } + + private static SdkPackages LoadPackageCorrelationsFromFile(string filePath) + { + var packageCorrelationJson = File.ReadAllText(filePath); + return JsonSerializer.Deserialize(packageCorrelationJson, Correlator.SerializerOptions)!; + } + private static void ProcessResolvedPackageReference( NamedNode node, Dictionary>> packagesPerProject, // projectPath -> tfm -> (packageName, packageVersion) From e69d9a93213584f97e99a871b91aace149a7821b Mon Sep 17 00:00:00 2001 From: "Brett V. Forsgren" Date: Mon, 16 Dec 2024 13:47:56 -0700 Subject: [PATCH 10/20] rename property --- .../DotNetPackageCorrelation.Cli/Program.cs | 4 ++-- .../DotNetPackageCorrelation.Test/CorrelatorTests.cs | 12 ++++++------ .../DotNetPackageCorrelation.Test/EndToEndTests.cs | 4 ++-- .../SdkPackagesTests.cs | 8 ++++---- .../DotNetPackageCorrelation/Correlator.cs | 6 +++--- .../DotNetPackageCorrelation/Model/SdkPackages.cs | 2 +- .../SdkPackagesExtensions.cs | 4 ++-- .../Discover/DiscoveryWorkerTests.Project.cs | 2 +- 8 files changed, 21 insertions(+), 21 deletions(-) diff --git a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Cli/Program.cs b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Cli/Program.cs index 3caca06364..b9e608c0ac 100644 --- a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Cli/Program.cs +++ b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Cli/Program.cs @@ -22,8 +22,8 @@ public static async Task Main(string[] args) // the tool is expected to be given the path to the .NET Core repository, but the correlator only needs a specific subdirectory var releaseNotesDirectory = new DirectoryInfo(Path.Combine(coreLocationDirectory.FullName, "release-notes")); var correlator = new Correlator(releaseNotesDirectory); - var (packages, _warnings) = await correlator.RunAsync(); - var json = JsonSerializer.Serialize(packages, Correlator.SerializerOptions); + var (sdkPackages, _warnings) = await correlator.RunAsync(); + var json = JsonSerializer.Serialize(sdkPackages, Correlator.SerializerOptions); await File.WriteAllTextAsync(output.FullName, json); }, coreLocationOption, outputOption); var exitCode = await command.InvokeAsync(args); diff --git a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/CorrelatorTests.cs b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/CorrelatorTests.cs index 82beda5a15..5efa04977f 100644 --- a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/CorrelatorTests.cs +++ b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/CorrelatorTests.cs @@ -10,7 +10,7 @@ public class CorrelatorTests public async Task FileHandling_AllFilesShapedAppropriately() { // the JSON and markdown are shaped as expected - var (packages, warnings) = await RunFromFilesAsync( + var (sdkPackages, warnings) = await RunFromFilesAsync( ("8.0/releases.json", """ { "releases": [ @@ -31,8 +31,8 @@ Package name | Version """) ); Assert.Empty(warnings); - AssertPackageVersion(packages, "8.0.100", "Package.A", "8.0.0"); - AssertPackageVersion(packages, "8.0.100", "Package.B", "1.2.3"); + AssertPackageVersion(sdkPackages, "8.0.100", "Package.A", "8.0.0"); + AssertPackageVersion(sdkPackages, "8.0.100", "Package.B", "1.2.3"); } [Theory] @@ -58,15 +58,15 @@ Package name | Version Assert.Equal(expectedPackageVersion, actualpackage.Version.ToString()); } - private static void AssertPackageVersion(SdkPackages packages, string sdkVersion, string packageName, string expectedPackageVersion) + private static void AssertPackageVersion(SdkPackages sdkPackages, string sdkVersion, string packageName, string expectedPackageVersion) { - Assert.True(packages.Packages.TryGetValue(SemVersion.Parse(sdkVersion), out var packageSet), $"Unable to find SDK version [{sdkVersion}]"); + Assert.True(sdkPackages.Sdks.TryGetValue(SemVersion.Parse(sdkVersion), out var packageSet), $"Unable to find SDK version [{sdkVersion}]"); Assert.True(packageSet.Packages.TryGetValue(packageName, out var packageVersion), $"Unable to find package [{packageName}] under SDK version [{sdkVersion}]"); var actualPackageVersion = packageVersion.ToString(); Assert.Equal(expectedPackageVersion, actualPackageVersion); } - private static async Task<(SdkPackages Packages, IEnumerable Warnings)> RunFromFilesAsync(params (string Path, string Content)[] files) + private static async Task<(SdkPackages SdkPackages, IEnumerable Warnings)> RunFromFilesAsync(params (string Path, string Content)[] files) { var testDirectory = Path.Combine(Path.GetDirectoryName(typeof(CorrelatorTests).Assembly.Location)!, "test-data", Guid.NewGuid().ToString("D")); Directory.CreateDirectory(testDirectory); diff --git a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/EndToEndTests.cs b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/EndToEndTests.cs index b04f882a81..3da1ceab2c 100644 --- a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/EndToEndTests.cs +++ b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/EndToEndTests.cs @@ -17,11 +17,11 @@ public async Task IntegrationTest() var correlator = new Correlator(new DirectoryInfo(Path.Combine(dotnetCoreDirectory, "release-notes"))); // act - var (packages, _warnings) = await correlator.RunAsync(); + var (sdkPackages, _warnings) = await correlator.RunAsync(); var sdkVersion = SemVersion.Parse("8.0.307"); // SDK 8.0.307 has no System.Text.Json, but 8.0.306 provides System.Text.Json 8.0.5 - var systemTextJsonPackageVersion = packages.GetReplacementPackageVersion(sdkVersion, "system.TEXT.json"); + var systemTextJsonPackageVersion = sdkPackages.GetReplacementPackageVersion(sdkVersion, "system.TEXT.json"); // assert Assert.Equal("8.0.5", systemTextJsonPackageVersion?.ToString()); diff --git a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/SdkPackagesTests.cs b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/SdkPackagesTests.cs index 9e1a0b136e..d6428e6683 100644 --- a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/SdkPackagesTests.cs +++ b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/SdkPackagesTests.cs @@ -26,7 +26,7 @@ public void CorrelatedPackageCanBeFound(SdkPackages packages, string sdkVersionS // packages new SdkPackages() { - Packages = new SortedDictionary(SemVerComparer.Instance) + Sdks = new SortedDictionary(SemVerComparer.Instance) { { SemVersion.Parse("1.0.100"), @@ -74,7 +74,7 @@ public void CorrelatedPackageCanBeFound(SdkPackages packages, string sdkVersionS // packages new SdkPackages() { - Packages = new SortedDictionary(SemVerComparer.Instance) + Sdks = new SortedDictionary(SemVerComparer.Instance) { { SemVersion.Parse("1.0.100"), @@ -112,7 +112,7 @@ public void CorrelatedPackageCanBeFound(SdkPackages packages, string sdkVersionS // packages new SdkPackages() { - Packages = new SortedDictionary(SemVerComparer.Instance) + Sdks = new SortedDictionary(SemVerComparer.Instance) { { SemVersion.Parse("1.0.100"), @@ -150,7 +150,7 @@ public void CorrelatedPackageCanBeFound(SdkPackages packages, string sdkVersionS // packages new SdkPackages() { - Packages = new SortedDictionary(SemVerComparer.Instance) + Sdks = new SortedDictionary(SemVerComparer.Instance) { { SemVersion.Parse("1.0.100"), diff --git a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Correlator.cs b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Correlator.cs index cf6e89bcd5..bf6df66445 100644 --- a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Correlator.cs +++ b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Correlator.cs @@ -21,7 +21,7 @@ public Correlator(DirectoryInfo releaseNotesDirectory) _releaseNotesDirectory = releaseNotesDirectory; } - public async Task<(SdkPackages Packages, IEnumerable Warnings)> RunAsync() + public async Task<(SdkPackages SdkPackages, IEnumerable Warnings)> RunAsync() { var runtimeVersions = new List(); foreach (var directory in Directory.EnumerateDirectories(_releaseNotesDirectory.FullName)) @@ -63,10 +63,10 @@ public Correlator(DirectoryInfo releaseNotesDirectory) continue; } - if (!sdkPackages.Packages.TryGetValue(sdk.Version, out var packagesAndVersions)) + if (!sdkPackages.Sdks.TryGetValue(sdk.Version, out var packagesAndVersions)) { packagesAndVersions = new PackageSet(); - sdkPackages.Packages[sdk.Version] = packagesAndVersions; + sdkPackages.Sdks[sdk.Version] = packagesAndVersions; } var runtimeDirectory = new DirectoryInfo(Path.Combine(_releaseNotesDirectory.FullName, version.ToString(), sdk.RuntimeVersion.ToString())); diff --git a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/SdkPackages.cs b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/SdkPackages.cs index ba51a349f9..8ebfa6dfa7 100644 --- a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/SdkPackages.cs +++ b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/SdkPackages.cs @@ -7,5 +7,5 @@ namespace DotNetPackageCorrelation; public record SdkPackages { [JsonObjectCreationHandling(JsonObjectCreationHandling.Populate)] - public SortedDictionary Packages { get; init; } = new(SemVerComparer.Instance); + public SortedDictionary Sdks { get; init; } = new(SemVerComparer.Instance); } diff --git a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/SdkPackagesExtensions.cs b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/SdkPackagesExtensions.cs index 308be1dd35..a8d20db66a 100644 --- a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/SdkPackagesExtensions.cs +++ b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/SdkPackagesExtensions.cs @@ -8,7 +8,7 @@ public static class SdkPackagesExtensions { public static SemVersion? GetReplacementPackageVersion(this SdkPackages packages, SemVersion sdkVersion, string packageName) { - var sdkVersionsToCheck = packages.Packages.Keys + var sdkVersionsToCheck = packages.Sdks.Keys .Where(v => v.Major == sdkVersion.Major) .Where(v => v.ComparePrecedenceTo(sdkVersion) <= 0) .OrderBy(v => v, SemVerComparer.Instance) @@ -16,7 +16,7 @@ public static class SdkPackagesExtensions .ToImmutableArray(); foreach (var sdkVersionToCheck in sdkVersionsToCheck) { - var sdkPackages = packages.Packages[sdkVersionToCheck]; + var sdkPackages = packages.Sdks[sdkVersionToCheck]; if (sdkPackages.Packages.TryGetValue(packageName, out var packageVersion)) { return packageVersion; diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Discover/DiscoveryWorkerTests.Project.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Discover/DiscoveryWorkerTests.Project.cs index 9d3cb0b0d1..4a8ccff93e 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Discover/DiscoveryWorkerTests.Project.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Discover/DiscoveryWorkerTests.Project.cs @@ -1274,7 +1274,7 @@ public async Task PackagesManagedAndRemovedByTheSdkAreReported() var packageCorrelationFile = Path.Combine(tempDirectory.DirectoryPath, "dotnet-package-correlation.json"); await File.WriteAllTextAsync(packageCorrelationFile, """ { - "Packages": { + "Sdks": { "8.0.100": { "Packages": { "Test.Only.Package": "1.0.99" From 7cfb9390798cce879fdead7338d24257e7b3cd47 Mon Sep 17 00:00:00 2001 From: "Brett V. Forsgren" Date: Mon, 16 Dec 2024 14:37:49 -0700 Subject: [PATCH 11/20] fix typo --- .../Discover/DiscoveryWorkerTests.Project.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Discover/DiscoveryWorkerTests.Project.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Discover/DiscoveryWorkerTests.Project.cs index 4a8ccff93e..04b9e6f274 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Discover/DiscoveryWorkerTests.Project.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Discover/DiscoveryWorkerTests.Project.cs @@ -1309,7 +1309,7 @@ await TestDiscoveryAsync( - + From 8d9dc4ac294e2cf3f393a8c1062cdee5109617fa Mon Sep 17 00:00:00 2001 From: "Brett V. Forsgren" Date: Mon, 16 Dec 2024 15:54:28 -0700 Subject: [PATCH 12/20] use local packages for test --- .../UpdateWorkerTests.PackageReference.cs | 72 ++++++++----------- 1 file changed, 29 insertions(+), 43 deletions(-) diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Update/UpdateWorkerTests.PackageReference.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Update/UpdateWorkerTests.PackageReference.cs index 13b5ca1d64..33ae992d2d 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Update/UpdateWorkerTests.PackageReference.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Update/UpdateWorkerTests.PackageReference.cs @@ -3491,55 +3491,42 @@ await TestUpdateForProject("Some.Package", "1.0.0", "1.1.0", [Fact] public async Task UpdateSdkManagedPackage_DirectDependency() { - // this test uses live packages - await TestUpdateForProject("System.Text.Json", "8.0.0", "8.0.5", - experimentsManager: new ExperimentsManager() { UseDirectDiscovery = true, InstallDotnetSdks = true }, - projectContents: """ - - - net8.0 - - - - - - """, - additionalFiles: [ - ("global.json", """ - { - "sdk": { - "version": "8.0.307", - "rollForward": "latestMinor" + // To avoid a unit test that's tightly coupled to the installed SDK, the package correlation file is faked. + // Doing this requires a temporary file and environment variable override. Note that SDK version 8.0.100 + // or greater is required. + using var tempDirectory = new TemporaryDirectory(); + var packageCorrelationFile = Path.Combine(tempDirectory.DirectoryPath, "dotnet-package-correlation.json"); + await File.WriteAllTextAsync(packageCorrelationFile, """ + { + "Sdks": { + "8.0.100": { + "Packages": { + "System.Text.Json": "8.0.98" } } - """) - ], - expectedProjectContents: """ - - - net8.0 - - - - - - """ - ); - } - - [Fact] - public async Task UpdateSdkManagedPackage_TransitiveDependency() - { - // this test uses live packages - await TestUpdateForProject("System.Text.Json", "6.0.9", "6.0.10", isTransitive: true, + } + } + """); + using var tempEnvironment = new TemporaryEnvironment([("DOTNET_PACKAGE_CORRELATION_FILE_PATH", packageCorrelationFile)]); + + // In the `packages` section below, we fake a `System.Text.Json` package with a low assembly version that + // will always trigger the replacement so that can be detected and then the equivalent version is pulled + // from the correlation file specified above. In the original project contents, package version `8.0.98` + // is reported which makes the update to `8.0.99` always possible. + await TestUpdateForProject("System.Text.Json", "8.0.0", "8.0.99", experimentsManager: new ExperimentsManager() { UseDirectDiscovery = true, InstallDotnetSdks = true }, + packages: + [ + MockNuGetPackage.CreatePackageWithAssembly("System.Text.Json", "8.0.0", "net8.0", assemblyVersion: "8.0.0.0"), // this assembly version is lower than what the SDK will have + MockNuGetPackage.CreatePackageWithAssembly("System.Text.Json", "8.0.99", "net8.0", assemblyVersion: "8.99.99.99"), // this assembly version is greater than what the SDK will have + ], projectContents: """ net8.0 - + """, @@ -3547,7 +3534,7 @@ await TestUpdateForProject("System.Text.Json", "6.0.9", "6.0.10", isTransitive: ("global.json", """ { "sdk": { - "version": "8.0.307", + "version": "8.0.100", "rollForward": "latestMinor" } } @@ -3559,8 +3546,7 @@ await TestUpdateForProject("System.Text.Json", "6.0.9", "6.0.10", isTransitive: net8.0 - - + """ From 6b6e9c9760f7d837eb54ee14fcb75c5bbeb935dc Mon Sep 17 00:00:00 2001 From: "Brett V. Forsgren" Date: Mon, 16 Dec 2024 19:21:11 -0700 Subject: [PATCH 13/20] maintain `global.json` during dependency resolution --- .../UpdateWorkerTests.PackageReference.cs | 68 ++++++++++++++++ .../Utilities/MSBuildHelper.cs | 78 ++++++++++++------- .../Utilities/NuGetHelper.cs | 2 +- 3 files changed, 120 insertions(+), 28 deletions(-) diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Update/UpdateWorkerTests.PackageReference.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Update/UpdateWorkerTests.PackageReference.cs index 33ae992d2d..70a126979e 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Update/UpdateWorkerTests.PackageReference.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Update/UpdateWorkerTests.PackageReference.cs @@ -3552,5 +3552,73 @@ await TestUpdateForProject("System.Text.Json", "8.0.0", "8.0.99", """ ); } + + [Fact(Skip = "https://github.com/dependabot/dependabot-core/issues/11140")] + public async Task UpdateSdkManagedPackage_TransitiveDependency() + { + // To avoid a unit test that's tightly coupled to the installed SDK, the package correlation file is faked. + // Doing this requires a temporary file and environment variable override. Note that SDK version 8.0.100 + // or greater is required. + using var tempDirectory = new TemporaryDirectory(); + var packageCorrelationFile = Path.Combine(tempDirectory.DirectoryPath, "dotnet-package-correlation.json"); + await File.WriteAllTextAsync(packageCorrelationFile, """ + { + "Sdks": { + "8.0.100": { + "Packages": { + "System.Text.Json": "8.0.98" + } + } + } + } + """); + using var tempEnvironment = new TemporaryEnvironment([("DOTNET_PACKAGE_CORRELATION_FILE_PATH", packageCorrelationFile)]); + + // In the `packages` section below, we fake a `System.Text.Json` package with a low assembly version that + // will always trigger the replacement so that can be detected and then the equivalent version is pulled + // from the correlation file specified above. In the original project contents, package version `8.0.98` + // is reported which makes the update to `8.0.99` always possible. + await TestUpdateForProject("System.Text.Json", "8.0.98", "8.0.99", + isTransitive: true, + experimentsManager: new ExperimentsManager() { UseDirectDiscovery = true, InstallDotnetSdks = true }, + packages: + [ + MockNuGetPackage.CreateSimplePackage("Some.Package", "1.0.0", "net8.0", [(null, [("System.Text.Json", "[8.0.0]")])]), + MockNuGetPackage.CreateSimplePackage("Some.Package", "2.0.0", "net8.0", [(null, [("System.Text.Json", "[8.0.99]")])]), + MockNuGetPackage.CreatePackageWithAssembly("System.Text.Json", "8.0.0", "net8.0", assemblyVersion: "8.0.0.0"), // this assembly version is lower than what the SDK will have + MockNuGetPackage.CreatePackageWithAssembly("System.Text.Json", "8.0.99", "net8.0", assemblyVersion: "8.99.99.99"), // this assembly version is greater than what the SDK will have + ], + projectContents: """ + + + net8.0 + + + + + + """, + additionalFiles: [ + ("global.json", """ + { + "sdk": { + "version": "8.0.100", + "rollForward": "latestMinor" + } + } + """) + ], + expectedProjectContents: """ + + + net8.0 + + + + + + """ + ); + } } } diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Utilities/MSBuildHelper.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Utilities/MSBuildHelper.cs index 83bd5d806b..14c97598bf 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Utilities/MSBuildHelper.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Utilities/MSBuildHelper.cs @@ -19,6 +19,7 @@ using NuGet.Versioning; using NuGetUpdater.Core.Analyze; +using NuGetUpdater.Core.Discover; using NuGetUpdater.Core.Utilities; namespace NuGetUpdater.Core; @@ -342,7 +343,7 @@ internal static async Task DependenciesAreCoherentAsync(string repoRoot, s var tempDirectory = Directory.CreateTempSubdirectory("package-dependency-coherence_"); try { - var tempProjectPath = await CreateTempProjectAsync(tempDirectory, repoRoot, projectPath, targetFramework, packages, logger); + var tempProjectPath = await CreateTempProjectAsync(tempDirectory, repoRoot, projectPath, targetFramework, packages, experimentsManager, logger); var (exitCode, stdOut, stdErr) = await ProcessEx.RunDotnetWithoutMSBuildEnvironmentVariablesAsync(["restore", tempProjectPath], tempDirectory.FullName, experimentsManager); // NU1608: Detected package version outside of dependency constraint @@ -362,7 +363,7 @@ internal static async Task DependenciesAreCoherentAsync(string repoRoot, s try { - string tempProjectPath = await CreateTempProjectAsync(tempDirectory, repoRoot, projectPath, targetFramework, packages, logger); + string tempProjectPath = await CreateTempProjectAsync(tempDirectory, repoRoot, projectPath, targetFramework, packages, experimentsManager, logger); var (exitCode, stdOut, stdErr) = await ProcessEx.RunDotnetWithoutMSBuildEnvironmentVariablesAsync(["restore", tempProjectPath], tempDirectory.FullName, experimentsManager); // Add Dependency[] packages to List existingPackages @@ -518,7 +519,7 @@ internal static async Task DependenciesAreCoherentAsync(string repoRoot, s var tempDirectory = Directory.CreateTempSubdirectory("package-dependency-coherence_"); try { - var tempProjectPath = await CreateTempProjectAsync(tempDirectory, repoRoot, projectPath, targetFramework, packages, logger); + var tempProjectPath = await CreateTempProjectAsync(tempDirectory, repoRoot, projectPath, targetFramework, packages, experimentsManager, logger); var (exitCode, stdOut, stdErr) = await ProcessEx.RunDotnetWithoutMSBuildEnvironmentVariablesAsync(["restore", tempProjectPath], tempDirectory.FullName, experimentsManager); ThrowOnUnauthenticatedFeed(stdOut); @@ -668,12 +669,22 @@ internal static async Task CreateTempProjectAsync( string projectPath, string targetFramework, IReadOnlyCollection packages, + ExperimentsManager experimentsManager, ILogger logger, bool usePackageDownload = false) { var projectDirectory = Path.GetDirectoryName(projectPath); projectDirectory ??= repoRoot; - var topLevelFiles = Directory.GetFiles(repoRoot); + + if (experimentsManager.InstallDotnetSdks) + { + var globalJsonPath = PathHelper.GetFileInDirectoryOrParent(projectPath, repoRoot, "global.json", caseSensitive: true); + if (globalJsonPath is not null) + { + File.Copy(globalJsonPath, Path.Combine(tempDir.FullName, "global.json")); + } + } + var nugetConfigPath = PathHelper.GetFileInDirectoryOrParent(projectPath, repoRoot, "NuGet.Config", caseSensitive: false); if (nugetConfigPath is not null) { @@ -832,40 +843,53 @@ internal static async Task GetAllPackageDependenciesAsync( string targetFramework, IReadOnlyCollection packages, ExperimentsManager experimentsManager, - ILogger logger) + ILogger logger + ) { var tempDirectory = Directory.CreateTempSubdirectory("package-dependency-resolution_"); try { var topLevelPackagesNames = packages.Select(p => p.Name).ToHashSet(StringComparer.OrdinalIgnoreCase); - var tempProjectPath = await CreateTempProjectAsync(tempDirectory, repoRoot, projectPath, targetFramework, packages, logger); + var tempProjectPath = await CreateTempProjectAsync(tempDirectory, repoRoot, projectPath, targetFramework, packages, experimentsManager, logger); - var (exitCode, stdout, stderr) = await ProcessEx.RunDotnetWithoutMSBuildEnvironmentVariablesAsync(["build", tempProjectPath, "/t:_ReportDependencies"], tempDirectory.FullName, experimentsManager); - ThrowOnUnauthenticatedFeed(stdout); - - if (exitCode == 0) + Dependency[] allDependencies; + if (experimentsManager.UseDirectDiscovery) { - ImmutableArray tfms = [targetFramework]; - var lines = stdout.Split('\n').Select(line => line.Trim()); - var pattern = PackagePattern(); - var allDependencies = lines - .Select(line => pattern.Match(line)) - .Where(match => match.Success) - .Select(match => - { - var PackageName = match.Groups["PackageName"].Value; - var isTransitive = !topLevelPackagesNames.Contains(PackageName); - return new Dependency(PackageName, match.Groups["PackageVersion"].Value, DependencyType.Unknown, TargetFrameworks: tfms, IsTransitive: isTransitive); - }) - .ToArray(); - - return allDependencies; + var projectDiscovery = await SdkProjectDiscovery.DiscoverAsync(repoRoot, tempDirectory.FullName, tempProjectPath, experimentsManager, logger); + allDependencies = projectDiscovery + .Where(p => p.FilePath == Path.GetFileName(tempProjectPath)) + .FirstOrDefault() + ?.Dependencies.ToArray() ?? []; } else { - logger?.Warn($"dotnet build in {nameof(GetAllPackageDependenciesAsync)} failed. STDOUT: {stdout} STDERR: {stderr}"); - return []; + var (exitCode, stdout, stderr) = await ProcessEx.RunDotnetWithoutMSBuildEnvironmentVariablesAsync(["build", tempProjectPath, "/t:_ReportDependencies"], tempDirectory.FullName, experimentsManager); + ThrowOnUnauthenticatedFeed(stdout); + + if (exitCode == 0) + { + ImmutableArray tfms = [targetFramework]; + var lines = stdout.Split('\n').Select(line => line.Trim()); + var pattern = PackagePattern(); + allDependencies = lines + .Select(line => pattern.Match(line)) + .Where(match => match.Success) + .Select(match => + { + var PackageName = match.Groups["PackageName"].Value; + var isTransitive = !topLevelPackagesNames.Contains(PackageName); + return new Dependency(PackageName, match.Groups["PackageVersion"].Value, DependencyType.Unknown, TargetFrameworks: tfms, IsTransitive: isTransitive); + }) + .ToArray(); + } + else + { + logger?.Warn($"dotnet build in {nameof(GetAllPackageDependenciesAsync)} failed. STDOUT: {stdout} STDERR: {stderr}"); + allDependencies = []; + } } + + return allDependencies; } finally { diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Utilities/NuGetHelper.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Utilities/NuGetHelper.cs index fb29c8e751..9fb12652d2 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Utilities/NuGetHelper.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Utilities/NuGetHelper.cs @@ -9,7 +9,7 @@ internal static async Task DownloadNuGetPackagesAsync(string repoRoot, str var tempDirectory = Directory.CreateTempSubdirectory("msbuild_sdk_restore_"); try { - var tempProjectPath = await MSBuildHelper.CreateTempProjectAsync(tempDirectory, repoRoot, projectPath, "netstandard2.0", packages, logger, usePackageDownload: true); + var tempProjectPath = await MSBuildHelper.CreateTempProjectAsync(tempDirectory, repoRoot, projectPath, "netstandard2.0", packages, experimentsManager, logger, usePackageDownload: true); var (exitCode, stdOut, stdErr) = await ProcessEx.RunDotnetWithoutMSBuildEnvironmentVariablesAsync(["restore", tempProjectPath], tempDirectory.FullName, experimentsManager); return exitCode == 0; From a309c5671953f1e624bbf9f3b8b412404a07bd4d Mon Sep 17 00:00:00 2001 From: "Brett V. Forsgren" Date: Thu, 19 Dec 2024 16:34:33 -0700 Subject: [PATCH 14/20] use runtime package version for replacement lookup --- .../CorrelatorTests.cs | 33 +-- .../EndToEndTests.cs | 11 +- .../RuntimePackagesTests.cs | 206 ++++++++++++++++++ .../SdkPackagesTests.cs | 205 ----------------- .../DotNetPackageCorrelation/Correlator.cs | 22 +- .../Model/PackageMapper.cs | 68 ++++++ .../{SdkPackages.cs => RuntimePackages.cs} | 4 +- .../SdkPackagesExtensions.cs | 28 --- .../Discover/DiscoveryWorkerTests.Project.cs | 50 +++-- .../UpdateWorkerTests.PackageReference.cs | 17 +- .../Analyze/AnalyzeWorker.cs | 2 + .../DependencyDiscovery.targets | 4 +- .../Discover/SdkProjectDiscovery.cs | 135 ++++++++---- .../Updater/PackageReferenceUpdater.cs | 5 + 14 files changed, 453 insertions(+), 337 deletions(-) create mode 100644 nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/RuntimePackagesTests.cs delete mode 100644 nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/SdkPackagesTests.cs create mode 100644 nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/PackageMapper.cs rename nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/{SdkPackages.cs => RuntimePackages.cs} (54%) delete mode 100644 nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/SdkPackagesExtensions.cs diff --git a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/CorrelatorTests.cs b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/CorrelatorTests.cs index 5efa04977f..0d42491536 100644 --- a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/CorrelatorTests.cs +++ b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/CorrelatorTests.cs @@ -10,7 +10,8 @@ public class CorrelatorTests public async Task FileHandling_AllFilesShapedAppropriately() { // the JSON and markdown are shaped as expected - var (sdkPackages, warnings) = await RunFromFilesAsync( + // we're able to determine from `Runtime.Package/8.0.0` that the corresponding version of `Some.Package` is `1.2.3` + var (packageMapper, warnings) = await PackageMapperFromFilesAsync( ("8.0/releases.json", """ { "releases": [ @@ -26,13 +27,12 @@ public async Task FileHandling_AllFilesShapedAppropriately() ("8.0/8.0.0/8.0.0.md", """ Package name | Version :-- | :-- - Package.A | 8.0.0 - Package.B | 1.2.3 + Runtime.Package | 8.0.0 + Some.Package | 1.2.3 """) ); Assert.Empty(warnings); - AssertPackageVersion(sdkPackages, "8.0.100", "Package.A", "8.0.0"); - AssertPackageVersion(sdkPackages, "8.0.100", "Package.B", "1.2.3"); + AssertPackageVersion(packageMapper, "Runtime.Package", "8.0.0", "Some.Package", "1.2.3"); } [Theory] @@ -58,15 +58,21 @@ Package name | Version Assert.Equal(expectedPackageVersion, actualpackage.Version.ToString()); } - private static void AssertPackageVersion(SdkPackages sdkPackages, string sdkVersion, string packageName, string expectedPackageVersion) + private static void AssertPackageVersion(PackageMapper packageMapper, string runtimePackageName, string runtimePackageVersion, string candidatePackageName, string? expectedPackageVersion) { - Assert.True(sdkPackages.Sdks.TryGetValue(SemVersion.Parse(sdkVersion), out var packageSet), $"Unable to find SDK version [{sdkVersion}]"); - Assert.True(packageSet.Packages.TryGetValue(packageName, out var packageVersion), $"Unable to find package [{packageName}] under SDK version [{sdkVersion}]"); - var actualPackageVersion = packageVersion.ToString(); - Assert.Equal(expectedPackageVersion, actualPackageVersion); + var actualPackageVersion = packageMapper.GetPackageVersionThatShippedWithOtherPackage(runtimePackageName, SemVersion.Parse(runtimePackageVersion), candidatePackageName); + if (expectedPackageVersion is null) + { + Assert.Null(actualPackageVersion); + } + else + { + Assert.NotNull(actualPackageVersion); + Assert.Equal(expectedPackageVersion, actualPackageVersion.ToString()); + } } - private static async Task<(SdkPackages SdkPackages, IEnumerable Warnings)> RunFromFilesAsync(params (string Path, string Content)[] files) + private static async Task<(PackageMapper PackageMapper, IEnumerable Warnings)> PackageMapperFromFilesAsync(params (string Path, string Content)[] files) { var testDirectory = Path.Combine(Path.GetDirectoryName(typeof(CorrelatorTests).Assembly.Location)!, "test-data", Guid.NewGuid().ToString("D")); Directory.CreateDirectory(testDirectory); @@ -81,8 +87,9 @@ private static void AssertPackageVersion(SdkPackages sdkPackages, string sdkVers } var correlator = new Correlator(new DirectoryInfo(testDirectory)); - var result = await correlator.RunAsync(); - return result; + var (runtimePackages, warnings) = await correlator.RunAsync(); + var packageMapper = PackageMapper.Load(runtimePackages); + return (packageMapper, warnings); } finally { diff --git a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/EndToEndTests.cs b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/EndToEndTests.cs index 3da1ceab2c..4695babffa 100644 --- a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/EndToEndTests.cs +++ b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/EndToEndTests.cs @@ -17,14 +17,13 @@ public async Task IntegrationTest() var correlator = new Correlator(new DirectoryInfo(Path.Combine(dotnetCoreDirectory, "release-notes"))); // act - var (sdkPackages, _warnings) = await correlator.RunAsync(); - var sdkVersion = SemVersion.Parse("8.0.307"); - - // SDK 8.0.307 has no System.Text.Json, but 8.0.306 provides System.Text.Json 8.0.5 - var systemTextJsonPackageVersion = sdkPackages.GetReplacementPackageVersion(sdkVersion, "system.TEXT.json"); + var (runtimePackages, _warnings) = await correlator.RunAsync(); + var packageMapper = PackageMapper.Load(runtimePackages); // assert - Assert.Equal("8.0.5", systemTextJsonPackageVersion?.ToString()); + // Microsoft.NETCore.App.Ref/8.0.8 didn't ship with System.Text.Json, but the previous version 8.0.7 shipped at the same time as System.Text.Json/8.0.4 + var systemTextJsonVersion = packageMapper.GetPackageVersionThatShippedWithOtherPackage("Microsoft.NETCore.App.Ref", SemVersion.Parse("8.0.8"), "System.Text.Json"); + Assert.Equal("8.0.4", systemTextJsonVersion?.ToString()); } private static string GetThisFilePath([CallerFilePath] string? path = null) => path ?? throw new ArgumentNullException(nameof(path)); diff --git a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/RuntimePackagesTests.cs b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/RuntimePackagesTests.cs new file mode 100644 index 0000000000..bb957738d5 --- /dev/null +++ b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/RuntimePackagesTests.cs @@ -0,0 +1,206 @@ +using Semver; + +using Xunit; + +namespace DotNetPackageCorrelation; + +public class RuntimePackagesTests +{ + [Theory] + [MemberData(nameof(CorrelatedPackageCanBeFoundData))] + public void CorrelatedPackageCanBeFound(RuntimePackages runtimePackages, string runtimePackageName, string runtimePackageVersion, string candidatePackageName, string? expectedPackageVersion) + { + var packageMapper = PackageMapper.Load(runtimePackages); + var actualPackageVersion = packageMapper.GetPackageVersionThatShippedWithOtherPackage(runtimePackageName, SemVersion.Parse(runtimePackageVersion), candidatePackageName); + if (expectedPackageVersion is null) + { + Assert.Null(actualPackageVersion); + } + else + { + Assert.NotNull(actualPackageVersion); + Assert.Equal(expectedPackageVersion, actualPackageVersion.ToString()); + } + } + + public static IEnumerable CorrelatedPackageCanBeFoundData() + { + // package not found in specified runtime, but it is in earlier runtime; more recent runtime has that package, but that's not returned + yield return + [ + // runtimePackages + new RuntimePackages() + { + Runtimes = new SortedDictionary(SemVerComparer.Instance) + { + { + SemVersion.Parse("1.0.100"), + new PackageSet() + { + Packages = new SortedDictionary(StringComparer.OrdinalIgnoreCase) + { + { "Runtime.Package", SemVersion.Parse("1.0.0") }, + { "Some.Package", SemVersion.Parse("1.0.1") } + } + } + }, + { + SemVersion.Parse("1.0.101"), + new PackageSet() + { + Packages = new SortedDictionary(StringComparer.OrdinalIgnoreCase) + { + // this runtime didn't ship with a new version of "Some.Package", but the earlier release did + { "Runtime.Package", SemVersion.Parse("1.0.1") } + } + } + }, + { + SemVersion.Parse("1.0.200"), + new PackageSet() + { + Packages = new SortedDictionary(StringComparer.OrdinalIgnoreCase) + { + // the requested package shipped with this runtime, but this runtime isn't the correct version so it's not returned + { "Runtime.Package", SemVersion.Parse("1.0.2") }, + { "Some.Package", SemVersion.Parse("1.0.2") } + } + } + }, + } + }, + // runtimePackageName + "Runtime.Package", + // runtimePackageVersion + "1.0.1", + // candidatePackageName + "Some.Package", + // expectedPackageVersion + "1.0.1" + ]; + + // package differing in case is found + yield return + [ + // runtimePackages + new RuntimePackages() + { + Runtimes = new SortedDictionary(SemVerComparer.Instance) + { + { + SemVersion.Parse("1.0.100"), + new PackageSet() + { + Packages = new SortedDictionary(StringComparer.OrdinalIgnoreCase) + { + { "runtime.package", SemVersion.Parse("1.0.0") }, + { "some.package", SemVersion.Parse("1.0.1") } + } + } + } + } + }, + // runtimePackageName + "Runtime.Package", + // runtimePackageVersion + "1.0.0", + // candidatePackageName + "Some.Package", + // expectedPackageVersion + "1.0.1" + ]; + + // runtime package not found by name + yield return + [ + // runtimePackages + new RuntimePackages() + { + Runtimes = new SortedDictionary(SemVerComparer.Instance) + { + { + SemVersion.Parse("1.0.100"), + new PackageSet() + { + Packages = new SortedDictionary(StringComparer.OrdinalIgnoreCase) + { + { "runtime.package", SemVersion.Parse("1.0.0") }, + { "some.package", SemVersion.Parse("1.0.1") } + } + } + } + } + }, + // runtimePackageName + "Different.Runtime.Package", + // runtimePackageVersion + "1.0.0", + // candidatePackageName + "Some.Package", + // expectedPackageVersion + null + ]; + + // runtime package not found by version + yield return + [ + // runtimePackages + new RuntimePackages() + { + Runtimes = new SortedDictionary(SemVerComparer.Instance) + { + { + SemVersion.Parse("1.0.100"), + new PackageSet() + { + Packages = new SortedDictionary(StringComparer.OrdinalIgnoreCase) + { + { "runtime.package", SemVersion.Parse("1.0.0") }, + { "some.package", SemVersion.Parse("1.0.1") } + } + } + } + } + }, + // runtimePackageName + "Runtime.Package", + // runtimePackageVersion + "9.9.9", + // candidatePackageName + "Some.Package", + // expectedPackageVersion + null + ]; + + // candidate package not found + yield return + [ + // runtimePackages + new RuntimePackages() + { + Runtimes = new SortedDictionary(SemVerComparer.Instance) + { + { + SemVersion.Parse("1.0.100"), + new PackageSet() + { + Packages = new SortedDictionary(StringComparer.OrdinalIgnoreCase) + { + { "runtime.package", SemVersion.Parse("1.0.0") }, + { "some.package", SemVersion.Parse("1.0.1") } + } + } + } + } + }, + // runtimePackageName + "Runtime.Package", + // runtimePackageVersion + "1.0.0", + // candidatePackageName + "Package.Not.In.This.Runtime", + // expectedPackageVersion + null + ]; + } +} diff --git a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/SdkPackagesTests.cs b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/SdkPackagesTests.cs deleted file mode 100644 index d6428e6683..0000000000 --- a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation.Test/SdkPackagesTests.cs +++ /dev/null @@ -1,205 +0,0 @@ -using Semver; - -using Xunit; - -namespace DotNetPackageCorrelation; - -public class SdkPackagesTests -{ - [Theory] - [MemberData(nameof(CorrelatedPackageCanBeFoundData))] - public void CorrelatedPackageCanBeFound(SdkPackages packages, string sdkVersionString, string packageName, string? expectedPackageVersionString) - { - var sdkVersion = SemVersion.Parse(sdkVersionString); - var actualReplacementPackageVersion = packages.GetReplacementPackageVersion(sdkVersion, packageName); - var expectedPackageVersion = expectedPackageVersionString is not null - ? SemVersion.Parse(expectedPackageVersionString) - : null; - Assert.Equal(expectedPackageVersion, actualReplacementPackageVersion); - } - - public static IEnumerable CorrelatedPackageCanBeFoundData() - { - // package not found in current sdk, but is in parent; more recent sdk has package, but that's not returned - yield return - [ - // packages - new SdkPackages() - { - Sdks = new SortedDictionary(SemVerComparer.Instance) - { - { - SemVersion.Parse("1.0.100"), - new PackageSet() - { - Packages = new SortedDictionary(StringComparer.OrdinalIgnoreCase) - { - { "Some.Package", SemVersion.Parse("1.0.1") } - } - } - }, - { - SemVersion.Parse("1.0.101"), - new PackageSet() - { - Packages = new SortedDictionary(StringComparer.OrdinalIgnoreCase) - { - // empty - } - } - }, - { - SemVersion.Parse("1.0.102"), - new PackageSet() - { - Packages = new SortedDictionary(StringComparer.OrdinalIgnoreCase) - { - { "Some.Package", SemVersion.Parse("1.0.2") } - } - } - }, - } - }, - // sdkVersionString - "1.0.101", - // packageName - "Some.Package", - // expectedPackageVersionString - "1.0.1" - ]; - - // package differing in case is found - yield return - [ - // packages - new SdkPackages() - { - Sdks = new SortedDictionary(SemVerComparer.Instance) - { - { - SemVersion.Parse("1.0.100"), - new PackageSet() - { - Packages = new SortedDictionary(StringComparer.OrdinalIgnoreCase) - { - { "some.package", SemVersion.Parse("1.0.1") } - } - } - }, - { - SemVersion.Parse("1.0.101"), - new PackageSet() - { - Packages = new SortedDictionary(StringComparer.OrdinalIgnoreCase) - { - // empty - } - } - }, - } - }, - // sdkVersionString - "1.0.101", - // packageName - "Some.Package", - // expectedPackageVersionString - "1.0.1" - ]; - - // package not found results in null version - yield return - [ - // packages - new SdkPackages() - { - Sdks = new SortedDictionary(SemVerComparer.Instance) - { - { - SemVersion.Parse("1.0.100"), - new PackageSet() - { - Packages = new SortedDictionary(StringComparer.OrdinalIgnoreCase) - { - { "Some.Package", SemVersion.Parse("1.0.1") } - } - } - }, - { - SemVersion.Parse("1.0.101"), - new PackageSet() - { - Packages = new SortedDictionary(StringComparer.OrdinalIgnoreCase) - { - // empty - } - } - }, - } - }, - // sdkVersionString - "1.0.101", - // packageName - "UnrelatedPackage", - // expectedPackageVersionString - null - ]; - - // only SDKs with matching major version are considered - yield return - [ - // packages - new SdkPackages() - { - Sdks = new SortedDictionary(SemVerComparer.Instance) - { - { - SemVersion.Parse("1.0.100"), - new PackageSet() - { - Packages = new SortedDictionary(StringComparer.OrdinalIgnoreCase) - { - { "Some.Package", SemVersion.Parse("1.0.0") } - } - } - }, - { - SemVersion.Parse("2.0.100"), - new PackageSet() - { - Packages = new SortedDictionary(StringComparer.OrdinalIgnoreCase) - { - { "Some.Package", SemVersion.Parse("2.0.1") } - } - } - }, - { - SemVersion.Parse("2.0.200"), - new PackageSet() - { - Packages = new SortedDictionary(StringComparer.OrdinalIgnoreCase) - { - // empty - } - } - }, - { - SemVersion.Parse("3.0.100"), - new PackageSet() - { - Packages = new SortedDictionary(StringComparer.OrdinalIgnoreCase) - { - { "Some.Package", SemVersion.Parse("3.0.1") } - } - } - }, - } - }, - // sdkVersionString - "2.0.200", - // packageName - "Some.Package", - // expectedPackageVersionString - "2.0.1" - ]; - } -} diff --git a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Correlator.cs b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Correlator.cs index bf6df66445..0a6e408c91 100644 --- a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Correlator.cs +++ b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Correlator.cs @@ -21,7 +21,7 @@ public Correlator(DirectoryInfo releaseNotesDirectory) _releaseNotesDirectory = releaseNotesDirectory; } - public async Task<(SdkPackages SdkPackages, IEnumerable Warnings)> RunAsync() + public async Task<(RuntimePackages RuntimePackages, IEnumerable Warnings)> RunAsync() { var runtimeVersions = new List(); foreach (var directory in Directory.EnumerateDirectories(_releaseNotesDirectory.FullName)) @@ -33,14 +33,14 @@ public Correlator(DirectoryInfo releaseNotesDirectory) } } - var sdkPackages = new SdkPackages(); + var runtimePackages = new RuntimePackages(); var warnings = new List(); - foreach (var version in runtimeVersions) + foreach (var majorVersion in runtimeVersions) { - var releasesJsonPath = Path.Combine(_releaseNotesDirectory.FullName, version.ToString(), "releases.json"); + var releasesJsonPath = Path.Combine(_releaseNotesDirectory.FullName, majorVersion.ToString(), "releases.json"); if (!File.Exists(releasesJsonPath)) { - warnings.Add($"Unable to find releases.json file for version {version}"); + warnings.Add($"Unable to find releases.json file for version {majorVersion}"); continue; } @@ -63,13 +63,15 @@ public Correlator(DirectoryInfo releaseNotesDirectory) continue; } - if (!sdkPackages.Sdks.TryGetValue(sdk.Version, out var packagesAndVersions)) + if (runtimePackages.Runtimes.ContainsKey(sdk.RuntimeVersion)) { - packagesAndVersions = new PackageSet(); - sdkPackages.Sdks[sdk.Version] = packagesAndVersions; + // already processed this runtime + continue; } - var runtimeDirectory = new DirectoryInfo(Path.Combine(_releaseNotesDirectory.FullName, version.ToString(), sdk.RuntimeVersion.ToString())); + var packagesAndVersions = new PackageSet(); + runtimePackages.Runtimes.Add(sdk.RuntimeVersion, packagesAndVersions); + var runtimeDirectory = new DirectoryInfo(Path.Combine(_releaseNotesDirectory.FullName, majorVersion.ToString(), sdk.RuntimeVersion.ToString())); var runtimeMarkdownPath = Path.Combine(runtimeDirectory.FullName, $"{sdk.RuntimeVersion}.md"); if (!File.Exists(runtimeMarkdownPath)) { @@ -87,7 +89,7 @@ public Correlator(DirectoryInfo releaseNotesDirectory) } } - return (sdkPackages, warnings); + return (runtimePackages, warnings); } public static ImmutableArray<(string Name, SemVersion Version)> GetPackagesFromMarkdown(string markdownPath, string markdownContent, List warnings) diff --git a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/PackageMapper.cs b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/PackageMapper.cs new file mode 100644 index 0000000000..ec2d8676fe --- /dev/null +++ b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/PackageMapper.cs @@ -0,0 +1,68 @@ +using Semver; + +namespace DotNetPackageCorrelation; + +public class PackageMapper +{ + private readonly RuntimePackages _runtimePackages; + + private PackageMapper(RuntimePackages runtimePackages) + { + _runtimePackages = runtimePackages; + } + + /// + /// Find the version of that shipped at the same time as + /// "/". + /// + public SemVersion? GetPackageVersionThatShippedWithOtherPackage(string packageName, SemVersion packageVersion, string candidatePackageName) + { + var runtimeVersion = GetRuntimeVersionFromPackage(packageName, packageVersion); + if (runtimeVersion is null) + { + // no runtime found that contains the package + return null; + } + + var candidateRuntimeVersions = _runtimePackages.Runtimes.Keys + .Where(v => v.Major == runtimeVersion.Major) + .Where(v => v.ComparePrecedenceTo(runtimeVersion) <= 0) + .OrderBy(v => v, SemVerComparer.Instance) + .Reverse() + .ToArray(); + foreach (var candidateRuntimeVersion in candidateRuntimeVersions) + { + if (!_runtimePackages.Runtimes.TryGetValue(candidateRuntimeVersion, out var packageSet)) + { + continue; + } + + if (packageSet.Packages.TryGetValue(candidatePackageName, out var foundPackageVersion)) + { + return foundPackageVersion; + } + } + + return null; + } + + private SemVersion? GetRuntimeVersionFromPackage(string packageName, SemVersion packageVersion) + { + // TODO: linear search is slow + foreach (var runtime in _runtimePackages.Runtimes) + { + if (runtime.Value.Packages.TryGetValue(packageName, out var foundPackageVersion) && + foundPackageVersion == packageVersion) + { + return runtime.Key; + } + } + + return null; + } + + public static PackageMapper Load(RuntimePackages runtimePackages) + { + return new PackageMapper(runtimePackages); + } +} diff --git a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/SdkPackages.cs b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/RuntimePackages.cs similarity index 54% rename from nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/SdkPackages.cs rename to nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/RuntimePackages.cs index 8ebfa6dfa7..eb025e9a67 100644 --- a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/SdkPackages.cs +++ b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/RuntimePackages.cs @@ -4,8 +4,8 @@ namespace DotNetPackageCorrelation; -public record SdkPackages +public record RuntimePackages { [JsonObjectCreationHandling(JsonObjectCreationHandling.Populate)] - public SortedDictionary Sdks { get; init; } = new(SemVerComparer.Instance); + public SortedDictionary Runtimes { get; init; } = new(SemVerComparer.Instance); } diff --git a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/SdkPackagesExtensions.cs b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/SdkPackagesExtensions.cs deleted file mode 100644 index a8d20db66a..0000000000 --- a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/SdkPackagesExtensions.cs +++ /dev/null @@ -1,28 +0,0 @@ -using System.Collections.Immutable; - -using Semver; - -namespace DotNetPackageCorrelation; - -public static class SdkPackagesExtensions -{ - public static SemVersion? GetReplacementPackageVersion(this SdkPackages packages, SemVersion sdkVersion, string packageName) - { - var sdkVersionsToCheck = packages.Sdks.Keys - .Where(v => v.Major == sdkVersion.Major) - .Where(v => v.ComparePrecedenceTo(sdkVersion) <= 0) - .OrderBy(v => v, SemVerComparer.Instance) - .Reverse() - .ToImmutableArray(); - foreach (var sdkVersionToCheck in sdkVersionsToCheck) - { - var sdkPackages = packages.Sdks[sdkVersionToCheck]; - if (sdkPackages.Packages.TryGetValue(packageName, out var packageVersion)) - { - return packageVersion; - } - } - - return null; - } -} diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Discover/DiscoveryWorkerTests.Project.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Discover/DiscoveryWorkerTests.Project.cs index 04b9e6f274..45747da42f 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Discover/DiscoveryWorkerTests.Project.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Discover/DiscoveryWorkerTests.Project.cs @@ -1274,9 +1274,16 @@ public async Task PackagesManagedAndRemovedByTheSdkAreReported() var packageCorrelationFile = Path.Combine(tempDirectory.DirectoryPath, "dotnet-package-correlation.json"); await File.WriteAllTextAsync(packageCorrelationFile, """ { - "Sdks": { - "8.0.100": { + "Runtimes": { + "1.0.0": { "Packages": { + "Dependabot.App.Core.Ref": "1.0.0", + "Test.Only.Package": "1.0.0" + } + }, + "1.0.1": { + "Packages": { + "Dependabot.App.Core.Ref": "1.0.1", "Test.Only.Package": "1.0.99" } } @@ -1286,8 +1293,8 @@ await File.WriteAllTextAsync(packageCorrelationFile, """ using var tempEnvironment = new TemporaryEnvironment([("DOTNET_PACKAGE_CORRELATION_FILE_PATH", packageCorrelationFile)]); // The SDK package handling is detected in a very specific circumstance; an assembly being removed from the - // `@(RuntimeCopyLocalItems)` item group in the `GenerateBuildDependencyFile` target. Since we don't want - // to involve the real SDK, we fake some required targets. + // `@(References)` item group in the `_HandlePackageFileConflicts` target. Since we don't want to involve + // the real SDK, we fake some required targets. await TestDiscoveryAsync( experimentsManager: new ExperimentsManager() { InstallDotnetSdks = true, UseDirectDiscovery = true }, packages: [], @@ -1296,28 +1303,42 @@ await TestDiscoveryAsync( [ ("project.csproj", """ - + - + - - 8.0.100 net8.0 - + + + + + + + + + + - - + + + + + - + + + + + - + @@ -1339,8 +1360,7 @@ await TestDiscoveryAsync( new("Test.Only.Package", "1.0.99", DependencyType.Unknown, TargetFrameworks: ["net8.0"], IsTransitive: true) ], Properties = [ - new("NETCoreSdkVersion", "8.0.100", "project.csproj"), - new("TargetFramework", "net8.0", "project.csproj"), + new("TargetFramework", "net8.0", "project.csproj") ], TargetFrameworks = ["net8.0"], ReferencedProjectPaths = [], diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Update/UpdateWorkerTests.PackageReference.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Update/UpdateWorkerTests.PackageReference.cs index 70a126979e..9ab8aaade7 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Update/UpdateWorkerTests.PackageReference.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Update/UpdateWorkerTests.PackageReference.cs @@ -3492,14 +3492,13 @@ await TestUpdateForProject("Some.Package", "1.0.0", "1.1.0", public async Task UpdateSdkManagedPackage_DirectDependency() { // To avoid a unit test that's tightly coupled to the installed SDK, the package correlation file is faked. - // Doing this requires a temporary file and environment variable override. Note that SDK version 8.0.100 - // or greater is required. + // Doing this requires a temporary file and environment variable override. using var tempDirectory = new TemporaryDirectory(); var packageCorrelationFile = Path.Combine(tempDirectory.DirectoryPath, "dotnet-package-correlation.json"); await File.WriteAllTextAsync(packageCorrelationFile, """ { - "Sdks": { - "8.0.100": { + "Runtimes": { + "1": { "Packages": { "System.Text.Json": "8.0.98" } @@ -3530,16 +3529,6 @@ await TestUpdateForProject("System.Text.Json", "8.0.0", "8.0.99", """, - additionalFiles: [ - ("global.json", """ - { - "sdk": { - "version": "8.0.100", - "rollForward": "latestMinor" - } - } - """) - ], expectedProjectContents: """ diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Analyze/AnalyzeWorker.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Analyze/AnalyzeWorker.cs index a560cbad94..21a934020b 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Analyze/AnalyzeWorker.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Analyze/AnalyzeWorker.cs @@ -79,6 +79,8 @@ internal async Task RunWithErrorHandlingAsync(string repoRoot, s public async Task RunAsync(string repoRoot, WorkspaceDiscoveryResult discovery, DependencyInfo dependencyInfo) { + MSBuildHelper.RegisterMSBuild(repoRoot, repoRoot); + var startingDirectory = PathHelper.JoinPath(repoRoot, discovery.Path); _logger.Info($"Starting analysis of {dependencyInfo.Name}..."); diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/DependencyDiscovery.targets b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/DependencyDiscovery.targets index e9ba35b51d..65ba7d4bbd 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/DependencyDiscovery.targets +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/DependencyDiscovery.targets @@ -1,9 +1,9 @@ - + diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Discover/SdkProjectDiscovery.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Discover/SdkProjectDiscovery.cs index 7131743238..6438296bc5 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Discover/SdkProjectDiscovery.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Discover/SdkProjectDiscovery.cs @@ -20,13 +20,14 @@ namespace NuGetUpdater.Core.Discover; internal static class SdkProjectDiscovery { - private static readonly SdkPackages _sdkPackages; - private static readonly Dictionary _sdkPackagesByOverrideFile = new(); + private static readonly PackageMapper _packageMapper; + private static readonly Dictionary _packageMapperByOverrideFile = new(); static SdkProjectDiscovery() { var packageCorrelationPath = Path.Combine(Path.GetDirectoryName(typeof(SdkProjectDiscovery).Assembly.Location)!, "dotnet-package-correlation.json"); - _sdkPackages = LoadPackageCorrelationsFromFile(packageCorrelationPath); + var runtimePackages = LoadRuntimePackagesFromFile(packageCorrelationPath); + _packageMapper = PackageMapper.Load(runtimePackages); } private static readonly HashSet TopLevelPackageItemNames = new HashSet(StringComparer.OrdinalIgnoreCase) @@ -86,6 +87,9 @@ public static async Task> DiscoverWithBin Dictionary> topLevelPackagesPerProject = new(PathComparer.Instance); // projectPath, packageNames + Dictionary>> packagesReplacedBySdkPerProject = new(PathComparer.Instance); + // projectPath tfm packageName, packageVersion + Dictionary> resolvedProperties = new(PathComparer.Instance); // projectPath propertyName, propertyValue @@ -217,6 +221,54 @@ public static async Task> DiscoverWithBin } } break; + case Target target when target.Name == "_HandlePackageFileConflicts": + // this only works if we've installed the exact SDK required + if (experimentsManager.InstallDotnetSdks) + { + var projectEvaluation = GetNearestProjectEvaluation(target); + if (projectEvaluation is not null) + { + var removedReferences = target.Children.OfType().FirstOrDefault(r => r.Name == "Reference"); + var addedReferences = target.Children.OfType().FirstOrDefault(r => r.Name == "Reference"); + if (removedReferences is not null && addedReferences is not null) + { + foreach (var removedAssembly in removedReferences.Children.OfType()) + { + var removedPackageName = GetChildMetadataValue(removedAssembly, "NuGetPackageId"); + var removedFileName = Path.GetFileName(removedAssembly.Name); + if (removedPackageName is not null && removedFileName is not null) + { + var existingProjectPackagesByTfm = packagesPerProject.GetOrAdd(projectEvaluation.ProjectFile, () => new(PathComparer.Instance)); + var existingProjectPackages = existingProjectPackagesByTfm.GetOrAdd(tfm, () => new(StringComparer.OrdinalIgnoreCase)); + if (existingProjectPackages.ContainsKey(removedPackageName)) + { + var correspondingAddedFile = addedReferences.Children.OfType() + .FirstOrDefault(i => removedFileName.Equals(Path.GetFileName(i.Name), StringComparison.OrdinalIgnoreCase)); + if (correspondingAddedFile is not null) + { + var runtimePackageName = GetChildMetadataValue(correspondingAddedFile, "NuGetPackageId"); + var runtimePackageVersion = GetChildMetadataValue(correspondingAddedFile, "NuGetPackageVersion"); + if (runtimePackageName is not null && + runtimePackageVersion is not null && + SemVersion.TryParse(runtimePackageVersion, out var parsedRuntimePackageVersion)) + { + var packageMapper = GetPackageMapper(); + var replacementPackageVersion = packageMapper.GetPackageVersionThatShippedWithOtherPackage(runtimePackageName, parsedRuntimePackageVersion, removedPackageName); + if (replacementPackageVersion is not null) + { + var packagesPerProject = packagesReplacedBySdkPerProject.GetOrAdd(projectEvaluation.ProjectFile, () => new(PathComparer.Instance)); + var packagesPerTfm = packagesPerProject.GetOrAdd(tfm, () => new(StringComparer.OrdinalIgnoreCase)); + packagesPerTfm[removedPackageName] = replacementPackageVersion.ToString(); + } + } + } + } + } + } + } + } + } + break; } }, takeChildrenSnapshot: true); } @@ -242,6 +294,33 @@ public static async Task> DiscoverWithBin { // gather some project-level information var packagesByTfm = packagesPerProject[projectPath]; + if (packagesReplacedBySdkPerProject.TryGetValue(projectPath, out var packagesReplacedBySdk)) + { + var consolidatedPackagesByTfm = new Dictionary>(StringComparer.OrdinalIgnoreCase); + + // copy the first dictionary + foreach (var kvp in packagesByTfm) + { + var tfm = kvp.Key; + var packages = kvp.Value; + consolidatedPackagesByTfm[tfm] = packages; + } + + // merge in the second + foreach (var kvp in packagesReplacedBySdk) + { + var tfm = kvp.Key; + var packages = kvp.Value; + var replacedPackages = consolidatedPackagesByTfm.GetOrAdd(tfm, () => new(StringComparer.OrdinalIgnoreCase)); + foreach (var packagePair in packages) + { + replacedPackages[packagePair.Key] = packagePair.Value; + } + } + + packagesByTfm = consolidatedPackagesByTfm; + } + var projectFullDirectory = Path.GetDirectoryName(projectPath)!; var doc = XDocument.Load(projectPath); var localPropertyDefinitionElements = doc.Root!.XPathSelectElements("/Project/PropertyGroup/*"); @@ -299,36 +378,30 @@ public static async Task> DiscoverWithBin return projectDiscoveryResults; } - private static string? GetCorrespondingSdkManagedPackageVersion(string packageName, string sdkVersionString) - { - var sdkVersion = SemVersion.Parse(sdkVersionString); - var replacementPackageVersion = GetSdkPackageCorrelations().GetReplacementPackageVersion(sdkVersion, packageName); - return replacementPackageVersion?.ToString(); - } - - private static SdkPackages GetSdkPackageCorrelations() + private static PackageMapper GetPackageMapper() { var packageCorrelationFileOverride = Environment.GetEnvironmentVariable("DOTNET_PACKAGE_CORRELATION_FILE_PATH"); if (packageCorrelationFileOverride is not null) { // this is used as a test hook to allow unit tests to be SDK agnostic - if (_sdkPackagesByOverrideFile.TryGetValue(packageCorrelationFileOverride, out var sdkPackages)) + if (_packageMapperByOverrideFile.TryGetValue(packageCorrelationFileOverride, out var packageMapper)) { - return sdkPackages; + return packageMapper; } - sdkPackages = LoadPackageCorrelationsFromFile(packageCorrelationFileOverride); - _sdkPackagesByOverrideFile[packageCorrelationFileOverride] = sdkPackages; - return sdkPackages; + var runtimePackages = LoadRuntimePackagesFromFile(packageCorrelationFileOverride); + packageMapper = PackageMapper.Load(runtimePackages); + _packageMapperByOverrideFile[packageCorrelationFileOverride] = packageMapper; + return packageMapper; } - return _sdkPackages; + return _packageMapper; } - private static SdkPackages LoadPackageCorrelationsFromFile(string filePath) + private static RuntimePackages LoadRuntimePackagesFromFile(string filePath) { var packageCorrelationJson = File.ReadAllText(filePath); - return JsonSerializer.Deserialize(packageCorrelationJson, Correlator.SerializerOptions)!; + return JsonSerializer.Deserialize(packageCorrelationJson, Correlator.SerializerOptions)!; } private static void ProcessResolvedPackageReference( @@ -395,29 +468,7 @@ ExperimentsManager experimentsManager if (doRemoveOperation) { - var wasRemoved = packagesPerTfm.Remove(packageName); - if (wasRemoved) - { - // we only want to track packages that were explicitly part of the SDK's conflict resolution - if (node.Name == "RuntimeCopyLocalItems" && - node.Parent is Target target && - target.Name == "GenerateBuildDependencyFile") - { - // dotnet package correlation correction requires specific dotnet sdk handling - if (experimentsManager.InstallDotnetSdks) - { - var sdkVersionString = GetPropertyValueFromProjectEvaluation(projectEvaluation, "NETCoreSdkVersion"); - if (sdkVersionString is not null) - { - var replacementVersion = GetCorrespondingSdkManagedPackageVersion(packageName, sdkVersionString); - if (replacementVersion is not null) - { - packagesPerTfm[packageName] = replacementVersion.ToString(); - } - } - } - } - } + packagesPerTfm.Remove(packageName); } if (doAddOperation) diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Updater/PackageReferenceUpdater.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Updater/PackageReferenceUpdater.cs index da0a0f8e99..4ad7045fef 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Updater/PackageReferenceUpdater.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Updater/PackageReferenceUpdater.cs @@ -95,6 +95,11 @@ public static async Task UpdateDependencyWithConflictResolution( var dependenciesToUpdate = new[] { new Dependency(dependencyName, newDependencyVersion, DependencyType.PackageReference) }; // update the initial dependency... + if (isDependencyTopLevel) + { + // TODO: is SDK replacement package + previousDependencyVersion = topLevelDependencies.First(d => d.Name.Equals(dependencyName, StringComparison.OrdinalIgnoreCase)).Version!; + } TryUpdateDependencyVersion(buildFiles, dependencyName, previousDependencyVersion, newDependencyVersion, logger); // ...and the peer dependencies... From 7a818eccfd17625e49b26a32d2d8e22af6136672 Mon Sep 17 00:00:00 2001 From: "Brett V. Forsgren" Date: Thu, 19 Dec 2024 16:48:47 -0700 Subject: [PATCH 15/20] move package correlator to common location --- .../Model/PackageMapper.cs | 10 ++-- .../Discover/SdkProjectDiscovery.cs | 41 +---------------- .../Updater/PackageReferenceUpdater.cs | 26 +++++++++-- .../DotNetPackageCorrelationManager.cs | 46 +++++++++++++++++++ 4 files changed, 73 insertions(+), 50 deletions(-) create mode 100644 nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Utilities/DotNetPackageCorrelationManager.cs diff --git a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/PackageMapper.cs b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/PackageMapper.cs index ec2d8676fe..931e560246 100644 --- a/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/PackageMapper.cs +++ b/nuget/helpers/lib/NuGetUpdater/DotNetPackageCorrelation/Model/PackageMapper.cs @@ -4,11 +4,11 @@ namespace DotNetPackageCorrelation; public class PackageMapper { - private readonly RuntimePackages _runtimePackages; + public RuntimePackages RuntimePackages { get; } private PackageMapper(RuntimePackages runtimePackages) { - _runtimePackages = runtimePackages; + RuntimePackages = runtimePackages; } /// @@ -24,7 +24,7 @@ private PackageMapper(RuntimePackages runtimePackages) return null; } - var candidateRuntimeVersions = _runtimePackages.Runtimes.Keys + var candidateRuntimeVersions = RuntimePackages.Runtimes.Keys .Where(v => v.Major == runtimeVersion.Major) .Where(v => v.ComparePrecedenceTo(runtimeVersion) <= 0) .OrderBy(v => v, SemVerComparer.Instance) @@ -32,7 +32,7 @@ private PackageMapper(RuntimePackages runtimePackages) .ToArray(); foreach (var candidateRuntimeVersion in candidateRuntimeVersions) { - if (!_runtimePackages.Runtimes.TryGetValue(candidateRuntimeVersion, out var packageSet)) + if (!RuntimePackages.Runtimes.TryGetValue(candidateRuntimeVersion, out var packageSet)) { continue; } @@ -49,7 +49,7 @@ private PackageMapper(RuntimePackages runtimePackages) private SemVersion? GetRuntimeVersionFromPackage(string packageName, SemVersion packageVersion) { // TODO: linear search is slow - foreach (var runtime in _runtimePackages.Runtimes) + foreach (var runtime in RuntimePackages.Runtimes) { if (runtime.Value.Packages.TryGetValue(packageName, out var foundPackageVersion) && foundPackageVersion == packageVersion) diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Discover/SdkProjectDiscovery.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Discover/SdkProjectDiscovery.cs index 6438296bc5..1d143d006c 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Discover/SdkProjectDiscovery.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Discover/SdkProjectDiscovery.cs @@ -1,11 +1,8 @@ using System.Collections.Immutable; using System.Reflection; -using System.Text.Json; using System.Xml.Linq; using System.Xml.XPath; -using DotNetPackageCorrelation; - using Microsoft.Build.Logging.StructuredLogger; using NuGet.Versioning; @@ -20,16 +17,6 @@ namespace NuGetUpdater.Core.Discover; internal static class SdkProjectDiscovery { - private static readonly PackageMapper _packageMapper; - private static readonly Dictionary _packageMapperByOverrideFile = new(); - - static SdkProjectDiscovery() - { - var packageCorrelationPath = Path.Combine(Path.GetDirectoryName(typeof(SdkProjectDiscovery).Assembly.Location)!, "dotnet-package-correlation.json"); - var runtimePackages = LoadRuntimePackagesFromFile(packageCorrelationPath); - _packageMapper = PackageMapper.Load(runtimePackages); - } - private static readonly HashSet TopLevelPackageItemNames = new HashSet(StringComparer.OrdinalIgnoreCase) { "PackageReference" @@ -252,7 +239,7 @@ public static async Task> DiscoverWithBin runtimePackageVersion is not null && SemVersion.TryParse(runtimePackageVersion, out var parsedRuntimePackageVersion)) { - var packageMapper = GetPackageMapper(); + var packageMapper = DotNetPackageCorrelationManager.GetPackageMapper(); var replacementPackageVersion = packageMapper.GetPackageVersionThatShippedWithOtherPackage(runtimePackageName, parsedRuntimePackageVersion, removedPackageName); if (replacementPackageVersion is not null) { @@ -378,32 +365,6 @@ runtimePackageVersion is not null && return projectDiscoveryResults; } - private static PackageMapper GetPackageMapper() - { - var packageCorrelationFileOverride = Environment.GetEnvironmentVariable("DOTNET_PACKAGE_CORRELATION_FILE_PATH"); - if (packageCorrelationFileOverride is not null) - { - // this is used as a test hook to allow unit tests to be SDK agnostic - if (_packageMapperByOverrideFile.TryGetValue(packageCorrelationFileOverride, out var packageMapper)) - { - return packageMapper; - } - - var runtimePackages = LoadRuntimePackagesFromFile(packageCorrelationFileOverride); - packageMapper = PackageMapper.Load(runtimePackages); - _packageMapperByOverrideFile[packageCorrelationFileOverride] = packageMapper; - return packageMapper; - } - - return _packageMapper; - } - - private static RuntimePackages LoadRuntimePackagesFromFile(string filePath) - { - var packageCorrelationJson = File.ReadAllText(filePath); - return JsonSerializer.Deserialize(packageCorrelationJson, Correlator.SerializerOptions)!; - } - private static void ProcessResolvedPackageReference( NamedNode node, Dictionary>> packagesPerProject, // projectPath -> tfm -> (packageName, packageVersion) diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Updater/PackageReferenceUpdater.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Updater/PackageReferenceUpdater.cs index 4ad7045fef..cbf7ba7d2a 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Updater/PackageReferenceUpdater.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Updater/PackageReferenceUpdater.cs @@ -4,6 +4,8 @@ using NuGet.Versioning; +using NuGetUpdater.Core.Utilities; + namespace NuGetUpdater.Core; /// @@ -36,6 +38,25 @@ public static async Task UpdateDependencyAsync( // Get the set of all top-level dependencies in the current project var topLevelDependencies = MSBuildHelper.GetTopLevelPackageDependencyInfos(buildFiles).ToArray(); + var isDependencyTopLevel = topLevelDependencies.Any(d => d.Name.Equals(dependencyName, StringComparison.OrdinalIgnoreCase)); + if (isDependencyTopLevel) + { + var packageMapper = DotNetPackageCorrelationManager.GetPackageMapper(); + // TODO: this is slow + var isSdkReplacementPackage = packageMapper.RuntimePackages.Runtimes.Any(r => + { + return r.Value.Packages.Any(p => dependencyName.Equals(p.Key, StringComparison.Ordinal)); + }); + if (isSdkReplacementPackage) + { + // If we're updating a top level SDK replacement package, the version listed in the project file won't + // necessarily match the resolved version that caused the update because the SDK might have replaced + // the package. To handle this scenario, we pretend the version we're searching for is the actual + // version in the file, not the resolved version. This allows us to keep a strict equality check when + // finding the file to update. + previousDependencyVersion = topLevelDependencies.First(d => d.Name.Equals(dependencyName, StringComparison.OrdinalIgnoreCase)).Version!; + } + } if (!await DoesDependencyRequireUpdateAsync(repoRootPath, projectPath, tfms, topLevelDependencies, dependencyName, newDependencyVersion, experimentsManager, logger)) { @@ -95,11 +116,6 @@ public static async Task UpdateDependencyWithConflictResolution( var dependenciesToUpdate = new[] { new Dependency(dependencyName, newDependencyVersion, DependencyType.PackageReference) }; // update the initial dependency... - if (isDependencyTopLevel) - { - // TODO: is SDK replacement package - previousDependencyVersion = topLevelDependencies.First(d => d.Name.Equals(dependencyName, StringComparison.OrdinalIgnoreCase)).Version!; - } TryUpdateDependencyVersion(buildFiles, dependencyName, previousDependencyVersion, newDependencyVersion, logger); // ...and the peer dependencies... diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Utilities/DotNetPackageCorrelationManager.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Utilities/DotNetPackageCorrelationManager.cs new file mode 100644 index 0000000000..6238767410 --- /dev/null +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Utilities/DotNetPackageCorrelationManager.cs @@ -0,0 +1,46 @@ +using System.Text.Json; + +using DotNetPackageCorrelation; + +using NuGetUpdater.Core.Discover; + +namespace NuGetUpdater.Core.Utilities; + +internal static class DotNetPackageCorrelationManager +{ + private static readonly PackageMapper _packageMapper; + private static readonly Dictionary _packageMapperByOverrideFile = new(); + + static DotNetPackageCorrelationManager() + { + var packageCorrelationPath = Path.Combine(Path.GetDirectoryName(typeof(SdkProjectDiscovery).Assembly.Location)!, "dotnet-package-correlation.json"); + var runtimePackages = LoadRuntimePackagesFromFile(packageCorrelationPath); + _packageMapper = PackageMapper.Load(runtimePackages); + } + + public static PackageMapper GetPackageMapper() + { + var packageCorrelationFileOverride = Environment.GetEnvironmentVariable("DOTNET_PACKAGE_CORRELATION_FILE_PATH"); + if (packageCorrelationFileOverride is not null) + { + // this is used as a test hook to allow unit tests to be SDK agnostic + if (_packageMapperByOverrideFile.TryGetValue(packageCorrelationFileOverride, out var packageMapper)) + { + return packageMapper; + } + + var runtimePackages = LoadRuntimePackagesFromFile(packageCorrelationFileOverride); + packageMapper = PackageMapper.Load(runtimePackages); + _packageMapperByOverrideFile[packageCorrelationFileOverride] = packageMapper; + return packageMapper; + } + + return _packageMapper; + } + + private static RuntimePackages LoadRuntimePackagesFromFile(string filePath) + { + var packageCorrelationJson = File.ReadAllText(filePath); + return JsonSerializer.Deserialize(packageCorrelationJson, Correlator.SerializerOptions)!; + } +} From 631b3ef1c07d6258a219da98c9c47cc189e3123e Mon Sep 17 00:00:00 2001 From: "Brett V. Forsgren" Date: Fri, 20 Dec 2024 12:01:51 -0700 Subject: [PATCH 16/20] make tests SDK agnostic --- .../MockNuGetPackage.cs | 43 +++----- .../UpdateWorkerTests.PackageReference.cs | 97 ++++++++++++------- .../Discover/SdkProjectDiscovery.cs | 4 +- 3 files changed, 77 insertions(+), 67 deletions(-) diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/MockNuGetPackage.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/MockNuGetPackage.cs index eae2e1237f..2ceeaf2d03 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/MockNuGetPackage.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/MockNuGetPackage.cs @@ -388,6 +388,17 @@ public static MockNuGetPackage WellKnownReferencePackage(string packageName, str return WellKnownPackages[key]; } + public static MockNuGetPackage GetMicrosoftNETCoreAppRefPackage(int majorRuntimeVersion) + { + return WellKnownReferencePackage("Microsoft.NETCore.App", $"net{majorRuntimeVersion}.0", + [ + ("data/FrameworkList.xml", Encoding.UTF8.GetBytes($""" + + + """)) + ]); + } + public static MockNuGetPackage WellKnownHostPackage(string packageName, string targetFramework, (string Path, byte[] Content)[]? files = null) { string key = $"{packageName}/{targetFramework}"; @@ -437,34 +448,10 @@ public static MockNuGetPackage WellKnownHostPackage(string packageName, string t WellKnownReferencePackage("Microsoft.AspNetCore.App", "net7.0"), WellKnownReferencePackage("Microsoft.AspNetCore.App", "net8.0"), WellKnownReferencePackage("Microsoft.AspNetCore.App", "net9.0"), - WellKnownReferencePackage("Microsoft.NETCore.App", "net6.0", - [ - ("data/FrameworkList.xml", Encoding.UTF8.GetBytes(""" - - - """)) - ]), - WellKnownReferencePackage("Microsoft.NETCore.App", "net7.0", - [ - ("data/FrameworkList.xml", Encoding.UTF8.GetBytes(""" - - - """)) - ]), - WellKnownReferencePackage("Microsoft.NETCore.App", "net8.0", - [ - ("data/FrameworkList.xml", Encoding.UTF8.GetBytes(""" - - - """)) - ]), - WellKnownReferencePackage("Microsoft.NETCore.App", "net9.0", - [ - ("data/FrameworkList.xml", Encoding.UTF8.GetBytes(""" - - - """)) - ]), + GetMicrosoftNETCoreAppRefPackage(6), + GetMicrosoftNETCoreAppRefPackage(7), + GetMicrosoftNETCoreAppRefPackage(8), + GetMicrosoftNETCoreAppRefPackage(9), WellKnownReferencePackage("Microsoft.WindowsDesktop.App", "net6.0"), WellKnownReferencePackage("Microsoft.WindowsDesktop.App", "net7.0"), WellKnownReferencePackage("Microsoft.WindowsDesktop.App", "net8.0"), diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Update/UpdateWorkerTests.PackageReference.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Update/UpdateWorkerTests.PackageReference.cs index 9ab8aaade7..f4b17749d3 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Update/UpdateWorkerTests.PackageReference.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Update/UpdateWorkerTests.PackageReference.cs @@ -3491,16 +3491,20 @@ await TestUpdateForProject("Some.Package", "1.0.0", "1.1.0", [Fact] public async Task UpdateSdkManagedPackage_DirectDependency() { - // To avoid a unit test that's tightly coupled to the installed SDK, the package correlation file is faked. - // Doing this requires a temporary file and environment variable override. + // To avoid a unit test that's tightly coupled to the installed SDK, several values are simulated, + // including the runtime major version, the current Microsoft.NETCore.App.Ref package, and the package + // correlation file. Doing this requires a temporary file and environment variable override. + var runtimeMajorVersion = Environment.Version.Major; + var netCoreAppRefPackage = MockNuGetPackage.GetMicrosoftNETCoreAppRefPackage(runtimeMajorVersion); using var tempDirectory = new TemporaryDirectory(); var packageCorrelationFile = Path.Combine(tempDirectory.DirectoryPath, "dotnet-package-correlation.json"); - await File.WriteAllTextAsync(packageCorrelationFile, """ + await File.WriteAllTextAsync(packageCorrelationFile, $$""" { "Runtimes": { - "1": { + "{{runtimeMajorVersion}}.0.0": { "Packages": { - "System.Text.Json": "8.0.98" + "{{netCoreAppRefPackage.Id}}": "{{netCoreAppRefPackage.Version}}", + "System.Text.Json": "{{runtimeMajorVersion}}.0.98" } } } @@ -3510,52 +3514,68 @@ await File.WriteAllTextAsync(packageCorrelationFile, """ // In the `packages` section below, we fake a `System.Text.Json` package with a low assembly version that // will always trigger the replacement so that can be detected and then the equivalent version is pulled - // from the correlation file specified above. In the original project contents, package version `8.0.98` - // is reported which makes the update to `8.0.99` always possible. - await TestUpdateForProject("System.Text.Json", "8.0.0", "8.0.99", + // from the correlation file specified above. In the original project contents, package version `x.0.98` + // is reported which makes the update to `x.0.99` always possible. + await TestUpdateForProject("System.Text.Json", $"{runtimeMajorVersion}.0.98", $"{runtimeMajorVersion}.0.99", experimentsManager: new ExperimentsManager() { UseDirectDiscovery = true, InstallDotnetSdks = true }, packages: [ - MockNuGetPackage.CreatePackageWithAssembly("System.Text.Json", "8.0.0", "net8.0", assemblyVersion: "8.0.0.0"), // this assembly version is lower than what the SDK will have - MockNuGetPackage.CreatePackageWithAssembly("System.Text.Json", "8.0.99", "net8.0", assemblyVersion: "8.99.99.99"), // this assembly version is greater than what the SDK will have + // this assembly version is lower than what the SDK will have + MockNuGetPackage.CreatePackageWithAssembly("System.Text.Json", $"{runtimeMajorVersion}.0.0", $"net{runtimeMajorVersion}.0", assemblyVersion: $"{runtimeMajorVersion}.0.0.0"), + // this assembly version is greater than what the SDK will have + MockNuGetPackage.CreatePackageWithAssembly("System.Text.Json", $"{runtimeMajorVersion}.0.99", $"net{runtimeMajorVersion}.0", assemblyVersion: $"{runtimeMajorVersion}.99.99.99"), ], - projectContents: """ + projectContents: $""" - net8.0 + net{runtimeMajorVersion}.0 - + """, - expectedProjectContents: """ + additionalFiles: [ + ("global.json", $$""" + { + "sdk": { + "version": "{{runtimeMajorVersion}}.0.100", + "allowPrerelease": true, + "rollForward": "latestMinor" + } + } + """) + ], + expectedProjectContents: $""" - net8.0 + net{runtimeMajorVersion}.0 - + """ ); } - [Fact(Skip = "https://github.com/dependabot/dependabot-core/issues/11140")] + [Fact] public async Task UpdateSdkManagedPackage_TransitiveDependency() { - // To avoid a unit test that's tightly coupled to the installed SDK, the package correlation file is faked. - // Doing this requires a temporary file and environment variable override. Note that SDK version 8.0.100 - // or greater is required. + // To avoid a unit test that's tightly coupled to the installed SDK, several values are simulated, + // including the runtime major version, the current Microsoft.NETCore.App.Ref package, and the package + // correlation file. Doing this requires a temporary file and environment variable override. + var runtimeMajorVersion = Environment.Version.Major; + var netCoreAppRefPackage = MockNuGetPackage.GetMicrosoftNETCoreAppRefPackage(runtimeMajorVersion); using var tempDirectory = new TemporaryDirectory(); var packageCorrelationFile = Path.Combine(tempDirectory.DirectoryPath, "dotnet-package-correlation.json"); - await File.WriteAllTextAsync(packageCorrelationFile, """ + await File.WriteAllTextAsync(packageCorrelationFile, $$""" { - "Sdks": { - "8.0.100": { + "Runtimes": { + "{{runtimeMajorVersion}}.0.0": { "Packages": { - "System.Text.Json": "8.0.98" + "{{netCoreAppRefPackage.Id}}": "{{netCoreAppRefPackage.Version}}", + "System.Text.Json": "{{runtimeMajorVersion}}.0.98" } } } @@ -3565,22 +3585,24 @@ await File.WriteAllTextAsync(packageCorrelationFile, """ // In the `packages` section below, we fake a `System.Text.Json` package with a low assembly version that // will always trigger the replacement so that can be detected and then the equivalent version is pulled - // from the correlation file specified above. In the original project contents, package version `8.0.98` - // is reported which makes the update to `8.0.99` always possible. - await TestUpdateForProject("System.Text.Json", "8.0.98", "8.0.99", + // from the correlation file specified above. In the original project contents, package version `x.0.98` + // is reported which makes the update to `x.0.99` always possible. + await TestUpdateForProject("System.Text.Json", $"{runtimeMajorVersion}.0.98", $"{runtimeMajorVersion}.0.99", isTransitive: true, experimentsManager: new ExperimentsManager() { UseDirectDiscovery = true, InstallDotnetSdks = true }, packages: [ - MockNuGetPackage.CreateSimplePackage("Some.Package", "1.0.0", "net8.0", [(null, [("System.Text.Json", "[8.0.0]")])]), - MockNuGetPackage.CreateSimplePackage("Some.Package", "2.0.0", "net8.0", [(null, [("System.Text.Json", "[8.0.99]")])]), - MockNuGetPackage.CreatePackageWithAssembly("System.Text.Json", "8.0.0", "net8.0", assemblyVersion: "8.0.0.0"), // this assembly version is lower than what the SDK will have - MockNuGetPackage.CreatePackageWithAssembly("System.Text.Json", "8.0.99", "net8.0", assemblyVersion: "8.99.99.99"), // this assembly version is greater than what the SDK will have + MockNuGetPackage.CreateSimplePackage("Some.Package", "1.0.0", $"net{runtimeMajorVersion}.0", [(null, [("System.Text.Json", $"[{runtimeMajorVersion}.0.0]")])]), + MockNuGetPackage.CreateSimplePackage("Some.Package", "2.0.0", $"net{runtimeMajorVersion}.0", [(null, [("System.Text.Json", $"[{runtimeMajorVersion}.0.99]")])]), + // this assembly version is lower than what the SDK will have + MockNuGetPackage.CreatePackageWithAssembly("System.Text.Json", $"{runtimeMajorVersion}.0.0", $"net{runtimeMajorVersion}.0", assemblyVersion: $"{runtimeMajorVersion}.0.0.0"), + // this assembly version is greater than what the SDK will have + MockNuGetPackage.CreatePackageWithAssembly("System.Text.Json", $"{runtimeMajorVersion}.0.99", $"net{runtimeMajorVersion}.0", assemblyVersion: $"{runtimeMajorVersion}.99.99.99"), ], - projectContents: """ + projectContents: $""" - net8.0 + net{runtimeMajorVersion}.0 @@ -3588,19 +3610,20 @@ await TestUpdateForProject("System.Text.Json", "8.0.98", "8.0.99", """, additionalFiles: [ - ("global.json", """ + ("global.json", $$""" { "sdk": { - "version": "8.0.100", + "version": "{{runtimeMajorVersion}}.0.100", + "allowPrerelease": true, "rollForward": "latestMinor" } } """) ], - expectedProjectContents: """ + expectedProjectContents: $""" - net8.0 + net{runtimeMajorVersion}.0 diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Discover/SdkProjectDiscovery.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Discover/SdkProjectDiscovery.cs index 1d143d006c..dc60a5f42a 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Discover/SdkProjectDiscovery.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Discover/SdkProjectDiscovery.cs @@ -243,8 +243,8 @@ runtimePackageVersion is not null && var replacementPackageVersion = packageMapper.GetPackageVersionThatShippedWithOtherPackage(runtimePackageName, parsedRuntimePackageVersion, removedPackageName); if (replacementPackageVersion is not null) { - var packagesPerProject = packagesReplacedBySdkPerProject.GetOrAdd(projectEvaluation.ProjectFile, () => new(PathComparer.Instance)); - var packagesPerTfm = packagesPerProject.GetOrAdd(tfm, () => new(StringComparer.OrdinalIgnoreCase)); + var packagesPerThisProject = packagesReplacedBySdkPerProject.GetOrAdd(projectEvaluation.ProjectFile, () => new(PathComparer.Instance)); + var packagesPerTfm = packagesPerThisProject.GetOrAdd(tfm, () => new(StringComparer.OrdinalIgnoreCase)); packagesPerTfm[removedPackageName] = replacementPackageVersion.ToString(); } } From 4fa86cb9a6b2b332e32f91258d2e97ed1eac3e72 Mon Sep 17 00:00:00 2001 From: "Brett V. Forsgren" Date: Fri, 20 Dec 2024 15:34:55 -0700 Subject: [PATCH 17/20] ensure consistent tfm value --- .../NuGetUpdater.Core/Analyze/AnalyzeWorker.cs | 1 + .../NuGetUpdater.Core/Analyze/DependencyFinder.cs | 8 ++++---- .../NuGetUpdater/NuGetUpdater.Core/Analyze/Extensions.cs | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Analyze/AnalyzeWorker.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Analyze/AnalyzeWorker.cs index 21a934020b..eb0b8fcd86 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Analyze/AnalyzeWorker.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Analyze/AnalyzeWorker.cs @@ -425,6 +425,7 @@ internal static async Task> FindUpdatedDependenciesAs .SelectMany(p => p.TargetFrameworks) .Select(NuGetFramework.Parse) .Distinct() + .Select(f => f.GetShortFolderName()) .ToImmutableArray(); // When updating peer dependencies, we only need to consider top-level dependencies. diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Analyze/DependencyFinder.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Analyze/DependencyFinder.cs index b47988f724..b761dd7c96 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Analyze/DependencyFinder.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Analyze/DependencyFinder.cs @@ -7,10 +7,10 @@ namespace NuGetUpdater.Core.Analyze; internal static class DependencyFinder { - public static async Task>> GetDependenciesAsync( + public static async Task>> GetDependenciesAsync( string repoRoot, string projectPath, - IEnumerable frameworks, + IEnumerable frameworks, ImmutableHashSet packageIds, NuGetVersion version, NuGetContext nugetContext, @@ -23,13 +23,13 @@ public static async Task new Dependency(id, versionString, DependencyType.Unknown)) .ToImmutableArray(); - var result = ImmutableDictionary.CreateBuilder>(); + var result = ImmutableDictionary.CreateBuilder>(); foreach (var framework in frameworks) { var dependencies = await MSBuildHelper.GetAllPackageDependenciesAsync( repoRoot, projectPath, - framework.ToString(), + framework, packages, experimentsManager, logger); diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Analyze/Extensions.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Analyze/Extensions.cs index 8089fd5e78..0312c6c8cc 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Analyze/Extensions.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Analyze/Extensions.cs @@ -7,7 +7,7 @@ internal static class Extensions { - public static ImmutableArray GetDependencies(this ImmutableDictionary> dependenciesByTfm) + public static ImmutableArray GetDependencies(this ImmutableDictionary> dependenciesByTfm) { Dictionary dependencies = []; foreach (var (_framework, dependenciesForTfm) in dependenciesByTfm) From 4660f3851403ee509cd1bc7c81508e2e8b6eda97 Mon Sep 17 00:00:00 2001 From: "Brett V. Forsgren" Date: Fri, 20 Dec 2024 15:37:26 -0700 Subject: [PATCH 18/20] log replaced packages --- .../NuGetUpdater.Core/Discover/SdkProjectDiscovery.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Discover/SdkProjectDiscovery.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Discover/SdkProjectDiscovery.cs index dc60a5f42a..048eaf622f 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Discover/SdkProjectDiscovery.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Discover/SdkProjectDiscovery.cs @@ -246,6 +246,8 @@ runtimePackageVersion is not null && var packagesPerThisProject = packagesReplacedBySdkPerProject.GetOrAdd(projectEvaluation.ProjectFile, () => new(PathComparer.Instance)); var packagesPerTfm = packagesPerThisProject.GetOrAdd(tfm, () => new(StringComparer.OrdinalIgnoreCase)); packagesPerTfm[removedPackageName] = replacementPackageVersion.ToString(); + var relativeProjectPath = Path.GetRelativePath(repoRootPath, projectEvaluation.ProjectFile).NormalizePathToUnix(); + logger.Info($"Re-added SDK managed package [{removedPackageName}/{replacementPackageVersion}] to project [{relativeProjectPath}]"); } } } From 8c3d9b8686d17d26dac2711381f814502d6cf78f Mon Sep 17 00:00:00 2001 From: "Brett V. Forsgren" Date: Tue, 7 Jan 2025 16:01:13 -0700 Subject: [PATCH 19/20] update submodule --- nuget/helpers/lib/dotnet-core | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nuget/helpers/lib/dotnet-core b/nuget/helpers/lib/dotnet-core index 649080cc3a..218ef74621 160000 --- a/nuget/helpers/lib/dotnet-core +++ b/nuget/helpers/lib/dotnet-core @@ -1 +1 @@ -Subproject commit 649080cc3a0e927abb88d05591256e23c29e2170 +Subproject commit 218ef74621d76d73c0a562893df875d37ce928ad From 00a4c8b4f83e68fb02c9a7a85677775feecdc57c Mon Sep 17 00:00:00 2001 From: "Brett V. Forsgren" Date: Tue, 7 Jan 2025 16:48:43 -0700 Subject: [PATCH 20/20] use `break` and `continue` to reduce nesting --- .../Discover/SdkProjectDiscovery.cs | 92 +++++++++++-------- 1 file changed, 53 insertions(+), 39 deletions(-) diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Discover/SdkProjectDiscovery.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Discover/SdkProjectDiscovery.cs index 048eaf622f..6f1f64f931 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Discover/SdkProjectDiscovery.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Discover/SdkProjectDiscovery.cs @@ -213,48 +213,62 @@ public static async Task> DiscoverWithBin if (experimentsManager.InstallDotnetSdks) { var projectEvaluation = GetNearestProjectEvaluation(target); - if (projectEvaluation is not null) + if (projectEvaluation is null) + { + break; + } + + var removedReferences = target.Children.OfType().FirstOrDefault(r => r.Name == "Reference"); + var addedReferences = target.Children.OfType().FirstOrDefault(r => r.Name == "Reference"); + if (removedReferences is null || addedReferences is null) + { + break; + } + + foreach (var removedAssembly in removedReferences.Children.OfType()) { - var removedReferences = target.Children.OfType().FirstOrDefault(r => r.Name == "Reference"); - var addedReferences = target.Children.OfType().FirstOrDefault(r => r.Name == "Reference"); - if (removedReferences is not null && addedReferences is not null) + var removedPackageName = GetChildMetadataValue(removedAssembly, "NuGetPackageId"); + var removedFileName = Path.GetFileName(removedAssembly.Name); + if (removedPackageName is null || removedFileName is null) { - foreach (var removedAssembly in removedReferences.Children.OfType()) - { - var removedPackageName = GetChildMetadataValue(removedAssembly, "NuGetPackageId"); - var removedFileName = Path.GetFileName(removedAssembly.Name); - if (removedPackageName is not null && removedFileName is not null) - { - var existingProjectPackagesByTfm = packagesPerProject.GetOrAdd(projectEvaluation.ProjectFile, () => new(PathComparer.Instance)); - var existingProjectPackages = existingProjectPackagesByTfm.GetOrAdd(tfm, () => new(StringComparer.OrdinalIgnoreCase)); - if (existingProjectPackages.ContainsKey(removedPackageName)) - { - var correspondingAddedFile = addedReferences.Children.OfType() - .FirstOrDefault(i => removedFileName.Equals(Path.GetFileName(i.Name), StringComparison.OrdinalIgnoreCase)); - if (correspondingAddedFile is not null) - { - var runtimePackageName = GetChildMetadataValue(correspondingAddedFile, "NuGetPackageId"); - var runtimePackageVersion = GetChildMetadataValue(correspondingAddedFile, "NuGetPackageVersion"); - if (runtimePackageName is not null && - runtimePackageVersion is not null && - SemVersion.TryParse(runtimePackageVersion, out var parsedRuntimePackageVersion)) - { - var packageMapper = DotNetPackageCorrelationManager.GetPackageMapper(); - var replacementPackageVersion = packageMapper.GetPackageVersionThatShippedWithOtherPackage(runtimePackageName, parsedRuntimePackageVersion, removedPackageName); - if (replacementPackageVersion is not null) - { - var packagesPerThisProject = packagesReplacedBySdkPerProject.GetOrAdd(projectEvaluation.ProjectFile, () => new(PathComparer.Instance)); - var packagesPerTfm = packagesPerThisProject.GetOrAdd(tfm, () => new(StringComparer.OrdinalIgnoreCase)); - packagesPerTfm[removedPackageName] = replacementPackageVersion.ToString(); - var relativeProjectPath = Path.GetRelativePath(repoRootPath, projectEvaluation.ProjectFile).NormalizePathToUnix(); - logger.Info($"Re-added SDK managed package [{removedPackageName}/{replacementPackageVersion}] to project [{relativeProjectPath}]"); - } - } - } - } - } - } + continue; + } + + var existingProjectPackagesByTfm = packagesPerProject.GetOrAdd(projectEvaluation.ProjectFile, () => new(PathComparer.Instance)); + var existingProjectPackages = existingProjectPackagesByTfm.GetOrAdd(tfm, () => new(StringComparer.OrdinalIgnoreCase)); + if (!existingProjectPackages.ContainsKey(removedPackageName)) + { + continue; + } + + var correspondingAddedFile = addedReferences.Children.OfType() + .FirstOrDefault(i => removedFileName.Equals(Path.GetFileName(i.Name), StringComparison.OrdinalIgnoreCase)); + if (correspondingAddedFile is null) + { + continue; + } + + var runtimePackageName = GetChildMetadataValue(correspondingAddedFile, "NuGetPackageId"); + var runtimePackageVersion = GetChildMetadataValue(correspondingAddedFile, "NuGetPackageVersion"); + if (runtimePackageName is null || + runtimePackageVersion is null || + !SemVersion.TryParse(runtimePackageVersion, out var parsedRuntimePackageVersion)) + { + continue; + } + + var packageMapper = DotNetPackageCorrelationManager.GetPackageMapper(); + var replacementPackageVersion = packageMapper.GetPackageVersionThatShippedWithOtherPackage(runtimePackageName, parsedRuntimePackageVersion, removedPackageName); + if (replacementPackageVersion is null) + { + continue; } + + var packagesPerThisProject = packagesReplacedBySdkPerProject.GetOrAdd(projectEvaluation.ProjectFile, () => new(PathComparer.Instance)); + var packagesPerTfm = packagesPerThisProject.GetOrAdd(tfm, () => new(StringComparer.OrdinalIgnoreCase)); + packagesPerTfm[removedPackageName] = replacementPackageVersion.ToString(); + var relativeProjectPath = Path.GetRelativePath(repoRootPath, projectEvaluation.ProjectFile).NormalizePathToUnix(); + logger.Info($"Re-added SDK managed package [{removedPackageName}/{replacementPackageVersion}] to project [{relativeProjectPath}]"); } } break;