From 30155928205acf18616f98c8ccd00ca2e3a0fb7e Mon Sep 17 00:00:00 2001 From: Paul Hebble Date: Fri, 6 Dec 2024 17:34:17 -0600 Subject: [PATCH] Don't raise inflation errors for single-hosted prereleases of multi-hosted mods --- Netkan/CmdLineOptions.cs | 5 +- Netkan/Processors/QueueHandler.cs | 7 ++ Netkan/Program.cs | 1 + Netkan/Transformers/EpochTransformer.cs | 77 ++++++++++++------- Netkan/Transformers/ITransformer.cs | 13 ++-- Tests/NetKAN/MainClass.cs | 2 +- Tests/NetKAN/NetkanOverride.cs | 2 +- .../Transformers/AvcKrefTransformerTests.cs | 2 +- .../Transformers/AvcTransformerTests.cs | 2 +- .../Transformers/CurseTransformerTests.cs | 2 +- .../DownloadAttributeTransformerTests.cs | 2 +- .../Transformers/EpochTransformerTests.cs | 54 +++++++++---- .../GeneratedByTransformerTests.cs | 2 +- .../Transformers/GithubTransformerTests.cs | 8 +- .../Transformers/HttpTransformerTests.cs | 2 +- .../InstallSizeTransformerTests.cs | 2 +- .../InternalCkanTransformerTests.cs | 2 +- .../MetaNetkanTransformerTests.cs | 2 +- .../Transformers/SpacedockTransformerTests.cs | 2 +- .../StripNetkanMetadataTransformerTests.cs | 2 +- .../VersionEditTransformerTests.cs | 2 +- 21 files changed, 128 insertions(+), 65 deletions(-) diff --git a/Netkan/CmdLineOptions.cs b/Netkan/CmdLineOptions.cs index dadb50926b..a1f27db3c0 100644 --- a/Netkan/CmdLineOptions.cs +++ b/Netkan/CmdLineOptions.cs @@ -46,9 +46,12 @@ internal class CmdLineOptions [Option("queues", HelpText = "Input,Output queue names for Queue Inflator mode")] public string? Queues { get; set; } - [Option("highest-version", HelpText = "Highest known version for auto-epoching")] + [Option("highest-version", HelpText = "Highest known non-prerelease version for auto-epoching")] public string? HighestVersion { get; set; } + [Option("highest-version-prerelease", HelpText = "Highest known prerelease version for auto-epoching")] + public string? HighestVersionPrerelease { get; set; } + [Option("validate-ckan", HelpText = "Name of .ckan file to check for errors")] public string? ValidateCkan { get; set; } diff --git a/Netkan/Processors/QueueHandler.cs b/Netkan/Processors/QueueHandler.cs index 4e272d57d6..7f0e61cd5e 100644 --- a/Netkan/Processors/QueueHandler.cs +++ b/Netkan/Processors/QueueHandler.cs @@ -152,6 +152,12 @@ private IEnumerable Inflate(Message msg) highVer = new ModuleVersion(highVerAttr.StringValue); } + ModuleVersion? highVerPre = null; + if (msg.MessageAttributes.TryGetValue("HighestVersionPrerelease", out MessageAttributeValue? highVerPreAttr)) + { + highVerPre = new ModuleVersion(highVerPreAttr.StringValue); + } + log.InfoFormat("Inflating {0}", netkans.First().Identifier); IEnumerable? ckans = null; bool caught = false; @@ -159,6 +165,7 @@ private IEnumerable Inflate(Message msg) var opts = new TransformOptions(releases, null, highVer, + highVerPre, netkans.First().Staged, netkans.First().StagingReason); try diff --git a/Netkan/Program.cs b/Netkan/Program.cs index 697b14989f..562e26bd7a 100644 --- a/Netkan/Program.cs +++ b/Netkan/Program.cs @@ -109,6 +109,7 @@ is string[] array ParseReleases(Options.Releases), ParseSkipReleases(Options.SkipReleases), ParseHighestVersion(Options.HighestVersion), + ParseHighestVersion(Options.HighestVersionPrerelease), netkans.First().Staged, netkans.First().StagingReason)) .ToArray(); diff --git a/Netkan/Transformers/EpochTransformer.cs b/Netkan/Transformers/EpochTransformer.cs index 9aa7feb8ef..29ece5eded 100644 --- a/Netkan/Transformers/EpochTransformer.cs +++ b/Netkan/Transformers/EpochTransformer.cs @@ -1,7 +1,10 @@ using System; +using System.Linq; using System.Collections.Generic; + using log4net; using Newtonsoft.Json.Linq; + using CKAN.Versioning; using CKAN.NetKAN.Model; @@ -29,7 +32,7 @@ public IEnumerable Transform(Metadata metadata, TransformOptions? opts if (uint.TryParse(epoch.ToString(), out uint epochNumber)) { - //Implicit if zero. No need to add + // Implicit if zero. No need to add if (epochNumber != 0) { json["version"] = epochNumber + ":" + json["version"]; @@ -43,40 +46,62 @@ public IEnumerable Transform(Metadata metadata, TransformOptions? opts } } - if (json.TryGetValue("x_netkan_allow_out_of_order", out JToken? allowOOO) && (bool)allowOOO) + if (json.TryGetValue("x_netkan_allow_out_of_order", out JToken? allowOOO) + && (bool)allowOOO) { Log.Debug("Out of order versions enabled in netkan, skipping OOO check"); } - else if (opts?.HighestVersion != null && (string?)json["version"] is string v) + else if (opts != null + && (string?)json["version"] is string v + && (metadata.Prerelease + ? new ModuleVersion?[] + { + opts.HighestVersionPrerelease, + opts.HighestVersion, + } + : new ModuleVersion?[] + { + opts.HighestVersion, + }) + .OfType() + .Max() + is ModuleVersion highest) + { + json["version"] = CheckOutOfOrder(opts, highest, + new ModuleVersion(v)) + .ToString(); + } + + yield return new Metadata(json); + } + + private static ModuleVersion CheckOutOfOrder(TransformOptions opts, + ModuleVersion highest, + ModuleVersion start) + { + // Ensure we are greater or equal to the previous max + ModuleVersion current = start; + while (current < highest) + { + Log.DebugFormat("Auto-epoching out of order version: {0} < {1}", + current, highest); + // Increment epoch if too small + current = current.IncrementEpoch(); + } + if (!highest.EpochEquals(current) && start < highest && highest < current) { - // Ensure we are greater or equal to the previous max - ModuleVersion startV = new ModuleVersion(v); - ModuleVersion currentV = startV; - while (currentV < opts.HighestVersion) + if (opts.FlakyAPI) { - Log.DebugFormat("Auto-epoching out of order version: {0} < {1}", - currentV, opts.HighestVersion); - // Increment epoch if too small - currentV = currentV.IncrementEpoch(); + throw new Kraken($"Out-of-order version found on unreliable server: {start} < {highest} < {current}"); } - if (!opts.HighestVersion.EpochEquals(currentV) - && startV < opts.HighestVersion && opts.HighestVersion < currentV) + else { - if (opts.FlakyAPI) - { - throw new Kraken($"Out-of-order version found on unreliable server: {startV} < {opts.HighestVersion} < {currentV}"); - } - else - { - // New file, tell the Indexer to be careful - opts.Staged = true; - opts.StagingReasons.Add($"Auto-epoching out of order version: {startV} < {opts.HighestVersion} < {currentV}"); - } + // New file, tell the Indexer to be careful + opts.Staged = true; + opts.StagingReasons.Add($"Auto-epoching out of order version: {start} < {highest} < {current}"); } - json["version"] = currentV.ToString(); } - - yield return new Metadata(json); + return current; } } } diff --git a/Netkan/Transformers/ITransformer.cs b/Netkan/Transformers/ITransformer.cs index 2af94c5c91..25c38e2fd1 100644 --- a/Netkan/Transformers/ITransformer.cs +++ b/Netkan/Transformers/ITransformer.cs @@ -9,14 +9,16 @@ internal class TransformOptions public TransformOptions(int? releases, int? skipReleases, ModuleVersion? highVer, + ModuleVersion? highVerPre, bool staged, string? stagingReason) { - Releases = releases; - SkipReleases = skipReleases; - HighestVersion = highVer; - Staged = staged; - StagingReasons = new List(); + Releases = releases; + SkipReleases = skipReleases; + HighestVersion = highVer; + HighestVersionPrerelease = highVerPre; + Staged = staged; + StagingReasons = new List(); if (stagingReason != null && !string.IsNullOrEmpty(stagingReason)) { StagingReasons.Add(stagingReason); @@ -26,6 +28,7 @@ public TransformOptions(int? releases, public readonly int? Releases; public readonly int? SkipReleases; public readonly ModuleVersion? HighestVersion; + public readonly ModuleVersion? HighestVersionPrerelease; public bool Staged; public readonly List StagingReasons; public bool FlakyAPI = false; diff --git a/Tests/NetKAN/MainClass.cs b/Tests/NetKAN/MainClass.cs index 6fdcb8fde4..640a940313 100644 --- a/Tests/NetKAN/MainClass.cs +++ b/Tests/NetKAN/MainClass.cs @@ -11,7 +11,7 @@ namespace Tests.NetKAN [TestFixture] public class MainClassTests { - private readonly TransformOptions opts = new TransformOptions(1, null, null, false, null); + private readonly TransformOptions opts = new TransformOptions(1, null, null, null, false, null); [Test] public void FixVersionStringsUnharmed() diff --git a/Tests/NetKAN/NetkanOverride.cs b/Tests/NetKAN/NetkanOverride.cs index 0f7992e647..8390facc61 100644 --- a/Tests/NetKAN/NetkanOverride.cs +++ b/Tests/NetKAN/NetkanOverride.cs @@ -13,7 +13,7 @@ namespace Tests.NetKAN public class NetkanOverride { private JObject? such_metadata; - private readonly TransformOptions opts = new TransformOptions(1, null, null, false, null); + private readonly TransformOptions opts = new TransformOptions(1, null, null, null, false, null); [SetUp] public void Setup() diff --git a/Tests/NetKAN/Transformers/AvcKrefTransformerTests.cs b/Tests/NetKAN/Transformers/AvcKrefTransformerTests.cs index 1368c7b81f..2f29d534b7 100644 --- a/Tests/NetKAN/Transformers/AvcKrefTransformerTests.cs +++ b/Tests/NetKAN/Transformers/AvcKrefTransformerTests.cs @@ -14,7 +14,7 @@ namespace Tests.NetKAN.Transformers [TestFixture] public sealed class AvcKrefTransformerTests { - private readonly TransformOptions opts = new TransformOptions(1, null, null, false, null); + private readonly TransformOptions opts = new TransformOptions(1, null, null, null, false, null); [Test, TestCase( diff --git a/Tests/NetKAN/Transformers/AvcTransformerTests.cs b/Tests/NetKAN/Transformers/AvcTransformerTests.cs index b221b9c005..af8808e303 100644 --- a/Tests/NetKAN/Transformers/AvcTransformerTests.cs +++ b/Tests/NetKAN/Transformers/AvcTransformerTests.cs @@ -18,7 +18,7 @@ namespace Tests.NetKAN.Transformers [TestFixture] public sealed class AvcTransformerTests { - private readonly TransformOptions opts = new TransformOptions(1, null, null, false, null); + private readonly TransformOptions opts = new TransformOptions(1, null, null, null, false, null); private readonly IGame game = new KerbalSpaceProgram(); [Test] diff --git a/Tests/NetKAN/Transformers/CurseTransformerTests.cs b/Tests/NetKAN/Transformers/CurseTransformerTests.cs index 04ea29dbf5..912062acc7 100644 --- a/Tests/NetKAN/Transformers/CurseTransformerTests.cs +++ b/Tests/NetKAN/Transformers/CurseTransformerTests.cs @@ -12,7 +12,7 @@ namespace Tests.NetKAN.Transformers [TestFixture] public sealed class CurseTransformerTests { - private readonly TransformOptions opts = new TransformOptions(1, null, null, false, null); + private readonly TransformOptions opts = new TransformOptions(1, null, null, null, false, null); // GH #199: Don't pre-fill KSP version fields if we see a ksp_min/max [Test] diff --git a/Tests/NetKAN/Transformers/DownloadAttributeTransformerTests.cs b/Tests/NetKAN/Transformers/DownloadAttributeTransformerTests.cs index 1005593816..ac0fd018d9 100644 --- a/Tests/NetKAN/Transformers/DownloadAttributeTransformerTests.cs +++ b/Tests/NetKAN/Transformers/DownloadAttributeTransformerTests.cs @@ -11,7 +11,7 @@ namespace Tests.NetKAN.Transformers [TestFixture] public sealed class DownloadAttributeTransformerTests { - private readonly TransformOptions opts = new TransformOptions(1, null, null, false, null); + private readonly TransformOptions opts = new TransformOptions(1, null, null, null, false, null); [Test] [Category("Cache")] diff --git a/Tests/NetKAN/Transformers/EpochTransformerTests.cs b/Tests/NetKAN/Transformers/EpochTransformerTests.cs index 3d41aa48ab..ac1d74b0e7 100644 --- a/Tests/NetKAN/Transformers/EpochTransformerTests.cs +++ b/Tests/NetKAN/Transformers/EpochTransformerTests.cs @@ -10,33 +10,56 @@ namespace Tests.NetKAN.Transformers [TestFixture] public sealed class EpochTransformerTests { - [Test, - TestCase("1.1", null, "1.1"), - TestCase("1.1", "1.1", "1.1"), - TestCase("1.1", "1.2", "1:1.1"), - TestCase("1.1", "5:1.1", "5:1.1"), - TestCase("1.1", "5:1.2", "6:1.1"), - TestCase("0.7", "0.65", "1:0.7"), - TestCase("2.5", "v2.4", "1:2.5"), - TestCase("2.5", "V2.4", "1:2.5"), - TestCase("v2.5", "vV2.4", "1:v2.5"), + [ + TestCase("1.1", null, null, false, "1.1", false), + TestCase("1.1", null, null, true, "1.1", false), + TestCase("1.1", "1.1", null, false, "1.1", false), + TestCase("1.1", "1.2", null, false, "1:1.1", true), + TestCase("1.1", "5:1.1", null, false, "5:1.1", false), + TestCase("1.1", "5:1.2", null, false, "6:1.1", true), + TestCase("0.7", "0.65", null, false, "1:0.7", true), + TestCase("2.5", "v2.4", null, false, "1:2.5", true), + TestCase("2.5", "V2.4", null, false, "1:2.5", true), + TestCase("v2.5", "vV2.4", null, false, "1:v2.5", true), + + // Non-prerelease with a prerelease on another host + TestCase("1.0", "1.0", "2.0", false, "1.0", false), + // Prerelease with no normal releases + TestCase("1.0", null, "1.0", true, "1.0", false), + // Updating a prerelease to a normal release + TestCase("2.0", "1.0", "2.0", false, "2.0", false), + // Prerelease needing an epoch boost + TestCase("2.0", "1.0", "3.0", true, "1:2.0", true), + // The previous prerelease is old + TestCase("1.5", "2.0", "1.0", true, "1:1.5", true), ] - public void Transform_WithHighVersionParam_MatchesExpected(string version, string highVer, string expected) + public void Transform_WithHighVersionParam_MatchesExpected(string version, + string? highVer, + string? highVerPre, + bool prerelease, + string expected, + bool staged) { // Arrange var json = new JObject() { - { "spec_version", "v1.4" }, - { "identifier", "AwesomeMod" }, - { "version", version }, + { "spec_version", "v1.4" }, + { "identifier", "AwesomeMod" }, + { "version", version }, + { "release_status", prerelease + ? "testing" + : "stable" }, }; ITransformer sut = new EpochTransformer(); TransformOptions opts = new TransformOptions( 1, null, - string.IsNullOrEmpty(highVer) + highVer is null or "" ? null : new ModuleVersion(highVer), + highVerPre is null or "" + ? null + : new ModuleVersion(highVerPre), false, null ); @@ -47,6 +70,7 @@ public void Transform_WithHighVersionParam_MatchesExpected(string version, strin // Assert Assert.AreEqual(expected, (string?)transformedJson["version"]); + Assert.AreEqual(staged, opts.Staged); } } } diff --git a/Tests/NetKAN/Transformers/GeneratedByTransformerTests.cs b/Tests/NetKAN/Transformers/GeneratedByTransformerTests.cs index 21ff87f2b7..4fd1d8288c 100644 --- a/Tests/NetKAN/Transformers/GeneratedByTransformerTests.cs +++ b/Tests/NetKAN/Transformers/GeneratedByTransformerTests.cs @@ -9,7 +9,7 @@ namespace Tests.NetKAN.Transformers [TestFixture] public sealed class GeneratedByTransformerTests { - private readonly TransformOptions opts = new TransformOptions(1, null, null, false, null); + private readonly TransformOptions opts = new TransformOptions(1, null, null, null, false, null); [Test] public void AddsGeneratedByProperty() diff --git a/Tests/NetKAN/Transformers/GithubTransformerTests.cs b/Tests/NetKAN/Transformers/GithubTransformerTests.cs index ac47501e03..1e631d6f3c 100644 --- a/Tests/NetKAN/Transformers/GithubTransformerTests.cs +++ b/Tests/NetKAN/Transformers/GithubTransformerTests.cs @@ -14,7 +14,7 @@ namespace Tests.NetKAN.Transformers [TestFixture] public sealed class GithubTransformerTests { - private readonly TransformOptions opts = new TransformOptions(1, null, null, false, null); + private readonly TransformOptions opts = new TransformOptions(1, null, null, null, false, null); private Mock? apiMockUp; @@ -192,7 +192,7 @@ public void Transform_MultipleReleases_TransformsAll() // Act var results = sut.Transform( new Metadata(json), - new TransformOptions(2, null, null, false, null) + new TransformOptions(2, null, null, null, false, null) ); var transformedJsons = results.Select(result => result.Json()).ToArray(); @@ -230,7 +230,7 @@ public void Transform_MultipleAssets_TransformsAll() // Act var results = sut.Transform( new Metadata(json), - new TransformOptions(1, 3, null, false, null) + new TransformOptions(1, 3, null, null, false, null) ); var transformedJsons = results.Select(result => result.Json()).ToArray(); @@ -268,7 +268,7 @@ public void Transform_SkipReleases_SkipsCorrectly() // Act var results = sut.Transform( new Metadata(json), - new TransformOptions(2, 1, null, false, null) + new TransformOptions(2, 1, null, null, false, null) ).ToArray(); // Assert diff --git a/Tests/NetKAN/Transformers/HttpTransformerTests.cs b/Tests/NetKAN/Transformers/HttpTransformerTests.cs index d2dba40551..30c58b233c 100644 --- a/Tests/NetKAN/Transformers/HttpTransformerTests.cs +++ b/Tests/NetKAN/Transformers/HttpTransformerTests.cs @@ -9,7 +9,7 @@ namespace Tests.NetKAN.Transformers [TestFixture] public sealed class HttpTransformerTests { - private readonly TransformOptions opts = new TransformOptions(1, null, null, false, null); + private readonly TransformOptions opts = new TransformOptions(1, null, null, null, false, null); [TestCase("#/ckan/github/foo/bar")] [TestCase("#/ckan/netkan/http://awesomemod.example/awesomemod.netkan")] diff --git a/Tests/NetKAN/Transformers/InstallSizeTransformerTests.cs b/Tests/NetKAN/Transformers/InstallSizeTransformerTests.cs index 376e7dfee5..27f4d0bd1a 100644 --- a/Tests/NetKAN/Transformers/InstallSizeTransformerTests.cs +++ b/Tests/NetKAN/Transformers/InstallSizeTransformerTests.cs @@ -38,6 +38,6 @@ public void Transform_NormalModule_CorrectInstallSize() Assert.AreEqual(52291, (int?)transformedJson["install_size"]); } - private readonly TransformOptions opts = new TransformOptions(1, null, null, false, null); + private readonly TransformOptions opts = new TransformOptions(1, null, null, null, false, null); } } diff --git a/Tests/NetKAN/Transformers/InternalCkanTransformerTests.cs b/Tests/NetKAN/Transformers/InternalCkanTransformerTests.cs index 9c495be162..0dff6ea2e5 100644 --- a/Tests/NetKAN/Transformers/InternalCkanTransformerTests.cs +++ b/Tests/NetKAN/Transformers/InternalCkanTransformerTests.cs @@ -15,7 +15,7 @@ namespace Tests.NetKAN.Transformers [TestFixture] public sealed class InternalCkanTransformerTests { - private readonly TransformOptions opts = new TransformOptions(1, null, null, false, null); + private readonly TransformOptions opts = new TransformOptions(1, null, null, null, false, null); [Test] public void AddsMissingProperties() diff --git a/Tests/NetKAN/Transformers/MetaNetkanTransformerTests.cs b/Tests/NetKAN/Transformers/MetaNetkanTransformerTests.cs index faccac90cd..87a478ff78 100644 --- a/Tests/NetKAN/Transformers/MetaNetkanTransformerTests.cs +++ b/Tests/NetKAN/Transformers/MetaNetkanTransformerTests.cs @@ -15,7 +15,7 @@ namespace Tests.NetKAN.Transformers [TestFixture] public sealed class MetaNetkanTransformerTests { - private readonly TransformOptions opts = new TransformOptions(1, null, null, false, null); + private readonly TransformOptions opts = new TransformOptions(1, null, null, null, false, null); [Test] public void DoesNothingWhenNoMatch() diff --git a/Tests/NetKAN/Transformers/SpacedockTransformerTests.cs b/Tests/NetKAN/Transformers/SpacedockTransformerTests.cs index 631dd38ac6..77bec175c9 100644 --- a/Tests/NetKAN/Transformers/SpacedockTransformerTests.cs +++ b/Tests/NetKAN/Transformers/SpacedockTransformerTests.cs @@ -15,7 +15,7 @@ namespace Tests.NetKAN.Transformers [TestFixture] public sealed class SpacedockTransformerTests { - private readonly TransformOptions opts = new TransformOptions(1, null, null, false, null); + private readonly TransformOptions opts = new TransformOptions(1, null, null, null, false, null); // GH #199: Don't pre-fill KSP version fields if we see a ksp_min/max [Test] diff --git a/Tests/NetKAN/Transformers/StripNetkanMetadataTransformerTests.cs b/Tests/NetKAN/Transformers/StripNetkanMetadataTransformerTests.cs index 3b1d1f5fa2..1063027edd 100644 --- a/Tests/NetKAN/Transformers/StripNetkanMetadataTransformerTests.cs +++ b/Tests/NetKAN/Transformers/StripNetkanMetadataTransformerTests.cs @@ -10,7 +10,7 @@ namespace Tests.NetKAN.Transformers [TestFixture] public sealed class StripNetkanMetadataTransformerTests { - private readonly TransformOptions opts = new TransformOptions(1, null, null, false, null); + private readonly TransformOptions opts = new TransformOptions(1, null, null, null, false, null); [TestCaseSource("StripNetkanMetadataTestCaseSource")] public void StripNetkanMetadata(string json, string expected) diff --git a/Tests/NetKAN/Transformers/VersionEditTransformerTests.cs b/Tests/NetKAN/Transformers/VersionEditTransformerTests.cs index ae65a79575..bc83e73d14 100644 --- a/Tests/NetKAN/Transformers/VersionEditTransformerTests.cs +++ b/Tests/NetKAN/Transformers/VersionEditTransformerTests.cs @@ -10,7 +10,7 @@ namespace Tests.NetKAN.Transformers [TestFixture] public sealed class VersionEditTransformerTests { - private readonly TransformOptions opts = new TransformOptions(1, null, null, false, null); + private readonly TransformOptions opts = new TransformOptions(1, null, null, null, false, null); [Test] public void DoesNothingWhenNoMatch()