From c29d3a25c35f96179f1a093278d5f7de5698b6d8 Mon Sep 17 00:00:00 2001 From: Ikiru Yoshizaki <3856350+guitarrapc@users.noreply.github.com> Date: Tue, 13 Aug 2024 17:04:16 +0900 Subject: [PATCH 1/2] chore: additional metrics tag --- .../PerformanceTest.Client/Program.cs | 19 ++++++++----------- .../Reporting/DatadogMetricsRecorder.cs | 19 ++++++++++++++++--- 2 files changed, 24 insertions(+), 14 deletions(-) diff --git a/perf/BenchmarkApp/PerformanceTest.Client/Program.cs b/perf/BenchmarkApp/PerformanceTest.Client/Program.cs index 8f3f8a90f..524b70d58 100644 --- a/perf/BenchmarkApp/PerformanceTest.Client/Program.cs +++ b/perf/BenchmarkApp/PerformanceTest.Client/Program.cs @@ -23,11 +23,12 @@ async Task Main( [Option("r")]string? report = null, uint rounds = 1, [Option("v")]bool verbose = false, - SerializationType serialization = SerializationType.MessagePack + SerializationType serialization = SerializationType.MessagePack, + string? tags = null ) { var config = new ScenarioConfiguration(url, warmup, duration, streams, channels, verbose); - var datadog = DatadogMetricsRecorder.Create(); + var datadog = DatadogMetricsRecorder.Create(tags); PrintStartupInformation(); @@ -39,6 +40,7 @@ async Task Main( WriteLog($"Channels: {config.Channels}"); WriteLog($"Rounds: {rounds}"); WriteLog($"Serialization: {serialization}"); + WriteLog($"Tags: {tags}"); // Setup serializer switch (serialization) @@ -265,17 +267,12 @@ public static class DatadogMetricsRecorderExtensions /// public static async Task PutClientBenchmarkMetricsAsync(this DatadogMetricsRecorder recorder, string scenario, ApplicationInformation applicationInfo, string serialization, PerformanceResult result) { - var tags = MetricsTagCache.Get((scenario, applicationInfo, serialization), static x => [$"app:MagicOnion", $"magiconion_version:{x.applicationInfo.MagicOnionVersion}", $"grpcdotnet_version:{x.applicationInfo.GrpcNetVersion}", $"messagepack_version:{x.applicationInfo.MessagePackVersion}", $"memorypack_version:{x.applicationInfo.MemoryPackVersion}", $"process_arch:{x.applicationInfo.ProcessArchitecture}", $"process_count:{x.applicationInfo.ProcessorCount}", $"scenario:{x.scenario}", $"serialization:{x.serialization}"]); + var tags = MetricsTagCache.Get((recorder.DefaultTags, scenario, applicationInfo, serialization), static x => [$"magiconion_version:{x.applicationInfo.MagicOnionVersion}", $"grpcdotnet_version:{x.applicationInfo.GrpcNetVersion}", $"messagepack_version:{x.applicationInfo.MessagePackVersion}", $"memorypack_version:{x.applicationInfo.MemoryPackVersion}", $"process_arch:{x.applicationInfo.ProcessArchitecture}", $"process_count:{x.applicationInfo.ProcessorCount}", $"scenario:{x.scenario}", $"serialization:{x.serialization}"]); // Don't want to await each put. Let's send it to queue and await when benchmark ends. - recorder.Record(recorder.SendAsync("benchmark.client.rps", result.RequestsPerSecond, DatadogMetricsType.Rate, tags, "request")); - recorder.Record(recorder.SendAsync("benchmark.client.total_requests", result.TotalRequests, DatadogMetricsType.Gauge, tags, "request")); - recorder.Record(recorder.SendAsync("benchmark.client.latency_mean", result.Latency.Mean, DatadogMetricsType.Gauge, tags, "millisecond")); - recorder.Record(recorder.SendAsync("benchmark.client.latency_max", result.Latency.Max, DatadogMetricsType.Gauge, tags, "millisecond")); - recorder.Record(recorder.SendAsync("benchmark.client.latency_p50", result.Latency.P50, DatadogMetricsType.Gauge, tags, "millisecond")); - recorder.Record(recorder.SendAsync("benchmark.client.latency_p75", result.Latency.P75, DatadogMetricsType.Gauge, tags, "millisecond")); - recorder.Record(recorder.SendAsync("benchmark.client.latency_p90", result.Latency.P90, DatadogMetricsType.Gauge, tags, "millisecond")); - recorder.Record(recorder.SendAsync("benchmark.client.latency_p99", result.Latency.P99, DatadogMetricsType.Gauge, tags, "millisecond")); + recorder.Record(recorder.SendAsync("benchmark.magiconion.client.rps", result.RequestsPerSecond, DatadogMetricsType.Rate, tags, "request")); + recorder.Record(recorder.SendAsync("benchmark.magiconion.client.total_requests", result.TotalRequests, DatadogMetricsType.Gauge, tags, "request")); + recorder.Record(recorder.SendAsync("benchmark.magiconion.client.latency_mean", result.Latency.Mean, DatadogMetricsType.Gauge, tags, "millisecond")); // wait until send complete await recorder.WaitSaveAsync(); diff --git a/perf/BenchmarkApp/PerformanceTest.Shared/Reporting/DatadogMetricsRecorder.cs b/perf/BenchmarkApp/PerformanceTest.Shared/Reporting/DatadogMetricsRecorder.cs index 89840b99a..4efd461ac 100644 --- a/perf/BenchmarkApp/PerformanceTest.Shared/Reporting/DatadogMetricsRecorder.cs +++ b/perf/BenchmarkApp/PerformanceTest.Shared/Reporting/DatadogMetricsRecorder.cs @@ -14,13 +14,15 @@ namespace PerformanceTest.Shared.Reporting; // * The full payload is approximately 100 bytes. public class DatadogMetricsRecorder { + public IReadOnlyList DefaultTags { get; } private readonly JsonSerializerOptions jsonSerializerOptions; private readonly TimeProvider timeProvider = TimeProvider.System; private readonly HttpClient client; private readonly ConcurrentQueue backgroundQueue; - private DatadogMetricsRecorder(string apiKey) + private DatadogMetricsRecorder(IReadOnlyList tags, string apiKey) { + DefaultTags = tags; jsonSerializerOptions = new JsonSerializerOptions() { DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, @@ -35,14 +37,25 @@ private DatadogMetricsRecorder(string apiKey) backgroundQueue = new ConcurrentQueue(); } - public static DatadogMetricsRecorder Create(bool validate = false) + public static DatadogMetricsRecorder Create(string? tagString, bool validate = false) { + List tags = []; + if (!string.IsNullOrEmpty(tagString)) + { + foreach (var item in tagString.Split(",")) + { + if (item.Contains(":")) + { + tags.Add(item); + } + } + } var apiKey = Environment.GetEnvironmentVariable("DD_API_KEY"); if (validate) { ArgumentException.ThrowIfNullOrEmpty(apiKey); } - return new DatadogMetricsRecorder(apiKey!); + return new DatadogMetricsRecorder(tags, apiKey!); } /// From 9483d1e9ffaeb791bc25cadc86afd379ad56f439 Mon Sep 17 00:00:00 2001 From: Ikiru Yoshizaki <3856350+guitarrapc@users.noreply.github.com> Date: Tue, 13 Aug 2024 17:10:26 +0900 Subject: [PATCH 2/2] chore: add tags --- .github/workflows/benchmark.yml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 44ca942a7..95e79ad31 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -20,16 +20,20 @@ jobs: matrix: include: # 1 - - channels: 28 + - tags: "streams:1" + channels: 28 streams: 1 # 1x1 - - channels: 1 + - tags: "streams:1x1" + channels: 1 streams: 1 # 70 - - channels: 28 + - tags: "streams:70" + channels: 28 streams: 70 # 70x1 - - channels: 1 + - tags: "streams:70x1" + channels: 1 streams: 70 uses: Cysharp/Actions/.github/workflows/benchmark.yaml@main with: @@ -38,7 +42,7 @@ jobs: benchmark-name: "magiconion-${{ github.event.issue.number || github.run_number }}" benchmark-timeout: 20 # 10min (env prepare) + 5min (clone & benchmark) + 5min (spare) client-benchmark-script-path: ".github/scripts/run-benchmark-client.sh" - client-benchmark-script-args: "--args \"-u http://${BENCHMARK_SERVER_NAME}:5000 -s CI --channels ${{ matrix.channels }} --streams ${{ matrix.streams }}\"" + client-benchmark-script-args: "--args \"-u http://${BENCHMARK_SERVER_NAME}:5000 -s CI --channels ${{ matrix.channels }} --streams ${{ matrix.streams }}\" --tags \"${{ matrix.tags }}\"" server-benchmark-script-path: ".github/scripts/run-benchmark-server.sh" server-benchmark-script-args: "" secrets: inherit