Merge pull request #821 from Cysharp/feature/benchmark

chore: benchmark additional metrics tag
Cysharp · Aug 13, 2024 · deb6086 · deb6086
2 parents 4f63800 + 9483d1e
commit deb6086
Show file tree

Hide file tree

Showing 3 changed files with 33 additions and 19 deletions.
diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
@@ -20,16 +20,20 @@ jobs:
       matrix:
         include:
           # 1
-          - channels: 28
+          - tags: "streams:1"
+            channels: 28
             streams: 1
           # 1x1
-          - channels: 1
+          - tags: "streams:1x1"
+            channels: 1
             streams: 1
           # 70
-          - channels: 28
+          - tags: "streams:70"
+            channels: 28
             streams: 70
           # 70x1
-          - channels: 1
+          - tags: "streams:70x1"
+            channels: 1
             streams: 70
     uses: Cysharp/Actions/.github/workflows/benchmark.yaml@main
     with:
@@ -38,7 +42,7 @@ jobs:
       benchmark-name: "magiconion-${{ github.event.issue.number || github.run_number }}"
       benchmark-timeout: 20 # 10min (env prepare) + 5min (clone & benchmark) + 5min (spare)
       client-benchmark-script-path: ".github/scripts/run-benchmark-client.sh"
-      client-benchmark-script-args: "--args \"-u http://${BENCHMARK_SERVER_NAME}:5000 -s CI --channels ${{ matrix.channels }} --streams ${{ matrix.streams }}\""
+      client-benchmark-script-args: "--args \"-u http://${BENCHMARK_SERVER_NAME}:5000 -s CI --channels ${{ matrix.channels }} --streams ${{ matrix.streams }}\" --tags \"${{ matrix.tags }}\""
       server-benchmark-script-path: ".github/scripts/run-benchmark-server.sh"
       server-benchmark-script-args: ""
     secrets: inherit
diff --git a/perf/BenchmarkApp/PerformanceTest.Client/Program.cs b/perf/BenchmarkApp/PerformanceTest.Client/Program.cs
@@ -23,11 +23,12 @@ async Task Main(
     [Option("r")]string? report = null,
     uint rounds = 1,
     [Option("v")]bool verbose = false,
-    SerializationType serialization = SerializationType.MessagePack
+    SerializationType serialization = SerializationType.MessagePack,
+    string? tags = null
 )
 {
     var config = new ScenarioConfiguration(url, warmup, duration, streams, channels, verbose);
-    var datadog = DatadogMetricsRecorder.Create();
+    var datadog = DatadogMetricsRecorder.Create(tags);
 
     PrintStartupInformation();
 
@@ -39,6 +40,7 @@ async Task Main(
     WriteLog($"Channels: {config.Channels}");
     WriteLog($"Rounds: {rounds}");
     WriteLog($"Serialization: {serialization}");
+    WriteLog($"Tags: {tags}");
 
     // Setup serializer
     switch (serialization)
@@ -265,17 +267,12 @@ public static class DatadogMetricsRecorderExtensions
     /// <param name="result"></param>
     public static async Task PutClientBenchmarkMetricsAsync(this DatadogMetricsRecorder recorder, string scenario, ApplicationInformation applicationInfo, string serialization, PerformanceResult result)
     {
-        var tags = MetricsTagCache.Get((scenario, applicationInfo, serialization), static x => [$"app:MagicOnion", $"magiconion_version:{x.applicationInfo.MagicOnionVersion}", $"grpcdotnet_version:{x.applicationInfo.GrpcNetVersion}", $"messagepack_version:{x.applicationInfo.MessagePackVersion}", $"memorypack_version:{x.applicationInfo.MemoryPackVersion}", $"process_arch:{x.applicationInfo.ProcessArchitecture}", $"process_count:{x.applicationInfo.ProcessorCount}", $"scenario:{x.scenario}", $"serialization:{x.serialization}"]);
+        var tags = MetricsTagCache.Get((recorder.DefaultTags, scenario, applicationInfo, serialization), static x => [$"magiconion_version:{x.applicationInfo.MagicOnionVersion}", $"grpcdotnet_version:{x.applicationInfo.GrpcNetVersion}", $"messagepack_version:{x.applicationInfo.MessagePackVersion}", $"memorypack_version:{x.applicationInfo.MemoryPackVersion}", $"process_arch:{x.applicationInfo.ProcessArchitecture}", $"process_count:{x.applicationInfo.ProcessorCount}", $"scenario:{x.scenario}", $"serialization:{x.serialization}"]);
 
         // Don't want to await each put. Let's send it to queue and await when benchmark ends.
-        recorder.Record(recorder.SendAsync("benchmark.client.rps", result.RequestsPerSecond, DatadogMetricsType.Rate, tags, "request"));
-        recorder.Record(recorder.SendAsync("benchmark.client.total_requests", result.TotalRequests, DatadogMetricsType.Gauge, tags, "request"));
-        recorder.Record(recorder.SendAsync("benchmark.client.latency_mean", result.Latency.Mean, DatadogMetricsType.Gauge, tags, "millisecond"));
-        recorder.Record(recorder.SendAsync("benchmark.client.latency_max", result.Latency.Max, DatadogMetricsType.Gauge, tags, "millisecond"));
-        recorder.Record(recorder.SendAsync("benchmark.client.latency_p50", result.Latency.P50, DatadogMetricsType.Gauge, tags, "millisecond"));
-        recorder.Record(recorder.SendAsync("benchmark.client.latency_p75", result.Latency.P75, DatadogMetricsType.Gauge, tags, "millisecond"));
-        recorder.Record(recorder.SendAsync("benchmark.client.latency_p90", result.Latency.P90, DatadogMetricsType.Gauge, tags, "millisecond"));
-        recorder.Record(recorder.SendAsync("benchmark.client.latency_p99", result.Latency.P99, DatadogMetricsType.Gauge, tags, "millisecond"));
+        recorder.Record(recorder.SendAsync("benchmark.magiconion.client.rps", result.RequestsPerSecond, DatadogMetricsType.Rate, tags, "request"));
+        recorder.Record(recorder.SendAsync("benchmark.magiconion.client.total_requests", result.TotalRequests, DatadogMetricsType.Gauge, tags, "request"));
+        recorder.Record(recorder.SendAsync("benchmark.magiconion.client.latency_mean", result.Latency.Mean, DatadogMetricsType.Gauge, tags, "millisecond"));
 
         // wait until send complete
         await recorder.WaitSaveAsync();

diff --git a/perf/BenchmarkApp/PerformanceTest.Shared/Reporting/DatadogMetricsRecorder.cs b/perf/BenchmarkApp/PerformanceTest.Shared/Reporting/DatadogMetricsRecorder.cs
@@ -14,13 +14,15 @@ namespace PerformanceTest.Shared.Reporting;
 // * The full payload is approximately 100 bytes.
 public class DatadogMetricsRecorder
 {
+    public IReadOnlyList<string> DefaultTags { get; }
     private readonly JsonSerializerOptions jsonSerializerOptions;
     private readonly TimeProvider timeProvider = TimeProvider.System;
     private readonly HttpClient client;
     private readonly ConcurrentQueue<Task> backgroundQueue;
 
-    private DatadogMetricsRecorder(string apiKey)
+    private DatadogMetricsRecorder(IReadOnlyList<string> tags, string apiKey)
     {
+        DefaultTags = tags;
         jsonSerializerOptions = new JsonSerializerOptions()
         {
             DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
@@ -35,14 +37,25 @@ private DatadogMetricsRecorder(string apiKey)
         backgroundQueue = new ConcurrentQueue<Task>();
     }
 
-    public static DatadogMetricsRecorder Create(bool validate = false)
+    public static DatadogMetricsRecorder Create(string? tagString, bool validate = false)
     {
+        List<string> tags = [];
+        if (!string.IsNullOrEmpty(tagString))
+        {
+            foreach (var item in tagString.Split(","))
+            {
+                if (item.Contains(":"))
+                {
+                    tags.Add(item);
+                }
+            }
+        }
         var apiKey = Environment.GetEnvironmentVariable("DD_API_KEY");
         if (validate)
         {
             ArgumentException.ThrowIfNullOrEmpty(apiKey);
         }
-        return new DatadogMetricsRecorder(apiKey!);
+        return new DatadogMetricsRecorder(tags, apiKey!);
     }
 
     /// <summary>