Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore: benchmark additional metrics tag #821

Merged
merged 2 commits into from
Aug 13, 2024
Merged

chore: benchmark additional metrics tag #821

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c29d3a2
chore: additional metrics tag
guitarrapc Aug 13, 2024
9483d1e
chore: add tags
guitarrapc Aug 13, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 9 additions & 5 deletions .github/workflows/benchmark.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,16 +20,20 @@ jobs:
matrix:
include:
# 1
- channels: 28
- tags: "streams:1"
channels: 28
streams: 1
# 1x1
- channels: 1
- tags: "streams:1x1"
channels: 1
streams: 1
# 70
- channels: 28
- tags: "streams:70"
channels: 28
streams: 70
# 70x1
- channels: 1
- tags: "streams:70x1"
channels: 1
streams: 70
uses: Cysharp/Actions/.github/workflows/benchmark.yaml@main
with:
Expand All @@ -38,7 +42,7 @@ jobs:
benchmark-name: "magiconion-${{ github.event.issue.number || github.run_number }}"
benchmark-timeout: 20 # 10min (env prepare) + 5min (clone & benchmark) + 5min (spare)
client-benchmark-script-path: ".github/scripts/run-benchmark-client.sh"
client-benchmark-script-args: "--args \"-u http://${BENCHMARK_SERVER_NAME}:5000 -s CI --channels ${{ matrix.channels }} --streams ${{ matrix.streams }}\""
client-benchmark-script-args: "--args \"-u http://${BENCHMARK_SERVER_NAME}:5000 -s CI --channels ${{ matrix.channels }} --streams ${{ matrix.streams }}\" --tags \"${{ matrix.tags }}\""
server-benchmark-script-path: ".github/scripts/run-benchmark-server.sh"
server-benchmark-script-args: ""
secrets: inherit
19 changes: 8 additions & 11 deletions perf/BenchmarkApp/PerformanceTest.Client/Program.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,11 +23,12 @@ async Task Main(
[Option("r")]string? report = null,
uint rounds = 1,
[Option("v")]bool verbose = false,
SerializationType serialization = SerializationType.MessagePack
SerializationType serialization = SerializationType.MessagePack,
string? tags = null
)
{
var config = new ScenarioConfiguration(url, warmup, duration, streams, channels, verbose);
var datadog = DatadogMetricsRecorder.Create();
var datadog = DatadogMetricsRecorder.Create(tags);

PrintStartupInformation();

Expand All @@ -39,6 +40,7 @@ async Task Main(
WriteLog($"Channels: {config.Channels}");
WriteLog($"Rounds: {rounds}");
WriteLog($"Serialization: {serialization}");
WriteLog($"Tags: {tags}");

// Setup serializer
switch (serialization)
Expand Down Expand Up @@ -265,17 +267,12 @@ public static class DatadogMetricsRecorderExtensions
/// <param name="result"></param>
public static async Task PutClientBenchmarkMetricsAsync(this DatadogMetricsRecorder recorder, string scenario, ApplicationInformation applicationInfo, string serialization, PerformanceResult result)
{
var tags = MetricsTagCache.Get((scenario, applicationInfo, serialization), static x => [$"app:MagicOnion", $"magiconion_version:{x.applicationInfo.MagicOnionVersion}", $"grpcdotnet_version:{x.applicationInfo.GrpcNetVersion}", $"messagepack_version:{x.applicationInfo.MessagePackVersion}", $"memorypack_version:{x.applicationInfo.MemoryPackVersion}", $"process_arch:{x.applicationInfo.ProcessArchitecture}", $"process_count:{x.applicationInfo.ProcessorCount}", $"scenario:{x.scenario}", $"serialization:{x.serialization}"]);
var tags = MetricsTagCache.Get((recorder.DefaultTags, scenario, applicationInfo, serialization), static x => [$"magiconion_version:{x.applicationInfo.MagicOnionVersion}", $"grpcdotnet_version:{x.applicationInfo.GrpcNetVersion}", $"messagepack_version:{x.applicationInfo.MessagePackVersion}", $"memorypack_version:{x.applicationInfo.MemoryPackVersion}", $"process_arch:{x.applicationInfo.ProcessArchitecture}", $"process_count:{x.applicationInfo.ProcessorCount}", $"scenario:{x.scenario}", $"serialization:{x.serialization}"]);

// Don't want to await each put. Let's send it to queue and await when benchmark ends.
recorder.Record(recorder.SendAsync("benchmark.client.rps", result.RequestsPerSecond, DatadogMetricsType.Rate, tags, "request"));
recorder.Record(recorder.SendAsync("benchmark.client.total_requests", result.TotalRequests, DatadogMetricsType.Gauge, tags, "request"));
recorder.Record(recorder.SendAsync("benchmark.client.latency_mean", result.Latency.Mean, DatadogMetricsType.Gauge, tags, "millisecond"));
recorder.Record(recorder.SendAsync("benchmark.client.latency_max", result.Latency.Max, DatadogMetricsType.Gauge, tags, "millisecond"));
recorder.Record(recorder.SendAsync("benchmark.client.latency_p50", result.Latency.P50, DatadogMetricsType.Gauge, tags, "millisecond"));
recorder.Record(recorder.SendAsync("benchmark.client.latency_p75", result.Latency.P75, DatadogMetricsType.Gauge, tags, "millisecond"));
recorder.Record(recorder.SendAsync("benchmark.client.latency_p90", result.Latency.P90, DatadogMetricsType.Gauge, tags, "millisecond"));
recorder.Record(recorder.SendAsync("benchmark.client.latency_p99", result.Latency.P99, DatadogMetricsType.Gauge, tags, "millisecond"));
recorder.Record(recorder.SendAsync("benchmark.magiconion.client.rps", result.RequestsPerSecond, DatadogMetricsType.Rate, tags, "request"));
recorder.Record(recorder.SendAsync("benchmark.magiconion.client.total_requests", result.TotalRequests, DatadogMetricsType.Gauge, tags, "request"));
recorder.Record(recorder.SendAsync("benchmark.magiconion.client.latency_mean", result.Latency.Mean, DatadogMetricsType.Gauge, tags, "millisecond"));

// wait until send complete
await recorder.WaitSaveAsync();
Expand Down
19 changes: 16 additions & 3 deletions perf/BenchmarkApp/PerformanceTest.Shared/Reporting/DatadogMetricsRecorder.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,13 +14,15 @@ namespace PerformanceTest.Shared.Reporting;
// * The full payload is approximately 100 bytes.
public class DatadogMetricsRecorder
{
public IReadOnlyList<string> DefaultTags { get; }
private readonly JsonSerializerOptions jsonSerializerOptions;
private readonly TimeProvider timeProvider = TimeProvider.System;
private readonly HttpClient client;
private readonly ConcurrentQueue<Task> backgroundQueue;

private DatadogMetricsRecorder(string apiKey)
private DatadogMetricsRecorder(IReadOnlyList<string> tags, string apiKey)
{
DefaultTags = tags;
jsonSerializerOptions = new JsonSerializerOptions()
{
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
Expand All @@ -35,14 +37,25 @@ private DatadogMetricsRecorder(string apiKey)
backgroundQueue = new ConcurrentQueue<Task>();
}

public static DatadogMetricsRecorder Create(bool validate = false)
public static DatadogMetricsRecorder Create(string? tagString, bool validate = false)
{
List<string> tags = [];
if (!string.IsNullOrEmpty(tagString))
{
foreach (var item in tagString.Split(","))
{
if (item.Contains(":"))
{
tags.Add(item);
}
}
}
var apiKey = Environment.GetEnvironmentVariable("DD_API_KEY");
if (validate)
{
ArgumentException.ThrowIfNullOrEmpty(apiKey);
}
return new DatadogMetricsRecorder(apiKey!);
return new DatadogMetricsRecorder(tags, apiKey!);
}

/// <summary>
Expand Down
Loading