Skip to content

Commit

Permalink
Adjust exporter to OTLP proto 1.5.0
Browse files Browse the repository at this point in the history
  • Loading branch information
Kielek committed Jan 3, 2025
1 parent ba45fef commit 3c07ffa
Show file tree
Hide file tree
Showing 6 changed files with 62 additions and 79 deletions.
14 changes: 7 additions & 7 deletions test/IntegrationTests/ContinuousProfilerTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ public void ExportAllocationSamples()
SetEnvironmentVariable("OTEL_DOTNET_AUTO_TRACES_ADDITIONAL_SOURCES", "TestApplication.ContinuousProfiler");
RunTestApplication();

collector.Expect(profileData => profileData.ResourceProfiles.Any(resourceProfiles => resourceProfiles.ScopeProfiles.Any(scopeProfile => scopeProfile.Profiles.Any(profileContainer => ContainAttributes(profileContainer, "allocation") && profileContainer.Profile.Sample[0].Value[0] != 0.0))));
collector.Expect(profileData => profileData.ResourceProfiles.Any(resourceProfiles => resourceProfiles.ScopeProfiles.Any(scopeProfile => scopeProfile.Profiles.Any(profile => ContainAttributes(profile, "allocation") && profile.Sample[0].Value[0] != 0.0))));
collector.ResourceExpector.Expect("todo.resource.detector.key", "todo.resource.detector.value");

collector.AssertExpectations();
Expand All @@ -47,16 +47,16 @@ public void ExportThreadSamples()

var expectedStackTrace = string.Join("\n", CreateExpectedStackTrace());

collector.Expect(profileData => profileData.ResourceProfiles.Any(resourceProfiles => resourceProfiles.ScopeProfiles.Any(scopeProfile => scopeProfile.Profiles.Any(profileContainer => ContainStackTraceForClassHierarchy(profileContainer.Profile, expectedStackTrace) && ContainAttributes(profileContainer, "cpu")))));
collector.Expect(profileData => profileData.ResourceProfiles.Any(resourceProfiles => resourceProfiles.ScopeProfiles.Any(scopeProfile => scopeProfile.Profiles.Any(profile => ContainStackTraceForClassHierarchy(profile, expectedStackTrace) && ContainAttributes(profile, "cpu")))));
collector.ResourceExpector.Expect("todo.resource.detector.key", "todo.resource.detector.value");

collector.AssertExpectations();
collector.ResourceExpector.AssertExpectations();
}

private static bool ContainAttributes(ProfileContainer profileContainer, string profilingDataType)
private static bool ContainAttributes(Profile profileContainer, string profilingDataType)
{
return profileContainer.Attributes.Any(x => x.Key == "todo.profiling.data.type" && x.Value.StringValue == profilingDataType);
return profileContainer.AttributeTable.Any(x => x.Key == "todo.profiling.data.type" && x.Value.StringValue == profilingDataType);
}

private static List<string> CreateExpectedStackTrace()
Expand Down Expand Up @@ -96,11 +96,11 @@ private static List<string> CreateExpectedStackTrace()

private bool ContainStackTraceForClassHierarchy(Profile profile, string expectedStackTrace)
{
var frames = profile.Location
var frames = profile.LocationTable
.SelectMany(location => location.Line)
.Select(line => line.FunctionIndex)
.Select(functionId => profile.Function[(int)functionId - 1])
.Select(function => profile.StringTable[(int)function.Name]);
.Select(functionId => profile.FunctionTable[functionId - 1])
.Select(function => profile.StringTable[function.NameStrindex]);

var stackTrace = string.Join("\n", frames);

Expand Down
2 changes: 0 additions & 2 deletions test/IntegrationTests/SmokeTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,7 @@
using System.Reflection;
using FluentAssertions;
using IntegrationTests.Helpers;
using OpenTelemetry.Logs;
using Xunit.Abstractions;
using LogRecord = OpenTelemetry.Proto.Logs.V1.LogRecord;

#if NETFRAMEWORK
using System.Net;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
// Copyright The OpenTelemetry Authors
// SPDX-License-Identifier: Apache-2.0

using System.Diagnostics;
using Google.Protobuf;
using OpenTelemetry.Proto.Common.V1;
using OpenTelemetry.Proto.Profiles.V1Development;
Expand All @@ -13,19 +14,28 @@ internal class ExtendedPprofBuilder
private readonly LinkCache _linkCache;
private readonly AttributeCache _attributeCache;

public ExtendedPprofBuilder()
public ExtendedPprofBuilder(string profilingDataType, long timestampNanoseconds)
{
Profile = new Profile();
var profileByteId = new byte[16];
ActivityTraceId.CreateRandom().CopyTo(profileByteId);
Profile = new Profile
{
ProfileId = ByteString.CopyFrom(profileByteId),
TimeNanos = timestampNanoseconds,
};
var stringCache = new StringCache(Profile);
var functionCache = new FunctionCache(Profile, stringCache);
_locationCache = new LocationCache(Profile, functionCache);
_linkCache = new LinkCache(Profile);
_attributeCache = new AttributeCache(Profile);

var profilingDataTypeAttributeId = _attributeCache.GetOrAdd("todo.profiling.data.type", value => value.StringValue = profilingDataType);
Profile.AttributeIndices.Add(profilingDataTypeAttributeId);
}

public Profile Profile { get; }

public ulong GetLocationId(string function) => _locationCache.Get(function);
public int AddLocationId(string function) => _locationCache.Add(function);

public void AddLink(SampleBuilder sampleBuilder, long spanId, long traceIdHigh, long traceIdLow)
{
Expand Down Expand Up @@ -53,16 +63,16 @@ private void AddAttribute(SampleBuilder sampleBuilder, string name, Action<AnyVa
private class StringCache
{
private readonly Profile _profile;
private readonly Dictionary<string, long> _table = new();
private long _index;
private readonly Dictionary<string, int> _table = new();
private int _index;

public StringCache(Profile profile)
{
_profile = profile;
GetOrAdd(string.Empty); // 0 is reserved for the empty string
}

public long GetOrAdd(string str)
public int GetOrAdd(string str)
{
if (_table.TryGetValue(str, out var value))
{
Expand All @@ -79,25 +89,27 @@ private class FunctionCache
{
private readonly Profile _profile;
private readonly StringCache _stringCache;
private readonly Dictionary<string, ulong> _table = new();
private ulong _index = 1; // 0 is reserved
private readonly Dictionary<string, int> _table = new();
private int _index = 1; // 0 is reserved

public FunctionCache(Profile profile, StringCache stringCache)
{
_profile = profile;
_stringCache = stringCache;
}

public ulong GetOrAdd(string functionName)
public int GetOrAdd(string functionName)
{
if (_table.TryGetValue(functionName, out var value))
{
return value;
}

var function = new Function { Id = _index, Filename = _stringCache.GetOrAdd("unknown"), Name = _stringCache.GetOrAdd(functionName) }; // for now, we don't support file name
// TODO How to handle SystemName in .NET
// TODO handle line number
var function = new Function { SystemNameStrindex = _stringCache.GetOrAdd("TODO How to handle SystemName in .NET?"), FilenameStrindex = _stringCache.GetOrAdd("unknown"), NameStrindex = _stringCache.GetOrAdd(functionName), }; // for now, we don't support file name

_profile.Function.Add(function);
_profile.FunctionTable.Add(function);
_table[functionName] = _index;
return _index++;
}
Expand All @@ -107,27 +119,20 @@ private class LocationCache
{
private readonly Profile _profile;
private readonly FunctionCache _functionCache;
private readonly Dictionary<string, ulong> _table = new();
private ulong _index = 1; // 0 is reserved
private int _index = 1; // 0 is reserved

public LocationCache(Profile profile, FunctionCache functionCache)
{
_profile = profile;
_functionCache = functionCache;
}

public ulong Get(string function)
public int Add(string function)
{
if (_table.TryGetValue(function, out var value))
{
return value;
}

var location = new Location { Id = _index };
var location = new Location();
location.Line.Add(new Line { FunctionIndex = _functionCache.GetOrAdd(function), Line_ = 0, Column = 0 }); // for now, we don't support line nor column number

_profile.Location.Add(location);
_table[function] = _index;
_profile.LocationTable.Add(location);

return _index++;
}
Expand All @@ -136,15 +141,15 @@ public ulong Get(string function)
private class LinkCache
{
private readonly Profile _profile;
private readonly Dictionary<Tuple<long, long, long>, ulong> _table = new();
private ulong _index = 1; // 0 is reserved
private readonly Dictionary<Tuple<long, long, long>, int> _table = new();
private int _index = 1; // 0 is reserved

public LinkCache(Profile profile)
{
_profile = profile;
}

public ulong GetOrAdd(long spanId, long traceIdHigh, long traceIdLow)
public int GetOrAdd(long spanId, long traceIdHigh, long traceIdLow)
{
var key = Tuple.Create(spanId, traceIdHigh, traceIdLow);

Expand Down Expand Up @@ -177,15 +182,15 @@ public ulong GetOrAdd(long spanId, long traceIdHigh, long traceIdLow)
private class AttributeCache
{
private readonly Profile _profile;
private readonly Dictionary<KeyValue, ulong> _table = new();
private ulong _index = 1; // 0 is reserved
private readonly Dictionary<KeyValue, int> _table = new();
private int _index = 1; // 0 is reserved

public AttributeCache(Profile profile)
{
_profile = profile;
}

public ulong GetOrAdd(string name, Action<AnyValue> setValue)
public int GetOrAdd(string name, Action<AnyValue> setValue)
{
var keyValue = new KeyValue
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,8 @@ public void ExportThreadSamples(byte[] buffer, int read, CancellationToken cance

try
{
var extendedPprofBuilder = new ExtendedPprofBuilder();
var timestampNanoseconds = threadSamples[0].TimestampNanoseconds; // all items in the batch have same timestamp
var extendedPprofBuilder = new ExtendedPprofBuilder("cpu", timestampNanoseconds);

for (var i = 0; i < threadSamples.Count; i++)
{
Expand All @@ -43,12 +44,8 @@ public void ExportThreadSamples(byte[] buffer, int read, CancellationToken cance
extendedPprofBuilder.Profile.Sample.Add(sampleBuilder.Build());
}

var timestampNanoseconds = threadSamples[0].TimestampNanoseconds; // all items in the batch have same timestamp

var profileContainer = CreateProfileContainer(extendedPprofBuilder.Profile, "cpu", timestampNanoseconds);

var scopeProfiles = CreateScopeProfiles();
scopeProfiles.Profiles.Add(profileContainer);
scopeProfiles.Profiles.Add(extendedPprofBuilder.Profile);

var resourceProfiles = CreateResourceProfiles(scopeProfiles);

Expand Down Expand Up @@ -81,20 +78,18 @@ public void ExportAllocationSamples(byte[] buffer, int read, CancellationToken c
var scopeProfiles = CreateScopeProfiles();

var lastTimestamp = allocationSamples[0].ThreadSample.TimestampNanoseconds;
var extendedPprofBuilder = new ExtendedPprofBuilder();
var profileContainer = CreateProfileContainer(extendedPprofBuilder.Profile, "allocation", lastTimestamp);
scopeProfiles.Profiles.Add(profileContainer);
var extendedPprofBuilder = new ExtendedPprofBuilder("allocation", lastTimestamp);
scopeProfiles.Profiles.Add(extendedPprofBuilder.Profile);

for (var i = 0; i < allocationSamples.Count; i++)
{
var allocationSample = allocationSamples[i];
if (allocationSample.ThreadSample.TimestampNanoseconds != lastTimestamp)
{
// TODO consider either putting each sample in separate profile or in one profile with min and max timestamp
extendedPprofBuilder = new ExtendedPprofBuilder();
lastTimestamp = allocationSample.ThreadSample.TimestampNanoseconds;
profileContainer = CreateProfileContainer(extendedPprofBuilder.Profile, "allocation", lastTimestamp);
scopeProfiles.Profiles.Add(profileContainer);
extendedPprofBuilder = new ExtendedPprofBuilder("allocation", lastTimestamp);
scopeProfiles.Profiles.Add(extendedPprofBuilder.Profile);
}

var sampleBuilder = CreateSampleBuilder(allocationSample.ThreadSample, extendedPprofBuilder);
Expand Down Expand Up @@ -133,7 +128,12 @@ private static SampleBuilder CreateSampleBuilder(ThreadSample threadSample, Exte
for (var index = 0; index < threadSample.Frames.Count; index++)
{
var methodName = threadSample.Frames[index];
sampleBuilder.AddLocationId(extendedPprofBuilder.GetLocationId(methodName));
var locationId = extendedPprofBuilder.AddLocationId(methodName);

if (index == 0)
{
sampleBuilder.SetLocationRange(locationId, threadSample.Frames.Count);
}
}

if (!string.IsNullOrEmpty(threadSample.ThreadName))
Expand Down Expand Up @@ -182,24 +182,6 @@ private static ScopeProfiles CreateScopeProfiles()
return scopeProfiles;
}

private ProfileContainer CreateProfileContainer(Profile profile, string profilingDataType, ulong timestampNanoseconds)
{
var profileByteId = new byte[16];
ActivityTraceId.CreateRandom().CopyTo(profileByteId);

var profileContainer = new ProfileContainer
{
Profile = profile,
ProfileId = UnsafeByteOperations.UnsafeWrap(profileByteId), // ProfileId should be same as TraceId - 16 bytes
StartTimeUnixNano = timestampNanoseconds,
EndTimeUnixNano = timestampNanoseconds
};

profileContainer.Attributes.Add(new KeyValue { Key = "todo.profiling.data.type", Value = new AnyValue { StringValue = profilingDataType } });

return profileContainer;
}

private HttpResponseMessage SendHttpRequest(HttpRequestMessage request, CancellationToken cancellationToken)
{
return _httpClient.Send(request, cancellationToken);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,11 @@ namespace TestApplication.ContinuousProfiler;
internal class SampleBuilder
{
private readonly Sample _sample = new();
private readonly IList<ulong> _locationIds = new List<ulong>();
private long? _value;

public SampleBuilder AddAttribute(ulong attributeId)
public SampleBuilder AddAttribute(int attributeId)
{
_sample.Attributes.Add(attributeId);
_sample.AttributeIndices.Add(attributeId);
return this;
}

Expand All @@ -23,22 +22,21 @@ public SampleBuilder SetValue(long val)
return this;
}

public SampleBuilder AddLocationId(ulong locationId)
public SampleBuilder SetLocationRange(int locationsStartIndex, int locationsLength)
{
_locationIds.Add(locationId);
_sample.LocationsStartIndex = locationsStartIndex;
_sample.LocationsLength = locationsLength;
return this;
}

public SampleBuilder SetLink(ulong linkId)
public SampleBuilder SetLink(int linkId)
{
_sample.Link = linkId;
_sample.LinkIndex = linkId;
return this;
}

public Sample Build()
{
_sample.LocationIndex.AddRange(_locationIds);

if (_value.HasValue)
{
_sample.Value.Add(_value.Value);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,15 +7,15 @@ internal class ThreadSample
{
public ThreadSample(long timestampMilliseconds, long traceIdHigh, long traceIdLow, long spanId, string? threadName, uint threadIndex = default)
{
TimestampNanoseconds = (ulong)timestampMilliseconds * 1_000_000u;
TimestampNanoseconds = timestampMilliseconds * 1_000_000;
TraceIdHigh = traceIdHigh;
TraceIdLow = traceIdLow;
SpanId = spanId;
ThreadName = threadName;
ThreadIndex = threadIndex;
}

public ulong TimestampNanoseconds { get; }
public long TimestampNanoseconds { get; }

public long SpanId { get; }

Expand Down

0 comments on commit 3c07ffa

Please sign in to comment.