From 849dd58229b86db575da3b717502060bb389f875 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Kie=C5=82kowicz?= Date: Tue, 7 Jan 2025 02:31:17 +0100 Subject: [PATCH] Bump OTLP to 1.5.0 (#3912) * Bump OTLP proto files to 1.5.0 * v1experimental -> v1development * Adjust exporter to OTLP proto 1.5.0 --- .../ContinuousProfilerTests.cs | 16 +- .../Helpers/MockProfilesCollector.cs | 4 +- test/IntegrationTests/SmokeTests.cs | 2 - .../profiles_service.proto | 12 +- .../profiles_service_http.yaml | 4 +- .../opentelemetry/proto/logs/v1/logs.proto | 20 +- .../proto/metrics/v1/metrics.proto | 39 +- .../profiles.proto} | 343 +++++++++++------- .../profiles/v1experimental/profiles.proto | 191 ---------- .../opentelemetry/proto/trace/v1/trace.proto | 6 +- .../Exporter/ExtendedPprofBuilder.cs | 63 ++-- .../Exporter/OtlpOverHttpExporter.cs | 48 +-- .../Exporter/SampleBuilder.cs | 18 +- .../Exporter/ThreadSample.cs | 4 +- .../TestApplication.ContinuousProfiler.csproj | 5 +- 15 files changed, 347 insertions(+), 428 deletions(-) rename test/IntegrationTests/opentelemetry/proto/collector/profiles/{v1experimental => v1development}/profiles_service.proto (91%) rename test/IntegrationTests/opentelemetry/proto/collector/profiles/{v1experimental => v1development}/profiles_service_http.yaml (63%) rename test/IntegrationTests/opentelemetry/proto/profiles/{v1experimental/pprofextended.proto => v1development/profiles.proto} (55%) delete mode 100644 test/IntegrationTests/opentelemetry/proto/profiles/v1experimental/profiles.proto diff --git a/test/IntegrationTests/ContinuousProfilerTests.cs b/test/IntegrationTests/ContinuousProfilerTests.cs index 04a2df6664..68d05d4cab 100644 --- a/test/IntegrationTests/ContinuousProfilerTests.cs +++ b/test/IntegrationTests/ContinuousProfilerTests.cs @@ -4,7 +4,7 @@ #if NET using IntegrationTests.Helpers; -using OpenTelemetry.Proto.Profiles.V1Experimental; +using OpenTelemetry.Proto.Profiles.V1Development; using Xunit.Abstractions; namespace IntegrationTests; @@ -27,7 +27,7 @@ public void ExportAllocationSamples() SetEnvironmentVariable("OTEL_DOTNET_AUTO_TRACES_ADDITIONAL_SOURCES", "TestApplication.ContinuousProfiler"); RunTestApplication(); - collector.Expect(profileData => profileData.ResourceProfiles.Any(resourceProfiles => resourceProfiles.ScopeProfiles.Any(scopeProfile => scopeProfile.Profiles.Any(profileContainer => ContainAttributes(profileContainer, "allocation") && profileContainer.Profile.Sample[0].Value[0] != 0.0)))); + collector.Expect(profileData => profileData.ResourceProfiles.Any(resourceProfiles => resourceProfiles.ScopeProfiles.Any(scopeProfile => scopeProfile.Profiles.Any(profile => ContainAttributes(profile, "allocation") && profile.Sample[0].Value[0] != 0.0)))); collector.ResourceExpector.Expect("todo.resource.detector.key", "todo.resource.detector.value"); collector.AssertExpectations(); @@ -47,16 +47,16 @@ public void ExportThreadSamples() var expectedStackTrace = string.Join("\n", CreateExpectedStackTrace()); - collector.Expect(profileData => profileData.ResourceProfiles.Any(resourceProfiles => resourceProfiles.ScopeProfiles.Any(scopeProfile => scopeProfile.Profiles.Any(profileContainer => ContainStackTraceForClassHierarchy(profileContainer.Profile, expectedStackTrace) && ContainAttributes(profileContainer, "cpu"))))); + collector.Expect(profileData => profileData.ResourceProfiles.Any(resourceProfiles => resourceProfiles.ScopeProfiles.Any(scopeProfile => scopeProfile.Profiles.Any(profile => ContainStackTraceForClassHierarchy(profile, expectedStackTrace) && ContainAttributes(profile, "cpu"))))); collector.ResourceExpector.Expect("todo.resource.detector.key", "todo.resource.detector.value"); collector.AssertExpectations(); collector.ResourceExpector.AssertExpectations(); } - private static bool ContainAttributes(ProfileContainer profileContainer, string profilingDataType) + private static bool ContainAttributes(Profile profileContainer, string profilingDataType) { - return profileContainer.Attributes.Any(x => x.Key == "todo.profiling.data.type" && x.Value.StringValue == profilingDataType); + return profileContainer.AttributeTable.Any(x => x.Key == "todo.profiling.data.type" && x.Value.StringValue == profilingDataType); } private static List CreateExpectedStackTrace() @@ -96,11 +96,11 @@ private static List CreateExpectedStackTrace() private bool ContainStackTraceForClassHierarchy(Profile profile, string expectedStackTrace) { - var frames = profile.Location + var frames = profile.LocationTable .SelectMany(location => location.Line) .Select(line => line.FunctionIndex) - .Select(functionId => profile.Function[(int)functionId - 1]) - .Select(function => profile.StringTable[(int)function.Name]); + .Select(functionId => profile.FunctionTable[functionId - 1]) + .Select(function => profile.StringTable[function.NameStrindex]); var stackTrace = string.Join("\n", frames); diff --git a/test/IntegrationTests/Helpers/MockProfilesCollector.cs b/test/IntegrationTests/Helpers/MockProfilesCollector.cs index 7993927c77..aa0ae3c434 100644 --- a/test/IntegrationTests/Helpers/MockProfilesCollector.cs +++ b/test/IntegrationTests/Helpers/MockProfilesCollector.cs @@ -6,8 +6,8 @@ using System.Collections.Concurrent; using System.Text; using Microsoft.AspNetCore.Http; -using OpenTelemetry.Proto.Collector.Profiles.V1Experimental; -using OpenTelemetry.Proto.Profiles.V1Experimental; +using OpenTelemetry.Proto.Collector.Profiles.V1Development; +using OpenTelemetry.Proto.Profiles.V1Development; using Xunit.Abstractions; namespace IntegrationTests.Helpers; diff --git a/test/IntegrationTests/SmokeTests.cs b/test/IntegrationTests/SmokeTests.cs index 292fb09349..3b487e9398 100644 --- a/test/IntegrationTests/SmokeTests.cs +++ b/test/IntegrationTests/SmokeTests.cs @@ -4,9 +4,7 @@ using System.Reflection; using FluentAssertions; using IntegrationTests.Helpers; -using OpenTelemetry.Logs; using Xunit.Abstractions; -using LogRecord = OpenTelemetry.Proto.Logs.V1.LogRecord; #if NETFRAMEWORK using System.Net; diff --git a/test/IntegrationTests/opentelemetry/proto/collector/profiles/v1experimental/profiles_service.proto b/test/IntegrationTests/opentelemetry/proto/collector/profiles/v1development/profiles_service.proto similarity index 91% rename from test/IntegrationTests/opentelemetry/proto/collector/profiles/v1experimental/profiles_service.proto rename to test/IntegrationTests/opentelemetry/proto/collector/profiles/v1development/profiles_service.proto index d0e7894b29..ab2433ed29 100644 --- a/test/IntegrationTests/opentelemetry/proto/collector/profiles/v1experimental/profiles_service.proto +++ b/test/IntegrationTests/opentelemetry/proto/collector/profiles/v1development/profiles_service.proto @@ -14,15 +14,15 @@ syntax = "proto3"; -package opentelemetry.proto.collector.profiles.v1experimental; +package opentelemetry.proto.collector.profiles.v1development; -import "opentelemetry/proto/profiles/v1experimental/profiles.proto"; +import "opentelemetry/proto/profiles/v1development/profiles.proto"; -option csharp_namespace = "OpenTelemetry.Proto.Collector.Profiles.V1Experimental"; +option csharp_namespace = "OpenTelemetry.Proto.Collector.Profiles.V1Development"; option java_multiple_files = true; -option java_package = "io.opentelemetry.proto.collector.profiles.v1experimental"; +option java_package = "io.opentelemetry.proto.collector.profiles.v1development"; option java_outer_classname = "ProfilesServiceProto"; -option go_package = "go.opentelemetry.io/proto/otlp/collector/profiles/v1experimental"; +option go_package = "go.opentelemetry.io/proto/otlp/collector/profiles/v1development"; // Service that can be used to push profiles between one Application instrumented with // OpenTelemetry and a collector, or between a collector and a central collector. @@ -38,7 +38,7 @@ message ExportProfilesServiceRequest { // element. Intermediary nodes (such as OpenTelemetry Collector) that receive // data from multiple origins typically batch the data before forwarding further and // in that case this array will contain multiple elements. - repeated opentelemetry.proto.profiles.v1experimental.ResourceProfiles resource_profiles = 1; + repeated opentelemetry.proto.profiles.v1development.ResourceProfiles resource_profiles = 1; } message ExportProfilesServiceResponse { diff --git a/test/IntegrationTests/opentelemetry/proto/collector/profiles/v1experimental/profiles_service_http.yaml b/test/IntegrationTests/opentelemetry/proto/collector/profiles/v1development/profiles_service_http.yaml similarity index 63% rename from test/IntegrationTests/opentelemetry/proto/collector/profiles/v1experimental/profiles_service_http.yaml rename to test/IntegrationTests/opentelemetry/proto/collector/profiles/v1development/profiles_service_http.yaml index ea501afbbb..6b3b91da07 100644 --- a/test/IntegrationTests/opentelemetry/proto/collector/profiles/v1experimental/profiles_service_http.yaml +++ b/test/IntegrationTests/opentelemetry/proto/collector/profiles/v1development/profiles_service_http.yaml @@ -4,6 +4,6 @@ type: google.api.Service config_version: 3 http: rules: - - selector: opentelemetry.proto.collector.profiles.v1experimental.ProfilesService.Export - post: /v1experimental/profiles + - selector: opentelemetry.proto.collector.profiles.v1development.ProfilesService.Export + post: /v1development/profiles body: "*" diff --git a/test/IntegrationTests/opentelemetry/proto/logs/v1/logs.proto b/test/IntegrationTests/opentelemetry/proto/logs/v1/logs.proto index f9b97dd745..261d22916b 100644 --- a/test/IntegrationTests/opentelemetry/proto/logs/v1/logs.proto +++ b/test/IntegrationTests/opentelemetry/proto/logs/v1/logs.proto @@ -56,7 +56,8 @@ message ResourceLogs { repeated ScopeLogs scope_logs = 2; // The Schema URL, if known. This is the identifier of the Schema that the resource data - // is recorded in. To learn more about Schema URL see + // is recorded in. Notably, the last part of the URL path is the version number of the + // schema: http[s]://server[:port]/path/. To learn more about Schema URL see // https://opentelemetry.io/docs/specs/otel/schemas/#schema-url // This schema_url applies to the data in the "resource" field. It does not apply // to the data in the "scope_logs" field which have their own schema_url field. @@ -74,7 +75,8 @@ message ScopeLogs { repeated LogRecord log_records = 2; // The Schema URL, if known. This is the identifier of the Schema that the log data - // is recorded in. To learn more about Schema URL see + // is recorded in. Notably, the last part of the URL path is the version number of the + // schema: http[s]://server[:port]/path/. To learn more about Schema URL see // https://opentelemetry.io/docs/specs/otel/schemas/#schema-url // This schema_url applies to all logs in the "logs" field. string schema_url = 3; @@ -208,4 +210,18 @@ message LogRecord { // - the field is not present, // - the field contains an invalid value. bytes span_id = 10; + + // A unique identifier of event category/type. + // All events with the same event_name are expected to conform to the same + // schema for both their attributes and their body. + // + // Recommended to be fully qualified and short (no longer than 256 characters). + // + // Presence of event_name on the log record identifies this record + // as an event. + // + // [Optional]. + // + // Status: [Development] + string event_name = 12; } diff --git a/test/IntegrationTests/opentelemetry/proto/metrics/v1/metrics.proto b/test/IntegrationTests/opentelemetry/proto/metrics/v1/metrics.proto index 19bb7ff8d5..00c5112ce8 100644 --- a/test/IntegrationTests/opentelemetry/proto/metrics/v1/metrics.proto +++ b/test/IntegrationTests/opentelemetry/proto/metrics/v1/metrics.proto @@ -29,6 +29,24 @@ option go_package = "go.opentelemetry.io/proto/otlp/metrics/v1"; // storage, OR can be embedded by other protocols that transfer OTLP metrics // data but do not implement the OTLP protocol. // +// MetricsData +// └─── ResourceMetrics +// ├── Resource +// ├── SchemaURL +// └── ScopeMetrics +// ├── Scope +// ├── SchemaURL +// └── Metric +// ├── Name +// ├── Description +// ├── Unit +// └── data +// ├── Gauge +// ├── Sum +// ├── Histogram +// ├── ExponentialHistogram +// └── Summary +// // The main difference between this message and collector protocol is that // in this message there will not be any "control" or "metadata" specific to // OTLP protocol. @@ -56,7 +74,8 @@ message ResourceMetrics { repeated ScopeMetrics scope_metrics = 2; // The Schema URL, if known. This is the identifier of the Schema that the resource data - // is recorded in. To learn more about Schema URL see + // is recorded in. Notably, the last part of the URL path is the version number of the + // schema: http[s]://server[:port]/path/. To learn more about Schema URL see // https://opentelemetry.io/docs/specs/otel/schemas/#schema-url // This schema_url applies to the data in the "resource" field. It does not apply // to the data in the "scope_metrics" field which have their own schema_url field. @@ -74,7 +93,8 @@ message ScopeMetrics { repeated Metric metrics = 2; // The Schema URL, if known. This is the identifier of the Schema that the metric data - // is recorded in. To learn more about Schema URL see + // is recorded in. Notably, the last part of the URL path is the version number of the + // schema: http[s]://server[:port]/path/. To learn more about Schema URL see // https://opentelemetry.io/docs/specs/otel/schemas/#schema-url // This schema_url applies to all metrics in the "metrics" field. string schema_url = 3; @@ -85,7 +105,6 @@ message ScopeMetrics { // // https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/metrics/data-model.md // -// // The data model and relation between entities is shown in the // diagram below. Here, "DataPoint" is the term used to refer to any // one of the specific data point value types, and "points" is the term used @@ -97,7 +116,7 @@ message ScopeMetrics { // - DataPoint contains timestamps, attributes, and one of the possible value type // fields. // -// Metric +// Metric // +------------+ // |name | // |description | @@ -251,6 +270,9 @@ message ExponentialHistogram { // data type. These data points cannot always be merged in a meaningful way. // While they can be useful in some applications, histogram data points are // recommended for new applications. +// Summary metrics do not have an aggregation temporality field. This is +// because the count and sum fields of a SummaryDataPoint are assumed to be +// cumulative values. message Summary { repeated SummaryDataPoint data_points = 1; } @@ -430,7 +452,7 @@ message HistogramDataPoint { // events, and is assumed to be monotonic over the values of these events. // Negative events *can* be recorded, but sum should not be filled out when // doing so. This is specifically to enforce compatibility w/ OpenMetrics, - // see: https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#histogram + // see: https://github.com/prometheus/OpenMetrics/blob/v1.0.0/specification/OpenMetrics.md#histogram optional double sum = 5; // bucket_counts is an optional field contains the count values of histogram @@ -509,7 +531,7 @@ message ExponentialHistogramDataPoint { // events, and is assumed to be monotonic over the values of these events. // Negative events *can* be recorded, but sum should not be filled out when // doing so. This is specifically to enforce compatibility w/ OpenMetrics, - // see: https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#histogram + // see: https://github.com/prometheus/OpenMetrics/blob/v1.0.0/specification/OpenMetrics.md#histogram optional double sum = 5; // scale describes the resolution of the histogram. Boundaries are @@ -589,7 +611,8 @@ message ExponentialHistogramDataPoint { } // SummaryDataPoint is a single data point in a timeseries that describes the -// time-varying values of a Summary metric. +// time-varying values of a Summary metric. The count and sum fields represent +// cumulative values. message SummaryDataPoint { reserved 1; @@ -622,7 +645,7 @@ message SummaryDataPoint { // events, and is assumed to be monotonic over the values of these events. // Negative events *can* be recorded, but sum should not be filled out when // doing so. This is specifically to enforce compatibility w/ OpenMetrics, - // see: https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#summary + // see: https://github.com/prometheus/OpenMetrics/blob/v1.0.0/specification/OpenMetrics.md#summary double sum = 5; // Represents the value at a given quantile of a distribution. diff --git a/test/IntegrationTests/opentelemetry/proto/profiles/v1experimental/pprofextended.proto b/test/IntegrationTests/opentelemetry/proto/profiles/v1development/profiles.proto similarity index 55% rename from test/IntegrationTests/opentelemetry/proto/profiles/v1experimental/pprofextended.proto rename to test/IntegrationTests/opentelemetry/proto/profiles/v1development/profiles.proto index b5b5b88fce..1cb20b05c1 100644 --- a/test/IntegrationTests/opentelemetry/proto/profiles/v1experimental/pprofextended.proto +++ b/test/IntegrationTests/opentelemetry/proto/profiles/v1development/profiles.proto @@ -28,6 +28,126 @@ // See the License for the specific language governing permissions and // limitations under the License. +syntax = "proto3"; + +package opentelemetry.proto.profiles.v1development; + +import "opentelemetry/proto/common/v1/common.proto"; +import "opentelemetry/proto/resource/v1/resource.proto"; + +option csharp_namespace = "OpenTelemetry.Proto.Profiles.V1Development"; +option java_multiple_files = true; +option java_package = "io.opentelemetry.proto.profiles.v1development"; +option java_outer_classname = "ProfilesProto"; +option go_package = "go.opentelemetry.io/proto/otlp/profiles/v1development"; + +// Relationships Diagram +// +// ┌──────────────────┐ LEGEND +// │ ProfilesData │ +// └──────────────────┘ ─────▶ embedded +// │ +// │ 1-n ─────▷ referenced by index +// ▼ +// ┌──────────────────┐ +// │ ResourceProfiles │ +// └──────────────────┘ +// │ +// │ 1-n +// ▼ +// ┌──────────────────┐ +// │ ScopeProfiles │ +// └──────────────────┘ +// │ +// │ 1-1 +// ▼ +// ┌──────────────────┐ +// │ Profile │ +// └──────────────────┘ +// │ n-1 +// │ 1-n ┌───────────────────────────────────────┐ +// ▼ │ ▽ +// ┌──────────────────┐ 1-n ┌──────────────┐ ┌──────────┐ +// │ Sample │ ──────▷ │ KeyValue │ │ Link │ +// └──────────────────┘ └──────────────┘ └──────────┘ +// │ 1-n △ △ +// │ 1-n ┌─────────────────┘ │ 1-n +// ▽ │ │ +// ┌──────────────────┐ n-1 ┌──────────────┐ +// │ Location │ ──────▷ │ Mapping │ +// └──────────────────┘ └──────────────┘ +// │ +// │ 1-n +// ▼ +// ┌──────────────────┐ +// │ Line │ +// └──────────────────┘ +// │ +// │ 1-1 +// ▽ +// ┌──────────────────┐ +// │ Function │ +// └──────────────────┘ +// + +// ProfilesData represents the profiles data that can be stored in persistent storage, +// OR can be embedded by other protocols that transfer OTLP profiles data but do not +// implement the OTLP protocol. +// +// The main difference between this message and collector protocol is that +// in this message there will not be any "control" or "metadata" specific to +// OTLP protocol. +// +// When new fields are added into this message, the OTLP request MUST be updated +// as well. +message ProfilesData { + // An array of ResourceProfiles. + // For data coming from a single resource this array will typically contain + // one element. Intermediary nodes that receive data from multiple origins + // typically batch the data before forwarding further and in that case this + // array will contain multiple elements. + repeated ResourceProfiles resource_profiles = 1; +} + + +// A collection of ScopeProfiles from a Resource. +message ResourceProfiles { + reserved 1000; + + // The resource for the profiles in this message. + // If this field is not set then no resource info is known. + opentelemetry.proto.resource.v1.Resource resource = 1; + + // A list of ScopeProfiles that originate from a resource. + repeated ScopeProfiles scope_profiles = 2; + + // The Schema URL, if known. This is the identifier of the Schema that the resource data + // is recorded in. Notably, the last part of the URL path is the version number of the + // schema: http[s]://server[:port]/path/. To learn more about Schema URL see + // https://opentelemetry.io/docs/specs/otel/schemas/#schema-url + // This schema_url applies to the data in the "resource" field. It does not apply + // to the data in the "scope_profiles" field which have their own schema_url field. + string schema_url = 3; +} + +// A collection of Profiles produced by an InstrumentationScope. +message ScopeProfiles { + // The instrumentation scope information for the profiles in this message. + // Semantically when InstrumentationScope isn't set, it is equivalent with + // an empty instrumentation scope name (unknown). + opentelemetry.proto.common.v1.InstrumentationScope scope = 1; + + // A list of Profiles that originate from an instrumentation scope. + repeated Profile profiles = 2; + + // The Schema URL, if known. This is the identifier of the Schema that the profile data + // is recorded in. Notably, the last part of the URL path is the version number of the + // schema: http[s]://server[:port]/path/. To learn more about Schema URL see + // https://opentelemetry.io/docs/specs/otel/schemas/#schema-url + // This schema_url applies to all profiles in the "profiles" field. + string schema_url = 3; +} + // Profile is a common stacktrace profile format. // // Measurements represented with this format should follow the @@ -52,20 +172,15 @@ // mappings. For every nonzero Location.mapping_id there must be a // unique Mapping with that index. -syntax = "proto3"; - -package opentelemetry.proto.profiles.v1experimental; - -import "opentelemetry/proto/common/v1/common.proto"; - -option csharp_namespace = "OpenTelemetry.Proto.Profiles.V1Experimental"; -option java_multiple_files = true; -option java_package = "io.opentelemetry.proto.profiles.v1experimental"; -option go_package = "go.opentelemetry.io/proto/otlp/profiles/v1experimental"; - // Represents a complete profile, including sample types, samples, // mappings to binaries, locations, functions, string table, and additional metadata. +// It modifies and annotates pprof Profile with OpenTelemetry specific fields. +// +// Note that whilst fields in this message retain the name and field id from pprof in most cases +// for ease of understanding data migration, it is not intended that pprof:Profile and +// OpenTelemetry:Profile encoding be wire compatible. message Profile { + // A description of the samples associated with each Sample.value. // For a cpu profile this might be: // [["cpu","nanoseconds"]] or [["wall","seconds"]] or [["syscall","count"]] @@ -79,58 +194,92 @@ message Profile { repeated Sample sample = 2; // Mapping from address ranges to the image/binary/library mapped // into that address range. mapping[0] will be the main binary. - repeated Mapping mapping = 3; + // If multiple binaries contribute to the Profile and no main + // binary can be identified, mapping[0] has no special meaning. + repeated Mapping mapping_table = 3; // Locations referenced by samples via location_indices. - repeated Location location = 4; + repeated Location location_table = 4; // Array of locations referenced by samples. - repeated int64 location_indices = 15; + repeated int32 location_indices = 5; // Functions referenced by locations. - repeated Function function = 5; + repeated Function function_table = 6; // Lookup table for attributes. - repeated opentelemetry.proto.common.v1.KeyValue attribute_table = 16; + repeated opentelemetry.proto.common.v1.KeyValue attribute_table = 7; // Represents a mapping between Attribute Keys and Units. - repeated AttributeUnit attribute_units = 17; + repeated AttributeUnit attribute_units = 8; // Lookup table for links. - repeated Link link_table = 18; + repeated Link link_table = 9; // A common table for strings referenced by various messages. // string_table[0] must always be "". - repeated string string_table = 6; - // frames with Function.function_name fully matching the following - // regexp will be dropped from the samples, along with their successors. - int64 drop_frames = 7; // Index into string table. - // frames with Function.function_name fully matching the following - // regexp will be kept, even if it matches drop_frames. - int64 keep_frames = 8; // Index into string table. - - // The following fields are informational, do not affect + repeated string string_table = 10; + + // The following fields 9-14 are informational, do not affect // interpretation of results. // Time of collection (UTC) represented as nanoseconds past the epoch. - int64 time_nanos = 9; + int64 time_nanos = 11; // Duration of the profile, if a duration makes sense. - int64 duration_nanos = 10; + int64 duration_nanos = 12; // The kind of events between sampled occurrences. // e.g [ "cpu","cycles" ] or [ "heap","bytes" ] - ValueType period_type = 11; + ValueType period_type = 13; // The number of events between sampled occurrences. - int64 period = 12; + int64 period = 14; // Free-form text associated with the profile. The text is displayed as is // to the user by the tools that read profiles (e.g. by pprof). This field // should not be used to store any machine-readable information, it is only // for human-friendly content. The profile must stay functional if this field // is cleaned. - repeated int64 comment = 13; // Indices into string table. + repeated int32 comment_strindices = 15; // Indices into string table. // Index into the string table of the type of the preferred sample // value. If unset, clients should default to the last sample value. - int64 default_sample_type = 14; + int32 default_sample_type_strindex = 16; + + + // A globally unique identifier for a profile. The ID is a 16-byte array. An ID with + // all zeroes is considered invalid. + // + // This field is required. + bytes profile_id = 17; + + // dropped_attributes_count is the number of attributes that were discarded. Attributes + // can be discarded because their keys are too long or because there are too many + // attributes. If this value is 0, then no attributes were dropped. + uint32 dropped_attributes_count = 19; + + // Specifies format of the original payload. Common values are defined in semantic conventions. [required if original_payload is present] + string original_payload_format = 20; + + // Original payload can be stored in this field. This can be useful for users who want to get the original payload. + // Formats such as JFR are highly extensible and can contain more information than what is defined in this spec. + // Inclusion of original payload should be configurable by the user. Default behavior should be to not include the original payload. + // If the original payload is in pprof format, it SHOULD not be included in this field. + // The field is optional, however if it is present then equivalent converted data should be populated in other fields + // of this message as far as is practicable. + bytes original_payload = 21; + + // References to attributes in attribute_table. [optional] + // It is a collection of key/value pairs. Note, global attributes + // like server name can be set using the resource API. Examples of attributes: + // + // "/http/user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36" + // "/http/server_latency": 300 + // "abc.com/myattribute": true + // "abc.com/score": 10.239 + // + // The OpenTelemetry API specification further restricts the allowed value types: + // https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/common/README.md#attribute + // Attribute keys MUST be unique (it is not allowed to have more than one + // attribute with the same key). + repeated int32 attribute_indices = 22; } // Represents a mapping between Attribute Keys and Units. message AttributeUnit { // Index into string table. - int64 attribute_key = 1; + int32 attribute_key_strindex = 1; // Index into string table. - int64 unit = 2; + int32 unit_strindex = 2; } // A pointer from a profile Sample to a trace Span. @@ -214,8 +363,8 @@ enum AggregationTemporality { // ValueType describes the type and units of a value, with an optional aggregation temporality. message ValueType { - int64 type = 1; // Index into string table. - int64 unit = 2; // Index into string table. + int32 type_strindex = 1; // Index into string table. + int32 unit_strindex = 2; // Index into string table. AggregationTemporality aggregation_temporality = 3; } @@ -225,120 +374,63 @@ message ValueType { // augmented with auxiliary information like the thread-id, some // indicator of a higher level request being handled etc. message Sample { - // The indices recorded here correspond to locations in Profile.location. - // The leaf is at location_index[0]. [deprecated, superseded by locations_start_index / locations_length] - repeated uint64 location_index = 1; - // locations_start_index along with locations_length refers to to a slice of locations in Profile.location. - // Supersedes location_index. - uint64 locations_start_index = 7; - // locations_length along with locations_start_index refers to a slice of locations in Profile.location. + // locations_start_index along with locations_length refers to to a slice of locations in Profile.location_indices. + int32 locations_start_index = 1; + // locations_length along with locations_start_index refers to a slice of locations in Profile.location_indices. // Supersedes location_index. - uint64 locations_length = 8; - // A 128bit id that uniquely identifies this stacktrace, globally. Index into string table. [optional] - uint32 stacktrace_id_index = 9; + int32 locations_length = 2; // The type and unit of each value is defined by the corresponding // entry in Profile.sample_type. All samples must have the same // number of values, the same as the length of Profile.sample_type. // When aggregating multiple samples into a single sample, the // result has a list of values that is the element-wise sum of the // lists of the originals. - repeated int64 value = 2; - // label includes additional context for this sample. It can include - // things like a thread id, allocation size, etc. - // - // NOTE: While possible, having multiple values for the same label key is - // strongly discouraged and should never be used. Most tools (e.g. pprof) do - // not have good (or any) support for multi-value labels. And an even more - // discouraged case is having a string label and a numeric label of the same - // name on a sample. Again, possible to express, but should not be used. - // [deprecated, superseded by attributes] - repeated Label label = 3; + repeated int64 value = 3; // References to attributes in Profile.attribute_table. [optional] - repeated uint64 attributes = 10; + repeated int32 attribute_indices = 4; // Reference to link in Profile.link_table. [optional] - uint64 link = 12; + optional int32 link_index = 5; // Timestamps associated with Sample represented in nanoseconds. These timestamps are expected // to fall within the Profile's time range. [optional] - repeated uint64 timestamps_unix_nano = 13; -} - -// Provides additional context for a sample, -// such as thread ID or allocation size, with optional units. [deprecated] -message Label { - int64 key = 1; // Index into string table - - // At most one of the following must be present - int64 str = 2; // Index into string table - int64 num = 3; - - // Should only be present when num is present. - // Specifies the units of num. - // Use arbitrary string (for example, "requests") as a custom count unit. - // If no unit is specified, consumer may apply heuristic to deduce the unit. - // Consumers may also interpret units like "bytes" and "kilobytes" as memory - // units and units like "seconds" and "nanoseconds" as time units, - // and apply appropriate unit conversions to these. - int64 num_unit = 4; // Index into string table -} - -// Indicates the semantics of the build_id field. -enum BuildIdKind { - // Linker-generated build ID, stored in the ELF binary notes. - BUILD_ID_LINKER = 0; - // Build ID based on the content hash of the binary. Currently no particular - // hashing approach is standardized, so a given producer needs to define it - // themselves and thus unlike BUILD_ID_LINKER this kind of hash is producer-specific. - // We may choose to provide a standardized stable hash recommendation later. - BUILD_ID_BINARY_HASH = 1; + repeated uint64 timestamps_unix_nano = 6; } // Describes the mapping of a binary in memory, including its address range, // file offset, and metadata like build ID message Mapping { - // Unique nonzero id for the mapping. [deprecated] - uint64 id = 1; // Address at which the binary (or DLL) is loaded into memory. - uint64 memory_start = 2; + uint64 memory_start = 1; // The limit of the address range occupied by this mapping. - uint64 memory_limit = 3; + uint64 memory_limit = 2; // Offset in the binary that corresponds to the first mapped address. - uint64 file_offset = 4; + uint64 file_offset = 3; // The object this entry is loaded from. This can be a filename on // disk for the main binary and shared libraries, or virtual // abstractions like "[vdso]". - int64 filename = 5; // Index into string table - // A string that uniquely identifies a particular program version - // with high probability. E.g., for binaries generated by GNU tools, - // it could be the contents of the .note.gnu.build-id field. - int64 build_id = 6; // Index into string table - // Specifies the kind of build id. See BuildIdKind enum for more details [optional] - BuildIdKind build_id_kind = 11; + int32 filename_strindex = 4; // Index into string table // References to attributes in Profile.attribute_table. [optional] - repeated uint64 attributes = 12; + repeated int32 attribute_indices = 5; // The following fields indicate the resolution of symbolic info. - bool has_functions = 7; - bool has_filenames = 8; - bool has_line_numbers = 9; - bool has_inline_frames = 10; + bool has_functions = 6; + bool has_filenames = 7; + bool has_line_numbers = 8; + bool has_inline_frames = 9; } // Describes function and line table debug information. message Location { - // Unique nonzero id for the location. A profile could use - // instruction addresses or any integer sequence as ids. [deprecated] - uint64 id = 1; - // The index of the corresponding profile.Mapping for this location. + // Reference to mapping in Profile.mapping_table. // It can be unset if the mapping is unknown or not applicable for // this profile type. - uint64 mapping_index = 2; + optional int32 mapping_index = 1; // The instruction address for this location, if available. It // should be within [Mapping.memory_start...Mapping.memory_limit] // for the corresponding mapping. A non-leaf address may be in the // middle of a call instruction. It is up to display tools to find // the beginning of the instruction if necessary. - uint64 address = 3; + uint64 address = 2; // Multiple line indicates this location has inlined functions, // where the last entry represents the caller into which the // preceding entries were inlined. @@ -346,25 +438,22 @@ message Location { // E.g., if memcpy() is inlined into printf: // line[0].function_name == "memcpy" // line[1].function_name == "printf" - repeated Line line = 4; + repeated Line line = 3; // Provides an indication that multiple symbols map to this location's // address, for example due to identical code folding by the linker. In that // case the line information above represents one of the multiple // symbols. This field must be recomputed when the symbolization state of the // profile changes. - bool is_folded = 5; - - // Type of frame (e.g. kernel, native, python, hotspot, php). Index into string table. - uint32 type_index = 6; + bool is_folded = 4; // References to attributes in Profile.attribute_table. [optional] - repeated uint64 attributes = 7; + repeated int32 attribute_indices = 5; } // Details a specific line in a source code, linked to a function. message Line { - // The index of the corresponding profile.Function for this line. - uint64 function_index = 1; + // Reference to function in Profile.function_table. + int32 function_index = 1; // Line number in source code. int64 line = 2; // Column number in source code. @@ -374,15 +463,13 @@ message Line { // Describes a function, including its human-readable name, system name, // source file, and starting line number in the source. message Function { - // Unique nonzero id for the function. [deprecated] - uint64 id = 1; // Name of the function, in human-readable form if available. - int64 name = 2; // Index into string table + int32 name_strindex = 1; // Index into string table // Name of the function, as identified by the system. // For instance, it can be a C++ mangled name. - int64 system_name = 3; // Index into string table + int32 system_name_strindex = 2; // Index into string table // Source file containing the function. - int64 filename = 4; // Index into string table + int32 filename_strindex = 3; // Index into string table // Line number in source file. - int64 start_line = 5; + int64 start_line = 4; } diff --git a/test/IntegrationTests/opentelemetry/proto/profiles/v1experimental/profiles.proto b/test/IntegrationTests/opentelemetry/proto/profiles/v1experimental/profiles.proto deleted file mode 100644 index 84b0108f86..0000000000 --- a/test/IntegrationTests/opentelemetry/proto/profiles/v1experimental/profiles.proto +++ /dev/null @@ -1,191 +0,0 @@ -// Copyright 2023, OpenTelemetry Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -package opentelemetry.proto.profiles.v1experimental; - -import "opentelemetry/proto/common/v1/common.proto"; -import "opentelemetry/proto/resource/v1/resource.proto"; -import "opentelemetry/proto/profiles/v1experimental/pprofextended.proto"; - -option csharp_namespace = "OpenTelemetry.Proto.Profiles.V1Experimental"; -option java_multiple_files = true; -option java_package = "io.opentelemetry.proto.profiles.v1experimental"; -option java_outer_classname = "ProfilesProto"; -option go_package = "go.opentelemetry.io/proto/otlp/profiles/v1experimental"; - -// Relationships Diagram -// -// ┌──────────────────┐ LEGEND -// │ ProfilesData │ -// └──────────────────┘ ─────▶ embedded -// │ -// │ 1-n ─────▷ referenced by index -// ▼ -// ┌──────────────────┐ -// │ ResourceProfiles │ -// └──────────────────┘ -// │ -// │ 1-n -// ▼ -// ┌──────────────────┐ -// │ ScopeProfiles │ -// └──────────────────┘ -// │ -// │ 1-n -// ▼ -// ┌──────────────────┐ -// │ ProfileContainer │ -// └──────────────────┘ -// │ -// │ 1-1 -// ▼ -// ┌──────────────────┐ -// │ Profile │ -// └──────────────────┘ -// │ n-1 -// │ 1-n ┌───────────────────────────────────────┐ -// ▼ │ ▽ -// ┌──────────────────┐ 1-n ┌──────────────┐ ┌──────────┐ -// │ Sample │ ──────▷ │ KeyValue │ │ Link │ -// └──────────────────┘ └──────────────┘ └──────────┘ -// │ 1-n △ △ -// │ 1-n ┌─────────────────┘ │ 1-n -// ▽ │ │ -// ┌──────────────────┐ n-1 ┌──────────────┐ -// │ Location │ ──────▷ │ Mapping │ -// └──────────────────┘ └──────────────┘ -// │ -// │ 1-n -// ▼ -// ┌──────────────────┐ -// │ Line │ -// └──────────────────┘ -// │ -// │ 1-1 -// ▽ -// ┌──────────────────┐ -// │ Function │ -// └──────────────────┘ -// - -// ProfilesData represents the profiles data that can be stored in persistent storage, -// OR can be embedded by other protocols that transfer OTLP profiles data but do not -// implement the OTLP protocol. -// -// The main difference between this message and collector protocol is that -// in this message there will not be any "control" or "metadata" specific to -// OTLP protocol. -// -// When new fields are added into this message, the OTLP request MUST be updated -// as well. -message ProfilesData { - // An array of ResourceProfiles. - // For data coming from a single resource this array will typically contain - // one element. Intermediary nodes that receive data from multiple origins - // typically batch the data before forwarding further and in that case this - // array will contain multiple elements. - repeated ResourceProfiles resource_profiles = 1; -} - - -// A collection of ScopeProfiles from a Resource. -message ResourceProfiles { - reserved 1000; - - // The resource for the profiles in this message. - // If this field is not set then no resource info is known. - opentelemetry.proto.resource.v1.Resource resource = 1; - - // A list of ScopeProfiles that originate from a resource. - repeated ScopeProfiles scope_profiles = 2; - - // The Schema URL, if known. This is the identifier of the Schema that the resource data - // is recorded in. To learn more about Schema URL see - // https://opentelemetry.io/docs/specs/otel/schemas/#schema-url - // This schema_url applies to the data in the "resource" field. It does not apply - // to the data in the "scope_profiles" field which have their own schema_url field. - string schema_url = 3; -} - -// A collection of ProfileContainers produced by an InstrumentationScope. -message ScopeProfiles { - // The instrumentation scope information for the profiles in this message. - // Semantically when InstrumentationScope isn't set, it is equivalent with - // an empty instrumentation scope name (unknown). - opentelemetry.proto.common.v1.InstrumentationScope scope = 1; - - // A list of ProfileContainers that originate from an instrumentation scope. - repeated ProfileContainer profiles = 2; - - // The Schema URL, if known. This is the identifier of the Schema that the metric data - // is recorded in. To learn more about Schema URL see - // https://opentelemetry.io/docs/specs/otel/schemas/#schema-url - // This schema_url applies to all profiles in the "profiles" field. - string schema_url = 3; -} - -// A ProfileContainer represents a single profile. It wraps pprof profile with OpenTelemetry specific metadata. -message ProfileContainer { - // A globally unique identifier for a profile. The ID is a 16-byte array. An ID with - // all zeroes is considered invalid. - // - // This field is required. - bytes profile_id = 1; - - // start_time_unix_nano is the start time of the profile. - // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January 1970. - // - // This field is semantically required and it is expected that end_time >= start_time. - fixed64 start_time_unix_nano = 2; - - // end_time_unix_nano is the end time of the profile. - // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January 1970. - // - // This field is semantically required and it is expected that end_time >= start_time. - fixed64 end_time_unix_nano = 3; - - // attributes is a collection of key/value pairs. Note, global attributes - // like server name can be set using the resource API. Examples of attributes: - // - // "/http/user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36" - // "/http/server_latency": 300 - // "abc.com/myattribute": true - // "abc.com/score": 10.239 - // - // The OpenTelemetry API specification further restricts the allowed value types: - // https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/common/README.md#attribute - // Attribute keys MUST be unique (it is not allowed to have more than one - // attribute with the same key). - repeated opentelemetry.proto.common.v1.KeyValue attributes = 4; - - // dropped_attributes_count is the number of attributes that were discarded. Attributes - // can be discarded because their keys are too long or because there are too many - // attributes. If this value is 0, then no attributes were dropped. - uint32 dropped_attributes_count = 5; - - // Specifies format of the original payload. Common values are defined in semantic conventions. [required if original_payload is present] - string original_payload_format = 6; - - // Original payload can be stored in this field. This can be useful for users who want to get the original payload. - // Formats such as JFR are highly extensible and can contain more information than what is defined in this spec. - // Inclusion of original payload should be configurable by the user. Default behavior should be to not include the original payload. - // If the original payload is in pprof format, it SHOULD not be included in this field. - // The field is optional, however if it is present `profile` MUST be present and contain the same profiling information. - bytes original_payload = 7; - - // This is a reference to a pprof profile. Required, even when original_payload is present. - opentelemetry.proto.profiles.v1experimental.Profile profile = 8; -} diff --git a/test/IntegrationTests/opentelemetry/proto/trace/v1/trace.proto b/test/IntegrationTests/opentelemetry/proto/trace/v1/trace.proto index 5cb2f3ce1c..24442853ed 100644 --- a/test/IntegrationTests/opentelemetry/proto/trace/v1/trace.proto +++ b/test/IntegrationTests/opentelemetry/proto/trace/v1/trace.proto @@ -56,7 +56,8 @@ message ResourceSpans { repeated ScopeSpans scope_spans = 2; // The Schema URL, if known. This is the identifier of the Schema that the resource data - // is recorded in. To learn more about Schema URL see + // is recorded in. Notably, the last part of the URL path is the version number of the + // schema: http[s]://server[:port]/path/. To learn more about Schema URL see // https://opentelemetry.io/docs/specs/otel/schemas/#schema-url // This schema_url applies to the data in the "resource" field. It does not apply // to the data in the "scope_spans" field which have their own schema_url field. @@ -74,7 +75,8 @@ message ScopeSpans { repeated Span spans = 2; // The Schema URL, if known. This is the identifier of the Schema that the span data - // is recorded in. To learn more about Schema URL see + // is recorded in. Notably, the last part of the URL path is the version number of the + // schema: http[s]://server[:port]/path/. To learn more about Schema URL see // https://opentelemetry.io/docs/specs/otel/schemas/#schema-url // This schema_url applies to all spans and span events in the "spans" field. string schema_url = 3; diff --git a/test/test-applications/integrations/TestApplication.ContinuousProfiler/Exporter/ExtendedPprofBuilder.cs b/test/test-applications/integrations/TestApplication.ContinuousProfiler/Exporter/ExtendedPprofBuilder.cs index 2a3e182484..3f82cd9bb7 100644 --- a/test/test-applications/integrations/TestApplication.ContinuousProfiler/Exporter/ExtendedPprofBuilder.cs +++ b/test/test-applications/integrations/TestApplication.ContinuousProfiler/Exporter/ExtendedPprofBuilder.cs @@ -1,9 +1,10 @@ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 +using System.Diagnostics; using Google.Protobuf; using OpenTelemetry.Proto.Common.V1; -using OpenTelemetry.Proto.Profiles.V1Experimental; +using OpenTelemetry.Proto.Profiles.V1Development; namespace TestApplication.ContinuousProfiler; @@ -13,19 +14,28 @@ internal class ExtendedPprofBuilder private readonly LinkCache _linkCache; private readonly AttributeCache _attributeCache; - public ExtendedPprofBuilder() + public ExtendedPprofBuilder(string profilingDataType, long timestampNanoseconds) { - Profile = new Profile(); + var profileByteId = new byte[16]; + ActivityTraceId.CreateRandom().CopyTo(profileByteId); + Profile = new Profile + { + ProfileId = ByteString.CopyFrom(profileByteId), + TimeNanos = timestampNanoseconds, + }; var stringCache = new StringCache(Profile); var functionCache = new FunctionCache(Profile, stringCache); _locationCache = new LocationCache(Profile, functionCache); _linkCache = new LinkCache(Profile); _attributeCache = new AttributeCache(Profile); + + var profilingDataTypeAttributeId = _attributeCache.GetOrAdd("todo.profiling.data.type", value => value.StringValue = profilingDataType); + Profile.AttributeIndices.Add(profilingDataTypeAttributeId); } public Profile Profile { get; } - public ulong GetLocationId(string function) => _locationCache.Get(function); + public int AddLocationId(string function) => _locationCache.Add(function); public void AddLink(SampleBuilder sampleBuilder, long spanId, long traceIdHigh, long traceIdLow) { @@ -53,8 +63,8 @@ private void AddAttribute(SampleBuilder sampleBuilder, string name, Action _table = new(); - private long _index; + private readonly Dictionary _table = new(); + private int _index; public StringCache(Profile profile) { @@ -62,7 +72,7 @@ public StringCache(Profile profile) GetOrAdd(string.Empty); // 0 is reserved for the empty string } - public long GetOrAdd(string str) + public int GetOrAdd(string str) { if (_table.TryGetValue(str, out var value)) { @@ -79,8 +89,8 @@ private class FunctionCache { private readonly Profile _profile; private readonly StringCache _stringCache; - private readonly Dictionary _table = new(); - private ulong _index = 1; // 0 is reserved + private readonly Dictionary _table = new(); + private int _index = 1; // 0 is reserved public FunctionCache(Profile profile, StringCache stringCache) { @@ -88,16 +98,18 @@ public FunctionCache(Profile profile, StringCache stringCache) _stringCache = stringCache; } - public ulong GetOrAdd(string functionName) + public int GetOrAdd(string functionName) { if (_table.TryGetValue(functionName, out var value)) { return value; } - var function = new Function { Id = _index, Filename = _stringCache.GetOrAdd("unknown"), Name = _stringCache.GetOrAdd(functionName) }; // for now, we don't support file name + // TODO How to handle SystemName in .NET + // TODO handle line number + var function = new Function { SystemNameStrindex = _stringCache.GetOrAdd("TODO How to handle SystemName in .NET?"), FilenameStrindex = _stringCache.GetOrAdd("unknown"), NameStrindex = _stringCache.GetOrAdd(functionName), }; // for now, we don't support file name - _profile.Function.Add(function); + _profile.FunctionTable.Add(function); _table[functionName] = _index; return _index++; } @@ -107,8 +119,7 @@ private class LocationCache { private readonly Profile _profile; private readonly FunctionCache _functionCache; - private readonly Dictionary _table = new(); - private ulong _index = 1; // 0 is reserved + private int _index = 1; // 0 is reserved public LocationCache(Profile profile, FunctionCache functionCache) { @@ -116,18 +127,12 @@ public LocationCache(Profile profile, FunctionCache functionCache) _functionCache = functionCache; } - public ulong Get(string function) + public int Add(string function) { - if (_table.TryGetValue(function, out var value)) - { - return value; - } - - var location = new Location { Id = _index }; + var location = new Location(); location.Line.Add(new Line { FunctionIndex = _functionCache.GetOrAdd(function), Line_ = 0, Column = 0 }); // for now, we don't support line nor column number - _profile.Location.Add(location); - _table[function] = _index; + _profile.LocationTable.Add(location); return _index++; } @@ -136,15 +141,15 @@ public ulong Get(string function) private class LinkCache { private readonly Profile _profile; - private readonly Dictionary, ulong> _table = new(); - private ulong _index = 1; // 0 is reserved + private readonly Dictionary, int> _table = new(); + private int _index = 1; // 0 is reserved public LinkCache(Profile profile) { _profile = profile; } - public ulong GetOrAdd(long spanId, long traceIdHigh, long traceIdLow) + public int GetOrAdd(long spanId, long traceIdHigh, long traceIdLow) { var key = Tuple.Create(spanId, traceIdHigh, traceIdLow); @@ -177,15 +182,15 @@ public ulong GetOrAdd(long spanId, long traceIdHigh, long traceIdLow) private class AttributeCache { private readonly Profile _profile; - private readonly Dictionary _table = new(); - private ulong _index = 1; // 0 is reserved + private readonly Dictionary _table = new(); + private int _index = 1; // 0 is reserved public AttributeCache(Profile profile) { _profile = profile; } - public ulong GetOrAdd(string name, Action setValue) + public int GetOrAdd(string name, Action setValue) { var keyValue = new KeyValue { diff --git a/test/test-applications/integrations/TestApplication.ContinuousProfiler/Exporter/OtlpOverHttpExporter.cs b/test/test-applications/integrations/TestApplication.ContinuousProfiler/Exporter/OtlpOverHttpExporter.cs index e94b54507e..1709f0f3aa 100644 --- a/test/test-applications/integrations/TestApplication.ContinuousProfiler/Exporter/OtlpOverHttpExporter.cs +++ b/test/test-applications/integrations/TestApplication.ContinuousProfiler/Exporter/OtlpOverHttpExporter.cs @@ -7,9 +7,9 @@ using System.Net.Http.Headers; using System.Runtime.CompilerServices; using Google.Protobuf; -using OpenTelemetry.Proto.Collector.Profiles.V1Experimental; +using OpenTelemetry.Proto.Collector.Profiles.V1Development; using OpenTelemetry.Proto.Common.V1; -using OpenTelemetry.Proto.Profiles.V1Experimental; +using OpenTelemetry.Proto.Profiles.V1Development; using OpenTelemetry.Proto.Resource.V1; namespace TestApplication.ContinuousProfiler; @@ -32,7 +32,8 @@ public void ExportThreadSamples(byte[] buffer, int read, CancellationToken cance try { - var extendedPprofBuilder = new ExtendedPprofBuilder(); + var timestampNanoseconds = threadSamples[0].TimestampNanoseconds; // all items in the batch have same timestamp + var extendedPprofBuilder = new ExtendedPprofBuilder("cpu", timestampNanoseconds); for (var i = 0; i < threadSamples.Count; i++) { @@ -43,12 +44,8 @@ public void ExportThreadSamples(byte[] buffer, int read, CancellationToken cance extendedPprofBuilder.Profile.Sample.Add(sampleBuilder.Build()); } - var timestampNanoseconds = threadSamples[0].TimestampNanoseconds; // all items in the batch have same timestamp - - var profileContainer = CreateProfileContainer(extendedPprofBuilder.Profile, "cpu", timestampNanoseconds); - var scopeProfiles = CreateScopeProfiles(); - scopeProfiles.Profiles.Add(profileContainer); + scopeProfiles.Profiles.Add(extendedPprofBuilder.Profile); var resourceProfiles = CreateResourceProfiles(scopeProfiles); @@ -81,9 +78,8 @@ public void ExportAllocationSamples(byte[] buffer, int read, CancellationToken c var scopeProfiles = CreateScopeProfiles(); var lastTimestamp = allocationSamples[0].ThreadSample.TimestampNanoseconds; - var extendedPprofBuilder = new ExtendedPprofBuilder(); - var profileContainer = CreateProfileContainer(extendedPprofBuilder.Profile, "allocation", lastTimestamp); - scopeProfiles.Profiles.Add(profileContainer); + var extendedPprofBuilder = new ExtendedPprofBuilder("allocation", lastTimestamp); + scopeProfiles.Profiles.Add(extendedPprofBuilder.Profile); for (var i = 0; i < allocationSamples.Count; i++) { @@ -91,10 +87,9 @@ public void ExportAllocationSamples(byte[] buffer, int read, CancellationToken c if (allocationSample.ThreadSample.TimestampNanoseconds != lastTimestamp) { // TODO consider either putting each sample in separate profile or in one profile with min and max timestamp - extendedPprofBuilder = new ExtendedPprofBuilder(); lastTimestamp = allocationSample.ThreadSample.TimestampNanoseconds; - profileContainer = CreateProfileContainer(extendedPprofBuilder.Profile, "allocation", lastTimestamp); - scopeProfiles.Profiles.Add(profileContainer); + extendedPprofBuilder = new ExtendedPprofBuilder("allocation", lastTimestamp); + scopeProfiles.Profiles.Add(extendedPprofBuilder.Profile); } var sampleBuilder = CreateSampleBuilder(allocationSample.ThreadSample, extendedPprofBuilder); @@ -133,7 +128,12 @@ private static SampleBuilder CreateSampleBuilder(ThreadSample threadSample, Exte for (var index = 0; index < threadSample.Frames.Count; index++) { var methodName = threadSample.Frames[index]; - sampleBuilder.AddLocationId(extendedPprofBuilder.GetLocationId(methodName)); + var locationId = extendedPprofBuilder.AddLocationId(methodName); + + if (index == 0) + { + sampleBuilder.SetLocationRange(locationId, threadSample.Frames.Count); + } } if (!string.IsNullOrEmpty(threadSample.ThreadName)) @@ -182,24 +182,6 @@ private static ScopeProfiles CreateScopeProfiles() return scopeProfiles; } - private ProfileContainer CreateProfileContainer(Profile profile, string profilingDataType, ulong timestampNanoseconds) - { - var profileByteId = new byte[16]; - ActivityTraceId.CreateRandom().CopyTo(profileByteId); - - var profileContainer = new ProfileContainer - { - Profile = profile, - ProfileId = UnsafeByteOperations.UnsafeWrap(profileByteId), // ProfileId should be same as TraceId - 16 bytes - StartTimeUnixNano = timestampNanoseconds, - EndTimeUnixNano = timestampNanoseconds - }; - - profileContainer.Attributes.Add(new KeyValue { Key = "todo.profiling.data.type", Value = new AnyValue { StringValue = profilingDataType } }); - - return profileContainer; - } - private HttpResponseMessage SendHttpRequest(HttpRequestMessage request, CancellationToken cancellationToken) { return _httpClient.Send(request, cancellationToken); diff --git a/test/test-applications/integrations/TestApplication.ContinuousProfiler/Exporter/SampleBuilder.cs b/test/test-applications/integrations/TestApplication.ContinuousProfiler/Exporter/SampleBuilder.cs index f7ba725e1a..ae91bffec2 100644 --- a/test/test-applications/integrations/TestApplication.ContinuousProfiler/Exporter/SampleBuilder.cs +++ b/test/test-applications/integrations/TestApplication.ContinuousProfiler/Exporter/SampleBuilder.cs @@ -1,19 +1,18 @@ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 -using OpenTelemetry.Proto.Profiles.V1Experimental; +using OpenTelemetry.Proto.Profiles.V1Development; namespace TestApplication.ContinuousProfiler; internal class SampleBuilder { private readonly Sample _sample = new(); - private readonly IList _locationIds = new List(); private long? _value; - public SampleBuilder AddAttribute(ulong attributeId) + public SampleBuilder AddAttribute(int attributeId) { - _sample.Attributes.Add(attributeId); + _sample.AttributeIndices.Add(attributeId); return this; } @@ -23,22 +22,21 @@ public SampleBuilder SetValue(long val) return this; } - public SampleBuilder AddLocationId(ulong locationId) + public SampleBuilder SetLocationRange(int locationsStartIndex, int locationsLength) { - _locationIds.Add(locationId); + _sample.LocationsStartIndex = locationsStartIndex; + _sample.LocationsLength = locationsLength; return this; } - public SampleBuilder SetLink(ulong linkId) + public SampleBuilder SetLink(int linkId) { - _sample.Link = linkId; + _sample.LinkIndex = linkId; return this; } public Sample Build() { - _sample.LocationIndex.AddRange(_locationIds); - if (_value.HasValue) { _sample.Value.Add(_value.Value); diff --git a/test/test-applications/integrations/TestApplication.ContinuousProfiler/Exporter/ThreadSample.cs b/test/test-applications/integrations/TestApplication.ContinuousProfiler/Exporter/ThreadSample.cs index e7e68292a1..c5f34857de 100644 --- a/test/test-applications/integrations/TestApplication.ContinuousProfiler/Exporter/ThreadSample.cs +++ b/test/test-applications/integrations/TestApplication.ContinuousProfiler/Exporter/ThreadSample.cs @@ -7,7 +7,7 @@ internal class ThreadSample { public ThreadSample(long timestampMilliseconds, long traceIdHigh, long traceIdLow, long spanId, string? threadName, uint threadIndex = default) { - TimestampNanoseconds = (ulong)timestampMilliseconds * 1_000_000u; + TimestampNanoseconds = timestampMilliseconds * 1_000_000; TraceIdHigh = traceIdHigh; TraceIdLow = traceIdLow; SpanId = spanId; @@ -15,7 +15,7 @@ public ThreadSample(long timestampMilliseconds, long traceIdHigh, long traceIdLo ThreadIndex = threadIndex; } - public ulong TimestampNanoseconds { get; } + public long TimestampNanoseconds { get; } public long SpanId { get; } diff --git a/test/test-applications/integrations/TestApplication.ContinuousProfiler/TestApplication.ContinuousProfiler.csproj b/test/test-applications/integrations/TestApplication.ContinuousProfiler/TestApplication.ContinuousProfiler.csproj index e1cd364b26..dabc261f2d 100644 --- a/test/test-applications/integrations/TestApplication.ContinuousProfiler/TestApplication.ContinuousProfiler.csproj +++ b/test/test-applications/integrations/TestApplication.ContinuousProfiler/TestApplication.ContinuousProfiler.csproj @@ -35,8 +35,7 @@ - - - + +