Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add gRPC error message for client spans #1528

Merged
merged 7 commits into from
Jan 7, 2025
Merged

Add gRPC error message for client spans #1528

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
eedd80c
Add gRPC error message for client spans
RonFed Dec 29, 2024
62be5e2
Merge branch 'main' into grpc_err_msg
RonFed Jan 3, 2025
16e579c
Add changelog entry
RonFed Jan 3, 2025
2296140
Apply suggestions from code review
RonFed Jan 6, 2025
997a49b
Merge branch 'main' into grpc_err_msg
RonFed Jan 6, 2025
91a192f
Merge branch 'main' into grpc_err_msg
RonFed Jan 7, 2025
d1d1cbe
Merge branch 'main' into grpc_err_msg
MrAlias Jan 7, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ OpenTelemetry Go Automatic Instrumentation adheres to [Semantic Versioning](http
- Support `google.golang.org/grpc` `1.68.2`. ([#1462](https://github.com/open-telemetry/opentelemetry-go-instrumentation/pull/1462))
- Support `google.golang.org/grpc` `1.69.2`. ([#1467](https://github.com/open-telemetry/opentelemetry-go-instrumentation/pull/1467))
- Support `golang.org/x/net` `0.33.0`. ([#1471](https://github.com/open-telemetry/opentelemetry-go-instrumentation/pull/1471))
- Include gRPC error message for client spans. ([#1528](https://github.com/open-telemetry/opentelemetry-go-instrumentation/pull/1528))
- Support `golang.org/x/net` `0.34.0`. ([#1552](https://github.com/open-telemetry/opentelemetry-go-instrumentation/pull/1552))

## [v0.19.0-alpha] - 2024-12-05
Expand Down
4 changes: 2 additions & 2 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -126,8 +126,8 @@ offsets: | $(OFFSETGEN)
$(OFFSETGEN) -output=$(OFFSETS_OUTPUT_FILE) -cache=$(OFFSETS_OUTPUT_FILE)

.PHONY: docker-offsets
docker-offsets:
docker run -e DOCKER_USERNAME=$(DOCKER_USERNAME) -e DOCKER_PASSWORD=$(DOCKER_PASSWORD) --rm -v /tmp:/tmp -v /var/run/docker.sock:/var/run/docker.sock -v $(shell pwd):/app golang:1.22 /bin/sh -c "cd ../app && make offsets"
docker-offsets: docker-build-base
docker run -e DOCKER_USERNAME=$(DOCKER_USERNAME) -e DOCKER_PASSWORD=$(DOCKER_PASSWORD) --rm -v /tmp:/tmp -v /var/run/docker.sock:/var/run/docker.sock -v $(shell pwd):/app $(IMG_NAME_BASE) /bin/sh -c "cd ../app && make offsets"

.PHONY: update-licenses
update-licenses: generate $(GOLICENSES)
Expand Down
217 changes: 217 additions & 0 deletions internal/pkg/inject/offset_results.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1880,6 +1880,223 @@
]
}
]
},
{
"field": "Message",
"offsets": [
{
"offset": null,
"versions": [
"1.0.0",
"1.0.1-GA",
"1.0.2",
"1.0.3",
"1.0.4",
"1.0.5",
"1.2.0",
"1.2.1",
"1.3.0",
"1.4.0",
"1.4.1",
"1.4.2",
"1.5.0",
"1.5.1",
"1.5.2",
"1.6.0",
"1.7.0",
"1.7.1",
"1.7.2",
"1.7.3",
"1.7.4",
"1.7.5",
"1.64.0"
]
},
{
"offset": 8,
"versions": [
"1.15.0",
"1.16.0",
"1.17.0",
"1.18.0",
"1.18.1",
"1.19.0",
"1.19.1",
"1.20.0",
"1.20.1",
"1.21.0",
"1.21.1",
"1.21.2",
"1.21.3",
"1.21.4",
"1.22.0",
"1.22.1",
"1.22.2",
"1.22.3",
"1.23.0",
"1.23.1",
"1.24.0",
"1.25.0",
"1.25.1",
"1.26.0",
"1.27.0-pre",
"1.27.0",
"1.27.1",
"1.28.0-pre",
"1.28.0",
"1.28.1",
"1.29.0-dev",
"1.29.0",
"1.29.1",
"1.30.0-dev",
"1.30.0-dev.1",
"1.30.0",
"1.30.1",
"1.31.0-dev",
"1.31.0",
"1.31.1",
"1.32.0-dev",
"1.32.0",
"1.33.0-dev",
"1.33.0",
"1.33.1",
"1.34.0-dev"
]
},
{
"offset": 48,
"versions": [
"1.8.0",
"1.8.2",
"1.9.0",
"1.9.1",
"1.9.2",
"1.10.0",
"1.10.1",
"1.11.0",
"1.11.1",
"1.11.2",
"1.11.3",
"1.12.0",
"1.12.1",
"1.12.2",
"1.13.0",
"1.14.0",
"1.33.2",
"1.33.3",
"1.34.0",
"1.34.1",
"1.34.2",
"1.35.0-dev",
"1.35.0",
"1.35.1",
"1.36.0-dev",
"1.36.0",
"1.36.1",
"1.37.0-dev",
"1.37.0",
"1.37.1",
"1.38.0-dev",
"1.38.0",
"1.38.1",
"1.39.0-dev",
"1.39.0",
"1.39.1",
"1.40.0-dev",
"1.40.0",
"1.40.1",
"1.41.0-dev",
"1.41.0",
"1.41.1",
"1.42.0-dev",
"1.42.0",
"1.43.0-dev",
"1.43.0",
"1.44.0-dev",
"1.44.0",
"1.45.0-dev",
"1.45.0",
"1.46.0-dev",
"1.46.0",
"1.46.1",
"1.46.2",
"1.47.0-dev",
"1.47.0",
"1.48.0-dev",
"1.48.0",
"1.49.0-dev",
"1.49.0",
"1.50.0-dev",
"1.50.0",
"1.50.1",
"1.51.0-dev",
"1.51.0",
"1.52.0-dev",
"1.52.0",
"1.52.1",
"1.52.3",
"1.53.0-dev",
"1.53.0",
"1.54.0",
"1.54.1",
"1.55.0-dev",
"1.55.0",
"1.55.1",
"1.56.0-dev",
"1.56.0",
"1.56.1",
"1.56.2",
"1.56.3",
"1.57.0-dev",
"1.57.0",
"1.57.1",
"1.57.2",
"1.58.0-dev",
"1.58.0",
"1.58.1",
"1.58.2",
"1.58.3",
"1.59.0-dev",
"1.59.0",
"1.60.0-dev",
"1.60.0",
"1.60.1",
"1.61.0-dev",
"1.61.0",
"1.61.1",
"1.61.2",
"1.62.0",
"1.62.1",
"1.62.2",
"1.63.0",
"1.63.1",
"1.63.2",
"1.63.3",
"1.64.1",
"1.65.0-dev",
"1.65.0",
"1.65.1",
"1.66.0-dev",
"1.66.0",
"1.66.1",
"1.66.2",
"1.66.3",
"1.67.0-dev",
"1.67.0",
"1.67.1",
"1.67.2",
"1.67.3",
"1.68.0-dev",
"1.68.0",
"1.68.1",
"1.68.2",
"1.69.0-dev",
"1.69.0",
"1.69.2",
"1.70.0-dev"
]
}
]
}
]
}
Expand Down
9 changes: 9 additions & 0 deletions internal/pkg/instrumentation/bpf/google.golang.org/grpc/client/bpf/probe.bpf.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,12 @@ char __license[] SEC("license") = "Dual MIT/GPL";

#define MAX_SIZE 50
#define MAX_CONCURRENT 50
#define MAX_ERROR_LEN 128

struct grpc_request_t
{
BASE_SPAN_PROPERTIES
char err_msg[MAX_ERROR_LEN];
char method[MAX_SIZE];
char target[MAX_SIZE];
u32 status_code;
Expand Down Expand Up @@ -51,6 +53,7 @@ volatile const u64 headerFrame_streamid_pos;
volatile const u64 headerFrame_hf_pos;
volatile const u64 error_status_pos;
volatile const u64 status_s_pos;
volatile const u64 status_message_pos;
volatile const u64 status_code_pos;

volatile const bool write_status_supported;
Expand Down Expand Up @@ -139,6 +142,11 @@ int uprobe_ClientConn_Invoke_Returns(struct pt_regs *ctx) {
// s *Status
// }
// The `Status` proto object contains a `Code` int32 field, which is what we want
// type Status struct {
// Code int32
// Message string
// Details []*anypb.Any
// }
void *resp_ptr = get_argument(ctx, 2);
if(resp_ptr == 0) {
// err == nil
Expand All @@ -152,6 +160,7 @@ int uprobe_ClientConn_Invoke_Returns(struct pt_regs *ctx) {
bpf_probe_read_user(&s_ptr, sizeof(s_ptr), (void *)(status_ptr + status_s_pos));
// Get status code from Status.s pointer
bpf_probe_read_user(&grpc_span->status_code, sizeof(grpc_span->status_code), (void *)(s_ptr + status_code_pos));
get_go_string_from_user_ptr((void *)(s_ptr + status_message_pos), grpc_span->err_msg, sizeof(grpc_span->err_msg));

done:
grpc_span->end_time = bpf_ktime_get_ns();
Expand Down
3 changes: 3 additions & 0 deletions internal/pkg/instrumentation/bpf/google.golang.org/grpc/client/bpf_arm64_bpfel.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 3 additions & 0 deletions internal/pkg/instrumentation/bpf/google.golang.org/grpc/client/bpf_x86_bpfel.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

12 changes: 12 additions & 0 deletions internal/pkg/instrumentation/bpf/google.golang.org/grpc/client/probe.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,13 @@ func New(logger *slog.Logger, version string) probe.Probe {
},
MinVersion: writeStatusMinVersion,
},
probe.StructFieldConstMinVersion{
StructField: probe.StructFieldConst{
Key: "status_message_pos",
Val: structfield.NewID("google.golang.org/grpc", "google.golang.org/genproto/googleapis/rpc/status", "Status", "Message"),
},
MinVersion: writeStatusMinVersion,
},
},
Uprobes: []probe.Uprobe{
{
Expand Down Expand Up @@ -137,6 +144,7 @@ func verifyAndLoadBpf() (*ebpf.CollectionSpec, error) {
// event represents an event in the gRPC client during a gRPC request.
type event struct {
context.BaseSpanProperties
ErrMsg [128]byte
Method [50]byte
Target [50]byte
StatusCode int32
Expand Down Expand Up @@ -184,6 +192,10 @@ func processFn(e *event) ptrace.SpanSlice {

if writeStatus && e.StatusCode > 0 {
span.Status().SetCode(ptrace.StatusCodeError)
errMsg := unix.ByteSliceToString(e.ErrMsg[:])
if errMsg != "" {
span.Status().SetMessage(errMsg)
}
}

return spans
Expand Down
11 changes: 7 additions & 4 deletions internal/pkg/instrumentation/bpf/google.golang.org/grpc/server/bpf/probe.bpf.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -273,9 +273,12 @@ SEC("uprobe/server_handleStream2")
int uprobe_server_handleStream2_Returns(struct pt_regs *ctx) {
u64 server_stream_pos = 4;
void *server_stream_ptr = get_argument(ctx, server_stream_pos);
struct go_iface go_context = {0};
void *key = NULL;
MrAlias marked this conversation as resolved.
Show resolved Hide resolved
if (server_stream_ptr == NULL) {
bpf_printk("grpc:server:uprobe/server_handleStream2Return: failed to get ServerStream arg");
return -1;
// We might fail to get the pointer for versions of Go which use register ABI, as this function does not return anything.
// This is not an error in that case so we can just go to the lookup which will happen by goroutine.
goto lookup;
}

void *stream_ptr;
Expand All @@ -285,7 +288,6 @@ int uprobe_server_handleStream2_Returns(struct pt_regs *ctx) {
return -2;
}

struct go_iface go_context = {0};
rc = bpf_probe_read_user(&go_context.type, sizeof(go_context.type), (void *)(stream_ptr + stream_ctx_pos));
if (rc != 0) {
bpf_printk("grpc:server:uprobe/server_handleStream2Return: failed to read context type");
Expand All @@ -298,7 +300,8 @@ int uprobe_server_handleStream2_Returns(struct pt_regs *ctx) {
return -4;
}

void *key = get_consistent_key(ctx, go_context.data);
lookup:
key = get_consistent_key(ctx, go_context.data);
struct grpc_request_t *event = bpf_map_lookup_elem(&grpc_events, &key);
if (event == NULL) {
bpf_printk("grpc:server:uprobe/server_handleStream2Return: event is NULL");
Expand Down
Loading
Loading