diff --git a/.chloggen/split-dimensions-optout.yaml b/.chloggen/split-dimensions-optout.yaml new file mode 100644 index 000000000000..dd28374428f3 --- /dev/null +++ b/.chloggen/split-dimensions-optout.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: receiver/azuremonitorreceiver + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: "Add dimensions.enabled and dimensions.overrides which allows to opt out from automatically split by all the dimensions of the resource type" + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [36240] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [] diff --git a/receiver/azuremonitorreceiver/README.md b/receiver/azuremonitorreceiver/README.md index ab18a2cc4b0c..0944133ebca6 100644 --- a/receiver/azuremonitorreceiver/README.md +++ b/receiver/azuremonitorreceiver/README.md @@ -31,6 +31,8 @@ The following settings are optional: - `maximum_number_of_records_per_resource` (default = 10): Maximum number of records to fetch per resource. - `initial_delay` (default = `1s`): defines how long this receiver waits before starting. - `cloud` (default = `AzureCloud`): defines which Azure cloud to use. Valid values: `AzureCloud`, `AzureUSGovernment`, `AzureChinaCloud`. +- `dimensions.enabled` (default = `true`): allows to opt out from automatically split by all the dimensions of the resource type. +- `dimensions.overrides` (default = `{}`): if dimensions are enabled, it allows you to specify a set of dimensions for a particular metric. This is a two levels map with first key being the resource type and second key being the metric name. Programmatic value should be used for metric name https://learn.microsoft.com/en-us/azure/azure-monitor/reference/metrics-index Authenticating using service principal requires following additional settings: @@ -101,6 +103,22 @@ receivers: auth: "default_credentials" ``` +Overriding dimensions for a particular metric: +```yaml +receivers: + azuremonitor: + dimensions: + enabled: true + overrides: + "Microsoft.Network/azureFirewalls": + # Real example of an Azure limitation here: + # Dimensions exposed are Reason, Status, Protocol, + # but when selecting Protocol in the filters, it returns nothing. + # Note here that the metric display name is ``Network rules hit count`` but it's programmatic value is ``NetworkRuleHit`` + # Ref: https://learn.microsoft.com/en-us/azure/azure-monitor/reference/supported-metrics/microsoft-network-azurefirewalls-metrics + "NetworkRuleHit": [Reason, Status] +``` + ## Metrics diff --git a/receiver/azuremonitorreceiver/config.go b/receiver/azuremonitorreceiver/config.go index 6e835003e781..2c421f9bbb30 100644 --- a/receiver/azuremonitorreceiver/config.go +++ b/receiver/azuremonitorreceiver/config.go @@ -228,6 +228,11 @@ var ( } ) +type DimensionsConfig struct { + Enabled *bool `mapstructure:"enabled"` + Overrides map[string]map[string][]string `mapstructure:"overrides"` +} + // Config defines the configuration for the various elements of the receiver agent. type Config struct { scraperhelper.ControllerConfig `mapstructure:",squash"` @@ -246,6 +251,7 @@ type Config struct { MaximumNumberOfMetricsInACall int `mapstructure:"maximum_number_of_metrics_in_a_call"` MaximumNumberOfRecordsPerResource int32 `mapstructure:"maximum_number_of_records_per_resource"` AppendTagsAsAttributes bool `mapstructure:"append_tags_as_attributes"` + Dimensions DimensionsConfig `mapstructure:"dimensions"` } const ( diff --git a/receiver/azuremonitorreceiver/dimension_test.go b/receiver/azuremonitorreceiver/dimension_test.go new file mode 100644 index 000000000000..50cacf994276 --- /dev/null +++ b/receiver/azuremonitorreceiver/dimension_test.go @@ -0,0 +1,171 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package azuremonitorreceiver // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/azuremonitorreceiver" + +import ( + "testing" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/monitor/armmonitor" + "github.com/stretchr/testify/require" +) + +func newDimension(value string) *armmonitor.LocalizableString { + return to.Ptr(armmonitor.LocalizableString{Value: to.Ptr(value)}) +} + +func TestFilterDimensions(t *testing.T) { + type args struct { + dimensions []*armmonitor.LocalizableString + cfg DimensionsConfig + resourceType string + metricName string + } + + tests := []struct { + name string + args args + expected []string + }{ + { + name: "always empty if dimensions disabled", + args: args{ + dimensions: []*armmonitor.LocalizableString{ + newDimension("foo"), + newDimension("bar"), + }, + cfg: DimensionsConfig{ + Enabled: to.Ptr(false), + }, + resourceType: "rt1", + metricName: "m1", + }, + expected: nil, + }, + { + name: "split by dimensions should be enabled by default", + args: args{ + dimensions: []*armmonitor.LocalizableString{ + newDimension("foo"), + newDimension("bar"), + }, + cfg: DimensionsConfig{}, // enabled by default + resourceType: "rt1", + metricName: "m1", + }, + expected: []string{"foo", "bar"}, + }, + { + name: "overrides takes precedence over input", + args: args{ + dimensions: []*armmonitor.LocalizableString{ + newDimension("foo"), + newDimension("bar"), + }, + cfg: DimensionsConfig{ + Enabled: to.Ptr(true), + Overrides: map[string]map[string][]string{ + "rt1": { + "m1": { + "foo", + }, + }, + }, + }, + resourceType: "rt1", + metricName: "m1", + }, + expected: []string{"foo"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + actual := filterDimensions(tt.args.dimensions, tt.args.cfg, tt.args.resourceType, tt.args.metricName) + require.Equal(t, tt.expected, actual) + }) + } +} + +func TestBuildDimensionsFilter(t *testing.T) { + type args struct { + dimensionsStr string + } + + tests := []struct { + name string + args args + expected *string + }{ + { + name: "empty given dimensions string", + args: args{ + dimensionsStr: "", + }, + expected: nil, + }, + { + name: "build dimensions filter", + args: args{ + dimensionsStr: "bar,foo", + }, + expected: to.Ptr("bar eq '*' and foo eq '*'"), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + actual := buildDimensionsFilter(tt.args.dimensionsStr) + require.EqualValues(t, tt.expected, actual) + }) + } +} + +func TestSerializeDimensions(t *testing.T) { + type args struct { + dimensions []string + } + + tests := []struct { + name string + args args + expected string + }{ + { + name: "empty given dimensions", + args: args{ + dimensions: []string{}, + }, + expected: "", + }, + { + name: "nil given dimensions", + args: args{ + dimensions: []string{}, + }, + expected: "", + }, + { + name: "reorder dimensions", + args: args{ + dimensions: []string{"foo", "bar"}, + }, + expected: "bar,foo", + }, + { + name: "trim spaces dimensions", + args: args{ + dimensions: []string{" bar", "foo "}, + }, + expected: "bar,foo", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + actual := serializeDimensions(tt.args.dimensions) + require.EqualValues(t, tt.expected, actual) + }) + } +} diff --git a/receiver/azuremonitorreceiver/dimensions.go b/receiver/azuremonitorreceiver/dimensions.go new file mode 100644 index 000000000000..9915df91ba00 --- /dev/null +++ b/receiver/azuremonitorreceiver/dimensions.go @@ -0,0 +1,66 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package azuremonitorreceiver // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/azuremonitorreceiver" + +import ( + "bytes" + "sort" + "strings" + + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/monitor/armmonitor" +) + +// filterDimensions transforms a list of azure dimensions into a list of string, taking in account the DimensionConfig +// given by the user. +func filterDimensions(dimensions []*armmonitor.LocalizableString, cfg DimensionsConfig, resourceType, metricName string) []string { + // Only skip if explicitly disabled. Enabled by default. + if cfg.Enabled != nil && !*cfg.Enabled { + return nil + } + + // If dimensions are overridden for that resource type and metric name, we take it + if _, resourceTypeFound := cfg.Overrides[resourceType]; resourceTypeFound { + if newDimensions, metricNameFound := cfg.Overrides[resourceType][metricName]; metricNameFound { + return newDimensions + } + } + // Otherwise we get all dimensions + var result []string + for _, dimension := range dimensions { + result = append(result, *dimension.Value) + } + return result +} + +// serializeDimensions build a comma separated string from trimmed, sorted dimensions list. +// It is designed to be used as a key in scraper maps. +func serializeDimensions(dimensions []string) string { + var dimensionsSlice []string + for _, dimension := range dimensions { + if trimmedDimension := strings.TrimSpace(dimension); len(trimmedDimension) > 0 { + dimensionsSlice = append(dimensionsSlice, trimmedDimension) + } + } + sort.Strings(dimensionsSlice) + return strings.Join(dimensionsSlice, ",") +} + +// buildDimensionsFilter takes a serialized dimensions input to build an Azure Request filter that will allow us to +// receive metrics values split by these dimensions. +func buildDimensionsFilter(dimensionsStr string) *string { + if len(dimensionsStr) == 0 { + return nil + } + var dimensionsFilter bytes.Buffer + dimensions := strings.Split(dimensionsStr, ",") + for i, dimension := range dimensions { + dimensionsFilter.WriteString(dimension) + dimensionsFilter.WriteString(" eq '*'") + if i < len(dimensions)-1 { + dimensionsFilter.WriteString(" and ") + } + } + result := dimensionsFilter.String() + return &result +} diff --git a/receiver/azuremonitorreceiver/scraper.go b/receiver/azuremonitorreceiver/scraper.go index 99d24600faa5..9446594a2e51 100644 --- a/receiver/azuremonitorreceiver/scraper.go +++ b/receiver/azuremonitorreceiver/scraper.go @@ -4,11 +4,9 @@ package azuremonitorreceiver // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/azuremonitorreceiver" import ( - "bytes" "context" "fmt" "regexp" - "sort" "strings" "sync" "time" @@ -64,6 +62,7 @@ type azureResource struct { metricsByCompositeKey map[metricsCompositeKey]*azureResourceMetrics metricsDefinitionsUpdated time.Time tags map[string]*string + resourceType *string } type metricsCompositeKey struct { @@ -281,8 +280,9 @@ func (s *azureScraper) getResources(ctx context.Context) { attributes[attributeLocation] = resource.Location } s.resources[*resource.ID] = &azureResource{ - attributes: attributes, - tags: resource.Tags, + attributes: attributes, + tags: resource.Tags, + resourceType: resource.Type, } } delete(existingResources, *resource.ID) @@ -338,20 +338,13 @@ func (s *azureScraper) getResourceMetricsDefinitions(ctx context.Context, resour for _, v := range nextResult.Value { timeGrain := *v.MetricAvailabilities[0].TimeGrain - name := *v.Name.Value - compositeKey := metricsCompositeKey{timeGrain: timeGrain} - - if len(v.Dimensions) > 0 { - var dimensionsSlice []string - for _, dimension := range v.Dimensions { - if len(strings.TrimSpace(*dimension.Value)) > 0 { - dimensionsSlice = append(dimensionsSlice, *dimension.Value) - } - } - sort.Strings(dimensionsSlice) - compositeKey.dimensions = strings.Join(dimensionsSlice, ",") + metricName := *v.Name.Value + dimensions := filterDimensions(v.Dimensions, s.cfg.Dimensions, *s.resources[resourceID].resourceType, metricName) + compositeKey := metricsCompositeKey{ + timeGrain: timeGrain, + dimensions: serializeDimensions(dimensions), } - s.storeMetricsDefinition(resourceID, name, compositeKey) + s.storeMetricsDefinition(resourceID, metricName, compositeKey) } } s.resources[resourceID].metricsDefinitionsUpdated = time.Now() @@ -439,30 +432,14 @@ func getResourceMetricsValuesRequestOptions( end int, top int32, ) armmonitor.MetricsClientListOptions { - resType := strings.Join(metrics[start:end], ",") - filter := armmonitor.MetricsClientListOptions{ - Metricnames: &resType, + return armmonitor.MetricsClientListOptions{ + Metricnames: to.Ptr(strings.Join(metrics[start:end], ",")), Interval: to.Ptr(timeGrain), Timespan: to.Ptr(timeGrain), Aggregation: to.Ptr(strings.Join(aggregations, ",")), Top: to.Ptr(top), + Filter: buildDimensionsFilter(dimensionsStr), } - - if len(dimensionsStr) > 0 { - var dimensionsFilter bytes.Buffer - dimensions := strings.Split(dimensionsStr, ",") - for i, dimension := range dimensions { - dimensionsFilter.WriteString(dimension) - dimensionsFilter.WriteString(" eq '*' ") - if i < len(dimensions)-1 { - dimensionsFilter.WriteString(" and ") - } - } - dimensionFilterString := dimensionsFilter.String() - filter.Filter = &dimensionFilterString - } - - return filter } func (s *azureScraper) processTimeseriesData(