Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: handle exists and nexists for mat columns and top level columns #6680

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
Open

fix: handle exists and nexists for mat columns and top level columns #6680

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
161f5fd
fix: handle exists and nexists for mat columns and top level columns
nityanandagohain Dec 19, 2024
2df8d40
fix: add comments
nityanandagohain Dec 20, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
70 changes: 35 additions & 35 deletions pkg/query-service/app/traces/v4/query_builder.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,8 @@ var tracesOperatorMappingV3 = map[v3.FilterOperator]string{
v3.FilterOperatorNotRegex: "NOT match(%s, %s)",
v3.FilterOperatorContains: "ILIKE",
v3.FilterOperatorNotContains: "NOT ILIKE",
v3.FilterOperatorExists: "mapContains(%s, '%s')",
v3.FilterOperatorNotExists: "NOT mapContains(%s, '%s')",
v3.FilterOperatorExists: "mapContains(%s_%s, '%s')",
v3.FilterOperatorNotExists: "NOT mapContains(%s_%s, '%s')",
}

func getClickHouseTracesColumnType(columnType v3.AttributeKeyType) string {
Expand Down Expand Up @@ -74,17 +74,38 @@ func getSelectLabels(groupBy []v3.AttributeKey) string {
return strings.Join(labels, ",")
}

// TODO(nitya): use the _exists columns as well in the future similar to logs
func existsSubQueryForFixedColumn(key v3.AttributeKey, op v3.FilterOperator) (string, error) {
if key.DataType == v3.AttributeKeyDataTypeString {
if op == v3.FilterOperatorExists {
return fmt.Sprintf("%s %s ''", getColumnName(key), tracesOperatorMappingV3[v3.FilterOperatorNotEqual]), nil
} else {
return fmt.Sprintf("%s %s ''", getColumnName(key), tracesOperatorMappingV3[v3.FilterOperatorEqual]), nil
func getExistsNexistsFilter(op v3.FilterOperator, item v3.FilterItem) string {
if _, ok := constants.StaticFieldsTraces[item.Key.Key]; ok {
chOp := "!="
if op == v3.FilterOperatorNotExists {
chOp = "="
}
} else {
return "", fmt.Errorf("unsupported operation, exists and not exists can only be applied on custom attributes or string type columns")
}
key := getColumnName(item.Key)
if item.Key.DataType == v3.AttributeKeyDataTypeString {
return fmt.Sprintf("%s %s ''", key, chOp)
}

// top level number columns are duration_nano, kind, status_code
if item.Key.DataType == v3.AttributeKeyDataTypeInt64 || item.Key.DataType == v3.AttributeKeyDataTypeFloat64 {
return fmt.Sprintf("%s %s 0", key, chOp)
}

// do noting for other types right now
return ""
} else if item.Key.IsColumn {
// get filter for materialized columns
val := true
if op == v3.FilterOperatorNotExists {
val = false
}
// trim suffix is added to add the _exists to the column name,
// eg: `resource_string_host` to `resource_string_host_exists`
return fmt.Sprintf("%s_exists` = %v", strings.TrimSuffix(getColumnName(item.Key), "`"), val)
}
// filter for non materialized attributes
columnType := getClickHouseTracesColumnType(item.Key.Type)
columnDataType := getClickHouseTracesColumnDataType(item.Key.DataType)
return fmt.Sprintf(tracesOperatorMappingV3[op], columnType, columnDataType, item.Key.Key)
}

func buildTracesFilterQuery(fs *v3.FilterSet) (string, error) {
Expand Down Expand Up @@ -122,19 +143,7 @@ func buildTracesFilterQuery(fs *v3.FilterSet) (string, error) {
case v3.FilterOperatorRegex, v3.FilterOperatorNotRegex:
conditions = append(conditions, fmt.Sprintf(operator, columnName, fmtVal))
case v3.FilterOperatorExists, v3.FilterOperatorNotExists:
if item.Key.IsColumn {
subQuery, err := existsSubQueryForFixedColumn(item.Key, item.Operator)
if err != nil {
return "", err
}
conditions = append(conditions, subQuery)
} else {
cType := getClickHouseTracesColumnType(item.Key.Type)
cDataType := getClickHouseTracesColumnDataType(item.Key.DataType)
col := fmt.Sprintf("%s_%s", cType, cDataType)
conditions = append(conditions, fmt.Sprintf(operator, col, item.Key.Key))
}

conditions = append(conditions, getExistsNexistsFilter(item.Operator, item))
default:
conditions = append(conditions, fmt.Sprintf("%s %s %s", columnName, operator, fmtVal))
}
Expand Down Expand Up @@ -362,16 +371,7 @@ func buildTracesQuery(start, end, step int64, mq *v3.BuilderQuery, panelType v3.
return query, nil
case v3.AggregateOperatorCount:
if mq.AggregateAttribute.Key != "" {
if mq.AggregateAttribute.IsColumn {
subQuery, err := existsSubQueryForFixedColumn(mq.AggregateAttribute, v3.FilterOperatorExists)
if err == nil {
filterSubQuery = fmt.Sprintf("%s AND %s", filterSubQuery, subQuery)
}
} else {
cType := getClickHouseTracesColumnType(mq.AggregateAttribute.Type)
cDataType := getClickHouseTracesColumnDataType(mq.AggregateAttribute.DataType)
filterSubQuery = fmt.Sprintf("%s AND mapContains(%s_%s, '%s')", filterSubQuery, cType, cDataType, mq.AggregateAttribute.Key)
}
filterSubQuery = filterSubQuery + " AND " + getExistsNexistsFilter(v3.FilterOperatorExists, v3.FilterItem{Key: mq.AggregateAttribute, Operator: v3.FilterOperatorExists})
}
op := "toFloat64(count())"
query := fmt.Sprintf(queryTmpl, op, filterSubQuery, groupBy, having, orderBy)
Expand Down
4 changes: 2 additions & 2 deletions pkg/query-service/app/traces/v4/query_builder_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -269,7 +269,7 @@ func Test_buildTracesFilterQuery(t *testing.T) {
{Key: v3.AttributeKey{Key: "http.route", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true}, Operator: v3.FilterOperatorNotExists},
}},
},
want: "mapContains(attributes_string, 'host') AND mapContains(attributes_number, 'duration') AND NOT mapContains(attributes_bool, 'isDone') AND NOT mapContains(attributes_string, 'host1') AND `attribute_string_path` = '' AND http_url = '' AND `attribute_string_http$$route` = ''",
want: "mapContains(attributes_string, 'host') AND mapContains(attributes_number, 'duration') AND NOT mapContains(attributes_bool, 'isDone') AND NOT mapContains(attributes_string, 'host1') AND `attribute_string_path_exists` = false AND http_url = '' AND `attribute_string_http$$route_exists` = false",
},
}
for _, tt := range tests {
Expand Down Expand Up @@ -478,7 +478,7 @@ func Test_buildTracesQuery(t *testing.T) {
},
},
want: "SELECT attributes_string['http.method'] as `http.method`, toFloat64(count()) as value from signoz_traces.distributed_signoz_index_v3 where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') " +
"AND (ts_bucket_start >= 1680064560 AND ts_bucket_start <= 1680066458) AND attributes_string['http.method'] = '100' AND mapContains(attributes_string, 'http.method') AND mapContains(attributes_string, 'name') " +
"AND (ts_bucket_start >= 1680064560 AND ts_bucket_start <= 1680066458) AND attributes_string['http.method'] = '100' AND mapContains(attributes_string, 'http.method') AND name != '' " +
"group by `http.method` order by `http.method` ASC",
},
{
Expand Down
Loading