From 4b0e1a8e971de652bc519919dc7ccffc7b5d8e21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wilson=20J=C3=BAnior?= Date: Thu, 17 Oct 2019 10:07:51 -0300 Subject: [PATCH] Avoid empty prometheus rules --- slo/slo.go | 22 ++++++---- slo/slo_test.go | 110 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 123 insertions(+), 9 deletions(-) diff --git a/slo/slo.go b/slo/slo.go index d9d65e1..dd62f19 100644 --- a/slo/slo.go +++ b/slo/slo.go @@ -116,7 +116,9 @@ func (slo SLO) GenerateGroupRules() []rulefmt.RuleGroup { ruleGroup.Rules = append(ruleGroup.Rules, slo.generateRules(bucket)...) } - rules = append(rules, ruleGroup) + if len(ruleGroup.Rules) > 0 { + rules = append(rules, ruleGroup) + } } return rules @@ -167,16 +169,18 @@ func (slo SLO) generateRules(bucket string) []rulefmt.Rule { } } - for _, latencyBucket := range slo.Objectives.Latency { - latencyRateRecord := rulefmt.Rule{ - Record: "slo:service_latency:ratio_rate_" + bucket, - Expr: slo.LatencyRecord.ComputeExpr(bucket, latencyBucket.LE), - Labels: slo.labels(), - } + if slo.LatencyRecord.Expr != "" { + for _, latencyBucket := range slo.Objectives.Latency { + latencyRateRecord := rulefmt.Rule{ + Record: "slo:service_latency:ratio_rate_" + bucket, + Expr: slo.LatencyRecord.ComputeExpr(bucket, latencyBucket.LE), + Labels: slo.labels(), + } - latencyRateRecord.Labels["le"] = latencyBucket.LE + latencyRateRecord.Labels["le"] = latencyBucket.LE - rules = append(rules, latencyRateRecord) + rules = append(rules, latencyRateRecord) + } } return rules diff --git a/slo/slo_test.go b/slo/slo_test.go index 4990ff0..6907b11 100644 --- a/slo/slo_test.go +++ b/slo/slo_test.go @@ -696,3 +696,113 @@ func TestSLOGenerateAlertRules(t *testing.T) { Annotations: slo.Annotations, }) } + +func TestSLOGenerateAlertRulesWithoutExpressions(t *testing.T) { + slo := &SLO{ + Name: "my-team.my-service.payment", + Objectives: Objectives{ + Availability: 99.9, + Latency: []methods.LatencyTarget{ + { + LE: "0.1", + Target: 95, + }, + { + LE: "0.5", + Target: 99, + }, + }, + }, + ErrorRateRecord: ExprBlock{ + AlertMethod: "multi-window", + }, + LatencyRecord: ExprBlock{ + AlertMethod: "multi-window", + }, + Labels: map[string]string{ + "channel": "my-channel", + }, + Annotations: map[string]string{ + "message": "Service A has lower SLI", + "link": "http://wiki.ops/1234", + "dashboard": "http://grafana.globo.com", + }, + } + + alertRules := slo.GenerateAlertRules() + assert.Len(t, alertRules, 4) + + assert.Equal(t, alertRules[0], rulefmt.Rule{ + Alert: "slo:my-team.my-service.payment.errors.page", + Expr: "(slo:service_errors_total:ratio_rate_1h{service=\"my-team.my-service.payment\"} > (14.4 * 0.001) and slo:service_errors_total:ratio_rate_5m{service=\"my-team.my-service.payment\"} > (14.4 * 0.001)) or (slo:service_errors_total:ratio_rate_6h{service=\"my-team.my-service.payment\"} > (6 * 0.001) and slo:service_errors_total:ratio_rate_30m{service=\"my-team.my-service.payment\"} > (6 * 0.001))", + Labels: map[string]string{ + "channel": "my-channel", + "severity": "page", + }, + Annotations: slo.Annotations, + }) + + assert.Equal(t, alertRules[1], rulefmt.Rule{ + Alert: "slo:my-team.my-service.payment.errors.ticket", + Expr: "(slo:service_errors_total:ratio_rate_1d{service=\"my-team.my-service.payment\"} > (3 * 0.001) and slo:service_errors_total:ratio_rate_2h{service=\"my-team.my-service.payment\"} > (3 * 0.001)) or (slo:service_errors_total:ratio_rate_3d{service=\"my-team.my-service.payment\"} > (1 * 0.001) and slo:service_errors_total:ratio_rate_6h{service=\"my-team.my-service.payment\"} > (1 * 0.001))", + Labels: map[string]string{ + "channel": "my-channel", + "severity": "ticket", + }, + Annotations: slo.Annotations, + }) + + assert.Equal(t, alertRules[2], rulefmt.Rule{ + Alert: "slo:my-team.my-service.payment.latency.page", + Expr: ("(" + + "slo:service_latency:ratio_rate_1h{le=\"0.1\", service=\"my-team.my-service.payment\"} < 0.28" + + " and " + + "slo:service_latency:ratio_rate_5m{le=\"0.1\", service=\"my-team.my-service.payment\"} < 0.28" + + ") or (" + + "slo:service_latency:ratio_rate_6h{le=\"0.1\", service=\"my-team.my-service.payment\"} < 0.7" + + " and " + + "slo:service_latency:ratio_rate_30m{le=\"0.1\", service=\"my-team.my-service.payment\"} < 0.7" + + ") or (" + + "slo:service_latency:ratio_rate_1h{le=\"0.5\", service=\"my-team.my-service.payment\"} < 0.856" + + " and " + + "slo:service_latency:ratio_rate_5m{le=\"0.5\", service=\"my-team.my-service.payment\"} < 0.856" + + ") or (" + + "slo:service_latency:ratio_rate_6h{le=\"0.5\", service=\"my-team.my-service.payment\"} < 0.94" + + " and " + + "slo:service_latency:ratio_rate_30m{le=\"0.5\", service=\"my-team.my-service.payment\"} < 0.94" + + ")"), + + Labels: map[string]string{ + "channel": "my-channel", + "severity": "page", + }, + Annotations: slo.Annotations, + }) + + assert.Equal(t, alertRules[3], rulefmt.Rule{ + Alert: "slo:my-team.my-service.payment.latency.ticket", + Expr: ("(" + + "slo:service_latency:ratio_rate_1d{le=\"0.1\", service=\"my-team.my-service.payment\"} < 0.85" + + " and " + + "slo:service_latency:ratio_rate_2h{le=\"0.1\", service=\"my-team.my-service.payment\"} < 0.85" + + ") or (" + + "slo:service_latency:ratio_rate_3d{le=\"0.1\", service=\"my-team.my-service.payment\"} < 0.95" + + " and " + + "slo:service_latency:ratio_rate_6h{le=\"0.1\", service=\"my-team.my-service.payment\"} < 0.95" + + ") or (" + + "slo:service_latency:ratio_rate_1d{le=\"0.5\", service=\"my-team.my-service.payment\"} < 0.97" + + " and " + + "slo:service_latency:ratio_rate_2h{le=\"0.5\", service=\"my-team.my-service.payment\"} < 0.97" + + ") or (" + + "slo:service_latency:ratio_rate_3d{le=\"0.5\", service=\"my-team.my-service.payment\"} < 0.99" + + " and " + + "slo:service_latency:ratio_rate_6h{le=\"0.5\", service=\"my-team.my-service.payment\"} < 0.99" + + ")"), + + Labels: map[string]string{ + "channel": "my-channel", + "severity": "ticket", + }, + Annotations: slo.Annotations, + }) +}