Add latency threshold to load test for node local dns measurement

kubernetes · Jan 18, 2021 · ece6960 · ece6960
1 parent d304ddf
commit ece6960
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 4 deletions.
diff --git a/clusterloader2/pkg/measurement/common/nodelocaldns_latency_prometheus.go b/clusterloader2/pkg/measurement/common/nodelocaldns_latency_prometheus.go
@@ -30,7 +30,7 @@ import (
 const (
 	nodelocaldnsLatencyPrometheusMeasurementName = "NodeLocalDNSLatencyPrometheus"
 	percLatencyQueryTemplate                     = `histogram_quantile(%v, sum(rate(coredns_dns_request_duration_seconds_bucket[%v])) by (le))`
-	defaultLatencyUpperBoundSeconds              = 5.0
+	defaultThreshold                             = 5 * time.Second
 )
 
 var (
@@ -70,14 +70,14 @@ func (n *nodelocaldnsLatencyGatherer) IsEnabled(config *measurement.Config) bool
 }
 
 func (n *nodelocaldnsLatencyGatherer) validateResult(config *measurement.Config, result *measurementutil.LatencyMetric) error {
-	latencyUpperBound, err := util.GetFloat64OrDefault(config.Params, "dns_latency_upper_bound_seconds", defaultLatencyUpperBoundSeconds)
+	latencyUpperBound, err := util.GetDurationOrDefault(config.Params, "threshold", defaultThreshold)
 	if err != nil {
 		return err
 	}
-	if result.Perc99 > time.Duration(latencyUpperBound)*time.Second {
+	if result.Perc99 > latencyUpperBound {
 		return errors.NewMetricViolationError(
 			"NodelocalDNS dns_request_duration_seconds",
-			fmt.Sprintf("99th Percentile Latency %v is higher than the upper bound of %f seconds", result.Perc99, latencyUpperBound))
+			fmt.Sprintf("99th Percentile Latency %v is higher than the upper bound of %s", result.Perc99, latencyUpperBound))
 	}
 	return nil
 }

diff --git a/clusterloader2/testing/load/config.yaml b/clusterloader2/testing/load/config.yaml
@@ -19,6 +19,7 @@
 {{$ENABLE_PVS := DefaultParam .CL2_ENABLE_PVS true}}
 {{$ENABLE_NETWORKPOLICIES := DefaultParam .CL2_ENABLE_NETWORKPOLICIES false}}
 {{$ENABLE_NODE_LOCAL_DNS_LATENCY := DefaultParam .CL2_ENABLE_NODE_LOCAL_DNS_LATENCY false}}
+{{$NODE_LOCAL_DNS_LATENCY_THRESHOLD := DefaultParam .CL2_NODE_LOCAL_DNS_LATENCY_THRESHOLD "5s"}}
 {{$ENABLE_DNSTESTS := DefaultParam .CL2_ENABLE_DNSTESTS false}}
 # LATENCY_POD_MEMORY and LATENCY_POD_CPU are calculated for 1-core 4GB node.
 # Increasing allocation of both memory and cpu by 10%
@@ -853,6 +854,7 @@ steps:
     Params:
       action: gather
       enableViolations: true
+      threshold: {{$NODE_LOCAL_DNS_LATENCY_THRESHOLD}}
 {{end}}
   - Identifier: SLOMeasurement
     Method: SLOMeasurement