From 2fee6835b42cc0692c9e2ed5f46dc15c8573bb84 Mon Sep 17 00:00:00 2001 From: Nathaniel Cook Date: Thu, 1 Feb 2024 16:06:16 -0700 Subject: [PATCH] fix: count both success and failures of reconile loop --- common/src/telemetry.rs | 2 +- operator/src/network/controller.rs | 31 +++++++++++++++++++++++--- operator/src/simulation/controller.rs | 32 ++++++++++++++++++++++++--- 3 files changed, 58 insertions(+), 7 deletions(-) diff --git a/common/src/telemetry.rs b/common/src/telemetry.rs index cc363658..47be0949 100644 --- a/common/src/telemetry.rs +++ b/common/src/telemetry.rs @@ -127,7 +127,7 @@ pub async fn init_metrics_prom( Ok((meter, shutdown, join)) } -// /metrics scrape endpoin +// /metrics scrape endpoint. async fn handle(_req: Request) -> Result, Infallible> { let mut data = Vec::new(); let encoder = TextEncoder::new(); diff --git a/operator/src/network/controller.rs b/operator/src/network/controller.rs index 8fa878fa..af003bca 100644 --- a/operator/src/network/controller.rs +++ b/operator/src/network/controller.rs @@ -31,7 +31,7 @@ use kube::{ }, Resource, }; -use opentelemetry::global; +use opentelemetry::{global, KeyValue}; use rand::RngCore; use tracing::{debug, error, info, trace, warn}; @@ -208,8 +208,33 @@ async fn reconcile( .u64_counter("network_reconcile_count") .with_description("Number of network reconciles") .init(); - runs.add(1, &[]); - + match reconcile_(network, cx).await { + Ok(action) => { + runs.add( + 1, + &[KeyValue { + key: "result".into(), + value: "ok".into(), + }], + ); + Ok(action) + } + Err(err) => { + runs.add( + 1, + &[KeyValue { + key: "result".into(), + value: "err".into(), + }], + ); + Err(err) + } + } +} +async fn reconcile_( + network: Arc, + cx: Arc>, +) -> Result { let spec = network.spec(); debug!(?spec, "reconcile"); diff --git a/operator/src/simulation/controller.rs b/operator/src/simulation/controller.rs index a0f7b8f1..848d7ed8 100644 --- a/operator/src/simulation/controller.rs +++ b/operator/src/simulation/controller.rs @@ -21,7 +21,7 @@ use kube::{ }, Resource, ResourceExt, }; -use opentelemetry::global; +use opentelemetry::{global, KeyValue}; use rand::{distributions::Alphanumeric, thread_rng, Rng, RngCore}; use tracing::{debug, error, info}; @@ -123,7 +123,6 @@ pub async fn run() { }) .await; } - /// Perform a reconile pass for the Simulation CRD async fn reconcile( simulation: Arc, @@ -134,8 +133,35 @@ async fn reconcile( .u64_counter("simulation_reconcile_count") .with_description("Number of simulation reconciles") .init(); - runs.add(1, &[]); + match reconcile_(simulation, cx).await { + Ok(action) => { + runs.add( + 1, + &[KeyValue { + key: "result".into(), + value: "ok".into(), + }], + ); + Ok(action) + } + Err(err) => { + runs.add( + 1, + &[KeyValue { + key: "result".into(), + value: "err".into(), + }], + ); + Err(err) + } + } +} +/// Perform a reconile pass for the Simulation CRD +async fn reconcile_( + simulation: Arc, + cx: Arc>, +) -> Result { let spec = simulation.spec(); let status = if let Some(status) = &simulation.status {