fix: count both success and failures of reconile loop

3box · Feb 2, 2024 · 2fee683 · 2fee683
1 parent 06bbea0
commit 2fee683
Show file tree

Hide file tree

Showing 3 changed files with 58 additions and 7 deletions.
diff --git a/common/src/telemetry.rs b/common/src/telemetry.rs
@@ -127,7 +127,7 @@ pub async fn init_metrics_prom(
     Ok((meter, shutdown, join))
 }
 
-// /metrics scrape endpoin
+// /metrics scrape endpoint.
 async fn handle(_req: Request<Body>) -> Result<Response<Body>, Infallible> {
     let mut data = Vec::new();
     let encoder = TextEncoder::new();

diff --git a/operator/src/network/controller.rs b/operator/src/network/controller.rs
@@ -31,7 +31,7 @@ use kube::{
     },
     Resource,
 };
-use opentelemetry::global;
+use opentelemetry::{global, KeyValue};
 use rand::RngCore;
 use tracing::{debug, error, info, trace, warn};
 
@@ -208,8 +208,33 @@ async fn reconcile(
         .u64_counter("network_reconcile_count")
         .with_description("Number of network reconciles")
         .init();
-    runs.add(1, &[]);
-
+    match reconcile_(network, cx).await {
+        Ok(action) => {
+            runs.add(
+                1,
+                &[KeyValue {
+                    key: "result".into(),
+                    value: "ok".into(),
+                }],
+            );
+            Ok(action)
+        }
+        Err(err) => {
+            runs.add(
+                1,
+                &[KeyValue {
+                    key: "result".into(),
+                    value: "err".into(),
+                }],
+            );
+            Err(err)
+        }
+    }
+}
+async fn reconcile_(
+    network: Arc<Network>,
+    cx: Arc<Context<impl IpfsRpcClient, impl RngCore, impl Clock>>,
+) -> Result<Action, Error> {
     let spec = network.spec();
     debug!(?spec, "reconcile");
 

diff --git a/operator/src/simulation/controller.rs b/operator/src/simulation/controller.rs
@@ -21,7 +21,7 @@ use kube::{
     },
     Resource, ResourceExt,
 };
-use opentelemetry::global;
+use opentelemetry::{global, KeyValue};
 use rand::{distributions::Alphanumeric, thread_rng, Rng, RngCore};
 
 use tracing::{debug, error, info};
@@ -123,7 +123,6 @@ pub async fn run() {
         })
         .await;
 }
-
 /// Perform a reconile pass for the Simulation CRD
 async fn reconcile(
     simulation: Arc<Simulation>,
@@ -134,8 +133,35 @@ async fn reconcile(
         .u64_counter("simulation_reconcile_count")
         .with_description("Number of simulation reconciles")
         .init();
-    runs.add(1, &[]);
 
+    match reconcile_(simulation, cx).await {
+        Ok(action) => {
+            runs.add(
+                1,
+                &[KeyValue {
+                    key: "result".into(),
+                    value: "ok".into(),
+                }],
+            );
+            Ok(action)
+        }
+        Err(err) => {
+            runs.add(
+                1,
+                &[KeyValue {
+                    key: "result".into(),
+                    value: "err".into(),
+                }],
+            );
+            Err(err)
+        }
+    }
+}
+/// Perform a reconile pass for the Simulation CRD
+async fn reconcile_(
+    simulation: Arc<Simulation>,
+    cx: Arc<Context<impl IpfsRpcClient, impl RngCore, impl Clock>>,
+) -> Result<Action, Error> {
     let spec = simulation.spec();
 
     let status = if let Some(status) = &simulation.status {