fix: New long tail assets configuration (#37)

* fix(long_tail_assets): New alerts for long tail assets * fix(long_tail_assets): Readable config * fix(long_tail_assets): misc * fix(long_tail_assets): Lint * fix(long_tail_assets): fixes from review * fix(long_tail_assets): Removed magic value * fix(long_tail_assets): Track individual sources deviation * fix(long_tail_assets): Alert name * fix(long_tail_assets): unused vec * fix(long_tail_assets): README
astraly-labs · Jul 29, 2024 · 1d0c6c4 · 1d0c6c4
1 parent 5435247
commit 1d0c6c4
Show file tree

Hide file tree

Showing 9 changed files with 300 additions and 227 deletions.
diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml
@@ -60,7 +60,7 @@ jobs:
         with:
           context: .
           file: Dockerfile
-          platforms:  ${{ format('linux/{0}', matrix.platform) }}
+          platforms: ${{ format('linux/{0}', matrix.platform) }}
           labels: ${{ steps.meta.outputs.labels }}
           cache-from: type=registry,ref=${{ env.REGISTRY_IMAGE }}:cache-${{ matrix.platform }},mode=max
           cache-to: type=registry,ref=${{ env.REGISTRY_IMAGE }}:cache-${{ matrix.platform }},mode=max

diff --git a/README.md b/README.md
@@ -13,8 +13,9 @@ It then processes the data and computes the following metrics:
 - `time_since_last_update_pair_id{network, pair, type}`: Time since an update has been published for a given pair. (in seconds)
 - `price_deviation{network, pair, source, type}`: Deviation of the price from a reference price (DefiLlama API) given source and pair. (in percents)
 - `price_deviation_source{network, pair, source, type}`: Deviation of the price from the on-chain aggregated median price given source and pair. (in percents)
-- `long_tail_asset_threshold{pair}`: Deviation threshold configuration for long tail assets.
-- `long_tail_asset_deviation{network, pair, type, source1, source2}`: Deviation between two sources for long tail assets.
+- `long_tail_asset_threshold{pair, type}`: Deviation threshold configuration for long tail assets. Type can be either "low" for when the pair has less than 7 sources else "high".
+- `long_tail_asset_source_deviation{network, pair, type}`: Deviation of a source from the on-chain aggregated median price given source and pair. (in percents)
+- `long_tail_asset_total_sources{network, pair, type}`: Current number of sources available for a given pair.
 - `publisher_balance{network, publisher}`: Balance of a publisher. (in ETH)
 - `vrf_balance{network}`: Balance of the VRF contract. (in ETH)
 - `vrf_requests_count{network, status}`: Number of VRF requests handled for a given network.

diff --git a/prometheus/alerts.rules.yml b/prometheus/alerts.rules.yml
@@ -64,14 +64,58 @@ groups:
           summary: "Price deviation is too high"
           description: "The median on-chain price of {{ $labels.pair }} has deviated for more than 2.5% with the reference price from DefiLlama."
 
-      - alert: LongTailAssetDeviation
-        expr: abs(long_tail_asset_deviation) > on(pair) group_left long_tail_asset_threshold
+      - alert: LongTailAssetSourceDeviation
+        expr: |
+          (
+            abs(long_tail_asset_source_deviation) > on(pair) group_left long_tail_asset_threshold{type="low"}
+            and
+            long_tail_asset_total_sources <= 6
+          )
+          or
+          (
+            abs(long_tail_asset_source_deviation) > on(pair) group_left long_tail_asset_threshold{type="high"}
+            and
+            long_tail_asset_total_sources > 6
+          )
         for: 5m
         labels:
           severity: warning
         annotations:
-          summary: "Long tail asset deviation is too high"
-          description: 'The deviation between sources for {{ $labels.pair }} ({{ $labels.type }}) from {{ $labels.source1 }} vs {{ $labels.source2 }} has exceeded the configured threshold of {{ $value | printf "%.2f" }}.'
+          summary: "Source deviation is too high"
+          description: |
+            {{ $total := long_tail_asset_total_sources }}
+            {{ $threshold := long_tail_asset_threshold{type="low"} }}
+            {{ if gt $total 6.0 }}
+              {{ $threshold = long_tail_asset_threshold{type="high"} }}
+            {{ end }}
+            Source {{ $labels.source }} for {{ $labels.pair }} ({{ $labels.type }}) has deviated from our price by {{ printf "%.2f" ($value * 100) }}%, which is beyond the threshold of {{ printf "%.2f" ($threshold * 100) }}%.
+
+      - alert: LongTailAssetMultipleSourcesDeviation
+        expr: |
+          (
+            count(abs(long_tail_asset_source_deviation) > on(pair) group_left long_tail_asset_threshold{type="low"}) by (network, pair, type) >= 2
+            and
+            long_tail_asset_total_sources <= 6
+          )
+          or
+          (
+            count(abs(long_tail_asset_source_deviation) > on(pair) group_left long_tail_asset_threshold{type="high"}) by (network, pair, type) / long_tail_asset_total_sources >= 0.25
+            and
+            long_tail_asset_total_sources > 6
+          )
+        for: 5m
+        labels:
+          severity: critical
+        annotations:
+          summary: "Too much sources have deviated"
+          description: |
+            {{ $deviating := count(abs(long_tail_asset_source_deviation) > on(pair) group_left long_tail_asset_threshold{type="low"}) by (network, pair, type) }}
+            {{ $total := long_tail_asset_total_sources }}
+            {{ $threshold := long_tail_asset_threshold{type="low"} }}
+            {{ if gt $total 6.0 }}
+              {{ $threshold = long_tail_asset_threshold{type="high"} }}
+            {{ end }}
+            {{ printf "%.0f" $deviating }} out of {{ printf "%.0f" $total }} sources for {{ $labels.pair }} ({{ $labels.type }}) have deviated from our price beyond {{ printf "%.2f" (100 * $threshold) }}%.
 
   - name: API
     rules:

diff --git a/src/config.rs b/src/config.rs
@@ -19,8 +19,10 @@ use tokio::sync::OnceCell;
 use url::Url;
 
 use crate::{
-    constants::{CONFIG_UPDATE_INTERVAL, LONG_TAIL_ASSETS, LONG_TAIL_ASSET_THRESHOLD},
-    utils::try_felt_to_u32,
+    constants::{
+        CONFIG_UPDATE_INTERVAL, LONG_TAIL_ASSETS, LONG_TAIL_ASSET_THRESHOLD, LOW_SOURCES_THRESHOLD,
+    },
+    utils::{is_long_tail_asset, try_felt_to_u32},
 };
 
 #[derive(Debug, Clone, EnumString, IntoStaticStr)]
@@ -451,10 +453,28 @@ async fn init_future_config(
 /// fetched from LONG_TAIL_ASSETS.
 /// TODO: LONG_TAIL_ASSETS should be an independent (db, yaml...) configuration?
 pub fn init_long_tail_asset_configuration() {
-    for (pair, threshold) in LONG_TAIL_ASSETS.iter() {
+    for (pair, (threshold_low, threshold_high)) in LONG_TAIL_ASSETS.iter() {
+        LONG_TAIL_ASSET_THRESHOLD
+            .with_label_values(&[pair, "low"])
+            .set(*threshold_low);
         LONG_TAIL_ASSET_THRESHOLD
-            .with_label_values(&[pair])
-            .set(*threshold);
+            .with_label_values(&[pair, "high"])
+            .set(*threshold_high);
+    }
+}
+
+#[allow(dead_code)]
+/// Retrieves the long tail asset threshold configuration depending on the number of sources.
+pub fn get_long_tail_threshold(pair: &str, number_of_sources: usize) -> Option<f64> {
+    if !is_long_tail_asset(pair) {
+        return None;
+    };
+    let (threshold_low, threshold_high) = LONG_TAIL_ASSETS.get(pair).unwrap();
+
+    if number_of_sources <= LOW_SOURCES_THRESHOLD {
+        Some(*threshold_low)
+    } else {
+        Some(*threshold_high)
     }
 }
 

diff --git a/src/constants.rs b/src/constants.rs
@@ -4,6 +4,8 @@ use lazy_static::lazy_static;
 use phf::phf_map;
 use prometheus::{opts, register_gauge_vec, register_int_gauge_vec, GaugeVec, IntGaugeVec};
 
+pub(crate) static LOW_SOURCES_THRESHOLD: usize = 6;
+
 #[allow(unused)]
 pub(crate) static COINGECKO_IDS: phf::Map<&'static str, &'static str> = phf_map! {
     "BTC/USD" => "bitcoin",
@@ -25,12 +27,16 @@ lazy_static! {
     /// We should probably store them either in a yaml config file or a
     /// database (cons of a database => update the threshold/pairs without restarting
     /// the monitoring service).
-    pub static ref LONG_TAIL_ASSETS: HashMap<String, f64> = {
+    ///
+    /// Stores the threshold for when:
+    ///     - `low`: the pair has 6 sources or less
+    ///     - `high`: the pair has more than 6 sources.
+    pub static ref LONG_TAIL_ASSETS: HashMap<String, (f64, f64)> = {
         let mut map = HashMap::new();
-        map.insert("ZEND/USD".to_string(), 0.05);
-        map.insert("NSTR/USD".to_string(), 0.05);
-        map.insert("LUSD/USD".to_string(), 0.05);
-        map.insert("LORDS/USD".to_string(), 0.05);
+        map.insert("ZEND/USD".to_string(), (0.05, 0.03));
+        map.insert("NSTR/USD".to_string(), (0.05, 0.03));
+        map.insert("LUSD/USD".to_string(), (0.05, 0.03));
+        map.insert("LORDS/USD".to_string(), (0.05, 0.03));
         map
     };
 
@@ -54,18 +60,23 @@ lazy_static! {
             "long_tail_asset_threshold",
             "Deviation threshold configuration for long tail assets"
         ),
-        &["pair"]
+        &["pair", "type"]
     )
     .unwrap();
-
-    pub static ref LONG_TAIL_ASSET_DEVIATION: GaugeVec = register_gauge_vec!(
+    pub static ref LONG_TAIL_ASSET_SOURCE_DEVIATION: GaugeVec = register_gauge_vec!(
         opts!(
-            "long_tail_asset_deviation",
-            "Deviation between two sources for long tail assets"
+            "long_tail_asset_source_deviation",
+            "Deviation of each source from our onchain aggregated price for long tail assets"
         ),
-        &["network", "pair", "type", "source1", "source2"]
-    )
-    .unwrap();
+        &["network", "pair", "type", "source"]
+    ).unwrap();
+    pub static ref LONG_TAIL_ASSET_TOTAL_SOURCES: GaugeVec = register_gauge_vec!(
+        opts!(
+            "long_tail_asset_total_sources",
+            "Total number of sources for long tail assets"
+        ),
+        &["network", "pair", "type"]
+    ).unwrap();
 
     // Regular metrics below
 

diff --git a/src/main.rs b/src/main.rs
@@ -179,6 +179,7 @@ pub(crate) async fn onchain_monitor(
                         ]
                     }
                 }
+                // TODO: Long tail assets aren't treated as such for Future data
                 DataType::Future => {
                     vec![
                         tokio::spawn(Box::pin(processing::future::process_data_by_pair(
-Original file line number
+Diff line change
@@ Expand Up / @@ -179,6 +179,7 @@ pub(crate) async fn onchain_monitor( @@
                             ]
                         }
                     }
+                    // TODO: Long tail assets aren't treated as such for Future data
                     DataType::Future => {
                         vec![
                             tokio::spawn(Box::pin(processing::future::process_data_by_pair(
@@ Expand Down @@