feat: devMode setting to override resource requests/limits

I implemented for the network using a global variable, which is ugly, but it applies to the entire network and doesn't seem totally wrong.
I think it could be done by passing things down, but it will probably require some refactoring to simplify CAS and other stateful spec declaration.
I considered adding this in the default impls, but I don't like having a parameterized default, even though it might make it more clear than having to call `Self::network_default()`

keramik/src/advanced_configuration.md

@@ -70,7 +70,25 @@ and adding the following environment variables to the `spec/template/spec/contai
 
 # Image Resources
 
-You can also use the [network](./setup_network.md) specification to specify resources for the pods that are running
+During local benchmarking, you may not have enough resources to run the cluster. A simple "fix" is to use the `devMode` flag on the network and simulation specs. This will override the resource requests and limits values to be none, which means it doesn't need available resources to deploy, and can consume as much as it desires. This would be problematic in production and should only be used for testing purposes.
+
+```yaml
+# network configuration
+---
+apiVersion: "keramik.3box.io/v1alpha1"
+kind: Network
+metadata:
+  name: small
+spec:
+  replicas: 2
+  devMode: true # ceramic will require specified resources but all other containers will be unconstrained
+  ceramic:
+    - resourceLimits:
+        cpu: "1"
+        memory: "1Gi"
+        storage: "1Gi"
+```
+
 
 ```yaml
 # network configuration

keramik/src/simulation.md

@@ -23,6 +23,7 @@ metadata:
   namespace: keramik-<unique-name>-small
 spec:
   scenario: ceramic-simple
+  devMode: true # optional to remove container resource limits and requirements for local benchmarking
   users: 10
   runTime: 4
 ```

operator/src/monitoring/jaeger.rs

@@ -14,7 +14,7 @@ use k8s_openapi::{
     },
 };
 
-use crate::labels::selector_labels;
+use crate::{labels::selector_labels, network::resource_limits::ResourceLimitsConfig};
 
 pub const JAEGER_APP: &str = "jaeger";
 
@@ -33,7 +33,7 @@ pub fn service_spec() -> ServiceSpec {
     }
 }
 
-pub fn stateful_set_spec() -> StatefulSetSpec {
+pub fn stateful_set_spec(dev_mode: bool) -> StatefulSetSpec {
     StatefulSetSpec {
         replicas: Some(1),
         selector: LabelSelector {
@@ -66,19 +66,7 @@ pub fn stateful_set_spec() -> StatefulSetSpec {
                         value: Some("true".to_owned()),
                         ..Default::default()
                     }]),
-                    resources: Some(ResourceRequirements {
-                        limits: Some(BTreeMap::from_iter(vec![
-                            ("cpu".to_owned(), Quantity("250m".to_owned())),
-                            ("ephemeral-storage".to_owned(), Quantity("1Gi".to_owned())),
-                            ("memory".to_owned(), Quantity("1Gi".to_owned())),
-                        ])),
-                        requests: Some(BTreeMap::from_iter(vec![
-                            ("cpu".to_owned(), Quantity("250m".to_owned())),
-                            ("ephemeral-storage".to_owned(), Quantity("1Gi".to_owned())),
-                            ("memory".to_owned(), Quantity("1Gi".to_owned())),
-                        ])),
-                        ..Default::default()
-                    }),
+                    resources: Some(resource_requirements(dev_mode)),
                     ..Default::default()
                 }],
                 ..Default::default()
@@ -87,3 +75,27 @@ pub fn stateful_set_spec() -> StatefulSetSpec {
         ..Default::default()
     }
 }
+
+fn resource_requirements(dev_mode: bool) -> ResourceRequirements {
+    if dev_mode {
+        ResourceRequirements {
+            limits: Some(ResourceLimitsConfig::dev_default().into()),
+            requests: Some(ResourceLimitsConfig::dev_default().into()),
+            ..Default::default()
+        }
+    } else {
+        ResourceRequirements {
+            limits: Some(BTreeMap::from_iter(vec![
+                ("cpu".to_owned(), Quantity("250m".to_owned())),
+                ("ephemeral-storage".to_owned(), Quantity("1Gi".to_owned())),
+                ("memory".to_owned(), Quantity("1Gi".to_owned())),
+            ])),
+            requests: Some(BTreeMap::from_iter(vec![
+                ("cpu".to_owned(), Quantity("250m".to_owned())),
+                ("ephemeral-storage".to_owned(), Quantity("1Gi".to_owned())),
+                ("memory".to_owned(), Quantity("1Gi".to_owned())),
+            ])),
+            ..Default::default()
+        }
+    }
+}

operator/src/monitoring/opentelemetry.rs

@@ -17,7 +17,7 @@ use k8s_openapi::{
     },
 };
 
-use crate::labels::selector_labels;
+use crate::{labels::selector_labels, network::resource_limits::ResourceLimitsConfig};
 
 use crate::simulation::controller::{OTEL_ACCOUNT, OTEL_CONFIG_MAP_NAME, OTEL_CR};
 
@@ -54,7 +54,31 @@ pub fn service_spec() -> ServiceSpec {
     }
 }
 
-pub fn stateful_set_spec() -> StatefulSetSpec {
+fn resource_requirements(dev_mode: bool) -> ResourceRequirements {
+    if dev_mode {
+        ResourceRequirements {
+            limits: Some(ResourceLimitsConfig::dev_default().into()),
+            requests: Some(ResourceLimitsConfig::dev_default().into()),
+            ..Default::default()
+        }
+    } else {
+        ResourceRequirements {
+            limits: Some(BTreeMap::from_iter(vec![
+                ("cpu".to_owned(), Quantity("250m".to_owned())),
+                ("ephemeral-storage".to_owned(), Quantity("1Gi".to_owned())),
+                ("memory".to_owned(), Quantity("1Gi".to_owned())),
+            ])),
+            requests: Some(BTreeMap::from_iter(vec![
+                ("cpu".to_owned(), Quantity("250m".to_owned())),
+                ("ephemeral-storage".to_owned(), Quantity("1Gi".to_owned())),
+                ("memory".to_owned(), Quantity("1Gi".to_owned())),
+            ])),
+            ..Default::default()
+        }
+    }
+}
+
+pub fn stateful_set_spec(dev_mode: bool) -> StatefulSetSpec {
     StatefulSetSpec {
         replicas: Some(1),
         service_name: OTEL_APP.to_owned(),
@@ -98,19 +122,7 @@ pub fn stateful_set_spec() -> StatefulSetSpec {
                             ..Default::default()
                         },
                     ]),
-                    resources: Some(ResourceRequirements {
-                        limits: Some(BTreeMap::from_iter(vec![
-                            ("cpu".to_owned(), Quantity("250m".to_owned())),
-                            ("ephemeral-storage".to_owned(), Quantity("1Gi".to_owned())),
-                            ("memory".to_owned(), Quantity("1Gi".to_owned())),
-                        ])),
-                        requests: Some(BTreeMap::from_iter(vec![
-                            ("cpu".to_owned(), Quantity("250m".to_owned())),
-                            ("ephemeral-storage".to_owned(), Quantity("1Gi".to_owned())),
-                            ("memory".to_owned(), Quantity("1Gi".to_owned())),
-                        ])),
-                        ..Default::default()
-                    }),
+                    resources: Some(resource_requirements(dev_mode)),
                     volume_mounts: Some(vec![
                         VolumeMount {
                             mount_path: "/config".to_owned(),

operator/src/monitoring/prometheus.rs

@@ -13,13 +13,39 @@ use k8s_openapi::{
     },
 };
 
-use crate::simulation::controller::PROM_CONFIG_MAP_NAME;
+use crate::{
+    network::resource_limits::ResourceLimitsConfig, simulation::controller::PROM_CONFIG_MAP_NAME,
+};
 
 use crate::labels::selector_labels;
 
 pub const PROM_APP: &str = "prometheus";
 
-pub fn stateful_set_spec() -> StatefulSetSpec {
+fn resource_requirements(dev_mode: bool) -> ResourceRequirements {
+    if dev_mode {
+        ResourceRequirements {
+            limits: Some(ResourceLimitsConfig::dev_default().into()),
+            requests: Some(ResourceLimitsConfig::dev_default().into()),
+            ..Default::default()
+        }
+    } else {
+        ResourceRequirements {
+            limits: Some(BTreeMap::from_iter(vec![
+                ("cpu".to_owned(), Quantity("250m".to_owned())),
+                ("ephemeral-storage".to_owned(), Quantity("1Gi".to_owned())),
+                ("memory".to_owned(), Quantity("1Gi".to_owned())),
+            ])),
+            requests: Some(BTreeMap::from_iter(vec![
+                ("cpu".to_owned(), Quantity("250m".to_owned())),
+                ("ephemeral-storage".to_owned(), Quantity("1Gi".to_owned())),
+                ("memory".to_owned(), Quantity("1Gi".to_owned())),
+            ])),
+            ..Default::default()
+        }
+    }
+}
+
+pub fn stateful_set_spec(dev_mode: bool) -> StatefulSetSpec {
     StatefulSetSpec {
         replicas: Some(1),
         selector: LabelSelector {
@@ -45,19 +71,7 @@ pub fn stateful_set_spec() -> StatefulSetSpec {
                         name: Some("webui".to_owned()),
                         ..Default::default()
                     }]),
-                    resources: Some(ResourceRequirements {
-                        limits: Some(BTreeMap::from_iter(vec![
-                            ("cpu".to_owned(), Quantity("250m".to_owned())),
-                            ("ephemeral-storage".to_owned(), Quantity("1Gi".to_owned())),
-                            ("memory".to_owned(), Quantity("1Gi".to_owned())),
-                        ])),
-                        requests: Some(BTreeMap::from_iter(vec![
-                            ("cpu".to_owned(), Quantity("250m".to_owned())),
-                            ("ephemeral-storage".to_owned(), Quantity("1Gi".to_owned())),
-                            ("memory".to_owned(), Quantity("1Gi".to_owned())),
-                        ])),
-                        ..Default::default()
-                    }),
+                    resources: Some(resource_requirements(dev_mode)),
                     volume_mounts: Some(vec![VolumeMount {
                         mount_path: "/config".to_owned(),
                         name: "config".to_owned(),

operator/src/network/cas.rs

@@ -21,7 +21,7 @@ use crate::network::{
     controller::{
         CAS_APP, CAS_IPFS_APP, CAS_IPFS_SERVICE_NAME, CAS_POSTGRES_APP, CAS_POSTGRES_SERVICE_NAME,
         CAS_SERVICE_NAME, GANACHE_APP, GANACHE_SERVICE_NAME, LOCALSTACK_APP,
-        LOCALSTACK_SERVICE_NAME,
+        LOCALSTACK_SERVICE_NAME, NETWORK_DEV_MODE_RESOURCES,
     },
     datadog::DataDogConfig,
     ipfs::{IpfsConfig, IPFS_DATA_PV_CLAIM},
@@ -45,31 +45,47 @@ pub struct CasConfig {
     pub localstack_resource_limits: ResourceLimitsConfig,
 }
 
+impl CasConfig {
+    pub fn network_default() -> Self {
+        if NETWORK_DEV_MODE_RESOURCES.load(std::sync::atomic::Ordering::Relaxed) {
+            Self {
+                cas_resource_limits: ResourceLimitsConfig::dev_default(),
+                ganache_resource_limits: ResourceLimitsConfig::dev_default(),
+                postgres_resource_limits: ResourceLimitsConfig::dev_default(),
+                localstack_resource_limits: ResourceLimitsConfig::dev_default(),
+                ..Default::default()
+            }
+        } else {
+            Self::default()
+        }
+    }
+}
+
 // Define clear defaults for this config
 impl Default for CasConfig {
     fn default() -> Self {
         Self {
             image: "ceramicnetwork/ceramic-anchor-service:latest".to_owned(),
             image_pull_policy: "Always".to_owned(),
             cas_resource_limits: ResourceLimitsConfig {
-                cpu: Quantity("250m".to_owned()),
-                memory: Quantity("1Gi".to_owned()),
+                cpu: Some(Quantity("250m".to_owned())),
+                memory: Some(Quantity("1Gi".to_owned())),
                 storage: Quantity("1Gi".to_owned()),
             },
             ipfs: Default::default(),
             ganache_resource_limits: ResourceLimitsConfig {
-                cpu: Quantity("250m".to_owned()),
-                memory: Quantity("1Gi".to_owned()),
+                cpu: Some(Quantity("250m".to_owned())),
+                memory: Some(Quantity("1Gi".to_owned())),
                 storage: Quantity("1Gi".to_owned()),
             },
             postgres_resource_limits: ResourceLimitsConfig {
-                cpu: Quantity("250m".to_owned()),
-                memory: Quantity("512Mi".to_owned()),
+                cpu: Some(Quantity("250m".to_owned())),
+                memory: Some(Quantity("512Mi".to_owned())),
                 storage: Quantity("1Gi".to_owned()),
             },
             localstack_resource_limits: ResourceLimitsConfig {
-                cpu: Quantity("250m".to_owned()),
-                memory: Quantity("1Gi".to_owned()),
+                cpu: Some(Quantity("250m".to_owned())),
+                memory: Some(Quantity("1Gi".to_owned())),
                 storage: Quantity("1Gi".to_owned()),
             },
         }
@@ -87,7 +103,7 @@ impl From<Option<CasSpec>> for CasConfig {
 
 impl From<CasSpec> for CasConfig {
     fn from(value: CasSpec) -> Self {
-        let default = Self::default();
+        let default = Self::network_default();
         Self {
             image: value.image.unwrap_or(default.image),
             image_pull_policy: value.image_pull_policy.unwrap_or(default.image_pull_policy),

operator/src/network/ceramic.rs

@@ -30,6 +30,8 @@ use crate::{
 
 use crate::network::controller::{CERAMIC_SERVICE_API_PORT, CERAMIC_SERVICE_IPFS_PORT};
 
+use super::controller::NETWORK_DEV_MODE_RESOURCES;
+
 pub fn config_maps(
     info: &CeramicInfo,
     config: &CeramicConfig,
@@ -139,6 +141,19 @@ pub struct CeramicConfig {
     pub env: Option<BTreeMap<String, String>>,
 }
 
+impl CeramicConfig {
+    pub fn network_default() -> Self {
+        if NETWORK_DEV_MODE_RESOURCES.load(std::sync::atomic::Ordering::Relaxed) {
+            Self {
+                resource_limits: ResourceLimitsConfig::dev_default(),
+                ..Default::default()
+            }
+        } else {
+            Self::default()
+        }
+    }
+}
+
 /// Bundles all relevant config for a ceramic spec.
 pub struct CeramicBundle<'a> {
     pub info: CeramicInfo,
@@ -235,8 +250,8 @@ impl Default for CeramicConfig {
             image_pull_policy: "Always".to_owned(),
             ipfs: IpfsConfig::default(),
             resource_limits: ResourceLimitsConfig {
-                cpu: Quantity("250m".to_owned()),
-                memory: Quantity("1Gi".to_owned()),
+                cpu: Some(Quantity("250m".to_owned())),
+                memory: Some(Quantity("1Gi".to_owned())),
                 storage: Quantity("1Gi".to_owned()),
             },
             env: None,
@@ -250,19 +265,19 @@ impl From<Option<Vec<CeramicSpec>>> for CeramicConfigs {
     fn from(value: Option<Vec<CeramicSpec>>) -> Self {
         if let Some(value) = value {
             if value.is_empty() {
-                Self(vec![CeramicConfig::default()])
+                Self(vec![CeramicConfig::network_default()])
             } else {
                 Self(value.into_iter().map(CeramicConfig::from).collect())
             }
         } else {
-            Self(vec![CeramicConfig::default()])
+            Self(vec![CeramicConfig::network_default()])
         }
     }
 }
 
 impl From<CeramicSpec> for CeramicConfig {
     fn from(value: CeramicSpec) -> Self {
-        let default = Self::default();
+        let default = Self::network_default();
         Self {
             weight: value.weight.unwrap_or(default.weight),
             init_config_map: value.init_config_map.unwrap_or(default.init_config_map),

operator/src/network/controller.rs

@@ -1,4 +1,10 @@
-use std::{cmp::min, collections::BTreeMap, str::from_utf8, sync::Arc, time::Duration};
+use std::{
+    cmp::min,
+    collections::BTreeMap,
+    str::from_utf8,
+    sync::{atomic::AtomicBool, Arc},
+    time::Duration,
+};
 
 use anyhow::anyhow;
 use futures::stream::StreamExt;
@@ -82,6 +88,8 @@ pub const LOCALSTACK_APP: &str = "localstack";
 
 pub const BOOTSTRAP_JOB_NAME: &str = "bootstrap";
 
+pub(crate) static NETWORK_DEV_MODE_RESOURCES: AtomicBool = AtomicBool::new(false);
+
 /// Handle errors during reconciliation.
 fn on_error(
     _network: Arc<Network>,
@@ -176,6 +184,11 @@ async fn reconcile(
     let spec = network.spec();
     debug!(?spec, "reconcile");
 
+    NETWORK_DEV_MODE_RESOURCES.store(
+        spec.dev_mode.unwrap_or(false),
+        std::sync::atomic::Ordering::SeqCst,
+    );
+
     let mut status = if let Some(status) = &network.status {
         status.clone()
     } else {