diff --git a/advanced_configuration.html b/advanced_configuration.html index 86f4a725..c0511e63 100644 --- a/advanced_configuration.html +++ b/advanced_configuration.html @@ -264,6 +264,21 @@
The CAS API environment variables can be set or overridden through the network configuration.
+# network configuration
+---
+apiVersion: "keramik.3box.io/v1alpha1"
+kind: Network
+metadata:
+ name: small
+spec:
+ replicas: 0
+ cas:
+ api:
+ env:
+ APP_PORT: "8080"
+
You can also use Recon for reconciliation by setting 'CERAMIC_ONE_RECON' env variable to true.
# network configuration
diff --git a/print.html b/print.html
index de183b22..73f40390 100644
--- a/print.html
+++ b/print.html
@@ -701,6 +701,21 @@ Image Resourc
cpu: "250m"
memory: "1Gi"
+The CAS API environment variables can be set or overridden through the network configuration.
+# network configuration
+---
+apiVersion: "keramik.3box.io/v1alpha1"
+kind: Network
+metadata:
+ name: small
+spec:
+ replicas: 0
+ cas:
+ api:
+ env:
+ APP_PORT: "8080"
+
You can also use Recon for reconciliation by setting 'CERAMIC_ONE_RECON' env variable to true.
# network configuration
diff --git a/searchindex.js b/searchindex.js
index 7a826cce..c4e903db 100644
--- a/searchindex.js
+++ b/searchindex.js
@@ -1 +1 @@
-Object.assign(window.search, {"doc_urls":["environment.html#environment","environment.html#local-environment","environment.html#eks","create_cluster.html#creating-a-cluster","deploy_keramik.html#deploy-keramik","deploy_keramik.html#deploy-crds","deploy_keramik.html#deploy-keramik--operator","setup_network.html#setting-up-a-network","setup_network.html#checking-the-status-of-the-network","simulation.html#simulation","simulation.html#simulating-specific-versions","simulation.html#example-custom-js-ceramic-image","simulation.html#example-custom-ipfs-image","analysis.html#analysis","analysis.html#quick-log-analysis","analysis.html#analysis-with-duckdb-or-jupyter","analysis.html#comparing-simulation-runs","analysis.html#performance-analysis","datadog.html#datadog","datadog.html#cleanup","developing.html#developing-keramik","developing_operator.html#operator","developing_runner.html#runner","developing_runner.html#build-and-load-the-runner-image","developing_runner.html#setup-network-with-runner-image","developing_runner.html#setup-simulation-with-runner-image","advanced.html#advanced-topics","advanced_configuration.html#advanced-cas-and-ceramic-configuration","advanced_configuration.html#adjusting-ceramic-environment","advanced_configuration.html#disabling-aws-functionality","advanced_configuration.html#image-resources","advanced_configuration.html#enabling-recon","monitoring.html#monitoring","monitoring.html#exposed-metrics","monitoring.html#pod-monitoring","ipfs.html#ipfs","ipfs.html#rust-ipfs","ipfs.html#ceramic","ipfs.html#cas","ipfs.html#kubo-ipfs","ipfs.html#ceramic","ipfs.html#cas","mixed_networks.html#mixed-networks","mixed_networks.html#examples","mixed_networks.html#mixed-ipfs","mixed_networks.html#mixed-js-ceramic","mixed_networks.html#weights","mixed_networks.html#examples","secrets.html#specifying-a-ceramic-admin-secret","operator.html#operator-patterns","operator.html#specs-statuses-and-configs","operator.html#assembled-nodes","migration.html#migration-tests","advanced_bootstrap.html#advanced-bootstrap-configuration","advanced_bootstrap.html#disable-bootstrap"],"index":{"documentStore":{"docInfo":{"0":{"body":5,"breadcrumbs":3,"title":1},"1":{"body":12,"breadcrumbs":4,"title":2},"10":{"body":14,"breadcrumbs":5,"title":3},"11":{"body":43,"breadcrumbs":7,"title":5},"12":{"body":36,"breadcrumbs":6,"title":4},"13":{"body":18,"breadcrumbs":4,"title":1},"14":{"body":68,"breadcrumbs":6,"title":3},"15":{"body":87,"breadcrumbs":6,"title":3},"16":{"body":99,"breadcrumbs":6,"title":3},"17":{"body":7,"breadcrumbs":5,"title":2},"18":{"body":278,"breadcrumbs":6,"title":1},"19":{"body":8,"breadcrumbs":6,"title":1},"2":{"body":59,"breadcrumbs":3,"title":1},"20":{"body":31,"breadcrumbs":4,"title":2},"21":{"body":86,"breadcrumbs":4,"title":1},"22":{"body":32,"breadcrumbs":4,"title":1},"23":{"body":30,"breadcrumbs":7,"title":4},"24":{"body":34,"breadcrumbs":7,"title":4},"25":{"body":29,"breadcrumbs":7,"title":4},"26":{"body":22,"breadcrumbs":4,"title":2},"27":{"body":111,"breadcrumbs":10,"title":4},"28":{"body":24,"breadcrumbs":9,"title":3},"29":{"body":38,"breadcrumbs":9,"title":3},"3":{"body":60,"breadcrumbs":4,"title":2},"30":{"body":181,"breadcrumbs":8,"title":2},"31":{"body":26,"breadcrumbs":8,"title":2},"32":{"body":64,"breadcrumbs":4,"title":1},"33":{"body":51,"breadcrumbs":5,"title":2},"34":{"body":43,"breadcrumbs":5,"title":2},"35":{"body":10,"breadcrumbs":4,"title":1},"36":{"body":0,"breadcrumbs":5,"title":2},"37":{"body":63,"breadcrumbs":4,"title":1},"38":{"body":64,"breadcrumbs":4,"title":1},"39":{"body":0,"breadcrumbs":5,"title":2},"4":{"body":11,"breadcrumbs":6,"title":2},"40":{"body":99,"breadcrumbs":4,"title":1},"41":{"body":99,"breadcrumbs":4,"title":1},"42":{"body":19,"breadcrumbs":6,"title":2},"43":{"body":0,"breadcrumbs":5,"title":1},"44":{"body":27,"breadcrumbs":6,"title":2},"45":{"body":31,"breadcrumbs":7,"title":3},"46":{"body":63,"breadcrumbs":5,"title":1},"47":{"body":109,"breadcrumbs":5,"title":1},"48":{"body":148,"breadcrumbs":7,"title":4},"49":{"body":5,"breadcrumbs":6,"title":2},"5":{"body":22,"breadcrumbs":6,"title":2},"50":{"body":360,"breadcrumbs":7,"title":3},"51":{"body":57,"breadcrumbs":6,"title":2},"52":{"body":59,"breadcrumbs":6,"title":2},"53":{"body":0,"breadcrumbs":8,"title":3},"54":{"body":26,"breadcrumbs":7,"title":2},"6":{"body":29,"breadcrumbs":7,"title":3},"7":{"body":57,"breadcrumbs":7,"title":3},"8":{"body":79,"breadcrumbs":7,"title":3},"9":{"body":221,"breadcrumbs":3,"title":1}},"docs":{"0":{"body":"Keramik can be setup in a local or eks environment.","breadcrumbs":"Environment Setup » Environment","id":"0","title":"Environment"},"1":{"body":"Requires rust kind kubectl docker protoc When using a local environment, you will need to create a cluster","breadcrumbs":"Environment Setup » Local Environment","id":"1","title":"Local Environment"},"10":{"body":"Often you will want to run a simulation against a specific version of software. To do this you will need to build the image and configure your network to run that image.","breadcrumbs":"Running a Simulation » Simulating Specific Versions","id":"10","title":"Simulating Specific Versions"},"11":{"body":"Use this example network definition with a custom js-ceramic image. # custom-js-ceramic.yaml\n---\napiVersion: \"keramik.3box.io/v1alpha1\"\nkind: Network\nmetadata: name: custom-js-ceramic\nspec: replicas: 2 monitoring: namespaced: true ceramic: - image: ceramicnetwork/composedb:dev imagePullPolicy: IfNotPresent kubectl apply -f custom-js-ceramic.yaml You can also run mixed networks and various other advanced configurations.","breadcrumbs":"Running a Simulation » Example Custom JS-Ceramic Image","id":"11","title":"Example Custom JS-Ceramic Image"},"12":{"body":"Use this example network definition with a custom IPFS image. # custom-ipfs.yaml\n---\napiVersion: \"keramik.3box.io/v1alpha1\"\nkind: Network\nmetadata: name: custom-ipfs\nspec: replicas: 2 monitoring: namespaced: true ceramic: - ipfs: rust: image: ceramicnetwork/rust-ceramic:dev imagePullPolicy: IfNotPresent kubectl apply -f custom-ipfs.yaml","breadcrumbs":"Running a Simulation » Example Custom IPFS Image","id":"12","title":"Example Custom IPFS Image"},"13":{"body":"Analysis of Keramik results depends on the purpose of the simulation. You may want to just see average latencies, or dive deeper into reported metrics. For profiling, you will want to use datadog .","breadcrumbs":"Analyzing Simulation Runs » Analysis","id":"13","title":"Analysis"},"14":{"body":"The simulation manager provides a very quick way to analyze the logs of a simulation run. You will need to know the name of the manager pod though. You will first need to see if the simulate-manager pod has completed, by running kubectl get pods If the pod has completed and is no longer in that list, you can see recently terminated pods using: kubectl get event -o custom-columns=NAME:.metadata.name | cut -d \".\" -f1 Once you have the name of the manager, you can retrieve its logs kubectl logs simulate-manager- If the simulate-manager pod is not in your pod list, you may need to get logs with the --previous flag: kubectl logs --previous simulate-manager-","breadcrumbs":"Analyzing Simulation Runs » Quick Log Analysis","id":"14","title":"Quick Log Analysis"},"15":{"body":"First you will need to install a few things: pip install duckdb duckdb-engine pandas jupyter jupysql matplotlib To analyze the results of a simulation first copy the metrics-TIMESTAMP.parquet file from the otel-0 pod. First restart opentelemetry-0 pod so it writes out the parquet file footer. kubectl delete pod opentelemetry-0\nkubectl wait --for=condition=Ready pod/opentelemetry-0 # make sure pod has restarted\nkubectl exec opentelemetry-0 -- ls -la /data # List files in the directly find the TIMESTAMP you need\nkubectl cp opentelemetry-0:data/metrics-TIMESTAMP.parquet ./analyze/metrics.parquet\ncd analyze Use duckdb to examine the data: duckdb\n> SELECT * FROM 'metrics.parquet' LIMIT 10; Alternatively start a jupyter notebook using analyze/sim.ipynb: jupyter notebook","breadcrumbs":"Analyzing Simulation Runs » Analysis with DuckDB or Jupyter","id":"15","title":"Analysis with DuckDB or Jupyter"},"16":{"body":"How do we conclude a simulation is better or worse that another run? Each simulation will likely be targeting a specific result however there are common results we should expect to see. Changes should not make correctness worse. Correctness is defined using two metrics: Percentage of events successfully persisted on the node that accepted the initial write. Percentage of events successfully replicated on nodes that observed the writes via the Ceramic protocol. Changes should not make performance worse. Performance is defined using these metrics: Writes/sec across all nodes in the cluster and by node p50,p90,p95,p99 and p99.9 of the duration of writes across all nodes in the cluster and by node Success/failure ratio of writes requests across all nodes in the cluster and by node p50,p90,p95,p99 and p99.9 of duration of time to become replicated. The time from when one node accepts the write to when another node has the same write available for read. For any simulation of the Ceramic protocol these metrics should apply. Any report about the results of a simulation should include these metrics and we compare them against the established a baseline.","breadcrumbs":"Analyzing Simulation Runs » Comparing Simulation Runs","id":"16","title":"Comparing Simulation Runs"},"17":{"body":"In addition to the above, we can also use datadog to dive further into performance.","breadcrumbs":"Analyzing Simulation Runs » Performance Analysis","id":"17","title":"Performance Analysis"},"18":{"body":"Keramik can also be configured to send metrics and telemetry data to datadog. You will first need to setup a barebones network that we can install the datadog operator into. An example barebones network from the above setup: apiVersion: \"keramik.3box.io/v1alpha1\"\nkind: Network\nmetadata: name: \nspec: replicas: 1 datadog: enabled: true version: \"unique_value\" profilingEnabled: true You will need to install the datadog k8s operator into the network. This requires installing helm, there doesn't seem to be any other way to install the operator without first installing helm. However once the datadog operator is installed helm is no longer needed. helm repo add datadog https://helm.datadoghq.com\nhelm install my-datadog-operator datadog/datadog-operator Now we will use that barebones network to setup secrets for datadog, and the datadog agent. Adjust the previously defined network definition to look like the following: # Network setup\n---\napiVersion: \"keramik.3box.io/v1alpha1\"\nkind: Network\nmetadata: name: small\nspec: replicas: 2 datadog: enabled: true version: \"unique_value\" profilingEnabled: true # Secrets Setup\n---\napiVersion: v1\nkind: Secret\nmetadata: name: datadog-secret\ntype: Opaque\nstringData: api-key: app-key: # Datadog Agent setup\n---\nkind: DatadogAgent\napiVersion: datadoghq.com/v2alpha1\nmetadata: name: datadog\nspec: global: kubelet: tlsVerify: false site: us3.datadoghq.com credentials: apiSecret: secretName: datadog-secret keyName: api-key appSecret: secretName: datadog-secret keyName: app-key override: clusterAgent: image: name: gcr.io/datadoghq/cluster-agent:latest nodeAgent: image: name: gcr.io/datadoghq/agent:latest features: npm: enabled: true apm: enabled: true hostPortConfig: enabled: true The Datadog API Key is found at the organization level, and should be the secret associated with the API Key. The Datadog application key can be found at the organization or user level, and should be the secret associated with the application key. You can now apply this with kubectl apply -f network.yaml Note If you are running locally, you will need to restart your CAS and Ceramic pods using kubectl delete pod ceramic-0 ceramic-1 cas-0 where the ceramic pods will depend on the replicas used. Make sure you delete all Ceramic and CAS pods. This only needs to be done the Anytime you need to change the network, change this file, then reapply it with kubectl apply -f network.yaml Telemetry data sent to datadog will have two properties to uniquely identifiy the data from other keramik networks. env - this is set based on the namespace of the keramik network. version - specified in the datadog config, may be any unique value.","breadcrumbs":"Analyzing Simulation Runs » Analysis with Datadog » Datadog","id":"18","title":"Datadog"},"19":{"body":"kubectl delete -f network.yaml\nhelm delete my-datadog-operator","breadcrumbs":"Analyzing Simulation Runs » Analysis with Datadog » Cleanup","id":"19","title":"Cleanup"},"2":{"body":"Requires eks - https://eksctl.io/introduction/#installation aws cli - https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html Once these are installed, you will need to login with aws cli via sso aws configure sso You will need to use https://3box.awsapps.com/start/ for the sso url with region us-east-2. Use account Benchmarking with role AWSAdministratorAccess. It is recommended to rename the profile to keramik or benchmarking. You can now find namespaces with aws eks update-kubeconfig --region=us-east-1 --profile=keramik --name=benchmarking-ceramic When using an eks environment, you do not need to create a cluster. You will need to setup a network .","breadcrumbs":"Environment Setup » EKS","id":"2","title":"EKS"},"20":{"body":"When you need to add features to Keramik networks or simulations you will need to run local builds of the operator and the runner. Operator - long lived process that manages the network custom resource. Runner - short lived process that performs various tasks within the network (i.e. bootstrapping)","breadcrumbs":"Developing Keramik » Developing Keramik","id":"20","title":"Developing Keramik"},"21":{"body":"The operator automates creating and manipulating networks via custom resource definition. Any changes to the operator require that you rebuild it and load it into kind again. docker buildx build --load -t keramik/operator:dev --target operator .\nkind load docker-image keramik/operator:dev Now we need to update the k8s operator definition to use our new image: Edit ./k8s/operator/kustomization.yaml to use the dev tag images: - name: keramik/operator newTag: dev Edit ./k8s/operator/manifests/operator.yaml to use IfNotPresent for the imagePullPolicy. # ... containers: - name: keramik-operator image: \"keramik/operator\" imagePullPolicy: IfNotPresent\n# ... Update the CRD definitions and apply the Keramik operator: cargo run --bin crdgen | kubectl apply -f -\nkubectl apply -k ./k8s/operator/ See the operator background for details on certain design patterns of the operator.","breadcrumbs":"Developing Keramik » Operator » Operator","id":"21","title":"Operator"},"22":{"body":"The runner is a utility for running various jobs to initialize the network and run workloads against it. Currently the runner provides two utilites: Bootstrap nodes Run simulations If you intend to develop either of these features you will need to build the runner image and configure your network or simulation to use your local image.","breadcrumbs":"Developing Keramik » Runner » Runner","id":"22","title":"Runner"},"23":{"body":"The runner is a utility for running various jobs to initialize the network and run workloads against it. Any changes to the runner require that you rebuild it and load it into kind again. docker buildx build --load -t keramik/runner:dev --target runner .\nkind load docker-image keramik/runner:dev","breadcrumbs":"Developing Keramik » Runner » Build and Load the Runner Image","id":"23","title":"Build and Load the Runner Image"},"24":{"body":"To use a custom runner image when you setup your network , you will need to adjust the yaml you use to specify how to bootstrap the runner. # small.yaml\n---\napiVersion: \"keramik.3box.io/v1alpha1\"\nkind: Network\nmetadata: name: small\nspec: replicas: 2 # Use custom runner image for bootstrapping bootstrap: image: keramik/runner:dev imagePullPolicy: IfNotPresent","breadcrumbs":"Developing Keramik » Runner » Setup network with Runner Image","id":"24","title":"Setup network with Runner Image"},"25":{"body":"You will also need to specify the image in your simulation yaml. # Custom runner\n---\napiVersion: \"keramik.3box.io/v1alpha1\"\nkind: Simulation\nmetadata: name: basic namespace: keramik-small\nspec: scenario: ceramic-simple users: 10 runTime: 4 image: keramik/runner:dev imagePullPolicy: IfNotPresent","breadcrumbs":"Developing Keramik » Runner » Setup simulation with Runner Image","id":"25","title":"Setup simulation with Runner Image"},"26":{"body":"For more advanced usage of keramik, please see Advanced CAS and Ceramic Configuration Monitoring IPFS Mixed Networks Secrets Operator Design Migration Tests Custom Bootstrap Configuration","breadcrumbs":"Advanced Topics » Advanced Topics","id":"26","title":"Advanced Topics"},"27":{"body":"By default, Keramik will instantiate all the resources required for a functional CAS service, including a Ganache blockchain. You can configure the Ceramic nodes to use an external instance of the CAS instead of one inside the cluster. If using a CAS running in 3Box Labs infrastructure, you will also need to specify the Ceramic network type associated with the node, e.g. dev-unstable. You may also specify an Ethereum RPC endpoint for the Ceramic nodes to be able to verify anchors, or set it to an empty string to clear it from the Ceramic configuration. In the latter case, the Ceramic nodes will come up but will not be able to verify anchors. If left unspecified, networkType will default to local, ethRpcUrl to http://ganache:8545, and casApiUrl to http://cas:8081. These defaults point to an internal CAS using a local pubsub topic in a fully isolated network. Additionally IPFS can be configured with custom images and resources for both CAS and Ceramic. # network configuration\n---\napiVersion: \"keramik.3box.io/v1alpha1\"\nkind: Network\nmetadata: name: small\nspec: replicas: 2 privateKeySecret: \"small\" networkType: \"dev-unstable\" ethRpcUrl: \"\" casApiUrl: \"https://some-anchor-service.com\"","breadcrumbs":"Advanced Topics » Advanced CAS and Ceramic Configuration » Advanced CAS and Ceramic Configuration","id":"27","title":"Advanced CAS and Ceramic Configuration"},"28":{"body":"Ceramic environment can be adjusted by specifying environment variables in the network configuration # network configuration\n---\napiVersion: \"keramik.3box.io/v1alpha1\"\nkind: Network\nmetadata: name: small\nspec: replicas: 2 ceramic: - env: CERAMIC_PUBSUB_QPS_LIMIT: \"500\"","breadcrumbs":"Advanced Topics » Advanced CAS and Ceramic Configuration » Adjusting Ceramic Environment","id":"28","title":"Adjusting Ceramic Environment"},"29":{"body":"Certain functionality in CAS depends on AWS services. If you are running Keramik in a non-AWS environment, you can disable this by editing the statefulset for CAS kubectl edit statefulsets cas and adding the following environment variables to the spec/template/spec/containers/env config: - name: SQS_QUEUE_URL value: \"\"\n- name: MERKLE_CAR_STORAGE_MODE value: disabled Note statefulsets must be edited every time the network is recreated.","breadcrumbs":"Advanced Topics » Advanced CAS and Ceramic Configuration » Disabling AWS Functionality","id":"29","title":"Disabling AWS Functionality"},"3":{"body":"Kind (Kubernetes in Docker) runs a local k8s cluster. Create and initialize a new kind cluster using this configuration: # kind.yaml\n---\nkind: Cluster\napiVersion: kind.x-k8s.io/v1alpha4\nfeatureGates: MaxUnavailableStatefulSet: true This configuration enables a feature that allows stateful sets to more rapidly redeploy pods on changes. While not required to use keramik it makes deploying and mutating networks significantly faster. # Create a new kind cluster (i.e. local k8s)\nkind create cluster --config kind.yaml Now you will need to deploy Keramik to the cluster.","breadcrumbs":"Creating a Cluster » Creating a Cluster","id":"3","title":"Creating a Cluster"},"30":{"body":"During local benchmarking, you may not have enough resources to run the cluster. A simple \"fix\" is to use the devMode flag on the network and simulation specs. This will override the resource requests and limits values to be none, which means it doesn't need available resources to deploy, and can consume as much as it desires. This would be problematic in production and should only be used for testing purposes. # network configuration\n---\napiVersion: \"keramik.3box.io/v1alpha1\"\nkind: Network\nmetadata: name: small\nspec: replicas: 2 devMode: true # ceramic will require specified resources but all other containers will be unconstrained ceramic: - resourceLimits: cpu: \"1\" memory: \"1Gi\" storage: \"1Gi\" # network configuration\n---\napiVersion: \"keramik.3box.io/v1alpha1\"\nkind: Network\nmetadata: name: small\nspec: replicas: 2 ceramic: - resourceLimits: cpu: \"4\" memory: \"8Gi\" storage: \"2Gi\" The above yaml will provide each ceramic pod with 4 cpu cores, 8GB of memory, and 2GB of storage. Dependent on the system you are running on you may run out of resources. You can check your resource usage with kubectl describe nodes You can also set resources for IPFS within ceramic similarly. # network configuration\n---\napiVersion: \"keramik.3box.io/v1alpha1\"\nkind: Network\nmetadata: name: small\nspec: replicas: 2 ceramic: - ipfs: go: resourceLimits: cpu: \"4\" memory: \"8Gi\" storage: \"2Gi\" storageClass: \"fastDisk\" Additionally the storage class can be set. The storage class must be created out of band but can be referenced as above. Setting resources for CAS is slightly different, using casResourceLimits to set CAS resources # network configuration\n---\napiVersion: \"keramik.3box.io/v1alpha1\"\nkind: Network\nmetadata: name: small\nspec: replicas: 2 cas: image: ceramicnetwork/ceramic-anchor-service:latest casResourceLimits: cpu: \"250m\" memory: \"1Gi\"","breadcrumbs":"Advanced Topics » Advanced CAS and Ceramic Configuration » Image Resources","id":"30","title":"Image Resources"},"31":{"body":"You can also use Recon for reconciliation by setting 'CERAMIC_ONE_RECON' env variable to true. # network configuration\n---\napiVersion: \"keramik.3box.io/v1alpha1\"\nkind: Network\nmetadata: name: small\nspec: replicas: 2 ceramic: - ipfs: rust: env: CERAMIC_ONE_RECON: \"true\"","breadcrumbs":"Advanced Topics » Advanced CAS and Ceramic Configuration » Enabling Recon","id":"31","title":"Enabling Recon"},"32":{"body":"You can enable monitoring on a network to deploy jaeger, prometheus and an opentelemetry collector into the network namespace. This is not the only way to monitor network resources but it is built in. Metrics from all pods in the network will be collected. Sample network resource with monitoring enabled. # basic.yaml\n---\napiVersion: \"keramik.3box.io/v1alpha1\"\nkind: Network\nmetadata: name: network-with-monitoring\nspec: replicas: 2 monitoring: namespaced: true podMonitor: true To view the metrics and traces port-forward the services: kubectl port-forward prometheus-0 9090\nkubectl port-forward jaeger-0 16686 Then navigate to http://localhost:9090 for metrics and http://localhost:16686 for traces.","breadcrumbs":"Advanced Topics » Monitoring » Monitoring","id":"32","title":"Monitoring"},"33":{"body":"The opentelemetry collector exposes metrics on two different ports under the otel service: otel:9464 - All metrics collected otel:9465 - Only simulation metrics Simulations will publish specific summary metrics about the simulation run. This is typically a collection of metrics per simulation run and is much lighter weight than all metrics from all pods in the network. Scrape the otel:9465 endpoint if you want on the simulation metrics. NOTE: The prometheus-0 pod will scrape all metrics so you can easily inspect all activity on the network.","breadcrumbs":"Advanced Topics » Monitoring » Exposed Metrics","id":"33","title":"Exposed Metrics"},"34":{"body":"This option expects the PodMonitor custom resource definition to already be installed in the network namespace. If podMonitor is enabled, the operator will create podmonitors.monitoring.coreos.com resources for collecting the metrics from the pods in the network. If you're using something like the grafana cloud agent, or prometheus-operator, the podmonitors.monitoring.coreos.com will be installed already. You can install the CRD directly from the operator: kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/main/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml","breadcrumbs":"Advanced Topics » Monitoring » Pod Monitoring","id":"34","title":"Pod Monitoring"},"35":{"body":"The IPFS behavior used by CAS and Ceramic can be customized using the same IPFS spec.","breadcrumbs":"Advanced Topics » IPFS » IPFS","id":"35","title":"IPFS"},"36":{"body":"","breadcrumbs":"Advanced Topics » IPFS » Rust IPFS","id":"36","title":"Rust IPFS"},"37":{"body":"Example network config that uses Rust based IPFS (i.e. ceramic-one) with its defaults for Ceramic. apiVersion: \"keramik.3box.io/v1alpha1\"\nkind: Network\nmetadata: name: example-vanilla-ceramic-one\nspec: replicas: 5 ceramic: - ipfs: rust: {} Example network config that uses Rust based IPFS (i.e. ceramic-one) with a specific image for Ceramic. apiVersion: \"keramik.3box.io/v1alpha1\"\nkind: Network\nmetadata: name: example-custom-ceramic-one\nspec: replicas: 5 ceramic: - ipfs: rust: image: rust-ceramic/ceramic-one:dev imagePullPolicy: IfNotPresent","breadcrumbs":"Advanced Topics » IPFS » Ceramic","id":"37","title":"Ceramic"},"38":{"body":"Example network config that uses Rust based IPFS (i.e. ceramic-one) with a specific image for CAS. apiVersion: \"keramik.3box.io/v1alpha1\"\nkind: Network\nmetadata: name: example-vanilla-ceramic-one\nspec: replicas: 5 cas: ipfs: rust: {} Example network config that uses Rust based IPFS (i.e. ceramic-one) with a specific image for CAS. apiVersion: \"keramik.3box.io/v1alpha1\"\nkind: Network\nmetadata: name: example-custom-ceramic-one\nspec: replicas: 5 cas: ipfs: rust: image: rust-ceramic/ceramic-one:dev imagePullPolicy: IfNotPresent","breadcrumbs":"Advanced Topics » IPFS » CAS","id":"38","title":"CAS"},"39":{"body":"","breadcrumbs":"Advanced Topics » IPFS » Kubo IPFS","id":"39","title":"Kubo IPFS"},"4":{"body":"To deploy keramik, we will need to deploy custom resource definitions (CRDs) and apply the Keramik operator.","breadcrumbs":"Creating a Cluster » Deploying Keramik » Deploy Keramik","id":"4","title":"Deploy Keramik"},"40":{"body":"Example network config that uses Go based IPFS (i.e. Kubo) with its defaults for Ceramic. apiVersion: \"keramik.3box.io/v1alpha1\"\nkind: Network\nmetadata: name: example-vanilla-kubo\nspec: replicas: 5 ceramic: - ipfs: go: {} Example network config that uses Go based IPFS (i.e. Kubo) with a specific image for Ceramic. apiVersion: \"keramik.3box.io/v1alpha1\"\nkind: Network\nmetadata: name: example-custom-kubo\nspec: replicas: 5 ceramic: - ipfs: go: image: ceramicnetwork/go-ipfs-daemon:develop imagePullPolicy: IfNotPresent Example network config that uses Go based IPFS (i.e. Kubo) with extra configuration commands for Ceramic. apiVersion: \"keramik.3box.io/v1alpha1\"\nkind: Network\nmetadata: name: example-custom-kubo\nspec: replicas: 5 ceramic: - ipfs: go: image: ceramicnetwork/go-ipfs-daemon:develop imagePullPolicy: IfNotPresent commands: - ipfs config --json Swarm.RelayClient.Enabled false","breadcrumbs":"Advanced Topics » IPFS » Ceramic","id":"40","title":"Ceramic"},"41":{"body":"Example network config that uses Go based IPFS (i.e. Kubo) with its defaults for CAS. apiVersion: \"keramik.3box.io/v1alpha1\"\nkind: Network\nmetadata: name: example-vanilla-kubo\nspec: replicas: 5 cas: ipfs: go: {} Example network config that uses Go based IPFS (i.e. Kubo) with a specific image for CAS. apiVersion: \"keramik.3box.io/v1alpha1\"\nkind: Network\nmetadata: name: example-custom-kubo\nspec: replicas: 5 cas: ipfs: go: image: ceramicnetwork/go-ipfs-daemon:develop imagePullPolicy: IfNotPresent Example network config that uses Go based IPFS (i.e. Kubo) with extra configuration commands for CAS. apiVersion: \"keramik.3box.io/v1alpha1\"\nkind: Network\nmetadata: name: example-custom-kubo\nspec: replicas: 5 cas: ipfs: go: image: ceramicnetwork/go-ipfs-daemon:develop imagePullPolicy: IfNotPresent commands: - ipfs config --json Swarm.RelayClient.Enabled false","breadcrumbs":"Advanced Topics » IPFS » CAS","id":"41","title":"CAS"},"42":{"body":"It is possible to configure multiple sets of Ceramic nodes that different from one another. For example a network where half of the nodes are running a different version of js-ceramic or IPFS.","breadcrumbs":"Advanced Topics » Mixed Networks » Mixed Networks","id":"42","title":"Mixed Networks"},"43":{"body":"","breadcrumbs":"Advanced Topics » Mixed Networks » Examples","id":"43","title":"Examples"},"44":{"body":"The following config creates a network with half of the nodes running Rust based IPFS and the other half Go. ---\napiVersion: \"keramik.3box.io/v1alpha1\"\nkind: Network\nmetadata: name: mixed\nspec: replicas: 5 ceramic: - ipfs: rust: {} - ipfs: go: {}","breadcrumbs":"Advanced Topics » Mixed Networks » Mixed IPFS","id":"44","title":"Mixed IPFS"},"45":{"body":"The following config creates a network with half of the nodes running a dev-0 of js-ceramic and the other half dev-1. ---\napiVersion: \"keramik.3box.io/v1alpha1\"\nkind: Network\nmetadata: name: mixed\nspec: replicas: 5 ceramic: - image: ceramicnetwork/composedb:dev-0 - image: ceramicnetwork/composedb:dev-1","breadcrumbs":"Advanced Topics » Mixed Networks » Mixed js-ceramic","id":"45","title":"Mixed js-ceramic"},"46":{"body":"Weights can be used to determine how many replicas of each Ceramic spec are created. The total network replicas are spread across each Ceramic spec according to its relative weight. The default weight is 1. The simplist way to get exact replica counts is to have the weights sum to the replica count. Then each Ceramic spec will have a number of replicas equal to its weight. However it can be tedious to ensure weights always add up to the replica count so this is not required. The total replicas all Ceramic specs will always sum to the configured replica count. As such some rounding will be applied to get a good approximation of the relative weights.","breadcrumbs":"Advanced Topics » Mixed Networks » Weights","id":"46","title":"Weights"},"47":{"body":"Create 2/3rd nodes with dev-0 and 1/3rd with dev-1. ---\napiVersion: \"keramik.3box.io/v1alpha1\"\nkind: Network\nmetadata: name: mixed\nspec: replicas: 3 ceramic: - weight: 2 image: ceramicnetwork/composedb:dev-0 # 2 replicas - image: ceramicnetwork/composedb:dev-1 # 1 replica Create 3/4ths nodes with dev-0 and 1/4rd with dev-1. ---\napiVersion: \"keramik.3box.io/v1alpha1\"\nkind: Network\nmetadata: name: mixed\nspec: replicas: 24 ceramic: - weight: 3 image: ceramicnetwork/composedb:dev-0 # 18 replicas - image: ceramicnetwork/composedb:dev-1 # 6 replicas Create three different version each having half the previous. In this case weights do not devide evenly so a close approximation is achived. ---\napiVersion: \"keramik.3box.io/v1alpha1\"\nkind: Network\nmetadata: name: mixed\nspec: replicas: 16 ceramic: - weight: 4 image: ceramicnetwork/composedb:dev-0 # 10 replicas - weight: 2 image: ceramicnetwork/composedb:dev-1 # 4 replicas - weight: 1 image: ceramicnetwork/composedb:dev-2 # 2 replicas","breadcrumbs":"Advanced Topics » Mixed Networks » Examples","id":"47","title":"Examples"},"48":{"body":"You can choose to specify a private key for the Ceramic nodes to use as their admin secret. This will allow you to set up the corresponding DID with CAS Auth. Leaving the private key unspecified will cause a new key to be randomly generated. This can be fine for simulation runs against CAS/Ganache running locally within the cluster but not for simulations that hit CAS running behind the AWS API Gateway. Using an unauthorized DID in that case will prevent the Ceramic nodes from starting up. apiVersion: \"keramik.3box.io/v1alpha1\"\nkind: Network\nmetadata: name: small\nspec: replicas: 2 privateKeySecret: \"small\" Note that privateKeySecret is the name of another k8s secret in the keramik namespace that has already been populated beforehand with the desired hex-encoded private key. This source secret MUST exist before it can be used to populate the Ceramic admin secret. kubectl create secret generic small --from-literal=private-key=0e3b57bb4d269b6707019f75fe82fe06b1180dd762f183e96cab634e38d6e57b The secret can also be created from a file containing the private key. kubectl create secret generic small --from-file=private-key=./my_secret Here's an example of the contents of the my_secret file. Please make sure that there are no newlines at the end of the file. 0e3b57bb4d269b6707019f75fe82fe06b1180dd762f183e96cab634e38d6e57b Alternatively, you can use a kustomization.yml file to create the secret from a file before creating the network, and using the name of the new secret in the network configuration. ---\nnamespace: keramik secretGenerator:\n- name: small envs: - .env.secret Here's an example of the contents of the .env.secret file. private-key=0e3b57bb4d269b6707019f75fe82fe06b1180dd762f183e96cab634e38d6e57b","breadcrumbs":"Advanced Topics » Secrets » Specifying a Ceramic admin secret","id":"48","title":"Specifying a Ceramic admin secret"},"49":{"body":"This document discusses some of the designs patterns of the operator.","breadcrumbs":"Advanced Topics » Operator Design » Operator Patterns","id":"49","title":"Operator Patterns"},"5":{"body":"Custom resource definitions tell k8s about our network and simulation resources. When deploying a new cluster and anytime they change you need to apply them: cargo run --bin crdgen | kubectl apply -f -","breadcrumbs":"Creating a Cluster » Deploying Keramik » Deploy CRDS","id":"5","title":"Deploy CRDS"},"50":{"body":"The operator is responsible to managing many resources and controlling how those resources can be customized. As a result the operator adopts a specs, statuses and configs pattern. Specs - Defines the desired state. Statuses - Reports the current state. Configs - Custom configuration to control creating a Spec. Both specs and statuses are native concepts to Kubernetes. A spec provides the user facing API for defining their desired state. A status reports on the actual state. This code base introduces the concept of a config. Naturally, operators wrap existing specs and hide some of their details. However some of those details should be exposed to the user. A config defines how the parts of a spec owned by the operator can be exposed. In turn the configs themselves have their own specs, i.e. the API into how to customize internal specs of the operator. For example the bootstrap job requires JobSpec to run the job. The bootstrap job is responsible for telling new peers in the network about existing peers. Exposing the JobSpec to the user puts too much onus on the user to create a functional job. Instead we define a BootstrapSpec, a BootstrapConfig and a function that can create the necessary JobSpec given a BootstrapConfig. The BootstrapSpec is the user API for controlling the bootstrap job. The BootstrapConfig controls which properties of the JobSpec can be customized and provides sane defaults. Let's see how this plays out in the code. Here is a simplified example of the bootstrap job that allows customizing only the image and bootstrap method: // BootstrapSpec defines how the network bootstrap process should proceed.\n#[derive(Serialize, Deserialize, Debug, PartialEq, Clone, JsonSchema)]\npub struct BootstrapSpec { // Note, both image and method are optional as the user // may want to specify only one or the other or both. pub image: Option, pub method: Option,\n}\n// BootstrapConfig defines which properties of the JobSpec can be customized.\npub struct BootstrapConfig { // Note, neither image nor method are optional as we need // valid values in order to build the JobSpec. pub image: String, pub method: String,\n}\n// Define clear defaults for the config.\nimpl Default for BootstrapConfig { fn default() -> Self { Self { image: \"public.ecr.aws/r5b3e0r5/3box/keramik-runner\".to_owned(), method: \"ring\".to_owned(), } }\n}\n// Implement a conversion from the spec to the config applying defaults.\nimpl From for BootstrapConfig { fn from(value: BootstrapSpec) -> Self { let default = Self::default(); Self { image: value.image.unwrap_or(default.image), method: value.method.unwrap_or(default.method), } }\n}\n// Additionally implement the conversion for the case we the entire spec was left undefined.\nimpl From