Add observability stack

platform/loki/Makefile

@@ -0,0 +1,48 @@
+.DEFAULT_GOAL := help
+
+# Default Loki version
+DEFAULT_VERSION := 6.22.0
+LOKI_VERSION ?= $(DEFAULT_VERSION)
+LOKI_OUTPUT_YAML := vendor/loki.yaml
+VALUES_FILE := $(CURDIR)/values.yaml
+
+.PHONY: all clean manifests check-tools
+
+all: check-tools clean manifests
+
+.PHONY: init
+init:
+	helm repo add grafana https://grafana.github.io/helm-charts
+	helm repo update
+
+.PHONY: manifests
+manifests:
+	@echo "Removing previous $(LOKI_OUTPUT_YAML)..."
+	rm -f $(LOKI_OUTPUT_YAML)
+	@echo "Pulling Loki YAML (Version: $(LOKI_VERSION)) from Helm chart..."
+	helm repo update
+	helm template loki --values $(VALUES_FILE) --namespace loki  grafana/loki --version $(LOKI_VERSION)  > $(LOKI_OUTPUT_YAML)
+
+.PHONY: upgrade
+upgrade:
+	@echo "Removing previous $(LOKI_OUTPUT_YAML)..."
+	rm -f $(LOKI_OUTPUT_YAML)
+	@echo "Pulling Loki YAML (Version: $(LOKI_VERSION)) from Helm chart..."
+	helm repo update
+	helm upgrade --values $(VALUES_FILE) --namespace loki grafana/loki  > $(LOKI_OUTPUT_YAML)
+
+.PHONY: check-tools
+check-tools:
+	@command -v helm >/dev/null 2>&1 || { echo >&2 "helm is not installed. Aborting."; exit 1; }
+
+# Help target to display usage information
+.PHONY: help
+help:
+	@echo "Makefile for pulling Loki YAML"
+	@echo ""
+	@echo "Targets:"
+	@echo "  init        : Add Loki helm chart repository"
+	@echo "  manifests   : Pull Loki YAML from Helm chart with specified values.yaml and remove the old Loki YAML"
+	@echo "  upgrade     : Pull Loki YAML from Helm chart with specified values.yaml and upgrade the old Loki YAML"
+	@echo "  check-tools : ensure required tools are installed"
+	@echo "  help        : Display this help message"

platform/loki/README.md

@@ -0,0 +1,104 @@
+# Loki
+
+The [Loki](https://github.com/grafana/loki) project is a log aggregation system
+designed to store and query logs from multiple sources. It's part of the
+[Grafana](https://github.com/grafana/grafana) ecosystem and optimizing for
+cost-effective and efficient log storage. Loki's design is to work seamlessly
+with Grafana for visualization and Prometheus for metrics.
+
+## Makefile Overview
+
+This Makefile manages the Loki Helm chart, which simplifies the deployment and
+management of Loki in a Kubernetes environment. The Makefile provides targets
+to automate common tasks such as pulling the latest Loki YAML configuration,
+upgrading Loki, and ensure required tools are present.
+
+## Prerequisites
+
+Before using the Makefile, ensure you have the following tools installed:
+
+- `helm`
+
+### Makefile Targets
+
+- **all**: This is the default target that runs `check-tools`, `clean`, and
+`manifests` targets in sequence.
+- **init**: Adds the Grafana Helm repository and updates the Helm repository
+index.
+- **manifests**: Removes the previous Loki YAML file and pulls the latest Loki
+YAML configuration from the Helm chart using the specified `values.yaml` file.
+- **upgrade**: Removes the previous Loki YAML file and upgrades the Loki
+deployment using the latest configuration from the Helm chart.
+- **check-tools**: Ensures that the required tools (e.g., Helm) are available
+on the system.
+- **help**: Displays usage information for the Makefile targets.
+
+### Usage
+
+1. **Initialize Helm Repository**
+
+   Before using the Makefile, you need to initialize the Helm repository:
+
+   ```sh
+   make init
+   ```
+
+2. **Generate Loki Manifests**
+
+   To generate the Loki YAML configuration file, run:
+
+   ```sh
+   make manifests
+   ```
+
+   This will remove the previous `loki.yaml` file and pull the latest
+   configuration from the Helm chart.
+
+3. **Upgrade Loki**
+
+   To upgrade the Loki deployment with the latest configuration, run:
+
+   ```sh
+   make upgrade
+   ```
+
+   This will remove the previous `loki.yaml` file and upgrade the Loki deployment.
+
+4. **Check Required Tools**
+
+   Ensure the installation of required tools by running:
+
+   ```sh
+   make check-tools
+   ```
+
+   This will check that Helm is available on your system.
+
+5. **Display Help**
+
+   To display the help message with information about the Makefile targets, run:
+
+   ```sh
+   make help
+   ```
+
+### Default Variables
+
+- **DEFAULT_VERSION**: The default version of Loki to use (e.g., `6.22.0`).
+- **LOKI_VERSION**: The version of Loki to use, which can be overridden by
+setting this variable.
+- **LOKI_OUTPUT_YAML**: The output file for the Loki YAML configuration
+(default: `vendor/loki.yaml`).
+- **VALUES_FILE**: The path to the `values.yaml` file used for Helm configuration.
+
+### Example
+
+To generate the Loki manifests using a specific version and values file, you
+can run:
+
+```sh
+make LOKI_VERSION=2.4.1 VALUES_FILE=/path/to/your/values.yaml manifests
+```
+
+This will pull the specified version of Loki and use the provided `values.yaml`
+file for configuration.

platform/loki/do/application.yaml

@@ -0,0 +1,18 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: loki
+  namespace: argocd
+spec:
+  project: default
+  source:
+    repoURL: https://github.com/metacpan/metacpan-k8s
+    targetRevision: main
+    path: platform/loki/do
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: loki
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: false

platform/loki/do/kustomization.yaml

@@ -0,0 +1,7 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - ./application.yaml
+  - ./sealedsecret-loki-secrets.yaml
+  - ../vendor/

platform/loki/do/sealedsecret-loki-secrets.yaml

@@ -0,0 +1,18 @@
+---
+apiVersion: bitnami.com/v1alpha1
+kind: SealedSecret
+metadata:
+  creationTimestamp: null
+  name: loki-secrets
+  namespace: loki
+spec:
+  encryptedData:
+    grafana-loki-s3-accessKeyId: AgAby4tq0CSboTv68HMUQlaM2JuArvmt35EMw1GTp/px3j5z5eY3eDJX8p9GLOTW4tTTgpvCUTwUYIbe1NC932iCkTDVo7DlJ6cpPTH8IJn0Bw/iCcX0ZWfYP5s/tQu2b7jrtyA4+TehhervYiSYS18Dh4LRZYs/CJIAQeSnX+XttZAsl2Kzfm9VT+7cXSzLVqgH/BXjvaVgTfx1Ijp+unRH+hj5fs7FPG7eQTcsGb5439r03f3QL59izhKMfa9FjSebnUhnLzmXVoic/o3vVn58/rVbJ+gSab1o7472Hwnd2Hx0uDkdzxsau/DjQPHXxjXtwtkROl2ida4MIYaVxT2RPDQPOoa8iphVLRGDo70R+2AXYv5rjP4iQcsLuTK9A9bG/Hlb2sZHG0CLJvMHBIHIQMtokN+ZJsRuCvR3sgWYBFhxStWxtM07PE+epbbEdOZK9V+WkbQEtgxU6Sfg4EG++r/nz2aXe/aji9QfuJpwCo8bLb3NtuOf2H5jrueOyicgGEv1IUTc+4mML50ZFdFU/JF/zU+y03cUoO8qzGJjrwL3epRfxwHapsVwaOLvcp5mAL4wR5Yfp6C5bTGQrq0mLrCDwEH+eyB1r//EqCAUUMUhesjSurb16CjTjthBEEX6LsS2wz7rXmj19gm44/G1OwPEpnwh6UHJRV9Aq6hNWhprAgesBEArAyCozLdWhk/7rsosnLHhkad3O8yi10FOX1RzJg==
+    grafana-loki-s3-endpoint: AgBTcLzWaCgc9SFaqQVbjUJDM2aFFI4zmHj0jgDiKgdPn8HlvKwhUU7b+ztyBHBczLY5FMF9PgMOHX8M+SXRn6aAXIvLQXns+poP9bipID4DghptoG1HE9NvxQ68KKJXCc+siJka8oXTFYSj9PXYEG7ntoRlfFhem858NyoMfCMQ6t2cnAbw3Qu8+boIBlmv9gh6fFm5LC9Hm9QF/zOUqkM/J/OVOh+SXEnbbxukDnINoeFybnl5ZOCfhfrXyaWruU2J+FxKBYq9th6VUJVcr3pTd7GZQ0Z5DZY/FMO4ecpDG397MrjysHu2oBstt/7Gzd2iWSMdvBTkUWxs1gktXT48FMS94hBlj1KhGnc5Qg1Y3xkAfaEA6DWQRwfJGWiueVyOQVG1ikpiog0fJ8Ga/vLsM+TtHZizIjioSfu1H4DmUsR9NlS5nOPkFQUqzqFqkxW9Da9xtGNDFuUX4kONUJL1P2WO0UmDnzG9OQYZnIOEq+qrQ01qXnzs3TjV3q6N0Cvpocn8ZDfj3G5fpg4GNMftiMInZZoTLXktiFpTteRhWaG8Ku0s6iGb8lTFrOEpcR1RgJMB97pyNGbN03VcDIW46YXc4CT0HGMcDzgILSISP/5ajLjUX2vHUfSkKXRe1aC+sTc46lld7Q7lRNfoO56mFhuneINF2pf0kS402GVpFFwZ+DUhW/xMVl1gbBUSHtsYVFLm+Fg2D+L4HC/D5Qh8xIdcSz2BPu2xhPQ=
+    grafana-loki-s3-secretAccessKey: AgCSPYP8neHgp8XNdAnB7OmvhLAgLvrGO3sa8cgC/kPsWltiDd1X+L+RE15WVeR/EHwWVkaAe8UQvLKcftpd9nW+vShPVJgECtg9HaMc4WVWx4cioNR8g9MHqV8XFuld/FBbrS+eurnUnmgb0anUZdc33creZW4XcqZ0REbGcPua50aRbrRSCoKHJFQbzHRakf/ek99gAP11qEUPt8pqDFABU9fnn13CjNVuI+QQhnt3FMmGaZnBXeszo9fNpU0eir4eTcSOw4xSHg+xVgVDa5i678ThMIXf3SUjYxC12TJysrW9Cnq1Vp5emYzCeNqhTPLC31SrID0h/e2FKizaRQdHkUqvzG44fGHt36EKbcA1JfjDoK0EWqGMkRhh2EeeM8vj1sUfS8i4nKJoBsHB9jHCJB5Fzm6/uHI8DY+A1RRMsiKoKQnh/RWTkbYW724Jz3wxEbmDK0UiBFuZcTWj/orLTAHK/yLriy/Bt9WGkAh76cYCoZvCSnoCUL4QShrs8ez3DAk/Fg2hfoUYOJNB/6Fxb6upd1j8OivngTNrwghMvhU6bUs42IKvmAYzuMIFiJPkmKrIG03D7GqxbjHtOmBJEv8pZEQNk5xBoyTCY12aIxmz2c9c8s8G42YyKV62dEZyfFPb3Gac6ppNxMqheVhgYKLOBsKCDBDRuVQpEHyWvpfK/G0QVutpa/PF87gKWOCrKs1llOGYU6NrX/A9qjl9r8yDx8BtoTAHqBetj4v5seE5e8jYoV6712Vz
+  template:
+    metadata:
+      creationTimestamp: null
+      name: loki-secrets
+      namespace: loki
+    type: Opaque

platform/loki/values.yaml

@@ -0,0 +1,116 @@
+loki:
+  auth_enabled: false
+  schemaConfig:
+    configs:
+      - from: "2024-04-01"
+        store: tsdb
+        object_store: s3
+        schema: v13
+        index:
+          prefix: loki_index_
+          period: 24h
+  storage_config:
+    aws:
+      region: fra1
+      bucketnames: mc-do-logs
+      s3forcepathstyle: true
+  pattern_ingester:
+    enabled: true
+  limits_config:
+    allow_structured_metadata: true
+    volume_enabled: true
+    retention_period: 672h # 28 days retention
+  querier:
+    max_concurrent: 4
+
+  storage:
+    type: s3
+    bucketNames:
+      chunks: mc-do-logs
+      ruler: mc-do-logs
+      admin: mc-do-logs
+    s3:
+      # AWS endpoint URL
+      endpoint: "${GRAFANA_LOKI_S3_ENDPOINT}"
+      # AWS region where the S3 bucket is located
+      region: fra1
+      # AWS secret access key
+      secretAccessKey: "${GRAFANA_LOKI_S3_SECRETACCESSKEY}"
+      # AWS access key ID
+      accessKeyId: "${GRAFANA_LOKI_S3_ACCESKEYID}"
+      # AWS signature version (e.g., v2 or v4)
+      signatureVersion: v4
+      # Forces the path style for S3 (true/false)
+      s3ForcePathStyle: true
+      # Allows insecure (HTTP) connections (true/false)
+      # HTTP configuration settings
+
+deploymentMode: SimpleScalable
+
+backend:
+  replicas: 3
+  extraArgs:
+    - "-config.expand-env=true"
+  extraEnv:
+    - name: GRAFANA_LOKI_S3_ENDPOINT
+      valueFrom:
+        secretKeyRef:
+          name: loki-secrets
+          key: grafana-loki-s3-endpoint
+    - name: GRAFANA_LOKI_S3_ACCESKEYID
+      valueFrom:
+        secretKeyRef:
+          name: loki-secrets
+          key: grafana-loki-s3-accessKeyId
+    - name: GRAFANA_LOKI_S3_SECRETACCESSKEY
+      valueFrom:
+        secretKeyRef:
+          name: loki-secrets
+          key: grafana-loki-s3-secretAccessKey
+read:
+  replicas: 3
+  extraArgs:
+    - "-config.expand-env=true"
+  extraEnv:
+    - name: GRAFANA_LOKI_S3_ENDPOINT
+      valueFrom:
+        secretKeyRef:
+          name: loki-secrets
+          key: grafana-loki-s3-endpoint
+    - name: GRAFANA_LOKI_S3_ACCESKEYID
+      valueFrom:
+        secretKeyRef:
+          name: loki-secrets
+          key: grafana-loki-s3-accessKeyId
+    - name: GRAFANA_LOKI_S3_SECRETACCESSKEY
+      valueFrom:
+        secretKeyRef:
+          name: loki-secrets
+          key: grafana-loki-s3-secretAccessKey
+write:
+  replicas: 3
+  extraArgs:
+    - "-config.expand-env=true"
+  extraEnv:
+    - name: GRAFANA_LOKI_S3_ENDPOINT
+      valueFrom:
+        secretKeyRef:
+          name: loki-secrets
+          key: grafana-loki-s3-endpoint
+    - name: GRAFANA_LOKI_S3_ACCESKEYID
+      valueFrom:
+        secretKeyRef:
+          name: loki-secrets
+          key: grafana-loki-s3-accessKeyId
+    - name: GRAFANA_LOKI_S3_SECRETACCESSKEY
+      valueFrom:
+        secretKeyRef:
+          name: loki-secrets
+          key: grafana-loki-s3-secretAccessKey
+
+chunksCache:
+  replicas: 1
+  allocatedMemory: 768
+# Disable minio storage
+minio:
+  enabled: false

platform/loki/vendor/kustomization.yaml

@@ -0,0 +1,5 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - ./loki.yaml