From eb5618b9af4013f7bef7e7462a9c6c28de3cd338 Mon Sep 17 00:00:00 2001 From: anmol thapar Date: Fri, 12 Jan 2024 07:25:11 +0000 Subject: [PATCH 01/17] feat: kubernetes setup --- README.md | 30 +++++++- haproxy/haproxy.cfg | 1 + k8s/base/deployment.yaml | 37 ++++++++++ k8s/base/ingress.yaml | 21 ++++++ k8s/base/kustomization.yaml | 13 ++++ k8s/base/namespace.yaml | 4 ++ k8s/base/persistance.yaml | 82 ++++++++++++++++++++++ k8s/base/service.yaml | 12 ++++ k8s/krsync | 26 +++++++ k8s/overlays/production/kustomization.yaml | 12 ++++ k8s/overlays/staging/kustomization.yaml | 12 ++++ k8s/shiny-sync | 11 +++ shiny/Dockerfile | 5 +- 13 files changed, 261 insertions(+), 5 deletions(-) create mode 100644 k8s/base/deployment.yaml create mode 100644 k8s/base/ingress.yaml create mode 100644 k8s/base/kustomization.yaml create mode 100644 k8s/base/namespace.yaml create mode 100644 k8s/base/persistance.yaml create mode 100644 k8s/base/service.yaml create mode 100755 k8s/krsync create mode 100644 k8s/overlays/production/kustomization.yaml create mode 100644 k8s/overlays/staging/kustomization.yaml create mode 100755 k8s/shiny-sync diff --git a/README.md b/README.md index 166dd8d..bb8dc9b 100644 --- a/README.md +++ b/README.md @@ -33,14 +33,14 @@ Some applications (copied over from the original deployment are in `apps`). Thes ## Bringing the bits up -``` +```bash docker network create twinkle 2> /dev/null || /bin/true docker volume create shiny_logs docker run -d --name haproxy --network twinkle mrcide/haproxy:dev docker run -d --name apache --network twinkle \ -p 80:80 \ -p 443:443 \ - -p 9000:9000 \ + -p 9005:9005 \ -v "${PWD}/apache/httpd.conf:/usr/local/apache2/conf/httpd.conf:ro" \ -v "${PWD}/apache/auth:/usr/local/apache2/conf/auth:ro" \ -v "${PWD}/apache/ssl:/usr/local/apache2/conf/ssl:ro" \ @@ -55,7 +55,31 @@ docker exec haproxy update_shiny_servers shiny 1 Teardown -``` +```bash docker rm -f haproxy apache shiny-1 docker network rm twinkle ``` + +## Running in Kubernetes + +The following is guide to run the shiny server in kubernetes. They are based on running a Kind cluster. The configuration may +nee to be adjusted for other k8s clusters. + +### Prerequisites + +A single node k8s cluster is needed. To setup a k8s cluster follow the guide [here](https://mrc-ide.myjetbrains.com/youtrack/articles/RESIDE-A-31/Setting-up-Kubernetes-k8s-Cluster). + +Run the following commands from the root of the repository + +1. `./shiny/build` +2. For KIND cluster load image: `kind load docker-image mrcide/shiny-server:dev` +3. Create /shiny/logs and /shiny/apps directories in k8s node. For kind exec into node via `docker exec -it kind-control-plane sh` + and then `mkdir -p /shiny/logs /shiny/apps` to create the directories. +3.`k apply -k k8s/overlays/production` for production and `k apply -k k8s/overlays/staging` for staging. +4.`k8s/shiny-sync` + +Great the shiny server is running and can be seen on the External IP of the ingress-nginx-controller LoadBalancer service. + +#### Teardown + +`k delete -k k8s/overlays/production` for production or `k delete -k k8s/overlays/staging` for staging diff --git a/haproxy/haproxy.cfg b/haproxy/haproxy.cfg index 3cb09b4..eafb204 100644 --- a/haproxy/haproxy.cfg +++ b/haproxy/haproxy.cfg @@ -25,6 +25,7 @@ frontend http-in option forwardfor default_backend servers +# point this to K8s master nodes (control planes) backend servers # I think that leastconn is probably the best bet here. balance leastconn diff --git a/k8s/base/deployment.yaml b/k8s/base/deployment.yaml new file mode 100644 index 0000000..d831735 --- /dev/null +++ b/k8s/base/deployment.yaml @@ -0,0 +1,37 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: shiny-deploy + labels: + app: shiny +spec: + replicas: 2 + selector: + matchLabels: + app: shiny + template: + metadata: + labels: + app: shiny + spec: + containers: + - name: shiny + image: mrcide/shiny-server:dev # todo: change to one in registery + volumeMounts: + - name: shiny-apps + mountPath: /shiny/apps + - name: shiny-logs + mountPath: /shiny/logs + # todo: create appropriate resource requests and limits + resources: + requests: + memory: "128Mi" + cpu: "250m" + volumes: + - name: shiny-apps + persistentVolumeClaim: + claimName: shiny-apps-pvc + - name: shiny-logs + persistentVolumeClaim: + claimName: shiny-logs-pvc + \ No newline at end of file diff --git a/k8s/base/ingress.yaml b/k8s/base/ingress.yaml new file mode 100644 index 0000000..0a384f8 --- /dev/null +++ b/k8s/base/ingress.yaml @@ -0,0 +1,21 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: ingress-shiny + annotations: + nginx.ingress.kubernetes.io/affinity: "cookie" + nginx.ingress.kubernetes.io/session-cookie-name: "shinycookie" + nginx.ingress.kubernetes.io/session-cookie-expires: "172800" + nginx.ingress.kubernetes.io/session-cookie-max-age: "172800" +spec: + ingressClassName: nginx + rules: + - http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: shiny-svc + port: + number: 3838 \ No newline at end of file diff --git a/k8s/base/kustomization.yaml b/k8s/base/kustomization.yaml new file mode 100644 index 0000000..9738aac --- /dev/null +++ b/k8s/base/kustomization.yaml @@ -0,0 +1,13 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +metadata: + name: shiny + +namespace: twinkle + +resources: +- namespace.yaml +- persistance.yaml +- deployment.yaml +- service.yaml +- ingress.yaml \ No newline at end of file diff --git a/k8s/base/namespace.yaml b/k8s/base/namespace.yaml new file mode 100644 index 0000000..78717dc --- /dev/null +++ b/k8s/base/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: twinkle \ No newline at end of file diff --git a/k8s/base/persistance.yaml b/k8s/base/persistance.yaml new file mode 100644 index 0000000..d9787f1 --- /dev/null +++ b/k8s/base/persistance.yaml @@ -0,0 +1,82 @@ +# todo: storage capacity as per requirements +# todo: For distributed nodes use another storage solution https://ceph.io +# storage class +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: local-storage +provisioner: kubernetes.io/no-provisioner +volumeBindingMode: WaitForFirstConsumer +--- +# shiny apps +apiVersion: v1 +kind: PersistentVolume +metadata: + name: shiny-apps-pv +spec: + storageClassName: local-storage + capacity: + storage: 10Gi + volumeMode: Filesystem + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Retain + local: + path: /shiny/apps + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - kind-control-plane # update this to the node name +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: shiny-apps-pvc +spec: + storageClassName: local-storage + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi +--- +# shiny logs +apiVersion: v1 +kind: PersistentVolume +metadata: + name: shiny-logs-pv +spec: + storageClassName: local-storage + capacity: + storage: 2Gi + volumeMode: Filesystem + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Retain + local: + path: /shiny/logs + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - kind-control-plane # update this to the node you want to use +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: shiny-logs-pvc +spec: + storageClassName: local-storage + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 2Gi +--- diff --git a/k8s/base/service.yaml b/k8s/base/service.yaml new file mode 100644 index 0000000..8db8181 --- /dev/null +++ b/k8s/base/service.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Service +metadata: + name: shiny-svc +spec: + type: ClusterIP + selector: + app: shiny + ports: + - protocol: TCP + port: 3838 + targetPort: 3838 \ No newline at end of file diff --git a/k8s/krsync b/k8s/krsync new file mode 100755 index 0000000..2426cdd --- /dev/null +++ b/k8s/krsync @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +# https://serverfault.com/questions/741670/rsync-files-to-a-kubernetes-pod +# example usage: +# ./krsync -av --progress --stats podName@namespace: + +if [ -z "$KRSYNC_STARTED" ]; then + export KRSYNC_STARTED=true + exec rsync -a --delete --blocking-io --rsh "$0" $@ +fi + +# Running as --rsh +namespace='' +pod=$1 +shift + +# If user uses pod@namespace, rsync passes args as: {us} -l pod namespace ... +if [ "X$pod" = "X-l" ]; then + pod=$1 + shift + namespace="-n $1" + shift +fi + +# customise this to cluster needs!! +exec kubectl $namespace exec -i $pod -- "$@" \ No newline at end of file diff --git a/k8s/overlays/production/kustomization.yaml b/k8s/overlays/production/kustomization.yaml new file mode 100644 index 0000000..082f513 --- /dev/null +++ b/k8s/overlays/production/kustomization.yaml @@ -0,0 +1,12 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../../base + +commonLabels: + env: production + +replicas: + - name: shiny-deploy + count: 7 \ No newline at end of file diff --git a/k8s/overlays/staging/kustomization.yaml b/k8s/overlays/staging/kustomization.yaml new file mode 100644 index 0000000..544d045 --- /dev/null +++ b/k8s/overlays/staging/kustomization.yaml @@ -0,0 +1,12 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../../base + +commonLabels: + env: staging + +replicas: + - name: shiny-deploy + count: 3 \ No newline at end of file diff --git a/k8s/shiny-sync b/k8s/shiny-sync new file mode 100755 index 0000000..b9ac3b2 --- /dev/null +++ b/k8s/shiny-sync @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +# uses krysnc to sync files to a kubernetes pod and thus into k8s cluster (via persistant volume) +# RUN this from repository root to ensure correct paths to app files +NAMESPACE=twinkle +SYNC_DIR=/shiny/apps + + +FIRST_SHINY_POD=$(kubectl get pods -l app=shiny -n $NAMESPACE -o jsonpath='{.items[0].metadata.name}') +echo "pod name: $FIRST_SHINY_POD" +$PWD/k8s/krsync -av --progress --stats $PWD/apps/* $FIRST_SHINY_POD@$NAMESPACE:$SYNC_DIR \ No newline at end of file diff --git a/shiny/Dockerfile b/shiny/Dockerfile index e922a30..c04f15a 100644 --- a/shiny/Dockerfile +++ b/shiny/Dockerfile @@ -3,5 +3,6 @@ FROM rocker/shiny:latest # Override the rocker defaults with our defaults COPY shiny-server.conf /etc/shiny-server/shiny-server.conf -VOLUME /shiny/apps -VOLUME /shiny/logs +# Install rsync +RUN apt-get update && apt-get install -y rsync + From 69d5f9899e69fe3084a9b3a45b25cc273f3ace2c Mon Sep 17 00:00:00 2001 From: absternator Date: Fri, 12 Jan 2024 15:24:21 +0000 Subject: [PATCH 02/17] feat: ssl ingress + persistance local --- .gitignore | 1 + k8s/base/deployment.yaml | 39 +++++----- k8s/base/ingress.yaml | 23 +++--- k8s/base/kustomization.yaml | 9 ++- k8s/base/namespace.yaml | 4 -- k8s/base/persistance.yaml | 140 ++++++++++++++++++------------------ k8s/configure_ssl | 15 ++++ shiny/build | 2 +- 8 files changed, 125 insertions(+), 108 deletions(-) delete mode 100644 k8s/base/namespace.yaml create mode 100755 k8s/configure_ssl diff --git a/.gitignore b/.gitignore index 4486983..92ab869 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ apache/ssl +k8s/ssl \ No newline at end of file diff --git a/k8s/base/deployment.yaml b/k8s/base/deployment.yaml index d831735..b06111f 100644 --- a/k8s/base/deployment.yaml +++ b/k8s/base/deployment.yaml @@ -14,24 +14,25 @@ spec: labels: app: shiny spec: + initContainers: + - name: init-shiny + image: busybox:1.28 + command: ["sh", "-c", "mkdir -p /shiny/logs /shiny/apps"] + volumeMounts: + - name: shiny-data + mountPath: /shiny containers: - - name: shiny - image: mrcide/shiny-server:dev # todo: change to one in registery - volumeMounts: - - name: shiny-apps - mountPath: /shiny/apps - - name: shiny-logs - mountPath: /shiny/logs - # todo: create appropriate resource requests and limits - resources: - requests: - memory: "128Mi" - cpu: "250m" + - name: shiny + image: absternator/shiny-server:dev # todo: change to actual one!!! mrcide- + volumeMounts: + - name: shiny-data + mountPath: /shiny + # todo: create appropriate resource requests and limits + resources: + requests: + memory: "128Mi" + cpu: "250m" volumes: - - name: shiny-apps - persistentVolumeClaim: - claimName: shiny-apps-pvc - - name: shiny-logs - persistentVolumeClaim: - claimName: shiny-logs-pvc - \ No newline at end of file + - name: shiny-data + persistentVolumeClaim: + claimName: local-path-pvc diff --git a/k8s/base/ingress.yaml b/k8s/base/ingress.yaml index 0a384f8..0509e52 100644 --- a/k8s/base/ingress.yaml +++ b/k8s/base/ingress.yaml @@ -9,13 +9,18 @@ metadata: nginx.ingress.kubernetes.io/session-cookie-max-age: "172800" spec: ingressClassName: nginx + tls: + - hosts: + - shiny-dev.dide.ic.ac.uk + secretName: tls-secret rules: - - http: - paths: - - path: / - pathType: Prefix - backend: - service: - name: shiny-svc - port: - number: 3838 \ No newline at end of file + - host: shiny-dev.dide.ic.ac.uk + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: shiny-svc + port: + number: 3838 diff --git a/k8s/base/kustomization.yaml b/k8s/base/kustomization.yaml index 9738aac..27a134b 100644 --- a/k8s/base/kustomization.yaml +++ b/k8s/base/kustomization.yaml @@ -6,8 +6,7 @@ metadata: namespace: twinkle resources: -- namespace.yaml -- persistance.yaml -- deployment.yaml -- service.yaml -- ingress.yaml \ No newline at end of file + - persistance.yaml + - deployment.yaml + - service.yaml + - ingress.yaml diff --git a/k8s/base/namespace.yaml b/k8s/base/namespace.yaml deleted file mode 100644 index 78717dc..0000000 --- a/k8s/base/namespace.yaml +++ /dev/null @@ -1,4 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: twinkle \ No newline at end of file diff --git a/k8s/base/persistance.yaml b/k8s/base/persistance.yaml index d9787f1..a5bf7c7 100644 --- a/k8s/base/persistance.yaml +++ b/k8s/base/persistance.yaml @@ -1,82 +1,82 @@ # todo: storage capacity as per requirements # todo: For distributed nodes use another storage solution https://ceph.io # storage class -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: local-storage -provisioner: kubernetes.io/no-provisioner -volumeBindingMode: WaitForFirstConsumer ---- +# apiVersion: storage.k8s.io/v1 +# kind: StorageClass +# metadata: +# name: local-storage +# provisioner: kubernetes.io/no-provisioner +# volumeBindingMode: WaitForFirstConsumer +# --- # shiny apps -apiVersion: v1 -kind: PersistentVolume -metadata: - name: shiny-apps-pv -spec: - storageClassName: local-storage - capacity: - storage: 10Gi - volumeMode: Filesystem - accessModes: - - ReadWriteOnce - persistentVolumeReclaimPolicy: Retain - local: - path: /shiny/apps - nodeAffinity: - required: - nodeSelectorTerms: - - matchExpressions: - - key: kubernetes.io/hostname - operator: In - values: - - kind-control-plane # update this to the node name ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: shiny-apps-pvc -spec: - storageClassName: local-storage - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 10Gi ---- -# shiny logs -apiVersion: v1 -kind: PersistentVolume -metadata: - name: shiny-logs-pv -spec: - storageClassName: local-storage - capacity: - storage: 2Gi - volumeMode: Filesystem - accessModes: - - ReadWriteOnce - persistentVolumeReclaimPolicy: Retain - local: - path: /shiny/logs - nodeAffinity: - required: - nodeSelectorTerms: - - matchExpressions: - - key: kubernetes.io/hostname - operator: In - values: - - kind-control-plane # update this to the node you want to use ---- +# apiVersion: v1 +# kind: PersistentVolume +# metadata: +# name: shiny-apps-pv +# spec: +# storageClassName: local-storage +# capacity: +# storage: 10Gi +# volumeMode: Filesystem +# accessModes: +# - ReadWriteOnce +# persistentVolumeReclaimPolicy: Retain +# local: +# path: /shiny/apps +# nodeAffinity: +# required: +# nodeSelectorTerms: +# - matchExpressions: +# - key: kubernetes.io/hostname +# operator: In +# values: +# - kind-control-plane # update this to the node name +# --- apiVersion: v1 kind: PersistentVolumeClaim metadata: - name: shiny-logs-pvc + name: local-path-pvc spec: - storageClassName: local-storage + storageClassName: local-path accessModes: - ReadWriteOnce resources: requests: - storage: 2Gi ---- + storage: 15Gi +# --- +# # shiny logs +# apiVersion: v1 +# kind: PersistentVolume +# metadata: +# name: shiny-logs-pv +# spec: +# storageClassName: local-storage +# capacity: +# storage: 2Gi +# volumeMode: Filesystem +# accessModes: +# - ReadWriteOnce +# persistentVolumeReclaimPolicy: Retain +# local: +# path: /shiny/logs +# nodeAffinity: +# required: +# nodeSelectorTerms: +# - matchExpressions: +# - key: kubernetes.io/hostname +# operator: In +# values: +# - kind-control-plane # update this to the node you want to use +# --- +# apiVersion: v1 +# kind: PersistentVolumeClaim +# metadata: +# name: shiny-logs-pvc +# spec: +# storageClassName: local-storage +# accessModes: +# - ReadWriteOnce +# resources: +# requests: +# storage: 2Gi +# --- diff --git a/k8s/configure_ssl b/k8s/configure_ssl new file mode 100755 index 0000000..4e3fb91 --- /dev/null +++ b/k8s/configure_ssl @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +export VAULT_ADDR=https://vault.dide.ic.ac.uk:8200 +export VAULT_TOKEN=$(vault login -method=github -token-only) + +DEST=${PWD}/k8s/ssl +mkdir -p $DEST + +vault read -field=value /secret/shiny.dide/dev/ssl/cert > \ + ${DEST}/certificate.pem +vault read -field=value /secret/shiny.dide/dev/ssl/key > \ + ${DEST}/key.pem + +kubectl create ns twinkle +kubectl -n twinkle create secret tls tls-secret --cert ${DEST}/certificate.pem --key ${DEST}/key.pem \ No newline at end of file diff --git a/shiny/build b/shiny/build index 9397d52..b5cd616 100755 --- a/shiny/build +++ b/shiny/build @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -e HERE=$(realpath $(dirname $0)) -IMAGE_NAME="mrcide/shiny-server" +IMAGE_NAME="absternator/shiny-server" TAG_VERSION=dev docker pull rocker/shiny:latest docker build --rm \ From 4fc22d82da51b31464446823ec647ef032e59c58 Mon Sep 17 00:00:00 2001 From: absternator Date: Sun, 14 Jan 2024 17:50:11 +0000 Subject: [PATCH 03/17] feat: add start script --- README.md | 2 +- k8s/base/kustomization.yaml | 2 +- k8s/configure_ssl | 1 - start-k8s-shiny | 25 +++++++++++++++++++++++++ 4 files changed, 27 insertions(+), 3 deletions(-) create mode 100755 start-k8s-shiny diff --git a/README.md b/README.md index bb8dc9b..348fd24 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,7 @@ Run the following commands from the root of the repository 1. `./shiny/build` 2. For KIND cluster load image: `kind load docker-image mrcide/shiny-server:dev` -3. Create /shiny/logs and /shiny/apps directories in k8s node. For kind exec into node via `docker exec -it kind-control-plane sh` +3. Create /shiny/logs and /shiny/apps directories in k8s node or run `./k8s/configure_ssl`. For kind exec into node via `docker exec -it kind-control-plane sh` and then `mkdir -p /shiny/logs /shiny/apps` to create the directories. 3.`k apply -k k8s/overlays/production` for production and `k apply -k k8s/overlays/staging` for staging. 4.`k8s/shiny-sync` diff --git a/k8s/base/kustomization.yaml b/k8s/base/kustomization.yaml index 27a134b..46e19bd 100644 --- a/k8s/base/kustomization.yaml +++ b/k8s/base/kustomization.yaml @@ -3,7 +3,7 @@ kind: Kustomization metadata: name: shiny -namespace: twinkle +namespace: twinkle # assume this namespace exists resources: - persistance.yaml diff --git a/k8s/configure_ssl b/k8s/configure_ssl index 4e3fb91..8769c61 100755 --- a/k8s/configure_ssl +++ b/k8s/configure_ssl @@ -11,5 +11,4 @@ vault read -field=value /secret/shiny.dide/dev/ssl/cert > \ vault read -field=value /secret/shiny.dide/dev/ssl/key > \ ${DEST}/key.pem -kubectl create ns twinkle kubectl -n twinkle create secret tls tls-secret --cert ${DEST}/certificate.pem --key ${DEST}/key.pem \ No newline at end of file diff --git a/start-k8s-shiny b/start-k8s-shiny new file mode 100755 index 0000000..ce9cfde --- /dev/null +++ b/start-k8s-shiny @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +# Set the environment variable to the first command-line argument or default to 'staging' +ENV=${1:-staging} + +# Make the script itself executable +chmod +x "$0" + +# Make the k8s directory and its contents executable +chmod +x k8s/ + +# Create a Kubernetes namespace named 'twinkle' +kubectl create ns twinkle + +# Run the script to configure SSL (assuming it's in the k8s directory) +k8s/configure_ssl + +# Apply Kubernetes manifests from overlays based on the specified or default environment +kubectl apply -k "k8s/overlays/$ENV" + +# Wait for the 'shiny-deploy' deployment to be in the 'available' condition within the 'twinkle' namespace +kubectl wait -n twinkle --for=condition=available --timeout=60s deployment/shiny-deploy + +# Run the script for 'shiny-sync' (assuming it's in the k8s directory) +k8s/shiny-sync From 6e4e6c782998c5a8d43460ef63a87fd2c0efc90d Mon Sep 17 00:00:00 2001 From: absternator Date: Mon, 15 Jan 2024 09:23:32 +0000 Subject: [PATCH 04/17] docs: update readme --- README.md | 19 ++++++++----------- k8s/base/persistance.yaml | 2 +- start-k8s-shiny | 1 + 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 348fd24..c462e30 100644 --- a/README.md +++ b/README.md @@ -67,19 +67,16 @@ nee to be adjusted for other k8s clusters. ### Prerequisites -A single node k8s cluster is needed. To setup a k8s cluster follow the guide [here](https://mrc-ide.myjetbrains.com/youtrack/articles/RESIDE-A-31/Setting-up-Kubernetes-k8s-Cluster). +A production kubernetes cluster using k3s is needed to be setup first. To setup a k8s cluster follow the guide [here](https://mrc-ide.myjetbrains.com/youtrack/articles/RESIDE-A-31/Setting-up-Kubernetes-k8s-Cluster). Note: If using dev cluster, the storageClassName: local-path is unavailable and you will need to provision own storage class and persistant volume. -Run the following commands from the root of the repository +Run `start-k8s-shiny ` to run the shint server in k8s. -1. `./shiny/build` -2. For KIND cluster load image: `kind load docker-image mrcide/shiny-server:dev` -3. Create /shiny/logs and /shiny/apps directories in k8s node or run `./k8s/configure_ssl`. For kind exec into node via `docker exec -it kind-control-plane sh` - and then `mkdir -p /shiny/logs /shiny/apps` to create the directories. -3.`k apply -k k8s/overlays/production` for production and `k apply -k k8s/overlays/staging` for staging. -4.`k8s/shiny-sync` - -Great the shiny server is running and can be seen on the External IP of the ingress-nginx-controller LoadBalancer service. +The server can be seen running on the External IP of the ingress-nginx-controller LoadBalancer service. +Note: `kubectl get svc -n ingress-nginx ingress-nginx-controller` command to get external IP. #### Teardown -`k delete -k k8s/overlays/production` for production or `k delete -k k8s/overlays/staging` for staging +Run the following: + +1. `kubectl delete -k k8s/overlays/`. Replace with staging or production. +2. `kubectl delete ns twinkle` to remove namespace. diff --git a/k8s/base/persistance.yaml b/k8s/base/persistance.yaml index a5bf7c7..a87f354 100644 --- a/k8s/base/persistance.yaml +++ b/k8s/base/persistance.yaml @@ -1,5 +1,5 @@ # todo: storage capacity as per requirements -# todo: For distributed nodes use another storage solution https://ceph.io +# todo: Switch out for longhorn if local does not work # storage class # apiVersion: storage.k8s.io/v1 # kind: StorageClass diff --git a/start-k8s-shiny b/start-k8s-shiny index ce9cfde..66e76ba 100755 --- a/start-k8s-shiny +++ b/start-k8s-shiny @@ -19,6 +19,7 @@ k8s/configure_ssl kubectl apply -k "k8s/overlays/$ENV" # Wait for the 'shiny-deploy' deployment to be in the 'available' condition within the 'twinkle' namespace +echo 'Waiting for pods to be ready...' kubectl wait -n twinkle --for=condition=available --timeout=60s deployment/shiny-deploy # Run the script for 'shiny-sync' (assuming it's in the k8s directory) From 7badd77575ac24126e49a972c777769c09369fed Mon Sep 17 00:00:00 2001 From: absternator Date: Mon, 15 Jan 2024 10:57:04 +0000 Subject: [PATCH 05/17] chore: comments for error + persistance --- k8s/base/persistance.yaml | 45 ++++----------------------------------- start-k8s-shiny | 7 +++++- 2 files changed, 10 insertions(+), 42 deletions(-) diff --git a/k8s/base/persistance.yaml b/k8s/base/persistance.yaml index a87f354..933d18e 100644 --- a/k8s/base/persistance.yaml +++ b/k8s/base/persistance.yaml @@ -1,6 +1,7 @@ # todo: storage capacity as per requirements # todo: Switch out for longhorn if local does not work -# storage class + +# Uncoment below if on dev cluster or dont have access to local-storage storageClass from k3s # apiVersion: storage.k8s.io/v1 # kind: StorageClass # metadata: @@ -8,15 +9,14 @@ # provisioner: kubernetes.io/no-provisioner # volumeBindingMode: WaitForFirstConsumer # --- -# shiny apps # apiVersion: v1 # kind: PersistentVolume # metadata: # name: shiny-apps-pv # spec: -# storageClassName: local-storage +# storageClassName: local-path # capacity: -# storage: 10Gi +# storage: 15Gi # volumeMode: Filesystem # accessModes: # - ReadWriteOnce @@ -43,40 +43,3 @@ spec: resources: requests: storage: 15Gi -# --- -# # shiny logs -# apiVersion: v1 -# kind: PersistentVolume -# metadata: -# name: shiny-logs-pv -# spec: -# storageClassName: local-storage -# capacity: -# storage: 2Gi -# volumeMode: Filesystem -# accessModes: -# - ReadWriteOnce -# persistentVolumeReclaimPolicy: Retain -# local: -# path: /shiny/logs -# nodeAffinity: -# required: -# nodeSelectorTerms: -# - matchExpressions: -# - key: kubernetes.io/hostname -# operator: In -# values: -# - kind-control-plane # update this to the node you want to use -# --- -# apiVersion: v1 -# kind: PersistentVolumeClaim -# metadata: -# name: shiny-logs-pvc -# spec: -# storageClassName: local-storage -# accessModes: -# - ReadWriteOnce -# resources: -# requests: -# storage: 2Gi -# --- diff --git a/start-k8s-shiny b/start-k8s-shiny index 66e76ba..cc5bb1c 100755 --- a/start-k8s-shiny +++ b/start-k8s-shiny @@ -1,8 +1,13 @@ #!/usr/bin/env bash +# example use case: ./start-k8s-shiny production + # Set the environment variable to the first command-line argument or default to 'staging' ENV=${1:-staging} - +if [[ "$ENV" != "staging" && "$ENV" != "production" ]]; then + echo "Error: env must be either 'staging' or 'production'" + exit 1 +fi # Make the script itself executable chmod +x "$0" From 1078b51394ed53a0a7df853e1b06d3be0b8d190a Mon Sep 17 00:00:00 2001 From: absternator Date: Wed, 17 Jan 2024 13:28:26 +0000 Subject: [PATCH 06/17] fix: longhorn and update scripts --- k8s/base/deployment.yaml | 10 ++++----- k8s/base/persistance.yaml | 26 +++++++++++----------- k8s/overlays/production/kustomization.yaml | 2 +- start-k8s-shiny | 4 ++-- 4 files changed, 21 insertions(+), 21 deletions(-) diff --git a/k8s/base/deployment.yaml b/k8s/base/deployment.yaml index b06111f..9f26a8c 100644 --- a/k8s/base/deployment.yaml +++ b/k8s/base/deployment.yaml @@ -28,11 +28,11 @@ spec: - name: shiny-data mountPath: /shiny # todo: create appropriate resource requests and limits - resources: - requests: - memory: "128Mi" - cpu: "250m" + # resources: + # requests: + # memory: "128Mi" + # cpu: "250m" volumes: - name: shiny-data persistentVolumeClaim: - claimName: local-path-pvc + claimName: shiny-pvc diff --git a/k8s/base/persistance.yaml b/k8s/base/persistance.yaml index 933d18e..70a5119 100644 --- a/k8s/base/persistance.yaml +++ b/k8s/base/persistance.yaml @@ -1,7 +1,18 @@ # todo: storage capacity as per requirements -# todo: Switch out for longhorn if local does not work - +# can use local-path if single node +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: shiny-pvc +spec: + storageClassName: longhorn + accessModes: + - ReadWriteMany + resources: + requests: + storage: 15Gi # Uncoment below if on dev cluster or dont have access to local-storage storageClass from k3s +# --- # apiVersion: storage.k8s.io/v1 # kind: StorageClass # metadata: @@ -32,14 +43,3 @@ # values: # - kind-control-plane # update this to the node name # --- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: local-path-pvc -spec: - storageClassName: local-path - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 15Gi diff --git a/k8s/overlays/production/kustomization.yaml b/k8s/overlays/production/kustomization.yaml index 082f513..2c06008 100644 --- a/k8s/overlays/production/kustomization.yaml +++ b/k8s/overlays/production/kustomization.yaml @@ -9,4 +9,4 @@ commonLabels: replicas: - name: shiny-deploy - count: 7 \ No newline at end of file + count: 9 diff --git a/start-k8s-shiny b/start-k8s-shiny index cc5bb1c..01c173a 100755 --- a/start-k8s-shiny +++ b/start-k8s-shiny @@ -18,14 +18,14 @@ chmod +x k8s/ kubectl create ns twinkle # Run the script to configure SSL (assuming it's in the k8s directory) -k8s/configure_ssl +k8s/configure_ssl || { echo "Error: Failed to configure SSL"; exit 1; } # Apply Kubernetes manifests from overlays based on the specified or default environment kubectl apply -k "k8s/overlays/$ENV" # Wait for the 'shiny-deploy' deployment to be in the 'available' condition within the 'twinkle' namespace echo 'Waiting for pods to be ready...' -kubectl wait -n twinkle --for=condition=available --timeout=60s deployment/shiny-deploy +kubectl wait -n twinkle --for=condition=available --timeout=300s deployment/shiny-deploy # Run the script for 'shiny-sync' (assuming it's in the k8s directory) k8s/shiny-sync From 55e8329c787e5d97ff99d584a33d31d55fc861f5 Mon Sep 17 00:00:00 2001 From: absternator Date: Wed, 17 Jan 2024 13:31:01 +0000 Subject: [PATCH 07/17] update readme --- README.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index c462e30..b0c0849 100644 --- a/README.md +++ b/README.md @@ -67,12 +67,11 @@ nee to be adjusted for other k8s clusters. ### Prerequisites -A production kubernetes cluster using k3s is needed to be setup first. To setup a k8s cluster follow the guide [here](https://mrc-ide.myjetbrains.com/youtrack/articles/RESIDE-A-31/Setting-up-Kubernetes-k8s-Cluster). Note: If using dev cluster, the storageClassName: local-path is unavailable and you will need to provision own storage class and persistant volume. +A production kubernetes cluster using k3s is needed to be setup first. To setup a k8s cluster follow the guide [here](https://mrc-ide.myjetbrains.com/youtrack/articles/RESIDE-A-31/Setting-up-Kubernetes-k8s-Cluster). Note: If using dev cluster (KIND), the storageClassName: local-path & longhorn is unavailable and you will need to provision own storage class and persistant volume. Run `start-k8s-shiny ` to run the shint server in k8s. -The server can be seen running on the External IP of the ingress-nginx-controller LoadBalancer service. -Note: `kubectl get svc -n ingress-nginx ingress-nginx-controller` command to get external IP. +The server can be seen running on https://shiny-dev.dide.ic.ac.uk. #### Teardown From 37c5cfd4d419258bd6bdb0476043d077ffd7e9eb Mon Sep 17 00:00:00 2001 From: absternator Date: Wed, 17 Jan 2024 13:34:39 +0000 Subject: [PATCH 08/17] chore: remove dead code --- k8s/base/persistance.yaml | 34 +--------------------------------- 1 file changed, 1 insertion(+), 33 deletions(-) diff --git a/k8s/base/persistance.yaml b/k8s/base/persistance.yaml index 70a5119..0edf096 100644 --- a/k8s/base/persistance.yaml +++ b/k8s/base/persistance.yaml @@ -1,5 +1,5 @@ # todo: storage capacity as per requirements -# can use local-path if single node +# can use local-path if single node or dont need distributed & replicated apiVersion: v1 kind: PersistentVolumeClaim metadata: @@ -11,35 +11,3 @@ spec: resources: requests: storage: 15Gi -# Uncoment below if on dev cluster or dont have access to local-storage storageClass from k3s -# --- -# apiVersion: storage.k8s.io/v1 -# kind: StorageClass -# metadata: -# name: local-storage -# provisioner: kubernetes.io/no-provisioner -# volumeBindingMode: WaitForFirstConsumer -# --- -# apiVersion: v1 -# kind: PersistentVolume -# metadata: -# name: shiny-apps-pv -# spec: -# storageClassName: local-path -# capacity: -# storage: 15Gi -# volumeMode: Filesystem -# accessModes: -# - ReadWriteOnce -# persistentVolumeReclaimPolicy: Retain -# local: -# path: /shiny/apps -# nodeAffinity: -# required: -# nodeSelectorTerms: -# - matchExpressions: -# - key: kubernetes.io/hostname -# operator: In -# values: -# - kind-control-plane # update this to the node name -# --- From 472a68923f1c33221fb05c2a8a110481ee1474a4 Mon Sep 17 00:00:00 2001 From: absternator Date: Wed, 17 Jan 2024 13:41:08 +0000 Subject: [PATCH 09/17] update readme extras --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index b0c0849..3247757 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ docker run -d --name haproxy --network twinkle mrcide/haproxy:dev docker run -d --name apache --network twinkle \ -p 80:80 \ -p 443:443 \ - -p 9005:9005 \ + -p 9000:9000 \ -v "${PWD}/apache/httpd.conf:/usr/local/apache2/conf/httpd.conf:ro" \ -v "${PWD}/apache/auth:/usr/local/apache2/conf/auth:ro" \ -v "${PWD}/apache/ssl:/usr/local/apache2/conf/ssl:ro" \ @@ -71,7 +71,6 @@ A production kubernetes cluster using k3s is needed to be setup first. To setup Run `start-k8s-shiny ` to run the shint server in k8s. -The server can be seen running on https://shiny-dev.dide.ic.ac.uk. #### Teardown From 31b18a4263f89d19d9f355e743abfef11532c947 Mon Sep 17 00:00:00 2001 From: anmol thapar Date: Fri, 19 Jan 2024 10:40:32 +0000 Subject: [PATCH 10/17] refactor: PR comments --- README.md | 61 +------------------ k8s/base/deployment.yaml | 2 +- k8s/configure_ssl | 1 + k8s/krsync | 4 +- .../{staging => testing}/kustomization.yaml | 2 +- k8s/shiny-sync | 2 +- start-k8s-shiny | 10 +-- 7 files changed, 13 insertions(+), 69 deletions(-) rename k8s/overlays/{staging => testing}/kustomization.yaml (91%) diff --git a/README.md b/README.md index 3247757..f1ff711 100644 --- a/README.md +++ b/README.md @@ -2,63 +2,6 @@ This is a temporary repo designed to give an easy-to-understand deployment of the apache / haproxy / shiny stack; it does not include configurable applications or anything like that. See it online at https://shiny-dev.dide.ic.ac.uk (DIDE network only). -## The components - -Some of these are docker images, and they are not set to pull, so you will want to rebuild. All build very quickly. - -### apache - -Running an unmodified httpd container (previously was 2.4, we'll update once we know this works). The configuration ([`apache/httpd.conf`](httpd/httpd.conf)) and certificates (`apache/ssl`) will be read-only mounted into the container. You need to fetch the ssl key and certificate, run `./apache/configure_ssl` to do this (only needs to be done if they change or if the `ssl` directory is deleted) - -### haproxy - -Build the image with `./haproxy/build` which builds `mrcide/haproxy` with a configuration that can be seen in [`haproxy/haproxy.cfg`](haproxy/haproxy.cfg) and some utilities which enable some degree of dynamic scaling of shiny servers. - -### shiny - -A lightly modified version of the official shiny container; the original version was more extensively modified - -### apps - -Some applications (copied over from the original deployment are in `apps`). These will want to be in a volume; run `./apps/create_volume` to copy them into the volume - -### Summary - -``` -./apache/configure_ssl -./haproxy/build -./shiny/build -./apps/create_volume -``` - -## Bringing the bits up - -```bash -docker network create twinkle 2> /dev/null || /bin/true -docker volume create shiny_logs -docker run -d --name haproxy --network twinkle mrcide/haproxy:dev -docker run -d --name apache --network twinkle \ - -p 80:80 \ - -p 443:443 \ - -p 9000:9000 \ - -v "${PWD}/apache/httpd.conf:/usr/local/apache2/conf/httpd.conf:ro" \ - -v "${PWD}/apache/auth:/usr/local/apache2/conf/auth:ro" \ - -v "${PWD}/apache/ssl:/usr/local/apache2/conf/ssl:ro" \ - httpd:2.4 -docker run -d --name shiny-1 --network=twinkle \ - -v twinkle_apps:/shiny/apps \ - -v twinkle_logs:/shiny/logs \ - -p 3838:3838 \ - mrcide/shiny-server:dev -docker exec haproxy update_shiny_servers shiny 1 -``` - -Teardown - -```bash -docker rm -f haproxy apache shiny-1 -docker network rm twinkle -``` ## Running in Kubernetes @@ -69,12 +12,12 @@ nee to be adjusted for other k8s clusters. A production kubernetes cluster using k3s is needed to be setup first. To setup a k8s cluster follow the guide [here](https://mrc-ide.myjetbrains.com/youtrack/articles/RESIDE-A-31/Setting-up-Kubernetes-k8s-Cluster). Note: If using dev cluster (KIND), the storageClassName: local-path & longhorn is unavailable and you will need to provision own storage class and persistant volume. -Run `start-k8s-shiny ` to run the shint server in k8s. +Run `start-k8s-shiny ` to run the shint server in k8s. #### Teardown Run the following: -1. `kubectl delete -k k8s/overlays/`. Replace with staging or production. +1. `kubectl delete -k k8s/overlays/`. Replace with testing or production. 2. `kubectl delete ns twinkle` to remove namespace. diff --git a/k8s/base/deployment.yaml b/k8s/base/deployment.yaml index 9f26a8c..3216d6b 100644 --- a/k8s/base/deployment.yaml +++ b/k8s/base/deployment.yaml @@ -27,7 +27,7 @@ spec: volumeMounts: - name: shiny-data mountPath: /shiny - # todo: create appropriate resource requests and limits + # todo: create appropriate resource requests # resources: # requests: # memory: "128Mi" diff --git a/k8s/configure_ssl b/k8s/configure_ssl index 8769c61..5262317 100755 --- a/k8s/configure_ssl +++ b/k8s/configure_ssl @@ -1,4 +1,5 @@ #!/usr/bin/env bash +set -eu export VAULT_ADDR=https://vault.dide.ic.ac.uk:8200 export VAULT_TOKEN=$(vault login -method=github -token-only) diff --git a/k8s/krsync b/k8s/krsync index 2426cdd..7dc6033 100755 --- a/k8s/krsync +++ b/k8s/krsync @@ -1,5 +1,5 @@ #!/usr/bin/env bash - +set -eu # https://serverfault.com/questions/741670/rsync-files-to-a-kubernetes-pod # example usage: # ./krsync -av --progress --stats podName@namespace: @@ -22,5 +22,5 @@ if [ "X$pod" = "X-l" ]; then shift fi -# customise this to cluster needs!! + exec kubectl $namespace exec -i $pod -- "$@" \ No newline at end of file diff --git a/k8s/overlays/staging/kustomization.yaml b/k8s/overlays/testing/kustomization.yaml similarity index 91% rename from k8s/overlays/staging/kustomization.yaml rename to k8s/overlays/testing/kustomization.yaml index 544d045..6fe1290 100644 --- a/k8s/overlays/staging/kustomization.yaml +++ b/k8s/overlays/testing/kustomization.yaml @@ -5,7 +5,7 @@ resources: - ../../base commonLabels: - env: staging + env: testing replicas: - name: shiny-deploy diff --git a/k8s/shiny-sync b/k8s/shiny-sync index b9ac3b2..ca9f3c0 100755 --- a/k8s/shiny-sync +++ b/k8s/shiny-sync @@ -1,5 +1,5 @@ #!/usr/bin/env bash - +set -eu # uses krysnc to sync files to a kubernetes pod and thus into k8s cluster (via persistant volume) # RUN this from repository root to ensure correct paths to app files NAMESPACE=twinkle diff --git a/start-k8s-shiny b/start-k8s-shiny index 01c173a..5a3a1db 100755 --- a/start-k8s-shiny +++ b/start-k8s-shiny @@ -1,11 +1,11 @@ #!/usr/bin/env bash - +set -eu # example use case: ./start-k8s-shiny production -# Set the environment variable to the first command-line argument or default to 'staging' -ENV=${1:-staging} -if [[ "$ENV" != "staging" && "$ENV" != "production" ]]; then - echo "Error: env must be either 'staging' or 'production'" +# Set the environment variable to the first command-line argument or default to 'testing' +ENV=${1:-testing} +if [[ "$ENV" != "testing" && "$ENV" != "production" ]]; then + echo "Error: env must be either 'testing' or 'production'" exit 1 fi # Make the script itself executable From a6f2fa58de829f85e6d77c23b1755421d8aae3de Mon Sep 17 00:00:00 2001 From: anmol thapar Date: Tue, 23 Jan 2024 14:52:53 +0000 Subject: [PATCH 11/17] Update configure_ssl script to use vault k8s auth --- k8s/configure_ssl | 1 + 1 file changed, 1 insertion(+) diff --git a/k8s/configure_ssl b/k8s/configure_ssl index 5262317..8d2312e 100755 --- a/k8s/configure_ssl +++ b/k8s/configure_ssl @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# TODO: update to use vault k8s auth set -eu export VAULT_ADDR=https://vault.dide.ic.ac.uk:8200 From 16ad36d95e378606c094c4d03c4cb90f7065adb3 Mon Sep 17 00:00:00 2001 From: absternator Date: Thu, 25 Jan 2024 12:13:52 +0000 Subject: [PATCH 12/17] cleanups + add gha --- .github/workflows/shiny-image-push.yaml | 24 +++++ README.md | 3 +- apache/configure_ssl | 9 -- apache/httpd.conf | 128 ------------------------ haproxy/Dockerfile | 5 - haproxy/bin/add_server | 11 -- haproxy/bin/drop_server | 6 -- haproxy/bin/update_shiny_servers | 17 ---- haproxy/build | 8 -- haproxy/haproxy.cfg | 44 -------- k8s/base/deployment.yaml | 4 +- shiny/build | 9 +- shiny/build-and-push | 7 ++ 13 files changed, 37 insertions(+), 238 deletions(-) create mode 100644 .github/workflows/shiny-image-push.yaml delete mode 100755 apache/configure_ssl delete mode 100644 apache/httpd.conf delete mode 100644 haproxy/Dockerfile delete mode 100755 haproxy/bin/add_server delete mode 100755 haproxy/bin/drop_server delete mode 100755 haproxy/bin/update_shiny_servers delete mode 100755 haproxy/build delete mode 100644 haproxy/haproxy.cfg create mode 100755 shiny/build-and-push diff --git a/.github/workflows/shiny-image-push.yaml b/.github/workflows/shiny-image-push.yaml new file mode 100644 index 0000000..215b048 --- /dev/null +++ b/.github/workflows/shiny-image-push.yaml @@ -0,0 +1,24 @@ +name: Build and push Shiny server image + +on: + push: + branches: + - "main" + +jobs: + docker: + runs-on: ubuntu-latest + steps: + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Checkout repository + uses: actions/checkout@v3 + - name: Build and push + run: shiny/build-and-push diff --git a/README.md b/README.md index f1ff711..6543494 100644 --- a/README.md +++ b/README.md @@ -10,11 +10,10 @@ nee to be adjusted for other k8s clusters. ### Prerequisites -A production kubernetes cluster using k3s is needed to be setup first. To setup a k8s cluster follow the guide [here](https://mrc-ide.myjetbrains.com/youtrack/articles/RESIDE-A-31/Setting-up-Kubernetes-k8s-Cluster). Note: If using dev cluster (KIND), the storageClassName: local-path & longhorn is unavailable and you will need to provision own storage class and persistant volume. +A k8s kubernetes cluster using k3s is needed to be setup first. To setup a k8s cluster follow the guide [here](https://mrc-ide.myjetbrains.com/youtrack/articles/RESIDE-A-31/Setting-up-Kubernetes-k8s-Cluster). Run `start-k8s-shiny ` to run the shint server in k8s. - #### Teardown Run the following: diff --git a/apache/configure_ssl b/apache/configure_ssl deleted file mode 100755 index 18985e0..0000000 --- a/apache/configure_ssl +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env bash -export VAULT_ADDR=https://vault.dide.ic.ac.uk:8200 -export VAULT_TOKEN=$(vault login -method=github -token-only) -DEST=apache/ssl -mkdir -p ${DEST} -vault read -field=value /secret/shiny.dide/dev/ssl/cert > \ - ${DEST}/certificate.pem -vault read -field=value /secret/shiny.dide/dev/ssl/key > \ - ${DEST}/key.pem diff --git a/apache/httpd.conf b/apache/httpd.conf deleted file mode 100644 index 3beeb59..0000000 --- a/apache/httpd.conf +++ /dev/null @@ -1,128 +0,0 @@ -# Starting point is the "Apache httpd v2.4 minimal configuration" -# https://wiki.apache.org/httpd/Minimal_Config -# https://support.rstudio.com/hc/en-us/articles/213733868-Running-Shiny-Server-with-a-Proxy - -ServerAdmin r.fitzjohn@imperial.ac.uk -ServerName shiny-dev.dide.ic.ac.uk - -ServerRoot "/usr/local/apache2" - -ServerSignature Off -ServerTokens Prod - -User daemon -Group daemon - -# Minimum modules needed -LoadModule mpm_event_module modules/mod_mpm_event.so -LoadModule log_config_module modules/mod_log_config.so -LoadModule mime_module modules/mod_mime.so -LoadModule dir_module modules/mod_dir.so -LoadModule authz_core_module modules/mod_authz_core.so -LoadModule unixd_module modules/mod_unixd.so - -# For proxying shiny: -LoadModule rewrite_module modules/mod_rewrite.so -LoadModule proxy_module modules/mod_proxy.so -LoadModule proxy_wstunnel_module modules/mod_proxy_wstunnel.so -LoadModule proxy_http_module modules/mod_proxy_http.so - -# For doing auth -LoadModule authn_core_module modules/mod_authn_core.so -LoadModule authn_file_module modules/mod_authn_file.so -LoadModule auth_basic_module modules/mod_auth_basic.so -LoadModule authz_groupfile_module modules/mod_authz_groupfile.so -LoadModule authz_user_module modules/mod_authz_user.so - -# SSL -LoadModule setenvif_module modules/mod_setenvif.so -LoadModule ssl_module modules/mod_ssl.so -LoadModule socache_shmcb_module modules/mod_socache_shmcb.so - -# https://aaronsilber.me/2016/11/02/disable-3des-ssl-ciphers-apache-nginx/ -SSLCipherSuite HIGH:MEDIUM:!SSLv3:!kRSA:!3DES -SSLProxyCipherSuite HIGH:MEDIUM:!SSLv3:!kRSA:!3DES -SSLPassPhraseDialog builtin -SSLSessionCache "shmcb:/usr/local/apache2/logs/ssl_scache(512000)" -SSLSessionCacheTimeout 300 - -## https://httpd.apache.org/docs/trunk/ssl/ssl_howto.html -SSLProtocol all -SSLv3 -TLSv1 -TLSv1.1 -SSLHonorCipherOrder on -SSLCompression off -SSLSessionTickets off - - -ErrorLog /proc/self/fd/2 -LogLevel warn - -LogFormat "%h %l %u %t \"%r\" %>s %b" common -CustomLog /proc/self/fd/1 common - -TypesConfig conf/mime.types - -PidFile logs/httpd.pid - -User nobody - -# Port to Listen on -Listen *:80 -Listen *:443 -Listen *:9000 -Listen *:9001 - -# In a basic setup httpd can only serve files from its document root -DocumentRoot "/usr/local/apache2/htdocs" - -# Default file to serve -DirectoryIndex index.html - -# Never change this block - - AllowOverride None - Require all denied - - -# Allow documents to be served from the DocumentRoot - - Require all granted - - - - RewriteEngine On - RewriteRule ^(.*)$ https://%{HTTP_HOST}$1 [R=301,L] - - - - SSLEngine on - # or server.crt - SSLCertificateFile "/usr/local/apache2/conf/ssl/certificate.pem" - # or server.key - SSLCertificateKeyFile "/usr/local/apache2/conf/ssl/key.pem" - CustomLog "/usr/local/apache2/logs/ssl_request_log" \ - "%t %h %{SSL_PROTOCOL}x %{SSL_CIPHER}x \"%r\" %b" - - RewriteEngine on - RewriteCond %{HTTP:Upgrade} =websocket - RewriteRule /(.*) ws://haproxy:8080/$1 [P,L] - RewriteCond %{HTTP:Upgrade} !=websocket - RewriteRule /(.*) http://haproxy:8080/$1 [P,L] - ProxyPass / http://haproxy:8080/ - ProxyPassReverse / http://haproxy:8080/ - - IncludeOptional conf/auth/*.conf - - - - SSLEngine on - # or server.crt - SSLCertificateFile "/usr/local/apache2/conf/ssl/certificate.pem" - # or server.key - SSLCertificateKeyFile "/usr/local/apache2/conf/ssl/key.pem" - CustomLog "/usr/local/apache2/logs/ssl_request_log" \ - "%t %h %{SSL_PROTOCOL}x %{SSL_CIPHER}x \"%r\" %b" - - RewriteEngine on - ProxyPass / http://haproxy:9001/ - ProxyPassReverse / http://haproxy:9001/ - diff --git a/haproxy/Dockerfile b/haproxy/Dockerfile deleted file mode 100644 index 98676ba..0000000 --- a/haproxy/Dockerfile +++ /dev/null @@ -1,5 +0,0 @@ -FROM haproxy:1.8 -RUN apt-get update && apt-get install -y \ - socat -COPY bin /usr/local/bin -COPY haproxy.cfg /usr/local/etc/haproxy/haproxy.cfg diff --git a/haproxy/bin/add_server b/haproxy/bin/add_server deleted file mode 100755 index cb4cacc..0000000 --- a/haproxy/bin/add_server +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash -set -e - -SERVER=$1 -TARGET=$2 -echo "Enabling $SERVER as $TARGET" -TARGET_IP=$(getent hosts $TARGET | cut -d ' ' -f1) -echo "set server servers/$SERVER addr $TARGET_IP" | \ - socat stdio /var/run/hapee-lb.sock > /dev/null -echo "set server servers/$SERVER state ready" | \ - socat stdio /var/run/hapee-lb.sock > /dev/null diff --git a/haproxy/bin/drop_server b/haproxy/bin/drop_server deleted file mode 100755 index 4d4c05b..0000000 --- a/haproxy/bin/drop_server +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash -set -e -SERVER=$1 -echo "Disabling $SERVER" -echo "disable server servers/$SERVER" | \ - socat stdio /var/run/hapee-lb.sock > /dev/null diff --git a/haproxy/bin/update_shiny_servers b/haproxy/bin/update_shiny_servers deleted file mode 100755 index 3e9626d..0000000 --- a/haproxy/bin/update_shiny_servers +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash -set -eu -SERVER_NAME_BASE=${1:-shiny-server-config} -SERVER_N=${2:-100} - -echo "Updating servers 1..${SERVER_N}" - -for i in $(seq $SERVER_N); do - SERVER_NAME="${SERVER_NAME_BASE}-${i}" - SERVER_CODE="shiny${i}" - SERVER_IP=$(getent hosts $SERVER_NAME | cut -d ' ' -f1) - if [[ -z $SERVER_IP ]]; then - drop_server "$SERVER_CODE" - else - add_server "$SERVER_CODE" "$SERVER_IP" - fi -done diff --git a/haproxy/build b/haproxy/build deleted file mode 100755 index 1efe4ee..0000000 --- a/haproxy/build +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash -set -e -HERE=$(realpath $(dirname $0)) -IMAGE_NAME="mrcide/haproxy" -TAG_VERSION=dev -docker build --rm \ - --tag "${IMAGE_NAME}:${TAG_VERSION}" \ - $HERE diff --git a/haproxy/haproxy.cfg b/haproxy/haproxy.cfg deleted file mode 100644 index eafb204..0000000 --- a/haproxy/haproxy.cfg +++ /dev/null @@ -1,44 +0,0 @@ -global - daemon - maxconn 256 - log 127.0.0.1 local0 notice - stats socket /var/run/hapee-lb.sock mode 666 level admin - stats socket ipv4@127.0.0.1:9999 level admin - stats timeout 2m - -defaults - log global - mode http - timeout connect 5000ms - timeout client 50000ms - timeout server 50000ms - # https://www.haproxy.com/blog/websockets-load-balancing-with-haproxy/ - timeout tunnel 3600s - timeout http-keep-alive 1s - timeout http-request 15s - timeout queue 30s - timeout tarpit 60s - -frontend http-in - bind *:8080 - option httplog - option forwardfor - default_backend servers - -# point this to K8s master nodes (control planes) -backend servers - # I think that leastconn is probably the best bet here. - balance leastconn - # https://www.haproxy.com/blog/whats-new-haproxy-1-8/ - dynamic-cookie-key MYKEY - cookie SRVID insert dynamic - # This sets us up for up to 100 workers which do not need to exist - # at the point where we start up the proxy - server-template shiny 1-100 127.0.0.1:3838 check disabled - -listen stats - bind *:9001 - mode http - stats enable - stats realm Haproxy\ Statistics - stats uri / diff --git a/k8s/base/deployment.yaml b/k8s/base/deployment.yaml index 3216d6b..5683780 100644 --- a/k8s/base/deployment.yaml +++ b/k8s/base/deployment.yaml @@ -23,11 +23,11 @@ spec: mountPath: /shiny containers: - name: shiny - image: absternator/shiny-server:dev # todo: change to actual one!!! mrcide- + image: mrcide/shiny-server:dev # todo: change to actual one!!! mrcide- volumeMounts: - name: shiny-data mountPath: /shiny - # todo: create appropriate resource requests + # todo: create appropriate resource requests # resources: # requests: # memory: "128Mi" diff --git a/shiny/build b/shiny/build index b5cd616..8b34ddf 100755 --- a/shiny/build +++ b/shiny/build @@ -1,9 +1,6 @@ #!/usr/bin/env bash set -e HERE=$(realpath $(dirname $0)) -IMAGE_NAME="absternator/shiny-server" -TAG_VERSION=dev -docker pull rocker/shiny:latest -docker build --rm \ - --tag "${IMAGE_NAME}:${TAG_VERSION}" \ - $HERE +IMAGE_NAME="mrcide/shiny-server" +TAG=dev +docker build -t "${IMAGE_NAME}:${TAG}" $HERE diff --git a/shiny/build-and-push b/shiny/build-and-push new file mode 100755 index 0000000..ef36356 --- /dev/null +++ b/shiny/build-and-push @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +set -e +HERE=$(realpath $(dirname $0)) + +. $HERE/build + +docker push $IMAGE_NAME:$TAG \ No newline at end of file From 8b19073d795d886555371127fa4903f043094a6c Mon Sep 17 00:00:00 2001 From: absternator Date: Thu, 25 Jan 2024 12:15:51 +0000 Subject: [PATCH 13/17] gha update --- .github/workflows/shiny-image-push.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/shiny-image-push.yaml b/.github/workflows/shiny-image-push.yaml index 215b048..4fb5937 100644 --- a/.github/workflows/shiny-image-push.yaml +++ b/.github/workflows/shiny-image-push.yaml @@ -4,6 +4,9 @@ on: push: branches: - "main" + pull_request: + branches: + - "main" jobs: docker: From d5fd1c44c0683695a14285cc6e00411feb20e4f6 Mon Sep 17 00:00:00 2001 From: absternator Date: Thu, 25 Jan 2024 12:20:32 +0000 Subject: [PATCH 14/17] remove PR into main docker hub push --- .github/workflows/shiny-image-push.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/shiny-image-push.yaml b/.github/workflows/shiny-image-push.yaml index 4fb5937..215b048 100644 --- a/.github/workflows/shiny-image-push.yaml +++ b/.github/workflows/shiny-image-push.yaml @@ -4,9 +4,6 @@ on: push: branches: - "main" - pull_request: - branches: - - "main" jobs: docker: From bc066f3c3644bfaeddf55151a38aa48d90fe9865 Mon Sep 17 00:00:00 2001 From: absternator Date: Thu, 25 Jan 2024 12:43:56 +0000 Subject: [PATCH 15/17] fix krsync issue --- k8s/krsync | 1 - 1 file changed, 1 deletion(-) diff --git a/k8s/krsync b/k8s/krsync index 7dc6033..5f869bf 100755 --- a/k8s/krsync +++ b/k8s/krsync @@ -1,5 +1,4 @@ #!/usr/bin/env bash -set -eu # https://serverfault.com/questions/741670/rsync-files-to-a-kubernetes-pod # example usage: # ./krsync -av --progress --stats podName@namespace: From 9a9c8f04461f82983c7ec50f7ca20cc717648c57 Mon Sep 17 00:00:00 2001 From: absternator Date: Thu, 25 Jan 2024 15:39:09 +0000 Subject: [PATCH 16/17] feat: cleanup kustomize for testing and prod --- README.md | 6 +----- k8s/base/deployment.yaml | 2 +- k8s/base/ingress.yaml | 7 +------ k8s/base/kustomization.yaml | 2 +- .../{persistance.yaml => persistence.yaml} | 5 ++--- k8s/overlays/production/ingress-patch.yaml | 21 +++++++++++++++++++ k8s/overlays/production/kustomization.yaml | 15 ++++++++----- .../production/persistence-patch.yaml | 8 +++++++ k8s/overlays/testing/kustomization.yaml | 8 ++++--- 9 files changed, 50 insertions(+), 24 deletions(-) rename k8s/base/{persistance.yaml => persistence.yaml} (59%) create mode 100644 k8s/overlays/production/ingress-patch.yaml create mode 100644 k8s/overlays/production/persistence-patch.yaml diff --git a/README.md b/README.md index 6543494..443bc75 100644 --- a/README.md +++ b/README.md @@ -2,17 +2,13 @@ This is a temporary repo designed to give an easy-to-understand deployment of the apache / haproxy / shiny stack; it does not include configurable applications or anything like that. See it online at https://shiny-dev.dide.ic.ac.uk (DIDE network only). - ## Running in Kubernetes -The following is guide to run the shiny server in kubernetes. They are based on running a Kind cluster. The configuration may -nee to be adjusted for other k8s clusters. - ### Prerequisites A k8s kubernetes cluster using k3s is needed to be setup first. To setup a k8s cluster follow the guide [here](https://mrc-ide.myjetbrains.com/youtrack/articles/RESIDE-A-31/Setting-up-Kubernetes-k8s-Cluster). -Run `start-k8s-shiny ` to run the shint server in k8s. +Run `./start-k8s-shiny ` to run the shiny server in k8s. #### Teardown diff --git a/k8s/base/deployment.yaml b/k8s/base/deployment.yaml index 5683780..bdb1647 100644 --- a/k8s/base/deployment.yaml +++ b/k8s/base/deployment.yaml @@ -23,7 +23,7 @@ spec: mountPath: /shiny containers: - name: shiny - image: mrcide/shiny-server:dev # todo: change to actual one!!! mrcide- + image: mrcide/shiny-server:dev volumeMounts: - name: shiny-data mountPath: /shiny diff --git a/k8s/base/ingress.yaml b/k8s/base/ingress.yaml index 0509e52..e46ae51 100644 --- a/k8s/base/ingress.yaml +++ b/k8s/base/ingress.yaml @@ -9,13 +9,8 @@ metadata: nginx.ingress.kubernetes.io/session-cookie-max-age: "172800" spec: ingressClassName: nginx - tls: - - hosts: - - shiny-dev.dide.ic.ac.uk - secretName: tls-secret rules: - - host: shiny-dev.dide.ic.ac.uk - http: + - http: paths: - path: / pathType: Prefix diff --git a/k8s/base/kustomization.yaml b/k8s/base/kustomization.yaml index 46e19bd..0d9ce2f 100644 --- a/k8s/base/kustomization.yaml +++ b/k8s/base/kustomization.yaml @@ -6,7 +6,7 @@ metadata: namespace: twinkle # assume this namespace exists resources: - - persistance.yaml + - persistence.yaml - deployment.yaml - service.yaml - ingress.yaml diff --git a/k8s/base/persistance.yaml b/k8s/base/persistence.yaml similarity index 59% rename from k8s/base/persistance.yaml rename to k8s/base/persistence.yaml index 0edf096..e4b8afd 100644 --- a/k8s/base/persistance.yaml +++ b/k8s/base/persistence.yaml @@ -1,13 +1,12 @@ # todo: storage capacity as per requirements -# can use local-path if single node or dont need distributed & replicated apiVersion: v1 kind: PersistentVolumeClaim metadata: name: shiny-pvc spec: - storageClassName: longhorn + storageClassName: local-path # will use `longhorn` for distributed system (prod) accessModes: - - ReadWriteMany + - ReadWriteOnce resources: requests: storage: 15Gi diff --git a/k8s/overlays/production/ingress-patch.yaml b/k8s/overlays/production/ingress-patch.yaml new file mode 100644 index 0000000..df1c4b9 --- /dev/null +++ b/k8s/overlays/production/ingress-patch.yaml @@ -0,0 +1,21 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: ingress-shiny +spec: + ingressClassName: nginx + tls: + - hosts: + - shiny-dev.dide.ic.ac.uk + secretName: tls-secret + rules: + - host: shiny-dev.dide.ic.ac.uk + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: shiny-svc + port: + number: 3838 diff --git a/k8s/overlays/production/kustomization.yaml b/k8s/overlays/production/kustomization.yaml index 2c06008..2a56d3f 100644 --- a/k8s/overlays/production/kustomization.yaml +++ b/k8s/overlays/production/kustomization.yaml @@ -4,9 +4,14 @@ kind: Kustomization resources: - ../../base -commonLabels: - env: production - replicas: - - name: shiny-deploy - count: 9 + - count: 9 + name: shiny-deploy + +labels: + - includeSelectors: true + pairs: + env: production +patches: + - path: persistence-patch.yaml # if 1 node cluster remove this patch which enables longhorn + - path: ingress-patch.yaml diff --git a/k8s/overlays/production/persistence-patch.yaml b/k8s/overlays/production/persistence-patch.yaml new file mode 100644 index 0000000..a2c85d4 --- /dev/null +++ b/k8s/overlays/production/persistence-patch.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: shiny-pvc +spec: + storageClassName: longhorn + accessModes: + - ReadWriteMany diff --git a/k8s/overlays/testing/kustomization.yaml b/k8s/overlays/testing/kustomization.yaml index 6fe1290..bbdf639 100644 --- a/k8s/overlays/testing/kustomization.yaml +++ b/k8s/overlays/testing/kustomization.yaml @@ -4,9 +4,11 @@ kind: Kustomization resources: - ../../base -commonLabels: - env: testing +labels: + - includeSelectors: true + pairs: + env: testing replicas: - name: shiny-deploy - count: 3 \ No newline at end of file + count: 3 From 526208e1f8a7bb6cb79aebca26bdaa61971fd9e8 Mon Sep 17 00:00:00 2001 From: absternator Date: Thu, 25 Jan 2024 15:54:10 +0000 Subject: [PATCH 17/17] readme updae for local --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 443bc75..7c4de2d 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,9 @@ A k8s kubernetes cluster using k3s is needed to be setup first. To setup a k8s c Run `./start-k8s-shiny ` to run the shiny server in k8s. +Note: If on testing enviroment the app will launch on the IP addr of the result of the following command: +`kubectl -n ingress-nginx get svc ingress-nginx-controller -o=jsonpath='{.status.loadBalancer.ingress[0].ip}'` + #### Teardown Run the following: