diff --git a/.github/workflows/shiny-image-push.yaml b/.github/workflows/shiny-image-push.yaml new file mode 100644 index 0000000..215b048 --- /dev/null +++ b/.github/workflows/shiny-image-push.yaml @@ -0,0 +1,24 @@ +name: Build and push Shiny server image + +on: + push: + branches: + - "main" + +jobs: + docker: + runs-on: ubuntu-latest + steps: + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Checkout repository + uses: actions/checkout@v3 + - name: Build and push + run: shiny/build-and-push diff --git a/.gitignore b/.gitignore index 4486983..92ab869 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ apache/ssl +k8s/ssl \ No newline at end of file diff --git a/README.md b/README.md index 166dd8d..7c4de2d 100644 --- a/README.md +++ b/README.md @@ -2,60 +2,20 @@ This is a temporary repo designed to give an easy-to-understand deployment of the apache / haproxy / shiny stack; it does not include configurable applications or anything like that. See it online at https://shiny-dev.dide.ic.ac.uk (DIDE network only). -## The components +## Running in Kubernetes -Some of these are docker images, and they are not set to pull, so you will want to rebuild. All build very quickly. +### Prerequisites -### apache +A k8s kubernetes cluster using k3s is needed to be setup first. To setup a k8s cluster follow the guide [here](https://mrc-ide.myjetbrains.com/youtrack/articles/RESIDE-A-31/Setting-up-Kubernetes-k8s-Cluster). -Running an unmodified httpd container (previously was 2.4, we'll update once we know this works). The configuration ([`apache/httpd.conf`](httpd/httpd.conf)) and certificates (`apache/ssl`) will be read-only mounted into the container. You need to fetch the ssl key and certificate, run `./apache/configure_ssl` to do this (only needs to be done if they change or if the `ssl` directory is deleted) +Run `./start-k8s-shiny ` to run the shiny server in k8s. -### haproxy +Note: If on testing enviroment the app will launch on the IP addr of the result of the following command: +`kubectl -n ingress-nginx get svc ingress-nginx-controller -o=jsonpath='{.status.loadBalancer.ingress[0].ip}'` -Build the image with `./haproxy/build` which builds `mrcide/haproxy` with a configuration that can be seen in [`haproxy/haproxy.cfg`](haproxy/haproxy.cfg) and some utilities which enable some degree of dynamic scaling of shiny servers. +#### Teardown -### shiny +Run the following: -A lightly modified version of the official shiny container; the original version was more extensively modified - -### apps - -Some applications (copied over from the original deployment are in `apps`). These will want to be in a volume; run `./apps/create_volume` to copy them into the volume - -### Summary - -``` -./apache/configure_ssl -./haproxy/build -./shiny/build -./apps/create_volume -``` - -## Bringing the bits up - -``` -docker network create twinkle 2> /dev/null || /bin/true -docker volume create shiny_logs -docker run -d --name haproxy --network twinkle mrcide/haproxy:dev -docker run -d --name apache --network twinkle \ - -p 80:80 \ - -p 443:443 \ - -p 9000:9000 \ - -v "${PWD}/apache/httpd.conf:/usr/local/apache2/conf/httpd.conf:ro" \ - -v "${PWD}/apache/auth:/usr/local/apache2/conf/auth:ro" \ - -v "${PWD}/apache/ssl:/usr/local/apache2/conf/ssl:ro" \ - httpd:2.4 -docker run -d --name shiny-1 --network=twinkle \ - -v twinkle_apps:/shiny/apps \ - -v twinkle_logs:/shiny/logs \ - -p 3838:3838 \ - mrcide/shiny-server:dev -docker exec haproxy update_shiny_servers shiny 1 -``` - -Teardown - -``` -docker rm -f haproxy apache shiny-1 -docker network rm twinkle -``` +1. `kubectl delete -k k8s/overlays/`. Replace with testing or production. +2. `kubectl delete ns twinkle` to remove namespace. diff --git a/apache/httpd.conf b/apache/httpd.conf deleted file mode 100644 index 3beeb59..0000000 --- a/apache/httpd.conf +++ /dev/null @@ -1,128 +0,0 @@ -# Starting point is the "Apache httpd v2.4 minimal configuration" -# https://wiki.apache.org/httpd/Minimal_Config -# https://support.rstudio.com/hc/en-us/articles/213733868-Running-Shiny-Server-with-a-Proxy - -ServerAdmin r.fitzjohn@imperial.ac.uk -ServerName shiny-dev.dide.ic.ac.uk - -ServerRoot "/usr/local/apache2" - -ServerSignature Off -ServerTokens Prod - -User daemon -Group daemon - -# Minimum modules needed -LoadModule mpm_event_module modules/mod_mpm_event.so -LoadModule log_config_module modules/mod_log_config.so -LoadModule mime_module modules/mod_mime.so -LoadModule dir_module modules/mod_dir.so -LoadModule authz_core_module modules/mod_authz_core.so -LoadModule unixd_module modules/mod_unixd.so - -# For proxying shiny: -LoadModule rewrite_module modules/mod_rewrite.so -LoadModule proxy_module modules/mod_proxy.so -LoadModule proxy_wstunnel_module modules/mod_proxy_wstunnel.so -LoadModule proxy_http_module modules/mod_proxy_http.so - -# For doing auth -LoadModule authn_core_module modules/mod_authn_core.so -LoadModule authn_file_module modules/mod_authn_file.so -LoadModule auth_basic_module modules/mod_auth_basic.so -LoadModule authz_groupfile_module modules/mod_authz_groupfile.so -LoadModule authz_user_module modules/mod_authz_user.so - -# SSL -LoadModule setenvif_module modules/mod_setenvif.so -LoadModule ssl_module modules/mod_ssl.so -LoadModule socache_shmcb_module modules/mod_socache_shmcb.so - -# https://aaronsilber.me/2016/11/02/disable-3des-ssl-ciphers-apache-nginx/ -SSLCipherSuite HIGH:MEDIUM:!SSLv3:!kRSA:!3DES -SSLProxyCipherSuite HIGH:MEDIUM:!SSLv3:!kRSA:!3DES -SSLPassPhraseDialog builtin -SSLSessionCache "shmcb:/usr/local/apache2/logs/ssl_scache(512000)" -SSLSessionCacheTimeout 300 - -## https://httpd.apache.org/docs/trunk/ssl/ssl_howto.html -SSLProtocol all -SSLv3 -TLSv1 -TLSv1.1 -SSLHonorCipherOrder on -SSLCompression off -SSLSessionTickets off - - -ErrorLog /proc/self/fd/2 -LogLevel warn - -LogFormat "%h %l %u %t \"%r\" %>s %b" common -CustomLog /proc/self/fd/1 common - -TypesConfig conf/mime.types - -PidFile logs/httpd.pid - -User nobody - -# Port to Listen on -Listen *:80 -Listen *:443 -Listen *:9000 -Listen *:9001 - -# In a basic setup httpd can only serve files from its document root -DocumentRoot "/usr/local/apache2/htdocs" - -# Default file to serve -DirectoryIndex index.html - -# Never change this block - - AllowOverride None - Require all denied - - -# Allow documents to be served from the DocumentRoot - - Require all granted - - - - RewriteEngine On - RewriteRule ^(.*)$ https://%{HTTP_HOST}$1 [R=301,L] - - - - SSLEngine on - # or server.crt - SSLCertificateFile "/usr/local/apache2/conf/ssl/certificate.pem" - # or server.key - SSLCertificateKeyFile "/usr/local/apache2/conf/ssl/key.pem" - CustomLog "/usr/local/apache2/logs/ssl_request_log" \ - "%t %h %{SSL_PROTOCOL}x %{SSL_CIPHER}x \"%r\" %b" - - RewriteEngine on - RewriteCond %{HTTP:Upgrade} =websocket - RewriteRule /(.*) ws://haproxy:8080/$1 [P,L] - RewriteCond %{HTTP:Upgrade} !=websocket - RewriteRule /(.*) http://haproxy:8080/$1 [P,L] - ProxyPass / http://haproxy:8080/ - ProxyPassReverse / http://haproxy:8080/ - - IncludeOptional conf/auth/*.conf - - - - SSLEngine on - # or server.crt - SSLCertificateFile "/usr/local/apache2/conf/ssl/certificate.pem" - # or server.key - SSLCertificateKeyFile "/usr/local/apache2/conf/ssl/key.pem" - CustomLog "/usr/local/apache2/logs/ssl_request_log" \ - "%t %h %{SSL_PROTOCOL}x %{SSL_CIPHER}x \"%r\" %b" - - RewriteEngine on - ProxyPass / http://haproxy:9001/ - ProxyPassReverse / http://haproxy:9001/ - diff --git a/haproxy/Dockerfile b/haproxy/Dockerfile deleted file mode 100644 index 98676ba..0000000 --- a/haproxy/Dockerfile +++ /dev/null @@ -1,5 +0,0 @@ -FROM haproxy:1.8 -RUN apt-get update && apt-get install -y \ - socat -COPY bin /usr/local/bin -COPY haproxy.cfg /usr/local/etc/haproxy/haproxy.cfg diff --git a/haproxy/bin/add_server b/haproxy/bin/add_server deleted file mode 100755 index cb4cacc..0000000 --- a/haproxy/bin/add_server +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash -set -e - -SERVER=$1 -TARGET=$2 -echo "Enabling $SERVER as $TARGET" -TARGET_IP=$(getent hosts $TARGET | cut -d ' ' -f1) -echo "set server servers/$SERVER addr $TARGET_IP" | \ - socat stdio /var/run/hapee-lb.sock > /dev/null -echo "set server servers/$SERVER state ready" | \ - socat stdio /var/run/hapee-lb.sock > /dev/null diff --git a/haproxy/bin/drop_server b/haproxy/bin/drop_server deleted file mode 100755 index 4d4c05b..0000000 --- a/haproxy/bin/drop_server +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash -set -e -SERVER=$1 -echo "Disabling $SERVER" -echo "disable server servers/$SERVER" | \ - socat stdio /var/run/hapee-lb.sock > /dev/null diff --git a/haproxy/bin/update_shiny_servers b/haproxy/bin/update_shiny_servers deleted file mode 100755 index 3e9626d..0000000 --- a/haproxy/bin/update_shiny_servers +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash -set -eu -SERVER_NAME_BASE=${1:-shiny-server-config} -SERVER_N=${2:-100} - -echo "Updating servers 1..${SERVER_N}" - -for i in $(seq $SERVER_N); do - SERVER_NAME="${SERVER_NAME_BASE}-${i}" - SERVER_CODE="shiny${i}" - SERVER_IP=$(getent hosts $SERVER_NAME | cut -d ' ' -f1) - if [[ -z $SERVER_IP ]]; then - drop_server "$SERVER_CODE" - else - add_server "$SERVER_CODE" "$SERVER_IP" - fi -done diff --git a/haproxy/build b/haproxy/build deleted file mode 100755 index 1efe4ee..0000000 --- a/haproxy/build +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash -set -e -HERE=$(realpath $(dirname $0)) -IMAGE_NAME="mrcide/haproxy" -TAG_VERSION=dev -docker build --rm \ - --tag "${IMAGE_NAME}:${TAG_VERSION}" \ - $HERE diff --git a/haproxy/haproxy.cfg b/haproxy/haproxy.cfg deleted file mode 100644 index 3cb09b4..0000000 --- a/haproxy/haproxy.cfg +++ /dev/null @@ -1,43 +0,0 @@ -global - daemon - maxconn 256 - log 127.0.0.1 local0 notice - stats socket /var/run/hapee-lb.sock mode 666 level admin - stats socket ipv4@127.0.0.1:9999 level admin - stats timeout 2m - -defaults - log global - mode http - timeout connect 5000ms - timeout client 50000ms - timeout server 50000ms - # https://www.haproxy.com/blog/websockets-load-balancing-with-haproxy/ - timeout tunnel 3600s - timeout http-keep-alive 1s - timeout http-request 15s - timeout queue 30s - timeout tarpit 60s - -frontend http-in - bind *:8080 - option httplog - option forwardfor - default_backend servers - -backend servers - # I think that leastconn is probably the best bet here. - balance leastconn - # https://www.haproxy.com/blog/whats-new-haproxy-1-8/ - dynamic-cookie-key MYKEY - cookie SRVID insert dynamic - # This sets us up for up to 100 workers which do not need to exist - # at the point where we start up the proxy - server-template shiny 1-100 127.0.0.1:3838 check disabled - -listen stats - bind *:9001 - mode http - stats enable - stats realm Haproxy\ Statistics - stats uri / diff --git a/k8s/base/deployment.yaml b/k8s/base/deployment.yaml new file mode 100644 index 0000000..bdb1647 --- /dev/null +++ b/k8s/base/deployment.yaml @@ -0,0 +1,38 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: shiny-deploy + labels: + app: shiny +spec: + replicas: 2 + selector: + matchLabels: + app: shiny + template: + metadata: + labels: + app: shiny + spec: + initContainers: + - name: init-shiny + image: busybox:1.28 + command: ["sh", "-c", "mkdir -p /shiny/logs /shiny/apps"] + volumeMounts: + - name: shiny-data + mountPath: /shiny + containers: + - name: shiny + image: mrcide/shiny-server:dev + volumeMounts: + - name: shiny-data + mountPath: /shiny + # todo: create appropriate resource requests + # resources: + # requests: + # memory: "128Mi" + # cpu: "250m" + volumes: + - name: shiny-data + persistentVolumeClaim: + claimName: shiny-pvc diff --git a/k8s/base/ingress.yaml b/k8s/base/ingress.yaml new file mode 100644 index 0000000..e46ae51 --- /dev/null +++ b/k8s/base/ingress.yaml @@ -0,0 +1,21 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: ingress-shiny + annotations: + nginx.ingress.kubernetes.io/affinity: "cookie" + nginx.ingress.kubernetes.io/session-cookie-name: "shinycookie" + nginx.ingress.kubernetes.io/session-cookie-expires: "172800" + nginx.ingress.kubernetes.io/session-cookie-max-age: "172800" +spec: + ingressClassName: nginx + rules: + - http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: shiny-svc + port: + number: 3838 diff --git a/k8s/base/kustomization.yaml b/k8s/base/kustomization.yaml new file mode 100644 index 0000000..0d9ce2f --- /dev/null +++ b/k8s/base/kustomization.yaml @@ -0,0 +1,12 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +metadata: + name: shiny + +namespace: twinkle # assume this namespace exists + +resources: + - persistence.yaml + - deployment.yaml + - service.yaml + - ingress.yaml diff --git a/k8s/base/persistence.yaml b/k8s/base/persistence.yaml new file mode 100644 index 0000000..e4b8afd --- /dev/null +++ b/k8s/base/persistence.yaml @@ -0,0 +1,12 @@ +# todo: storage capacity as per requirements +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: shiny-pvc +spec: + storageClassName: local-path # will use `longhorn` for distributed system (prod) + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 15Gi diff --git a/k8s/base/service.yaml b/k8s/base/service.yaml new file mode 100644 index 0000000..8db8181 --- /dev/null +++ b/k8s/base/service.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Service +metadata: + name: shiny-svc +spec: + type: ClusterIP + selector: + app: shiny + ports: + - protocol: TCP + port: 3838 + targetPort: 3838 \ No newline at end of file diff --git a/apache/configure_ssl b/k8s/configure_ssl similarity index 61% rename from apache/configure_ssl rename to k8s/configure_ssl index 18985e0..8d2312e 100755 --- a/apache/configure_ssl +++ b/k8s/configure_ssl @@ -1,9 +1,16 @@ #!/usr/bin/env bash +# TODO: update to use vault k8s auth +set -eu + export VAULT_ADDR=https://vault.dide.ic.ac.uk:8200 export VAULT_TOKEN=$(vault login -method=github -token-only) -DEST=apache/ssl -mkdir -p ${DEST} + +DEST=${PWD}/k8s/ssl +mkdir -p $DEST + vault read -field=value /secret/shiny.dide/dev/ssl/cert > \ ${DEST}/certificate.pem vault read -field=value /secret/shiny.dide/dev/ssl/key > \ ${DEST}/key.pem + +kubectl -n twinkle create secret tls tls-secret --cert ${DEST}/certificate.pem --key ${DEST}/key.pem \ No newline at end of file diff --git a/k8s/krsync b/k8s/krsync new file mode 100755 index 0000000..5f869bf --- /dev/null +++ b/k8s/krsync @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# https://serverfault.com/questions/741670/rsync-files-to-a-kubernetes-pod +# example usage: +# ./krsync -av --progress --stats podName@namespace: + +if [ -z "$KRSYNC_STARTED" ]; then + export KRSYNC_STARTED=true + exec rsync -a --delete --blocking-io --rsh "$0" $@ +fi + +# Running as --rsh +namespace='' +pod=$1 +shift + +# If user uses pod@namespace, rsync passes args as: {us} -l pod namespace ... +if [ "X$pod" = "X-l" ]; then + pod=$1 + shift + namespace="-n $1" + shift +fi + + +exec kubectl $namespace exec -i $pod -- "$@" \ No newline at end of file diff --git a/k8s/overlays/production/ingress-patch.yaml b/k8s/overlays/production/ingress-patch.yaml new file mode 100644 index 0000000..df1c4b9 --- /dev/null +++ b/k8s/overlays/production/ingress-patch.yaml @@ -0,0 +1,21 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: ingress-shiny +spec: + ingressClassName: nginx + tls: + - hosts: + - shiny-dev.dide.ic.ac.uk + secretName: tls-secret + rules: + - host: shiny-dev.dide.ic.ac.uk + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: shiny-svc + port: + number: 3838 diff --git a/k8s/overlays/production/kustomization.yaml b/k8s/overlays/production/kustomization.yaml new file mode 100644 index 0000000..2a56d3f --- /dev/null +++ b/k8s/overlays/production/kustomization.yaml @@ -0,0 +1,17 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../../base + +replicas: + - count: 9 + name: shiny-deploy + +labels: + - includeSelectors: true + pairs: + env: production +patches: + - path: persistence-patch.yaml # if 1 node cluster remove this patch which enables longhorn + - path: ingress-patch.yaml diff --git a/k8s/overlays/production/persistence-patch.yaml b/k8s/overlays/production/persistence-patch.yaml new file mode 100644 index 0000000..a2c85d4 --- /dev/null +++ b/k8s/overlays/production/persistence-patch.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: shiny-pvc +spec: + storageClassName: longhorn + accessModes: + - ReadWriteMany diff --git a/k8s/overlays/testing/kustomization.yaml b/k8s/overlays/testing/kustomization.yaml new file mode 100644 index 0000000..bbdf639 --- /dev/null +++ b/k8s/overlays/testing/kustomization.yaml @@ -0,0 +1,14 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../../base + +labels: + - includeSelectors: true + pairs: + env: testing + +replicas: + - name: shiny-deploy + count: 3 diff --git a/k8s/shiny-sync b/k8s/shiny-sync new file mode 100755 index 0000000..ca9f3c0 --- /dev/null +++ b/k8s/shiny-sync @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +set -eu +# uses krysnc to sync files to a kubernetes pod and thus into k8s cluster (via persistant volume) +# RUN this from repository root to ensure correct paths to app files +NAMESPACE=twinkle +SYNC_DIR=/shiny/apps + + +FIRST_SHINY_POD=$(kubectl get pods -l app=shiny -n $NAMESPACE -o jsonpath='{.items[0].metadata.name}') +echo "pod name: $FIRST_SHINY_POD" +$PWD/k8s/krsync -av --progress --stats $PWD/apps/* $FIRST_SHINY_POD@$NAMESPACE:$SYNC_DIR \ No newline at end of file diff --git a/shiny/Dockerfile b/shiny/Dockerfile index e922a30..c04f15a 100644 --- a/shiny/Dockerfile +++ b/shiny/Dockerfile @@ -3,5 +3,6 @@ FROM rocker/shiny:latest # Override the rocker defaults with our defaults COPY shiny-server.conf /etc/shiny-server/shiny-server.conf -VOLUME /shiny/apps -VOLUME /shiny/logs +# Install rsync +RUN apt-get update && apt-get install -y rsync + diff --git a/shiny/build b/shiny/build index 9397d52..8b34ddf 100755 --- a/shiny/build +++ b/shiny/build @@ -2,8 +2,5 @@ set -e HERE=$(realpath $(dirname $0)) IMAGE_NAME="mrcide/shiny-server" -TAG_VERSION=dev -docker pull rocker/shiny:latest -docker build --rm \ - --tag "${IMAGE_NAME}:${TAG_VERSION}" \ - $HERE +TAG=dev +docker build -t "${IMAGE_NAME}:${TAG}" $HERE diff --git a/shiny/build-and-push b/shiny/build-and-push new file mode 100755 index 0000000..ef36356 --- /dev/null +++ b/shiny/build-and-push @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +set -e +HERE=$(realpath $(dirname $0)) + +. $HERE/build + +docker push $IMAGE_NAME:$TAG \ No newline at end of file diff --git a/start-k8s-shiny b/start-k8s-shiny new file mode 100755 index 0000000..5a3a1db --- /dev/null +++ b/start-k8s-shiny @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +set -eu +# example use case: ./start-k8s-shiny production + +# Set the environment variable to the first command-line argument or default to 'testing' +ENV=${1:-testing} +if [[ "$ENV" != "testing" && "$ENV" != "production" ]]; then + echo "Error: env must be either 'testing' or 'production'" + exit 1 +fi +# Make the script itself executable +chmod +x "$0" + +# Make the k8s directory and its contents executable +chmod +x k8s/ + +# Create a Kubernetes namespace named 'twinkle' +kubectl create ns twinkle + +# Run the script to configure SSL (assuming it's in the k8s directory) +k8s/configure_ssl || { echo "Error: Failed to configure SSL"; exit 1; } + +# Apply Kubernetes manifests from overlays based on the specified or default environment +kubectl apply -k "k8s/overlays/$ENV" + +# Wait for the 'shiny-deploy' deployment to be in the 'available' condition within the 'twinkle' namespace +echo 'Waiting for pods to be ready...' +kubectl wait -n twinkle --for=condition=available --timeout=300s deployment/shiny-deploy + +# Run the script for 'shiny-sync' (assuming it's in the k8s directory) +k8s/shiny-sync