Fix the local dev environemnt (#325)

dev_utils/README.md

@@ -1,20 +1,33 @@
 # Dev environment setup recomendations
 
-This guide uses the
-[minio client](https://docs.min.io/minio/baremetal/reference/minio-cli/minio-mc.html)
-(mc) for testing.
+## Deploy a stack locally
 
-## minio s3 server
+To start the S3Proxy development environment locally with docker compose, run the following command from the directory `dev_utils`
+
+```bash
+docker compose run local
+```
+
+After that, you can use [s3cmd](https://s3tools.org/s3cmd) to manually interact with the s3 server with proxy by 
+
+```bash
+s3cmd -c proxyS3 put README.md s3://dummy ## Upload a file using the proxy
+s3cmd -c proxyS3 ls s3://dummy ## List all files of the user using the proxy 
+```
 
-The S3Proxy development environment is run through docker compose, and can be
-started from this directory using:
+>Note that the content of the file `proxyS3` will be modified since the string `TOKEN` will be replaced by the actual token during the local deployment. Make sure not to commit this change.
 
+If the above commands fail, you may also test if the interaction with the s3 server works without the proxy by
 ```bash
-docker compose up
+s3cmd -c directS3 ls s3 ## For access without using the proxy
 ```
-(use the `-d` flag if you wish to run docker in the background).
 
-Then it's possible to trace all the requests that come to minio by first
+## Trace requests to the minio server
+This guide uses the
+[minio client](https://docs.min.io/minio/baremetal/reference/minio-cli/minio-mc.html)
+(mc) for testing.
+
+Once the stack is deployed locally with docker compose, it's possible to trace all the requests that come to minio by first
 putting the following in the hosts array of your `~/.mc/config.json` file:
 
 ```json
@@ -27,8 +40,7 @@ putting the following in the hosts array of your `~/.mc/config.json` file:
 }
 ```
 
-and then in one terminal it's possible to see all requests comming to and all
-responses from minio by running
+and then run the following command in a terminal
 
 ```bash
 mc admin trace -v proxydev
@@ -45,16 +57,6 @@ go build main.go
 ./main
 ```
 
-## Test with s3 configuration file
-To test the implementation locally use the `proxyS3` file located
-in the `dev_utils` folder or a file downloaded from the login portal.
-
-## For example use s3tools to interact with the proxy
-
-```bash
-s3cmd -c dev_utils/directS3 ls s3 ## For access without using the proxy
-s3cmd -c dev_utils/proxyS3 ls s3  ## For access with using the proxy
-```
 
 it's of course also possible to use the `mc` command from minio to access
 through the proxy or directly but then you have to configure that in the

dev_utils/certfixer/make_certs.sh

@@ -17,7 +17,7 @@ s3_certs="/s3_certs/CAs/public.crt /s3_certs/public.crt /s3_certs/private.key"
 mq_certs="/mq_certs/ca.crt /mq_certs/mq.crt /mq_certs/mq.key"
 pub_cert="/pubcert/public.crt"
 proxy_certs="/proxy_certs/ca.crt /proxy_certs/client.crt /proxy_certs/client.key /proxy_certs/proxy.crt /proxy_certs/proxy.key"
-keys="/keys/jwt.key /keys/jwt.pub"
+keys="/keys/jwt.key /keys/sda-sda-svc-auth.pub"
 targets="$s3_certs $mq_certs $pub_cert $proxy_certs $keys"
 
 echo ""

dev_utils/config.yaml

@@ -7,10 +7,10 @@ aws:
   secretKey: "987654321"
   bucket: "test"
   region: "us-east-1"
-  cacert: "/certs/ca.crt"
+  cacert: "/tmp/certs/ca.crt"
 
 broker:
-  host: "mq"
+  host: "localhost"
   port: "5671"
   user: "test"
   password: "test"
@@ -19,17 +19,26 @@ broker:
   routingKey: "files.inbox"
   ssl: "true"
   verifyPeer: "true"
-  cacert: "/certs/ca.crt"
-  clientCert: "/certs/client.crt"
-  clientKey: "/certs/client.key"
+  cacert: "/tmp/certs/ca.crt"
+  clientCert: "/tmp/certs/client.crt"
+  clientKey: "/tmp/certs/client.key"
 # If the FQDN and hostname of the broker differ
 # serverName can be set to the SAN name in the certificate
   #  serverName: ""
 
+db:
+  host: "localhost"
+  port: "2345"
+  user: "lega_in"
+  password: "lega_in"
+  database: "lega"
+  sslmode: "disable"
+
+
 server:
-  cert: "/certs/proxy.crt"
-  key: "/certs/proxy.key"
-  jwtpubkeypath: "./dev_utils/keys/"
-  jwtpubkeyurl: "https://login.elixir-czech.org/oidc/jwk"
+  cert: "/tmp/certs/proxy.crt"
+  key: "/tmp/certs/proxy.key"
+  jwtpubkeypath: "/tmp/certs/"
+  jwtpubkeyurl: ""
 
 

dev_utils/directS3

@@ -1,13 +1,12 @@
 [default]
 access_key=ElixirID
 secret_key=987654321
-#access_token=FooBar123
 check_ssl_certificate = False
 encoding = UTF-8
 encrypt = False
 guess_mime_type = True
-host_base = s3:9000
-host_bucket = s3:9000
+host_base = localhost:9000
+host_bucket = localhost:9000
 human_readable_sizes = True
 multipart_chunk_size_mb = 5
 use_https = True

dev_utils/docker-compose.yml

@@ -28,6 +28,7 @@ services:
       certfixer:
         condition: service_completed_successfully
     ports:
+      - "9000:9000"
       - "9001:9001"
     volumes:
       - s3_certs:/root/.minio/certs
@@ -130,12 +131,20 @@ services:
 
   database:
     container_name: db
-    image: neicnordic/sda-db:v2.0.0
+    image: neicnordic/sda-db:v2.0.7
+    depends_on:
+      certfixer:
+        condition: service_completed_successfully
     environment:
       - DB_LEGA_IN_PASSWORD=lega_in
       - DB_LEGA_OUT_PASSWORD=lega_out
       - PGVOLUME=/var/lib/postgresql
       - NOTLS=true
+    healthcheck:
+      test: [ "CMD", "pg_isready", "-U", "lega_in", "-h", "localhost" ]
+      interval: 30s
+      timeout: 20s
+      retries: 3
     volumes:
       - psqldata:/var/lib/postgresql
     ports:
@@ -190,6 +199,31 @@ services:
       - ..:/app
       - keys:/keys
 
+  local:
+    image: python:3.9.15-buster
+    container_name: local-tests
+    profiles:
+      - local
+    command:
+      - "/bin/sh"
+      - "/dev_utils/local.sh"
+    depends_on:
+      mq:
+        condition: service_healthy
+      s3:
+        condition: service_healthy
+      s3_proxy:
+        condition: service_started
+      certfixer:
+        condition: service_completed_successfully
+      createbucket:
+        condition: service_completed_successfully
+    volumes:
+      - proxy_certs:/certs
+      - .:/dev_utils
+      - keys:/keys
+      - /tmp:/local_tmp
+
 volumes:
   keys:
   pubcert:

dev_utils/local.sh

@@ -0,0 +1,19 @@
+#!/bin/sh
+
+if [ "$(id -u)" -eq 0 ]; then
+    apt-get -qq update && apt-get -qq install -y jq xxd
+fi
+
+cd dev_utils || exit 1
+
+local_uid=$(stat -c '%u' .)
+
+token="$(bash keys/sign_jwt.sh ES256 /keys/jwt.key)"
+sed -i "s/^access_token=.*/access_token=$token/" proxyS3
+
+mkdir -p /local_tmp/certs
+cp /certs/* /local_tmp/certs/
+cp /keys/*pub /local_tmp/certs/
+
+chown "$local_uid":"$local_uid" /local_tmp/certs/*
+chmod 600 /local_tmp/certs/*.key

dev_utils/proxyS3

@@ -7,10 +7,9 @@ check_ssl_hostname = False
 encoding = UTF-8
 encrypt = False
 guess_mime_type = True
-host_base = s3_proxy:8000
-host_bucket = s3_proxy:8000
+host_base = localhost:8000
+host_bucket = localhost:8000
 human_readable_sizes = true
 multipart_chunk_size_mb = 5
 use_https = True
 socket_timeout = 30
-ca_certs_file = /certs/ca.crt

tests/tests.sh

@@ -17,7 +17,11 @@ function check_output_status() {
 cd dev_utils || exit 1
 
 token="$(bash keys/sign_jwt.sh ES256 /keys/jwt.key)"
-sed -i "s/TOKEN/$token/" proxyS3
+sed -i "s/^access_token=.*/access_token=$token/" proxyS3
+
+# set correct host for S3 and proxy
+sed -i "s/localhost:9000/s3:9000/g" directS3
+sed -i "s/localhost:8000/s3_proxy:8000/g" proxyS3
 
 s3cmd -c directS3 put README.md s3://test/some_user/ >/dev/null 2>&1 || exit 1