diff --git a/dev_utils/README.md b/dev_utils/README.md index e3c5af2..12ba859 100644 --- a/dev_utils/README.md +++ b/dev_utils/README.md @@ -1,20 +1,33 @@ # Dev environment setup recomendations -This guide uses the -[minio client](https://docs.min.io/minio/baremetal/reference/minio-cli/minio-mc.html) -(mc) for testing. +## Deploy a stack locally -## minio s3 server +To start the S3Proxy development environment locally with docker compose, run the following command from the directory `dev_utils` + +```bash +docker compose run local +``` + +After that, you can use [s3cmd](https://s3tools.org/s3cmd) to manually interact with the s3 server with proxy by + +```bash +s3cmd -c proxyS3 put README.md s3://dummy ## Upload a file using the proxy +s3cmd -c proxyS3 ls s3://dummy ## List all files of the user using the proxy +``` -The S3Proxy development environment is run through docker compose, and can be -started from this directory using: +>Note that the content of the file `proxyS3` will be modified since the string `TOKEN` will be replaced by the actual token during the local deployment. Make sure not to commit this change. +If the above commands fail, you may also test if the interaction with the s3 server works without the proxy by ```bash -docker compose up +s3cmd -c directS3 ls s3 ## For access without using the proxy ``` -(use the `-d` flag if you wish to run docker in the background). -Then it's possible to trace all the requests that come to minio by first +## Trace requests to the minio server +This guide uses the +[minio client](https://docs.min.io/minio/baremetal/reference/minio-cli/minio-mc.html) +(mc) for testing. + +Once the stack is deployed locally with docker compose, it's possible to trace all the requests that come to minio by first putting the following in the hosts array of your `~/.mc/config.json` file: ```json @@ -27,8 +40,7 @@ putting the following in the hosts array of your `~/.mc/config.json` file: } ``` -and then in one terminal it's possible to see all requests comming to and all -responses from minio by running +and then run the following command in a terminal ```bash mc admin trace -v proxydev @@ -45,16 +57,6 @@ go build main.go ./main ``` -## Test with s3 configuration file -To test the implementation locally use the `proxyS3` file located -in the `dev_utils` folder or a file downloaded from the login portal. - -## For example use s3tools to interact with the proxy - -```bash -s3cmd -c dev_utils/directS3 ls s3 ## For access without using the proxy -s3cmd -c dev_utils/proxyS3 ls s3 ## For access with using the proxy -``` it's of course also possible to use the `mc` command from minio to access through the proxy or directly but then you have to configure that in the diff --git a/dev_utils/certfixer/make_certs.sh b/dev_utils/certfixer/make_certs.sh index d9c0d91..3eebd46 100644 --- a/dev_utils/certfixer/make_certs.sh +++ b/dev_utils/certfixer/make_certs.sh @@ -17,7 +17,7 @@ s3_certs="/s3_certs/CAs/public.crt /s3_certs/public.crt /s3_certs/private.key" mq_certs="/mq_certs/ca.crt /mq_certs/mq.crt /mq_certs/mq.key" pub_cert="/pubcert/public.crt" proxy_certs="/proxy_certs/ca.crt /proxy_certs/client.crt /proxy_certs/client.key /proxy_certs/proxy.crt /proxy_certs/proxy.key" -keys="/keys/jwt.key /keys/jwt.pub" +keys="/keys/jwt.key /keys/sda-sda-svc-auth.pub" targets="$s3_certs $mq_certs $pub_cert $proxy_certs $keys" echo "" diff --git a/dev_utils/config.yaml b/dev_utils/config.yaml index 8d6ce1a..f0b74f0 100644 --- a/dev_utils/config.yaml +++ b/dev_utils/config.yaml @@ -7,10 +7,10 @@ aws: secretKey: "987654321" bucket: "test" region: "us-east-1" - cacert: "/certs/ca.crt" + cacert: "/tmp/certs/ca.crt" broker: - host: "mq" + host: "localhost" port: "5671" user: "test" password: "test" @@ -19,17 +19,26 @@ broker: routingKey: "files.inbox" ssl: "true" verifyPeer: "true" - cacert: "/certs/ca.crt" - clientCert: "/certs/client.crt" - clientKey: "/certs/client.key" + cacert: "/tmp/certs/ca.crt" + clientCert: "/tmp/certs/client.crt" + clientKey: "/tmp/certs/client.key" # If the FQDN and hostname of the broker differ # serverName can be set to the SAN name in the certificate # serverName: "" +db: + host: "localhost" + port: "2345" + user: "lega_in" + password: "lega_in" + database: "lega" + sslmode: "disable" + + server: - cert: "/certs/proxy.crt" - key: "/certs/proxy.key" - jwtpubkeypath: "./dev_utils/keys/" - jwtpubkeyurl: "https://login.elixir-czech.org/oidc/jwk" + cert: "/tmp/certs/proxy.crt" + key: "/tmp/certs/proxy.key" + jwtpubkeypath: "/tmp/certs/" + jwtpubkeyurl: "" diff --git a/dev_utils/directS3 b/dev_utils/directS3 index 453bbce..e88e019 100644 --- a/dev_utils/directS3 +++ b/dev_utils/directS3 @@ -1,13 +1,12 @@ [default] access_key=ElixirID secret_key=987654321 -#access_token=FooBar123 check_ssl_certificate = False encoding = UTF-8 encrypt = False guess_mime_type = True -host_base = s3:9000 -host_bucket = s3:9000 +host_base = localhost:9000 +host_bucket = localhost:9000 human_readable_sizes = True multipart_chunk_size_mb = 5 use_https = True diff --git a/dev_utils/docker-compose.yml b/dev_utils/docker-compose.yml index 1d1430a..38ddaf5 100644 --- a/dev_utils/docker-compose.yml +++ b/dev_utils/docker-compose.yml @@ -28,6 +28,7 @@ services: certfixer: condition: service_completed_successfully ports: + - "9000:9000" - "9001:9001" volumes: - s3_certs:/root/.minio/certs @@ -130,12 +131,20 @@ services: database: container_name: db - image: neicnordic/sda-db:v2.0.0 + image: neicnordic/sda-db:v2.0.7 + depends_on: + certfixer: + condition: service_completed_successfully environment: - DB_LEGA_IN_PASSWORD=lega_in - DB_LEGA_OUT_PASSWORD=lega_out - PGVOLUME=/var/lib/postgresql - NOTLS=true + healthcheck: + test: [ "CMD", "pg_isready", "-U", "lega_in", "-h", "localhost" ] + interval: 30s + timeout: 20s + retries: 3 volumes: - psqldata:/var/lib/postgresql ports: @@ -190,6 +199,31 @@ services: - ..:/app - keys:/keys + local: + image: python:3.9.15-buster + container_name: local-tests + profiles: + - local + command: + - "/bin/sh" + - "/dev_utils/local.sh" + depends_on: + mq: + condition: service_healthy + s3: + condition: service_healthy + s3_proxy: + condition: service_started + certfixer: + condition: service_completed_successfully + createbucket: + condition: service_completed_successfully + volumes: + - proxy_certs:/certs + - .:/dev_utils + - keys:/keys + - /tmp:/local_tmp + volumes: keys: pubcert: diff --git a/dev_utils/local.sh b/dev_utils/local.sh new file mode 100644 index 0000000..13f4711 --- /dev/null +++ b/dev_utils/local.sh @@ -0,0 +1,19 @@ +#!/bin/sh + +if [ "$(id -u)" -eq 0 ]; then + apt-get -qq update && apt-get -qq install -y jq xxd +fi + +cd dev_utils || exit 1 + +local_uid=$(stat -c '%u' .) + +token="$(bash keys/sign_jwt.sh ES256 /keys/jwt.key)" +sed -i "s/^access_token=.*/access_token=$token/" proxyS3 + +mkdir -p /local_tmp/certs +cp /certs/* /local_tmp/certs/ +cp /keys/*pub /local_tmp/certs/ + +chown "$local_uid":"$local_uid" /local_tmp/certs/* +chmod 600 /local_tmp/certs/*.key diff --git a/dev_utils/proxyS3 b/dev_utils/proxyS3 index d0d601b..c5111e1 100644 --- a/dev_utils/proxyS3 +++ b/dev_utils/proxyS3 @@ -7,10 +7,9 @@ check_ssl_hostname = False encoding = UTF-8 encrypt = False guess_mime_type = True -host_base = s3_proxy:8000 -host_bucket = s3_proxy:8000 +host_base = localhost:8000 +host_bucket = localhost:8000 human_readable_sizes = true multipart_chunk_size_mb = 5 use_https = True socket_timeout = 30 -ca_certs_file = /certs/ca.crt diff --git a/tests/tests.sh b/tests/tests.sh index c66c29d..e5a82f5 100644 --- a/tests/tests.sh +++ b/tests/tests.sh @@ -17,7 +17,11 @@ function check_output_status() { cd dev_utils || exit 1 token="$(bash keys/sign_jwt.sh ES256 /keys/jwt.key)" -sed -i "s/TOKEN/$token/" proxyS3 +sed -i "s/^access_token=.*/access_token=$token/" proxyS3 + +# set correct host for S3 and proxy +sed -i "s/localhost:9000/s3:9000/g" directS3 +sed -i "s/localhost:8000/s3_proxy:8000/g" proxyS3 s3cmd -c directS3 put README.md s3://test/some_user/ >/dev/null 2>&1 || exit 1