From 23420a0816746e63729c2e13172c6399a4446ed0 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Sat, 21 Jan 2023 14:30:29 +0100 Subject: [PATCH 01/21] expose port 9000 for the service s3 --- dev_utils/docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/dev_utils/docker-compose.yml b/dev_utils/docker-compose.yml index 1d1430a..a170b42 100644 --- a/dev_utils/docker-compose.yml +++ b/dev_utils/docker-compose.yml @@ -28,6 +28,7 @@ services: certfixer: condition: service_completed_successfully ports: + - "9000:9000" - "9001:9001" volumes: - s3_certs:/root/.minio/certs From 8dcaa6b9a1132010008c8e845c47e7fbf18615df Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Sat, 21 Jan 2023 14:31:47 +0100 Subject: [PATCH 02/21] Change host setting in config files for direct access --- dev_utils/directS3 | 5 ++--- dev_utils/proxyS3 | 5 ++--- tests/tests.sh | 6 +++++- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/dev_utils/directS3 b/dev_utils/directS3 index 453bbce..e88e019 100644 --- a/dev_utils/directS3 +++ b/dev_utils/directS3 @@ -1,13 +1,12 @@ [default] access_key=ElixirID secret_key=987654321 -#access_token=FooBar123 check_ssl_certificate = False encoding = UTF-8 encrypt = False guess_mime_type = True -host_base = s3:9000 -host_bucket = s3:9000 +host_base = localhost:9000 +host_bucket = localhost:9000 human_readable_sizes = True multipart_chunk_size_mb = 5 use_https = True diff --git a/dev_utils/proxyS3 b/dev_utils/proxyS3 index d0d601b..c5111e1 100644 --- a/dev_utils/proxyS3 +++ b/dev_utils/proxyS3 @@ -7,10 +7,9 @@ check_ssl_hostname = False encoding = UTF-8 encrypt = False guess_mime_type = True -host_base = s3_proxy:8000 -host_bucket = s3_proxy:8000 +host_base = localhost:8000 +host_bucket = localhost:8000 human_readable_sizes = true multipart_chunk_size_mb = 5 use_https = True socket_timeout = 30 -ca_certs_file = /certs/ca.crt diff --git a/tests/tests.sh b/tests/tests.sh index c66c29d..7c65969 100644 --- a/tests/tests.sh +++ b/tests/tests.sh @@ -17,7 +17,11 @@ function check_output_status() { cd dev_utils || exit 1 token="$(bash keys/sign_jwt.sh ES256 /keys/jwt.key)" -sed -i "s/TOKEN/$token/" proxyS3 +sed -i "s/^access_token=.*/access_token=$token/" proxyS3 + +# set correct hostanles for S3 and proxy +sed -i "s/localhost:9000/s3:9000/g" directS3 +sed -i "s/localhost:8000/s3_proxy:8000/g" proxyS3 s3cmd -c directS3 put README.md s3://test/some_user/ >/dev/null 2>&1 || exit 1 From d68f04445eb64efc4179ab3ca1f8d77d7f763121 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Sat, 21 Jan 2023 15:18:15 +0100 Subject: [PATCH 03/21] [mkCerts] use correct name for the pub key when checking for existence --- dev_utils/certfixer/make_certs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev_utils/certfixer/make_certs.sh b/dev_utils/certfixer/make_certs.sh index d9c0d91..3eebd46 100644 --- a/dev_utils/certfixer/make_certs.sh +++ b/dev_utils/certfixer/make_certs.sh @@ -17,7 +17,7 @@ s3_certs="/s3_certs/CAs/public.crt /s3_certs/public.crt /s3_certs/private.key" mq_certs="/mq_certs/ca.crt /mq_certs/mq.crt /mq_certs/mq.key" pub_cert="/pubcert/public.crt" proxy_certs="/proxy_certs/ca.crt /proxy_certs/client.crt /proxy_certs/client.key /proxy_certs/proxy.crt /proxy_certs/proxy.key" -keys="/keys/jwt.key /keys/jwt.pub" +keys="/keys/jwt.key /keys/sda-sda-svc-auth.pub" targets="$s3_certs $mq_certs $pub_cert $proxy_certs $keys" echo "" From e77a1183bd47e6692eb6c6ee8374c163b0779944 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Sat, 21 Jan 2023 15:19:15 +0100 Subject: [PATCH 04/21] [local] add profile for generating local config --- dev_utils/docker-compose.yml | 25 +++++++++++++++++++++++++ dev_utils/local.sh | 19 +++++++++++++++++++ 2 files changed, 44 insertions(+) create mode 100644 dev_utils/local.sh diff --git a/dev_utils/docker-compose.yml b/dev_utils/docker-compose.yml index a170b42..a864b99 100644 --- a/dev_utils/docker-compose.yml +++ b/dev_utils/docker-compose.yml @@ -191,6 +191,31 @@ services: - ..:/app - keys:/keys + local: + image: python:3.9.15-buster + container_name: local-tests + profiles: + - local + command: + - "/bin/sh" + - "/dev_utils/local.sh" + depends_on: + mq: + condition: service_healthy + s3: + condition: service_healthy + s3_proxy: + condition: service_started + certfixer: + condition: service_completed_successfully + createbucket: + condition: service_completed_successfully + volumes: + - proxy_certs:/certs + - .:/dev_utils + - keys:/keys + - /tmp:/local_tmp + volumes: keys: pubcert: diff --git a/dev_utils/local.sh b/dev_utils/local.sh new file mode 100644 index 0000000..13f4711 --- /dev/null +++ b/dev_utils/local.sh @@ -0,0 +1,19 @@ +#!/bin/sh + +if [ "$(id -u)" -eq 0 ]; then + apt-get -qq update && apt-get -qq install -y jq xxd +fi + +cd dev_utils || exit 1 + +local_uid=$(stat -c '%u' .) + +token="$(bash keys/sign_jwt.sh ES256 /keys/jwt.key)" +sed -i "s/^access_token=.*/access_token=$token/" proxyS3 + +mkdir -p /local_tmp/certs +cp /certs/* /local_tmp/certs/ +cp /keys/*pub /local_tmp/certs/ + +chown "$local_uid":"$local_uid" /local_tmp/certs/* +chmod 600 /local_tmp/certs/*.key From f33c5f3289abcdf6f5143dfba37f8086f3606c73 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Sat, 21 Jan 2023 15:28:40 +0100 Subject: [PATCH 05/21] [local] update proxy config file As long as the proxy container is not running the proxy can be started with this command: `SERVER_CONFFILE=dev_utils/config.yaml go run .` --- dev_utils/config.yaml | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/dev_utils/config.yaml b/dev_utils/config.yaml index 8d6ce1a..f0b74f0 100644 --- a/dev_utils/config.yaml +++ b/dev_utils/config.yaml @@ -7,10 +7,10 @@ aws: secretKey: "987654321" bucket: "test" region: "us-east-1" - cacert: "/certs/ca.crt" + cacert: "/tmp/certs/ca.crt" broker: - host: "mq" + host: "localhost" port: "5671" user: "test" password: "test" @@ -19,17 +19,26 @@ broker: routingKey: "files.inbox" ssl: "true" verifyPeer: "true" - cacert: "/certs/ca.crt" - clientCert: "/certs/client.crt" - clientKey: "/certs/client.key" + cacert: "/tmp/certs/ca.crt" + clientCert: "/tmp/certs/client.crt" + clientKey: "/tmp/certs/client.key" # If the FQDN and hostname of the broker differ # serverName can be set to the SAN name in the certificate # serverName: "" +db: + host: "localhost" + port: "2345" + user: "lega_in" + password: "lega_in" + database: "lega" + sslmode: "disable" + + server: - cert: "/certs/proxy.crt" - key: "/certs/proxy.key" - jwtpubkeypath: "./dev_utils/keys/" - jwtpubkeyurl: "https://login.elixir-czech.org/oidc/jwk" + cert: "/tmp/certs/proxy.crt" + key: "/tmp/certs/proxy.key" + jwtpubkeypath: "/tmp/certs/" + jwtpubkeyurl: "" From 3a2774ca859ace3505e6bc17ab009b619a033334 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Mon, 23 Jan 2023 09:48:37 +0100 Subject: [PATCH 06/21] Update DB version to 2.0.7 --- dev_utils/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev_utils/docker-compose.yml b/dev_utils/docker-compose.yml index a864b99..707d9c9 100644 --- a/dev_utils/docker-compose.yml +++ b/dev_utils/docker-compose.yml @@ -131,7 +131,7 @@ services: database: container_name: db - image: neicnordic/sda-db:v2.0.0 + image: neicnordic/sda-db:v2.0.7 environment: - DB_LEGA_IN_PASSWORD=lega_in - DB_LEGA_OUT_PASSWORD=lega_out From 1af0065ffe52b63a7c6f7ada4abc23e9b584367c Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Mon, 23 Jan 2023 09:59:29 +0100 Subject: [PATCH 07/21] [dev] add dependency to DB statup --- dev_utils/docker-compose.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dev_utils/docker-compose.yml b/dev_utils/docker-compose.yml index 707d9c9..3128715 100644 --- a/dev_utils/docker-compose.yml +++ b/dev_utils/docker-compose.yml @@ -132,6 +132,9 @@ services: database: container_name: db image: neicnordic/sda-db:v2.0.7 + depends_on: + certfixer: + condition: service_completed_successfully environment: - DB_LEGA_IN_PASSWORD=lega_in - DB_LEGA_OUT_PASSWORD=lega_out From bb9989e010eb01b00081e6c2ec235852d3b9ca73 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Mon, 23 Jan 2023 10:19:39 +0100 Subject: [PATCH 08/21] [dev] ad healthcheck to DB container --- dev_utils/docker-compose.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/dev_utils/docker-compose.yml b/dev_utils/docker-compose.yml index 3128715..38ddaf5 100644 --- a/dev_utils/docker-compose.yml +++ b/dev_utils/docker-compose.yml @@ -140,6 +140,11 @@ services: - DB_LEGA_OUT_PASSWORD=lega_out - PGVOLUME=/var/lib/postgresql - NOTLS=true + healthcheck: + test: [ "CMD", "pg_isready", "-U", "lega_in", "-h", "localhost" ] + interval: 30s + timeout: 20s + retries: 3 volumes: - psqldata:/var/lib/postgresql ports: From 9ec2748e22041ec60dd4534aa7d6c08e6eb34496 Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Mon, 23 Jan 2023 12:03:02 +0100 Subject: [PATCH 09/21] update README for deploying stack locally --- dev_utils/README.md | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/dev_utils/README.md b/dev_utils/README.md index e3c5af2..b27c687 100644 --- a/dev_utils/README.md +++ b/dev_utils/README.md @@ -1,20 +1,31 @@ # Dev environment setup recomendations -This guide uses the -[minio client](https://docs.min.io/minio/baremetal/reference/minio-cli/minio-mc.html) -(mc) for testing. +## Deploy a stack locally -## minio s3 server +To start the S3Proxy development environment locally with docker compose, run the following command from the directory `dev_utils` -The S3Proxy development environment is run through docker compose, and can be -started from this directory using: +```bash +docker compose run local +``` + +After that, you can use [s3cmd](https://s3tools.org/s3cmd) to manually interact with the s3 server with proxy by + +```bash +s3cmd -c proxyS3 put README.md s3://dummy ## Upload a file using the proxy +s3cmd -c proxyS3 ls s3://dummy ## List all files of the user using the proxy +``` +If the above commands fail, you may also test if the interaction with the s3 server works without the proxy by ```bash -docker compose up +s3cmd -c directS3 ls s3 ## For access without using the proxy ``` -(use the `-d` flag if you wish to run docker in the background). -Then it's possible to trace all the requests that come to minio by first +## Trace requests to the minio server +This guide uses the +[minio client](https://docs.min.io/minio/baremetal/reference/minio-cli/minio-mc.html) +(mc) for testing. + +Once the stack is deployed locally with docker compose, it's possible to trace all the requests that come to minio by first putting the following in the hosts array of your `~/.mc/config.json` file: ```json @@ -27,8 +38,7 @@ putting the following in the hosts array of your `~/.mc/config.json` file: } ``` -and then in one terminal it's possible to see all requests comming to and all -responses from minio by running +and then run the following command in a terminal ```bash mc admin trace -v proxydev @@ -45,16 +55,6 @@ go build main.go ./main ``` -## Test with s3 configuration file -To test the implementation locally use the `proxyS3` file located -in the `dev_utils` folder or a file downloaded from the login portal. - -## For example use s3tools to interact with the proxy - -```bash -s3cmd -c dev_utils/directS3 ls s3 ## For access without using the proxy -s3cmd -c dev_utils/proxyS3 ls s3 ## For access with using the proxy -``` it's of course also possible to use the `mc` command from minio to access through the proxy or directly but then you have to configure that in the From 72fe9e5092fe697cd0f7776cf0a5e5f3dfa4b0b2 Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Mon, 23 Jan 2023 12:09:11 +0100 Subject: [PATCH 10/21] add note for the change of the file proxyS3 during deployment --- dev_utils/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dev_utils/README.md b/dev_utils/README.md index b27c687..8df0d1b 100644 --- a/dev_utils/README.md +++ b/dev_utils/README.md @@ -15,6 +15,8 @@ s3cmd -c proxyS3 put README.md s3://dummy ## Upload a file using the proxy s3cmd -c proxyS3 ls s3://dummy ## List all files of the user using the proxy ``` +>Note that the content of the file `proxyS3` will be modified since the the string `TOKEN` will be replaced by the actual token during the local deployment. Make sure not to commit this change. + If the above commands fail, you may also test if the interaction with the s3 server works without the proxy by ```bash s3cmd -c directS3 ls s3 ## For access without using the proxy From 2dfd640a6578a1467e000eaecf2b0989239ba842 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Sat, 21 Jan 2023 14:30:29 +0100 Subject: [PATCH 11/21] expose port 9000 for the service s3 --- dev_utils/docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/dev_utils/docker-compose.yml b/dev_utils/docker-compose.yml index 1d1430a..a170b42 100644 --- a/dev_utils/docker-compose.yml +++ b/dev_utils/docker-compose.yml @@ -28,6 +28,7 @@ services: certfixer: condition: service_completed_successfully ports: + - "9000:9000" - "9001:9001" volumes: - s3_certs:/root/.minio/certs From 38e5c28b638d010a2a6df29150903f989c97290b Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Sat, 21 Jan 2023 14:31:47 +0100 Subject: [PATCH 12/21] Change host setting in config files for direct access --- dev_utils/directS3 | 5 ++--- dev_utils/proxyS3 | 5 ++--- tests/tests.sh | 6 +++++- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/dev_utils/directS3 b/dev_utils/directS3 index 453bbce..e88e019 100644 --- a/dev_utils/directS3 +++ b/dev_utils/directS3 @@ -1,13 +1,12 @@ [default] access_key=ElixirID secret_key=987654321 -#access_token=FooBar123 check_ssl_certificate = False encoding = UTF-8 encrypt = False guess_mime_type = True -host_base = s3:9000 -host_bucket = s3:9000 +host_base = localhost:9000 +host_bucket = localhost:9000 human_readable_sizes = True multipart_chunk_size_mb = 5 use_https = True diff --git a/dev_utils/proxyS3 b/dev_utils/proxyS3 index d0d601b..c5111e1 100644 --- a/dev_utils/proxyS3 +++ b/dev_utils/proxyS3 @@ -7,10 +7,9 @@ check_ssl_hostname = False encoding = UTF-8 encrypt = False guess_mime_type = True -host_base = s3_proxy:8000 -host_bucket = s3_proxy:8000 +host_base = localhost:8000 +host_bucket = localhost:8000 human_readable_sizes = true multipart_chunk_size_mb = 5 use_https = True socket_timeout = 30 -ca_certs_file = /certs/ca.crt diff --git a/tests/tests.sh b/tests/tests.sh index c66c29d..7c65969 100644 --- a/tests/tests.sh +++ b/tests/tests.sh @@ -17,7 +17,11 @@ function check_output_status() { cd dev_utils || exit 1 token="$(bash keys/sign_jwt.sh ES256 /keys/jwt.key)" -sed -i "s/TOKEN/$token/" proxyS3 +sed -i "s/^access_token=.*/access_token=$token/" proxyS3 + +# set correct hostanles for S3 and proxy +sed -i "s/localhost:9000/s3:9000/g" directS3 +sed -i "s/localhost:8000/s3_proxy:8000/g" proxyS3 s3cmd -c directS3 put README.md s3://test/some_user/ >/dev/null 2>&1 || exit 1 From 693d0181004cca69f858eaa06ffe0ba96233d821 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Sat, 21 Jan 2023 15:18:15 +0100 Subject: [PATCH 13/21] [mkCerts] use correct name for the pub key when checking for existence --- dev_utils/certfixer/make_certs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev_utils/certfixer/make_certs.sh b/dev_utils/certfixer/make_certs.sh index d9c0d91..3eebd46 100644 --- a/dev_utils/certfixer/make_certs.sh +++ b/dev_utils/certfixer/make_certs.sh @@ -17,7 +17,7 @@ s3_certs="/s3_certs/CAs/public.crt /s3_certs/public.crt /s3_certs/private.key" mq_certs="/mq_certs/ca.crt /mq_certs/mq.crt /mq_certs/mq.key" pub_cert="/pubcert/public.crt" proxy_certs="/proxy_certs/ca.crt /proxy_certs/client.crt /proxy_certs/client.key /proxy_certs/proxy.crt /proxy_certs/proxy.key" -keys="/keys/jwt.key /keys/jwt.pub" +keys="/keys/jwt.key /keys/sda-sda-svc-auth.pub" targets="$s3_certs $mq_certs $pub_cert $proxy_certs $keys" echo "" From 0a4b657e8763ed6b55a44ad5d20950770e974421 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Sat, 21 Jan 2023 15:19:15 +0100 Subject: [PATCH 14/21] [local] add profile for generating local config --- dev_utils/docker-compose.yml | 25 +++++++++++++++++++++++++ dev_utils/local.sh | 19 +++++++++++++++++++ 2 files changed, 44 insertions(+) create mode 100644 dev_utils/local.sh diff --git a/dev_utils/docker-compose.yml b/dev_utils/docker-compose.yml index a170b42..a864b99 100644 --- a/dev_utils/docker-compose.yml +++ b/dev_utils/docker-compose.yml @@ -191,6 +191,31 @@ services: - ..:/app - keys:/keys + local: + image: python:3.9.15-buster + container_name: local-tests + profiles: + - local + command: + - "/bin/sh" + - "/dev_utils/local.sh" + depends_on: + mq: + condition: service_healthy + s3: + condition: service_healthy + s3_proxy: + condition: service_started + certfixer: + condition: service_completed_successfully + createbucket: + condition: service_completed_successfully + volumes: + - proxy_certs:/certs + - .:/dev_utils + - keys:/keys + - /tmp:/local_tmp + volumes: keys: pubcert: diff --git a/dev_utils/local.sh b/dev_utils/local.sh new file mode 100644 index 0000000..13f4711 --- /dev/null +++ b/dev_utils/local.sh @@ -0,0 +1,19 @@ +#!/bin/sh + +if [ "$(id -u)" -eq 0 ]; then + apt-get -qq update && apt-get -qq install -y jq xxd +fi + +cd dev_utils || exit 1 + +local_uid=$(stat -c '%u' .) + +token="$(bash keys/sign_jwt.sh ES256 /keys/jwt.key)" +sed -i "s/^access_token=.*/access_token=$token/" proxyS3 + +mkdir -p /local_tmp/certs +cp /certs/* /local_tmp/certs/ +cp /keys/*pub /local_tmp/certs/ + +chown "$local_uid":"$local_uid" /local_tmp/certs/* +chmod 600 /local_tmp/certs/*.key From fb23d3f3a75a4681d5c2c4e34159cdcf18faca1f Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Sat, 21 Jan 2023 15:28:40 +0100 Subject: [PATCH 15/21] [local] update proxy config file As long as the proxy container is not running the proxy can be started with this command: `SERVER_CONFFILE=dev_utils/config.yaml go run .` --- dev_utils/config.yaml | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/dev_utils/config.yaml b/dev_utils/config.yaml index 8d6ce1a..f0b74f0 100644 --- a/dev_utils/config.yaml +++ b/dev_utils/config.yaml @@ -7,10 +7,10 @@ aws: secretKey: "987654321" bucket: "test" region: "us-east-1" - cacert: "/certs/ca.crt" + cacert: "/tmp/certs/ca.crt" broker: - host: "mq" + host: "localhost" port: "5671" user: "test" password: "test" @@ -19,17 +19,26 @@ broker: routingKey: "files.inbox" ssl: "true" verifyPeer: "true" - cacert: "/certs/ca.crt" - clientCert: "/certs/client.crt" - clientKey: "/certs/client.key" + cacert: "/tmp/certs/ca.crt" + clientCert: "/tmp/certs/client.crt" + clientKey: "/tmp/certs/client.key" # If the FQDN and hostname of the broker differ # serverName can be set to the SAN name in the certificate # serverName: "" +db: + host: "localhost" + port: "2345" + user: "lega_in" + password: "lega_in" + database: "lega" + sslmode: "disable" + + server: - cert: "/certs/proxy.crt" - key: "/certs/proxy.key" - jwtpubkeypath: "./dev_utils/keys/" - jwtpubkeyurl: "https://login.elixir-czech.org/oidc/jwk" + cert: "/tmp/certs/proxy.crt" + key: "/tmp/certs/proxy.key" + jwtpubkeypath: "/tmp/certs/" + jwtpubkeyurl: "" From 80060f4f9f2e2930b4db40f8cc2ae2e540de33c8 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Mon, 23 Jan 2023 09:48:37 +0100 Subject: [PATCH 16/21] Update DB version to 2.0.7 --- dev_utils/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev_utils/docker-compose.yml b/dev_utils/docker-compose.yml index a864b99..707d9c9 100644 --- a/dev_utils/docker-compose.yml +++ b/dev_utils/docker-compose.yml @@ -131,7 +131,7 @@ services: database: container_name: db - image: neicnordic/sda-db:v2.0.0 + image: neicnordic/sda-db:v2.0.7 environment: - DB_LEGA_IN_PASSWORD=lega_in - DB_LEGA_OUT_PASSWORD=lega_out From e97ef1e237f6e0e067df4cead9fe53c41a2b93ab Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Mon, 23 Jan 2023 09:59:29 +0100 Subject: [PATCH 17/21] [dev] add dependency to DB statup --- dev_utils/docker-compose.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dev_utils/docker-compose.yml b/dev_utils/docker-compose.yml index 707d9c9..3128715 100644 --- a/dev_utils/docker-compose.yml +++ b/dev_utils/docker-compose.yml @@ -132,6 +132,9 @@ services: database: container_name: db image: neicnordic/sda-db:v2.0.7 + depends_on: + certfixer: + condition: service_completed_successfully environment: - DB_LEGA_IN_PASSWORD=lega_in - DB_LEGA_OUT_PASSWORD=lega_out From 890e83c6328a101a40fb3d8a6477dd9dccafee4c Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Mon, 23 Jan 2023 10:19:39 +0100 Subject: [PATCH 18/21] [dev] ad healthcheck to DB container --- dev_utils/docker-compose.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/dev_utils/docker-compose.yml b/dev_utils/docker-compose.yml index 3128715..38ddaf5 100644 --- a/dev_utils/docker-compose.yml +++ b/dev_utils/docker-compose.yml @@ -140,6 +140,11 @@ services: - DB_LEGA_OUT_PASSWORD=lega_out - PGVOLUME=/var/lib/postgresql - NOTLS=true + healthcheck: + test: [ "CMD", "pg_isready", "-U", "lega_in", "-h", "localhost" ] + interval: 30s + timeout: 20s + retries: 3 volumes: - psqldata:/var/lib/postgresql ports: From d2b4f1fa825d5dcaf645f303604df5965b4de7b4 Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Mon, 23 Jan 2023 12:03:02 +0100 Subject: [PATCH 19/21] update README for deploying stack locally --- dev_utils/README.md | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/dev_utils/README.md b/dev_utils/README.md index e3c5af2..b27c687 100644 --- a/dev_utils/README.md +++ b/dev_utils/README.md @@ -1,20 +1,31 @@ # Dev environment setup recomendations -This guide uses the -[minio client](https://docs.min.io/minio/baremetal/reference/minio-cli/minio-mc.html) -(mc) for testing. +## Deploy a stack locally -## minio s3 server +To start the S3Proxy development environment locally with docker compose, run the following command from the directory `dev_utils` -The S3Proxy development environment is run through docker compose, and can be -started from this directory using: +```bash +docker compose run local +``` + +After that, you can use [s3cmd](https://s3tools.org/s3cmd) to manually interact with the s3 server with proxy by + +```bash +s3cmd -c proxyS3 put README.md s3://dummy ## Upload a file using the proxy +s3cmd -c proxyS3 ls s3://dummy ## List all files of the user using the proxy +``` +If the above commands fail, you may also test if the interaction with the s3 server works without the proxy by ```bash -docker compose up +s3cmd -c directS3 ls s3 ## For access without using the proxy ``` -(use the `-d` flag if you wish to run docker in the background). -Then it's possible to trace all the requests that come to minio by first +## Trace requests to the minio server +This guide uses the +[minio client](https://docs.min.io/minio/baremetal/reference/minio-cli/minio-mc.html) +(mc) for testing. + +Once the stack is deployed locally with docker compose, it's possible to trace all the requests that come to minio by first putting the following in the hosts array of your `~/.mc/config.json` file: ```json @@ -27,8 +38,7 @@ putting the following in the hosts array of your `~/.mc/config.json` file: } ``` -and then in one terminal it's possible to see all requests comming to and all -responses from minio by running +and then run the following command in a terminal ```bash mc admin trace -v proxydev @@ -45,16 +55,6 @@ go build main.go ./main ``` -## Test with s3 configuration file -To test the implementation locally use the `proxyS3` file located -in the `dev_utils` folder or a file downloaded from the login portal. - -## For example use s3tools to interact with the proxy - -```bash -s3cmd -c dev_utils/directS3 ls s3 ## For access without using the proxy -s3cmd -c dev_utils/proxyS3 ls s3 ## For access with using the proxy -``` it's of course also possible to use the `mc` command from minio to access through the proxy or directly but then you have to configure that in the From 901c6b4f41c6a84631b4b66fd01ff79017c72e9b Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Mon, 23 Jan 2023 12:09:11 +0100 Subject: [PATCH 20/21] add note for the change of the file proxyS3 during deployment --- dev_utils/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dev_utils/README.md b/dev_utils/README.md index b27c687..8df0d1b 100644 --- a/dev_utils/README.md +++ b/dev_utils/README.md @@ -15,6 +15,8 @@ s3cmd -c proxyS3 put README.md s3://dummy ## Upload a file using the proxy s3cmd -c proxyS3 ls s3://dummy ## List all files of the user using the proxy ``` +>Note that the content of the file `proxyS3` will be modified since the the string `TOKEN` will be replaced by the actual token during the local deployment. Make sure not to commit this change. + If the above commands fail, you may also test if the interaction with the s3 server works without the proxy by ```bash s3cmd -c directS3 ls s3 ## For access without using the proxy From 5836f2e4e087b73f3d124195ad1dd2b2eac6898f Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Thu, 2 Feb 2023 14:05:33 +0100 Subject: [PATCH 21/21] fix typos in bash script and README --- dev_utils/README.md | 2 +- tests/tests.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dev_utils/README.md b/dev_utils/README.md index 8df0d1b..12ba859 100644 --- a/dev_utils/README.md +++ b/dev_utils/README.md @@ -15,7 +15,7 @@ s3cmd -c proxyS3 put README.md s3://dummy ## Upload a file using the proxy s3cmd -c proxyS3 ls s3://dummy ## List all files of the user using the proxy ``` ->Note that the content of the file `proxyS3` will be modified since the the string `TOKEN` will be replaced by the actual token during the local deployment. Make sure not to commit this change. +>Note that the content of the file `proxyS3` will be modified since the string `TOKEN` will be replaced by the actual token during the local deployment. Make sure not to commit this change. If the above commands fail, you may also test if the interaction with the s3 server works without the proxy by ```bash diff --git a/tests/tests.sh b/tests/tests.sh index 7c65969..e5a82f5 100644 --- a/tests/tests.sh +++ b/tests/tests.sh @@ -19,7 +19,7 @@ cd dev_utils || exit 1 token="$(bash keys/sign_jwt.sh ES256 /keys/jwt.key)" sed -i "s/^access_token=.*/access_token=$token/" proxyS3 -# set correct hostanles for S3 and proxy +# set correct host for S3 and proxy sed -i "s/localhost:9000/s3:9000/g" directS3 sed -i "s/localhost:8000/s3_proxy:8000/g" proxyS3