Update integration tests to run in docker compose

.github/workflows/functionality.yml

@@ -17,29 +17,5 @@ jobs:
       - name: Check out source code
         uses: actions/checkout@v3
 
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.9'
-
-      - name: install s3cmd
-        run: pip3 install s3cmd
-
-      - name: Deploy containers
-        run: cd dev_utils && GOLANG_VERSION=${{ matrix.go-version }} docker-compose up -d --build
-
-      - name: Wait for containers to start
-        run: |
-          RETRY_TIMES=0
-          for p in mq s3
-          do
-          until docker ps -f name=$p --format {{.Status}} | grep "(healthy)"
-          do echo "waiting for $p to become ready"
-          RETRY_TIMES=$((RETRY_TIMES+1));
-          if [ $RETRY_TIMES -eq 30 ]; then exit 1; fi
-          sleep 10;
-          done
-          done
-
-      - name: Run tests
-        run: bash -x .github/integration/tests/tests.sh
+      - name: Run integration tests
+        run: cd dev_utils && GOLANG_VERSION=${{ matrix.go-version }} docker-compose run integration_tests

dev_utils/directS3

@@ -6,8 +6,8 @@ check_ssl_certificate = False
 encoding = UTF-8
 encrypt = False
 guess_mime_type = True
-host_base = localhost:9000
-host_bucket = localhost:9000
+host_base = s3:9000
+host_bucket = s3:9000
 human_readable_sizes = True
 multipart_chunk_size_mb = 5
 use_https = True

dev_utils/docker-compose.yml

@@ -138,6 +138,29 @@ services:
       - ./users.csv:/users.csv
       - ..:/app
 
+  integration_tests:
+    image: python:3.9.15-buster
+    container_name: s3proxy-integration-tests
+    profiles:
+      - test
+    command:
+      - "/bin/sh"
+      - "-c"
+      - "cd /app; pip install s3cmd && bash ./tests/tests.sh"
+    depends_on:
+      mq:
+        condition: service_healthy
+      s3:
+        condition: service_healthy
+      s3_proxy:
+        condition: service_started
+      certfixer:
+        condition: service_completed_successfully
+    volumes:
+      - proxy_certs:/certs
+      - ./users.csv:/users.csv
+      - ..:/app
+
 volumes:
   pubcert:
   s3_certs:

dev_utils/proxyS3

@@ -7,8 +7,8 @@ check_ssl_hostname = False
 encoding = UTF-8
 encrypt = False
 guess_mime_type = True
-host_base = localhost:8000
-host_bucket = localhost:8000
+host_base = s3_proxy:8000
+host_bucket = s3_proxy:8000
 human_readable_sizes = true
 multipart_chunk_size_mb = 5
 use_https = True

tests/tests.sh

@@ -0,0 +1,117 @@
+#!/bin/bash
+
+# Function checking that a file was uploaded to the S3 backend
+function check_output_status() {
+    if [[ $1 -eq 0 ]]; then
+        echo -e "\u2705 Test passed, expected response found"
+    else
+        echo -e "\u274c Test failed, expected response not found"
+        exit 1
+    fi
+}
+
+cd dev_utils || exit 1
+
+s3cmd -c directS3 put README.md s3://test/some_user/ >/dev/null 2>&1 || exit 1
+
+echo "- Testing allowed actions"
+
+# Put file into bucket
+echo "Trying to upload a file to user's bucket"
+output=$(s3cmd -c proxyS3 put README.md s3://dummy/ >/dev/null 2>&1)
+check_output_status "$output"
+
+# List objects
+echo "Trying to list user's bucket"
+output=$(s3cmd -c proxyS3 ls s3://dummy 2>&1 | grep -q "README.md")
+check_output_status "$output"
+
+# ---------- Test forbidden actions ----------
+forbidden="Forbidden"
+unauthorized="Unauthorized"
+nobucket="NoSuchBucket"
+notfound="Not Found"
+
+echo "- Testing forbidden actions"
+
+# Make bucket
+echo "Trying to create bucket"
+output=$(s3cmd -c proxyS3 mb s3://test_bucket 2>&1 | grep -q $forbidden)
+check_output_status "$output"
+
+# Remove bucket
+echo "Trying to remove bucket"
+output=$(s3cmd -c proxyS3 rb s3://test 2>&1 | grep -q $forbidden)
+check_output_status "$output"
+
+# List buckets
+echo "Trying to list all buckets"
+output=$(s3cmd -c proxyS3 ls s3:// 2>&1 | grep -q $forbidden)
+check_output_status "$output"
+
+# List all objects in all buckets
+echo "Trying to list all objects in all buckets"
+output=$(s3cmd -c proxyS3 la s3:// 2>&1 | grep -q $forbidden)
+check_output_status "$output"
+
+# Put file into another user's bucket
+echo "Trying to upload a file to another user's bucket"
+output=$(s3cmd -c proxyS3 put README.md s3://some_user/ 2>&1 | grep -q $unauthorized)
+check_output_status "$output"
+
+# Get file from another user's bucket
+echo "Trying to get a file from another user's bucket"
+output=$(s3cmd -c proxyS3 get s3://some_user/README.md local_file.md 2>&1 | grep -q $unauthorized)
+check_output_status "$output"
+
+# Get file from own bucket
+echo "Trying to get a file from user's bucket"
+output=$(s3cmd -c proxyS3 get s3://dummy/README.md local_file.md 2>&1 | grep -q $nobucket)
+check_output_status "$output"
+
+# Delete file from bucket
+echo "Trying to delete a file from user's bucket"
+output=$(s3cmd -c proxyS3 del s3://dummy/README.md 2>&1 | grep -q $forbidden)
+check_output_status "$output"
+
+# Disk usage by buckets
+echo "Trying to get disk usage for user's bucket"
+output=$(s3cmd -c proxyS3 du s3://dummy 2>&1 | grep -q $forbidden)
+check_output_status "$output"
+
+# Get various information about user's bucket
+echo "Trying to get information about for user's bucket"
+output=$(s3cmd -c proxyS3 info s3://dummy 2>&1 | grep -q $forbidden)
+check_output_status "$output"
+
+# Get various information about user's file
+echo "Trying to get information about user's file"
+output=$(s3cmd -c proxyS3 info s3://dummy/README.md 2>&1 | grep -q "$notfound")
+check_output_status "$output"
+
+# Move object
+echo "Trying to move file to another location"
+output=$(s3cmd -c proxyS3 mv s3://dummy/README.md s3://dummy/test 2>&1 | grep -q $forbidden)
+check_output_status "$output"
+
+# Copy object
+echo "Trying to copy file to another location"
+output=$(s3cmd -c proxyS3 cp s3://dummy/README.md s3://dummy/test 2>&1 | grep -q $forbidden)
+check_output_status "$output"
+
+# Modify access control list for file
+echo "Trying to modify acl for user's file"
+output=$(s3cmd -c proxyS3 setacl s3://dummy/README.md --acl-public 2>&1 | grep -q $forbidden)
+check_output_status "$output"
+
+# Show multipart uploads - when multipart enabled, add all relevant tests
+echo "Trying to list multipart uploads"
+output=$(s3cmd -c proxyS3 multipart s3://dummy/ 2>&1 | grep -q $nobucket)
+check_output_status "$output"
+
+# Enable/disable bucket access logging
+echo "Trying to change the access logging for a bucket"
+output=$(s3cmd -c proxyS3 accesslog s3://dummy/ 2>&1 | grep -q $nobucket)
+check_output_status "$output"
+
+echo "All tests have passed"