Merge pull request #289 from neicnordic/fix/dev-utils-certs

Fix/dev utils certs

.github/workflows/functionality.yml

@@ -1,5 +1,5 @@
 name: Functionality test
-on: [push, pull_request]
+on: [push]
 jobs:
 
   test:
@@ -17,32 +17,5 @@ jobs:
       - name: Check out source code
         uses: actions/checkout@v3
 
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.9'
-
-      - name: install s3cmd
-        run: pip3 install s3cmd
-
-      - name: generate certificates
-        run: cd dev_utils && /bin/sh make_certs.sh
-
-      - name: Deploy containers
-        run: cd dev_utils && GOLANG_VERSION=${{ matrix.go-version }} docker-compose up -d --build
-
-      - name: Wait for containers to start
-        run: |
-          RETRY_TIMES=0
-          for p in mq s3
-          do
-          until docker ps -f name=$p --format {{.Status}} | grep "(healthy)"
-          do echo "waiting for $p to become ready"
-          RETRY_TIMES=$((RETRY_TIMES+1));
-          if [ $RETRY_TIMES -eq 30 ]; then exit 1; fi
-          sleep 10;
-          done
-          done
-
-      - name: Run tests
-        run: bash -x .github/integration/tests/tests.sh
+      - name: Run integration tests
+        run: cd dev_utils && GOLANG_VERSION=${{ matrix.go-version }} docker-compose run integration_tests

.github/workflows/golint.yml

@@ -1,5 +1,5 @@
 name: linting check
-on: [push, pull_request]
+on: [push]
 jobs:
 
   lint:

.github/workflows/gotest.yml

@@ -1,5 +1,5 @@
 name: Go tests
-on: [push, pull_request]
+on: [push]
 jobs:
 
   test:
@@ -17,27 +17,8 @@ jobs:
       - name: Check out source code
         uses: actions/checkout@v3
 
-      - name: generate certificates
-        run: cd dev_utils && /bin/sh make_certs.sh
-
-      - name: Deploy containers
-        run: cd dev_utils && docker-compose up -d s3_backend mq_server
-
-      - name: Wait for containers to start
-        run: |
-          RETRY_TIMES=0
-          for p in mq s3
-          do
-          until docker ps -f name=$p --format {{.Status}} | grep "(healthy)"
-          do echo "waiting for $p to become ready"
-          RETRY_TIMES=$((RETRY_TIMES+1));
-          if [ $RETRY_TIMES -eq 30 ]; then exit 1; fi
-          sleep 10;
-          done
-          done
-
-      - name: Calc coverage 
-        run: go test -tags live -coverprofile=coverage.txt -covermode=atomic
+      - name: Run test container
+        run: cd dev_utils && GOLANG_VERSION=${{ matrix.go-version }} docker compose run tests
 
       - name: Codecov
         uses: codecov/[email protected]

.gitignore

@@ -12,4 +12,7 @@
 *.out
 
 # log dumps
-*.dump
+*.dump
+
+# coverage report
+coverage.txt

Dockerfile

@@ -1,5 +1,4 @@
-ARG GOLANG_VERSION=1.17
-FROM "golang:${GOLANG_VERSION}-alpine"
+FROM "golang:${GOLANG_VERSION:-1.19}-alpine"
 RUN apk add --no-cache git
 COPY . .
 ENV GO111MODULE=on

config_test.go

@@ -2,9 +2,11 @@ package main
 
 import (
 	"fmt"
+	"os"
 	"path/filepath"
 	"testing"
 
+	"github.com/NBISweden/S3-Upload-Proxy/helper"
 	log "github.com/sirupsen/logrus"
 
 	"github.com/spf13/viper"
@@ -16,7 +18,12 @@ type TestSuite struct {
 	suite.Suite
 }
 
+var certPath string
+
 func (suite *TestSuite) SetupTest() {
+	certPath, _ = os.MkdirTemp("", "gocerts")
+	helper.MakeCerts(certPath)
+
 	viper.Set("broker.host", "testhost")
 	viper.Set("broker.port", 123)
 	viper.Set("broker.user", "testuser")
@@ -33,6 +40,7 @@ func (suite *TestSuite) SetupTest() {
 
 func (suite *TestSuite) TearDownTest() {
 	viper.Reset()
+	defer os.RemoveAll(certPath)
 }
 
 func TestConfigTestSuite(t *testing.T) {
@@ -123,7 +131,7 @@ func (suite *TestSuite) TestConfigBroker() {
 func (suite *TestSuite) TestTLSConfigBroker() {
 	viper.Set("broker.serverName", "broker")
 	viper.Set("broker.ssl", true)
-	viper.Set("broker.cacert", "dev_utils/certs/ca.crt")
+	viper.Set("broker.cacert", certPath+"/ca.crt")
 	config, err := NewConfig()
 	assert.NotNil(suite.T(), config)
 	assert.NoError(suite.T(), err)
@@ -132,28 +140,27 @@ func (suite *TestSuite) TestTLSConfigBroker() {
 	assert.NoError(suite.T(), err)
 
 	viper.Set("broker.verifyPeer", true)
-	viper.Set("broker.clientCert", "./dev_utils/certs/client.crt")
-	viper.Set("broker.clientKey", "./dev_utils/certs/client.key")
+	viper.Set("broker.clientCert", certPath+"/tls.crt")
+	viper.Set("broker.clientKey", certPath+"/tls.key")
 	config, err = NewConfig()
 	assert.NotNil(suite.T(), config)
 	assert.NoError(suite.T(), err)
 	tlsBroker, err = TLSConfigBroker(config)
 	assert.NotNil(suite.T(), tlsBroker)
 	assert.NoError(suite.T(), err)
 
-	viper.Set("broker.clientCert", "./dev_utils/certs/client.pem")
-	viper.Set("broker.clientKey", "./dev_utils/certs/client-key.pem")
+	viper.Set("broker.clientCert", certPath+"tls.crt")
+	viper.Set("broker.clientKey", certPath+"/tls.key")
 	config, err = NewConfig()
 	assert.NotNil(suite.T(), config)
 	assert.NoError(suite.T(), err)
 	tlsBroker, err = TLSConfigBroker(config)
 	assert.Nil(suite.T(), tlsBroker)
 	assert.Error(suite.T(), err)
-
 }
 
 func (suite *TestSuite) TestTLSConfigProxy() {
-	viper.Set("aws.cacert", "dev_utils/certs/ca.crt")
+	viper.Set("aws.cacert", certPath+"/ca.crt")
 	config, err := NewConfig()
 	assert.NotNil(suite.T(), config)
 	assert.NoError(suite.T(), err)

dev_utils/certfixer/make_certs.sh

@@ -0,0 +1,84 @@
+#!/bin/sh
+
+set -e
+
+out_dir="/cert_gen"
+
+# install openssl if it's missing
+if [ ! "$(command -v openssl)" ];
+then
+    apk add openssl
+fi
+
+script_dir="$(dirname "$0")"
+mkdir -p "$out_dir"
+
+# list all certificates we want, so that we can check if they already exist
+s3_certs="/s3_certs/CAs/public.crt /s3_certs/public.crt /s3_certs/private.key"
+mq_certs="/mq_certs/ca.crt /mq_certs/mq.crt /mq_certs/mq.key"
+pub_cert="/pubcert/public.crt"
+proxy_certs="/proxy_certs/ca.crt /proxy_certs/client.crt /proxy_certs/client.key /proxy_certs/proxy.crt /proxy_certs/proxy.key"
+targets="$s3_certs $mq_certs $pub_cert $proxy_certs"
+
+echo ""
+echo "Checking certificates"
+recreate="false"
+# check if certificates exist
+for target in $targets
+do
+    if [ ! -f "$target" ]
+    then
+        recreate="true"
+        break
+    fi
+done
+
+# only recreate certificates if any certificate is missing
+if [ "$recreate" = "false" ]
+then
+    echo "certificates already exists"
+    exit 0
+fi
+
+# create CA certificate
+openssl req -config "$script_dir/ssl.cnf" -new -sha256 -nodes -extensions v3_ca -out "$out_dir/ca.csr" -keyout "$out_dir/ca-key.pem"
+openssl req -config "$script_dir/ssl.cnf" -key "$out_dir/ca-key.pem" -x509 -new -days 7300 -sha256 -nodes -extensions v3_ca -out "$out_dir/ca.crt"
+
+# Create certificate for MQ
+openssl req -config "$script_dir/ssl.cnf" -new -nodes -newkey rsa:4096 -keyout "$out_dir/mq.key" -out "$out_dir/mq.csr" -extensions server_cert
+openssl x509 -req -in "$out_dir/mq.csr" -days 1200 -CA "$out_dir/ca.crt" -CAkey "$out_dir/ca-key.pem" -set_serial 01 -out "$out_dir/mq.crt" -extensions server_cert -extfile "$script_dir/ssl.cnf"
+
+# Create certificate for Proxy
+openssl req -config "$script_dir/ssl.cnf" -new -nodes -newkey rsa:4096 -keyout "$out_dir/proxy.key" -out "$out_dir/proxy.csr" -extensions server_cert
+openssl x509 -req -in "$out_dir/proxy.csr" -days 1200 -CA "$out_dir/ca.crt" -CAkey "$out_dir/ca-key.pem" -set_serial 01 -out "$out_dir/proxy.crt" -extensions server_cert -extfile "$script_dir/ssl.cnf"
+
+# Create certificate for minio
+openssl req -config "$script_dir/ssl.cnf" -new -nodes -newkey rsa:4096 -keyout "$out_dir/s3.key" -out "$out_dir/s3.csr" -extensions server_cert
+openssl x509 -req -in "$out_dir/s3.csr" -days 1200 -CA "$out_dir/ca.crt" -CAkey "$out_dir/ca-key.pem" -set_serial 01 -out "$out_dir/s3.crt" -extensions server_cert -extfile "$script_dir/ssl.cnf"
+
+# Create client certificate
+openssl req -config "$script_dir/ssl.cnf" -new -nodes -newkey rsa:4096 -keyout "$out_dir/client.key" -out "$out_dir/client.csr" -extensions client_cert -subj "/CN=admin"
+openssl x509 -req -in "$out_dir/client.csr" -days 1200 -CA "$out_dir/ca.crt" -CAkey "$out_dir/ca-key.pem" -set_serial 01 -out "$out_dir/client.crt" -extensions client_cert -extfile "$script_dir/ssl.cnf"
+
+# fix permissions
+chmod 644 "$out_dir"/*
+chown -R root:root "$out_dir"/*
+chmod 600 "$out_dir"/*-key.pem
+
+# move certificates to volumes
+mkdir -p /s3_certs/CAs
+cp -p "$out_dir/ca.crt" /s3_certs/CAs/public.crt
+cp -p "$out_dir/s3.crt" /s3_certs/public.crt
+cp -p "$out_dir/s3.key" /s3_certs/private.key
+
+cp -p "$out_dir/ca.crt" /mq_certs/ca.crt
+cp -p "$out_dir/mq.crt" /mq_certs/mq.crt
+cp -p "$out_dir/mq.key" /mq_certs/mq.key
+
+cp -p "$out_dir/ca.crt" /pubcert/public.crt
+
+cp -p "$out_dir/ca.crt" /proxy_certs/ca.crt
+cp -p "$out_dir/client.crt" /proxy_certs/client.crt
+cp -p "$out_dir/client.key" /proxy_certs/client.key
+cp -p "$out_dir/proxy.crt" /proxy_certs/proxy.crt
+cp -p "$out_dir/proxy.key" /proxy_certs/proxy.key

dev_utils/config.yaml

@@ -7,10 +7,10 @@ aws:
   secretKey: "987654321"
   bucket: "test"
   region: "us-east-1"
-  cacert: "./dev_utils/certs/ca.crt"
+  cacert: "/certs/ca.crt"
 
 broker:
-  host: "localhost"
+  host: "mq"
   port: "5671"
   user: "test"
   password: "test"
@@ -19,16 +19,16 @@ broker:
   routingKey: "files.inbox"
   ssl: "true"
   verifyPeer: "true"
-  cacert: "./dev_utils/certs/ca.crt"
-  clientCert: "./dev_utils/certs/client.crt"
-  clientKey: "./dev_utils/certs/client.key"
+  cacert: "/certs/ca.crt"
+  clientCert: "/certs/client.crt"
+  clientKey: "/certs/client.key"
 # If the FQDN and hostname of the broker differ
 # serverName can be set to the SAN name in the certificate
   #  serverName: ""
 
 server:
-  cert: "./dev_utils/certs/proxy.crt"
-  key: "./dev_utils/certs/proxy.key"
+  cert: "/certs/proxy.crt"
+  key: "/certs/proxy.key"
   users: "./dev_utils/users.csv"
   jwtpubkeypath: "./dev_utils/keys/"
   jwtpubkeyurl: "https://login.elixir-czech.org/oidc/jwk"

dev_utils/directS3

@@ -6,8 +6,8 @@ check_ssl_certificate = False
 encoding = UTF-8
 encrypt = False
 guess_mime_type = True
-host_base = localhost:9000
-host_bucket = localhost:9000
+host_base = s3:9000
+host_bucket = s3:9000
 human_readable_sizes = True
 multipart_chunk_size_mb = 5
 use_https = True