Feature/db connection (#302)

.github/workflows/functionality.yml

@@ -7,7 +7,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        go-version: [1.17, 1.18]
+        go-version: [1.18, 1.19]
     steps:
       - name: Set up Go ${{ matrix.go-version }}
         uses: actions/setup-go@v3

.github/workflows/golint.yml

@@ -8,7 +8,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        go-version: [1.17, 1.18]
+        go-version: [1.18, 1.19]
     steps:
     - name: Set up Go ${{ matrix.go-version }}
       uses: actions/setup-go@v3
@@ -18,5 +18,4 @@ jobs:
     - name: Run golangci-lint
       uses: golangci/[email protected]
       with:
-        version: v1.45
-        args: -E gosec,nestif,bodyclose,rowserrcheck -e G107
+        args: -E bodyclose,gocritic,gofmt,gosec,govet,nestif,nlreturn,revive -e G107

.github/workflows/gotest.yml

@@ -7,7 +7,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        go-version: [1.17, 1.18]
+        go-version: [1.18, 1.19]
     steps:
       - name: Set up Go ${{ matrix.go-version }}
         uses: actions/setup-go@v3

config.go

@@ -9,6 +9,7 @@ import (
 	"reflect"
 	"strings"
 
+	"github.com/neicnordic/sda-common/database"
 	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/viper"
@@ -62,6 +63,7 @@ type Config struct {
 	S3     S3Config
 	Broker BrokerConfig
 	Server ServerConfig
+	DB     database.DBConf
 }
 
 // NewConfig initializes and parses the config file and/or environment using
@@ -196,6 +198,25 @@ func (c *Config) readConfig() error {
 
 	c.Broker = b
 
+	// Setup psql db
+	c.DB.Host = viper.GetString("db.host")
+	c.DB.Port = viper.GetInt("db.port")
+	c.DB.User = viper.GetString("db.user")
+	c.DB.Password = viper.GetString("db.password")
+	c.DB.Database = viper.GetString("db.database")
+	if viper.IsSet("db.cacert") {
+		c.DB.CACert = viper.GetString("db.cacert")
+	}
+	c.DB.SslMode = viper.GetString("db.sslmode")
+	if c.DB.SslMode == "verify-full" {
+		// Since verify-full is specified, these are required.
+		if !(viper.IsSet("db.clientCert") && viper.IsSet("db.clientKey")) {
+			return errors.New("when db.sslMode is set to verify-full both db.clientCert and db.clientKey are needed")
+		}
+		c.DB.ClientCert = viper.GetString("db.clientcert")
+		c.DB.ClientKey = viper.GetString("db.clientkey")
+	}
+
 	// Setup server
 	s := ServerConfig{}
 

dev_utils/docker-compose.yml

@@ -79,8 +79,12 @@ services:
         condition: service_healthy
       s3:
         condition: service_healthy
+      database:
+        condition: service_healthy
       certfixer:
         condition: service_completed_successfully
+      createbucket:
+        condition: service_completed_successfully
     restart: always
     environment:
       - LOG_LEVEL=info
@@ -91,6 +95,15 @@ services:
       - AWS_REGION=us-east-1
       - AWS_READYPATH=/minio/health/ready
       - AWS_CACERT=/certs/ca.crt
+      - DB_HOST=db
+      - DB_PORT=5432
+      - DB_USER=lega_in
+      - DB_PASSWORD=lega_in
+      - DB_DATABASE=lega
+      - DB_CACERT=
+      - DB_SSLMODE=disable
+      - DB_CLIENTCERT=
+      - DB_CLIENTKEY=
       - BROKER_HOST=mq
       - BROKER_USER=test
       - BROKER_PASSWORD=test
@@ -115,22 +128,37 @@ services:
       - "8000:8000"
       - "8001:8001"
 
+  database:
+    container_name: db
+    image: neicnordic/sda-db:v2.0.0
+    environment:
+      - DB_LEGA_IN_PASSWORD=lega_in
+      - DB_LEGA_OUT_PASSWORD=lega_out
+      - PGVOLUME=/var/lib/postgresql
+      - NOTLS=true
+    volumes:
+      - psqldata:/var/lib/postgresql
+    ports:
+      - 2345:5432
+
   tests:
-    image: golang:${GOLANG_VERSION:-1.18}
+    image: golang:${GOLANG_VERSION:-1.19}
     container_name: s3proxy-tests
     profiles:
       - test
     command:
       - "/bin/sh"
       - "-c"
-      - "cd /app; echo 'Running go ${GOLANG_VERSION:-1.18} tests';
+      - "cd /app; echo 'Running go ${GOLANG_VERSION:-1.19} tests';
          go install 2>/dev/null
-         && go test . -v -coverprofile=coverage.txt -covermode=atomic"
+         && go test ./... -v -coverprofile=coverage.txt -covermode=atomic"
     depends_on:
       mq:
         condition: service_healthy
       s3:
         condition: service_healthy
+      database:
+        condition: service_healthy
       certfixer:
         condition: service_completed_successfully
     volumes:
@@ -156,6 +184,8 @@ services:
         condition: service_started
       certfixer:
         condition: service_completed_successfully
+      createbucket:
+        condition: service_completed_successfully
     volumes:
       - proxy_certs:/certs
       - ./users.csv:/users.csv
@@ -166,6 +196,7 @@ volumes:
   s3_certs:
   mq_certs:
   proxy_certs:
+  psqldata:
   data:
     # These settings only work on linux (including WSL2), and can be used to
     # test when the disk is full.

go.mod

@@ -6,10 +6,11 @@ require (
 	github.com/aws/aws-sdk-go v1.44.153
 	github.com/golang-jwt/jwt/v4 v4.4.3
 	github.com/google/uuid v1.3.0
-	github.com/heptiolabs/healthcheck v0.0.0-20180807145615-6ff867650f40
-	github.com/johannesboyne/gofakes3 v0.0.0-20210608054100-92d5d4af5fde
+	github.com/heptiolabs/healthcheck v0.0.0-20211123025425-613501dd5deb
+	github.com/johannesboyne/gofakes3 v0.0.0-20220627085814-c3ac35da23b2
 	github.com/lestrrat/go-jwx v0.0.0-20180221005942-b7d4802280ae
 	github.com/minio/minio-go/v6 v6.0.43
+	github.com/neicnordic/sda-common v0.0.3-0.20221122104056-37b54aea80a7
 	github.com/pkg/errors v0.9.1
 	github.com/sirupsen/logrus v1.9.0
 	github.com/spf13/viper v1.14.0
@@ -26,6 +27,7 @@ require (
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/lestrrat/go-pdebug v0.0.0-20180220043741-569c97477ae8 // indirect
+	github.com/lib/pq v1.10.7 // indirect
 	github.com/magiconair/properties v1.8.6 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
 	github.com/minio/sha256-simd v0.1.1 // indirect

go.sum

@@ -156,17 +156,17 @@ github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
-github.com/heptiolabs/healthcheck v0.0.0-20180807145615-6ff867650f40 h1:GT4RsKmHh1uZyhmTkWJTDALRjSHYQp6FRKrotf0zhAs=
-github.com/heptiolabs/healthcheck v0.0.0-20180807145615-6ff867650f40/go.mod h1:NtmN9h8vrTveVQRLHcX2HQ5wIPBDCsZ351TGbZWgg38=
+github.com/heptiolabs/healthcheck v0.0.0-20211123025425-613501dd5deb h1:tsEKRC3PU9rMw18w/uAptoijhgG4EvlA5kfJPtwrMDk=
+github.com/heptiolabs/healthcheck v0.0.0-20211123025425-613501dd5deb/go.mod h1:NtmN9h8vrTveVQRLHcX2HQ5wIPBDCsZ351TGbZWgg38=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
 github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
-github.com/johannesboyne/gofakes3 v0.0.0-20210608054100-92d5d4af5fde h1:ekNURlaug3SgiS0KQzL/5oiYPUJPozt1C+ajLBWk7/E=
-github.com/johannesboyne/gofakes3 v0.0.0-20210608054100-92d5d4af5fde/go.mod h1:LIAXxPvcUXwOcTIj9LSNSUpE9/eMHalTWxsP/kmWxQI=
+github.com/johannesboyne/gofakes3 v0.0.0-20220627085814-c3ac35da23b2 h1:V5q1Mx2WTE5coXLG2QpkRZ7LsJvgkedm6Ib4AwC1Lfg=
+github.com/johannesboyne/gofakes3 v0.0.0-20220627085814-c3ac35da23b2/go.mod h1:LIAXxPvcUXwOcTIj9LSNSUpE9/eMHalTWxsP/kmWxQI=
 github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
@@ -191,6 +191,8 @@ github.com/lestrrat/go-jwx v0.0.0-20180221005942-b7d4802280ae h1:XoMPFIGibcPKgLr
 github.com/lestrrat/go-jwx v0.0.0-20180221005942-b7d4802280ae/go.mod h1:T+yHdCP6MJKtzoVQMHvVCeam5VFwX1+rWzn5zZgKYMI=
 github.com/lestrrat/go-pdebug v0.0.0-20180220043741-569c97477ae8 h1:ttJD8hTqvrPEUBoAG5hJKbDOJ84u7zmbnZsUL4V9430=
 github.com/lestrrat/go-pdebug v0.0.0-20180220043741-569c97477ae8/go.mod h1:VXFH11P7fHn2iPBsfSW1JacR59rttTcafJnwYcI/IdY=
+github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw=
+github.com/lib/pq v1.10.7/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
 github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo=
 github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60=
 github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
@@ -209,6 +211,12 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/neicnordic/sda-common v0.0.3-0.20221107140128-a5c86e921aa1 h1:K/kwvbH1QFwoICMAbeQi9+FMVpGsmKYCz1lETZWqkrY=
+github.com/neicnordic/sda-common v0.0.3-0.20221107140128-a5c86e921aa1/go.mod h1:idzRZoISiXSkdVDrRPmNtF89kxFZtGz7p8QUQBGd7ZA=
+github.com/neicnordic/sda-common v0.0.3-0.20221114114925-a020b3bce09d h1:ADI6ETvCJluWP9N9HHB4f0MX7Qfi9ZEDBlHWIv8cznM=
+github.com/neicnordic/sda-common v0.0.3-0.20221114114925-a020b3bce09d/go.mod h1:idzRZoISiXSkdVDrRPmNtF89kxFZtGz7p8QUQBGd7ZA=
+github.com/neicnordic/sda-common v0.0.3-0.20221122104056-37b54aea80a7 h1:P+Qs2se4Y+I4+KZY6BavumsU6cMZ4ar2KiuvrswzToQ=
+github.com/neicnordic/sda-common v0.0.3-0.20221122104056-37b54aea80a7/go.mod h1:0+D7zVXKh15bkLgir4EmtSwzoo5fs70Ko9nYR8d18QE=
 github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8=
 github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
 github.com/pelletier/go-toml/v2 v2.0.5 h1:ipoSadvV8oGUjnUbMub59IDPPwfxF694nG/jwbMiyQg=

main.go

@@ -4,6 +4,7 @@ import (
 	"net/http"
 	"time"
 
+	common "github.com/neicnordic/sda-common/database"
 	log "github.com/sirupsen/logrus"
 )
 
@@ -28,6 +29,15 @@ func main() {
 		log.Panic(err)
 	}
 
+	sdaDB, err := common.NewSDAdb(config.DB)
+	if err != nil {
+		log.Panic(err)
+	}
+
+	defer sdaDB.Close()
+
+	log.Debugf("Connected to sda-db (v%v)", sdaDB.Version)
+
 	err = checkS3Bucket(config.S3)
 	if err != nil {
 		log.Panic(err)
@@ -50,7 +60,7 @@ func main() {
 			log.Panicf("Error while getting key %s: %v", config.Server.jwtpubkeypath, err)
 		}
 	}
-	proxy := NewProxy(config.S3, auth, messenger, tlsProxy)
+	proxy := NewProxy(config.S3, auth, messenger, sdaDB, tlsProxy)
 
 	log.Debug("got the proxy ", proxy)
 

proxy.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"crypto/sha256"
 	"crypto/tls"
+	"encoding/json"
 	"fmt"
 	"io"
 	"net/http"
@@ -13,6 +14,8 @@ import (
 	"strings"
 	"time"
 
+	common "github.com/neicnordic/sda-common/database"
+
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/awserr"
 	"github.com/aws/aws-sdk-go/aws/credentials"
@@ -28,7 +31,9 @@ type Proxy struct {
 	s3        S3Config
 	auth      Authenticator
 	messenger Messenger
+	database  *common.SDAdb
 	client    *http.Client
+	fileIds   map[string]string
 }
 
 // S3RequestType is the type of request that we are currently proxying to the
@@ -49,11 +54,11 @@ const (
 )
 
 // NewProxy creates a new S3Proxy. This implements the ServerHTTP interface.
-func NewProxy(s3conf S3Config, auth Authenticator, messenger Messenger, tls *tls.Config) *Proxy {
+func NewProxy(s3conf S3Config, auth Authenticator, messenger Messenger, database *common.SDAdb, tls *tls.Config) *Proxy {
 	tr := &http.Transport{TLSClientConfig: tls}
 	client := &http.Client{Transport: tr}
 
-	return &Proxy{s3conf, auth, messenger, client}
+	return &Proxy{s3conf, auth, messenger, database, client, make(map[string]string)}
 }
 
 func (p *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
@@ -99,6 +104,20 @@ func (p *Proxy) allowedResponse(w http.ResponseWriter, r *http.Request) {
 	log.Debug("prepend")
 	p.prependBucketToHostPath(r)
 
+	username := fmt.Sprintf("%v", claims["sub"])
+	filepath := strings.Replace(r.URL.Path, "/"+p.s3.bucket+"/", "", 1)
+	// register file in database if it's the start of an upload
+	if p.detectRequestType(r) == Put && p.fileIds[r.URL.Path] == "" {
+		log.Debugf("registering file %v in the database", r.URL.Path)
+		p.fileIds[r.URL.Path], err = p.database.RegisterFile(filepath, username)
+		log.Debugf("fileId: %v", p.fileIds[r.URL.Path])
+		if err != nil {
+			log.Errorf("failed to register file in database: %v", err)
+
+			return
+		}
+	}
+
 	log.Debug("Forwarding to backend")
 	s3response, err := p.forwardToBackend(r)
 
@@ -110,13 +129,27 @@ func (p *Proxy) allowedResponse(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	// Send message to upstream
+	// Send message to upstream and set file as uploaded in the database
 	if p.uploadFinishedSuccessfully(r, s3response) {
 		log.Debug("create message")
 		message, _ := p.CreateMessageFromRequest(r, claims)
 		if err = p.messenger.SendMessage(message); err != nil {
 			log.Debug("error when sending message")
-			log.Debug(err)
+			log.Error(err)
+		}
+
+		jsonMessage, err := json.Marshal(message)
+		if err != nil {
+			log.Errorf("failed to marshal rabbitmq message to json: %v", err)
+
+			return
+		}
+		fileID := p.fileIds[r.URL.Path]
+		delete(p.fileIds, r.URL.Path)
+		log.Debugf("marking file %v as 'uploaded' in database", fileID)
+		err = p.database.MarkFileAsUploaded(fileID, username, string(jsonMessage))
+		if err != nil {
+			log.Error(err)
 		}
 	}