From 2a91fff8fdf1be2bd9b2228a0382c6075fcfe5ad Mon Sep 17 00:00:00 2001 From: Martin Norling Date: Wed, 23 Nov 2022 13:48:22 +0100 Subject: [PATCH 01/16] Add database settings to config --- config.go | 13 +++++++++++++ dev_utils/docker-compose.yml | 9 +++++++++ 2 files changed, 22 insertions(+) diff --git a/config.go b/config.go index 1ef7f34..93621f5 100644 --- a/config.go +++ b/config.go @@ -9,6 +9,7 @@ import ( "reflect" "strings" + "github.com/neicnordic/sda-common/database" "github.com/pkg/errors" log "github.com/sirupsen/logrus" "github.com/spf13/viper" @@ -62,6 +63,7 @@ type Config struct { S3 S3Config Broker BrokerConfig Server ServerConfig + DB database.DBConf } // NewConfig initializes and parses the config file and/or environment using @@ -196,6 +198,17 @@ func (c *Config) readConfig() error { c.Broker = b + // Setup psql db + c.DB.Host = viper.GetString("db.host") + c.DB.Port = viper.GetInt("db.port") + c.DB.User = viper.GetString("db.user") + c.DB.Password = viper.GetString("db.password") + c.DB.Database = viper.GetString("db.database") + c.DB.CACert = viper.GetString("db.cacert") + c.DB.SslMode = viper.GetString("db.sslmode") + c.DB.ClientCert = viper.GetString("db.clientcert") + c.DB.ClientKey = viper.GetString("db.clientkey") + // Setup server s := ServerConfig{} diff --git a/dev_utils/docker-compose.yml b/dev_utils/docker-compose.yml index ec32320..a9311cb 100644 --- a/dev_utils/docker-compose.yml +++ b/dev_utils/docker-compose.yml @@ -91,6 +91,15 @@ services: - AWS_REGION=us-east-1 - AWS_READYPATH=/minio/health/ready - AWS_CACERT=/certs/ca.crt + - DB_HOST=db + - DB_PORT=5432 + - DB_USER=lega_in + - DB_PASSWORD=inpass + - DB_DATABASE=lega + - DB_CACERT= + - DB_SSLMODE=disable + - DB_CLIENTCERT= + - DB_CLIENTKEY= - BROKER_HOST=mq - BROKER_USER=test - BROKER_PASSWORD=test From e06d58bf6a3af8ced67540b8f857cb289bea6ac3 Mon Sep 17 00:00:00 2001 From: Martin Norling Date: Wed, 23 Nov 2022 13:49:55 +0100 Subject: [PATCH 02/16] Add database connection to proxy --- dev_utils/docker-compose.yml | 14 ++++++++++++++ go.mod | 6 ++++-- go.sum | 14 ++++++++++---- main.go | 12 +++++++++++- proxy.go | 32 ++++++++++++++++++++++++++++---- 5 files changed, 67 insertions(+), 11 deletions(-) diff --git a/dev_utils/docker-compose.yml b/dev_utils/docker-compose.yml index a9311cb..7fa86f2 100644 --- a/dev_utils/docker-compose.yml +++ b/dev_utils/docker-compose.yml @@ -124,6 +124,19 @@ services: - "8000:8000" - "8001:8001" + database: + container_name: db + image: neicnordic/sda-db:v2.0.0 + environment: + - DB_LEGA_IN_PASSWORD=inpass + - DB_LEGA_OUT_PASSWORD=outpass + - PGVOLUME=/var/lib/postgresql + - NOTLS=true + volumes: + - psqldata:/var/lib/postgresql + ports: + - 5432:5432 + tests: image: golang:${GOLANG_VERSION:-1.18} container_name: s3proxy-tests @@ -175,6 +188,7 @@ volumes: s3_certs: mq_certs: proxy_certs: + psqldata: data: # These settings only work on linux (including WSL2), and can be used to # test when the disk is full. diff --git a/go.mod b/go.mod index a76f76b..77d9c32 100644 --- a/go.mod +++ b/go.mod @@ -6,10 +6,11 @@ require ( github.com/aws/aws-sdk-go v1.44.153 github.com/golang-jwt/jwt/v4 v4.4.3 github.com/google/uuid v1.3.0 - github.com/heptiolabs/healthcheck v0.0.0-20180807145615-6ff867650f40 - github.com/johannesboyne/gofakes3 v0.0.0-20210608054100-92d5d4af5fde + github.com/heptiolabs/healthcheck v0.0.0-20211123025425-613501dd5deb + github.com/johannesboyne/gofakes3 v0.0.0-20220627085814-c3ac35da23b2 github.com/lestrrat/go-jwx v0.0.0-20180221005942-b7d4802280ae github.com/minio/minio-go/v6 v6.0.43 + github.com/neicnordic/sda-common v0.0.3-0.20221114114925-a020b3bce09d github.com/pkg/errors v0.9.1 github.com/sirupsen/logrus v1.9.0 github.com/spf13/viper v1.14.0 @@ -26,6 +27,7 @@ require ( github.com/hashicorp/hcl v1.0.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/lestrrat/go-pdebug v0.0.0-20180220043741-569c97477ae8 // indirect + github.com/lib/pq v1.10.7 // indirect github.com/magiconair/properties v1.8.6 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect github.com/minio/sha256-simd v0.1.1 // indirect diff --git a/go.sum b/go.sum index 13b13c9..46454fc 100644 --- a/go.sum +++ b/go.sum @@ -156,8 +156,8 @@ github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/heptiolabs/healthcheck v0.0.0-20180807145615-6ff867650f40 h1:GT4RsKmHh1uZyhmTkWJTDALRjSHYQp6FRKrotf0zhAs= -github.com/heptiolabs/healthcheck v0.0.0-20180807145615-6ff867650f40/go.mod h1:NtmN9h8vrTveVQRLHcX2HQ5wIPBDCsZ351TGbZWgg38= +github.com/heptiolabs/healthcheck v0.0.0-20211123025425-613501dd5deb h1:tsEKRC3PU9rMw18w/uAptoijhgG4EvlA5kfJPtwrMDk= +github.com/heptiolabs/healthcheck v0.0.0-20211123025425-613501dd5deb/go.mod h1:NtmN9h8vrTveVQRLHcX2HQ5wIPBDCsZ351TGbZWgg38= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= @@ -165,8 +165,8 @@ github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9Y github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= -github.com/johannesboyne/gofakes3 v0.0.0-20210608054100-92d5d4af5fde h1:ekNURlaug3SgiS0KQzL/5oiYPUJPozt1C+ajLBWk7/E= -github.com/johannesboyne/gofakes3 v0.0.0-20210608054100-92d5d4af5fde/go.mod h1:LIAXxPvcUXwOcTIj9LSNSUpE9/eMHalTWxsP/kmWxQI= +github.com/johannesboyne/gofakes3 v0.0.0-20220627085814-c3ac35da23b2 h1:V5q1Mx2WTE5coXLG2QpkRZ7LsJvgkedm6Ib4AwC1Lfg= +github.com/johannesboyne/gofakes3 v0.0.0-20220627085814-c3ac35da23b2/go.mod h1:LIAXxPvcUXwOcTIj9LSNSUpE9/eMHalTWxsP/kmWxQI= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -191,6 +191,8 @@ github.com/lestrrat/go-jwx v0.0.0-20180221005942-b7d4802280ae h1:XoMPFIGibcPKgLr github.com/lestrrat/go-jwx v0.0.0-20180221005942-b7d4802280ae/go.mod h1:T+yHdCP6MJKtzoVQMHvVCeam5VFwX1+rWzn5zZgKYMI= github.com/lestrrat/go-pdebug v0.0.0-20180220043741-569c97477ae8 h1:ttJD8hTqvrPEUBoAG5hJKbDOJ84u7zmbnZsUL4V9430= github.com/lestrrat/go-pdebug v0.0.0-20180220043741-569c97477ae8/go.mod h1:VXFH11P7fHn2iPBsfSW1JacR59rttTcafJnwYcI/IdY= +github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw= +github.com/lib/pq v1.10.7/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo= github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= @@ -209,6 +211,10 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/neicnordic/sda-common v0.0.3-0.20221107140128-a5c86e921aa1 h1:K/kwvbH1QFwoICMAbeQi9+FMVpGsmKYCz1lETZWqkrY= +github.com/neicnordic/sda-common v0.0.3-0.20221107140128-a5c86e921aa1/go.mod h1:idzRZoISiXSkdVDrRPmNtF89kxFZtGz7p8QUQBGd7ZA= +github.com/neicnordic/sda-common v0.0.3-0.20221114114925-a020b3bce09d h1:ADI6ETvCJluWP9N9HHB4f0MX7Qfi9ZEDBlHWIv8cznM= +github.com/neicnordic/sda-common v0.0.3-0.20221114114925-a020b3bce09d/go.mod h1:idzRZoISiXSkdVDrRPmNtF89kxFZtGz7p8QUQBGd7ZA= github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml/v2 v2.0.5 h1:ipoSadvV8oGUjnUbMub59IDPPwfxF694nG/jwbMiyQg= diff --git a/main.go b/main.go index 9e25f66..205400e 100644 --- a/main.go +++ b/main.go @@ -4,6 +4,7 @@ import ( "net/http" "time" + common "github.com/neicnordic/sda-common/database" log "github.com/sirupsen/logrus" ) @@ -28,6 +29,15 @@ func main() { log.Panic(err) } + sdaDB, err := common.NewSDAdb(config.DB) + if err != nil { + log.Fatal(err) + } + + defer sdaDB.Close() + + log.Infof("Connected to sda-db (v%v)", sdaDB.Version) + err = checkS3Bucket(config.S3) if err != nil { log.Panic(err) @@ -50,7 +60,7 @@ func main() { log.Panicf("Error while getting key %s: %v", config.Server.jwtpubkeypath, err) } } - proxy := NewProxy(config.S3, auth, messenger, tlsProxy) + proxy := NewProxy(config.S3, auth, messenger, sdaDB, tlsProxy) log.Debug("got the proxy ", proxy) diff --git a/proxy.go b/proxy.go index b61f1ce..4f1179c 100644 --- a/proxy.go +++ b/proxy.go @@ -4,6 +4,7 @@ import ( "bytes" "crypto/sha256" "crypto/tls" + "encoding/json" "fmt" "io" "net/http" @@ -13,6 +14,8 @@ import ( "strings" "time" + common "github.com/neicnordic/sda-common/database" + "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/aws/credentials" @@ -28,6 +31,7 @@ type Proxy struct { s3 S3Config auth Authenticator messenger Messenger + database *common.SDAdb client *http.Client } @@ -49,11 +53,11 @@ const ( ) // NewProxy creates a new S3Proxy. This implements the ServerHTTP interface. -func NewProxy(s3conf S3Config, auth Authenticator, messenger Messenger, tls *tls.Config) *Proxy { +func NewProxy(s3conf S3Config, auth Authenticator, messenger Messenger, database *common.SDAdb, tls *tls.Config) *Proxy { tr := &http.Transport{TLSClientConfig: tls} client := &http.Client{Transport: tr} - return &Proxy{s3conf, auth, messenger, client} + return &Proxy{s3conf, auth, messenger, database, client} } func (p *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) { @@ -99,6 +103,15 @@ func (p *Proxy) allowedResponse(w http.ResponseWriter, r *http.Request) { log.Debug("prepend") p.prependBucketToHostPath(r) + // register file in database + username := fmt.Sprintf("%v", claims["sub"]) + filepath := strings.Replace(r.URL.Path, "/"+p.s3.bucket+"/", "", 1) + fileId, err := p.database.RegisterFile(filepath, username) + if err != nil { + log.Errorf("failed to register file in database: %v", err) + return + } + log.Debug("Forwarding to backend") s3response, err := p.forwardToBackend(r) @@ -110,13 +123,24 @@ func (p *Proxy) allowedResponse(w http.ResponseWriter, r *http.Request) { return } - // Send message to upstream + // Send message to upstream and set file as uploaded in the database if p.uploadFinishedSuccessfully(r, s3response) { log.Debug("create message") message, _ := p.CreateMessageFromRequest(r, claims) if err = p.messenger.SendMessage(message); err != nil { log.Debug("error when sending message") - log.Debug(err) + log.Error(err) + } + + jsonMessage, err := json.Marshal(message) + if err != nil { + log.Errorf("failed to marshal rabbitmq message to json: %v", err) + return + } + log.Debug("marking file as 'uploaded' in database") + err = p.database.MarkFileAsUploaded(fileId, username, string(jsonMessage)) + if err != nil { + log.Error(err) } } From 23d00ccdc0e6926e74078f363578cc2fe65acd1d Mon Sep 17 00:00:00 2001 From: Martin Norling Date: Wed, 23 Nov 2022 13:50:27 +0100 Subject: [PATCH 03/16] Add tests for database connection --- dev_utils/docker-compose.yml | 6 +-- proxy_test.go | 81 ++++++++++++++++++++++++++++++++++-- 2 files changed, 80 insertions(+), 7 deletions(-) diff --git a/dev_utils/docker-compose.yml b/dev_utils/docker-compose.yml index 7fa86f2..1918fe9 100644 --- a/dev_utils/docker-compose.yml +++ b/dev_utils/docker-compose.yml @@ -138,16 +138,16 @@ services: - 5432:5432 tests: - image: golang:${GOLANG_VERSION:-1.18} + image: golang:${GOLANG_VERSION:-1.19} container_name: s3proxy-tests profiles: - test command: - "/bin/sh" - "-c" - - "cd /app; echo 'Running go ${GOLANG_VERSION:-1.18} tests'; + - "cd /app; echo 'Running go ${GOLANG_VERSION:-1.19} tests'; go install 2>/dev/null - && go test . -v -coverprofile=coverage.txt -covermode=atomic" + && go test ./... -v -coverprofile=coverage.txt -covermode=atomic" depends_on: mq: condition: service_healthy diff --git a/proxy_test.go b/proxy_test.go index dfd696c..68cf86a 100644 --- a/proxy_test.go +++ b/proxy_test.go @@ -2,14 +2,19 @@ package main import ( "crypto/tls" + "database/sql" "encoding/json" "fmt" "net" "net/http" "net/http/httptest" "net/url" + "os" "testing" + common "github.com/neicnordic/sda-common/database" + log "github.com/sirupsen/logrus" + "github.com/golang-jwt/jwt/v4" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" @@ -18,8 +23,10 @@ import ( type ProxyTests struct { suite.Suite S3conf S3Config + DBConf common.DBConf fakeServer *FakeServer messenger *MockMessenger + database *common.SDAdb } func TestProxyTestSuite(t *testing.T) { @@ -42,10 +49,37 @@ func (suite *ProxyTests) SetupTest() { // Create a mock messenger suite.messenger = NewMockMessenger() + + // Create a database configuration for the fake database + suite.DBConf = common.DBConf{ + Host: "localhost", + Port: 5432, + User: "lega_in", + Password: "inpass", + Database: "lega", + CACert: "", + SslMode: "disable", + ClientCert: "", + ClientKey: "", + } + + var err error + + _, err = os.Stat("/.dockerenv") + if err == nil { + suite.DBConf.Host = "db" + } + + // Create a database, but don't care if it can't connect + suite.database, err = common.NewSDAdb(suite.DBConf) + if err != nil { + log.Infof("couldn't connect to database") + } } func (suite *ProxyTests) TearDownTest() { suite.fakeServer.Close() + suite.database.Close() } type FakeServer struct { @@ -121,7 +155,7 @@ func (u *AlwaysDeny) Authenticate(r *http.Request) (jwt.MapClaims, error) { func (suite *ProxyTests) TestServeHTTP_disallowed() { // Start mock messenger that denies everything - proxy := NewProxy(suite.S3conf, &AlwaysDeny{}, suite.messenger, new(tls.Config)) + proxy := NewProxy(suite.S3conf, &AlwaysDeny{}, suite.messenger, suite.database, new(tls.Config)) r, _ := http.NewRequest("", "", nil) w := httptest.NewRecorder() @@ -196,7 +230,7 @@ func (suite *ProxyTests) TestServeHTTPS3Unresponsive() { bucket: "buckbuck", region: "us-east-1", } - proxy := NewProxy(s3conf, &AlwaysAllow{}, suite.messenger, new(tls.Config)) + proxy := NewProxy(s3conf, &AlwaysAllow{}, suite.messenger, suite.database, new(tls.Config)) r, _ := http.NewRequest("", "", nil) w := httptest.NewRecorder() @@ -213,7 +247,7 @@ func (suite *ProxyTests) TestServeHTTPS3Unresponsive() { func (suite *ProxyTests) TestServeHTTP_allowed() { // Start proxy that allows everything - proxy := NewProxy(suite.S3conf, NewAlwaysAllow(), suite.messenger, new(tls.Config)) + proxy := NewProxy(suite.S3conf, NewAlwaysAllow(), suite.messenger, suite.database, new(tls.Config)) // List files works r, _ := http.NewRequest("GET", "/username/file", nil) @@ -311,7 +345,7 @@ func (suite *ProxyTests) TestMessageFormatting() { claims["sub"] = "user@host.domain" // start proxy that denies everything - proxy := NewProxy(suite.S3conf, &AlwaysDeny{}, suite.messenger, new(tls.Config)) + proxy := NewProxy(suite.S3conf, &AlwaysDeny{}, suite.messenger, suite.database, new(tls.Config)) suite.fakeServer.resp = "test/user/new_file.txt12false/user/new_file.txt2020-03-10T13:20:15.000Z"0a44282bd39178db9680f24813c41aec-1"1234STANDARD" msg, err := proxy.CreateMessageFromRequest(r, claims) assert.Nil(suite.T(), err) @@ -334,3 +368,42 @@ func (suite *ProxyTests) TestMessageFormatting() { assert.IsType(suite.T(), Event{}, msg) assert.Equal(suite.T(), "upload", msg.Operation) } + +func (suite *ProxyTests) TestDatabaseConnection() { + + // Start proxy that allows everything + proxy := NewProxy(suite.S3conf, NewAlwaysAllow(), suite.messenger, suite.database, new(tls.Config)) + + // PUT a file into the system + filename := "/username/db-test-file" + r, _ := http.NewRequest("PUT", filename, nil) + w := httptest.NewRecorder() + suite.fakeServer.resp = "test/elixirid/db-test-file.txt12false/elixirid/file.txt2020-03-10T13:20:15.000Z"0a44282bd39178db9680f24813c41aec-1"5STANDARD" + proxy.ServeHTTP(w, r) + assert.Equal(suite.T(), 200, w.Result().StatusCode) + assert.Equal(suite.T(), true, suite.fakeServer.PingedAndRestore()) + assert.Equal(suite.T(), true, suite.messenger.CheckAndRestore()) + assert.Equal(suite.T(), false, suite.messenger.CheckAndRestore()) + + // Check that the file is registered and uploaded in the database + // connect to the database + db, err := sql.Open(suite.DBConf.PgDataSource()) + assert.Nil(suite.T(), err, "Failed to connect to database") + + // Check that the file is in the database + var fileId string + query := "SELECT id FROM sda.files WHERE submission_file_path = $1" + err = db.QueryRow(query, filename[1:]).Scan(&fileId) + assert.Nil(suite.T(), err, "Failed to query database") + assert.NotNil(suite.T(), fileId, "File not found in database") + + // Check that the "registered" status is in the database for this file + for _, status := range []string{"registered", "uploaded"} { + var exists int + query = "SELECT 1 FROM sda.file_event_log WHERE event = $1 AND file_id = $2" + err = db.QueryRow(query, status, fileId).Scan(&exists) + assert.Nil(suite.T(), err, "Failed to find '%v' event in database", status) + assert.Equal(suite.T(), exists, 1, "File '%v' event does not exist", status) + } + +} From 5ba51acc766a23aa5086f9704efdb58b5ce4cd72 Mon Sep 17 00:00:00 2001 From: Martin Norling Date: Wed, 23 Nov 2022 13:54:05 +0100 Subject: [PATCH 04/16] Update sda-common version --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 77d9c32..b8c9693 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/johannesboyne/gofakes3 v0.0.0-20220627085814-c3ac35da23b2 github.com/lestrrat/go-jwx v0.0.0-20180221005942-b7d4802280ae github.com/minio/minio-go/v6 v6.0.43 - github.com/neicnordic/sda-common v0.0.3-0.20221114114925-a020b3bce09d + github.com/neicnordic/sda-common v0.0.3-0.20221122104056-37b54aea80a7 github.com/pkg/errors v0.9.1 github.com/sirupsen/logrus v1.9.0 github.com/spf13/viper v1.14.0 diff --git a/go.sum b/go.sum index 46454fc..a3d5168 100644 --- a/go.sum +++ b/go.sum @@ -215,6 +215,8 @@ github.com/neicnordic/sda-common v0.0.3-0.20221107140128-a5c86e921aa1 h1:K/kwvbH github.com/neicnordic/sda-common v0.0.3-0.20221107140128-a5c86e921aa1/go.mod h1:idzRZoISiXSkdVDrRPmNtF89kxFZtGz7p8QUQBGd7ZA= github.com/neicnordic/sda-common v0.0.3-0.20221114114925-a020b3bce09d h1:ADI6ETvCJluWP9N9HHB4f0MX7Qfi9ZEDBlHWIv8cznM= github.com/neicnordic/sda-common v0.0.3-0.20221114114925-a020b3bce09d/go.mod h1:idzRZoISiXSkdVDrRPmNtF89kxFZtGz7p8QUQBGd7ZA= +github.com/neicnordic/sda-common v0.0.3-0.20221122104056-37b54aea80a7 h1:P+Qs2se4Y+I4+KZY6BavumsU6cMZ4ar2KiuvrswzToQ= +github.com/neicnordic/sda-common v0.0.3-0.20221122104056-37b54aea80a7/go.mod h1:0+D7zVXKh15bkLgir4EmtSwzoo5fs70Ko9nYR8d18QE= github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml/v2 v2.0.5 h1:ipoSadvV8oGUjnUbMub59IDPPwfxF694nG/jwbMiyQg= From 19deefab507d582681816224cfcdb062d04403e7 Mon Sep 17 00:00:00 2001 From: Martin Norling Date: Wed, 30 Nov 2022 09:47:03 +0100 Subject: [PATCH 05/16] update tests to golang 1.18 and 1.19 --- .github/workflows/functionality.yml | 2 +- .github/workflows/gotest.yml | 2 +- dev_utils/docker-compose.yml | 4 ++++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/functionality.yml b/.github/workflows/functionality.yml index 6cd7283..ec35582 100644 --- a/.github/workflows/functionality.yml +++ b/.github/workflows/functionality.yml @@ -7,7 +7,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - go-version: [1.17, 1.18] + go-version: [1.18, 1.19] steps: - name: Set up Go ${{ matrix.go-version }} uses: actions/setup-go@v3 diff --git a/.github/workflows/gotest.yml b/.github/workflows/gotest.yml index 50324eb..a99ea34 100644 --- a/.github/workflows/gotest.yml +++ b/.github/workflows/gotest.yml @@ -7,7 +7,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - go-version: [1.17, 1.18] + go-version: [1.18, 1.19] steps: - name: Set up Go ${{ matrix.go-version }} uses: actions/setup-go@v3 diff --git a/dev_utils/docker-compose.yml b/dev_utils/docker-compose.yml index 1918fe9..5374fc6 100644 --- a/dev_utils/docker-compose.yml +++ b/dev_utils/docker-compose.yml @@ -153,6 +153,8 @@ services: condition: service_healthy s3: condition: service_healthy + database: + condition: service_healthy certfixer: condition: service_completed_successfully volumes: @@ -178,6 +180,8 @@ services: condition: service_started certfixer: condition: service_completed_successfully + createbucket: + condition: service_completed_successfully volumes: - proxy_certs:/certs - ./users.csv:/users.csv From 8d551a0987573a14e89ee877d4138f2054665916 Mon Sep 17 00:00:00 2001 From: Martin Norling Date: Wed, 30 Nov 2022 11:25:21 +0100 Subject: [PATCH 06/16] fix: only register files on Put requests --- dev_utils/docker-compose.yml | 4 ++++ proxy.go | 17 ++++++++++------- proxy_test.go | 1 - 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/dev_utils/docker-compose.yml b/dev_utils/docker-compose.yml index 5374fc6..459a049 100644 --- a/dev_utils/docker-compose.yml +++ b/dev_utils/docker-compose.yml @@ -79,8 +79,12 @@ services: condition: service_healthy s3: condition: service_healthy + database: + condition: service_healthy certfixer: condition: service_completed_successfully + createbucket: + condition: service_completed_successfully restart: always environment: - LOG_LEVEL=info diff --git a/proxy.go b/proxy.go index 4f1179c..869223e 100644 --- a/proxy.go +++ b/proxy.go @@ -103,13 +103,16 @@ func (p *Proxy) allowedResponse(w http.ResponseWriter, r *http.Request) { log.Debug("prepend") p.prependBucketToHostPath(r) + var fileId string // register file in database - username := fmt.Sprintf("%v", claims["sub"]) - filepath := strings.Replace(r.URL.Path, "/"+p.s3.bucket+"/", "", 1) - fileId, err := p.database.RegisterFile(filepath, username) - if err != nil { - log.Errorf("failed to register file in database: %v", err) - return + if p.detectRequestType(r) == Put { + username := fmt.Sprintf("%v", claims["sub"]) + filepath := strings.Replace(r.URL.Path, "/"+p.s3.bucket+"/", "", 1) + fileId, err = p.database.RegisterFile(filepath, username) + if err != nil { + log.Errorf("failed to register file in database: %v", err) + return + } } log.Debug("Forwarding to backend") @@ -138,7 +141,7 @@ func (p *Proxy) allowedResponse(w http.ResponseWriter, r *http.Request) { return } log.Debug("marking file as 'uploaded' in database") - err = p.database.MarkFileAsUploaded(fileId, username, string(jsonMessage)) + err = p.database.MarkFileAsUploaded(fileId, message.Username, string(jsonMessage)) if err != nil { log.Error(err) } diff --git a/proxy_test.go b/proxy_test.go index 68cf86a..5452707 100644 --- a/proxy_test.go +++ b/proxy_test.go @@ -70,7 +70,6 @@ func (suite *ProxyTests) SetupTest() { suite.DBConf.Host = "db" } - // Create a database, but don't care if it can't connect suite.database, err = common.NewSDAdb(suite.DBConf) if err != nil { log.Infof("couldn't connect to database") From 0035921e4e5eced8ccabb0ca4f4ae84769fbacfa Mon Sep 17 00:00:00 2001 From: Martin Norling Date: Thu, 1 Dec 2022 09:46:06 +0100 Subject: [PATCH 07/16] Update config.go Co-authored-by: Joakim Bygdell --- config.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/config.go b/config.go index 93621f5..c687c24 100644 --- a/config.go +++ b/config.go @@ -204,10 +204,18 @@ func (c *Config) readConfig() error { c.DB.User = viper.GetString("db.user") c.DB.Password = viper.GetString("db.password") c.DB.Database = viper.GetString("db.database") - c.DB.CACert = viper.GetString("db.cacert") + if viper.IsSet("db.cacert") { + c.DB.CACert = viper.GetString("db.cacert") + } c.DB.SslMode = viper.GetString("db.sslmode") - c.DB.ClientCert = viper.GetString("db.clientcert") - c.DB.ClientKey = viper.GetString("db.clientkey") + if db.SslMode == "verify-full" { + // Since verify-full is specified, these are required. + if !(viper.IsSet("db.clientCert") && viper.IsSet("db.clientKey")) { + return errors.New("when db.sslMode is set to verify-full both db.clientCert and db.clientKey are needed") + } + c.DB.ClientCert = viper.GetString("db.clientcert") + c.DB.ClientKey = viper.GetString("db.clientkey") + } // Setup server s := ServerConfig{} From 3dae5d736b2f9b5a53f3aac163e243c0ce26f6ff Mon Sep 17 00:00:00 2001 From: Martin Norling Date: Thu, 1 Dec 2022 09:47:21 +0100 Subject: [PATCH 08/16] Set db-connect message as debug level Co-authored-by: Joakim Bygdell --- main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.go b/main.go index 205400e..35e162a 100644 --- a/main.go +++ b/main.go @@ -36,7 +36,7 @@ func main() { defer sdaDB.Close() - log.Infof("Connected to sda-db (v%v)", sdaDB.Version) + log.Debugf("Connected to sda-db (v%v)", sdaDB.Version) err = checkS3Bucket(config.S3) if err != nil { From e2d073fa6322dd4065c44f0a86ae91f85e9846c8 Mon Sep 17 00:00:00 2001 From: Martin Norling Date: Thu, 1 Dec 2022 09:49:17 +0100 Subject: [PATCH 09/16] Apply suggestions from code review Co-authored-by: Joakim Bygdell --- .github/workflows/golint.yml | 2 +- config.go | 2 +- main.go | 2 +- proxy.go | 2 +- proxy_test.go | 30 ++++++++++++++++++------------ 5 files changed, 22 insertions(+), 16 deletions(-) diff --git a/.github/workflows/golint.yml b/.github/workflows/golint.yml index 886bb88..f684080 100644 --- a/.github/workflows/golint.yml +++ b/.github/workflows/golint.yml @@ -8,7 +8,7 @@ jobs: strategy: fail-fast: false matrix: - go-version: [1.17, 1.18] + go-version: [1.18, 1.19] steps: - name: Set up Go ${{ matrix.go-version }} uses: actions/setup-go@v3 diff --git a/config.go b/config.go index c687c24..f31a310 100644 --- a/config.go +++ b/config.go @@ -208,7 +208,7 @@ func (c *Config) readConfig() error { c.DB.CACert = viper.GetString("db.cacert") } c.DB.SslMode = viper.GetString("db.sslmode") - if db.SslMode == "verify-full" { + if c.DB.SslMode == "verify-full" { // Since verify-full is specified, these are required. if !(viper.IsSet("db.clientCert") && viper.IsSet("db.clientKey")) { return errors.New("when db.sslMode is set to verify-full both db.clientCert and db.clientKey are needed") diff --git a/main.go b/main.go index 35e162a..3c1d6d0 100644 --- a/main.go +++ b/main.go @@ -31,7 +31,7 @@ func main() { sdaDB, err := common.NewSDAdb(config.DB) if err != nil { - log.Fatal(err) + log.Panic(err) } defer sdaDB.Close() diff --git a/proxy.go b/proxy.go index 869223e..9279b89 100644 --- a/proxy.go +++ b/proxy.go @@ -140,7 +140,7 @@ func (p *Proxy) allowedResponse(w http.ResponseWriter, r *http.Request) { log.Errorf("failed to marshal rabbitmq message to json: %v", err) return } - log.Debug("marking file as 'uploaded' in database") + log.Debugf("marking file %v as 'uploaded' in database", fileId) err = p.database.MarkFileAsUploaded(fileId, message.Username, string(jsonMessage)) if err != nil { log.Error(err) diff --git a/proxy_test.go b/proxy_test.go index 5452707..fea16f4 100644 --- a/proxy_test.go +++ b/proxy_test.go @@ -13,7 +13,6 @@ import ( "testing" common "github.com/neicnordic/sda-common/database" - log "github.com/sirupsen/logrus" "github.com/golang-jwt/jwt/v4" "github.com/stretchr/testify/assert" @@ -70,10 +69,7 @@ func (suite *ProxyTests) SetupTest() { suite.DBConf.Host = "db" } - suite.database, err = common.NewSDAdb(suite.DBConf) - if err != nil { - log.Infof("couldn't connect to database") - } + suite.database = &common.SDAdb{} } func (suite *ProxyTests) TearDownTest() { @@ -246,7 +242,11 @@ func (suite *ProxyTests) TestServeHTTPS3Unresponsive() { func (suite *ProxyTests) TestServeHTTP_allowed() { // Start proxy that allows everything - proxy := NewProxy(suite.S3conf, NewAlwaysAllow(), suite.messenger, suite.database, new(tls.Config)) + database, err := common.NewSDAdb(suite.DBConf) + if err != nil { + suite.T().Skip("skip TestShutdown since broker not present") + } + proxy := NewProxy(suite.S3conf, NewAlwaysAllow(), suite.messenger, database, new(tls.Config)) // List files works r, _ := http.NewRequest("GET", "/username/file", nil) @@ -369,9 +369,13 @@ func (suite *ProxyTests) TestMessageFormatting() { } func (suite *ProxyTests) TestDatabaseConnection() { + database, err := common.NewSDAdb(suite.DBConf) + if err != nil { + suite.T().Skip("skip TestShutdown since broker not present") + } // Start proxy that allows everything - proxy := NewProxy(suite.S3conf, NewAlwaysAllow(), suite.messenger, suite.database, new(tls.Config)) + proxy := NewProxy(suite.S3conf, NewAlwaysAllow(), suite.messenger, database, new(tls.Config)) // PUT a file into the system filename := "/username/db-test-file" @@ -379,7 +383,9 @@ func (suite *ProxyTests) TestDatabaseConnection() { w := httptest.NewRecorder() suite.fakeServer.resp = "test/elixirid/db-test-file.txt12false/elixirid/file.txt2020-03-10T13:20:15.000Z"0a44282bd39178db9680f24813c41aec-1"5STANDARD" proxy.ServeHTTP(w, r) - assert.Equal(suite.T(), 200, w.Result().StatusCode) + res := w.Result() + defer res.Body.Close() + assert.Equal(suite.T(), 200, res.StatusCode) assert.Equal(suite.T(), true, suite.fakeServer.PingedAndRestore()) assert.Equal(suite.T(), true, suite.messenger.CheckAndRestore()) assert.Equal(suite.T(), false, suite.messenger.CheckAndRestore()) @@ -390,17 +396,17 @@ func (suite *ProxyTests) TestDatabaseConnection() { assert.Nil(suite.T(), err, "Failed to connect to database") // Check that the file is in the database - var fileId string + var fileID string query := "SELECT id FROM sda.files WHERE submission_file_path = $1" - err = db.QueryRow(query, filename[1:]).Scan(&fileId) + err = db.QueryRow(query, filename[1:]).Scan(&fileID) assert.Nil(suite.T(), err, "Failed to query database") - assert.NotNil(suite.T(), fileId, "File not found in database") + assert.NotNil(suite.T(), fileID, "File not found in database") // Check that the "registered" status is in the database for this file for _, status := range []string{"registered", "uploaded"} { var exists int query = "SELECT 1 FROM sda.file_event_log WHERE event = $1 AND file_id = $2" - err = db.QueryRow(query, status, fileId).Scan(&exists) + err = db.QueryRow(query, status, fileID).Scan(&exists) assert.Nil(suite.T(), err, "Failed to find '%v' event in database", status) assert.Equal(suite.T(), exists, 1, "File '%v' event does not exist", status) } From 4c6e59530ad60348e3360381e4841fdd46df44c6 Mon Sep 17 00:00:00 2001 From: Martin Norling Date: Thu, 1 Dec 2022 12:30:25 +0100 Subject: [PATCH 10/16] fix: Make multipart files log correctly in db --- proxy.go | 39 +++++++++++++++++++++++++++++---------- 1 file changed, 29 insertions(+), 10 deletions(-) diff --git a/proxy.go b/proxy.go index 9279b89..5096379 100644 --- a/proxy.go +++ b/proxy.go @@ -33,6 +33,7 @@ type Proxy struct { messenger Messenger database *common.SDAdb client *http.Client + fileIds map[string]string } // S3RequestType is the type of request that we are currently proxying to the @@ -57,7 +58,7 @@ func NewProxy(s3conf S3Config, auth Authenticator, messenger Messenger, database tr := &http.Transport{TLSClientConfig: tls} client := &http.Client{Transport: tr} - return &Proxy{s3conf, auth, messenger, database, client} + return &Proxy{s3conf, auth, messenger, database, client, make(map[string]string)} } func (p *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) { @@ -103,15 +104,31 @@ func (p *Proxy) allowedResponse(w http.ResponseWriter, r *http.Request) { log.Debug("prepend") p.prependBucketToHostPath(r) - var fileId string - // register file in database + username := fmt.Sprintf("%v", claims["sub"]) + filepath := strings.Replace(r.URL.Path, "/"+p.s3.bucket+"/", "", 1) + // register file in database if it's the start of an upload if p.detectRequestType(r) == Put { - username := fmt.Sprintf("%v", claims["sub"]) - filepath := strings.Replace(r.URL.Path, "/"+p.s3.bucket+"/", "", 1) - fileId, err = p.database.RegisterFile(filepath, username) - if err != nil { - log.Errorf("failed to register file in database: %v", err) - return + // Check if this is part of a multipart upload + partNumberParam := r.URL.Query().Get("partNumber") + partNumber := 1 + if partNumberParam != "" { + partNumber, err = strconv.Atoi(partNumberParam) + if err != nil { + log.Errorf("failure parsing part number from %v", partNumberParam) + return + } + } + + // if it's the first part, or not multipart, register file, and add it + // to the fileId store + if partNumberParam == "" || partNumber == 1 { + log.Debugf("registering file %v in the database", filepath) + p.fileIds[filepath], err = p.database.RegisterFile(filepath, username) + log.Debugf("fileId: %v", p.fileIds[filepath]) + if err != nil { + log.Errorf("failed to register file in database: %v", err) + return + } } } @@ -140,8 +157,10 @@ func (p *Proxy) allowedResponse(w http.ResponseWriter, r *http.Request) { log.Errorf("failed to marshal rabbitmq message to json: %v", err) return } + fileId := p.fileIds[filepath] + delete(p.fileIds, filepath) log.Debugf("marking file %v as 'uploaded' in database", fileId) - err = p.database.MarkFileAsUploaded(fileId, message.Username, string(jsonMessage)) + err = p.database.MarkFileAsUploaded(fileId, username, string(jsonMessage)) if err != nil { log.Error(err) } From a933040ab6b4ec3e5ae119ee94018f3c590b5461 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Thu, 1 Dec 2022 10:19:28 +0100 Subject: [PATCH 11/16] Use latest version of the golang-ci-linter --- .github/workflows/golint.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/golint.yml b/.github/workflows/golint.yml index f684080..26896da 100644 --- a/.github/workflows/golint.yml +++ b/.github/workflows/golint.yml @@ -18,5 +18,4 @@ jobs: - name: Run golangci-lint uses: golangci/golangci-lint-action@v3.3.1 with: - version: v1.45 args: -E gosec,nestif,bodyclose,rowserrcheck -e G107 From bee98f1d772084620ee326b923059275628440ac Mon Sep 17 00:00:00 2001 From: Martin Norling Date: Thu, 1 Dec 2022 13:10:29 +0100 Subject: [PATCH 12/16] Cleanup code --- proxy.go | 33 +++++++++------------------------ 1 file changed, 9 insertions(+), 24 deletions(-) diff --git a/proxy.go b/proxy.go index 5096379..7feaf77 100644 --- a/proxy.go +++ b/proxy.go @@ -107,28 +107,13 @@ func (p *Proxy) allowedResponse(w http.ResponseWriter, r *http.Request) { username := fmt.Sprintf("%v", claims["sub"]) filepath := strings.Replace(r.URL.Path, "/"+p.s3.bucket+"/", "", 1) // register file in database if it's the start of an upload - if p.detectRequestType(r) == Put { - // Check if this is part of a multipart upload - partNumberParam := r.URL.Query().Get("partNumber") - partNumber := 1 - if partNumberParam != "" { - partNumber, err = strconv.Atoi(partNumberParam) - if err != nil { - log.Errorf("failure parsing part number from %v", partNumberParam) - return - } - } - - // if it's the first part, or not multipart, register file, and add it - // to the fileId store - if partNumberParam == "" || partNumber == 1 { - log.Debugf("registering file %v in the database", filepath) - p.fileIds[filepath], err = p.database.RegisterFile(filepath, username) - log.Debugf("fileId: %v", p.fileIds[filepath]) - if err != nil { - log.Errorf("failed to register file in database: %v", err) - return - } + if p.detectRequestType(r) == Put && p.fileIds[r.URL.Path] == "" { + log.Debugf("registering file %v in the database", r.URL.Path) + p.fileIds[r.URL.Path], err = p.database.RegisterFile(filepath, username) + log.Debugf("fileId: %v", p.fileIds[r.URL.Path]) + if err != nil { + log.Errorf("failed to register file in database: %v", err) + return } } @@ -157,8 +142,8 @@ func (p *Proxy) allowedResponse(w http.ResponseWriter, r *http.Request) { log.Errorf("failed to marshal rabbitmq message to json: %v", err) return } - fileId := p.fileIds[filepath] - delete(p.fileIds, filepath) + fileId := p.fileIds[r.URL.Path] + delete(p.fileIds, r.URL.Path) log.Debugf("marking file %v as 'uploaded' in database", fileId) err = p.database.MarkFileAsUploaded(fileId, username, string(jsonMessage)) if err != nil { From be16ab0a34c7e43126a84341995c8c048e8ab8d4 Mon Sep 17 00:00:00 2001 From: Martin Norling Date: Thu, 1 Dec 2022 12:31:13 +0100 Subject: [PATCH 13/16] Update dev_utils/docker-compose.yml Co-authored-by: Joakim Bygdell --- dev_utils/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev_utils/docker-compose.yml b/dev_utils/docker-compose.yml index 459a049..b5ef33e 100644 --- a/dev_utils/docker-compose.yml +++ b/dev_utils/docker-compose.yml @@ -139,7 +139,7 @@ services: volumes: - psqldata:/var/lib/postgresql ports: - - 5432:5432 + - 2345:5432 tests: image: golang:${GOLANG_VERSION:-1.19} From f0dc2f49a1890bc2bff83202db65444aeafa26ee Mon Sep 17 00:00:00 2001 From: Martin Norling Date: Mon, 5 Dec 2022 08:59:21 +0100 Subject: [PATCH 14/16] linter fixes Co-authored-by: Alex Aperis <76202622+aaperis@users.noreply.github.com> --- .github/workflows/golint.yml | 2 +- proxy.go | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/golint.yml b/.github/workflows/golint.yml index 26896da..774281b 100644 --- a/.github/workflows/golint.yml +++ b/.github/workflows/golint.yml @@ -18,4 +18,4 @@ jobs: - name: Run golangci-lint uses: golangci/golangci-lint-action@v3.3.1 with: - args: -E gosec,nestif,bodyclose,rowserrcheck -e G107 + args: -E bodyclose,gocritic,gofmt,gosec,govet,nestif,nlreturn,revive -e G107 diff --git a/proxy.go b/proxy.go index 7feaf77..25ac4e5 100644 --- a/proxy.go +++ b/proxy.go @@ -113,6 +113,7 @@ func (p *Proxy) allowedResponse(w http.ResponseWriter, r *http.Request) { log.Debugf("fileId: %v", p.fileIds[r.URL.Path]) if err != nil { log.Errorf("failed to register file in database: %v", err) + return } } @@ -140,12 +141,13 @@ func (p *Proxy) allowedResponse(w http.ResponseWriter, r *http.Request) { jsonMessage, err := json.Marshal(message) if err != nil { log.Errorf("failed to marshal rabbitmq message to json: %v", err) + return } - fileId := p.fileIds[r.URL.Path] + fileID := p.fileIds[r.URL.Path] delete(p.fileIds, r.URL.Path) - log.Debugf("marking file %v as 'uploaded' in database", fileId) - err = p.database.MarkFileAsUploaded(fileId, username, string(jsonMessage)) + log.Debugf("marking file %v as 'uploaded' in database", fileID) + err = p.database.MarkFileAsUploaded(fileID, username, string(jsonMessage)) if err != nil { log.Error(err) } From 453a141bc56751d03384c7dd6b316bf15f96b6db Mon Sep 17 00:00:00 2001 From: Martin Norling Date: Wed, 7 Dec 2022 18:49:57 +0100 Subject: [PATCH 15/16] Update proxy_test.go Co-authored-by: Dimitris --- proxy_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proxy_test.go b/proxy_test.go index fea16f4..728021f 100644 --- a/proxy_test.go +++ b/proxy_test.go @@ -54,7 +54,7 @@ func (suite *ProxyTests) SetupTest() { Host: "localhost", Port: 5432, User: "lega_in", - Password: "inpass", + Password: "lega_in", Database: "lega", CACert: "", SslMode: "disable", From 40100d615f7825a3d5ca849d4d32efa554426e25 Mon Sep 17 00:00:00 2001 From: Martin Norling Date: Thu, 8 Dec 2022 08:41:48 +0100 Subject: [PATCH 16/16] standardize dev db passwords Co-authored-by: Dimitris --- dev_utils/docker-compose.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dev_utils/docker-compose.yml b/dev_utils/docker-compose.yml index b5ef33e..95ed3ed 100644 --- a/dev_utils/docker-compose.yml +++ b/dev_utils/docker-compose.yml @@ -98,7 +98,7 @@ services: - DB_HOST=db - DB_PORT=5432 - DB_USER=lega_in - - DB_PASSWORD=inpass + - DB_PASSWORD=lega_in - DB_DATABASE=lega - DB_CACERT= - DB_SSLMODE=disable @@ -132,8 +132,8 @@ services: container_name: db image: neicnordic/sda-db:v2.0.0 environment: - - DB_LEGA_IN_PASSWORD=inpass - - DB_LEGA_OUT_PASSWORD=outpass + - DB_LEGA_IN_PASSWORD=lega_in + - DB_LEGA_OUT_PASSWORD=lega_out - PGVOLUME=/var/lib/postgresql - NOTLS=true volumes: