diff --git a/.github/workflows/functionality.yml b/.github/workflows/functionality.yml index 6cd7283..ec35582 100644 --- a/.github/workflows/functionality.yml +++ b/.github/workflows/functionality.yml @@ -7,7 +7,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - go-version: [1.17, 1.18] + go-version: [1.18, 1.19] steps: - name: Set up Go ${{ matrix.go-version }} uses: actions/setup-go@v3 diff --git a/.github/workflows/golint.yml b/.github/workflows/golint.yml index 886bb88..774281b 100644 --- a/.github/workflows/golint.yml +++ b/.github/workflows/golint.yml @@ -8,7 +8,7 @@ jobs: strategy: fail-fast: false matrix: - go-version: [1.17, 1.18] + go-version: [1.18, 1.19] steps: - name: Set up Go ${{ matrix.go-version }} uses: actions/setup-go@v3 @@ -18,5 +18,4 @@ jobs: - name: Run golangci-lint uses: golangci/golangci-lint-action@v3.3.1 with: - version: v1.45 - args: -E gosec,nestif,bodyclose,rowserrcheck -e G107 + args: -E bodyclose,gocritic,gofmt,gosec,govet,nestif,nlreturn,revive -e G107 diff --git a/.github/workflows/gotest.yml b/.github/workflows/gotest.yml index 50324eb..a99ea34 100644 --- a/.github/workflows/gotest.yml +++ b/.github/workflows/gotest.yml @@ -7,7 +7,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - go-version: [1.17, 1.18] + go-version: [1.18, 1.19] steps: - name: Set up Go ${{ matrix.go-version }} uses: actions/setup-go@v3 diff --git a/config.go b/config.go index 1ef7f34..f31a310 100644 --- a/config.go +++ b/config.go @@ -9,6 +9,7 @@ import ( "reflect" "strings" + "github.com/neicnordic/sda-common/database" "github.com/pkg/errors" log "github.com/sirupsen/logrus" "github.com/spf13/viper" @@ -62,6 +63,7 @@ type Config struct { S3 S3Config Broker BrokerConfig Server ServerConfig + DB database.DBConf } // NewConfig initializes and parses the config file and/or environment using @@ -196,6 +198,25 @@ func (c *Config) readConfig() error { c.Broker = b + // Setup psql db + c.DB.Host = viper.GetString("db.host") + c.DB.Port = viper.GetInt("db.port") + c.DB.User = viper.GetString("db.user") + c.DB.Password = viper.GetString("db.password") + c.DB.Database = viper.GetString("db.database") + if viper.IsSet("db.cacert") { + c.DB.CACert = viper.GetString("db.cacert") + } + c.DB.SslMode = viper.GetString("db.sslmode") + if c.DB.SslMode == "verify-full" { + // Since verify-full is specified, these are required. + if !(viper.IsSet("db.clientCert") && viper.IsSet("db.clientKey")) { + return errors.New("when db.sslMode is set to verify-full both db.clientCert and db.clientKey are needed") + } + c.DB.ClientCert = viper.GetString("db.clientcert") + c.DB.ClientKey = viper.GetString("db.clientkey") + } + // Setup server s := ServerConfig{} diff --git a/dev_utils/docker-compose.yml b/dev_utils/docker-compose.yml index ec32320..95ed3ed 100644 --- a/dev_utils/docker-compose.yml +++ b/dev_utils/docker-compose.yml @@ -79,8 +79,12 @@ services: condition: service_healthy s3: condition: service_healthy + database: + condition: service_healthy certfixer: condition: service_completed_successfully + createbucket: + condition: service_completed_successfully restart: always environment: - LOG_LEVEL=info @@ -91,6 +95,15 @@ services: - AWS_REGION=us-east-1 - AWS_READYPATH=/minio/health/ready - AWS_CACERT=/certs/ca.crt + - DB_HOST=db + - DB_PORT=5432 + - DB_USER=lega_in + - DB_PASSWORD=lega_in + - DB_DATABASE=lega + - DB_CACERT= + - DB_SSLMODE=disable + - DB_CLIENTCERT= + - DB_CLIENTKEY= - BROKER_HOST=mq - BROKER_USER=test - BROKER_PASSWORD=test @@ -115,22 +128,37 @@ services: - "8000:8000" - "8001:8001" + database: + container_name: db + image: neicnordic/sda-db:v2.0.0 + environment: + - DB_LEGA_IN_PASSWORD=lega_in + - DB_LEGA_OUT_PASSWORD=lega_out + - PGVOLUME=/var/lib/postgresql + - NOTLS=true + volumes: + - psqldata:/var/lib/postgresql + ports: + - 2345:5432 + tests: - image: golang:${GOLANG_VERSION:-1.18} + image: golang:${GOLANG_VERSION:-1.19} container_name: s3proxy-tests profiles: - test command: - "/bin/sh" - "-c" - - "cd /app; echo 'Running go ${GOLANG_VERSION:-1.18} tests'; + - "cd /app; echo 'Running go ${GOLANG_VERSION:-1.19} tests'; go install 2>/dev/null - && go test . -v -coverprofile=coverage.txt -covermode=atomic" + && go test ./... -v -coverprofile=coverage.txt -covermode=atomic" depends_on: mq: condition: service_healthy s3: condition: service_healthy + database: + condition: service_healthy certfixer: condition: service_completed_successfully volumes: @@ -156,6 +184,8 @@ services: condition: service_started certfixer: condition: service_completed_successfully + createbucket: + condition: service_completed_successfully volumes: - proxy_certs:/certs - ./users.csv:/users.csv @@ -166,6 +196,7 @@ volumes: s3_certs: mq_certs: proxy_certs: + psqldata: data: # These settings only work on linux (including WSL2), and can be used to # test when the disk is full. diff --git a/go.mod b/go.mod index a76f76b..b8c9693 100644 --- a/go.mod +++ b/go.mod @@ -6,10 +6,11 @@ require ( github.com/aws/aws-sdk-go v1.44.153 github.com/golang-jwt/jwt/v4 v4.4.3 github.com/google/uuid v1.3.0 - github.com/heptiolabs/healthcheck v0.0.0-20180807145615-6ff867650f40 - github.com/johannesboyne/gofakes3 v0.0.0-20210608054100-92d5d4af5fde + github.com/heptiolabs/healthcheck v0.0.0-20211123025425-613501dd5deb + github.com/johannesboyne/gofakes3 v0.0.0-20220627085814-c3ac35da23b2 github.com/lestrrat/go-jwx v0.0.0-20180221005942-b7d4802280ae github.com/minio/minio-go/v6 v6.0.43 + github.com/neicnordic/sda-common v0.0.3-0.20221122104056-37b54aea80a7 github.com/pkg/errors v0.9.1 github.com/sirupsen/logrus v1.9.0 github.com/spf13/viper v1.14.0 @@ -26,6 +27,7 @@ require ( github.com/hashicorp/hcl v1.0.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/lestrrat/go-pdebug v0.0.0-20180220043741-569c97477ae8 // indirect + github.com/lib/pq v1.10.7 // indirect github.com/magiconair/properties v1.8.6 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect github.com/minio/sha256-simd v0.1.1 // indirect diff --git a/go.sum b/go.sum index 13b13c9..a3d5168 100644 --- a/go.sum +++ b/go.sum @@ -156,8 +156,8 @@ github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/heptiolabs/healthcheck v0.0.0-20180807145615-6ff867650f40 h1:GT4RsKmHh1uZyhmTkWJTDALRjSHYQp6FRKrotf0zhAs= -github.com/heptiolabs/healthcheck v0.0.0-20180807145615-6ff867650f40/go.mod h1:NtmN9h8vrTveVQRLHcX2HQ5wIPBDCsZ351TGbZWgg38= +github.com/heptiolabs/healthcheck v0.0.0-20211123025425-613501dd5deb h1:tsEKRC3PU9rMw18w/uAptoijhgG4EvlA5kfJPtwrMDk= +github.com/heptiolabs/healthcheck v0.0.0-20211123025425-613501dd5deb/go.mod h1:NtmN9h8vrTveVQRLHcX2HQ5wIPBDCsZ351TGbZWgg38= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= @@ -165,8 +165,8 @@ github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9Y github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= -github.com/johannesboyne/gofakes3 v0.0.0-20210608054100-92d5d4af5fde h1:ekNURlaug3SgiS0KQzL/5oiYPUJPozt1C+ajLBWk7/E= -github.com/johannesboyne/gofakes3 v0.0.0-20210608054100-92d5d4af5fde/go.mod h1:LIAXxPvcUXwOcTIj9LSNSUpE9/eMHalTWxsP/kmWxQI= +github.com/johannesboyne/gofakes3 v0.0.0-20220627085814-c3ac35da23b2 h1:V5q1Mx2WTE5coXLG2QpkRZ7LsJvgkedm6Ib4AwC1Lfg= +github.com/johannesboyne/gofakes3 v0.0.0-20220627085814-c3ac35da23b2/go.mod h1:LIAXxPvcUXwOcTIj9LSNSUpE9/eMHalTWxsP/kmWxQI= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -191,6 +191,8 @@ github.com/lestrrat/go-jwx v0.0.0-20180221005942-b7d4802280ae h1:XoMPFIGibcPKgLr github.com/lestrrat/go-jwx v0.0.0-20180221005942-b7d4802280ae/go.mod h1:T+yHdCP6MJKtzoVQMHvVCeam5VFwX1+rWzn5zZgKYMI= github.com/lestrrat/go-pdebug v0.0.0-20180220043741-569c97477ae8 h1:ttJD8hTqvrPEUBoAG5hJKbDOJ84u7zmbnZsUL4V9430= github.com/lestrrat/go-pdebug v0.0.0-20180220043741-569c97477ae8/go.mod h1:VXFH11P7fHn2iPBsfSW1JacR59rttTcafJnwYcI/IdY= +github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw= +github.com/lib/pq v1.10.7/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo= github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= @@ -209,6 +211,12 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/neicnordic/sda-common v0.0.3-0.20221107140128-a5c86e921aa1 h1:K/kwvbH1QFwoICMAbeQi9+FMVpGsmKYCz1lETZWqkrY= +github.com/neicnordic/sda-common v0.0.3-0.20221107140128-a5c86e921aa1/go.mod h1:idzRZoISiXSkdVDrRPmNtF89kxFZtGz7p8QUQBGd7ZA= +github.com/neicnordic/sda-common v0.0.3-0.20221114114925-a020b3bce09d h1:ADI6ETvCJluWP9N9HHB4f0MX7Qfi9ZEDBlHWIv8cznM= +github.com/neicnordic/sda-common v0.0.3-0.20221114114925-a020b3bce09d/go.mod h1:idzRZoISiXSkdVDrRPmNtF89kxFZtGz7p8QUQBGd7ZA= +github.com/neicnordic/sda-common v0.0.3-0.20221122104056-37b54aea80a7 h1:P+Qs2se4Y+I4+KZY6BavumsU6cMZ4ar2KiuvrswzToQ= +github.com/neicnordic/sda-common v0.0.3-0.20221122104056-37b54aea80a7/go.mod h1:0+D7zVXKh15bkLgir4EmtSwzoo5fs70Ko9nYR8d18QE= github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml/v2 v2.0.5 h1:ipoSadvV8oGUjnUbMub59IDPPwfxF694nG/jwbMiyQg= diff --git a/main.go b/main.go index 9e25f66..3c1d6d0 100644 --- a/main.go +++ b/main.go @@ -4,6 +4,7 @@ import ( "net/http" "time" + common "github.com/neicnordic/sda-common/database" log "github.com/sirupsen/logrus" ) @@ -28,6 +29,15 @@ func main() { log.Panic(err) } + sdaDB, err := common.NewSDAdb(config.DB) + if err != nil { + log.Panic(err) + } + + defer sdaDB.Close() + + log.Debugf("Connected to sda-db (v%v)", sdaDB.Version) + err = checkS3Bucket(config.S3) if err != nil { log.Panic(err) @@ -50,7 +60,7 @@ func main() { log.Panicf("Error while getting key %s: %v", config.Server.jwtpubkeypath, err) } } - proxy := NewProxy(config.S3, auth, messenger, tlsProxy) + proxy := NewProxy(config.S3, auth, messenger, sdaDB, tlsProxy) log.Debug("got the proxy ", proxy) diff --git a/proxy.go b/proxy.go index b61f1ce..25ac4e5 100644 --- a/proxy.go +++ b/proxy.go @@ -4,6 +4,7 @@ import ( "bytes" "crypto/sha256" "crypto/tls" + "encoding/json" "fmt" "io" "net/http" @@ -13,6 +14,8 @@ import ( "strings" "time" + common "github.com/neicnordic/sda-common/database" + "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/aws/credentials" @@ -28,7 +31,9 @@ type Proxy struct { s3 S3Config auth Authenticator messenger Messenger + database *common.SDAdb client *http.Client + fileIds map[string]string } // S3RequestType is the type of request that we are currently proxying to the @@ -49,11 +54,11 @@ const ( ) // NewProxy creates a new S3Proxy. This implements the ServerHTTP interface. -func NewProxy(s3conf S3Config, auth Authenticator, messenger Messenger, tls *tls.Config) *Proxy { +func NewProxy(s3conf S3Config, auth Authenticator, messenger Messenger, database *common.SDAdb, tls *tls.Config) *Proxy { tr := &http.Transport{TLSClientConfig: tls} client := &http.Client{Transport: tr} - return &Proxy{s3conf, auth, messenger, client} + return &Proxy{s3conf, auth, messenger, database, client, make(map[string]string)} } func (p *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) { @@ -99,6 +104,20 @@ func (p *Proxy) allowedResponse(w http.ResponseWriter, r *http.Request) { log.Debug("prepend") p.prependBucketToHostPath(r) + username := fmt.Sprintf("%v", claims["sub"]) + filepath := strings.Replace(r.URL.Path, "/"+p.s3.bucket+"/", "", 1) + // register file in database if it's the start of an upload + if p.detectRequestType(r) == Put && p.fileIds[r.URL.Path] == "" { + log.Debugf("registering file %v in the database", r.URL.Path) + p.fileIds[r.URL.Path], err = p.database.RegisterFile(filepath, username) + log.Debugf("fileId: %v", p.fileIds[r.URL.Path]) + if err != nil { + log.Errorf("failed to register file in database: %v", err) + + return + } + } + log.Debug("Forwarding to backend") s3response, err := p.forwardToBackend(r) @@ -110,13 +129,27 @@ func (p *Proxy) allowedResponse(w http.ResponseWriter, r *http.Request) { return } - // Send message to upstream + // Send message to upstream and set file as uploaded in the database if p.uploadFinishedSuccessfully(r, s3response) { log.Debug("create message") message, _ := p.CreateMessageFromRequest(r, claims) if err = p.messenger.SendMessage(message); err != nil { log.Debug("error when sending message") - log.Debug(err) + log.Error(err) + } + + jsonMessage, err := json.Marshal(message) + if err != nil { + log.Errorf("failed to marshal rabbitmq message to json: %v", err) + + return + } + fileID := p.fileIds[r.URL.Path] + delete(p.fileIds, r.URL.Path) + log.Debugf("marking file %v as 'uploaded' in database", fileID) + err = p.database.MarkFileAsUploaded(fileID, username, string(jsonMessage)) + if err != nil { + log.Error(err) } } diff --git a/proxy_test.go b/proxy_test.go index dfd696c..728021f 100644 --- a/proxy_test.go +++ b/proxy_test.go @@ -2,14 +2,18 @@ package main import ( "crypto/tls" + "database/sql" "encoding/json" "fmt" "net" "net/http" "net/http/httptest" "net/url" + "os" "testing" + common "github.com/neicnordic/sda-common/database" + "github.com/golang-jwt/jwt/v4" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" @@ -18,8 +22,10 @@ import ( type ProxyTests struct { suite.Suite S3conf S3Config + DBConf common.DBConf fakeServer *FakeServer messenger *MockMessenger + database *common.SDAdb } func TestProxyTestSuite(t *testing.T) { @@ -42,10 +48,33 @@ func (suite *ProxyTests) SetupTest() { // Create a mock messenger suite.messenger = NewMockMessenger() + + // Create a database configuration for the fake database + suite.DBConf = common.DBConf{ + Host: "localhost", + Port: 5432, + User: "lega_in", + Password: "lega_in", + Database: "lega", + CACert: "", + SslMode: "disable", + ClientCert: "", + ClientKey: "", + } + + var err error + + _, err = os.Stat("/.dockerenv") + if err == nil { + suite.DBConf.Host = "db" + } + + suite.database = &common.SDAdb{} } func (suite *ProxyTests) TearDownTest() { suite.fakeServer.Close() + suite.database.Close() } type FakeServer struct { @@ -121,7 +150,7 @@ func (u *AlwaysDeny) Authenticate(r *http.Request) (jwt.MapClaims, error) { func (suite *ProxyTests) TestServeHTTP_disallowed() { // Start mock messenger that denies everything - proxy := NewProxy(suite.S3conf, &AlwaysDeny{}, suite.messenger, new(tls.Config)) + proxy := NewProxy(suite.S3conf, &AlwaysDeny{}, suite.messenger, suite.database, new(tls.Config)) r, _ := http.NewRequest("", "", nil) w := httptest.NewRecorder() @@ -196,7 +225,7 @@ func (suite *ProxyTests) TestServeHTTPS3Unresponsive() { bucket: "buckbuck", region: "us-east-1", } - proxy := NewProxy(s3conf, &AlwaysAllow{}, suite.messenger, new(tls.Config)) + proxy := NewProxy(s3conf, &AlwaysAllow{}, suite.messenger, suite.database, new(tls.Config)) r, _ := http.NewRequest("", "", nil) w := httptest.NewRecorder() @@ -213,7 +242,11 @@ func (suite *ProxyTests) TestServeHTTPS3Unresponsive() { func (suite *ProxyTests) TestServeHTTP_allowed() { // Start proxy that allows everything - proxy := NewProxy(suite.S3conf, NewAlwaysAllow(), suite.messenger, new(tls.Config)) + database, err := common.NewSDAdb(suite.DBConf) + if err != nil { + suite.T().Skip("skip TestShutdown since broker not present") + } + proxy := NewProxy(suite.S3conf, NewAlwaysAllow(), suite.messenger, database, new(tls.Config)) // List files works r, _ := http.NewRequest("GET", "/username/file", nil) @@ -311,7 +344,7 @@ func (suite *ProxyTests) TestMessageFormatting() { claims["sub"] = "user@host.domain" // start proxy that denies everything - proxy := NewProxy(suite.S3conf, &AlwaysDeny{}, suite.messenger, new(tls.Config)) + proxy := NewProxy(suite.S3conf, &AlwaysDeny{}, suite.messenger, suite.database, new(tls.Config)) suite.fakeServer.resp = "test/user/new_file.txt12false/user/new_file.txt2020-03-10T13:20:15.000Z"0a44282bd39178db9680f24813c41aec-1"1234STANDARD" msg, err := proxy.CreateMessageFromRequest(r, claims) assert.Nil(suite.T(), err) @@ -334,3 +367,48 @@ func (suite *ProxyTests) TestMessageFormatting() { assert.IsType(suite.T(), Event{}, msg) assert.Equal(suite.T(), "upload", msg.Operation) } + +func (suite *ProxyTests) TestDatabaseConnection() { + database, err := common.NewSDAdb(suite.DBConf) + if err != nil { + suite.T().Skip("skip TestShutdown since broker not present") + } + + // Start proxy that allows everything + proxy := NewProxy(suite.S3conf, NewAlwaysAllow(), suite.messenger, database, new(tls.Config)) + + // PUT a file into the system + filename := "/username/db-test-file" + r, _ := http.NewRequest("PUT", filename, nil) + w := httptest.NewRecorder() + suite.fakeServer.resp = "test/elixirid/db-test-file.txt12false/elixirid/file.txt2020-03-10T13:20:15.000Z"0a44282bd39178db9680f24813c41aec-1"5STANDARD" + proxy.ServeHTTP(w, r) + res := w.Result() + defer res.Body.Close() + assert.Equal(suite.T(), 200, res.StatusCode) + assert.Equal(suite.T(), true, suite.fakeServer.PingedAndRestore()) + assert.Equal(suite.T(), true, suite.messenger.CheckAndRestore()) + assert.Equal(suite.T(), false, suite.messenger.CheckAndRestore()) + + // Check that the file is registered and uploaded in the database + // connect to the database + db, err := sql.Open(suite.DBConf.PgDataSource()) + assert.Nil(suite.T(), err, "Failed to connect to database") + + // Check that the file is in the database + var fileID string + query := "SELECT id FROM sda.files WHERE submission_file_path = $1" + err = db.QueryRow(query, filename[1:]).Scan(&fileID) + assert.Nil(suite.T(), err, "Failed to query database") + assert.NotNil(suite.T(), fileID, "File not found in database") + + // Check that the "registered" status is in the database for this file + for _, status := range []string{"registered", "uploaded"} { + var exists int + query = "SELECT 1 FROM sda.file_event_log WHERE event = $1 AND file_id = $2" + err = db.QueryRow(query, status, fileID).Scan(&exists) + assert.Nil(suite.T(), err, "Failed to find '%v' event in database", status) + assert.Equal(suite.T(), exists, 1, "File '%v' event does not exist", status) + } + +}