Add support for Google Cloud storage as a DataStore (tus#106)

* Add support for Google cloud storage as a DataStore

* Use memorylocker in GCS composer

* Add documentation and recursive compose

* Offset being calculated in getInfo. Tests not working

* Fix tests. Attach metadata to composed object.

* Add dependencies using govendor. Add new package to README. Add test instructions for AppVeyor

* Add google cloud internal and transport packages with govendor

* Adding more dependencies to pass travis integration

* Update w/ internal packages

* Skip tests on go1.5/1.6

* Gcsstore tests (tus#2)

* Refactor as a struct of clojures for testing

* Refactor to support tests

* Add docs

* Struct members cannot fufill the interface, adding wrapper functions

* update tests

* Update documentation

* add more tests

* Add gock and redo some tests

* Add gock

* remove unneeded clojures

.scripts/test_all.sh

@@ -15,6 +15,9 @@ if [[ "$goversion" == *"go1.5"* ]] ||
 
   # Exclude consullocker since this may not be run on all Go versions.
   packages=$(echo "$packages" | sed '/consul/d')
+
+  echo "Skipping tests requiring GCSStore, which is not supported on $goversion"
+  packages=$(echo "$packages" | sed '/gcsstore/d')
 else
   # Install the Consul packages which are not vendored.
   go get -u github.com/hashicorp/consul/...

README.md

@@ -125,6 +125,7 @@ useful tools:
 
 * [**s3store**](https://godoc.org/github.com/tus/tusd/s3store): A storage backend using AWS S3
 * [**filestore**](https://godoc.org/github.com/tus/tusd/filestore): A storage backend using the local file system
+* [**gcsstore**](https://godoc.org/github.com/tus/tusd/gcsstore): A storage backend using Google cloud storage
 * [**memorylocker**](https://godoc.org/github.com/tus/tusd/memorylocker): An in-memory locker for handling concurrent uploads
 * [**consullocker**](https://godoc.org/github.com/tus/tusd/consullocker): A locker using the distributed Consul service
 * [**limitedstore**](https://godoc.org/github.com/tus/tusd/limitedstore): A storage wrapper limiting the total used space for uploads

appveyor.yml

@@ -22,5 +22,5 @@ test_script:
   - go test ./limitedstore
   - go test ./memorylocker
   - go test ./s3store
-
   - go vet ./prometheuscollector
+  - go test ./gcsstore

cmd/tusd/cli/composer.go

@@ -8,6 +8,7 @@ import (
 	"github.com/tus/tusd/limitedstore"
 	"github.com/tus/tusd/memorylocker"
 	"github.com/tus/tusd/s3store"
+	"github.com/tus/tusd/gcsstore"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/credentials"
@@ -21,17 +22,7 @@ func CreateComposer() {
 	// Attempt to use S3 as a backend if the -s3-bucket option has been supplied.
 	// If not, we default to storing them locally on disk.
 	Composer = tusd.NewStoreComposer()
-	if Flags.S3Bucket == "" {
-		dir := Flags.UploadDir
-
-		stdout.Printf("Using '%s' as directory storage.\n", dir)
-		if err := os.MkdirAll(dir, os.FileMode(0774)); err != nil {
-			stderr.Fatalf("Unable to ensure directory exists: %s", err)
-		}
-
-		store := filestore.New(dir)
-		store.UseIn(Composer)
-	} else {
+	if Flags.S3Bucket != "" {
 		s3Config := aws.NewConfig()
 
 		if Flags.S3Endpoint == "" {
@@ -50,6 +41,34 @@ func CreateComposer() {
 
 		locker := memorylocker.New()
 		locker.UseIn(Composer)
+	} else if Flags.GCSBucket != "" {
+		// Derivce credentials from service account file path passed in
+		// GCS_SERVICE_ACCOUNT_FILE environment variable.
+		gcsSAF := os.Getenv("GCS_SERVICE_ACCOUNT_FILE")
+		if gcsSAF == "" {
+			stderr.Fatalf("No service account file provided for Google Cloud Storage\n")
+		}
+
+		service, err := gcsstore.NewGCSService(gcsSAF)
+		if err != nil {
+			stderr.Fatalf("Unable to create Google Cloud Storage service: %s\n", err)
+		}
+
+		store := gcsstore.New(Flags.GCSBucket, service)
+		store.UseIn(Composer)
+
+		locker := memorylocker.New()
+		locker.UseIn(Composer)
+	} else {
+		dir := Flags.UploadDir
+
+		stdout.Printf("Using '%s' as directory storage.\n", dir)
+		if err := os.MkdirAll(dir, os.FileMode(0774)); err != nil {
+			stderr.Fatalf("Unable to ensure directory exists: %s", err)
+		}
+
+		store := filestore.New(dir)
+		store.UseIn(Composer)
 	}
 
 	storeSize := Flags.StoreSize

cmd/tusd/cli/flags.go

@@ -15,6 +15,7 @@ var Flags struct {
 	Timeout           int64
 	S3Bucket          string
 	S3Endpoint        string
+	GCSBucket     	  string
 	FileHooksDir      string
 	HttpHooksEndpoint string
 	HttpHooksRetry    int
@@ -38,6 +39,7 @@ func ParseFlags() {
 	flag.Int64Var(&Flags.Timeout, "timeout", 30*1000, "Read timeout for connections in milliseconds.  A zero value means that reads will not timeout")
 	flag.StringVar(&Flags.S3Bucket, "s3-bucket", "", "Use AWS S3 with this bucket as storage backend (requires the AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY and AWS_REGION environment variables to be set)")
 	flag.StringVar(&Flags.S3Endpoint, "s3-endpoint", "", "Endpoint to use S3 compatible implementations like minio (requires s3-bucket to be pass)")
+	flag.StringVar(&Flags.GCSBucket, "gcs-bucket", "", "Use Google Cloud Storage with this bucket as storage backend (requires the GCS_SERVICE_ACCOUNT_FILE environment variable to be set)")
 	flag.StringVar(&Flags.FileHooksDir, "hooks-dir", "", "Directory to search for available hooks scripts")
 	flag.StringVar(&Flags.HttpHooksEndpoint, "hooks-http", "", "An HTTP endpoint to which hook events will be sent to")
 	flag.IntVar(&Flags.HttpHooksRetry, "hooks-http-retry", 3, "Number of times to retry on a 500 or network timeout")