Merge pull request #543 from akutz/feature/mount-retry

Integration driver mount retry logic
thecodeteam · May 3, 2017 · c3b1dc1 · c3b1dc1
2 parents 1937a13 + a406c09
commit c3b1dc1
Show file tree

Hide file tree

Showing 3 changed files with 36 additions and 5 deletions.
diff --git a/api/registry/registry_integration.go b/api/registry/registry_integration.go
@@ -2,6 +2,7 @@ package registry
 
 import (
 	"sync"
+	"time"
 
 	log "github.com/Sirupsen/logrus"
 	gofig "github.com/akutz/gofig/types"
@@ -14,9 +15,11 @@ import (
 type idm struct {
 	types.IntegrationDriver
 	sync.RWMutex
-	ctx    types.Context
-	config gofig.Config
-	used   map[string]int
+	ctx        types.Context
+	config     gofig.Config
+	used       map[string]int
+	retryCount int
+	retryWait  time.Duration
 }
 
 // NewIntegrationDriverManager returns a new integration driver manager.
@@ -37,6 +40,14 @@ func (d *idm) Init(ctx types.Context, config gofig.Config) error {
 	d.ctx = ctx
 	d.config = config
 	d.used = map[string]int{}
+	d.retryCount = config.GetInt(types.ConfigIgVolOpsMountRetryCount)
+	if v := config.GetString(types.ConfigIgVolOpsMountRetryWait); v != "" {
+		var err error
+		d.retryWait, err = time.ParseDuration(v)
+		if err != nil {
+			return err
+		}
+	}
 
 	d.initPathCache(ctx)
 
@@ -143,10 +154,22 @@ func (d *idm) Mount(
 		"opts":       opts}
 	ctx.WithFields(fields).Debug("mounting volume")
 
+	ctx = ctx.Join(d.ctx)
+
 	mp, vol, err := d.IntegrationDriver.Mount(
-		ctx.Join(d.ctx), volumeID, volumeName, opts)
+		ctx, volumeID, volumeName, opts)
 	if err != nil {
-		return "", nil, err
+		for x := 0; x < d.retryCount; x++ {
+			time.Sleep(d.retryWait)
+			mp, vol, err = d.IntegrationDriver.Mount(
+				ctx, volumeID, volumeName, opts)
+			if err == nil {
+				break
+			}
+		}
+		if err != nil {
+			return "", nil, err
+		}
 	}
 
 	// if the volume has attachments assign the new mount point to the

diff --git a/api/types/types_config_integration.go b/api/types/types_config_integration.go
@@ -22,6 +22,12 @@ const (
 	//ConfigIgVolOpsMountRootPath is a config key.
 	ConfigIgVolOpsMountRootPath = ConfigIgVolOpsMount + ".rootPath"
 
+	//ConfigIgVolOpsMountRetryCount is a config key.
+	ConfigIgVolOpsMountRetryCount = ConfigIgVolOpsMount + ".retryCount"
+
+	//ConfigIgVolOpsMountRetryWait is a config key.
+	ConfigIgVolOpsMountRetryWait = ConfigIgVolOpsMount + ".retryWait"
+
 	//ConfigIgVolOpsUnmount is a config key.
 	ConfigIgVolOpsUnmount = ConfigIgVolOps + ".unmount"
 

diff --git a/imports/config/imports_config_99_gofig.go b/imports/config/imports_config_99_gofig.go
@@ -84,6 +84,8 @@ func init() {
 
 			rk(gofig.Bool, false, "", types.ConfigExecutorNoDownload)
 			rk(gofig.Bool, false, "", types.ConfigIgVolOpsMountPreempt)
+			rk(gofig.Int, 0, "", types.ConfigIgVolOpsMountRetryCount)
+			rk(gofig.String, "5s", "", types.ConfigIgVolOpsMountRetryWait)
 			rk(gofig.Bool, false, "", types.ConfigIgVolOpsCreateDisable)
 			rk(gofig.Bool, false, "", types.ConfigIgVolOpsRemoveDisable)
 			rk(gofig.Bool, false, "", types.ConfigIgVolOpsUnmountIgnoreUsed)