Merge pull request #7598 from tstromberg/pause-rettry

Add retry to pause/unpause functions

pkg/minikube/cluster/pause.go

@@ -17,15 +17,32 @@ limitations under the License.
 package cluster
 
 import (
+	"time"
+
 	"github.com/golang/glog"
 	"github.com/pkg/errors"
 	"k8s.io/minikube/pkg/minikube/command"
 	"k8s.io/minikube/pkg/minikube/cruntime"
 	"k8s.io/minikube/pkg/minikube/sysinit"
+	"k8s.io/minikube/pkg/util/retry"
 )
 
-// Pause pauses a Kubernetes cluster
+// Pause pauses a Kubernetes cluster, retrying if necessary
 func Pause(cr cruntime.Manager, r command.Runner, namespaces []string) ([]string, error) {
+	var ids []string
+	tryPause := func() (err error) {
+		ids, err = pause(cr, r, namespaces)
+		return err
+	}
+
+	if err := retry.Expo(tryPause, 250*time.Millisecond, 2*time.Second); err != nil {
+		return ids, err
+	}
+	return ids, nil
+}
+
+// pause pauses a Kubernetes cluster
+func pause(cr cruntime.Manager, r command.Runner, namespaces []string) ([]string, error) {
 	ids := []string{}
 
 	// Disable the kubelet so it does not attempt to restart paused pods
@@ -49,11 +66,24 @@ func Pause(cr cruntime.Manager, r command.Runner, namespaces []string) ([]string
 	}
 
 	return ids, cr.PauseContainers(ids)
-
 }
 
-// Unpause unpauses a Kubernetes cluster
+// Unpause unpauses a Kubernetes cluster, retrying if necessary
 func Unpause(cr cruntime.Manager, r command.Runner, namespaces []string) ([]string, error) {
+	var ids []string
+	tryUnpause := func() (err error) {
+		ids, err = unpause(cr, r, namespaces)
+		return err
+	}
+
+	if err := retry.Expo(tryUnpause, 250*time.Millisecond, 2*time.Second); err != nil {
+		return ids, err
+	}
+	return ids, nil
+}
+
+// unpause unpauses a Kubernetes cluster
+func unpause(cr cruntime.Manager, r command.Runner, namespaces []string) ([]string, error) {
 	ids, err := cr.ListContainers(cruntime.ListOptions{State: cruntime.Paused, Namespaces: namespaces})
 	if err != nil {
 		return ids, errors.Wrap(err, "list paused")