Merge pull request #7173 from medyagh/stop_kic

stop kubernetes containers on stopping docker/podman driver

cmd/minikube/cmd/pause.go

@@ -24,6 +24,7 @@ import (
 	"github.com/spf13/viper"
 
 	"k8s.io/minikube/pkg/minikube/cluster"
+	"k8s.io/minikube/pkg/minikube/constants"
 	"k8s.io/minikube/pkg/minikube/cruntime"
 	"k8s.io/minikube/pkg/minikube/driver"
 	"k8s.io/minikube/pkg/minikube/exit"
@@ -84,6 +85,6 @@ func runPause(cmd *cobra.Command, args []string) {
 }
 
 func init() {
-	pauseCmd.Flags().StringSliceVarP(&namespaces, "--namespaces", "n", cluster.DefaultNamespaces, "namespaces to pause")
+	pauseCmd.Flags().StringSliceVarP(&namespaces, "--namespaces", "n", constants.DefaultNamespaces, "namespaces to pause")
 	pauseCmd.Flags().BoolVarP(&allNamespaces, "all-namespaces", "A", false, "If set, pause all namespaces")
 }

cmd/minikube/cmd/unpause.go

@@ -24,6 +24,7 @@ import (
 	"github.com/spf13/viper"
 
 	"k8s.io/minikube/pkg/minikube/cluster"
+	"k8s.io/minikube/pkg/minikube/constants"
 	"k8s.io/minikube/pkg/minikube/cruntime"
 	"k8s.io/minikube/pkg/minikube/driver"
 	"k8s.io/minikube/pkg/minikube/exit"
@@ -82,6 +83,6 @@ var unpauseCmd = &cobra.Command{
 }
 
 func init() {
-	unpauseCmd.Flags().StringSliceVarP(&namespaces, "--namespaces", "n", cluster.DefaultNamespaces, "namespaces to unpause")
+	unpauseCmd.Flags().StringSliceVarP(&namespaces, "--namespaces", "n", constants.DefaultNamespaces, "namespaces to unpause")
 	unpauseCmd.Flags().BoolVarP(&allNamespaces, "all-namespaces", "A", false, "If set, unpause all namespaces")
 }

pkg/drivers/kic/kic.go

@@ -36,7 +36,9 @@ import (
 	"k8s.io/minikube/pkg/minikube/assets"
 	"k8s.io/minikube/pkg/minikube/command"
 	"k8s.io/minikube/pkg/minikube/constants"
+	"k8s.io/minikube/pkg/minikube/cruntime"
 	"k8s.io/minikube/pkg/minikube/download"
+	"k8s.io/minikube/pkg/minikube/kubelet"
 )
 
 // Driver represents a kic driver https://minikube.sigs.k8s.io/docs/reference/drivers/docker
@@ -246,6 +248,11 @@ func (d *Driver) GetState() (state.State, error) {
 
 // Kill stops a host forcefully, including any containers that we are managing.
 func (d *Driver) Kill() error {
+	// on init this doesn't get filled when called from cmd
+	d.exec = command.NewKICRunner(d.MachineName, d.OCIBinary)
+	if err := kubelet.ForceStop(d.exec); err != nil {
+		glog.Warningf("couldn't force stop kubelet. will continue with kill anyways: %v", err)
+	}
 	cmd := exec.Command(d.NodeConfig.OCIBinary, "kill", d.MachineName)
 	if err := cmd.Run(); err != nil {
 		return errors.Wrapf(err, "killing kic node %s", d.MachineName)
@@ -312,6 +319,35 @@ func (d *Driver) Start() error {
 
 // Stop a host gracefully, including any containers that we are managing.
 func (d *Driver) Stop() error {
+	// on init this doesn't get filled when called from cmd
+	d.exec = command.NewKICRunner(d.MachineName, d.OCIBinary)
+	// docker does not send right SIG for systemd to know to stop the systemd.
+	// to avoid bind address be taken on an upgrade. more info https://github.com/kubernetes/minikube/issues/7171
+	if err := kubelet.Stop(d.exec); err != nil {
+		glog.Warningf("couldn't stop kubelet. will continue with stop anyways: %v", err)
+		if err := kubelet.ForceStop(d.exec); err != nil {
+			glog.Warningf("couldn't force stop kubelet. will continue with stop anyways: %v", err)
+		}
+	}
+
+	runtime, err := cruntime.New(cruntime.Config{Type: d.NodeConfig.ContainerRuntime, Runner: d.exec})
+	if err != nil { // won't return error because:
+		// even though we can't stop the cotainers inside, we still wanna stop the minikube container itself
+		glog.Errorf("unable to get container runtime: %v", err)
+	} else {
+		containers, err := runtime.ListContainers(cruntime.ListOptions{Namespaces: constants.DefaultNamespaces})
+		if err != nil {
+			glog.Infof("unable list containers : %v", err)
+		}
+		if len(containers) > 0 {
+			if err := runtime.StopContainers(containers); err != nil {
+				glog.Errorf("unable to stop containers : %v", err)
+			}
+		}
+		glog.Infof("successfully stopped kubernetes!")
+
+	}
+
 	cmd := exec.Command(d.NodeConfig.OCIBinary, "stop", d.MachineName)
 	if err := cmd.Run(); err != nil {
 		return errors.Wrapf(err, "stopping %s", d.MachineName)

pkg/drivers/none/none.go

@@ -20,8 +20,6 @@ import (
 	"fmt"
 	"net"
 	"os/exec"
-	"strings"
-	"time"
 
 	"github.com/docker/machine/libmachine/drivers"
 	"github.com/docker/machine/libmachine/state"
@@ -34,8 +32,8 @@ import (
 	"k8s.io/minikube/pkg/minikube/constants"
 	"k8s.io/minikube/pkg/minikube/cruntime"
 	"k8s.io/minikube/pkg/minikube/kubeconfig"
+	"k8s.io/minikube/pkg/minikube/kubelet"
 	"k8s.io/minikube/pkg/minikube/vmpath"
-	"k8s.io/minikube/pkg/util/retry"
 )
 
 // cleanupPaths are paths to be removed by cleanup, and are used by both kubeadm and minikube.
@@ -156,8 +154,8 @@ func (d *Driver) GetState() (state.State, error) {
 
 // Kill stops a host forcefully, including any containers that we are managing.
 func (d *Driver) Kill() error {
-	if err := stopKubelet(d.exec); err != nil {
-		return errors.Wrap(err, "kubelet")
+	if err := kubelet.ForceStop(d.exec); err != nil {
+		glog.Warningf("couldn't force stop kubelet. will continue with kill anyways: %v", err)
 	}
 
 	// First try to gracefully stop containers
@@ -220,8 +218,11 @@ func (d *Driver) Start() error {
 
 // Stop a host gracefully, including any containers that we are managing.
 func (d *Driver) Stop() error {
-	if err := stopKubelet(d.exec); err != nil {
-		return errors.Wrap(err, "stop kubelet")
+	if err := kubelet.Stop(d.exec); err != nil {
+		glog.Warningf("couldn't stop kubelet. will continue with stop anyways: %v", err)
+		if err := kubelet.ForceStop(d.exec); err != nil {
+			glog.Warningf("couldn't force stop kubelet. will continue with stop anyways: %v", err)
+		}
 	}
 	containers, err := d.runtime.ListContainers(cruntime.ListOptions{})
 	if err != nil {
@@ -241,32 +242,6 @@ func (d *Driver) RunSSHCommandFromDriver() error {
 	return fmt.Errorf("driver does not support ssh commands")
 }
 
-// stopKubelet idempotently stops the kubelet
-func stopKubelet(cr command.Runner) error {
-	glog.Infof("stopping kubelet.service ...")
-	stop := func() error {
-		cmd := exec.Command("sudo", "systemctl", "stop", "kubelet.service")
-		if rr, err := cr.RunCmd(cmd); err != nil {
-			glog.Errorf("temporary error for %q : %v", rr.Command(), err)
-		}
-		cmd = exec.Command("sudo", "systemctl", "show", "-p", "SubState", "kubelet")
-		rr, err := cr.RunCmd(cmd)
-		if err != nil {
-			glog.Errorf("temporary error: for %q : %v", rr.Command(), err)
-		}
-		if !strings.Contains(rr.Stdout.String(), "dead") && !strings.Contains(rr.Stdout.String(), "failed") {
-			return fmt.Errorf("unexpected kubelet state: %q", rr.Stdout.String())
-		}
-		return nil
-	}
-
-	if err := retry.Expo(stop, 2*time.Second, time.Minute*3, 5); err != nil {
-		return errors.Wrapf(err, "error stopping kubelet")
-	}
-
-	return nil
-}
-
 // restartKubelet restarts the kubelet
 func restartKubelet(cr command.Runner) error {
 	glog.Infof("restarting kubelet.service ...")

pkg/minikube/cluster/pause.go

@@ -24,14 +24,6 @@ import (
 	"k8s.io/minikube/pkg/minikube/kubelet"
 )
 
-// DefaultNamespaces are namespaces used by minikube, including addons
-var DefaultNamespaces = []string{
-	"kube-system",
-	"kubernetes-dashboard",
-	"storage-gluster",
-	"istio-operator",
-}
-
 // Pause pauses a Kubernetes cluster
 func Pause(cr cruntime.Manager, r command.Runner, namespaces []string) ([]string, error) {
 	ids := []string{}

pkg/minikube/constants/constants.go

@@ -92,4 +92,12 @@ var (
 	KubernetesReleaseBinaries = []string{"kubelet", "kubeadm", "kubectl"}
 	// ImageCacheDir is the path to the image cache directory
 	ImageCacheDir = localpath.MakeMiniPath("cache", "images")
+
+	// DefaultNamespaces are kubernetes namespaces used by minikube, including addons
+	DefaultNamespaces = []string{
+		"kube-system",
+		"kubernetes-dashboard",
+		"storage-gluster",
+		"istio-operator",
+	}
 )

pkg/minikube/kubelet/kubelet.go

@@ -30,16 +30,29 @@ import (
 
 // Stop idempotently stops the kubelet
 func Stop(cr command.Runner) error {
+	return stop(cr, false)
+}
+
+// ForceStop idempotently force stops the kubelet
+func ForceStop(cr command.Runner) error {
+	return stop(cr, true)
+}
+
+// stop dempotently stops the kubelet
+func stop(cr command.Runner, force bool) error {
 	glog.Infof("stopping kubelet ...")
-	stop := func() error {
+	stp := func() error {
 		cmd := exec.Command("sudo", "systemctl", "stop", "kubelet.service")
+		if force {
+			cmd = exec.Command("sudo", "systemctl", "stop", "-f", "kubelet.service")
+		}
 		if rr, err := cr.RunCmd(cmd); err != nil {
-			glog.Errorf("temporary error for %q : %v", rr.Command(), err)
+			return fmt.Errorf("temporary error for %q : %v", rr.Command(), err)
 		}
 		cmd = exec.Command("sudo", "systemctl", "show", "-p", "SubState", "kubelet")
 		rr, err := cr.RunCmd(cmd)
 		if err != nil {
-			glog.Errorf("temporary error: for %q : %v", rr.Command(), err)
+			return fmt.Errorf("temporary error: for %q : %v", rr.Command(), err)
 		}
 		if !strings.Contains(rr.Stdout.String(), "dead") && !strings.Contains(rr.Stdout.String(), "failed") {
 			return fmt.Errorf("unexpected kubelet state: %q", rr.Stdout.String())
@@ -47,10 +60,9 @@ func Stop(cr command.Runner) error {
 		return nil
 	}
 
-	if err := retry.Expo(stop, 2*time.Second, time.Minute*3, 5); err != nil {
+	if err := retry.Expo(stp, 1*time.Second, time.Minute, 2); err != nil {
 		return errors.Wrapf(err, "error stopping kubelet")
 	}
-
 	return nil
 }