Merge branch 'master' into consistent_err_formatting

.github/ISSUE_TEMPLATE

@@ -33,7 +33,7 @@ echo "";
 echo "OS:";
 cat /etc/os-release
 echo "";
-echo "VM driver": 
+echo "VM driver:"; 
 grep DriverName ~/.minikube/machines/minikube/config.json
 echo "";
 echo "ISO version";

CHANGELOG.md

 # Minikube Release Notes
 
+# Version 0.30.0 - 10/04/2018
+
+* Initial support for Kubernetes 1.12+ [#3180](https://github.com/kubernetes/minikube/pull/3180)
+* Use "kubectl proxy" instead of a NodePort to expose the dashboard [#3210](https://github.com/kubernetes/minikube/pull/3210)
+* Enhance the Ingress Addon [#3099](https://github.com/kubernetes/minikube/pull/3099)
+* Upgrade cni and cni-plugins to release version [#3152](https://github.com/kubernetes/minikube/pull/3152)
+* ensure that /dev has settled before operating [#3195](https://github.com/kubernetes/minikube/pull/3195)
+* Upgrade gluster client in ISO to 4.1.5 [#3162](https://github.com/kubernetes/minikube/pull/3162)
+* update nginx ingress controller version to 0.19.0 [#3123](https://github.com/kubernetes/minikube/pull/3123)
+* Install crictl from binary instead of from source [#3160](https://github.com/kubernetes/minikube/pull/3160)
+* Switch the source of libmachine to machine-drivers. [#3185](https://github.com/kubernetes/minikube/pull/3185)
+* Add psmisc package, for pstree command [#3161](https://github.com/kubernetes/minikube/pull/3161)
+
+Huge thank you for this release towards our contributors:
+- Anders F Björklund
+- Bob Killen
+- David Genest
+- Denis Gladkikh
+- dlorenc
+- Fernando Diaz
+- oilbeater
+- Raunak Ramakrishnan
+- Rui Cao
+- samuela
+- Sven Anderson
+- Thomas Strömberg
+
 # Version 0.29.0 - 09/27/2018
 * Issue #3037 change dependency management to dep [#3136](https://github.com/kubernetes/minikube/pull/3136)
 * Update dashboard version to v1.10.0 [#3122](https://github.com/kubernetes/minikube/pull/3122)
@@ -17,7 +44,7 @@
 * Revert "Remove untainting logic." [#3050](https://github.com/kubernetes/minikube/pull/3050)
 * Upgrade kpod 0.1 to podman 0.4.1 [#3026](https://github.com/kubernetes/minikube/pull/3026)
 * Linux install: Set owner to root [#3021](https://github.com/kubernetes/minikube/pull/3021)
-* Rm localkube [#2911](https://github.com/kubernetes/minikube/pull/2911)
+* Remove localkube bootstrapper and associated `get-k8s-versions` command [#2911](https://github.com/kubernetes/minikube/pull/2911)
 * Update to go 1.10.1 everywhere. [#2777](https://github.com/kubernetes/minikube/pull/2777)
 * Allow to override build date with SOURCE_DATE_EPOCH [#3009](https://github.com/kubernetes/minikube/pull/3009)
 
@@ -55,8 +82,6 @@ Huge Thank You for this release to our contributors:
 - wangxy518
 - yanxuean
 
-
-
 # Version 0.28.2 - 7/20/2018
 * Nvidia driver installation fixed [#2996](https://github.com/kubernetes/minikube/pull/2986)
 

Gopkg.toml

@@ -31,6 +31,7 @@
 
 [[constraint]]
   branch = "master"
+  source = "github.com/machine-drivers/machine"
   name = "github.com/docker/machine"
 
 [[constraint]]

Makefile

@@ -14,7 +14,7 @@
 
 # Bump these on release
 VERSION_MAJOR ?= 0
-VERSION_MINOR ?= 29
+VERSION_MINOR ?= 30
 VERSION_BUILD ?= 0
 VERSION ?= v$(VERSION_MAJOR).$(VERSION_MINOR).$(VERSION_BUILD)
 DEB_VERSION ?= $(VERSION_MAJOR).$(VERSION_MINOR)-$(VERSION_BUILD)
@@ -26,7 +26,7 @@ HYPERKIT_BUILD_IMAGE 	?= karalabe/xgo-1.10.x
 BUILD_IMAGE 	?= k8s.gcr.io/kube-cross:v1.10.1-1
 ISO_BUILD_IMAGE ?= $(REGISTRY)/buildroot-image
 
-ISO_VERSION ?= v0.29.0
+ISO_VERSION ?= v0.30.0
 ISO_BUCKET ?= minikube/iso
 
 MINIKUBE_VERSION ?= $(ISO_VERSION)
@@ -282,10 +282,6 @@ install-hyperkit-driver: out/docker-machine-driver-hyperkit
 check-release:
 	go test -v ./deploy/minikube/release_sanity_test.go -tags=release
 
-.PHONY: update-releases
-update-releases:
-	gsutil cp deploy/minikube/k8s_releases.json gs://minikube/k8s_releases.json
-
 buildroot-image: $(ISO_BUILD_IMAGE) # convenient alias to build the docker container
 $(ISO_BUILD_IMAGE): deploy/iso/minikube-iso/Dockerfile
 	docker build $(ISO_DOCKER_EXTRA_ARGS) -t $@ -f $< $(dir $<)

README.md

@@ -76,6 +76,7 @@ export MINIKUBE_WANTREPORTERRORPROMPT=false
 export MINIKUBE_HOME=$HOME
 export CHANGE_MINIKUBE_NONE_USER=true
 mkdir -p $HOME/.kube
+mkdir -p $HOME/.minikube
 touch $HOME/.kube/config
 
 export KUBECONFIG=$HOME/.kube/config
@@ -219,5 +220,5 @@ For more information about Minikube, see the [proposal](https://github.com/kuber
 ## Community
 
 * [**#minikube on Kubernetes Slack**](https://kubernetes.slack.com)
-* [**kubernetes-users mailing list** ](https://groups.google.com/forum/#!forum/kubernetes-users)
-(If you are posting to the list, please prefix your subject with "minikube: ")
+* [**Kubernetes Official Forum** ](https://discuss.kubernetes.io)
+(If you are posting to the forum, please tag your post with "minikube")

cmd/minikube/cmd/config/util.go

@@ -114,7 +114,7 @@ func EnableOrDisableAddon(name string, val string) error {
 	if err != nil {
 		return err
 	}
-	host, err := cluster.CheckIfApiExistsAndLoad(api)
+	host, err := cluster.CheckIfHostExistsAndLoad(api, config.GetMachineName())
 	if err != nil {
 		return errors.Wrap(err, "getting host")
 	}

cmd/minikube/cmd/dashboard.go

@@ -17,68 +17,138 @@ limitations under the License.
 package cmd
 
 import (
+	"bufio"
 	"fmt"
+	"net/http"
 	"os"
-	"text/template"
+	"os/exec"
+	"regexp"
 	"time"
 
 	"github.com/golang/glog"
 	"github.com/pkg/browser"
+	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
 	"k8s.io/minikube/pkg/minikube/cluster"
+	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/machine"
 	"k8s.io/minikube/pkg/minikube/service"
 
-	commonutil "k8s.io/minikube/pkg/util"
+	"k8s.io/minikube/pkg/util"
 )
 
 var (
 	dashboardURLMode bool
+	// Matches: 127.0.0.1:8001
+	// TODO(tstromberg): Get kubectl to implement a stable supported output format.
+	hostPortRe = regexp.MustCompile(`127.0.0.1:\d{4,}`)
 )
 
 // dashboardCmd represents the dashboard command
 var dashboardCmd = &cobra.Command{
 	Use:   "dashboard",
-	Short: "Opens/displays the kubernetes dashboard URL for your local cluster",
-	Long:  `Opens/displays the kubernetes dashboard URL for your local cluster`,
+	Short: "Access the kubernetes dashboard running within the minikube cluster",
+	Long:  `Access the kubernetes dashboard running within the minikube cluster`,
 	Run: func(cmd *cobra.Command, args []string) {
 		api, err := machine.NewAPIClient()
+		defer func() {
+			err := api.Close()
+			if err != nil {
+				glog.Warningf("Failed to close API: %v", err)
+			}
+		}()
+
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "Error getting client: %v\n", err)
 			os.Exit(1)
 		}
-		defer api.Close()
-
 		cluster.EnsureMinikubeRunningOrExit(api, 1)
-		namespace := "kube-system"
-		svc := "kubernetes-dashboard"
 
-		if err = commonutil.RetryAfter(20, func() error { return service.CheckService(namespace, svc) }, 6*time.Second); err != nil {
-			fmt.Fprintf(os.Stderr, "Could not find finalized endpoint being pointed to by %s: %v\n", svc, err)
+		ns := "kube-system"
+		svc := "kubernetes-dashboard"
+		if err = util.RetryAfter(30, func() error { return service.CheckService(ns, svc) }, 1*time.Second); err != nil {
+			fmt.Fprintf(os.Stderr, "%s:%s is not running: %v\n", ns, svc, err)
 			os.Exit(1)
 		}
 
-		urls, err := service.GetServiceURLsForService(api, namespace, svc, template.Must(template.New("dashboardServiceFormat").Parse(defaultServiceFormatTemplate)))
+		p, hostPort, err := kubectlProxy()
 		if err != nil {
-			fmt.Fprintln(os.Stderr, err)
-			fmt.Fprintln(os.Stderr, "Check that minikube is running.")
-			os.Exit(1)
+			glog.Fatalf("kubectl proxy: %v", err)
 		}
-		if len(urls) == 0 {
-			errMsg := "There appears to be no url associated with dashboard, this is not expected, exiting"
-			glog.Infoln(errMsg)
+		url := dashboardURL(hostPort, ns, svc)
+
+		if err = util.RetryAfter(60, func() error { return checkURL(url) }, 1*time.Second); err != nil {
+			fmt.Fprintf(os.Stderr, "%s is not responding properly: %v\n", url, err)
 			os.Exit(1)
 		}
+
 		if dashboardURLMode {
-			fmt.Fprintln(os.Stdout, urls[0])
+			fmt.Fprintln(os.Stdout, url)
 		} else {
-			fmt.Fprintln(os.Stdout, "Opening kubernetes dashboard in default browser...")
-			browser.OpenURL(urls[0])
+			fmt.Fprintln(os.Stdout, fmt.Sprintf("Opening %s in your default browser...", url))
+			if err = browser.OpenURL(url); err != nil {
+				fmt.Fprintf(os.Stderr, fmt.Sprintf("failed to open browser: %v", err))
+			}
+		}
+
+		glog.Infof("Waiting forever for kubectl proxy to exit ...")
+		if err = p.Wait(); err != nil {
+			glog.Errorf("Wait: %v", err)
 		}
 	},
 }
 
+// kubectlProxy runs "kubectl proxy", returning host:port
+func kubectlProxy() (*exec.Cmd, string, error) {
+	path, err := exec.LookPath("kubectl")
+	if err != nil {
+		return nil, "", errors.Wrap(err, "kubectl not found in PATH")
+	}
+
+	// port=0 picks a random system port
+	// config.GetMachineName() respects the -p (profile) flag
+	cmd := exec.Command(path, "--context", config.GetMachineName(), "proxy", "--port=0")
+	stdoutPipe, err := cmd.StdoutPipe()
+	if err != nil {
+		return nil, "", errors.Wrap(err, "cmd stdout")
+	}
+
+	glog.Infof("Executing: %s %s", cmd.Path, cmd.Args)
+	if err := cmd.Start(); err != nil {
+		return nil, "", errors.Wrap(err, "proxy start")
+	}
+	reader := bufio.NewReader(stdoutPipe)
+	glog.Infof("proxy started, reading stdout pipe ...")
+	out, err := reader.ReadString('\n')
+	if err != nil {
+		return nil, "", errors.Wrap(err, "reading stdout pipe")
+	}
+	glog.Infof("proxy stdout: %s", out)
+	return cmd, hostPortRe.FindString(out), nil
+}
+
+// dashboardURL generates a URL for accessing the dashboard service
+func dashboardURL(proxy string, ns string, svc string) string {
+	// Reference: https://github.com/kubernetes/dashboard/wiki/Accessing-Dashboard---1.7.X-and-above
+	return fmt.Sprintf("http://%s/api/v1/namespaces/%s/services/http:%s:/proxy/", proxy, ns, svc)
+}
+
+// checkURL checks if a URL returns 200 HTTP OK
+func checkURL(url string) error {
+	resp, err := http.Get(url)
+	glog.Infof("%s response: %v %+v", url, err, resp)
+	if err != nil {
+		return errors.Wrap(err, "checkURL")
+	}
+	if resp.StatusCode != http.StatusOK {
+		return &util.RetriableError{
+			Err: fmt.Errorf("unexpected response code: %d", resp.StatusCode),
+		}
+	}
+	return nil
+}
+
 func init() {
-	dashboardCmd.Flags().BoolVar(&dashboardURLMode, "url", false, "Display the kubernetes dashboard in the CLI instead of opening it in the default browser")
+	dashboardCmd.Flags().BoolVar(&dashboardURLMode, "url", false, "Display dashboard URL instead of opening a browser")
 	RootCmd.AddCommand(dashboardCmd)
 }

cmd/minikube/cmd/env.go

@@ -306,7 +306,7 @@ var dockerEnvCmd = &cobra.Command{
 			os.Exit(1)
 		}
 		defer api.Close()
-		host, err := cluster.CheckIfApiExistsAndLoad(api)
+		host, err := cluster.CheckIfHostExistsAndLoad(api, config.GetMachineName())
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "Error getting host: %v\n", err)
 			os.Exit(1)

cmd/minikube/cmd/env_test.go

@@ -45,10 +45,12 @@ func (f FakeNoProxyGetter) GetNoProxyVar() (string, string) {
 }
 
 var defaultAPI = &tests.MockAPI{
-	Hosts: map[string]*host.Host{
-		config.GetMachineName(): {
-			Name:   config.GetMachineName(),
-			Driver: &tests.MockDriver{},
+	FakeStore: tests.FakeStore{
+		Hosts: map[string]*host.Host{
+			config.GetMachineName(): {
+				Name:   config.GetMachineName(),
+				Driver: &tests.MockDriver{},
+			},
 		},
 	},
 }
@@ -81,7 +83,9 @@ func TestShellCfgSet(t *testing.T) {
 		{
 			description: "no host specified",
 			api: &tests.MockAPI{
-				Hosts: make(map[string]*host.Host),
+				FakeStore: tests.FakeStore{
+					Hosts: make(map[string]*host.Host),
+				},
 			},
 			shell:            "bash",
 			expectedShellCfg: nil,

cmd/minikube/cmd/ssh.go

@@ -24,6 +24,7 @@ import (
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
 	"k8s.io/minikube/pkg/minikube/cluster"
+	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/machine"
 )
 
@@ -39,7 +40,7 @@ var sshCmd = &cobra.Command{
 			os.Exit(1)
 		}
 		defer api.Close()
-		host, err := cluster.CheckIfApiExistsAndLoad(api)
+		host, err := cluster.CheckIfHostExistsAndLoad(api, config.GetMachineName())
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "Error getting host: %v\n", err)
 			os.Exit(1)

cmd/minikube/cmd/start.go

@@ -187,7 +187,7 @@ func runStart(cmd *cobra.Command, args []string) {
 		selectedKubernetesVersion = constants.DefaultKubernetesVersion
 	}
 	// Load profile cluster config from file
-	cc, err := loadConfigFromFile(viper.GetString(cfg.MachineProfile))
+	cc, err := cfg.Load()
 	if err != nil && !os.IsNotExist(err) {
 		glog.Errorln("Error loading profile config: ", err)
 	}
@@ -463,23 +463,3 @@ func saveConfigToFile(data []byte, file string) error {
 	}
 	return nil
 }
-
-func loadConfigFromFile(profile string) (cfg.Config, error) {
-	var cc cfg.Config
-
-	profileConfigFile := constants.GetProfileFile(profile)
-
-	if _, err := os.Stat(profileConfigFile); os.IsNotExist(err) {
-		return cc, err
-	}
-
-	data, err := ioutil.ReadFile(profileConfigFile)
-	if err != nil {
-		return cc, err
-	}
-
-	if err := json.Unmarshal(data, &cc); err != nil {
-		return cc, err
-	}
-	return cc, nil
-}

cmd/minikube/cmd/status.go

@@ -88,7 +88,7 @@ var statusCmd = &cobra.Command{
 				returnCode |= clusterNotRunningStatusFlag
 			}
 
-			ip, err := cluster.GetHostDriverIP(api)
+			ip, err := cluster.GetHostDriverIP(api, config.GetMachineName())
 			if err != nil {
 				glog.Errorln("Error host driver ip status:", err)
 				cmdUtil.MaybeReportErrorAndExitWithCode(err, internalErrorCode)

cmd/minikube/cmd/tunnel.go

+/*
+Copyright 2018 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cmd
+
+import (
+	"context"
+	"os"
+	"os/signal"
+
+	"github.com/golang/glog"
+	"github.com/spf13/cobra"
+	"k8s.io/minikube/pkg/minikube/config"
+	"k8s.io/minikube/pkg/minikube/machine"
+	"k8s.io/minikube/pkg/minikube/service"
+	"k8s.io/minikube/pkg/minikube/tunnel"
+)
+
+var cleanup bool
+
+// tunnelCmd represents the tunnel command
+var tunnelCmd = &cobra.Command{
+	Use:   "tunnel",
+	Short: "tunnel makes services of type LoadBalancer accessible on localhost",
+	Long:  `tunnel creates a route to services deployed with type LoadBalancer and sets their Ingress to their ClusterIP`,
+	PersistentPreRun: func(cmd *cobra.Command, args []string) {
+		RootCmd.PersistentPreRun(cmd, args)
+	},
+	Run: func(cmd *cobra.Command, args []string) {
+		manager := tunnel.NewManager()
+
+		if cleanup {
+			glog.Info("Checking for tunnels to cleanup...")
+			if err := manager.CleanupNotRunningTunnels(); err != nil {
+				glog.Errorf("error cleaning up: %s", err)
+			}
+			return
+		}
+
+		glog.Infof("Creating docker machine client...")
+		api, err := machine.NewAPIClient()
+		if err != nil {
+			glog.Fatalf("error creating dockermachine client: %s", err)
+		}
+		glog.Infof("Creating k8s client...")
+		clientset, err := service.K8s.GetClientset()
+		if err != nil {
+			glog.Fatalf("error creating K8S clientset: %s", err)
+		}
+
+		ctrlC := make(chan os.Signal, 1)
+		signal.Notify(ctrlC, os.Interrupt)
+		ctx, cancel := context.WithCancel(context.Background())
+		go func() {
+			<-ctrlC
+			cancel()
+		}()
+
+		done, err := manager.StartTunnel(ctx, config.GetMachineName(), api, config.DefaultLoader, clientset.CoreV1())
+		if err != nil {
+			glog.Fatalf("error starting tunnel: %s", err)
+		}
+		<-done
+	},
+}
+
+func init() {
+	tunnelCmd.Flags().BoolVarP(&cleanup, "cleanup", "c", false, "call with cleanup=true to remove old tunnels")
+	RootCmd.AddCommand(tunnelCmd)
+}

cmd/minikube/cmd/update-context.go

@@ -43,17 +43,18 @@ var updateContextCmd = &cobra.Command{
 			os.Exit(1)
 		}
 		defer api.Close()
-		ip, err := cluster.GetHostDriverIP(api)
+		machineName := config.GetMachineName()
+		ip, err := cluster.GetHostDriverIP(api, machineName)
 		if err != nil {
 			glog.Errorln("Error host driver ip status:", err)
 			cmdUtil.MaybeReportErrorAndExit(err)
 		}
-		kstatus, err := kcfg.UpdateKubeconfigIP(ip, constants.KubeconfigPath, config.GetMachineName())
+		ok, err := kcfg.UpdateKubeconfigIP(ip, constants.KubeconfigPath, machineName)
 		if err != nil {
 			glog.Errorln("Error kubeconfig status:", err)
 			cmdUtil.MaybeReportErrorAndExit(err)
 		}
-		if kstatus {
+		if ok {
 			fmt.Println("Reconfigured kubeconfig IP, now pointing at " + ip.String())
 		} else {
 			fmt.Println("Kubeconfig IP correctly configured, pointing at " + ip.String())

deploy/addons/dashboard/dashboard-svc.yaml

@@ -24,10 +24,8 @@ metadata:
     kubernetes.io/minikube-addons: dashboard
     kubernetes.io/minikube-addons-endpoint: dashboard
 spec:
-  type: NodePort
   ports:
   - port: 80
     targetPort: 9090
-    nodePort: 30000
   selector:
     app: kubernetes-dashboard

deploy/addons/ingress/ingress-configmap.yaml

@@ -29,9 +29,13 @@ kind: ConfigMap
 metadata:
   name: tcp-services
   namespace: kube-system
+  labels:
+    addonmanager.kubernetes.io/mode: EnsureExists
 ---
 apiVersion: v1
 kind: ConfigMap
 metadata:
   name: udp-services
   namespace: kube-system
+  labels:
+    addonmanager.kubernetes.io/mode: EnsureExists

deploy/addons/ingress/ingress-dp.yaml

@@ -18,17 +18,19 @@ metadata:
   name: default-http-backend
   namespace: kube-system
   labels:
+    app.kubernetes.io/name: default-http-backend
+    app.kubernetes.io/part-of: kube-system
     addonmanager.kubernetes.io/mode: Reconcile
 spec:
   replicas: 1
   selector:
     matchLabels:
-      app: default-http-backend
+      app.kubernetes.io/name: default-http-backend
       addonmanager.kubernetes.io/mode: Reconcile
   template:
     metadata:
       labels:
-        app: default-http-backend
+        app.kubernetes.io/name: default-http-backend
         addonmanager.kubernetes.io/mode: Reconcile
     spec:
       terminationGracePeriodSeconds: 60
@@ -37,7 +39,7 @@ spec:
         # Any image is permissible as long as:
         # 1. It serves a 404 page at /
         # 2. It serves 200 on a /healthz endpoint
-        image: k8s.gcr.io/defaultbackend:1.4
+        image: gcr.io/google_containers/defaultbackend:1.4
         imagePullPolicy: IfNotPresent
         livenessProbe:
           httpGet:
@@ -50,11 +52,11 @@ spec:
         - containerPort: 8080
         resources:
           limits:
-            cpu: 10m
-            memory: 20Mi
+            cpu: 20m
+            memory: 30Mi
           requests:
-            cpu: 10m
-            memory: 20Mi
+            cpu: 20m
+            memory: 30Mi
 ---
 apiVersion: extensions/v1beta1
 kind: Deployment
@@ -62,24 +64,30 @@ metadata:
   name: nginx-ingress-controller
   namespace: kube-system
   labels:
-    app: nginx-ingress-controller
+    app.kubernetes.io/name: nginx-ingress-controller
+    app.kubernetes.io/part-of: kube-system
     addonmanager.kubernetes.io/mode: Reconcile
 spec:
   replicas: 1
   selector:
     matchLabels:
-      app: nginx-ingress-controller
+      app.kubernetes.io/name: nginx-ingress-controller
+      app.kubernetes.io/part-of: kube-system
       addonmanager.kubernetes.io/mode: Reconcile
   template:
     metadata:
       labels:
-        app: nginx-ingress-controller
-        name: nginx-ingress-controller
+        app.kubernetes.io/name: nginx-ingress-controller
+        app.kubernetes.io/part-of: kube-system
         addonmanager.kubernetes.io/mode: Reconcile
+      annotations:
+        prometheus.io/port: '10254'
+        prometheus.io/scrape: 'true'
     spec:
+      serviceAccountName: nginx-ingress
       terminationGracePeriodSeconds: 60
       containers:
-      - image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.16.2
+      - image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.19.0
         name: nginx-ingress-controller
         imagePullPolicy: IfNotPresent
         readinessProbe:
@@ -108,8 +116,7 @@ spec:
           hostPort: 80
         - containerPort: 443
           hostPort: 443
-        # we expose 18080 to access nginx stats in url /nginx-status
-        # this is optional
+        # (Optional) we expose 18080 to access nginx stats in url /nginx-status
         - containerPort: 18080
           hostPort: 18080
         args:
@@ -118,6 +125,7 @@ spec:
         - --configmap=$(POD_NAMESPACE)/nginx-load-balancer-conf
         - --tcp-services-configmap=$(POD_NAMESPACE)/tcp-services
         - --udp-services-configmap=$(POD_NAMESPACE)/udp-services
+        - --annotations-prefix=nginx.ingress.kubernetes.io
         # use minikube IP address in ingress status field
         - --report-node-internal-ip-address
         securityContext:

deploy/addons/ingress/ingress-rbac.yaml

+---
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: nginx-ingress
+  namespace: kube-system
+  labels:
+    addonmanager.kubernetes.io/mode: Reconcile
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRole
+metadata:
+  name: system:nginx-ingress
+  labels:
+    kubernetes.io/bootstrapping: rbac-defaults
+    addonmanager.kubernetes.io/mode: Reconcile
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  - endpoints
+  - nodes
+  - pods
+  - secrets
+  verbs:
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - get
+- apiGroups:
+  - ""
+  resources:
+  - services
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - "extensions"
+  resources:
+  - ingresses
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - create
+  - patch
+- apiGroups:
+  - "extensions"
+  resources:
+  - ingresses/status
+  verbs:
+  - update
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: Role
+metadata:
+  name: system::nginx-ingress-role
+  namespace: kube-system
+  labels:
+    kubernetes.io/bootstrapping: rbac-defaults
+    addonmanager.kubernetes.io/mode: Reconcile
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  - pods
+  - secrets
+  - namespaces
+  verbs:
+  - get
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  resourceNames:
+  # Defaults to "<election-id>-<ingress-class>"
+  # Here: "<ingress-controller-leader>-<nginx>"
+  # This has to be adapted if you change either parameter
+  # when launching the nginx-ingress-controller.
+  - ingress-controller-leader-nginx
+  verbs:
+  - get
+  - update
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  verbs:
+  - create
+- apiGroups:
+  - ""
+  resources:
+  - endpoints
+  verbs:
+  - get
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: RoleBinding
+metadata:
+  name: system::nginx-ingress-role-binding
+  namespace: kube-system
+  labels:
+    kubernetes.io/bootstrapping: rbac-defaults
+    addonmanager.kubernetes.io/mode: EnsureExists
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: system::nginx-ingress-role
+subjects:
+- kind: ServiceAccount
+  name: nginx-ingress
+  namespace: kube-system
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+  name: system:nginx-ingress
+  labels:
+    kubernetes.io/bootstrapping: rbac-defaults
+    addonmanager.kubernetes.io/mode: EnsureExists
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:nginx-ingress
+subjects:
+- kind: ServiceAccount
+  name: nginx-ingress
+  namespace: kube-system
\ No newline at end of file

deploy/addons/ingress/ingress-svc.yaml

@@ -18,7 +18,8 @@ metadata:
   name: default-http-backend
   namespace: kube-system
   labels:
-    app: default-http-backend
+    app.kubernetes.io/name: default-http-backend
+    app.kubernetes.io/part-of: kube-system
     kubernetes.io/minikube-addons: ingress
     kubernetes.io/minikube-addons-endpoint: ingress
     addonmanager.kubernetes.io/mode: Reconcile
@@ -29,4 +30,4 @@ spec:
     targetPort: 8080
     nodePort: 30001
   selector:
-    app: default-http-backend
+    app.kubernetes.io/name: default-http-backend
\ No newline at end of file

deploy/iso/minikube-iso/package/Config.in

@@ -3,7 +3,7 @@ menu "System tools"
     source "$BR2_EXTERNAL_MINIKUBE_PATH/package/runc-master/Config.in"
     source "$BR2_EXTERNAL_MINIKUBE_PATH/package/podman/Config.in"
     source "$BR2_EXTERNAL_MINIKUBE_PATH/package/crio-bin/Config.in"
-    source "$BR2_EXTERNAL_MINIKUBE_PATH/package/cri-tools/Config.in"
+    source "$BR2_EXTERNAL_MINIKUBE_PATH/package/crictl-bin/Config.in"
     source "$BR2_EXTERNAL_MINIKUBE_PATH/package/automount/Config.in"
     source "$BR2_EXTERNAL_MINIKUBE_PATH/package/docker-bin/Config.in"
     source "$BR2_EXTERNAL_MINIKUBE_PATH/package/cni-bin/Config.in"

deploy/iso/minikube-iso/package/automount/minikube-automount.service

 [Unit]
 Description=minikube automount
+Requires=systemd-udev-settle.service
 Before=docker.service rkt-api.service rkt-metadata.service
+After=systemd-udev-settle.service
 
 [Service]
 ExecStart=/usr/sbin/minikube-automount

deploy/iso/minikube-iso/package/cni-bin/cni-bin.hash

 sha256 b1ae09833a238c51161918a8849031efdb46cf0068ea5b752e362d9836e2af7d  cni-v0.3.0.tgz
 sha256 84c9a0a41b59211d560bef14bf3f53bb370156f9ac7762270b3848fed96e1be8  cni-v0.4.0.tgz
-sha256 d1e3c693903d498fcb89076f66410167eaa6d81df4a1051eba7565672f896543  cni-amd64-v0.6.0-rc1.tgz
+sha256 a7f84a742c8f3a95843b3cc636444742554a4853835649ec371a07c841daebab  cni-amd64-v0.6.0.tgz

deploy/iso/minikube-iso/package/cni-bin/cni-bin.mk

@@ -4,7 +4,7 @@
 #
 ################################################################################
 
-CNI_BIN_VERSION = v0.6.0-rc1
+CNI_BIN_VERSION = v0.6.0
 CNI_BIN_SITE = https://github.com/containernetworking/cni/releases/download/$(CNI_BIN_VERSION)
 CNI_BIN_SOURCE = cni-amd64-$(CNI_BIN_VERSION).tgz
 

deploy/iso/minikube-iso/package/cni-plugins-bin/cni-plugins-bin.hash

-sha256 49f14413c62f77d0ce5751934a28b291c4f04af47e591631d00e787ebcd07875  cni-plugins-amd64-v0.6.0-rc1.tgz
+sha256 f04339a21b8edf76d415e7f17b620e63b8f37a76b2f706671587ab6464411f2d  cni-plugins-amd64-v0.6.0.tgz

deploy/iso/minikube-iso/package/cni-plugins-bin/cni-plugins-bin.mk

@@ -4,7 +4,7 @@
 #
 ################################################################################
 
-CNI_PLUGINS_BIN_VERSION = v0.6.0-rc1
+CNI_PLUGINS_BIN_VERSION = v0.6.0
 CNI_PLUGINS_BIN_SITE = https://github.com/containernetworking/plugins/releases/download/$(CNI_PLUGINS_BIN_VERSION)
 CNI_PLUGINS_BIN_SOURCE = cni-plugins-amd64-$(CNI_PLUGINS_BIN_VERSION).tgz
 

deploy/iso/minikube-iso/package/cri-tools/Config.in

-config BR2_PACKAGE_CRI_TOOLS
-	bool "cri-tools"
-	default y
-	depends on BR2_PACKAGE_HOST_GO_ARCH_SUPPORTS
-	depends on BR2_PACKAGE_HOST_GO_CGO_LINKING_SUPPORTS

deploy/iso/minikube-iso/package/cri-tools/cri-tools.hash

-sha256 a357c67c891896032865f7a34f7ec330e5a00fe7f20b6d8be50399b91c99a4ac v1.11.1.tar.gz

deploy/iso/minikube-iso/package/cri-tools/cri-tools.mk

-################################################################################
-#
-# cri-tools
-#
-################################################################################
-
-CRI_TOOLS_VERSION = v1.11.1
-CRI_TOOLS_SITE = https://github.com/kubernetes-incubator/cri-tools/archive
-CRI_TOOLS_SOURCE = $(CRI_TOOLS_VERSION).tar.gz
-CRI_TOOLS_LICENSE = Apache-2.0
-CRI_TOOLS_LICENSE_FILES = LICENSE
-CRI_TOOLS_DEPENDENCIES = host-go
-CRI_TOOLS_GOPATH = $(@D)/_output
-CRI_TOOLS_ENV = \
-	CGO_ENABLED=1 \
-	GOPATH="$(CRI_TOOLS_GOPATH)" \
-	GOBIN="$(CRI_TOOLS_GOPATH)/bin" \
-	PATH=$(CRI_TOOLS_GOPATH)/bin:$(BR_PATH)
-
-
-define CRI_TOOLS_CONFIGURE_CMDS
-	mkdir -p $(CRI_TOOLS_GOPATH)/src/github.com/kubernetes-incubator
-	ln -sf $(@D) $(CRI_TOOLS_GOPATH)/src/github.com/kubernetes-incubator/cri-tools
-endef
-
-define CRI_TOOLS_BUILD_CMDS
-	$(CRI_TOOLS_ENV) $(MAKE) $(TARGET_CONFIGURE_OPTS) -C $(@D) crictl
-endef
-
-define CRI_TOOLS_INSTALL_TARGET_CMDS
-	$(INSTALL) -Dm755 \
-		$(CRI_TOOLS_GOPATH)/bin/crictl \
-		$(TARGET_DIR)/usr/bin/crictl
-endef
-
-$(eval $(generic-package))

deploy/iso/minikube-iso/package/crictl-bin/Config.in

+config BR2_PACKAGE_CRICTL_BIN
+	bool "crictl-bin"
+	default y
+	depends on BR2_x86_64

deploy/iso/minikube-iso/package/crictl-bin/crictl-bin.hash

+sha256 ccf83574556793ceb01717dc91c66b70f183c60c2bbec70283939aae8fdef768  crictl-v1.11.1-linux-amd64.tar.gz

deploy/iso/minikube-iso/package/crictl-bin/crictl-bin.mk

+################################################################################
+#
+# crictl-bin
+#
+################################################################################
+
+CRICTL_BIN_VERSION = v1.11.1
+CRICTL_BIN_SITE = https://github.com/kubernetes-sigs/cri-tools/releases/download/$(CRICTL_BIN_VERSION)
+CRICTL_BIN_SOURCE = crictl-$(CRICTL_BIN_VERSION)-linux-amd64.tar.gz
+CRICTL_BIN_STRIP_COMPONENTS = 0
+
+define CRICTL_BIN_INSTALL_TARGET_CMDS
+	$(INSTALL) -D -m 0755 \
+		$(@D)/crictl \
+		$(TARGET_DIR)/usr/bin/crictl
+endef
+
+$(eval $(generic-package))

deploy/iso/minikube-iso/package/docker-bin/docker-bin.hash

@@ -5,3 +5,4 @@ sha256 a9e90a73c3cdfbf238f148e1ec0eaff5eb181f92f35bdd938fd7dab18e1c4647  docker-
 sha256 77d3eaa72f2b63c94ea827b548f4a8b572b754a431c59258e3f2730411f64be7  docker-17.09.1-ce.tgz
 sha256 692e1c72937f6214b1038def84463018d8e320c8eaf8530546c84c2f8f9c767d  docker-17.12.0-ce.tgz
 sha256 1270dce1bd7e1838d62ae21d2505d87f16efc1d9074645571daaefdfd0c14054  docker-17.12.1-ce.tgz
+sha256 83be159cf0657df9e1a1a4a127d181725a982714a983b2bdcc0621244df93687  docker-18.06.1-ce.tgz

deploy/iso/minikube-iso/package/docker-bin/docker-bin.mk

@@ -4,7 +4,7 @@
 #
 ################################################################################
 
-DOCKER_BIN_VERSION = 17.12.1-ce
+DOCKER_BIN_VERSION = 18.06.1-ce
 DOCKER_BIN_SITE = https://download.docker.com/linux/static/stable/x86_64
 DOCKER_BIN_SOURCE = docker-$(DOCKER_BIN_VERSION).tgz
 

deploy/iso/minikube-iso/package/gluster/0002-fix-xdr-files.patch

---- a/rpc/xdr/src/changelog-xdr.x	2018-04-24 19:55:36.000000000 +0530
-+++ b/rpc/xdr/src/changelog-xdr.x	2018-06-21 19:41:11.146680931 +0530
-@@ -27,16 +27,16 @@
- /* XDR: changelog -> libgfchangelog */
- struct changelog_event_req {
-        /* sequence number for the buffer */
--       unsigned long seq;
-+       unsigned int seq;
- 
-        /* time of dispatch */
--       unsigned long tv_sec;
--       unsigned long tv_usec;
-+       unsigned int tv_sec;
-+       unsigned int tv_usec;
- };
- 
- struct changelog_event_rsp {
-        int op_ret;
- 
-        /* ack'd buffers sequence number */
--       unsigned long seq;
-+       unsigned int seq;
- };

deploy/iso/minikube-iso/package/gluster/0002-use-gnu-rpcgen.patch

+diff -ru gluster-4.1.5/rpc/xdr/gen/Makefile.am gluster-4.1.5.patched/rpc/xdr/gen/Makefile.am
+--- gluster-4.1.5/rpc/xdr/gen/Makefile.am	2018-09-21 16:13:45.958611272 +0200
++++ gluster-4.1.5.patched/rpc/xdr/gen/Makefile.am	2018-09-24 19:50:32.798967652 +0200
+@@ -19,14 +19,14 @@
+ # in the build. Or we do this crufty thing instead.
+ $(XDRSOURCES): $(XDRGENFILES)
+ 	@if [ ! -e $(xdrdst)/$@ -o $(@:.c=.x) -nt $(xdrdst)/$@ ]; then \
+-		rpcgen -c -o $(xdrdst)/$@ $(@:.c=.x) ;\
++		/usr/bin/rpcgen -c -o $(xdrdst)/$@ $(@:.c=.x) ;\
+ 	fi
+ 
+ # d*mn sed in netbsd6 doesn't do -i (inline)
+ # (why are we still running smoke on netbsd6 and not netbsd7?)
+ $(XDRHEADERS): $(XDRGENFILES)
+ 	@if [ ! -e $(xdrdst)/$@ -o $(@:.h=.x) -nt $(xdrdst)/$@ ]; then \
+-		rpcgen -h -o $(@:.h=.tmp) $(@:.h=.x) && \
++		/usr/bin/rpcgen -h -o $(@:.h=.tmp) $(@:.h=.x) && \
+ 		sed -e '/#ifndef/ s/-/_/g' -e '/#define/ s/-/_/g' \
+ 			-e '/#endif/ s/-/_/' -e 's/TMP_/H_/g' \
+ 			$(@:.h=.tmp) > $(xdrdst)/$@ && \
+Only in gluster-4.1.5.patched/rpc/xdr/gen: Makefile.am~
+diff -ru gluster-4.1.5/rpc/xdr/gen/Makefile.in gluster-4.1.5.patched/rpc/xdr/gen/Makefile.in
+--- gluster-4.1.5/rpc/xdr/gen/Makefile.in	2018-09-21 16:13:56.087638030 +0200
++++ gluster-4.1.5.patched/rpc/xdr/gen/Makefile.in	2018-09-24 19:51:06.198109046 +0200
+@@ -558,14 +558,14 @@
+ # in the build. Or we do this crufty thing instead.
+ $(XDRSOURCES): $(XDRGENFILES)
+ 	@if [ ! -e $(xdrdst)/$@ -o $(@:.c=.x) -nt $(xdrdst)/$@ ]; then \
+-		rpcgen -c -o $(xdrdst)/$@ $(@:.c=.x) ;\
++		/usr/bin/rpcgen -c -o $(xdrdst)/$@ $(@:.c=.x) ;\
+ 	fi
+ 
+ # d*mn sed in netbsd6 doesn't do -i (inline)
+ # (why are we still running smoke on netbsd6 and not netbsd7?)
+ $(XDRHEADERS): $(XDRGENFILES)
+ 	@if [ ! -e $(xdrdst)/$@ -o $(@:.h=.x) -nt $(xdrdst)/$@ ]; then \
+-		rpcgen -h -o $(@:.h=.tmp) $(@:.h=.x) && \
++		/usr/bin/rpcgen -h -o $(@:.h=.tmp) $(@:.h=.x) && \
+ 		sed -e '/#ifndef/ s/-/_/g' -e '/#define/ s/-/_/g' \
+ 			-e '/#endif/ s/-/_/' -e 's/TMP_/H_/g' \
+ 			$(@:.h=.tmp) > $(xdrdst)/$@ && \
+Only in gluster-4.1.5.patched/rpc/xdr/gen: Makefile.in~

deploy/iso/minikube-iso/package/gluster/gluster.hash

-sha512 52043cb298831281b96869a6b4bcc25277493a50f95e8fda7fa26cbfba80b5e5d204ba83b0f13299599eefb29f058ed8cfc1d54188695f76323567df03c0501d glusterfs-3.10.12.tar.gz
+sha512 ae557472b6a263e815b8c4d630b606f8e1334b6604799e499e6f53ea6ff60c2a696160fa427943cc3d01ffee91a57787c91f93a1b914179679038e63d291401f  glusterfs-4.1.5.tar.gz

deploy/iso/minikube-iso/package/gluster/gluster.mk

@@ -4,9 +4,9 @@
 #
 ################################################################################
 
-GLUSTER_VERSION = 3.10.12
-GLUSTER_SITE = https://download.gluster.org/pub/gluster/glusterfs/3.10/$(GLUSTER_VERSION)
+GLUSTER_VERSION = 4.1.5
+GLUSTER_SITE = https://download.gluster.org/pub/gluster/glusterfs/4.1/$(GLUSTER_VERSION)
 GLUSTER_SOURCE = glusterfs-$(GLUSTER_VERSION).tar.gz
-GLUSTER_CONF_OPTS = --disable-tiering --disable-ec-dynamic --disable-xmltest --disable-crypt-xlator --disable-georeplication --disable-ibverbs
+GLUSTER_CONF_OPTS = --disable-tiering --disable-ec-dynamic --disable-xmltest --disable-crypt-xlator --disable-georeplication --disable-ibverbs --disable-glupy --disable-gnfs --disable-cmocka --without-server
 GLUSTER_INSTALL_TARGET_OPTS = DESTDIR=$(TARGET_DIR) install
 $(eval $(autotools-package))

deploy/minikube/k8s_releases.json

-[
-  {
-    "version": "v1.10.0"
-  },  
-  {
-    "version": "v1.9.4"
-  },  
-  {
-    "version": "v1.9.0"
-  },
-  {
-    "version": "v1.8.0"
-  },
-  {
-    "version": "v1.7.5"
-  },
-  {
-    "version": "v1.7.4"
-  },
-  {
-    "version": "v1.7.3"
-  },
-  {
-    "version": "v1.7.2"
-  },
-  {
-    "version": "v1.7.0"
-  },
-  {
-    "version": "v1.7.0-rc.1"
-  },
-  {
-    "version": "v1.7.0-alpha.2"
-  },
-  {
-    "version": "v1.6.4"
-  },
-  {
-    "version": "v1.6.3"
-  },
-  {
-    "version": "v1.6.0"
-  },
-  {
-    "version": "v1.6.0-rc.1"
-  },
-  {
-    "version": "v1.6.0-beta.4"
-  },
-  {
-    "version": "v1.6.0-beta.3"
-  },
-  {
-    "version": "v1.6.0-beta.2"
-  },
-  {
-    "version": "v1.6.0-alpha.1"
-  },
-  {
-    "version": "v1.6.0-alpha.0"
-  },
-  {
-    "version": "v1.5.3"
-  },
-  {
-    "version": "v1.5.2"
-  },
-  {
-    "version": "v1.5.1"
-  },
-  {
-    "version": "v1.4.5"
-  },
-  {
-    "version": "v1.4.3"
-  },
-  {
-    "version": "v1.4.2"
-  },
-  {
-    "version": "v1.4.1"
-  },
-  {
-    "version": "v1.4.0"
-  },
-  {
-    "version": "v1.3.7"
-  },
-  {
-    "version": "v1.3.6"
-  },
-  {
-    "version": "v1.3.5"
-  },
-  {
-    "version": "v1.3.4"
-  },
-  {
-    "version": "v1.3.3"
-  },
-  {
-    "version": "v1.3.0"
-  }
-]
\ No newline at end of file

deploy/minikube/k8s_schema.json

-{
-  "$schema": "http://json-schema.org/draft-04/schema#",
-  "type": "array",
-  "items": {
-    "type": "object",
-    "properties": {
-      "version": {
-        "type": "string"
-      }
-    },
-    "required": [
-      "version"
-    ]
-  }
-}

deploy/minikube/releases.json

 [
+  {
+      "name": "v0.30.0",
+      "checksums": {
+          "darwin": "e09789c4eb751969f712947a43effd79cf73488163563e79d98bc3d15d06831e",
+          "linux": "f6fcd916adbdabc84fceb4ff3cadd58586f0ef6e576233b1bd03ead1f8f04afa",
+          "windows": "8f09d63c64a2a0c4810c492066b16ccd4bd63e2f3c2d0eb55e49c51c915493f6"
+      }
+  },
   {
       "name": "v0.29.0",
       "checksums": {

deploy/minikube/schema_check.go

@@ -26,7 +26,6 @@ import (
 
 func main() {
 	validateSchema("deploy/minikube/schema.json", "deploy/minikube/releases.json")
-	validateSchema("deploy/minikube/k8s_schema.json", "deploy/minikube/k8s_releases.json")
 	os.Exit(0)
 }
 

docs/alternative_runtimes.md

@@ -16,8 +16,7 @@ To use [CRI-O](https://github.com/kubernetes-incubator/cri-o) as the container r
 ```shell
 $ minikube start \
     --network-plugin=cni \
-    --container-runtime=cri-o \
-    --bootstrapper=kubeadm
+    --container-runtime=cri-o
 ```
 
 Or you can use the extended version:
@@ -27,8 +26,7 @@ $ minikube start \
     --network-plugin=cni \
     --extra-config=kubelet.container-runtime=remote \
     --extra-config=kubelet.container-runtime-endpoint=/var/run/crio/crio.sock \
-    --extra-config=kubelet.image-service-endpoint=/var/run/crio/crio.sock \
-    --bootstrapper=kubeadm
+    --extra-config=kubelet.image-service-endpoint=/var/run/crio/crio.sock
 ```
 
 ### Using containerd
@@ -38,8 +36,7 @@ To use [containerd](https://github.com/containerd/containerd) as the container r
 ```shell
 $ minikube start \
     --network-plugin=cni \
-    --container-runtime=containerd \
-    --bootstrapper=kubeadm
+    --container-runtime=containerd
 ```
 
 Or you can use the extended version:
@@ -49,6 +46,5 @@ $ minikube start \
     --network-plugin=cni \
     --extra-config=kubelet.container-runtime=remote \
     --extra-config=kubelet.container-runtime-endpoint=unix:///run/containerd/containerd.sock \
-    --extra-config=kubelet.image-service-endpoint=unix:///run/containerd/containerd.sock \
-    --bootstrapper=kubeadm
-```
\ No newline at end of file
+    --extra-config=kubelet.image-service-endpoint=unix:///run/containerd/containerd.sock
+```

docs/contributors/build_guide.md

@@ -21,12 +21,15 @@ $ cd $GOPATH/src/k8s.io/minikube
 $ make
 ```
 
+Note: Make sure that you uninstall any previous versions of minikube before building
+from the source.
+
 ### Building from Source in Docker (using Debian stretch image with golang)
 Clone minikube:
 ```shell
 $ git clone https://github.com/kubernetes/minikube.git
 ```
-Build (cross complile for linux / OS X and Windows) using make:
+Build (cross compile for linux / OS X and Windows) using make:
 ```shell
 $ cd minikube
 $ docker run --rm -v "$PWD":/go/src/k8s.io/minikube -w /go/src/k8s.io/minikube golang:stretch make cross

docs/contributors/releasing_minikube.md

@@ -2,8 +2,13 @@
 
 ## Create a Release Notes PR
 
-Assemble all the meaningful changes since the last release into the CHANGELOG.md file.
-See [this PR](https://github.com/kubernetes/minikube/pull/164) for an example.
+Collect the release notes, and edit them as necessary:
+
+```shell
+hack/release_notes.sh
+```
+
+Then merge into the CHANGELOG.md file.  See [this PR](https://github.com/kubernetes/minikube/pull/164) for an example.
 
 ## Build and Release a New ISO
 

docs/networking.md

@@ -10,3 +10,53 @@ To determine the NodePort for your service, you can use a `kubectl` command like
 We also have a shortcut for fetching the minikube IP and a service's `NodePort`:
 
 `minikube service --url $SERVICE`
+
+### LoadBalancer emulation (`minikube tunnel`)
+
+Services of type `LoadBalancer` can be exposed via the `minikube tunnel` command. 
+
+````shell
+minikube tunnel
+```` 
+
+Will output:
+
+```
+out/minikube tunnel
+Password: *****
+Status: 
+        machine: minikube
+        pid: 59088
+        route: 10.96.0.0/12 -> 192.168.99.101
+        minikube: Running
+        services: []
+    errors: 
+                minikube: no errors
+                router: no errors
+                loadbalancer emulator: no errors
+
+
+```` 
+
+Tunnel might ask you for password for creating and deleting network routes.
+ 
+# Cleaning up orphaned routes 
+
+If the `minikube tunnel` shuts down in an unclean way, it might leave a network route around.
+This case the ~/.minikube/tunnels.json file will contain an entry for that tunnel. 
+To cleanup orphaned routes, run: 
+````
+minikube tunnel --cleanup
+````
+
+# (Advanced) Running tunnel as root to avoid entering password multiple times 
+
+`minikube tunnel` runs as a separate daemon, creates a network route on the host to the service CIDR of the cluster using the cluster's IP address as a gateway. 
+Adding a route requires root privileges for the user, and thus there are differences in how to run `minikube tunnel` depending on the OS.    
+
+Recommended way to use on Linux with KVM2 driver and MacOSX with Hyperkit driver: 
+
+`sudo -E minikube tunnel`
+
+Using VirtualBox on Windows, Mac and Linux _both_ `minikube start` and `minikube tunnel` needs to be started from the same Administrator user session otherwise [VBoxManage can't recognize the created VM](https://forums.virtualbox.org/viewtopic.php?f=6&t=81551).     
+

docs/tunnel.md

+# Minikube Tunnel Design Doc
+
+## Background
+
+Minikube today only exposes a single IP address for all cluster and VM communication.
+This effectively requires users to connect to any running Pods, Services or LoadBalancers over ClusterIPs, which can require modifications to workflows when compared to developing against a production cluster.
+
+A main goal of Minikube is to minimize the differences required in code and configuration between development and production, so this is not ideal.
+If all cluster IP addresses and Load Balancers were made available on the minikube host machine, these modifications would not be necessary and users would get the "magic" experience of developing from inside a cluster.
+
+Tools like telepresence.io, sshuttle, and the OpenVPN chart provide similar capabilities already.
+
+Also, Steve Sloka has provided a very detailed guide on how to setup a similar configuration [manually](https://stevesloka.com/2017/06/12/access-minikube-service-from-linux-host/).
+
+Elson Rodriguez has provided a similar guide, including a Minikube [external LB controller](https://github.com/elsonrodriguez/minikube-lb-patch).
+
+## Example usage
+
+```shell
+$ minikube tunnel
+Starting minikube tunnel process. Press Ctrl+C to exit.
+All cluster IPs and load balancers are now available from your host machine.
+```
+
+## Overview
+
+We will introduce a new command, `minikube tunnel`, that must be run with root permissions.
+This command will:
+
+* Establish networking routes from the host into the VM for all IP ranges used by Kubernetes.
+* Enable a cluster controller that allocates IPs to services external `LoadBalancer` IPs.
+* Clean up routes and IPs when stopped, or when `minikube` stops.
+
+Additionally, we will introduce a Minikube LoadBalancer controller that manages a CIDR of IPs and assigns them to services of type `LoadBalancer`.
+These IPs will also be made available on the host machine.
+
+## Network Routes
+
+Minikube drivers usually establish "host-only" IP addresses (192.168.1.1, for example) that route into the running VM
+from the host.
+
+The new `minikube tunnel` command will create a static routing table entry that maps the CIDRs used by Pods, Services and LoadBalancers to the host-only IP, obtainable via the `minikube ip` command.
+
+The commands below detail adding routes for the entire `/8` block, we should probably add individual entries for each CIDR we manage instead.
+
+### Linux
+
+Route entries for the entire 10.* block can be added via:
+
+```shell
+sudo ip route add 10.0.0.0/8 via $(minikube ip)
+```
+
+and deleted via:
+
+```shell
+sudo ip route delete 10.0.0.0/8
+```
+
+The routing table can be queried with `netstat -nr -f inet` 
+
+### OSX
+
+Route entries can be added via:
+
+```shell
+sudo route -n add 10.0.0.0/8 $(minikube ip)
+```
+
+and deleted via:
+
+```shell
+sudo route -n delete 10.0.0.0/8
+
+```
+
+The routing table can be queried with `netstat -nr -f inet`
+
+### Windows
+
+Route entries can be added via:
+
+```shell
+route ADD 10.0.0.0 MASK 255.0.0.0 <minikube ip>
+```
+
+and deleted via:
+
+```shell
+route DELETE 10.0.0.0 
+```
+
+The routing table can be queried with `route print -4`
+
+### Handling unclean shutdowns 
+
+Unclean shutdowns of the tunnel process can result in partially executed cleanup process, leaving network routes in the routing table. 
+We will keep track of the routes created by each tunnel in a centralized location in the main minikube config directory. 
+This list serves as a registry for tunnels containing information about 
+- machine profile 
+- process ID 
+- and the route that was created
+
+The cleanup command cleans the routes from both the routing table and the registry for tunnels that are not running: 
+ 
+``` 
+minikube tunnel --cleanup
+```
+
+Updating the tunnel registry and the routing table is an atomic transaction: 
+
+- create route in the routing table + create registry entry if both are successful, otherwise rollback  
+- delete route in the routing table + remove registry entry if both are successful, otherwise rollback 
+
+*Note*: because we don't support currently real multi cluster setup (due to overlapping CIDRs), the handling of running/not-running processes is not strictly required however it is forward looking.  
+
+### Handling routing table conflicts  
+
+A routing table conflict happens when a destination CIDR of the route required by the tunnel overlaps with an existing route. 
+Minikube tunnel will warn the user if this happens and should not create the rule. 
+There should not be any automated removal of conflicting routes.
+
+*Note*: If the user removes the minikube config directory, this might leave conflicting rules in the network routing table that will have to be cleaned up manually.  
+
+
+## Load Balancer Controller
+
+In addition to making IPs routable, minikube tunnel will assign an external IP (the ClusterIP) to all services of type `LoadBalancer`.
+
+The logic of this controller will be, roughly:
+
+```python
+for service in services:
+  if service.type == "LoadBalancer" and len(service.ingress) == 0:
+    add_ip_to_service(ClusterIP, service)
+sleep
+```
+
+Note that the Minikube ClusterIP can change over time (during system reboots) and this loop should also handle reconcilliation of those changes.
+
+## Handling multiple clusters 
+
+Multiple clusters are currently not supported due to our inability to specify ServiceCIDR. 
+This causes conflicting routes having the same destination CIDR.

hack/boilerplate/boilerplate.py

@@ -115,7 +115,7 @@ def normalize_files(files):
         newfiles.append(pathname)
     for i, pathname in enumerate(newfiles):
         if not os.path.isabs(pathname):
-            newfiles[i] = os.path.join(args.rootdir, pathname)
+            newfiles[i] = os.path.join(rootdir, pathname)
     return newfiles
 
 def get_files(extensions):

hack/boilerplate/fix.sh

+#!/usr/bin/env bash
+
+# Copyright 2018 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+export readonly ROOT_DIR=${1:-$(pwd)}
+
+function prepend() {
+  local ignore="vendor\|\_gopath\|assets.go"
+  local pattern=$1
+  local ref=$2
+  local headers=$3
+  local files=$(hack/boilerplate/boilerplate.py --rootdir ${ROOT_DIR} | grep -v "$ignore" | grep "$pattern")
+  for f in ${files}; do
+    echo ${f};
+    local copyright="$(cat hack/boilerplate/boilerplate.${ref}.txt | sed s/YEAR/$(date +%Y)/g)"
+    local file_headers=""
+
+    if [ "${headers}" != "" ]; then
+      file_headers="$(cat ${f} | grep ${headers})"
+    fi
+
+    if [ "${file_headers}" != "" ]; then
+        fileContent="$(cat ${f} | grep -v ${headers})"
+        printf '%s\n\n%s\n%s\n' "$file_headers" "${copyright}" "$fileContent" > ${f}
+    else
+      fileContent="$(cat ${f})"
+      printf '%s\n\n%s\n' "${copyright}" "$fileContent" > ${f}
+    fi
+
+    done
+}
+
+prepend "\.go" "go" "+build"
+prepend "\.py" "py"
+prepend "\.sh" "sh" "#!"
+prepend Makefile Makefile
+prepend Dockerfile Dockerfile

hack/jenkins/common.sh

@@ -36,14 +36,10 @@ fi
 
 # Add the out/ directory to the PATH, for using new drivers.
 export PATH="$(pwd)/out/":$PATH
-gsutil cp gs://minikube-builds/${MINIKUBE_LOCATION}/minikube-${OS_ARCH} out/
-gsutil cp gs://minikube-builds/${MINIKUBE_LOCATION}/docker-machine-driver-* out/
-gsutil cp gs://minikube-builds/${MINIKUBE_LOCATION}/e2e-${OS_ARCH} out/
-gsutil cp gs://minikube-builds/${MINIKUBE_LOCATION}/testdata/busybox.yaml testdata/
-gsutil cp gs://minikube-builds/${MINIKUBE_LOCATION}/testdata/pvc.yaml testdata/
-gsutil cp gs://minikube-builds/${MINIKUBE_LOCATION}/testdata/busybox-mount-test.yaml testdata/
-gsutil cp gs://minikube-builds/${MINIKUBE_LOCATION}/testdata/nginx-pod-svc.yaml testdata/
-gsutil cp gs://minikube-builds/${MINIKUBE_LOCATION}/testdata/nginx-ing.yaml testdata/
+gsutil -m cp gs://minikube-builds/${MINIKUBE_LOCATION}/minikube-${OS_ARCH} out/
+gsutil -m cp gs://minikube-builds/${MINIKUBE_LOCATION}/docker-machine-driver-* out/
+gsutil -m cp gs://minikube-builds/${MINIKUBE_LOCATION}/e2e-${OS_ARCH} out/
+gsutil -m cp gs://minikube-builds/${MINIKUBE_LOCATION}/testdata/* testdata/
 
 # Set the executable bit on the e2e binary and out binary
 chmod +x out/e2e-${OS_ARCH}

hack/jenkins/minishift_linux_integration_tests_kvm.sh

@@ -35,8 +35,8 @@ EXTRA_FLAGS="--show-libmachine-logs"
 mkdir -p out/
 
 curl -L http://artifacts.ci.centos.org/minishift/minishift/master/latest/linux-amd64/minishift -o out/minishift
-gsutil cp gs://minikube-builds/${MINIKUBE_LOCATION}/docker-machine-driver-* out/ || true
-gsutil cp gs://minikube-builds/${MINIKUBE_LOCATION}/minikube-testing.iso out/ || true
+gsutil -m cp gs://minikube-builds/${MINIKUBE_LOCATION}/docker-machine-driver-* out/ || true
+gsutil -m cp gs://minikube-builds/${MINIKUBE_LOCATION}/minikube-testing.iso out/ || true
 
 # Set the executable bit on the minishift and driver binary
 chmod +x out/minishift

hack/jenkins/unix_white_box_integration_tests.sh

+#!/bin/bash
+
+# Copyright 2018 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+for pkg in $(go list ./pkg/minikube/...); do
+    #we are assuming that the runner has NOPASSWD sudoer
+    go test -v $pkg -tags integration
+done

hack/jenkins/windows_integration_test_virtualbox.ps1

@@ -13,9 +13,9 @@
 # limitations under the License.
 
 mkdir -p out
-gsutil.cmd cp gs://minikube-builds/$env:MINIKUBE_LOCATION/minikube-windows-amd64.exe out/
-gsutil.cmd cp gs://minikube-builds/$env:MINIKUBE_LOCATION/e2e-windows-amd64.exe out/
-gsutil.cmd cp -r gs://minikube-builds/$env:MINIKUBE_LOCATION/testdata .
+gsutil.cmd -m cp gs://minikube-builds/$env:MINIKUBE_LOCATION/minikube-windows-amd64.exe out/
+gsutil.cmd -m cp gs://minikube-builds/$env:MINIKUBE_LOCATION/e2e-windows-amd64.exe out/
+gsutil.cmd -m cp -r gs://minikube-builds/$env:MINIKUBE_LOCATION/testdata .
 
 ./out/minikube-windows-amd64.exe delete
 

hack/jenkins/windows_integration_test_virtualbox_kubeadm_crio.ps1

@@ -13,9 +13,9 @@
 # limitations under the License.
 
 mkdir -p out
-gsutil.cmd cp gs://minikube-builds/$env:MINIKUBE_LOCATION/minikube-windows-amd64.exe out/
-gsutil.cmd cp gs://minikube-builds/$env:MINIKUBE_LOCATION/e2e-windows-amd64.exe out/
-gsutil.cmd cp -r gs://minikube-builds/$env:MINIKUBE_LOCATION/testdata .
+gsutil.cmd -m cp gs://minikube-builds/$env:MINIKUBE_LOCATION/minikube-windows-amd64.exe out/
+gsutil.cmd -m cp gs://minikube-builds/$env:MINIKUBE_LOCATION/e2e-windows-amd64.exe out/
+gsutil.cmd -m cp -r gs://minikube-builds/$env:MINIKUBE_LOCATION/testdata .
 
 
 ./out/minikube-windows-amd64.exe delete

hack/release_notes.sh

+#!/bin/bash
+
+# Copyright 2018 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eux -o pipefail
+
+DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
+
+go run "${DIR}/release_notes/listpullreqs.go"
+
+echo "Huge thank you for this release towards our contributors: "
+git log "$(git describe  --abbrev=0)".. --format="%aN" --reverse | sort | uniq | awk '{printf "- %s\n", $0 }'

hack/release_notes/listpullreqs.go

+/*
+Copyright 2018 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// listpullreqs.go lists pull requests since the last release.
+package main
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/google/go-github/github"
+	"github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+	"golang.org/x/oauth2"
+)
+
+var (
+	token   string
+	fromTag string
+	toTag   string
+)
+
+var rootCmd = &cobra.Command{
+	Use:        "listpullreqs fromTag toTag",
+	Short:      "Lists pull requests between two versions in our changelog markdown format",
+	ArgAliases: []string{"fromTag", "toTag"},
+	Run: func(cmd *cobra.Command, args []string) {
+		printPullRequests()
+	},
+}
+
+const org = "kubernetes"
+const repo = "minikube"
+
+func main() {
+	rootCmd.Flags().StringVar(&token, "token", "", "Specify personal Github Token if you are hitting a rate limit anonymously. https://github.com/settings/tokens")
+	rootCmd.Flags().StringVar(&fromTag, "fromTag", "", "comparison of commits is based on this tag (defaults to the latest tag in the repo)")
+	rootCmd.Flags().StringVar(&toTag, "toTag", "master", "this is the commit that is compared with fromTag")
+	if err := rootCmd.Execute(); err != nil {
+		logrus.Fatal(err)
+	}
+}
+
+func printPullRequests() {
+	client := getClient()
+
+	releases, _, err := client.Repositories.ListReleases(context.Background(), org, repo, &github.ListOptions{})
+	if err != nil {
+		logrus.Fatal(err)
+	}
+	lastReleaseTime := *releases[0].PublishedAt
+	fmt.Println(fmt.Sprintf("Collecting pull request that were merged since the last release: %s (%s)", *releases[0].TagName, lastReleaseTime))
+
+	listSize := 1
+	for page := 1; listSize > 0; page++ {
+		pullRequests, _, err := client.PullRequests.List(context.Background(), org, repo, &github.PullRequestListOptions{
+			State:     "closed",
+			Sort:      "updated",
+			Direction: "desc",
+			ListOptions: github.ListOptions{
+				PerPage: 100,
+				Page:    page,
+			},
+		})
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		seen := 0
+		for idx := range pullRequests {
+			pr := pullRequests[idx]
+			if pr.MergedAt != nil {
+				if pr.GetMergedAt().After(lastReleaseTime.Time) {
+					fmt.Printf("* %s [#%d](https://github.com/%s/%s/pull/%d)\n", pr.GetTitle(), *pr.Number, org, repo, *pr.Number)
+					seen++
+				}
+			}
+		}
+		if seen == 0 {
+			break
+		}
+		listSize = len(pullRequests)
+	}
+}
+
+func getClient() *github.Client {
+	if len(token) <= 0 {
+		return github.NewClient(nil)
+	}
+	ctx := context.Background()
+	ts := oauth2.StaticTokenSource(
+		&oauth2.Token{AccessToken: token},
+	)
+	tc := oauth2.NewClient(ctx, ts)
+	return github.NewClient(tc)
+}

pkg/minikube/assets/addons.go

@@ -203,6 +203,11 @@ var Addons = map[string]*Addon{
 			constants.AddonsPath,
 			"ingress-configmap.yaml",
 			"0640"),
+		NewBinDataAsset(
+			"deploy/addons/ingress/ingress-rbac.yaml",
+			constants.AddonsPath,
+			"ingress-rbac.yaml",
+			"0640"),
 		NewBinDataAsset(
 			"deploy/addons/ingress/ingress-dp.yaml",
 			constants.AddonsPath,

pkg/minikube/bootstrapper/kubeadm/kubeadm.go

@@ -119,12 +119,17 @@ func (k *KubeadmBootstrapper) StartCluster(k8s config.KubernetesConfig) error {
 		KubeadmConfigFile   string
 		SkipPreflightChecks bool
 		Preflights          []string
+		DNSAddon            string
 	}{
 		KubeadmConfigFile: constants.KubeadmConfigFile,
 		SkipPreflightChecks: !VersionIsBetween(version,
 			semver.MustParse("1.9.0-alpha.0"),
 			semver.Version{}),
 		Preflights: constants.Preflights,
+		DNSAddon:   "kube-dns",
+	}
+	if version.GTE(semver.MustParse("1.12.0")) {
+		templateContext.DNSAddon = "coredns"
 	}
 	if err := kubeadmInitTemplate.Execute(&b, templateContext); err != nil {
 		return err
@@ -380,7 +385,7 @@ func generateConfig(k8s config.KubernetesConfig) (string, error) {
 		NodeName:          k8s.NodeName,
 		ExtraArgs:         extraComponentConfig,
 		FeatureArgs:       kubeadmFeatureArgs,
-		NoTaintMaster:     false,
+		NoTaintMaster:     false, // That does not work with k8s 1.12+
 	}
 
 	if version.GTE(semver.MustParse("1.10.0-alpha.0")) {
@@ -388,6 +393,10 @@ func generateConfig(k8s config.KubernetesConfig) (string, error) {
 	}
 
 	b := bytes.Buffer{}
+	kubeadmConfigTemplate := kubeadmConfigTemplateV1Alpha1
+	if version.GTE(semver.MustParse("1.12.0")) {
+		kubeadmConfigTemplate = kubeadmConfigTemplateV1Alpha3
+	}
 	if err := kubeadmConfigTemplate.Execute(&b, opts); err != nil {
 		return "", err
 	}

pkg/minikube/bootstrapper/kubeadm/templates.go

@@ -22,7 +22,7 @@ import (
 	"text/template"
 )
 
-var kubeadmConfigTemplate = template.Must(template.New("kubeadmConfigTemplate").Funcs(template.FuncMap{
+var kubeadmConfigTemplateV1Alpha1 = template.Must(template.New("kubeadmConfigTemplate-v1alpha1").Funcs(template.FuncMap{
 	"printMapInOrder": printMapInOrder,
 }).Parse(`apiVersion: kubeadm.k8s.io/v1alpha1
 kind: MasterConfiguration
@@ -44,6 +44,44 @@ nodeName: {{.NodeName}}
   {{$i}}: {{$val}}{{end}}
 {{end}}`))
 
+var kubeadmConfigTemplateV1Alpha3 = template.Must(template.New("kubeadmConfigTemplate-v1alpha3").Funcs(template.FuncMap{
+	"printMapInOrder": printMapInOrder,
+}).Parse(`apiEndpoint:
+  advertiseAddress: {{.AdvertiseAddress}}
+  bindPort: {{.APIServerPort}}
+apiVersion: kubeadm.k8s.io/v1alpha3
+bootstrapTokens:
+- groups:
+  - system:bootstrappers:kubeadm:default-node-token
+  ttl: 24h0m0s
+  usages:
+  - signing
+  - authentication
+kind: InitConfiguration
+nodeRegistration:
+  criSocket: /var/run/dockershim.sock
+  name: {{.NodeName}}
+  taints: []
+---
+{{range .ExtraArgs}}{{.Component}}:{{range $i, $val := printMapInOrder .Options ": " }}
+  {{$val}}{{end}}
+{{end}}{{if .FeatureArgs}}featureGates: {{range $i, $val := .FeatureArgs}}
+  {{$i}}: {{$val}}{{end}}
+{{end}}
+apiVersion: kubeadm.k8s.io/v1alpha3
+certificatesDir: {{.CertDir}}
+clusterName: kubernetes
+controlPlaneEndpoint: localhost:{{.APIServerPort}}
+etcd:
+  local:
+    dataDir: {{.EtcdDataDir}}
+kind: ClusterConfiguration
+kubernetesVersion: {{.KubernetesVersion}}
+networking:
+  dnsDomain: cluster.local
+  podSubnet: ""
+  serviceSubnet: {{.ServiceCIDR}}`))
+
 var kubeletSystemdTemplate = template.Must(template.New("kubeletSystemdTemplate").Parse(`
 [Unit]
 {{if or (eq .ContainerRuntime "cri-o") (eq .ContainerRuntime "cri")}}Wants=crio.service{{else}}Wants=docker.socket{{end}}
@@ -79,7 +117,7 @@ sudo /usr/bin/kubeadm alpha phase etcd local --config {{.KubeadmConfigFile}}
 
 var kubeadmInitTemplate = template.Must(template.New("kubeadmInitTemplate").Parse(`
 sudo /usr/bin/kubeadm init --config {{.KubeadmConfigFile}} {{if .SkipPreflightChecks}}--skip-preflight-checks{{else}}{{range .Preflights}}--ignore-preflight-errors={{.}} {{end}}{{end}} &&
-sudo /usr/bin/kubeadm alpha phase addon kube-dns
+sudo /usr/bin/kubeadm alpha phase addon {{ .DNSAddon }}
 `))
 
 // printMapInOrder sorts the keys and prints the map in order, combining key

pkg/minikube/bootstrapper/kubeadm/versions.go

@@ -241,7 +241,6 @@ var versionSpecificOpts = []VersionedExtraOption{
 	NewUnversionedOption(Kubelet, "client-ca-file", path.Join(util.DefaultCertPath, "ca.crt")),
 
 	// Cgroup args
-	NewUnversionedOption(Kubelet, "cadvisor-port", "0"),
 	NewUnversionedOption(Kubelet, "cgroup-driver", "cgroupfs"),
 	{
 		Option: util.ExtraOption{
@@ -260,6 +259,14 @@ var versionSpecificOpts = []VersionedExtraOption{
 		},
 		GreaterThanOrEqual: semver.MustParse("1.11.0-alpha.0"),
 	},
+	{
+		Option: util.ExtraOption{
+			Component: Kubelet,
+			Key:       "cadvisor-port",
+			Value:     "0",
+		},
+		LessThanOrEqual: semver.MustParse("1.11.1000"),
+	},
 }
 
 func VersionIsBetween(version, gte, lte semver.Version) bool {

pkg/minikube/cluster/cluster.go

@@ -69,10 +69,10 @@ func StartHost(api libmachine.API, config cfg.MachineConfig) (*host.Host, error)
 		glog.Infoln("Machine does not exist... provisioning new machine")
 		glog.Infof("Provisioning machine with config: %+v", config)
 		return createHost(api, config)
-	} else {
-		glog.Infoln("Skipping create...Using existing machine configuration")
 	}
 
+	glog.Infoln("Skipping create...Using existing machine configuration")
+
 	h, err := api.Load(cfg.GetMachineName())
 	if err != nil {
 		return nil, errors.Wrap(err, "Error loading existing host. Please try running [minikube delete], then run [minikube start] again.")
@@ -152,8 +152,8 @@ func GetHostStatus(api libmachine.API) (string, error) {
 }
 
 // GetHostDriverIP gets the ip address of the current minikube cluster
-func GetHostDriverIP(api libmachine.API) (net.IP, error) {
-	host, err := CheckIfApiExistsAndLoad(api)
+func GetHostDriverIP(api libmachine.API, machineName string) (net.IP, error) {
+	host, err := CheckIfHostExistsAndLoad(api, machineName)
 	if err != nil {
 		return nil, err
 	}
@@ -164,7 +164,7 @@ func GetHostDriverIP(api libmachine.API) (net.IP, error) {
 	}
 	ip := net.ParseIP(ipStr)
 	if ip == nil {
-		return nil, errors.Wrap(err, "Error parsing IP")
+		return nil, fmt.Errorf("error parsing IP: %s", ipStr)
 	}
 	return ip, nil
 }
@@ -251,7 +251,7 @@ func createHost(api libmachine.API, config cfg.MachineConfig) (*host.Host, error
 
 // GetHostDockerEnv gets the necessary docker env variables to allow the use of docker through minikube's vm
 func GetHostDockerEnv(api libmachine.API) (map[string]string, error) {
-	host, err := CheckIfApiExistsAndLoad(api)
+	host, err := CheckIfHostExistsAndLoad(api, cfg.GetMachineName())
 	if err != nil {
 		return nil, errors.Wrap(err, "Error checking that api exists and loading it")
 	}
@@ -273,7 +273,7 @@ func GetHostDockerEnv(api libmachine.API) (map[string]string, error) {
 
 // MountHost runs the mount command from the 9p client on the VM to the 9p server on the host
 func MountHost(api libmachine.API, ip net.IP, path, port, mountVersion string, uid, gid, msize int) error {
-	host, err := CheckIfApiExistsAndLoad(api)
+	host, err := CheckIfHostExistsAndLoad(api, cfg.GetMachineName())
 	if err != nil {
 		return errors.Wrap(err, "Error checking that api exists and loading it")
 	}
@@ -344,24 +344,25 @@ func getIPForInterface(name string) (net.IP, error) {
 	return nil, errors.Errorf("Error finding IPV4 address for %s", name)
 }
 
-func CheckIfApiExistsAndLoad(api libmachine.API) (*host.Host, error) {
-	exists, err := api.Exists(cfg.GetMachineName())
+func CheckIfHostExistsAndLoad(api libmachine.API, machineName string) (*host.Host, error) {
+	exists, err := api.Exists(machineName)
 	if err != nil {
-		return nil, errors.Wrapf(err, "Error checking that api exists for: %s", cfg.GetMachineName())
+		return nil, errors.Wrapf(err, "Error checking that machine exists: %s", machineName)
 	}
 	if !exists {
-		return nil, errors.Errorf("Machine does not exist for api.Exists(%s)", cfg.GetMachineName())
+		return nil, errors.Errorf("Machine does not exist for api.Exists(%s)", machineName)
 	}
 
-	host, err := api.Load(cfg.GetMachineName())
+	host, err := api.Load(machineName)
 	if err != nil {
-		return nil, errors.Wrapf(err, "Error loading api for: %s", cfg.GetMachineName())
+		return nil, errors.Wrapf(err, "Error loading store for: %s", machineName)
 	}
 	return host, nil
 }
 
 func CreateSSHShell(api libmachine.API, args []string) error {
-	host, err := CheckIfApiExistsAndLoad(api)
+	machineName := cfg.GetMachineName()
+	host, err := CheckIfHostExistsAndLoad(api, machineName)
 	if err != nil {
 		return errors.Wrap(err, "Error checking if api exist and loading it")
 	}
@@ -372,7 +373,7 @@ func CreateSSHShell(api libmachine.API, args []string) error {
 	}
 
 	if currentState != state.Running {
-		return errors.Errorf("Error: Cannot run ssh command: Host %q is not running", cfg.GetMachineName())
+		return errors.Errorf("Error: Cannot run ssh command: Host %q is not running", machineName)
 	}
 
 	client, err := host.CreateSSHClient()

pkg/minikube/cluster/cluster_test.go

@@ -271,7 +271,7 @@ func TestDeleteHostMultipleErrors(t *testing.T) {
 		t.Fatal("Expected error deleting host, didn't get one.")
 	}
 
-	expectedErrors := []string{"Error removing " + config.GetMachineName(), "Error deleting machine"}
+	expectedErrors := []string{"error removing " + config.GetMachineName(), "error deleting machine"}
 	for _, expectedError := range expectedErrors {
 		if !strings.Contains(err.Error(), expectedError) {
 			t.Fatalf("Error %v expected to contain: %s.", err, expectedError)

pkg/minikube/config/config.go

@@ -23,6 +23,8 @@ import (
 	"io"
 	"os"
 
+	"io/ioutil"
+
 	"github.com/spf13/viper"
 	"k8s.io/minikube/pkg/minikube/constants"
 )
@@ -88,3 +90,37 @@ func GetMachineName() string {
 	}
 	return viper.GetString(MachineProfile)
 }
+
+// Load loads the kubernetes and machine config for the current machine
+func Load() (Config, error) {
+	return DefaultLoader.LoadConfigFromFile(GetMachineName())
+}
+
+// Loader loads the kubernetes and machine config based on the machine profile name
+type Loader interface {
+	LoadConfigFromFile(profile string) (Config, error)
+}
+
+type simpleConfigLoader struct{}
+
+var DefaultLoader Loader = &simpleConfigLoader{}
+
+func (c *simpleConfigLoader) LoadConfigFromFile(profile string) (Config, error) {
+	var cc Config
+
+	path := constants.GetProfileFile(profile)
+
+	if _, err := os.Stat(path); os.IsNotExist(err) {
+		return cc, err
+	}
+
+	data, err := ioutil.ReadFile(path)
+	if err != nil {
+		return cc, err
+	}
+
+	if err := json.Unmarshal(data, &cc); err != nil {
+		return cc, err
+	}
+	return cc, nil
+}

pkg/minikube/constants/constants.go

@@ -87,6 +87,10 @@ const DefaultStorageClassProvisioner = "standard"
 // Used to modify the cache field in the config file
 const Cache = "cache"
 
+func TunnelRegistryPath() string {
+	return filepath.Join(GetMinipath(), "tunnels.json")
+}
+
 // MakeMiniPath is a utility to calculate a relative path to our directory.
 func MakeMiniPath(fileName ...string) string {
 	args := []string{GetMinipath()}
@@ -110,7 +114,6 @@ const (
 	DefaultConfigViewFormat    = "- {{.ConfigKey}}: {{.ConfigValue}}\n"
 	DefaultCacheListFormat     = "{{.CacheImage}}\n"
 	GithubMinikubeReleasesURL  = "https://storage.googleapis.com/minikube/releases.json"
-	KubernetesVersionGCSURL    = "https://storage.googleapis.com/minikube/k8s_releases.json"
 	DefaultWait                = 20
 	DefaultInterval            = 6
 	DefaultClusterBootstrapper = "kubeadm"

pkg/minikube/service/service.go

@@ -25,6 +25,8 @@ import (
 	"time"
 
 	"github.com/docker/machine/libmachine"
+	"github.com/golang/glog"
+
 	"github.com/pkg/browser"
 	"github.com/pkg/errors"
 	"k8s.io/api/core/v1"
@@ -78,6 +80,7 @@ func (*K8sClientGetter) GetClientset() (*kubernetes.Clientset, error) {
 	if err != nil {
 		return nil, fmt.Errorf("Error creating kubeConfig: %v", err)
 	}
+	clientConfig.Timeout = 1 * time.Second
 	client, err := kubernetes.NewForConfig(clientConfig)
 	if err != nil {
 		return nil, errors.Wrap(err, "Error creating new client from kubeConfig.ClientConfig()")
@@ -86,18 +89,18 @@ func (*K8sClientGetter) GetClientset() (*kubernetes.Clientset, error) {
 	return client, nil
 }
 
-type ServiceURL struct {
+type URL struct {
 	Namespace string
 	Name      string
 	URLs      []string
 }
 
-type ServiceURLs []ServiceURL
+type URLs []URL
 
 // Returns all the node port URLs for every service in a particular namespace
 // Accepts a template for formatting
-func GetServiceURLs(api libmachine.API, namespace string, t *template.Template) (ServiceURLs, error) {
-	host, err := cluster.CheckIfApiExistsAndLoad(api)
+func GetServiceURLs(api libmachine.API, namespace string, t *template.Template) (URLs, error) {
+	host, err := cluster.CheckIfHostExistsAndLoad(api, config.GetMachineName())
 	if err != nil {
 		return nil, err
 	}
@@ -119,13 +122,13 @@ func GetServiceURLs(api libmachine.API, namespace string, t *template.Template)
 		return nil, err
 	}
 
-	var serviceURLs []ServiceURL
+	var serviceURLs []URL
 	for _, svc := range svcs.Items {
 		urls, err := printURLsForService(client, ip, svc.Name, svc.Namespace, t)
 		if err != nil {
 			return nil, err
 		}
-		serviceURLs = append(serviceURLs, ServiceURL{Namespace: svc.Namespace, Name: svc.Name, URLs: urls})
+		serviceURLs = append(serviceURLs, URL{Namespace: svc.Namespace, Name: svc.Name, URLs: urls})
 	}
 
 	return serviceURLs, nil
@@ -134,7 +137,7 @@ func GetServiceURLs(api libmachine.API, namespace string, t *template.Template)
 // Returns all the node ports for a service in a namespace
 // with optional formatting
 func GetServiceURLsForService(api libmachine.API, namespace, service string, t *template.Template) ([]string, error) {
-	host, err := cluster.CheckIfApiExistsAndLoad(api)
+	host, err := cluster.CheckIfHostExistsAndLoad(api, config.GetMachineName())
 	if err != nil {
 		return nil, errors.Wrap(err, "Error checking if api exist and loading it")
 	}
@@ -189,61 +192,39 @@ func printURLsForService(c corev1.CoreV1Interface, ip, service, namespace string
 	return urls, nil
 }
 
-// CheckService waits for the specified service to be ready by returning an error until the service is up
-// The check is done by polling the endpoint associated with the service and when the endpoint exists, returning no error->service-online
+// CheckService checks if a service is listening on a port.
 func CheckService(namespace string, service string) error {
 	client, err := K8s.GetCoreClient()
 	if err != nil {
 		return errors.Wrap(err, "Error getting kubernetes client")
 	}
-	services := client.Services(namespace)
-	err = validateService(services, service)
-	if err != nil {
-		return errors.Wrap(err, "Error validating service")
-	}
-	endpoints := client.Endpoints(namespace)
-	return checkEndpointReady(endpoints, service)
-}
-
-func validateService(s corev1.ServiceInterface, service string) error {
-	if _, err := s.Get(service, metav1.GetOptions{}); err != nil {
-		return errors.Wrapf(err, "Error getting service %s", service)
-	}
-	return nil
-}
 
-func checkEndpointReady(endpoints corev1.EndpointsInterface, service string) error {
-	endpoint, err := endpoints.Get(service, metav1.GetOptions{})
+	svc, err := client.Services(namespace).Get(service, metav1.GetOptions{})
 	if err != nil {
-		return &util.RetriableError{Err: errors.Errorf("Error getting endpoints for service %s", service)}
-	}
-	const notReadyMsg = "Waiting, endpoint for service is not ready yet...\n"
-	if len(endpoint.Subsets) == 0 {
-		fmt.Fprintf(os.Stderr, notReadyMsg)
-		return &util.RetriableError{Err: errors.New("Endpoint for service is not ready yet")}
-	}
-	for _, subset := range endpoint.Subsets {
-		if len(subset.Addresses) == 0 {
-			fmt.Fprintf(os.Stderr, notReadyMsg)
-			return &util.RetriableError{Err: errors.New("No endpoints for service are ready yet")}
+		return &util.RetriableError{
+			Err: errors.Wrapf(err, "Error getting service %s", service),
 		}
 	}
+	if len(svc.Spec.Ports) == 0 {
+		return fmt.Errorf("%s:%s has no ports", namespace, service)
+	}
+	glog.Infof("Found service: %+v", svc)
 	return nil
 }
 
-func OptionallyHttpsFormattedUrlString(bareUrlString string, https bool) (string, bool) {
-	httpsFormattedString := bareUrlString
-	isHttpSchemedURL := false
+func OptionallyHTTPSFormattedURLString(bareURLString string, https bool) (string, bool) {
+	httpsFormattedString := bareURLString
+	isHTTPSchemedURL := false
 
-	if u, parseErr := url.Parse(bareUrlString); parseErr == nil {
-		isHttpSchemedURL = u.Scheme == "http"
+	if u, parseErr := url.Parse(bareURLString); parseErr == nil {
+		isHTTPSchemedURL = u.Scheme == "http"
 	}
 
-	if isHttpSchemedURL && https {
-		httpsFormattedString = strings.Replace(bareUrlString, "http", "https", 1)
+	if isHTTPSchemedURL && https {
+		httpsFormattedString = strings.Replace(bareURLString, "http", "https", 1)
 	}
 
-	return httpsFormattedString, isHttpSchemedURL
+	return httpsFormattedString, isHTTPSchemedURL
 }
 
 func WaitAndMaybeOpenService(api libmachine.API, namespace string, service string, urlTemplate *template.Template, urlMode bool, https bool,
@@ -256,10 +237,10 @@ func WaitAndMaybeOpenService(api libmachine.API, namespace string, service strin
 	if err != nil {
 		return errors.Wrap(err, "Check that minikube is running and that you have specified the correct namespace")
 	}
-	for _, bareUrlString := range urls {
-		urlString, isHttpSchemedURL := OptionallyHttpsFormattedUrlString(bareUrlString, https)
+	for _, bareURLString := range urls {
+		urlString, isHTTPSchemedURL := OptionallyHTTPSFormattedURLString(bareURLString, https)
 
-		if urlMode || !isHttpSchemedURL {
+		if urlMode || !isHTTPSchemedURL {
 			fmt.Fprintln(os.Stdout, urlString)
 		} else {
 			fmt.Fprintln(os.Stderr, "Opening kubernetes service "+namespace+"/"+service+" in default browser...")

pkg/minikube/service/service_test.go

@@ -133,44 +133,6 @@ func (e MockEndpointsInterface) Get(name string, _ metav1.GetOptions) (*v1.Endpo
 	return endpoint, nil
 }
 
-func TestCheckEndpointReady(t *testing.T) {
-	var tests = []struct {
-		description string
-		service     string
-		err         bool
-	}{
-		{
-			description: "Endpoint with no subsets should return an error",
-			service:     "no-subsets",
-			err:         true,
-		},
-		{
-			description: "Endpoint with no ready endpoints should return an error",
-			service:     "not-ready",
-			err:         true,
-		},
-		{
-			description: "Endpoint with at least one ready endpoint should not return an error",
-			service:     "one-ready",
-			err:         false,
-		},
-	}
-
-	for _, test := range tests {
-		test := test
-		t.Run(test.description, func(t *testing.T) {
-			t.Parallel()
-			err := checkEndpointReady(&MockEndpointsInterface{}, test.service)
-			if err != nil && !test.err {
-				t.Errorf("Check endpoints returned an error: %+v", err)
-			}
-			if err == nil && test.err {
-				t.Errorf("Check endpoints should have returned an error but returned nil")
-			}
-		})
-	}
-}
-
 type MockServiceInterface struct {
 	fake.FakeServices
 	ServiceList *v1.ServiceList
@@ -290,12 +252,13 @@ func TestOptionallyHttpsFormattedUrlString(t *testing.T) {
 
 	var tests = []struct {
 		description                     string
-		bareUrlString                   string
+		bareURLString                   string
 		https                           bool
-		expectedHttpsFormattedUrlString string
-		expectedIsHttpSchemedURL        bool
+		expectedHTTPSFormattedURLString string
+		expectedIsHTTPSchemedURL        bool
 	}{
 		{
+<<<<<<< HEAD
 			description:                     "no https for http schemed with no https option",
 			bareUrlString:                   "http://192.168.99.100:30563",
 			https:                           false,
@@ -322,6 +285,34 @@ func TestOptionallyHttpsFormattedUrlString(t *testing.T) {
 			https:                           true,
 			expectedHttpsFormattedUrlString: "xyz.http.myservice:30563",
 			expectedIsHttpSchemedURL:        false,
+=======
+			description:   "no https for http schemed with no https option",
+			bareURLString: "http://192.168.99.100:30563",
+			https:         false,
+			expectedHTTPSFormattedURLString: "http://192.168.99.100:30563",
+			expectedIsHTTPSchemedURL:        true,
+		},
+		{
+			description:   "no https for non-http schemed with no https option",
+			bareURLString: "xyz.http.myservice:30563",
+			https:         false,
+			expectedHTTPSFormattedURLString: "xyz.http.myservice:30563",
+			expectedIsHTTPSchemedURL:        false,
+		},
+		{
+			description:   "https for http schemed with https option",
+			bareURLString: "http://192.168.99.100:30563",
+			https:         true,
+			expectedHTTPSFormattedURLString: "https://192.168.99.100:30563",
+			expectedIsHTTPSchemedURL:        true,
+		},
+		{
+			description:   "no https for non-http schemed with https option and http substring",
+			bareURLString: "xyz.http.myservice:30563",
+			https:         true,
+			expectedHTTPSFormattedURLString: "xyz.http.myservice:30563",
+			expectedIsHTTPSchemedURL:        false,
+>>>>>>> master
 		},
 	}
 
@@ -329,15 +320,15 @@ func TestOptionallyHttpsFormattedUrlString(t *testing.T) {
 		test := test
 		t.Run(test.description, func(t *testing.T) {
 			t.Parallel()
-			httpsFormattedUrlString, isHttpSchemedURL := OptionallyHttpsFormattedUrlString(test.bareUrlString, test.https)
+			httpsFormattedURLString, isHTTPSchemedURL := OptionallyHTTPSFormattedURLString(test.bareURLString, test.https)
 
-			if httpsFormattedUrlString != test.expectedHttpsFormattedUrlString {
-				t.Errorf("\nhttpsFormattedUrlString, Expected %v \nActual: %v \n\n", test.expectedHttpsFormattedUrlString, httpsFormattedUrlString)
+			if httpsFormattedURLString != test.expectedHTTPSFormattedURLString {
+				t.Errorf("\nhttpsFormattedURLString, Expected %v \nActual: %v \n\n", test.expectedHTTPSFormattedURLString, httpsFormattedURLString)
 			}
 
-			if isHttpSchemedURL != test.expectedIsHttpSchemedURL {
-				t.Errorf("\nisHttpSchemedURL, Expected %v \nActual: %v \n\n",
-					test.expectedHttpsFormattedUrlString, httpsFormattedUrlString)
+			if isHTTPSchemedURL != test.expectedIsHTTPSchemedURL {
+				t.Errorf("\nisHTTPSchemedURL, Expected %v \nActual: %v \n\n",
+					test.expectedHTTPSFormattedURLString, httpsFormattedURLString)
 			}
 		})
 	}
@@ -345,10 +336,12 @@ func TestOptionallyHttpsFormattedUrlString(t *testing.T) {
 
 func TestGetServiceURLs(t *testing.T) {
 	defaultAPI := &tests.MockAPI{
-		Hosts: map[string]*host.Host{
-			config.GetMachineName(): {
-				Name:   config.GetMachineName(),
-				Driver: &tests.MockDriver{},
+		FakeStore: tests.FakeStore{
+			Hosts: map[string]*host.Host{
+				config.GetMachineName(): {
+					Name:   config.GetMachineName(),
+					Driver: &tests.MockDriver{},
+				},
 			},
 		},
 	}
@@ -358,13 +351,15 @@ func TestGetServiceURLs(t *testing.T) {
 		description string
 		api         libmachine.API
 		namespace   string
-		expected    ServiceURLs
+		expected    URLs
 		err         bool
 	}{
 		{
 			description: "no host",
 			api: &tests.MockAPI{
-				Hosts: make(map[string]*host.Host),
+				FakeStore: tests.FakeStore{
+					Hosts: make(map[string]*host.Host),
+				},
 			},
 			err: true,
 		},
@@ -372,7 +367,7 @@ func TestGetServiceURLs(t *testing.T) {
 			description: "correctly return serviceURLs",
 			namespace:   "default",
 			api:         defaultAPI,
-			expected: []ServiceURL{
+			expected: []URL{
 				{
 					Namespace: "default",
 					Name:      "mock-dashboard",
@@ -412,10 +407,12 @@ func TestGetServiceURLs(t *testing.T) {
 
 func TestGetServiceURLsForService(t *testing.T) {
 	defaultAPI := &tests.MockAPI{
-		Hosts: map[string]*host.Host{
-			config.GetMachineName(): {
-				Name:   config.GetMachineName(),
-				Driver: &tests.MockDriver{},
+		FakeStore: tests.FakeStore{
+			Hosts: map[string]*host.Host{
+				config.GetMachineName(): {
+					Name:   config.GetMachineName(),
+					Driver: &tests.MockDriver{},
+				},
 			},
 		},
 	}
@@ -432,7 +429,9 @@ func TestGetServiceURLsForService(t *testing.T) {
 		{
 			description: "no host",
 			api: &tests.MockAPI{
-				Hosts: make(map[string]*host.Host),
+				FakeStore: tests.FakeStore{
+					Hosts: make(map[string]*host.Host),
+				},
 			},
 			err: true,
 		},

pkg/minikube/tests/api_mock.go

@@ -20,17 +20,14 @@ import (
 	"encoding/json"
 	"fmt"
 
-	"github.com/docker/machine/libmachine"
 	"github.com/docker/machine/libmachine/auth"
 	"github.com/docker/machine/libmachine/host"
-	"github.com/docker/machine/libmachine/mcnerror"
-	"github.com/docker/machine/libmachine/state"
 	"github.com/pkg/errors"
 )
 
 // MockAPI is a struct used to mock out libmachine.API
 type MockAPI struct {
-	Hosts       map[string]*host.Host
+	FakeStore
 	CreateError bool
 	RemoveError bool
 	SaveCalled  bool
@@ -38,7 +35,9 @@ type MockAPI struct {
 
 func NewMockAPI() *MockAPI {
 	m := MockAPI{
-		Hosts: make(map[string]*host.Host),
+		FakeStore: FakeStore{
+			Hosts: make(map[string]*host.Host),
+		},
 	}
 	return &m
 }
@@ -52,7 +51,7 @@ func (api *MockAPI) Close() error {
 func (api *MockAPI) NewHost(driverName string, rawDriver []byte) (*host.Host, error) {
 	var driver MockDriver
 	if err := json.Unmarshal(rawDriver, &driver); err != nil {
-		return nil, errors.Wrap(err, "Error unmarshalling json")
+		return nil, errors.Wrap(err, "error unmarshalling json")
 	}
 	h := &host.Host{
 		DriverName:  driverName,
@@ -67,38 +66,20 @@ func (api *MockAPI) NewHost(driverName string, rawDriver []byte) (*host.Host, er
 // Create creates the actual host.
 func (api *MockAPI) Create(h *host.Host) error {
 	if api.CreateError {
-		return fmt.Errorf("Error creating host.")
+		return errors.New("error creating host")
 	}
 	return h.Driver.Create()
 }
 
-// Exists determines if the host already exists.
-func (api *MockAPI) Exists(name string) (bool, error) {
-	_, ok := api.Hosts[name]
-	return ok, nil
-}
-
 // List the existing hosts.
 func (api *MockAPI) List() ([]string, error) {
 	return []string{}, nil
 }
 
-// Load loads a host from disk.
-func (api *MockAPI) Load(name string) (*host.Host, error) {
-	h, ok := api.Hosts[name]
-	if !ok {
-		return nil, mcnerror.ErrHostDoesNotExist{
-			Name: name,
-		}
-
-	}
-	return h, nil
-}
-
 // Remove a host.
 func (api *MockAPI) Remove(name string) error {
 	if api.RemoveError {
-		return fmt.Errorf("Error removing %s", name)
+		return fmt.Errorf("error removing %s", name)
 	}
 
 	delete(api.Hosts, name)
@@ -107,25 +88,11 @@ func (api *MockAPI) Remove(name string) error {
 
 // Save saves a host to disk.
 func (api *MockAPI) Save(host *host.Host) error {
-	api.Hosts[host.Name] = host
 	api.SaveCalled = true
-	return nil
+	return api.FakeStore.Save(host)
 }
 
 // GetMachinesDir returns the directory to store machines in.
 func (api MockAPI) GetMachinesDir() string {
 	return ""
 }
-
-// State returns the state of a host.
-func State(api libmachine.API, name string) state.State {
-	host, _ := api.Load(name)
-	machineState, _ := host.Driver.GetState()
-	return machineState
-}
-
-// Exists tells whether a named host exists.
-func Exists(api libmachine.API, name string) bool {
-	exists, _ := api.Exists(name)
-	return exists
-}

pkg/minikube/tests/driver_mock.go

@@ -17,11 +17,10 @@ limitations under the License.
 package tests
 
 import (
-	"fmt"
-
 	"github.com/docker/machine/libmachine/drivers"
 	"github.com/docker/machine/libmachine/mcnflag"
 	"github.com/docker/machine/libmachine/state"
+	"github.com/pkg/errors"
 )
 
 // MockDriver is a struct used to mock out libmachine.Driver
@@ -31,6 +30,7 @@ type MockDriver struct {
 	RemoveError  bool
 	HostError    bool
 	Port         int
+	IP           string
 }
 
 // Create creates a MockDriver instance
@@ -40,6 +40,9 @@ func (driver *MockDriver) Create() error {
 }
 
 func (driver *MockDriver) GetIP() (string, error) {
+	if driver.IP != "" {
+		return driver.IP, nil
+	}
 	if driver.BaseDriver.IPAddress != "" {
 		return driver.BaseDriver.IPAddress, nil
 	}
@@ -58,7 +61,7 @@ func (driver *MockDriver) GetSSHPort() (int, error) {
 // GetSSHHostname returns the hostname for SSH
 func (driver *MockDriver) GetSSHHostname() (string, error) {
 	if driver.HostError {
-		return "", fmt.Errorf("Error getting host!")
+		return "", errors.New("error getting host")
 	}
 	return "localhost", nil
 }
@@ -87,7 +90,7 @@ func (driver *MockDriver) Kill() error {
 // Remove removes the machine
 func (driver *MockDriver) Remove() error {
 	if driver.RemoveError {
-		return fmt.Errorf("Error deleting machine.")
+		return errors.New("error deleting machine")
 	}
 	return nil
 }

pkg/minikube/tests/fake_store.go

+/*
+Copyright 2018 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tests
+
+import (
+	"github.com/docker/machine/libmachine/host"
+	"github.com/docker/machine/libmachine/mcnerror"
+)
+
+//implements persist.Store from libmachine
+type FakeStore struct {
+	Hosts map[string]*host.Host
+}
+
+// Exists determines if the host already exists.
+func (s *FakeStore) Exists(name string) (bool, error) {
+	_, ok := s.Hosts[name]
+	return ok, nil
+}
+
+func (s *FakeStore) List() ([]string, error) {
+	hostNames := []string{}
+	for h := range s.Hosts {
+		hostNames = append(hostNames, h)
+	}
+	return hostNames, nil
+}
+
+// Load loads a host from disk.
+func (s *FakeStore) Load(name string) (*host.Host, error) {
+	h, ok := s.Hosts[name]
+	if !ok {
+		return nil, mcnerror.ErrHostDoesNotExist{
+			Name: name,
+		}
+
+	}
+	return h, nil
+}
+
+// Remove removes a machine from the store
+func (s *FakeStore) Remove(name string) error {
+	_, ok := s.Hosts[name]
+	if !ok {
+		return mcnerror.ErrHostDoesNotExist{
+			Name: name,
+		}
+
+	}
+	delete(s.Hosts, name)
+	return nil
+}
+
+// Save persists a machine in the store
+func (s *FakeStore) Save(host *host.Host) error {
+	s.Hosts[host.Name] = host
+	return nil
+}

pkg/minikube/tunnel/cluster_inspector.go

+/*
+Copyright 2018 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tunnel
+
+import (
+	"fmt"
+	"net"
+
+	"github.com/docker/machine/libmachine"
+	"github.com/docker/machine/libmachine/host"
+	"github.com/docker/machine/libmachine/state"
+	"github.com/pkg/errors"
+	"k8s.io/minikube/pkg/minikube/cluster"
+	"k8s.io/minikube/pkg/minikube/config"
+)
+
+type clusterInspector struct {
+	machineAPI   libmachine.API
+	configLoader config.Loader
+	machineName  string
+}
+
+func (m *clusterInspector) getStateAndHost() (HostState, *host.Host, error) {
+
+	h, err := cluster.CheckIfHostExistsAndLoad(m.machineAPI, m.machineName)
+
+	if err != nil {
+		err = errors.Wrapf(err, "error loading docker-machine host for: %s", m.machineName)
+		return Unknown, nil, err
+	}
+
+	var s state.State
+	s, err = h.Driver.GetState()
+	if err != nil {
+		err = errors.Wrapf(err, "error getting host status for %s", m.machineName)
+		return Unknown, nil, err
+	}
+
+	if s == state.Running {
+		return Running, h, nil
+	}
+
+	return Stopped, h, nil
+}
+
+func (m *clusterInspector) getStateAndRoute() (HostState, *Route, error) {
+	hostState, h, err := m.getStateAndHost()
+	defer m.machineAPI.Close()
+	if err != nil {
+		return hostState, nil, err
+	}
+	var c config.Config
+	c, err = m.configLoader.LoadConfigFromFile(m.machineName)
+	if err != nil {
+		err = errors.Wrapf(err, "error loading config for %s", m.machineName)
+		return hostState, nil, err
+	}
+
+	var route *Route
+	route, err = getRoute(h, c)
+	if err != nil {
+		err = errors.Wrapf(err, "error getting Route info for %s", m.machineName)
+		return hostState, nil, err
+	}
+	return hostState, route, nil
+}
+
+func getRoute(host *host.Host, clusterConfig config.Config) (*Route, error) {
+	hostDriverIP, err := host.Driver.GetIP()
+	if err != nil {
+		return nil, errors.Wrapf(err, "error getting host IP for %s", host.Name)
+	}
+
+	_, ipNet, err := net.ParseCIDR(clusterConfig.KubernetesConfig.ServiceCIDR)
+	if err != nil {
+		return nil, fmt.Errorf("error parsing service CIDR: %s", err)
+	}
+	ip := net.ParseIP(hostDriverIP)
+	if ip == nil {
+		return nil, fmt.Errorf("invalid IP for host %s", hostDriverIP)
+	}
+
+	return &Route{
+		Gateway:  ip,
+		DestCIDR: ipNet,
+	}, nil
+}

pkg/minikube/tunnel/cluster_inspector_test.go

+/*
+Copyright 2018 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tunnel
+
+import (
+	"testing"
+
+	"net"
+	"reflect"
+	"strings"
+
+	"github.com/docker/machine/libmachine/host"
+	"github.com/docker/machine/libmachine/state"
+	"k8s.io/minikube/pkg/minikube/config"
+	"k8s.io/minikube/pkg/minikube/tests"
+)
+
+func TestAPIError(t *testing.T) {
+	machineName := "nonexistentmachine"
+
+	machineAPI := tests.NewMockAPI()
+	configLoader := &stubConfigLoader{}
+	inspector := &clusterInspector{
+		machineAPI, configLoader, machineName,
+	}
+
+	s, r, err := inspector.getStateAndRoute()
+
+	if err == nil || !strings.Contains(err.Error(), "Machine does not exist") {
+		t.Errorf("cluster inspector should propagate errors from API, getStateAndRoute() returned \"%v, %v\", %v", s, r, err)
+	}
+}
+
+func TestMinikubeCheckReturnsHostInformation(t *testing.T) {
+	machineAPI := &tests.MockAPI{
+		FakeStore: tests.FakeStore{
+			Hosts: map[string]*host.Host{
+				"testmachine": {
+					Driver: &tests.MockDriver{
+						CurrentState: state.Running,
+						IP:           "1.2.3.4",
+					},
+				},
+			},
+		},
+	}
+
+	configLoader := &stubConfigLoader{
+		c: config.Config{
+			KubernetesConfig: config.KubernetesConfig{
+				ServiceCIDR: "96.0.0.0/12",
+			},
+		},
+	}
+	inspector := &clusterInspector{
+		machineAPI, configLoader, "testmachine",
+	}
+
+	s, r, err := inspector.getStateAndRoute()
+
+	if err != nil {
+		t.Errorf("`error` is not nil")
+	}
+
+	ip := net.ParseIP("1.2.3.4")
+	_, ipNet, _ := net.ParseCIDR("96.0.0.0/12")
+
+	expectedRoute := &Route{
+		Gateway:  ip,
+		DestCIDR: ipNet,
+	}
+
+	if s != Running {
+		t.Errorf("expected running, got %s", s)
+	}
+	if !reflect.DeepEqual(r, expectedRoute) {
+		t.Errorf("expected %v, got %v", expectedRoute, r)
+	}
+}
+
+func TestUnparseableCIDR(t *testing.T) {
+	cfg := config.Config{
+		KubernetesConfig: config.KubernetesConfig{
+			ServiceCIDR: "bad.cidr.0.0/12",
+		}}
+	h := &host.Host{
+		Driver: &tests.MockDriver{
+			IP: "192.168.1.1",
+		},
+	}
+
+	_, err := getRoute(h, cfg)
+
+	if err == nil {
+		t.Errorf("expected non nil error, instead got %s", err)
+	}
+}
+
+func TestRouteIPDetection(t *testing.T) {
+	expectedTargetCIDR := "10.96.0.0/12"
+
+	cfg := config.Config{
+		KubernetesConfig: config.KubernetesConfig{
+			ServiceCIDR: expectedTargetCIDR,
+		},
+	}
+
+	expectedGatewayIP := "192.168.1.1"
+	h := &host.Host{
+		Driver: &tests.MockDriver{
+			IP: expectedGatewayIP,
+		},
+	}
+
+	routerConfig, err := getRoute(h, cfg)
+
+	if err != nil {
+		t.Errorf("expected no errors but got: %s", err)
+	}
+
+	if routerConfig.DestCIDR.String() != expectedTargetCIDR {
+		t.Errorf("addTargetCIDR doesn't match, expected '%s', got '%s'", expectedTargetCIDR, routerConfig.DestCIDR)
+	}
+
+	if routerConfig.Gateway.String() != expectedGatewayIP {
+		t.Errorf("add gateway IP doesn't match, expected '%s', got '%s'", expectedGatewayIP, routerConfig.Gateway)
+	}
+
+}

pkg/minikube/tunnel/loadbalancer_patcher.go

+/*
+Copyright 2018 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tunnel
+
+import (
+	"fmt"
+
+	"github.com/golang/glog"
+	core_v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	k8s_types "k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/kubernetes/typed/core/v1"
+	"k8s.io/client-go/rest"
+)
+
+//requestSender is an interface exposed for testing what requests are sent through the k8s REST client
+type requestSender interface {
+	send(request *rest.Request) ([]byte, error)
+}
+
+//patchConverter is an interface exposed for testing what patches are sent through the k8s REST client
+type patchConverter interface {
+	convert(restClient rest.Interface, patch *Patch) *rest.Request
+}
+
+//loadBalancerEmulator is the main struct for emulating the loadbalancer behavior. it sets the ingress to the cluster IP
+type loadBalancerEmulator struct {
+	coreV1Client   v1.CoreV1Interface
+	requestSender  requestSender
+	patchConverter patchConverter
+}
+
+func (l *loadBalancerEmulator) PatchServices() ([]string, error) {
+	return l.applyOnLBServices(func(restClient rest.Interface, svc core_v1.Service) ([]byte, error) {
+		return l.updateService(restClient, svc)
+	})
+}
+
+func (l *loadBalancerEmulator) Cleanup() ([]string, error) {
+	return l.applyOnLBServices(func(restClient rest.Interface, svc core_v1.Service) ([]byte, error) {
+		return l.cleanupService(restClient, svc)
+	})
+}
+
+func (l *loadBalancerEmulator) applyOnLBServices(action func(restClient rest.Interface, svc core_v1.Service) ([]byte, error)) ([]string, error) {
+	services := l.coreV1Client.Services("")
+	serviceList, err := services.List(metav1.ListOptions{})
+	if err != nil {
+		return nil, err
+	}
+	restClient := l.coreV1Client.RESTClient()
+
+	var managedServices []string
+
+	for _, svc := range serviceList.Items {
+		if svc.Spec.Type != "LoadBalancer" {
+			glog.V(3).Infof("%s is not type LoadBalancer, skipping.", svc.Name)
+			continue
+		}
+		glog.Infof("%s is type LoadBalancer.", svc.Name)
+		managedServices = append(managedServices, svc.Name)
+		result, err := action(restClient, svc)
+		if err != nil {
+			glog.Errorf("%s", result)
+			glog.Errorf("error patching service %s/%s: %s", svc.Namespace, svc.Name, err)
+			continue
+		}
+
+	}
+	return managedServices, nil
+}
+func (l *loadBalancerEmulator) updateService(restClient rest.Interface, svc core_v1.Service) ([]byte, error) {
+	clusterIP := svc.Spec.ClusterIP
+	ingresses := svc.Status.LoadBalancer.Ingress
+	if len(ingresses) == 1 && ingresses[0].IP == clusterIP {
+		return nil, nil
+	}
+	glog.V(3).Infof("[%s] setting ClusterIP as the LoadBalancer Ingress", svc.Name)
+	jsonPatch := fmt.Sprintf(`[{"op": "add", "path": "/status/loadBalancer/ingress", "value":  [ { "ip": "%s" } ] }]`, clusterIP)
+	patch := &Patch{
+		Type:         k8s_types.JSONPatchType,
+		ResourceName: svc.Name,
+		NameSpaceSet: true,
+		NameSpace:    svc.Namespace,
+		Subresource:  "status",
+		Resource:     "services",
+		BodyContent:  jsonPatch,
+	}
+	request := l.patchConverter.convert(restClient, patch)
+	result, err := l.requestSender.send(request)
+	if err != nil {
+		glog.Infof("Patched %s with IP %s", svc.Name, clusterIP)
+	} else {
+		glog.Errorf("error patching %s with IP %s: %s", svc.Name, clusterIP, err)
+	}
+	return result, err
+}
+
+func (l *loadBalancerEmulator) cleanupService(restClient rest.Interface, svc core_v1.Service) ([]byte, error) {
+	ingresses := svc.Status.LoadBalancer.Ingress
+	if len(ingresses) == 0 {
+		return nil, nil
+	}
+	glog.V(3).Infof("[%s] cleanup: unset load balancer ingress", svc.Name)
+	jsonPatch := `[{"op": "remove", "path": "/status/loadBalancer/ingress" }]`
+	patch := &Patch{
+		Type:         k8s_types.JSONPatchType,
+		ResourceName: svc.Name,
+		NameSpaceSet: true,
+		NameSpace:    svc.Namespace,
+		Subresource:  "status",
+		Resource:     "services",
+		BodyContent:  jsonPatch,
+	}
+	request := l.patchConverter.convert(restClient, patch)
+	result, err := l.requestSender.send(request)
+	glog.Infof("Removed load balancer ingress from %s.", svc.Name)
+	return result, err
+
+}
+
+func newLoadBalancerEmulator(corev1Client v1.CoreV1Interface) loadBalancerEmulator {
+	return loadBalancerEmulator{
+		coreV1Client:   corev1Client,
+		requestSender:  &defaultRequestSender{},
+		patchConverter: &defaultPatchConverter{},
+	}
+}
+
+type defaultPatchConverter struct{}
+
+func (c *defaultPatchConverter) convert(restClient rest.Interface, patch *Patch) *rest.Request {
+	request := restClient.Patch(patch.Type)
+	request.Name(patch.ResourceName)
+	request.Resource(patch.Resource)
+	request.SubResource(patch.Subresource)
+	if patch.NameSpaceSet {
+		request.Namespace(patch.NameSpace)
+	}
+	request.Body([]byte(patch.BodyContent))
+	return request
+}
+
+type defaultRequestSender struct{}
+
+func (r *defaultRequestSender) send(request *rest.Request) ([]byte, error) {
+	return request.Do().Raw()
+}

pkg/minikube/tunnel/loadbalancer_patcher_test.go

+/*
+Copyright 2018 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tunnel
+
+import (
+	"testing"
+
+	"reflect"
+
+	apiV1 "k8s.io/api/core/v1"
+	metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes/typed/core/v1"
+	"k8s.io/client-go/kubernetes/typed/core/v1/fake"
+	"k8s.io/client-go/rest"
+)
+
+type stubCoreClient struct {
+	fake.FakeCoreV1
+	servicesList *apiV1.ServiceList
+	restClient   *rest.RESTClient
+}
+
+func (c *stubCoreClient) Services(namespace string) v1.ServiceInterface {
+	return &stubServices{
+		fake.FakeServices{Fake: &c.FakeCoreV1},
+		c.servicesList,
+	}
+}
+
+func (c *stubCoreClient) RESTClient() rest.Interface {
+	return c.restClient
+}
+
+type stubServices struct {
+	fake.FakeServices
+	servicesList *apiV1.ServiceList
+}
+
+func (s *stubServices) List(opts metaV1.ListOptions) (*apiV1.ServiceList, error) {
+	return s.servicesList, nil
+}
+
+func newStubCoreClient(servicesList *apiV1.ServiceList) *stubCoreClient {
+	if servicesList == nil {
+		servicesList = &apiV1.ServiceList{
+			Items: []apiV1.Service{}}
+	}
+	return &stubCoreClient{
+		servicesList: servicesList,
+		restClient:   nil,
+	}
+}
+
+type countingRequestSender struct {
+	requests int
+}
+
+func (s *countingRequestSender) send(request *rest.Request) (result []byte, err error) {
+	s.requests++
+	return nil, nil
+}
+
+type recordingPatchConverter struct {
+	patches []*Patch
+}
+
+func (r *recordingPatchConverter) convert(restClient rest.Interface, patch *Patch) *rest.Request {
+	r.patches = append(r.patches, patch)
+	return nil
+}
+
+func TestEmptyListOfServicesDoesNothing(t *testing.T) {
+	client := newStubCoreClient(&apiV1.ServiceList{
+		Items: []apiV1.Service{}})
+
+	patcher := newLoadBalancerEmulator(client)
+
+	serviceNames, err := patcher.PatchServices()
+
+	if len(serviceNames) > 0 || err != nil {
+		t.Errorf("Expected: [], nil\n Got: %v, %s", serviceNames, err)
+	}
+
+}
+
+func TestServicesWithNoLoadbalancerType(t *testing.T) {
+	client := newStubCoreClient(&apiV1.ServiceList{
+		Items: []apiV1.Service{
+			{
+				Spec: apiV1.ServiceSpec{
+					Type: "ClusterIP",
+				},
+			},
+			{
+				Spec: apiV1.ServiceSpec{
+					Type: "NodeIP",
+				},
+			},
+		},
+	})
+
+	patcher := newLoadBalancerEmulator(client)
+
+	serviceNames, err := patcher.PatchServices()
+
+	if len(serviceNames) > 0 || err != nil {
+		t.Errorf("Expected: [], nil\n Got: %v, %s", serviceNames, err)
+	}
+
+}
+
+func TestServicesWithLoadbalancerType(t *testing.T) {
+	client := newStubCoreClient(&apiV1.ServiceList{
+		Items: []apiV1.Service{
+			{
+				ObjectMeta: metaV1.ObjectMeta{
+					Name:      "svc1-up-to-date",
+					Namespace: "ns1",
+				},
+				Spec: apiV1.ServiceSpec{
+					Type:      "LoadBalancer",
+					ClusterIP: "10.96.0.3",
+				},
+				Status: apiV1.ServiceStatus{
+					LoadBalancer: apiV1.LoadBalancerStatus{
+						Ingress: []apiV1.LoadBalancerIngress{
+							{
+								IP: "10.96.0.3",
+							},
+						},
+					},
+				},
+			},
+			{
+				ObjectMeta: metaV1.ObjectMeta{
+					Name:      "svc2-out-of-date",
+					Namespace: "ns2",
+				},
+				Spec: apiV1.ServiceSpec{
+					Type:      "LoadBalancer",
+					ClusterIP: "10.96.0.4",
+				},
+				Status: apiV1.ServiceStatus{
+					LoadBalancer: apiV1.LoadBalancerStatus{
+						Ingress: []apiV1.LoadBalancerIngress{
+							{
+								IP: "10.96.0.5",
+							},
+						},
+					},
+				},
+			},
+			{
+				ObjectMeta: metaV1.ObjectMeta{
+					Name:      "svc3-empty-ingress",
+					Namespace: "ns3",
+				},
+				Spec: apiV1.ServiceSpec{
+					Type:      "LoadBalancer",
+					ClusterIP: "10.96.0.2",
+				},
+				Status: apiV1.ServiceStatus{
+					LoadBalancer: apiV1.LoadBalancerStatus{
+						Ingress: []apiV1.LoadBalancerIngress{},
+					},
+				},
+			},
+			{
+				ObjectMeta: metaV1.ObjectMeta{
+					Name: "svc4-not-lb",
+				},
+				Spec: apiV1.ServiceSpec{
+					Type: "NodeIP",
+				},
+			},
+		},
+	})
+
+	expectedPatches := []*Patch{
+		{
+			Type:         "application/json-patch+json",
+			NameSpace:    "ns2",
+			NameSpaceSet: true,
+			Resource:     "services",
+			Subresource:  "status",
+			ResourceName: "svc2-out-of-date",
+			BodyContent:  `[{"op": "add", "path": "/status/loadBalancer/ingress", "value":  [ { "ip": "10.96.0.4" } ] }]`,
+		},
+		{
+			Type:         "application/json-patch+json",
+			NameSpace:    "ns3",
+			NameSpaceSet: true,
+			Resource:     "services",
+			Subresource:  "status",
+			ResourceName: "svc3-empty-ingress",
+			BodyContent:  `[{"op": "add", "path": "/status/loadBalancer/ingress", "value":  [ { "ip": "10.96.0.2" } ] }]`,
+		},
+	}
+
+	requestSender := &countingRequestSender{}
+	patchConverter := &recordingPatchConverter{}
+
+	patcher := newLoadBalancerEmulator(client)
+	patcher.requestSender = requestSender
+	patcher.patchConverter = patchConverter
+
+	serviceNames, err := patcher.PatchServices()
+
+	expectedServices := []string{"svc1-up-to-date", "svc2-out-of-date", "svc3-empty-ingress"}
+
+	if !reflect.DeepEqual(serviceNames, expectedServices) || err != nil {
+		t.Errorf("error.\nExpected: %s, <nil>\nGot: %v, %v", expectedServices, serviceNames, err)
+	}
+
+	if !reflect.DeepEqual(patchConverter.patches, expectedPatches) {
+		t.Errorf("error in patches.\nExpected: %v, <nil>\nGot: %v", expectedPatches, patchConverter.patches)
+	}
+
+	if requestSender.requests != 2 {
+		t.Errorf("error in number of requests sent.\nExpected: %v, <nil>\nGot: %v", 2, requestSender.requests)
+	}
+
+}
+
+func TestCleanupPatchedIPs(t *testing.T) {
+	expectedPatches := []*Patch{
+		{
+			Type:         "application/json-patch+json",
+			NameSpace:    "ns1",
+			NameSpaceSet: true,
+			Resource:     "services",
+			Subresource:  "status",
+			ResourceName: "svc1-up-to-date",
+			BodyContent:  `[{"op": "remove", "path": "/status/loadBalancer/ingress" }]`,
+		},
+
+		{
+			Type:         "application/json-patch+json",
+			NameSpace:    "ns2",
+			NameSpaceSet: true,
+			Resource:     "services",
+			Subresource:  "status",
+			ResourceName: "svc2-out-of-date",
+			BodyContent:  `[{"op": "remove", "path": "/status/loadBalancer/ingress" }]`,
+		},
+	}
+
+	client := newStubCoreClient(&apiV1.ServiceList{
+		Items: []apiV1.Service{
+			{
+				ObjectMeta: metaV1.ObjectMeta{
+					Name:      "svc1-up-to-date",
+					Namespace: "ns1",
+				},
+				Spec: apiV1.ServiceSpec{
+					Type:      "LoadBalancer",
+					ClusterIP: "10.96.0.3",
+				},
+				Status: apiV1.ServiceStatus{
+					LoadBalancer: apiV1.LoadBalancerStatus{
+						Ingress: []apiV1.LoadBalancerIngress{
+							{
+								IP: "10.96.0.3",
+							},
+						},
+					},
+				},
+			},
+			{
+				ObjectMeta: metaV1.ObjectMeta{
+					Name:      "svc2-out-of-date",
+					Namespace: "ns2",
+				},
+				Spec: apiV1.ServiceSpec{
+					Type:      "LoadBalancer",
+					ClusterIP: "10.96.0.4",
+				},
+				Status: apiV1.ServiceStatus{
+					LoadBalancer: apiV1.LoadBalancerStatus{
+						Ingress: []apiV1.LoadBalancerIngress{
+							{
+								IP: "10.96.0.5",
+							},
+						},
+					},
+				},
+			},
+			{
+				ObjectMeta: metaV1.ObjectMeta{
+					Name:      "svc3-empty-ingress",
+					Namespace: "ns3",
+				},
+				Spec: apiV1.ServiceSpec{
+					Type:      "LoadBalancer",
+					ClusterIP: "10.96.0.2",
+				},
+				Status: apiV1.ServiceStatus{
+					LoadBalancer: apiV1.LoadBalancerStatus{
+						Ingress: []apiV1.LoadBalancerIngress{},
+					},
+				},
+			},
+			{
+				ObjectMeta: metaV1.ObjectMeta{
+					Name: "svc4-not-lb",
+				},
+				Spec: apiV1.ServiceSpec{
+					Type: "NodeIP",
+				},
+			},
+		},
+	})
+
+	requestSender := &countingRequestSender{}
+	patchConverter := &recordingPatchConverter{}
+
+	patcher := newLoadBalancerEmulator(client)
+	patcher.requestSender = requestSender
+	patcher.patchConverter = patchConverter
+
+	serviceNames, err := patcher.Cleanup()
+	expectedServices := []string{"svc1-up-to-date", "svc2-out-of-date", "svc3-empty-ingress"}
+
+	if !reflect.DeepEqual(serviceNames, expectedServices) || err != nil {
+		t.Errorf("error.\nExpected: %s, <nil>\nGot: %v, %v", expectedServices, serviceNames, err)
+	}
+	if !reflect.DeepEqual(patchConverter.patches, expectedPatches) {
+		t.Errorf("error in patches.\nExpected: %v, <nil>\nGot: %v", expectedPatches, patchConverter.patches)
+	}
+	if requestSender.requests != 2 {
+		t.Errorf("error in number of requests sent.\nExpected: %v, <nil>\nGot: %v", 2, requestSender.requests)
+	}
+}

pkg/minikube/tunnel/process.go

+/*
+Copyright 2018 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tunnel
+
+import (
+	"fmt"
+	"os"
+	"runtime"
+	"syscall"
+)
+
+var checkIfRunning func(pid int) (bool, error)
+var getPid func() int
+
+func init() {
+	checkIfRunning = osCheckIfRunning
+	getPid = osGetPid
+}
+
+func osGetPid() int {
+	return os.Getpid()
+}
+
+//TODO(balintp): this is vulnerable to pid reuse we should include process name in the check
+func osCheckIfRunning(pid int) (bool, error) {
+	p, err := os.FindProcess(pid)
+	if runtime.GOOS == "windows" {
+		return err == nil, nil
+	}
+	//on unix systems further checking is required, as findProcess is noop
+	if err != nil {
+		return false, fmt.Errorf("error finding process %d: %s", pid, err)
+	}
+	if err := p.Signal(syscall.Signal(0)); err != nil {
+		return false, nil
+	}
+	if p == nil {
+		return false, nil
+	}
+	return true, nil
+}

pkg/minikube/tunnel/registry.go

+/*
+Copyright 2018 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tunnel
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+
+	"github.com/golang/glog"
+	"github.com/pkg/errors"
+)
+
+// There is one tunnel registry per user, shared across multiple vms.
+// It can register, list and check for existing and running tunnels
+type ID struct {
+	//Route is the key
+	Route *Route
+	//the rest is metadata
+	MachineName string
+	Pid         int
+}
+
+func (t *ID) Equal(other *ID) bool {
+	return t.Route.Equal(other.Route) &&
+		t.MachineName == other.MachineName &&
+		t.Pid == other.Pid
+}
+
+func (t *ID) String() string {
+	return fmt.Sprintf("ID { Route: %v, machineName: %s, Pid: %d }", t.Route, t.MachineName, t.Pid)
+}
+
+type persistentRegistry struct {
+	path string
+}
+
+func (r *persistentRegistry) IsAlreadyDefinedAndRunning(tunnel *ID) (*ID, error) {
+	tunnels, err := r.List()
+	if err != nil {
+		return nil, fmt.Errorf("failed to list: %s", err)
+	}
+
+	for _, t := range tunnels {
+		if t.Route.Equal(tunnel.Route) {
+			isRunning, err := checkIfRunning(t.Pid)
+			if err != nil {
+				return nil, fmt.Errorf("error checking whether conflicting tunnel (%v) is running: %s", t, err)
+			}
+			if isRunning {
+				return t, nil
+			}
+		}
+	}
+	return nil, nil
+}
+
+func (r *persistentRegistry) Register(tunnel *ID) error {
+	glog.V(3).Infof("registering tunnel: %s", tunnel)
+	if tunnel.Route == nil {
+		return errors.New("tunnel.Route should not be nil")
+	}
+
+	tunnels, err := r.List()
+	if err != nil {
+		return fmt.Errorf("failed to list: %s", err)
+	}
+
+	alreadyExists := false
+	for i, t := range tunnels {
+		if t.Route.Equal(tunnel.Route) {
+			isRunning, err := checkIfRunning(t.Pid)
+			if err != nil {
+				return fmt.Errorf("error checking whether conflicting tunnel (%v) is running: %s", t, err)
+			}
+			if isRunning {
+				return errorTunnelAlreadyExists(t)
+			}
+			tunnels[i] = tunnel
+			alreadyExists = true
+		}
+	}
+
+	if !alreadyExists {
+		tunnels = append(tunnels, tunnel)
+	}
+
+	bytes, err := json.Marshal(tunnels)
+	if err != nil {
+		return fmt.Errorf("error marshalling json %s", err)
+	}
+
+	glog.V(5).Infof("json marshalled: %v, %s\n", tunnels, bytes)
+
+	f, err := os.OpenFile(r.path, os.O_RDWR|os.O_TRUNC, 0600)
+	if err != nil {
+		if os.IsNotExist(err) {
+			f, err = os.Create(r.path)
+			if err != nil {
+				return fmt.Errorf("error creating registry file (%s): %s", r.path, err)
+			}
+		} else {
+			return err
+		}
+	}
+	defer func() {
+		err := f.Close()
+		if err != nil {
+			fmt.Errorf("error closing registry file: %s", err)
+		}
+	}()
+
+	n, err := f.Write(bytes)
+	if n < len(bytes) || err != nil {
+		return fmt.Errorf("error registering tunnel while writing tunnels file: %s", err)
+	}
+
+	return nil
+}
+
+func (r *persistentRegistry) Remove(route *Route) error {
+	glog.V(3).Infof("removing tunnel from registry: %s", route)
+	tunnels, err := r.List()
+	if err != nil {
+		return err
+	}
+	idx := -1
+	for i := range tunnels {
+		if tunnels[i].Route.Equal(route) {
+			idx = i
+			break
+		}
+	}
+	if idx == -1 {
+		return fmt.Errorf("can't remove route: %s not found in tunnel registry", route)
+	}
+	tunnels = append(tunnels[:idx], tunnels[idx+1:]...)
+	glog.V(4).Infof("tunnels after remove: %s", tunnels)
+	f, err := os.OpenFile(r.path, os.O_RDWR|os.O_TRUNC, 0600)
+	if err != nil {
+		return fmt.Errorf("error removing tunnel %s", err)
+	}
+	defer func() {
+		err := f.Close()
+		if err != nil {
+			fmt.Errorf("error closing tunnel registry file: %s", err)
+		}
+	}()
+
+	var bytes []byte
+	bytes, err = json.Marshal(tunnels)
+	if err != nil {
+		return fmt.Errorf("error removing tunnel %s", err)
+	}
+	n, err := f.Write(bytes)
+	if n < len(bytes) || err != nil {
+		return fmt.Errorf("error removing tunnel %s", err)
+	}
+
+	return nil
+}
+func (r *persistentRegistry) List() ([]*ID, error) {
+	f, err := os.Open(r.path)
+	if err != nil {
+		if !os.IsNotExist(err) {
+			return nil, err
+		}
+		return []*ID{}, nil
+	}
+	byteValue, _ := ioutil.ReadAll(f)
+	var tunnels []*ID
+	if len(byteValue) == 0 {
+		return tunnels, nil
+	}
+	if err = json.Unmarshal(byteValue, &tunnels); err != nil {
+		return nil, err
+	}
+
+	return tunnels, nil
+}

pkg/minikube/tunnel/registry_test.go

+/*
+Copyright 2018 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tunnel
+
+import (
+	"io/ioutil"
+	"os"
+	"reflect"
+	"testing"
+)
+
+func TestPersistentRegistryWithNoKey(t *testing.T) {
+	registry, cleanup := createTestRegistry(t)
+	defer cleanup()
+
+	route := &ID{}
+	err := registry.Register(route)
+
+	if err == nil {
+		t.Errorf("attempting to register ID without key should throw error")
+	}
+}
+
+func TestPersistentRegistryNullableMetadata(t *testing.T) {
+	registry, cleanup := createTestRegistry(t)
+	defer cleanup()
+
+	route := &ID{
+		Route: unsafeParseRoute("1.2.3.4", "10.96.0.0/12"),
+	}
+	err := registry.Register(route)
+
+	if err != nil {
+		t.Errorf("metadata should be nullable, expected no error, got %s", err)
+	}
+}
+
+func TestListOnEmptyRegistry(t *testing.T) {
+	reg := &persistentRegistry{
+		path: "nonexistent.txt",
+	}
+
+	info, err := reg.List()
+	expectedInfo := []*ID{}
+	if !reflect.DeepEqual(info, expectedInfo) || err != nil {
+		t.Errorf("expected %s, nil error, got %s, %s", expectedInfo, info, err)
+	}
+}
+
+func TestRemoveOnEmptyRegistry(t *testing.T) {
+	reg := &persistentRegistry{
+		path: "nonexistent.txt",
+	}
+
+	e := reg.Remove(unsafeParseRoute("1.2.3.4", "1.2.3.4/5"))
+	if e == nil {
+		t.Errorf("expected error, got %s", e)
+	}
+}
+
+func TestRegisterOnEmptyRegistry(t *testing.T) {
+	reg := &persistentRegistry{
+		path: "nonexistent.txt",
+	}
+
+	err := reg.Register(&ID{Route: unsafeParseRoute("1.2.3.4", "1.2.3.4/5")})
+	if err != nil {
+		t.Errorf("expected no error, got %s", err)
+	}
+	f, err := os.Open("nonexistent.txt")
+	if err != nil {
+		t.Errorf("expected file to exist, got: %s", err)
+		return
+	}
+	f.Close()
+	err = os.Remove("nonexistent.txt")
+	if err != nil {
+		t.Errorf("error removing nonexistent.txt: %s", err)
+	}
+}
+
+func TestRemoveOnNonExistentTunnel(t *testing.T) {
+	file := tmpFile(t)
+	reg := &persistentRegistry{
+		path: file,
+	}
+	defer os.Remove(file)
+
+	err := reg.Register(&ID{Route: unsafeParseRoute("1.2.3.4", "1.2.3.4/5")})
+	if err != nil {
+		t.Errorf("expected no error, got %s", err)
+	}
+
+	err = reg.Remove(unsafeParseRoute("5.6.7.8", "1.2.3.4/5"))
+	if err == nil {
+		t.Errorf("expected error, got nil")
+	}
+}
+
+func TestListAfterRegister(t *testing.T) {
+	file := tmpFile(t)
+	reg := &persistentRegistry{
+		path: file,
+	}
+	defer os.Remove(file)
+	err := reg.Register(&ID{
+		Route:       unsafeParseRoute("1.2.3.4", "1.2.3.4/5"),
+		MachineName: "testmachine",
+		Pid:         1234,
+	})
+	if err != nil {
+		t.Errorf("failed to register: expected no error, got %s", err)
+	}
+
+	tunnelList, err := reg.List()
+	if err != nil {
+		t.Errorf("failed to list: expected no error, got %s", err)
+	}
+
+	expectedList := []*ID{
+		{
+			Route:       unsafeParseRoute("1.2.3.4", "1.2.3.4/5"),
+			MachineName: "testmachine",
+			Pid:         1234,
+		},
+	}
+
+	if len(tunnelList) != 1 || !tunnelList[0].Equal(expectedList[0]) {
+		t.Errorf("\nexpected %+v,\ngot      %+v", expectedList, tunnelList)
+	}
+}
+func TestRegisterRemoveList(t *testing.T) {
+	file := tmpFile(t)
+	reg := &persistentRegistry{
+		path: file,
+	}
+	defer os.Remove(file)
+
+	err := reg.Register(&ID{
+		Route:       unsafeParseRoute("192.168.1.25", "10.96.0.0/12"),
+		MachineName: "testmachine",
+		Pid:         1234,
+	})
+	if err != nil {
+		t.Errorf("failed to register: expected no error, got %s", err)
+	}
+
+	err = reg.Remove(unsafeParseRoute("192.168.1.25", "10.96.0.0/12"))
+
+	if err != nil {
+		t.Errorf("failed to remove: expected no error, got %s", err)
+	}
+
+	tunnelList, err := reg.List()
+	if err != nil {
+		t.Errorf("failed to list: expected no error, got %s", err)
+	}
+
+	expectedList := []*ID{}
+
+	if len(tunnelList) != 0 {
+		t.Errorf("\nexpected %+v,\ngot      %+v", expectedList, tunnelList)
+	}
+}
+
+func TestDuplicateRouteError(t *testing.T) {
+	file := tmpFile(t)
+	reg := &persistentRegistry{
+		path: file,
+	}
+	defer os.Remove(file)
+
+	err := reg.Register(&ID{
+		Route:       unsafeParseRoute("192.168.1.25", "10.96.0.0/12"),
+		MachineName: "testmachine",
+		Pid:         os.Getpid(),
+	})
+	if err != nil {
+		t.Errorf("failed to register: expected no error, got %s", err)
+	}
+	err = reg.Register(&ID{
+		Route:       unsafeParseRoute("192.168.1.25", "10.96.0.0/12"),
+		MachineName: "testmachine",
+		Pid:         5678,
+	})
+	if err == nil {
+		t.Error("expected error on duplicate route, got nil")
+	}
+}
+
+func TestTunnelTakeoverFromNonRunningProcess(t *testing.T) {
+	file := tmpFile(t)
+	reg := &persistentRegistry{
+		path: file,
+	}
+	defer os.Remove(file)
+
+	err := reg.Register(&ID{
+		Route:       unsafeParseRoute("192.168.1.25", "10.96.0.0/12"),
+		MachineName: "testmachine",
+		Pid:         12341234,
+	})
+	if err != nil {
+		t.Errorf("failed to register: expected no error, got %s", err)
+	}
+	err = reg.Register(&ID{
+		Route:       unsafeParseRoute("192.168.1.25", "10.96.0.0/12"),
+		MachineName: "testmachine",
+		Pid:         5678,
+	})
+	if err != nil {
+		t.Errorf("failed to register: expected no error, got %s", err)
+	}
+
+	tunnelList, err := reg.List()
+	if err != nil {
+		t.Errorf("failed to list: expected no error, got %s", err)
+	}
+
+	expectedList := []*ID{
+		{
+			Route:       unsafeParseRoute("192.168.1.25", "10.96.0.0/12"),
+			MachineName: "testmachine",
+			Pid:         5678,
+		},
+	}
+
+	if len(tunnelList) != 1 || !tunnelList[0].Equal(expectedList[0]) {
+		t.Errorf("\nexpected %+v,\ngot      %+v", expectedList, tunnelList)
+	}
+}
+
+func tmpFile(t *testing.T) string {
+	t.Helper()
+	f, err := ioutil.TempFile(os.TempDir(), "reg_")
+	f.Close()
+	if err != nil {
+		t.Errorf("failed to create temp file %s", err)
+	}
+	return f.Name()
+}
+
+func createTestRegistry(t *testing.T) (reg *persistentRegistry, cleanup func()) {
+	f, err := ioutil.TempFile(os.TempDir(), "reg_")
+	f.Close()
+	if err != nil {
+		t.Errorf("failed to create temp file %s", err)
+	}
+
+	registry := &persistentRegistry{
+		path: f.Name(),
+	}
+	return registry, func() { os.Remove(f.Name()) }
+}

pkg/minikube/tunnel/reporter.go

+/*
+Copyright 2018 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tunnel
+
+import (
+	"fmt"
+
+	"io"
+	"strings"
+
+	"github.com/golang/glog"
+)
+
+//reporter that reports the status of a tunnel
+type reporter interface {
+	Report(tunnelState *Status)
+}
+
+type simpleReporter struct {
+	out       io.Writer
+	lastState *Status
+}
+
+const noErrors = "no errors"
+
+func (r *simpleReporter) Report(tunnelState *Status) {
+	if r.lastState == tunnelState {
+		return
+	}
+	r.lastState = tunnelState
+	minikubeState := tunnelState.MinikubeState.String()
+
+	managedServices := fmt.Sprintf("[%s]", strings.Join(tunnelState.PatchedServices, ", "))
+
+	lbError := noErrors
+	if tunnelState.LoadBalancerEmulatorError != nil {
+		lbError = tunnelState.LoadBalancerEmulatorError.Error()
+	}
+
+	minikubeError := noErrors
+	if tunnelState.MinikubeError != nil {
+		minikubeError = tunnelState.MinikubeError.Error()
+	}
+
+	routerError := noErrors
+	if tunnelState.RouteError != nil {
+		routerError = tunnelState.RouteError.Error()
+	}
+
+	errors := fmt.Sprintf(`    errors: 
+		minikube: %s
+		router: %s
+		loadbalancer emulator: %s
+`, minikubeError, routerError, lbError)
+
+	_, err := r.out.Write([]byte(fmt.Sprintf(
+		`Status:	
+	machine: %s
+	pid: %d
+	route: %s
+	minikube: %s
+	services: %s
+%s`, tunnelState.TunnelID.MachineName,
+		tunnelState.TunnelID.Pid,
+		tunnelState.TunnelID.Route,
+		minikubeState,
+		managedServices,
+		errors)))
+	if err != nil {
+		glog.Errorf("failed to report state %s", err)
+	}
+}
+
+func newReporter(out io.Writer) reporter {
+	return &simpleReporter{
+		out: out,
+	}
+}

pkg/minikube/tunnel/reporter_test.go

+/*
+Copyright 2018 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tunnel
+
+import (
+	"errors"
+	"fmt"
+
+	"testing"
+)
+
+func TestReporter(t *testing.T) {
+	testCases := []struct {
+		name           string
+		tunnelState    *Status
+		expectedOutput string
+	}{
+		{
+			name: "simple",
+			tunnelState: &Status{
+				TunnelID: ID{
+					Route:       unsafeParseRoute("1.2.3.4", "10.96.0.0/12"),
+					MachineName: "testmachine",
+					Pid:         1234,
+				},
+				MinikubeState: Running,
+				MinikubeError: nil,
+
+				RouteError: nil,
+
+				PatchedServices:           []string{"svc1", "svc2"},
+				LoadBalancerEmulatorError: nil,
+			},
+			expectedOutput: `Status:	
+	machine: testmachine
+	pid: 1234
+	route: 10.96.0.0/12 -> 1.2.3.4
+	minikube: Running
+	services: [svc1, svc2]
+    errors: 
+		minikube: no errors
+		router: no errors
+		loadbalancer emulator: no errors
+`,
+		},
+		{
+			name: "errors",
+			tunnelState: &Status{
+				TunnelID: ID{
+					Route:       unsafeParseRoute("1.2.3.4", "10.96.0.0/12"),
+					MachineName: "testmachine",
+					Pid:         1234,
+				},
+				MinikubeState: Unknown,
+				MinikubeError: errors.New("minikubeerror"),
+
+				RouteError: errors.New("routeerror"),
+
+				PatchedServices:           nil,
+				LoadBalancerEmulatorError: errors.New("lberror"),
+			},
+			expectedOutput: `Status:	
+	machine: testmachine
+	pid: 1234
+	route: 10.96.0.0/12 -> 1.2.3.4
+	minikube: Unknown
+	services: []
+    errors: 
+		minikube: minikubeerror
+		router: routeerror
+		loadbalancer emulator: lberror
+`,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			out := &recordingWriter{}
+			reporter := newReporter(out)
+			reporter.Report(tc.tunnelState)
+			if tc.expectedOutput != out.output {
+				t.Errorf(`%s [FAIL].
+Expected: "%s" 
+Got:	  "%s"`, tc.name, tc.expectedOutput, out.output)
+			}
+		})
+	}
+
+	//testing deduplication
+	out := &recordingWriter{}
+	reporter := newReporter(out)
+	reporter.Report(testCases[0].tunnelState)
+	reporter.Report(testCases[0].tunnelState)
+	reporter.Report(testCases[1].tunnelState)
+	reporter.Report(testCases[1].tunnelState)
+	reporter.Report(testCases[0].tunnelState)
+
+	expectedOutput := fmt.Sprintf("%s%s%s",
+		testCases[0].expectedOutput,
+		testCases[1].expectedOutput,
+		testCases[0].expectedOutput)
+
+	if out.output != expectedOutput {
+		t.Errorf(`Deduplication test [FAIL].
+Expected: "%s" 
+Got:	  "%s"`, expectedOutput, out.output)
+	}
+}
+
+type recordingWriter struct {
+	output string
+}
+
+func (w *recordingWriter) Write(p []byte) (n int, err error) {
+	w.output = fmt.Sprintf("%s%s", w.output, p)
+	return 0, nil
+}

pkg/minikube/tunnel/route.go

+/*
+Copyright 2018 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tunnel
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/golang/glog"
+)
+
+//router manages the routing table on the host, implementations should cater for OS specific methods
+type router interface {
+	//Inspect checks if the given route exists or not in the routing table
+	//conflict is defined as: same destination CIDR, different Gateway
+	//overlaps are defined as: routes that have overlapping but not exactly matching destination CIDR
+	Inspect(route *Route) (exists bool, conflict string, overlaps []string, err error)
+
+	//EnsureRouteIsAdded is an idempotent way to add a route to the routing table
+	//it fails if there is a conflict
+	EnsureRouteIsAdded(route *Route) error
+
+	//Cleanup is an idempotent way to remove a route from the routing table
+	//it fails if there is a conflict
+	Cleanup(route *Route) error
+}
+
+type osRouter struct{}
+
+type routingTableLine struct {
+	route *Route
+	line  string
+}
+
+func isValidToAddOrDelete(router router, r *Route) (bool, error) {
+	exists, conflict, overlaps, err := router.Inspect(r)
+	if err != nil {
+		return false, err
+	}
+
+	if len(overlaps) > 0 {
+		glog.Warningf("overlapping CIDR detected in routing table with minikube tunnel (CIDR: %s). It is advisable to remove these rules:\n%v", r.DestCIDR, strings.Join(overlaps, "\n"))
+	}
+
+	if exists {
+		return true, nil
+	}
+
+	if len(conflict) > 0 {
+		return false, fmt.Errorf("conflicting rule in routing table: %s", conflict)
+	}
+
+	return false, nil
+}
+
+//a partial representation of the routing table on the host
+//tunnel only requires the destination CIDR, the gateway and the actual textual representation per line
+type routingTable []routingTableLine
+
+func (t *routingTable) Check(route *Route) (exists bool, conflict string, overlaps []string) {
+	conflict = ""
+	exists = false
+	overlaps = []string{}
+	for _, tableLine := range *t {
+		if route.Equal(tableLine.route) {
+			exists = true
+		} else if route.DestCIDR.String() == tableLine.route.DestCIDR.String() &&
+			route.Gateway.String() != tableLine.route.Gateway.String() {
+			conflict = tableLine.line
+		} else if route.DestCIDR.Contains(tableLine.route.DestCIDR.IP) || tableLine.route.DestCIDR.Contains(route.DestCIDR.IP) {
+			overlaps = append(overlaps, tableLine.line)
+		}
+	}
+	return
+}
+
+func (t *routingTable) String() string {
+	result := fmt.Sprintf("table (%d routes)", len(*t))
+	for _, l := range *t {
+		result = fmt.Sprintf("%s\n  %s\t|%s", result, l.route.String(), l.line)
+	}
+	return result
+}
+
+func (t *routingTable) Equal(other *routingTable) bool {
+	if other == nil || len(*t) != len(*other) {
+		return false
+	}
+
+	for i := range *t {
+		routesEqual := (*t)[i].route.Equal((*other)[i].route)
+		linesEqual := (*t)[i].line == ((*other)[i].line)
+		if !(routesEqual && linesEqual) {
+			return false
+		}
+	}
+	return true
+}

pkg/minikube/tunnel/route_darwin.go

+/*
+Copyright 2018 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tunnel
+
+import (
+	"fmt"
+	"net"
+	"os/exec"
+	"regexp"
+	"strings"
+
+	"github.com/golang/glog"
+)
+
+func (router *osRouter) EnsureRouteIsAdded(route *Route) error {
+	exists, err := isValidToAddOrDelete(router, route)
+	if err != nil {
+		return err
+	}
+	if exists {
+		return nil
+	}
+
+	serviceCIDR := route.DestCIDR.String()
+	gatewayIP := route.Gateway.String()
+
+	glog.Infof("Adding Route for CIDR %s to gateway %s", serviceCIDR, gatewayIP)
+	command := exec.Command("sudo", "route", "-n", "add", serviceCIDR, gatewayIP)
+	glog.Infof("About to run command: %s", command.Args)
+	stdInAndOut, err := command.CombinedOutput()
+	message := fmt.Sprintf("%s", stdInAndOut)
+	re := regexp.MustCompile(fmt.Sprintf("add net (.*): gateway %s\n", gatewayIP))
+	if !re.MatchString(message) {
+		return fmt.Errorf("error adding Route: %s, %d", message, len(strings.Split(message, "\n")))
+	}
+	glog.Infof("%s", stdInAndOut)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (router *osRouter) Inspect(route *Route) (exists bool, conflict string, overlaps []string, err error) {
+	cmd := exec.Command("netstat", "-nr", "-f", "inet")
+	cmd.Env = append(cmd.Env, "LC_ALL=C")
+	stdInAndOut, err := cmd.CombinedOutput()
+	if err != nil {
+		err = fmt.Errorf("error running '%v': %s", cmd, err)
+		return
+	}
+
+	rt := router.parseTable(stdInAndOut)
+
+	exists, conflict, overlaps = rt.Check(route)
+
+	return
+}
+
+func (router *osRouter) parseTable(table []byte) routingTable {
+	t := routingTable{}
+	skip := true
+	for _, line := range strings.Split(string(table), "\n") {
+		//header
+		if strings.HasPrefix(line, "Destination") {
+			skip = false
+			continue
+		}
+		//don't care about the 0.0.0.0 routes
+		if skip || strings.HasPrefix(line, "default") {
+			continue
+		}
+		fields := strings.Fields(line)
+
+		if len(fields) <= 2 {
+			continue
+		}
+		dstCIDRString := router.padCIDR(fields[0])
+		gatewayIPString := fields[1]
+		gatewayIP := net.ParseIP(gatewayIPString)
+
+		_, ipNet, err := net.ParseCIDR(dstCIDRString)
+		if err != nil {
+			glog.V(4).Infof("skipping line: can't parse CIDR from routing table: %s", dstCIDRString)
+		} else if gatewayIP == nil {
+			glog.V(4).Infof("skipping line: can't parse IP from routing table: %s", gatewayIPString)
+		} else {
+			tableLine := routingTableLine{
+				route: &Route{
+					DestCIDR: ipNet,
+					Gateway:  gatewayIP,
+				},
+				line: line,
+			}
+			t = append(t, tableLine)
+		}
+	}
+
+	return t
+}
+
+func (router *osRouter) padCIDR(origCIDR string) string {
+	s := ""
+	dots := 0
+	slash := false
+	for i, c := range origCIDR {
+		if c == '.' {
+			dots++
+		}
+		if c == '/' {
+			for dots < 3 {
+				s += ".0"
+				dots++
+			}
+			slash = true
+		}
+		if i == len(origCIDR)-1 {
+			s += string(c)
+			bits := 32 - 8*(3-dots)
+			for dots < 3 {
+				s += ".0"
+				dots++
+			}
+			if !slash {
+				s += fmt.Sprintf("/%d", bits)
+			}
+		} else {
+			s += string(c)
+		}
+	}
+	return s
+}
+
+func (router *osRouter) Cleanup(route *Route) error {
+	glog.V(3).Infof("Cleaning up %s\n", route)
+	exists, err := isValidToAddOrDelete(router, route)
+	if err != nil {
+		return err
+	}
+	if !exists {
+		return nil
+	}
+	command := exec.Command("sudo", "route", "-n", "delete", route.DestCIDR.String())
+	stdInAndOut, err := command.CombinedOutput()
+	if err != nil {
+		return err
+	}
+	message := fmt.Sprintf("%s", stdInAndOut)
+	glog.V(4).Infof("%s", message)
+	re := regexp.MustCompile("^delete net ([^:]*)$")
+	if !re.MatchString(message) {
+		return fmt.Errorf("error deleting route: %s, %d", message, len(strings.Split(message, "\n")))
+	}
+	return nil
+}

pkg/minikube/tunnel/route_darwin_test.go

+// +build darwin,integration
+
+/*
+Copyright 2018 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tunnel
+
+import (
+	"net"
+	"os/exec"
+	"testing"
+
+	"fmt"
+
+	"reflect"
+)
+
+func TestDarwinRouteFailsOnConflictIntegrationTest(t *testing.T) {
+	cfg := &Route{
+		Gateway: net.IPv4(127, 0, 0, 1),
+		DestCIDR: &net.IPNet{
+			IP:   net.IPv4(10, 96, 0, 0),
+			Mask: net.IPv4Mask(255, 240, 0, 0),
+		},
+	}
+
+	addRoute(t, "10.96.0.0/12", "127.0.0.2")
+	err := (&osRouter{}).EnsureRouteIsAdded(cfg)
+	if err == nil {
+		t.Errorf("add should have error, but it is nil")
+	}
+}
+
+func TestDarwinRouteIdempotentIntegrationTest(t *testing.T) {
+	cfg := &Route{
+		Gateway: net.IPv4(127, 0, 0, 1),
+		DestCIDR: &net.IPNet{
+			IP:   net.IPv4(10, 96, 0, 0),
+			Mask: net.IPv4Mask(255, 240, 0, 0),
+		},
+	}
+
+	cleanRoute(t, "10.96.0.0/12")
+	err := (&osRouter{}).EnsureRouteIsAdded(cfg)
+	if err != nil {
+		t.Errorf("add error: %s", err)
+	}
+
+	err = (&osRouter{}).EnsureRouteIsAdded(cfg)
+	if err != nil {
+		t.Errorf("add error: %s", err)
+	}
+
+	cleanRoute(t, "10.96.0.0/12")
+}
+
+func TestDarwinRouteCleanupIdempontentIntegrationTest(t *testing.T) {
+
+	cfg := &Route{
+		Gateway: net.IPv4(192, 168, 1, 1),
+		DestCIDR: &net.IPNet{
+			IP:   net.IPv4(10, 96, 0, 0),
+			Mask: net.IPv4Mask(255, 240, 0, 0),
+		},
+	}
+
+	cleanRoute(t, "10.96.0.0/12")
+	addRoute(t, "10.96.0.0/12", "192.168.1.1")
+	err := (&osRouter{}).Cleanup(cfg)
+	if err != nil {
+		t.Errorf("cleanup failed with %s", err)
+	}
+	err = (&osRouter{}).Cleanup(cfg)
+	if err != nil {
+		t.Errorf("cleanup failed with %s", err)
+	}
+}
+
+func addRoute(t *testing.T, cidr string, gw string) {
+	command := exec.Command("sudo", "route", "-n", "add", cidr, gw)
+	_, err := command.CombinedOutput()
+	if err != nil {
+		t.Logf("add Route error (should be ok): %s", err)
+	}
+}
+
+func cleanRoute(t *testing.T, cidr string) {
+	command := exec.Command("sudo", "route", "-n", "delete", cidr)
+	_, err := command.CombinedOutput()
+	if err != nil {
+		t.Logf("cleanup error (should be ok): %s", err)
+	}
+}
+
+func TestCIDRPadding(t *testing.T) {
+	testCases := []struct {
+		inputCIDR  string
+		paddedCIDR string
+	}{
+		{inputCIDR: "10", paddedCIDR: "10.0.0.0/8"},
+		{inputCIDR: "10.96/12", paddedCIDR: "10.96.0.0/12"},
+		{inputCIDR: "192.168.43", paddedCIDR: "192.168.43.0/24"},
+		{inputCIDR: "192.168.43.1/32", paddedCIDR: "192.168.43.1/32"},
+		{inputCIDR: "127.0.0.1", paddedCIDR: "127.0.0.1/32"},
+	}
+
+	for _, test := range testCases {
+		testName := fmt.Sprintf("pad(%s) should be %s", test.inputCIDR, test.paddedCIDR)
+		t.Run(testName, func(t *testing.T) {
+			cidr := (&osRouter{}).padCIDR(test.inputCIDR)
+			if cidr != test.paddedCIDR {
+				t.Errorf("%s got %s", testName, cidr)
+			}
+		})
+	}
+}
+
+func TestRoutingTableParser(t *testing.T) {
+	table := `Routing tables
+
+Internet:
+Destination        Gateway            Flags        Refs      Use   Netif Expire
+127                127.0.0.1          UCS             0        0     lo0
+127.0.0.1          127.0.0.1          UH             13    30917     lo0
+172.16.128/24      link#17            UC              1        0  vmnet1
+192.168.246        link#18            UC              1        0  vmnet8
+224.0.0            link#1             UmCS            0        0     lo0
+`
+	rt := (&osRouter{}).parseTable([]byte(table))
+
+	expectedRt := routingTable{
+		routingTableLine{
+			route: unsafeParseRoute("127.0.0.1", "127.0.0.0/8"),
+			line:  "127                127.0.0.1          UCS             0        0     lo0",
+		},
+		routingTableLine{
+			route: unsafeParseRoute("127.0.0.1", "127.0.0.1/32"),
+			line:  "127.0.0.1          127.0.0.1          UH             13    30917     lo0",
+		},
+	}
+	if !reflect.DeepEqual(rt, expectedRt) {
+		t.Errorf("expected:\n %s\ngot\n %s", expectedRt.String(), rt.String())
+	}
+}

pkg/minikube/tunnel/route_linux.go

+/*
+Copyright 2018 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tunnel
+
+import (
+	"fmt"
+	"net"
+	"os/exec"
+	"strings"
+
+	"github.com/golang/glog"
+)
+
+func (router *osRouter) EnsureRouteIsAdded(route *Route) error {
+	exists, err := isValidToAddOrDelete(router, route)
+	if err != nil {
+		return err
+	}
+	if exists {
+		return nil
+	}
+
+	serviceCIDR := route.DestCIDR.String()
+	gatewayIP := route.Gateway.String()
+
+	glog.Infof("Adding Route for CIDR %s to gateway %s", serviceCIDR, gatewayIP)
+	command := exec.Command("sudo", "ip", "route", "add", serviceCIDR, "via", gatewayIP)
+	glog.Infof("About to run command: %s", command.Args)
+	stdInAndOut, err := command.CombinedOutput()
+	message := string(stdInAndOut)
+	if len(message) > 0 {
+		return fmt.Errorf("error adding Route: %s, %d", message, len(strings.Split(message, "\n")))
+	}
+	glog.Info(stdInAndOut)
+	if err != nil {
+		glog.Errorf("error adding Route: %s, %d", message, len(strings.Split(message, "\n")))
+		return err
+	}
+	return nil
+}
+
+func (router *osRouter) Inspect(route *Route) (exists bool, conflict string, overlaps []string, err error) {
+	cmd := exec.Command("netstat", "-nr", "-f", "inet")
+	cmd.Env = append(cmd.Env, "LC_ALL=C")
+	stdInAndOut, err := cmd.CombinedOutput()
+	if err != nil {
+		err = fmt.Errorf("error running '%v': %s", cmd, err)
+		return
+	}
+	rt := router.parseTable(stdInAndOut)
+
+	exists, conflict, overlaps = rt.Check(route)
+
+	return
+}
+
+func (router *osRouter) parseTable(table []byte) routingTable {
+	t := routingTable{}
+	skip := true
+	for _, line := range strings.Split(string(table), "\n") {
+		//after first line of header we can start consuming
+		if strings.HasPrefix(line, "Destination") {
+			skip = false
+			continue
+		}
+
+		fields := strings.Fields(line)
+		//don't care about the 0.0.0.0 routes
+		if skip || len(fields) == 0 || len(fields) > 0 && (fields[0] == "default" || fields[0] == "0.0.0.0") {
+			continue
+		}
+		if len(fields) > 2 {
+			dstCIDRIP := net.ParseIP(fields[0])
+			dstCIDRMask := fields[2]
+			dstMaskIP := net.ParseIP(dstCIDRMask)
+			gatewayIP := net.ParseIP(fields[1])
+
+			if dstCIDRIP == nil || gatewayIP == nil || dstMaskIP == nil {
+				glog.V(8).Infof("skipping line: can't parse: %s", line)
+			} else {
+
+				dstCIDR := &net.IPNet{
+					IP:   dstCIDRIP,
+					Mask: net.IPv4Mask(dstMaskIP[12], dstMaskIP[13], dstMaskIP[14], dstMaskIP[15]),
+				}
+
+				tableLine := routingTableLine{
+					route: &Route{
+						DestCIDR: dstCIDR,
+						Gateway:  gatewayIP,
+					},
+					line: line,
+				}
+				t = append(t, tableLine)
+			}
+		}
+	}
+
+	return t
+}
+
+func (router *osRouter) Cleanup(route *Route) error {
+	exists, err := isValidToAddOrDelete(router, route)
+	if err != nil {
+		return err
+	}
+	if !exists {
+		return nil
+	}
+	serviceCIDR := route.DestCIDR.String()
+	gatewayIP := route.Gateway.String()
+
+	glog.Infof("Cleaning up Route for CIDR %s to gateway %s\n", serviceCIDR, gatewayIP)
+	command := exec.Command("sudo", "ip", "route", "delete", serviceCIDR)
+	stdInAndOut, err := command.CombinedOutput()
+	message := fmt.Sprintf("%s", stdInAndOut)
+	glog.Infof("%s", message)
+	if err != nil {
+		return fmt.Errorf("error deleting Route: %s, %s", message, err)
+	}
+	return nil
+}

pkg/minikube/tunnel/route_linux_test.go

+// +build linux,integration
+
+/*
+Copyright 2018 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tunnel
+
+import (
+	"net"
+	"os/exec"
+	"testing"
+)
+
+func TestLinuxRouteFailsOnConflictIntegrationTest(t *testing.T) {
+	r := &osRouter{}
+
+	cleanRoute(t, "10.96.0.0/12")
+	addRoute(t, "10.96.0.0/12", "127.0.0.1")
+	err := r.EnsureRouteIsAdded(&Route{
+		Gateway: net.IPv4(127, 0, 0, 2),
+		DestCIDR: &net.IPNet{
+			IP:   net.IPv4(10, 96, 0, 0),
+			Mask: net.IPv4Mask(255, 240, 0, 0),
+		}})
+	if err == nil {
+		t.Errorf("add should have error, but it is nil")
+	}
+	cleanRoute(t, "10.96.0.0/12")
+}
+
+func TestLinuxRouteIdempotentIntegrationTest(t *testing.T) {
+	r := &osRouter{}
+
+	cleanRoute(t, "10.96.0.0/12")
+	route := &Route{
+		Gateway: net.IPv4(127, 0, 0, 1),
+		DestCIDR: &net.IPNet{
+			IP:   net.IPv4(10, 96, 0, 0),
+			Mask: net.IPv4Mask(255, 240, 0, 0),
+		},
+	}
+	err := r.EnsureRouteIsAdded(route)
+	if err != nil {
+		t.Errorf("add error: %s", err)
+	}
+
+	err = r.EnsureRouteIsAdded(route)
+	if err != nil {
+		t.Errorf("add error: %s", err)
+	}
+
+	cleanRoute(t, "10.96.0.0/12")
+}
+
+func TestLinuxRouteCleanupIdempontentIntegrationTest(t *testing.T) {
+
+	r := &osRouter{}
+	route := &Route{
+		Gateway: net.IPv4(127, 0, 0, 1),
+		DestCIDR: &net.IPNet{
+			IP:   net.IPv4(10, 96, 0, 0),
+			Mask: net.IPv4Mask(255, 240, 0, 0),
+		},
+	}
+
+	cleanRoute(t, "10.96.0.0/12")
+	addRoute(t, "10.96.0.0/12", "127.0.0.1")
+	err := r.Cleanup(route)
+	if err != nil {
+		t.Errorf("cleanup failed: %s", err)
+	}
+	err = r.Cleanup(route)
+	if err != nil {
+		t.Errorf("cleanup failed: %s", err)
+	}
+}
+
+func TestParseTable(t *testing.T) {
+
+	const table = `Kernel IP routing table
+Destination     Gateway         Genmask         Flags   MSS Window  irtt Iface
+0.0.0.0         172.31.126.254  0.0.0.0         UG        0 0          0 eno1
+10.96.0.0       127.0.0.1   		255.240.0.0			UG        0 0          0 eno1
+172.31.126.0    0.0.0.0         255.255.255.0   U         0 0          0 eno1
+`
+
+	rt := (&osRouter{}).parseTable([]byte(table))
+
+	expectedRt := routingTable{
+		routingTableLine{
+			route: unsafeParseRoute("127.0.0.1", "10.96.0.0/12"),
+			line: "10.96.0.0       127.0.0.1   		255.240.0.0			UG        0 0          0 eno1",
+		},
+		routingTableLine{
+			route: unsafeParseRoute("0.0.0.0", "172.31.126.0/24"),
+			line:  "172.31.126.0    0.0.0.0         255.255.255.0   U         0 0          0 eno1",
+		},
+	}
+	if !expectedRt.Equal(&rt) {
+		t.Errorf("expected:\n %s\ngot\n %s", expectedRt.String(), rt.String())
+	}
+}
+
+func addRoute(t *testing.T, cidr string, gw string) {
+	command := exec.Command("sudo", "ip", "route", "add", cidr, "via", gw)
+	sout, err := command.CombinedOutput()
+	if err != nil {
+		t.Logf("assertion add Route error (should be ok): %s, error: %s", sout, err)
+	} else {
+		t.Logf("assertion - successfully added %s -> %s", cidr, gw)
+	}
+}
+
+func cleanRoute(t *testing.T, cidr string) {
+	command := exec.Command("sudo", "ip", "route", "delete", cidr)
+	sout, err := command.CombinedOutput()
+	if err != nil {
+		t.Logf("integration test cleanup error (should be ok): %s, error: %s", sout, err)
+	} else {
+		t.Logf("integration test successfully cleaned %s", cidr)
+	}
+}

pkg/minikube/tunnel/route_test.go

+/*
+Copyright 2018 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tunnel
+
+import (
+	"net"
+	"reflect"
+	"testing"
+)
+
+func TestRoutingTable(t *testing.T) {
+	tcs := []struct {
+		name  string
+		table routingTable
+		route *Route
+
+		exists   bool
+		conflict string
+		overlaps []string
+	}{
+		{
+			name: "doesn't exist, no complication",
+			table: routingTable{
+				{
+					route: unsafeParseRoute("127.0.0.1", "10.96.0.0/12"),
+					line:  "line1",
+				},
+			},
+			route: unsafeParseRoute("127.0.0.1", "10.112.0.0/12"),
+
+			exists:   false,
+			conflict: "",
+			overlaps: []string{},
+		},
+
+		{
+			name: "doesn't exist, and has overlap and a conflict",
+			table: routingTable{
+				{
+					route: unsafeParseRoute("127.0.0.1", "10.96.0.0/12"),
+					line:  "conflicting line",
+				},
+				{
+					route: unsafeParseRoute("127.0.0.1", "10.98.0.0/8"),
+					line:  "overlap line1",
+				},
+				{
+					route: unsafeParseRoute("127.0.0.1", "10.100.0.0/24"),
+					line:  "overlap line2",
+				},
+				{
+					route: unsafeParseRoute("127.0.0.1", "192.96.0.0/12"),
+					line:  "no overlap",
+				},
+			},
+			route: unsafeParseRoute("192.168.1.1", "10.96.0.0/12"),
+
+			exists:   false,
+			conflict: "conflicting line",
+			overlaps: []string{
+				"overlap line1",
+				"overlap line2",
+			},
+		},
+
+		{
+			name: "exists, and has overlap and no conflict",
+			table: routingTable{
+				{
+					route: unsafeParseRoute("127.0.0.1", "10.96.0.0/12"),
+					line:  "same",
+				},
+				{
+					route: unsafeParseRoute("127.0.0.1", "10.98.0.0/8"),
+					line:  "overlap line1",
+				},
+				{
+					route: unsafeParseRoute("127.0.0.1", "10.100.0.0/24"),
+					line:  "overlap line2",
+				},
+				{
+					route: unsafeParseRoute("127.0.0.1", "192.96.0.0/12"),
+					line:  "no overlap",
+				},
+			},
+			route: unsafeParseRoute("127.0.0.1", "10.96.0.0/12"),
+
+			exists:   true,
+			conflict: "",
+			overlaps: []string{
+				"overlap line1",
+				"overlap line2",
+			},
+		},
+	}
+
+	for _, tc := range tcs {
+		t.Run(tc.name, func(t *testing.T) {
+			exists, conflict, overlaps := tc.table.Check(tc.route)
+			if tc.exists != exists || tc.conflict != conflict || !reflect.DeepEqual(tc.overlaps, overlaps) {
+				t.Errorf(`expected
+  exists: %v
+  conflict: %s
+  overlaps: %s
+got
+  exists: %v
+  conflict: %s
+  overlaps: %s
+`, tc.exists, tc.conflict, tc.overlaps,
+					exists, conflict, overlaps)
+			}
+		})
+	}
+}
+
+func unsafeParseRoute(gatewayIP string, destCIDR string) *Route {
+	ip := net.ParseIP(gatewayIP)
+	_, ipNet, _ := net.ParseCIDR(destCIDR)
+
+	expectedRoute := &Route{
+		Gateway:  ip,
+		DestCIDR: ipNet,
+	}
+	return expectedRoute
+}

pkg/minikube/tunnel/route_windows.go

+/*
+Copyright 2018 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tunnel
+
+import (
+	"fmt"
+	"net"
+	"os/exec"
+	"strings"
+
+	"github.com/golang/glog"
+)
+
+func (router *osRouter) EnsureRouteIsAdded(route *Route) error {
+	exists, err := isValidToAddOrDelete(router, route)
+	if err != nil {
+		return err
+	}
+	if exists {
+		return nil
+	}
+
+	serviceCIDR := route.DestCIDR.String()
+	destinationIP := route.DestCIDR.IP.String()
+	destinationMask := fmt.Sprintf("%d.%d.%d.%d",
+		route.DestCIDR.Mask[0],
+		route.DestCIDR.Mask[1],
+		route.DestCIDR.Mask[2],
+		route.DestCIDR.Mask[3])
+
+	gatewayIP := route.Gateway.String()
+
+	glog.Infof("Adding Route for CIDR %s to gateway %s", serviceCIDR, gatewayIP)
+	command := exec.Command("route", "ADD", destinationIP, "MASK", destinationMask, gatewayIP)
+	glog.Infof("About to run command: %s", command.Args)
+	stdInAndOut, err := command.CombinedOutput()
+	message := string(stdInAndOut)
+	if message != " OK!\r\n" {
+		return fmt.Errorf("error adding route: %s, %d", message, len(strings.Split(message, "\n")))
+	}
+	glog.Infof("%s", stdInAndOut)
+	if err != nil {
+		glog.Errorf("error adding Route: %s, %d", message, len(strings.Split(message, "\n")))
+		return err
+	}
+	return nil
+}
+
+func (router *osRouter) parseTable(table []byte) routingTable {
+	t := routingTable{}
+	skip := true
+	for _, line := range strings.Split(string(table), "\n") {
+		//after first line of header we can start consuming
+		if strings.HasPrefix(line, "Network Destination") {
+			skip = false
+			continue
+		}
+
+		fields := strings.Fields(line)
+		//don't care about the 0.0.0.0 routes
+		if skip || len(fields) == 0 || len(fields) > 0 && (fields[0] == "default" || fields[0] == "0.0.0.0") {
+			continue
+		}
+		if len(fields) > 2 {
+			dstCIDRIP := net.ParseIP(fields[0])
+			dstCIDRMask := fields[1]
+			dstMaskIP := net.ParseIP(dstCIDRMask)
+			gatewayIP := net.ParseIP(fields[2])
+			if dstCIDRIP == nil || dstMaskIP == nil || gatewayIP == nil {
+				glog.V(4).Infof("skipping line: can't parse all IPs from routing table: %s", line)
+			} else {
+				tableLine := routingTableLine{
+					route: &Route{
+						DestCIDR: &net.IPNet{
+							IP:   dstCIDRIP,
+							Mask: net.IPMask(dstMaskIP.To4()),
+						},
+						Gateway: gatewayIP,
+					},
+					line: line,
+				}
+				glog.V(4).Infof("adding line %s", tableLine)
+				t = append(t, tableLine)
+			}
+		}
+	}
+
+	return t
+}
+
+func (router *osRouter) Inspect(route *Route) (exists bool, conflict string, overlaps []string, err error) {
+	command := exec.Command("route", "print", "-4")
+	stdInAndOut, err := command.CombinedOutput()
+	if err != nil {
+		err = fmt.Errorf("error running '%s': %s", command.Args, err)
+		return
+	}
+	rt := router.parseTable(stdInAndOut)
+
+	exists, conflict, overlaps = rt.Check(route)
+
+	return
+}
+
+func (router *osRouter) Cleanup(route *Route) error {
+	exists, err := isValidToAddOrDelete(router, route)
+	if err != nil {
+		return err
+	}
+	if !exists {
+		return nil
+	}
+	serviceCIDR := route.DestCIDR.String()
+	gatewayIP := route.Gateway.String()
+
+	glog.Infof("Cleaning up Route for CIDR %s to gateway %s\n", serviceCIDR, gatewayIP)
+	command := exec.Command("route", "delete", serviceCIDR)
+	stdInAndOut, err := command.CombinedOutput()
+	if err != nil {
+		return err
+	}
+	message := string(stdInAndOut)
+	glog.Infof("'%s'", message)
+	if message != " OK!\r\n" {
+		return fmt.Errorf("error deleting route: %s, %d", message, len(strings.Split(message, "\n")))
+	}
+	return nil
+}

pkg/util/utils.go

@@ -97,14 +97,17 @@ func Retry(attempts int, callback func() error) (err error) {
 func RetryAfter(attempts int, callback func() error, d time.Duration) (err error) {
 	m := MultiError{}
 	for i := 0; i < attempts; i++ {
+		glog.V(1).Infof("retry loop %d", i)
 		err = callback()
 		if err == nil {
 			return nil
 		}
 		m.Collect(err)
 		if _, ok := err.(*RetriableError); !ok {
+			glog.Infof("non-retriable error: %v", err)
 			return m.ToError()
 		}
+		glog.V(2).Infof("sleeping %s", d)
 		time.Sleep(d)
 	}
 	return m.ToError()

test.sh

@@ -43,7 +43,7 @@ done
 rm out/$COV_TMP_FILE
 
 # Ignore these paths in the following tests.
-ignore="vendor\|\_gopath\|assets.go\|out"
+ignore="vendor\|\_gopath\|assets.go\|out\/"
 
 # Check gofmt
 echo "Checking gofmt..."
@@ -51,6 +51,7 @@ set +e
 files=$(gofmt -l -s . | grep -v ${ignore})
 set -e
 if [[ $files ]]; then
+  gofmt -d ${files}
   echo "Gofmt errors in files: $files"
   exit 1
 fi

test/integration/cluster_status_test.go

@@ -42,7 +42,6 @@ func testClusterStatus(t *testing.T) {
 				if c.Type != api.ComponentHealthy {
 					continue
 				}
-				t.Logf("Component: %v, Healthy: %s.\n", i.GetName(), c.Status)
 				status = c.Status
 			}
 			if status != api.ConditionTrue {

test/integration/functional_test.go

@@ -36,6 +36,7 @@ func TestFunctional(t *testing.T) {
 	t.Run("Dashboard", testDashboard)
 	t.Run("ServicesList", testServicesList)
 	t.Run("Provisioning", testProvisioning)
+	t.Run("Tunnel", testTunnel)
 
 	if !strings.Contains(minikubeRunner.StartArgs, "--vm-driver=none") {
 		t.Run("EnvVars", testClusterEnv)

test/integration/mount_test.go

@@ -46,8 +46,13 @@ func testMounting(t *testing.T) {
 	defer os.RemoveAll(tempDir)
 
 	mountCmd := fmt.Sprintf("mount %s:/mount-9p", tempDir)
-	cmd := minikubeRunner.RunDaemon(mountCmd)
-	defer cmd.Process.Kill()
+	cmd, _ := minikubeRunner.RunDaemon(mountCmd)
+	defer func() {
+		err := cmd.Process.Kill()
+		if err != nil {
+			t.Logf("Failed to kill mount command: %v", err)
+		}
+	}()
 
 	kubectlRunner := util.NewKubectlRunner(t)
 	podName := "busybox-mount"