Merge branch 'master' into f-fix-5144

.github/workflows/main.yml

+name: CI
+
+on: [pull_request]
+
+jobs:
+  docker_ubuntu_16_04:
+    runs-on: ubuntu-16.04
+    steps:
+    - uses: actions/checkout@v2
+    - name: build binaries
+      run : |
+        make minikube-linux-amd64
+        make e2e-linux-amd64
+        mkdir -p report
+    - name: install gopogh
+      run: |
+        cd /tmp
+        GO111MODULE="on" go get github.com/medyagh/gopogh@v0.0.17 || true
+        cd -
+    - name: run integration test
+      run: |
+        mkdir -p /tmp/testhome
+        MINIKUBE_HOME=/tmp/testhome ./out/e2e-linux-amd64 -minikube-start-args=--vm-driver=docker -expected-default-driver= -test.timeout=70m -test.v -binary=out/minikube-linux-amd64 2>&1 | tee ./report/testout.txt
+    - name: generate gopogh report
+      run: |
+        export PATH=${PATH}:`go env GOPATH`/bin
+        go tool test2json -t < ./report/testout.txt > ./report/testout.json || true
+        gopogh -in ./report/testout.json -out ./report/testout.html -name "docker ubuntu" -repo github.com/kubernetes/minikube/  || true
+    - uses: actions/upload-artifact@v1
+      with:
+        name: docker_on_ubuntu_16_04_report
+        path: report
+  docker_ubuntu_18_04:
+    runs-on: ubuntu-18.04
+    steps:
+    - uses: actions/checkout@v2
+    - name: build binaries
+      run : |
+        make minikube-linux-amd64
+        make e2e-linux-amd64
+        mkdir -p report
+    - name: install gopogh
+      run: |
+        cd /tmp
+        GO111MODULE="on" go get github.com/medyagh/gopogh@v0.0.17 || true
+        cd -
+    - name: run integration test
+      run: |
+        mkdir -p /tmp/testhome 
+        MINIKUBE_HOME=/tmp/testhome ./out/e2e-linux-amd64 -minikube-start-args=--vm-driver=docker -expected-default-driver= -test.timeout=70m -test.v -binary=out/minikube-linux-amd64 2>&1 | tee ./report/testout.txt
+    - name: generate gopogh report
+      run: |
+        export PATH=${PATH}:`go env GOPATH`/bin
+        go tool test2json -t < ./report/testout.txt > ./report/testout.json || true
+        gopogh -in ./report/testout.json -out ./report/testout.html -name "docker ubuntu" -repo github.com/kubernetes/minikube/  || true
+    - uses: actions/upload-artifact@v1
+      with:
+        name: docker_on_ubuntu_18_04_report
+        path: report
+  docker_macos:
+    runs-on: macos-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: build binaries
+      run : |
+        make minikube-darwin-amd64
+        make e2e-darwin-amd64
+        mkdir -p report
+    - name: install docker
+      run: |
+          brew install docker-machine docker || true
+          brew services start docker-machine || true
+          docker version || true
+    - name: install gopogh
+      run: |
+        cd /tmp
+        GO111MODULE="on" go get github.com/medyagh/gopogh@v0.0.17 || true
+        cd -
+    - name: run integration test
+      run: |
+        mkdir -p /tmp/testhome
+        MINIKUBE_HOME=/tmp/testhome ./out/e2e-darwin-amd64 -minikube-start-args=--vm-driver=docker -expected-default-driver= -test.timeout=70m -test.v -binary=./out/minikube-darwin-amd64 2>&1 | tee ./report/testout.txt
+    - name: generate gopogh report
+      run: |
+        export PATH=${PATH}:`go env GOPATH`/bin
+        go tool test2json -t < ./report/testout.txt > ./report/testout.json || true
+        gopogh -in ./report/testout.json -out ./report/testout.html -name "docker macos" -repo github.com/kubernetes/minikube/  || true
+    - uses: actions/upload-artifact@v1
+      with:
+        name: docker_on_macos_report
+        path: ./report
+  none_ubuntu16_04:
+    runs-on: ubuntu-16.04
+    steps:
+    - uses: actions/checkout@v2
+    - name: build binaries
+      run : |
+        make minikube-linux-amd64
+        make e2e-linux-amd64
+        mkdir -p report
+    - name: install gopogh
+      run: |
+        cd /tmp
+        GO111MODULE="on" go get github.com/medyagh/gopogh@v0.0.17 || true
+        cd -
+    - name: run integration test
+      run: |
+        mkdir -p /tmp/testhome
+        MINIKUBE_HOME=/tmp/testhome sudo -E ./out/e2e-linux-amd64 -minikube-start-args=--vm-driver=none -expected-default-driver= -test.timeout=70m -test.v -binary=out/minikube-linux-amd64 2>&1 | tee ./report/testout.txt
+    - name: generate gopogh report
+      run: |
+        export PATH=${PATH}:`go env GOPATH`/bin
+        go tool test2json -t < ./report/testout.txt > ./report/testout.json || true
+        gopogh -in ./report/testout.json -out ./report/testout.html -name "docker ubuntu" -repo github.com/kubernetes/minikube/  || true
+    - uses: actions/upload-artifact@v1
+      with:
+        name: none_on_ubuntu_16_04
+        path: report
+  none_ubuntu_18_04:
+    runs-on: ubuntu-18.04
+    steps:
+    - uses: actions/checkout@v2
+    - name: build binaries
+      run : |
+        make minikube-linux-amd64
+        make e2e-linux-amd64
+    - name: install gopogh
+      run: |
+        cd /tmp
+        GO111MODULE="on" go get github.com/medyagh/gopogh@v0.0.17 || true
+        cd -
+    - name: run integration test
+      run: |
+        mkdir -p /tmp/testhome
+        MINIKUBE_HOME=/tmp/testhome sudo -E ./out/e2e-linux-amd64 -minikube-start-args=--vm-driver=none -expected-default-driver= -test.timeout=70m -test.v -binary=out/minikube-linux-amd64 2>&1 | tee ./report/testout.txt
+    - name: generate gopogh report
+      run: |
+        export PATH=${PATH}:`go env GOPATH`/bin
+        go tool test2json -t < ./report/testout.txt > ./report/testout.json || true
+        gopogh -in ./report/testout.json -out ./report/testout.html -name "docker ubuntu" -repo github.com/kubernetes/minikube/  || true
+    - uses: actions/upload-artifact@v1
+      with:
+        name: none_on_ubuntu_latest_report
+        path: report

CHANGELOG.md

 # Release Notes
 
+## Version 1.7.0 - 2020-02-04
+
+* Add Azure Container Registry support [#6483](https://github.com/kubernetes/minikube/pull/6483)
+* Support --force for overriding the ssh check [#6237](https://github.com/kubernetes/minikube/pull/6237)
+* Update translation files with new strings [#6491](https://github.com/kubernetes/minikube/pull/6491)
+* fix docker-env for kic drivers [#6487](https://github.com/kubernetes/minikube/pull/6487)
+* Fix bugs that prevented previously-enabled addons from starting up [#6471](https://github.com/kubernetes/minikube/pull/6471)
+* Fix none driver bugs with "pause"  [#6452](https://github.com/kubernetes/minikube/pull/6452)
+
+Thank you to those brave souls who made the final push toward this release:
+
+- Medya Gh
+- Priya Wadhwa
+- Sharif Elgamal
+- Thomas Strömberg
+
+## Version 1.7.0-beta.2 - 2020-01-31
+
+* Add docker run-time for kic driver [#6436](https://github.com/kubernetes/minikube/pull/6436)
+* Configure etcd and kube-proxy metrics to listen on minikube node IP [#6322](https://github.com/kubernetes/minikube/pull/6322)
+* add container runtime info to profile list [#6409](https://github.com/kubernetes/minikube/pull/6409)
+* status: Explicitly state that the cluster does not exist [#6438](https://github.com/kubernetes/minikube/pull/6438)
+* Do not use an arch suffix for the coredns name [#6243](https://github.com/kubernetes/minikube/pull/6243)
+* Prevent registry-creds configure from failing when a secret does not exist.  [#6380](https://github.com/kubernetes/minikube/pull/6380)
+* improve checking modprob netfilter [#6427](https://github.com/kubernetes/minikube/pull/6427)
+
+Huge thank you for this release towards our contributors: 
+
+- Anders Björklund
+- Bjørn Harald Fotland
+- Chance Zibolski
+- Kim Bao Long
+- Medya Ghazizadeh
+- Priya Wadhwa
+- Sharif Elgamal
+- Thomas Strömberg
+- akshay
+
+## Version 1.7.0-beta.1 - 2020-01-24
+
+* Add 'pause' command to freeze Kubernetes cluster [#5962](https://github.com/kubernetes/minikube/pull/5962)
+* kic driver: add multiple profiles and ssh [#6390](https://github.com/kubernetes/minikube/pull/6390)
+* Update DefaultKubernetesVersion to v1.17.2 [#6392](https://github.com/kubernetes/minikube/pull/6392)
+* Add varlink program for using with podman-remote [#6349](https://github.com/kubernetes/minikube/pull/6349)
+* Update Kubernetes libraries to v1.17.2 [#6374](https://github.com/kubernetes/minikube/pull/6374)
+* Remove addon manager [#6334](https://github.com/kubernetes/minikube/pull/6334)
+* Remove unnecessary crio restart to improve start latency [#6369](https://github.com/kubernetes/minikube/pull/6369)
+* Check for nil ref and img before passing them into go-containerregistry [#6236](https://github.com/kubernetes/minikube/pull/6236)
+* Change the compression methods used on the iso [#6341](https://github.com/kubernetes/minikube/pull/6341)
+* Update the crio.conf instead of overwriting it [#6219](https://github.com/kubernetes/minikube/pull/6219)
+* Update Japanese translation [#6339](https://github.com/kubernetes/minikube/pull/6339)
+* Stop minikube dashboard from crashing at start [#6325](https://github.com/kubernetes/minikube/pull/6325)
+
+Thanks you to the following contributors:
+
+- Anders F Björklund
+- inductor
+- Medya Ghazizadeh
+- Naoki Oketani
+- Priya Wadhwa
+- Sharif Elgamal
+- sshukun
+- Thomas Strömberg
+
+## Version 1.7.0-beta.0 - 2020-01-15
+
+* Use CGroupDriver function from cruntime for kubelet [#6287](https://github.com/kubernetes/minikube/pull/6287)
+* Experimental Docker support (kic) using the Kind image [#6151](https://github.com/kubernetes/minikube/pull/6151)
+* disable istio provisioner by default [#6315](https://github.com/kubernetes/minikube/pull/6315)
+* Add --dry-run option to start [#6256](https://github.com/kubernetes/minikube/pull/6256)
+* Improve "addon list" by viewing as a table  [#6274](https://github.com/kubernetes/minikube/pull/6274)
+* Disable IPv6 in the minikube VM until it can be properly supported [#6241](https://github.com/kubernetes/minikube/pull/6241)
+* Fixes IPv6 address handling in kubeadm [#6214](https://github.com/kubernetes/minikube/pull/6214)
+* Upgrade crio to 1.16.1 [#6210](https://github.com/kubernetes/minikube/pull/6210)
+* Upgrade podman to 1.6.4 [#6208](https://github.com/kubernetes/minikube/pull/6208)
+* Enable or disable addons per profile [#6124](https://github.com/kubernetes/minikube/pull/6124)
+* Upgrade buildroot minor version [#6199](https://github.com/kubernetes/minikube/pull/6199)
+* Add systemd patch for booting on AMD Ryzen [#6183](https://github.com/kubernetes/minikube/pull/6183)
+* update zh translation [#6176](https://github.com/kubernetes/minikube/pull/6176)
+* Add istio addon for minikube [#6154](https://github.com/kubernetes/minikube/pull/6154)
+
+Huge thank you for this release towards our contributors:
+- Anders Björklund
+- andylibrian
+- Dao Cong Tien
+- Dominic Yin
+- fenglixa
+- GennadySpb
+- Kenta Iso
+- Kim Bao Long
+- Medya Ghazizadeh
+- Nguyen Hai Truong
+- Priya Wadhwa
+- Sharif Elgamal
+- Thomas Strömberg
+- ttonline6
+- Zhongcheng Lao
+- Zhou Hao
 
 ## Version 1.6.2  - 2019-12-19
 
@@ -11,7 +109,7 @@
 * start: Remove create/delete retry loop [#6129](https://github.com/kubernetes/minikube/pull/6129)
 * Change error text to encourage better issue reports [#6121](https://github.com/kubernetes/minikube/pull/6121)
 
-Huge thank you for this release towards our contributors: 
+Huge thank you for this release towards our contributors:
 - Anukul Sangwan
 - Aresforchina
 - Curtis Carter

Makefile

@@ -15,12 +15,12 @@
 # Bump these on release - and please check ISO_VERSION for correctness.
 VERSION_MAJOR ?= 1
 VERSION_MINOR ?= 7
-VERSION_BUILD ?= 0-beta.0
+VERSION_BUILD ?= 0
 RAW_VERSION=$(VERSION_MAJOR).$(VERSION_MINOR).${VERSION_BUILD}
 VERSION ?= v$(RAW_VERSION)
 
 # Default to .0 for higher cache hit rates, as build increments typically don't require new ISO versions
-ISO_VERSION ?= v$(VERSION_MAJOR).$(VERSION_MINOR).0-beta.0
+ISO_VERSION ?= v$(VERSION_MAJOR).$(VERSION_MINOR).$(VERSION_BUILD)
 # Dashes are valid in semver, but not Linux packaging. Use ~ to delimit alpha/beta
 DEB_VERSION ?= $(subst -,~,$(RAW_VERSION))
 RPM_VERSION ?= $(DEB_VERSION)
@@ -51,7 +51,7 @@ MINIKUBE_RELEASES_URL=https://github.com/kubernetes/minikube/releases/download
 
 KERNEL_VERSION ?= 4.19.88
 # latest from https://github.com/golangci/golangci-lint/releases
-GOLINT_VERSION ?= v1.21.0
+GOLINT_VERSION ?= v1.23.2
 # Limit number of default jobs, to avoid the CI builds running out of memory
 GOLINT_JOBS ?= 4
 # see https://github.com/golangci/golangci-lint#memory-usage-of-golangci-lint
@@ -73,6 +73,7 @@ GOARCH ?= $(shell go env GOARCH)
 GOPATH ?= $(shell go env GOPATH)
 BUILD_DIR ?= ./out
 $(shell mkdir -p $(BUILD_DIR))
+CURRENT_GIT_BRANCH ?= $(shell git branch | grep \* | cut -d ' ' -f2)
 
 # Use system python if it exists, otherwise use Docker.
 PYTHON := $(shell command -v python || echo "docker run --rm -it -v $(shell pwd):/minikube -w /minikube python python")
@@ -81,10 +82,16 @@ BUILD_OS := $(shell uname -s)
 SHA512SUM=$(shell command -v sha512sum || echo "shasum -a 512")
 
 STORAGE_PROVISIONER_TAG := v1.8.1
+# TODO: multi-arch manifest
+ifeq ($(GOARCH),amd64)
+STORAGE_PROVISIONER_IMAGE ?= $(REGISTRY)/storage-provisioner:$(STORAGE_PROVISIONER_TAG)
+else
+STORAGE_PROVISIONER_IMAGE ?= $(REGISTRY)/storage-provisioner-$(GOARCH):$(STORAGE_PROVISIONER_TAG)
+endif
 
 # Set the version information for the Kubernetes servers
 MINIKUBE_LDFLAGS := -X k8s.io/minikube/pkg/version.version=$(VERSION) -X k8s.io/minikube/pkg/version.isoVersion=$(ISO_VERSION) -X k8s.io/minikube/pkg/version.isoPath=$(ISO_BUCKET) -X k8s.io/minikube/pkg/version.gitCommitID=$(COMMIT)
-PROVISIONER_LDFLAGS := "$(MINIKUBE_LDFLAGS) -s -w -extldflags '-static'"
+PROVISIONER_LDFLAGS := "-X k8s.io/minikube/pkg/storage.version=$(STORAGE_PROVISIONER_TAG) -s -w -extldflags '-static'"
 
 MINIKUBEFILES := ./cmd/minikube/
 HYPERKIT_FILES := ./cmd/drivers/hyperkit
@@ -368,6 +375,9 @@ mdlint:
 out/docs/minikube.md: $(shell find "cmd") $(shell find "pkg/minikube/constants") pkg/minikube/assets/assets.go pkg/minikube/translate/translations.go
 	go run -ldflags="$(MINIKUBE_LDFLAGS)" -tags gendocs hack/help_text/gen_help_text.go
 
+deb_version:
+	@echo $(DEB_VERSION)
+
 out/minikube_$(DEB_VERSION).deb: out/minikube_$(DEB_VERSION)-0_amd64.deb
 	cp $< $@
 
@@ -381,6 +391,9 @@ out/minikube_$(DEB_VERSION)-0_%.deb: out/minikube-linux-%
 	fakeroot dpkg-deb --build out/minikube_$(DEB_VERSION) $@
 	rm -rf out/minikube_$(DEB_VERSION)
 
+rpm_version:
+	@echo $(RPM_VERSION)
+
 out/minikube-$(RPM_VERSION).rpm: out/minikube-$(RPM_VERSION)-0.x86_64.rpm
 	cp $< $@
 
@@ -472,31 +485,30 @@ $(ISO_BUILD_IMAGE): deploy/iso/minikube-iso/Dockerfile
 	@echo ""
 	@echo "$(@) successfully built"
 
-out/storage-provisioner:
-	CGO_ENABLED=0 GOOS=linux go build -o $@ -ldflags=$(PROVISIONER_LDFLAGS) cmd/storage-provisioner/main.go
+out/storage-provisioner: out/storage-provisioner-$(GOARCH)
+	cp $< $@
 
-.PHONY: storage-provisioner-image
-storage-provisioner-image: out/storage-provisioner ## Build storage-provisioner docker image
-ifeq ($(GOARCH),amd64)
-	docker build -t $(REGISTRY)/storage-provisioner:$(STORAGE_PROVISIONER_TAG) -f deploy/storage-provisioner/Dockerfile  .
+out/storage-provisioner-%: cmd/storage-provisioner/main.go pkg/storage/storage_provisioner.go
+ifeq ($(MINIKUBE_BUILD_IN_DOCKER),y)
+	$(call DOCKER,$(BUILD_IMAGE),/usr/bin/make $@)
 else
-	docker build -t $(REGISTRY)/storage-provisioner-$(GOARCH):$(STORAGE_PROVISIONER_TAG) -f deploy/storage-provisioner/Dockerfile-$(GOARCH) .
+	CGO_ENABLED=0 GOOS=linux GOARCH=$* go build -o $@ -ldflags=$(PROVISIONER_LDFLAGS) cmd/storage-provisioner/main.go
 endif
 
+.PHONY: storage-provisioner-image
+storage-provisioner-image: out/storage-provisioner-$(GOARCH) ## Build storage-provisioner docker image
+	docker build -t $(STORAGE_PROVISIONER_IMAGE) -f deploy/storage-provisioner/Dockerfile  --build-arg arch=$(GOARCH) .
+
 .PHONY: kic-base-image
 kic-base-image: ## builds the base image used for kic.
-	docker rmi -f $(REGISTRY)/kicbase:v0.0.1-snapshot || true
-	docker build -f ./hack/images/kicbase.Dockerfile -t $(REGISTRY)/kicbase:v0.0.1-snapshot  --build-arg COMMIT_SHA=${VERSION}-$(COMMIT)  .
+	docker rmi -f $(REGISTRY)/kicbase:v0.0.5-snapshot || true
+	docker build -f ./hack/images/kicbase.Dockerfile -t $(REGISTRY)/kicbase:v0.0.5-snapshot  --build-arg COMMIT_SHA=${VERSION}-$(COMMIT)  .
 
 
 
 .PHONY: push-storage-provisioner-image
 push-storage-provisioner-image: storage-provisioner-image ## Push storage-provisioner docker image using gcloud
-ifeq ($(GOARCH),amd64)
-	gcloud docker -- push $(REGISTRY)/storage-provisioner:$(STORAGE_PROVISIONER_TAG)
-else
-	gcloud docker -- push $(REGISTRY)/storage-provisioner-$(GOARCH):$(STORAGE_PROVISIONER_TAG)
-endif
+	gcloud docker -- push $(STORAGE_PROVISIONER_IMAGE)
 
 .PHONY: out/gvisor-addon
 out/gvisor-addon: pkg/minikube/assets/assets.go pkg/minikube/translate/translations.go ## Build gvisor addon
@@ -520,13 +532,23 @@ release-minikube: out/minikube checksum ## Minikube release
 	gsutil cp out/minikube-$(GOOS)-$(GOARCH) $(MINIKUBE_UPLOAD_LOCATION)/$(MINIKUBE_VERSION)/minikube-$(GOOS)-$(GOARCH)
 	gsutil cp out/minikube-$(GOOS)-$(GOARCH).sha256 $(MINIKUBE_UPLOAD_LOCATION)/$(MINIKUBE_VERSION)/minikube-$(GOOS)-$(GOARCH).sha256
 
-out/docker-machine-driver-kvm2:
+out/docker-machine-driver-kvm2: out/docker-machine-driver-kvm2-amd64
+	cp $< $@
+
+out/docker-machine-driver-kvm2-x86_64: out/docker-machine-driver-kvm2-amd64
+	cp $< $@
+
+out/docker-machine-driver-kvm2-aarch64: out/docker-machine-driver-kvm2-arm64
+	cp $< $@
+
+out/docker-machine-driver-kvm2-%:
 ifeq ($(MINIKUBE_BUILD_IN_DOCKER),y)
 	docker inspect -f '{{.Id}} {{.RepoTags}}' $(KVM_BUILD_IMAGE) || $(MAKE) kvm-image
 	$(call DOCKER,$(KVM_BUILD_IMAGE),/usr/bin/make $@ COMMIT=$(COMMIT))
 	# make extra sure that we are linking with the older version of libvirt (1.3.1)
 	test "`strings $@ | grep '^LIBVIRT_[0-9]' | sort | tail -n 1`" = "LIBVIRT_1.2.9"
 else
+	GOARCH=$* \
 	go build \
 		-installsuffix "static" \
 		-ldflags="$(KVM2_LDFLAGS)" \
@@ -536,21 +558,29 @@ else
 endif
 	chmod +X $@
 
-out/docker-machine-driver-kvm2_$(DEB_VERSION).deb: out/docker-machine-driver-kvm2
+out/docker-machine-driver-kvm2_$(DEB_VERSION).deb: out/docker-machine-driver-kvm2_$(DEB_VERSION)-0_amd64.deb
+	cp $< $@
+
+out/docker-machine-driver-kvm2_$(DEB_VERSION)-0_%.deb: out/docker-machine-driver-kvm2-%
 	cp -r installers/linux/deb/kvm2_deb_template out/docker-machine-driver-kvm2_$(DEB_VERSION)
 	chmod 0755 out/docker-machine-driver-kvm2_$(DEB_VERSION)/DEBIAN
 	sed -E -i 's/--VERSION--/'$(DEB_VERSION)'/g' out/docker-machine-driver-kvm2_$(DEB_VERSION)/DEBIAN/control
+	sed -E -i 's/--ARCH--/'$*'/g' out/docker-machine-driver-kvm2_$(DEB_VERSION)/DEBIAN/control
 	mkdir -p out/docker-machine-driver-kvm2_$(DEB_VERSION)/usr/bin
-	cp out/docker-machine-driver-kvm2 out/docker-machine-driver-kvm2_$(DEB_VERSION)/usr/bin/docker-machine-driver-kvm2
-	fakeroot dpkg-deb --build out/docker-machine-driver-kvm2_$(DEB_VERSION)
+	cp $< out/docker-machine-driver-kvm2_$(DEB_VERSION)/usr/bin/docker-machine-driver-kvm2
+	fakeroot dpkg-deb --build out/docker-machine-driver-kvm2_$(DEB_VERSION) $@
 	rm -rf out/docker-machine-driver-kvm2_$(DEB_VERSION)
 
-out/docker-machine-driver-kvm2-$(RPM_VERSION).rpm: out/docker-machine-driver-kvm2
+out/docker-machine-driver-kvm2-$(RPM_VERSION).rpm: out/docker-machine-driver-kvm2-$(RPM_VERSION)-0.x86_64.deb
+	cp $< $@
+
+out/docker-machine-driver-kvm2-$(RPM_VERSION)-0.%.rpm: out/docker-machine-driver-kvm2-%
 	cp -r installers/linux/rpm/kvm2_rpm_template out/docker-machine-driver-kvm2-$(RPM_VERSION)
 	sed -E -i 's/--VERSION--/'$(RPM_VERSION)'/g' out/docker-machine-driver-kvm2-$(RPM_VERSION)/docker-machine-driver-kvm2.spec
 	sed -E -i 's|--OUT--|'$(PWD)/out'|g' out/docker-machine-driver-kvm2-$(RPM_VERSION)/docker-machine-driver-kvm2.spec
-	rpmbuild -bb -D "_rpmdir $(PWD)/out" -D "_rpmfilename docker-machine-driver-kvm2-$(RPM_VERSION).rpm" \
+	rpmbuild -bb -D "_rpmdir $(PWD)/out" --target $* \
 		out/docker-machine-driver-kvm2-$(RPM_VERSION)/docker-machine-driver-kvm2.spec
+	@mv out/$*/docker-machine-driver-kvm2-$(RPM_VERSION)-0.$*.rpm out/ && rmdir out/$*
 	rm -rf out/docker-machine-driver-kvm2-$(RPM_VERSION)
 
 .PHONY: kvm-image
@@ -598,6 +628,15 @@ out/mkcmp:
 out/performance-monitor:
 	GOOS=$(GOOS) GOARCH=$(GOARCH) go build -o $@ cmd/performance/monitor/monitor.go
 
+.PHONY: compare
+compare: out/mkcmp out/minikube
+	mv out/minikube out/$(CURRENT_GIT_BRANCH).minikube
+	git checkout master
+	make out/minikube
+	mv out/minikube out/master.minikube
+	git checkout $(CURRENT_GIT_BRANCH)
+	out/mkcmp out/master.minikube out/$(CURRENT_GIT_BRANCH).minikube
+	
 
 .PHONY: help
 help:

cmd/minikube/cmd/config/addons_list.go

@@ -102,9 +102,9 @@ var printAddonsList = func() {
 
 	for _, addonName := range addonNames {
 		addonBundle := assets.Addons[addonName]
-		addonStatus, err := addonBundle.IsEnabled()
+		addonStatus, err := addonBundle.IsEnabled(pName)
 		if err != nil {
-			exit.WithError("Error getting addons status", err)
+			out.WarningT("Unable to get addon status for {{.name}}: {{.error}}", out.V{"name": addonName, "error": err})
 		}
 		tData = append(tData, []string{addonName, pName, fmt.Sprintf("%s %s", stringFromStatus(addonStatus), iconFromStatus(addonStatus))})
 	}
@@ -114,12 +114,11 @@ var printAddonsList = func() {
 
 	v, _, err := config.ListProfiles()
 	if err != nil {
-		glog.Infof("error getting list of porfiles: %v", err)
+		glog.Errorf("list profiles returned error: %v", err)
 	}
 	if len(v) > 1 {
 		out.T(out.Tip, "To see addons list for other profiles use: `minikube addons -p name list`")
 	}
-
 }
 
 var printAddonsJSON = func() {
@@ -135,9 +134,10 @@ var printAddonsJSON = func() {
 	for _, addonName := range addonNames {
 		addonBundle := assets.Addons[addonName]
 
-		addonStatus, err := addonBundle.IsEnabled()
+		addonStatus, err := addonBundle.IsEnabled(pName)
 		if err != nil {
-			exit.WithError("Error getting addons status", err)
+			glog.Errorf("Unable to get addon status for %s: %v", addonName, err)
+			continue
 		}
 
 		addonsMap[addonName] = map[string]interface{}{

cmd/minikube/cmd/config/configure.go

@@ -53,6 +53,9 @@ var addonsConfigureCmd = &cobra.Command{
 			dockerUser := "changeme"
 			dockerPass := "changeme"
 			gcrURL := "https://gcr.io"
+			acrURL := "changeme"
+			acrClientID := "changeme"
+			acrPassword := "changeme"
 
 			enableAWSECR := AskForYesNoConfirmation("\nDo you want to enable AWS Elastic Container Registry?", posResponses, negResponses)
 			if enableAWSECR {
@@ -90,6 +93,13 @@ var addonsConfigureCmd = &cobra.Command{
 				dockerPass = AskForPasswordValue("-- Enter docker registry password: ")
 			}
 
+			enableACR := AskForYesNoConfirmation("\nDo you want to enable Azure Container Registry?", posResponses, negResponses)
+			if enableACR {
+				acrURL = AskForStaticValue("-- Enter Azure Container Registry (ACR) URL: ")
+				acrClientID = AskForStaticValue("-- Enter client ID (service principal ID) to access ACR: ")
+				acrPassword = AskForPasswordValue("-- Enter service principal password to access Azure Container Registry: ")
+			}
+
 			// Create ECR Secret
 			err := service.CreateSecret(
 				"kube-system",
@@ -148,6 +158,26 @@ var addonsConfigureCmd = &cobra.Command{
 			if err != nil {
 				out.WarningT("ERROR creating `registry-creds-dpr` secret")
 			}
+
+			// Create Azure Container Registry Secret
+			err = service.CreateSecret(
+				"kube-system",
+				"registry-creds-acr",
+				map[string]string{
+					"ACR_URL":       acrURL,
+					"ACR_CLIENT_ID": acrClientID,
+					"ACR_PASSWORD":  acrPassword,
+				},
+				map[string]string{
+					"app":                           "registry-creds",
+					"cloud":                         "acr",
+					"kubernetes.io/minikube-addons": "registry-creds",
+				})
+
+			if err != nil {
+				out.WarningT("ERROR creating `registry-creds-acr` secret")
+			}
+
 		default:
 			out.FailureT("{{.name}} has no available configuration options", out.V{"name": addon})
 			return

cmd/minikube/cmd/config/disable.go

@@ -39,7 +39,7 @@ var addonsDisableCmd = &cobra.Command{
 		if err != nil {
 			exit.WithError("disable failed", err)
 		}
-		out.SuccessT(`"{{.minikube_addon}}" was successfully disabled`, out.V{"minikube_addon": addon})
+		out.T(out.AddonDisable, `"The '{{.minikube_addon}}' addon is disabled`, out.V{"minikube_addon": addon})
 	},
 }
 

cmd/minikube/cmd/config/enable.go

@@ -33,13 +33,12 @@ var addonsEnableCmd = &cobra.Command{
 		if len(args) != 1 {
 			exit.UsageT("usage: minikube addons enable ADDON_NAME")
 		}
-
 		addon := args[0]
 		err := addons.Set(addon, "true", viper.GetString(config.MachineProfile))
 		if err != nil {
 			exit.WithError("enable failed", err)
 		}
-		out.SuccessT("{{.addonName}} was successfully enabled", out.V{"addonName": addon})
+		out.T(out.AddonEnable, "The '{{.addonName}}' addon is enabled", out.V{"addonName": addon})
 	},
 }
 

cmd/minikube/cmd/config/open.go

@@ -24,8 +24,10 @@ import (
 	"github.com/pkg/browser"
 
 	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
 	"k8s.io/minikube/pkg/minikube/assets"
 	"k8s.io/minikube/pkg/minikube/cluster"
+	pkg_config "k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/exit"
 	"k8s.io/minikube/pkg/minikube/machine"
 	"k8s.io/minikube/pkg/minikube/out"
@@ -66,7 +68,8 @@ var addonsOpenCmd = &cobra.Command{
 		}
 		defer api.Close()
 
-		if !cluster.IsMinikubeRunning(api) {
+		profileName := viper.GetString(pkg_config.MachineProfile)
+		if !cluster.IsHostRunning(api, profileName) {
 			os.Exit(1)
 		}
 		addon, ok := assets.Addons[addonName] // validate addon input
@@ -75,7 +78,7 @@ var addonsOpenCmd = &cobra.Command{
 To see the list of available addons run:
 minikube addons list`, out.V{"name": addonName})
 		}
-		ok, err = addon.IsEnabled()
+		ok, err = addon.IsEnabled(profileName)
 		if err != nil {
 			exit.WithError("IsEnabled failed", err)
 		}

cmd/minikube/cmd/config/profile.go

@@ -22,6 +22,7 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 	pkgConfig "k8s.io/minikube/pkg/minikube/config"
+	pkg_config "k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/constants"
 	"k8s.io/minikube/pkg/minikube/exit"
 	"k8s.io/minikube/pkg/minikube/kubeconfig"
@@ -78,7 +79,7 @@ var ProfileCmd = &cobra.Command{
 		}
 		cc, err := pkgConfig.Load(profile)
 		// might err when loading older version of cfg file that doesn't have KeepContext field
-		if err != nil && !os.IsNotExist(err) {
+		if err != nil && !pkg_config.IsNotExist(err) {
 			out.ErrT(out.Sad, `Error loading profile config: {{.error}}`, out.V{"error": err})
 		}
 		if err == nil {

cmd/minikube/cmd/config/profile_list.go

@@ -60,7 +60,7 @@ var printProfilesTable = func() {
 
 	var validData [][]string
 	table := tablewriter.NewWriter(os.Stdout)
-	table.SetHeader([]string{"Profile", "VM Driver", "NodeIP", "Node Port", "Kubernetes Version", "Status"})
+	table.SetHeader([]string{"Profile", "VM Driver", "Runtime", "IP", "Port", "Version", "Status"})
 	table.SetAutoFormatHeaders(false)
 	table.SetBorders(tablewriter.Border{Left: true, Top: true, Right: true, Bottom: true})
 	table.SetCenterSeparator("|")
@@ -71,16 +71,21 @@ var printProfilesTable = func() {
 	}
 	api, err := machine.NewAPIClient()
 	if err != nil {
-		glog.Infof("failed to get machine api client %v", err)
+		glog.Errorf("failed to get machine api client %v", err)
 	}
 	defer api.Close()
 
 	for _, p := range validProfiles {
 		p.Status, err = cluster.GetHostStatus(api, p.Name)
 		if err != nil {
-			glog.Infof("error getting host status for %v", err)
+			glog.Warningf("error getting host status for %s: %v", p.Name, err)
 		}
-		validData = append(validData, []string{p.Name, p.Config[0].VMDriver, p.Config[0].KubernetesConfig.NodeIP, strconv.Itoa(p.Config[0].KubernetesConfig.NodePort), p.Config[0].KubernetesConfig.KubernetesVersion, p.Status})
+		cp, err := config.PrimaryControlPlane(*p.Config)
+		if err != nil {
+			glog.Errorf("%q has no control plane: %v", p.Name, err)
+			// Print the data we know about anyways
+		}
+		validData = append(validData, []string{p.Name, p.Config.VMDriver, p.Config.KubernetesConfig.ContainerRuntime, cp.IP, strconv.Itoa(cp.Port), p.Config.KubernetesConfig.KubernetesVersion, p.Status})
 	}
 
 	table.AppendBulk(validData)
@@ -107,7 +112,7 @@ var printProfilesTable = func() {
 var printProfilesJSON = func() {
 	api, err := machine.NewAPIClient()
 	if err != nil {
-		glog.Infof("failed to get machine api client %v", err)
+		glog.Errorf("failed to get machine api client %v", err)
 	}
 	defer api.Close()
 
@@ -115,7 +120,7 @@ var printProfilesJSON = func() {
 	for _, v := range validProfiles {
 		status, err := cluster.GetHostStatus(api, v.Name)
 		if err != nil {
-			glog.Infof("error getting host status for  %v", err)
+			glog.Warningf("error getting host status for %s: %v", v.Name, err)
 		}
 		v.Status = status
 	}
@@ -137,7 +142,7 @@ var printProfilesJSON = func() {
 
 	var body = map[string]interface{}{}
 
-	if err == nil || os.IsNotExist(err) {
+	if err == nil || config.IsNotExist(err) {
 		body["valid"] = valid
 		body["invalid"] = invalid
 		jsonString, _ := json.Marshal(body)

cmd/minikube/cmd/config/set_test.go

@@ -16,9 +16,18 @@ limitations under the License.
 
 package config
 
-import "testing"
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"k8s.io/minikube/pkg/minikube/config"
+	"k8s.io/minikube/pkg/minikube/localpath"
+)
 
 func TestNotFound(t *testing.T) {
+	createTestProfile(t)
 	err := Set("nonexistent", "10")
 	if err == nil {
 		t.Fatalf("Set did not return error for unknown property")
@@ -26,6 +35,7 @@ func TestNotFound(t *testing.T) {
 }
 
 func TestSetNotAllowed(t *testing.T) {
+	createTestProfile(t)
 	err := Set("vm-driver", "123456")
 	if err == nil || err.Error() != "[driver \"123456\" is not supported]" {
 		t.Fatalf("Set did not return error for unallowed value")
@@ -33,7 +43,14 @@ func TestSetNotAllowed(t *testing.T) {
 }
 
 func TestSetOK(t *testing.T) {
+	createTestProfile(t)
 	err := Set("vm-driver", "virtualbox")
+	defer func() {
+		err = Unset("vm-driver")
+		if err != nil {
+			t.Errorf("failed to unset vm-driver")
+		}
+	}()
 	if err != nil {
 		t.Fatalf("Set returned error for valid property value")
 	}
@@ -45,3 +62,25 @@ func TestSetOK(t *testing.T) {
 		t.Fatalf("Get returned %s, expected \"virtualbox\"", val)
 	}
 }
+
+func createTestProfile(t *testing.T) {
+	t.Helper()
+	td, err := ioutil.TempDir("", "profile")
+	if err != nil {
+		t.Fatalf("tempdir: %v", err)
+	}
+
+	err = os.Setenv(localpath.MinikubeHome, td)
+	if err != nil {
+		t.Errorf("error setting up test environment. could not set %s", localpath.MinikubeHome)
+	}
+
+	// Not necessary, but it is a handy random alphanumeric
+	name := filepath.Base(td)
+	if err := os.MkdirAll(config.ProfileFolderPath(name), 0777); err != nil {
+		t.Fatalf("error creating temporary directory")
+	}
+	if err := config.DefaultLoader.WriteConfigToFile(name, &config.MachineConfig{}); err != nil {
+		t.Fatalf("error creating temporary profile config: %v", err)
+	}
+}

cmd/minikube/cmd/dashboard.go

@@ -36,7 +36,6 @@ import (
 	pkgaddons "k8s.io/minikube/pkg/addons"
 	"k8s.io/minikube/pkg/minikube/assets"
 	"k8s.io/minikube/pkg/minikube/cluster"
-	"k8s.io/minikube/pkg/minikube/config"
 	pkg_config "k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/exit"
 	"k8s.io/minikube/pkg/minikube/machine"
@@ -59,11 +58,17 @@ var dashboardCmd = &cobra.Command{
 	Short: "Access the kubernetes dashboard running within the minikube cluster",
 	Long:  `Access the kubernetes dashboard running within the minikube cluster`,
 	Run: func(cmd *cobra.Command, args []string) {
-		cc, err := pkg_config.Load(viper.GetString(config.MachineProfile))
-		if err != nil && !os.IsNotExist(err) {
+		profileName := viper.GetString(pkg_config.MachineProfile)
+		cc, err := pkg_config.Load(profileName)
+		if err != nil && !pkg_config.IsNotExist(err) {
 			exit.WithError("Error loading profile config", err)
 		}
 
+		if err != nil {
+			out.ErrT(out.Meh, `"{{.name}}" profile does not exist`, out.V{"name": profileName})
+			os.Exit(1)
+		}
+
 		api, err := machine.NewAPIClient()
 		defer func() {
 			err := api.Close()
@@ -85,9 +90,11 @@ var dashboardCmd = &cobra.Command{
 			}
 		}
 
-		err = proxy.ExcludeIP(cc.KubernetesConfig.NodeIP) // to be used for http get calls
-		if err != nil {
-			glog.Errorf("Error excluding IP from proxy: %s", err)
+		for _, n := range cc.Nodes {
+			err = proxy.ExcludeIP(n.IP) // to be used for http get calls
+			if err != nil {
+				glog.Errorf("Error excluding IP from proxy: %s", err)
+			}
 		}
 
 		kubectl, err := exec.LookPath("kubectl")
@@ -95,18 +102,18 @@ var dashboardCmd = &cobra.Command{
 			exit.WithCodeT(exit.NoInput, "kubectl not found in PATH, but is required for the dashboard. Installation guide: https://kubernetes.io/docs/tasks/tools/install-kubectl/")
 		}
 
-		if !cluster.IsMinikubeRunning(api) {
+		if !cluster.IsHostRunning(api, profileName) {
 			os.Exit(1)
 		}
 
 		// Check dashboard status before enabling it
 		dashboardAddon := assets.Addons["dashboard"]
-		dashboardStatus, _ := dashboardAddon.IsEnabled()
+		dashboardStatus, _ := dashboardAddon.IsEnabled(profileName)
 		if !dashboardStatus {
 			// Send status messages to stderr for folks re-using this output.
 			out.ErrT(out.Enabling, "Enabling dashboard ...")
 			// Enable the dashboard add-on
-			err = pkgaddons.Set("dashboard", "true", viper.GetString(config.MachineProfile))
+			err = pkgaddons.Set("dashboard", "true", profileName)
 			if err != nil {
 				exit.WithError("Unable to enable dashboard", err)
 			}

cmd/minikube/cmd/delete.go

@@ -35,6 +35,7 @@ import (
 	"k8s.io/minikube/pkg/minikube/cluster"
 	pkg_config "k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/constants"
+	"k8s.io/minikube/pkg/minikube/cruntime"
 	"k8s.io/minikube/pkg/minikube/driver"
 	"k8s.io/minikube/pkg/minikube/exit"
 	"k8s.io/minikube/pkg/minikube/kubeconfig"
@@ -131,14 +132,12 @@ func runDelete(cmd *cobra.Command, args []string) {
 		profileName := viper.GetString(pkg_config.MachineProfile)
 		profile, err := pkg_config.LoadProfile(profileName)
 		if err != nil {
-			out.ErrT(out.Meh, `"{{.name}}" profile does not exist`, out.V{"name": profileName})
+			out.ErrT(out.Meh, `"{{.name}}" profile does not exist, trying anyways.`, out.V{"name": profileName})
 		}
 
 		errs := DeleteProfiles([]*pkg_config.Profile{profile})
 		if len(errs) > 0 {
 			HandleDeletionErrors(errs)
-		} else {
-			out.T(out.DeletingHost, "Successfully deleted profile \"{{.name}}\"", out.V{"name": profileName})
 		}
 	}
 
@@ -153,7 +152,7 @@ func purgeMinikubeDirectory() {
 	if err := os.RemoveAll(localpath.MiniPath()); err != nil {
 		exit.WithError("unable to delete minikube config folder", err)
 	}
-	out.T(out.Crushed, "Successfully purged minikube directory located at - [{{.minikubeDirectory}}]", out.V{"minikubeDirectory": localpath.MiniPath()})
+	out.T(out.Deleted, "Successfully purged minikube directory located at - [{{.minikubeDirectory}}]", out.V{"minikubeDirectory": localpath.MiniPath()})
 }
 
 // DeleteProfiles deletes one or more profiles
@@ -188,13 +187,13 @@ func deleteProfile(profile *pkg_config.Profile) error {
 	}
 	defer api.Close()
 	cc, err := pkg_config.Load(profile.Name)
-	if err != nil && !os.IsNotExist(err) {
+	if err != nil && !pkg_config.IsNotExist(err) {
 		delErr := profileDeletionErr(profile.Name, fmt.Sprintf("error loading profile config: %v", err))
 		return DeletionError{Err: delErr, Errtype: MissingProfile}
 	}
 
 	if err == nil && driver.BareMetal(cc.VMDriver) {
-		if err := uninstallKubernetes(api, cc.KubernetesConfig, viper.GetString(cmdcfg.Bootstrapper)); err != nil {
+		if err := uninstallKubernetes(api, profile.Name, cc.KubernetesConfig, viper.GetString(cmdcfg.Bootstrapper)); err != nil {
 			deletionError, ok := err.(DeletionError)
 			if ok {
 				delErr := profileDeletionErr(profile.Name, fmt.Sprintf("%v", err))
@@ -212,7 +211,7 @@ func deleteProfile(profile *pkg_config.Profile) error {
 	if err = cluster.DeleteHost(api, profile.Name); err != nil {
 		switch errors.Cause(err).(type) {
 		case mcnerror.ErrHostDoesNotExist:
-			out.T(out.Meh, `"{{.name}}" cluster does not exist. Proceeding ahead with cleanup.`, out.V{"name": profile.Name})
+			glog.Infof("%s cluster does not exist. Proceeding ahead with cleanup.", profile.Name)
 		default:
 			out.T(out.FailureType, "Failed to delete cluster: {{.error}}", out.V{"error": err})
 			out.T(out.Notice, `You may need to manually remove the "{{.name}}" VM from your hypervisor`, out.V{"name": profile.Name})
@@ -223,7 +222,7 @@ func deleteProfile(profile *pkg_config.Profile) error {
 	deleteProfileDirectory(profile.Name)
 
 	if err := pkg_config.DeleteProfile(profile.Name); err != nil {
-		if os.IsNotExist(err) {
+		if pkg_config.IsNotExist(err) {
 			delErr := profileDeletionErr(profile.Name, fmt.Sprintf("\"%s\" profile does not exist", profile.Name))
 			return DeletionError{Err: delErr, Errtype: MissingProfile}
 		}
@@ -231,11 +230,10 @@ func deleteProfile(profile *pkg_config.Profile) error {
 		return DeletionError{Err: delErr, Errtype: Fatal}
 	}
 
-	out.T(out.Crushed, `The "{{.name}}" cluster has been deleted.`, out.V{"name": profile.Name})
-
 	if err := deleteContext(profile.Name); err != nil {
 		return err
 	}
+	out.T(out.Deleted, `Removed all traces of the "{{.name}}" cluster.`, out.V{"name": profile.Name})
 	return nil
 }
 
@@ -276,12 +274,34 @@ func profileDeletionErr(profileName string, additionalInfo string) error {
 	return fmt.Errorf("error deleting profile \"%s\": %s", profileName, additionalInfo)
 }
 
-func uninstallKubernetes(api libmachine.API, kc pkg_config.KubernetesConfig, bsName string) error {
+func uninstallKubernetes(api libmachine.API, profile string, kc pkg_config.KubernetesConfig, bsName string) error {
 	out.T(out.Resetting, "Uninstalling Kubernetes {{.kubernetes_version}} using {{.bootstrapper_name}} ...", out.V{"kubernetes_version": kc.KubernetesVersion, "bootstrapper_name": bsName})
 	clusterBootstrapper, err := getClusterBootstrapper(api, bsName)
 	if err != nil {
 		return DeletionError{Err: fmt.Errorf("unable to get bootstrapper: %v", err), Errtype: Fatal}
-	} else if err = clusterBootstrapper.DeleteCluster(kc); err != nil {
+	}
+
+	host, err := cluster.CheckIfHostExistsAndLoad(api, profile)
+	if err != nil {
+		exit.WithError("Error getting host", err)
+	}
+	r, err := machine.CommandRunner(host)
+	if err != nil {
+		exit.WithError("Failed to get command runner", err)
+	}
+
+	cr, err := cruntime.New(cruntime.Config{Type: kc.ContainerRuntime, Runner: r})
+	if err != nil {
+		exit.WithError("Failed runtime", err)
+	}
+
+	// Unpause the cluster if necessary to avoid hung kubeadm
+	_, err = cluster.Unpause(cr, r, nil)
+	if err != nil {
+		glog.Errorf("unpause failed: %v", err)
+	}
+
+	if err = clusterBootstrapper.DeleteCluster(kc); err != nil {
 		return DeletionError{Err: fmt.Errorf("failed to delete cluster: %v", err), Errtype: Fatal}
 	}
 	return nil

cmd/minikube/cmd/env.go

@@ -36,14 +36,15 @@ import (
 	"github.com/spf13/viper"
 	"k8s.io/minikube/pkg/minikube/cluster"
 	"k8s.io/minikube/pkg/minikube/config"
+	"k8s.io/minikube/pkg/minikube/constants"
 	"k8s.io/minikube/pkg/minikube/driver"
 	"k8s.io/minikube/pkg/minikube/exit"
 	"k8s.io/minikube/pkg/minikube/machine"
 )
 
-const (
-	envTmpl = `{{ .Prefix }}DOCKER_TLS_VERIFY{{ .Delimiter }}{{ .DockerTLSVerify }}{{ .Suffix }}{{ .Prefix }}DOCKER_HOST{{ .Delimiter }}{{ .DockerHost }}{{ .Suffix }}{{ .Prefix }}DOCKER_CERT_PATH{{ .Delimiter }}{{ .DockerCertPath }}{{ .Suffix }}{{ if .NoProxyVar }}{{ .Prefix }}{{ .NoProxyVar }}{{ .Delimiter }}{{ .NoProxyValue }}{{ .Suffix }}{{end}}{{ .UsageHint }}`
+var envTmpl = fmt.Sprintf("{{ .Prefix }}%s{{ .Delimiter }}{{ .DockerTLSVerify }}{{ .Suffix }}{{ .Prefix }}%s{{ .Delimiter }}{{ .DockerHost }}{{ .Suffix }}{{ .Prefix }}%s{{ .Delimiter }}{{ .DockerCertPath }}{{ .Suffix }}{{ .Prefix }}%s{{ .Delimiter }}{{ .MinikubeDockerdProfile }}{{ .Suffix }}{{ if .NoProxyVar }}{{ .Prefix }}{{ .NoProxyVar }}{{ .Delimiter }}{{ .NoProxyValue }}{{ .Suffix }}{{end}}{{ .UsageHint }}", constants.DockerTLSVerifyEnv, constants.DockerHostEnv, constants.DockerCertPathEnv, constants.MinikubeActiveDockerdEnv)
 
+const (
 	fishSetPfx   = "set -gx "
 	fishSetSfx   = "\";\n"
 	fishSetDelim = " \""
@@ -89,35 +90,18 @@ const (
 	noneDelim = "="
 )
 
-var usageHintMap = map[string]string{
-	"bash": `# Run this command to configure your shell:
-# eval $(minikube docker-env)
-`,
-	"fish": `# Run this command to configure your shell:
-# eval (minikube docker-env)
-`,
-	"powershell": `# Run this command to configure your shell:
-# & minikube docker-env | Invoke-Expression
-`,
-	"cmd": `REM Run this command to configure your shell:
-REM @FOR /f "tokens=*" %i IN ('minikube docker-env') DO @%i
-`,
-	"emacs": `;; Run this command to configure your shell:
-;; (with-temp-buffer (shell-command "minikube docker-env" (current-buffer)) (eval-buffer))
-`,
-}
-
 // ShellConfig represents the shell config
 type ShellConfig struct {
-	Prefix          string
-	Delimiter       string
-	Suffix          string
-	DockerCertPath  string
-	DockerHost      string
-	DockerTLSVerify string
-	UsageHint       string
-	NoProxyVar      string
-	NoProxyValue    string
+	Prefix                 string
+	Delimiter              string
+	Suffix                 string
+	DockerCertPath         string
+	DockerHost             string
+	DockerTLSVerify        string
+	MinikubeDockerdProfile string
+	UsageHint              string
+	NoProxyVar             string
+	NoProxyValue           string
 }
 
 var (
@@ -144,7 +128,32 @@ type NoProxyGetter interface {
 // EnvNoProxyGetter gets the no_proxy variable, using environment
 type EnvNoProxyGetter struct{}
 
-func generateUsageHint(userShell string) string {
+func generateUsageHint(profile string, userShell string) string {
+	const usgPlz = "Please run command bellow to point your shell to minikube's docker-daemon :"
+	var usgCmd = fmt.Sprintf("minikube -p %s docker-env", profile)
+	var usageHintMap = map[string]string{
+		"bash": fmt.Sprintf(`
+# %s
+# eval $(%s)
+`, usgPlz, usgCmd),
+		"fish": fmt.Sprintf(`
+# %s
+# eval (%s)
+`, usgPlz, usgCmd),
+		"powershell": fmt.Sprintf(`
+# %s
+# & %s | Invoke-Expression
+	`, usgPlz, usgCmd),
+		"cmd": fmt.Sprintf(`
+REM %s
+REM @FOR /f "tokens=*" %%i IN ('%s') DO @%%i
+	`, usgPlz, usgCmd),
+		"emacs": fmt.Sprintf(`
+;; %s
+;; (with-temp-buffer (shell-command "%s" (current-buffer)) (eval-buffer))
+	`, usgPlz, usgCmd),
+	}
+
 	hint, ok := usageHintMap[userShell]
 	if !ok {
 		return usageHintMap["bash"]
@@ -154,7 +163,7 @@ func generateUsageHint(userShell string) string {
 
 func shellCfgSet(api libmachine.API) (*ShellConfig, error) {
 
-	envMap, err := cluster.GetHostDockerEnv(api)
+	envMap, err := cluster.GetNodeDockerEnv(api)
 	if err != nil {
 		return nil, err
 	}
@@ -165,10 +174,11 @@ func shellCfgSet(api libmachine.API) (*ShellConfig, error) {
 	}
 
 	shellCfg := &ShellConfig{
-		DockerCertPath:  envMap["DOCKER_CERT_PATH"],
-		DockerHost:      envMap["DOCKER_HOST"],
-		DockerTLSVerify: envMap["DOCKER_TLS_VERIFY"],
-		UsageHint:       generateUsageHint(userShell),
+		DockerCertPath:         envMap[constants.DockerCertPathEnv],
+		DockerHost:             envMap[constants.DockerHostEnv],
+		DockerTLSVerify:        envMap[constants.DockerTLSVerifyEnv],
+		MinikubeDockerdProfile: envMap[constants.MinikubeActiveDockerdEnv],
+		UsageHint:              generateUsageHint(viper.GetString(config.MachineProfile), userShell),
 	}
 
 	if noProxy {
@@ -237,7 +247,7 @@ func shellCfgUnset() (*ShellConfig, error) {
 	}
 
 	shellCfg := &ShellConfig{
-		UsageHint: generateUsageHint(userShell),
+		UsageHint: generateUsageHint(viper.GetString(config.MachineProfile), userShell),
 	}
 
 	if noProxy {

cmd/minikube/cmd/env_test.go

@@ -59,13 +59,14 @@ var defaultAPI = &tests.MockAPI{
 // Most of the shell cfg isn't configurable
 func newShellCfg(shell, prefix, suffix, delim string) *ShellConfig {
 	return &ShellConfig{
-		DockerCertPath:  localpath.MakeMiniPath("certs"),
-		DockerTLSVerify: "1",
-		DockerHost:      "tcp://127.0.0.1:2376",
-		UsageHint:       generateUsageHint(shell),
-		Prefix:          prefix,
-		Suffix:          suffix,
-		Delimiter:       delim,
+		DockerCertPath:         localpath.MakeMiniPath("certs"),
+		DockerTLSVerify:        "1",
+		DockerHost:             "tcp://127.0.0.1:2376",
+		UsageHint:              generateUsageHint("minikube", shell),
+		Prefix:                 prefix,
+		Suffix:                 suffix,
+		Delimiter:              delim,
+		MinikubeDockerdProfile: "minikube",
 	}
 }
 
@@ -141,15 +142,16 @@ func TestShellCfgSet(t *testing.T) {
 			noProxyValue: "",
 			noProxyFlag:  true,
 			expectedShellCfg: &ShellConfig{
-				DockerCertPath:  localpath.MakeMiniPath("certs"),
-				DockerTLSVerify: "1",
-				DockerHost:      "tcp://127.0.0.1:2376",
-				UsageHint:       usageHintMap["bash"],
-				Prefix:          bashSetPfx,
-				Suffix:          bashSetSfx,
-				Delimiter:       bashSetDelim,
-				NoProxyVar:      "NO_PROXY",
-				NoProxyValue:    "127.0.0.1",
+				DockerCertPath:         localpath.MakeMiniPath("certs"),
+				DockerTLSVerify:        "1",
+				DockerHost:             "tcp://127.0.0.1:2376",
+				UsageHint:              generateUsageHint("minikube", "bash"),
+				Prefix:                 bashSetPfx,
+				Suffix:                 bashSetSfx,
+				Delimiter:              bashSetDelim,
+				NoProxyVar:             "NO_PROXY",
+				NoProxyValue:           "127.0.0.1",
+				MinikubeDockerdProfile: "minikube",
 			},
 		},
 		{
@@ -160,15 +162,16 @@ func TestShellCfgSet(t *testing.T) {
 			noProxyValue: "",
 			noProxyFlag:  true,
 			expectedShellCfg: &ShellConfig{
-				DockerCertPath:  localpath.MakeMiniPath("certs"),
-				DockerTLSVerify: "1",
-				DockerHost:      "tcp://127.0.0.1:2376",
-				UsageHint:       usageHintMap["bash"],
-				Prefix:          bashSetPfx,
-				Suffix:          bashSetSfx,
-				Delimiter:       bashSetDelim,
-				NoProxyVar:      "no_proxy",
-				NoProxyValue:    "127.0.0.1",
+				DockerCertPath:         localpath.MakeMiniPath("certs"),
+				DockerTLSVerify:        "1",
+				DockerHost:             "tcp://127.0.0.1:2376",
+				UsageHint:              generateUsageHint("minikube", "bash"),
+				Prefix:                 bashSetPfx,
+				Suffix:                 bashSetSfx,
+				Delimiter:              bashSetDelim,
+				NoProxyVar:             "no_proxy",
+				NoProxyValue:           "127.0.0.1",
+				MinikubeDockerdProfile: "minikube",
 			},
 		},
 		{
@@ -179,15 +182,16 @@ func TestShellCfgSet(t *testing.T) {
 			noProxyValue: "127.0.0.1",
 			noProxyFlag:  true,
 			expectedShellCfg: &ShellConfig{
-				DockerCertPath:  localpath.MakeMiniPath("certs"),
-				DockerTLSVerify: "1",
-				DockerHost:      "tcp://127.0.0.1:2376",
-				UsageHint:       usageHintMap["bash"],
-				Prefix:          bashSetPfx,
-				Suffix:          bashSetSfx,
-				Delimiter:       bashSetDelim,
-				NoProxyVar:      "no_proxy",
-				NoProxyValue:    "127.0.0.1",
+				DockerCertPath:         localpath.MakeMiniPath("certs"),
+				DockerTLSVerify:        "1",
+				DockerHost:             "tcp://127.0.0.1:2376",
+				UsageHint:              generateUsageHint("minikube", "bash"),
+				Prefix:                 bashSetPfx,
+				Suffix:                 bashSetSfx,
+				Delimiter:              bashSetDelim,
+				NoProxyVar:             "no_proxy",
+				NoProxyValue:           "127.0.0.1",
+				MinikubeDockerdProfile: "minikube",
 			},
 		},
 		{
@@ -198,15 +202,16 @@ func TestShellCfgSet(t *testing.T) {
 			noProxyValue: "0.0.0.0",
 			noProxyFlag:  true,
 			expectedShellCfg: &ShellConfig{
-				DockerCertPath:  localpath.MakeMiniPath("certs"),
-				DockerTLSVerify: "1",
-				DockerHost:      "tcp://127.0.0.1:2376",
-				UsageHint:       usageHintMap["bash"],
-				Prefix:          bashSetPfx,
-				Suffix:          bashSetSfx,
-				Delimiter:       bashSetDelim,
-				NoProxyVar:      "no_proxy",
-				NoProxyValue:    "0.0.0.0,127.0.0.1",
+				DockerCertPath:         localpath.MakeMiniPath("certs"),
+				DockerTLSVerify:        "1",
+				DockerHost:             "tcp://127.0.0.1:2376",
+				UsageHint:              generateUsageHint("minikube", "bash"),
+				Prefix:                 bashSetPfx,
+				Suffix:                 bashSetSfx,
+				Delimiter:              bashSetDelim,
+				NoProxyVar:             "no_proxy",
+				NoProxyValue:           "0.0.0.0,127.0.0.1",
+				MinikubeDockerdProfile: "minikube",
 			},
 		},
 		{
@@ -217,15 +222,16 @@ func TestShellCfgSet(t *testing.T) {
 			noProxyValue: "0.0.0.0,127.0.0.1",
 			noProxyFlag:  true,
 			expectedShellCfg: &ShellConfig{
-				DockerCertPath:  localpath.MakeMiniPath("certs"),
-				DockerTLSVerify: "1",
-				DockerHost:      "tcp://127.0.0.1:2376",
-				UsageHint:       usageHintMap["bash"],
-				Prefix:          bashSetPfx,
-				Suffix:          bashSetSfx,
-				Delimiter:       bashSetDelim,
-				NoProxyVar:      "no_proxy",
-				NoProxyValue:    "0.0.0.0,127.0.0.1",
+				DockerCertPath:         localpath.MakeMiniPath("certs"),
+				DockerTLSVerify:        "1",
+				DockerHost:             "tcp://127.0.0.1:2376",
+				UsageHint:              generateUsageHint("minikube", "bash"),
+				Prefix:                 bashSetPfx,
+				Suffix:                 bashSetSfx,
+				Delimiter:              bashSetDelim,
+				NoProxyVar:             "no_proxy",
+				NoProxyValue:           "0.0.0.0,127.0.0.1",
+				MinikubeDockerdProfile: "minikube",
 			},
 		},
 	}
@@ -254,6 +260,7 @@ func TestShellCfgSet(t *testing.T) {
 }
 
 func TestShellCfgUnset(t *testing.T) {
+
 	var tests = []struct {
 		description      string
 		shell            string
@@ -266,7 +273,7 @@ func TestShellCfgUnset(t *testing.T) {
 				Prefix:    bashUnsetPfx,
 				Suffix:    bashUnsetSfx,
 				Delimiter: bashUnsetDelim,
-				UsageHint: usageHintMap["bash"],
+				UsageHint: generateUsageHint("minikube", "bash"),
 			},
 		},
 		{
@@ -276,7 +283,7 @@ func TestShellCfgUnset(t *testing.T) {
 				Prefix:    bashUnsetPfx,
 				Suffix:    bashUnsetSfx,
 				Delimiter: bashUnsetDelim,
-				UsageHint: usageHintMap["bash"],
+				UsageHint: generateUsageHint("minikube", "bash"),
 			},
 		},
 		{
@@ -286,7 +293,7 @@ func TestShellCfgUnset(t *testing.T) {
 				Prefix:    fishUnsetPfx,
 				Suffix:    fishUnsetSfx,
 				Delimiter: fishUnsetDelim,
-				UsageHint: usageHintMap["fish"],
+				UsageHint: generateUsageHint("minikube", "fish"),
 			},
 		},
 		{
@@ -296,7 +303,7 @@ func TestShellCfgUnset(t *testing.T) {
 				Prefix:    psUnsetPfx,
 				Suffix:    psUnsetSfx,
 				Delimiter: psUnsetDelim,
-				UsageHint: usageHintMap["powershell"],
+				UsageHint: generateUsageHint("minikube", "powershell"),
 			},
 		},
 		{
@@ -306,7 +313,7 @@ func TestShellCfgUnset(t *testing.T) {
 				Prefix:    cmdUnsetPfx,
 				Suffix:    cmdUnsetSfx,
 				Delimiter: cmdUnsetDelim,
-				UsageHint: usageHintMap["cmd"],
+				UsageHint: generateUsageHint("minikube", "cmd"),
 			},
 		},
 		{
@@ -316,7 +323,7 @@ func TestShellCfgUnset(t *testing.T) {
 				Prefix:    emacsUnsetPfx,
 				Suffix:    emacsUnsetSfx,
 				Delimiter: emacsUnsetDelim,
-				UsageHint: usageHintMap["emacs"],
+				UsageHint: generateUsageHint("minikube", "emacs"),
 			},
 		},
 	}

cmd/minikube/cmd/kubectl.go

@@ -27,7 +27,6 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 	"k8s.io/minikube/pkg/minikube/config"
-	pkg_config "k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/constants"
 	"k8s.io/minikube/pkg/minikube/exit"
 	"k8s.io/minikube/pkg/minikube/machine"
@@ -38,10 +37,11 @@ import (
 var kubectlCmd = &cobra.Command{
 	Use:   "kubectl",
 	Short: "Run kubectl",
-	Long: `Run the kubernetes client, download it if necessary.
+	Long: `Run the kubernetes client, download it if necessary. Remember -- after kubectl!
+
 Examples:
 minikube kubectl -- --help
-kubectl get pods --namespace kube-system`,
+minikube kubectl -- get pods --namespace kube-system`,
 	Run: func(cmd *cobra.Command, args []string) {
 		api, err := machine.NewAPIClient()
 		if err != nil {
@@ -50,8 +50,8 @@ kubectl get pods --namespace kube-system`,
 		}
 		defer api.Close()
 
-		cc, err := pkg_config.Load(viper.GetString(config.MachineProfile))
-		if err != nil && !os.IsNotExist(err) {
+		cc, err := config.Load(viper.GetString(config.MachineProfile))
+		if err != nil && !config.IsNotExist(err) {
 			out.ErrLn("Error loading profile config: %v", err)
 		}
 

cmd/minikube/cmd/mount.go

@@ -108,7 +108,6 @@ var mountCmd = &cobra.Command{
 			exit.WithError("Error getting config", err)
 		}
 		host, err := api.Load(cc.Name)
-
 		if err != nil {
 			exit.WithError("Error loading api", err)
 		}

cmd/minikube/cmd/pause.go

+/*
+Copyright 2020 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cmd
+
+import (
+	"os"
+	"strings"
+
+	"github.com/golang/glog"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+
+	"k8s.io/minikube/pkg/minikube/cluster"
+	"k8s.io/minikube/pkg/minikube/config"
+	"k8s.io/minikube/pkg/minikube/cruntime"
+	"k8s.io/minikube/pkg/minikube/exit"
+	"k8s.io/minikube/pkg/minikube/machine"
+	"k8s.io/minikube/pkg/minikube/out"
+)
+
+var (
+	namespaces    []string
+	allNamespaces bool
+)
+
+// pauseCmd represents the docker-pause command
+var pauseCmd = &cobra.Command{
+	Use:   "pause",
+	Short: "pause containers",
+	Run:   runPause,
+}
+
+func runPause(cmd *cobra.Command, args []string) {
+	cname := viper.GetString(config.MachineProfile)
+	api, err := machine.NewAPIClient()
+	if err != nil {
+		exit.WithError("Error getting client", err)
+	}
+	defer api.Close()
+	cc, err := config.Load(cname)
+
+	if err != nil && !config.IsNotExist(err) {
+		exit.WithError("Error loading profile config", err)
+	}
+
+	if err != nil {
+		out.ErrT(out.Meh, `"{{.name}}" profile does not exist`, out.V{"name": cname})
+		os.Exit(1)
+	}
+
+	glog.Infof("config: %+v", cc)
+	host, err := cluster.CheckIfHostExistsAndLoad(api, cname)
+	if err != nil {
+		exit.WithError("Error getting host", err)
+	}
+
+	r, err := machine.CommandRunner(host)
+	if err != nil {
+		exit.WithError("Failed to get command runner", err)
+	}
+
+	cr, err := cruntime.New(cruntime.Config{Type: cc.KubernetesConfig.ContainerRuntime, Runner: r})
+	if err != nil {
+		exit.WithError("Failed runtime", err)
+	}
+
+	glog.Infof("namespaces: %v keys: %v", namespaces, viper.AllSettings())
+	if allNamespaces {
+		namespaces = nil //all
+	} else if len(namespaces) == 0 {
+		exit.WithCodeT(exit.BadUsage, "Use -A to specify all namespaces")
+	}
+
+	ids, err := cluster.Pause(cr, r, namespaces)
+	if err != nil {
+		exit.WithError("Pause", err)
+	}
+
+	if namespaces == nil {
+		out.T(out.Unpause, "Paused kubelet and {{.count}} containers", out.V{"count": len(ids)})
+	} else {
+		out.T(out.Unpause, "Paused kubelet and {{.count}} containers in: {{.namespaces}}", out.V{"count": len(ids), "namespaces": strings.Join(namespaces, ", ")})
+	}
+}
+
+func init() {
+	pauseCmd.Flags().StringSliceVarP(&namespaces, "--namespaces", "n", cluster.DefaultNamespaces, "namespaces to pause")
+	pauseCmd.Flags().BoolVarP(&allNamespaces, "all-namespaces", "A", false, "If set, pause all namespaces")
+}

cmd/minikube/cmd/root.go

@@ -32,7 +32,6 @@ import (
 	"k8s.io/kubectl/pkg/util/templates"
 	configCmd "k8s.io/minikube/cmd/minikube/cmd/config"
 	"k8s.io/minikube/pkg/minikube/bootstrapper"
-	"k8s.io/minikube/pkg/minikube/bootstrapper/kicbs"
 	"k8s.io/minikube/pkg/minikube/bootstrapper/kubeadm"
 	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/constants"
@@ -173,6 +172,8 @@ func init() {
 				stopCmd,
 				deleteCmd,
 				dashboardCmd,
+				pauseCmd,
+				unpauseCmd,
 			},
 		},
 		{
@@ -274,16 +275,9 @@ func getClusterBootstrapper(api libmachine.API, bootstrapperName string) (bootst
 		if err != nil {
 			return nil, errors.Wrap(err, "getting a new kubeadm bootstrapper")
 		}
-	case bootstrapper.KIC:
-		b, err = kicbs.NewBootstrapper(api)
-		if err != nil {
-			return nil, errors.Wrap(err, "getting a new kic bootstrapper")
-		}
-
 	default:
 		return nil, fmt.Errorf("unknown bootstrapper: %s", bootstrapperName)
 	}
-
 	return b, nil
 }
 

cmd/minikube/cmd/service.go

@@ -25,8 +25,10 @@ import (
 	"github.com/golang/glog"
 	"github.com/pkg/browser"
 	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
 
 	"k8s.io/minikube/pkg/minikube/cluster"
+	pkg_config "k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/exit"
 	"k8s.io/minikube/pkg/minikube/machine"
 	"k8s.io/minikube/pkg/minikube/out"
@@ -71,7 +73,8 @@ var serviceCmd = &cobra.Command{
 		}
 		defer api.Close()
 
-		if !cluster.IsMinikubeRunning(api) {
+		profileName := viper.GetString(pkg_config.MachineProfile)
+		if !cluster.IsHostRunning(api, profileName) {
 			os.Exit(1)
 		}
 

cmd/minikube/cmd/start_test.go

@@ -121,7 +121,7 @@ func TestGenerateCfgFromFlagsHTTPProxyHandling(t *testing.T) {
 			if err := os.Setenv("HTTP_PROXY", test.proxy); err != nil {
 				t.Fatalf("Unexpected error setting HTTP_PROXY: %v", err)
 			}
-			config, err := generateCfgFromFlags(cmd, k8sVersion, "none")
+			config, _, err := generateCfgFromFlags(cmd, k8sVersion, "none")
 			if err != nil {
 				t.Fatalf("Got unexpected error %v during config generation", err)
 			}

cmd/minikube/cmd/status.go

@@ -19,15 +19,18 @@ package cmd
 import (
 	"encoding/json"
 	"fmt"
+	"io"
 	"os"
 	"strings"
 	"text/template"
 
+	"github.com/docker/machine/libmachine"
 	"github.com/docker/machine/libmachine/state"
 	"github.com/golang/glog"
+	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
-	cmdcfg "k8s.io/minikube/cmd/minikube/cmd/config"
+	"k8s.io/minikube/pkg/minikube/bootstrapper/bsutil/kverify"
 	"k8s.io/minikube/pkg/minikube/cluster"
 	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/constants"
@@ -40,16 +43,15 @@ import (
 var statusFormat string
 var output string
 
-// KubeconfigStatus represents the kubeconfig status
-var KubeconfigStatus = struct {
-	Configured    string
-	Misconfigured string
-}{
-	Configured:    `Configured`,
-	Misconfigured: `Misconfigured`,
-}
+const (
+	// Additional states used by kubeconfig
+	Configured    = "Configured"    // ~state.Saved
+	Misconfigured = "Misconfigured" // ~state.Error
+	// Additional states used for clarity
+	Nonexistent = "Nonexistent" // ~state.None
+)
 
-// Status represents the status
+// Status holds string representations of component states
 type Status struct {
 	Host       string
 	Kubelet    string
@@ -81,7 +83,6 @@ var statusCmd = &cobra.Command{
 			exit.UsageT("Cannot use both --output and --format options")
 		}
 
-		var returnCode = 0
 		api, err := machine.NewAPIClient()
 		if err != nil {
 			exit.WithCodeT(exit.Unavailable, "Error getting client: {{.error}}", out.V{"error": err})
@@ -89,81 +90,128 @@ var statusCmd = &cobra.Command{
 		defer api.Close()
 
 		machineName := viper.GetString(config.MachineProfile)
-
-		hostSt, err := cluster.GetHostStatus(api, machineName)
+		st, err := status(api, machineName)
 		if err != nil {
-			exit.WithError("Error getting host status", err)
-		}
-
-		kubeletSt := state.None.String()
-		kubeconfigSt := state.None.String()
-		apiserverSt := state.None.String()
-
-		if hostSt == state.Running.String() {
-			clusterBootstrapper, err := getClusterBootstrapper(api, viper.GetString(cmdcfg.Bootstrapper))
-			if err != nil {
-				exit.WithError("Error getting bootstrapper", err)
-			}
-			kubeletSt, err = clusterBootstrapper.GetKubeletStatus()
-			if err != nil {
-				glog.Warningf("kubelet err: %v", err)
-				returnCode |= clusterNotRunningStatusFlag
-			} else if kubeletSt != state.Running.String() {
-				returnCode |= clusterNotRunningStatusFlag
-			}
-
-			ip, err := cluster.GetHostDriverIP(api, machineName)
-			if err != nil {
-				glog.Errorln("Error host driver ip status:", err)
-			}
-
-			apiserverPort, err := kubeconfig.Port(machineName)
-			if err != nil {
-				// Fallback to presuming default apiserver port
-				apiserverPort = constants.APIServerPort
-			}
-
-			apiserverSt, err = clusterBootstrapper.GetAPIServerStatus(ip, apiserverPort)
-			if err != nil {
-				glog.Errorln("Error apiserver status:", err)
-			} else if apiserverSt != state.Running.String() {
-				returnCode |= clusterNotRunningStatusFlag
-			}
-
-			ks, err := kubeconfig.IsClusterInConfig(ip, machineName)
-			if err != nil {
-				glog.Errorln("Error kubeconfig status:", err)
-			}
-			if ks {
-				kubeconfigSt = KubeconfigStatus.Configured
-			} else {
-				kubeconfigSt = KubeconfigStatus.Misconfigured
-				returnCode |= k8sNotRunningStatusFlag
-			}
-		} else {
-			returnCode |= minikubeNotRunningStatusFlag
+			glog.Errorf("status error: %v", err)
 		}
-
-		status := Status{
-			Host:       hostSt,
-			Kubelet:    kubeletSt,
-			APIServer:  apiserverSt,
-			Kubeconfig: kubeconfigSt,
+		if st.Host == Nonexistent {
+			glog.Errorf("The %q cluster does not exist!", machineName)
 		}
 
 		switch strings.ToLower(output) {
 		case "text":
-			printStatusText(status)
+			if err := statusText(st, os.Stdout); err != nil {
+				exit.WithError("status text failure", err)
+			}
 		case "json":
-			printStatusJSON(status)
+			if err := statusJSON(st, os.Stdout); err != nil {
+				exit.WithError("status json failure", err)
+			}
 		default:
 			exit.WithCodeT(exit.BadUsage, fmt.Sprintf("invalid output format: %s. Valid values: 'text', 'json'", output))
 		}
 
-		os.Exit(returnCode)
+		os.Exit(exitCode(st))
 	},
 }
 
+func exitCode(st *Status) int {
+	c := 0
+	if st.Host != state.Running.String() {
+		c |= minikubeNotRunningStatusFlag
+	}
+	if st.APIServer != state.Running.String() || st.Kubelet != state.Running.String() {
+		c |= clusterNotRunningStatusFlag
+	}
+	if st.Kubeconfig != Configured {
+		c |= k8sNotRunningStatusFlag
+	}
+	return c
+}
+
+func status(api libmachine.API, name string) (*Status, error) {
+	st := &Status{
+		Host:       Nonexistent,
+		APIServer:  Nonexistent,
+		Kubelet:    Nonexistent,
+		Kubeconfig: Nonexistent,
+	}
+
+	hs, err := cluster.GetHostStatus(api, name)
+	glog.Infof("%s host status = %q (err=%v)", name, hs, err)
+	if err != nil {
+		return st, errors.Wrap(err, "host")
+	}
+
+	// We have no record of this host. Return nonexistent struct
+	if hs == state.None.String() {
+		return st, nil
+	}
+	st.Host = hs
+
+	// If it's not running, quickly bail out rather than delivering conflicting messages
+	if st.Host != state.Running.String() {
+		glog.Infof("host is not running, skipping remaining checks")
+		st.APIServer = st.Host
+		st.Kubelet = st.Host
+		st.Kubeconfig = st.Host
+		return st, nil
+	}
+
+	// We have a fully operational host, now we can check for details
+	ip, err := cluster.GetHostDriverIP(api, name)
+	if err != nil {
+		glog.Errorln("Error host driver ip status:", err)
+		st.APIServer = state.Error.String()
+		return st, err
+	}
+
+	port, err := kubeconfig.Port(name)
+	if err != nil {
+		glog.Warningf("unable to get port: %v", err)
+		port = constants.APIServerPort
+	}
+
+	st.Kubeconfig = Misconfigured
+	ok, err := kubeconfig.IsClusterInConfig(ip, name)
+	glog.Infof("%s is in kubeconfig at ip %s: %v (err=%v)", name, ip, ok, err)
+	if ok {
+		st.Kubeconfig = Configured
+	}
+
+	host, err := cluster.CheckIfHostExistsAndLoad(api, name)
+	if err != nil {
+		return st, err
+	}
+
+	cr, err := machine.CommandRunner(host)
+	if err != nil {
+		return st, err
+	}
+
+	stk, err := kverify.KubeletStatus(cr)
+	glog.Infof("%s kubelet status = %s (err=%v)", name, stk, err)
+
+	if err != nil {
+		glog.Warningf("kubelet err: %v", err)
+		st.Kubelet = state.Error.String()
+	} else {
+		st.Kubelet = stk.String()
+	}
+
+	sta, err := kverify.APIServerStatus(cr, ip, port)
+	glog.Infof("%s apiserver status = %s (err=%v)", name, stk, err)
+
+	if err != nil {
+		glog.Errorln("Error apiserver status:", err)
+		st.APIServer = state.Error.String()
+	} else {
+		st.APIServer = sta.String()
+	}
+
+	return st, nil
+}
+
 func init() {
 	statusCmd.Flags().StringVarP(&statusFormat, "format", "f", defaultStatusFormat,
 		`Go template format string for the status output.  The format for Go templates can be found here: https://golang.org/pkg/text/template/
@@ -172,25 +220,26 @@ For the list accessible variables for the template, see the struct values here:
 		`minikube status --output OUTPUT. json, text`)
 }
 
-var printStatusText = func(status Status) {
+func statusText(st *Status, w io.Writer) error {
 	tmpl, err := template.New("status").Parse(statusFormat)
 	if err != nil {
-		exit.WithError("Error creating status template", err)
+		return err
 	}
-	err = tmpl.Execute(os.Stdout, status)
-	if err != nil {
-		exit.WithError("Error executing status template", err)
+	if err := tmpl.Execute(w, st); err != nil {
+		return err
 	}
-	if status.Kubeconfig == KubeconfigStatus.Misconfigured {
-		out.WarningT("Warning: Your kubectl is pointing to stale minikube-vm.\nTo fix the kubectl context, run `minikube update-context`")
+	if st.Kubeconfig == Misconfigured {
+		_, err := w.Write([]byte("\nWARNING: Your kubectl is pointing to stale minikube-vm.\nTo fix the kubectl context, run `minikube update-context`\n"))
+		return err
 	}
+	return nil
 }
 
-var printStatusJSON = func(status Status) {
-
-	jsonString, err := json.Marshal(status)
+func statusJSON(st *Status, w io.Writer) error {
+	js, err := json.Marshal(st)
 	if err != nil {
-		exit.WithError("Error converting status to json", err)
+		return err
 	}
-	out.String(string(jsonString))
+	_, err = w.Write(js)
+	return err
 }

cmd/minikube/cmd/status_test.go

+/*
+Copyright 2016 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cmd
+
+import (
+	"bytes"
+	"encoding/json"
+	"testing"
+)
+
+func TestExitCode(t *testing.T) {
+	var tests = []struct {
+		name  string
+		want  int
+		state *Status
+	}{
+		{"ok", 0, &Status{Host: "Running", Kubelet: "Running", APIServer: "Running", Kubeconfig: Configured}},
+		{"paused", 2, &Status{Host: "Running", Kubelet: "Stopped", APIServer: "Paused", Kubeconfig: Configured}},
+		{"down", 7, &Status{Host: "Stopped", Kubelet: "Stopped", APIServer: "Stopped", Kubeconfig: Misconfigured}},
+		{"missing", 7, &Status{Host: "Nonexistent", Kubelet: "Nonexistent", APIServer: "Nonexistent", Kubeconfig: "Nonexistent"}},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			got := exitCode(tc.state)
+			if got != tc.want {
+				t.Errorf("exitcode(%+v) = %d, want: %d", tc.state, got, tc.want)
+			}
+		})
+	}
+}
+
+func TestStatusText(t *testing.T) {
+	var tests = []struct {
+		name  string
+		state *Status
+		want  string
+	}{
+		{
+			name:  "ok",
+			state: &Status{Host: "Running", Kubelet: "Running", APIServer: "Running", Kubeconfig: Configured},
+			want:  "host: Running\nkubelet: Running\napiserver: Running\nkubeconfig: Configured\n",
+		},
+		{
+			name:  "paused",
+			state: &Status{Host: "Running", Kubelet: "Stopped", APIServer: "Paused", Kubeconfig: Configured},
+			want:  "host: Running\nkubelet: Stopped\napiserver: Paused\nkubeconfig: Configured\n",
+		},
+		{
+			name:  "down",
+			state: &Status{Host: "Stopped", Kubelet: "Stopped", APIServer: "Stopped", Kubeconfig: Misconfigured},
+			want:  "host: Stopped\nkubelet: Stopped\napiserver: Stopped\nkubeconfig: Misconfigured\n\nWARNING: Your kubectl is pointing to stale minikube-vm.\nTo fix the kubectl context, run `minikube update-context`\n",
+		},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			var b bytes.Buffer
+			err := statusText(tc.state, &b)
+			if err != nil {
+				t.Errorf("text(%+v) error: %v", tc.state, err)
+			}
+
+			got := b.String()
+			if got != tc.want {
+				t.Errorf("text(%+v) = %q, want: %q", tc.state, got, tc.want)
+			}
+		})
+	}
+}
+
+func TestStatusJSON(t *testing.T) {
+	var tests = []struct {
+		name  string
+		state *Status
+	}{
+		{"ok", &Status{Host: "Running", Kubelet: "Running", APIServer: "Running", Kubeconfig: Configured}},
+		{"paused", &Status{Host: "Running", Kubelet: "Stopped", APIServer: "Paused", Kubeconfig: Configured}},
+		{"down", &Status{Host: "Stopped", Kubelet: "Stopped", APIServer: "Stopped", Kubeconfig: Misconfigured}},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			var b bytes.Buffer
+			err := statusJSON(tc.state, &b)
+			if err != nil {
+				t.Errorf("json(%+v) error: %v", tc.state, err)
+			}
+
+			st := &Status{}
+			if err := json.Unmarshal(b.Bytes(), st); err != nil {
+				t.Errorf("json(%+v) unmarshal error: %v", tc.state, err)
+			}
+		})
+	}
+}

cmd/minikube/cmd/stop.go

@@ -20,6 +20,7 @@ import (
 	"time"
 
 	"github.com/docker/machine/libmachine/mcnerror"
+	"github.com/golang/glog"
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
@@ -54,6 +55,11 @@ func runStop(cmd *cobra.Command, args []string) {
 	nonexistent := false
 	stop := func() (err error) {
 		err = cluster.StopHost(api)
+		if err == nil {
+			return nil
+		}
+		glog.Warningf("stop host returned error: %v", err)
+
 		switch err := errors.Cause(err).(type) {
 		case mcnerror.ErrHostDoesNotExist:
 			out.T(out.Meh, `"{{.profile_name}}" VM does not exist, nothing to stop`, out.V{"profile_name": profile})

cmd/minikube/cmd/unpause.go

+/*
+Copyright 2020 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cmd
+
+import (
+	"os"
+	"strings"
+
+	"github.com/golang/glog"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+
+	"k8s.io/minikube/pkg/minikube/cluster"
+	"k8s.io/minikube/pkg/minikube/config"
+	"k8s.io/minikube/pkg/minikube/cruntime"
+	"k8s.io/minikube/pkg/minikube/exit"
+	"k8s.io/minikube/pkg/minikube/machine"
+	"k8s.io/minikube/pkg/minikube/out"
+)
+
+// unpauseCmd represents the docker-pause command
+var unpauseCmd = &cobra.Command{
+	Use:   "unpause",
+	Short: "unpause Kubernetes",
+	Run: func(cmd *cobra.Command, args []string) {
+		cname := viper.GetString(config.MachineProfile)
+		api, err := machine.NewAPIClient()
+		if err != nil {
+			exit.WithError("Error getting client", err)
+		}
+		defer api.Close()
+		cc, err := config.Load(cname)
+
+		if err != nil && !config.IsNotExist(err) {
+			exit.WithError("Error loading profile config", err)
+		}
+
+		if err != nil {
+			out.ErrT(out.Meh, `"{{.name}}" profile does not exist`, out.V{"name": cname})
+			os.Exit(1)
+		}
+		glog.Infof("config: %+v", cc)
+		host, err := cluster.CheckIfHostExistsAndLoad(api, cname)
+		if err != nil {
+			exit.WithError("Error getting host", err)
+		}
+
+		r, err := machine.CommandRunner(host)
+		if err != nil {
+			exit.WithError("Failed to get command runner", err)
+		}
+
+		cr, err := cruntime.New(cruntime.Config{Type: cc.KubernetesConfig.ContainerRuntime, Runner: r})
+		if err != nil {
+			exit.WithError("Failed runtime", err)
+		}
+
+		glog.Infof("namespaces: %v keys: %v", namespaces, viper.AllSettings())
+		if allNamespaces {
+			namespaces = nil //all
+		} else {
+			if len(namespaces) == 0 {
+				exit.WithCodeT(exit.BadUsage, "Use -A to specify all namespaces")
+			}
+		}
+
+		ids, err := cluster.Unpause(cr, r, namespaces)
+		if err != nil {
+			exit.WithError("Pause", err)
+		}
+
+		if namespaces == nil {
+			out.T(out.Pause, "Unpaused kubelet and {{.count}} containers", out.V{"count": len(ids)})
+		} else {
+			out.T(out.Pause, "Unpaused kubelet and {{.count}} containers in: {{.namespaces}}", out.V{"count": len(ids), "namespaces": strings.Join(namespaces, ", ")})
+		}
+
+	},
+}
+
+func init() {
+	unpauseCmd.Flags().StringSliceVarP(&namespaces, "--namespaces", "n", cluster.DefaultNamespaces, "namespaces to unpause")
+	unpauseCmd.Flags().BoolVarP(&allNamespaces, "all-namespaces", "A", false, "If set, unpause all namespaces")
+}

cmd/performance/mkcmp/cmd/mkcmp.go

@@ -17,20 +17,26 @@ limitations under the License.
 package cmd
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"os"
 
 	"github.com/spf13/cobra"
+	"k8s.io/minikube/pkg/minikube/perf"
 )
 
 var rootCmd = &cobra.Command{
-	Use:   "mkcmp [path to first binary] [path to second binary]",
-	Short: "mkcmp is used to compare performance of two minikube binaries",
+	Use:           "mkcmp [path to first binary] [path to second binary]",
+	Short:         "mkcmp is used to compare performance of two minikube binaries",
+	SilenceUsage:  true,
+	SilenceErrors: true,
 	PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
 		return validateArgs(args)
 	},
-	Run: func(cmd *cobra.Command, args []string) {},
+	RunE: func(cmd *cobra.Command, args []string) error {
+		return perf.CompareMinikubeStart(context.Background(), os.Stdout, args)
+	},
 }
 
 func validateArgs(args []string) error {
@@ -43,7 +49,7 @@ func validateArgs(args []string) error {
 // Execute runs the mkcmp command
 func Execute() {
 	if err := rootCmd.Execute(); err != nil {
-		fmt.Println(err)
+		fmt.Println("Error:", err)
 		os.Exit(1)
 	}
 }

deploy/addons/addon-manager.yaml.tmpl

-# Copyright 2016 The Kubernetes Authors All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-apiVersion: v1
-kind: Pod
-metadata:
-  name: kube-addon-manager
-  namespace: kube-system
-  labels:
-    component: kube-addon-manager
-    version: v9.0.2
-    kubernetes.io/minikube-addons: addon-manager
-spec:
-  hostNetwork: true
-  containers:
-  - name: kube-addon-manager
-    image: {{default "k8s.gcr.io" .ImageRepository}}/kube-addon-manager{{.ExoticArch}}:v9.0.2
-    env:
-    - name: KUBECONFIG
-      value: /var/lib/minikube/kubeconfig
-    - name: TEST_ADDON_CHECK_INTERVAL_SEC
-      value: "5"
-    - name: ADDON_MANAGER_LEADER_ELECTION
-      value: "false"
-    - name: KUBECTL_EXTRA_PRUNE_WHITELIST
-      value: install.istio.io/v1alpha2/IstioControlPlane
-    imagePullPolicy: IfNotPresent
-    resources:
-      requests:
-        cpu: 5m
-        memory: 50Mi
-    volumeMounts:
-    - mountPath: /etc/kubernetes/
-      name: addons
-      readOnly: true
-    - mountPath: /var/lib/minikube/
-      name: kubeconfig
-      readOnly: true
-  volumes:
-  - hostPath:
-      path: /etc/kubernetes/
-    name: addons
-  - hostPath:
-      path: /var/lib/minikube/
-    name: kubeconfig

deploy/addons/registry-creds/registry-creds-rc.yaml.tmpl

@@ -18,7 +18,7 @@ spec:
         addonmanager.kubernetes.io/mode: Reconcile
     spec:
       containers:
-      - image: upmcenterprises/registry-creds:1.9
+      - image: upmcenterprises/registry-creds:1.10
         name: registry-creds
         imagePullPolicy: Always
         env:
@@ -77,6 +77,21 @@ spec:
               secretKeyRef:
                 name: registry-creds-gcr
                 key: gcrurl
+          - name: ACR_PASSWORD
+            valueFrom:
+              secretKeyRef:
+                name: registry-creds-acr
+                key: ACR_PASSWORD
+          - name: ACR_URL
+            valueFrom:
+              secretKeyRef:
+                name: registry-creds-acr
+                key: ACR_URL
+          - name: ACR_CLIENT_ID
+            valueFrom:
+              secretKeyRef:
+                name: registry-creds-acr
+                key: ACR_CLIENT_ID
         volumeMounts:
         - name: gcr-creds
           mountPath: "/root/.config/gcloud"

deploy/iso/minikube-iso/configs/minikube_defconfig

@@ -19,6 +19,8 @@ BR2_ROOTFS_USERS_TABLES="$(BR2_EXTERNAL_MINIKUBE_PATH)/board/coreos/minikube/use
 BR2_ROOTFS_OVERLAY="$(BR2_EXTERNAL_MINIKUBE_PATH)/board/coreos/minikube/rootfs-overlay"
 BR2_GLOBAL_PATCH_DIR="$(BR2_EXTERNAL_MINIKUBE_PATH)/board/coreos/minikube/patches"
 BR2_LINUX_KERNEL=y
+BR2_LINUX_KERNEL_BZIMAGE=y
+BR2_LINUX_KERNEL_LZ4=y
 BR2_LINUX_KERNEL_USE_CUSTOM_CONFIG=y
 BR2_LINUX_KERNEL_CUSTOM_CONFIG_FILE="$(BR2_EXTERNAL_MINIKUBE_PATH)/board/coreos/minikube/linux_defconfig"
 BR2_LINUX_KERNEL_NEEDS_HOST_LIBELF=y
@@ -55,7 +57,8 @@ BR2_PACKAGE_SYSTEMD_MACHINED=y
 BR2_PACKAGE_SYSTEMD_VCONSOLE=y
 BR2_PACKAGE_UTIL_LINUX_NSENTER=y
 BR2_PACKAGE_UTIL_LINUX_SCHEDUTILS=y
-BR2_TARGET_ROOTFS_CPIO_BZIP2=y
+BR2_TARGET_ROOTFS_CPIO=y
+BR2_TARGET_ROOTFS_CPIO_GZIP=y
 BR2_TARGET_ROOTFS_ISO9660=y
 BR2_TARGET_ROOTFS_ISO9660_BOOT_MENU="$(BR2_EXTERNAL_MINIKUBE_PATH)/board/coreos/minikube/isolinux.cfg"
 BR2_TARGET_SYSLINUX=y

deploy/iso/minikube-iso/package/Config.in

 menu "System tools"
     source "$BR2_EXTERNAL_MINIKUBE_PATH/package/runc-master/Config.in"
     source "$BR2_EXTERNAL_MINIKUBE_PATH/package/podman/Config.in"
+    source "$BR2_EXTERNAL_MINIKUBE_PATH/package/varlink/Config.in"
     source "$BR2_EXTERNAL_MINIKUBE_PATH/package/conmon-master/Config.in"
     source "$BR2_EXTERNAL_MINIKUBE_PATH/package/crio-bin/Config.in"
     source "$BR2_EXTERNAL_MINIKUBE_PATH/package/crictl-bin/Config.in"

deploy/iso/minikube-iso/package/automount/minikube-automount

@@ -29,7 +29,7 @@ if [ ! -n "$BOOT2DOCKER_DATA" ]; then
             # Let kernel re-read partition table
             partprobe
             # Add the data partition
-            (echo n; echo p; echo 1; echo ; echo ; echo w) | fdisk $UNPARTITIONED_HD
+            (echo g; echo n; echo 1; echo ; echo ; echo w) | fdisk $UNPARTITIONED_HD
             # Let kernel re-read partition table
             partprobe
             # wait for the partition to actually exist, timeout after about 5 seconds
@@ -57,7 +57,7 @@ if [ ! -n "$BOOT2DOCKER_DATA" ]; then
                 if [ $NON_NUL == 0 ]; then
                     # Create the partition, format it and then mount it
                     echo "NEW VMware boot2docker managed disk image ($UNPARTITIONED_HD): formatting it for use"
-                    (echo n; echo p; echo 1; echo ; echo ; echo w) | fdisk $UNPARTITIONED_HD
+                    (echo g; echo n; echo 1; echo ; echo ; echo w) | fdisk $UNPARTITIONED_HD
                     BOOT2DOCKER_DATA=`echo "${UNPARTITIONED_HD}1"`
                     mkfs.ext4 -i 2048 -L $LABEL $BOOT2DOCKER_DATA
                 else

deploy/iso/minikube-iso/package/varlink/Config.in

+config BR2_PACKAGE_VARLINK
+	bool "varlink"
+	default y

deploy/iso/minikube-iso/package/varlink/varlink.hash

+sha256 3857f109574750403b233b5fdf73f1852d8decc33dac8f73bd49f2003b69ad22 16.tar.gz
+sha256 0dcb451f32033154c56710c216e67f245923fe2b011321271f6670e5a2285ce6 17.tar.gz
+sha256 7a32543643116ad105da4ddb2f8030de7dcad1cdb3feb1a214ae5e7b65a6a198 18.tar.gz

deploy/iso/minikube-iso/package/varlink/varlink.mk

+VARLINK_VERSION = 18
+VARLINK_SITE = https://github.com/varlink/libvarlink/archive
+VARLINK_SOURCE = $(VARLINK_VERSION).tar.gz
+VARLINK_LICENSE = Apache-2.0
+VARLINK_LICENSE_FILES = LICENSE
+
+$(eval $(meson-package))

deploy/minikube/releases.json

 [
+  {
+      "name": "v1.7.0",
+      "checksums": {
+          "darwin": "1f3785e9521eabe241df0481fa41887a6a3873307bac8a89fd0e48aa7612be29",
+          "linux": "f1fcab9f161a64f19b618a901e50488ed6f1c6ab20695c82623586a701d2d261",
+          "windows": "2f4448f32e505bf38ba52cd3678c73622a6bb452c63a4179d590f6da26520c68"
+      }
+  },
   {
       "name": "v1.6.2",
       "checksums": {

deploy/storage-provisioner/Dockerfile

@@ -13,5 +13,6 @@
 # limitations under the License.
 
 FROM scratch
-COPY out/storage-provisioner /storage-provisioner
-CMD ["/storage-provisioner"]
\ No newline at end of file
+ARG arch
+COPY out/storage-provisioner-${arch} /storage-provisioner
+CMD ["/storage-provisioner"]

go.mod

@@ -2,10 +2,6 @@ module k8s.io/minikube
 
 go 1.13
 
-require github.com/google/go-containerregistry v0.0.0-20180731221751-697ee0b3d46e
-
-require k8s.io/kubernetes v1.15.2
-
 require (
 	github.com/Parallels/docker-machine-parallels v1.3.0
 	github.com/Sirupsen/logrus v0.0.0-20170822132746-89742aefa4b2 // indirect
@@ -17,7 +13,7 @@ require (
 	github.com/cloudfoundry-attic/jibber_jabber v0.0.0-20151120183258-bcc4c8345a21
 	github.com/cloudfoundry/jibber_jabber v0.0.0-20151120183258-bcc4c8345a21 // indirect
 	github.com/docker/docker v1.13.1
-	github.com/docker/go-units v0.3.3
+	github.com/docker/go-units v0.4.0
 	github.com/docker/machine v0.7.1-0.20190718054102-a555e4f7a8f5 // version is 0.7.1 to pin to a555e4f7a8f5
 	github.com/elazarl/goproxy v0.0.0-20190421051319-9d40249d3c2f
 	github.com/elazarl/goproxy/ext v0.0.0-20190421051319-9d40249d3c2f // indirect
@@ -26,14 +22,16 @@ require (
 	github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3
 	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
 	github.com/google/go-cmp v0.3.0
+	github.com/google/go-containerregistry v0.0.0-20180731221751-697ee0b3d46e
 	github.com/googleapis/gnostic v0.3.0 // indirect
-	github.com/gorilla/mux v1.7.1 // indirect
+	github.com/gorilla/mux v1.7.3 // indirect
 	github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce // indirect
 	github.com/hashicorp/go-getter v1.4.0
 	github.com/hashicorp/go-multierror v0.0.0-20160811015721-8c5f0ad93604 // indirect
 	github.com/hashicorp/go-retryablehttp v0.5.4
 	github.com/hooklift/assert v0.0.0-20170704181755-9d1defd6d214 // indirect
 	github.com/hooklift/iso9660 v0.0.0-20170318115843-1cf07e5970d8
+	github.com/imdario/mergo v0.3.8 // indirect
 	github.com/intel-go/cpuid v0.0.0-20181003105527-1a4a6f06a1c6 // indirect
 	github.com/jimmidyson/go-download v0.0.0-20161028105827-7f9a90c8c95b
 	github.com/johanneswuerbach/nfsexports v0.0.0-20181204082207-1aa528dcb345
@@ -48,10 +46,10 @@ require (
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
 	github.com/libvirt/libvirt-go v3.4.0+incompatible
 	github.com/machine-drivers/docker-machine-driver-vmware v0.1.1
-	github.com/mattn/go-isatty v0.0.8
+	github.com/mattn/go-isatty v0.0.9
 	github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936
 	github.com/moby/hyperkit v0.0.0-20171020124204-a12cd7250bcd
-	github.com/olekukonko/tablewriter v0.0.0-20160923125401-bdcc175572fd
+	github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5
 	github.com/otiai10/copy v1.0.2
 	github.com/pborman/uuid v1.2.0
 	github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2
@@ -61,30 +59,25 @@ require (
 	github.com/pmezard/go-difflib v1.0.0
 	github.com/samalba/dockerclient v0.0.0-20160414174713-91d7393ff859 // indirect
 	github.com/shirou/gopsutil v2.18.12+incompatible
-	github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 // indirect
-	github.com/sirupsen/logrus v1.4.1 // indirect
 	github.com/spf13/cobra v0.0.5
-	github.com/spf13/pflag v1.0.3
+	github.com/spf13/pflag v1.0.5
 	github.com/spf13/viper v1.3.2
 	github.com/xeipuuv/gojsonpointer v0.0.0-20151027082146-e0fe6f683076 // indirect
 	github.com/xeipuuv/gojsonreference v0.0.0-20150808065054-e02fc20de94c // indirect
 	github.com/xeipuuv/gojsonschema v0.0.0-20160623135812-c539bca196be
 	github.com/zchee/go-vmnet v0.0.0-20161021174912-97ebf9174097
-	golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d
-	golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4
-	golang.org/x/sync v0.0.0-20190423024810-112230192c58
-	golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb
+	golang.org/x/build v0.0.0-20190927031335-2835ba2e683f
+	golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586
+	golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e
+	golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456
 	golang.org/x/text v0.3.2
-	gopkg.in/airbrake/gobrake.v2 v2.0.9 // indirect
-	gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
-	gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2 // indirect
 	gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 // indirect
-	gotest.tools v2.2.0+incompatible
-	k8s.io/api v0.0.0
-	k8s.io/apimachinery v0.0.0
+	k8s.io/api v0.17.2
+	k8s.io/apimachinery v0.17.2
 	k8s.io/client-go v11.0.0+incompatible
-	k8s.io/klog v0.3.3 // indirect
-	k8s.io/kubectl v0.0.0-00010101000000-000000000000
+	k8s.io/kubectl v0.0.0
+	k8s.io/kubernetes v1.17.2
+	k8s.io/utils v0.0.0-20200122174043-1e243dd1a584 // indirect
 	sigs.k8s.io/sig-storage-lib-external-provisioner v4.0.0+incompatible
 )
 
@@ -92,28 +85,28 @@ replace (
 	git.apache.org/thrift.git => github.com/apache/thrift v0.0.0-20180902110319-2566ecd5d999
 	github.com/docker/machine => github.com/machine-drivers/machine v0.7.1-0.20191109154235-b39d5b50de51
 	github.com/hashicorp/go-getter => github.com/afbjorklund/go-getter v1.4.1-0.20190910175809-eb9f6c26742c
-	k8s.io/api => k8s.io/kubernetes/staging/src/k8s.io/api v0.0.0-20190623232353-8c3b7d7679cc
-	k8s.io/apiextensions-apiserver => k8s.io/kubernetes/staging/src/k8s.io/apiextensions-apiserver v0.0.0-20190623232353-8c3b7d7679cc
-	k8s.io/apimachinery => k8s.io/kubernetes/staging/src/k8s.io/apimachinery v0.0.0-20190623232353-8c3b7d7679cc
-	k8s.io/apiserver => k8s.io/kubernetes/staging/src/k8s.io/apiserver v0.0.0-20190623232353-8c3b7d7679cc
-	k8s.io/cli-runtime => k8s.io/kubernetes/staging/src/k8s.io/cli-runtime v0.0.0-20190623232353-8c3b7d7679cc
-	k8s.io/client-go => k8s.io/kubernetes/staging/src/k8s.io/client-go v0.0.0-20190623232353-8c3b7d7679cc
-	k8s.io/cloud-provider => k8s.io/kubernetes/staging/src/k8s.io/cloud-provider v0.0.0-20190623232353-8c3b7d7679cc
-	k8s.io/cluster-bootstrap => k8s.io/kubernetes/staging/src/k8s.io/cluster-bootstrap v0.0.0-20190623232353-8c3b7d7679cc
-	k8s.io/code-generator => k8s.io/kubernetes/staging/src/k8s.io/code-generator v0.0.0-20190623232353-8c3b7d7679cc
-	k8s.io/component-base => k8s.io/kubernetes/staging/src/k8s.io/component-base v0.0.0-20190623232353-8c3b7d7679cc
-	k8s.io/cri-api => k8s.io/kubernetes/staging/src/k8s.io/cri-api v0.0.0-20190623232353-8c3b7d7679cc
-	k8s.io/csi-translation-lib => k8s.io/kubernetes/staging/src/k8s.io/csi-translation-lib v0.0.0-20190623232353-8c3b7d7679cc
-	k8s.io/kube-aggregator => k8s.io/kubernetes/staging/src/k8s.io/kube-aggregator v0.0.0-20190623232353-8c3b7d7679cc
-	k8s.io/kube-controller-manager => k8s.io/kubernetes/staging/src/k8s.io/kube-controller-manager v0.0.0-20190623232353-8c3b7d7679cc
-	k8s.io/kube-proxy => k8s.io/kubernetes/staging/src/k8s.io/kube-proxy v0.0.0-20190623232353-8c3b7d7679cc
-	k8s.io/kube-scheduler => k8s.io/kubernetes/staging/src/k8s.io/kube-scheduler v0.0.0-20190623232353-8c3b7d7679cc
-	k8s.io/kubectl => k8s.io/kubernetes/staging/src/k8s.io/kubectl v0.0.0-20190623232353-8c3b7d7679cc
-	k8s.io/kubelet => k8s.io/kubernetes/staging/src/k8s.io/kubelet v0.0.0-20190623232353-8c3b7d7679cc
-	k8s.io/legacy-cloud-providers => k8s.io/kubernetes/staging/src/k8s.io/legacy-cloud-providers v0.0.0-20190623232353-8c3b7d7679cc
-	k8s.io/metrics => k8s.io/kubernetes/staging/src/k8s.io/metrics v0.0.0-20190623232353-8c3b7d7679cc
-	k8s.io/node-api => k8s.io/kubernetes/staging/src/k8s.io/node-api v0.0.0-20190623232353-8c3b7d7679cc
-	k8s.io/sample-apiserver => k8s.io/kubernetes/staging/src/k8s.io/sample-apiserver v0.0.0-20190623232353-8c3b7d7679cc
-	k8s.io/sample-cli-plugin => k8s.io/kubernetes/staging/src/k8s.io/sample-cli-plugin v0.0.0-20190623232353-8c3b7d7679cc
-	k8s.io/sample-controller => k8s.io/kubernetes/staging/src/k8s.io/sample-controller v0.0.0-20190623232353-8c3b7d7679cc
+	k8s.io/api => k8s.io/api v0.17.2
+	k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.17.2
+	k8s.io/apimachinery => k8s.io/apimachinery v0.17.2
+	k8s.io/apiserver => k8s.io/apiserver v0.17.2
+	k8s.io/cli-runtime => k8s.io/cli-runtime v0.17.2
+	k8s.io/client-go => k8s.io/client-go v0.17.2
+	k8s.io/cloud-provider => k8s.io/cloud-provider v0.17.2
+	k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.17.2
+	k8s.io/code-generator => k8s.io/code-generator v0.17.2
+	k8s.io/component-base => k8s.io/component-base v0.17.2
+	k8s.io/cri-api => k8s.io/cri-api v0.17.2
+	k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.17.2
+	k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.17.2
+	k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.17.2
+	k8s.io/kube-proxy => k8s.io/kube-proxy v0.17.2
+	k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.17.2
+	k8s.io/kubectl => k8s.io/kubectl v0.17.2
+	k8s.io/kubelet => k8s.io/kubelet v0.17.2
+	k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.17.2
+	k8s.io/metrics => k8s.io/metrics v0.17.2
+	k8s.io/node-api => k8s.io/node-api v0.17.2
+	k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.17.2
+	k8s.io/sample-cli-plugin => k8s.io/sample-cli-plugin v0.17.2
+	k8s.io/sample-controller => k8s.io/sample-controller v0.17.2
 )

hack/images/kicbase.Dockerfile

 ARG COMMIT_SHA
-FROM kindest/node:v1.16.2
+# using base image created by kind https://github.com/kubernetes-sigs/kind
+# which is an ubuntu 19.10 with an entry-point that helps running systemd
+# could be changed to any debian that can run systemd
+FROM kindest/base:v20200122-2dfe64b2
 USER root
 RUN apt-get update && apt-get install -y \
   sudo \
   dnsutils \
+  openssh-server \
+  docker.io \
   && apt-get clean -y 
+# disable containerd by default
+RUN systemctl disable containerd
+RUN rm /etc/crictl.yaml
+# enable docker which is default
+RUN systemctl enable docker
+# making SSH work for docker container 
+# based on https://github.com/rastasheep/ubuntu-sshd/blob/master/18.04/Dockerfile
+RUN mkdir /var/run/sshd
+RUN echo 'root:root' |chpasswd
+RUN sed -ri 's/^#?PermitRootLogin\s+.*/PermitRootLogin yes/' /etc/ssh/sshd_config
+RUN sed -ri 's/UsePAM yes/#UsePAM yes/g' /etc/ssh/sshd_config
+EXPOSE 22
+# for minikube ssh. to match VM using "docker" as username
+RUN adduser --ingroup docker --disabled-password --gecos '' docker 
+RUN adduser docker sudo
+RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
+USER docker
+RUN mkdir /home/docker/.ssh
+# Deleting leftovers
+USER root
+# kind base-image entry-point expects a "kind" folder for product_name,product_uuid
+# https://github.com/kubernetes-sigs/kind/blob/master/images/base/files/usr/local/bin/entrypoint
+RUN mkdir -p /kind
 RUN rm -rf \
     /var/cache/debconf/* \
     /var/lib/apt/lists/* \
@@ -14,5 +42,4 @@ RUN rm -rf \
     /usr/share/doc/* \
     /usr/share/man/* \
     /usr/share/local/* \
-    /kind/bin/kubeadm /kind/bin/kubelet /kind/systemd /kind/images /kind/manifests
 RUN echo "kic! Build: ${COMMIT_SHA} Time :$(date)" > "/kic.txt"

deploy/storage-provisioner/Dockerfile-s390x → hack/jenkins/linux_integration_tests_docker.sh

-# Copyright 2016 The Kubernetes Authors All rights reserved.
+#!/bin/bash
+
+# Copyright 2019 The Kubernetes Authors All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,6 +14,22 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM s390x/ubuntu:16.04
-COPY out/storage-provisioner storage-provisioner
-CMD ["/storage-provisioner"]
+
+# This script runs the integration tests on a Linux machine for the KVM Driver
+
+# The script expects the following env variables:
+# MINIKUBE_LOCATION: GIT_COMMIT from upstream build.
+# COMMIT: Actual commit ID from upstream build
+# EXTRA_BUILD_ARGS (optional): Extra args to be passed into the minikube integrations tests
+# access_token: The Github API access token. Injected by the Jenkins credential provider. 
+
+set -e
+
+OS_ARCH="linux-amd64"
+VM_DRIVER="docker"
+JOB_NAME="Docker_Linux"
+
+mkdir -p cron && gsutil -qm rsync "gs://minikube-builds/${MINIKUBE_LOCATION}/cron" cron || echo "FAILED TO GET CRON FILES"
+sudo install cron/cleanup_and_reboot_Linux.sh /etc/cron.hourly/cleanup_and_reboot || echo "FAILED TO INSTALL CLEANUP"
+
+source ./common.sh

hack/jenkins/minikube_set_pending.sh

@@ -40,6 +40,7 @@ jobs=(
      # 'KVM-GPU_Linux' - Disabled
      'KVM_Linux'
      'none_Linux'
+     'Docker_Linux'
 )
 
 # retry_github_status provides reliable github status updates

hack/jenkins/release_build_and_upload.sh

@@ -35,18 +35,18 @@ grep -E "^VERSION_MAJOR \\?=" Makefile | grep "${VERSION_MAJOR}"
 grep -E "^VERSION_MINOR \\?=" Makefile | grep "${VERSION_MINOR}"
 grep -E "^VERSION_BUILD \\?=" Makefile | grep "${VERSION_BUILD}"
 
-# Diagnostics
-go env GOPATH
+# Force go packages to the Jekins home directory
+export GOPATH=$HOME/go
 
 # Build and upload
-env GOPATH=$HOME/go BUILD_IN_DOCKER=y \
+env BUILD_IN_DOCKER=y \
   make -j 16 \
   all \
   out/minikube-installer.exe \
-  "out/minikube_${DEB_VERSION}.deb" \
-  "out/minikube-${RPM_VERSION}.rpm" \
-  "out/docker-machine-driver-kvm2_${DEB_VERSION}.deb" \
-  "out/docker-machine-driver-kvm2-${RPM_VERSION}.rpm"
+  "out/minikube_${DEB_VERSION}-0_amd64.deb" \
+  "out/minikube-${RPM_VERSION}-0.x86_64.rpm" \
+  "out/docker-machine-driver-kvm2_${DEB_VERSION}-0_amd64.deb" \
+  "out/docker-machine-driver-kvm2-${RPM_VERSION}-0.x86_64.rpm"
 
 make checksum
 

hack/jenkins/release_github_page.sh

@@ -61,14 +61,14 @@ See [Getting Started](https://minikube.sigs.k8s.io/docs/start/)
 
 # ================================================================================
 # Deleting release from github before creating new one
-github-release delete \
+github-release -v delete \
   --user "${GITHUB_ORGANIZATION}" \
   --repo "${GITHUB_REPO}" \
   --tag "${TAGNAME}" \
   || true
 
 # Creating a new release in github
-github-release release ${RELEASE_FLAGS} \
+github-release -v release ${RELEASE_FLAGS} \
     --user "${GITHUB_ORGANIZATION}" \
     --repo "${GITHUB_REPO}" \
     --tag "${TAGNAME}" \
@@ -84,8 +84,8 @@ FILES_TO_UPLOAD=(
     'minikube-windows-amd64.exe'
     'minikube-windows-amd64.exe.sha256'
     'minikube-installer.exe'
-    "minikube_${DEB_VERSION}.deb"
-    "minikube-${RPM_VERSION}.rpm"
+    "minikube_${DEB_VERSION}-0_amd64.deb"
+    "minikube-${RPM_VERSION}-0.x86_64.rpm"
     'docker-machine-driver-kvm2'
     'docker-machine-driver-kvm2.sha256'
     'docker-machine-driver-hyperkit'
@@ -106,7 +106,7 @@ do
     n=0
     until [ $n -ge 5 ]
     do
-        github-release upload \
+        github-release -v upload \
           --user "${GITHUB_ORGANIZATION}" \
           --repo "${GITHUB_REPO}" \
           --tag "${TAGNAME}" \

installers/linux/deb/kvm2_deb_template/DEBIAN/control

@@ -2,7 +2,7 @@ Package: docker-machine-driver-kvm2
 Version: --VERSION--
 Section: base
 Priority: optional
-Architecture: amd64
+Architecture: --ARCH--
 Depends: libvirt0 (>= 1.3.1)
 Recommends: minikube
 Maintainer: Thomas Strömberg <t+minikube@stromberg.org>

installers/linux/rpm/kvm2_rpm_template/docker-machine-driver-kvm2.spec

@@ -18,7 +18,7 @@ a consistent way to manage various VM providers.
 %prep
 mkdir -p %{name}-%{version}
 cd %{name}-%{version}
-cp --OUT--/docker-machine-driver-kvm2 .
+cp --OUT--/docker-machine-driver-kvm2-%{_arch} docker-machine-driver-kvm2
 
 %install
 cd %{name}-%{version}

pkg/addons/addons.go

@@ -18,9 +18,13 @@ package addons
 
 import (
 	"fmt"
-	"os"
+	"path"
+	"sort"
 	"strconv"
+	"strings"
+	"time"
 
+	"github.com/golang/glog"
 	"github.com/pkg/errors"
 	"github.com/spf13/viper"
 	"k8s.io/minikube/pkg/minikube/assets"
@@ -39,6 +43,7 @@ const defaultStorageClassProvisioner = "standard"
 
 // Set sets a value
 func Set(name, value, profile string) error {
+	glog.Infof("Setting %s=%s in profile %q", name, value, profile)
 	a, valid := isAddonValid(name)
 	if !valid {
 		return errors.Errorf("%s is not a valid addon", name)
@@ -64,7 +69,7 @@ func Set(name, value, profile string) error {
 		return errors.Wrap(err, "running callbacks")
 	}
 
-	// Write the value
+	glog.Infof("Writing out %q config to set %s=%v...", profile, name, value)
 	return config.Write(profile, c)
 }
 
@@ -98,6 +103,7 @@ func SetBool(m *config.MachineConfig, name string, val string) error {
 
 // enableOrDisableAddon updates addon status executing any commands necessary
 func enableOrDisableAddon(name, val, profile string) error {
+	glog.Infof("Setting addon %s=%s in %q", name, val, profile)
 	enable, err := strconv.ParseBool(val)
 	if err != nil {
 		return errors.Wrapf(err, "parsing bool: %s", name)
@@ -105,14 +111,14 @@ func enableOrDisableAddon(name, val, profile string) error {
 	addon := assets.Addons[name]
 
 	// check addon status before enabling/disabling it
-	alreadySet, err := isAddonAlreadySet(addon, enable)
+	alreadySet, err := isAddonAlreadySet(addon, enable, profile)
 	if err != nil {
 		out.ErrT(out.Conflict, "{{.error}}", out.V{"error": err})
 		return err
 	}
-	//if addon is already enabled or disabled, do nothing
+
 	if alreadySet {
-		return nil
+		glog.Warningf("addon %s should already be in state %v", name, val)
 	}
 
 	if name == "istio" && enable {
@@ -132,20 +138,15 @@ func enableOrDisableAddon(name, val, profile string) error {
 	}
 	defer api.Close()
 
-	//if minikube is not running, we return and simply update the value in the addon
-	//config and rewrite the file
-	if !cluster.IsMinikubeRunning(api) {
-		return nil
-	}
-
 	cfg, err := config.Load(profile)
-	if err != nil && !os.IsNotExist(err) {
+	if err != nil && !config.IsNotExist(err) {
 		exit.WithCodeT(exit.Data, "Unable to load config: {{.error}}", out.V{"error": err})
 	}
 
-	host, err := cluster.CheckIfHostExistsAndLoad(api, cfg.Name)
-	if err != nil {
-		return errors.Wrap(err, "getting host")
+	host, err := cluster.CheckIfHostExistsAndLoad(api, profile)
+	if err != nil || !cluster.IsHostRunning(api, profile) {
+		glog.Warningf("%q is not running, writing %s=%v to disk and skipping enablement (err=%v)", profile, addon.Name(), enable, err)
+		return nil
 	}
 
 	cmd, err := machine.CommandRunner(host)
@@ -154,14 +155,13 @@ func enableOrDisableAddon(name, val, profile string) error {
 	}
 
 	data := assets.GenerateTemplateData(cfg.KubernetesConfig)
-	return enableOrDisableAddonInternal(addon, cmd, data, enable)
+	return enableOrDisableAddonInternal(addon, cmd, data, enable, profile)
 }
 
-func isAddonAlreadySet(addon *assets.Addon, enable bool) (bool, error) {
-	addonStatus, err := addon.IsEnabled()
-
+func isAddonAlreadySet(addon *assets.Addon, enable bool, profile string) (bool, error) {
+	addonStatus, err := addon.IsEnabled(profile)
 	if err != nil {
-		return false, errors.Wrap(err, "get the addon status")
+		return false, errors.Wrap(err, "is enabled")
 	}
 
 	if addonStatus && enable {
@@ -173,34 +173,53 @@ func isAddonAlreadySet(addon *assets.Addon, enable bool) (bool, error) {
 	return false, nil
 }
 
-func enableOrDisableAddonInternal(addon *assets.Addon, cmd command.Runner, data interface{}, enable bool) error {
-	var err error
-
-	updateFile := cmd.Copy
-	if !enable {
-		updateFile = cmd.Remove
-	}
-
+func enableOrDisableAddonInternal(addon *assets.Addon, cmd command.Runner, data interface{}, enable bool, profile string) error {
+	files := []string{}
 	for _, addon := range addon.Assets {
-		var addonFile assets.CopyableFile
+		var f assets.CopyableFile
+		var err error
 		if addon.IsTemplate() {
-			addonFile, err = addon.Evaluate(data)
+			f, err = addon.Evaluate(data)
 			if err != nil {
 				return errors.Wrapf(err, "evaluate bundled addon %s asset", addon.GetAssetName())
 			}
 
 		} else {
-			addonFile = addon
+			f = addon
 		}
-		if err := updateFile(addonFile); err != nil {
-			return errors.Wrapf(err, "updating addon %s", addon.AssetName)
+		fPath := path.Join(f.GetTargetDir(), f.GetTargetName())
+
+		if enable {
+			glog.Infof("installing %s", fPath)
+			if err := cmd.Copy(f); err != nil {
+				return err
+			}
+		} else {
+			glog.Infof("Removing %+v", fPath)
+			defer func() {
+				if err := cmd.Remove(f); err != nil {
+					glog.Warningf("error removing %s; addon should still be disabled as expected", fPath)
+				}
+			}()
 		}
+		files = append(files, fPath)
+	}
+	command, err := kubectlCommand(profile, files, enable)
+	if err != nil {
+		return err
 	}
+	glog.Infof("Running: %s", command)
+	rr, err := cmd.RunCmd(command)
+	if err != nil {
+		return errors.Wrapf(err, "addon apply")
+	}
+	glog.Infof("output:\n%s", rr.Output())
 	return nil
 }
 
 // enableOrDisableStorageClasses enables or disables storage classes
 func enableOrDisableStorageClasses(name, val, profile string) error {
+	glog.Infof("enableOrDisableStorageClasses %s=%v on %q", name, val, profile)
 	enable, err := strconv.ParseBool(val)
 	if err != nil {
 		return errors.Wrap(err, "Error parsing boolean")
@@ -215,6 +234,17 @@ func enableOrDisableStorageClasses(name, val, profile string) error {
 		return errors.Wrapf(err, "Error getting storagev1 interface %v ", err)
 	}
 
+	api, err := machine.NewAPIClient()
+	if err != nil {
+		return errors.Wrap(err, "machine client")
+	}
+	defer api.Close()
+
+	if !cluster.IsHostRunning(api, profile) {
+		glog.Warningf("%q is not running, writing %s=%v to disk and skipping enablement", profile, name, val)
+		return enableOrDisableAddon(name, val, profile)
+	}
+
 	if enable {
 		// Only StorageClass for 'name' should be marked as default
 		err = storageclass.SetDefaultStorageClass(storagev1, class)
@@ -231,3 +261,48 @@ func enableOrDisableStorageClasses(name, val, profile string) error {
 
 	return enableOrDisableAddon(name, val, profile)
 }
+
+// Start enables the default addons for a profile, plus any additional
+func Start(profile string, toEnable map[string]bool, additional []string) {
+	start := time.Now()
+	glog.Infof("enableAddons start: toEnable=%v, additional=%s", toEnable, additional)
+	defer func() {
+		glog.Infof("enableAddons completed in %s", time.Since(start))
+	}()
+
+	// Get the default values of any addons not saved to our config
+	for name, a := range assets.Addons {
+		defaultVal, err := a.IsEnabled(profile)
+		if err != nil {
+			glog.Errorf("is-enabled failed for %q: %v", a.Name(), err)
+			continue
+		}
+
+		_, exists := toEnable[name]
+		if !exists {
+			toEnable[name] = defaultVal
+		}
+	}
+
+	// Apply new addons
+	for _, name := range additional {
+		toEnable[name] = true
+	}
+
+	toEnableList := []string{}
+	for k, v := range toEnable {
+		if v {
+			toEnableList = append(toEnableList, k)
+		}
+	}
+	sort.Strings(toEnableList)
+
+	out.T(out.AddonEnable, "Enabling addons: {{.addons}}", out.V{"addons": strings.Join(toEnableList, ", ")})
+	for _, a := range toEnableList {
+		err := Set(a, "true", profile)
+		if err != nil {
+			// Intentionally non-fatal
+			out.WarningT("Enabling '{{.name}}' returned an error: {{.error}}", out.V{"name": a, "error": err})
+		}
+	}
+}

pkg/addons/addons_test.go

@@ -17,93 +17,115 @@ limitations under the License.
 package addons
 
 import (
-	"fmt"
+	"io/ioutil"
 	"os"
+	"path/filepath"
 	"testing"
 
-	"gotest.tools/assert"
 	"k8s.io/minikube/pkg/minikube/assets"
 	"k8s.io/minikube/pkg/minikube/config"
+	"k8s.io/minikube/pkg/minikube/localpath"
 )
 
+func createTestProfile(t *testing.T) string {
+	t.Helper()
+	td, err := ioutil.TempDir("", "profile")
+	if err != nil {
+		t.Fatalf("tempdir: %v", err)
+	}
+
+	err = os.Setenv(localpath.MinikubeHome, td)
+	if err != nil {
+		t.Errorf("error setting up test environment. could not set %s", localpath.MinikubeHome)
+	}
+
+	// Not necessary, but it is a handy random alphanumeric
+	name := filepath.Base(td)
+	if err := os.MkdirAll(config.ProfileFolderPath(name), 0777); err != nil {
+		t.Fatalf("error creating temporary directory")
+	}
+	if err := config.DefaultLoader.WriteConfigToFile(name, &config.MachineConfig{}); err != nil {
+		t.Fatalf("error creating temporary profile config: %v", err)
+	}
+	return name
+}
+
 func TestIsAddonAlreadySet(t *testing.T) {
-	testCases := []struct {
-		addonName string
-	}{
-		{
-			addonName: "ingress",
-		},
-
-		{
-			addonName: "registry",
-		},
-	}
-
-	for _, test := range testCases {
-		addon := assets.Addons[test.addonName]
-		addonStatus, _ := addon.IsEnabled()
-
-		alreadySet, err := isAddonAlreadySet(addon, addonStatus)
-		if !alreadySet {
-			if addonStatus {
-				t.Errorf("Did not get expected status, \n\n expected %+v already enabled", test.addonName)
-			} else {
-				t.Errorf("Did not get expected status, \n\n expected %+v already disabled", test.addonName)
-			}
-		}
-		if err != nil {
-			t.Errorf("Got unexpected error: %+v", err)
-		}
+	profile := createTestProfile(t)
+	if err := Set("registry", "true", profile); err != nil {
+		t.Errorf("unable to set registry true: %v", err)
+	}
+	enabled, err := assets.Addons["registry"].IsEnabled(profile)
+	if err != nil {
+		t.Errorf("registry: %v", err)
 	}
+	if !enabled {
+		t.Errorf("expected registry to be enabled")
+	}
+
+	enabled, err = assets.Addons["ingress"].IsEnabled(profile)
+	if err != nil {
+		t.Errorf("ingress: %v", err)
+	}
+	if enabled {
+		t.Errorf("expected ingress to not be enabled")
+	}
+
 }
 
 func TestDisableUnknownAddon(t *testing.T) {
-	tmpProfile := "temp-minikube-profile"
-	if err := Set("InvalidAddon", "false", tmpProfile); err == nil {
+	profile := createTestProfile(t)
+	if err := Set("InvalidAddon", "false", profile); err == nil {
 		t.Fatalf("Disable did not return error for unknown addon")
 	}
 }
 
 func TestEnableUnknownAddon(t *testing.T) {
-	tmpProfile := "temp-minikube-profile"
-	if err := Set("InvalidAddon", "true", tmpProfile); err == nil {
+	profile := createTestProfile(t)
+	if err := Set("InvalidAddon", "true", profile); err == nil {
 		t.Fatalf("Enable did not return error for unknown addon")
 	}
 }
 
 func TestEnableAndDisableAddon(t *testing.T) {
-	tests := []struct {
-		name   string
-		enable bool
-	}{
-		{
-			name:   "test enable",
-			enable: true,
-		}, {
-			name:   "test disable",
-			enable: false,
-		},
-	}
-
-	for _, test := range tests {
-		t.Run(test.name, func(t *testing.T) {
-			tmpProfile := "temp-minikube-profile"
-			if err := os.MkdirAll(config.ProfileFolderPath(tmpProfile), 0777); err != nil {
-				t.Fatalf("error creating temporary directory")
-			}
-			defer os.RemoveAll(config.ProfileFolderPath(tmpProfile))
-
-			if err := config.DefaultLoader.WriteConfigToFile(tmpProfile, &config.MachineConfig{}); err != nil {
-				t.Fatalf("error creating temporary profile config: %v", err)
-			}
-			if err := Set("dashboard", fmt.Sprintf("%t", test.enable), tmpProfile); err != nil {
-				t.Fatalf("Disable returned unexpected error: " + err.Error())
-			}
-			c, err := config.DefaultLoader.LoadConfigFromFile(tmpProfile)
-			if err != nil {
-				t.Fatalf("error loading config: %v", err)
-			}
-			assert.Equal(t, c.Addons["dashboard"], test.enable)
-		})
+	profile := createTestProfile(t)
+
+	// enable
+	if err := Set("dashboard", "true", profile); err != nil {
+		t.Errorf("Disable returned unexpected error: " + err.Error())
+	}
+
+	c, err := config.DefaultLoader.LoadConfigFromFile(profile)
+	if err != nil {
+		t.Errorf("unable to load profile: %v", err)
+	}
+	if c.Addons["dashboard"] != true {
+		t.Errorf("expected dashboard to be enabled")
+	}
+
+	// disable
+	if err := Set("dashboard", "false", profile); err != nil {
+		t.Errorf("Disable returned unexpected error: " + err.Error())
+	}
+
+	c, err = config.DefaultLoader.LoadConfigFromFile(profile)
+	if err != nil {
+		t.Errorf("unable to load profile: %v", err)
+	}
+	if c.Addons["dashboard"] != false {
+		t.Errorf("expected dashboard to be enabled")
+	}
+}
+
+func TestStart(t *testing.T) {
+	profile := createTestProfile(t)
+	Start(profile, map[string]bool{}, []string{"dashboard"})
+
+	enabled, err := assets.Addons["dashboard"].IsEnabled(profile)
+	if err != nil {
+		t.Errorf("dashboard: %v", err)
+	}
+	if !enabled {
+		t.Errorf("expected dashboard to be enabled")
 	}
 }

pkg/addons/config.go

@@ -30,11 +30,6 @@ type Addon struct {
 
 // Addons is a list of all addons
 var Addons = []*Addon{
-	{
-		name:      "addon-manager",
-		set:       SetBool,
-		callbacks: []setFn{enableOrDisableAddon},
-	},
 	{
 		name:      "dashboard",
 		set:       SetBool,

pkg/addons/kubectl.go

+/*
+Copyright 2016 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package addons
+
+import (
+	"os/exec"
+	"path"
+
+	"k8s.io/minikube/pkg/minikube/config"
+	"k8s.io/minikube/pkg/minikube/constants"
+)
+
+var (
+	// For testing
+	k8sVersion = kubernetesVersion
+)
+
+func kubectlCommand(profile string, files []string, enable bool) (*exec.Cmd, error) {
+	v, err := k8sVersion(profile)
+	if err != nil {
+		return nil, err
+	}
+	kubectlBinary := kubectlBinaryPath(v)
+
+	kubectlAction := "apply"
+	if !enable {
+		kubectlAction = "delete"
+	}
+
+	args := []string{"KUBECONFIG=/var/lib/minikube/kubeconfig", kubectlBinary, kubectlAction}
+	for _, f := range files {
+		args = append(args, []string{"-f", f}...)
+	}
+
+	cmd := exec.Command("sudo", args...)
+	return cmd, nil
+}
+
+func kubernetesVersion(profile string) (string, error) {
+	cc, err := config.Load(profile)
+	if err != nil && !config.IsNotExist(err) {
+		return "", err
+	}
+	version := constants.DefaultKubernetesVersion
+	if cc != nil {
+		version = cc.KubernetesConfig.KubernetesVersion
+	}
+	return version, nil
+}
+
+func kubectlBinaryPath(version string) string {
+	return path.Join("/var/lib/minikube/binaries", version, "kubectl")
+}

pkg/addons/kubectl_test.go

+/*
+Copyright 2016 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package addons
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestKubectlCommand(t *testing.T) {
+	tests := []struct {
+		description string
+		files       []string
+		enable      bool
+		expected    string
+	}{
+		{
+			description: "enable an addon",
+			files:       []string{"a", "b"},
+			enable:      true,
+			expected:    "sudo KUBECONFIG=/var/lib/minikube/kubeconfig /var/lib/minikube/binaries/v1.17.0/kubectl apply -f a -f b",
+		}, {
+			description: "disable an addon",
+			files:       []string{"a", "b"},
+			enable:      false,
+			expected:    "sudo KUBECONFIG=/var/lib/minikube/kubeconfig /var/lib/minikube/binaries/v1.17.0/kubectl delete -f a -f b",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.description, func(t *testing.T) {
+			originalK8sVersion := k8sVersion
+			defer func() { k8sVersion = originalK8sVersion }()
+			k8sVersion = func(_ string) (string, error) {
+				return "v1.17.0", nil
+			}
+
+			command, err := kubectlCommand("", test.files, test.enable)
+			if err != nil {
+				t.Fatalf("error getting kubectl command: %v", err)
+			}
+			actual := strings.Join(command.Args, " ")
+
+			if actual != test.expected {
+				t.Fatalf("expected does not match actual\nExpected: %s\nActual: %s", test.expected, actual)
+			}
+		})
+	}
+}

pkg/drivers/kic/kic.go

@@ -18,29 +18,24 @@ package kic
 
 import (
 	"fmt"
+	"net"
 	"os/exec"
 	"strconv"
 	"strings"
 
 	"github.com/docker/machine/libmachine/drivers"
+	"github.com/docker/machine/libmachine/log"
+	"github.com/docker/machine/libmachine/ssh"
 	"github.com/docker/machine/libmachine/state"
+	"github.com/golang/glog"
 	"github.com/pkg/errors"
 	pkgdrivers "k8s.io/minikube/pkg/drivers"
-	"k8s.io/minikube/pkg/drivers/kic/node"
 	"k8s.io/minikube/pkg/drivers/kic/oci"
+	"k8s.io/minikube/pkg/minikube/assets"
 	"k8s.io/minikube/pkg/minikube/command"
 	"k8s.io/minikube/pkg/minikube/constants"
 )
 
-// DefaultPodCIDR is The CIDR to be used for pods inside the node.
-const DefaultPodCIDR = "10.244.0.0/16"
-
-// DefaultBindIPV4 is The default IP the container will bind to.
-const DefaultBindIPV4 = "127.0.0.1"
-
-// BaseImage is the base image is used to spin up kic containers
-const BaseImage = "gcr.io/k8s-minikube/kicbase:v0.0.1@sha256:c4ad2938877d2ae0d5b7248a5e7182ff58c0603165c3bedfe9d503e2d380a0db"
-
 // Driver represents a kic driver https://minikube.sigs.k8s.io/docs/reference/drivers/kic/
 type Driver struct {
 	*drivers.BaseDriver
@@ -51,20 +46,6 @@ type Driver struct {
 	OCIBinary  string // docker,podman
 }
 
-// Config is configuration for the kic driver used by registry
-type Config struct {
-	MachineName  string            // maps to the container name being created
-	CPU          int               // Number of CPU cores assigned to the container
-	Memory       int               // max memory in MB
-	StorePath    string            // libmachine store path
-	OCIBinary    string            // oci tool to use (docker, podman,...)
-	ImageDigest  string            // image name with sha to use for the node
-	HostBindPort int               // port to connect to forward from container to user's machine
-	Mounts       []oci.Mount       // mounts
-	PortMappings []oci.PortMapping // container port mappings
-	Envs         map[string]string // key,value of environment variables passed to the node
-}
-
 // NewDriver returns a fully configured Kic driver
 func NewDriver(c Config) *Driver {
 	d := &Driver{
@@ -81,28 +62,63 @@ func NewDriver(c Config) *Driver {
 
 // Create a host using the driver's config
 func (d *Driver) Create() error {
-	params := node.CreateConfig{
-		Name:         d.NodeConfig.MachineName,
-		Image:        d.NodeConfig.ImageDigest,
-		ClusterLabel: node.ClusterLabelKey + "=" + d.MachineName,
-		CPUs:         strconv.Itoa(d.NodeConfig.CPU),
-		Memory:       strconv.Itoa(d.NodeConfig.Memory) + "mb",
-		Envs:         d.NodeConfig.Envs,
-		ExtraArgs:    []string{"--expose", fmt.Sprintf("%d", d.NodeConfig.HostBindPort)},
-		OCIBinary:    d.NodeConfig.OCIBinary,
+	params := oci.CreateParams{
+		Name:          d.NodeConfig.MachineName,
+		Image:         d.NodeConfig.ImageDigest,
+		ClusterLabel:  oci.ClusterLabelKey + "=" + d.MachineName,
+		CPUs:          strconv.Itoa(d.NodeConfig.CPU),
+		Memory:        strconv.Itoa(d.NodeConfig.Memory) + "mb",
+		Envs:          d.NodeConfig.Envs,
+		ExtraArgs:     []string{"--expose", fmt.Sprintf("%d", d.NodeConfig.APIServerPort)},
+		OCIBinary:     d.NodeConfig.OCIBinary,
+		APIServerPort: d.NodeConfig.APIServerPort,
 	}
 
 	// control plane specific options
 	params.PortMappings = append(params.PortMappings, oci.PortMapping{
-		ListenAddress: "127.0.0.1",
-		HostPort:      int32(d.NodeConfig.HostBindPort),
+		ListenAddress: DefaultBindIPV4,
 		ContainerPort: constants.APIServerPort,
-	})
-
-	_, err := node.CreateNode(params)
+	},
+		oci.PortMapping{
+			ListenAddress: DefaultBindIPV4,
+			ContainerPort: constants.SSHPort,
+		},
+		oci.PortMapping{
+			ListenAddress: DefaultBindIPV4,
+			ContainerPort: constants.DockerDaemonPort,
+		},
+	)
+	err := oci.CreateContainerNode(params)
 	if err != nil {
 		return errors.Wrap(err, "create kic node")
 	}
+
+	if err := d.prepareSSH(); err != nil {
+		return errors.Wrap(err, "prepare kic ssh")
+	}
+	return nil
+}
+
+// prepareSSH will generate keys and copy to the container so minikube ssh works
+func (d *Driver) prepareSSH() error {
+	keyPath := d.GetSSHKeyPath()
+	glog.Infof("Creating ssh key for kic: %s...", keyPath)
+	if err := ssh.GenerateSSHKey(keyPath); err != nil {
+		return errors.Wrap(err, "generate ssh key")
+	}
+
+	cmder := command.NewKICRunner(d.NodeConfig.MachineName, d.NodeConfig.OCIBinary)
+	f, err := assets.NewFileAsset(d.GetSSHKeyPath()+".pub", "/home/docker/.ssh/", "authorized_keys", "0644")
+	if err != nil {
+		return errors.Wrap(err, "create pubkey assetfile ")
+	}
+	if err := cmder.Copy(f); err != nil {
+		return errors.Wrap(err, "copying pub key")
+	}
+	if rr, err := cmder.RunCmd(exec.Command("chown", "docker:docker", "/home/docker/.ssh/authorized_keys")); err != nil {
+		return errors.Wrapf(err, "apply authorized_keys file ownership, output %s", rr.Output())
+	}
+
 	return nil
 }
 
@@ -116,36 +132,63 @@ func (d *Driver) DriverName() string {
 
 // GetIP returns an IP or hostname that this host is available at
 func (d *Driver) GetIP() (string, error) {
-	node, err := node.Find(d.OCIBinary, d.MachineName, d.exec)
-	if err != nil {
-		return "", fmt.Errorf("ip not found for nil node")
-	}
-	ip, _, err := node.IP()
+	ip, _, err := oci.ContainerIPs(d.OCIBinary, d.MachineName)
 	return ip, err
 }
 
+// GetExternalIP returns an IP which is accissble from outside
+func (d *Driver) GetExternalIP() (string, error) {
+	return DefaultBindIPV4, nil
+}
+
 // GetSSHHostname returns hostname for use with ssh
 func (d *Driver) GetSSHHostname() (string, error) {
-	return "", fmt.Errorf("driver does not have SSHHostName")
+	return DefaultBindIPV4, nil
 }
 
 // GetSSHPort returns port for use with ssh
 func (d *Driver) GetSSHPort() (int, error) {
-	return 0, fmt.Errorf("driver does not support GetSSHPort")
+	p, err := oci.HostPortBinding(d.OCIBinary, d.MachineName, constants.SSHPort)
+	if err != nil {
+		return p, errors.Wrap(err, "get ssh host-port")
+	}
+	return p, nil
+}
+
+// GetSSHUsername returns the ssh username
+func (d *Driver) GetSSHUsername() string {
+	return "docker"
+}
+
+// GetSSHKeyPath returns the ssh key path
+func (d *Driver) GetSSHKeyPath() string {
+	if d.SSHKeyPath == "" {
+		d.SSHKeyPath = d.ResolveStorePath("id_rsa")
+	}
+	return d.SSHKeyPath
 }
 
 // GetURL returns ip of the container running kic control-panel
 func (d *Driver) GetURL() (string, error) {
-	return d.GetIP()
+	p, err := oci.HostPortBinding(d.NodeConfig.OCIBinary, d.MachineName, d.NodeConfig.APIServerPort)
+	url := fmt.Sprintf("https://%s", net.JoinHostPort("127.0.0.1", fmt.Sprint(p)))
+	if err != nil {
+		return url, errors.Wrap(err, "api host port binding")
+	}
+	return url, nil
 }
 
 // GetState returns the state that the host is in (running, stopped, etc)
 func (d *Driver) GetState() (state.State, error) {
+	if err := oci.PointToHostDockerDaemon(); err != nil {
+		return state.Error, errors.Wrap(err, "point host docker-daemon")
+	}
+
 	cmd := exec.Command(d.NodeConfig.OCIBinary, "inspect", "-f", "{{.State.Status}}", d.MachineName)
 	out, err := cmd.CombinedOutput()
 	o := strings.Trim(string(out), "\n")
 	if err != nil {
-		return state.Error, errors.Wrapf(err, "error stop node %s", d.MachineName)
+		return state.Error, errors.Wrapf(err, "get container %s status", d.MachineName)
 	}
 	switch o {
 	case "running":
@@ -174,12 +217,17 @@ func (d *Driver) Kill() error {
 
 // Remove will delete the Kic Node Container
 func (d *Driver) Remove() error {
-	if _, err := d.nodeID(d.MachineName); err != nil {
-		return errors.Wrapf(err, "not found node %s", d.MachineName)
+	if _, err := oci.ContainerID(d.OCIBinary, d.MachineName); err != nil {
+		log.Warnf("could not find the container %s to remove it.", d.MachineName)
 	}
 	cmd := exec.Command(d.NodeConfig.OCIBinary, "rm", "-f", "-v", d.MachineName)
-	if err := cmd.Run(); err != nil {
-		return errors.Wrapf(err, "error removing node %s", d.MachineName)
+	o, err := cmd.CombinedOutput()
+	out := strings.Trim(string(o), "\n")
+	if err != nil {
+		if strings.Contains(out, "is already in progress") {
+			log.Warnf("Docker engine is stuck. please restart docker daemon on your computer.", d.MachineName)
+		}
+		return errors.Wrapf(err, "removing container %s, output %s", d.MachineName, out)
 	}
 	return nil
 }
@@ -248,13 +296,3 @@ func (d *Driver) Stop() error {
 func (d *Driver) RunSSHCommandFromDriver() error {
 	return fmt.Errorf("driver does not support RunSSHCommandFromDriver commands")
 }
-
-// looks up for a container node by name, will return error if not found.
-func (d *Driver) nodeID(nameOrID string) (string, error) {
-	cmd := exec.Command(d.NodeConfig.OCIBinary, "inspect", "-f", "{{.Id}}", nameOrID)
-	id, err := cmd.CombinedOutput()
-	if err != nil {
-		id = []byte{}
-	}
-	return string(id), err
-}

pkg/drivers/kic/oci/types.go

@@ -19,8 +19,38 @@ package oci
 const (
 	Docker = "docker"
 	Podman = "podman"
+	// ClusterLabelKey is applied to each node docker container for identification
+	ClusterLabelKey = "io.x-k8s.kic.cluster"
+	// NodeRoleKey is used to identify if it is control plane or worker
+	nodeRoleKey = "io.k8s.sigs.kic.role"
 )
 
+type CreateParams struct {
+	Name          string            // used for container name and hostname
+	Image         string            // container image to use to create the node.
+	ClusterLabel  string            // label the containers we create using minikube so we can clean up
+	Role          string            // currently only role supported is control-plane
+	Mounts        []Mount           // volume mounts
+	APIServerPort int               // kubernetes api server port
+	PortMappings  []PortMapping     // ports to map to container from host
+	CPUs          string            // number of cpu cores assign to container
+	Memory        string            // memory (mbs) to assign to the container
+	Envs          map[string]string // environment variables to pass to the container
+	ExtraArgs     []string          // a list of any extra option to pass to oci binary during creation time, for example --expose 8080...
+	OCIBinary     string            // docker or podman
+}
+
+// createOpt is an option for Create
+type createOpt func(*createOpts) *createOpts
+
+// actual options struct
+type createOpts struct {
+	RunArgs       []string
+	ContainerArgs []string
+	Mounts        []Mount
+	PortMappings  []PortMapping
+}
+
 /*
 These types are from
 https://github.com/kubernetes/kubernetes/blob/063e7ff358fdc8b0916e6f39beedc0d025734cb1/pkg/kubelet/apis/cri/runtime/v1alpha2/api.pb.go#L183

pkg/drivers/kic/types.go

+/*
+Copyright 2019 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kic
+
+import "k8s.io/minikube/pkg/drivers/kic/oci"
+
+const (
+	// Docker default bridge network is named "bridge" (https://docs.docker.com/network/bridge/#use-the-default-bridge-network)
+	DefaultNetwork = "bridge"
+	// DefaultPodCIDR is The CIDR to be used for pods inside the node.
+	DefaultPodCIDR = "10.244.0.0/16"
+	// DefaultBindIPV4 is The default IP the container will bind to.
+	DefaultBindIPV4 = "127.0.0.1"
+	// BaseImage is the base image is used to spin up kic containers. it uses same base-image as kind.
+	BaseImage = "gcr.io/k8s-minikube/kicbase:v0.0.5@sha256:3ddd8461dfb5c3e452ccc44d87750b87a574ec23fc425da67dccc1f0c57d428a"
+	// OverlayImage is the cni plugin used for overlay image, created by kind.
+	// CNI plugin image used for kic drivers created by kind.
+	OverlayImage = "kindest/kindnetd:0.5.3"
+)
+
+// Config is configuration for the kic driver used by registry
+type Config struct {
+	MachineName   string            // maps to the container name being created
+	CPU           int               // Number of CPU cores assigned to the container
+	Memory        int               // max memory in MB
+	StorePath     string            // libmachine store path
+	OCIBinary     string            // oci tool to use (docker, podman,...)
+	ImageDigest   string            // image name with sha to use for the node
+	Mounts        []oci.Mount       // mounts
+	APIServerPort int               // kubernetes api server port inside the container
+	PortMappings  []oci.PortMapping // container port mappings
+	Envs          map[string]string // key,value of environment variables passed to the node
+}

pkg/minikube/bootstrapper/bootstrapper.go

@@ -37,13 +37,13 @@ type LogOptions struct {
 type Bootstrapper interface {
 	// PullImages pulls images necessary for a cluster. Success should not be required.
 	PullImages(config.KubernetesConfig) error
-	StartCluster(config.KubernetesConfig) error
+	StartCluster(config.MachineConfig) error
 	UpdateCluster(config.MachineConfig) error
 	DeleteCluster(config.KubernetesConfig) error
-	WaitForCluster(config.KubernetesConfig, time.Duration) error
+	WaitForCluster(config.MachineConfig, time.Duration) error
 	// LogCommands returns a map of log type to a command which will display that log.
 	LogCommands(LogOptions) map[string]string
-	SetupCerts(cfg config.KubernetesConfig) error
+	SetupCerts(config.KubernetesConfig, config.Node) error
 	GetKubeletStatus() (string, error)
 	GetAPIServerStatus(net.IP, int) (string, error)
 }
@@ -51,12 +51,11 @@ type Bootstrapper interface {
 const (
 	// Kubeadm is the kubeadm bootstrapper type
 	Kubeadm = "kubeadm"
-	KIC     = "kic"
 )
 
 // GetCachedBinaryList returns the list of binaries
 func GetCachedBinaryList(bootstrapper string) []string {
-	return constants.KubeadmBinaries
+	return constants.KubernetesReleaseBinaries
 }
 
 // GetCachedImageList returns the list of images for a version