first version

cmd/minikube/cmd/delete.go

@@ -260,6 +260,11 @@ func deletePossibleKicLeftOver(cname string, driverName string) {
 		glog.Warningf("error deleting volumes (might be okay).\nTo see the list of volumes run: 'docker volume ls'\n:%v", errs)
 	}
 
+	errs = oci.DeleteAllNetworksByKIC()
+	if errs != nil {
+		glog.Warningf("error deleting left over networks (might be okay).\nTo see the list of netowrks: 'docker network ls'\n:%v", errs)
+	}
+
 	if bin == oci.Podman {
 		// podman prune does not support --filter
 		return

cmd/minikube/cmd/mount.go

@@ -110,7 +110,7 @@ var mountCmd = &cobra.Command{
 		var ip net.IP
 		var err error
 		if mountIP == "" {
-			ip, err = cluster.HostIP(co.CP.Host)
+			ip, err = cluster.HostIP(co.CP.Host, co.Config.Name)
 			if err != nil {
 				exit.Error(reason.IfHostIP, "Error getting the host IP address to use from within the VM", err)
 			}

hack/preload-images/generate.go

@@ -39,6 +39,7 @@ import (
 
 func generateTarball(kubernetesVersion, containerRuntime, tarballFilename string) error {
 	driver := kic.NewDriver(kic.Config{
+		ClusterName:       profile,
 		KubernetesVersion: kubernetesVersion,
 		ContainerRuntime:  containerRuntime,
 		OCIBinary:         oci.Docker,

pkg/drivers/kic/kic.go

@@ -37,6 +37,7 @@ import (
 	"k8s.io/minikube/pkg/minikube/constants"
 	"k8s.io/minikube/pkg/minikube/cruntime"
 	"k8s.io/minikube/pkg/minikube/download"
+	"k8s.io/minikube/pkg/minikube/out"
 	"k8s.io/minikube/pkg/minikube/sysinit"
 	"k8s.io/minikube/pkg/util/retry"
 )
@@ -65,6 +66,18 @@ func NewDriver(c Config) *Driver {
 	return d
 }
 
+// machineOrder returns the order of the container based on it is name
+func machineOrder(machineName string) int {
+	// minikube-m02
+	sp := strings.Split(machineName, "-")
+	m := strings.Trim(sp[len(sp)-1], "m") // m02
+	i, err := strconv.Atoi(m)
+	if err != nil {
+		return 1
+	}
+	return i
+}
+
 // Create a host using the driver's config
 func (d *Driver) Create() error {
 	params := oci.CreateParams{
@@ -81,6 +94,20 @@ func (d *Driver) Create() error {
 		APIServerPort: d.NodeConfig.APIServerPort,
 	}
 
+	// one network bridge per cluster.
+	defaultNetwork := d.NodeConfig.ClusterName
+	if gateway, err := oci.CreateNetwork(d.OCIBinary, defaultNetwork); err != nil {
+		glog.Warningf("failed to create network: %v", err)
+		out.WarningT("Unable to create dedicated network, This might result in cluster IP change after restart.")
+	} else {
+		params.Network = defaultNetwork
+		ip := gateway.To4()
+		// calculate the container IP based on its machine order
+		ip[3] += byte(machineOrder(d.NodeConfig.MachineName))
+		glog.Infof("calculated static IP %q for the %q container ", ip.String(), d.NodeConfig.MachineName)
+		params.IP = ip.String()
+	}
+
 	// control plane specific options
 	params.PortMappings = append(params.PortMappings, oci.PortMapping{
 		ListenAddress: oci.DefaultBindIPV4,
@@ -289,6 +316,11 @@ func (d *Driver) Remove() error {
 	if id, err := oci.ContainerID(d.OCIBinary, d.MachineName); err == nil && id != "" {
 		return fmt.Errorf("expected no container ID be found for %q after delete. but got %q", d.MachineName, id)
 	}
+
+	if err := oci.RemoveNetwork(d.NodeConfig.ClusterName); err != nil {
+		//TODO: Ingore error if this is a multinode cluster and first container is trying to delete network while other containers are attached to it
+		glog.Warningf("failed to remove network (which might be okay) %s: %v", d.NodeConfig.ClusterName, err)
+	}
 	return nil
 }
 

pkg/drivers/kic/kic_test.go

+/*
+Copyright 2019 The Kubernetes Authors All rights reserved.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kic
+
+import (
+	"testing"
+)
+
+func TestMachineOrder(t *testing.T) {
+	testCases := []struct {
+		Name        string
+		MachineName string
+		Want        int
+	}{
+		{
+			Name:        "default",
+			MachineName: "minikube",
+			Want:        1},
+		{
+			Name:        "second-node",
+			MachineName: "minikube-m02",
+			Want:        2},
+		{
+			Name:        "dash-profile",
+			MachineName: "my-dashy-minikube",
+			Want:        1},
+
+		{
+			Name:        "dash-profile-second-node",
+			MachineName: "my-dashy-minikube-m02",
+			Want:        2},
+		{
+			Name:        "michivious-user",
+			MachineName: "michivious-user-m02-m03",
+			Want:        3},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.Name, func(t *testing.T) {
+			got := machineOrder(tc.MachineName)
+			if got != tc.Want {
+				t.Errorf("want order %q but got %q", tc.Want, got)
+
+			}
+		})
+
+	}
+}

pkg/drivers/kic/oci/network.go

@@ -31,17 +31,21 @@ import (
 
 // RoutableHostIPFromInside returns the ip/dns of the host that container lives on
 // is routable from inside the container
-func RoutableHostIPFromInside(ociBin string, containerName string) (net.IP, error) {
+func RoutableHostIPFromInside(ociBin string, clusterName string, containerName string) (net.IP, error) {
 	if ociBin == Docker {
 		if runtime.GOOS == "linux" {
-			return dockerGatewayIP(containerName)
+			_, gateway, err := dockerNetworkInspect(clusterName)
+			if err != nil {
+				return gateway, errors.Wrap(err, "network inspect")
+			}
+			return gateway, nil
 		}
 		// for windows and mac, the gateway ip is not routable so we use dns trick.
 		return digDNS(ociBin, containerName, "host.docker.internal")
 	}
 
 	if runtime.GOOS == "linux" {
-		return containerGatewayIP(ociBin, containerName)
+		return podmanGatewayIP(containerName)
 	}
 
 	return nil, fmt.Errorf("RoutableHostIPFromInside is currently only implemented for linux")
@@ -59,57 +63,9 @@ func digDNS(ociBin, containerName, dns string) (net.IP, error) {
 	return ip, nil
 }
 
-// profileInContainers checks whether the profile is within the containers list
-func profileInContainers(profile string, containers []string) bool {
-	for _, container := range containers {
-		if container == profile {
-			return true
-		}
-	}
-	return false
-}
-
-// dockerGatewayIP gets the default gateway ip for the docker bridge on the user's host machine
-// gets the ip from user's host docker
-func dockerGatewayIP(profile string) (net.IP, error) {
-	var bridgeID string
-	rr, err := runCmd(exec.Command(Docker, "network", "ls", "--filter", "name=bridge", "--format", "{{.ID}}"))
-	if err != nil {
-		return nil, errors.Wrapf(err, "get network bridge")
-	}
-	networksOutput := strings.TrimSpace(rr.Stdout.String())
-	networksSlice := strings.Fields(networksOutput)
-	// Look for the minikube container within each docker network
-	for _, net := range networksSlice {
-		// get all containers in the network
-		rs, err := runCmd(exec.Command(Docker, "network", "inspect", net, "-f", "{{range $k, $v := .Containers}}{{$v.Name}} {{end}}"))
-		if err != nil {
-			return nil, errors.Wrapf(err, "get containers in network")
-		}
-		containersSlice := strings.Fields(rs.Stdout.String())
-		if profileInContainers(profile, containersSlice) {
-			bridgeID = net
-			break
-		}
-	}
-
-	if bridgeID == "" {
-		return nil, errors.Errorf("unable to determine bridge network id from %q", networksOutput)
-	}
-	rr, err = runCmd(exec.Command(Docker, "network", "inspect",
-		"--format", "{{(index .IPAM.Config 0).Gateway}}", bridgeID))
-	if err != nil {
-		return nil, errors.Wrapf(err, "inspect IP bridge network %q.", bridgeID)
-	}
-
-	ip := net.ParseIP(strings.TrimSpace(rr.Stdout.String()))
-	glog.Infof("got host ip for mount in container by inspect docker network: %s", ip.String())
-	return ip, nil
-}
-
-// containerGatewayIP gets the default gateway ip for the container
-func containerGatewayIP(ociBin, containerName string) (net.IP, error) {
-	rr, err := runCmd(exec.Command(ociBin, "container", "inspect", "--format", "{{.NetworkSettings.Gateway}}", containerName))
+// podmanGatewayIP gets the default gateway ip for the container
+func podmanGatewayIP(containerName string) (net.IP, error) {
+	rr, err := runCmd(exec.Command(Podman, "container", "inspect", "--format", "{{.NetworkSettings.Gateway}}", containerName))
 	if err != nil {
 		return nil, errors.Wrapf(err, "inspect gateway")
 	}

pkg/drivers/kic/oci/network_create.go

@@ -17,6 +17,8 @@ limitations under the License.
 package oci
 
 import (
+	"bufio"
+	"bytes"
 	"fmt"
 	"net"
 	"os/exec"
@@ -123,3 +125,72 @@ func dockerNetworkInspect(name string) (*net.IPNet, net.IP, error) {
 	}
 	return subnet, gateway, nil
 }
+
+// RemoveNetwork removes a network
+func RemoveNetwork(name string) error {
+	if !networkExists(name) {
+		return nil
+	}
+	rr, err := runCmd(exec.Command(Docker, "network", "remove", name))
+	if err != nil {
+		if strings.Contains(rr.Output(), "No such network:") {
+			return ErrNetworkNotFound
+		}
+		// Error response from daemon: error while removing network: network mynet123 id f9e1c50b89feb0b8f4b687f3501a81b618252c9907bc20666e386d0928322387 has active endpoints
+		if strings.Contains(rr.Output(), "has active endpoints") {
+			return ErrNetworkInUse
+		}
+	}
+
+	return err
+}
+
+func networkExists(name string) bool {
+	if _, _, err := dockerNetworkInspect(name); err != nil {
+		if err == ErrNetworkNotFound {
+			return false
+		}
+		glog.Warningf("error inspecting network %s: %v", name, err)
+		return false
+	}
+	return true
+}
+
+// returns all network names created by a label
+func allNetworkByLabel(ociBin string, label string) ([]string, error) {
+	if ociBin != Docker {
+		return nil, fmt.Errorf("%s not supported", ociBin)
+	}
+
+	// docker network ls --filter='label=created_by.minikube.sigs.k8s.io=true' --format '{{.Name}}
+	rr, err := runCmd(exec.Command(Docker, "network", "ls", fmt.Sprintf("--filter=label=%s", label), "--format", "{{.Name}}"))
+	if err != nil {
+		return nil, err
+	}
+	var lines []string
+	scanner := bufio.NewScanner(bytes.NewReader(rr.Stdout.Bytes()))
+	for scanner.Scan() {
+		lines = append(lines, strings.TrimSpace(scanner.Text()))
+	}
+
+	return lines, nil
+}
+
+// DeleteAllNetworksByKIC deletes all networks created by kic
+func DeleteAllNetworksByKIC() []error {
+	var errs []error
+	ns, err := allNetworkByLabel(Docker, CreatedByLabelKey+"=true")
+	if err != nil {
+		return []error{errors.Wrap(err, "list all volume")}
+	}
+	for _, n := range ns {
+		err := RemoveNetwork(n)
+		if err != nil {
+			errs = append(errs, err)
+		}
+	}
+	if len(errs) > 0 {
+		return errs
+	}
+	return nil
+}

pkg/drivers/kic/oci/oci.go

@@ -169,6 +169,11 @@ func CreateContainerNode(p CreateParams) error {
 		virtualization = "podman" // VIRTUALIZATION_PODMAN
 	}
 	if p.OCIBinary == Docker {
+		// to provide a static IP for docker
+		if p.Network != "" && p.IP != "" {
+			runArgs = append(runArgs, "--network", p.Network)
+			runArgs = append(runArgs, "--ip", p.IP)
+		}
 		runArgs = append(runArgs, "--volume", fmt.Sprintf("%s:/var", p.Name))
 		// ignore apparmore github actions docker: https://github.com/kubernetes/minikube/issues/7624
 		runArgs = append(runArgs, "--security-opt", "apparmor=unconfined")
@@ -285,6 +290,10 @@ func createContainer(ociBin string, image string, opts ...createOpt) error {
 		if strings.Contains(rr.Output(), "Range of CPUs is from") && strings.Contains(rr.Output(), "CPUs available") { // CPUs available
 			return ErrCPUCountLimit
 		}
+		// example: docker: Error response from daemon: Address already in use.
+		if strings.Contains(rr.Output(), "Address already in use") {
+			return ErrIPinUse
+		}
 		return err
 	}
 

pkg/drivers/kic/oci/types.go

@@ -43,6 +43,7 @@ const (
 
 // CreateParams are parameters needed to create a container
 type CreateParams struct {
+	ClusterName   string            // cluster(profile name) that this container belongs to
 	Name          string            // used for container name and hostname
 	Image         string            // container image to use to create the node.
 	ClusterLabel  string            // label the clusters we create using minikube so we can clean up
@@ -56,6 +57,8 @@ type CreateParams struct {
 	Envs          map[string]string // environment variables to pass to the container
 	ExtraArgs     []string          // a list of any extra option to pass to oci binary during creation time, for example --expose 8080...
 	OCIBinary     string            // docker or podman
+	Network       string            // network to use for the containe
+	IP            string            // IP to assign for th container in the work
 }
 
 // createOpt is an option for Create

pkg/drivers/kic/types.go

@@ -48,6 +48,7 @@ var (
 
 // Config is configuration for the kic driver used by registry
 type Config struct {
+	ClusterName       string            // The cluster the container belongs to
 	MachineName       string            // maps to the container name being created
 	CPU               int               // Number of CPU cores assigned to the container
 	Memory            int               // max memory in MB

pkg/minikube/cluster/ip.go

@@ -34,12 +34,12 @@ import (
 )
 
 // HostIP gets the ip address to be used for mapping host -> VM and VM -> host
-func HostIP(host *host.Host) (net.IP, error) {
+func HostIP(host *host.Host, clusterName string) (net.IP, error) {
 	switch host.DriverName {
 	case driver.Docker:
-		return oci.RoutableHostIPFromInside(oci.Docker, host.Name)
+		return oci.RoutableHostIPFromInside(oci.Docker, clusterName, host.Name)
 	case driver.Podman:
-		return oci.RoutableHostIPFromInside(oci.Podman, host.Name)
+		return oci.RoutableHostIPFromInside(oci.Podman, clusterName, host.Name)
 	case driver.KVM2:
 		return net.ParseIP("192.168.39.1"), nil
 	case driver.HyperV:

pkg/minikube/node/start.go

@@ -94,7 +94,7 @@ func Start(starter Starter, apiServer bool) (*kubeconfig.Settings, error) {
 	showVersionInfo(starter.Node.KubernetesVersion, cr)
 
 	// Add "host.minikube.internal" DNS alias (intentionally non-fatal)
-	hostIP, err := cluster.HostIP(starter.Host)
+	hostIP, err := cluster.HostIP(starter.Host, starter.Cfg.Name)
 	if err != nil {
 		glog.Errorf("Unable to get host IP: %v", err)
 	} else if err := machine.AddHostAlias(starter.Runner, constants.HostAlias, hostIP); err != nil {

pkg/minikube/registry/drvs/docker/docker.go

@@ -61,6 +61,7 @@ func configure(cc config.ClusterConfig, n config.Node) (interface{}, error) {
 	}
 
 	return kic.NewDriver(kic.Config{
+		ClusterName:       cc.Name,
 		MachineName:       driver.MachineName(cc, n),
 		StorePath:         localpath.MiniPath(),
 		ImageDigest:       cc.KicBaseImage,

pkg/minikube/registry/drvs/podman/podman.go

@@ -73,6 +73,7 @@ func configure(cc config.ClusterConfig, n config.Node) (interface{}, error) {
 	}
 
 	return kic.NewDriver(kic.Config{
+		ClusterName:       cc.Name,
 		MachineName:       driver.MachineName(cc, n),
 		StorePath:         localpath.MiniPath(),
 		ImageDigest:       strings.Split(cc.KicBaseImage, "@")[0], // for podman does not support docker images references with both a tag and digest.