diff --git a/cmd/minikube/cmd/delete.go b/cmd/minikube/cmd/delete.go index 6b50e46316201fa0649a2776cc7c1471abedf63d..e819fec0f36409c1884ef7adf86b22c5e090ba87 100644 --- a/cmd/minikube/cmd/delete.go +++ b/cmd/minikube/cmd/delete.go @@ -260,6 +260,11 @@ func deletePossibleKicLeftOver(cname string, driverName string) { glog.Warningf("error deleting volumes (might be okay).\nTo see the list of volumes run: 'docker volume ls'\n:%v", errs) } + errs = oci.DeleteKICNetworks() + if errs != nil { + glog.Warningf("error deleting leftover networks (might be okay).\nTo see the list of networks: 'docker network ls'\n:%v", errs) + } + if bin == oci.Podman { // podman prune does not support --filter return diff --git a/cmd/minikube/cmd/mount.go b/cmd/minikube/cmd/mount.go index 8d268d1cc28f85881d074c40bd0ea55a3b3a96d5..f59b3ca4a7816cedf7f8faa4afd82d2754ebc108 100644 --- a/cmd/minikube/cmd/mount.go +++ b/cmd/minikube/cmd/mount.go @@ -110,7 +110,7 @@ var mountCmd = &cobra.Command{ var ip net.IP var err error if mountIP == "" { - ip, err = cluster.HostIP(co.CP.Host) + ip, err = cluster.HostIP(co.CP.Host, co.Config.Name) if err != nil { exit.Error(reason.IfHostIP, "Error getting the host IP address to use from within the VM", err) } diff --git a/hack/preload-images/generate.go b/hack/preload-images/generate.go index ec3de2ece5066fd12de0c7c20b9627aa7d809c84..e1d6061d8bd641b592c77da640420f157cf31122 100644 --- a/hack/preload-images/generate.go +++ b/hack/preload-images/generate.go @@ -39,6 +39,7 @@ import ( func generateTarball(kubernetesVersion, containerRuntime, tarballFilename string) error { driver := kic.NewDriver(kic.Config{ + ClusterName: profile, KubernetesVersion: kubernetesVersion, ContainerRuntime: containerRuntime, OCIBinary: oci.Docker, diff --git a/pkg/drivers/kic/kic.go b/pkg/drivers/kic/kic.go index e296b327b83fba51868ca70569536235810721b9..120d64a9b646b098ff4bcdf3267ead15869c4512 100644 --- a/pkg/drivers/kic/kic.go +++ b/pkg/drivers/kic/kic.go @@ -37,6 +37,8 @@ import ( "k8s.io/minikube/pkg/minikube/constants" "k8s.io/minikube/pkg/minikube/cruntime" "k8s.io/minikube/pkg/minikube/download" + "k8s.io/minikube/pkg/minikube/driver" + "k8s.io/minikube/pkg/minikube/out" "k8s.io/minikube/pkg/minikube/sysinit" "k8s.io/minikube/pkg/util/retry" ) @@ -81,6 +83,17 @@ func (d *Driver) Create() error { APIServerPort: d.NodeConfig.APIServerPort, } + if gateway, err := oci.CreateNetwork(d.OCIBinary, d.NodeConfig.ClusterName); err != nil { + out.WarningT("Unable to create dedicated network, this might result in cluster IP change after restart: {{.error}}", out.V{"error": err}) + } else { + params.Network = d.NodeConfig.ClusterName + ip := gateway.To4() + // calculate the container IP based on guessing the machine index + ip[3] += byte(driver.IndexFromMachineName(d.NodeConfig.MachineName)) + glog.Infof("calculated static IP %q for the %q container", ip.String(), d.NodeConfig.MachineName) + params.IP = ip.String() + } + // control plane specific options params.PortMappings = append(params.PortMappings, oci.PortMapping{ ListenAddress: oci.DefaultBindIPV4, @@ -289,6 +302,10 @@ func (d *Driver) Remove() error { if id, err := oci.ContainerID(d.OCIBinary, d.MachineName); err == nil && id != "" { return fmt.Errorf("expected no container ID be found for %q after delete. but got %q", d.MachineName, id) } + + if err := oci.RemoveNetwork(d.NodeConfig.ClusterName); err != nil { + glog.Warningf("failed to remove network (which might be okay) %s: %v", d.NodeConfig.ClusterName, err) + } return nil } diff --git a/pkg/drivers/kic/oci/errors.go b/pkg/drivers/kic/oci/errors.go index 8c30201f978cf533d0c7d38a4f1d3d1028290a6a..f8e65b7b1bb605945906a7214b9351daa82a830b 100644 --- a/pkg/drivers/kic/oci/errors.go +++ b/pkg/drivers/kic/oci/errors.go @@ -39,12 +39,27 @@ var ErrWindowsContainers = &FailFastError{errors.New("docker container type is w // ErrCPUCountLimit is thrown when docker daemon doesn't have enough CPUs for the requested container var ErrCPUCountLimit = &FailFastError{errors.New("not enough CPUs is available for container")} +// ErrIPinUse is thrown when the container been given an IP used by another container +var ErrIPinUse = &FailFastError{errors.New("can't create with that IP, address already in use")} + // ErrExitedUnexpectedly is thrown when container is created/started without error but later it exists and it's status is not running anymore. var ErrExitedUnexpectedly = errors.New("container exited unexpectedly") // ErrDaemonInfo is thrown when docker/podman info is failing or not responding var ErrDaemonInfo = errors.New("daemon info not responding") +// ErrNetworkSubnetTaken is thrown when a subnet is taken by another network +var ErrNetworkSubnetTaken = errors.New("subnet is taken") + +// ErrNetworkNotFound is when given network was not found +var ErrNetworkNotFound = errors.New("kic network not found") + +// ErrNetworkGatewayTaken is when given network gatway is taken +var ErrNetworkGatewayTaken = errors.New("network gateway is taken") + +// ErrNetworkInUse is when trying to delete a network which is attached to another container +var ErrNetworkInUse = errors.New("unable to delete a network that is attached to a running container") + // LogContainerDebug will print relevant docker/podman infos after a container fails func LogContainerDebug(ociBin string, name string) string { rr, err := containerInspect(ociBin, name) diff --git a/pkg/drivers/kic/oci/network.go b/pkg/drivers/kic/oci/network.go index ca9e53bdb627112478b1eb9f8f16cc5da8f4adb6..137063996ae984859e374a2c358d66b16768ea0f 100644 --- a/pkg/drivers/kic/oci/network.go +++ b/pkg/drivers/kic/oci/network.go @@ -31,17 +31,26 @@ import ( // RoutableHostIPFromInside returns the ip/dns of the host that container lives on // is routable from inside the container -func RoutableHostIPFromInside(ociBin string, containerName string) (net.IP, error) { +func RoutableHostIPFromInside(ociBin string, clusterName string, containerName string) (net.IP, error) { if ociBin == Docker { if runtime.GOOS == "linux" { - return dockerGatewayIP(containerName) + _, gateway, err := dockerNetworkInspect(clusterName) + if err != nil { + if errors.Is(err, ErrNetworkNotFound) { + glog.Infof("The container %s is not attached to a network, this could be because the cluster was created by minikube 0 { + gateway = net.ParseIP(ips[1]) + } + return subnet, gateway, nil +} + +// RemoveNetwork removes a network +func RemoveNetwork(name string) error { + if !networkExists(name) { + return nil + } + rr, err := runCmd(exec.Command(Docker, "network", "remove", name)) + if err != nil { + if strings.Contains(rr.Output(), "No such network") { + return ErrNetworkNotFound + } + // Error response from daemon: error while removing network: network mynet123 id f9e1c50b89feb0b8f4b687f3501a81b618252c9907bc20666e386d0928322387 has active endpoints + if strings.Contains(rr.Output(), "has active endpoints") { + return ErrNetworkInUse + } + } + + return err +} + +func networkExists(name string) bool { + _, _, err := dockerNetworkInspect(name) + if err != nil && !errors.Is(err, ErrNetworkNotFound) { // log unexpected error + glog.Warningf("Error inspecting docker network %s: %v", name, err) + } + return err == nil +} + +// networkNamesByLabel returns all network names created by a label +func networkNamesByLabel(ociBin string, label string) ([]string, error) { + if ociBin != Docker { + return nil, fmt.Errorf("%s not supported", ociBin) + } + + // docker network ls --filter='label=created_by.minikube.sigs.k8s.io=true' --format '{{.Name}}' + rr, err := runCmd(exec.Command(Docker, "network", "ls", fmt.Sprintf("--filter=label=%s", label), "--format", "{{.Name}}")) + if err != nil { + return nil, err + } + var lines []string + scanner := bufio.NewScanner(bytes.NewReader(rr.Stdout.Bytes())) + for scanner.Scan() { + lines = append(lines, strings.TrimSpace(scanner.Text())) + } + + return lines, nil +} + +// DeleteKICNetworks deletes all networks created by kic +func DeleteKICNetworks() []error { + var errs []error + ns, err := networkNamesByLabel(Docker, CreatedByLabelKey+"=true") + if err != nil { + return []error{errors.Wrap(err, "list all volume")} + } + for _, n := range ns { + err := RemoveNetwork(n) + if err != nil { + errs = append(errs, err) + } + } + if len(errs) > 0 { + return errs + } + return nil +} diff --git a/pkg/drivers/kic/oci/oci.go b/pkg/drivers/kic/oci/oci.go index 878189e3872715af36f425f12d4c4ce0679875d9..ac7c3ee87b063f94e091f085ff9a4e1d02f97334 100644 --- a/pkg/drivers/kic/oci/oci.go +++ b/pkg/drivers/kic/oci/oci.go @@ -169,6 +169,11 @@ func CreateContainerNode(p CreateParams) error { virtualization = "podman" // VIRTUALIZATION_PODMAN } if p.OCIBinary == Docker { + // to provide a static IP for docker + if p.Network != "" && p.IP != "" { + runArgs = append(runArgs, "--network", p.Network) + runArgs = append(runArgs, "--ip", p.IP) + } runArgs = append(runArgs, "--volume", fmt.Sprintf("%s:/var", p.Name)) // ignore apparmore github actions docker: https://github.com/kubernetes/minikube/issues/7624 runArgs = append(runArgs, "--security-opt", "apparmor=unconfined") @@ -285,6 +290,10 @@ func createContainer(ociBin string, image string, opts ...createOpt) error { if strings.Contains(rr.Output(), "Range of CPUs is from") && strings.Contains(rr.Output(), "CPUs available") { // CPUs available return ErrCPUCountLimit } + // example: docker: Error response from daemon: Address already in use. + if strings.Contains(rr.Output(), "Address already in use") { + return ErrIPinUse + } return err } diff --git a/pkg/drivers/kic/oci/types.go b/pkg/drivers/kic/oci/types.go index b2dad8de4ade713595877add77231ce2f16c9911..b21da438ec5ac6993040862513076579197b5ad9 100644 --- a/pkg/drivers/kic/oci/types.go +++ b/pkg/drivers/kic/oci/types.go @@ -43,6 +43,7 @@ const ( // CreateParams are parameters needed to create a container type CreateParams struct { + ClusterName string // cluster(profile name) that this container belongs to Name string // used for container name and hostname Image string // container image to use to create the node. ClusterLabel string // label the clusters we create using minikube so we can clean up @@ -56,6 +57,8 @@ type CreateParams struct { Envs map[string]string // environment variables to pass to the container ExtraArgs []string // a list of any extra option to pass to oci binary during creation time, for example --expose 8080... OCIBinary string // docker or podman + Network string // network name that the container will attach to + IP string // static IP to assign for th container in the cluster network } // createOpt is an option for Create diff --git a/pkg/drivers/kic/types.go b/pkg/drivers/kic/types.go index 7e51ce945cc07f9dbed7efed69d0ecfad5f077e8..42ec88749a44de1dbbe40163506637c082340a9a 100644 --- a/pkg/drivers/kic/types.go +++ b/pkg/drivers/kic/types.go @@ -48,6 +48,7 @@ var ( // Config is configuration for the kic driver used by registry type Config struct { + ClusterName string // The cluster the container belongs to MachineName string // maps to the container name being created CPU int // Number of CPU cores assigned to the container Memory int // max memory in MB diff --git a/pkg/minikube/cluster/ip.go b/pkg/minikube/cluster/ip.go index 2dd4548020bdae5f29eb5712fc2fa80d37b4e936..caf34e2bb239b5b0fc6d30f0614a754a55e2f7d1 100644 --- a/pkg/minikube/cluster/ip.go +++ b/pkg/minikube/cluster/ip.go @@ -34,12 +34,12 @@ import ( ) // HostIP gets the ip address to be used for mapping host -> VM and VM -> host -func HostIP(host *host.Host) (net.IP, error) { +func HostIP(host *host.Host, clusterName string) (net.IP, error) { switch host.DriverName { case driver.Docker: - return oci.RoutableHostIPFromInside(oci.Docker, host.Name) + return oci.RoutableHostIPFromInside(oci.Docker, clusterName, host.Name) case driver.Podman: - return oci.RoutableHostIPFromInside(oci.Podman, host.Name) + return oci.RoutableHostIPFromInside(oci.Podman, clusterName, host.Name) case driver.KVM2: return net.ParseIP("192.168.39.1"), nil case driver.HyperV: diff --git a/pkg/minikube/driver/driver.go b/pkg/minikube/driver/driver.go index 791b3d0bd3b602c206ec8b1feb1dc13fb366ae6b..2002ea3afdea7916535de2e2795a96b4a5e2e2bc 100644 --- a/pkg/minikube/driver/driver.go +++ b/pkg/minikube/driver/driver.go @@ -21,6 +21,7 @@ import ( "os" "runtime" "sort" + "strconv" "strings" "github.com/golang/glog" @@ -297,3 +298,15 @@ func MachineName(cc config.ClusterConfig, n config.Node) string { } return fmt.Sprintf("%s-%s", cc.Name, n.Name) } + +// IndexFromMachineName returns the order of the container based on it is name +func IndexFromMachineName(machineName string) int { + // minikube-m02 + sp := strings.Split(machineName, "-") + m := strings.Trim(sp[len(sp)-1], "m") // m02 + i, err := strconv.Atoi(m) + if err != nil { + return 1 + } + return i +} diff --git a/pkg/minikube/driver/driver_test.go b/pkg/minikube/driver/driver_test.go index b6afd6a62c9eabafcf4d606b1950524ee5ab2d2f..953ff53691dcaf7c88f72dd1f3a903c7b0c44e1d 100644 --- a/pkg/minikube/driver/driver_test.go +++ b/pkg/minikube/driver/driver_test.go @@ -23,6 +23,7 @@ import ( "testing" "github.com/google/go-cmp/cmp" + "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/registry" ) @@ -201,3 +202,160 @@ func TestSuggest(t *testing.T) { }) } } + +func TestMachineName(t *testing.T) { + testsCases := []struct { + ClusterConfig config.ClusterConfig + Want string + }{ + { + ClusterConfig: config.ClusterConfig{Name: "minikube", + Nodes: []config.Node{ + config.Node{ + Name: "", + IP: "172.17.0.3", + Port: 8443, + KubernetesVersion: "v1.19.2", + ControlPlane: true, + Worker: true, + }, + }, + }, + Want: "minikube", + }, + + { + ClusterConfig: config.ClusterConfig{Name: "p2", + Nodes: []config.Node{ + config.Node{ + Name: "", + IP: "172.17.0.3", + Port: 8443, + KubernetesVersion: "v1.19.2", + ControlPlane: true, + Worker: true, + }, + config.Node{ + Name: "m2", + IP: "172.17.0.4", + Port: 0, + KubernetesVersion: "v1.19.2", + ControlPlane: false, + Worker: true, + }, + }, + }, + Want: "p2-m2", + }, + } + + for _, tc := range testsCases { + got := MachineName(tc.ClusterConfig, tc.ClusterConfig.Nodes[len(tc.ClusterConfig.Nodes)-1]) + if got != tc.Want { + t.Errorf("Expected MachineName to be %q but got %q", tc.Want, got) + } + } +} + +func TestIndexFromMachineName(t *testing.T) { + testCases := []struct { + Name string + MachineName string + Want int + }{ + { + Name: "default", + MachineName: "minikube", + Want: 1}, + { + Name: "second-node", + MachineName: "minikube-m02", + Want: 2}, + { + Name: "funny", + MachineName: "hahaha", + Want: 1}, + + { + Name: "dash-profile", + MachineName: "my-dashy-minikube", + Want: 1}, + + { + Name: "dash-profile-second-node", + MachineName: "my-dashy-minikube-m02", + Want: 2}, + { + Name: "michivious-user", + MachineName: "michivious-user-m02-m03", + Want: 3}, + } + + for _, tc := range testCases { + t.Run(tc.Name, func(t *testing.T) { + got := IndexFromMachineName(tc.MachineName) + if got != tc.Want { + t.Errorf("want order %q but got %q", tc.Want, got) + + } + }) + + } +} + +// test indexFroMachine against cluster config +func TestIndexFromMachineNameClusterConfig(t *testing.T) { + + testsCases := []struct { + ClusterConfig config.ClusterConfig + Want int + }{ + { + ClusterConfig: config.ClusterConfig{Name: "minikube", + Nodes: []config.Node{ + config.Node{ + Name: "", + IP: "172.17.0.3", + Port: 8443, + KubernetesVersion: "v1.19.2", + ControlPlane: true, + Worker: true, + }, + }, + }, + Want: 1, + }, + + { + ClusterConfig: config.ClusterConfig{Name: "p2", + Nodes: []config.Node{ + config.Node{ + Name: "", + IP: "172.17.0.3", + Port: 8443, + KubernetesVersion: "v1.19.2", + ControlPlane: true, + Worker: true, + }, + config.Node{ + Name: "m2", + IP: "172.17.0.4", + Port: 0, + KubernetesVersion: "v1.19.2", + ControlPlane: false, + Worker: true, + }, + }, + }, + Want: 2, + }, + } + + for _, tc := range testsCases { + got := IndexFromMachineName(MachineName(tc.ClusterConfig, tc.ClusterConfig.Nodes[len(tc.ClusterConfig.Nodes)-1])) + if got != tc.Want { + t.Errorf("expected IndexFromMachineName to be %d but got %d", tc.Want, got) + } + + } +} diff --git a/pkg/minikube/node/start.go b/pkg/minikube/node/start.go index 6982f1013514f6549054065a0555a4a5b794481f..c00aa6cadca36551ae64752df66d0c1212ebd500 100644 --- a/pkg/minikube/node/start.go +++ b/pkg/minikube/node/start.go @@ -94,7 +94,7 @@ func Start(starter Starter, apiServer bool) (*kubeconfig.Settings, error) { showVersionInfo(starter.Node.KubernetesVersion, cr) // Add "host.minikube.internal" DNS alias (intentionally non-fatal) - hostIP, err := cluster.HostIP(starter.Host) + hostIP, err := cluster.HostIP(starter.Host, starter.Cfg.Name) if err != nil { glog.Errorf("Unable to get host IP: %v", err) } else if err := machine.AddHostAlias(starter.Runner, constants.HostAlias, hostIP); err != nil { diff --git a/pkg/minikube/registry/drvs/docker/docker.go b/pkg/minikube/registry/drvs/docker/docker.go index 38a325a6aa7449a23461d069f85f501097843ca8..98d961e1d23de03c0f0695cba521d4485c4dc3c4 100644 --- a/pkg/minikube/registry/drvs/docker/docker.go +++ b/pkg/minikube/registry/drvs/docker/docker.go @@ -61,6 +61,7 @@ func configure(cc config.ClusterConfig, n config.Node) (interface{}, error) { } return kic.NewDriver(kic.Config{ + ClusterName: cc.Name, MachineName: driver.MachineName(cc, n), StorePath: localpath.MiniPath(), ImageDigest: cc.KicBaseImage, diff --git a/pkg/minikube/registry/drvs/podman/podman.go b/pkg/minikube/registry/drvs/podman/podman.go index 166fd9e6d5a336f48ef1bffefd33ada37cc7d284..fdf912f5e23789922ac1641b2781ae3e9ae681e3 100644 --- a/pkg/minikube/registry/drvs/podman/podman.go +++ b/pkg/minikube/registry/drvs/podman/podman.go @@ -73,6 +73,7 @@ func configure(cc config.ClusterConfig, n config.Node) (interface{}, error) { } return kic.NewDriver(kic.Config{ + ClusterName: cc.Name, MachineName: driver.MachineName(cc, n), StorePath: localpath.MiniPath(), ImageDigest: strings.Split(cc.KicBaseImage, "@")[0], // for podman does not support docker images references with both a tag and digest. diff --git a/site/content/en/docs/handbook/vpn_and_proxy.md b/site/content/en/docs/handbook/vpn_and_proxy.md index 3d430b10b46f0994bb8ad175318626a82921e696..bcd92ee5c8e53a713aca162a01fe1028be1c1704 100644 --- a/site/content/en/docs/handbook/vpn_and_proxy.md +++ b/site/content/en/docs/handbook/vpn_and_proxy.md @@ -22,6 +22,7 @@ The NO_PROXY variable here is important: Without setting it, minikube may not be * **192.168.99.0/24**: Used by the minikube VM. Configurable for some hypervisors via `--host-only-cidr` * **192.168.39.0/24**: Used by the minikube kvm2 driver. +* **192.168.49.0/24**: Used by the minikube docker driver's first cluster. * **10.96.0.0/12**: Used by service cluster IP's. Configurable via `--service-cluster-ip-range` One important note: If NO_PROXY is required by non-Kubernetes applications, such as Firefox or Chrome, you may want to specifically add the minikube IP to the comma-separated list, as they may not understand IP ranges ([#3827](https://github.com/kubernetes/minikube/issues/3827)).