未验证 提交 0fc0c823 编写于 作者: M Medya Ghazizadeh 提交者: GitHub

Merge pull request #9294 from medyagh/docker_network

add dedicated network for docker driver
......@@ -260,6 +260,11 @@ func deletePossibleKicLeftOver(cname string, driverName string) {
glog.Warningf("error deleting volumes (might be okay).\nTo see the list of volumes run: 'docker volume ls'\n:%v", errs)
}
errs = oci.DeleteKICNetworks()
if errs != nil {
glog.Warningf("error deleting leftover networks (might be okay).\nTo see the list of networks: 'docker network ls'\n:%v", errs)
}
if bin == oci.Podman {
// podman prune does not support --filter
return
......
......@@ -110,7 +110,7 @@ var mountCmd = &cobra.Command{
var ip net.IP
var err error
if mountIP == "" {
ip, err = cluster.HostIP(co.CP.Host)
ip, err = cluster.HostIP(co.CP.Host, co.Config.Name)
if err != nil {
exit.Error(reason.IfHostIP, "Error getting the host IP address to use from within the VM", err)
}
......
......@@ -39,6 +39,7 @@ import (
func generateTarball(kubernetesVersion, containerRuntime, tarballFilename string) error {
driver := kic.NewDriver(kic.Config{
ClusterName: profile,
KubernetesVersion: kubernetesVersion,
ContainerRuntime: containerRuntime,
OCIBinary: oci.Docker,
......
......@@ -37,6 +37,8 @@ import (
"k8s.io/minikube/pkg/minikube/constants"
"k8s.io/minikube/pkg/minikube/cruntime"
"k8s.io/minikube/pkg/minikube/download"
"k8s.io/minikube/pkg/minikube/driver"
"k8s.io/minikube/pkg/minikube/out"
"k8s.io/minikube/pkg/minikube/sysinit"
"k8s.io/minikube/pkg/util/retry"
)
......@@ -81,6 +83,17 @@ func (d *Driver) Create() error {
APIServerPort: d.NodeConfig.APIServerPort,
}
if gateway, err := oci.CreateNetwork(d.OCIBinary, d.NodeConfig.ClusterName); err != nil {
out.WarningT("Unable to create dedicated network, this might result in cluster IP change after restart: {{.error}}", out.V{"error": err})
} else {
params.Network = d.NodeConfig.ClusterName
ip := gateway.To4()
// calculate the container IP based on guessing the machine index
ip[3] += byte(driver.IndexFromMachineName(d.NodeConfig.MachineName))
glog.Infof("calculated static IP %q for the %q container", ip.String(), d.NodeConfig.MachineName)
params.IP = ip.String()
}
// control plane specific options
params.PortMappings = append(params.PortMappings, oci.PortMapping{
ListenAddress: oci.DefaultBindIPV4,
......@@ -289,6 +302,10 @@ func (d *Driver) Remove() error {
if id, err := oci.ContainerID(d.OCIBinary, d.MachineName); err == nil && id != "" {
return fmt.Errorf("expected no container ID be found for %q after delete. but got %q", d.MachineName, id)
}
if err := oci.RemoveNetwork(d.NodeConfig.ClusterName); err != nil {
glog.Warningf("failed to remove network (which might be okay) %s: %v", d.NodeConfig.ClusterName, err)
}
return nil
}
......
......@@ -39,12 +39,27 @@ var ErrWindowsContainers = &FailFastError{errors.New("docker container type is w
// ErrCPUCountLimit is thrown when docker daemon doesn't have enough CPUs for the requested container
var ErrCPUCountLimit = &FailFastError{errors.New("not enough CPUs is available for container")}
// ErrIPinUse is thrown when the container been given an IP used by another container
var ErrIPinUse = &FailFastError{errors.New("can't create with that IP, address already in use")}
// ErrExitedUnexpectedly is thrown when container is created/started without error but later it exists and it's status is not running anymore.
var ErrExitedUnexpectedly = errors.New("container exited unexpectedly")
// ErrDaemonInfo is thrown when docker/podman info is failing or not responding
var ErrDaemonInfo = errors.New("daemon info not responding")
// ErrNetworkSubnetTaken is thrown when a subnet is taken by another network
var ErrNetworkSubnetTaken = errors.New("subnet is taken")
// ErrNetworkNotFound is when given network was not found
var ErrNetworkNotFound = errors.New("kic network not found")
// ErrNetworkGatewayTaken is when given network gatway is taken
var ErrNetworkGatewayTaken = errors.New("network gateway is taken")
// ErrNetworkInUse is when trying to delete a network which is attached to another container
var ErrNetworkInUse = errors.New("unable to delete a network that is attached to a running container")
// LogContainerDebug will print relevant docker/podman infos after a container fails
func LogContainerDebug(ociBin string, name string) string {
rr, err := containerInspect(ociBin, name)
......
......@@ -31,17 +31,26 @@ import (
// RoutableHostIPFromInside returns the ip/dns of the host that container lives on
// is routable from inside the container
func RoutableHostIPFromInside(ociBin string, containerName string) (net.IP, error) {
func RoutableHostIPFromInside(ociBin string, clusterName string, containerName string) (net.IP, error) {
if ociBin == Docker {
if runtime.GOOS == "linux" {
return dockerGatewayIP(containerName)
_, gateway, err := dockerNetworkInspect(clusterName)
if err != nil {
if errors.Is(err, ErrNetworkNotFound) {
glog.Infof("The container %s is not attached to a network, this could be because the cluster was created by minikube <v1.14, will try to get the IP using container gatway", containerName)
return containerGatewayIP(Docker, containerName)
}
return gateway, errors.Wrap(err, "network inspect")
}
return gateway, nil
}
// for windows and mac, the gateway ip is not routable so we use dns trick.
return digDNS(ociBin, containerName, "host.docker.internal")
}
// podman
if runtime.GOOS == "linux" {
return containerGatewayIP(ociBin, containerName)
return containerGatewayIP(Podman, containerName)
}
return nil, fmt.Errorf("RoutableHostIPFromInside is currently only implemented for linux")
......@@ -59,56 +68,8 @@ func digDNS(ociBin, containerName, dns string) (net.IP, error) {
return ip, nil
}
// profileInContainers checks whether the profile is within the containers list
func profileInContainers(profile string, containers []string) bool {
for _, container := range containers {
if container == profile {
return true
}
}
return false
}
// dockerGatewayIP gets the default gateway ip for the docker bridge on the user's host machine
// gets the ip from user's host docker
func dockerGatewayIP(profile string) (net.IP, error) {
var bridgeID string
rr, err := runCmd(exec.Command(Docker, "network", "ls", "--filter", "name=bridge", "--format", "{{.ID}}"))
if err != nil {
return nil, errors.Wrapf(err, "get network bridge")
}
networksOutput := strings.TrimSpace(rr.Stdout.String())
networksSlice := strings.Fields(networksOutput)
// Look for the minikube container within each docker network
for _, net := range networksSlice {
// get all containers in the network
rs, err := runCmd(exec.Command(Docker, "network", "inspect", net, "-f", "{{range $k, $v := .Containers}}{{$v.Name}} {{end}}"))
if err != nil {
return nil, errors.Wrapf(err, "get containers in network")
}
containersSlice := strings.Fields(rs.Stdout.String())
if profileInContainers(profile, containersSlice) {
bridgeID = net
break
}
}
if bridgeID == "" {
return nil, errors.Errorf("unable to determine bridge network id from %q", networksOutput)
}
rr, err = runCmd(exec.Command(Docker, "network", "inspect",
"--format", "{{(index .IPAM.Config 0).Gateway}}", bridgeID))
if err != nil {
return nil, errors.Wrapf(err, "inspect IP bridge network %q.", bridgeID)
}
ip := net.ParseIP(strings.TrimSpace(rr.Stdout.String()))
glog.Infof("got host ip for mount in container by inspect docker network: %s", ip.String())
return ip, nil
}
// containerGatewayIP gets the default gateway ip for the container
func containerGatewayIP(ociBin, containerName string) (net.IP, error) {
func containerGatewayIP(ociBin string, containerName string) (net.IP, error) {
rr, err := runCmd(exec.Command(ociBin, "container", "inspect", "--format", "{{.NetworkSettings.Gateway}}", containerName))
if err != nil {
return nil, errors.Wrapf(err, "inspect gateway")
......
/*
Copyright 2020 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package oci
import (
"bufio"
"bytes"
"fmt"
"net"
"os/exec"
"strings"
"github.com/golang/glog"
"github.com/pkg/errors"
)
// firstSubnetAddr subnet to be used on first kic cluster
// it is one octet more than the one used by KVM to avoid possible conflict
const firstSubnetAddr = "192.168.49.0"
// big enough for a cluster of 254 nodes
const defaultSubnetMask = 24
// CreateNetwork creates a network returns gateway and error, minikube creates one network per cluster
func CreateNetwork(ociBin string, name string) (net.IP, error) {
if ociBin != Docker {
return nil, fmt.Errorf("%s network not implemented yet", ociBin)
}
return createDockerNetwork(name)
}
func createDockerNetwork(clusterName string) (net.IP, error) {
// check if the network already exists
subnet, gateway, err := dockerNetworkInspect(clusterName)
if err == nil {
glog.Infof("Found existing network with subnet %s and gateway %s.", subnet, gateway)
return gateway, nil
}
attempts := 0
subnetAddr := firstSubnetAddr
// Rather than iterate through all of the valid subnets, give up at 20 to avoid a lengthy user delay for something that is unlikely to work.
// will be like 192.168.49.0/24 ,...,192.168.239.0/24
for attempts < 20 {
gateway, err = tryCreateDockerNetwork(subnetAddr, defaultSubnetMask, clusterName)
if err == nil {
return gateway, nil
}
// don't retry if error is not adddress is taken
if !(errors.Is(err, ErrNetworkSubnetTaken) || errors.Is(err, ErrNetworkGatewayTaken)) {
glog.Errorf("error while trying to create network %v", err)
return nil, errors.Wrap(err, "un-retryable")
}
attempts++
// Find an open subnet by incrementing the 3rd octet by 10 for each try
// 13 times adding 10 firstSubnetAddr "192.168.49.0/24"
// at most it will add up to 169 which is still less than max allowed 255
// this is large enough to try more and not too small to not try enough
// can be tuned in the next iterations
newSubnet := net.ParseIP(subnetAddr).To4()
newSubnet[2] += byte(9 + attempts)
subnetAddr = newSubnet.String()
}
return gateway, fmt.Errorf("failed to create network after 20 attempts")
}
func tryCreateDockerNetwork(subnetAddr string, subnetMask int, name string) (net.IP, error) {
gateway := net.ParseIP(subnetAddr)
gateway.To4()[3]++ // first ip for gateway
glog.Infof("attempt to create network %s/%d with subnet: %s and gateway %s...", subnetAddr, subnetMask, name, gateway)
// options documentation https://docs.docker.com/engine/reference/commandline/network_create/#bridge-driver-options
rr, err := runCmd(exec.Command(Docker, "network", "create", "--driver=bridge", fmt.Sprintf("--subnet=%s", fmt.Sprintf("%s/%d", subnetAddr, subnetMask)), fmt.Sprintf("--gateway=%s", gateway), "-o", "--ip-masq", "-o", "--icc", fmt.Sprintf("--label=%s=%s", CreatedByLabelKey, "true"), name))
if err != nil {
// Pool overlaps with other one on this address space
if strings.Contains(rr.Output(), "Pool overlaps") {
return nil, ErrNetworkSubnetTaken
}
if strings.Contains(rr.Output(), "failed to allocate gateway") && strings.Contains(rr.Output(), "Address already in use") {
return nil, ErrNetworkGatewayTaken
}
return nil, errors.Wrapf(err, "create network %s", fmt.Sprintf("%s %s/%d", name, subnetAddr, subnetMask))
}
return gateway, nil
}
// returns subnet and gate if exists
func dockerNetworkInspect(name string) (*net.IPNet, net.IP, error) {
rr, err := runCmd(exec.Command(Docker, "network", "inspect", name, "--format", "{{(index .IPAM.Config 0).Subnet}},{{(index .IPAM.Config 0).Gateway}}"))
if err != nil {
if strings.Contains(rr.Output(), "No such network") {
return nil, nil, ErrNetworkNotFound
}
return nil, nil, err
}
// results looks like 172.17.0.0/16,172.17.0.1
ips := strings.Split(strings.TrimSpace(rr.Stdout.String()), ",")
if len(ips) == 0 {
return nil, nil, fmt.Errorf("empty IP list parsed from: %q", rr.Output())
}
_, subnet, err := net.ParseCIDR(ips[0])
if err != nil {
return nil, nil, errors.Wrapf(err, "parse subnet for %s", name)
}
var gateway net.IP
if len(ips) > 0 {
gateway = net.ParseIP(ips[1])
}
return subnet, gateway, nil
}
// RemoveNetwork removes a network
func RemoveNetwork(name string) error {
if !networkExists(name) {
return nil
}
rr, err := runCmd(exec.Command(Docker, "network", "remove", name))
if err != nil {
if strings.Contains(rr.Output(), "No such network") {
return ErrNetworkNotFound
}
// Error response from daemon: error while removing network: network mynet123 id f9e1c50b89feb0b8f4b687f3501a81b618252c9907bc20666e386d0928322387 has active endpoints
if strings.Contains(rr.Output(), "has active endpoints") {
return ErrNetworkInUse
}
}
return err
}
func networkExists(name string) bool {
_, _, err := dockerNetworkInspect(name)
if err != nil && !errors.Is(err, ErrNetworkNotFound) { // log unexpected error
glog.Warningf("Error inspecting docker network %s: %v", name, err)
}
return err == nil
}
// networkNamesByLabel returns all network names created by a label
func networkNamesByLabel(ociBin string, label string) ([]string, error) {
if ociBin != Docker {
return nil, fmt.Errorf("%s not supported", ociBin)
}
// docker network ls --filter='label=created_by.minikube.sigs.k8s.io=true' --format '{{.Name}}'
rr, err := runCmd(exec.Command(Docker, "network", "ls", fmt.Sprintf("--filter=label=%s", label), "--format", "{{.Name}}"))
if err != nil {
return nil, err
}
var lines []string
scanner := bufio.NewScanner(bytes.NewReader(rr.Stdout.Bytes()))
for scanner.Scan() {
lines = append(lines, strings.TrimSpace(scanner.Text()))
}
return lines, nil
}
// DeleteKICNetworks deletes all networks created by kic
func DeleteKICNetworks() []error {
var errs []error
ns, err := networkNamesByLabel(Docker, CreatedByLabelKey+"=true")
if err != nil {
return []error{errors.Wrap(err, "list all volume")}
}
for _, n := range ns {
err := RemoveNetwork(n)
if err != nil {
errs = append(errs, err)
}
}
if len(errs) > 0 {
return errs
}
return nil
}
......@@ -169,6 +169,11 @@ func CreateContainerNode(p CreateParams) error {
virtualization = "podman" // VIRTUALIZATION_PODMAN
}
if p.OCIBinary == Docker {
// to provide a static IP for docker
if p.Network != "" && p.IP != "" {
runArgs = append(runArgs, "--network", p.Network)
runArgs = append(runArgs, "--ip", p.IP)
}
runArgs = append(runArgs, "--volume", fmt.Sprintf("%s:/var", p.Name))
// ignore apparmore github actions docker: https://github.com/kubernetes/minikube/issues/7624
runArgs = append(runArgs, "--security-opt", "apparmor=unconfined")
......@@ -285,6 +290,10 @@ func createContainer(ociBin string, image string, opts ...createOpt) error {
if strings.Contains(rr.Output(), "Range of CPUs is from") && strings.Contains(rr.Output(), "CPUs available") { // CPUs available
return ErrCPUCountLimit
}
// example: docker: Error response from daemon: Address already in use.
if strings.Contains(rr.Output(), "Address already in use") {
return ErrIPinUse
}
return err
}
......
......@@ -43,6 +43,7 @@ const (
// CreateParams are parameters needed to create a container
type CreateParams struct {
ClusterName string // cluster(profile name) that this container belongs to
Name string // used for container name and hostname
Image string // container image to use to create the node.
ClusterLabel string // label the clusters we create using minikube so we can clean up
......@@ -56,6 +57,8 @@ type CreateParams struct {
Envs map[string]string // environment variables to pass to the container
ExtraArgs []string // a list of any extra option to pass to oci binary during creation time, for example --expose 8080...
OCIBinary string // docker or podman
Network string // network name that the container will attach to
IP string // static IP to assign for th container in the cluster network
}
// createOpt is an option for Create
......
......@@ -48,6 +48,7 @@ var (
// Config is configuration for the kic driver used by registry
type Config struct {
ClusterName string // The cluster the container belongs to
MachineName string // maps to the container name being created
CPU int // Number of CPU cores assigned to the container
Memory int // max memory in MB
......
......@@ -34,12 +34,12 @@ import (
)
// HostIP gets the ip address to be used for mapping host -> VM and VM -> host
func HostIP(host *host.Host) (net.IP, error) {
func HostIP(host *host.Host, clusterName string) (net.IP, error) {
switch host.DriverName {
case driver.Docker:
return oci.RoutableHostIPFromInside(oci.Docker, host.Name)
return oci.RoutableHostIPFromInside(oci.Docker, clusterName, host.Name)
case driver.Podman:
return oci.RoutableHostIPFromInside(oci.Podman, host.Name)
return oci.RoutableHostIPFromInside(oci.Podman, clusterName, host.Name)
case driver.KVM2:
return net.ParseIP("192.168.39.1"), nil
case driver.HyperV:
......
......@@ -21,6 +21,7 @@ import (
"os"
"runtime"
"sort"
"strconv"
"strings"
"github.com/golang/glog"
......@@ -297,3 +298,15 @@ func MachineName(cc config.ClusterConfig, n config.Node) string {
}
return fmt.Sprintf("%s-%s", cc.Name, n.Name)
}
// IndexFromMachineName returns the order of the container based on it is name
func IndexFromMachineName(machineName string) int {
// minikube-m02
sp := strings.Split(machineName, "-")
m := strings.Trim(sp[len(sp)-1], "m") // m02
i, err := strconv.Atoi(m)
if err != nil {
return 1
}
return i
}
......@@ -23,6 +23,7 @@ import (
"testing"
"github.com/google/go-cmp/cmp"
"k8s.io/minikube/pkg/minikube/config"
"k8s.io/minikube/pkg/minikube/registry"
)
......@@ -201,3 +202,160 @@ func TestSuggest(t *testing.T) {
})
}
}
func TestMachineName(t *testing.T) {
testsCases := []struct {
ClusterConfig config.ClusterConfig
Want string
}{
{
ClusterConfig: config.ClusterConfig{Name: "minikube",
Nodes: []config.Node{
config.Node{
Name: "",
IP: "172.17.0.3",
Port: 8443,
KubernetesVersion: "v1.19.2",
ControlPlane: true,
Worker: true,
},
},
},
Want: "minikube",
},
{
ClusterConfig: config.ClusterConfig{Name: "p2",
Nodes: []config.Node{
config.Node{
Name: "",
IP: "172.17.0.3",
Port: 8443,
KubernetesVersion: "v1.19.2",
ControlPlane: true,
Worker: true,
},
config.Node{
Name: "m2",
IP: "172.17.0.4",
Port: 0,
KubernetesVersion: "v1.19.2",
ControlPlane: false,
Worker: true,
},
},
},
Want: "p2-m2",
},
}
for _, tc := range testsCases {
got := MachineName(tc.ClusterConfig, tc.ClusterConfig.Nodes[len(tc.ClusterConfig.Nodes)-1])
if got != tc.Want {
t.Errorf("Expected MachineName to be %q but got %q", tc.Want, got)
}
}
}
func TestIndexFromMachineName(t *testing.T) {
testCases := []struct {
Name string
MachineName string
Want int
}{
{
Name: "default",
MachineName: "minikube",
Want: 1},
{
Name: "second-node",
MachineName: "minikube-m02",
Want: 2},
{
Name: "funny",
MachineName: "hahaha",
Want: 1},
{
Name: "dash-profile",
MachineName: "my-dashy-minikube",
Want: 1},
{
Name: "dash-profile-second-node",
MachineName: "my-dashy-minikube-m02",
Want: 2},
{
Name: "michivious-user",
MachineName: "michivious-user-m02-m03",
Want: 3},
}
for _, tc := range testCases {
t.Run(tc.Name, func(t *testing.T) {
got := IndexFromMachineName(tc.MachineName)
if got != tc.Want {
t.Errorf("want order %q but got %q", tc.Want, got)
}
})
}
}
// test indexFroMachine against cluster config
func TestIndexFromMachineNameClusterConfig(t *testing.T) {
testsCases := []struct {
ClusterConfig config.ClusterConfig
Want int
}{
{
ClusterConfig: config.ClusterConfig{Name: "minikube",
Nodes: []config.Node{
config.Node{
Name: "",
IP: "172.17.0.3",
Port: 8443,
KubernetesVersion: "v1.19.2",
ControlPlane: true,
Worker: true,
},
},
},
Want: 1,
},
{
ClusterConfig: config.ClusterConfig{Name: "p2",
Nodes: []config.Node{
config.Node{
Name: "",
IP: "172.17.0.3",
Port: 8443,
KubernetesVersion: "v1.19.2",
ControlPlane: true,
Worker: true,
},
config.Node{
Name: "m2",
IP: "172.17.0.4",
Port: 0,
KubernetesVersion: "v1.19.2",
ControlPlane: false,
Worker: true,
},
},
},
Want: 2,
},
}
for _, tc := range testsCases {
got := IndexFromMachineName(MachineName(tc.ClusterConfig, tc.ClusterConfig.Nodes[len(tc.ClusterConfig.Nodes)-1]))
if got != tc.Want {
t.Errorf("expected IndexFromMachineName to be %d but got %d", tc.Want, got)
}
}
}
......@@ -94,7 +94,7 @@ func Start(starter Starter, apiServer bool) (*kubeconfig.Settings, error) {
showVersionInfo(starter.Node.KubernetesVersion, cr)
// Add "host.minikube.internal" DNS alias (intentionally non-fatal)
hostIP, err := cluster.HostIP(starter.Host)
hostIP, err := cluster.HostIP(starter.Host, starter.Cfg.Name)
if err != nil {
glog.Errorf("Unable to get host IP: %v", err)
} else if err := machine.AddHostAlias(starter.Runner, constants.HostAlias, hostIP); err != nil {
......
......@@ -61,6 +61,7 @@ func configure(cc config.ClusterConfig, n config.Node) (interface{}, error) {
}
return kic.NewDriver(kic.Config{
ClusterName: cc.Name,
MachineName: driver.MachineName(cc, n),
StorePath: localpath.MiniPath(),
ImageDigest: cc.KicBaseImage,
......
......@@ -73,6 +73,7 @@ func configure(cc config.ClusterConfig, n config.Node) (interface{}, error) {
}
return kic.NewDriver(kic.Config{
ClusterName: cc.Name,
MachineName: driver.MachineName(cc, n),
StorePath: localpath.MiniPath(),
ImageDigest: strings.Split(cc.KicBaseImage, "@")[0], // for podman does not support docker images references with both a tag and digest.
......
......@@ -22,6 +22,7 @@ The NO_PROXY variable here is important: Without setting it, minikube may not be
* **192.168.99.0/24**: Used by the minikube VM. Configurable for some hypervisors via `--host-only-cidr`
* **192.168.39.0/24**: Used by the minikube kvm2 driver.
* **192.168.49.0/24**: Used by the minikube docker driver's first cluster.
* **10.96.0.0/12**: Used by service cluster IP's. Configurable via `--service-cluster-ip-range`
One important note: If NO_PROXY is required by non-Kubernetes applications, such as Firefox or Chrome, you may want to specifically add the minikube IP to the comma-separated list, as they may not understand IP ranges ([#3827](https://github.com/kubernetes/minikube/issues/3827)).
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册