Merge pull request #1975 from aaron-prindle/sshd-dind

Added sshd, kubeconfig and insecure-serving flags to localkube for di…

cmd/localkube/cmd/options.go

@@ -39,8 +39,9 @@ func NewLocalkubeServer() *localkube.LocalkubeServer {
 		APIServerAddress:         net.ParseIP("0.0.0.0"),
 		APIServerPort:            util.APIServerPort,
 		APIServerInsecureAddress: net.ParseIP("127.0.0.1"),
-		APIServerInsecurePort:    8080,
+		APIServerInsecurePort:    0,
 		APIServerName:            constants.APIServerName,
+		ShouldGenerateKubeconfig: false,
 		ShouldGenerateCerts:      true,
 		ShowVersion:              false,
 		RuntimeConfig:            map[string]string{"api/all": "true"},
@@ -62,6 +63,7 @@ func AddFlags(s *localkube.LocalkubeServer) {
 	flag.IntVar(&s.APIServerInsecurePort, "apiserver-insecure-port", s.APIServerInsecurePort, "The port the apiserver will listen insecurely on")
 	flag.StringVar(&s.APIServerName, "apiserver-name", s.APIServerName, "The apiserver name which is used in the generated certificate for localkube/kubernetes.  This can be used if you want to make the API server available from outside the machine")
 
+	flag.BoolVar(&s.ShouldGenerateKubeconfig, "generate-kubeconfig", s.ShouldGenerateKubeconfig, "If localkube should generate its own kubeconfig")
 	flag.BoolVar(&s.ShouldGenerateCerts, "generate-certs", s.ShouldGenerateCerts, "If localkube should generate it's own certificates")
 	flag.BoolVar(&s.ShowVersion, "show-version", s.ShowVersion, "If localkube should just print the version and exit.")
 	flag.BoolVar(&s.ShowHostIP, "host-ip", s.ShowHostIP, "If localkube should just print the host IP and exit.")

cmd/localkube/cmd/start.go

@@ -79,6 +79,12 @@ func SetupServer(s *localkube.LocalkubeServer) {
 			panic(err)
 		}
 	}
+	if s.ShouldGenerateKubeconfig {
+		if err := s.GenerateKubeconfig(); err != nil {
+			fmt.Println("Failed to create kubeconfig!")
+			panic(err)
+		}
+	}
 
 	// Set feature gates
 	if s.FeatureGates != "" {

deploy/docker/localkube-dind/Dockerfile

@@ -55,6 +55,18 @@ RUN add-apt-repository \
    stable"
 RUN apt-get update && apt-get install -yy -q docker-ce
 
+# Install sshd
+RUN apt-get update && apt-get install -y openssh-server
+RUN mkdir /var/run/sshd
+
+# SSH login fix. Otherwise user is kicked off after login
+RUN sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd
+# Change SSH Port to not conflict with Devshell
+# TODO(aaron-prindle) make this configurable, perhaps including sshd at all
+RUN sed -i 's/Port 22/Port 26/g' /etc/ssh/sshd_config
+RUN sed -i 's/#AuthorizedKeysFile/AuthorizedKeysFile/g' /etc/ssh/sshd_config
+RUN sed -i 's|authorized_keys|authorized_keys /etc/localkube-ssh/localkube_rsa.pub|g' /etc/ssh/sshd_config
+
 # Copy over important files
 COPY out/localkube /localkube
 COPY deploy/docker/localkube-dind/start.sh /start.sh
@@ -67,4 +79,4 @@ RUN chmod +x /localkube
 RUN chmod +x /start.sh
 RUN chmod +x /dindnet
 
-CMD /start.sh
+CMD /usr/sbin/sshd -D

deploy/docker/localkube-dind/start.sh

@@ -19,25 +19,28 @@ mount --make-shared /
 export CNI_BRIDGE_NETWORK_OFFSET="0.0.1.0"
 /dindnet &> /dev/null &
 
-
-
-mkdir -p /etc/localkube
-base=/etc/localkube/docker
+mkdir -p /etc/localkube-docker
+base=/etc/localkube-docker/docker
 /usr/bin/dockerd \
   --exec-root=$base.exec \
   --graph=$base.graph \
   --host=unix://$base.socket \
   --pidfile=$base.pid &> /dev/null &
 
+mkdir -p /etc/localkube-docker
 
 mkdir -p /etc/kubernetes/manifests
+mkdir -p /var/log/localkube
 /localkube start \
 --apiserver-insecure-address=0.0.0.0 \
---apiserver-insecure-port=8080 \
+--apiserver-insecure-port=4321 \
 --network-plugin=cni \
+--generate-kubeconfig=true \
 --extra-config=kubelet.DockerEndpoint=unix:///$base.socket \
 --extra-config=kubelet.KubeletFlags.ContainerRuntimeOptions.CNIConfDir="/etc/cni/net.d" \
 --extra-config=kubelet.KubeletFlags.ContainerRuntimeOptions.CNIBinDir="/opt/cni/bin" \
 --extra-config=kubelet.ClusterDNS="10.96.0.10" \
 --extra-config=kubelet.ClusterDomain="cluster.local" \
---extra-config=kubelet.AllowPrivileged="true"
+--extra-config=kubelet.AllowPrivileged="true" \
+--v=100 \
+--logtostderr &> /var/log/localkube/logs.txt

pkg/localkube/apiserver.go

@@ -41,8 +41,8 @@ func StartAPIServer(lk LocalkubeServer) func() error {
 	config.SecureServing.BindAddress = lk.APIServerAddress
 	config.SecureServing.BindPort = lk.APIServerPort
 
-	// 0 turns off insecure serving.
-	config.InsecureServing.BindPort = 0
+	config.InsecureServing.BindAddress = lk.APIServerInsecureAddress
+	config.InsecureServing.BindPort = lk.APIServerInsecurePort
 
 	config.Authentication.ClientCert.ClientCA = lk.GetCAPublicKeyCertPath()
 
@@ -99,7 +99,6 @@ func StartAPIServer(lk LocalkubeServer) func() error {
 }
 
 func readyFunc(lk LocalkubeServer) HealthCheck {
-	hostport := net.JoinHostPort("localhost", strconv.Itoa(lk.APIServerPort))
-	addr := "https://" + path.Join(hostport, "healthz")
-	return healthCheck(addr, lk)
+	return healthCheck(lk.GetAPIServerProtocol()+path.Join(
+		net.JoinHostPort("localhost", strconv.Itoa(lk.APIServerPort)), "healthz"), lk)
 }

pkg/localkube/localkube.go

@@ -17,17 +17,21 @@ limitations under the License.
 package localkube
 
 import (
+	"crypto/tls"
 	"crypto/x509"
 	"encoding/pem"
 	"fmt"
 	"io/ioutil"
 	"net"
+	"net/http"
 	"path"
+	"strconv"
 
 	"github.com/golang/glog"
 
 	utilnet "k8s.io/apimachinery/pkg/util/net"
 	"k8s.io/apiserver/pkg/util/flag"
+	"k8s.io/minikube/pkg/util/kubeconfig"
 
 	"k8s.io/minikube/pkg/util"
 )
@@ -52,6 +56,7 @@ type LocalkubeServer struct {
 	APIServerInsecurePort    int
 	APIServerName            string
 	ShouldGenerateCerts      bool
+	ShouldGenerateKubeconfig bool
 	ShowVersion              bool
 	ShowHostIP               bool
 	RuntimeConfig            flag.ConfigurationMap
@@ -110,7 +115,43 @@ func (lk LocalkubeServer) GetAPIServerSecureURL() string {
 }
 
 func (lk LocalkubeServer) GetAPIServerInsecureURL() string {
-	return fmt.Sprintf("http://%s:%d", lk.APIServerInsecureAddress.String(), lk.APIServerInsecurePort)
+	if lk.APIServerInsecurePort != 0 {
+		return fmt.Sprintf("http://%s:%d", lk.APIServerInsecureAddress.String(), lk.APIServerInsecurePort)
+	}
+	return ""
+}
+
+func (lk LocalkubeServer) GetAPIServerProtocol() string {
+	if lk.APIServerInsecurePort != 0 {
+		return "http://"
+	}
+	return "https://"
+}
+
+func (lk LocalkubeServer) GetTransport() (*http.Transport, error) {
+	if lk.APIServerInsecurePort != 0 {
+		return &http.Transport{}, nil
+	}
+	cert, err := tls.LoadX509KeyPair(lk.GetPublicKeyCertPath(), lk.GetPrivateKeyCertPath())
+	if err != nil {
+		glog.Error(err)
+		return &http.Transport{}, err
+	}
+
+	// Load CA cert
+	caCert, err := ioutil.ReadFile(lk.GetCAPublicKeyCertPath())
+	if err != nil {
+		glog.Warning(err)
+		return &http.Transport{}, err
+	}
+	caCertPool := x509.NewCertPool()
+	caCertPool.AppendCertsFromPEM(caCert)
+	tlsConfig := &tls.Config{
+		Certificates: []tls.Certificate{cert},
+		RootCAs:      caCertPool,
+	}
+	tlsConfig.BuildNameToCertificate()
+	return &http.Transport{TLSClientConfig: tlsConfig}, nil
 }
 
 // Get the host's public IP address
@@ -194,6 +235,33 @@ func (lk LocalkubeServer) shouldGenerateCACerts() bool {
 	return false
 }
 
+func (lk LocalkubeServer) GenerateKubeconfig() error {
+	if !lk.ShouldGenerateKubeconfig {
+		return nil
+	}
+
+	// setup kubeconfig
+	kubeConfigFile := util.DefaultKubeConfigPath
+	glog.Infof("Setting up kubeconfig at: %s", kubeConfigFile)
+	kubeHost := "http://127.0.0.1:" + strconv.Itoa(lk.APIServerInsecurePort)
+
+	//TODO(aaron-prindle) configure this so that it can generate secure certs as well
+	kubeCfgSetup := &kubeconfig.KubeConfigSetup{
+		ClusterName:          lk.APIServerName,
+		ClusterServerAddress: kubeHost,
+		KeepContext:          false,
+	}
+
+	kubeCfgSetup.SetKubeConfigFile(kubeConfigFile)
+
+	if err := kubeconfig.SetupKubeConfig(kubeCfgSetup); err != nil {
+		glog.Errorln("Error setting up kubeconfig: ", err)
+		return err
+	}
+
+	return nil
+}
+
 func (lk LocalkubeServer) getAllIPs() ([]net.IP, error) {
 	ips := []net.IP{net.ParseIP(util.DefaultServiceClusterIP)}
 	addrs, err := net.InterfaceAddrs()

pkg/localkube/proxy.go

@@ -38,6 +38,10 @@ func (lk LocalkubeServer) NewProxyServer() Server {
 }
 
 func StartProxyServer(lk LocalkubeServer) func() error {
+	bindaddress := lk.APIServerAddress.String()
+	if lk.APIServerInsecurePort != 0 {
+		bindaddress = lk.APIServerInsecureAddress.String()
+	}
 	config := &componentconfig.KubeProxyConfiguration{
 		OOMScoreAdj: &OOMScoreAdj,
 		ClientConnection: componentconfig.ClientConnectionConfiguration{
@@ -51,7 +55,7 @@ func StartProxyServer(lk LocalkubeServer) func() error {
 			SyncPeriod:    v1.Duration{Duration: 30 * time.Second},
 			MinSyncPeriod: v1.Duration{Duration: 5 * time.Second},
 		},
-		BindAddress:  lk.APIServerAddress.String(),
+		BindAddress:  bindaddress,
 		Mode:         componentconfig.ProxyModeIPTables,
 		FeatureGates: lk.FeatureGates,
 		// Disable the healthz check
@@ -62,7 +66,7 @@ func StartProxyServer(lk LocalkubeServer) func() error {
 
 	return func() error {
 		// Creating this config requires the API Server to be up, so do it in the start function itself.
-		server, err := kubeproxy.NewProxyServer(config, false, runtime.NewScheme(), "")
+		server, err := kubeproxy.NewProxyServer(config, false, runtime.NewScheme(), lk.GetAPIServerInsecureURL())
 		if err != nil {
 			panic(err)
 		}

pkg/localkube/ready.go

@@ -17,8 +17,6 @@ limitations under the License.
 package localkube
 
 import (
-	"crypto/tls"
-	"crypto/x509"
 	"io/ioutil"
 	"net/http"
 
@@ -31,27 +29,11 @@ func healthCheck(addr string, lk LocalkubeServer) HealthCheck {
 	return func() bool {
 		glog.Infof("Performing healthcheck on %s\n", addr)
 
-		cert, err := tls.LoadX509KeyPair(lk.GetPublicKeyCertPath(), lk.GetPrivateKeyCertPath())
+		transport, err := lk.GetTransport()
 		if err != nil {
-			glog.Error(err)
 			return false
 		}
-
-		// Load CA cert
-		caCert, err := ioutil.ReadFile(lk.GetCAPublicKeyCertPath())
-		if err != nil {
-			glog.Warning(err)
-			return false
-		}
-		caCertPool := x509.NewCertPool()
-		caCertPool.AppendCertsFromPEM(caCert)
-		tlsConfig := &tls.Config{
-			Certificates: []tls.Certificate{cert},
-			RootCAs:      caCertPool,
-		}
-		tlsConfig.BuildNameToCertificate()
-		transport := &http.Transport{TLSClientConfig: tlsConfig}
-		client := &http.Client{Transport: transport}
+		client := http.Client{Transport: transport}
 
 		resp, err := client.Get(addr)
 		if err != nil {