提交 88370459 编写于 作者: D dlorenc 提交者: dlorenc

Stop using insecure serving.

上级 e9795ca1
......@@ -26,6 +26,9 @@ spec:
containers:
- name: kube-addon-manager
image: gcr.io/google-containers/kube-addon-manager:v6.4-beta.2
env:
- name: KUBECONFIG
value: /var/lib/localkube/kubeconfig
imagePullPolicy: IfNotPresent
resources:
requests:
......@@ -35,7 +38,13 @@ spec:
- mountPath: /etc/kubernetes/
name: addons
readOnly: true
- mountPath: /var/lib/localkube
name: kubeconfig
readOnly: true
volumes:
- hostPath:
path: /etc/kubernetes/
name: addons
- hostPath:
path: /var/lib/localkube
name: kubeconfig
......@@ -75,6 +75,9 @@ ${SUDO_PREFIX}out/e2e-${OS_ARCH} -minikube-args="--vm-driver=${VM_DRIVER} --v=10
result=$?
set -e
# See the KUBECONFIG file for debugging
sudo cat $KUBECONFIG
MINIKUBE_WANTREPORTERRORPROMPT=False sudo ./out/minikube-${OS_ARCH} delete \
|| MINIKUBE_WANTREPORTERRORPROMPT=False ./out/minikube-${OS_ARCH} delete \
|| true
......
......@@ -30,7 +30,8 @@ OS_ARCH="linux-amd64"
VM_DRIVER="none"
JOB_NAME="Linux-None"
EXTRA_BUILD_ARGS="$EXTRA_BUILD_ARGS --use-vendored-driver"
SUDO_PREFIX="sudo "
SUDO_PREFIX="sudo -E "
export KUBECONFIG="/root/.kube/config"
# Download files and set permissions
source common.sh
......@@ -20,11 +20,14 @@
# the cluster state.
set +e
env
${SUDO_PREFIX} cat $KUBECONFIG
kubectl get pods --all-namespaces
kubectl cluster-info dump
# For the none driver
journalctl -u localkube
journalctl -u localkube -n 500
${SUDO_PREFIX}cat $KUBECONFIG
cat $HOME/.kube/config
......
......@@ -41,8 +41,8 @@ func StartAPIServer(lk LocalkubeServer) func() error {
config.SecureServing.BindAddress = lk.APIServerAddress
config.SecureServing.BindPort = lk.APIServerPort
config.InsecureServing.BindAddress = lk.APIServerInsecureAddress
config.InsecureServing.BindPort = lk.APIServerInsecurePort
// 0 turns off insecure serving.
config.InsecureServing.BindPort = 0
config.Authentication.ClientCert.ClientCA = lk.GetCAPublicKeyCertPath()
......@@ -86,7 +86,7 @@ func StartAPIServer(lk LocalkubeServer) func() error {
}
func readyFunc(lk LocalkubeServer) HealthCheck {
hostport := net.JoinHostPort(lk.APIServerInsecureAddress.String(), strconv.Itoa(lk.APIServerInsecurePort))
addr := "http://" + path.Join(hostport, "healthz")
return healthCheck(addr)
hostport := net.JoinHostPort("localhost", strconv.Itoa(lk.APIServerPort))
addr := "https://" + path.Join(hostport, "healthz")
return healthCheck(addr, lk)
}
......@@ -19,6 +19,7 @@ package localkube
import (
controllerManager "k8s.io/kubernetes/cmd/kube-controller-manager/app"
"k8s.io/kubernetes/cmd/kube-controller-manager/app/options"
"k8s.io/minikube/pkg/util"
)
func (lk LocalkubeServer) NewControllerManagerServer() Server {
......@@ -28,7 +29,7 @@ func (lk LocalkubeServer) NewControllerManagerServer() Server {
func StartControllerManagerServer(lk LocalkubeServer) func() error {
config := options.NewCMServer()
config.Master = lk.GetAPIServerInsecureURL()
config.Kubeconfig = util.DefaultKubeConfigPath
// defaults from command
config.DeletingPodsQps = 0.1
......
......@@ -17,8 +17,10 @@ limitations under the License.
package localkube
import (
"k8s.io/apiserver/pkg/util/flag"
kubelet "k8s.io/kubernetes/cmd/kubelet/app"
"k8s.io/kubernetes/cmd/kubelet/app/options"
"k8s.io/minikube/pkg/util"
)
func (lk LocalkubeServer) NewKubeletServer() Server {
......@@ -29,7 +31,8 @@ func StartKubeletServer(lk LocalkubeServer) func() error {
config := options.NewKubeletServer()
// Master details
config.APIServerList = []string{lk.GetAPIServerInsecureURL()}
config.KubeConfig = flag.NewStringFlag(util.DefaultKubeConfigPath)
config.RequireKubeConfig = true
// Set containerized based on the flag
config.Containerized = lk.Containerized
......
......@@ -18,6 +18,7 @@ package localkube
import (
kubeproxy "k8s.io/kubernetes/cmd/kube-proxy/app"
"k8s.io/minikube/pkg/util"
"time"
......@@ -40,8 +41,9 @@ func StartProxyServer(lk LocalkubeServer) func() error {
config := &componentconfig.KubeProxyConfiguration{
OOMScoreAdj: &OOMScoreAdj,
ClientConnection: componentconfig.ClientConnectionConfiguration{
Burst: 10,
QPS: 5,
Burst: 10,
QPS: 5,
KubeConfigFile: util.DefaultKubeConfigPath,
},
ConfigSyncPeriod: v1.Duration{Duration: 15 * time.Minute},
IPTables: componentconfig.KubeProxyIPTablesConfiguration{
......@@ -49,7 +51,7 @@ func StartProxyServer(lk LocalkubeServer) func() error {
SyncPeriod: v1.Duration{Duration: 30 * time.Second},
MinSyncPeriod: v1.Duration{Duration: 5 * time.Second},
},
BindAddress: lk.APIServerInsecureAddress.String(),
BindAddress: lk.APIServerAddress.String(),
Mode: componentconfig.ProxyModeIPTables,
FeatureGates: lk.FeatureGates,
// Disable the healthz check
......@@ -60,7 +62,7 @@ func StartProxyServer(lk LocalkubeServer) func() error {
return func() error {
// Creating this config requires the API Server to be up, so do it in the start function itself.
server, err := kubeproxy.NewProxyServer(config, false, runtime.NewScheme(), lk.GetAPIServerInsecureURL())
server, err := kubeproxy.NewProxyServer(config, false, runtime.NewScheme(), "")
if err != nil {
panic(err)
}
......
......@@ -17,6 +17,8 @@ limitations under the License.
package localkube
import (
"crypto/tls"
"crypto/x509"
"io/ioutil"
"net/http"
......@@ -25,10 +27,33 @@ import (
type HealthCheck func() bool
func healthCheck(addr string) HealthCheck {
func healthCheck(addr string, lk LocalkubeServer) HealthCheck {
return func() bool {
glog.Infof("Performing healthcheck on %s\n", addr)
resp, err := http.Get(addr)
cert, err := tls.LoadX509KeyPair(lk.GetPublicKeyCertPath(), lk.GetPrivateKeyCertPath())
if err != nil {
glog.Error(err)
return false
}
// Load CA cert
caCert, err := ioutil.ReadFile(lk.GetCAPublicKeyCertPath())
if err != nil {
glog.Warning(err)
return false
}
caCertPool := x509.NewCertPool()
caCertPool.AppendCertsFromPEM(caCert)
tlsConfig := &tls.Config{
Certificates: []tls.Certificate{cert},
RootCAs: caCertPool,
}
tlsConfig.BuildNameToCertificate()
transport := &http.Transport{TLSClientConfig: tlsConfig}
client := &http.Client{Transport: transport}
resp, err := client.Get(addr)
if err != nil {
glog.Errorf("Error performing healthcheck: %s", err)
return false
......
......@@ -17,15 +17,21 @@ limitations under the License.
package localkube
import (
"crypto/tls"
"crypto/x509"
"io"
"io/ioutil"
"net/http"
"net/http/httptest"
"os"
"testing"
"k8s.io/minikube/pkg/minikube/tests"
)
func TestBasicHealthCheck(t *testing.T) {
tests := []struct {
tcs := []struct {
body string
statusCode int
shouldSucceed bool
......@@ -34,17 +40,43 @@ func TestBasicHealthCheck(t *testing.T) {
{"notok", 200, false},
}
for _, tc := range tests {
tempDir := tests.MakeTempDir()
defer os.RemoveAll(tempDir)
lk := LocalkubeServer{LocalkubeDirectory: tempDir}
lk.GenerateCerts()
cert, err := tls.LoadX509KeyPair(lk.GetPublicKeyCertPath(), lk.GetPrivateKeyCertPath())
if err != nil {
t.Fatalf("Unable to load server certs.")
}
caCert, err := ioutil.ReadFile(lk.GetCAPublicKeyCertPath())
if err != nil {
t.Fatalf("Unable to load CA certs.")
}
caCertPool := x509.NewCertPool()
caCertPool.AppendCertsFromPEM(caCert)
tls := tls.Config{
Certificates: []tls.Certificate{cert},
ClientCAs: caCertPool,
}
tls.BuildNameToCertificate()
for _, tc := range tcs {
// Do this in a func so we can use defer.
doTest := func() {
handler := func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(tc.statusCode)
io.WriteString(w, tc.body)
}
server := httptest.NewServer(http.HandlerFunc(handler))
server := httptest.NewUnstartedServer(http.HandlerFunc(handler))
defer server.Close()
server.TLS = &tls
server.StartTLS()
hcFunc := healthCheck(server.URL)
hcFunc := healthCheck(server.URL, lk)
result := hcFunc()
if result != tc.shouldSucceed {
t.Errorf("Expected healthcheck to return %v. Got %v", result, tc.shouldSucceed)
......
......@@ -19,6 +19,7 @@ package localkube
import (
scheduler "k8s.io/kubernetes/plugin/cmd/kube-scheduler/app"
"k8s.io/kubernetes/plugin/cmd/kube-scheduler/app/options"
"k8s.io/minikube/pkg/util"
)
func (lk LocalkubeServer) NewSchedulerServer() Server {
......@@ -29,7 +30,7 @@ func StartSchedulerServer(lk LocalkubeServer) func() error {
config := options.NewSchedulerServer()
// master details
config.Master = lk.GetAPIServerInsecureURL()
config.Kubeconfig = util.DefaultKubeConfigPath
// defaults from command
config.EnableProfiling = true
......
......@@ -19,10 +19,8 @@ package localkube
import (
"errors"
"fmt"
"net"
"os"
"path"
"strconv"
"time"
"github.com/golang/glog"
......@@ -34,7 +32,8 @@ import (
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/pkg/api/v1"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
"k8s.io/minikube/pkg/util"
)
const (
......@@ -123,12 +122,12 @@ func (lk LocalkubeServer) NewStorageProvisionerServer() Server {
func StartStorageProvisioner(lk LocalkubeServer) func() error {
// Create an InClusterConfig and use it to create a client for the controller
// to use to communicate with Kubernetes
config := rest.Config{Host: net.JoinHostPort("localhost", strconv.Itoa(lk.APIServerInsecurePort))}
return func() error {
clientset, err := kubernetes.NewForConfig(&config)
config, err := clientcmd.BuildConfigFromFlags("", util.DefaultKubeConfigPath)
if err != nil {
return err
}
clientset, err := kubernetes.NewForConfig(config)
if err != nil {
glog.Fatalf("Failed to create client: %v", err)
}
......
......@@ -28,12 +28,12 @@ import (
)
type Addon struct {
Assets []*MemoryAsset
Assets []*BinDataAsset
enabled bool
addonName string
}
func NewAddon(assets []*MemoryAsset, enabled bool, addonName string) *Addon {
func NewAddon(assets []*BinDataAsset, enabled bool, addonName string) *Addon {
a := &Addon{
Assets: assets,
enabled: enabled,
......@@ -55,107 +55,107 @@ func (a *Addon) IsEnabled() (bool, error) {
}
var Addons = map[string]*Addon{
"addon-manager": NewAddon([]*MemoryAsset{
NewMemoryAsset(
"addon-manager": NewAddon([]*BinDataAsset{
NewBinDataAsset(
"deploy/addons/addon-manager.yaml",
"/etc/kubernetes/manifests/",
"addon-manager.yaml",
"0640"),
}, true, "addon-manager"),
"dashboard": NewAddon([]*MemoryAsset{
NewMemoryAsset(
"dashboard": NewAddon([]*BinDataAsset{
NewBinDataAsset(
"deploy/addons/dashboard/dashboard-rc.yaml",
constants.AddonsPath,
"dashboard-rc.yaml",
"0640"),
NewMemoryAsset(
NewBinDataAsset(
"deploy/addons/dashboard/dashboard-svc.yaml",
constants.AddonsPath,
"dashboard-svc.yaml",
"0640"),
}, true, "dashboard"),
"default-storageclass": NewAddon([]*MemoryAsset{
NewMemoryAsset(
"default-storageclass": NewAddon([]*BinDataAsset{
NewBinDataAsset(
"deploy/addons/storageclass/storageclass.yaml",
constants.AddonsPath,
"storageclass.yaml",
"0640"),
}, true, "default-storageclass"),
"kube-dns": NewAddon([]*MemoryAsset{
NewMemoryAsset(
"kube-dns": NewAddon([]*BinDataAsset{
NewBinDataAsset(
"deploy/addons/kube-dns/kube-dns-controller.yaml",
constants.AddonsPath,
"kube-dns-controller.yaml",
"0640"),
NewMemoryAsset(
NewBinDataAsset(
"deploy/addons/kube-dns/kube-dns-cm.yaml",
constants.AddonsPath,
"kube-dns-cm.yaml",
"0640"),
NewMemoryAsset(
NewBinDataAsset(
"deploy/addons/kube-dns/kube-dns-svc.yaml",
constants.AddonsPath,
"kube-dns-svc.yaml",
"0640"),
}, true, "kube-dns"),
"heapster": NewAddon([]*MemoryAsset{
NewMemoryAsset(
"heapster": NewAddon([]*BinDataAsset{
NewBinDataAsset(
"deploy/addons/heapster/influxGrafana-rc.yaml",
constants.AddonsPath,
"influxGrafana-rc.yaml",
"0640"),
NewMemoryAsset(
NewBinDataAsset(
"deploy/addons/heapster/grafana-svc.yaml",
constants.AddonsPath,
"grafana-svc.yaml",
"0640"),
NewMemoryAsset(
NewBinDataAsset(
"deploy/addons/heapster/influxdb-svc.yaml",
constants.AddonsPath,
"influxdb-svc.yaml",
"0640"),
NewMemoryAsset(
NewBinDataAsset(
"deploy/addons/heapster/heapster-rc.yaml",
constants.AddonsPath,
"heapster-rc.yaml",
"0640"),
NewMemoryAsset(
NewBinDataAsset(
"deploy/addons/heapster/heapster-svc.yaml",
constants.AddonsPath,
"heapster-svc.yaml",
"0640"),
}, false, "heapster"),
"ingress": NewAddon([]*MemoryAsset{
NewMemoryAsset(
"ingress": NewAddon([]*BinDataAsset{
NewBinDataAsset(
"deploy/addons/ingress/ingress-configmap.yaml",
constants.AddonsPath,
"ingress-configmap.yaml",
"0640"),
NewMemoryAsset(
NewBinDataAsset(
"deploy/addons/ingress/ingress-rc.yaml",
constants.AddonsPath,
"ingress-rc.yaml",
"0640"),
NewMemoryAsset(
NewBinDataAsset(
"deploy/addons/ingress/ingress-svc.yaml",
constants.AddonsPath,
"ingress-svc.yaml",
"0640"),
}, false, "ingress"),
"registry": NewAddon([]*MemoryAsset{
NewMemoryAsset(
"registry": NewAddon([]*BinDataAsset{
NewBinDataAsset(
"deploy/addons/registry/registry-rc.yaml",
constants.AddonsPath,
"registry-rc.yaml",
"0640"),
NewMemoryAsset(
NewBinDataAsset(
"deploy/addons/registry/registry-svc.yaml",
constants.AddonsPath,
"registry-svc.yaml",
"0640"),
}, false, "registry"),
"registry-creds": NewAddon([]*MemoryAsset{
NewMemoryAsset(
"registry-creds": NewAddon([]*BinDataAsset{
NewBinDataAsset(
"deploy/addons/registry-creds/registry-creds-rc.yaml",
constants.AddonsPath,
"registry-creds-rc.yaml",
......
......@@ -106,8 +106,35 @@ type MemoryAsset struct {
BaseAsset
}
func NewMemoryAsset(assetName, targetDir, targetName, permissions string) *MemoryAsset {
func (m *MemoryAsset) GetLength() int {
return m.Length
}
func (m *MemoryAsset) Read(p []byte) (int, error) {
return m.reader.Read(p)
}
func NewMemoryAsset(d []byte, targetDir, targetName, permissions string) *MemoryAsset {
m := &MemoryAsset{
BaseAsset{
TargetDir: targetDir,
TargetName: targetName,
Permissions: permissions,
},
}
m.data = d
m.Length = len(m.data)
m.reader = bytes.NewReader(m.data)
return m
}
type BinDataAsset struct {
BaseAsset
}
func NewBinDataAsset(assetName, targetDir, targetName, permissions string) *BinDataAsset {
m := &BinDataAsset{
BaseAsset{
AssetName: assetName,
TargetDir: targetDir,
......@@ -119,7 +146,7 @@ func NewMemoryAsset(assetName, targetDir, targetName, permissions string) *Memor
return m
}
func (m *MemoryAsset) loadData() error {
func (m *BinDataAsset) loadData() error {
contents, err := Asset(m.AssetName)
if err != nil {
return err
......@@ -130,11 +157,11 @@ func (m *MemoryAsset) loadData() error {
return nil
}
func (m *MemoryAsset) GetLength() int {
func (m *BinDataAsset) GetLength() int {
return m.Length
}
func (m *MemoryAsset) Read(p []byte) (int, error) {
func (m *BinDataAsset) Read(p []byte) (int, error) {
return m.reader.Read(p)
}
......
......@@ -38,12 +38,16 @@ import (
"github.com/docker/machine/libmachine/state"
"github.com/golang/glog"
"github.com/pkg/errors"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/tools/clientcmd/api"
"k8s.io/client-go/tools/clientcmd/api/latest"
"k8s.io/minikube/pkg/minikube/assets"
cfg "k8s.io/minikube/pkg/minikube/config"
"k8s.io/minikube/pkg/minikube/constants"
"k8s.io/minikube/pkg/minikube/sshutil"
"k8s.io/minikube/pkg/util"
"k8s.io/minikube/pkg/util/kubeconfig"
)
var (
......@@ -221,7 +225,7 @@ func UpdateCluster(d drivers.Driver, config KubernetesConfig) error {
return errors.Wrap(err, "Error updating localkube from uri")
}
} else {
localkubeFile = assets.NewMemoryAsset("out/localkube", "/usr/local/bin", "localkube", "0777")
localkubeFile = assets.NewBinDataAsset("out/localkube", "/usr/local/bin", "localkube", "0777")
}
copyableFiles = append(copyableFiles, localkubeFile)
......@@ -301,6 +305,23 @@ func SetupCerts(d drivers.Driver, apiServerName string, clusterDnsDomain string)
copyableFiles = append(copyableFiles, certFile)
}
kubeCfgSetup := &kubeconfig.KubeConfigSetup{
ClusterName: cfg.GetMachineName(),
ClusterServerAddress: "https://localhost:8443",
ClientCertificate: filepath.Join(util.DefaultCertPath, "apiserver.crt"),
ClientKey: filepath.Join(util.DefaultCertPath, "apiserver.key"),
CertificateAuthority: filepath.Join(util.DefaultCertPath, "ca.crt"),
KeepContext: false,
}
kubeCfg := api.NewConfig()
kubeconfig.PopulateKubeConfig(kubeCfgSetup, kubeCfg)
data, err := runtime.Encode(latest.Codec, kubeCfg)
kubeCfgFile := assets.NewMemoryAsset(data,
util.DefaultLocalkubeDirectory, "kubeconfig", "0644")
copyableFiles = append(copyableFiles, kubeCfgFile)
if d.DriverName() == "none" {
// transfer files to correct place on filesystem
for _, f := range copyableFiles {
......
......@@ -93,7 +93,7 @@ func (d *Driver) GetSSHUsername() string {
}
func (d *Driver) GetURL() (string, error) {
return "127.0.0.1:8080", nil
return "tcp://127.0.0.1:2376", nil
}
func (d *Driver) GetState() (state.State, error) {
......
......@@ -21,6 +21,7 @@ const (
APIServerPort = 8443
DefaultLocalkubeDirectory = "/var/lib/localkube"
DefaultCertPath = DefaultLocalkubeDirectory + "/certs/"
DefaultKubeConfigPath = DefaultLocalkubeDirectory + "/kubeconfig"
DefaultServiceClusterIP = "10.0.0.1"
DefaultDNSDomain = "cluster.local"
DefaultDNSIP = "10.0.0.10"
......@@ -28,5 +29,5 @@ const (
)
func GetAlternateDNS(domain string) []string {
return []string{"kubernetes.default.svc." + domain, "kubernetes.default.svc", "kubernetes.default", "kubernetes"}
return []string{"kubernetes.default.svc." + domain, "kubernetes.default.svc", "kubernetes.default", "kubernetes", "localhost"}
}
......@@ -66,43 +66,48 @@ func (k *KubeConfigSetup) GetKubeConfigFile() string {
return k.kubeConfigFile.Load().(string)
}
// SetupKubeconfig reads config from disk, adds the minikube settings, and writes it back.
// activeContext is true when minikube is the CurrentContext
// If no CurrentContext is set, the given name will be used.
func SetupKubeConfig(cfg *KubeConfigSetup) error {
glog.Infoln("Using kubeconfig: ", cfg.GetKubeConfigFile())
// read existing config or create new if does not exist
config, err := ReadConfigOrNew(cfg.GetKubeConfigFile())
if err != nil {
return err
}
// PopulateKubeConfig populates an api.Config object.
func PopulateKubeConfig(cfg *KubeConfigSetup, kubecfg *api.Config) {
clusterName := cfg.ClusterName
cluster := api.NewCluster()
cluster.Server = cfg.ClusterServerAddress
cluster.CertificateAuthority = cfg.CertificateAuthority
config.Clusters[clusterName] = cluster
kubecfg.Clusters[clusterName] = cluster
// user
userName := cfg.ClusterName
user := api.NewAuthInfo()
user.ClientCertificate = cfg.ClientCertificate
user.ClientKey = cfg.ClientKey
config.AuthInfos[userName] = user
kubecfg.AuthInfos[userName] = user
// context
contextName := cfg.ClusterName
context := api.NewContext()
context.Cluster = cfg.ClusterName
context.AuthInfo = userName
config.Contexts[contextName] = context
kubecfg.Contexts[contextName] = context
// Only set current context to minikube if the user has not used the keepContext flag
if !cfg.KeepContext {
config.CurrentContext = contextName
kubecfg.CurrentContext = cfg.ClusterName
}
}
// SetupKubeConfig reads config from disk, adds the minikube settings, and writes it back.
// activeContext is true when minikube is the CurrentContext
// If no CurrentContext is set, the given name will be used.
func SetupKubeConfig(cfg *KubeConfigSetup) error {
glog.Infoln("Using kubeconfig: ", cfg.GetKubeConfigFile())
// read existing config or create new if does not exist
config, err := ReadConfigOrNew(cfg.GetKubeConfigFile())
if err != nil {
return err
}
PopulateKubeConfig(cfg, config)
// write back to disk
if err := WriteConfig(config, cfg.GetKubeConfigFile()); err != nil {
return err
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册