/* Copyright 2020 KubeSphere Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package v1alpha1 import ( "bytes" "context" "encoding/json" "fmt" "net/http" "net/http/httptest" "net/url" "testing" "github.com/google/go-cmp/cmp" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/cli-runtime/pkg/printers" k8s "k8s.io/client-go/kubernetes" k8sfake "k8s.io/client-go/kubernetes/fake" "k8s.io/client-go/tools/clientcmd" "sigs.k8s.io/controller-runtime/pkg/envtest" "kubesphere.io/kubesphere/pkg/apis/cluster/v1alpha1" "kubesphere.io/kubesphere/pkg/client/clientset/versioned/fake" "kubesphere.io/kubesphere/pkg/informers" "kubesphere.io/kubesphere/pkg/version" ) const ( proxyAddress = "http://139.198.121.121:8080" agentImage = "kubesphere/tower:v1.0" proxyService = "tower.kubesphere-system.svc" ) var cluster = &v1alpha1.Cluster{ ObjectMeta: metav1.ObjectMeta{ Name: "gondor", }, Spec: v1alpha1.ClusterSpec{ Connection: v1alpha1.Connection{ Type: v1alpha1.ConnectionTypeProxy, Token: "randomtoken", }, }, } var service = &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "tower", Namespace: "kubesphere-system", }, Spec: corev1.ServiceSpec{ Ports: []corev1.ServicePort{ { Port: 8080, Protocol: corev1.ProtocolTCP, }, }, }, Status: corev1.ServiceStatus{ LoadBalancer: corev1.LoadBalancerStatus{ Ingress: []corev1.LoadBalancerIngress{ { IP: "139.198.121.121", Hostname: "foo.bar", }, }, }, }, } var hostMap = map[string]string{ "kubesphere.yaml": ` monitoring: endpoint: http://prometheus-operated.kubesphere-monitoring-system.svc:9090 authentication: jwtSecret: sQh3JOqNbmci6Gu94TeV10AY7ipltwjp oauthOptions: accessTokenMaxAge: 0s accessTokenInactivityTimeout: 0s `, } var memberMap = map[string]string{ "kubesphere.yaml": ` monitoring: endpoint: http://prometheus-operated.kubesphere-monitoring-system.svc:9090 authentication: jwtSecret: sQh3JOqNbmci6Gu94TeV10AY7ipltwj oauthOptions: accessTokenMaxAge: 0s accessTokenInactivityTimeout: 0s `, } var hostCm = &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Namespace: KubesphereNamespace, Name: KubeSphereConfigName, }, Data: hostMap, } var memberCm = &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Namespace: KubesphereNamespace, Name: KubeSphereConfigName, }, Data: memberMap, } var ksApiserverDeploy = ` { "apiVersion": "apps/v1", "kind": "Deployment", "metadata": { "labels": { "app": "ks-apiserver", "tier": "backend", "version": "v3.0.0" }, "name": "ks-apiserver", "namespace": "kubesphere-system" }, "spec": { "replicas": 1, "selector": { "matchLabels": { "app": "ks-apiserver", "tier": "backend", "version": "v3.0.0" } }, "template": { "metadata": { "creationTimestamp": null, "labels": { "app": "ks-apiserver", "tier": "backend", "version": "v3.0.0" } }, "spec": { "containers": [ { "command": [ "ks-apiserver", "--logtostderr=true" ], "image": "kubesphere/ks-apiserver:v3.0.0", "name": "ks-apiserver" } ] } } } }` var expected = `apiVersion: apps/v1 kind: Deployment metadata: creationTimestamp: null name: cluster-agent namespace: kubesphere-system spec: selector: matchLabels: app: agent app.kubernetes.io/part-of: tower strategy: {} template: metadata: creationTimestamp: null labels: app: agent app.kubernetes.io/part-of: tower spec: containers: - command: - /agent - --name=gondor - --token=randomtoken - --proxy-server=http://139.198.121.121:8080 - --keepalive=10s - --kubesphere-service=ks-apiserver.kubesphere-system.svc:80 - --kubernetes-service=kubernetes.default.svc:443 - --v=0 image: kubesphere/tower:v1.0 name: agent resources: limits: cpu: "1" memory: 200M requests: cpu: 100m memory: 100M serviceAccountName: kubesphere status: {} ` func TestGeranteAgentDeployment(t *testing.T) { k8sclient := k8sfake.NewSimpleClientset(service) ksclient := fake.NewSimpleClientset(cluster) informersFactory := informers.NewInformerFactories(k8sclient, ksclient, nil, nil, nil, nil) informersFactory.KubernetesSharedInformerFactory().Core().V1().Services().Informer().GetIndexer().Add(service) informersFactory.KubeSphereSharedInformerFactory().Cluster().V1alpha1().Clusters().Informer().GetIndexer().Add(cluster) directConnectionCluster := cluster.DeepCopy() directConnectionCluster.Spec.Connection.Type = v1alpha1.ConnectionTypeDirect var testCases = []struct { description string expectingError bool expectedError error cluster *v1alpha1.Cluster expected string }{ { description: "test normal case", expectingError: false, expected: expected, cluster: cluster, }, { description: "test direct connection cluster", expectingError: true, expectedError: errClusterConnectionIsNotProxy, cluster: directConnectionCluster, }, } for _, testCase := range testCases { t.Run(testCase.description, func(t *testing.T) { h := newHandler(informersFactory.KubernetesSharedInformerFactory(), informersFactory.KubeSphereSharedInformerFactory(), proxyService, "", agentImage) var buf bytes.Buffer err := h.populateProxyAddress() if err != nil { t.Error(err) } err = h.generateDefaultDeployment(testCase.cluster, &buf) if testCase.expectingError { if err == nil { t.Fatalf("expecting error %v, got nil", testCase.expectedError) } else if err != testCase.expectedError { t.Fatalf("expecting error %v, got %v", testCase.expectedError, err) } } if diff := cmp.Diff(testCase.expected, buf.String()); len(diff) != 0 { t.Errorf("%T, got +, expected -, %s", testCase.expected, diff) } }) } } func TestInnerGenerateAgentDeployment(t *testing.T) { h := &handler{ proxyAddress: proxyAddress, agentImage: agentImage, yamlPrinter: &printers.YAMLPrinter{}, } var buf bytes.Buffer err := h.generateDefaultDeployment(cluster, &buf) if err != nil { t.Error(err) } if diff := cmp.Diff(buf.String(), expected); len(diff) != 0 { t.Error(diff) } } var base64EncodedKubeConfig = ` apiVersion: v1 clusters: - cluster: certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUN5RENDQWJDZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKYzNCb1pYSmxNQjRYRFRJd01EVXlOekEzTWpFME1Gb1hEVE13TURVeU5UQTNNakUwTUZvd0ZURVRNQkVHQTFVRQpBeE1LYTNWaVpYTndhR1Z5WlRDQ0FTSXdEUVlKS29aSWh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTWVxCjRwNldVQVJmeUtRcEpZM1ZGcEVhT3Y5REZBQkZhQXBpbUswQTBBL05KV25yaGl0MjhnL3ZXMno5bmxkeHQwZzgKc1J0ZEp4TUxmOHF5WkEramZudU5jUDBLUTJ5a3VxZE41c29MdE1TUmt1K1dHZFNkWlJvTVpEakdKbHRSUEdVRQpnOHd6OE9zdWRMcmZ5Zlcxdy8vUFRPWnRLNmsrNUhQZGtuWU5KcU9UZGJDTEVma2RZbFB1ZXNZTTFKamRacXlNClJQRDM1RXpUSEowR05jYlBzbUlyZ05WZGR4Nmh5RmIxTFZ0QXRqa0tWY2lNR1k1UlFTQWlQdVVGaGQreUcrOHUKUWlqQlgvV09ESlFOelJCQWFPdWRWdURqSkhIZ0lBV3FzcC9qSllWQkdRYnNad3djTTRUSEJPb2k3N1krYVR3SAphcHRaMitVMzgwbFY4d0tJR3NFQ0F3RUFBYU1qTUNFd0RnWURWUjBQQVFIL0JBUURBZ0trTUE4R0ExVWRFd0VCCi93UUZNQU1CQWY4d0RRWUpLb1pJaHZjTkFRRUxCUUFEZ2dFQkFDVjY0VVpCZkcxc1d4ampJWTZ5Vm0rdzhxTFgKS0ZYanR3VElGb1Y3WENOT2RDbEQ0a2NsN1pyTnl5UEtUaTFOV3BDbUVzSXZPVjRSRXV6a2ZWZ2Rzc0tHL2dYVwpGNGV2dStZTFh1UEk1RHJFejNGaW5OMGxVcFNZMVg3b1Y5N2JyU1lmdE53aWJQbUVFTEVHbWJvMnNHS3VoL21BCjRJZ0tmZklqWVdSYjE3TlZLb2s0am1WTnhkMmNCL09GeFlsY2xndlc2THpxc1BDdnpWbDdDRWErRElyNHZLamgKRlhvOERXejMzclUySCs0RjNMOThzWGE1OWNITWZPb1kzZ3pzUE5LYnQwbWsxeTNOUmZocTZYTTJXemkrUFZkcAppbUVqUlY4UUl5c3Zhc25sTjFIY1FtcWtMaFdLYlRoOEc5MGhiTzlwdTFRNE0wMmZ3STg0ZVlSUWZNUT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= server: 127.0.0.1:6578 name: kubernetes contexts: - context: cluster: kubernetes user: kubernetes-admin name: kubernetes-admin@kubernetes current-context: kubernetes-admin@kubernetes kind: Config preferences: {} users: - name: kubernetes-admin user: client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM4akNDQWRxZ0F3SUJBZ0lJZkcyYVlKYnRYT2N3RFFZSktvWklodmNOQVFFTEJRQXdGVEVUTUJFR0ExVUUKQXhNS2EzVmlaWE53YUdWeVpUQWVGdzB5TURBMU1qY3dOekl4TkRCYUZ3MHpNREExTWpVd056SXhOREJhTURReApGekFWQmdOVkJBb1REbk41YzNSbGJUcHRZWE4wWlhKek1Sa3dGd1lEVlFRREV4QnJkV0psY201bGRHVnpMV0ZrCmJXbHVNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DQVE4QU1JSUJDZ0tDQVFFQXRiR3JpeFBxOWF6QU1wNGsKM0VsaElmanhOZXhkY2VORGJENkFMMXVGSWJUMk1uVnRXeFFXOU5ueVFwVU9NMzN4clNJMkJlVW1uU0U5RlR6cAo1bTBmaGVBOGhtRHRSZWVOckNxVnBJQy9kNUV2eDJ3NTJ2NXVCYUNXd09ZUGpQMi9uL0Y3THQwMmN3TkFXWE5pCmFRTi9uRGl6TjJrRXFoSmZiL0tyNGx3eEEzTDExVXJhMDNTRUp0U3FXSzBKQ1pnL0lzUnRFNFFqZXp1WWhiVWkKcWU0TmdqZjN1ZFBMMXQyeVpCK3hSTE1sNTFqenhYaXQ0U2pHSFJ2UEt1VHlkc1AxdEtINXdYdnhqaEZTZjN3UQpMSHFQd3hQWXVKSG5MWlhPaElnTnEzNnk4Rlp0WkdQRUhDVDFlUEh5cWhQaEFZZGlBRlRXT3pRN2FlS1puZWJzClpSblJMUUlEQVFBQm95Y3dKVEFPQmdOVkhROEJBZjhFQkFNQ0JhQXdFd1lEVlIwbEJBd3dDZ1lJS3dZQkJRVUgKQXdJd0RRWUpLb1pJaHZjTkFRRUxCUUFEZ2dFQkFCZGNMN2I4akVuYjRSSndqZ3ZrVUxSQXJLVGk1WFhRSzllUwpsMlNkNEF3Ris5QWg3WElUazJFQ3J0WVR2WEM0K2trS1BxK2FuYjlOOFptenA0R0ovd3N5VEN2dHo5eGlQMTd3ClFCUXREdFA3eS9venlLc24rUzYvSDg2Y0JqdGZGT0dMYm5CekRBY3J0S3YyeUxxY3pyNlYzSDBnZDI3MVdlSkQKcU81U3czSEZoTHhERDRXSVVSQnFLOTJPVUhnSzVQOWRHaWdkK2MvQ0xWMFdJS1kzN3JGR2MrU3VUa2JOQXNmaQpmVmhBYXRsYlpQdE1QekJoV1hkM0JWcTMxTmtBM1F4aWUzdWc4Tm9OZ0czUHFPanJkZHA2aEs0Sk5wMGtkSGFvClZHRXMxcUdzOGJjekxNTTVzeUdkSndNbXVNUEdnaGxheUZrZlJ2RmpZWDlwbllJUStpTT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= client-key-data: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFcEFJQkFBS0NBUUVBdGJHcml4UHE5YXpBTXA0azNFbGhJZmp4TmV4ZGNlTkRiRDZBTDF1RkliVDJNblZ0Cld4UVc5Tm55UXBVT00zM3hyU0kyQmVVbW5TRTlGVHpwNW0wZmhlQThobUR0UmVlTnJDcVZwSUMvZDVFdngydzUKMnY1dUJhQ1d3T1lQalAyL24vRjdMdDAyY3dOQVdYTmlhUU4vbkRpek4ya0VxaEpmYi9LcjRsd3hBM0wxMVVyYQowM1NFSnRTcVdLMEpDWmcvSXNSdEU0UWplenVZaGJVaXFlNE5namYzdWRQTDF0MnlaQit4UkxNbDUxanp4WGl0CjRTakdIUnZQS3VUeWRzUDF0S0g1d1h2eGpoRlNmM3dRTEhxUHd4UFl1SkhuTFpYT2hJZ05xMzZ5OEZadFpHUEUKSENUMWVQSHlxaFBoQVlkaUFGVFdPelE3YWVLWm5lYnNaUm5STFFJREFRQUJBb0lCQVFDSDFnQ054YUpQY1l0dgpURlA2Yk5HMWVFdTlLS3pqekNoSDhLSWN4YXRPZTkvajhXNkVQUXk4bVlSSXl1OEhDQTE2aHEwazB5Qi9NSzVlCkJtQkg2U1U4RFZ5eWloeFp1cmRzRTVvMGxoeU80M2g0K3l4MTBPbW9RMXJ4ZEE0RU5tRGd6c1J0VU95NEo2SWcKUGVkQTQyQ3dCcVBWdFNuTGpGalZkUE9VRTZDQkZsa01nTVR2R1I5SE9EaVovdmhqRXhFQWg0UFFmMkl3ZXZhVwpxTXFsOVJNVFprVEUyVUJEbHZMWVRNcG5pdGlSSWdLdEdmSk1WTW81WWR4SWpOMXRnUHcwZmN5ZXkrM2JhdDZQCkVUSjk5K21KNDJMZGYzaTlNNkNvTDVaa1BjcEE0ckVSR1RKamtaT3FESzhPMVZXSDNtQ21WYTFyVGN0bmpJbVMKaGRGWElJWUJBb0dCQU84R25uekhzSVFXaExBSkw5U3VWWjhBODl5WHZuWHBFYUUxaHBRdEViRFlEaTBPcGg1UAp0cVVISlZ0T0ViaTYvai9tY1pwTnNNVzR2QitiV3hkbDA4U0g1Qnp2eW9qOEkvbUlVNDFCQkYya3hxMk8yNlJRCjRkdFQrN3NWUUFCT1ArUE0vcWtMOWVKdFI3blYwamRGd3BvcmtzcFo5cE9BZHNnUGVnUzlxUHFOQW9HQkFNS1kKeU5OQWdYRjhBdEV3ZEk5OVB2ZTVoYzN6QjZUclVOMStYNVNUZWNYWnFVSUNwMlJGTFNtL3RiVkRuay9VaXBXeQpYSjgzT084VGFIUjIrYk5OZU5NK2REK0diakVWdDNVMk1XSzN2azFjcy9oWjZrK0F2aE9iSXBrLzR2VEVRUW1FCjh4bkl5bkpPNWJleFQybkEyUWVYblVPbjZzdWtxVnk3YlBLSnNOa2hBb0dBV1YzNUxhQWZvQk1uUXdYOFN5RnYKUThiQVptNlp1RTRPMkY1QjFlN1AyWFcrUHh4bUFaaytLWTkxYVNEVVFXUXdvVVdRbmVlRU96aXBwWXVaVURNegpMUnk5cmcvOWdwLzY5MVlBSHlUNjgrUWlvRXQwVllna0diUFp2NFhmYXYzV3AxNUNySU9iU0RBaGpCcWt3U09rCjhhMXU4WmNYT09qa0FFTEJGVHF3RGhVQ2dZQVVqQjFvY1A4NkJHWW53SDRPU0tORmRRbHozWjJKQkcvZGMyS1UKUlo0dURmV1pTcjV5RC92YzFLbFRJbmlzNVR4YzRpQjFqMWNycDFqNE16ZmFmdXVySW9VVDBCWUNpTkIrUitLZgpFZGUrUTNPZFhhRW9FK2YrR2Z0bFF5R3J4cTAzWEJwdk5veHAxWHJjRXBUWURjemN5RjJLcjBoVGlHZDVxekN0Cnkyd3BBUUtCZ1FDWTFKdlp0YkFlZll2RXQ4c3JETVZsb1FOWVVKTURxWnBLenpqR0w5S3FqdlFwbjJOL2EwNmsKUzNsMndWWExXNy8xb0RMV2gxZGZLbUJlUlJhZUxJMXIvS3FyeTRjUStiVEIzTzBwS2R3WjFFYXNQajBDTnlaUAp0YnFkOEFWa09pRkFNYlRWaVZuR3RzSnJ3c1V2N1NSTUU4ckloMVJva0ZtRGRpN0J1UVl5emc9PQotLS0tLUVORCBSU0EgUFJJVkFURSBLRVktLS0tLQo= ` func TestValidateKubeConfig(t *testing.T) { k8sclient := k8sfake.NewSimpleClientset(service) ksclient := fake.NewSimpleClientset(cluster) informersFactory := informers.NewInformerFactories(k8sclient, ksclient, nil, nil, nil, nil) informersFactory.KubernetesSharedInformerFactory().Core().V1().Services().Informer().GetIndexer().Add(service) informersFactory.KubeSphereSharedInformerFactory().Cluster().V1alpha1().Clusters().Informer().GetIndexer().Add(cluster) h := newHandler(informersFactory.KubernetesSharedInformerFactory(), informersFactory.KubeSphereSharedInformerFactory(), proxyService, "", agentImage) config, err := loadKubeConfigFromBytes([]byte(base64EncodedKubeConfig)) if err != nil { t.Fatal(err) } // config.Host is schemaless, we need to add schema manually u, err := url.Parse(fmt.Sprintf("http://%s", config.Host)) if err != nil { t.Fatal(err) } // we need to specify apiserver port to match above kubeconfig env := &envtest.Environment{ Config: config, ControlPlane: envtest.ControlPlane{ APIServer: &envtest.APIServer{ Args: envtest.DefaultKubeAPIServerFlags, URL: u, }, }, } cfg, err := env.Start() if err != nil { t.Log(cfg) t.Fatal(err) } defer func() { _ = env.Stop() }() err = h.validateKubeConfig([]byte(base64EncodedKubeConfig)) if err != nil { t.Fatal(err) } } var ver = version.Get() func endpoint(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(ver) } func TestValidateKubeSphereEndpoint(t *testing.T) { svr := httptest.NewServer(http.HandlerFunc(endpoint)) defer svr.Close() got, err := validateKubeSphereAPIServer(svr.URL, nil) if err != nil { t.Fatal(err) } if diff := cmp.Diff(&ver, got); len(diff) != 0 { t.Errorf("%T +got, -expected %v", ver, diff) } } func TestValidateMemberClusterConfiguration(t *testing.T) { k8sclient := k8sfake.NewSimpleClientset(service) ksclient := fake.NewSimpleClientset(cluster) informersFactory := informers.NewInformerFactories(k8sclient, ksclient, nil, nil, nil, nil) informersFactory.KubernetesSharedInformerFactory().Core().V1().Services().Informer().GetIndexer().Add(service) informersFactory.KubeSphereSharedInformerFactory().Cluster().V1alpha1().Clusters().Informer().GetIndexer().Add(cluster) informersFactory.KubernetesSharedInformerFactory().Core().V1().ConfigMaps().Informer().GetIndexer().Add(hostCm) h := newHandler(informersFactory.KubernetesSharedInformerFactory(), informersFactory.KubeSphereSharedInformerFactory(), proxyService, "", agentImage) config, err := loadKubeConfigFromBytes([]byte(base64EncodedKubeConfig)) if err != nil { t.Fatal(err) } // config.Host is schemaless, we need to add schema manually u, err := url.Parse(fmt.Sprintf("http://%s", config.Host)) if err != nil { t.Fatal(err) } // we need to specify apiserver port to match above kubeconfig env := &envtest.Environment{ Config: config, ControlPlane: envtest.ControlPlane{ APIServer: &envtest.APIServer{ Args: envtest.DefaultKubeAPIServerFlags, URL: u, }, }, } cfg, err := env.Start() if err != nil { t.Log(cfg) t.Fatal(err) } defer func() { _ = env.Stop() }() addMemberClusterResource(hostCm, t) err = h.validateMemberClusterConfiguration([]byte(base64EncodedKubeConfig)) if err != nil { t.Fatal(err) } addMemberClusterResource(memberCm, t) err = h.validateMemberClusterConfiguration([]byte(base64EncodedKubeConfig)) if err == nil { t.Fatal() } t.Log(err) } func addMemberClusterResource(targetCm *corev1.ConfigMap, t *testing.T) { con, err := clientcmd.NewClientConfigFromBytes([]byte(base64EncodedKubeConfig)) if err != nil { t.Fatal(err) } cli, err := con.ClientConfig() if err != nil { t.Fatal(err) } c, err := k8s.NewForConfig(cli) if err != nil { t.Fatal(err) } _, err = c.CoreV1().Namespaces().Create(context.Background(), &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: KubesphereNamespace}}, metav1.CreateOptions{}) if err != nil && !errors.IsAlreadyExists(err) { t.Fatal(err) } _, err = c.CoreV1().ConfigMaps(KubesphereNamespace).Create(context.Background(), targetCm, metav1.CreateOptions{}) if err != nil && errors.IsAlreadyExists(err) { _, err = c.CoreV1().ConfigMaps(KubesphereNamespace).Update(context.Background(), targetCm, metav1.UpdateOptions{}) if err != nil { t.Fatal(err) } } else if err != nil { t.Fatal(err) } deploy := appsv1.Deployment{} err = json.Unmarshal([]byte(ksApiserverDeploy), &deploy) if err != nil { t.Fatal(err) } _, err = c.AppsV1().Deployments(KubesphereNamespace).Create(context.Background(), &deploy, metav1.CreateOptions{}) if err != nil && !errors.IsAlreadyExists(err) { t.Fatal(err) } }