From 69c6d91fdbbd9a1b2d69da4ed162a39ed2bf5679 Mon Sep 17 00:00:00 2001 From: zryfish Date: Fri, 10 Apr 2020 12:37:05 +0800 Subject: [PATCH] fix cluster controller (#1996) --- Makefile | 2 +- .../bases/cluster.kubesphere.io_clusters.yaml | 67 ++++++---- pkg/apis/cluster/v1alpha1/agent_types.go | 4 +- pkg/apis/cluster/v1alpha1/cluster_types.go | 35 ++++- .../cluster/v1alpha1/zz_generated.deepcopy.go | 37 ++++-- .../iam/v1alpha2/zz_generated.deepcopy.go | 7 +- pkg/apiserver/apiserver.go | 4 +- pkg/apiserver/config/config.go | 24 ---- pkg/apiserver/dispatch/dispatch.go | 43 +++++- pkg/apiserver/request/requestinfo.go | 24 ++-- pkg/apiserver/request/requestinfo_test.go | 65 +++++---- pkg/controller/cluster/cluster_controller.go | 123 +++++++++++++++++- .../destinationrule_controller.go | 8 +- .../virtualservice_controller.go | 2 +- 14 files changed, 317 insertions(+), 128 deletions(-) diff --git a/Makefile b/Makefile index 56684a95b..96b090982 100644 --- a/Makefile +++ b/Makefile @@ -63,7 +63,7 @@ vet: generate # Generate manifests e.g. CRD, RBAC etc. manifests: - go run ./vendor/sigs.k8s.io/controller-tools/cmd/controller-gen/main.go object:headerFile=./hack/boilerplate.go.txt paths=./pkg/apis/... + go run ./vendor/sigs.k8s.io/controller-tools/cmd/controller-gen/main.go object:headerFile=./hack/boilerplate.go.txt paths=./pkg/apis/... rbac:roleName=controller-perms ${CRD_OPTIONS} output:crd:artifacts:config=config/crd/bases deploy: manifests kubectl apply -f config/crds diff --git a/config/crd/bases/cluster.kubesphere.io_clusters.yaml b/config/crd/bases/cluster.kubesphere.io_clusters.yaml index 2992c8885..c1289c8e4 100644 --- a/config/crd/bases/cluster.kubesphere.io_clusters.yaml +++ b/config/crd/bases/cluster.kubesphere.io_clusters.yaml @@ -14,7 +14,7 @@ spec: listKind: ClusterList plural: clusters singular: cluster - scope: Namespaced + scope: Cluster validation: openAPIV3Schema: description: Cluster is the schema for the clusters API @@ -37,35 +37,52 @@ spec: description: Desired state of the cluster type: boolean federated: - description: Join cluster as kubefed cluster + description: Join cluster as a kubefed cluster type: boolean + provider: + description: Provider of the cluster, this field is just for description + type: string type: object status: properties: - lastTransitionTime: - description: Last time the condition transitioned from one status to - another. - format: date-time - type: string - lastUpdateTime: - description: The last time this condition was updated. - format: date-time - type: string - message: - description: A human readable message indicating details about the transition. - type: string - reason: - description: The reason for the condition's last transition. - type: string - status: - description: Status of the condition, one of True, False, Unknown. - type: string - type: - description: Type of the condition + conditions: + description: Represents the latest available observations of a cluster's + current state. + items: + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. + format: date-time + type: string + lastUpdateTime: + description: The last time this condition was updated. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. + type: string + reason: + description: The reason for the condition's last transition. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of the condition + type: string + required: + - status + - type + type: object + type: array + count: + type: integer + version: + description: GitVersion of the kubernetes cluster, this field is set + by cluster controller type: string - required: - - status - - type type: object type: object version: v1alpha1 diff --git a/pkg/apis/cluster/v1alpha1/agent_types.go b/pkg/apis/cluster/v1alpha1/agent_types.go index 2be87bbb9..82f407bbe 100644 --- a/pkg/apis/cluster/v1alpha1/agent_types.go +++ b/pkg/apis/cluster/v1alpha1/agent_types.go @@ -82,8 +82,6 @@ type AgentCondition struct { // AgentStatus defines the observed state of Agent type AgentStatus struct { - // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster - // Important: Run "make" to regenerate code after modifying this file // Represents the latest available observations of a agent's current state. Conditions []AgentCondition `json:"conditions,omitempty"` @@ -99,6 +97,8 @@ type AgentStatus struct { // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object // +k8s:openapi-gen=true // +genclient:nonNamespaced +// +kubebuilder:printcolumn:name="Paused",type="bool",JSONPath=".spec.Paused" +// +kubebuilder:resource:scope=Cluster // Agent is the Schema for the agents API type Agent struct { diff --git a/pkg/apis/cluster/v1alpha1/cluster_types.go b/pkg/apis/cluster/v1alpha1/cluster_types.go index 834ba7d50..bdf69f3ac 100644 --- a/pkg/apis/cluster/v1alpha1/cluster_types.go +++ b/pkg/apis/cluster/v1alpha1/cluster_types.go @@ -9,31 +9,38 @@ const ( ResourceKindCluster = "Cluster" ResourcesSingularCluster = "cluster" ResourcesPluralCluster = "clusters" + + IsHostCluster = "cluster.kubesphere.io/is-host-cluster" ) type ClusterSpec struct { - // Join cluster as kubefed cluster + // Join cluster as a kubefed cluster + // +optional Federated bool `json:"federated,omitempty"` // Desired state of the cluster Active bool `json:"active,omitempty"` + + // Provider of the cluster, this field is just for description + // +optional + Provider string `json:"provider,omitempty"` } type ClusterConditionType string const ( // Cluster agent is initialized and waiting for connecting - ClusterAgentInitialized ClusterConditionType = "AgentInitialized" + ClusterInitialized ClusterConditionType = "Initialized" // Cluster agent is available ClusterAgentAvailable ClusterConditionType = "AgentAvailable" - // - + // Cluster has been one of federated clusters + ClusterFederated ClusterConditionType = "Federated" ) -type ClusterStatus struct { +type ClusterCondition struct { // Type of the condition Type ClusterConditionType `json:"type"` // Status of the condition, one of True, False, Unknown. @@ -48,10 +55,28 @@ type ClusterStatus struct { Message string `json:"message,omitempty"` } +type ClusterStatus struct { + + // Represents the latest available observations of a cluster's current state. + Conditions []ClusterCondition `json:"conditions,omitempty"` + + // GitVersion of the kubernetes cluster, this field is set by cluster controller + // +optional + KubernetesVersion string `json:"version,omitempty"` + + // + NodeCount int `json:"count,omitempty"` +} + // +genclient // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object // +k8s:openapi-gen=true // +genclient:nonNamespaced +// +groupName +// +kubebuilder:printcolumn:name="Federated",type="bool",JSONPath=".spec.Federated" +// +kubebuilder:printcolumn:name="Provider",type="string",JSONPath=".spec.Provider" +// +kubebuilder:printcolumn:name="Active",type="bool",JSONPath=".spec.Active" +// +kubebuilder:resource:scope=Cluster // Cluster is the schema for the clusters API type Cluster struct { diff --git a/pkg/apis/cluster/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/cluster/v1alpha1/zz_generated.deepcopy.go index cbe0824e5..9ad579cb8 100644 --- a/pkg/apis/cluster/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/cluster/v1alpha1/zz_generated.deepcopy.go @@ -16,7 +16,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -// Code generated by deepcopy-gen. DO NOT EDIT. +// Code generated by controller-gen. DO NOT EDIT. package v1alpha1 @@ -31,7 +31,6 @@ func (in *Agent) DeepCopyInto(out *Agent) { in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) out.Spec = in.Spec in.Status.DeepCopyInto(&out.Status) - return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Agent. @@ -57,7 +56,6 @@ func (in *AgentCondition) DeepCopyInto(out *AgentCondition) { *out = *in in.LastUpdateTime.DeepCopyInto(&out.LastUpdateTime) in.LastTransitionTime.DeepCopyInto(&out.LastTransitionTime) - return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AgentCondition. @@ -82,7 +80,6 @@ func (in *AgentList) DeepCopyInto(out *AgentList) { (*in)[i].DeepCopyInto(&(*out)[i]) } } - return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AgentList. @@ -106,7 +103,6 @@ func (in *AgentList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *AgentSpec) DeepCopyInto(out *AgentSpec) { *out = *in - return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AgentSpec. @@ -134,7 +130,6 @@ func (in *AgentStatus) DeepCopyInto(out *AgentStatus) { *out = make([]byte, len(*in)) copy(*out, *in) } - return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AgentStatus. @@ -154,7 +149,6 @@ func (in *Cluster) DeepCopyInto(out *Cluster) { in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) out.Spec = in.Spec in.Status.DeepCopyInto(&out.Status) - return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Cluster. @@ -175,6 +169,23 @@ func (in *Cluster) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClusterCondition) DeepCopyInto(out *ClusterCondition) { + *out = *in + in.LastUpdateTime.DeepCopyInto(&out.LastUpdateTime) + in.LastTransitionTime.DeepCopyInto(&out.LastTransitionTime) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterCondition. +func (in *ClusterCondition) DeepCopy() *ClusterCondition { + if in == nil { + return nil + } + out := new(ClusterCondition) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ClusterList) DeepCopyInto(out *ClusterList) { *out = *in @@ -187,7 +198,6 @@ func (in *ClusterList) DeepCopyInto(out *ClusterList) { (*in)[i].DeepCopyInto(&(*out)[i]) } } - return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterList. @@ -211,7 +221,6 @@ func (in *ClusterList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ClusterSpec) DeepCopyInto(out *ClusterSpec) { *out = *in - return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterSpec. @@ -227,9 +236,13 @@ func (in *ClusterSpec) DeepCopy() *ClusterSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ClusterStatus) DeepCopyInto(out *ClusterStatus) { *out = *in - in.LastUpdateTime.DeepCopyInto(&out.LastUpdateTime) - in.LastTransitionTime.DeepCopyInto(&out.LastTransitionTime) - return + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]ClusterCondition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterStatus. diff --git a/pkg/apis/iam/v1alpha2/zz_generated.deepcopy.go b/pkg/apis/iam/v1alpha2/zz_generated.deepcopy.go index f963f4f8e..a70563680 100644 --- a/pkg/apis/iam/v1alpha2/zz_generated.deepcopy.go +++ b/pkg/apis/iam/v1alpha2/zz_generated.deepcopy.go @@ -16,7 +16,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -// Code generated by deepcopy-gen. DO NOT EDIT. +// Code generated by controller-gen. DO NOT EDIT. package v1alpha2 @@ -31,7 +31,6 @@ func (in *User) DeepCopyInto(out *User) { in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) in.Spec.DeepCopyInto(&out.Spec) in.Status.DeepCopyInto(&out.Status) - return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new User. @@ -56,7 +55,6 @@ func (in *User) DeepCopyObject() runtime.Object { func (in *UserCondition) DeepCopyInto(out *UserCondition) { *out = *in in.LastTransitionTime.DeepCopyInto(&out.LastTransitionTime) - return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new UserCondition. @@ -81,7 +79,6 @@ func (in *UserList) DeepCopyInto(out *UserList) { (*in)[i].DeepCopyInto(&(*out)[i]) } } - return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new UserList. @@ -115,7 +112,6 @@ func (in *UserSpec) DeepCopyInto(out *UserSpec) { *out = make([]FinalizerName, len(*in)) copy(*out, *in) } - return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new UserSpec. @@ -138,7 +134,6 @@ func (in *UserStatus) DeepCopyInto(out *UserStatus) { (*in)[i].DeepCopyInto(&(*out)[i]) } } - return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new UserStatus. diff --git a/pkg/apiserver/apiserver.go b/pkg/apiserver/apiserver.go index 7c4c701d1..176495c76 100644 --- a/pkg/apiserver/apiserver.go +++ b/pkg/apiserver/apiserver.go @@ -182,7 +182,9 @@ func (s *APIServer) buildHandlerChain() { handler := s.Server.Handler handler = filters.WithKubeAPIServer(handler, s.KubernetesClient.Config(), &errorResponder{}) - handler = filters.WithMultipleClusterDispatcher(handler, dispatch.NewClusterDispatch(s.InformerFactory.KubeSphereSharedInformerFactory().Cluster().V1alpha1().Agents().Lister())) + + clusterDispatcher := dispatch.NewClusterDispatch(s.InformerFactory.KubeSphereSharedInformerFactory().Cluster().V1alpha1().Agents().Lister(), s.InformerFactory.KubeSphereSharedInformerFactory().Cluster().V1alpha1().Clusters().Lister()) + handler = filters.WithMultipleClusterDispatcher(handler, clusterDispatcher) excludedPaths := []string{"/oauth/*", "/kapis/config.kubesphere.io/*"} pathAuthorizer, _ := path.NewAuthorizer(excludedPaths) diff --git a/pkg/apiserver/config/config.go b/pkg/apiserver/config/config.go index 574194b2d..01fd3d575 100644 --- a/pkg/apiserver/config/config.go +++ b/pkg/apiserver/config/config.go @@ -2,11 +2,8 @@ package config import ( "fmt" - "github.com/emicklei/go-restful" "github.com/spf13/viper" - "k8s.io/apimachinery/pkg/runtime/schema" authoptions "kubesphere.io/kubesphere/pkg/apiserver/authentication/options" - "kubesphere.io/kubesphere/pkg/apiserver/runtime" "kubesphere.io/kubesphere/pkg/simple/client/alerting" "kubesphere.io/kubesphere/pkg/simple/client/cache" "kubesphere.io/kubesphere/pkg/simple/client/devops/jenkins" @@ -20,7 +17,6 @@ import ( "kubesphere.io/kubesphere/pkg/simple/client/s3" "kubesphere.io/kubesphere/pkg/simple/client/servicemesh" "kubesphere.io/kubesphere/pkg/simple/client/sonarqube" - "net/http" "reflect" "strings" ) @@ -126,26 +122,6 @@ func TryLoadFromDisk() (*Config, error) { return conf, nil } -// InstallAPI installs api for config -func (conf *Config) InstallAPI(c *restful.Container) { - ws := runtime.NewWebService(schema.GroupVersion{ - Group: "", - Version: "v1alpha1", - }) - - ws.Route(ws.GET("/configz"). - To(func(request *restful.Request, response *restful.Response) { - conf.stripEmptyOptions() - response.WriteAsJson(conf.ToMap()) - }). - Doc("Get system components configuration"). - Produces(restful.MIME_JSON). - Writes(Config{}). - Returns(http.StatusOK, "ok", Config{})) - - c.Add(ws) -} - // convertToMap simply converts config to map[string]bool // to hide sensitive information func (conf *Config) ToMap() map[string]bool { diff --git a/pkg/apiserver/dispatch/dispatch.go b/pkg/apiserver/dispatch/dispatch.go index fd079b134..6948114d2 100644 --- a/pkg/apiserver/dispatch/dispatch.go +++ b/pkg/apiserver/dispatch/dispatch.go @@ -6,6 +6,7 @@ import ( "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/util/proxy" "k8s.io/apiserver/pkg/endpoints/handlers/responsewriters" + "k8s.io/klog" clusterv1alpha1 "kubesphere.io/kubesphere/pkg/apis/cluster/v1alpha1" "kubesphere.io/kubesphere/pkg/apiserver/request" "kubesphere.io/kubesphere/pkg/client/listers/cluster/v1alpha1" @@ -13,27 +14,46 @@ import ( "strings" ) -const defaultMultipleClusterAgentNamespace = "kubesphere-system" - // Dispatcher defines how to forward request to designated cluster based on cluster name type Dispatcher interface { Dispatch(w http.ResponseWriter, req *http.Request, handler http.Handler) } type clusterDispatch struct { - agentLister v1alpha1.AgentLister + agentLister v1alpha1.AgentLister + clusterLister v1alpha1.ClusterLister } -func NewClusterDispatch(agentLister v1alpha1.AgentLister) Dispatcher { +func NewClusterDispatch(agentLister v1alpha1.AgentLister, clusterLister v1alpha1.ClusterLister) Dispatcher { return &clusterDispatch{ - agentLister: agentLister, + agentLister: agentLister, + clusterLister: clusterLister, } } func (c *clusterDispatch) Dispatch(w http.ResponseWriter, req *http.Request, handler http.Handler) { info, _ := request.RequestInfoFrom(req.Context()) - if info.Cluster == "" { // fallback to host cluster if cluster name if empty + + if len(info.Cluster) == 0 { + klog.Warningf("Request with empty cluster, %v", req.URL) + http.Error(w, fmt.Sprintf("Bad request, empty cluster"), http.StatusBadRequest) + return + } + + cluster, err := c.clusterLister.Get(info.Cluster) + if err != nil { + if errors.IsNotFound(err) { + http.Error(w, fmt.Sprintf("cluster %s not found", info.Cluster), http.StatusNotFound) + } else { + http.Error(w, err.Error(), http.StatusInternalServerError) + } + return + } + + // request cluster is host cluster, no need go through agent + if isClusterHostCluster(cluster) { + req.URL.Path = strings.Replace(req.URL.Path, fmt.Sprintf("/clusters/%s", info.Cluster), "", 1) handler.ServeHTTP(w, req) return } @@ -74,3 +94,14 @@ func isAgentReady(agent *clusterv1alpha1.Agent) bool { return false } + +// +func isClusterHostCluster(cluster *clusterv1alpha1.Cluster) bool { + for key, value := range cluster.Annotations { + if key == clusterv1alpha1.IsHostCluster && value == "true" { + return true + } + } + + return false +} diff --git a/pkg/apiserver/request/requestinfo.go b/pkg/apiserver/request/requestinfo.go index 1645be19a..ca94487ee 100644 --- a/pkg/apiserver/request/requestinfo.go +++ b/pkg/apiserver/request/requestinfo.go @@ -78,8 +78,8 @@ type RequestInfoFactory struct { // /kapis/{api-group}/{version}/namespaces/{namespace}/{resource} // /kapis/{api-group}/{version}/namespaces/{namespace}/{resource}/{resourceName} // With workspaces: -// /kapis/{api-group}/{version}/clusters/{cluster}/namespaces/{namespace}/{resource} -// /kapis/{api-group}/{version}/clusters/{cluster}/namespaces/{namespace}/{resource}/{resourceName} +// /kapis/clusters/{cluster}/{api-group}/{version}/namespaces/{namespace}/{resource} +// /kapis/clusters/{cluster}/{api-group}/{version}/namespaces/{namespace}/{resource}/{resourceName} // func (r *RequestInfoFactory) NewRequestInfo(req *http.Request) (*RequestInfo, error) { @@ -111,6 +111,16 @@ func (r *RequestInfoFactory) NewRequestInfo(req *http.Request) (*RequestInfo, er requestInfo.APIPrefix = currentParts[0] currentParts = currentParts[1:] + // URL forms: /clusters/{cluster}/* + if currentParts[0] == "clusters" { + if len(currentParts) > 1 { + requestInfo.Cluster = currentParts[1] + } + if len(currentParts) > 2 { + currentParts = currentParts[2:] + } + } + if !r.GrouplessAPIPrefixes.Has(requestInfo.APIPrefix) { // one part (APIPrefix) has already been consumed, so this is actually "do we have four parts?" if len(currentParts) < 3 { @@ -150,16 +160,6 @@ func (r *RequestInfoFactory) NewRequestInfo(req *http.Request) (*RequestInfo, er } } - // URL forms: /clusters/{cluster}/* - if currentParts[0] == "clusters" { - if len(currentParts) > 1 { - requestInfo.Cluster = currentParts[1] - } - if len(currentParts) > 2 { - currentParts = currentParts[2:] - } - } - // URL forms: /workspaces/{workspace}/* if currentParts[0] == "workspaces" { if len(currentParts) > 1 { diff --git a/pkg/apiserver/request/requestinfo_test.go b/pkg/apiserver/request/requestinfo_test.go index 8e87a116a..c7d63a134 100644 --- a/pkg/apiserver/request/requestinfo_test.go +++ b/pkg/apiserver/request/requestinfo_test.go @@ -59,59 +59,48 @@ func TestRequestInfoFactory_NewRequestInfo(t *testing.T) { expectedKubernetesRequest: false, }, { - name: "list cluster roles", - url: "/apis/rbac.authorization.k8s.io/v1/clusters/cluster1/clusterroles", + name: "list clusterRoles of cluster gondor", + url: "/apis/clusters/gondor/rbac.authorization.k8s.io/v1/clusterroles", method: http.MethodGet, expectedErr: nil, expectedVerb: "list", expectedResource: "clusterroles", expectedIsResourceRequest: true, - expectedCluster: "cluster1", + expectedCluster: "gondor", expectedKubernetesRequest: true, }, { - name: "list cluster nodes", - url: "/api/v1/clusters/cluster1/nodes", - method: http.MethodGet, - expectedErr: nil, - expectedVerb: "list", - expectedResource: "nodes", - expectedIsResourceRequest: true, - expectedCluster: "cluster1", - expectedKubernetesRequest: true, - }, - { - name: "list cluster nodes", - url: "/api/v1/clusters/cluster1/nodes", + name: "list nodes", + url: "/api/v1/nodes", method: http.MethodGet, expectedErr: nil, expectedVerb: "list", expectedResource: "nodes", expectedIsResourceRequest: true, - expectedCluster: "cluster1", + expectedCluster: "", expectedKubernetesRequest: true, }, { - name: "list cluster nodes", - url: "/api/v1/nodes", + name: "list nodes of cluster gondor", + url: "/api/clusters/gondor/v1/nodes", method: http.MethodGet, expectedErr: nil, expectedVerb: "list", expectedResource: "nodes", expectedIsResourceRequest: true, - expectedCluster: "", + expectedCluster: "gondor", expectedKubernetesRequest: true, }, { - name: "list roles", - url: "/apis/rbac.authorization.k8s.io/v1/clusters/cluster1/namespaces/namespace1/roles", + name: "list roles of cluster gondor", + url: "/apis/clusters/gondor/rbac.authorization.k8s.io/v1/namespaces/namespace1/roles", method: http.MethodGet, expectedErr: nil, expectedVerb: "list", expectedResource: "roles", expectedIsResourceRequest: true, expectedNamespace: "namespace1", - expectedCluster: "cluster1", + expectedCluster: "gondor", expectedKubernetesRequest: true, }, { @@ -139,17 +128,41 @@ func TestRequestInfoFactory_NewRequestInfo(t *testing.T) { expectedKubernetesRequest: false, }, { - name: "list namespaces", - url: "/kapis/resources.kubesphere.io/v1alpha3/clusters/cluster1/workspaces/workspace1/namespaces", + name: "list namespaces of cluster gondor", + url: "/kapis/clusters/gondor/resources.kubesphere.io/v1alpha3/workspaces/workspace1/namespaces", method: http.MethodGet, expectedErr: nil, expectedVerb: "list", expectedResource: "namespaces", expectedIsResourceRequest: true, expectedWorkspace: "workspace1", - expectedCluster: "cluster1", + expectedCluster: "gondor", expectedKubernetesRequest: false, }, + { + name: "list clusters", + url: "/apis/cluster.kubesphere.io/v1alpha1/clusters", + method: http.MethodGet, + expectedErr: nil, + expectedVerb: "list", + expectedResource: "clusters", + expectedIsResourceRequest: true, + expectedWorkspace: "", + expectedCluster: "", + expectedKubernetesRequest: true, + }, + { + name: "get cluster gondor", + url: "/apis/cluster.kubesphere.io/v1alpha1/clusters/gondor", + method: http.MethodGet, + expectedErr: nil, + expectedVerb: "get", + expectedResource: "clusters", + expectedIsResourceRequest: true, + expectedWorkspace: "", + expectedCluster: "", + expectedKubernetesRequest: true, + }, { name: "random query", url: "/foo/bar", diff --git a/pkg/controller/cluster/cluster_controller.go b/pkg/controller/cluster/cluster_controller.go index 7fc3abe80..c46a7926e 100644 --- a/pkg/controller/cluster/cluster_controller.go +++ b/pkg/controller/cluster/cluster_controller.go @@ -11,6 +11,7 @@ import ( "k8s.io/client-go/kubernetes/scheme" v1core "k8s.io/client-go/kubernetes/typed/core/v1" "k8s.io/client-go/tools/cache" + "k8s.io/client-go/tools/clientcmd" "k8s.io/client-go/tools/record" "k8s.io/client-go/util/retry" "k8s.io/client-go/util/workqueue" @@ -87,6 +88,12 @@ func NewClusterController( DeleteFunc: c.addCluster, }) + agentInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: nil, + UpdateFunc: nil, + DeleteFunc: nil, + }) + return c } @@ -141,7 +148,7 @@ func (c *ClusterController) syncCluster(key string) error { } defer func() { - klog.V(4).Info("Finished syncing cluster.", "name", name, "duration", time.Since(startTime)) + klog.V(4).Infof("Finished syncing cluster %s in %s", name, time.Since(startTime)) }() cluster, err := c.clusterLister.Get(name) @@ -209,6 +216,66 @@ func (c *ClusterController) syncCluster(key string) error { }) } + // agent connection is ready, update cluster status + // set + if len(agent.Status.KubeConfig) != 0 && c.isAgentReady(agent) { + clientConfig, err := clientcmd.NewClientConfigFromBytes(agent.Status.KubeConfig) + if err != nil { + klog.Errorf("Unable to create client config from kubeconfig bytes, %#v", err) + return err + } + + config, err := clientConfig.ClientConfig() + if err != nil { + klog.Errorf("Failed to get client config, %#v", err) + return err + } + + clientSet, err := kubernetes.NewForConfig(config) + if err != nil { + klog.Errorf("Failed to create ClientSet from config, %#v", err) + return nil + } + + version, err := clientSet.Discovery().ServerVersion() + if err != nil { + klog.Errorf("Failed to get kubernetes version, %#v", err) + return err + } + + cluster.Status.KubernetesVersion = version.GitVersion + + nodes, err := clientSet.CoreV1().Nodes().List(metav1.ListOptions{}) + if err != nil { + klog.Errorf("Failed to get cluster nodes, %#v", err) + return err + } + + cluster.Status.NodeCount = len(nodes.Items) + } + + agentReadyCondition := clusterv1alpha1.ClusterCondition{ + Type: clusterv1alpha1.ClusterAgentAvailable, + LastUpdateTime: metav1.NewTime(time.Now()), + LastTransitionTime: metav1.NewTime(time.Now()), + Reason: "", + Message: "Cluster agent is available now.", + } + + if c.isAgentReady(agent) { + agentReadyCondition.Status = v1.ConditionTrue + } else { + agentReadyCondition.Status = v1.ConditionFalse + } + + c.addClusterCondition(cluster, agentReadyCondition) + + _, err = c.clusterClient.Update(cluster) + if err != nil { + klog.Errorf("Failed to update cluster status, %#v", err) + return err + } + return nil } @@ -217,14 +284,64 @@ func (c *ClusterController) addCluster(obj interface{}) { key, err := cache.MetaNamespaceKeyFunc(obj) if err != nil { - utilruntime.HandleError(fmt.Errorf("get cluster key %s/%s failed", cluster.Namespace, cluster.Name)) + utilruntime.HandleError(fmt.Errorf("get cluster key %s failed", cluster.Name)) return } c.queue.Add(key) - } func (c *ClusterController) handleErr(err error, key interface{}) { + if err == nil { + c.queue.Forget(key) + return + } + + if c.queue.NumRequeues(key) < maxRetries { + klog.V(2).Infof("Error syncing virtualservice %s for service retrying, %#v", key, err) + c.queue.AddRateLimited(key) + return + } + + klog.V(4).Infof("Dropping service %s out of the queue.", key) + c.queue.Forget(key) + utilruntime.HandleError(err) +} + +func (c *ClusterController) addAgent(obj interface{}) { + agent := obj.(*clusterv1alpha1.Agent) + key, err := cache.MetaNamespaceKeyFunc(obj) + if err != nil { + utilruntime.HandleError(fmt.Errorf("get agent key %s failed", agent.Name)) + return + } + + c.queue.Add(key) +} + +func (c *ClusterController) isAgentReady(agent *clusterv1alpha1.Agent) bool { + for _, condition := range agent.Status.Conditions { + if condition.Type == clusterv1alpha1.AgentConnected && condition.Status == v1.ConditionTrue { + return true + } + } + return false +} + +// addClusterCondition add condition +func (c *ClusterController) addClusterCondition(cluster *clusterv1alpha1.Cluster, condition clusterv1alpha1.ClusterCondition) { + if cluster.Status.Conditions == nil { + cluster.Status.Conditions = make([]clusterv1alpha1.ClusterCondition, 0) + } + + newConditions := make([]clusterv1alpha1.ClusterCondition, 0) + for _, cond := range cluster.Status.Conditions { + if cond.Type == condition.Type { + continue + } + newConditions = append(newConditions, cond) + } + newConditions = append(newConditions, condition) + cluster.Status.Conditions = newConditions } diff --git a/pkg/controller/destinationrule/destinationrule_controller.go b/pkg/controller/destinationrule/destinationrule_controller.go index dc5c37371..db9bc7d98 100644 --- a/pkg/controller/destinationrule/destinationrule_controller.go +++ b/pkg/controller/destinationrule/destinationrule_controller.go @@ -199,7 +199,7 @@ func (v *DestinationRuleController) processNextWorkItem() bool { func (v *DestinationRuleController) syncService(key string) error { startTime := time.Now() defer func() { - log.V(4).Info("Finished syncing service destinationrule.", "key", key, "duration", time.Since(startTime)) + log.V(4).Infof("Finished syncing service destinationrule %s in %s.", key, time.Since(startTime)) }() namespace, name, err := cache.SplitMetaNamespaceKey(key) @@ -212,14 +212,14 @@ func (v *DestinationRuleController) syncService(key string) error { // delete the corresponding destinationrule if there is any, as the service has been deleted. err = v.destinationRuleClient.NetworkingV1alpha3().DestinationRules(namespace).Delete(name, nil) if err != nil && !errors.IsNotFound(err) { - log.Error(err, "delete destination rule failed", "namespace", namespace, "name", name) + log.Errorf("delete destination rule failed %s/%s, error %v.", namespace, name, err) return err } // delete orphan service policy if there is any err = v.servicemeshClient.ServicemeshV1alpha2().ServicePolicies(namespace).Delete(name, nil) if err != nil && !errors.IsNotFound(err) { - log.Error(err, "delete orphan service policy failed", "namespace", namespace, "name", name) + log.Errorf("delete orphan service policy %s/%s failed, %#v", namespace, name, err) return err } @@ -259,7 +259,7 @@ func (v *DestinationRuleController) syncService(key string) error { version := util.GetComponentVersion(&deployment.ObjectMeta) if len(version) == 0 { - log.V(4).Info("Deployment doesn't have a version label", "key", types.NamespacedName{Namespace: deployment.Namespace, Name: deployment.Name}.String()) + log.V(4).Infof("Deployment %s doesn't have a version label", types.NamespacedName{Namespace: deployment.Namespace, Name: deployment.Name}.String()) continue } diff --git a/pkg/controller/virtualservice/virtualservice_controller.go b/pkg/controller/virtualservice/virtualservice_controller.go index a47f6925d..3f10b6a24 100644 --- a/pkg/controller/virtualservice/virtualservice_controller.go +++ b/pkg/controller/virtualservice/virtualservice_controller.go @@ -213,7 +213,7 @@ func (v *VirtualServiceController) syncService(key string) error { appName := name defer func() { - log.V(4).Info("Finished syncing service virtualservice.", "namespace", namespace, "name", name, "duration", time.Since(startTime)) + log.V(4).Infof("Finished syncing service virtualservice %s/%s in %s.", namespace, name, time.Since(startTime)) }() service, err := v.serviceLister.Services(namespace).Get(name) -- GitLab