From 5c695a1c90af127cdaf72341fbe84d94fb032dbd Mon Sep 17 00:00:00 2001 From: huanggze Date: Fri, 22 May 2020 22:03:25 +0800 Subject: [PATCH] monitor: add platform metrics Signed-off-by: huanggze --- pkg/apiserver/apiserver.go | 2 +- pkg/constants/constants.go | 1 + pkg/kapis/monitoring/v1alpha3/handler.go | 19 +- pkg/kapis/monitoring/v1alpha3/helper_test.go | 4 +- pkg/kapis/monitoring/v1alpha3/register.go | 15 +- pkg/models/monitoring/monitoring.go | 208 ++++++++++++++++++- pkg/models/monitoring/named_metrics.go | 12 ++ tools/cmd/doc-gen/main.go | 2 +- 8 files changed, 252 insertions(+), 11 deletions(-) diff --git a/pkg/apiserver/apiserver.go b/pkg/apiserver/apiserver.go index 919a4957..28febe81 100644 --- a/pkg/apiserver/apiserver.go +++ b/pkg/apiserver/apiserver.go @@ -152,7 +152,7 @@ func (s *APIServer) installKubeSphereAPIs() { urlruntime.Must(configv1alpha2.AddToContainer(s.container, s.Config)) urlruntime.Must(resourcev1alpha3.AddToContainer(s.container, s.InformerFactory)) urlruntime.Must(loggingv1alpha2.AddToContainer(s.container, s.KubernetesClient, s.LoggingClient)) - urlruntime.Must(monitoringv1alpha3.AddToContainer(s.container, s.KubernetesClient.Kubernetes(), s.MonitoringClient)) + urlruntime.Must(monitoringv1alpha3.AddToContainer(s.container, s.KubernetesClient.Kubernetes(), s.MonitoringClient, s.InformerFactory, s.OpenpitrixClient)) urlruntime.Must(openpitrixv1.AddToContainer(s.container, s.InformerFactory, s.OpenpitrixClient)) urlruntime.Must(networkv1alpha2.AddToContainer(s.container, s.Config.NetworkOptions.WeaveScopeHost)) urlruntime.Must(operationsv1alpha2.AddToContainer(s.container, s.KubernetesClient.Kubernetes())) diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go index 280da902..bf57c4cd 100644 --- a/pkg/constants/constants.go +++ b/pkg/constants/constants.go @@ -69,6 +69,7 @@ const ( DevOpsWebhookTag = "DevOps Webhook" DevOpsJenkinsfileTag = "DevOps Jenkinsfile" DevOpsScmTag = "DevOps Scm" + KubeSphereMetricsTag = "KubeSphere Metrics" ClusterMetricsTag = "Cluster Metrics" NodeMetricsTag = "Node Metrics" NamespaceMetricsTag = "Namespace Metrics" diff --git a/pkg/kapis/monitoring/v1alpha3/handler.go b/pkg/kapis/monitoring/v1alpha3/handler.go index 45333cd6..93618448 100644 --- a/pkg/kapis/monitoring/v1alpha3/handler.go +++ b/pkg/kapis/monitoring/v1alpha3/handler.go @@ -23,8 +23,10 @@ import ( "github.com/emicklei/go-restful" "k8s.io/client-go/kubernetes" "kubesphere.io/kubesphere/pkg/api" + "kubesphere.io/kubesphere/pkg/informers" model "kubesphere.io/kubesphere/pkg/models/monitoring" "kubesphere.io/kubesphere/pkg/simple/client/monitoring" + "kubesphere.io/kubesphere/pkg/simple/client/openpitrix" "regexp" ) @@ -33,8 +35,13 @@ type handler struct { mo model.MonitoringOperator } -func newHandler(k kubernetes.Interface, m monitoring.Interface) *handler { - return &handler{k, model.NewMonitoringOperator(m)} +func newHandler(k kubernetes.Interface, m monitoring.Interface, f informers.InformerFactory, o openpitrix.Client) *handler { + return &handler{k, model.NewMonitoringOperator(m, k, f, o)} +} + +func (h handler) handleKubeSphereMetricsQuery(req *restful.Request, resp *restful.Response) { + res := h.mo.GetKubeSphereStats() + resp.WriteAsJson(res) } func (h handler) handleClusterMetricsQuery(req *restful.Request, resp *restful.Response) { @@ -64,7 +71,13 @@ func (h handler) handleWorkspaceMetricsQuery(req *restful.Request, resp *restful api.HandleBadRequest(resp, nil, err) return } - h.handleNamedMetricsQuery(resp, opt) + + if req.QueryParameter("type") == "statistics" { + res := h.mo.GetWorkspaceStats(params.workspaceName) + resp.WriteAsJson(res) + } else { + h.handleNamedMetricsQuery(resp, opt) + } } func (h handler) handleNamespaceMetricsQuery(req *restful.Request, resp *restful.Response) { diff --git a/pkg/kapis/monitoring/v1alpha3/helper_test.go b/pkg/kapis/monitoring/v1alpha3/helper_test.go index b7aed1b2..59befb1c 100644 --- a/pkg/kapis/monitoring/v1alpha3/helper_test.go +++ b/pkg/kapis/monitoring/v1alpha3/helper_test.go @@ -6,6 +6,7 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/fake" + "kubesphere.io/kubesphere/pkg/informers" model "kubesphere.io/kubesphere/pkg/models/monitoring" "kubesphere.io/kubesphere/pkg/simple/client/monitoring" "testing" @@ -209,7 +210,8 @@ func TestParseRequestParams(t *testing.T) { for i, tt := range tests { t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { client := fake.NewSimpleClientset(&tt.namespace) - handler := newHandler(client, nil) + fakeInformerFactory := informers.NewInformerFactories(client, nil, nil, nil, nil, nil) + handler := newHandler(client, nil, fakeInformerFactory, nil) result, err := handler.makeQueryOptions(tt.params, tt.lvl) if err != nil { diff --git a/pkg/kapis/monitoring/v1alpha3/register.go b/pkg/kapis/monitoring/v1alpha3/register.go index d6a06051..332517f0 100644 --- a/pkg/kapis/monitoring/v1alpha3/register.go +++ b/pkg/kapis/monitoring/v1alpha3/register.go @@ -24,8 +24,10 @@ import ( "k8s.io/client-go/kubernetes" "kubesphere.io/kubesphere/pkg/apiserver/runtime" "kubesphere.io/kubesphere/pkg/constants" + "kubesphere.io/kubesphere/pkg/informers" model "kubesphere.io/kubesphere/pkg/models/monitoring" "kubesphere.io/kubesphere/pkg/simple/client/monitoring" + "kubesphere.io/kubesphere/pkg/simple/client/openpitrix" "net/http" ) @@ -36,10 +38,18 @@ const ( var GroupVersion = schema.GroupVersion{Group: GroupName, Version: "v1alpha3"} -func AddToContainer(c *restful.Container, k8sClient kubernetes.Interface, monitoringClient monitoring.Interface) error { +func AddToContainer(c *restful.Container, k8sClient kubernetes.Interface, monitoringClient monitoring.Interface, factory informers.InformerFactory, opClient openpitrix.Client) error { ws := runtime.NewWebService(GroupVersion) - h := newHandler(k8sClient, monitoringClient) + h := newHandler(k8sClient, monitoringClient, factory, opClient) + + ws.Route(ws.GET("/kubesphere"). + To(h.handleKubeSphereMetricsQuery). + Doc("Get platform-level metric data."). + Metadata(restfulspec.KeyOpenAPITags, []string{constants.KubeSphereMetricsTag}). + Writes(model.Metrics{}). + Returns(http.StatusOK, RespOK, model.Metrics{})). + Produces(restful.MIME_JSON) ws.Route(ws.GET("/cluster"). To(h.handleClusterMetricsQuery). @@ -113,6 +123,7 @@ func AddToContainer(c *restful.Container, k8sClient kubernetes.Interface, monito Param(ws.QueryParameter("end", "End time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1561939200. ").DataType("string").Required(false)). Param(ws.QueryParameter("step", "Time interval. Retrieve metric data at a fixed interval within the time range of start and end. It requires both **start** and **end** are provided. The format is [0-9]+[smhdwy]. Defaults to 10m (i.e. 10 min).").DataType("string").DefaultValue("10m").Required(false)). Param(ws.QueryParameter("time", "A timestamp in Unix time format. Retrieve metric data at a single point in time. Defaults to now. Time and the combination of start, end, step are mutually exclusive.").DataType("string").Required(false)). + Param(ws.QueryParameter("type", "Additional operations. Currently available types is statistics. It retrieves the total number of namespaces, devops projects, members and roles in this workspace at the moment.").DataType("string").Required(false)). Metadata(restfulspec.KeyOpenAPITags, []string{constants.WorkspaceMetricsTag}). Writes(model.Metrics{}). Returns(http.StatusOK, RespOK, model.Metrics{})). diff --git a/pkg/models/monitoring/monitoring.go b/pkg/models/monitoring/monitoring.go index 22607200..186360a2 100644 --- a/pkg/models/monitoring/monitoring.go +++ b/pkg/models/monitoring/monitoring.go @@ -19,9 +19,18 @@ package monitoring import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/kubernetes" "k8s.io/klog" + ksinformers "kubesphere.io/kubesphere/pkg/client/informers/externalversions" + "kubesphere.io/kubesphere/pkg/constants" + "kubesphere.io/kubesphere/pkg/informers" "kubesphere.io/kubesphere/pkg/models/monitoring/expressions" + "kubesphere.io/kubesphere/pkg/models/openpitrix" + "kubesphere.io/kubesphere/pkg/server/params" "kubesphere.io/kubesphere/pkg/simple/client/monitoring" + opclient "kubesphere.io/kubesphere/pkg/simple/client/openpitrix" "time" ) @@ -32,14 +41,26 @@ type MonitoringOperator interface { GetNamedMetricsOverTime(metrics []string, start, end time.Time, step time.Duration, opt monitoring.QueryOption) Metrics GetMetadata(namespace string) Metadata GetMetricLabelSet(metric, namespace string, start, end time.Time) MetricLabelSet + + // TODO: refactor + GetKubeSphereStats() Metrics + GetWorkspaceStats(workspace string) Metrics } type monitoringOperator struct { - c monitoring.Interface + c monitoring.Interface + k8s kubernetes.Interface + ks ksinformers.SharedInformerFactory + op openpitrix.Interface } -func NewMonitoringOperator(client monitoring.Interface) MonitoringOperator { - return &monitoringOperator{client} +func NewMonitoringOperator(client monitoring.Interface, k8s kubernetes.Interface, factory informers.InformerFactory, opClient opclient.Client) MonitoringOperator { + return &monitoringOperator{ + c: client, + k8s: k8s, + ks: factory.KubeSphereSharedInformerFactory(), + op: openpitrix.NewOpenpitrixOperator(factory.KubernetesSharedInformerFactory(), opClient), + } } func (mo monitoringOperator) GetMetric(expr, namespace string, time time.Time) (monitoring.Metric, error) { @@ -94,3 +115,184 @@ func (mo monitoringOperator) GetMetricLabelSet(metric, namespace string, start, data := mo.c.GetMetricLabelSet(expr, start, end) return MetricLabelSet{Data: data} } + +func (mo monitoringOperator) GetKubeSphereStats() Metrics { + var res Metrics + now := float64(time.Now().Unix()) + + clusterList, err := mo.ks.Cluster().V1alpha1().Clusters().Lister().List(labels.Everything()) + clusterTotal := len(clusterList) + if clusterTotal == 0 { + clusterTotal = 1 + } + if err != nil { + res.Results = append(res.Results, monitoring.Metric{ + MetricName: KubeSphereClusterCount, + Error: err.Error(), + }) + } else { + res.Results = append(res.Results, monitoring.Metric{ + MetricName: KubeSphereClusterCount, + MetricData: monitoring.MetricData{ + MetricType: monitoring.MetricTypeVector, + MetricValues: []monitoring.MetricValue{ + { + Sample: &monitoring.Point{now, float64(clusterTotal)}, + }, + }, + }, + }) + } + + wkList, err := mo.ks.Tenant().V1alpha1().Workspaces().Lister().List(labels.Everything()) + if err != nil { + res.Results = append(res.Results, monitoring.Metric{ + MetricName: KubeSphereWorkspaceCount, + Error: err.Error(), + }) + } else { + res.Results = append(res.Results, monitoring.Metric{ + MetricName: KubeSphereWorkspaceCount, + MetricData: monitoring.MetricData{ + MetricType: monitoring.MetricTypeVector, + MetricValues: []monitoring.MetricValue{ + { + Sample: &monitoring.Point{now, float64(len(wkList))}, + }, + }, + }, + }) + } + + usrList, err := mo.ks.Iam().V1alpha2().Users().Lister().List(labels.Everything()) + if err != nil { + res.Results = append(res.Results, monitoring.Metric{ + MetricName: KubeSphereUserCount, + Error: err.Error(), + }) + } else { + res.Results = append(res.Results, monitoring.Metric{ + MetricName: KubeSphereUserCount, + MetricData: monitoring.MetricData{ + MetricType: monitoring.MetricTypeVector, + MetricValues: []monitoring.MetricValue{ + { + Sample: &monitoring.Point{now, float64(len(usrList))}, + }, + }, + }, + }) + } + + tmpls, err := mo.op.ListApps(¶ms.Conditions{}, "", false, 0, 0) + if err != nil { + res.Results = append(res.Results, monitoring.Metric{ + MetricName: KubeSphereAppTmplCount, + Error: err.Error(), + }) + } else { + res.Results = append(res.Results, monitoring.Metric{ + MetricName: KubeSphereAppTmplCount, + MetricData: monitoring.MetricData{ + MetricType: monitoring.MetricTypeVector, + MetricValues: []monitoring.MetricValue{ + { + Sample: &monitoring.Point{now, float64(tmpls.TotalCount)}, + }, + }, + }, + }) + } + + return res +} + +func (mo monitoringOperator) GetWorkspaceStats(workspace string) Metrics { + var res Metrics + now := float64(time.Now().Unix()) + + selector := labels.SelectorFromSet(labels.Set{constants.WorkspaceLabelKey: workspace}) + opt := metav1.ListOptions{LabelSelector: selector.String()} + + nsList, err := mo.k8s.CoreV1().Namespaces().List(opt) + if err != nil { + res.Results = append(res.Results, monitoring.Metric{ + MetricName: WorkspaceNamespaceCount, + Error: err.Error(), + }) + } else { + res.Results = append(res.Results, monitoring.Metric{ + MetricName: WorkspaceNamespaceCount, + MetricData: monitoring.MetricData{ + MetricType: monitoring.MetricTypeVector, + MetricValues: []monitoring.MetricValue{ + { + Sample: &monitoring.Point{now, float64(len(nsList.Items))}, + }, + }, + }, + }) + } + + devopsList, err := mo.ks.Devops().V1alpha3().DevOpsProjects().Lister().List(selector) + if err != nil { + res.Results = append(res.Results, monitoring.Metric{ + MetricName: WorkspaceDevopsCount, + Error: err.Error(), + }) + } else { + res.Results = append(res.Results, monitoring.Metric{ + MetricName: WorkspaceDevopsCount, + MetricData: monitoring.MetricData{ + MetricType: monitoring.MetricTypeVector, + MetricValues: []monitoring.MetricValue{ + { + Sample: &monitoring.Point{now, float64(len(devopsList))}, + }, + }, + }, + }) + } + + memberList, err := mo.ks.Iam().V1alpha2().WorkspaceRoleBindings().Lister().List(selector) + if err != nil { + res.Results = append(res.Results, monitoring.Metric{ + MetricName: WorkspaceMemberCount, + Error: err.Error(), + }) + } else { + res.Results = append(res.Results, monitoring.Metric{ + MetricName: WorkspaceMemberCount, + MetricData: monitoring.MetricData{ + MetricType: monitoring.MetricTypeVector, + MetricValues: []monitoring.MetricValue{ + { + Sample: &monitoring.Point{now, float64(len(memberList))}, + }, + }, + }, + }) + } + + roleList, err := mo.ks.Iam().V1alpha2().WorkspaceRoles().Lister().List(selector) + if err != nil { + res.Results = append(res.Results, monitoring.Metric{ + MetricName: WorkspaceRoleCount, + Error: err.Error(), + }) + } else { + res.Results = append(res.Results, monitoring.Metric{ + MetricName: WorkspaceRoleCount, + MetricData: monitoring.MetricData{ + MetricType: monitoring.MetricTypeVector, + MetricValues: []monitoring.MetricValue{ + { + Sample: &monitoring.Point{now, float64(len(roleList))}, + }, + }, + }, + }) + } + + return res +} diff --git a/pkg/models/monitoring/named_metrics.go b/pkg/models/monitoring/named_metrics.go index 322e01f9..89aa218d 100644 --- a/pkg/models/monitoring/named_metrics.go +++ b/pkg/models/monitoring/named_metrics.go @@ -1,5 +1,17 @@ package monitoring +const ( + KubeSphereWorkspaceCount = "kubesphere_workspace_count" + KubeSphereUserCount = "kubesphere_user_count" + KubeSphereClusterCount = "kubesphere_cluser_count" + KubeSphereAppTmplCount = "kubesphere_app_template_count" + + WorkspaceNamespaceCount = "workspace_namespace_count" + WorkspaceDevopsCount = "workspace_devops_project_count" + WorkspaceMemberCount = "workspace_member_count" + WorkspaceRoleCount = "workspace_role_count" +) + var ClusterMetrics = []string{ "cluster_cpu_utilisation", "cluster_cpu_usage", diff --git a/tools/cmd/doc-gen/main.go b/tools/cmd/doc-gen/main.go index 24d36e00..1886a711 100644 --- a/tools/cmd/doc-gen/main.go +++ b/tools/cmd/doc-gen/main.go @@ -115,7 +115,7 @@ func generateSwaggerJson() []byte { urlruntime.Must(devopsv1alpha2.AddToContainer(container, informerFactory.KubeSphereSharedInformerFactory(), &fake.Devops{}, nil, clientsets.KubeSphere(), fakes3.NewFakeS3())) urlruntime.Must(iamv1alpha2.AddToContainer(container, im.NewOperator(clientsets.KubeSphere(), informerFactory), am.NewAMOperator(informerFactory), authoptions.NewAuthenticateOptions())) urlruntime.Must(loggingv1alpha2.AddToContainer(container, clientsets, nil)) - urlruntime.Must(monitoringv1alpha3.AddToContainer(container, clientsets.Kubernetes(), nil)) + urlruntime.Must(monitoringv1alpha3.AddToContainer(container, clientsets.Kubernetes(), nil, informerFactory, nil)) urlruntime.Must(openpitrixv1.AddToContainer(container, informerFactory, nil)) urlruntime.Must(operationsv1alpha2.AddToContainer(container, clientsets.Kubernetes())) urlruntime.Must(resourcesv1alpha2.AddToContainer(container, clientsets.Kubernetes(), informerFactory)) -- GitLab