Merge pull request #2104 from huanggze/monitoring

monitor: add platform metrics

pkg/apiserver/apiserver.go

@@ -150,7 +150,7 @@ func (s *APIServer) installKubeSphereAPIs() {
 	urlruntime.Must(configv1alpha2.AddToContainer(s.container, s.Config))
 	urlruntime.Must(resourcev1alpha3.AddToContainer(s.container, s.InformerFactory))
 	urlruntime.Must(loggingv1alpha2.AddToContainer(s.container, s.KubernetesClient, s.LoggingClient))
-	urlruntime.Must(monitoringv1alpha3.AddToContainer(s.container, s.KubernetesClient.Kubernetes(), s.MonitoringClient))
+	urlruntime.Must(monitoringv1alpha3.AddToContainer(s.container, s.KubernetesClient.Kubernetes(), s.MonitoringClient, s.InformerFactory, s.OpenpitrixClient))
 	urlruntime.Must(openpitrixv1.AddToContainer(s.container, s.InformerFactory, s.OpenpitrixClient))
 	urlruntime.Must(networkv1alpha2.AddToContainer(s.container, s.Config.NetworkOptions.WeaveScopeHost))
 	urlruntime.Must(operationsv1alpha2.AddToContainer(s.container, s.KubernetesClient.Kubernetes()))

pkg/constants/constants.go

@@ -70,6 +70,7 @@ const (
 	DevOpsWebhookTag           = "DevOps Webhook"
 	DevOpsJenkinsfileTag       = "DevOps Jenkinsfile"
 	DevOpsScmTag               = "DevOps Scm"
+	KubeSphereMetricsTag       = "KubeSphere Metrics"
 	ClusterMetricsTag          = "Cluster Metrics"
 	NodeMetricsTag             = "Node Metrics"
 	NamespaceMetricsTag        = "Namespace Metrics"

pkg/kapis/monitoring/v1alpha3/handler.go

@@ -23,8 +23,10 @@ import (
 	"github.com/emicklei/go-restful"
 	"k8s.io/client-go/kubernetes"
 	"kubesphere.io/kubesphere/pkg/api"
+	"kubesphere.io/kubesphere/pkg/informers"
 	model "kubesphere.io/kubesphere/pkg/models/monitoring"
 	"kubesphere.io/kubesphere/pkg/simple/client/monitoring"
+	"kubesphere.io/kubesphere/pkg/simple/client/openpitrix"
 	"regexp"
 )
 
@@ -33,8 +35,13 @@ type handler struct {
 	mo model.MonitoringOperator
 }
 
-func newHandler(k kubernetes.Interface, m monitoring.Interface) *handler {
-	return &handler{k, model.NewMonitoringOperator(m)}
+func newHandler(k kubernetes.Interface, m monitoring.Interface, f informers.InformerFactory, o openpitrix.Client) *handler {
+	return &handler{k, model.NewMonitoringOperator(m, k, f, o)}
+}
+
+func (h handler) handleKubeSphereMetricsQuery(req *restful.Request, resp *restful.Response) {
+	res := h.mo.GetKubeSphereStats()
+	resp.WriteAsJson(res)
 }
 
 func (h handler) handleClusterMetricsQuery(req *restful.Request, resp *restful.Response) {
@@ -64,7 +71,13 @@ func (h handler) handleWorkspaceMetricsQuery(req *restful.Request, resp *restful
 		api.HandleBadRequest(resp, nil, err)
 		return
 	}
-	h.handleNamedMetricsQuery(resp, opt)
+
+	if req.QueryParameter("type") == "statistics" {
+		res := h.mo.GetWorkspaceStats(params.workspaceName)
+		resp.WriteAsJson(res)
+	} else {
+		h.handleNamedMetricsQuery(resp, opt)
+	}
 }
 
 func (h handler) handleNamespaceMetricsQuery(req *restful.Request, resp *restful.Response) {

pkg/kapis/monitoring/v1alpha3/helper_test.go

@@ -6,6 +6,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes/fake"
+	"kubesphere.io/kubesphere/pkg/informers"
 	model "kubesphere.io/kubesphere/pkg/models/monitoring"
 	"kubesphere.io/kubesphere/pkg/simple/client/monitoring"
 	"testing"
@@ -209,7 +210,8 @@ func TestParseRequestParams(t *testing.T) {
 	for i, tt := range tests {
 		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
 			client := fake.NewSimpleClientset(&tt.namespace)
-			handler := newHandler(client, nil)
+			fakeInformerFactory := informers.NewInformerFactories(client, nil, nil, nil, nil, nil)
+			handler := newHandler(client, nil, fakeInformerFactory, nil)
 
 			result, err := handler.makeQueryOptions(tt.params, tt.lvl)
 			if err != nil {

pkg/kapis/monitoring/v1alpha3/register.go

@@ -24,8 +24,10 @@ import (
 	"k8s.io/client-go/kubernetes"
 	"kubesphere.io/kubesphere/pkg/apiserver/runtime"
 	"kubesphere.io/kubesphere/pkg/constants"
+	"kubesphere.io/kubesphere/pkg/informers"
 	model "kubesphere.io/kubesphere/pkg/models/monitoring"
 	"kubesphere.io/kubesphere/pkg/simple/client/monitoring"
+	"kubesphere.io/kubesphere/pkg/simple/client/openpitrix"
 	"net/http"
 )
 
@@ -36,10 +38,18 @@ const (
 
 var GroupVersion = schema.GroupVersion{Group: GroupName, Version: "v1alpha3"}
 
-func AddToContainer(c *restful.Container, k8sClient kubernetes.Interface, monitoringClient monitoring.Interface) error {
+func AddToContainer(c *restful.Container, k8sClient kubernetes.Interface, monitoringClient monitoring.Interface, factory informers.InformerFactory, opClient openpitrix.Client) error {
 	ws := runtime.NewWebService(GroupVersion)
 
-	h := newHandler(k8sClient, monitoringClient)
+	h := newHandler(k8sClient, monitoringClient, factory, opClient)
+
+	ws.Route(ws.GET("/kubesphere").
+		To(h.handleKubeSphereMetricsQuery).
+		Doc("Get platform-level metric data.").
+		Metadata(restfulspec.KeyOpenAPITags, []string{constants.KubeSphereMetricsTag}).
+		Writes(model.Metrics{}).
+		Returns(http.StatusOK, RespOK, model.Metrics{})).
+		Produces(restful.MIME_JSON)
 
 	ws.Route(ws.GET("/cluster").
 		To(h.handleClusterMetricsQuery).
@@ -113,6 +123,7 @@ func AddToContainer(c *restful.Container, k8sClient kubernetes.Interface, monito
 		Param(ws.QueryParameter("end", "End time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1561939200. ").DataType("string").Required(false)).
 		Param(ws.QueryParameter("step", "Time interval. Retrieve metric data at a fixed interval within the time range of start and end. It requires both **start** and **end** are provided. The format is [0-9]+[smhdwy]. Defaults to 10m (i.e. 10 min).").DataType("string").DefaultValue("10m").Required(false)).
 		Param(ws.QueryParameter("time", "A timestamp in Unix time format. Retrieve metric data at a single point in time. Defaults to now. Time and the combination of start, end, step are mutually exclusive.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("type", "Additional operations. Currently available types is statistics. It retrieves the total number of namespaces, devops projects, members and roles in this workspace at the moment.").DataType("string").Required(false)).
 		Metadata(restfulspec.KeyOpenAPITags, []string{constants.WorkspaceMetricsTag}).
 		Writes(model.Metrics{}).
 		Returns(http.StatusOK, RespOK, model.Metrics{})).

pkg/models/monitoring/monitoring.go

@@ -19,9 +19,18 @@
 package monitoring
 
 import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/client-go/kubernetes"
 	"k8s.io/klog"
+	ksinformers "kubesphere.io/kubesphere/pkg/client/informers/externalversions"
+	"kubesphere.io/kubesphere/pkg/constants"
+	"kubesphere.io/kubesphere/pkg/informers"
 	"kubesphere.io/kubesphere/pkg/models/monitoring/expressions"
+	"kubesphere.io/kubesphere/pkg/models/openpitrix"
+	"kubesphere.io/kubesphere/pkg/server/params"
 	"kubesphere.io/kubesphere/pkg/simple/client/monitoring"
+	opclient "kubesphere.io/kubesphere/pkg/simple/client/openpitrix"
 	"time"
 )
 
@@ -32,14 +41,26 @@ type MonitoringOperator interface {
 	GetNamedMetricsOverTime(metrics []string, start, end time.Time, step time.Duration, opt monitoring.QueryOption) Metrics
 	GetMetadata(namespace string) Metadata
 	GetMetricLabelSet(metric, namespace string, start, end time.Time) MetricLabelSet
+
+	// TODO: refactor
+	GetKubeSphereStats() Metrics
+	GetWorkspaceStats(workspace string) Metrics
 }
 
 type monitoringOperator struct {
-	c monitoring.Interface
+	c   monitoring.Interface
+	k8s kubernetes.Interface
+	ks  ksinformers.SharedInformerFactory
+	op  openpitrix.Interface
 }
 
-func NewMonitoringOperator(client monitoring.Interface) MonitoringOperator {
-	return &monitoringOperator{client}
+func NewMonitoringOperator(client monitoring.Interface, k8s kubernetes.Interface, factory informers.InformerFactory, opClient opclient.Client) MonitoringOperator {
+	return &monitoringOperator{
+		c:   client,
+		k8s: k8s,
+		ks:  factory.KubeSphereSharedInformerFactory(),
+		op:  openpitrix.NewOpenpitrixOperator(factory.KubernetesSharedInformerFactory(), opClient),
+	}
 }
 
 func (mo monitoringOperator) GetMetric(expr, namespace string, time time.Time) (monitoring.Metric, error) {
@@ -94,3 +115,184 @@ func (mo monitoringOperator) GetMetricLabelSet(metric, namespace string, start,
 	data := mo.c.GetMetricLabelSet(expr, start, end)
 	return MetricLabelSet{Data: data}
 }
+
+func (mo monitoringOperator) GetKubeSphereStats() Metrics {
+	var res Metrics
+	now := float64(time.Now().Unix())
+
+	clusterList, err := mo.ks.Cluster().V1alpha1().Clusters().Lister().List(labels.Everything())
+	clusterTotal := len(clusterList)
+	if clusterTotal == 0 {
+		clusterTotal = 1
+	}
+	if err != nil {
+		res.Results = append(res.Results, monitoring.Metric{
+			MetricName: KubeSphereClusterCount,
+			Error:      err.Error(),
+		})
+	} else {
+		res.Results = append(res.Results, monitoring.Metric{
+			MetricName: KubeSphereClusterCount,
+			MetricData: monitoring.MetricData{
+				MetricType: monitoring.MetricTypeVector,
+				MetricValues: []monitoring.MetricValue{
+					{
+						Sample: &monitoring.Point{now, float64(clusterTotal)},
+					},
+				},
+			},
+		})
+	}
+
+	wkList, err := mo.ks.Tenant().V1alpha1().Workspaces().Lister().List(labels.Everything())
+	if err != nil {
+		res.Results = append(res.Results, monitoring.Metric{
+			MetricName: KubeSphereWorkspaceCount,
+			Error:      err.Error(),
+		})
+	} else {
+		res.Results = append(res.Results, monitoring.Metric{
+			MetricName: KubeSphereWorkspaceCount,
+			MetricData: monitoring.MetricData{
+				MetricType: monitoring.MetricTypeVector,
+				MetricValues: []monitoring.MetricValue{
+					{
+						Sample: &monitoring.Point{now, float64(len(wkList))},
+					},
+				},
+			},
+		})
+	}
+
+	usrList, err := mo.ks.Iam().V1alpha2().Users().Lister().List(labels.Everything())
+	if err != nil {
+		res.Results = append(res.Results, monitoring.Metric{
+			MetricName: KubeSphereUserCount,
+			Error:      err.Error(),
+		})
+	} else {
+		res.Results = append(res.Results, monitoring.Metric{
+			MetricName: KubeSphereUserCount,
+			MetricData: monitoring.MetricData{
+				MetricType: monitoring.MetricTypeVector,
+				MetricValues: []monitoring.MetricValue{
+					{
+						Sample: &monitoring.Point{now, float64(len(usrList))},
+					},
+				},
+			},
+		})
+	}
+
+	tmpls, err := mo.op.ListApps(&params.Conditions{}, "", false, 0, 0)
+	if err != nil {
+		res.Results = append(res.Results, monitoring.Metric{
+			MetricName: KubeSphereAppTmplCount,
+			Error:      err.Error(),
+		})
+	} else {
+		res.Results = append(res.Results, monitoring.Metric{
+			MetricName: KubeSphereAppTmplCount,
+			MetricData: monitoring.MetricData{
+				MetricType: monitoring.MetricTypeVector,
+				MetricValues: []monitoring.MetricValue{
+					{
+						Sample: &monitoring.Point{now, float64(tmpls.TotalCount)},
+					},
+				},
+			},
+		})
+	}
+
+	return res
+}
+
+func (mo monitoringOperator) GetWorkspaceStats(workspace string) Metrics {
+	var res Metrics
+	now := float64(time.Now().Unix())
+
+	selector := labels.SelectorFromSet(labels.Set{constants.WorkspaceLabelKey: workspace})
+	opt := metav1.ListOptions{LabelSelector: selector.String()}
+
+	nsList, err := mo.k8s.CoreV1().Namespaces().List(opt)
+	if err != nil {
+		res.Results = append(res.Results, monitoring.Metric{
+			MetricName: WorkspaceNamespaceCount,
+			Error:      err.Error(),
+		})
+	} else {
+		res.Results = append(res.Results, monitoring.Metric{
+			MetricName: WorkspaceNamespaceCount,
+			MetricData: monitoring.MetricData{
+				MetricType: monitoring.MetricTypeVector,
+				MetricValues: []monitoring.MetricValue{
+					{
+						Sample: &monitoring.Point{now, float64(len(nsList.Items))},
+					},
+				},
+			},
+		})
+	}
+
+	devopsList, err := mo.ks.Devops().V1alpha3().DevOpsProjects().Lister().List(selector)
+	if err != nil {
+		res.Results = append(res.Results, monitoring.Metric{
+			MetricName: WorkspaceDevopsCount,
+			Error:      err.Error(),
+		})
+	} else {
+		res.Results = append(res.Results, monitoring.Metric{
+			MetricName: WorkspaceDevopsCount,
+			MetricData: monitoring.MetricData{
+				MetricType: monitoring.MetricTypeVector,
+				MetricValues: []monitoring.MetricValue{
+					{
+						Sample: &monitoring.Point{now, float64(len(devopsList))},
+					},
+				},
+			},
+		})
+	}
+
+	memberList, err := mo.ks.Iam().V1alpha2().WorkspaceRoleBindings().Lister().List(selector)
+	if err != nil {
+		res.Results = append(res.Results, monitoring.Metric{
+			MetricName: WorkspaceMemberCount,
+			Error:      err.Error(),
+		})
+	} else {
+		res.Results = append(res.Results, monitoring.Metric{
+			MetricName: WorkspaceMemberCount,
+			MetricData: monitoring.MetricData{
+				MetricType: monitoring.MetricTypeVector,
+				MetricValues: []monitoring.MetricValue{
+					{
+						Sample: &monitoring.Point{now, float64(len(memberList))},
+					},
+				},
+			},
+		})
+	}
+
+	roleList, err := mo.ks.Iam().V1alpha2().WorkspaceRoles().Lister().List(selector)
+	if err != nil {
+		res.Results = append(res.Results, monitoring.Metric{
+			MetricName: WorkspaceRoleCount,
+			Error:      err.Error(),
+		})
+	} else {
+		res.Results = append(res.Results, monitoring.Metric{
+			MetricName: WorkspaceRoleCount,
+			MetricData: monitoring.MetricData{
+				MetricType: monitoring.MetricTypeVector,
+				MetricValues: []monitoring.MetricValue{
+					{
+						Sample: &monitoring.Point{now, float64(len(roleList))},
+					},
+				},
+			},
+		})
+	}
+
+	return res
+}

pkg/models/monitoring/named_metrics.go

 package monitoring
 
+const (
+	KubeSphereWorkspaceCount = "kubesphere_workspace_count"
+	KubeSphereUserCount      = "kubesphere_user_count"
+	KubeSphereClusterCount   = "kubesphere_cluser_count"
+	KubeSphereAppTmplCount   = "kubesphere_app_template_count"
+
+	WorkspaceNamespaceCount = "workspace_namespace_count"
+	WorkspaceDevopsCount    = "workspace_devops_project_count"
+	WorkspaceMemberCount    = "workspace_member_count"
+	WorkspaceRoleCount      = "workspace_role_count"
+)
+
 var ClusterMetrics = []string{
 	"cluster_cpu_utilisation",
 	"cluster_cpu_usage",

tools/cmd/doc-gen/main.go

@@ -115,7 +115,7 @@ func generateSwaggerJson() []byte {
 	urlruntime.Must(devopsv1alpha2.AddToContainer(container, informerFactory.KubeSphereSharedInformerFactory(), &fake.Devops{}, nil, clientsets.KubeSphere(), fakes3.NewFakeS3()))
 	urlruntime.Must(iamv1alpha2.AddToContainer(container, im.NewOperator(clientsets.KubeSphere(), informerFactory), am.NewReadOnlyOperator(informerFactory), authoptions.NewAuthenticateOptions()))
 	urlruntime.Must(loggingv1alpha2.AddToContainer(container, clientsets, nil))
-	urlruntime.Must(monitoringv1alpha3.AddToContainer(container, clientsets.Kubernetes(), nil))
+	urlruntime.Must(monitoringv1alpha3.AddToContainer(container, clientsets.Kubernetes(), nil, informerFactory, nil))
 	urlruntime.Must(openpitrixv1.AddToContainer(container, informerFactory, nil))
 	urlruntime.Must(operationsv1alpha2.AddToContainer(container, clientsets.Kubernetes()))
 	urlruntime.Must(resourcesv1alpha2.AddToContainer(container, clientsets.Kubernetes(), informerFactory))