add pvc stats api

Signed-off-by: NXin Wang <wileywang@yunify.com>

pkg/apis/monitoring/v1alpha2/register.go

@@ -154,6 +154,59 @@ func addWebService(c *restful.Container) error {
 		Consumes(restful.MIME_JSON, restful.MIME_XML).
 		Produces(restful.MIME_JSON)
 
+	ws.Route(ws.GET("/storageclasses/{storageclass}/persistentvolumeclaims").To(monitoring.MonitorAllPVCsOfSpecificStorageClass).
+		Doc("Get PVC-level metric data of the specific storageclass's PVCs.").
+		Param(ws.PathParameter("storageclass", "The name of the storageclass.").DataType("string").Required(true)).
+		Param(ws.PathParameter("metrics_filter", "The metric name filter consists of a regexp pattern. It specifies which metric data to return. For example, the following filter matches both PVC available and used inodes: `pvc_inodes_available|pvc_inodes_used`. View available metrics at [kubesphere.io](https://docs.kubesphere.io/advanced-v2.0/zh-CN/api-reference/monitoring-metrics/).").DataType("string").Required(false)).
+		Param(ws.QueryParameter("resources_filter", "The PVC filter consists of a regexp pattern. It specifies which PVC data to return. For example, the following filter matches any pod whose name begins with redis: `redis.*`.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("start", "Start time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1559347200. ").DataType("string").Required(false)).
+		Param(ws.QueryParameter("end", "End time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1561939200. ").DataType("string").Required(false)).
+		Param(ws.QueryParameter("step", "Time interval. Retrieve metric data at a fixed interval within the time range of start and end. It requires both **start** and **end** are provided. The format is [0-9]+[smhdwy]. Defaults to 10m (i.e. 10 min).").DataType("string").DefaultValue("10m").Required(false)).
+		Param(ws.QueryParameter("time", "A timestamp in Unix time format. Retrieve metric data at a single point in time. Defaults to now. Time and the combination of start, end, step are mutually exclusive.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("sort_metric", "Sort PVCs by the specified metric. Not applicable if **start** and **end** are provided.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("sort_type", "Sort order. One of asc, desc.").DefaultValue("desc.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("page", "The page number. This field paginates result data of each metric, then returns a specific page. For example, setting **page** to 2 returns the second page. It only applies to sorted metric data.").DataType("integer").Required(false)).
+		Param(ws.QueryParameter("limit", "Page size, the maximum number of results in a single page. Defaults to 5.").DataType("integer").Required(false).DefaultValue("5")).
+		Metadata(restfulspec.KeyOpenAPITags, []string{constants.PVCMetricsTag}).
+		Writes(metrics.FormatedLevelMetric{}).
+		Returns(http.StatusOK, RespOK, metrics.FormatedLevelMetric{})).
+		Consumes(restful.MIME_JSON, restful.MIME_XML).
+		Produces(restful.MIME_JSON)
+
+	ws.Route(ws.GET("/namespaces/{namespace}/persistentvolumeclaims").To(monitoring.MonitorAllPVCsOfSpecificNamespace).
+		Doc("Get PVC-level metric data of the specific namespace's PVCs.").
+		Param(ws.PathParameter("namespace", "The name of the namespace.").DataType("string").Required(true)).
+		Param(ws.PathParameter("metrics_filter", "The metric name filter consists of a regexp pattern. It specifies which metric data to return. For example, the following filter matches both PVC available and used inodes: `pvc_inodes_available|pvc_inodes_used`. View available metrics at [kubesphere.io](https://docs.kubesphere.io/advanced-v2.0/zh-CN/api-reference/monitoring-metrics/).").DataType("string").Required(false)).
+		Param(ws.QueryParameter("resources_filter", "The PVC filter consists of a regexp pattern. It specifies which PVC data to return. For example, the following filter matches any pod whose name begins with redis: `redis.*`.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("start", "Start time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1559347200. ").DataType("string").Required(false)).
+		Param(ws.QueryParameter("end", "End time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1561939200. ").DataType("string").Required(false)).
+		Param(ws.QueryParameter("step", "Time interval. Retrieve metric data at a fixed interval within the time range of start and end. It requires both **start** and **end** are provided. The format is [0-9]+[smhdwy]. Defaults to 10m (i.e. 10 min).").DataType("string").DefaultValue("10m").Required(false)).
+		Param(ws.QueryParameter("time", "A timestamp in Unix time format. Retrieve metric data at a single point in time. Defaults to now. Time and the combination of start, end, step are mutually exclusive.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("sort_metric", "Sort PVCs by the specified metric. Not applicable if **start** and **end** are provided.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("sort_type", "Sort order. One of asc, desc.").DefaultValue("desc.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("page", "The page number. This field paginates result data of each metric, then returns a specific page. For example, setting **page** to 2 returns the second page. It only applies to sorted metric data.").DataType("integer").Required(false)).
+		Param(ws.QueryParameter("limit", "Page size, the maximum number of results in a single page. Defaults to 5.").DataType("integer").Required(false).DefaultValue("5")).
+		Metadata(restfulspec.KeyOpenAPITags, []string{constants.PVCMetricsTag}).
+		Writes(metrics.FormatedLevelMetric{}).
+		Returns(http.StatusOK, RespOK, metrics.FormatedLevelMetric{})).
+		Consumes(restful.MIME_JSON, restful.MIME_XML).
+		Produces(restful.MIME_JSON)
+
+	ws.Route(ws.GET("/namespaces/{namespace}/persistentvolumeclaims/{pvc}").To(monitoring.MonitorSpecificPVCofSpecificNamespace).
+		Doc("Get PVC-level metric data of a specific PVC. Navigate to the PVC by the PVC's namespace.").
+		Param(ws.PathParameter("namespace", "The name of the namespace.").DataType("string").Required(true)).
+		Param(ws.PathParameter("pvc", "PVC name.").DataType("string").Required(true)).
+		Param(ws.PathParameter("metrics_filter", "The metric name filter consists of a regexp pattern. It specifies which metric data to return. For example, the following filter matches both PVC available and used inodes: `pvc_inodes_available|pvc_inodes_used`. View available metrics at [kubesphere.io](https://docs.kubesphere.io/advanced-v2.0/zh-CN/api-reference/monitoring-metrics/).").DataType("string").Required(false)).
+		Param(ws.QueryParameter("start", "Start time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1559347200. ").DataType("string").Required(false)).
+		Param(ws.QueryParameter("end", "End time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1561939200. ").DataType("string").Required(false)).
+		Param(ws.QueryParameter("step", "Time interval. Retrieve metric data at a fixed interval within the time range of start and end. It requires both **start** and **end** are provided. The format is [0-9]+[smhdwy]. Defaults to 10m (i.e. 10 min).").DataType("string").DefaultValue("10m").Required(false)).
+		Param(ws.QueryParameter("time", "A timestamp in Unix time format. Retrieve metric data at a single point in time. Defaults to now. Time and the combination of start, end, step are mutually exclusive.").DataType("string").Required(false)).
+		Metadata(restfulspec.KeyOpenAPITags, []string{constants.PVCMetricsTag}).
+		Writes(metrics.FormatedLevelMetric{}).
+		Returns(http.StatusOK, RespOK, metrics.FormatedLevelMetric{})).
+		Consumes(restful.MIME_JSON, restful.MIME_XML).
+		Produces(restful.MIME_JSON)
+
 	ws.Route(ws.GET("/nodes/{node}/pods").To(monitoring.MonitorAllPodsOnSpecificNode).
 		Doc("Get pod-level metric data of all pods on a specific node.").
 		Param(ws.PathParameter("node", "Node name.").DataType("string").Required(true)).

pkg/apiserver/monitoring/monitoring.go

@@ -235,6 +235,33 @@ func MonitorNode(request *restful.Request, response *restful.Response) {
 	}
 }
 
+func MonitorAllPVCsOfSpecificNamespace(request *restful.Request, response *restful.Response) {
+	MonitorPVC(request, response)
+}
+
+func MonitorAllPVCsOfSpecificStorageClass(request *restful.Request, response *restful.Response) {
+	MonitorPVC(request, response)
+}
+
+func MonitorSpecificPVCofSpecificNamespace(request *restful.Request, response *restful.Response) {
+	MonitorPVC(request, response)
+}
+
+func MonitorPVC(request *restful.Request, response *restful.Response) {
+	requestParams := prometheus.ParseMonitoringRequestParams(request)
+	pvcName := requestParams.PVCName
+	if pvcName != "" {
+		requestParams.ResourcesFilter = fmt.Sprintf("^%s$", requestParams.PVCName)
+	}
+
+	rawMetrics := metrics.GetPVCLevelMetrics(requestParams)
+	// sorting
+	sortedMetrics, maxMetricCount := metrics.Sort(requestParams.SortMetricName, requestParams.SortType, rawMetrics)
+	// paging
+	pagedMetrics := metrics.Page(requestParams.PageNum, requestParams.LimitNum, sortedMetrics, maxMetricCount)
+	response.WriteAsJson(pagedMetrics)
+}
+
 func MonitorComponent(request *restful.Request, response *restful.Response) {
 	requestParams := prometheus.ParseMonitoringRequestParams(request)
 

pkg/constants/constants.go

@@ -68,6 +68,7 @@ const (
 	NodeMetricsTag             = "Node Metrics"
 	NamespaceMetricsTag        = "Namespace Metrics"
 	PodMetricsTag              = "Pod Metrics"
+	PVCMetricsTag              = "PVC Metrics"
 	ContainerMetricsTag        = "Container Metrics"
 	WorkloadMetricsTag         = "Workload Metrics"
 	WorkspaceMetricsTag        = "Workspace Metrics"

pkg/models/metrics/metrics.go

@@ -276,6 +276,16 @@ func AssemblePodMetricRequestInfo(monitoringRequest *client.MonitoringRequestPar
 	return queryType, params, rule == ""
 }
 
+func AssemblePVCMetricRequestInfo(monitoringRequest *client.MonitoringRequestParams, metricName string) (string, string, bool) {
+	queryType := monitoringRequest.QueryType
+
+	paramValues := monitoringRequest.Params
+
+	rule := MakePVCPromQL(metricName, monitoringRequest.NsName, monitoringRequest.PVCName, monitoringRequest.StorageClassName, monitoringRequest.ResourcesFilter)
+	params := makeRequestParamString(rule, paramValues)
+	return queryType, params, rule == ""
+}
+
 func GetNodeAddressInfo() *map[string][]v1.NodeAddress {
 	nodeLister := informers.SharedInformerFactory().Core().V1().Nodes().Lister()
 	nodes, err := nodeLister.List(labels.Everything())
@@ -901,6 +911,49 @@ func GetContainerLevelMetrics(monitoringRequest *client.MonitoringRequestParams)
 	}
 }
 
+func GetPVCLevelMetrics(monitoringRequest *client.MonitoringRequestParams) *FormatedLevelMetric {
+	metricsFilter := monitoringRequest.MetricsFilter
+	if metricsFilter == "" {
+		metricsFilter = ".*"
+	}
+
+	var ch = make(chan *FormatedMetric, ChannelMaxCapacity)
+	var wg sync.WaitGroup
+
+	for _, metricName := range PVCMetricsNames {
+		matched, err := regexp.MatchString(metricsFilter, metricName)
+		if err == nil && matched {
+			wg.Add(1)
+			go func(metricName string) {
+				queryType, params, nullRule := AssemblePVCMetricRequestInfo(monitoringRequest, metricName)
+				if !nullRule {
+					metricsStr := client.SendMonitoringRequest(client.PrometheusEndpoint, queryType, params)
+					ch <- ReformatJson(metricsStr, metricName, map[string]string{MetricLevelPVC: ""})
+				} else {
+					ch <- nil
+				}
+				wg.Done()
+			}(metricName)
+		}
+	}
+
+	wg.Wait()
+	close(ch)
+
+	var metricsArray []FormatedMetric
+
+	for oneMetric := range ch {
+		if oneMetric != nil {
+			metricsArray = append(metricsArray, *oneMetric)
+		}
+	}
+
+	return &FormatedLevelMetric{
+		MetricsLevel: MetricLevelPVC,
+		Results:      metricsArray,
+	}
+}
+
 func GetComponentLevelMetrics(monitoringRequest *client.MonitoringRequestParams) *FormatedLevelMetric {
 	metricsFilter := monitoringRequest.MetricsFilter
 	if metricsFilter == "" {

pkg/models/metrics/metricsrule.go

@@ -198,6 +198,37 @@ func MakePodPromQL(metricName, nsName, nodeID, podName, podFilter string) string
 	return promql
 }
 
+func MakePVCPromQL(metricName, nsName, pvcName, scName, pvcFilter string) string {
+	if pvcFilter == "" {
+		pvcFilter = ".*"
+	}
+
+	var promql = ""
+	if nsName != "" {
+		// get pvc metrics by namespace
+		if pvcName != "" {
+			// specific pvc
+			promql = RulePromQLTmplMap[metricName]
+			promql = strings.Replace(promql, "$1", nsName, -1)
+			promql = strings.Replace(promql, "$2", pvcName, -1)
+		} else {
+			// all pvc in a specific namespace
+			metricName += "_ns"
+			promql = RulePromQLTmplMap[metricName]
+			promql = strings.Replace(promql, "$1", nsName, -1)
+			promql = strings.Replace(promql, "$2", pvcFilter, -1)
+		}
+	} else {
+		if scName != "" {
+			// all pvc in a specific storageclass
+			metricName += "_sc"
+			promql = RulePromQLTmplMap[metricName]
+			promql = strings.Replace(promql, "$1", scName, -1)
+		}
+	}
+	return promql
+}
+
 func MakeNamespacePromQL(nsName string, nsFilter string, metricsName string) string {
 	var recordingRule = RulePromQLTmplMap[metricsName]
 

pkg/models/metrics/metricsruleconst.go

@@ -67,6 +67,7 @@ const (
 	MetricLevelPodName          = "pod_name"
 	MetricLevelContainer        = "container"
 	MetricLevelContainerName    = "container_name"
+	MetricLevelPVC              = "persistentvolumeclaim"
 	MetricLevelWorkload         = "workload"
 	MetricLevelComponent        = "component"
 )
@@ -330,6 +331,15 @@ var ContainerMetricsNames = []string{
 	//"container_net_bytes_received",
 }
 
+var PVCMetricsNames = []string{
+	"pvc_inodes_available",
+	"pvc_inodes_used",
+	"pvc_inodes_total",
+	"pvc_bytes_available",
+	"pvc_bytes_used",
+	"pvc_bytes_total",
+}
+
 var ComponentMetricsNames = []string{
 	"etcd_server_list",
 	"etcd_server_total",
@@ -684,6 +694,26 @@ var RulePromQLTmplMap = MetricMap{
 	// New in ks 2.0
 	"workspace_pod_abnormal_ratio": `count((kube_pod_info{node!="", namespace$1} unless on (pod, namespace) (kube_pod_status_phase{job="kube-state-metrics", phase="Succeeded"}>0) unless on (pod, namespace) ((kube_pod_status_ready{job="kube-state-metrics", condition="true"}>0) and on (pod, namespace) (kube_pod_status_phase{job="kube-state-metrics", phase="Running"}>0)) unless on (pod, namespace) (kube_pod_container_status_waiting_reason{job="kube-state-metrics", reason="ContainerCreating"}>0)) / sum(kube_pod_status_phase{phase!~"Succeeded", namespace!="", namespace$1}) * on (namespace) group_left(label_kubesphere_io_workspace)(kube_namespace_labels{label_kubesphere_io_workspace$2}))`,
 
+	// PVC
+	"pvc_inodes_available":    `max (kubelet_volume_stats_inodes_free{namespace="$1",persistentvolumeclaim="$2"})by(namespace, persistentvolumeclaim)*on(namespace, persistentvolumeclaim)group_left(storageclass)kube_persistentvolumeclaim_info`,
+	"pvc_inodes_used":         `max (kubelet_volume_stats_inodes_used{namespace="$1", persistentvolumeclaim="$2"})by(namespace, persistentvolumeclaim)*on(namespace, persistentvolumeclaim)group_left(storageclass)kube_persistentvolumeclaim_info`,
+	"pvc_inodes_total":        `max (kubelet_volume_stats_inodes{namespace="$1", persistentvolumeclaim="$2"})by(namespace, persistentvolumeclaim)*on(namespace, persistentvolumeclaim)group_left(storageclass)kube_persistentvolumeclaim_info`,
+	"pvc_bytes_available":     `max (kubelet_volume_stats_available_bytes{namespace="$1", persistentvolumeclaim="$2"})by(namespace, persistentvolumeclaim)*on(namespace, persistentvolumeclaim)group_left(storageclass)kube_persistentvolumeclaim_info`,
+	"pvc_bytes_used":          `max (kubelet_volume_stats_used_bytes{namespace="$1", persistentvolumeclaim="$2"})by(namespace, persistentvolumeclaim)*on(namespace, persistentvolumeclaim)group_left(storageclass)kube_persistentvolumeclaim_info`,
+	"pvc_bytes_total":         `max (kubelet_volume_stats_capacity_bytes{namespace="$1", persistentvolumeclaim="$2"})by(namespace, persistentvolumeclaim)*on(namespace, persistentvolumeclaim)group_left(storageclass)kube_persistentvolumeclaim_info`,
+	"pvc_inodes_available_ns": `max (kubelet_volume_stats_inodes_free{namespace="$1",persistentvolumeclaim=~"$2"})by(namespace, persistentvolumeclaim)*on(namespace, persistentvolumeclaim)group_left(storageclass)kube_persistentvolumeclaim_info`,
+	"pvc_inodes_used_ns":      `max (kubelet_volume_stats_inodes_used{namespace="$1",persistentvolumeclaim=~"$2"})by(namespace, persistentvolumeclaim)*on(namespace, persistentvolumeclaim)group_left(storageclass)kube_persistentvolumeclaim_info`,
+	"pvc_inodes_total_ns":     `max (kubelet_volume_stats_inodes{namespace="$1",persistentvolumeclaim=~"$2"})by(namespace, persistentvolumeclaim)*on(namespace, persistentvolumeclaim)group_left(storageclass)kube_persistentvolumeclaim_info`,
+	"pvc_bytes_available_ns":  `max (kubelet_volume_stats_available_bytes{namespace="$1",persistentvolumeclaim=~"$2"})by(namespace, persistentvolumeclaim)*on(namespace, persistentvolumeclaim)group_left(storageclass)kube_persistentvolumeclaim_info`,
+	"pvc_bytes_used_ns":       `max (kubelet_volume_stats_used_bytes{namespace="$1",persistentvolumeclaim=~"$2"})by(namespace, persistentvolumeclaim)*on(namespace, persistentvolumeclaim)group_left(storageclass)kube_persistentvolumeclaim_info`,
+	"pvc_bytes_total_ns":      `max (kubelet_volume_stats_capacity_bytes{namespace="$1",persistentvolumeclaim=~"$2"})by(namespace, persistentvolumeclaim)*on(namespace, persistentvolumeclaim)group_left(storageclass)kube_persistentvolumeclaim_info`,
+	"pvc_inodes_available_sc": `max (kubelet_volume_stats_inodes_free)by(namespace,persistentvolumeclaim)*on(namespace, persistentvolumeclaim)group_left(storageclass)kube_persistentvolumeclaim_info{storageclass="$1"}`,
+	"pvc_inodes_used_sc":      `max (kubelet_volume_stats_inodes_used)by(namespace, persistentvolumeclaim)*on(namespace, persistentvolumeclaim)group_left(storageclass)kube_persistentvolumeclaim_info{storageclass="$1"}`,
+	"pvc_inodes_total_sc":     `max (kubelet_volume_stats_inodes)by(namespace, persistentvolumeclaim)*on(namespace, persistentvolumeclaim)group_left(storageclass)kube_persistentvolumeclaim_info{storageclass="$1"}`,
+	"pvc_bytes_available_sc":  `max (kubelet_volume_stats_available_bytes)by(namespace, persistentvolumeclaim)*on(namespace, persistentvolumeclaim)group_left(storageclass)kube_persistentvolumeclaim_info{storageclass="$1"}`,
+	"pvc_bytes_used_sc":       `max (kubelet_volume_stats_used_bytes)by(namespace, persistentvolumeclaim)*on(namespace, persistentvolumeclaim)group_left(storageclass)kube_persistentvolumeclaim_info{storageclass="$1"}`,
+	"pvc_bytes_total_sc":      `max (kubelet_volume_stats_capacity_bytes)by(namespace, persistentvolumeclaim)*on(namespace, persistentvolumeclaim)group_left(storageclass)kube_persistentvolumeclaim_info{storageclass="$1"}`,
+
 	// component
 	"etcd_server_list":                           `label_replace(up{job="etcd"}, "node_ip", "$1", "instance", "(.*):.*")`,
 	"etcd_server_total":                          `count(up{job="etcd"})`,

pkg/simple/client/prometheus/prometheusclient.go

@@ -65,6 +65,8 @@ type MonitoringRequestParams struct {
 	WsName           string
 	NsName           string
 	PodName          string
+	PVCName          string
+	StorageClassName string
 	ContainerName    string
 	WorkloadKind     string
 	ComponentName    string
@@ -113,6 +115,8 @@ func ParseMonitoringRequestParams(request *restful.Request) *MonitoringRequestPa
 	wsName := strings.Trim(request.PathParameter("workspace"), " ")
 	nsName := strings.Trim(request.PathParameter("namespace"), " ")
 	podName := strings.Trim(request.PathParameter("pod"), " ")
+	pvcName := strings.Trim(request.PathParameter("pvc"), " ")
+	storageClassName := strings.Trim(request.PathParameter("storageclass"), " ")
 	containerName := strings.Trim(request.PathParameter("container"), " ")
 	workloadKind := strings.Trim(request.PathParameter("kind"), " ")
 	componentName := strings.Trim(request.PathParameter("component"), " ")
@@ -131,6 +135,8 @@ func ParseMonitoringRequestParams(request *restful.Request) *MonitoringRequestPa
 		WsName:           wsName,
 		NsName:           nsName,
 		PodName:          podName,
+		PVCName:          pvcName,
+		StorageClassName: storageClassName,
 		ContainerName:    containerName,
 		WorkloadKind:     workloadKind,
 		ComponentName:    componentName,