Merge pull request #214 from carmanzhang/monitor

change workload metrics, add node-pod-container metrics api

pkg/apis/v1alpha/monitoring/monitor_handler.go

@@ -51,9 +51,21 @@ func (u Monitor) monitorPod(request *restful.Request, response *restful.Response
 
 func (u Monitor) monitorContainer(request *restful.Request, response *restful.Response) {
 	requestParams := client.ParseMonitoringRequestParams(request)
-	res := metrics.MonitorContainer(requestParams)
+	metricName := requestParams.MetricsName
+	if requestParams.MetricsFilter != "" {
+		rawMetrics := metrics.MonitorAllMetrics(requestParams, metrics.MetricLevelContainer)
+		// sorting
+		sortedMetrics, maxMetricCount := metrics.Sort(requestParams.SortMetricName, requestParams.SortType, rawMetrics, metrics.MetricLevelContainerName)
+		// paging
+		pagedMetrics := metrics.Page(requestParams.PageNum, requestParams.LimitNum, sortedMetrics, maxMetricCount)
+
+		response.WriteAsJson(pagedMetrics)
+
+	} else {
+		res := metrics.MonitorContainer(requestParams, metricName)
+		response.WriteAsJson(res)
+	}
 
-	response.WriteAsJson(res)
 }
 
 func (u Monitor) monitorWorkload(request *restful.Request, response *restful.Response) {
@@ -323,13 +335,36 @@ func Register(ws *restful.WebService, subPath string) {
 		Consumes(restful.MIME_JSON, restful.MIME_XML).
 		Produces(restful.MIME_JSON)
 
+	ws.Route(ws.GET(subPath+"/nodes/{node_id}/pods/{pod_name}/containers").To(u.monitorContainer).
+		Filter(route.RouteLogging).
+		Doc("monitor specific pod level metrics by nodeid").
+		Param(ws.PathParameter("node_id", "specific node").DataType("string").Required(true)).
+		Param(ws.PathParameter("pod_name", "specific pod").DataType("string").Required(true)).
+		Param(ws.QueryParameter("containers_filter", "container re2 expression filter").DataType("string").Required(false).DefaultValue("")).
+		Param(ws.QueryParameter("metrics_filter", "metrics name cpu memory...").DataType("string").Required(false)).
+		Param(ws.QueryParameter("metrics_name", "metrics name cpu memory...").DataType("string").Required(true).DefaultValue("pod_memory_utilisation_wo_cache")).
+		Param(ws.QueryParameter("sort_metric", "sort metric").DataType("string").Required(false)).
+		Param(ws.QueryParameter("sort_type", "ascending descending order").DataType("string").Required(false)).
+		Param(ws.QueryParameter("page", "page number").DataType("string").Required(false).DefaultValue("1")).
+		Param(ws.QueryParameter("limit", "metrics name cpu memory...in re2 regex").DataType("string").Required(false).DefaultValue("4")).
+		Param(ws.QueryParameter("type", "rank, statistic").DataType("string").Required(false).DefaultValue("rank")).
+		Metadata(restfulspec.KeyOpenAPITags, tags)).
+		Consumes(restful.MIME_JSON, restful.MIME_XML).
+		Produces(restful.MIME_JSON)
+
 	ws.Route(ws.GET(subPath+"/namespaces/{ns_name}/pods/{pod_name}/containers").To(u.monitorContainer).
 		Filter(route.RouteLogging).
 		Doc("monitor containers level metrics").
 		Param(ws.PathParameter("ns_name", "specific namespace").DataType("string").Required(true).DefaultValue("monitoring")).
 		Param(ws.PathParameter("pod_name", "specific pod").DataType("string").Required(true).DefaultValue("")).
 		Param(ws.QueryParameter("containers_filter", "container re2 expression filter").DataType("string").Required(false).DefaultValue("")).
+		Param(ws.QueryParameter("metrics_filter", "metrics name cpu memory...").DataType("string").Required(false)).
 		Param(ws.QueryParameter("metrics_name", "metrics name cpu memory...").DataType("string").Required(true).DefaultValue("container_memory_utilisation_wo_cache")).
+		Param(ws.QueryParameter("sort_metric", "sort metric").DataType("string").Required(false)).
+		Param(ws.QueryParameter("sort_type", "ascending descending order").DataType("string").Required(false)).
+		Param(ws.QueryParameter("page", "page number").DataType("string").Required(false).DefaultValue("1")).
+		Param(ws.QueryParameter("limit", "metrics name cpu memory...in re2 regex").DataType("string").Required(false).DefaultValue("4")).
+		Param(ws.QueryParameter("type", "rank, statistic").DataType("string").Required(false).DefaultValue("rank")).
 		Metadata(restfulspec.KeyOpenAPITags, tags)).
 		Consumes(restful.MIME_JSON, restful.MIME_XML).
 		Produces(restful.MIME_JSON)

pkg/models/metrics/metrics.go

@@ -120,51 +120,6 @@ type OneComponentStatus struct {
 	Error string `json:"error,omitempty"`
 }
 
-func renameWorkload(formatedMetric *FormatedMetric, relationMap map[string]string) {
-	if formatedMetric.Status == MetricStatusSuccess {
-		for i := 0; i < len(formatedMetric.Data.Result); i++ {
-			metricDesc := formatedMetric.Data.Result[i][ResultItemMetric]
-			metricDescMap, ensure := metricDesc.(map[string]interface{})
-			if ensure {
-				if wl, exist := metricDescMap[MetricLevelWorkload]; exist {
-					if deployName, exist := relationMap[wl.(string)]; exist {
-						metricDescMap[MetricLevelWorkload] = deployName
-					}
-				}
-			}
-		}
-	}
-}
-
-func getReplicaAndDeployRelation(nsName string) map[string]string {
-	rule := strings.Replace(WorkloadReplicaSetOwnerRule, "$1", nsName, -1)
-
-	params := makeRequestParamString(rule, make(url.Values))
-
-	res := client.SendMonitoringRequest(client.DefaultQueryType, params)
-	formatedMetric := ReformatJson(res, "")
-	var relationMap = make(map[string]string)
-	if formatedMetric.Status == MetricStatusSuccess {
-		for i := 0; i < len(formatedMetric.Data.Result); i++ {
-			metricDesc := formatedMetric.Data.Result[i][ResultItemMetric]
-			metricDescMap, ensure := metricDesc.(map[string]interface{})
-			if ensure {
-				if ownerKind, exist := metricDescMap["owner_kind"]; exist && ownerKind == ReplicaSet {
-					if ownerName, exist := metricDescMap["owner_name"]; exist {
-						replicaName, sure := ownerName.(string)
-						if sure {
-							deployName := replicaName[:strings.LastIndex(replicaName, "-")]
-							relationMap[replicaName] = deployName
-						}
-					}
-				}
-			}
-
-		}
-	}
-	return relationMap
-}
-
 func getPodNameRegexInWorkload(res string) string {
 
 	data := []byte(res)
@@ -237,7 +192,7 @@ func unifyMetricHistoryTimeRange(fmtMetrics *FormatedMetric) {
 	}
 }
 
-func AssembleSpecificWorkloadMetricRequestInfo(monitoringRequest *client.MonitoringRequestParams, metricName string) (string, string) {
+func AssembleSpecificWorkloadMetricRequestInfo(monitoringRequest *client.MonitoringRequestParams, metricName string) (string, string, bool) {
 
 	nsName := monitoringRequest.NsName
 	wkName := monitoringRequest.WorkloadName
@@ -254,7 +209,7 @@ func AssembleSpecificWorkloadMetricRequestInfo(monitoringRequest *client.Monitor
 	rule = MakePodPromQL(metricName, nsName, "", "", podNamesFilter)
 	params = makeRequestParamString(rule, paramValues)
 
-	return queryType, params
+	return queryType, params, rule == ""
 }
 
 func AssembleAllWorkloadMetricRequestInfo(monitoringRequest *client.MonitoringRequestParams, metricName string) (string, string) {
@@ -311,15 +266,20 @@ func AddNodeAddressMetric(nodeMetric *FormatedMetric, nodeAddress *map[string][]
 	}
 }
 
-func MonitorContainer(monitoringRequest *client.MonitoringRequestParams) *FormatedMetric {
+func MonitorContainer(monitoringRequest *client.MonitoringRequestParams, metricName string) *FormatedMetric {
+	queryType, params := AssembleContainerMetricRequestInfo(monitoringRequest, metricName)
+	res := GetMetric(queryType, params, metricName)
+	return res
+}
+
+func AssembleContainerMetricRequestInfo(monitoringRequest *client.MonitoringRequestParams, metricName string) (string, string) {
 	queryType := monitoringRequest.QueryType
 
 	paramValues := monitoringRequest.Params
-	rule := MakeContainerPromQL(monitoringRequest.NsName, monitoringRequest.PodName, monitoringRequest.ContainerName, monitoringRequest.MetricsName, monitoringRequest.ContainersFilter)
+	rule := MakeContainerPromQL(monitoringRequest.NsName, monitoringRequest.NodeId, monitoringRequest.PodName, monitoringRequest.ContainerName, metricName, monitoringRequest.ContainersFilter)
 	params := makeRequestParamString(rule, paramValues)
 
-	res := GetMetric(queryType, params, monitoringRequest.MetricsName)
-	return res
+	return queryType, params
 }
 
 func AssembleNamespaceMetricRequestInfo(monitoringRequest *client.MonitoringRequestParams, metricName string) (string, string) {
@@ -646,20 +606,14 @@ func MonitorAllMetrics(monitoringRequest *client.MonitoringRequestParams, resour
 	case MetricLevelWorkload:
 		{
 			if monitoringRequest.Tp == "rank" {
-				// get relationship between replicaset and deployment
-				relationMap := getReplicaAndDeployRelation(monitoringRequest.NsName)
+
 				for _, metricName := range WorkloadMetricsNames {
 					bol, err := regexp.MatchString(metricsFilter, metricName)
 					if err == nil && bol {
 						wg.Add(1)
 						go func(metricName string) {
 							queryType, params := AssembleAllWorkloadMetricRequestInfo(monitoringRequest, metricName)
-							fmtMetrics := GetMetric(queryType, params, metricName)
-
-							// rename replica workload name
-							renameWorkload(fmtMetrics, relationMap)
-
-							ch <- fmtMetrics
+							ch <- GetMetric(queryType, params, metricName)
 							wg.Done()
 						}(metricName)
 					}
@@ -671,10 +625,12 @@ func MonitorAllMetrics(monitoringRequest *client.MonitoringRequestParams, resour
 						wg.Add(1)
 						go func(metricName string) {
 							metricName = strings.TrimLeft(metricName, "workload_")
-							queryType, params := AssembleSpecificWorkloadMetricRequestInfo(monitoringRequest, metricName)
-							fmtMetrics := GetMetric(queryType, params, metricName)
-							unifyMetricHistoryTimeRange(fmtMetrics)
-							ch <- fmtMetrics
+							queryType, params, nullRule := AssembleSpecificWorkloadMetricRequestInfo(monitoringRequest, metricName)
+							if !nullRule {
+								fmtMetrics := GetMetric(queryType, params, metricName)
+								unifyMetricHistoryTimeRange(fmtMetrics)
+								ch <- fmtMetrics
+							}
 							wg.Done()
 						}(metricName)
 					}
@@ -699,6 +655,20 @@ func MonitorAllMetrics(monitoringRequest *client.MonitoringRequestParams, resour
 				}
 			}
 		}
+	case MetricLevelContainer:
+		{
+			for _, metricName := range ContainerMetricsNames {
+				bol, err := regexp.MatchString(metricsFilter, metricName)
+				if err == nil && bol {
+					wg.Add(1)
+					go func(metricName string) {
+						queryType, params := AssembleContainerMetricRequestInfo(monitoringRequest, metricName)
+						ch <- GetMetric(queryType, params, metricName)
+						wg.Done()
+					}(metricName)
+				}
+			}
+		}
 	}
 
 	wg.Wait()

pkg/models/metrics/metricsrule.go

@@ -84,27 +84,31 @@ func MakeSpecificWorkspacePromQL(metricsName, nsFilter string) string {
 	return promql
 }
 
-func MakeContainerPromQL(nsName, podName, containerName, metricName, containerFilter string) string {
-	var promql = ""
-	if containerName == "" {
-		// all containers maybe use filter
-		metricName += "_all"
+func MakeContainerPromQL(nsName, nodeId, podName, containerName, metricName, containerFilter string) string {
+	var promql string
+
+	if nsName != "" {
+		// get container metrics from namespace-pod
 		promql = RulePromQLTmplMap[metricName]
 		promql = strings.Replace(promql, "$1", nsName, -1)
-		promql = strings.Replace(promql, "$2", podName, -1)
+	} else {
+		// get container metrics from node-pod
+		promql = RulePromQLTmplMap[metricName+"_node"]
+		promql = strings.Replace(promql, "$1", nodeId, -1)
+	}
+
+	promql = strings.Replace(promql, "$2", podName, -1)
+
+	if containerName == "" {
 
 		if containerFilter == "" {
 			containerFilter = ".*"
 		}
-
 		promql = strings.Replace(promql, "$3", containerFilter, -1)
-		return promql
+	} else {
+		promql = strings.Replace(promql, "$3", containerName, -1)
 	}
-	promql = RulePromQLTmplMap[metricName]
 
-	promql = strings.Replace(promql, "$1", nsName, -1)
-	promql = strings.Replace(promql, "$2", podName, -1)
-	promql = strings.Replace(promql, "$3", containerName, -1)
 	return promql
 }
 

pkg/models/metrics/util.go

@@ -113,9 +113,13 @@ func Sort(sortMetricName string, sortType string, fmtLevelMetric *FormatedLevelM
 					// for some reasons, 'metric' may not contain `resourceType` field
 					// example: {"metric":{},"value":[1541142931.731,"3"]}
 					k, exist := r[ResultItemMetric].(map[string]interface{})[resourceType]
+					key := k.(string)
 					if exist {
-						indexMap[k.(string)] = i
-						i = i + 1
+						if _, exist := indexMap[key]; !exist {
+							indexMap[key] = i
+							i = i + 1
+						}
+
 					}
 				}
 			}