Merge pull request #210 from carmanzhang/monitor

changed promqls and fixed several monitoring bugs

pkg/apis/v1alpha/monitoring/monitor_handler.go

@@ -73,12 +73,14 @@ func (u Monitor) monitorAllWorkspaces(request *restful.Request, response *restfu
 
 	requestParams := client.ParseMonitoringRequestParams(request)
 
-	if requestParams.Tp == "_statistics" {
+	tp := requestParams.Tp
+	if tp == "_statistics" {
 		// merge multiple metric: all-devops, all-roles, all-projects...this api is designed for admin
 		res := metrics.MonitorAllWorkspacesStatistics()
 
 		response.WriteAsJson(res)
-	} else {
+
+	} else if tp == "rank" {
 		rawMetrics := metrics.MonitorAllWorkspaces(requestParams)
 		// sorting
 		sortedMetrics, maxMetricCount := metrics.Sort(requestParams.SortMetricName, requestParams.SortType, rawMetrics, metrics.MetricLevelWorkspace)
@@ -86,6 +88,9 @@ func (u Monitor) monitorAllWorkspaces(request *restful.Request, response *restfu
 		pagedMetrics := metrics.Page(requestParams.PageNum, requestParams.LimitNum, sortedMetrics, maxMetricCount)
 
 		response.WriteAsJson(pagedMetrics)
+	} else {
+		res := metrics.MonitorAllMetrics(requestParams, metrics.MetricLevelWorkspace)
+		response.WriteAsJson(res)
 	}
 }
 
@@ -145,10 +150,6 @@ func (u Monitor) monitorCluster(request *restful.Request, response *restful.Resp
 		queryType, params := metrics.AssembleClusterMetricRequestInfo(requestParams, metricName)
 		res := metrics.GetMetric(queryType, params, metricName)
 
-		if metricName == metrics.MetricNameWorkspaceAllProjectCount {
-			res = metrics.MonitorWorkspaceNamespaceHistory(res)
-		}
-
 		response.WriteAsJson(res)
 	} else {
 		// multiple

pkg/models/metrics/metrics.go

@@ -277,12 +277,28 @@ func AssembleNamespaceMetricRequestInfo(monitoringRequest *client.MonitoringRequ
 	return queryType, params
 }
 
-func AssembleWorkspaceMetricRequestInfo(monitoringRequest *client.MonitoringRequestParams, namespaceList []string, metricName string) (string, string) {
+func AssembleSpecificWorkspaceMetricRequestInfo(monitoringRequest *client.MonitoringRequestParams, namespaceList []string, metricName string) (string, string) {
+
 	nsFilter := "^(" + strings.Join(namespaceList, "|") + ")$"
 
 	queryType := monitoringRequest.QueryType
 
-	rule := MakeWorkspacePromQL(metricName, nsFilter)
+	rule := MakeSpecificWorkspacePromQL(metricName, nsFilter)
+	paramValues := monitoringRequest.Params
+	params := makeRequestParamString(rule, paramValues)
+	return queryType, params
+}
+
+func AssembleAllWorkspaceMetricRequestInfo(monitoringRequest *client.MonitoringRequestParams, namespaceList []string, metricName string) (string, string) {
+	var nsFilter = "^()$"
+
+	if namespaceList != nil {
+		nsFilter = "^(" + strings.Join(namespaceList, "|") + ")$"
+	}
+
+	queryType := monitoringRequest.QueryType
+
+	rule := MakeAllWorkspacesPromQL(metricName, nsFilter)
 	paramValues := monitoringRequest.Params
 	params := makeRequestParamString(rule, paramValues)
 	return queryType, params
@@ -330,6 +346,9 @@ func MonitorAllWorkspaces(monitoringRequest *client.MonitoringRequestParams) *Fo
 	}
 	var filterMetricsName []string
 	for _, metricName := range WorkspaceMetricsNames {
+		if metricName == MetricNameWorkspaceAllProjectCount {
+			continue
+		}
 		bol, err := regexp.MatchString(metricsFilter, metricName)
 		if err == nil && bol {
 			filterMetricsName = append(filterMetricsName, metricName)
@@ -400,7 +419,7 @@ func collectWorkspaceMetric(monitoringRequest *client.MonitoringRequestParams, w
 		wg.Add(1)
 		go func(metricName string) {
 
-			queryType, params := AssembleWorkspaceMetricRequestInfo(monitoringRequest, namespaceArray, metricName)
+			queryType, params := AssembleSpecificWorkspaceMetricRequestInfo(monitoringRequest, namespaceArray, metricName)
 			ch <- GetMetric(queryType, params, metricName)
 
 			wg.Done()
@@ -448,11 +467,6 @@ func MonitorAllMetrics(monitoringRequest *client.MonitoringRequestParams, resour
 						queryType, params := AssembleClusterMetricRequestInfo(monitoringRequest, metricName)
 						clusterMetrics := GetMetric(queryType, params, metricName)
 
-						// for this special case, get namespace history which in a workspace by determining namespace label
-						if metricName == MetricNameWorkspaceAllProjectCount {
-							clusterMetrics = MonitorWorkspaceNamespaceHistory(clusterMetrics)
-						}
-
 						ch <- clusterMetrics
 
 						wg.Done()
@@ -476,6 +490,8 @@ func MonitorAllMetrics(monitoringRequest *client.MonitoringRequestParams, resour
 		}
 	case MetricLevelWorkspace:
 		{
+			// a specific workspace's metrics
+			if monitoringRequest.WsName != "" {
 				namespaceArray, err := workspaces.WorkspaceNamespaces(monitoringRequest.WsName)
 				if err != nil {
 					glog.Errorln(err.Error())
@@ -484,6 +500,10 @@ func MonitorAllMetrics(monitoringRequest *client.MonitoringRequestParams, resour
 
 				if monitoringRequest.Tp == "rank" {
 					for _, metricName := range NamespaceMetricsNames {
+						if metricName == MetricNameWorkspaceAllProjectCount {
+							continue
+						}
+
 						bol, err := regexp.MatchString(metricsFilter, metricName)
 						ns := "^(" + strings.Join(namespaceArray, "|") + ")$"
 						monitoringRequest.NsFilter = ns
@@ -498,13 +518,56 @@ func MonitorAllMetrics(monitoringRequest *client.MonitoringRequestParams, resour
 					}
 
 				} else {
+					for _, metricName := range WorkspaceMetricsNames {
+
+						if metricName == MetricNameWorkspaceAllProjectCount {
+							continue
+						}
+
+						bol, err := regexp.MatchString(metricsFilter, metricName)
+						if err == nil && bol {
+							wg.Add(1)
+							go func(metricName string) {
+								queryType, params := AssembleSpecificWorkspaceMetricRequestInfo(monitoringRequest, namespaceArray, metricName)
+								ch <- GetMetric(queryType, params, metricName)
+								wg.Done()
+							}(metricName)
+						}
+					}
+				}
+			} else {
+				// sum all workspaces
+				_, namespaceWorkspaceMap, err := workspaces.GetAllOrgAndProjList()
+
+				if err != nil {
+					glog.Errorln(err.Error())
+				}
+
+				nsList := make([]string, 0)
+				for ns := range namespaceWorkspaceMap {
+					if namespaceWorkspaceMap[ns] == "" {
+						nsList = append(nsList, ns)
+					}
+				}
+
 				for _, metricName := range WorkspaceMetricsNames {
 					bol, err := regexp.MatchString(metricsFilter, metricName)
 					if err == nil && bol {
+
 						wg.Add(1)
+
 						go func(metricName string) {
-							queryType, params := AssembleWorkspaceMetricRequestInfo(monitoringRequest, namespaceArray, metricName)
+							queryType, params := AssembleAllWorkspaceMetricRequestInfo(monitoringRequest, nil, metricName)
+
+							if metricName == MetricNameWorkspaceAllProjectCount {
+								res := GetMetric(queryType, params, metricName)
+								res = MonitorWorkspaceNamespaceHistory(res)
+								ch <- res
+
+							} else {
 								ch <- GetMetric(queryType, params, metricName)
+							}
+
 							wg.Done()
 						}(metricName)
 					}

pkg/models/metrics/metricsrule.go

@@ -57,8 +57,18 @@ func MakeWorkloadRule(wkKind, wkName, namespace string) string {
 	return rule
 }
 
-func MakeWorkspacePromQL(metricsName string, nsFilter string) string {
-	promql := RulePromQLTmplMap[metricsName]
+func MakeAllWorkspacesPromQL(metricsName, nsFilter string) string {
+
+	var promql = RulePromQLTmplMap[metricsName]
+	nsFilter = "!~\"" + nsFilter + "\""
+	promql = strings.Replace(promql, "$1", nsFilter, -1)
+	return promql
+}
+
+func MakeSpecificWorkspacePromQL(metricsName, nsFilter string) string {
+
+	var promql = RulePromQLTmplMap[metricsName]
+	nsFilter = "=~\"" + nsFilter + "\""
 	promql = strings.Replace(promql, "$1", nsFilter, -1)
 	return promql
 }

pkg/models/metrics/util.go

@@ -23,6 +23,8 @@ import (
 	"strconv"
 	"unicode"
 
+	"runtime/debug"
+
 	"github.com/golang/glog"
 )
 
@@ -50,6 +52,12 @@ func (wrapper FormatedMetricDataWrapper) Swap(i, j int) {
 
 // sorted metric by ascending or descending order
 func Sort(sortMetricName string, sortType string, fmtLevelMetric *FormatedLevelMetric, resourceType string) (*FormatedLevelMetric, int) {
+	defer func() {
+		if err := recover(); err != nil {
+			glog.Errorln(err)
+			debug.PrintStack()
+		}
+	}()
 
 	if sortMetricName == "" {
 		return fmtLevelMetric, -1
@@ -75,6 +83,12 @@ func Sort(sortMetricName string, sortType string, fmtLevelMetric *FormatedLevelM
 						value2 := (*q)[ResultItemValue].([]interface{})
 						v1, _ := strconv.ParseFloat(value1[len(value1)-1].(string), 64)
 						v2, _ := strconv.ParseFloat(value2[len(value2)-1].(string), 64)
+						if v1 == v2 {
+							resourceName1 := (*p)[ResultItemMetric].(map[string]interface{})[resourceType]
+							resourceName2 := (*q)[ResultItemMetric].(map[string]interface{})[resourceType]
+							return resourceName1.(string) < resourceName2.(string)
+						}
+
 						return v1 < v2
 					}})
 				} else {
@@ -84,6 +98,13 @@ func Sort(sortMetricName string, sortType string, fmtLevelMetric *FormatedLevelM
 						value2 := (*q)[ResultItemValue].([]interface{})
 						v1, _ := strconv.ParseFloat(value1[len(value1)-1].(string), 64)
 						v2, _ := strconv.ParseFloat(value2[len(value2)-1].(string), 64)
+
+						if v1 == v2 {
+							resourceName1 := (*p)[ResultItemMetric].(map[string]interface{})[resourceType]
+							resourceName2 := (*q)[ResultItemMetric].(map[string]interface{})[resourceType]
+							return resourceName1.(string) > resourceName2.(string)
+						}
+
 						return v1 > v2
 					}})
 				}
@@ -110,7 +131,13 @@ func Sort(sortMetricName string, sortType string, fmtLevelMetric *FormatedLevelM
 		}
 	}
 
-	for resource, _ := range currentResourceMap {
+	var keys []string
+	for k := range currentResourceMap {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+
+	for _, resource := range keys {
 		if _, exist := indexMap[resource]; !exist {
 			indexMap[resource] = i
 			i = i + 1