Merge pull request #2076 from huanggze/custom-monitoring

api: list metric labels and values

pkg/kapis/monitoring/v1alpha3/handler.go

@@ -19,6 +19,7 @@
 package v1alpha3
 
 import (
+	"errors"
 	"github.com/emicklei/go-restful"
 	"k8s.io/client-go/kubernetes"
 	"kubesphere.io/kubesphere/pkg/api"
@@ -198,6 +199,30 @@ func (h handler) handleMetadataQuery(req *restful.Request, resp *restful.Respons
 	resp.WriteAsJson(res)
 }
 
+func (h handler) handleMetricLabelSetQuery(req *restful.Request, resp *restful.Response) {
+	var res model.MetricLabelSet
+
+	params := parseRequestParams(req)
+	if params.metric == "" || params.start == "" || params.end == "" {
+		api.HandleBadRequest(resp, nil, errors.New("required fields are missing: [metric, start, end]"))
+		return
+	}
+
+	opt, err := h.makeQueryOptions(params, 0)
+	if err != nil {
+		if err.Error() == ErrNoHit {
+			resp.WriteAsJson(res)
+			return
+		}
+
+		api.HandleBadRequest(resp, nil, err)
+		return
+	}
+
+	res = h.mo.GetMetricLabelSet(params.metric, params.namespaceName, opt.start, opt.end)
+	resp.WriteAsJson(res)
+}
+
 func (h handler) handleAdhocQuery(req *restful.Request, resp *restful.Response) {
 	var res monitoring.Metric
 

pkg/kapis/monitoring/v1alpha3/helper.go

@@ -50,6 +50,7 @@ type reqParams struct {
 	storageClassName string
 	componentType    string
 	expression       string
+	metric           string
 }
 
 type queryOptions struct {
@@ -101,6 +102,7 @@ func parseRequestParams(req *restful.Request) reqParams {
 	r.storageClassName = req.PathParameter("storageclass")
 	r.componentType = req.PathParameter("component")
 	r.expression = req.QueryParameter("expr")
+	r.metric = req.QueryParameter("metric")
 	return r
 }
 

pkg/kapis/monitoring/v1alpha3/register.go

@@ -409,12 +409,24 @@ func AddToContainer(c *restful.Container, k8sClient kubernetes.Interface, monito
 		Returns(http.StatusOK, RespOK, model.Metadata{})).
 		Produces(restful.MIME_JSON)
 
+	ws.Route(ws.GET("/namespaces/{namespace}/targets/labelsets").
+		To(h.handleMetricLabelSetQuery).
+		Doc("List all available labels and values of a metric within a specific time span.").
+		Param(ws.PathParameter("namespace", "The name of the namespace.").DataType("string").Required(true)).
+		Param(ws.QueryParameter("metric", "The name of the metric").DataType("string").Required(true)).
+		Param(ws.QueryParameter("start", "Start time of query. It is a string with Unix time format, eg. 1559347200. ").DataType("string").Required(true)).
+		Param(ws.QueryParameter("end", "End time of query. It is a string with Unix time format, eg. 1561939200. ").DataType("string").Required(true)).
+		Metadata(restfulspec.KeyOpenAPITags, []string{constants.CustomMetricsTag}).
+		Writes(model.MetricLabelSet{}).
+		Returns(http.StatusOK, RespOK, model.MetricLabelSet{})).
+		Produces(restful.MIME_JSON)
+
 	ws.Route(ws.GET("/namespaces/{namespace}/targets/query").
 		To(h.handleAdhocQuery).
 		Doc("Make an ad-hoc query in the specific namespace.").
 		Param(ws.PathParameter("namespace", "The name of the namespace.").DataType("string").Required(true)).
 		Param(ws.QueryParameter("expr", "The expression to be evaluated.").DataType("string").Required(false)).
-		Param(ws.QueryParameter("start", "Start time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1559347200. ").DataType("string").Required(true)).
+		Param(ws.QueryParameter("start", "Start time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1559347200. ").DataType("string").Required(false)).
 		Param(ws.QueryParameter("end", "End time of query. Use **start** and **end** to retrieve metric data over a time span. It is a string with Unix time format, eg. 1561939200. ").DataType("string").Required(false)).
 		Param(ws.QueryParameter("step", "Time interval. Retrieve metric data at a fixed interval within the time range of start and end. It requires both **start** and **end** are provided. The format is [0-9]+[smhdwy]. Defaults to 10m (i.e. 10 min).").DataType("string").DefaultValue("10m").Required(false)).
 		Param(ws.QueryParameter("time", "A timestamp in Unix time format. Retrieve metric data at a single point in time. Defaults to now. Time and the combination of start, end, step are mutually exclusive.").DataType("string").Required(false)).

pkg/models/monitoring/monitoring.go

@@ -19,6 +19,7 @@
 package monitoring
 
 import (
+	"k8s.io/klog"
 	"kubesphere.io/kubesphere/pkg/models/monitoring/expressions"
 	"kubesphere.io/kubesphere/pkg/simple/client/monitoring"
 	"time"
@@ -30,6 +31,7 @@ type MonitoringOperator interface {
 	GetNamedMetrics(metrics []string, time time.Time, opt monitoring.QueryOption) Metrics
 	GetNamedMetricsOverTime(metrics []string, start, end time.Time, step time.Duration, opt monitoring.QueryOption) Metrics
 	GetMetadata(namespace string) Metadata
+	GetMetricLabelSet(metric, namespace string, start, end time.Time) MetricLabelSet
 }
 
 type monitoringOperator struct {
@@ -78,3 +80,17 @@ func (mo monitoringOperator) GetMetadata(namespace string) Metadata {
 	data := mo.c.GetMetadata(namespace)
 	return Metadata{Data: data}
 }
+
+func (mo monitoringOperator) GetMetricLabelSet(metric, namespace string, start, end time.Time) MetricLabelSet {
+	// Different monitoring backend implementations have different ways to enforce namespace isolation.
+	// Each implementation should register itself to `ReplaceNamespaceFns` during init().
+	// We hard code "prometheus" here because we only support this datasource so far.
+	// In the future, maybe the value should be returned from a method like `mo.c.GetMonitoringServiceName()`.
+	expr, err := expressions.ReplaceNamespaceFns["prometheus"](metric, namespace)
+	if err != nil {
+		klog.Error(err)
+		return MetricLabelSet{}
+	}
+	data := mo.c.GetMetricLabelSet(expr, start, end)
+	return MetricLabelSet{Data: data}
+}

pkg/models/monitoring/types.go

@@ -12,3 +12,7 @@ type Metrics struct {
 type Metadata struct {
 	Data []monitoring.Metadata `json:"data" description:"actual array of results"`
 }
+
+type MetricLabelSet struct {
+	Data []map[string]string `json:"data" description:"actual array of results"`
+}

pkg/simple/client/monitoring/interface.go

@@ -8,4 +8,5 @@ type Interface interface {
 	GetNamedMetrics(metrics []string, time time.Time, opt QueryOption) []Metric
 	GetNamedMetricsOverTime(metrics []string, start, end time.Time, step time.Duration, opt QueryOption) []Metric
 	GetMetadata(namespace string) []Metadata
+	GetMetricLabelSet(expr string, start, end time.Time) []map[string]string
 }

pkg/simple/client/monitoring/prometheus/prometheus.go

@@ -6,6 +6,7 @@ import (
 	"github.com/prometheus/client_golang/api"
 	apiv1 "github.com/prometheus/client_golang/api/prometheus/v1"
 	"github.com/prometheus/common/model"
+	"k8s.io/klog"
 	"kubesphere.io/kubesphere/pkg/simple/client/monitoring"
 	"sync"
 	"time"
@@ -135,6 +136,7 @@ func (p prometheus) GetMetadata(namespace string) []monitoring.Metadata {
 	matchTarget := fmt.Sprintf("{namespace=\"%s\"}", namespace)
 	items, err := p.client.TargetsMetadata(context.Background(), matchTarget, "", "")
 	if err != nil {
+		klog.Error(err)
 		return meta
 	}
 
@@ -155,6 +157,30 @@ func (p prometheus) GetMetadata(namespace string) []monitoring.Metadata {
 	return meta
 }
 
+func (p prometheus) GetMetricLabelSet(expr string, start, end time.Time) []map[string]string {
+	var res []map[string]string
+
+	labelSet, err := p.client.Series(context.Background(), []string{expr}, start, end)
+	if err != nil {
+		klog.Error(err)
+		return []map[string]string{}
+	}
+
+	for _, item := range labelSet {
+		var tmp = map[string]string{}
+		for key, val := range item {
+			if key == "__name__" {
+				continue
+			}
+			tmp[string(key)] = string(val)
+		}
+
+		res = append(res, tmp)
+	}
+
+	return res
+}
+
 func parseQueryRangeResp(value model.Value) monitoring.MetricData {
 	res := monitoring.MetricData{MetricType: monitoring.MetricTypeMatrix}
 

pkg/simple/client/monitoring/prometheus/prometheus_test.go

@@ -120,6 +120,41 @@ func TestGetMetadata(t *testing.T) {
 	}
 }
 
+func TestGetMetricLabelSet(t *testing.T) {
+	tests := []struct {
+		fakeResp string
+		expected string
+	}{
+		{
+			fakeResp: "labels-prom.json",
+			expected: "labels-res.json",
+		},
+		{
+			fakeResp: "labels-error-prom.json",
+			expected: "labels-error-res.json",
+		},
+	}
+
+	for i, tt := range tests {
+		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
+			var expected []map[string]string
+			err := jsonFromFile(tt.expected, &expected)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			srv := mockPrometheusService("/api/v1/series", tt.fakeResp)
+			defer srv.Close()
+
+			client, _ := NewPrometheus(&Options{Endpoint: srv.URL})
+			result := client.GetMetricLabelSet("default", time.Now(), time.Now())
+			if diff := cmp.Diff(result, expected); diff != "" {
+				t.Fatalf("%T differ (-got, +want): %s", expected, diff)
+			}
+		})
+	}
+}
+
 func mockPrometheusService(pattern, fakeResp string) *httptest.Server {
 	mux := http.NewServeMux()
 	mux.HandleFunc(pattern, func(res http.ResponseWriter, req *http.Request) {

pkg/simple/client/monitoring/prometheus/testdata/labels-error-prom.json

+{
+  "status":"error",
+  "errorType":"bad_data",
+  "error":"1:6: parse error: unexpected left brace '{'"
+}
\ No newline at end of file

pkg/simple/client/monitoring/prometheus/testdata/labels-error-res.json

+[]
\ No newline at end of file

pkg/simple/client/monitoring/prometheus/testdata/labels-prom.json

+{
+  "status":"success",
+  "data":[
+    {
+      "__name__":"kube_configmap_info",
+      "configmap":"grafana-dashboard-cluster-total",
+      "endpoint":"https-main",
+      "instance":"10.233.99.140:8443",
+      "job":"kube-state-metrics",
+      "namespace":"kubesphere-monitoring-system",
+      "pod":"kube-state-metrics-869dc86c5b-prwbc",
+      "service":"kube-state-metrics"
+    },
+    {
+      "__name__":"kube_configmap_info",
+      "configmap":"grafana-dashboard-cluster-total",
+      "endpoint":"https-main",
+      "instance":"10.233.99.172:8443",
+      "job":"kube-state-metrics",
+      "namespace":"kubesphere-monitoring-system",
+      "pod":"kube-state-metrics-869dc86c5b-6x2gr",
+      "service":"kube-state-metrics"
+    },
+    {
+      "__name__":"kube_configmap_info",
+      "configmap":"grafana-dashboard-cluster-total",
+      "endpoint":"https-main",
+      "instance":"10.233.99.173:8443",
+      "job":"kube-state-metrics",
+      "namespace":"kubesphere-monitoring-system",
+      "pod":"kube-state-metrics-6646f8489d-2l482",
+      "service":"kube-state-metrics"
+    },
+    {
+      "__name__":"kube_configmap_info",
+      "configmap":"grafana-dashboard-cluster-total",
+      "instance":"10.233.99.140:8443",
+      "job":"kube-state-metrics",
+      "namespace":"kubesphere-monitoring-system",
+      "pod":"kube-state-metrics-869dc86c5b-prwbc"
+    },
+    {
+      "__name__":"kube_configmap_info",
+      "configmap":"grafana-dashboard-k8s-resources-workload",
+      "endpoint":"https-main",
+      "instance":"10.233.99.173:8443",
+      "job":"kube-state-metrics",
+      "namespace":"kubesphere-monitoring-system",
+      "pod":"kube-state-metrics-6646f8489d-2l482",
+      "service":"kube-state-metrics"
+    },
+    {
+      "__name__":"kube_configmap_info",
+      "configmap":"grafana-dashboard-k8s-resources-workload",
+      "instance":"10.233.99.140:8443",
+      "job":"kube-state-metrics",
+      "namespace":"kubesphere-monitoring-system",
+      "pod":"kube-state-metrics-869dc86c5b-prwbc"
+    },
+    {
+      "__name__":"kube_configmap_info",
+      "configmap":"grafana-dashboard-k8s-resources-workload",
+      "instance":"10.233.99.172:8443",
+      "job":"kube-state-metrics",
+      "namespace":"kubesphere-monitoring-system",
+      "pod":"kube-state-metrics-869dc86c5b-6x2gr"
+    },
+    {
+      "__name__":"kube_configmap_info",
+      "configmap":"grafana-dashboard-namespace-by-pod",
+      "instance":"10.233.99.172:8443",
+      "job":"kube-state-metrics",
+      "namespace":"kubesphere-monitoring-system",
+      "pod":"kube-state-metrics-869dc86c5b-6x2gr"
+    },
+    {
+      "__name__":"kube_configmap_info",
+      "configmap":"grafana-dashboard-persistentvolumesusage",
+      "endpoint":"https-main",
+      "instance":"10.233.99.140:8443",
+      "job":"kube-state-metrics",
+      "namespace":"kubesphere-monitoring-system",
+      "pod":"kube-state-metrics-869dc86c5b-prwbc",
+      "service":"kube-state-metrics"
+    },
+    {
+      "__name__":"kube_configmap_info",
+      "configmap":"grafana-dashboard-persistentvolumesusage",
+      "endpoint":"https-main",
+      "instance":"10.233.99.172:8443",
+      "job":"kube-state-metrics",
+      "namespace":"kubesphere-monitoring-system",
+      "pod":"kube-state-metrics-869dc86c5b-6x2gr",
+      "service":"kube-state-metrics"
+    },
+    {
+      "__name__":"kube_configmap_info",
+      "configmap":"grafana-dashboard-persistentvolumesusage",
+      "endpoint":"https-main",
+      "instance":"10.233.99.173:8443",
+      "job":"kube-state-metrics",
+      "namespace":"kubesphere-monitoring-system",
+      "pod":"kube-state-metrics-6646f8489d-2l482",
+      "service":"kube-state-metrics"
+    },
+    {
+      "__name__":"kube_configmap_info",
+      "configmap":"grafana-dashboard-proxy",
+      "instance":"10.233.99.172:8443",
+      "job":"kube-state-metrics",
+      "namespace":"kubesphere-monitoring-system",
+      "pod":"kube-state-metrics-869dc86c5b-6x2gr"
+    },
+    {
+      "__name__":"kube_configmap_info",
+      "configmap":"grafana-dashboard-scheduler",
+      "endpoint":"https-main",
+      "instance":"10.233.99.140:8443",
+      "job":"kube-state-metrics",
+      "namespace":"kubesphere-monitoring-system",
+      "pod":"kube-state-metrics-869dc86c5b-prwbc",
+      "service":"kube-state-metrics"
+    }
+  ]
+}
\ No newline at end of file

pkg/simple/client/monitoring/prometheus/testdata/labels-res.json

+[
+  {
+    "configmap":"grafana-dashboard-cluster-total",
+    "endpoint":"https-main",
+    "instance":"10.233.99.140:8443",
+    "job":"kube-state-metrics",
+    "namespace":"kubesphere-monitoring-system",
+    "pod":"kube-state-metrics-869dc86c5b-prwbc",
+    "service":"kube-state-metrics"
+  },
+  {
+    "configmap":"grafana-dashboard-cluster-total",
+    "endpoint":"https-main",
+    "instance":"10.233.99.172:8443",
+    "job":"kube-state-metrics",
+    "namespace":"kubesphere-monitoring-system",
+    "pod":"kube-state-metrics-869dc86c5b-6x2gr",
+    "service":"kube-state-metrics"
+  },
+  {
+    "configmap":"grafana-dashboard-cluster-total",
+    "endpoint":"https-main",
+    "instance":"10.233.99.173:8443",
+    "job":"kube-state-metrics",
+    "namespace":"kubesphere-monitoring-system",
+    "pod":"kube-state-metrics-6646f8489d-2l482",
+    "service":"kube-state-metrics"
+  },
+  {
+    "configmap":"grafana-dashboard-cluster-total",
+    "instance":"10.233.99.140:8443",
+    "job":"kube-state-metrics",
+    "namespace":"kubesphere-monitoring-system",
+    "pod":"kube-state-metrics-869dc86c5b-prwbc"
+  },
+  {
+    "configmap":"grafana-dashboard-k8s-resources-workload",
+    "endpoint":"https-main",
+    "instance":"10.233.99.173:8443",
+    "job":"kube-state-metrics",
+    "namespace":"kubesphere-monitoring-system",
+    "pod":"kube-state-metrics-6646f8489d-2l482",
+    "service":"kube-state-metrics"
+  },
+  {
+    "configmap":"grafana-dashboard-k8s-resources-workload",
+    "instance":"10.233.99.140:8443",
+    "job":"kube-state-metrics",
+    "namespace":"kubesphere-monitoring-system",
+    "pod":"kube-state-metrics-869dc86c5b-prwbc"
+  },
+  {
+    "configmap":"grafana-dashboard-k8s-resources-workload",
+    "instance":"10.233.99.172:8443",
+    "job":"kube-state-metrics",
+    "namespace":"kubesphere-monitoring-system",
+    "pod":"kube-state-metrics-869dc86c5b-6x2gr"
+  },
+  {
+    "configmap":"grafana-dashboard-namespace-by-pod",
+    "instance":"10.233.99.172:8443",
+    "job":"kube-state-metrics",
+    "namespace":"kubesphere-monitoring-system",
+    "pod":"kube-state-metrics-869dc86c5b-6x2gr"
+  },
+  {
+    "configmap":"grafana-dashboard-persistentvolumesusage",
+    "endpoint":"https-main",
+    "instance":"10.233.99.140:8443",
+    "job":"kube-state-metrics",
+    "namespace":"kubesphere-monitoring-system",
+    "pod":"kube-state-metrics-869dc86c5b-prwbc",
+    "service":"kube-state-metrics"
+  },
+  {
+    "configmap":"grafana-dashboard-persistentvolumesusage",
+    "endpoint":"https-main",
+    "instance":"10.233.99.172:8443",
+    "job":"kube-state-metrics",
+    "namespace":"kubesphere-monitoring-system",
+    "pod":"kube-state-metrics-869dc86c5b-6x2gr",
+    "service":"kube-state-metrics"
+  },
+  {
+    "configmap":"grafana-dashboard-persistentvolumesusage",
+    "endpoint":"https-main",
+    "instance":"10.233.99.173:8443",
+    "job":"kube-state-metrics",
+    "namespace":"kubesphere-monitoring-system",
+    "pod":"kube-state-metrics-6646f8489d-2l482",
+    "service":"kube-state-metrics"
+  },
+  {
+    "configmap":"grafana-dashboard-proxy",
+    "instance":"10.233.99.172:8443",
+    "job":"kube-state-metrics",
+    "namespace":"kubesphere-monitoring-system",
+    "pod":"kube-state-metrics-869dc86c5b-6x2gr"
+  },
+  {
+    "configmap":"grafana-dashboard-scheduler",
+    "endpoint":"https-main",
+    "instance":"10.233.99.140:8443",
+    "job":"kube-state-metrics",
+    "namespace":"kubesphere-monitoring-system",
+    "pod":"kube-state-metrics-869dc86c5b-prwbc",
+    "service":"kube-state-metrics"
+  }
+]
\ No newline at end of file