From 3d36b254b0f3ce900c055e5791e7fbd3a072b7a7 Mon Sep 17 00:00:00 2001 From: huanggze Date: Fri, 10 May 2019 15:23:14 +0800 Subject: [PATCH] fix: ensure namespces to be filterred correctly in workspace metrics Signed-off-by: huanggze --- pkg/models/metrics/metrics.go | 8 +- pkg/models/metrics/metricsrule.go | 10 ++- pkg/models/metrics/metricsruleconst.go | 80 +++++++++---------- .../client/prometheus/prometheusclient.go | 17 ++++ 4 files changed, 70 insertions(+), 45 deletions(-) diff --git a/pkg/models/metrics/metrics.go b/pkg/models/metrics/metrics.go index ae7bd3e6..ed15face 100644 --- a/pkg/models/metrics/metrics.go +++ b/pkg/models/metrics/metrics.go @@ -343,13 +343,13 @@ func AssembleNamespaceMetricRequestInfoByNamesapce(monitoringRequest *client.Mon return queryType, params } -func AssembleSpecificWorkspaceMetricRequestInfo(monitoringRequest *client.MonitoringRequestParams, namespaceList []string, metricName string) (string, string) { +func AssembleSpecificWorkspaceMetricRequestInfo(monitoringRequest *client.MonitoringRequestParams, namespaceList []string, workspace string, metricName string) (string, string) { nsFilter := "^(" + strings.Join(namespaceList, "|") + ")$" queryType := monitoringRequest.QueryType - rule := MakeSpecificWorkspacePromQL(metricName, nsFilter) + rule := MakeSpecificWorkspacePromQL(metricName, nsFilter, workspace) paramValues := monitoringRequest.Params params := makeRequestParamString(rule, paramValues) return queryType, params @@ -480,7 +480,7 @@ func collectWorkspaceMetric(monitoringRequest *client.MonitoringRequestParams, w wg.Add(1) go func(metricName string) { - queryType, params := AssembleSpecificWorkspaceMetricRequestInfo(monitoringRequest, namespaceArray, metricName) + queryType, params := AssembleSpecificWorkspaceMetricRequestInfo(monitoringRequest, namespaceArray, "", metricName) metricsStr := client.SendMonitoringRequest(client.PrometheusEndpoint, queryType, params) ch <- ReformatJson(metricsStr, metricName, map[string]string{ResultItemMetricResourceName: ws}) wg.Done() @@ -668,7 +668,7 @@ func GetWorkspaceLevelMetrics(monitoringRequest *client.MonitoringRequestParams) if err == nil && matched { wg.Add(1) go func(metricName string, workspace string) { - queryType, params := AssembleSpecificWorkspaceMetricRequestInfo(monitoringRequest, namespaceArray, metricName) + queryType, params := AssembleSpecificWorkspaceMetricRequestInfo(monitoringRequest, namespaceArray, workspace, metricName) metricsStr := client.SendMonitoringRequest(client.PrometheusEndpoint, queryType, params) ch <- ReformatJson(metricsStr, metricName, map[string]string{ResultItemMetricResourceName: workspace}) wg.Done() diff --git a/pkg/models/metrics/metricsrule.go b/pkg/models/metrics/metricsrule.go index 16fc8a78..9b1a0f6a 100644 --- a/pkg/models/metrics/metricsrule.go +++ b/pkg/models/metrics/metricsrule.go @@ -116,11 +116,19 @@ func MakeAllWorkspacesPromQL(metricsName, nsFilter string) string { return promql } -func MakeSpecificWorkspacePromQL(metricsName, nsFilter string) string { +func MakeSpecificWorkspacePromQL(metricsName, nsFilter string, workspace string) string { var promql = RulePromQLTmplMap[metricsName] nsFilter = "=~\"" + nsFilter + "\"" + + if workspace == "" { + workspace = "=~\".*\"" + } else { + workspace = "=\"" + workspace + "\"" + } + promql = strings.Replace(promql, "$1", nsFilter, -1) + promql = strings.Replace(promql, "$2", workspace, -1) return promql } diff --git a/pkg/models/metrics/metricsruleconst.go b/pkg/models/metrics/metricsruleconst.go index 276b4ab9..0522196e 100644 --- a/pkg/models/metrics/metricsruleconst.go +++ b/pkg/models/metrics/metricsruleconst.go @@ -631,53 +631,53 @@ var RulePromQLTmplMap = MetricMap{ // workspace "workspace_cpu_usage": `round(sum(namespace:container_cpu_usage_seconds_total:sum_rate{namespace!="", namespace$1}), 0.001)`, - "workspace_memory_usage": `sum(namespace:container_memory_usage_bytes:sum{namespace!="", namespace$1})`, + "workspace_memory_usage": `sum(namespace:container_memory_usage_bytes:sum{namespace!="", namespace$1} * on (namespace) group_left(label_kubesphere_io_workspace)(kube_namespace_labels{label_kubesphere_io_workspace$2}))`, "workspace_memory_usage_wo_cache": `sum(namespace:container_memory_usage_bytes_wo_cache:sum{namespace!="", namespace$1})`, "workspace_net_bytes_transmitted": `sum(sum by (namespace) (irate(container_network_transmit_bytes_total{namespace!="", namespace$1, pod_name!="", ` + ExcludedVirtualNetworkInterfaces + `, job="kubelet"}[5m])))`, "workspace_net_bytes_received": `sum(sum by (namespace) (irate(container_network_receive_bytes_total{namespace!="", namespace$1, pod_name!="", ` + ExcludedVirtualNetworkInterfaces + `, job="kubelet"}[5m])))`, - "workspace_pod_count": `sum(kube_pod_status_phase{phase!~"Failed|Succeeded", namespace!="", namespace$1})`, - "workspace_pod_running_count": `sum(kube_pod_status_phase{phase="Running", namespace!="", namespace$1})`, - "workspace_pod_succeeded_count": `sum(kube_pod_status_phase{phase="Succeeded", namespace!="", namespace$1})`, - "workspace_pod_abnormal_count": `count(kube_pod_info{node!="", namespace$1} unless on (pod, namespace) (kube_pod_status_phase{job="kube-state-metrics", phase="Succeeded"}>0) unless on (pod, namespace) ((kube_pod_status_ready{job="kube-state-metrics", condition="true"}>0) and on (pod, namespace) (kube_pod_status_phase{job="kube-state-metrics", phase="Running"}>0)) unless on (pod, namespace) (kube_pod_container_status_waiting_reason{job="kube-state-metrics", reason="ContainerCreating"}>0))`, - - "workspace_configmap_count_used": `sum(kube_resourcequota{resourcequota!="quota", type="used", namespace!="", namespace$1, resource="count/configmaps"}) by (resource, type)`, - "workspace_jobs_batch_count_used": `sum(kube_resourcequota{resourcequota!="quota", type="used", namespace!="", namespace$1, resource="count/jobs.batch"}) by (resource, type)`, - "workspace_roles_count_used": `sum(kube_resourcequota{resourcequota!="quota", type="used", namespace!="", namespace$1, resource="count/roles.rbac.authorization.k8s.io"}) by (resource, type)`, - "workspace_memory_limit_used": `sum(kube_resourcequota{resourcequota!="quota", type="used", namespace!="", namespace$1, resource="limits.memory"}) by (resource, type)`, - "workspace_pvc_used": `sum(kube_resourcequota{resourcequota!="quota", type="used", namespace!="", namespace$1, resource="persistentvolumeclaims"}) by (resource, type)`, - "workspace_memory_request_used": `sum(kube_resourcequota{resourcequota!="quota", type="used", namespace!="", namespace$1, resource="requests.memory"}) by (resource, type)`, - "workspace_pvc_count_used": `sum(kube_resourcequota{resourcequota!="quota", type="used", namespace!="", namespace$1, resource="count/persistentvolumeclaims"}) by (resource, type)`, - "workspace_cronjobs_batch_count_used": `sum(kube_resourcequota{resourcequota!="quota", type="used", namespace!="", namespace$1, resource="count/cronjobs.batch"}) by (resource, type)`, - "workspace_ingresses_extensions_count_used": `sum(kube_resourcequota{resourcequota!="quota", type="used", namespace!="", namespace$1, resource="count/ingresses.extensions"}) by (resource, type)`, - "workspace_cpu_limit_used": `sum(kube_resourcequota{resourcequota!="quota", type="used", namespace!="", namespace$1, resource="limits.cpu"}) by (resource, type)`, - "workspace_storage_request_used": `sum(kube_resourcequota{resourcequota!="quota", type="used", namespace!="", namespace$1, resource="requests.storage"}) by (resource, type)`, - "workspace_deployment_count_used": `sum(kube_resourcequota{resourcequota!="quota", type="used", namespace!="", namespace$1, resource="count/deployments.apps"}) by (resource, type)`, - "workspace_pod_count_used": `sum(kube_resourcequota{resourcequota!="quota", type="used", namespace!="", namespace$1, resource="count/pods"}) by (resource, type)`, - "workspace_statefulset_count_used": `sum(kube_resourcequota{resourcequota!="quota", type="used", namespace!="", namespace$1, resource="count/statefulsets.apps"}) by (resource, type)`, - "workspace_daemonset_count_used": `sum(kube_resourcequota{resourcequota!="quota", type="used", namespace!="", namespace$1, resource="count/daemonsets.apps"}) by (resource, type)`, - "workspace_secret_count_used": `sum(kube_resourcequota{resourcequota!="quota", type="used", namespace!="", namespace$1, resource="count/secrets"}) by (resource, type)`, - "workspace_service_count_used": `sum(kube_resourcequota{resourcequota!="quota", type="used", namespace!="", namespace$1, resource="count/services"}) by (resource, type)`, - "workspace_cpu_request_used": `sum(kube_resourcequota{resourcequota!="quota", type="used", namespace!="", namespace$1, resource="requests.cpu"}) by (resource, type)`, - "workspace_service_loadbalancer_used": `sum(kube_resourcequota{resourcequota!="quota", type="used", namespace!="", namespace$1, resource="services.loadbalancers"}) by (resource, type)`, - - "workspace_ingresses_extensions_count": `sum(kube_ingress_labels{namespace!="", namespace$1})`, - - "workspace_cronjob_count": `sum(kube_cronjob_labels{namespace!="", namespace$1})`, - "workspace_pvc_count": `sum(kube_persistentvolumeclaim_info{namespace!="", namespace$1})`, - "workspace_daemonset_count": `sum(kube_daemonset_labels{namespace!="", namespace$1})`, - "workspace_deployment_count": `sum(kube_deployment_labels{namespace!="", namespace$1})`, - "workspace_endpoint_count": `sum(kube_endpoint_labels{namespace!="", namespace$1})`, - "workspace_hpa_count": `sum(kube_hpa_labels{namespace!="", namespace$1})`, - "workspace_job_count": `sum(kube_job_labels{namespace!="", namespace$1})`, - "workspace_statefulset_count": `sum(kube_statefulset_labels{namespace!="", namespace$1})`, - "workspace_replicaset_count": `count(kube_replicaset_created{namespace!="", namespace$1})`, - "workspace_service_count": `sum(kube_service_info{namespace!="", namespace$1})`, - "workspace_secret_count": `sum(kube_secret_info{namespace!="", namespace$1})`, + "workspace_pod_count": `sum(kube_pod_status_phase{phase!~"Failed|Succeeded", namespace!="", namespace$1} * on (namespace) group_left(label_kubesphere_io_workspace)(kube_namespace_labels{label_kubesphere_io_workspace$2}))`, + "workspace_pod_running_count": `sum(kube_pod_status_phase{phase="Running", namespace!="", namespace$1} * on (namespace) group_left(label_kubesphere_io_workspace)(kube_namespace_labels{label_kubesphere_io_workspace$2}))`, + "workspace_pod_succeeded_count": `sum(kube_pod_status_phase{phase="Succeeded", namespace!="", namespace$1} * on (namespace) group_left(label_kubesphere_io_workspace)(kube_namespace_labels{label_kubesphere_io_workspace$2}))`, + "workspace_pod_abnormal_count": `count((kube_pod_info{node!="", namespace$1} unless on (pod, namespace) (kube_pod_status_phase{job="kube-state-metrics", phase="Succeeded"}>0) unless on (pod, namespace) ((kube_pod_status_ready{job="kube-state-metrics", condition="true"}>0) and on (pod, namespace) (kube_pod_status_phase{job="kube-state-metrics", phase="Running"}>0)) unless on (pod, namespace) (kube_pod_container_status_waiting_reason{job="kube-state-metrics", reason="ContainerCreating"}>0)) * on (namespace) group_left(label_kubesphere_io_workspace)(kube_namespace_labels{label_kubesphere_io_workspace$2}))`, + + "workspace_configmap_count_used": `sum(kube_resourcequota{resourcequota!="quota", type="used", namespace!="", namespace$1, resource="count/configmaps"} * on (namespace) group_left(label_kubesphere_io_workspace)(kube_namespace_labels{label_kubesphere_io_workspace$2})) by (resource, type)`, + "workspace_jobs_batch_count_used": `sum(kube_resourcequota{resourcequota!="quota", type="used", namespace!="", namespace$1, resource="count/jobs.batch"} * on (namespace) group_left(label_kubesphere_io_workspace)(kube_namespace_labels{label_kubesphere_io_workspace$2})) by (resource, type)`, + "workspace_roles_count_used": `sum(kube_resourcequota{resourcequota!="quota", type="used", namespace!="", namespace$1, resource="count/roles.rbac.authorization.k8s.io"} * on (namespace) group_left(label_kubesphere_io_workspace)(kube_namespace_labels{label_kubesphere_io_workspace$2})) by (resource, type)`, + "workspace_memory_limit_used": `sum(kube_resourcequota{resourcequota!="quota", type="used", namespace!="", namespace$1, resource="limits.memory"} * on (namespace) group_left(label_kubesphere_io_workspace)(kube_namespace_labels{label_kubesphere_io_workspace$2})) by (resource, type)`, + "workspace_pvc_used": `sum(kube_resourcequota{resourcequota!="quota", type="used", namespace!="", namespace$1, resource="persistentvolumeclaims"} * on (namespace) group_left(label_kubesphere_io_workspace)(kube_namespace_labels{label_kubesphere_io_workspace$2})) by (resource, type)`, + "workspace_memory_request_used": `sum(kube_resourcequota{resourcequota!="quota", type="used", namespace!="", namespace$1, resource="requests.memory"} * on (namespace) group_left(label_kubesphere_io_workspace)(kube_namespace_labels{label_kubesphere_io_workspace$2})) by (resource, type)`, + "workspace_pvc_count_used": `sum(kube_resourcequota{resourcequota!="quota", type="used", namespace!="", namespace$1, resource="count/persistentvolumeclaims"} * on (namespace) group_left(label_kubesphere_io_workspace)(kube_namespace_labels{label_kubesphere_io_workspace$2})) by (resource, type)`, + "workspace_cronjobs_batch_count_used": `sum(kube_resourcequota{resourcequota!="quota", type="used", namespace!="", namespace$1, resource="count/cronjobs.batch"} * on (namespace) group_left(label_kubesphere_io_workspace)(kube_namespace_labels{label_kubesphere_io_workspace$2})) by (resource, type)`, + "workspace_ingresses_extensions_count_used": `sum(kube_resourcequota{resourcequota!="quota", type="used", namespace!="", namespace$1, resource="count/ingresses.extensions"} * on (namespace) group_left(label_kubesphere_io_workspace)(kube_namespace_labels{label_kubesphere_io_workspace$2})) by (resource, type)`, + "workspace_cpu_limit_used": `sum(kube_resourcequota{resourcequota!="quota", type="used", namespace!="", namespace$1, resource="limits.cpu"} * on (namespace) group_left(label_kubesphere_io_workspace)(kube_namespace_labels{label_kubesphere_io_workspace$2})) by (resource, type)`, + "workspace_storage_request_used": `sum(kube_resourcequota{resourcequota!="quota", type="used", namespace!="", namespace$1, resource="requests.storage"} * on (namespace) group_left(label_kubesphere_io_workspace)(kube_namespace_labels{label_kubesphere_io_workspace$2})) by (resource, type)`, + "workspace_deployment_count_used": `sum(kube_resourcequota{resourcequota!="quota", type="used", namespace!="", namespace$1, resource="count/deployments.apps"} * on (namespace) group_left(label_kubesphere_io_workspace)(kube_namespace_labels{label_kubesphere_io_workspace$2})) by (resource, type)`, + "workspace_pod_count_used": `sum(kube_resourcequota{resourcequota!="quota", type="used", namespace!="", namespace$1, resource="count/pods"} * on (namespace) group_left(label_kubesphere_io_workspace)(kube_namespace_labels{label_kubesphere_io_workspace$2})) by (resource, type)`, + "workspace_statefulset_count_used": `sum(kube_resourcequota{resourcequota!="quota", type="used", namespace!="", namespace$1, resource="count/statefulsets.apps"} * on (namespace) group_left(label_kubesphere_io_workspace)(kube_namespace_labels{label_kubesphere_io_workspace$2})) by (resource, type)`, + "workspace_daemonset_count_used": `sum(kube_resourcequota{resourcequota!="quota", type="used", namespace!="", namespace$1, resource="count/daemonsets.apps"} * on (namespace) group_left(label_kubesphere_io_workspace)(kube_namespace_labels{label_kubesphere_io_workspace$2})) by (resource, type)`, + "workspace_secret_count_used": `sum(kube_resourcequota{resourcequota!="quota", type="used", namespace!="", namespace$1, resource="count/secrets"} * on (namespace) group_left(label_kubesphere_io_workspace)(kube_namespace_labels{label_kubesphere_io_workspace$2})) by (resource, type)`, + "workspace_service_count_used": `sum(kube_resourcequota{resourcequota!="quota", type="used", namespace!="", namespace$1, resource="count/services"} * on (namespace) group_left(label_kubesphere_io_workspace)(kube_namespace_labels{label_kubesphere_io_workspace$2})) by (resource, type)`, + "workspace_cpu_request_used": `sum(kube_resourcequota{resourcequota!="quota", type="used", namespace!="", namespace$1, resource="requests.cpu"} * on (namespace) group_left(label_kubesphere_io_workspace)(kube_namespace_labels{label_kubesphere_io_workspace$2})) by (resource, type)`, + "workspace_service_loadbalancer_used": `sum(kube_resourcequota{resourcequota!="quota", type="used", namespace!="", namespace$1, resource="services.loadbalancers"} * on (namespace) group_left(label_kubesphere_io_workspace)(kube_namespace_labels{label_kubesphere_io_workspace$2})) by (resource, type)`, + + "workspace_ingresses_extensions_count": `sum(kube_ingress_labels{namespace!="", namespace$1} * on (namespace) group_left(label_kubesphere_io_workspace)(kube_namespace_labels{label_kubesphere_io_workspace$2}))`, + + "workspace_cronjob_count": `sum(kube_cronjob_labels{namespace!="", namespace$1} * on (namespace) group_left(label_kubesphere_io_workspace)(kube_namespace_labels{label_kubesphere_io_workspace$2}))`, + "workspace_pvc_count": `sum(kube_persistentvolumeclaim_info{namespace!="", namespace$1} * on (namespace) group_left(label_kubesphere_io_workspace)(kube_namespace_labels{label_kubesphere_io_workspace$2}))`, + "workspace_daemonset_count": `sum(kube_daemonset_labels{namespace!="", namespace$1} * on (namespace) group_left(label_kubesphere_io_workspace)(kube_namespace_labels{label_kubesphere_io_workspace$2}))`, + "workspace_deployment_count": `sum(kube_deployment_labels{namespace!="", namespace$1} * on (namespace) group_left(label_kubesphere_io_workspace)(kube_namespace_labels{label_kubesphere_io_workspace$2}))`, + "workspace_endpoint_count": `sum(kube_endpoint_labels{namespace!="", namespace$1} * on (namespace) group_left(label_kubesphere_io_workspace)(kube_namespace_labels{label_kubesphere_io_workspace$2}))`, + "workspace_hpa_count": `sum(kube_hpa_labels{namespace!="", namespace$1} * on (namespace) group_left(label_kubesphere_io_workspace)(kube_namespace_labels{label_kubesphere_io_workspace$2}))`, + "workspace_job_count": `sum(kube_job_labels{namespace!="", namespace$1} * on (namespace) group_left(label_kubesphere_io_workspace)(kube_namespace_labels{label_kubesphere_io_workspace$2}))`, + "workspace_statefulset_count": `sum(kube_statefulset_labels{namespace!="", namespace$1} * on (namespace) group_left(label_kubesphere_io_workspace)(kube_namespace_labels{label_kubesphere_io_workspace$2}))`, + "workspace_replicaset_count": `count(kube_replicaset_created{namespace!="", namespace$1} * on (namespace) group_left(label_kubesphere_io_workspace)(kube_namespace_labels{label_kubesphere_io_workspace$2}))`, + "workspace_service_count": `sum(kube_service_info{namespace!="", namespace$1} * on (namespace) group_left(label_kubesphere_io_workspace)(kube_namespace_labels{label_kubesphere_io_workspace$2}))`, + "workspace_secret_count": `sum(kube_secret_info{namespace!="", namespace$1} * on (namespace) group_left(label_kubesphere_io_workspace)(kube_namespace_labels{label_kubesphere_io_workspace$2}))`, "workspace_all_project_count": `count(kube_namespace_annotations)`, // New in ks 2.0 - "workspace_pod_abnormal_ratio": `count(kube_pod_info{node!="", namespace$1} unless on (pod, namespace) (kube_pod_status_phase{job="kube-state-metrics", phase="Succeeded"}>0) unless on (pod, namespace) ((kube_pod_status_ready{job="kube-state-metrics", condition="true"}>0) and on (pod, namespace) (kube_pod_status_phase{job="kube-state-metrics", phase="Running"}>0)) unless on (pod, namespace) (kube_pod_container_status_waiting_reason{job="kube-state-metrics", reason="ContainerCreating"}>0)) / sum(kube_pod_status_phase{phase!~"Succeeded", namespace!="", namespace$1})`, + "workspace_pod_abnormal_ratio": `count((kube_pod_info{node!="", namespace$1} unless on (pod, namespace) (kube_pod_status_phase{job="kube-state-metrics", phase="Succeeded"}>0) unless on (pod, namespace) ((kube_pod_status_ready{job="kube-state-metrics", condition="true"}>0) and on (pod, namespace) (kube_pod_status_phase{job="kube-state-metrics", phase="Running"}>0)) unless on (pod, namespace) (kube_pod_container_status_waiting_reason{job="kube-state-metrics", reason="ContainerCreating"}>0)) / sum(kube_pod_status_phase{phase!~"Succeeded", namespace!="", namespace$1}) * on (namespace) group_left(label_kubesphere_io_workspace)(kube_namespace_labels{label_kubesphere_io_workspace$2}))`, // component "etcd_server_list": `label_replace(up{job="etcd"}, "node_ip", "$1", "instance", "(.*):.*")`, diff --git a/pkg/simple/client/prometheus/prometheusclient.go b/pkg/simple/client/prometheus/prometheusclient.go index 9fd03ae6..4ed4b525 100644 --- a/pkg/simple/client/prometheus/prometheusclient.go +++ b/pkg/simple/client/prometheus/prometheusclient.go @@ -20,6 +20,7 @@ package prometheus import ( "flag" "io/ioutil" + "kubesphere.io/kubesphere/pkg/informers" "net/http" "net/url" "strconv" @@ -145,12 +146,28 @@ func ParseMonitoringRequestParams(request *restful.Request) *MonitoringRequestPa u := url.Values{} if start != "" && end != "" { + u.Set("start", convertTimeGranularity(start)) u.Set("end", convertTimeGranularity(end)) u.Set("step", step) u.Set("timeout", timeout) + + // range query start time must be greater than the namespace creation time + if nsName != "" { + nsLister := informers.SharedInformerFactory().Core().V1().Namespaces().Lister() + ns, err := nsLister.Get(nsName) + if err == nil { + queryStartTime := u.Get("start") + nsCreationTime := strconv.FormatInt(ns.CreationTimestamp.Unix(), 10) + if nsCreationTime > queryStartTime { + u.Set("start", nsCreationTime) + } + } + } + requestParams.QueryType = RangeQueryType requestParams.Params = u + return &requestParams } if instantTime != "" { -- GitLab