未验证 提交 6cdbf336 编写于 作者: K KubeSphere CI Bot 提交者: GitHub

Merge pull request #3526 from zhu733756/add-pod-level-metrics

Add pod level metrics for edge node
......@@ -19,9 +19,12 @@ package metricsserver
import (
"context"
"errors"
"regexp"
"strings"
"time"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/clientcmd"
......@@ -103,12 +106,28 @@ func (m metricsServer) filterEdgeNodeNames(edgeNodes map[string]v1.Node, opts *m
return edgeNodeNamesFiltered
}
func (m metricsServer) parseEdgePods(opts *monitoring.QueryOptions) map[string]bool {
edgePods := make(map[string]bool)
r, _ := regexp.Compile(`\s*\|\s*|\$`)
filters := r.Split(opts.ResourceFilter, -1)
for _, p := range filters {
if p != "" {
edgePods[p] = true
}
}
return edgePods
}
// node metrics of edge nodes
func (m metricsServer) getNodeMetricsFromMetricsAPI() (*metricsapi.NodeMetricsList, error) {
var err error
versionedMetrics := &metricsV1beta1.NodeMetricsList{}
mc := m.metricsClient.MetricsV1beta1()
nm := mc.NodeMetricses()
versionedMetrics, err = nm.List(context.TODO(), metav1.ListOptions{LabelSelector: edgeNodeLabel})
versionedMetrics, err := nm.List(context.TODO(), metav1.ListOptions{LabelSelector: edgeNodeLabel})
if err != nil {
return nil, err
}
......@@ -120,6 +139,87 @@ func (m metricsServer) getNodeMetricsFromMetricsAPI() (*metricsapi.NodeMetricsLi
return metrics, nil
}
// pods metrics of edge nodes
func (m metricsServer) getPodMetricsFromMetricsAPI(edgePods map[string]bool, podName string, ns string) ([]metricsapi.PodMetrics, error) {
mc := m.metricsClient.MetricsV1beta1()
// single pod request
if ns != "" && podName != "" {
pm := mc.PodMetricses(ns)
versionedMetrics, err := pm.Get(context.TODO(), podName, metav1.GetOptions{})
if err != nil {
klog.Error("Get pod metrics on edge node error:", err)
return nil, err
}
metrics := &metricsapi.PodMetrics{}
err = metricsV1beta1.Convert_v1beta1_PodMetrics_To_metrics_PodMetrics(versionedMetrics, metrics, nil)
if err != nil {
klog.Error("Convert pod metrics on edge node error:", err)
return nil, err
}
return []metricsapi.PodMetrics{*metrics}, nil
}
if len(edgePods) == 0 {
return nil, nil
}
var isNamespacedEdgePod bool
for p, _ := range edgePods {
if ok := strings.Contains(p, "/"); ok {
isNamespacedEdgePod = true
}
break
}
combinedPodMetrics := []metricsapi.PodMetrics{}
// handle cases with when edgePodName contains namespaceName
if isNamespacedEdgePod {
for p, _ := range edgePods {
splitedPodName := strings.Split(p, "/")
ns, p = strings.ReplaceAll(splitedPodName[0], " ", ""), strings.ReplaceAll(splitedPodName[1], " ", "")
pm := mc.PodMetricses(ns)
versionedMetrics, err := pm.Get(context.TODO(), p, metav1.GetOptions{})
if err != nil {
klog.Error("Get pod metrics on edge node error:", err)
continue
}
metrics := &metricsapi.PodMetrics{}
err = metricsV1beta1.Convert_v1beta1_PodMetrics_To_metrics_PodMetrics(versionedMetrics, metrics, nil)
if err != nil {
klog.Error("Convert pod metrics on edge node error:", err)
continue
}
combinedPodMetrics = append(combinedPodMetrics, *metrics)
}
return combinedPodMetrics, nil
}
// use list request in other cases
pm := mc.PodMetricses(ns)
versionedMetricsList, err := pm.List(context.TODO(), metav1.ListOptions{})
if err != nil {
klog.Error("List pod metrics on edge node error:", err)
return nil, err
}
podMetrics := &metricsapi.PodMetricsList{}
err = metricsV1beta1.Convert_v1beta1_PodMetricsList_To_metrics_PodMetricsList(versionedMetricsList, podMetrics, nil)
if err != nil {
klog.Error("Convert pod metrics on edge node error:", err)
return nil, err
}
for _, podMetric := range podMetrics.Items {
if _, ok := edgePods[podMetric.Name]; !ok {
continue
}
combinedPodMetrics = append(combinedPodMetrics, podMetric)
}
return combinedPodMetrics, nil
}
func NewMetricsClient(k kubernetes.Interface, options *k8s.KubernetesOptions) monitoring.Interface {
config, err := clientcmd.BuildConfigFromFlags("", options.KubeConfig)
if err != nil {
......@@ -172,6 +272,7 @@ func (m metricsServer) GetMetricOverTime(expr string, start, end time.Time, step
return parsedResp
}
// node metrics definition
const (
metricsNodeCPUUsage = "node_cpu_usage"
metricsNodeCPUTotal = "node_cpu_total"
......@@ -183,6 +284,20 @@ const (
var edgeNodeMetrics = []string{metricsNodeCPUUsage, metricsNodeCPUTotal, metricsNodeCPUUltilisation, metricsNodeMemoryUsageWoCache, metricsNodeMemoryTotal, metricsNodeMemoryUltilisation}
// pod metrics definition
const (
metricsPodCPUUsage = "pod_cpu_usage"
metricsPodMemoryUsage = "pod_memory_usage_wo_cache"
)
var (
edgePodMetrics = []string{metricsPodCPUUsage, metricsPodMemoryUsage}
MeasuredResources = []v1.ResourceName{
v1.ResourceCPU,
v1.ResourceMemory,
}
)
func (m metricsServer) parseErrorResp(metrics []string, err error) []monitoring.Metric {
var res []monitoring.Metric
......@@ -199,118 +314,213 @@ func (m metricsServer) GetNamedMetrics(metrics []string, ts time.Time, o monitor
opts := monitoring.NewQueryOptions()
o.Apply(opts)
if opts.Level == monitoring.LevelNode {
if !m.metricsAPIAvailable {
klog.Warningf("Metrics API not available.")
return m.parseErrorResp(metrics, errors.New("Metrics API not available."))
}
edgeNodes, err := m.listEdgeNodes()
if err != nil {
klog.Errorf("List edge nodes error %v\n", err)
return m.parseErrorResp(metrics, err)
}
if !m.metricsAPIAvailable {
klog.Warningf("Metrics API not available.")
return m.parseErrorResp(metrics, errors.New("Metrics API not available."))
}
edgeNodeNamesFiltered := m.filterEdgeNodeNames(edgeNodes, opts)
if len(edgeNodeNamesFiltered) == 0 {
klog.V(4).Infof("No edge node metrics is requested")
return res
switch opts.Level {
case monitoring.LevelNode:
return m.GetNodeLevelNamedMetrics(metrics, ts, opts)
case monitoring.LevelPod:
return m.GetPodLevelNamedMetrics(metrics, ts, opts)
default:
return res
}
}
func (m metricsServer) GetNodeLevelNamedMetrics(metrics []string, ts time.Time, opts *monitoring.QueryOptions) []monitoring.Metric {
var res []monitoring.Metric
edgeNodes, err := m.listEdgeNodes()
if err != nil {
klog.Errorf("List edge nodes error %v\n", err)
return m.parseErrorResp(metrics, err)
}
edgeNodeNamesFiltered := m.filterEdgeNodeNames(edgeNodes, opts)
if len(edgeNodeNamesFiltered) == 0 {
klog.V(4).Infof("No edge node metrics is requested")
return res
}
status := make(map[string]v1.NodeStatus)
for n, _ := range edgeNodeNamesFiltered {
status[n] = edgeNodes[n].Status
}
metricsResult, err := m.getNodeMetricsFromMetricsAPI()
if err != nil {
klog.Errorf("Get edge node metrics error %v\n", err)
return m.parseErrorResp(metrics, err)
}
metricsMap := make(map[string]bool)
for _, m := range metrics {
metricsMap[m] = true
}
nodeMetrics := make(map[string]*monitoring.MetricData)
for _, enm := range edgeNodeMetrics {
_, ok := metricsMap[enm]
if ok {
nodeMetrics[enm] = &monitoring.MetricData{MetricType: monitoring.MetricTypeVector}
}
}
metricsResult, err := m.getNodeMetricsFromMetricsAPI()
if err != nil {
klog.Errorf("Get edge node metrics error %v\n", err)
return m.parseErrorResp(metrics, err)
var usage v1.ResourceList
var cap v1.ResourceList
for _, m := range metricsResult.Items {
_, ok := edgeNodeNamesFiltered[m.Name]
if !ok {
continue
}
metricsMap := make(map[string]bool)
for _, m := range metrics {
metricsMap[m] = true
m.Usage.DeepCopyInto(&usage)
status[m.Name].Capacity.DeepCopyInto(&cap)
metricValues := make(map[string]*monitoring.MetricValue)
for _, enm := range edgeNodeMetrics {
metricValues[enm] = &monitoring.MetricValue{
Metadata: make(map[string]string),
}
metricValues[enm].Metadata["node"] = m.Name
metricValues[enm].Metadata["role"] = "edge"
}
status := make(map[string]v1.NodeStatus)
for n, _ := range edgeNodeNamesFiltered {
status[n] = edgeNodes[n].Status
for k, v := range metricsMap {
switch k {
case metricsNodeCPUUsage:
if v {
metricValues[metricsNodeCPUUsage].Sample = &monitoring.Point{float64(m.Timestamp.Unix()), float64(usage.Cpu().MilliValue()) / 1000}
}
case metricsNodeCPUTotal:
if v {
metricValues[metricsNodeCPUTotal].Sample = &monitoring.Point{float64(m.Timestamp.Unix()), float64(cap.Cpu().MilliValue()) / 1000}
}
case metricsNodeCPUUltilisation:
if v {
metricValues[metricsNodeCPUUltilisation].Sample = &monitoring.Point{float64(m.Timestamp.Unix()), float64(usage.Cpu().MilliValue()) / float64(cap.Cpu().MilliValue())}
}
case metricsNodeMemoryUsageWoCache:
if v {
metricValues[metricsNodeMemoryUsageWoCache].Sample = &monitoring.Point{float64(m.Timestamp.Unix()), float64(usage.Memory().Value())}
}
case metricsNodeMemoryTotal:
if v {
metricValues[metricsNodeMemoryTotal].Sample = &monitoring.Point{float64(m.Timestamp.Unix()), float64(cap.Memory().Value())}
}
case metricsNodeMemoryUltilisation:
if v {
metricValues[metricsNodeMemoryUltilisation].Sample = &monitoring.Point{float64(m.Timestamp.Unix()), float64(usage.Memory().Value()) / float64(cap.Memory().Value())}
}
}
}
nodeMetrics := make(map[string]*monitoring.MetricData)
for _, enm := range edgeNodeMetrics {
_, ok := metricsMap[enm]
_, ok = metricsMap[enm]
if ok {
nodeMetrics[enm] = &monitoring.MetricData{MetricType: monitoring.MetricTypeVector}
nodeMetrics[enm].MetricValues = append(nodeMetrics[enm].MetricValues, *metricValues[enm])
}
}
}
var usage v1.ResourceList
var cap v1.ResourceList
for _, m := range metricsResult.Items {
_, ok := edgeNodeNamesFiltered[m.Name]
if !ok {
continue
}
for _, enm := range edgeNodeMetrics {
_, ok := metricsMap[enm]
if ok {
res = append(res, monitoring.Metric{MetricName: enm, MetricData: *nodeMetrics[enm]})
}
}
return res
}
m.Usage.DeepCopyInto(&usage)
status[m.Name].Capacity.DeepCopyInto(&cap)
func (m metricsServer) GetPodLevelNamedMetrics(metrics []string, ts time.Time, opts *monitoring.QueryOptions) []monitoring.Metric {
var res []monitoring.Metric
metricValues := make(map[string]*monitoring.MetricValue)
edgePods := m.parseEdgePods(opts)
if len(edgePods) == 0 && opts.PodName == "" {
klog.Errorf("Edge node filter regexp error: %v\n", errors.New("no edge node pods metrics is requested or resource filter invalid"))
return res
}
for _, enm := range edgeNodeMetrics {
metricValues[enm] = &monitoring.MetricValue{
Metadata: make(map[string]string),
}
metricValues[enm].Metadata["node"] = m.Name
metricValues[enm].Metadata["role"] = "edge"
}
for _, addr := range status[m.Name].Addresses {
if addr.Type == v1.NodeInternalIP {
for _, enm := range edgeNodeMetrics {
metricValues[enm].Metadata["host_ip"] = addr.Address
}
break
}
}
podMetricsFromMetricsAPI, err := m.getPodMetricsFromMetricsAPI(edgePods, opts.PodName, opts.NamespaceName)
if err != nil {
klog.Errorf("Get pod metrics of edge nodes error %v\n", err)
return m.parseErrorResp(metrics, err)
}
_, ok = metricsMap[metricsNodeCPUUsage]
if ok {
metricValues[metricsNodeCPUUsage].Sample = &monitoring.Point{float64(m.Timestamp.Unix()), float64(usage.Cpu().MilliValue()) / 1000}
}
_, ok = metricsMap[metricsNodeCPUTotal]
if ok {
metricValues[metricsNodeCPUTotal].Sample = &monitoring.Point{float64(m.Timestamp.Unix()), float64(cap.Cpu().MilliValue()) / 1000}
}
_, ok = metricsMap[metricsNodeCPUUltilisation]
if ok {
metricValues[metricsNodeCPUUltilisation].Sample = &monitoring.Point{float64(m.Timestamp.Unix()), float64(usage.Cpu().MilliValue()) / float64(cap.Cpu().MilliValue())}
}
_, ok = metricsMap[metricsNodeMemoryUsageWoCache]
if ok {
metricValues[metricsNodeMemoryUsageWoCache].Sample = &monitoring.Point{float64(m.Timestamp.Unix()), float64(usage.Memory().Value())}
}
_, ok = metricsMap[metricsNodeMemoryTotal]
if ok {
metricValues[metricsNodeMemoryTotal].Sample = &monitoring.Point{float64(m.Timestamp.Unix()), float64(cap.Memory().Value())}
metricsMap := make(map[string]bool)
for _, m := range metrics {
metricsMap[m] = true
}
// init
podMetrics := make(map[string]*monitoring.MetricData)
for _, epm := range edgePodMetrics {
_, ok := metricsMap[epm]
if ok {
podMetrics[epm] = &monitoring.MetricData{MetricType: monitoring.MetricTypeVector}
}
}
for _, p := range podMetricsFromMetricsAPI {
metricValues := make(map[string]*monitoring.MetricValue)
for _, epm := range edgePodMetrics {
metricValues[epm] = &monitoring.MetricValue{
Metadata: make(map[string]string),
}
_, ok = metricsMap[metricsNodeMemoryUltilisation]
if ok {
metricValues[metricsNodeMemoryUltilisation].Sample = &monitoring.Point{float64(m.Timestamp.Unix()), float64(usage.Memory().Value()) / float64(cap.Memory().Value())}
metricValues[epm].Metadata["pod"] = p.Name
metricValues[epm].Metadata["namespace"] = p.Namespace
}
podMetricsUsge := make(v1.ResourceList)
for _, res := range MeasuredResources {
podMetricsUsge[res], _ = resource.ParseQuantity("0")
}
for _, podContainer := range p.Containers {
for _, res := range MeasuredResources {
quantity := podMetricsUsge[res]
quantity.Add(podContainer.Usage[res])
podMetricsUsge[res] = quantity
}
}
for _, enm := range edgeNodeMetrics {
_, ok = metricsMap[enm]
if ok {
nodeMetrics[enm].MetricValues = append(nodeMetrics[enm].MetricValues, *metricValues[enm])
for k, v := range metricsMap {
switch k {
case metricsPodCPUUsage:
if v {
cpuQuantity := podMetricsUsge[v1.ResourceCPU]
metricValues[metricsPodCPUUsage].Sample = &monitoring.Point{float64(p.Timestamp.Unix()), float64(cpuQuantity.MilliValue()) / 1000}
}
case metricsPodMemoryUsage:
if v {
memoryQuantity := podMetricsUsge[v1.ResourceMemory]
metricValues[metricsPodMemoryUsage].Sample = &monitoring.Point{float64(p.Timestamp.Unix()), float64(memoryQuantity.Value()) / (1024 * 1024)}
}
}
}
for _, enm := range edgeNodeMetrics {
_, ok := metricsMap[enm]
for _, epm := range edgePodMetrics {
_, ok := metricsMap[epm]
if ok {
res = append(res, monitoring.Metric{MetricName: enm, MetricData: *nodeMetrics[enm]})
podMetrics[epm].MetricValues = append(podMetrics[epm].MetricValues, *metricValues[epm])
}
}
}
for _, epm := range edgePodMetrics {
_, ok := metricsMap[epm]
if ok {
res = append(res, monitoring.Metric{MetricName: epm, MetricData: *podMetrics[epm]})
}
}
return res
}
......@@ -319,118 +529,220 @@ func (m metricsServer) GetNamedMetricsOverTime(metrics []string, start, end time
opts := monitoring.NewQueryOptions()
o.Apply(opts)
if opts.Level == monitoring.LevelNode {
if !m.metricsAPIAvailable {
klog.Warningf("Metrics API not available.")
return m.parseErrorResp(metrics, errors.New("Metrics API not available."))
}
edgeNodes, err := m.listEdgeNodes()
if err != nil {
klog.Errorf("List edge nodes error %v\n", err)
return m.parseErrorResp(metrics, err)
}
if !m.metricsAPIAvailable {
klog.Warningf("Metrics API not available.")
return m.parseErrorResp(metrics, errors.New("Metrics API not available."))
}
switch opts.Level {
case monitoring.LevelNode:
return m.GetNodeLevelNamedMetricsOverTime(metrics, start, end, step, opts)
case monitoring.LevelPod:
return m.GetPodLevelNamedMetricsOverTime(metrics, start, end, step, opts)
default:
return res
}
}
func (m metricsServer) GetNodeLevelNamedMetricsOverTime(metrics []string, start, end time.Time, step time.Duration, opts *monitoring.QueryOptions) []monitoring.Metric {
var res []monitoring.Metric
edgeNodes, err := m.listEdgeNodes()
if err != nil {
klog.Errorf("List edge nodes error %v\n", err)
return m.parseErrorResp(metrics, err)
}
edgeNodeNamesFiltered := m.filterEdgeNodeNames(edgeNodes, opts)
if len(edgeNodeNamesFiltered) == 0 {
klog.V(4).Infof("No edge node metrics is requested")
return res
}
metricsResult, err := m.getNodeMetricsFromMetricsAPI()
if err != nil {
klog.Errorf("Get edge node metrics error %v\n", err)
return m.parseErrorResp(metrics, err)
}
metricsMap := make(map[string]bool)
for _, m := range metrics {
metricsMap[m] = true
}
edgeNodeNamesFiltered := m.filterEdgeNodeNames(edgeNodes, opts)
if len(edgeNodeNamesFiltered) == 0 {
klog.V(4).Infof("No edge node metrics is requested")
return res
status := make(map[string]v1.NodeStatus)
for n, _ := range edgeNodeNamesFiltered {
status[n] = edgeNodes[n].Status
}
nodeMetrics := make(map[string]*monitoring.MetricData)
for _, enm := range edgeNodeMetrics {
_, ok := metricsMap[enm]
if ok {
nodeMetrics[enm] = &monitoring.MetricData{MetricType: monitoring.MetricTypeMatrix}
}
}
metricsResult, err := m.getNodeMetricsFromMetricsAPI()
if err != nil {
klog.Errorf("Get edge node metrics error %v\n", err)
return m.parseErrorResp(metrics, err)
var usage v1.ResourceList
var cap v1.ResourceList
for _, m := range metricsResult.Items {
_, ok := edgeNodeNamesFiltered[m.Name]
if !ok {
continue
}
metricsMap := make(map[string]bool)
for _, m := range metrics {
metricsMap[m] = true
m.Usage.DeepCopyInto(&usage)
status[m.Name].Capacity.DeepCopyInto(&cap)
metricValues := make(map[string]*monitoring.MetricValue)
for _, enm := range edgeNodeMetrics {
metricValues[enm] = &monitoring.MetricValue{
Metadata: make(map[string]string),
}
metricValues[enm].Metadata["node"] = m.Name
metricValues[enm].Metadata["role"] = "edge"
}
for _, addr := range status[m.Name].Addresses {
if addr.Type == v1.NodeInternalIP {
for _, enm := range edgeNodeMetrics {
metricValues[enm].Metadata["host_ip"] = addr.Address
}
break
}
}
status := make(map[string]v1.NodeStatus)
for n, _ := range edgeNodeNamesFiltered {
status[n] = edgeNodes[n].Status
for k, v := range metricsMap {
switch k {
case metricsNodeCPUUsage:
if v {
metricValues[metricsNodeCPUUsage].Series = append(metricValues[metricsNodeCPUUsage].Series, monitoring.Point{float64(m.Timestamp.Unix()), float64(usage.Cpu().MilliValue()) / 1000})
}
case metricsNodeCPUTotal:
if v {
metricValues[metricsNodeCPUTotal].Series = append(metricValues[metricsNodeCPUTotal].Series, monitoring.Point{float64(m.Timestamp.Unix()), float64(cap.Cpu().MilliValue()) / 1000})
}
case metricsNodeCPUUltilisation:
if v {
metricValues[metricsNodeCPUUltilisation].Series = append(metricValues[metricsNodeCPUUltilisation].Series, monitoring.Point{float64(m.Timestamp.Unix()), float64(usage.Cpu().MilliValue()) / float64(cap.Cpu().MilliValue())})
}
case metricsNodeMemoryUsageWoCache:
if v {
metricValues[metricsNodeMemoryUsageWoCache].Series = append(metricValues[metricsNodeMemoryUsageWoCache].Series, monitoring.Point{float64(m.Timestamp.Unix()), float64(usage.Memory().Value())})
}
case metricsNodeMemoryTotal:
if v {
metricValues[metricsNodeMemoryTotal].Series = append(metricValues[metricsNodeMemoryTotal].Series, monitoring.Point{float64(m.Timestamp.Unix()), float64(cap.Memory().Value())})
}
case metricsNodeMemoryUltilisation:
if v {
metricValues[metricsNodeMemoryUltilisation].Series = append(metricValues[metricsNodeMemoryUltilisation].Series, monitoring.Point{float64(m.Timestamp.Unix()), float64(usage.Memory().Value()) / float64(cap.Memory().Value())})
}
}
}
nodeMetrics := make(map[string]*monitoring.MetricData)
for _, enm := range edgeNodeMetrics {
_, ok := metricsMap[enm]
if ok {
nodeMetrics[enm] = &monitoring.MetricData{MetricType: monitoring.MetricTypeMatrix}
nodeMetrics[enm].MetricValues = append(nodeMetrics[enm].MetricValues, *metricValues[enm])
}
}
}
var usage v1.ResourceList
var cap v1.ResourceList
for _, m := range metricsResult.Items {
_, ok := edgeNodeNamesFiltered[m.Name]
if !ok {
continue
}
for _, enm := range edgeNodeMetrics {
_, ok := metricsMap[enm]
if ok {
res = append(res, monitoring.Metric{MetricName: enm, MetricData: *nodeMetrics[enm]})
}
}
return res
}
m.Usage.DeepCopyInto(&usage)
status[m.Name].Capacity.DeepCopyInto(&cap)
func (m metricsServer) GetPodLevelNamedMetricsOverTime(metrics []string, start, end time.Time, step time.Duration, opts *monitoring.QueryOptions) []monitoring.Metric {
var res []monitoring.Metric
metricValues := make(map[string]*monitoring.MetricValue)
edgePods := m.parseEdgePods(opts)
if len(edgePods) == 0 && opts.PodName == "" {
klog.Errorf("Edge node filter regexp error: %v\n", errors.New("no edge node pods metrics is requested or resource filter invalid"))
return res
}
for _, enm := range edgeNodeMetrics {
metricValues[enm] = &monitoring.MetricValue{
Metadata: make(map[string]string),
}
metricValues[enm].Metadata["node"] = m.Name
metricValues[enm].Metadata["role"] = "edge"
}
for _, addr := range status[m.Name].Addresses {
if addr.Type == v1.NodeInternalIP {
for _, enm := range edgeNodeMetrics {
metricValues[enm].Metadata["host_ip"] = addr.Address
}
break
}
}
podMetricsFromMetricsAPI, err := m.getPodMetricsFromMetricsAPI(edgePods, opts.PodName, opts.NamespaceName)
if err != nil {
klog.Errorf("Get pod metrics of edge nodes error %v\n", err)
return m.parseErrorResp(metrics, err)
}
_, ok = metricsMap[metricsNodeCPUUsage]
if ok {
metricValues[metricsNodeCPUUsage].Series = append(metricValues[metricsNodeCPUUsage].Series, monitoring.Point{float64(m.Timestamp.Unix()), float64(usage.Cpu().MilliValue()) / 1000})
}
_, ok = metricsMap[metricsNodeCPUTotal]
if ok {
metricValues[metricsNodeCPUTotal].Series = append(metricValues[metricsNodeCPUTotal].Series, monitoring.Point{float64(m.Timestamp.Unix()), float64(cap.Cpu().MilliValue()) / 1000})
}
_, ok = metricsMap[metricsNodeCPUUltilisation]
if ok {
metricValues[metricsNodeCPUUltilisation].Series = append(metricValues[metricsNodeCPUUltilisation].Series, monitoring.Point{float64(m.Timestamp.Unix()), float64(usage.Cpu().MilliValue()) / float64(cap.Cpu().MilliValue())})
}
_, ok = metricsMap[metricsNodeMemoryUsageWoCache]
if ok {
metricValues[metricsNodeMemoryUsageWoCache].Series = append(metricValues[metricsNodeMemoryUsageWoCache].Series, monitoring.Point{float64(m.Timestamp.Unix()), float64(usage.Memory().Value())})
}
_, ok = metricsMap[metricsNodeMemoryTotal]
if ok {
metricValues[metricsNodeMemoryTotal].Series = append(metricValues[metricsNodeMemoryTotal].Series, monitoring.Point{float64(m.Timestamp.Unix()), float64(cap.Memory().Value())})
metricsMap := make(map[string]bool)
for _, m := range metrics {
metricsMap[m] = true
}
// init
podMetrics := make(map[string]*monitoring.MetricData)
for _, epm := range edgePodMetrics {
_, ok := metricsMap[epm]
if ok {
podMetrics[epm] = &monitoring.MetricData{MetricType: monitoring.MetricTypeMatrix}
}
}
for _, p := range podMetricsFromMetricsAPI {
metricValues := make(map[string]*monitoring.MetricValue)
for _, epm := range edgePodMetrics {
metricValues[epm] = &monitoring.MetricValue{
Metadata: make(map[string]string),
}
_, ok = metricsMap[metricsNodeMemoryUltilisation]
if ok {
metricValues[metricsNodeMemoryUltilisation].Series = append(metricValues[metricsNodeMemoryUltilisation].Series, monitoring.Point{float64(m.Timestamp.Unix()), float64(usage.Memory().Value()) / float64(cap.Memory().Value())})
metricValues[epm].Metadata["pod"] = p.Name
metricValues[epm].Metadata["namespace"] = p.Namespace
}
podMetricsUsge := make(v1.ResourceList)
for _, res := range MeasuredResources {
podMetricsUsge[res], _ = resource.ParseQuantity("0")
}
for _, podContainer := range p.Containers {
for _, res := range MeasuredResources {
quantity := podMetricsUsge[res]
quantity.Add(podContainer.Usage[res])
podMetricsUsge[res] = quantity
}
}
for _, enm := range edgeNodeMetrics {
_, ok = metricsMap[enm]
if ok {
nodeMetrics[enm].MetricValues = append(nodeMetrics[enm].MetricValues, *metricValues[enm])
for k, v := range metricsMap {
switch k {
case metricsPodCPUUsage:
if v {
cpuQuantity := podMetricsUsge[v1.ResourceCPU]
metricValues[metricsPodCPUUsage].Series = append(metricValues[metricsPodCPUUsage].Series, monitoring.Point{float64(p.Timestamp.Unix()), float64(cpuQuantity.MilliValue()) / 1000})
}
case metricsPodMemoryUsage:
if v {
memoryQuantity := podMetricsUsge[v1.ResourceMemory]
metricValues[metricsPodMemoryUsage].Series = append(metricValues[metricsPodMemoryUsage].Series, monitoring.Point{float64(p.Timestamp.Unix()), float64(memoryQuantity.Value()) / (1024 * 1024)})
}
}
}
for _, enm := range edgeNodeMetrics {
_, ok := metricsMap[enm]
for _, epm := range edgePodMetrics {
_, ok := metricsMap[epm]
if ok {
res = append(res, monitoring.Metric{MetricName: enm, MetricData: *nodeMetrics[enm]})
podMetrics[epm].MetricValues = append(podMetrics[epm].MetricValues, *metricValues[epm])
}
}
}
for _, epm := range edgePodMetrics {
_, ok := metricsMap[epm]
if ok {
res = append(res, monitoring.Metric{MetricName: epm, MetricData: *podMetrics[epm]})
}
}
return res
}
......
......@@ -2,6 +2,7 @@ package metricsserver
import (
"fmt"
"strings"
"testing"
"time"
......@@ -71,8 +72,126 @@ var node2 = &v1.Node{
},
}
var pod1 = &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "pod1",
Namespace: "kubeedge",
},
}
var pod2 = &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "pod2",
Namespace: "kubeedge",
},
}
const (
layout = "2006-01-02T15:04:05.000Z"
str = "2021-03-25T12:34:56.789Z"
)
var (
metricsTime, _ = time.Parse(layout, str)
nodeMetric1 = metricsV1beta1.NodeMetrics{
ObjectMeta: metav1.ObjectMeta{
Name: "edgenode-1",
Labels: map[string]string{
"node-role.kubernetes.io/edge": "",
},
},
Timestamp: metav1.Time{Time: metricsTime},
Window: metav1.Duration{Duration: time.Minute},
Usage: v1.ResourceList{
v1.ResourceCPU: *resource.NewMilliQuantity(
int64(1000),
resource.DecimalSI),
v1.ResourceMemory: *resource.NewQuantity(
int64(1024*1024),
resource.BinarySI),
},
}
nodeMetric2 = metricsV1beta1.NodeMetrics{
ObjectMeta: metav1.ObjectMeta{
Name: "edgenode-2",
Labels: map[string]string{
"node-role.kubernetes.io/edge": "",
},
},
Timestamp: metav1.Time{Time: metricsTime},
Window: metav1.Duration{Duration: time.Minute},
Usage: v1.ResourceList{
v1.ResourceCPU: *resource.NewMilliQuantity(
int64(2000),
resource.DecimalSI),
v1.ResourceMemory: *resource.NewQuantity(
int64(2*1024*1024),
resource.BinarySI),
},
}
podMetric1 = metricsV1beta1.PodMetrics{
TypeMeta: metav1.TypeMeta{
Kind: "DaemonSet",
},
ObjectMeta: metav1.ObjectMeta{
Name: "pod1",
Namespace: "kubeedge",
},
Timestamp: metav1.Time{Time: metricsTime},
Window: metav1.Duration{Duration: time.Minute},
Containers: []metricsV1beta1.ContainerMetrics{
metricsV1beta1.ContainerMetrics{
Name: "containers-1",
Usage: v1.ResourceList{
v1.ResourceCPU: *resource.NewMilliQuantity(
1,
resource.DecimalSI),
v1.ResourceMemory: *resource.NewQuantity(
int64(1024*1024),
resource.DecimalSI),
},
},
},
}
podMetric2 = metricsV1beta1.PodMetrics{
TypeMeta: metav1.TypeMeta{
Kind: "DaemonSet",
},
ObjectMeta: metav1.ObjectMeta{
Name: "pod2",
Namespace: "kubeedge",
},
Timestamp: metav1.Time{Time: metricsTime},
Window: metav1.Duration{Duration: time.Minute},
Containers: []metricsV1beta1.ContainerMetrics{
metricsV1beta1.ContainerMetrics{
Name: "containers-1",
Usage: v1.ResourceList{
v1.ResourceCPU: *resource.NewMilliQuantity(
1,
resource.DecimalSI),
v1.ResourceMemory: *resource.NewQuantity(
int64(1024*1024),
resource.DecimalSI),
},
},
metricsV1beta1.ContainerMetrics{
Name: "containers-2",
Usage: v1.ResourceList{
v1.ResourceCPU: *resource.NewMilliQuantity(
1,
resource.DecimalSI),
v1.ResourceMemory: *resource.NewQuantity(
int64(1024*1024),
resource.DecimalSI),
},
},
},
}
)
func TestGetNamedMetrics(t *testing.T) {
tests := []struct {
nodeMetricsTests := []struct {
metrics []string
filter string
expected string
......@@ -93,62 +212,63 @@ func TestGetNamedMetrics(t *testing.T) {
expected: "metrics-vector-3.json",
},
}
podMetricsTests := []struct {
metrics []string
filter string
expected string
podName string
namespaceName string
}{
{
metrics: []string{"pod_cpu_usage", "pod_memory_usage_wo_cache"},
filter: "pod1$",
expected: "metrics-vector-4.json",
podName: "",
namespaceName: "",
},
{
metrics: []string{"pod_cpu_usage", "pod_memory_usage_wo_cache"},
filter: "default/pod2$",
expected: "metrics-vector-5.json",
podName: "",
namespaceName: "",
},
{
metrics: []string{"pod_cpu_usage", "pod_memory_usage_wo_cache"},
filter: "",
expected: "metrics-vector-6.json",
podName: "pod1",
namespaceName: "kubeedge",
},
}
fakeK8sClient := fakek8s.NewSimpleClientset(node1, node2)
fakeK8sClient := fakek8s.NewSimpleClientset(node1, node2, pod1, pod2)
informer := informers.NewSharedInformerFactory(fakeK8sClient, 0)
informer.Core().V1().Nodes().Informer().GetIndexer().Add(node1)
informer.Core().V1().Nodes().Informer().GetIndexer().Add(node2)
informer.Core().V1().Pods().Informer().GetIndexer().Add(pod1)
informer.Core().V1().Pods().Informer().GetIndexer().Add(pod2)
fakeMetricsclient := &fakemetricsclient.Clientset{}
layout := "2006-01-02T15:04:05.000Z"
str := "2021-01-25T12:34:56.789Z"
metricsTime, _ := time.Parse(layout, str)
fakeMetricsclient.AddReactor("list", "nodes", func(action core.Action) (handled bool, ret runtime.Object, err error) {
metrics := &metricsV1beta1.NodeMetricsList{}
nodeMetric1 := metricsV1beta1.NodeMetrics{
ObjectMeta: metav1.ObjectMeta{
Name: "edgenode-1",
Labels: map[string]string{
"node-role.kubernetes.io/edge": "",
},
},
Timestamp: metav1.Time{Time: metricsTime},
Window: metav1.Duration{Duration: time.Minute},
Usage: v1.ResourceList{
v1.ResourceCPU: *resource.NewMilliQuantity(
int64(1000),
resource.DecimalSI),
v1.ResourceMemory: *resource.NewQuantity(
int64(1024*1024),
resource.BinarySI),
},
}
nodeMetric2 := metricsV1beta1.NodeMetrics{
ObjectMeta: metav1.ObjectMeta{
Name: "edgenode-2",
Labels: map[string]string{
"node-role.kubernetes.io/edge": "",
},
},
Timestamp: metav1.Time{Time: metricsTime},
Window: metav1.Duration{Duration: time.Minute},
Usage: v1.ResourceList{
v1.ResourceCPU: *resource.NewMilliQuantity(
int64(2000),
resource.DecimalSI),
v1.ResourceMemory: *resource.NewQuantity(
int64(2*1024*1024),
resource.BinarySI),
},
}
metrics.Items = append(metrics.Items, nodeMetric1)
metrics.Items = append(metrics.Items, nodeMetric2)
return true, metrics, nil
})
for i, tt := range tests {
fakeMetricsclient.AddReactor("list", "pods", func(action core.Action) (handled bool, ret runtime.Object, err error) {
metrics := &metricsV1beta1.PodMetricsList{}
metrics.Items = append(metrics.Items, podMetric1)
metrics.Items = append(metrics.Items, podMetric2)
return true, metrics, nil
})
// test for node edge
for i, tt := range nodeMetricsTests {
t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
expected := make([]monitoring.Metric, 0)
err := jsonFromFile(tt.expected, &expected)
......@@ -163,10 +283,36 @@ func TestGetNamedMetrics(t *testing.T) {
}
})
}
//test for pods on the node edges
for i, tt := range podMetricsTests {
t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
expected := make([]monitoring.Metric, 0)
err := jsonFromFile(tt.expected, &expected)
if err != nil {
t.Fatal(err)
}
if tt.podName == "pod1" || strings.Contains(tt.filter, "pod1") {
fakeMetricsclient.PrependReactor("get", "pods", func(action core.Action) (handled bool, ret runtime.Object, err error) {
return true, &podMetric1, nil
})
} else {
fakeMetricsclient.PrependReactor("get", "pods", func(action core.Action) (handled bool, ret runtime.Object, err error) {
return true, &podMetric2, nil
})
}
client := NewMetricsServer(fakeK8sClient, true, fakeMetricsclient)
result := client.GetNamedMetrics(tt.metrics, time.Now(), monitoring.PodOption{ResourceFilter: tt.filter, PodName: tt.podName, NamespaceName: tt.namespaceName})
if diff := cmp.Diff(result, expected); diff != "" {
t.Fatalf("%T differ (-got, +want): %s", expected, diff)
}
})
}
}
func TestGetNamedMetricsOverTime(t *testing.T) {
tests := []struct {
nodeMetricsTests := []struct {
metrics []string
filter string
expected string
......@@ -188,61 +334,62 @@ func TestGetNamedMetricsOverTime(t *testing.T) {
},
}
fakeK8sClient := fakek8s.NewSimpleClientset(node1, node2)
podMetricsTests := []struct {
metrics []string
filter string
expected string
podName string
namespaceName string
}{
{
metrics: []string{"pod_cpu_usage", "pod_memory_usage_wo_cache"},
filter: "pod1$",
expected: "metrics-matrix-4.json",
podName: "",
namespaceName: "",
},
{
metrics: []string{"pod_cpu_usage", "pod_memory_usage_wo_cache"},
filter: "default/pod2$",
expected: "metrics-matrix-5.json",
podName: "",
namespaceName: "",
},
{
metrics: []string{"pod_cpu_usage", "pod_memory_usage_wo_cache"},
filter: "",
expected: "metrics-matrix-6.json",
podName: "pod1",
namespaceName: "kubeedge",
},
}
fakeK8sClient := fakek8s.NewSimpleClientset(node1, node2, pod1, pod2)
informer := informers.NewSharedInformerFactory(fakeK8sClient, 0)
informer.Core().V1().Nodes().Informer().GetIndexer().Add(node1)
informer.Core().V1().Nodes().Informer().GetIndexer().Add(node2)
informer.Core().V1().Nodes().Informer().GetIndexer().Add(pod1)
informer.Core().V1().Nodes().Informer().GetIndexer().Add(pod2)
fakeMetricsclient := &fakemetricsclient.Clientset{}
layout := "2006-01-02T15:04:05.000Z"
str := "2021-01-25T12:34:56.789Z"
metricsTime, _ := time.Parse(layout, str)
fakeMetricsclient.AddReactor("list", "nodes", func(action core.Action) (handled bool, ret runtime.Object, err error) {
metrics := &metricsV1beta1.NodeMetricsList{}
nodeMetric1 := metricsV1beta1.NodeMetrics{
ObjectMeta: metav1.ObjectMeta{
Name: "edgenode-1",
Labels: map[string]string{
"node-role.kubernetes.io/edge": "",
},
},
Timestamp: metav1.Time{Time: metricsTime},
Window: metav1.Duration{Duration: time.Minute},
Usage: v1.ResourceList{
v1.ResourceCPU: *resource.NewMilliQuantity(
int64(1000),
resource.DecimalSI),
v1.ResourceMemory: *resource.NewQuantity(
int64(1024*1024),
resource.BinarySI),
},
}
nodeMetric2 := metricsV1beta1.NodeMetrics{
ObjectMeta: metav1.ObjectMeta{
Name: "edgenode-2",
Labels: map[string]string{
"node-role.kubernetes.io/edge": "",
},
},
Timestamp: metav1.Time{Time: metricsTime},
Window: metav1.Duration{Duration: time.Minute},
Usage: v1.ResourceList{
v1.ResourceCPU: *resource.NewMilliQuantity(
int64(2000),
resource.DecimalSI),
v1.ResourceMemory: *resource.NewQuantity(
int64(2*1024*1024),
resource.BinarySI),
},
}
metrics.Items = append(metrics.Items, nodeMetric1)
metrics.Items = append(metrics.Items, nodeMetric2)
return true, metrics, nil
})
for i, tt := range tests {
fakeMetricsclient.AddReactor("list", "pods", func(action core.Action) (handled bool, ret runtime.Object, err error) {
metrics := &metricsV1beta1.PodMetricsList{}
metrics.Items = append(metrics.Items, podMetric1)
metrics.Items = append(metrics.Items, podMetric2)
return true, metrics, nil
})
for i, tt := range nodeMetricsTests {
fakeMetricsclient.Fake.ClearActions()
t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
expected := make([]monitoring.Metric, 0)
err := jsonFromFile(tt.expected, &expected)
......@@ -257,6 +404,32 @@ func TestGetNamedMetricsOverTime(t *testing.T) {
}
})
}
for i, tt := range podMetricsTests {
t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
expected := make([]monitoring.Metric, 0)
err := jsonFromFile(tt.expected, &expected)
if err != nil {
t.Fatal(err)
}
if tt.podName == "pod1" || strings.Contains(tt.filter, "pod1") {
fakeMetricsclient.PrependReactor("get", "pods", func(action core.Action) (handled bool, ret runtime.Object, err error) {
return true, &podMetric1, nil
})
} else {
fakeMetricsclient.PrependReactor("get", "pods", func(action core.Action) (handled bool, ret runtime.Object, err error) {
return true, &podMetric2, nil
})
}
client := NewMetricsServer(fakeK8sClient, true, fakeMetricsclient)
result := client.GetNamedMetricsOverTime(tt.metrics, time.Now().Add(-time.Minute*3), time.Now(), time.Minute, monitoring.PodOption{ResourceFilter: tt.filter, PodName: tt.podName, NamespaceName: tt.namespaceName})
if diff := cmp.Diff(result, expected); diff != "" {
t.Fatalf("%T differ (-got, +want): %s", expected, diff)
}
})
}
}
func jsonFromFile(expectedFile string, expectedJsonPtr interface{}) error {
......
......@@ -11,7 +11,7 @@
},
"values": [
[
1611578096,
1616675696,
"1"
]
]
......@@ -23,7 +23,7 @@
},
"values": [
[
1611578096,
1616675696,
"2"
]
]
......@@ -43,7 +43,7 @@
},
"values": [
[
1611578096,
1616675696,
"1048576"
]
]
......@@ -55,7 +55,7 @@
},
"values": [
[
1611578096,
1616675696,
"2097152"
]
]
......
......@@ -11,7 +11,7 @@
},
"values": [
[
1611578096,
1616675696,
"2"
]
]
......@@ -31,7 +31,7 @@
},
"values": [
[
1611578096,
1616675696,
"0.25"
]
]
......
......@@ -11,7 +11,7 @@
},
"values": [
[
1611578096,
1616675696,
"1048576"
]
]
......@@ -23,7 +23,7 @@
},
"values": [
[
1611578096,
1616675696,
"2097152"
]
]
......@@ -43,7 +43,7 @@
},
"values": [
[
1611578096,
1616675696,
"0.0001220703125"
]
]
......@@ -55,7 +55,7 @@
},
"values": [
[
1611578096,
1616675696,
"0.000244140625"
]
]
......
[
{
"metric_name": "pod_cpu_usage",
"data": {
"resultType": "matrix",
"result": [
{
"metric": {
"namespace": "kubeedge",
"pod":"pod1"
},
"values":[
[
1616675696,
"0.001"
]
]
}
]
}
},
{
"metric_name": "pod_memory_usage_wo_cache",
"data": {
"resultType": "matrix",
"result": [
{
"metric": {
"namespace": "kubeedge",
"pod":"pod1"
},
"values": [
[
1616675696,
"1"
]
]
}
]
}
}
]
\ No newline at end of file
[
{
"metric_name": "pod_cpu_usage",
"data": {
"resultType": "matrix",
"result": [
{
"metric": {
"namespace": "kubeedge",
"pod":"pod2"
},
"values": [
[
1616675696,
"0.002"
]
]
}
]
}
},
{
"metric_name": "pod_memory_usage_wo_cache",
"data": {
"resultType": "matrix",
"result": [
{
"metric": {
"namespace": "kubeedge",
"pod":"pod2"
},
"values": [
[
1616675696,
"2"
]
]
}
]
}
}
]
\ No newline at end of file
[
{
"metric_name": "pod_cpu_usage",
"data": {
"resultType": "matrix",
"result": [
{
"metric": {
"namespace": "kubeedge",
"pod":"pod1"
},
"values":[
[
1616675696,
"0.001"
]
]
}
]
}
},
{
"metric_name": "pod_memory_usage_wo_cache",
"data": {
"resultType": "matrix",
"result": [
{
"metric": {
"namespace": "kubeedge",
"pod":"pod1"
},
"values": [
[
1616675696,
"1"
]
]
}
]
}
}
]
\ No newline at end of file
......@@ -10,7 +10,7 @@
"role": "edge"
},
"value": [
1611578096,
1616675696,
"1"
]
},
......@@ -20,7 +20,7 @@
"role": "edge"
},
"value": [
1611578096,
1616675696,
"2"
]
}
......@@ -38,7 +38,7 @@
"role": "edge"
},
"value": [
1611578096,
1616675696,
"1048576"
]
},
......@@ -48,7 +48,7 @@
"role": "edge"
},
"value": [
1611578096,
1616675696,
"2097152"
]
}
......
......@@ -10,7 +10,7 @@
"role": "edge"
},
"value": [
1611578096,
1616675696,
"2"
]
}
......@@ -28,7 +28,7 @@
"role": "edge"
},
"value": [
1611578096,
1616675696,
"0.25"
]
}
......
......@@ -10,7 +10,7 @@
"role": "edge"
},
"value": [
1611578096,
1616675696,
"1048576"
]
},
......@@ -20,7 +20,7 @@
"role": "edge"
},
"value": [
1611578096,
1616675696,
"2097152"
]
}
......@@ -38,7 +38,7 @@
"role": "edge"
},
"value": [
1611578096,
1616675696,
"0.0001220703125"
]
},
......@@ -48,7 +48,7 @@
"role": "edge"
},
"value": [
1611578096,
1616675696,
"0.000244140625"
]
}
......
[
{
"metric_name": "pod_cpu_usage",
"data": {
"resultType": "vector",
"result": [
{
"metric": {
"namespace": "kubeedge",
"pod":"pod1"
},
"value": [
1616675696,
"0.001"
]
}
]
}
},
{
"metric_name": "pod_memory_usage_wo_cache",
"data": {
"resultType": "vector",
"result": [
{
"metric": {
"namespace": "kubeedge",
"pod":"pod1"
},
"value": [
1616675696,
"1"
]
}
]
}
}
]
\ No newline at end of file
[
{
"metric_name": "pod_cpu_usage",
"data": {
"resultType": "vector",
"result": [
{
"metric": {
"namespace": "kubeedge",
"pod":"pod2"
},
"value": [
1616675696,
"0.002"
]
}
]
}
},
{
"metric_name": "pod_memory_usage_wo_cache",
"data": {
"resultType": "vector",
"result": [
{
"metric": {
"namespace": "kubeedge",
"pod":"pod2"
},
"value": [
1616675696,
"2"
]
}
]
}
}
]
\ No newline at end of file
[
{
"metric_name": "pod_cpu_usage",
"data": {
"resultType": "vector",
"result": [
{
"metric": {
"namespace": "kubeedge",
"pod":"pod1"
},
"value": [
1616675696,
"0.001"
]
}
]
}
},
{
"metric_name": "pod_memory_usage_wo_cache",
"data": {
"resultType": "vector",
"result": [
{
"metric": {
"namespace": "kubeedge",
"pod":"pod1"
},
"value": [
1616675696,
"1"
]
}
]
}
}
]
\ No newline at end of file
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册