Support Prometheus node-exporter VM metrics monitoring (#6277)

CHANGES.md

@@ -81,6 +81,7 @@ Release Notes.
 * Chore:  Remove duplicate codes in Envoy ALS handler.
 * Remove the strict rule of OAL disable statement parameter.
 * Fix a legal metric query adoption bug. Don't support global level metric query.
+* Add VM MAL and ui-template configration, support Prometheus node-exporter VM metrics that pushed from OpenTelemetry-collector.
 * Remove unused log query parameters.
 
 #### UI

docs/en/setup/backend/backend-receivers.md

@@ -123,6 +123,7 @@ to be the identification of the metric data.
 |----|----|-----|----|
 |istio-controlplane| Metrics of Istio control panel | otel-oc-rules/istio-controlplane.yaml | Istio Control Panel -> OpenTelemetry Collector --OC format--> SkyWalking OAP Server |
 |oap| Metrics of SkyWalking OAP server itself | otel-oc-rules/oap.yaml | SkyWalking OAP Server(SelfObservability) -> OpenTelemetry Collector --OC format--> SkyWalking OAP Server |
+|vm| Metrics of VMs | otel-oc-rules/vm.yaml | Prometheus node-exporter(VMs) -> OpenTelemetry Collector --OC format--> SkyWalking OAP Server |
 
 ## Meter receiver
 

oap-server/server-bootstrap/src/main/resources/otel-oc-rules/vm.yaml

+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This will parse a textual representation of a duration. The formats
+# accepted are based on the ISO-8601 duration format {@code PnDTnHnMn.nS}
+# with days considered to be exactly 24 hours.
+# <p>
+# Examples:
+# <pre>
+#    "PT20.345S" -- parses as "20.345 seconds"
+#    "PT15M"     -- parses as "15 minutes" (where a minute is 60 seconds)
+#    "PT10H"     -- parses as "10 hours" (where an hour is 3600 seconds)
+#    "P2D"       -- parses as "2 days" (where a day is 24 hours or 86400 seconds)
+#    "P2DT3H4M"  -- parses as "2 days, 3 hours and 4 minutes"
+#    "P-6H3M"    -- parses as "-6 hours and +3 minutes"
+#    "-P6H3M"    -- parses as "-6 hours and -3 minutes"
+#    "-P-6H+3M"  -- parses as "+6 hours and -3 minutes"
+# </pre>
+expSuffix: tag({tags -> tags.node_identifier_host_name = 'vm::' + tags.node_identifier_host_name}).service(['node_identifier_host_name'])
+metricPrefix: meter_vm
+metricsRules:
+
+  #node cpu
+  - name: cpu_total_percentage
+    exp: (node_cpu_seconds_total * 100).tagNotEqual('mode' , 'idle').sum(['node_identifier_host_name']).rate('PT1M')
+  - name: cpu_average_used
+    exp: (node_cpu_seconds_total * 100).sum(['node_identifier_host_name' , 'mode']).rate('PT1M')
+  - name: cpu_load1
+    exp: node_load1 * 100
+  - name: cpu_load5
+    exp: node_load5 * 100
+  - name: cpu_load15
+    exp: node_load15 * 100
+
+    #node Memory
+  - name: memory_total
+    exp: node_memory_MemTotal_bytes
+  - name: memory_available
+    exp: node_memory_MemAvailable_bytes
+  - name: memory_used
+    exp: node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes
+  - name: memory_swap_free
+    exp: node_memory_SwapFree_bytes
+  - name: memory_swap_total
+    exp: node_memory_SwapTotal_bytes
+  - name: memory_swap_percentage
+    exp: 100 - ((node_memory_SwapFree_bytes * 100) / node_memory_SwapTotal_bytes)
+
+    #node filesystem
+  - name: filesystem_percentage
+    exp: 100 - ((node_filesystem_avail_bytes * 100).sum(['node_identifier_host_name' , 'mountpoint']) / node_filesystem_size_bytes.sum(['node_identifier_host_name' , 'mountpoint']))
+
+    #node disk
+  - name: disk_read
+    exp: node_disk_read_bytes_total.sum(['node_identifier_host_name']).rate('PT1M')
+  - name: disk_written
+    exp: node_disk_written_bytes_total.sum(['node_identifier_host_name']).rate('PT1M')
+
+    #node network
+  - name: network_receive
+    exp: node_network_receive_bytes_total.sum(['node_identifier_host_name']).irate()
+  - name: network_transmit
+    exp: node_network_transmit_bytes_total.sum(['node_identifier_host_name']).irate()
+
+    #node netstat
+  - name: tcp_curr_estab
+    exp: node_netstat_Tcp_CurrEstab
+  - name: tcp_tw
+    exp: node_sockstat_TCP_tw
+  - name: tcp_alloc
+    exp: node_sockstat_TCP_alloc
+  - name: sockets_used
+    exp: node_sockstat_sockets_used
+  - name: udp_inuse
+    exp: node_sockstat_UDP_inuse
+
+    #node filefd
+  - name: filefd_allocated
+    exp: node_filefd_allocated
+  - name: filefd_maximum
+    exp: node_filefd_maximum
+
+
+

oap-server/server-bootstrap/src/main/resources/ui-initialized-templates/vm.yml

+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# UI templates initialized file includes the default template when the SkyWalking OAP starts up at the first time.
+#
+# Also, SkyWalking would detect the existing templates in the database, once they are missing, all templates in this file
+# could be added automatically.
+
+templates:
+  - name: "VM"
+    # The type includes DASHBOARD, TOPOLOGY_INSTANCE, TOPOLOGY_ENDPOINT.
+    # DASHBOARD type templates could have multiple definitions, by using different names.
+    # TOPOLOGY_INSTANCE, TOPOLOGY_ENDPOINT type templates should be defined once, as they are used in the topology page only.
+    type: "DASHBOARD"
+    # Configuration could be defined through UI, and use `export` to format in the standard JSON.
+    configuration: |-
+      [
+        {
+          "name": "VM",
+          "type": "service",
+          "children": [
+            {
+              "name": "VM",
+              "children": [
+                {
+                  "width": "4",
+                  "title": "CPU Usage",
+                  "height": "100",
+                  "entityType": "Service",
+                  "independentSelector": false,
+                  "metricType": "REGULAR_VALUE",
+                  "metricName": "meter_vm_cpu_total_percentage",
+                  "queryMetricType": "readMetricsValue",
+                  "chartType": "ChartNum",
+                  "unit": "%"
+                },
+                {
+                  "width": "4",
+                  "title": "Memory RAM Usage",
+                  "height": "100",
+                  "entityType": "Service",
+                  "independentSelector": false,
+                  "metricType": "REGULAR_VALUE",
+                  "metricName": "meter_vm_memory_used",
+                  "queryMetricType": "readMetricsValue",
+                  "chartType": "ChartNum",
+                  "unit": "MB",
+                  "aggregation": "/",
+                  "aggregationNum": "1048576"
+                },
+                {
+                  "width": "4",
+                  "title": "Memory Swap Usage",
+                  "height": "100",
+                  "entityType": "Service",
+                  "independentSelector": false,
+                  "metricType": "REGULAR_VALUE",
+                  "metricName": "meter_vm_memory_swap_percentage",
+                  "queryMetricType": "readMetricsValue",
+                  "chartType": "ChartNum",
+                  "unit": "%"
+                },
+                {
+                  "width": "4",
+                  "title": "CPU Average Used",
+                  "height": 350,
+                  "entityType": "Service",
+                  "independentSelector": false,
+                  "metricType": "LABELED_VALUE",
+                  "queryMetricType": "readLabeledMetricsValues",
+                  "chartType": "ChartLine",
+                  "metricName": "meter_vm_cpu_average_used",
+                  "metricLabels": "",
+                  "unit": "%",
+                  "labelsIndex": ""
+                },
+                {
+                  "width": "4",
+                  "title": "CPU Load",
+                  "height": 350,
+                  "entityType": "Service",
+                  "independentSelector": false,
+                  "metricType": "REGULAR_VALUE",
+                  "metricName": "meter_vm_cpu_load1,meter_vm_cpu_load5,meter_vm_cpu_load15",
+                  "queryMetricType": "readMetricsValues",
+                  "chartType": "ChartLine",
+                  "aggregation": "/",
+                  "aggregationNum": "100"
+                },
+                {
+                  "width": "4",
+                  "title": "Memory RAM",
+                  "height": 350,
+                  "entityType": "Service",
+                  "independentSelector": false,
+                  "metricType": "REGULAR_VALUE",
+                  "metricName": "meter_vm_memory_total,meter_vm_memory_available,meter_vm_memory_used",
+                  "queryMetricType": "readMetricsValues",
+                  "chartType": "ChartArea",
+                  "aggregation": "/",
+                  "aggregationNum": "1048576",
+                  "unit": "MB"
+                },
+                {
+                  "width": "4",
+                  "title": "Memory Swap",
+                  "height": 350,
+                  "entityType": "Service",
+                  "independentSelector": false,
+                  "metricType": "REGULAR_VALUE",
+                  "metricName": "meter_vm_memory_swap_free,meter_vm_memory_swap_total",
+                  "queryMetricType": "readMetricsValues",
+                  "chartType": "ChartArea",
+                  "aggregation": "/",
+                  "aggregationNum": "1048576",
+                  "unit": "MB"
+                },
+                {
+                  "width": "4",
+                  "title": "File System Mountpoint Usage",
+                  "height": 350,
+                  "entityType": "Service",
+                  "independentSelector": false,
+                  "metricType": "LABELED_VALUE",
+                  "metricName": "meter_vm_filesystem_percentage",
+                  "queryMetricType": "readLabeledMetricsValues",
+                  "chartType": "ChartLine",
+                  "aggregation": "+",
+                  "aggregationNum": "0",
+                  "unit": "%"
+                },
+                {
+                  "width": "4",
+                  "title": "Disk R/W",
+                  "height": 350,
+                  "entityType": "Service",
+                  "independentSelector": false,
+                  "metricType": "REGULAR_VALUE",
+                  "metricName": "meter_vm_disk_read,meter_vm_disk_written",
+                  "queryMetricType": "readMetricsValues",
+                  "chartType": "ChartLine",
+                  "aggregation": "/",
+                  "aggregationNum": "1024",
+                  "unit": "KB/s"
+                },
+                {
+                  "width": "4",
+                  "title": "Network Bandwidth Usage",
+                  "height": 350,
+                  "entityType": "Service",
+                  "independentSelector": false,
+                  "metricType": "REGULAR_VALUE",
+                  "metricName": "meter_vm_network_receive,meter_vm_network_transmit",
+                  "queryMetricType": "readMetricsValues",
+                  "chartType": "ChartArea",
+                  "aggregation": "/",
+                  "aggregationNum": "1024",
+                  "unit": "KB/s"
+                },
+                {
+                  "width": "4",
+                  "title": "Network Status",
+                  "height": 350,
+                  "entityType": "Service",
+                  "independentSelector": false,
+                  "metricType": "REGULAR_VALUE",
+                  "metricName": "meter_vm_tcp_curr_estab,meter_vm_tcp_tw,meter_vm_tcp_alloc,meter_vm_sockets_used,meter_vm_udp_inuse",
+                  "queryMetricType": "readMetricsValues",
+                  "chartType": "ChartLine"
+                },
+                {
+                  "width": "4",
+                  "title": "Filefd Allocated",
+                  "height": 350,
+                  "entityType": "Service",
+                  "independentSelector": false,
+                  "metricType": "REGULAR_VALUE",
+                  "metricName": "meter_vm_filefd_allocated",
+                  "queryMetricType": "readMetricsValues",
+                  "chartType": "ChartLine"
+                }
+              ]
+            }
+          ]
+        }
+      ]
+    # Activated as the DASHBOARD type, makes this templates added into the UI page automatically.
+    # False means providing a basic template, user needs to add it manually.
+    activated: true
+    # True means wouldn't show up on the dashboard. Only keeps the definition in the storage.
+    disabled: false
\ No newline at end of file