Support default value in labeled-value and heatmap query. (#4711)

* Support default value in labeled-value and heatmap query.

* Update a little document.

README.md

@@ -38,7 +38,7 @@ including
 1. Java, [.NET Core](https://github.com/SkyAPM/SkyAPM-dotnet), [NodeJS](https://github.com/SkyAPM/SkyAPM-nodejs) and [PHP](https://github.com/SkyAPM/SkyAPM-php-sdk) auto-instrument agents.
 1. [Go agent](https://github.com/tetratelabs/go2sky).
 1. [LUA agent](https://github.com/apache/skywalking-nginx-lua), especially for Nginx, OpenResty.
-1. Service Mesh Observability, including Envoy gRPC Access Log Service (ALS) format in Istio controlled service mesh, Istio telemetry. 
+1. Service Mesh Observability. Support Mixer telemetry. Recommend to use Envoy Access Log Service (ALS) for better performance, first introduced at [KubeCon 2019](https://www.youtube.com/watch?v=tERm39ju9ew).
 1. Metrics system, including Prometheus, Spring Sleuth(Micrometer).
 1. Zipkin v1/v2 and Jaeger gRPC format with limited topology and metrics analysis.(Experimental).
 

docs/en/setup/envoy/als_setting.md

@@ -2,7 +2,10 @@
 Envoy [ALS(access log service)](https://www.envoyproxy.io/docs/envoy/latest/api-v2/service/accesslog/v2/als.proto) provides
 fully logs about RPC routed, including HTTP and TCP.
 
-**If solution initialized and first implemented by [Sheng Wu](https://github.com/wu-sheng), [Hongtao Gao](https://github.com/hanahmily), [Lizan Zhou](https://github.com/lizan) and [Dhi Aurrahman](https://github.com/dio) at 17 May. 2019, and presented on [KubeCon China 2019](https://kccncosschn19eng.sched.com/event/NroB/observability-in-service-mesh-powered-by-envoy-and-apache-skywalking-sheng-wu-lizan-zhou-tetrate).**
+If solution initialized and first implemented by [Sheng Wu](https://github.com/wu-sheng), [Hongtao Gao](https://github.com/hanahmily), [Lizan Zhou](https://github.com/lizan), 
+and [Dhi Aurrahman](https://github.com/dio) at 17 May. 2019, 
+and presented on [KubeCon China 2019](https://kccncosschn19eng.sched.com/event/NroB/observability-in-service-mesh-powered-by-envoy-and-apache-skywalking-sheng-wu-lizan-zhou-tetrate).
+Here is the recorded [Video](https://www.youtube.com/watch?v=tERm39ju9ew).
 
 SkyWalking is the first open source project introducing this ALS based solution to the world. This provides a new way with very low payload to service mesh, but the same observability.
 
@@ -17,4 +20,4 @@ envoy-metric:
 ```
 Note multiple value，please use `,` symbol split
 
-Notice, only use this when envoy under Istio controlled, also in k8s env.
+Notice, only use this when envoy under Istio controlled, also in k8s env. The OAP requires the read right to k8s API server for all pods IPs.

docs/en/setup/istio/README.md

@@ -13,6 +13,8 @@ Follow the [deploying backend in kubernetes](../backend/backend-k8s.md) to insta
 
 ## Setup Istio to send metrics to oap
 
+Our scripts are wrote based on Istio 1.3.3.
+
 1. Install Istio metric template
 
 `kubectl apply -f https://raw.githubusercontent.com/istio/istio/1.3.3/mixer/template/metric/template.yaml`
@@ -22,3 +24,5 @@ Follow the [deploying backend in kubernetes](../backend/backend-k8s.md) to insta
 `kubectl apply -f skywalkingadapter.yml`
 
 Find the `skywalkingadapter.yml` at [here](yaml/skywalkingadapter.yml).
+
+NOTICE, due to Istio Mixer is default OFF, we recommend you to consider our [ALS solution](../envoy/als_setting.md)
\ No newline at end of file

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/DataTable.java

@@ -71,6 +71,10 @@ public class DataTable implements StorageDataComplexObject<DataTable> {
         return !data.isEmpty();
     }
 
+    public boolean hasKey(String key) {
+        return data.containsKey(key);
+    }
+
     public int size() {
         return data.size();
     }

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/HistogramMetrics.java

@@ -41,7 +41,7 @@ public abstract class HistogramMetrics extends Metrics {
 
     @Getter
     @Setter
-    @Column(columnName = DATASET, dataType = Column.ValueDataType.HISTOGRAM, storageOnly = true)
+    @Column(columnName = DATASET, dataType = Column.ValueDataType.HISTOGRAM, storageOnly = true, defaultValue = 0)
     private DataTable dataset = new DataTable(30);
 
     /**

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/HeatMap.java

@@ -45,7 +45,7 @@ public class HeatMap {
      * @param id      of the row
      * @param rawdata literal string, represent a {@link DataTable}
      */
-    public void buildColumn(String id, String rawdata) {
+    public void buildColumn(String id, String rawdata, int defaultValue) {
         DataTable dataset = new DataTable(rawdata);
 
         final List<String> sortedKeys = dataset.sortedKeys(
@@ -70,12 +70,16 @@ public class HeatMap {
         HeatMap.HeatMapColumn column = new HeatMap.HeatMapColumn();
         column.setId(id);
         sortedKeys.forEach(key -> {
-            column.addValue(dataset.get(key));
+            if (dataset.hasKey(key)) {
+                column.addValue(dataset.get(key));
+            } else {
+                column.addValue((long) defaultValue);
+            }
         });
         values.add(column);
     }
 
-    public void fixMissingColumns(List<String> ids) {
+    public void fixMissingColumns(List<String> ids, int defaultValue) {
         for (int i = 0; i < ids.size(); i++) {
             final String expectedId = ids.get(i);
             boolean found = false;
@@ -85,17 +89,17 @@ public class HeatMap {
                 }
             }
             if (!found) {
-                final HeatMapColumn emptyColumn = buildMissingColumn(expectedId);
+                final HeatMapColumn emptyColumn = buildMissingColumn(expectedId, defaultValue);
                 values.add(i, emptyColumn);
             }
         }
     }
 
-    private HeatMapColumn buildMissingColumn(String id) {
+    private HeatMapColumn buildMissingColumn(String id, int defaultValue) {
         HeatMapColumn column = new HeatMapColumn();
         column.setId(id);
         buckets.forEach(bucket -> {
-            column.addValue(0L);
+            column.addValue((long) defaultValue);
         });
         return column;
     }

oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/MetricsQueryEsDAO.java

@@ -148,14 +148,18 @@ public class MetricsQueryEsDAO extends EsDAO implements IMetricsQueryDAO {
             labeledValues.put(label, labelValue);
         });
 
+        final int defaultValue = ValueColumnMetadata.INSTANCE.getDefaultValue(condition.getName());
         for (String id : ids) {
             if (idMap.containsKey(id)) {
                 Map<String, Object> source = idMap.get(id);
                 DataTable multipleValues = new DataTable((String) source.getOrDefault(valueColumnName, ""));
 
                 labels.forEach(label -> {
-                    final Long data = multipleValues.get(label);
                     final IntValues values = labeledValues.get(label).getValues();
+                    Long data = multipleValues.get(label);
+                    if (data == null) {
+                        data = (long) defaultValue;
+                    }
                     KVInt kv = new KVInt();
                     kv.setId(id);
                     kv.setValue(data);
@@ -168,7 +172,7 @@ public class MetricsQueryEsDAO extends EsDAO implements IMetricsQueryDAO {
         return Util.sortValues(
             new ArrayList<>(labeledValues.values()),
             ids,
-            ValueColumnMetadata.INSTANCE.getDefaultValue(condition.getName())
+            defaultValue
         );
     }
 
@@ -187,15 +191,16 @@ public class MetricsQueryEsDAO extends EsDAO implements IMetricsQueryDAO {
 
         HeatMap heatMap = new HeatMap();
 
+        final int defaultValue = ValueColumnMetadata.INSTANCE.getDefaultValue(condition.getName());
         for (String id : ids) {
             Map<String, Object> source = idMap.get(id);
             if (source != null) {
                 String value = (String) source.get(HistogramMetrics.DATASET);
-                heatMap.buildColumn(id, value);
+                heatMap.buildColumn(id, value, defaultValue);
             }
         }
 
-        heatMap.fixMissingColumns(ids);
+        heatMap.fixMissingColumns(ids, defaultValue);
 
         return heatMap;
     }

oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/MetricsQuery.java

@@ -183,6 +183,7 @@ public class MetricsQuery implements IMetricsQueryDAO {
             labeledValues.put(label, labelValue);
         });
 
+        final int defaultValue = ValueColumnMetadata.INSTANCE.getDefaultValue(condition.getName());
         if (!CollectionUtils.isEmpty(series)) {
             series.get(0).getValues().forEach(values -> {
                 final String id = (String) values.get(1);
@@ -190,7 +191,10 @@ public class MetricsQuery implements IMetricsQueryDAO {
                 multipleValues.toObject((String) values.get(2));
 
                 labels.forEach(label -> {
-                    final Long data = multipleValues.get(label);
+                    Long data = multipleValues.get(label);
+                    if (data == null) {
+                        data = (long) defaultValue;
+                    }
                     final IntValues intValues = labeledValues.get(label).getValues();
                     KVInt kv = new KVInt();
                     kv.setId(id);
@@ -203,7 +207,7 @@ public class MetricsQuery implements IMetricsQueryDAO {
         return Util.sortValues(
             new ArrayList<>(labeledValues.values()),
             ids,
-            ValueColumnMetadata.INSTANCE.getDefaultValue(condition.getName())
+            defaultValue
         );
     }
 
@@ -229,14 +233,16 @@ public class MetricsQuery implements IMetricsQueryDAO {
             log.debug("SQL: {} result set: {}", query.getCommand(), series);
         }
 
+        final int defaultValue = ValueColumnMetadata.INSTANCE.getDefaultValue(condition.getName());
+
         HeatMap heatMap = new HeatMap();
         if (series != null) {
             for (List<Object> values : series.getValues()) {
-                heatMap.buildColumn(values.get(1).toString(), values.get(2).toString());
+                heatMap.buildColumn(values.get(1).toString(), values.get(2).toString(), defaultValue);
             }
         }
 
-        heatMap.fixMissingColumns(ids);
+        heatMap.fixMissingColumns(ids, defaultValue);
 
         return heatMap;
     }

oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2MetricsQueryDAO.java

@@ -170,6 +170,8 @@ public class H2MetricsQueryDAO extends H2SQLExecutor implements IMetricsQueryDAO
             labeledValues.put(label, labelValue);
         });
 
+        final int defaultValue = ValueColumnMetadata.INSTANCE.getDefaultValue(condition.getName());
+
         try (Connection connection = h2Client.getConnection()) {
             try (ResultSet resultSet = h2Client.executeQuery(
                 connection, sql.toString(), parameters.toArray(new Object[0]))) {
@@ -180,7 +182,10 @@ public class H2MetricsQueryDAO extends H2SQLExecutor implements IMetricsQueryDAO
                     multipleValues.toObject(resultSet.getString(valueColumnName));
 
                     labels.forEach(label -> {
-                        final Long data = multipleValues.get(label);
+                        Long data = multipleValues.get(label);
+                        if (data == null) {
+                            data = (long) defaultValue;
+                        }
                         final IntValues values = labeledValues.get(label).getValues();
                         KVInt kv = new KVInt();
                         kv.setId(id);
@@ -196,7 +201,7 @@ public class H2MetricsQueryDAO extends H2SQLExecutor implements IMetricsQueryDAO
         return Util.sortValues(
             new ArrayList<>(labeledValues.values()),
             ids,
-            ValueColumnMetadata.INSTANCE.getDefaultValue(condition.getName())
+            defaultValue
         );
     }
 
@@ -223,17 +228,20 @@ public class H2MetricsQueryDAO extends H2SQLExecutor implements IMetricsQueryDAO
         }
         sql.append(")");
 
+        final int defaultValue = ValueColumnMetadata.INSTANCE.getDefaultValue(condition.getName());
+
         try (Connection connection = h2Client.getConnection()) {
             HeatMap heatMap = new HeatMap();
             try (ResultSet resultSet = h2Client.executeQuery(
                 connection, sql.toString(), parameters.toArray(new Object[0]))) {
 
                 while (resultSet.next()) {
-                    heatMap.buildColumn(resultSet.getString("id"), resultSet.getString("dataset"));
+                    heatMap.buildColumn(
+                        resultSet.getString("id"), resultSet.getString("dataset"), defaultValue);
                 }
             }
 
-            heatMap.fixMissingColumns(ids);
+            heatMap.fixMissingColumns(ids, defaultValue);
 
             return heatMap;
         } catch (SQLException e) {