Query all of labels once selected labels condition is absent. (#5202)

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/query/IMetricsQueryDAO.java

@@ -19,7 +19,14 @@
 package org.apache.skywalking.oap.server.core.storage.query;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.TreeSet;
+import java.util.stream.Collectors;
+import org.apache.skywalking.oap.server.core.analysis.metrics.DataTable;
 import org.apache.skywalking.oap.server.core.query.input.Duration;
 import org.apache.skywalking.oap.server.core.query.input.MetricsCondition;
 import org.apache.skywalking.oap.server.core.query.type.HeatMap;
@@ -27,6 +34,9 @@ import org.apache.skywalking.oap.server.core.query.type.IntValues;
 import org.apache.skywalking.oap.server.core.query.type.KVInt;
 import org.apache.skywalking.oap.server.core.query.type.MetricsValues;
 import org.apache.skywalking.oap.server.core.storage.DAO;
+import org.apache.skywalking.oap.server.core.storage.annotation.ValueColumnMetadata;
+
+import static java.util.stream.Collectors.toList;
 
 /**
  * Query metrics values in different ways.
@@ -76,5 +86,56 @@ public interface IMetricsQueryDAO extends DAO {
             }
             return origin;
         }
+
+        /**
+         * Compose the multiple metric result based on conditions.
+         */
+        public static List<MetricsValues> composeLabelValue(final MetricsCondition condition,
+            final List<String> labels,
+            final List<String> ids,
+            final Map<String, DataTable> idMap) {
+            List<String> allLabels;
+            if (Objects.isNull(labels) || labels.size() < 1) {
+                allLabels = idMap.values().stream()
+                    .flatMap(dataTable -> dataTable.keys().stream())
+                    .distinct().collect(Collectors.toList());
+            } else {
+                allLabels = labels;
+            }
+            final int defaultValue = ValueColumnMetadata.INSTANCE.getDefaultValue(condition.getName());
+            List<LabeledValue> labeledValues = new TreeSet<>(allLabels).stream()
+                .flatMap(label -> ids.stream().map(id ->
+                    new LabeledValue(
+                        label,
+                        id,
+                        Optional.ofNullable(idMap.getOrDefault(id, new DataTable()).get(label)).orElse((long) defaultValue))))
+                .collect(toList());
+            MetricsValues current = new MetricsValues();
+            List<MetricsValues> result = new ArrayList<>();
+            for (LabeledValue each : labeledValues) {
+                if (Objects.equals(current.getLabel(), each.label)) {
+                    current.getValues().addKVInt(each.kv);
+                } else {
+                    current = new MetricsValues();
+                    current.setLabel(each.label);
+                    current.getValues().addKVInt(each.kv);
+                    result.add(current);
+                }
+            }
+            return result;
+        }
+    }
+
+    class LabeledValue {
+        private final String label;
+        private final KVInt kv;
+
+        public LabeledValue(String label, String id, long value) {
+            this.label = label;
+            KVInt kv = new KVInt();
+            kv.setId(id);
+            kv.setValue(value);
+            this.kv = kv;
+        }
     }
 }

oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/storage/query/MetricsQueryUtilTest.java

+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.oap.server.core.storage.query;
+
+import com.google.gson.Gson;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import lombok.RequiredArgsConstructor;
+import org.apache.skywalking.oap.server.core.analysis.metrics.DataTable;
+import org.apache.skywalking.oap.server.core.query.input.MetricsCondition;
+import org.apache.skywalking.oap.server.core.query.sql.Function;
+import org.apache.skywalking.oap.server.core.query.type.MetricsValues;
+import org.apache.skywalking.oap.server.core.storage.annotation.ValueColumnMetadata;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import static com.google.common.collect.ImmutableMap.of;
+import static java.util.Arrays.asList;
+import static org.apache.skywalking.oap.server.core.storage.annotation.Column.ValueDataType.LABELED_VALUE;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.core.Is.is;
+
+@RunWith(Parameterized.class)
+@RequiredArgsConstructor
+public class MetricsQueryUtilTest {
+
+    private static final int DEFAULT_VALUE = -1;
+
+    private static final String MODULE_NAME = "meter-test";
+
+    private final List<String> queryConditionLabels;
+
+    private final List<String> datePoints;
+
+    private final Map<String, DataTable> valueColumnData;
+
+    private final String expectedResult;
+
+    @Parameterized.Parameters
+    public static Collection<Object[]> data() {
+        return Arrays.asList(new Object[][] {
+            {
+                asList("200", "400"),
+                asList("202007291425", "202007291426"),
+                of("202007291425", new DataTable("200,1|400,2"), "202007291426", new DataTable("200,3|400,8")),
+                "[{\"label\":\"200\",\"values\":{\"values\":[{\"id\":\"202007291425\",\"value\":1},{\"id\":\"202007291426\",\"value\":3}]}}," +
+                    "{\"label\":\"400\",\"values\":{\"values\":[{\"id\":\"202007291425\",\"value\":2},{\"id\":\"202007291426\",\"value\":8}]}}]"
+            },
+            {
+                asList("400", "200"),
+                asList("202007291425", "202007291426"),
+                of("202007291425", new DataTable("200,1|400,2"), "202007291426", new DataTable("200,3|400,8")),
+                "[{\"label\":\"200\",\"values\":{\"values\":[{\"id\":\"202007291425\",\"value\":1},{\"id\":\"202007291426\",\"value\":3}]}}," +
+                    "{\"label\":\"400\",\"values\":{\"values\":[{\"id\":\"202007291425\",\"value\":2},{\"id\":\"202007291426\",\"value\":8}]}}]"
+            },
+            {
+                Collections.emptyList(),
+                asList("202007291425", "202007291426"),
+                of("202007291425", new DataTable("200,1|400,2"), "202007291426", new DataTable("200,3|400,8")),
+                "[{\"label\":\"200\",\"values\":{\"values\":[{\"id\":\"202007291425\",\"value\":1},{\"id\":\"202007291426\",\"value\":3}]}}," +
+                    "{\"label\":\"400\",\"values\":{\"values\":[{\"id\":\"202007291425\",\"value\":2},{\"id\":\"202007291426\",\"value\":8}]}}]"
+            },
+            {
+                Collections.singletonList("200"),
+                asList("202007291425", "202007291426"),
+                of("202007291425", new DataTable("200,1|400,2"), "202007291426", new DataTable("200,3|400,8")),
+                "[{\"label\":\"200\",\"values\":{\"values\":[{\"id\":\"202007291425\",\"value\":1},{\"id\":\"202007291426\",\"value\":3}]}}]"
+            },
+            {
+                asList("200", "400", "500"),
+                asList("202007291425", "202007291426"),
+                of("202007291425", new DataTable("200,1|400,2"), "202007291426", new DataTable("200,3|400,8")),
+                "[{\"label\":\"200\",\"values\":{\"values\":[{\"id\":\"202007291425\",\"value\":1},{\"id\":\"202007291426\",\"value\":3}]}}," +
+                    "{\"label\":\"400\",\"values\":{\"values\":[{\"id\":\"202007291425\",\"value\":2},{\"id\":\"202007291426\",\"value\":8}]}}," +
+                    "{\"label\":\"500\",\"values\":{\"values\":[{\"id\":\"202007291425\",\"value\":" + DEFAULT_VALUE + "},{\"id\":\"202007291426\",\"value\":" + DEFAULT_VALUE + "}]}}]"
+            },
+            {
+                asList("200", "400"),
+                asList("202007291425", "202007291426"),
+                of("202007291425", new DataTable("200,1|400,2")),
+                "[{\"label\":\"200\",\"values\":{\"values\":[{\"id\":\"202007291425\",\"value\":1},{\"id\":\"202007291426\",\"value\":" + DEFAULT_VALUE + "}]}}," +
+                    "{\"label\":\"400\",\"values\":{\"values\":[{\"id\":\"202007291425\",\"value\":2},{\"id\":\"202007291426\",\"value\":" + DEFAULT_VALUE + "}]}}]"
+            },
+        });
+    }
+
+    @Before
+    public void setup() {
+        ValueColumnMetadata.INSTANCE.putIfAbsent(
+            MODULE_NAME, "value", LABELED_VALUE, Function.None, DEFAULT_VALUE
+        );
+    }
+
+    @Test
+    public void testComposeLabelValue() {
+        MetricsCondition condition = new MetricsCondition();
+        condition.setName(MODULE_NAME);
+        List<MetricsValues> result = IMetricsQueryDAO.Util.composeLabelValue(condition, queryConditionLabels, datePoints, valueColumnData);
+        assertThat(new Gson().toJson(result), is(expectedResult));
+    }
+
+}
\ No newline at end of file

oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/MetricsQueryEsDAO.java

@@ -138,42 +138,12 @@ public class MetricsQueryEsDAO extends EsDAO implements IMetricsQueryDAO {
         });
 
         SearchResponse response = getClient().ids(condition.getName(), ids.toArray(new String[0]));
-        Map<String, Map<String, Object>> idMap = toMap(response);
-
-        Map<String, MetricsValues> labeledValues = new HashMap<>(labels.size());
-        labels.forEach(label -> {
-            MetricsValues labelValue = new MetricsValues();
-            labelValue.setLabel(label);
-
-            labeledValues.put(label, labelValue);
-        });
-
-        final int defaultValue = ValueColumnMetadata.INSTANCE.getDefaultValue(condition.getName());
-        for (String id : ids) {
-            if (idMap.containsKey(id)) {
-                Map<String, Object> source = idMap.get(id);
-                DataTable multipleValues = new DataTable((String) source.getOrDefault(valueColumnName, ""));
-
-                labels.forEach(label -> {
-                    final IntValues values = labeledValues.get(label).getValues();
-                    Long data = multipleValues.get(label);
-                    if (data == null) {
-                        data = (long) defaultValue;
-                    }
-                    KVInt kv = new KVInt();
-                    kv.setId(id);
-                    kv.setValue(data);
-                    values.addKVInt(kv);
-                });
-            }
-
+        Map<String, DataTable> idMap = new HashMap<>();
+        SearchHit[] hits = response.getHits().getHits();
+        for (SearchHit hit : hits) {
+            idMap.put(hit.getId(), new DataTable((String) hit.getSourceAsMap().getOrDefault(valueColumnName, "")));
         }
-
-        return Util.sortValues(
-            new ArrayList<>(labeledValues.values()),
-            ids,
-            defaultValue
-        );
+        return Util.composeLabelValue(condition, labels, ids, idMap);
     }
 
     @Override

oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/MetricsQuery.java

@@ -175,40 +175,16 @@ public class MetricsQuery implements IMetricsQueryDAO {
             log.debug("SQL: {} result set: {}", query.getCommand(), series);
         }
 
-        Map<String, MetricsValues> labeledValues = new HashMap<>(labels.size());
-        labels.forEach(label -> {
-            MetricsValues labelValue = new MetricsValues();
-            labelValue.setLabel(label);
-
-            labeledValues.put(label, labelValue);
-        });
-
-        final int defaultValue = ValueColumnMetadata.INSTANCE.getDefaultValue(condition.getName());
+        Map<String, DataTable> idMap = new HashMap<>();
         if (!CollectionUtils.isEmpty(series)) {
             series.get(0).getValues().forEach(values -> {
                 final String id = (String) values.get(1);
                 DataTable multipleValues = new DataTable(5);
                 multipleValues.toObject((String) values.get(2));
-
-                labels.forEach(label -> {
-                    Long data = multipleValues.get(label);
-                    if (data == null) {
-                        data = (long) defaultValue;
-                    }
-                    final IntValues intValues = labeledValues.get(label).getValues();
-                    KVInt kv = new KVInt();
-                    kv.setId(id);
-                    kv.setValue(data);
-                    intValues.addKVInt(kv);
-                });
+                idMap.put(id, multipleValues);
             });
         }
-
-        return Util.sortValues(
-            new ArrayList<>(labeledValues.values()),
-            ids,
-            defaultValue
-        );
+        return Util.composeLabelValue(condition, labels, ids, idMap);
     }
 
     @Override

oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2MetricsQueryDAO.java

@@ -162,16 +162,7 @@ public class H2MetricsQueryDAO extends H2SQLExecutor implements IMetricsQueryDAO
         }
         sql.append(")");
 
-        Map<String, MetricsValues> labeledValues = new HashMap<>(labels.size());
-        labels.forEach(label -> {
-            MetricsValues labelValue = new MetricsValues();
-            labelValue.setLabel(label);
-
-            labeledValues.put(label, labelValue);
-        });
-
-        final int defaultValue = ValueColumnMetadata.INSTANCE.getDefaultValue(condition.getName());
-
+        Map<String, DataTable> idMap = new HashMap<>();
         try (Connection connection = h2Client.getConnection()) {
             try (ResultSet resultSet = h2Client.executeQuery(
                 connection, sql.toString(), parameters.toArray(new Object[0]))) {
@@ -181,28 +172,13 @@ public class H2MetricsQueryDAO extends H2SQLExecutor implements IMetricsQueryDAO
                     DataTable multipleValues = new DataTable(5);
                     multipleValues.toObject(resultSet.getString(valueColumnName));
 
-                    labels.forEach(label -> {
-                        Long data = multipleValues.get(label);
-                        if (data == null) {
-                            data = (long) defaultValue;
-                        }
-                        final IntValues values = labeledValues.get(label).getValues();
-                        KVInt kv = new KVInt();
-                        kv.setId(id);
-                        kv.setValue(data);
-                        values.addKVInt(kv);
-                    });
+                   idMap.put(id, multipleValues);
                 }
             }
         } catch (SQLException e) {
             throw new IOException(e);
         }
-
-        return Util.sortValues(
-            new ArrayList<>(labeledValues.values()),
-            ids,
-            defaultValue
-        );
+        return Util.composeLabelValue(condition, labels, ids, idMap);
     }
 
     @Override