Fix tags store of log and trace on h2/mysql/pg storage (#6505)

CHANGES.md

@@ -46,6 +46,7 @@ Release Notes.
 * Fix wrong `service_instance_sla` setting in the `topology-instance.yml`.
 * Fix wrong metrics name setting in the `self-observability.yml`.
 * Add telemetry data about metrics in, metrics scraping and trace in metrics to zipkin receiver.
+* Fix tags store of log and trace on h2/mysql/pg storage.
 
 #### UI
 * Update selector scroller to show in all pages.

oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/AbstractSearchTagBuilder.java

@@ -50,8 +50,8 @@ public abstract class AbstractSearchTagBuilder<T extends Record> implements Stor
             int tagInx = 0;
             final String tagExpression = tag.toString();
             for (int i = 0; i < numOfSearchableValuesPerTag; i++) {
-                tagInx = index + numOfSearchableValuesPerTag + i;
-                final String previousValue = (String) dbMap.get(tagColumn);
+                tagInx = index * numOfSearchableValuesPerTag + i;
+                final String previousValue = (String) dbMap.get(tagColumn + "_" + tagInx);
                 if (previousValue == null) {
                     // Still have at least one available slot, add directly.
                     shouldAdd = true;

test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/SimpleQueryClient.java

@@ -18,6 +18,7 @@
 
 package org.apache.skywalking.e2e;
 
+import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.io.Resources;
 import java.net.URI;
 import java.net.URL;
@@ -75,6 +76,7 @@ import org.springframework.web.client.RestTemplate;
 @Slf4j
 public class SimpleQueryClient {
     protected final RestTemplate restTemplate = new RestTemplate();
+    protected final ObjectMapper objectMapper = new ObjectMapper();
 
     protected final String endpointUrl;
 
@@ -99,7 +101,8 @@ public class SimpleQueryClient {
                                             .replace("{pageNum}", query.pageNum())
                                             .replace("{pageSize}", query.pageSize())
                                             .replace("{needTotal}", query.needTotal())
-                                            .replace("{queryOrder}", query.queryOrder());
+                                            .replace("{queryOrder}", query.queryOrder())
+                                            .replace("{tags}", objectMapper.writeValueAsString(query.tags()));
         final ResponseEntity<GQLResponse<TracesData>> responseEntity = restTemplate.exchange(
             new RequestEntity<>(queryString, HttpMethod.POST, URI.create(endpointUrl)),
             new ParameterizedTypeReference<GQLResponse<TracesData>>() {
@@ -415,7 +418,8 @@ public class SimpleQueryClient {
                                             .replace("{needTotal}", query.needTotal())
                                             .replace("{keywordsOfContent}", query.keywordsOfContent())
                                             .replace(
-                                                "{excludingKeywordsOfContent}", query.excludingKeywordsOfContent());
+                                                "{excludingKeywordsOfContent}", query.excludingKeywordsOfContent())
+                                            .replace("{tags}", objectMapper.writeValueAsString(query.tags()));
         LOGGER.info("Query: {}", queryString);
         final ResponseEntity<GQLResponse<LogData>> responseEntity = restTemplate.exchange(
             new RequestEntity<>(queryString, HttpMethod.POST, URI.create(endpointUrl)),

test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/log/LogsQuery.java

@@ -19,6 +19,12 @@ package org.apache.skywalking.e2e.log;
 
 import org.apache.skywalking.e2e.AbstractQuery;
 
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
 public class LogsQuery extends AbstractQuery<LogsQuery> {
 
     private String serviceId;
@@ -29,6 +35,7 @@ public class LogsQuery extends AbstractQuery<LogsQuery> {
     private String needTotal = "true";
     private String keywordsOfContent = "";
     private String excludingKeywordsOfContent = "";
+    private List<Map<String, String>> tags = Collections.emptyList();
 
     public String serviceId() {
         return serviceId;
@@ -109,4 +116,24 @@ public class LogsQuery extends AbstractQuery<LogsQuery> {
         }
         return String.join(",", keywords);
     }
+
+    public List<Map<String, String>> tags() {
+        return tags;
+    }
+
+    public LogsQuery tags(List<Map<String, String>> tags) {
+        this.tags = tags;
+        return this;
+    }
+
+    public LogsQuery addTag(String key, String value) {
+        if (Collections.EMPTY_LIST.equals(tags)) {
+            tags = new ArrayList<>();
+        }
+        Map<String, String> tag = new HashMap<>();
+        tag.put("key", key);
+        tag.put("value", value);
+        tags.add(tag);
+        return this;
+    }
 }

test/e2e/e2e-data/src/main/java/org/apache/skywalking/e2e/trace/TracesQuery.java

@@ -20,12 +20,19 @@ package org.apache.skywalking.e2e.trace;
 
 import org.apache.skywalking.e2e.AbstractQuery;
 
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
 public class TracesQuery extends AbstractQuery<TracesQuery> {
     private String traceState = "ALL";
     private String pageNum = "1";
     private String pageSize = "15";
     private String needTotal = "true";
     private String queryOrder = "BY_DURATION";
+    private List<Map<String, String>> tags = Collections.emptyList();
 
     public String traceState() {
         return traceState;
@@ -59,6 +66,11 @@ public class TracesQuery extends AbstractQuery<TracesQuery> {
         return this;
     }
 
+    public TracesQuery pageSize(int pageSize) {
+        this.pageSize = String.valueOf(pageSize);
+        return this;
+    }
+
     public String needTotal() {
         return needTotal;
     }
@@ -87,8 +99,23 @@ public class TracesQuery extends AbstractQuery<TracesQuery> {
         return this;
     }
 
-    public TracesQuery pageSize(int pageSize) {
-        this.pageSize = String.valueOf(pageSize);
+    public List<Map<String, String>> tags() {
+        return tags;
+    }
+
+    public TracesQuery tags(List<Map<String, String>> tags) {
+        this.tags = tags;
+        return this;
+    }
+
+    public TracesQuery addTag(String key, String value) {
+        if (Collections.EMPTY_LIST.equals(tags)) {
+            tags = new ArrayList<>();
+        }
+        Map<String, String> tag = new HashMap<>();
+        tag.put("key", key);
+        tag.put("value", value);
+        tags.add(tag);
         return this;
     }
 }

test/e2e/e2e-data/src/main/resources/logs.gql

@@ -53,7 +53,8 @@
                 "pageNum": {pageNum},
                 "pageSize": {pageSize},
                 "needTotal": {needTotal}
-            }
+            },
+            "tags": {tags}
         }
     }
 }

test/e2e/e2e-data/src/main/resources/traces.gql

@@ -35,7 +35,8 @@
                 "pageSize": {pageSize},
                 "needTotal": {needTotal}
             },
-            "queryOrder": "{queryOrder}"
+            "queryOrder": "{queryOrder}",
+            "tags": {tags}
         }
     }
 }

test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/log/LogE2E.java

@@ -94,7 +94,8 @@ public class LogE2E extends SkyWalkingTestAdapter {
     public void verifyLog4jLog() throws Exception {
         LogsQuery logsQuery = new LogsQuery().serviceId("WW91cl9BcHBsaWNhdGlvbk5hbWU=.1")
                                              .start(startTime)
-                                             .end(Times.now());
+                                             .end(Times.now())
+                                             .addTag("level", "INFO");
         if (graphql.supportQueryLogsByKeywords()) {
             logsQuery.keywordsOfContent("log4j message");
         }
@@ -108,7 +109,8 @@ public class LogE2E extends SkyWalkingTestAdapter {
     public void verifyLog4j2Log() throws Exception {
         LogsQuery logsQuery = new LogsQuery().serviceId("WW91cl9BcHBsaWNhdGlvbk5hbWU=.1")
                                              .start(startTime)
-                                             .end(Times.now());
+                                             .end(Times.now())
+                                             .addTag("level", "INFO");
         if (graphql.supportQueryLogsByKeywords()) {
             logsQuery.keywordsOfContent("log4j2 message");
         }
@@ -122,7 +124,8 @@ public class LogE2E extends SkyWalkingTestAdapter {
     public void verifyLogbackLog() throws Exception {
         LogsQuery logsQuery = new LogsQuery().serviceId("WW91cl9BcHBsaWNhdGlvbk5hbWU=.1")
                                              .start(startTime)
-                                             .end(Times.now());
+                                             .end(Times.now())
+                                             .addTag("level", "INFO");
         if (graphql.supportQueryLogsByKeywords()) {
             logsQuery.keywordsOfContent("logback message");
         }

test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/storage/StorageE2E.java

@@ -147,7 +147,11 @@ public class StorageE2E extends SkyWalkingTestAdapter {
 
     @RetryableTest
     void traces() throws Exception {
-        final List<Trace> traces = graphql.traces(new TracesQuery().start(startTime).end(now()).orderByDuration());
+        final TracesQuery query = new TracesQuery().start(startTime)
+                                                   .end(now())
+                                                   .orderByDuration()
+                                                   .addTag("http.method", "POST");
+        final List<Trace> traces = graphql.traces(query);
 
         LOGGER.info("traces: {}", traces);