Apply tags from Correlation Context key/values to Spans (#5685)

Co-authored-by: 吴晟 Wu Sheng <wu.sheng@foxmail.com>
Co-authored-by: Nkezhenxu94 <kezhenxu94@163.com>

.github/workflows/plugins-test.0.yaml

@@ -36,6 +36,7 @@ jobs:
           - canal-scenario
           - cassandra-java-driver-3.x-scenario
           - customize-scenario
+          - correlation-autotag-scenario
           - dubbo-2.5.x-scenario
           - dubbo-2.7.x-scenario
           - ehcache-2.x-scenario

CHANGES.md

@@ -7,16 +7,19 @@ Release Notes.
 #### Project
 
 #### Java Agent
+* Support propagate the sending timestamp in MQ plugins to calculate the transfer latency in the async MQ scenarios.
+* Support auto-tag with the fixed values propagated in the correlation context.   
 * Make HttpClient 3.x, 4.x, and HttpAsyncClient 3.x plugins to support collecting HTTP parameters.
 * Make the Feign plugin to support Java 14
 * Make the okhttp3 plugin to support Java 14
 
 #### OAP-Backend
 * Add the `@SuperDataset` annotation for BrowserErrorLog.
+* Add the thread pool to the Kafka fetcher to increase the performance.
+* Add `contain` and `not contain` OPS in OAL.
 * Support keeping collecting the slowly segments in the sampling mechanism.
 * Support choose files to active the meter analyzer.
 * Improve Kubernetes service registry for ALS analysis.
-* Add the thread pool to the Kafka fetcher to increase the performance.
 
 #### UI
 

apm-sniffer/apm-agent-core/pom.xml

@@ -68,7 +68,6 @@
         <dependency>
             <groupId>net.bytebuddy</groupId>
             <artifactId>byte-buddy</artifactId>
-            <version>${bytebuddy.version}</version>
         </dependency>
         <dependency>
             <groupId>com.google.code.gson</groupId>
@@ -87,6 +86,20 @@
             <artifactId>wiremock</artifactId>
             <version>${wiremock.version}</version>
             <scope>test</scope>
+            <exclusions>
+                <exclusion>
+                    <artifactId>jackson-annotations</artifactId>
+                    <groupId>com.fasterxml.jackson.core</groupId>
+                </exclusion>
+                <exclusion>
+                    <artifactId>jackson-core</artifactId>
+                    <groupId>com.fasterxml.jackson.core</groupId>
+                </exclusion>
+                <exclusion>
+                    <artifactId>jackson-databind</artifactId>
+                    <groupId>com.fasterxml.jackson.core</groupId>
+                </exclusion>
+            </exclusions>
         </dependency>
         <dependency>
             <groupId>io.grpc</groupId>
@@ -119,7 +132,6 @@
         <dependency>
             <groupId>org.slf4j</groupId>
             <artifactId>slf4j-api</artifactId>
-            <version>${slf4j.version}</version>
         </dependency>
     </dependencies>
     <dependencyManagement>
@@ -129,6 +141,16 @@
                 <artifactId>guava</artifactId>
                 <version>${guava.version}</version>
             </dependency>
+            <dependency>
+                <groupId>net.bytebuddy</groupId>
+                <artifactId>byte-buddy</artifactId>
+                <version>${bytebuddy.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>org.slf4j</groupId>
+                <artifactId>slf4j-api</artifactId>
+                <version>${slf4j.version}</version>
+            </dependency>
         </dependencies>
     </dependencyManagement>
     <build>

apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/conf/Config.java

@@ -314,5 +314,10 @@ public class Config {
          * Max value length of each element.
          */
         public static int VALUE_MAX_LENGTH = 128;
+
+        /**
+         * Tag the span by the key/value in the correlation context, when the keys listed here exist.
+         */
+        public static String AUTO_TAG_KEYS = "";
     }
 }

apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/context/ContextCarrier.java

@@ -94,6 +94,14 @@ public class ContextCarrier implements Serializable {
      */
     void extractExtensionTo(AbstractSpan span) {
         this.extensionContext.handle(span);
+
+    }
+
+    /**
+     * Extract the correlation context to the given span
+     */
+    void extractCorrelationTo(AbstractSpan span) {
+        this.correlationContext.handle(span);
     }
 
     /**

apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/context/CorrelationContext.java

@@ -17,13 +17,18 @@
 
 package org.apache.skywalking.apm.agent.core.context;
 
+import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.stream.Collectors;
 import org.apache.skywalking.apm.agent.core.base64.Base64;
 import org.apache.skywalking.apm.agent.core.conf.Config;
+import org.apache.skywalking.apm.agent.core.context.tag.StringTag;
+import org.apache.skywalking.apm.agent.core.context.trace.AbstractSpan;
 import org.apache.skywalking.apm.util.StringUtil;
 
 /**
@@ -33,6 +38,16 @@ public class CorrelationContext {
 
     private final Map<String, String> data;
 
+    private static final List<String> AUTO_TAG_KEYS;
+
+    static {
+        if (StringUtil.isNotEmpty(Config.Correlation.AUTO_TAG_KEYS)) {
+            AUTO_TAG_KEYS = Arrays.asList(Config.Correlation.AUTO_TAG_KEYS.split(","));
+        } else {
+            AUTO_TAG_KEYS = new ArrayList<>();
+        }
+    }
+
     public CorrelationContext() {
         this.data = new HashMap<>(Config.Correlation.ELEMENT_MAX_NUMBER);
     }
@@ -70,7 +85,9 @@ public class CorrelationContext {
         if (data.size() >= Config.Correlation.ELEMENT_MAX_NUMBER) {
             return Optional.empty();
         }
-
+        if (AUTO_TAG_KEYS.contains(key) && ContextManager.isActive()) {
+            ContextManager.activeSpan().tag(new StringTag(key), value);
+        }
         // setting
         data.put(key, value);
         return Optional.empty();
@@ -160,6 +177,10 @@ public class CorrelationContext {
         this.data.putAll(snapshot.getCorrelationContext().data);
     }
 
+    void handle(AbstractSpan span) {
+        AUTO_TAG_KEYS.forEach(key -> this.get(key).ifPresent(val -> span.tag(new StringTag(key), val)));
+    }
+
     @Override
     public boolean equals(Object o) {
         if (this == o)

apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/context/TracingContext.java

@@ -188,6 +188,7 @@ public class TracingContext implements AbstractTracerContext {
         }
 
         carrier.extractExtensionTo(span);
+        carrier.extractCorrelationTo(span);
         this.correlationContext.extract(carrier);
         this.extensionContext.extract(carrier);
     }

apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/context/tag/AbstractTag.java

@@ -18,6 +18,7 @@
 
 package org.apache.skywalking.apm.agent.core.context.tag;
 
+import java.util.Objects;
 import org.apache.skywalking.apm.agent.core.context.trace.AbstractSpan;
 
 public abstract class AbstractTag<T> {
@@ -60,4 +61,21 @@ public abstract class AbstractTag<T> {
     public boolean isCanOverwrite() {
         return canOverwrite;
     }
+
+    @Override
+    public boolean equals(final Object o) {
+        if (this == o)
+            return true;
+        if (!(o instanceof AbstractTag))
+            return false;
+        final AbstractTag<?> that = (AbstractTag<?>) o;
+        return getId() == that.getId() &&
+            isCanOverwrite() == that.isCanOverwrite() &&
+            key.equals(that.key);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(getId(), isCanOverwrite(), key);
+    }
 }

apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/context/trace/AbstractSpan.java

@@ -124,5 +124,4 @@ public interface AbstractSpan extends AsyncSpan {
      * Should skip analysis in the backend.
      */
     void skipAnalysis();
-
 }

apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/context/util/TagValuePair.java

@@ -17,6 +17,7 @@
 
 package org.apache.skywalking.apm.agent.core.context.util;
 
+import java.util.Objects;
 import org.apache.skywalking.apm.agent.core.context.tag.AbstractTag;
 import org.apache.skywalking.apm.network.common.v3.KeyStringValuePair;
 
@@ -53,4 +54,20 @@ public class TagValuePair {
     public void setValue(String value) {
         this.value = value;
     }
+
+    @Override
+    public boolean equals(final Object o) {
+        if (this == o)
+            return true;
+        if (!(o instanceof TagValuePair))
+            return false;
+        final TagValuePair that = (TagValuePair) o;
+        return Objects.equals(getKey(), that.getKey()) &&
+            Objects.equals(getValue(), that.getValue());
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(getKey(), getValue());
+    }
 }
\ No newline at end of file

apm-sniffer/apm-agent-core/src/test/java/org/apache/skywalking/apm/agent/core/context/CorrelationContextTest.java

@@ -120,5 +120,4 @@ public class CorrelationContextTest {
         context.deserialize(null);
         Assert.assertNull(context.get("test1").orElse(null));
     }
-
 }

docs/en/setup/service-agent/java-agent/README.md

@@ -103,6 +103,9 @@ property key | Description | Default |
 `logging.max_history_files`|The max history log files. When rollover happened, if log files exceed this number,then the oldest file will be delete. Negative or zero means off, by default.|`-1`|
 `statuscheck.ignored_exceptions`|Listed exceptions would not be treated as an error. Because in some codes, the exception is being used as a way of controlling business flow.|`""`|
 `statuscheck.max_recursive_depth`|The max recursive depth when checking the exception traced by the agent. Typically, we don't recommend setting this more than 10, which could cause a performance issue. Negative value and 0 would be ignored, which means all exceptions would make the span tagged in error status.|`1`|
+`correlation.element_max_number`|Max element count in the correlation context.|3|
+`correlation.value_max_length`|Max value length of each element.|`128`|
+`correlation.auto_tag_keys`|Tag the span by the key/value in the correlation context, when the keys listed here exist.|`""`|
 `jvm.buffer_size`|The buffer size of collected JVM info.|`60 * 10`|
 `buffer.channel_size`|The buffer channel size.|`5`|
 `buffer.buffer_size`|The buffer size.|`300`|
@@ -141,8 +144,6 @@ property key | Description | Default |
 `plugin.feign.filter_length_limit`| When `COLLECT_REQUEST_BODY` is enabled, how many characters to keep and send to the OAP backend, use negative values to keep and send the complete body.  | `1024` |
 `plugin.feign.supported_content_types_prefix`| When `COLLECT_REQUEST_BODY` is enabled and content-type start with SUPPORTED_CONTENT_TYPES_PREFIX, collect the body of the request , multiple paths should be separated by `,`  | `application/json,text/` |
 `plugin.influxdb.trace_influxql`|If true, trace all the influxql(query and write) in InfluxDB access, default is true.|`true`|
-`correlation.element_max_number`|Max element count of the correlation context.|`3`|
-`correlation.value_max_length`|Max value length of correlation context element.|`128`|
 `plugin.dubbo.collect_consumer_arguments`| Apache Dubbo consumer collect `arguments` in RPC call, use `Object#toString` to collect `arguments`. |`false`| 
 `plugin.dubbo.consumer_arguments_length_threshold`| When `plugin.dubbo.collect_consumer_arguments` is `true`, Arguments of length from the front will to the OAP backend |`256`| 
 `plugin.dubbo.collect_provider_arguments`| Apache Dubbo provider collect `arguments` in RPC call, use `Object#toString` to collect `arguments`. |`false`| 

test/plugin/pom.xml

@@ -15,11 +15,7 @@
   ~ See the License for the specific language governing permissions and
   ~ limitations under the License.
   ~
-  -->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0"
-         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  --><project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
 
     <modelVersion>4.0.0</modelVersion>
 
@@ -34,7 +30,7 @@
         <module>runner-helper</module>
         <module>agent-test-tools</module>
         <module>containers</module>
-    </modules>
+  </modules>
 
     <properties>
         <java.version>8</java.version>
@@ -106,4 +102,4 @@
         </plugins>
     </build>
 
-</project>
+</project>
\ No newline at end of file

test/plugin/scenarios/correlation-autotag-scenario/bin/startup.sh

+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+home="$(cd "$(dirname $0)"; pwd)"
+
+java -jar ${agent_opts} "-Dskywalking.correlation.auto_tag_keys=autotag1,autotag2" ${home}/../libs/correlation-autotag-scenario.jar &
\ No newline at end of file

test/plugin/scenarios/correlation-autotag-scenario/config/expectedData.yaml

+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+segmentItems:
+  - serviceName: correlation-autotag-scenario
+    segmentSize: 12
+    segments:
+      - segmentId: not null
+        spans:
+          - operationName: /correlation-autotag-scenario/case/healthCheck
+            operationId: 0
+            parentSpanId: -1
+            spanId: 0
+            spanLayer: Http
+            startTime: gt 0
+            endTime: gt 0
+            componentId: 1
+            isError: false
+            spanType: Entry
+            peer: ''
+            skipAnalysis: false
+            tags:
+              - {key: url, value: 'http://localhost:8080/correlation-autotag-scenario/case/healthCheck'}
+              - {key: http.method, value: HEAD}
+      - segmentId: 78a0cdef42a74b4ba80b48bf72b8558d.37.16042096947740000
+        spans:
+          - {operationName: Greeter.sayHello, operationId: 0, parentSpanId: 0, spanId: 1,
+             spanLayer: RPCFramework, startTime: gt 0, endTime: gt 0, componentId: 23,
+             isError: false, spanType: Exit, peer: '127.0.0.1:18080', skipAnalysis: false}
+          - operationName: /correlation-autotag-scenario/case/correlation-autotag-scenario
+            operationId: 0
+            parentSpanId: -1
+            spanId: 0
+            spanLayer: Http
+            startTime: gt 0
+            endTime: gt 0
+            componentId: 1
+            isError: false
+            spanType: Entry
+            peer: ''
+            skipAnalysis: false
+            tags:
+              - {key: url, value: 'http://localhost:8080/correlation-autotag-scenario/case/correlation-autotag-scenario'}
+              - {key: http.method, value: GET}
+              - {key: autotag1, value: '1'}
+              - {key: autotag2, value: '1'}
+      - segmentId: not null
+        spans:
+          - operationName: Greeter.sayHello
+            operationId: 0
+            parentSpanId: -1
+            spanId: 0
+            spanLayer: RPCFramework
+            startTime: gt 0
+            endTime: gt 0
+            componentId: 23
+            isError: false
+            spanType: Entry
+            peer: ''
+            skipAnalysis: false
+            tags:
+              - {key: autotag1, value: '1'}
+              - {key: autotag2, value: '1'}
+            refs:
+              - {parentEndpoint: /correlation-autotag-scenario/case/correlation-autotag-scenario,
+                 networkAddress: '127.0.0.1:18080', refType: CrossProcess, parentSpanId: 1,
+                 parentTraceSegmentId: not null,
+                 parentServiceInstance: not null, parentService: correlation-autotag-scenario,
+                 traceId: not null}
+      - segmentId: not null
+        spans:
+          - operationName: Greeter.sayHello/client/Request/onComplete
+            operationId: 0
+            parentSpanId: -1
+            spanId: 0
+            spanLayer: RPCFramework
+            startTime: gt 0
+            endTime: gt 0
+            componentId: 23
+            isError: false
+            spanType: Local
+            peer: ''
+            skipAnalysis: false
+            refs:
+              - {parentEndpoint: /correlation-autotag-scenario/case/correlation-autotag-scenario,
+                 networkAddress: '', refType: CrossThread, parentSpanId: 1, parentTraceSegmentId: not null,
+                 parentServiceInstance: not null, parentService: correlation-autotag-scenario,
+                 traceId: not null}
+      - segmentId: not null
+        spans:
+          - operationName: Greeter.sayHello/client/Request/onMessage
+            operationId: 0
+            parentSpanId: -1
+            spanId: 0
+            spanLayer: RPCFramework
+            startTime: gt 0
+            endTime: gt 0
+            componentId: 23
+            isError: false
+            spanType: Local
+            peer: ''
+            skipAnalysis: false
+            refs:
+              - {parentEndpoint: /correlation-autotag-scenario/case/correlation-autotag-scenario,
+                 networkAddress: '', refType: CrossThread, parentSpanId: 1, parentTraceSegmentId: not null,
+                 parentServiceInstance: not null, parentService: correlation-autotag-scenario,
+                 traceId: not null}
+      - segmentId: not null
+        spans:
+          - operationName: Greeter.sayHello/client/Request/onMessage
+            operationId: 0
+            parentSpanId: -1
+            spanId: 0
+            spanLayer: RPCFramework
+            startTime: gt 0
+            endTime: gt 0
+            componentId: 23
+            isError: false
+            spanType: Local
+            peer: ''
+            skipAnalysis: false
+            refs:
+              - {parentEndpoint: /correlation-autotag-scenario/case/correlation-autotag-scenario,
+                 networkAddress: '', refType: CrossThread, parentSpanId: 1, parentTraceSegmentId: not null,
+                 parentServiceInstance: not null, parentService: correlation-autotag-scenario,
+                 traceId: not null}
+      - segmentId: not null
+        spans:
+          - operationName: Greeter.sayHello/server/Response/onMessage
+            operationId: 0
+            parentSpanId: 0
+            spanId: 1
+            spanLayer: RPCFramework
+            startTime: gt 0
+            endTime: gt 0
+            componentId: 23
+            isError: false
+            spanType: Local
+            peer: ''
+            skipAnalysis: false
+            refs:
+              - {parentEndpoint: Greeter.sayHello, networkAddress: '', refType: CrossThread,
+                 parentSpanId: 0, parentTraceSegmentId: not null,
+                 parentServiceInstance: not null, parentService: correlation-autotag-scenario,
+                 traceId: not null}
+          - operationName: Greeter.sayHello/server/Request/onMessage
+            operationId: 0
+            parentSpanId: -1
+            spanId: 0
+            spanLayer: RPCFramework
+            startTime: gt 0
+            endTime: gt 0
+            componentId: 23
+            isError: false
+            spanType: Local
+            peer: ''
+            skipAnalysis: false
+            refs:
+              - {parentEndpoint: Greeter.sayHello, networkAddress: '', refType: CrossThread,
+                 parentSpanId: 0, parentTraceSegmentId: not null,
+                 parentServiceInstance: not null, parentService: correlation-autotag-scenario,
+                 traceId: not null}
+      - segmentId: not null
+        spans:
+          - operationName: Greeter.sayHello/server/Response/onMessage
+            operationId: 0
+            parentSpanId: 0
+            spanId: 1
+            spanLayer: RPCFramework
+            startTime: gt 0
+            endTime: gt 0
+            componentId: 23
+            isError: false
+            spanType: Local
+            peer: ''
+            skipAnalysis: false
+            refs:
+              - {parentEndpoint: Greeter.sayHello, networkAddress: '', refType: CrossThread,
+                 parentSpanId: 0, parentTraceSegmentId: not null,
+                 parentServiceInstance: not null, parentService: correlation-autotag-scenario,
+                 traceId: not null}
+          - operationName: Greeter.sayHello/server/Request/onMessage
+            operationId: 0
+            parentSpanId: -1
+            spanId: 0
+            spanLayer: RPCFramework
+            startTime: gt 0
+            endTime: gt 0
+            componentId: 23
+            isError: false
+            spanType: Local
+            peer: ''
+            skipAnalysis: false
+            refs:
+              - {parentEndpoint: Greeter.sayHello, networkAddress: '', refType: CrossThread,
+                 parentSpanId: 0, parentTraceSegmentId: not null,
+                 parentServiceInstance: not null, parentService: correlation-autotag-scenario,
+                 traceId: not null}
+      - segmentId: not null
+        spans:
+          - operationName: Greeter.sayHello/server/Response/onClose
+            operationId: 0
+            parentSpanId: 0
+            spanId: 1
+            spanLayer: RPCFramework
+            startTime: gt 0
+            endTime: gt 0
+            componentId: 23
+            isError: false
+            spanType: Local
+            peer: ''
+            skipAnalysis: false
+            tags:
+              - {key: status_code, value: OK}
+            refs:
+              - {parentEndpoint: Greeter.sayHello, networkAddress: '', refType: CrossThread,
+                 parentSpanId: 0, parentTraceSegmentId: not null,
+                 parentServiceInstance: not null, parentService: correlation-autotag-scenario,
+                 traceId: not null}
+          - operationName: Greeter.sayHello/server/Request/onComplete
+            operationId: 0
+            parentSpanId: -1
+            spanId: 0
+            spanLayer: RPCFramework
+            startTime: gt 0
+            endTime: gt 0
+            componentId: 23
+            isError: false
+            spanType: Local
+            peer: ''
+            skipAnalysis: false
+            refs:
+              - {parentEndpoint: Greeter.sayHello, networkAddress: '', refType: CrossThread,
+                 parentSpanId: 0, parentTraceSegmentId: not null,
+                 parentServiceInstance: not null, parentService: correlation-autotag-scenario,
+                 traceId: not null}
+      - segmentId: not null
+        spans:
+          - operationName: Greeter.sayHello/client/Response/onMessage
+            operationId: 0
+            parentSpanId: -1
+            spanId: 0
+            spanLayer: RPCFramework
+            startTime: gt 0
+            endTime: gt 0
+            componentId: 23
+            isError: false
+            spanType: Local
+            peer: ''
+            skipAnalysis: false
+            refs:
+              - {parentEndpoint: /correlation-autotag-scenario/case/correlation-autotag-scenario,
+                 networkAddress: '', refType: CrossThread, parentSpanId: 1, parentTraceSegmentId: not null,
+                 parentServiceInstance: not null, parentService: correlation-autotag-scenario,
+                 traceId: not null}
+      - segmentId: not null
+        spans:
+          - operationName: Greeter.sayHello/client/Response/onClose
+            operationId: 0
+            parentSpanId: -1
+            spanId: 0
+            spanLayer: RPCFramework
+            startTime: gt 0
+            endTime: gt 0
+            componentId: 23
+            isError: false
+            spanType: Local
+            peer: ''
+            skipAnalysis: false
+            refs:
+              - {parentEndpoint: /correlation-autotag-scenario/case/correlation-autotag-scenario,
+                 networkAddress: '', refType: CrossThread, parentSpanId: 1, parentTraceSegmentId: not null,
+                 parentServiceInstance: not null, parentService: correlation-autotag-scenario,
+                 traceId: not null}
+      - segmentId: not null
+        spans:
+          - operationName: Greeter.sayHello/client/Response/onMessage
+            operationId: 0
+            parentSpanId: -1
+            spanId: 0
+            spanLayer: RPCFramework
+            startTime: gt 0
+            endTime: gt 0
+            componentId: 23
+            isError: false
+            spanType: Local
+            peer: ''
+            skipAnalysis: false
+            refs:
+              - {parentEndpoint: /correlation-autotag-scenario/case/correlation-autotag-scenario,
+                 networkAddress: '', refType: CrossThread, parentSpanId: 1, parentTraceSegmentId: not null,
+                 parentServiceInstance: not null, parentService: correlation-autotag-scenario,
+                 traceId: not null}
+meterItems: []
+

test/plugin/scenarios/correlation-autotag-scenario/configuration.yml

+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+type: jvm
+entryService: http://localhost:8080/correlation-autotag-scenario/case/correlation-autotag-scenario
+healthCheck: http://localhost:8080/correlation-autotag-scenario/case/healthCheck
+startScript: ./bin/startup.sh
+environment:
+dependencies:

test/plugin/scenarios/correlation-autotag-scenario/pom.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  ~
+  -->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+    <groupId>org.apache.skywalking.apm.testcase</groupId>
+    <artifactId>correlation-autotag-scenario</artifactId>
+    <version>1.0.0</version>
+    <packaging>jar</packaging>
+
+    <modelVersion>4.0.0</modelVersion>
+
+    <properties>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+        <compiler.version>1.8</compiler.version>
+        <os-maven-plugin.version>1.6.2</os-maven-plugin.version>
+
+        <test.framework.version>1.6.0</test.framework.version>
+
+        <spring-boot-version>2.1.6.RELEASE</spring-boot-version>
+    </properties>
+
+    <name>skywalking-correlation-autotag-scenario</name>
+
+    <dependencyManagement>
+        <dependencies>
+            <dependency>
+                <groupId>org.springframework.boot</groupId>
+                <artifactId>spring-boot-dependencies</artifactId>
+                <version>${spring-boot-version}</version>
+                <type>pom</type>
+                <scope>import</scope>
+            </dependency>
+        </dependencies>
+    </dependencyManagement>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-web</artifactId>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.springframework.boot</groupId>
+                    <artifactId>spring-boot-starter-logging</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-log4j2</artifactId>
+        </dependency>
+
+        <!--    grpc    -->
+        <dependency>
+            <groupId>io.grpc</groupId>
+            <artifactId>grpc-all</artifactId>
+            <version>${test.framework.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.skywalking</groupId>
+            <artifactId>apm-toolkit-trace</artifactId>
+            <version>8.2.0</version>
+            <scope>compile</scope>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <finalName>correlation-autotag-scenario</finalName>
+        <plugins>
+            <plugin>
+                <groupId>kr.motd.maven</groupId>
+                <artifactId>os-maven-plugin</artifactId>
+                <version>${os-maven-plugin.version}</version>
+                <executions>
+                    <execution>
+                        <phase>initialize</phase>
+                        <goals>
+                            <goal>detect</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.springframework.boot</groupId>
+                <artifactId>spring-boot-maven-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <goals>
+                            <goal>repackage</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <configuration>
+                    <source>${compiler.version}</source>
+                    <target>${compiler.version}</target>
+                    <encoding>${project.build.sourceEncoding}</encoding>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-assembly-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <id>assemble</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>single</goal>
+                        </goals>
+                        <configuration>
+                            <descriptors>
+                                <descriptor>src/main/assembly/assembly.xml</descriptor>
+                            </descriptors>
+                            <outputDirectory>./target/</outputDirectory>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.xolstice.maven.plugins</groupId>
+                <artifactId>protobuf-maven-plugin</artifactId>
+                <version>0.5.0</version>
+                <configuration>
+                    <!--
+                      The version of protoc must match protobuf-java. If you don't depend on
+                      protobuf-java directly, you will be transitively depending on the
+                      protobuf-java version that grpc depends on.
+                    -->
+                    <protocArtifact>com.google.protobuf:protoc:3.3.0:exe:${os.detected.classifier}
+                    </protocArtifact>
+                    <pluginId>grpc-java</pluginId>
+                    <pluginArtifact>io.grpc:protoc-gen-grpc-java:${test.framework.version}:exe:${os.detected.classifier}
+                    </pluginArtifact>
+                </configuration>
+                <executions>
+                    <execution>
+                        <goals>
+                            <goal>compile</goal>
+                            <goal>compile-custom</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+</project>

test/plugin/scenarios/correlation-autotag-scenario/src/main/assembly/assembly.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  ~
+  -->
+<assembly
+    xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
+    <formats>
+        <format>zip</format>
+    </formats>
+
+    <fileSets>
+        <fileSet>
+            <directory>./bin</directory>
+            <fileMode>0775</fileMode>
+        </fileSet>
+    </fileSets>
+
+    <files>
+        <file>
+            <source>${project.build.directory}/correlation-autotag-scenario.jar</source>
+            <outputDirectory>./libs</outputDirectory>
+            <fileMode>0775</fileMode>
+        </file>
+    </files>
+</assembly>

test/plugin/scenarios/correlation-autotag-scenario/src/main/java/org/apache/skywalking/apm/testcase/grpc/Application.java

+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.apm.testcase.grpc;
+
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+
+@SpringBootApplication
+public class Application {
+
+    public static void main(String[] args) {
+        try {
+            SpringApplication.run(Application.class, args);
+        } catch (Exception e) {
+            // Never do this
+        }
+    }
+}

test/plugin/scenarios/correlation-autotag-scenario/src/main/java/org/apache/skywalking/apm/testcase/grpc/consumr/ConsumerInterceptor.java

+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.apm.testcase.grpc.consumr;
+
+import io.grpc.CallOptions;
+import io.grpc.Channel;
+import io.grpc.ClientCall;
+import io.grpc.ClientInterceptor;
+import io.grpc.ForwardingClientCall;
+import io.grpc.ForwardingClientCallListener;
+import io.grpc.Metadata;
+import io.grpc.MethodDescriptor;
+import io.grpc.Status;
+import javax.annotation.Nullable;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+public class ConsumerInterceptor implements ClientInterceptor {
+
+    private static final Logger LOGGER = LogManager.getLogger(ConsumerInterceptor.class);
+
+    @Override
+    public <ReqT, RespT> ClientCall<ReqT, RespT> interceptCall(MethodDescriptor<ReqT, RespT> descriptor,
+        CallOptions options, Channel channel) {
+        LOGGER.info("start interceptor!");
+        LOGGER.info("method type: {}", descriptor.getType());
+        return new ForwardingClientCall.SimpleForwardingClientCall<ReqT, RespT>(channel.newCall(descriptor, options)) {
+            @Override
+            public void start(Listener<RespT> responseListener, Metadata headers) {
+                LOGGER.info("Peer: {}", channel.authority());
+                LOGGER.info("Operation Name : {}", descriptor.getFullMethodName());
+                Interceptor<RespT> tracingResponseListener = new Interceptor(responseListener);
+                tracingResponseListener.contextSnapshot = "contextSnapshot";
+                delegate().start(tracingResponseListener, headers);
+            }
+
+            @Override
+            public void cancel(@Nullable String message, @Nullable Throwable cause) {
+                LOGGER.info("cancel");
+                super.cancel(message, cause);
+            }
+
+            @Override
+            public void halfClose() {
+                LOGGER.info("halfClose");
+                super.halfClose();
+            }
+
+            @Override
+            public void sendMessage(ReqT message) {
+                LOGGER.info("sendMessage ....");
+                super.sendMessage(message);
+            }
+        };
+    }
+
+    private static class Interceptor<RespT> extends ForwardingClientCallListener.SimpleForwardingClientCallListener<RespT> {
+        private static final Logger LOGGER = LogManager.getLogger(Interceptor.class);
+
+        private Object contextSnapshot;
+
+        protected Interceptor(ClientCall.Listener<RespT> delegate) {
+            super(delegate);
+        }
+
+        @Override
+        public void onHeaders(Metadata headers) {
+            LOGGER.info("on Headers");
+            for (String key : headers.keys()) {
+                LOGGER.info("Receive key: {}", key);
+            }
+            delegate().onHeaders(headers);
+        }
+
+        @Override
+        public void onMessage(RespT message) {
+            LOGGER.info("contextSnapshot: {}", contextSnapshot);
+            delegate().onMessage(message);
+        }
+
+        @Override
+        public void onClose(Status status, Metadata trailers) {
+            LOGGER.info("on close");
+            delegate().onClose(status, trailers);
+        }
+
+        @Override
+        public void onReady() {
+            LOGGER.info("on Ready");
+            super.onReady();
+        }
+    }
+}

test/plugin/scenarios/correlation-autotag-scenario/src/main/java/org/apache/skywalking/apm/testcase/grpc/controller/CaseController.java

+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.apm.testcase.grpc.controller;
+
+import io.grpc.ClientInterceptors;
+import io.grpc.ManagedChannel;
+import io.grpc.ManagedChannelBuilder;
+import io.grpc.stub.ClientCallStreamObserver;
+import io.grpc.stub.ClientResponseObserver;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.List;
+import javax.annotation.PostConstruct;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.skywalking.apm.testcase.grpc.consumr.ConsumerInterceptor;
+import org.apache.skywalking.apm.testcase.grpc.proto.GreeterBlockingErrorGrpc;
+import org.apache.skywalking.apm.testcase.grpc.proto.GreeterBlockingGrpc;
+import org.apache.skywalking.apm.testcase.grpc.proto.GreeterGrpc;
+import org.apache.skywalking.apm.testcase.grpc.proto.HelloReply;
+import org.apache.skywalking.apm.testcase.grpc.proto.HelloRequest;
+import org.apache.skywalking.apm.toolkit.trace.TraceContext;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.ResponseBody;
+import org.springframework.web.bind.annotation.RestController;
+
+@RestController
+@RequestMapping("/case")
+public class CaseController {
+
+    private static final Logger LOGGER = LogManager.getLogger(CaseController.class);
+
+    private static final String SUCCESS = "Success";
+
+    private final String gprcProviderHost = "127.0.0.1";
+    private final int grpcProviderPort = 18080;
+    private ManagedChannel channel;
+    private GreeterGrpc.GreeterStub greeterStub;
+    private GreeterBlockingGrpc.GreeterBlockingBlockingStub greeterBlockingStub;
+    private GreeterBlockingErrorGrpc.GreeterBlockingErrorBlockingStub greeterBlockingErrorStub;
+
+    @PostConstruct
+    public void up() {
+        channel = ManagedChannelBuilder.forAddress(gprcProviderHost, grpcProviderPort).usePlaintext(true).build();
+        greeterStub = GreeterGrpc.newStub(ClientInterceptors.intercept(channel, new ConsumerInterceptor()));
+        greeterBlockingStub = GreeterBlockingGrpc.newBlockingStub(ClientInterceptors.intercept(channel, new ConsumerInterceptor()));
+        greeterBlockingErrorStub = GreeterBlockingErrorGrpc.newBlockingStub(ClientInterceptors.intercept(channel, new ConsumerInterceptor()));
+    }
+
+    @RequestMapping("/correlation-autotag-scenario")
+    @ResponseBody
+    public String testcase() {
+        TraceContext.putCorrelation("autotag1", "1");
+        TraceContext.putCorrelation("autotag2", "1");
+        greetService();
+        return SUCCESS;
+    }
+
+    @RequestMapping("/healthCheck")
+    @ResponseBody
+    public String healthCheck() {
+        // your codes
+        return SUCCESS;
+    }
+
+    private static List<String> names() {
+        return Arrays.asList("Sophia", "Jackson");
+    }
+
+    private void greetService() {
+        ClientResponseObserver<HelloRequest, HelloReply> helloReplyStreamObserver = new ClientResponseObserver<HelloRequest, HelloReply>() {
+            private ClientCallStreamObserver<HelloRequest> requestStream;
+
+            @Override
+            public void beforeStart(ClientCallStreamObserver observer) {
+                this.requestStream = observer;
+                this.requestStream.setOnReadyHandler(new Runnable() {
+                    Iterator<String> iterator = names().iterator();
+
+                    @Override
+                    public void run() {
+                        while (requestStream.isReady()) {
+                            if (iterator.hasNext()) {
+                                String name = iterator.next();
+                                HelloRequest request = HelloRequest.newBuilder().setName(name).build();
+                                requestStream.onNext(request);
+                            } else {
+                                requestStream.onCompleted();
+                            }
+                        }
+                    }
+                });
+            }
+
+            @Override
+            public void onNext(HelloReply reply) {
+                LOGGER.info("Receive an message from provider. message: {}", reply.getMessage());
+                requestStream.request(1);
+            }
+
+            public void onError(Throwable throwable) {
+                LOGGER.error("Failed to send data", throwable);
+            }
+
+            public void onCompleted() {
+                LOGGER.info("All Done");
+            }
+        };
+
+        greeterStub.sayHello(helloReplyStreamObserver);
+    }
+}

test/plugin/scenarios/correlation-autotag-scenario/src/main/java/org/apache/skywalking/apm/testcase/grpc/provider/ProviderConfiguration.java

+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.apm.testcase.grpc.provider;
+
+import io.grpc.Server;
+import io.grpc.ServerBuilder;
+import io.grpc.ServerInterceptors;
+import org.apache.skywalking.apm.testcase.grpc.provider.interceptor.ProviderInterceptor;
+import org.apache.skywalking.apm.testcase.grpc.provider.service.GreeterServiceImpl;
+import org.springframework.beans.factory.annotation.Configurable;
+import org.springframework.context.annotation.Bean;
+import org.springframework.stereotype.Component;
+
+@Configurable
+@Component
+public class ProviderConfiguration {
+
+    @Bean(initMethod = "start", destroyMethod = "shutdown")
+    public Server server() {
+        return ServerBuilder.forPort(18080)
+                            .addService(ServerInterceptors.intercept(new GreeterServiceImpl(), new ProviderInterceptor()))
+                            .build();
+    }
+}

test/plugin/scenarios/correlation-autotag-scenario/src/main/java/org/apache/skywalking/apm/testcase/grpc/provider/interceptor/ProviderInterceptor.java

+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.apm.testcase.grpc.provider.interceptor;
+
+import io.grpc.ForwardingServerCall;
+import io.grpc.ForwardingServerCallListener;
+import io.grpc.Metadata;
+import io.grpc.ServerCall;
+import io.grpc.ServerCallHandler;
+import io.grpc.ServerInterceptor;
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+public class ProviderInterceptor implements ServerInterceptor {
+    private static final Logger LOGGER = LogManager.getLogger(ProviderInterceptor.class);
+
+    @Override
+    public <ReqT, RespT> ServerCall.Listener<ReqT> interceptCall(ServerCall<ReqT, RespT> call, Metadata metadata,
+        ServerCallHandler<ReqT, RespT> handler) {
+        Map<String, String> headerMap = new HashMap<String, String>();
+        for (String key : metadata.keys()) {
+            LOGGER.info("Receive key: {}", key);
+            if (!key.endsWith(Metadata.BINARY_HEADER_SUFFIX)) {
+                String value = metadata.get(Metadata.Key.of(key, Metadata.ASCII_STRING_MARSHALLER));
+
+                headerMap.put(key, value);
+            }
+        }
+        LOGGER.info("authority : {}", call.getAuthority());
+        return new ForwardingServerCallListener.SimpleForwardingServerCallListener<ReqT>(handler.startCall(new ForwardingServerCall.SimpleForwardingServerCall<ReqT, RespT>(call) {
+            @Override
+            public void sendHeaders(Metadata responseHeaders) {
+                LOGGER.info("sendHeaders....");
+                Metadata.Key<String> headerKey = Metadata.Key.of("test-server", Metadata.ASCII_STRING_MARSHALLER);
+                responseHeaders.put(headerKey, "test-server");
+                delegate().sendHeaders(responseHeaders);
+            }
+
+            @Override
+            public void sendMessage(RespT message) {
+                delegate().sendMessage(message);
+            }
+
+        }, metadata)) {
+            @Override
+            public void onReady() {
+                LOGGER.info("onReady....");
+                delegate().onReady();
+            }
+
+            @Override
+            public void onCancel() {
+                LOGGER.info("onCancel....");
+                delegate().onCancel();
+            }
+
+            @Override
+            public void onComplete() {
+                LOGGER.info("onComplete....");
+                delegate().onComplete();
+            }
+
+            @Override
+            public void onHalfClose() {
+                LOGGER.info("onHalfClose....");
+                delegate().onHalfClose();
+            }
+
+            @Override
+            public void onMessage(ReqT message) {
+                LOGGER.info("onMessage....");
+                delegate().onMessage(message);
+            }
+        };
+    }
+}

test/plugin/scenarios/correlation-autotag-scenario/src/main/java/org/apache/skywalking/apm/testcase/grpc/provider/service/GreeterServiceImpl.java

+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.apm.testcase.grpc.provider.service;
+
+import io.grpc.stub.StreamObserver;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.skywalking.apm.testcase.grpc.proto.GreeterGrpc;
+import org.apache.skywalking.apm.testcase.grpc.proto.HelloReply;
+import org.apache.skywalking.apm.testcase.grpc.proto.HelloRequest;
+
+public class GreeterServiceImpl extends GreeterGrpc.GreeterImplBase {
+
+    private static final Logger LOGGER = LogManager.getLogger(GreeterServiceImpl.class);
+
+    @Override
+    public StreamObserver<HelloRequest> sayHello(final StreamObserver<HelloReply> responseObserver) {
+        StreamObserver<HelloRequest> requestStreamObserver = new StreamObserver<HelloRequest>() {
+
+            public void onNext(HelloRequest request) {
+                LOGGER.info("Receive an message from client. Message: {}", request.getName());
+                responseObserver.onNext(HelloReply.newBuilder().setMessage("Hi," + request.getName()).build());
+            }
+
+            public void onError(Throwable throwable) {
+                responseObserver.onError(throwable);
+            }
+
+            public void onCompleted() {
+                LOGGER.info("End the stream.");
+                responseObserver.onCompleted();
+            }
+        };
+        return requestStreamObserver;
+    }
+}

test/plugin/scenarios/correlation-autotag-scenario/src/main/proto/GreetService.proto

+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+syntax = "proto3";
+
+option java_multiple_files = true;
+option java_package = "org.apache.skywalking.apm.testcase.grpc.proto";
+
+service Greeter {
+    rpc SayHello (stream HelloRequest) returns (stream HelloReply) {
+    }
+}
+
+service GreeterBlocking {
+    rpc SayHello (HelloRequest) returns (HelloReply) {
+    }
+}
+service GreeterBlockingError {
+    rpc SayHello (HelloRequest) returns (HelloReply) {
+    }
+}
+
+message HelloRequest {
+    string name = 1;
+}
+
+message HelloReply {
+    string message = 1;
+}
\ No newline at end of file

test/plugin/scenarios/correlation-autotag-scenario/src/main/resources/application.yaml

+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+server:
+  port: 8080
+  servlet:
+    context-path: /correlation-autotag-scenario
+logging:
+  config: classpath:log4j2.xml
\ No newline at end of file

test/plugin/scenarios/correlation-autotag-scenario/src/main/resources/log4j2.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  ~
+  -->
+<Configuration status="WARN">
+    <Appenders>
+        <Console name="Console" target="SYSTEM_ERR">
+            <PatternLayout charset="UTF-8" pattern="[%d{yyyy-MM-dd HH:mm:ss:SSS}] [%p] - %l - %m%n"/>
+        </Console>
+    </Appenders>
+    <Loggers>
+        <Root level="WARN">
+            <AppenderRef ref="Console"/>
+        </Root>
+    </Loggers>
+</Configuration>
\ No newline at end of file

test/plugin/scenarios/correlation-autotag-scenario/support-version.list

+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# INTERNAL: HTTP/2 error code: INTERNAL_ERROR Received Goaway occur in test cases 1.0.0 to 1.5.0
+# So these versions were not included in support-version.list. if you know what caused it, please help us.
+
+# Contains only the last version number of each minor version
+
+1.25.0
\ No newline at end of file

tools/coverage/report.sh

@@ -41,4 +41,4 @@ for exec_data in "${JACOCO_HOME}"/*.exec; do
     "${JACOCO_HOME}"/"$exec_data".exec
 done
 
-bash <(curl -s https://codecov.io/bash) -X fix -f /tmp/report-*.xml
+bash <(curl -s https://codecov.io/bash) -X fix -f /tmp/report-*.xml || true