Add kotlin coroutine plugin (#4165)

* Kotlin coroutine plugin write in Java
Co-authored-by: 吴晟 Wu Sheng <wu.sheng@foxmail.com>

.github/workflows/plugins-test.yaml

@@ -182,7 +182,7 @@ jobs:
       - name: Run spring 3.0.x (8)
         run: bash test/plugin/run.sh spring-3.0.x-scenario
 
-  SpringAsync_gRPC:
+  SpringAsync_gRPC_KotlinCoroutine:
     runs-on: ubuntu-18.04
     timeout-minutes: 90
     strategy:
@@ -208,6 +208,8 @@ jobs:
         run: bash test/plugin/run.sh spring-async-scenario
       - name: Run grpc 1.6.0-1.25.0 (22)
         run: bash test/plugin/run.sh grpc-scenario
+      - name: Run kotlin coroutine 1.0.1-1.3.3 (4)
+        run: bash test/plugin/run.sh kotlin-coroutine-scenario
 
   Ehcache_Undertow_Jedis:
     runs-on: ubuntu-18.04

apm-protocol/apm-network/src/main/java/org/apache/skywalking/apm/network/trace/component/ComponentsDefine.java

@@ -151,4 +151,6 @@ public class ComponentsDefine {
     public static final OfficialComponent ARMERIA = new OfficialComponent(79, "Armeria");
 
     public static final OfficialComponent JDK_THREADING = new OfficialComponent(80, "JdkThreading");
+
+    public static final OfficialComponent KT_COROUTINE = new OfficialComponent(81, "KotlinCoroutine");
 }

apm-sniffer/optional-plugins/kotlin-coroutine-plugin/pom.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  ~
+  -->
+
+<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>org.apache.skywalking</groupId>
+        <artifactId>optional-plugins</artifactId>
+        <version>7.0.0-SNAPSHOT</version>
+    </parent>
+
+    <artifactId>apm-kotlin-coroutine-plugin</artifactId>
+    <name>kotlin-coroutine-plugin</name>
+    <packaging>jar</packaging>
+
+    <properties>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+        <kotlin.version>1.3.61</kotlin.version>
+        <kotlinx.coroutine.version>1.3.3</kotlinx.coroutine.version>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.jetbrains.kotlinx</groupId>
+            <artifactId>kotlinx-coroutines-core</artifactId>
+            <version>${kotlinx.coroutine.version}</version>
+            <scope>provided</scope>
+        </dependency>
+    </dependencies>
+</project>

apm-sniffer/optional-plugins/kotlin-coroutine-plugin/src/main/java/org/apache/skywalking/apm/plugin/kotlin/coroutine/DispatcherInterceptor.java

+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.apm.plugin.kotlin.coroutine;
+
+import org.apache.skywalking.apm.agent.core.plugin.interceptor.enhance.EnhancedInstance;
+import org.apache.skywalking.apm.agent.core.plugin.interceptor.enhance.InstanceMethodsAroundInterceptor;
+import org.apache.skywalking.apm.agent.core.plugin.interceptor.enhance.MethodInterceptResult;
+
+import java.lang.reflect.Method;
+
+public class DispatcherInterceptor implements InstanceMethodsAroundInterceptor {
+    @Override
+    public void beforeMethod(EnhancedInstance objInst, Method method, Object[] allArguments, Class<?>[] argumentsTypes, MethodInterceptResult result) {
+        // Wrapping runnable with current context snapshot
+        allArguments[1] = TracingRunnable.wrapOrNot((Runnable) allArguments[1]);
+    }
+
+    @Override
+    public Object afterMethod(EnhancedInstance objInst, Method method, Object[] allArguments, Class<?>[] argumentsTypes, Object ret) {
+        return ret;
+    }
+
+    @Override
+    public void handleMethodException(EnhancedInstance objInst, Method method, Object[] allArguments, Class<?>[] argumentsTypes, Throwable t) {
+    }
+}

apm-sniffer/optional-plugins/kotlin-coroutine-plugin/src/main/java/org/apache/skywalking/apm/plugin/kotlin/coroutine/TracingRunnable.java

+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.apm.plugin.kotlin.coroutine;
+
+import org.apache.skywalking.apm.agent.core.context.ContextManager;
+import org.apache.skywalking.apm.agent.core.context.ContextSnapshot;
+import org.apache.skywalking.apm.agent.core.context.trace.AbstractSpan;
+import org.apache.skywalking.apm.network.trace.component.ComponentsDefine;
+
+/**
+ * {@link Runnable} wrapper with trace context snapshot, it will create span
+ * with context snapshot around {@link Runnable} runs.
+ * <p>
+ * A class implementation will be cheaper cost than lambda with captured
+ * variables implementation.
+ */
+class TracingRunnable implements Runnable {
+    private static final String COROUTINE = "/Kotlin/Coroutine";
+
+    private ContextSnapshot snapshot;
+    private Runnable delegate;
+
+    private TracingRunnable(ContextSnapshot snapshot, Runnable delegate) {
+        this.snapshot = snapshot;
+        this.delegate = delegate;
+    }
+
+    /**
+     * Wrap {@link Runnable} by {@link TracingRunnable} if active trace context
+     * existed.
+     *
+     * @param delegate {@link Runnable} to wrap.
+     *
+     * @return Wrapped {@link TracingRunnable} or original {@link Runnable} if
+     * trace context not existed.
+     */
+    public static Runnable wrapOrNot(Runnable delegate) {
+        // Just wrap continuation with active trace context
+        if (ContextManager.isActive()) {
+            return new TracingRunnable(ContextManager.capture(), delegate);
+        } else {
+            return delegate;
+        }
+    }
+
+    @Override
+    public void run() {
+        if (ContextManager.isActive() && snapshot.isFromCurrent()) {
+            // Thread not switched, skip restore snapshot.
+            delegate.run();
+            return;
+        }
+
+        // Create local coroutine span
+        AbstractSpan span = ContextManager.createLocalSpan(COROUTINE);
+        span.setComponent(ComponentsDefine.KT_COROUTINE);
+
+        // Recover with snapshot
+        ContextManager.continued(snapshot);
+
+        try {
+            delegate.run();
+        } finally {
+            ContextManager.stopSpan(span);
+        }
+    }
+}

apm-sniffer/optional-plugins/kotlin-coroutine-plugin/src/main/java/org/apache/skywalking/apm/plugin/kotlin/coroutine/define/DispatcherInstrumentation.java

+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.apm.plugin.kotlin.coroutine.define;
+
+import net.bytebuddy.description.method.MethodDescription;
+import net.bytebuddy.matcher.ElementMatcher;
+import org.apache.skywalking.apm.agent.core.plugin.interceptor.ConstructorInterceptPoint;
+import org.apache.skywalking.apm.agent.core.plugin.interceptor.InstanceMethodsInterceptPoint;
+import org.apache.skywalking.apm.agent.core.plugin.interceptor.enhance.ClassInstanceMethodsEnhancePluginDefine;
+import org.apache.skywalking.apm.agent.core.plugin.match.ClassMatch;
+import org.apache.skywalking.apm.agent.core.plugin.match.HierarchyMatch;
+
+import static net.bytebuddy.matcher.ElementMatchers.named;
+import static org.apache.skywalking.apm.agent.core.plugin.bytebuddy.ArgumentTypeNameMatch.takesArgumentWithType;
+
+public class DispatcherInstrumentation extends ClassInstanceMethodsEnhancePluginDefine {
+    public static final String INTERCEPTOR_CLASS = "org.apache.skywalking.apm.plugin.kotlin.coroutine.DispatcherInterceptor";
+    public static final String ENHANCE_CLASS = "kotlinx.coroutines.CoroutineDispatcher";
+    public static final String ENHANCE_METHOD_DISPATCH = "dispatch";
+
+    @Override
+    protected ClassMatch enhanceClass() {
+        return HierarchyMatch.byHierarchyMatch(ENHANCE_CLASS);
+    }
+
+    @Override
+    public ConstructorInterceptPoint[] getConstructorsInterceptPoints() {
+        return new ConstructorInterceptPoint[0];
+    }
+
+    @Override
+    public InstanceMethodsInterceptPoint[] getInstanceMethodsInterceptPoints() {
+        return new InstanceMethodsInterceptPoint[]{
+            new InstanceMethodsInterceptPoint() {
+                @Override
+                public ElementMatcher<MethodDescription> getMethodsMatcher() {
+                    return named(ENHANCE_METHOD_DISPATCH)
+                            .and(takesArgumentWithType(0, "kotlin.coroutines.CoroutineContext"))
+                            .and(takesArgumentWithType(1, "java.lang.Runnable"));
+                }
+
+                @Override
+                public String getMethodsInterceptor() {
+                    return INTERCEPTOR_CLASS;
+                }
+
+                @Override
+                public boolean isOverrideArgs() {
+                    return true;
+                }
+            }
+        };
+    }
+}

apm-sniffer/optional-plugins/kotlin-coroutine-plugin/src/main/resources/skywalking-plugin.def

+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+kotlin-coroutine=org.apache.skywalking.apm.plugin.kotlin.coroutine.define.DispatcherInstrumentation
\ No newline at end of file

apm-sniffer/optional-plugins/pom.xml

@@ -46,6 +46,7 @@
         <module>gson-2.8.x-plugin</module>
         <module>zookeeper-3.4.x-plugin</module>
         <module>customize-enhance-plugin</module>
+        <module>kotlin-coroutine-plugin</module>
     </modules>
 
     <dependencies>

docs/en/setup/service-agent/java-agent/README.md

@@ -135,6 +135,7 @@ Now, we have the following known optional plugins.
 * [Customize enhance](Customize-enhance-trace.md) Trace methods based on description files, rather than write plugin or change source codes.
 * Plugin of Spring Cloud Gateway 2.1.x in optional plugin folder. Please only active this plugin when you install agent in Spring Gateway. spring-cloud-gateway-2.x-plugin and spring-webflux-5.x-plugin are both required.
 * Plugin of Spring Transaction in optional plugin folder. The reason of being optional plugin is, many local span are generated, which also spend more CPU, memory and network.
+* [Plugin of Kotlin coroutine](agent-optional-plugins/Kotlin-Coroutine-plugin.md) provides the tracing across coroutines automatically. As it will add local spans to all across routines scenarios, Please assess the performance impact.
 
 ## Bootstrap class plugins
 All bootstrap plugins are optional, due to unexpected risk. Bootstrap plugins are provided in `bootstrap-plugins` folder.

docs/en/setup/service-agent/java-agent/Supported-list.md

@@ -86,6 +86,8 @@
   * [Spring @Async](https://github.com/spring-projects/spring-framework) 4.x and 5.x
 * Cache
   * [Ehcache](https://www.ehcache.org/) 2.x
+* Kotlin
+  * [Coroutine](https://kotlinlang.org/docs/reference/coroutines-overview.html) 1.0.1 -> 1.3.x (Optional²)
 
 
 ¹Due to license incompatibilities/restrictions these plugins are hosted and released in 3rd part repository, 

docs/en/setup/service-agent/java-agent/agent-optional-plugins/Kotlin-Coroutine-plugin.md

+# Skywalking with Kotlin coroutine
+This Plugin provides an auto instrument support plugin for Kotlin coroutine based on context snapshot.
+
+## Description
+SkyWalking provide tracing context propagation inside thread. In order to support Kotlin Coroutine, we provide this additional plugin.
+
+## Implementation principle
+As we know, Kotlin coroutine switches the execution thread by `CoroutineDispatcher`.
+
+01. Create a snapshot of the current context before dispatch the continuation.
+02. Then create a coroutine span after thread switched, mark the span continued with the snapshot.
+03. Every new span which created in the new thread will be a child of this coroutine span. So we can link those span together in a tracing.
+04. After the original runnable executed, we need to stop the coroutine span for cleaning thread state.
+
+## Some screenshots
+### Run without the plugin
+We run a Kotlin coroutine based gRPC server without this coroutine plugin.  
+You can find, the one call (client -> server1 -> server2) has been split two tracing paths.
+
+01. Server1 without exit span and server2 tracing path.
+![Without kotlin plugin1](http://skywalking.apache.org/screenshots/7.0.0/kotlin/coroutine/without-coroutine-plugin-server1.jpg)
+02. Server2 tracing path.
+![Without kotlin plugin2](http://skywalking.apache.org/screenshots/7.0.0/kotlin/coroutine/without-coroutine-plugin-server2.jpg)
+
+### Run with the plugin
+Without changing codes manually, just install the plugin. We can find the spans be connected together. We can get all info of one client call.
+
+![With kotlin plugin](http://skywalking.apache.org/screenshots/7.0.0/kotlin/coroutine/run-with-coroutine-plugin.jpg)

test/plugin/scenarios/kotlin-coroutine-scenario/bin/startup.sh

+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+home="$(cd "$(dirname $0)"; pwd)"
+
+java -jar ${agent_opts} ${home}/../libs/kotlin-coroutine-scenario.jar &
\ No newline at end of file

test/plugin/scenarios/kotlin-coroutine-scenario/config/expectedData.yaml

+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+registryItems:
+  applications:
+    - {kotlin-coroutine-scenario: 2}
+  instances:
+    - {kotlin-coroutine-scenario: 1}
+  operationNames:
+    - kotlin-coroutine-scenario: [/kotlin-coroutine-scenario/case/h2]
+  heartbeat: []
+segmentItems:
+  - applicationCode: kotlin-coroutine-scenario
+    segmentSize: gt 1
+    segments:
+      - segmentId: not null
+        spans:
+          - operationName: H2/JDBI/PreparedStatement/executeQuery
+            operationId: 0
+            parentSpanId: 0
+            spanId: 1
+            spanLayer: Database
+            startTime: nq 0
+            endTime: nq 0
+            componentId: 32
+            componentName: ''
+            isError: false
+            spanType: Exit
+            peer: localhost:-1
+            peerId: 0
+            tags:
+              - {key: db.type, value: 'sql'}
+              - {key: db.instance, value: 'demo'}
+              - {key: db.statement, value: 'SELECT * FROM PERSON WHERE ID = ?'}
+          - operationName: H2/JDBI/Connection/commit
+            operationId: 0
+            parentSpanId: 0
+            spanId: 2
+            spanLayer: Database
+            startTime: nq 0
+            endTime: nq 0
+            componentId: 32
+            componentName: ''
+            isError: false
+            spanType: Exit
+            peer: localhost:-1
+            peerId: 0
+            tags:
+              - {key: db.type, value: 'sql'}
+              - {key: db.instance, value: 'demo'}
+              - {key: db.statement, value: ''}
+          - operationName: /Kotlin/Coroutine
+            operationId: 0
+            parentSpanId: -1
+            spanId: 0
+            startTime: nq 0
+            endTime: nq 0
+            componentId: 81
+            componentName: ''
+            isError: false
+            spanType: Local
+            peer: ''
+            peerId: 0
+            refs:
+              - {parentEndpointId: 0, parentEndpoint: /kotlin-coroutine-scenario/case/h2,
+                 networkAddressId: 0, entryEndpointId: 0, refType: CrossThread, parentSpanId: 0,
+                 parentTraceSegmentId: "${kotlin-coroutine-scenario[1]}", parentServiceInstanceId: 1,
+                 networkAddress: '', entryEndpoint: /kotlin-coroutine-scenario/case/h2, entryServiceInstanceId: 1}
+      - segmentId: not null
+        spans:
+          - operationName: H2/JDBI/PreparedStatement/executeQuery
+            operationId: 0
+            parentSpanId: 0
+            spanId: 1
+            spanLayer: Database
+            startTime: nq 0
+            endTime: nq 0
+            componentId: 32
+            componentName: ''
+            isError: false
+            spanType: Exit
+            peer: localhost:-1
+            peerId: 0
+            tags:
+              - {key: db.type, value: 'sql'}
+              - {key: db.instance, value: 'demo'}
+              - {key: db.statement, value: 'SELECT * FROM PERSON WHERE ID = ?'}
+          - operationName: H2/JDBI/Connection/commit
+            operationId: 0
+            parentSpanId: 0
+            spanId: 2
+            spanLayer: Database
+            startTime: nq 0
+            endTime: nq 0
+            componentId: 32
+            componentName: ''
+            isError: false
+            spanType: Exit
+            peer: localhost:-1
+            peerId: 0
+            tags:
+              - {key: db.type, value: 'sql'}
+              - {key: db.instance, value: 'demo'}
+              - {key: db.statement, value: ''}
+          - operationName: /kotlin-coroutine-scenario/case/h2
+            operationId: 0
+            parentSpanId: -1
+            spanId: 0
+            spanLayer: Http
+            startTime: nq 0
+            endTime: nq 0
+            componentId: 1
+            componentName: ''
+            isError: false
+            spanType: Entry
+            peer: ''
+            peerId: 0
+            tags:
+              - {key: url, value: 'http://localhost:8080/kotlin-coroutine-scenario/case/h2'}
+              - {key: http.method, value: GET}
\ No newline at end of file

test/plugin/scenarios/kotlin-coroutine-scenario/configuration.yml

+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+type: jvm
+entryService: http://localhost:8080/kotlin-coroutine-scenario/case/h2
+healthCheck: http://localhost:8080/kotlin-coroutine-scenario/case/healthCheck
+startScript: ./bin/startup.sh
+framework: kt-coroutine
+runningMode: with_optional
+withPlugins: apm-kotlin-coroutine-plugin-*.jar
\ No newline at end of file

test/plugin/scenarios/kotlin-coroutine-scenario/pom.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  ~
+  -->
+<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xmlns="http://maven.apache.org/POM/4.0.0"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+    <groupId>org.apache.skywalking.apm.testcase</groupId>
+    <artifactId>kotlin-coroutine-scenario</artifactId>
+    <version>1.0.0</version>
+    <packaging>jar</packaging>
+
+    <modelVersion>4.0.0</modelVersion>
+
+    <properties>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+        <compiler.version>1.8</compiler.version>
+        <test.framework.version>1.3.3</test.framework.version>
+
+        <spring-boot-version>2.1.6.RELEASE</spring-boot-version>
+        <kotlin.version>1.3.61</kotlin.version>
+        <kotlinx.coroutine.version>1.3.3</kotlinx.coroutine.version>
+    </properties>
+
+    <name>skywalking-kotlin-coroutine-scenario</name>
+
+    <dependencyManagement>
+        <dependencies>
+            <dependency>
+                <groupId>org.springframework.boot</groupId>
+                <artifactId>spring-boot-dependencies</artifactId>
+                <version>${spring-boot-version}</version>
+                <type>pom</type>
+                <scope>import</scope>
+            </dependency>
+        </dependencies>
+    </dependencyManagement>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-web</artifactId>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.springframework.boot</groupId>
+                    <artifactId>spring-boot-starter-logging</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-log4j2</artifactId>
+            <exclusions>
+                <exclusion>
+                    <artifactId>jul-to-slf4j</artifactId>
+                    <groupId>org.slf4j</groupId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>com.h2database</groupId>
+            <artifactId>h2</artifactId>
+            <version>1.4.196</version>
+        </dependency>
+        <dependency>
+            <groupId>commons-dbcp</groupId>
+            <artifactId>commons-dbcp</artifactId>
+            <version>1.4</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.jetbrains.kotlin</groupId>
+            <artifactId>kotlin-stdlib</artifactId>
+            <version>${kotlin.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.jetbrains.kotlinx</groupId>
+            <artifactId>kotlinx-coroutines-core</artifactId>
+            <version>${test.framework.version}</version>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <finalName>kotlin-coroutine-scenario</finalName>
+        <sourceDirectory>${project.basedir}/src/main/kotlin</sourceDirectory>
+        <plugins>
+            <plugin>
+                <groupId>org.springframework.boot</groupId>
+                <artifactId>spring-boot-maven-plugin</artifactId>
+                <version>${spring-boot-version}</version>
+                <executions>
+                    <execution>
+                        <goals>
+                            <goal>repackage</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.jetbrains.kotlin</groupId>
+                <artifactId>kotlin-maven-plugin</artifactId>
+                <version>${kotlin.version}</version>
+
+                <executions>
+                    <execution>
+                        <id>compile</id>
+                        <goals>
+                            <goal>compile</goal>
+                        </goals>
+                    </execution>
+
+                    <execution>
+                        <id>test-compile</id>
+                        <goals>
+                            <goal>test-compile</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-assembly-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <id>assemble</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>single</goal>
+                        </goals>
+                        <configuration>
+                            <descriptors>
+                                <descriptor>src/main/assembly/assembly.xml</descriptor>
+                            </descriptors>
+                            <outputDirectory>./target/</outputDirectory>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+</project>
\ No newline at end of file

test/plugin/scenarios/kotlin-coroutine-scenario/src/main/assembly/assembly.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  ~
+  -->
+<assembly
+        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+        xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
+        xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
+    <formats>
+        <format>zip</format>
+    </formats>
+
+    <fileSets>
+        <fileSet>
+            <directory>./bin</directory>
+            <fileMode>0775</fileMode>
+        </fileSet>
+    </fileSets>
+
+    <files>
+        <file>
+            <source>${project.build.directory}/kotlin-coroutine-scenario.jar</source>
+            <outputDirectory>./libs</outputDirectory>
+            <fileMode>0775</fileMode>
+        </file>
+    </files>
+</assembly>

test/plugin/scenarios/kotlin-coroutine-scenario/src/main/kotlin/org/apache/skywalking/apm/testcase/kotlin/coroutine/Application.kt

+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.apm.testcase.kotlin.coroutine
+
+import org.springframework.boot.SpringApplication
+import org.springframework.boot.autoconfigure.SpringBootApplication
+
+@SpringBootApplication
+open class Application
+
+fun main(args: Array<String>) {
+    SpringApplication.run(Application::class.java, *args)
+}
\ No newline at end of file

test/plugin/scenarios/kotlin-coroutine-scenario/src/main/kotlin/org/apache/skywalking/apm/testcase/kotlin/coroutine/controller/DemoController.kt

+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.apm.testcase.kotlin.coroutine.controller
+
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.runBlocking
+import org.apache.skywalking.apm.testcase.kotlin.coroutine.service.DemoService
+import org.springframework.beans.factory.annotation.Autowired
+import org.springframework.stereotype.Controller
+import org.springframework.web.bind.annotation.RequestMapping
+import org.springframework.web.bind.annotation.ResponseBody
+
+@Controller
+@RequestMapping("/case")
+class DemoController {
+    @Autowired
+    private lateinit var service: DemoService
+
+    @ResponseBody
+    @RequestMapping("/healthCheck")
+    fun healthCheck(): String {
+        return "Success"
+    }
+
+    @ResponseBody
+    @RequestMapping("/h2")
+    fun h2ThreadSwitchedCase(): String {
+        val threadId = Thread.currentThread().id
+
+        runBlocking {
+            // Run blocking with default context will not switch thread
+            if (threadId != Thread.currentThread().id) {
+                throw IllegalStateException("Prerequisite failed")
+            }
+
+            service.work()
+        }
+
+        runBlocking(Dispatchers.IO) {
+            // Run blocking with IO dispatcher will switch to IO thread
+            if (threadId == Thread.currentThread().id) {
+                throw IllegalStateException("Prerequisite failed")
+            }
+
+            service.work()
+        }
+        return "Success"
+    }
+}
\ No newline at end of file

test/plugin/scenarios/kotlin-coroutine-scenario/src/main/kotlin/org/apache/skywalking/apm/testcase/kotlin/coroutine/respository/DemoRepository.kt

+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.apm.testcase.kotlin.coroutine.respository
+
+import org.apache.commons.dbcp.BasicDataSource
+import org.apache.skywalking.apm.testcase.kotlin.coroutine.util.use
+import org.springframework.stereotype.Service
+import javax.sql.DataSource
+
+@Service
+class DemoRepository {
+    private val datasource: DataSource = BasicDataSource().apply {
+        driverClassName = "org.h2.Driver"
+        url = String.format("jdbc:h2:mem:demo")
+        username = ""
+        password = ""
+    }
+
+    init {
+        datasource.connection.use {
+            prepareStatement("CREATE TABLE PERSON(ID INT auto_increment primary key, NAME varchar(255))").execute()
+            prepareStatement("INSERT INTO PERSON (NAME) VALUES (?)").apply {
+                setString(1, "kanro")
+                execute()
+            }
+        }
+    }
+
+    fun getPersonName(id: Int): String? {
+        return datasource.connection.use {
+            prepareStatement("SELECT * FROM PERSON WHERE ID = ?").run {
+                setInt(1, id)
+                val result = executeQuery()
+                if (result.next()) {
+                    result.getString("NAME")
+                } else {
+                    null
+                }
+            }
+        }
+    }
+}
\ No newline at end of file

test/plugin/scenarios/kotlin-coroutine-scenario/src/main/kotlin/org/apache/skywalking/apm/testcase/kotlin/coroutine/service/DemoService.kt

+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.apm.testcase.kotlin.coroutine.service
+
+import org.apache.skywalking.apm.testcase.kotlin.coroutine.respository.DemoRepository
+import org.springframework.beans.factory.annotation.Autowired
+import org.springframework.stereotype.Service
+
+@Service
+class DemoService {
+    @Autowired
+    private lateinit var repository: DemoRepository
+
+    fun work() {
+        if (repository.getPersonName(1) == null) {
+            throw IllegalStateException()
+        }
+    }
+}
\ No newline at end of file

test/plugin/scenarios/kotlin-coroutine-scenario/src/main/kotlin/org/apache/skywalking/apm/testcase/kotlin/coroutine/util/Connection.kt

+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.apm.testcase.kotlin.coroutine.util
+
+import java.sql.Connection
+
+inline fun <T> Connection.use(block: Connection.() -> T): T {
+    try {
+        return block()
+    } finally {
+        commit()
+        close()
+    }
+}
\ No newline at end of file

test/plugin/scenarios/kotlin-coroutine-scenario/src/main/resources/application.yaml

+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+server:
+  port: 8080
+  servlet:
+    context-path: /kotlin-coroutine-scenario
+logging:
+  config: classpath:log4j2.xml

test/plugin/scenarios/kotlin-coroutine-scenario/src/main/resources/log4j2.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  ~
+  -->
+<Configuration status="WARN">
+    <Appenders>
+        <Console name="Console" target="SYSTEM_ERR">
+            <PatternLayout charset="UTF-8" pattern="[%d{yyyy-MM-dd HH:mm:ss:SSS}] [%p] - %l - %m%n"/>
+        </Console>
+    </Appenders>
+    <Loggers>
+        <Root level="WARN">
+            <AppenderRef ref="Console"/>
+        </Root>
+    </Loggers>
+</Configuration>
\ No newline at end of file

test/plugin/scenarios/kotlin-coroutine-scenario/support-version.list

+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+1.3.3
+1.2.2
+1.1.1
+1.0.1
\ No newline at end of file