Add E2E test for python Kafka report protocol (#5640)

test/e2e/e2e-test/docker/python/Dockerfile.python

@@ -15,7 +15,7 @@
 
 FROM python:3.7
 
-ENV COMMIT_HASH=2056144925a574d04c4a90914a82b8efce2375be
+ENV COMMIT_HASH=50388c55428d742d73d9733278f04173585de80d
 
 WORKDIR /app
 
@@ -24,6 +24,8 @@ RUN git clone https://github.com/apache/skywalking-python.git $(pwd)
 RUN git reset --hard ${COMMIT_HASH} && git submodule update --init
 
 RUN make setup install
+RUN python3 -m pip install kafka-python
 
 ADD ./consumer.py /consumer.py
 ADD ./provider.py /provider.py
+ADD ./provider-kafka.py /provider-kafka.py

test/e2e/e2e-test/docker/python/consumer.py

@@ -43,6 +43,12 @@ if __name__ == '__main__':
             with request.urlopen(req, data):
                 self.wfile.write(data)
 
+            req2 = request.Request("http://provider-kafka:9089/users")
+            req2.add_header('Content-Type', 'application/json; charset=utf-8')
+            req2.add_header('Content-Length', str(len(data)))
+            with request.urlopen(req2, data):
+                self.wfile.write(data)
+
     PORT = 9090
     Handler = SimpleHTTPRequestHandler
 

test/e2e/e2e-test/docker/python/docker-compose.yml

@@ -16,10 +16,77 @@
 version: '2.1'
 
 services:
+  zookeeper:
+    image: zookeeper:3.4
+    hostname: zookeeper
+    expose:
+      - 2181
+    networks:
+      - e2e
+    environment:
+      - ALLOW_ANONYMOUS_LOGIN=yes
+    healthcheck:
+      test: ["CMD", "sh", "-c", "nc -nz 127.0.0.1 2181"]
+      interval: 5s
+      timeout: 60s
+      retries: 120
+
+  broker-a:
+    image: bitnami/kafka:2.4.1
+    hostname: broker-a
+    expose:
+      - 9092
+    networks:
+      - e2e
+    environment:
+      - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
+      - KAFKA_BROKER_ID=10
+      - ALLOW_PLAINTEXT_LISTENER=yes
+    depends_on:
+      zookeeper:
+        condition: service_healthy
+    healthcheck:
+      test: ["CMD", "kafka-topics.sh", "--list", "--zookeeper", "zookeeper:2181"]
+      interval: 5s
+      timeout: 60s
+      retries: 120
+
+  broker-b:
+    image: bitnami/kafka:2.4.1
+    hostname: broker-b
+    expose:
+      - 9092
+    networks:
+      - e2e
+    environment:
+      - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
+      - KAFKA_BROKER_ID=24
+      - ALLOW_PLAINTEXT_LISTENER=yes
+    depends_on:
+      zookeeper:
+        condition: service_healthy
+    healthcheck:
+      test: ["CMD", "kafka-topics.sh", "--list", "--zookeeper", "zookeeper:2181"]
+      interval: 5s
+      timeout: 60s
+      retries: 120
+
   oap:
     extends:
       file: ../base-compose.yml
       service: oap
+    environment:
+      SW_PROMETHEUS_FETCHER_ACTIVE: "true"
+      SW_TELEMETRY: prometheus
+      SW_KAFKA_FETCHER: default
+      SW_KAFKA_FETCHER_SERVERS: broker-a:9092,broker-b:9092
+      SW_KAFKA_FETCHER_PARTITIONS: 2
+      SW_KAFKA_FETCHER_PARTITIONS_FACTOR: 1
+    depends_on:
+      broker-a:
+        condition: service_healthy
+      broker-b:
+        condition: service_healthy
 
   ui:
     extends:
@@ -62,6 +129,29 @@ services:
       provider:
         condition: service_healthy
 
+  provider-kafka:
+    build:
+      context: .
+      dockerfile: Dockerfile.python
+    networks:
+      - e2e
+    expose:
+      - 9089
+    environment:
+      SW_AGENT_COLLECTOR_BACKEND_SERVICES: oap:11800
+      SW_AGENT_INSTANCE: provider-kafka-instance
+      SW_KAFKA_REPORTER_BOOTSTRAP_SERVERS: broker-a:9092,broker-b:9092
+    depends_on:
+      oap:
+        condition: service_healthy
+    healthcheck:
+      test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/9089"]
+      interval: 5s
+      timeout: 60s
+      retries: 120
+    entrypoint: ['python3', '/provider-kafka.py']
+
+
   consumer:
     build:
       context: .
@@ -79,6 +169,8 @@ services:
         condition: service_healthy
       medium:
         condition: service_healthy
+      provider-kafka:
+        condition: service_healthy
     healthcheck:
       test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/9090"]
       interval: 5s

test/e2e/e2e-test/docker/python/provider-kafka.py

+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+
+from urllib import request
+
+from skywalking import agent, config
+
+if __name__ == '__main__':
+    config.service_name = 'provider-kafka'
+    config.logging_level = 'DEBUG'
+    config.protocol = "kafka"
+    agent.start()
+
+    import socketserver
+    from http.server import BaseHTTPRequestHandler
+
+    class SimpleHTTPRequestHandler(BaseHTTPRequestHandler):
+
+        def do_POST(self):
+            time.sleep(0.15)
+            self.send_response(200)
+            self.send_header('Content-Type', 'application/json')
+            self.end_headers()
+            self.wfile.write('{"name": "whatever"}'.encode('ascii'))
+
+    PORT = 9089
+    Handler = SimpleHTTPRequestHandler
+
+    with socketserver.TCPServer(("", PORT), Handler) as httpd:
+        print("serving at port", PORT)
+        httpd.serve_forever()

test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/PythonE2E.java

@@ -18,7 +18,6 @@
 
 package org.apache.skywalking.e2e;
 
-import java.util.List;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.skywalking.e2e.annotation.ContainerHostAndPort;
 import org.apache.skywalking.e2e.annotation.DockerCompose;
@@ -55,6 +54,9 @@ import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.BeforeAll;
 import org.testcontainers.containers.DockerComposeContainer;
 
+import java.util.List;
+import java.util.stream.Collectors;
+
 import static org.apache.skywalking.e2e.metrics.MetricsMatcher.verifyMetrics;
 import static org.apache.skywalking.e2e.metrics.MetricsQuery.ALL_ENDPOINT_METRICS;
 import static org.apache.skywalking.e2e.metrics.MetricsQuery.ALL_INSTANCE_METRICS;
@@ -95,8 +97,8 @@ public class PythonE2E extends SkyWalkingTestAdapter {
 
     @RetryableTest
     void services() throws Exception {
-        final List<Service> services = graphql.services(new ServicesQuery().start(startTime).end(now()));
-
+        List<Service> services = graphql.services(new ServicesQuery().start(startTime).end(now()));
+        services = services.stream().filter(s -> !s.getLabel().equals("oap-server")).collect(Collectors.toList());
         LOGGER.info("services: {}", services);
 
         load("expected/python/services.yml").as(ServicesMatcher.class).verify(services);
@@ -105,6 +107,7 @@ public class PythonE2E extends SkyWalkingTestAdapter {
             if ("Your_ApplicationName".equals(service.getLabel())) {
                 continue;
             }
+
             LOGGER.info("verifying service instances: {}", service);
 
             verifyServiceMetrics(service);

test/e2e/e2e-test/src/test/resources/expected/python/provider-kafka-endpoints.yml

+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+endpoints:
+  - key: not null
+    label: /users

test/e2e/e2e-test/src/test/resources/expected/python/provider-kafka-instances.yml

+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+instances:
+  - key: not null
+    label: not null

test/e2e/e2e-test/src/test/resources/expected/python/services.yml

@@ -16,6 +16,8 @@
 services:
   - key: not null
     label: provider
+  - key: not null
+    label: provider-kafka
   - key: not null
     label: consumer
   - key: not null

test/e2e/e2e-test/src/test/resources/expected/python/topo.yml

@@ -26,6 +26,10 @@ nodes:
     name: provider
     type: Python
     isReal: true
+  - id: not null
+    name: provider-kafka
+    type: Python
+    isReal: true
   - id: not null
     name: Your_ApplicationName
     type: Tomcat
@@ -42,6 +46,12 @@ calls:
       - CLIENT
       - SERVER
     target: ${Your_ApplicationName[0]}
+  - id: not null
+    source: ${consumer[0]}
+    detectPoints:
+      - CLIENT
+      - SERVER
+    target: ${provider-kafka[0]}
   - id: not null
     source: ${Your_ApplicationName[0]}
     detectPoints:

test/e2e/e2e-test/src/test/resources/expected/python/traces.yml

@@ -22,6 +22,14 @@ traces:
     isError: false
     traceIds:
       - not null
+  - key: not null
+    endpointNames:
+      - /users
+    duration: ge 0
+    start: gt 0
+    isError: false
+    traceIds:
+      - not null
   - key: not null
     endpointNames:
       - /test