Support topology with multiple selected services (#4408)

* 1. Format codes

2. Add comments for storage interface of topology

3. Narrow the scope of secondary query to get the component name. Only query the nodes as client out of the given service Ids.

* Support topology with multiple selected services

* Sync the typo fix in the protocol

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/TopologyBuilder.java

@@ -82,7 +82,8 @@ class TopologyBuilder {
                 nodes.put(target.getSequence(), buildNode(target));
                 if (BooleanUtils.valueToBoolean(target.getIsAddress())) {
                     nodes.get(target.getSequence())
-                         .setType(componentLibraryCatalogService.getServerNameBasedOnComponent(clientCall.getComponentId()));
+                         .setType(
+                             componentLibraryCatalogService.getServerNameBasedOnComponent(clientCall.getComponentId()));
                 }
             }
 
@@ -129,7 +130,8 @@ class TopologyBuilder {
                     Node conjecturalNode = new Node();
                     conjecturalNode.setId(source.getSequence());
                     conjecturalNode.setName(source.getName());
-                    conjecturalNode.setType(componentLibraryCatalogService.getServerNameBasedOnComponent(serverCall.getComponentId()));
+                    conjecturalNode.setType(
+                        componentLibraryCatalogService.getServerNameBasedOnComponent(serverCall.getComponentId()));
                     conjecturalNode.setReal(true);
                     nodes.put(source.getSequence(), conjecturalNode);
                 }

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/TopologyQueryService.java

@@ -90,30 +90,44 @@ public class TopologyQueryService implements Service {
     }
 
     public Topology getGlobalTopology(final Downsampling downsampling, final long startTB,
-        final long endTB) throws IOException {
+                                      final long endTB) throws IOException {
         logger.debug("Downsampling: {}, startTimeBucket: {}, endTimeBucket: {}", downsampling, startTB, endTB);
-        List<Call.CallDetail> serviceRelationServerCalls = getTopologyQueryDAO().loadServerSideServiceRelations(downsampling, startTB, endTB);
-        List<Call.CallDetail> serviceRelationClientCalls = getTopologyQueryDAO().loadClientSideServiceRelations(downsampling, startTB, endTB);
+        List<Call.CallDetail> serviceRelationServerCalls = getTopologyQueryDAO().loadServerSideServiceRelations(
+            downsampling, startTB, endTB);
+        List<Call.CallDetail> serviceRelationClientCalls = getTopologyQueryDAO().loadClientSideServiceRelations(
+            downsampling, startTB, endTB);
 
         TopologyBuilder builder = new TopologyBuilder(moduleManager);
         return builder.build(serviceRelationClientCalls, serviceRelationServerCalls);
     }
 
     public Topology getServiceTopology(final Downsampling downsampling, final long startTB, final long endTB,
-        final int serviceId) throws IOException {
-        List<Integer> serviceIds = new ArrayList<>();
-        serviceIds.add(serviceId);
-
-        List<Call.CallDetail> serviceRelationClientCalls = getTopologyQueryDAO().loadSpecifiedClientSideServiceRelations(downsampling, startTB, endTB, serviceIds);
-        List<Call.CallDetail> serviceRelationServerCalls = getTopologyQueryDAO().loadSpecifiedServerSideServiceRelations(downsampling, startTB, endTB, serviceIds);
+                                       final List<Integer> serviceIds) throws IOException {
+        List<Call.CallDetail> serviceRelationClientCalls = getTopologyQueryDAO().loadSpecifiedClientSideServiceRelations(
+            downsampling, startTB, endTB, serviceIds);
+        List<Call.CallDetail> serviceRelationServerCalls = getTopologyQueryDAO().loadSpecifiedServerSideServiceRelations(
+            downsampling, startTB, endTB, serviceIds);
 
         TopologyBuilder builder = new TopologyBuilder(moduleManager);
         Topology topology = builder.build(serviceRelationClientCalls, serviceRelationServerCalls);
 
-        List<Integer> sourceServiceIds = new ArrayList<>();
-        serviceRelationClientCalls.forEach(call -> sourceServiceIds.add(call.getSource()));
-        if (CollectionUtils.isNotEmpty(sourceServiceIds)) {
-            List<Call.CallDetail> sourceCalls = getTopologyQueryDAO().loadSpecifiedServerSideServiceRelations(downsampling, startTB, endTB, sourceServiceIds);
+        /**
+         * The topology built above is complete.
+         * There is a special case, there may be a node of the `serviceIds` call these services as and only as a client, so it is included in the topology,
+         * its component name could be missed as not being queried before. We add another query about this.
+         */
+        List<Integer> outScopeSourceServiceIds = new ArrayList<>();
+        serviceRelationClientCalls.forEach(call -> {
+            // Client side relationships exclude the given services(#serviceIds)
+            // The given services(#serviceIds)'s component names have been included inside `serviceRelationServerCalls`
+            if (!serviceIds.contains(call.getSource())) {
+                outScopeSourceServiceIds.add(call.getSource());
+            }
+        });
+        if (CollectionUtils.isNotEmpty(outScopeSourceServiceIds)) {
+            // If exist, query them as the server side to get the target's component.
+            List<Call.CallDetail> sourceCalls = getTopologyQueryDAO().loadSpecifiedServerSideServiceRelations(
+                downsampling, startTB, endTB, outScopeSourceServiceIds);
             topology.getNodes().forEach(node -> {
                 if (Strings.isNullOrEmpty(node.getType())) {
                     for (Call.CallDetail call : sourceCalls) {
@@ -129,20 +143,29 @@ public class TopologyQueryService implements Service {
         return topology;
     }
 
-    public ServiceInstanceTopology getServiceInstanceTopology(final int clientServiceId, final int serverServiceId,
-        final Downsampling downsampling, final long startTB, final long endTB) throws IOException {
-        logger.debug("ClientServiceId: {}, ServerServiceId: {}, Downsampling: {}, startTimeBucket: {}, endTimeBucket: {}", clientServiceId, serverServiceId, downsampling, startTB, endTB);
-
-        List<Call.CallDetail> serviceInstanceRelationClientCalls = getTopologyQueryDAO().loadClientSideServiceInstanceRelations(clientServiceId, serverServiceId, downsampling, startTB, endTB);
-        List<Call.CallDetail> serviceInstanceRelationServerCalls = getTopologyQueryDAO().loadServerSideServiceInstanceRelations(clientServiceId, serverServiceId, downsampling, startTB, endTB);
+    public ServiceInstanceTopology getServiceInstanceTopology(final int clientServiceId,
+                                                              final int serverServiceId,
+                                                              final Downsampling downsampling,
+                                                              final long startTB,
+                                                              final long endTB) throws IOException {
+        logger.debug(
+            "ClientServiceId: {}, ServerServiceId: {}, Downsampling: {}, startTimeBucket: {}, endTimeBucket: {}",
+            clientServiceId, serverServiceId, downsampling, startTB, endTB
+        );
+
+        List<Call.CallDetail> serviceInstanceRelationClientCalls = getTopologyQueryDAO().loadClientSideServiceInstanceRelations(
+            clientServiceId, serverServiceId, downsampling, startTB, endTB);
+        List<Call.CallDetail> serviceInstanceRelationServerCalls = getTopologyQueryDAO().loadServerSideServiceInstanceRelations(
+            clientServiceId, serverServiceId, downsampling, startTB, endTB);
 
         ServiceInstanceTopologyBuilder builder = new ServiceInstanceTopologyBuilder(moduleManager);
         return builder.build(serviceInstanceRelationClientCalls, serviceInstanceRelationServerCalls);
     }
 
     public Topology getEndpointTopology(final Downsampling downsampling, final long startTB, final long endTB,
-        final int endpointId) throws IOException {
-        List<Call.CallDetail> serverSideCalls = getTopologyQueryDAO().loadSpecifiedDestOfServerSideEndpointRelations(downsampling, startTB, endTB, endpointId);
+                                        final int endpointId) throws IOException {
+        List<Call.CallDetail> serverSideCalls = getTopologyQueryDAO().loadSpecifiedDestOfServerSideEndpointRelations(
+            downsampling, startTB, endTB, endpointId);
 
         Topology topology = new Topology();
         serverSideCalls.forEach(callDetail -> {

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/query/ITopologyQueryDAO.java

@@ -21,29 +21,63 @@ package org.apache.skywalking.oap.server.core.storage.query;
 import java.io.IOException;
 import java.util.List;
 import org.apache.skywalking.oap.server.core.analysis.Downsampling;
+import org.apache.skywalking.oap.server.core.analysis.manual.relation.instance.ServiceInstanceRelationClientSideMetrics;
+import org.apache.skywalking.oap.server.core.analysis.manual.relation.instance.ServiceInstanceRelationServerSideMetrics;
+import org.apache.skywalking.oap.server.core.analysis.manual.relation.service.ServiceRelationClientSideMetrics;
+import org.apache.skywalking.oap.server.core.analysis.manual.relation.service.ServiceRelationServerSideMetrics;
 import org.apache.skywalking.oap.server.core.query.entity.Call;
 import org.apache.skywalking.oap.server.library.module.Service;
 
 public interface ITopologyQueryDAO extends Service {
-
+    /**
+     * Query {@link ServiceRelationServerSideMetrics} through the given conditions
+     */
     List<Call.CallDetail> loadSpecifiedServerSideServiceRelations(Downsampling downsampling, long startTB, long endTB,
-        List<Integer> serviceIds) throws IOException;
+                                                                  List<Integer> serviceIds) throws IOException;
 
+    /**
+     * Query {@link ServiceRelationClientSideMetrics} through the given conditions
+     */
     List<Call.CallDetail> loadSpecifiedClientSideServiceRelations(Downsampling downsampling, long startTB, long endTB,
-        List<Integer> serviceIds) throws IOException;
+                                                                  List<Integer> serviceIds) throws IOException;
 
+    /**
+     * Query {@link ServiceRelationServerSideMetrics} globally, without given serviceIds
+     */
     List<Call.CallDetail> loadServerSideServiceRelations(Downsampling downsampling, long startTB,
-        long endTB) throws IOException;
+                                                         long endTB) throws IOException;
 
+    /**
+     * Query {@link ServiceRelationClientSideMetrics} globally, without given serviceIds
+     */
     List<Call.CallDetail> loadClientSideServiceRelations(Downsampling downsampling, long startTB,
-        long endTB) throws IOException;
+                                                         long endTB) throws IOException;
 
-    List<Call.CallDetail> loadServerSideServiceInstanceRelations(int clientServiceId, int serverServiceId,
-        Downsampling downsampling, long startTB, long endTB) throws IOException;
+    /**
+     * Query {@link ServiceInstanceRelationServerSideMetrics} through given conditions, including the specific
+     * clientServiceId and serverServiceId
+     */
+    List<Call.CallDetail> loadServerSideServiceInstanceRelations(int clientServiceId,
+                                                                 int serverServiceId,
+                                                                 Downsampling downsampling,
+                                                                 long startTB,
+                                                                 long endTB) throws IOException;
 
-    List<Call.CallDetail> loadClientSideServiceInstanceRelations(int clientServiceId, int serverServiceId,
-        Downsampling downsampling, long startTB, long endTB) throws IOException;
+    /**
+     * Query {@link ServiceInstanceRelationClientSideMetrics} through given conditions, including the specific
+     * clientServiceId and serverServiceId
+     */
+    List<Call.CallDetail> loadClientSideServiceInstanceRelations(int clientServiceId,
+                                                                 int serverServiceId,
+                                                                 Downsampling downsampling,
+                                                                 long startTB,
+                                                                 long endTB) throws IOException;
 
-    List<Call.CallDetail> loadSpecifiedDestOfServerSideEndpointRelations(Downsampling downsampling, long startTB,
-        long endTB, int destEndpointId) throws IOException;
+    /**
+     * Query the endpoint relationship. Endpoint dependency is not detected from server side agent.
+     */
+    List<Call.CallDetail> loadSpecifiedDestOfServerSideEndpointRelations(Downsampling downsampling,
+                                                                         long startTB,
+                                                                         long endTB,
+                                                                         int destEndpointId) throws IOException;
 }

oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/TopologyQuery.java

@@ -20,6 +20,8 @@ package org.apache.skywalking.oap.query.graphql.resolver;
 
 import com.coxautodev.graphql.tools.GraphQLQueryResolver;
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
 import org.apache.skywalking.oap.query.graphql.type.Duration;
 import org.apache.skywalking.oap.server.core.CoreModule;
 import org.apache.skywalking.oap.server.core.query.DurationUtils;
@@ -49,29 +51,41 @@ public class TopologyQuery implements GraphQLQueryResolver {
         long startTimeBucket = DurationUtils.INSTANCE.exchangeToTimeBucket(duration.getStart());
         long endTimeBucket = DurationUtils.INSTANCE.exchangeToTimeBucket(duration.getEnd());
 
-        return getQueryService().getGlobalTopology(StepToDownsampling.transform(duration.getStep()), startTimeBucket, endTimeBucket);
+        return getQueryService().getGlobalTopology(
+            StepToDownsampling.transform(duration.getStep()), startTimeBucket, endTimeBucket);
     }
 
     public Topology getServiceTopology(final int serviceId, final Duration duration) throws IOException {
+        List<Integer> selectedServiceList = new ArrayList<>(1);
+        selectedServiceList.add(serviceId);
+        return this.getServicesTopology(selectedServiceList, duration);
+    }
+
+    public Topology getServicesTopology(final List<Integer> serviceIds, final Duration duration) throws IOException {
         long startTimeBucket = DurationUtils.INSTANCE.exchangeToTimeBucket(duration.getStart());
         long endTimeBucket = DurationUtils.INSTANCE.exchangeToTimeBucket(duration.getEnd());
 
-        return getQueryService().getServiceTopology(StepToDownsampling.transform(duration.getStep()), startTimeBucket, endTimeBucket, serviceId);
+        return getQueryService().getServiceTopology(
+            StepToDownsampling.transform(duration.getStep()), startTimeBucket, endTimeBucket, serviceIds);
     }
 
     public ServiceInstanceTopology getServiceInstanceTopology(final int clientServiceId, final int serverServiceId,
-        final Duration duration) throws IOException {
+                                                              final Duration duration) throws IOException {
         long startTimeBucket = DurationUtils.INSTANCE.exchangeToTimeBucket(duration.getStart());
         long endTimeBucket = DurationUtils.INSTANCE.exchangeToTimeBucket(duration.getEnd());
 
-        return getQueryService().getServiceInstanceTopology(clientServiceId, serverServiceId, StepToDownsampling.transform(duration
-            .getStep()), startTimeBucket, endTimeBucket);
+        return getQueryService().getServiceInstanceTopology(
+            clientServiceId, serverServiceId, StepToDownsampling.transform(duration
+                                                                               .getStep()), startTimeBucket,
+            endTimeBucket
+        );
     }
 
     public Topology getEndpointTopology(final int endpointId, final Duration duration) throws IOException {
         long startTimeBucket = DurationUtils.INSTANCE.exchangeToTimeBucket(duration.getStart());
         long endTimeBucket = DurationUtils.INSTANCE.exchangeToTimeBucket(duration.getEnd());
 
-        return getQueryService().getEndpointTopology(StepToDownsampling.transform(duration.getStep()), startTimeBucket, endTimeBucket, endpointId);
+        return getQueryService().getEndpointTopology(
+            StepToDownsampling.transform(duration.getStep()), startTimeBucket, endTimeBucket, endpointId);
     }
 }

query-protocol @ 1018b795

-Subproject commit 6104df8f0debce4c81142e5b1fc696d61bb673c5
+Subproject commit 1018b795021e0d96e6c131c8695fb9ddc9d66916

oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2TopologyQueryDAO.java

@@ -56,7 +56,7 @@ public class H2TopologyQueryDAO implements ITopologyQueryDAO {
     public List<Call.CallDetail> loadSpecifiedClientSideServiceRelations(Downsampling downsampling, long startTB,
         long endTB, List<Integer> serviceIds) throws IOException {
         String tableName = ModelName.build(downsampling, ServiceRelationClientSideMetrics.INDEX_NAME);
-        return loadServiceCalls(tableName, startTB, endTB, ServiceRelationServerSideMetrics.SOURCE_SERVICE_ID, ServiceRelationServerSideMetrics.DEST_SERVICE_ID, serviceIds, true);
+        return loadServiceCalls(tableName, startTB, endTB, ServiceRelationClientSideMetrics.SOURCE_SERVICE_ID, ServiceRelationClientSideMetrics.DEST_SERVICE_ID, serviceIds, true);
     }
 
     @Override
@@ -70,7 +70,7 @@ public class H2TopologyQueryDAO implements ITopologyQueryDAO {
     public List<Call.CallDetail> loadClientSideServiceRelations(Downsampling downsampling, long startTB,
         long endTB) throws IOException {
         String tableName = ModelName.build(downsampling, ServiceRelationClientSideMetrics.INDEX_NAME);
-        return loadServiceCalls(tableName, startTB, endTB, ServiceRelationServerSideMetrics.SOURCE_SERVICE_ID, ServiceRelationServerSideMetrics.DEST_SERVICE_ID, new ArrayList<>(0), true);
+        return loadServiceCalls(tableName, startTB, endTB, ServiceRelationClientSideMetrics.SOURCE_SERVICE_ID, ServiceRelationClientSideMetrics.DEST_SERVICE_ID, new ArrayList<>(0), true);
     }
 
     @Override