Add service type from register and support browser as service type (#4157)

* Add service type from register and support browser as service type

* Add service type at agent side.

* Fix format.

* Fix a test.

* Fix formats.
Co-authored-by: Nkezhenxu94 <kezhenxu94@163.com>

proto @ c331e840

-Subproject commit 7b244ff7ec350910295eee85633e02d92a6f6b1c
+Subproject commit c331e840908905460c0d8df7573fa33ea6764873

apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/remote/ServiceAndEndpointRegisterClient.java

@@ -41,6 +41,7 @@ import org.apache.skywalking.apm.agent.core.os.OSUtil;
 import org.apache.skywalking.apm.network.common.Commands;
 import org.apache.skywalking.apm.network.common.KeyIntValuePair;
 import org.apache.skywalking.apm.network.common.KeyStringValuePair;
+import org.apache.skywalking.apm.network.common.ServiceType;
 import org.apache.skywalking.apm.network.register.v2.RegisterGrpc;
 import org.apache.skywalking.apm.network.register.v2.Service;
 import org.apache.skywalking.apm.network.register.v2.ServiceInstance;
@@ -141,7 +142,7 @@ public class ServiceAndEndpointRegisterClient implements BootService, Runnable,
                 if (RemoteDownstreamConfig.Agent.SERVICE_ID == DictionaryUtil.nullValue()) {
                     if (registerBlockingStub != null) {
                         ServiceRegisterMapping serviceRegisterMapping = registerBlockingStub.withDeadlineAfter(GRPC_UPSTREAM_TIMEOUT, TimeUnit.SECONDS).doServiceRegister(
-                            Services.newBuilder().addServices(Service.newBuilder().setServiceName(Config.Agent.SERVICE_NAME)).build());
+                            Services.newBuilder().addServices(Service.newBuilder().setServiceName(Config.Agent.SERVICE_NAME).setType(ServiceType.normal)).build());
                         if (serviceRegisterMapping != null) {
                             for (KeyIntValuePair registered : serviceRegisterMapping.getServicesList()) {
                                 if (Config.Agent.SERVICE_NAME.equals(registered.getKey())) {

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/MetadataQueryService.java

@@ -77,6 +77,10 @@ public class MetadataQueryService implements org.apache.skywalking.oap.server.li
         return getMetadataQueryDAO().getAllServices(startTimestamp, endTimestamp);
     }
 
+    public List<Service> getAllBrowserServices(final long startTimestamp, final long endTimestamp) throws IOException {
+        return getMetadataQueryDAO().getAllBrowserServices(startTimestamp, endTimestamp);
+    }
+
     public List<Database> getAllDatabases() throws IOException {
         return getMetadataQueryDAO().getAllDatabases();
     }

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/register/NodeType.java

@@ -18,6 +18,7 @@
 
 package org.apache.skywalking.oap.server.core.register;
 
+import org.apache.skywalking.apm.network.common.ServiceType;
 import org.apache.skywalking.oap.server.core.UnexpectedException;
 
 /**
@@ -32,7 +33,7 @@ public enum NodeType {
     /**
      * <code>Unknown = 0;</code>
      */
-    Unknown(0),
+    Normal(0),
     /**
      * <code>Database = 1;</code>
      */
@@ -53,6 +54,10 @@ public enum NodeType {
      * <code>Cache = 5;</code>
      */
     Cache(5),
+    /**
+     * <code>Browser = 6;</code>
+     */
+    Browser(6),
     UNRECOGNIZED(-1);
 
     private final int value;
@@ -68,7 +73,7 @@ public enum NodeType {
     public static NodeType get(int value) {
         switch (value) {
             case 0:
-                return Unknown;
+                return Normal;
             case 1:
                 return Database;
             case 2:
@@ -95,4 +100,21 @@ public enum NodeType {
     public static NodeType fromSpanLayerValue(int spanLayerValue) {
         return get(spanLayerValue);
     }
+
+    public static NodeType fromRegisterServiceType(ServiceType serviceType) {
+        switch (serviceType) {
+            case normal:
+                return Normal;
+            case database:
+                return Database;
+            case cache:
+                return Cache;
+            case mq:
+                return MQ;
+            case browser:
+                return Browser;
+            default:
+                return Normal;
+        }
+    }
 }

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/register/ServiceInventory.java

@@ -58,7 +58,10 @@ public class ServiceInventory extends RegisterSource {
     @Setter @Getter @Column(columnName = NAME, matchQuery = true) private String name = Const.EMPTY_STRING;
     @Setter @Getter @Column(columnName = IS_ADDRESS) private int isAddress;
     @Setter @Getter @Column(columnName = ADDRESS_ID) private int addressId;
-    @Setter(AccessLevel.PRIVATE) @Getter(AccessLevel.PRIVATE) @Column(columnName = NODE_TYPE) private int nodeType;
+    /**
+     * Node type of the service, default {@link NodeType#Normal}
+     */
+    @Setter(AccessLevel.PRIVATE) @Getter(AccessLevel.PRIVATE) @Column(columnName = NODE_TYPE) private int nodeType = 0;
     @Setter @Getter @Column(columnName = MAPPING_SERVICE_ID) private int mappingServiceId;
     @Getter(AccessLevel.PRIVATE) @Column(columnName = PROPERTIES) private String prop = Const.EMPTY_JSON_OBJECT_STRING;
     @Getter private JsonObject properties;

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/register/service/IServiceInventoryRegister.java

@@ -29,6 +29,8 @@ public interface IServiceInventoryRegister extends Service {
 
     int getOrCreate(String serviceName, JsonObject properties);
 
+    int getOrCreate(String serviceName, NodeType nodeType, JsonObject properties);
+
     int getOrCreate(int addressId, String serviceName, JsonObject properties);
 
     void update(int serviceId, NodeType nodeType, JsonObject properties);

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/register/service/ServiceInventoryRegister.java

@@ -55,7 +55,7 @@ public class ServiceInventoryRegister implements IServiceInventoryRegister {
         return serviceInventoryCache;
     }
 
-    @Override public int getOrCreate(String serviceName, JsonObject properties) {
+    @Override public int getOrCreate(String serviceName, NodeType nodeType, JsonObject properties) {
         int serviceId = getServiceInventoryCache().getServiceId(serviceName);
 
         if (serviceId == Const.NONE) {
@@ -67,6 +67,7 @@ public class ServiceInventoryRegister implements IServiceInventoryRegister {
             long now = System.currentTimeMillis();
             serviceInventory.setRegisterTime(now);
             serviceInventory.setHeartbeatTime(now);
+            serviceInventory.setServiceNodeType(nodeType);
             serviceInventory.setMappingServiceId(Const.NONE);
             serviceInventory.setLastUpdateTime(now);
             serviceInventory.setProperties(properties);
@@ -76,6 +77,10 @@ public class ServiceInventoryRegister implements IServiceInventoryRegister {
         return serviceId;
     }
 
+    @Override public int getOrCreate(String serviceName, JsonObject properties) {
+        return getOrCreate(serviceName, NodeType.Normal, properties);
+    }
+
     @Override public int getOrCreate(int addressId, String serviceName, JsonObject properties) {
         int serviceId = getServiceInventoryCache().getServiceId(addressId);
 

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/query/IMetadataQueryDAO.java

@@ -36,6 +36,8 @@ public interface IMetadataQueryDAO extends DAO {
 
     List<Service> getAllServices(final long startTimestamp, final long endTimestamp) throws IOException;
 
+    List<Service> getAllBrowserServices(long startTimestamp, long endTimestamp) throws IOException;
+
     List<Database> getAllDatabases() throws IOException;
 
     List<Service> searchServices(final long startTimestamp, final long endTimestamp,

oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/resolver/MetadataQuery.java

@@ -64,6 +64,13 @@ public class MetadataQuery implements GraphQLQueryResolver {
         return getMetadataQueryService().getAllServices(startTimestamp, endTimestamp);
     }
 
+    public List<Service> getAllBrowserServices(final Duration duration) throws IOException, ParseException {
+        long startTimestamp = DurationUtils.INSTANCE.startTimeToTimestamp(duration.getStep(), duration.getStart());
+        long endTimestamp = DurationUtils.INSTANCE.endTimeToTimestamp(duration.getStep(), duration.getEnd());
+
+        return getMetadataQueryService().getAllBrowserServices(startTimestamp, endTimestamp);
+    }
+
     public List<Service> searchServices(final Duration duration, final String keyword)
         throws IOException, ParseException {
         long startTimestamp = DurationUtils.INSTANCE.startTimeToTimestamp(duration.getStep(), duration.getStart());

query-protocol @ a9b48130

-Subproject commit a52c7a957f02a37b9409b52fc55db774f20291bf
+Subproject commit a9b48130626e4b4dcf46bb8268c7125cc6f50814

oap-server/server-receiver-plugin/skywalking-register-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/register/provider/handler/v6/grpc/RegisterServiceHandler.java

@@ -25,6 +25,7 @@ import java.util.List;
 import org.apache.skywalking.apm.network.common.Commands;
 import org.apache.skywalking.apm.network.common.KeyIntValuePair;
 import org.apache.skywalking.apm.network.common.KeyStringValuePair;
+import org.apache.skywalking.apm.network.common.ServiceType;
 import org.apache.skywalking.apm.network.register.v2.EndpointMapping;
 import org.apache.skywalking.apm.network.register.v2.EndpointMappingElement;
 import org.apache.skywalking.apm.network.register.v2.Endpoints;
@@ -41,6 +42,7 @@ import org.apache.skywalking.oap.server.core.Const;
 import org.apache.skywalking.oap.server.core.CoreModule;
 import org.apache.skywalking.oap.server.core.cache.ServiceInstanceInventoryCache;
 import org.apache.skywalking.oap.server.core.cache.ServiceInventoryCache;
+import org.apache.skywalking.oap.server.core.register.NodeType;
 import org.apache.skywalking.oap.server.core.register.ServiceInstanceInventory;
 import org.apache.skywalking.oap.server.core.register.ServiceInventory;
 import org.apache.skywalking.oap.server.core.register.service.IEndpointInventoryRegister;
@@ -90,7 +92,14 @@ public class RegisterServiceHandler extends RegisterGrpc.RegisterImplBase implem
             if (logger.isDebugEnabled()) {
                 logger.debug("Register service, service code: {}", serviceName);
             }
-            int serviceId = serviceInventoryRegister.getOrCreate(serviceName, null);
+
+            ServiceType serviceType = service.getType();
+            if (serviceType == null) {
+                // All service register from agents before 7.0.0, should be be null.
+                serviceType = ServiceType.normal;
+            }
+
+            int serviceId = serviceInventoryRegister.getOrCreate(serviceName, NodeType.fromRegisterServiceType(serviceType), null);
 
             if (serviceId != Const.NONE) {
                 KeyIntValuePair value = KeyIntValuePair.newBuilder().setKey(serviceName).setValue(serviceId).build();

oap-server/server-receiver-plugin/zipkin-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/zipkin/analysis/transform/SpringSleuthSegmentBuilderTest.java

@@ -56,6 +56,10 @@ public class SpringSleuthSegmentBuilderTest implements SegmentListener {
                 }
             }
 
+            @Override public int getOrCreate(String serviceName, NodeType nodeType, JsonObject properties) {
+                return getOrCreate(serviceName, properties);
+            }
+
             @Override public int getOrCreate(int addressId, String serviceName, JsonObject properties) {
                 String key = "Address:" + serviceName;
                 if (applicationRegister.containsKey(key)) {

oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/MetadataQueryEsDAO.java

@@ -76,6 +76,7 @@ public class MetadataQueryEsDAO extends EsDAO implements IMetadataQueryDAO {
         boolQueryBuilder.must().add(timeRangeQueryBuild(startTimestamp, endTimestamp));
 
         boolQueryBuilder.must().add(QueryBuilders.termQuery(ServiceInventory.IS_ADDRESS, BooleanUtils.FALSE));
+        boolQueryBuilder.must().add(QueryBuilders.termQuery(ServiceInventory.NODE_TYPE, NodeType.Normal.value()));
 
         sourceBuilder.query(boolQueryBuilder);
         sourceBuilder.size(0);
@@ -118,6 +119,25 @@ public class MetadataQueryEsDAO extends EsDAO implements IMetadataQueryDAO {
         boolQueryBuilder.must().add(timeRangeQueryBuild(startTimestamp, endTimestamp));
 
         boolQueryBuilder.must().add(QueryBuilders.termQuery(ServiceInventory.IS_ADDRESS, BooleanUtils.FALSE));
+        boolQueryBuilder.must().add(QueryBuilders.termQuery(ServiceInventory.NODE_TYPE, NodeType.Normal.value()));
+
+        sourceBuilder.query(boolQueryBuilder);
+        sourceBuilder.size(queryMaxSize);
+
+        SearchResponse response = getClient().search(ServiceInventory.INDEX_NAME, sourceBuilder);
+
+        return buildServices(response);
+    }
+
+    @Override
+    public List<Service> getAllBrowserServices(long startTimestamp, long endTimestamp) throws IOException {
+        SearchSourceBuilder sourceBuilder = SearchSourceBuilder.searchSource();
+
+        BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
+        boolQueryBuilder.must().add(timeRangeQueryBuild(startTimestamp, endTimestamp));
+
+        boolQueryBuilder.must().add(QueryBuilders.termQuery(ServiceInventory.IS_ADDRESS, BooleanUtils.FALSE));
+        boolQueryBuilder.must().add(QueryBuilders.termQuery(ServiceInventory.NODE_TYPE, NodeType.Browser.value()));
 
         sourceBuilder.query(boolQueryBuilder);
         sourceBuilder.size(queryMaxSize);
@@ -166,6 +186,7 @@ public class MetadataQueryEsDAO extends EsDAO implements IMetadataQueryDAO {
         BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
         boolQueryBuilder.must().add(timeRangeQueryBuild(startTimestamp, endTimestamp));
         boolQueryBuilder.must().add(QueryBuilders.termQuery(ServiceInventory.IS_ADDRESS, BooleanUtils.FALSE));
+        boolQueryBuilder.must().add(QueryBuilders.termQuery(ServiceInventory.NODE_TYPE, NodeType.Normal.value()));
 
         if (!Strings.isNullOrEmpty(keyword)) {
             String matchCName = MatchCNameBuilder.INSTANCE.build(ServiceInventory.NAME);

oap-server/server-storage-plugin/storage-elasticsearch7-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch7/query/MetadataQueryEs7DAO.java

@@ -18,7 +18,9 @@
 
 package org.apache.skywalking.oap.server.storage.plugin.elasticsearch7.query;
 
+import java.io.IOException;
 import org.apache.skywalking.oap.server.core.register.EndpointInventory;
+import org.apache.skywalking.oap.server.core.register.NodeType;
 import org.apache.skywalking.oap.server.core.register.ServiceInventory;
 import org.apache.skywalking.oap.server.core.source.DetectPoint;
 import org.apache.skywalking.oap.server.library.client.elasticsearch.ElasticSearchClient;
@@ -29,8 +31,6 @@ import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 
-import java.io.IOException;
-
 /**
  * @author peng-yongsheng
  */
@@ -48,12 +48,13 @@ public class MetadataQueryEs7DAO extends MetadataQueryEsDAO {
         boolQueryBuilder.must().add(timeRangeQueryBuild(startTimestamp, endTimestamp));
 
         boolQueryBuilder.must().add(QueryBuilders.termQuery(ServiceInventory.IS_ADDRESS, BooleanUtils.FALSE));
+        boolQueryBuilder.must().add(QueryBuilders.termQuery(ServiceInventory.NODE_TYPE, NodeType.Normal.value()));
 
         sourceBuilder.query(boolQueryBuilder);
         sourceBuilder.size(0);
 
         SearchResponse response = getClient().search(ServiceInventory.INDEX_NAME, sourceBuilder);
-        return (int) response.getHits().getTotalHits().value;
+        return (int)response.getHits().getTotalHits().value;
     }
 
     @Override
@@ -68,7 +69,7 @@ public class MetadataQueryEs7DAO extends MetadataQueryEsDAO {
         sourceBuilder.size(0);
 
         SearchResponse response = getClient().search(EndpointInventory.INDEX_NAME, sourceBuilder);
-        return (int) response.getHits().getTotalHits().value;
+        return (int)response.getHits().getTotalHits().value;
     }
 
     @Override
@@ -80,7 +81,7 @@ public class MetadataQueryEs7DAO extends MetadataQueryEsDAO {
 
         SearchResponse response = getClient().search(ServiceInventory.INDEX_NAME, sourceBuilder);
 
-        return (int) response.getHits().getTotalHits().value;
+        return (int)response.getHits().getTotalHits().value;
     }
 
 }

oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2MetadataQueryDAO.java

@@ -71,7 +71,8 @@ public class H2MetadataQueryDAO implements IMetadataQueryDAO {
         List<Object> condition = new ArrayList<>(5);
         sql.append("select count(*) num from ").append(ServiceInventory.INDEX_NAME).append(" where ");
         setTimeRangeCondition(sql, condition, startTimestamp, endTimestamp);
-        sql.append(" and ").append(ServiceInventory.IS_ADDRESS).append("=0");
+        sql.append(" and ").append(ServiceInventory.IS_ADDRESS).append("=" + BooleanUtils.FALSE);
+        sql.append(" and ").append(ServiceInventory.NODE_TYPE).append("=" + NodeType.Normal.value());
 
         try (Connection connection = h2Client.getConnection()) {
             try (ResultSet resultSet = h2Client.executeQuery(connection, sql.toString(), condition.toArray(new Object[0]))) {
@@ -131,8 +132,32 @@ public class H2MetadataQueryDAO implements IMetadataQueryDAO {
         List<Object> condition = new ArrayList<>(5);
         sql.append("select * from ").append(ServiceInventory.INDEX_NAME).append(" where ");
         setTimeRangeCondition(sql, condition, startTimestamp, endTimestamp);
-        sql.append(" and ").append(ServiceInventory.IS_ADDRESS).append("=? limit ").append(metadataQueryMaxSize);
+        sql.append(" and ").append(ServiceInventory.IS_ADDRESS).append("=?");
+        condition.add(BooleanUtils.FALSE);
+        sql.append(" and ").append(ServiceInventory.NODE_TYPE).append("=?");
+        condition.add(NodeType.Normal.value());
+        sql.append(" limit ").append(metadataQueryMaxSize);
+
+        try (Connection connection = h2Client.getConnection()) {
+            try (ResultSet resultSet = h2Client.executeQuery(connection, sql.toString(), condition.toArray(new Object[0]))) {
+                return buildServices(resultSet);
+            }
+        } catch (SQLException e) {
+            throw new IOException(e);
+        }
+    }
+
+    @Override
+    public List<Service> getAllBrowserServices(long startTimestamp, long endTimestamp) throws IOException {
+        StringBuilder sql = new StringBuilder();
+        List<Object> condition = new ArrayList<>(5);
+        sql.append("select * from ").append(ServiceInventory.INDEX_NAME).append(" where ");
+        setTimeRangeCondition(sql, condition, startTimestamp, endTimestamp);
+        sql.append(" and ").append(ServiceInventory.IS_ADDRESS).append("=?");
         condition.add(BooleanUtils.FALSE);
+        sql.append(" and ").append(ServiceInventory.NODE_TYPE).append("=?");
+        condition.add(NodeType.Browser.value());
+        sql.append(" limit ").append(metadataQueryMaxSize);
 
         try (Connection connection = h2Client.getConnection()) {
             try (ResultSet resultSet = h2Client.executeQuery(connection, sql.toString(), condition.toArray(new Object[0]))) {
@@ -185,6 +210,8 @@ public class H2MetadataQueryDAO implements IMetadataQueryDAO {
         setTimeRangeCondition(sql, condition, startTimestamp, endTimestamp);
         sql.append(" and ").append(ServiceInventory.IS_ADDRESS).append("=?");
         condition.add(BooleanUtils.FALSE);
+        sql.append(" and ").append(ServiceInventory.NODE_TYPE).append("=?");
+        condition.add(NodeType.Normal.value());
         if (!Strings.isNullOrEmpty(keyword)) {
             sql.append(" and ").append(ServiceInventory.NAME).append(" like \"%").append(keyword).append("%\"");
         }