diff --git a/docs/en/guides/README.md b/docs/en/guides/README.md index 26f0033664349caf0fa981b18fad0a661fba6e36..af3c1926efdc3b6ca0916b95ec5bf76a66755667 100644 --- a/docs/en/guides/README.md +++ b/docs/en/guides/README.md @@ -37,6 +37,8 @@ and private plugin developer should read this. - [Storage extension development guide](storage-extention.md). Help potential contributors to build a new storage implementor besides the official. - [Customize analysis by oal script](write-oal.md). Guide you to use oal script to make your own metric available. +- [Backend Inventory entity extension](inventory-extension.md). If you want to extend SkyWalking inventory entities, and +want to push upstream back to our Apache OSS repo, please read these principles. ### UI developer Our UI is constituted by static pages and web container. diff --git a/docs/en/guides/inventory-extension.md b/docs/en/guides/inventory-extension.md new file mode 100644 index 0000000000000000000000000000000000000000..cee70eb2884c593c66362b35f91b3bdae9bbb252 --- /dev/null +++ b/docs/en/guides/inventory-extension.md @@ -0,0 +1,31 @@ +# Backend Inventory Entity Extension +SkyWalking includes four inventory entities. +- Service Inventory +- Service Instance Inventory +- Endpoint Inventory +- Network Address Inventory + +All metric, topology, trace and alarm are related to these entity IDs. + +For understanding the **Service**, **Service Instance** and **Endpoint** concepts, +please read [Project Overview](../concepts-and-designs/overview.md#why-use-skywalking). + +For **Network Address Inventory**, it represents all network address, in IP:port, hostname, domain name +formats, which are detected by language agents or other probes. + +## Extension +Right now, only **Service Inventory** extension is already supported in backend core. +Service provides field `properties` in Json format, which is usually used for specific service +rather than normal business services, such as Database, Cache, MQ, etc. + +For keeping code consistent and friendly in query and visualization, the Json properties +need to follow the rules. + +### Database +1. NodeType == **Database(1)** +1. Json properties include following keys. + - `database`. Database name, such as MySQL, PostgreSQL + - `db.type`. Database type, such as sql db, redis db. + - `db.instance`. Database instance name. + + diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/register/NetworkAddressInventory.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/register/NetworkAddressInventory.java index 0647a2c3861aebc0af687712e73aa375289620d7..3fc26cde1392bf2339a206884fd6bb38c71a6e2f 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/register/NetworkAddressInventory.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/register/NetworkAddressInventory.java @@ -82,6 +82,17 @@ public class NetworkAddressInventory extends RegisterSource { return true; } + public NetworkAddressInventory getClone() { + NetworkAddressInventory inventory = new NetworkAddressInventory(); + inventory.setSequence(getSequence()); + inventory.setRegisterTime(getRegisterTime()); + inventory.setHeartbeatTime(getHeartbeatTime()); + inventory.setName(name); + inventory.setNodeType(nodeType); + + return inventory; + } + @Override public void combine(RegisterSource registerSource) { super.combine(registerSource); NetworkAddressInventory inventory = (NetworkAddressInventory)registerSource; diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/register/ServiceInventory.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/register/ServiceInventory.java index 068d7b44f188d764815c7908d8e4e2d832477f55..676ba52b5ea935e89a5765b2c2862d300833b995 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/register/ServiceInventory.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/register/ServiceInventory.java @@ -18,6 +18,7 @@ package org.apache.skywalking.oap.server.core.register; +import com.google.gson.*; import java.util.*; import lombok.*; import org.apache.skywalking.oap.server.core.Const; @@ -46,6 +47,7 @@ public class ServiceInventory extends RegisterSource { public static final String NODE_TYPE = "node_type"; public static final String MAPPING_SERVICE_ID = "mapping_service_id"; public static final String MAPPING_LAST_UPDATE_TIME = "mapping_last_update_time"; + public static final String PROPERTIES = "properties"; @Setter @Getter @Column(columnName = NAME, matchQuery = true) private String name = Const.EMPTY_STRING; @Setter @Getter @Column(columnName = IS_ADDRESS) private int isAddress; @@ -53,6 +55,8 @@ public class ServiceInventory extends RegisterSource { @Setter(AccessLevel.PRIVATE) @Getter(AccessLevel.PRIVATE) @Column(columnName = NODE_TYPE) private int nodeType; @Setter @Getter @Column(columnName = MAPPING_SERVICE_ID) private int mappingServiceId; @Setter @Getter @Column(columnName = MAPPING_LAST_UPDATE_TIME) private long mappingLastUpdateTime; + @Getter(AccessLevel.PRIVATE) @Column(columnName = PROPERTIES) private String prop; + @Getter private JsonObject properties; public NodeType getServiceNodeType() { return NodeType.get(this.nodeType); @@ -86,6 +90,24 @@ public class ServiceInventory extends RegisterSource { return result; } + public void setProperties(JsonObject properties) { + this.properties = properties; + if (properties != null && properties.keySet().size() > 0) { + this.prop = properties.toString(); + } + } + + private void setProp(String prop) { + this.prop = prop; + if (!Strings.isNullOrEmpty(prop)) { + this.properties = new Gson().fromJson(prop, JsonObject.class); + } + } + + public boolean hasProperties() { + return prop != null && prop.length() > 0; + } + public ServiceInventory getClone() { ServiceInventory inventory = new ServiceInventory(); inventory.setSequence(getSequence()); @@ -97,6 +119,7 @@ public class ServiceInventory extends RegisterSource { inventory.setAddressId(addressId); inventory.setMappingLastUpdateTime(mappingLastUpdateTime); inventory.setMappingServiceId(mappingServiceId); + inventory.setProp(prop); return inventory; } @@ -133,6 +156,7 @@ public class ServiceInventory extends RegisterSource { remoteBuilder.addDataLongs(getMappingLastUpdateTime()); remoteBuilder.addDataStrings(Strings.isNullOrEmpty(name) ? Const.EMPTY_STRING : name); + remoteBuilder.addDataStrings(Strings.isNullOrEmpty(prop) ? Const.EMPTY_STRING : prop); return remoteBuilder; } @@ -148,6 +172,8 @@ public class ServiceInventory extends RegisterSource { setMappingLastUpdateTime(remoteData.getDataLongs(2)); setName(remoteData.getDataStrings(0)); + setProp(remoteData.getDataStrings(1)); + } @Override public int remoteHashCode() { @@ -158,6 +184,7 @@ public class ServiceInventory extends RegisterSource { super.combine(registerSource); ServiceInventory serviceInventory = (ServiceInventory)registerSource; nodeType = serviceInventory.nodeType; + setProp(serviceInventory.getProp()); if (Const.NONE != serviceInventory.getMappingServiceId() && serviceInventory.getMappingLastUpdateTime() >= this.getMappingLastUpdateTime()) { this.mappingServiceId = serviceInventory.getMappingServiceId(); this.mappingLastUpdateTime = serviceInventory.getMappingLastUpdateTime(); @@ -177,6 +204,7 @@ public class ServiceInventory extends RegisterSource { inventory.setRegisterTime((Long)dbMap.get(REGISTER_TIME)); inventory.setHeartbeatTime((Long)dbMap.get(HEARTBEAT_TIME)); inventory.setMappingLastUpdateTime((Long)dbMap.get(MAPPING_LAST_UPDATE_TIME)); + inventory.setProp((String)dbMap.get(PROPERTIES)); return inventory; } @@ -191,6 +219,7 @@ public class ServiceInventory extends RegisterSource { map.put(REGISTER_TIME, storageData.getRegisterTime()); map.put(HEARTBEAT_TIME, storageData.getHeartbeatTime()); map.put(MAPPING_LAST_UPDATE_TIME, storageData.getMappingLastUpdateTime()); + map.put(PROPERTIES, storageData.getProp()); return map; } } diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/register/service/INetworkAddressInventoryRegister.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/register/service/INetworkAddressInventoryRegister.java index d5f190ff60081297346b33dbdb249a533461f287..32e78ea522217efa51f375f27fd0d41af1f482e0 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/register/service/INetworkAddressInventoryRegister.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/register/service/INetworkAddressInventoryRegister.java @@ -18,6 +18,7 @@ package org.apache.skywalking.oap.server.core.register.service; +import com.google.gson.JsonObject; import org.apache.skywalking.oap.server.core.register.NodeType; import org.apache.skywalking.oap.server.library.module.Service; @@ -25,7 +26,7 @@ import org.apache.skywalking.oap.server.library.module.Service; * @author peng-yongsheng */ public interface INetworkAddressInventoryRegister extends Service { - int getOrCreate(String networkAddress); + int getOrCreate(String networkAddress, JsonObject properties); int get(String networkAddress); diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/register/service/IServiceInventoryRegister.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/register/service/IServiceInventoryRegister.java index 398149c1e0aa06eee6ed6fe80b2df6a8222d5c66..c4d7ef02fd343803439b36bf09c849b1c56702b6 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/register/service/IServiceInventoryRegister.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/register/service/IServiceInventoryRegister.java @@ -18,6 +18,7 @@ package org.apache.skywalking.oap.server.core.register.service; +import com.google.gson.JsonObject; import org.apache.skywalking.oap.server.library.module.Service; /** @@ -25,9 +26,11 @@ import org.apache.skywalking.oap.server.library.module.Service; */ public interface IServiceInventoryRegister extends Service { - int getOrCreate(String serviceName); + int getOrCreate(String serviceName, JsonObject properties); - int getOrCreate(int addressId, String serviceName); + int getOrCreate(int addressId, String serviceName, JsonObject properties); + + void updateProperties(int serviceId, JsonObject properties); void heartbeat(int serviceId, long heartBeatTime); diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/register/service/NetworkAddressInventoryRegister.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/register/service/NetworkAddressInventoryRegister.java index 18d05b200a867d0f1a577bf854942909e8cf98b4..3910299716869e13ec9ff1cc7413766667efee67 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/register/service/NetworkAddressInventoryRegister.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/register/service/NetworkAddressInventoryRegister.java @@ -18,6 +18,7 @@ package org.apache.skywalking.oap.server.core.register.service; +import com.google.gson.JsonObject; import java.util.Objects; import org.apache.skywalking.oap.server.core.*; import org.apache.skywalking.oap.server.core.cache.*; @@ -73,11 +74,11 @@ public class NetworkAddressInventoryRegister implements INetworkAddressInventory return this.serviceInstanceInventoryRegister; } - @Override public int getOrCreate(String networkAddress) { + @Override public int getOrCreate(String networkAddress, JsonObject properties) { int addressId = getNetworkAddressInventoryCache().getAddressId(networkAddress); if (addressId != Const.NONE) { - int serviceId = getServiceInventoryRegister().getOrCreate(addressId, networkAddress); + int serviceId = getServiceInventoryRegister().getOrCreate(addressId, networkAddress, properties); if (serviceId != Const.NONE) { int serviceInstanceId = getServiceInstanceInventoryRegister().getOrCreate(serviceId, addressId, System.currentTimeMillis()); @@ -107,6 +108,7 @@ public class NetworkAddressInventoryRegister implements INetworkAddressInventory @Override public void heartbeat(int addressId, long heartBeatTime) { NetworkAddressInventory networkAddress = getNetworkAddressInventoryCache().get(addressId); if (Objects.nonNull(networkAddress)) { + networkAddress = networkAddress.getClone(); networkAddress.setHeartbeatTime(heartBeatTime); InventoryProcess.INSTANCE.in(networkAddress); @@ -119,7 +121,7 @@ public class NetworkAddressInventoryRegister implements INetworkAddressInventory NetworkAddressInventory networkAddress = getNetworkAddressInventoryCache().get(addressId); if (!this.compare(networkAddress, nodeType)) { - NetworkAddressInventory newNetworkAddress = getNetworkAddressInventoryCache().get(addressId); + NetworkAddressInventory newNetworkAddress = networkAddress.getClone(); newNetworkAddress.setNetworkAddressNodeType(nodeType); newNetworkAddress.setHeartbeatTime(System.currentTimeMillis()); @@ -128,6 +130,7 @@ public class NetworkAddressInventoryRegister implements INetworkAddressInventory ServiceInventory newServiceInventory = getServiceInventoryCache().get(getServiceInventoryCache().getServiceId(networkAddress.getSequence())); if (!this.compare(newServiceInventory, nodeType)) { + newServiceInventory = newServiceInventory.getClone(); newServiceInventory.setServiceNodeType(nodeType); newServiceInventory.setHeartbeatTime(System.currentTimeMillis()); diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/register/service/ServiceInventoryRegister.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/register/service/ServiceInventoryRegister.java index 04b60dba76e7ffb7411a9d1ea1e3e5d9563d5c1f..25e06ab7d2f6274a3e73de96734bf1c5c3e69dae 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/register/service/ServiceInventoryRegister.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/register/service/ServiceInventoryRegister.java @@ -18,10 +18,11 @@ package org.apache.skywalking.oap.server.core.register.service; +import com.google.gson.JsonObject; import java.util.Objects; import org.apache.skywalking.oap.server.core.*; import org.apache.skywalking.oap.server.core.cache.ServiceInventoryCache; -import org.apache.skywalking.oap.server.core.register.ServiceInventory; +import org.apache.skywalking.oap.server.core.register.*; import org.apache.skywalking.oap.server.core.register.worker.InventoryProcess; import org.apache.skywalking.oap.server.library.module.ModuleManager; import org.apache.skywalking.oap.server.library.util.BooleanUtils; @@ -50,7 +51,7 @@ public class ServiceInventoryRegister implements IServiceInventoryRegister { return serviceInventoryCache; } - @Override public int getOrCreate(String serviceName) { + @Override public int getOrCreate(String serviceName, JsonObject properties) { int serviceId = getServiceInventoryCache().getServiceId(serviceName); if (serviceId == Const.NONE) { @@ -64,13 +65,14 @@ public class ServiceInventoryRegister implements IServiceInventoryRegister { serviceInventory.setHeartbeatTime(now); serviceInventory.setMappingServiceId(Const.NONE); serviceInventory.setMappingLastUpdateTime(now); + serviceInventory.setProperties(properties); InventoryProcess.INSTANCE.in(serviceInventory); } return serviceId; } - @Override public int getOrCreate(int addressId, String serviceName) { + @Override public int getOrCreate(int addressId, String serviceName, JsonObject properties) { int serviceId = getServiceInventoryCache().getServiceId(addressId); if (serviceId == Const.NONE) { @@ -89,9 +91,23 @@ public class ServiceInventoryRegister implements IServiceInventoryRegister { return serviceId; } + @Override public void updateProperties(int serviceId, JsonObject properties) { + ServiceInventory serviceInventory = getServiceInventoryCache().get(serviceId); + if (Objects.nonNull(serviceInventory)) { + serviceInventory = serviceInventory.getClone(); + serviceInventory.setProperties(properties); + serviceInventory.setMappingLastUpdateTime(System.currentTimeMillis()); + + InventoryProcess.INSTANCE.in(serviceInventory); + } else { + logger.warn("Service {} properties update, but not found in storage."); + } + } + @Override public void heartbeat(int serviceId, long heartBeatTime) { ServiceInventory serviceInventory = getServiceInventoryCache().get(serviceId); if (Objects.nonNull(serviceInventory)) { + serviceInventory = serviceInventory.getClone(); serviceInventory.setHeartbeatTime(heartBeatTime); InventoryProcess.INSTANCE.in(serviceInventory); diff --git a/oap-server/server-receiver-plugin/skywalking-mesh-receiver-plugin/src/main/java/org/apache/skywalking/aop/server/receiver/mesh/ServiceMeshMetricDataDecorator.java b/oap-server/server-receiver-plugin/skywalking-mesh-receiver-plugin/src/main/java/org/apache/skywalking/aop/server/receiver/mesh/ServiceMeshMetricDataDecorator.java index 4861955871fca95444ec0bd607c04296a3706002..abdbb10bea4d0a637729eaea24fa815d1bcc9b0b 100644 --- a/oap-server/server-receiver-plugin/skywalking-mesh-receiver-plugin/src/main/java/org/apache/skywalking/aop/server/receiver/mesh/ServiceMeshMetricDataDecorator.java +++ b/oap-server/server-receiver-plugin/skywalking-mesh-receiver-plugin/src/main/java/org/apache/skywalking/aop/server/receiver/mesh/ServiceMeshMetricDataDecorator.java @@ -44,7 +44,7 @@ public class ServiceMeshMetricDataDecorator { boolean isRegistered = true; sourceServiceId = origin.getSourceServiceId(); if (sourceServiceId == Const.NONE) { - sourceServiceId = CoreRegisterLinker.getServiceInventoryRegister().getOrCreate(origin.getSourceServiceName()); + sourceServiceId = CoreRegisterLinker.getServiceInventoryRegister().getOrCreate(origin.getSourceServiceName(), null); if (sourceServiceId != Const.NONE) { getNewDataBuilder().setSourceServiceId(sourceServiceId); } else { @@ -65,7 +65,7 @@ public class ServiceMeshMetricDataDecorator { } destServiceId = origin.getDestServiceId(); if (destServiceId == Const.NONE) { - destServiceId = CoreRegisterLinker.getServiceInventoryRegister().getOrCreate(origin.getDestServiceName()); + destServiceId = CoreRegisterLinker.getServiceInventoryRegister().getOrCreate(origin.getDestServiceName(), null); if (destServiceId != Const.NONE) { getNewDataBuilder().setDestServiceId(destServiceId); } else { diff --git a/oap-server/server-receiver-plugin/skywalking-register-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/register/provider/handler/v5/grpc/ApplicationRegisterHandler.java b/oap-server/server-receiver-plugin/skywalking-register-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/register/provider/handler/v5/grpc/ApplicationRegisterHandler.java index 44eec839a95f8b8ed6af26d49fb415f9494621cb..f0e2b5ef1a814e3b09fef4cf4e9db9e6526008bb 100644 --- a/oap-server/server-receiver-plugin/skywalking-register-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/register/provider/handler/v5/grpc/ApplicationRegisterHandler.java +++ b/oap-server/server-receiver-plugin/skywalking-register-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/register/provider/handler/v5/grpc/ApplicationRegisterHandler.java @@ -47,7 +47,7 @@ public class ApplicationRegisterHandler extends ApplicationRegisterServiceGrpc.A ApplicationMapping.Builder builder = ApplicationMapping.newBuilder(); String serviceName = request.getApplicationCode(); - int serviceId = serviceInventoryRegister.getOrCreate(serviceName); + int serviceId = serviceInventoryRegister.getOrCreate(serviceName, null); if (serviceId != Const.NONE) { KeyWithIntegerValue value = KeyWithIntegerValue.newBuilder().setKey(serviceName).setValue(serviceId).build(); diff --git a/oap-server/server-receiver-plugin/skywalking-register-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/register/provider/handler/v5/grpc/NetworkAddressRegisterServiceHandler.java b/oap-server/server-receiver-plugin/skywalking-register-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/register/provider/handler/v5/grpc/NetworkAddressRegisterServiceHandler.java index 5c88d63fe043c147c6cec699ecdc54bee90c8c6b..a1adb9e57bbfeee81a49427abddab0c71a9ada70 100644 --- a/oap-server/server-receiver-plugin/skywalking-register-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/register/provider/handler/v5/grpc/NetworkAddressRegisterServiceHandler.java +++ b/oap-server/server-receiver-plugin/skywalking-register-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/register/provider/handler/v5/grpc/NetworkAddressRegisterServiceHandler.java @@ -50,7 +50,7 @@ public class NetworkAddressRegisterServiceHandler extends NetworkAddressRegister NetworkAddressMappings.Builder builder = NetworkAddressMappings.newBuilder(); for (String networkAddress : addressesList) { - int addressId = networkAddressInventoryRegister.getOrCreate(networkAddress); + int addressId = networkAddressInventoryRegister.getOrCreate(networkAddress, null); if (addressId != Const.NONE) { KeyWithIntegerValue value = KeyWithIntegerValue.newBuilder().setKey(networkAddress).setValue(addressId).build(); diff --git a/oap-server/server-receiver-plugin/skywalking-register-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/register/provider/handler/v5/rest/ApplicationRegisterServletHandler.java b/oap-server/server-receiver-plugin/skywalking-register-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/register/provider/handler/v5/rest/ApplicationRegisterServletHandler.java index 49f4f35fea261425fb8833d48353546c645bc286..b08512bc99e5438050560d15c413700550859716 100644 --- a/oap-server/server-receiver-plugin/skywalking-register-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/register/provider/handler/v5/rest/ApplicationRegisterServletHandler.java +++ b/oap-server/server-receiver-plugin/skywalking-register-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/register/provider/handler/v5/rest/ApplicationRegisterServletHandler.java @@ -57,7 +57,7 @@ public class ApplicationRegisterServletHandler extends JettyJsonHandler { JsonArray applicationCodes = gson.fromJson(req.getReader(), JsonArray.class); for (int i = 0; i < applicationCodes.size(); i++) { String applicationCode = applicationCodes.get(i).getAsString(); - int applicationId = serviceInventoryRegister.getOrCreate(applicationCode); + int applicationId = serviceInventoryRegister.getOrCreate(applicationCode, null); JsonObject mapping = new JsonObject(); mapping.addProperty(APPLICATION_CODE, applicationCode); mapping.addProperty(APPLICATION_ID, applicationId); diff --git a/oap-server/server-receiver-plugin/skywalking-register-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/register/provider/handler/v5/rest/NetworkAddressRegisterServletHandler.java b/oap-server/server-receiver-plugin/skywalking-register-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/register/provider/handler/v5/rest/NetworkAddressRegisterServletHandler.java index d1a4632130ccd70a9d3d1842ba9b569f77f928d8..b873d08e47b66e07975706a6259da48df3d391c8 100644 --- a/oap-server/server-receiver-plugin/skywalking-register-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/register/provider/handler/v5/rest/NetworkAddressRegisterServletHandler.java +++ b/oap-server/server-receiver-plugin/skywalking-register-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/register/provider/handler/v5/rest/NetworkAddressRegisterServletHandler.java @@ -62,7 +62,7 @@ public class NetworkAddressRegisterServletHandler extends JettyJsonHandler { logger.debug("network getAddress register, network getAddress: {}", networkAddress); } - int addressId = networkAddressInventoryRegister.getOrCreate(networkAddress); + int addressId = networkAddressInventoryRegister.getOrCreate(networkAddress, null); JsonObject mapping = new JsonObject(); mapping.addProperty(ADDRESS_ID, addressId); mapping.addProperty(NETWORK_ADDRESS, networkAddress); diff --git a/oap-server/server-receiver-plugin/skywalking-register-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/register/provider/handler/v6/grpc/RegisterServiceHandler.java b/oap-server/server-receiver-plugin/skywalking-register-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/register/provider/handler/v6/grpc/RegisterServiceHandler.java index 05c11d1714af16d8f10a4f7d53cf094525e7ff23..f31f27faef0f37704bd14601b82db1fad13b8eca 100644 --- a/oap-server/server-receiver-plugin/skywalking-register-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/register/provider/handler/v6/grpc/RegisterServiceHandler.java +++ b/oap-server/server-receiver-plugin/skywalking-register-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/register/provider/handler/v6/grpc/RegisterServiceHandler.java @@ -63,7 +63,7 @@ public class RegisterServiceHandler extends RegisterGrpc.RegisterImplBase implem if (logger.isDebugEnabled()) { logger.debug("Register service, service code: {}", serviceName); } - int serviceId = serviceInventoryRegister.getOrCreate(serviceName); + int serviceId = serviceInventoryRegister.getOrCreate(serviceName, null); if (serviceId != Const.NONE) { KeyIntValuePair value = KeyIntValuePair.newBuilder().setKey(serviceName).setValue(serviceId).build(); @@ -149,7 +149,7 @@ public class RegisterServiceHandler extends RegisterGrpc.RegisterImplBase implem NetAddressMapping.Builder builder = NetAddressMapping.newBuilder(); request.getAddressesList().forEach(networkAddress -> { - int addressId = networkAddressInventoryRegister.getOrCreate(networkAddress); + int addressId = networkAddressInventoryRegister.getOrCreate(networkAddress, null); if (addressId != Const.NONE) { builder.addAddressIds(KeyIntValuePair.newBuilder().setKey(networkAddress).setValue(addressId)); @@ -186,7 +186,7 @@ public class RegisterServiceHandler extends RegisterGrpc.RegisterImplBase implem return; } - networkAddressId = networkAddressInventoryRegister.getOrCreate(address); + networkAddressId = networkAddressInventoryRegister.getOrCreate(address, null); if (networkAddressId == Const.NONE) { return; } diff --git a/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/decorator/SpanDecorator.java b/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/decorator/SpanDecorator.java index fb4cf364c2bc8bd7261cf3d5e17b5d1773c2fbb8..76fefa6f1cfbb2aeda754d4eb93f22ca0835972a 100644 --- a/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/decorator/SpanDecorator.java +++ b/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/decorator/SpanDecorator.java @@ -18,9 +18,9 @@ package org.apache.skywalking.oap.server.receiver.trace.provider.parser.decorator; -import org.apache.skywalking.apm.network.language.agent.SpanLayer; -import org.apache.skywalking.apm.network.language.agent.SpanObject; -import org.apache.skywalking.apm.network.language.agent.SpanType; +import java.util.*; +import org.apache.skywalking.apm.network.common.KeyStringValuePair; +import org.apache.skywalking.apm.network.language.agent.*; import org.apache.skywalking.apm.network.language.agent.v2.SpanObjectV2; import static java.util.Objects.isNull; @@ -281,6 +281,14 @@ public class SpanDecorator implements StandardBuilder { return referenceDecorators[index]; } + public List getAllTags() { + if (isOrigin) { + return isV2 ? spanObjectV2.getTagsList() : convert(spanObject.getTagsList()); + } else { + return isV2 ? spanBuilderV2.getTagsList() : convert(spanBuilder.getTagsList()); + } + } + @Override public void toBuilder() { if (this.isOrigin) { this.isOrigin = false; @@ -292,4 +300,16 @@ public class SpanDecorator implements StandardBuilder { standardBuilder.toBuilder(); } } + + private List convert(List list) { + List result = new ArrayList<>(); + if (list != null) { + list.forEach(element -> { + result.add(KeyStringValuePair.newBuilder() + .setKey(element.getKey()) + .setValue(element.getValue()).build()); + }); + } + return result; + } } diff --git a/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/standardization/ReferenceIdExchanger.java b/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/standardization/ReferenceIdExchanger.java index 31d6368bf663c6116bd080cc7337411762c9c200..c41253c457186adb0836469a7664b34a3cd84b58 100644 --- a/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/standardization/ReferenceIdExchanger.java +++ b/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/standardization/ReferenceIdExchanger.java @@ -88,7 +88,7 @@ public class ReferenceIdExchanger implements IdExchanger { } if (standardBuilder.getNetworkAddressId() == 0 && !Strings.isNullOrEmpty(standardBuilder.getNetworkAddress())) { - int networkAddressId = networkAddressInventoryRegister.getOrCreate(standardBuilder.getNetworkAddress()); + int networkAddressId = networkAddressInventoryRegister.getOrCreate(standardBuilder.getNetworkAddress(), null); if (networkAddressId == 0) { if (logger.isDebugEnabled()) { diff --git a/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/standardization/SpanIdExchanger.java b/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/standardization/SpanIdExchanger.java index 8a1af6b74e89f3ead6e3ec72e622d4ae1767012a..f9bcf9c87ab7d14ac31c1c9dff3d01a4be6bea69 100644 --- a/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/standardization/SpanIdExchanger.java +++ b/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/standardization/SpanIdExchanger.java @@ -19,9 +19,14 @@ package org.apache.skywalking.oap.server.receiver.trace.provider.parser.standardization; import com.google.common.base.Strings; +import com.google.gson.JsonObject; +import java.util.List; +import org.apache.skywalking.apm.network.common.KeyStringValuePair; +import org.apache.skywalking.apm.network.language.agent.SpanLayer; import org.apache.skywalking.oap.server.core.*; +import org.apache.skywalking.oap.server.core.cache.ServiceInventoryCache; import org.apache.skywalking.oap.server.core.config.IComponentLibraryCatalogService; -import org.apache.skywalking.oap.server.core.register.NodeType; +import org.apache.skywalking.oap.server.core.register.*; import org.apache.skywalking.oap.server.core.register.service.*; import org.apache.skywalking.oap.server.core.source.DetectPoint; import org.apache.skywalking.oap.server.library.module.ModuleManager; @@ -36,6 +41,8 @@ public class SpanIdExchanger implements IdExchanger { private static final Logger logger = LoggerFactory.getLogger(SpanIdExchanger.class); private static SpanIdExchanger EXCHANGER; + private final ServiceInventoryCache serviceInventoryCacheDAO; + private final IServiceInventoryRegister serviceInventoryRegister; private final IEndpointInventoryRegister endpointInventoryRegister; private final INetworkAddressInventoryRegister networkAddressInventoryRegister; private final IComponentLibraryCatalogService componentLibraryCatalogService; @@ -48,6 +55,8 @@ public class SpanIdExchanger implements IdExchanger { } private SpanIdExchanger(ModuleManager moduleManager) { + this.serviceInventoryCacheDAO = moduleManager.find(CoreModule.NAME).provider().getService(ServiceInventoryCache.class); + this.serviceInventoryRegister = moduleManager.find(CoreModule.NAME).provider().getService(IServiceInventoryRegister.class); this.endpointInventoryRegister = moduleManager.find(CoreModule.NAME).provider().getService(IEndpointInventoryRegister.class); this.networkAddressInventoryRegister = moduleManager.find(CoreModule.NAME).provider().getService(INetworkAddressInventoryRegister.class); this.componentLibraryCatalogService = moduleManager.find(CoreModule.NAME).provider().getService(IComponentLibraryCatalogService.class); @@ -69,10 +78,11 @@ public class SpanIdExchanger implements IdExchanger { } } + int peerId = 0; if (standardBuilder.getPeerId() == 0 && !Strings.isNullOrEmpty(standardBuilder.getPeer())) { - int peerId = networkAddressInventoryRegister.getOrCreate(standardBuilder.getPeer()); + peerId = networkAddressInventoryRegister.getOrCreate(standardBuilder.getPeer(), buildServiceProperties(standardBuilder)); - if (peerId == 0) { + if (peerId == Const.NONE) { if (logger.isDebugEnabled()) { logger.debug("peer: {} in service: {} exchange failed", standardBuilder.getPeer(), serviceId); } @@ -81,13 +91,27 @@ public class SpanIdExchanger implements IdExchanger { standardBuilder.toBuilder(); standardBuilder.setPeerId(peerId); standardBuilder.setPeer(Const.EMPTY_STRING); + } + } + + if (peerId != Const.NONE) { + int spanLayerValue = standardBuilder.getSpanLayerValue(); + networkAddressInventoryRegister.update(peerId, NodeType.fromSpanLayerValue(spanLayerValue)); - int spanLayerValue = standardBuilder.getSpanLayerValue(); - networkAddressInventoryRegister.update(peerId, NodeType.fromSpanLayerValue(spanLayerValue)); + /** + * In some case, conjecture node, such as Database node, could be registered by agents. + * At here, if the target service properties need to be updated, + * it will only be updated at the first time for now. + */ + if (SpanLayer.Database.equals(standardBuilder.getSpanLayer())) { + ServiceInventory newServiceInventory = serviceInventoryCacheDAO.get(serviceInventoryCacheDAO.getServiceId(peerId)); + if (!newServiceInventory.hasProperties()) { + serviceInventoryRegister.updateProperties(newServiceInventory.getSequence(), buildServiceProperties(standardBuilder)); + } } } - if (standardBuilder.getOperationNameId() == 0) { + if (standardBuilder.getOperationNameId() == Const.NONE) { String endpointName = Strings.isNullOrEmpty(standardBuilder.getOperationName()) ? Const.DOMAIN_OPERATION_NAME : standardBuilder.getOperationName(); int endpointId = endpointInventoryRegister.getOrCreate(serviceId, endpointName, DetectPoint.fromSpanType(standardBuilder.getSpanType())); @@ -104,4 +128,28 @@ public class SpanIdExchanger implements IdExchanger { } return true; } + + private JsonObject buildServiceProperties(SpanDecorator standardBuilder) { + JsonObject properties = new JsonObject(); + if (SpanLayer.Database.equals(standardBuilder.getSpanLayer())) { + List tags = standardBuilder.getAllTags(); + tags.forEach(tag -> { + if ("db.type".equals(tag.getKey())) { + properties.addProperty("type", tag.getValue()); + } else if ("db.instance".equals(tag.getKey())) { + properties.addProperty("instance", tag.getValue()); + } + }); + String componentName; + int id = standardBuilder.getComponentId(); + if (id != Const.NONE) { + componentName = componentLibraryCatalogService.getServerNameBasedOnComponent(id); + } else { + componentName = "UNKNOWN"; + } + properties.addProperty("database", componentName); + } + + return properties; + } } diff --git a/oap-server/server-receiver-plugin/zipkin-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/zipkin/handler/SpanProcessor.java b/oap-server/server-receiver-plugin/zipkin-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/zipkin/handler/SpanProcessor.java index f5e97be51cabfc445ba708721f5476ee6b8a391a..48774f92378c5e5fb4e8161faa2e2c5d4c5097a6 100644 --- a/oap-server/server-receiver-plugin/zipkin-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/zipkin/handler/SpanProcessor.java +++ b/oap-server/server-receiver-plugin/zipkin-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/zipkin/handler/SpanProcessor.java @@ -48,7 +48,7 @@ public class SpanProcessor { // In Zipkin, the local service name represents the application owner. String applicationCode = span.localServiceName(); if (applicationCode != null) { - int applicationId = CoreRegisterLinker.getServiceInventoryRegister().getOrCreate(applicationCode); + int applicationId = CoreRegisterLinker.getServiceInventoryRegister().getOrCreate(applicationCode, null); if (applicationId != 0) { CoreRegisterLinker.getServiceInstanceInventoryRegister().getOrCreate(applicationId, applicationCode, applicationCode, span.timestampAsLong(), diff --git a/oap-server/server-receiver-plugin/zipkin-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/zipkin/transform/SegmentBuilder.java b/oap-server/server-receiver-plugin/zipkin-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/zipkin/transform/SegmentBuilder.java index be1f1bb6ef2d5e24a685a634bdb2af726e976f40..3f687a6cc7a375a67f33bd40da7807d621411054 100644 --- a/oap-server/server-receiver-plugin/zipkin-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/zipkin/transform/SegmentBuilder.java +++ b/oap-server/server-receiver-plugin/zipkin-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/zipkin/transform/SegmentBuilder.java @@ -301,7 +301,7 @@ public class SegmentBuilder { private Segment addApp(String serviceCode, long registerTime) throws Exception { int serviceId = waitForExchange(() -> - CoreRegisterLinker.getServiceInventoryRegister().getOrCreate(serviceCode), + CoreRegisterLinker.getServiceInventoryRegister().getOrCreate(serviceCode, null), 10 ); diff --git a/oap-server/server-receiver-plugin/zipkin-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/zipkin/transform/SpringSleuthSegmentBuilderTest.java b/oap-server/server-receiver-plugin/zipkin-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/zipkin/transform/SpringSleuthSegmentBuilderTest.java index 1fbdeb3ea40c709a890b5057e1e3a4beca3d0f4a..5539046058a87fda7f5e2e5e5143b387ad6a38e4 100644 --- a/oap-server/server-receiver-plugin/zipkin-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/zipkin/transform/SpringSleuthSegmentBuilderTest.java +++ b/oap-server/server-receiver-plugin/zipkin-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/zipkin/transform/SpringSleuthSegmentBuilderTest.java @@ -18,6 +18,7 @@ package org.apache.skywalking.oap.server.receiver.zipkin.transform; +import com.google.gson.JsonObject; import java.io.UnsupportedEncodingException; import java.util.HashMap; import java.util.LinkedList; @@ -27,7 +28,7 @@ import org.apache.skywalking.apm.network.language.agent.SpanType; import org.apache.skywalking.apm.network.language.agent.v2.SegmentObject; import org.apache.skywalking.apm.network.language.agent.v2.SegmentReference; import org.apache.skywalking.apm.network.language.agent.v2.SpanObjectV2; -import org.apache.skywalking.oap.server.core.register.ServiceInstanceInventory; +import org.apache.skywalking.oap.server.core.register.*; import org.apache.skywalking.oap.server.core.register.service.IServiceInstanceInventoryRegister; import org.apache.skywalking.oap.server.core.register.service.IServiceInventoryRegister; import org.apache.skywalking.oap.server.receiver.zipkin.CoreRegisterLinker; @@ -52,7 +53,7 @@ public class SpringSleuthSegmentBuilderTest implements SegmentListener { public void testTransform() throws Exception { IServiceInventoryRegister applicationIDService = new IServiceInventoryRegister() { - @Override public int getOrCreate(String serviceName) { + @Override public int getOrCreate(String serviceName, JsonObject properties) { String key = "AppCode:" + serviceName; if (applicationRegister.containsKey(key)) { return applicationRegister.get(key); @@ -63,7 +64,7 @@ public class SpringSleuthSegmentBuilderTest implements SegmentListener { } } - @Override public int getOrCreate(int addressId, String serviceName) { + @Override public int getOrCreate(int addressId, String serviceName, JsonObject properties) { String key = "Address:" + serviceName; if (applicationRegister.containsKey(key)) { return applicationRegister.get(key); @@ -74,6 +75,9 @@ public class SpringSleuthSegmentBuilderTest implements SegmentListener { } } + @Override public void updateProperties(int serviceId, JsonObject properties) { + } + @Override public void heartbeat(int serviceId, long heartBeatTime) { }