diff --git a/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/application/ApplicationIDService.java b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/application/ApplicationIDService.java index 2cf836e90763956a69c39ae36c6264e08db0377b..fd169c8f78a330b265413ae244abfb8cfc85563a 100644 --- a/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/application/ApplicationIDService.java +++ b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/application/ApplicationIDService.java @@ -1,10 +1,9 @@ package org.skywalking.apm.collector.agentregister.application; +import org.skywalking.apm.collector.agentstream.worker.cache.ApplicationCache; import org.skywalking.apm.collector.agentstream.worker.register.application.ApplicationDataDefine; import org.skywalking.apm.collector.agentstream.worker.register.application.ApplicationRegisterRemoteWorker; -import org.skywalking.apm.collector.agentstream.worker.register.application.dao.IApplicationDAO; import org.skywalking.apm.collector.core.framework.CollectorContextHelper; -import org.skywalking.apm.collector.storage.dao.DAOContainer; import org.skywalking.apm.collector.stream.StreamModuleContext; import org.skywalking.apm.collector.stream.StreamModuleGroupDefine; import org.skywalking.apm.collector.stream.worker.WorkerInvokeException; @@ -20,8 +19,7 @@ public class ApplicationIDService { private final Logger logger = LoggerFactory.getLogger(ApplicationIDService.class); public int getOrCreate(String applicationCode) { - IApplicationDAO dao = (IApplicationDAO)DAOContainer.INSTANCE.get(IApplicationDAO.class.getName()); - int applicationId = dao.getApplicationId(applicationCode); + int applicationId = ApplicationCache.get(applicationCode); if (applicationId == 0) { StreamModuleContext context = (StreamModuleContext)CollectorContextHelper.INSTANCE.getContext(StreamModuleGroupDefine.GROUP_NAME); diff --git a/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/jetty/AgentRegisterJettyConfig.java b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/jetty/AgentRegisterJettyConfig.java new file mode 100644 index 0000000000000000000000000000000000000000..cf1e452076cf7c9cac5515345b805fc5ee3c16b9 --- /dev/null +++ b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/jetty/AgentRegisterJettyConfig.java @@ -0,0 +1,10 @@ +package org.skywalking.apm.collector.agentregister.jetty; + +/** + * @author pengys5 + */ +public class AgentRegisterJettyConfig { + public static String HOST; + public static int PORT; + public static String CONTEXT_PATH; +} diff --git a/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/jetty/AgentRegisterJettyConfigParser.java b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/jetty/AgentRegisterJettyConfigParser.java new file mode 100644 index 0000000000000000000000000000000000000000..65a10bebead885f4d379c370ad5e623c48fc75bf --- /dev/null +++ b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/jetty/AgentRegisterJettyConfigParser.java @@ -0,0 +1,36 @@ +package org.skywalking.apm.collector.agentregister.jetty; + +import java.util.Map; +import org.skywalking.apm.collector.core.config.ConfigParseException; +import org.skywalking.apm.collector.core.module.ModuleConfigParser; +import org.skywalking.apm.collector.core.util.ObjectUtils; +import org.skywalking.apm.collector.core.util.StringUtils; + +/** + * @author pengys5 + */ +public class AgentRegisterJettyConfigParser implements ModuleConfigParser { + + private static final String HOST = "host"; + private static final String PORT = "port"; + public static final String CONTEXT_PATH = "contextPath"; + + @Override public void parse(Map config) throws ConfigParseException { + AgentRegisterJettyConfig.CONTEXT_PATH = "/"; + + if (ObjectUtils.isEmpty(config) || StringUtils.isEmpty(config.get(HOST))) { + AgentRegisterJettyConfig.HOST = "localhost"; + } else { + AgentRegisterJettyConfig.HOST = (String)config.get(HOST); + } + + if (ObjectUtils.isEmpty(config) || StringUtils.isEmpty(config.get(PORT))) { + AgentRegisterJettyConfig.PORT = 12800; + } else { + AgentRegisterJettyConfig.PORT = (Integer)config.get(PORT); + } + if (ObjectUtils.isNotEmpty(config) && StringUtils.isNotEmpty(config.get(CONTEXT_PATH))) { + AgentRegisterJettyConfig.CONTEXT_PATH = (String)config.get(CONTEXT_PATH); + } + } +} diff --git a/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/jetty/AgentRegisterJettyDataListener.java b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/jetty/AgentRegisterJettyDataListener.java new file mode 100644 index 0000000000000000000000000000000000000000..a15ff860e633e1712dc6d21231eed0d7cd21bd6c --- /dev/null +++ b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/jetty/AgentRegisterJettyDataListener.java @@ -0,0 +1,20 @@ +package org.skywalking.apm.collector.agentregister.jetty; + +import org.skywalking.apm.collector.agentregister.AgentRegisterModuleGroupDefine; +import org.skywalking.apm.collector.cluster.ClusterModuleDefine; +import org.skywalking.apm.collector.core.cluster.ClusterDataListener; + +/** + * @author pengys5 + */ +public class AgentRegisterJettyDataListener extends ClusterDataListener { + + public static final String PATH = ClusterModuleDefine.BASE_CATALOG + "." + AgentRegisterModuleGroupDefine.GROUP_NAME + "." + AgentRegisterJettyModuleDefine.MODULE_NAME; + + @Override public String path() { + return PATH; + } + + @Override public void addressChangedNotify() { + } +} diff --git a/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/jetty/AgentRegisterJettyModuleDefine.java b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/jetty/AgentRegisterJettyModuleDefine.java new file mode 100644 index 0000000000000000000000000000000000000000..413149d715ab8494de111316d644794dfb760d03 --- /dev/null +++ b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/jetty/AgentRegisterJettyModuleDefine.java @@ -0,0 +1,53 @@ +package org.skywalking.apm.collector.agentregister.jetty; + +import java.util.LinkedList; +import java.util.List; +import org.skywalking.apm.collector.agentregister.AgentRegisterModuleDefine; +import org.skywalking.apm.collector.agentregister.AgentRegisterModuleGroupDefine; +import org.skywalking.apm.collector.agentregister.jetty.handler.ApplicationRegisterServletHandler; +import org.skywalking.apm.collector.agentregister.jetty.handler.InstanceDiscoveryServletHandler; +import org.skywalking.apm.collector.core.cluster.ClusterDataListener; +import org.skywalking.apm.collector.core.framework.Handler; +import org.skywalking.apm.collector.core.module.ModuleConfigParser; +import org.skywalking.apm.collector.core.module.ModuleRegistration; +import org.skywalking.apm.collector.core.server.Server; +import org.skywalking.apm.collector.server.jetty.JettyServer; + +/** + * @author pengys5 + */ +public class AgentRegisterJettyModuleDefine extends AgentRegisterModuleDefine { + + public static final String MODULE_NAME = "jetty"; + + @Override protected String group() { + return AgentRegisterModuleGroupDefine.GROUP_NAME; + } + + @Override public String name() { + return MODULE_NAME; + } + + @Override protected ModuleConfigParser configParser() { + return new AgentRegisterJettyConfigParser(); + } + + @Override protected Server server() { + return new JettyServer(AgentRegisterJettyConfig.HOST, AgentRegisterJettyConfig.PORT, AgentRegisterJettyConfig.CONTEXT_PATH); + } + + @Override protected ModuleRegistration registration() { + return new AgentRegisterJettyModuleRegistration(); + } + + @Override public ClusterDataListener listener() { + return new AgentRegisterJettyDataListener(); + } + + @Override public List handlerList() { + List handlers = new LinkedList<>(); + handlers.add(new ApplicationRegisterServletHandler()); + handlers.add(new InstanceDiscoveryServletHandler()); + return handlers; + } +} diff --git a/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/jetty/AgentRegisterJettyModuleRegistration.java b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/jetty/AgentRegisterJettyModuleRegistration.java new file mode 100644 index 0000000000000000000000000000000000000000..d6e2d260a4d48500c9c0a8fd6a2233a0b53958c9 --- /dev/null +++ b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/jetty/AgentRegisterJettyModuleRegistration.java @@ -0,0 +1,13 @@ +package org.skywalking.apm.collector.agentregister.jetty; + +import org.skywalking.apm.collector.core.module.ModuleRegistration; + +/** + * @author pengys5 + */ +public class AgentRegisterJettyModuleRegistration extends ModuleRegistration { + + @Override public Value buildValue() { + return new Value(AgentRegisterJettyConfig.HOST, AgentRegisterJettyConfig.PORT, AgentRegisterJettyConfig.CONTEXT_PATH); + } +} diff --git a/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/jetty/handler/ApplicationRegisterServletHandler.java b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/jetty/handler/ApplicationRegisterServletHandler.java new file mode 100644 index 0000000000000000000000000000000000000000..7c7f2a99b6dff0f82e48ed385b3d4c9139912338 --- /dev/null +++ b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/jetty/handler/ApplicationRegisterServletHandler.java @@ -0,0 +1,52 @@ +package org.skywalking.apm.collector.agentregister.jetty.handler; + +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import java.io.IOException; +import javax.servlet.http.HttpServletRequest; +import org.skywalking.apm.collector.agentregister.application.ApplicationIDService; +import org.skywalking.apm.collector.server.jetty.ArgumentsParseException; +import org.skywalking.apm.collector.server.jetty.JettyHandler; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public class ApplicationRegisterServletHandler extends JettyHandler { + + private final Logger logger = LoggerFactory.getLogger(ApplicationRegisterServletHandler.class); + + private ApplicationIDService applicationIDService = new ApplicationIDService(); + private Gson gson = new Gson(); + private static final String APPLICATION_CODE = "c"; + private static final String APPLICATION_ID = "i"; + + @Override public String pathSpec() { + return "/application/register"; + } + + @Override protected JsonElement doGet(HttpServletRequest req) throws ArgumentsParseException { + throw new UnsupportedOperationException(); + } + + @Override protected JsonElement doPost(HttpServletRequest req) throws ArgumentsParseException { + JsonArray responseArray = new JsonArray(); + try { + JsonArray applicationCodes = gson.fromJson(req.getReader(), JsonArray.class); + for (int i = 0; i < applicationCodes.size(); i++) { + String applicationCode = applicationCodes.get(i).getAsString(); + int applicationId = applicationIDService.getOrCreate(applicationCode); + JsonObject mapping = new JsonObject(); + mapping.addProperty(APPLICATION_CODE, applicationCode); + mapping.addProperty(APPLICATION_ID, applicationId); + responseArray.add(mapping); + } + } catch (IOException e) { + logger.error(e.getMessage(), e); + } + return responseArray; + } +} diff --git a/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/jetty/handler/InstanceDiscoveryServletHandler.java b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/jetty/handler/InstanceDiscoveryServletHandler.java new file mode 100644 index 0000000000000000000000000000000000000000..ffddfb94dbfc63c146a2038009d9de1ee20099d6 --- /dev/null +++ b/apm-collector/apm-collector-agentregister/src/main/java/org/skywalking/apm/collector/agentregister/jetty/handler/InstanceDiscoveryServletHandler.java @@ -0,0 +1,53 @@ +package org.skywalking.apm.collector.agentregister.jetty.handler; + +import com.google.gson.Gson; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import java.io.IOException; +import javax.servlet.http.HttpServletRequest; +import org.skywalking.apm.collector.agentregister.instance.InstanceIDService; +import org.skywalking.apm.collector.server.jetty.ArgumentsParseException; +import org.skywalking.apm.collector.server.jetty.JettyHandler; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public class InstanceDiscoveryServletHandler extends JettyHandler { + + private final Logger logger = LoggerFactory.getLogger(InstanceDiscoveryServletHandler.class); + + private InstanceIDService instanceIDService = new InstanceIDService(); + private Gson gson = new Gson(); + + private static final String APPLICATION_ID = "ai"; + private static final String AGENT_UUID = "au"; + private static final String REGISTER_TIME = "rt"; + private static final String INSTANCE_ID = "ii"; + + @Override public String pathSpec() { + return "/instance/register"; + } + + @Override protected JsonElement doGet(HttpServletRequest req) throws ArgumentsParseException { + throw new UnsupportedOperationException(); + } + + @Override protected JsonElement doPost(HttpServletRequest req) throws ArgumentsParseException { + JsonObject responseJson = new JsonObject(); + try { + JsonObject instance = gson.fromJson(req.getReader(), JsonObject.class); + int applicationId = instance.get(APPLICATION_ID).getAsInt(); + String agentUUID = instance.get(AGENT_UUID).getAsString(); + long registerTime = instance.get(REGISTER_TIME).getAsLong(); + + int instanceId = instanceIDService.getOrCreate(applicationId, agentUUID, registerTime); + responseJson.addProperty(APPLICATION_ID, applicationId); + responseJson.addProperty(INSTANCE_ID, instanceId); + } catch (IOException e) { + logger.error(e.getMessage(), e); + } + return responseJson; + } +} diff --git a/apm-collector/apm-collector-agentregister/src/main/resources/META-INF/defines/module.define b/apm-collector/apm-collector-agentregister/src/main/resources/META-INF/defines/module.define index 70f99f0a2c14c36579604f04eea0ea5bb52abb42..e826d41609b2c8b3731a2731b2c80d5795ee33ce 100644 --- a/apm-collector/apm-collector-agentregister/src/main/resources/META-INF/defines/module.define +++ b/apm-collector/apm-collector-agentregister/src/main/resources/META-INF/defines/module.define @@ -1 +1,2 @@ -org.skywalking.apm.collector.agentregister.grpc.AgentRegisterGRPCModuleDefine \ No newline at end of file +org.skywalking.apm.collector.agentregister.grpc.AgentRegisterGRPCModuleDefine +org.skywalking.apm.collector.agentregister.jetty.AgentRegisterJettyModuleDefine \ No newline at end of file diff --git a/apm-collector/apm-collector-agentserver/src/main/java/org/skywalking/apm/collector/agentserver/jetty/handler/AgentStreamGRPCServerHandler.java b/apm-collector/apm-collector-agentserver/src/main/java/org/skywalking/apm/collector/agentserver/jetty/handler/AgentStreamGRPCServerHandler.java index 2c4e4bebb4b260269b630a15f19b6644a5210ac8..f1cd494617a33c72377d146e4c77771aa2bbbbc9 100644 --- a/apm-collector/apm-collector-agentserver/src/main/java/org/skywalking/apm/collector/agentserver/jetty/handler/AgentStreamGRPCServerHandler.java +++ b/apm-collector/apm-collector-agentserver/src/main/java/org/skywalking/apm/collector/agentserver/jetty/handler/AgentStreamGRPCServerHandler.java @@ -25,9 +25,11 @@ public class AgentStreamGRPCServerHandler extends JettyHandler { ClusterModuleRegistrationReader reader = ((ClusterModuleContext)CollectorContextHelper.INSTANCE.getContext(ClusterModuleGroupDefine.GROUP_NAME)).getReader(); List servers = reader.read(AgentStreamGRPCDataListener.PATH); JsonArray serverArray = new JsonArray(); - servers.forEach(server -> { - serverArray.add(server); - }); + servers.forEach(server -> serverArray.add(server)); return serverArray; } + + @Override protected JsonElement doPost(HttpServletRequest req) throws ArgumentsParseException { + throw new UnsupportedOperationException(); + } } diff --git a/apm-collector/apm-collector-agentserver/src/main/java/org/skywalking/apm/collector/agentserver/jetty/handler/AgentStreamJettyServerHandler.java b/apm-collector/apm-collector-agentserver/src/main/java/org/skywalking/apm/collector/agentserver/jetty/handler/AgentStreamJettyServerHandler.java index dcaf05cfab7880ddbb4164bda37b8294eff41536..bfcecdbf7196c082461cb1c43bb4ac4e78f4cf72 100644 --- a/apm-collector/apm-collector-agentserver/src/main/java/org/skywalking/apm/collector/agentserver/jetty/handler/AgentStreamJettyServerHandler.java +++ b/apm-collector/apm-collector-agentserver/src/main/java/org/skywalking/apm/collector/agentserver/jetty/handler/AgentStreamJettyServerHandler.java @@ -30,4 +30,8 @@ public class AgentStreamJettyServerHandler extends JettyHandler { }); return serverArray; } + + @Override protected JsonElement doPost(HttpServletRequest req) throws ArgumentsParseException { + throw new UnsupportedOperationException(); + } } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/AgentStreamModuleInstaller.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/AgentStreamModuleInstaller.java index 2b53fa68ac77ff6657826f5b98335ebd0af53f1a..bf5227d48553e12629da2c24be73364a2bc68bab 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/AgentStreamModuleInstaller.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/AgentStreamModuleInstaller.java @@ -2,7 +2,6 @@ package org.skywalking.apm.collector.agentstream; import java.util.Iterator; import java.util.Map; -import org.skywalking.apm.collector.agentstream.worker.storage.IDNameExchangeTimer; import org.skywalking.apm.collector.agentstream.worker.storage.PersistenceTimer; import org.skywalking.apm.collector.core.client.ClientException; import org.skywalking.apm.collector.core.framework.CollectorContextHelper; @@ -36,6 +35,5 @@ public class AgentStreamModuleInstaller implements ModuleInstaller { } new PersistenceTimer().start(); - new IDNameExchangeTimer().start(); } } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/AgentStreamJettyModuleDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/AgentStreamJettyModuleDefine.java index d20996ebd4f862c8acb435b12b4a9a56d28249a1..87428c339954e24ec7dd08ae620422ff337c43cf 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/AgentStreamJettyModuleDefine.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/AgentStreamJettyModuleDefine.java @@ -1,8 +1,10 @@ package org.skywalking.apm.collector.agentstream.jetty; +import java.util.LinkedList; import java.util.List; import org.skywalking.apm.collector.agentstream.AgentStreamModuleDefine; import org.skywalking.apm.collector.agentstream.AgentStreamModuleGroupDefine; +import org.skywalking.apm.collector.agentstream.jetty.handler.TraceSegmentServletHandler; import org.skywalking.apm.collector.core.cluster.ClusterDataListener; import org.skywalking.apm.collector.core.framework.Handler; import org.skywalking.apm.collector.core.module.ModuleConfigParser; @@ -42,6 +44,8 @@ public class AgentStreamJettyModuleDefine extends AgentStreamModuleDefine { } @Override public List handlerList() { - return null; + List handlers = new LinkedList<>(); + handlers.add(new TraceSegmentServletHandler()); + return handlers; } } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/TraceSegmentServletHandler.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/TraceSegmentServletHandler.java new file mode 100644 index 0000000000000000000000000000000000000000..7622aac48fd3e21969bb49ed3cf2d053e931ab04 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/TraceSegmentServletHandler.java @@ -0,0 +1,54 @@ +package org.skywalking.apm.collector.agentstream.jetty.handler; + +import com.google.gson.JsonElement; +import com.google.gson.stream.JsonReader; +import java.io.BufferedReader; +import java.io.IOException; +import javax.servlet.http.HttpServletRequest; +import org.skywalking.apm.collector.agentstream.jetty.handler.reader.TraceSegment; +import org.skywalking.apm.collector.agentstream.jetty.handler.reader.TraceSegmentJsonReader; +import org.skywalking.apm.collector.agentstream.worker.segment.SegmentParse; +import org.skywalking.apm.collector.server.jetty.ArgumentsParseException; +import org.skywalking.apm.collector.server.jetty.JettyHandler; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public class TraceSegmentServletHandler extends JettyHandler { + + private final Logger logger = LoggerFactory.getLogger(TraceSegmentServletHandler.class); + + @Override public String pathSpec() { + return "/segments"; + } + + @Override protected JsonElement doGet(HttpServletRequest req) throws ArgumentsParseException { + throw new UnsupportedOperationException(); + } + + @Override protected JsonElement doPost(HttpServletRequest req) throws ArgumentsParseException { + logger.debug("receive stream segment"); + try { + BufferedReader bufferedReader = req.getReader(); + read(bufferedReader); + } catch (IOException e) { + logger.error(e.getMessage(), e); + } + return null; + } + + private TraceSegmentJsonReader jsonReader = new TraceSegmentJsonReader(); + + private void read(BufferedReader bufferedReader) throws IOException { + JsonReader reader = new JsonReader(bufferedReader); + reader.beginArray(); + while (reader.hasNext()) { + SegmentParse segmentParse = new SegmentParse(); + TraceSegment traceSegment = jsonReader.read(reader); + segmentParse.parse(traceSegment.getGlobalTraceIds(), traceSegment.getTraceSegmentObject()); + } + reader.endArray(); + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/KeyWithStringValueJsonReader.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/KeyWithStringValueJsonReader.java new file mode 100644 index 0000000000000000000000000000000000000000..f0f1bb144442982c5c7a8800e15c91a893c7834a --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/KeyWithStringValueJsonReader.java @@ -0,0 +1,36 @@ +package org.skywalking.apm.collector.agentstream.jetty.handler.reader; + +import com.google.gson.stream.JsonReader; +import java.io.IOException; +import org.skywalking.apm.network.proto.KeyWithStringValue; + +/** + * @author pengys5 + */ +public class KeyWithStringValueJsonReader implements StreamJsonReader { + + private static final String KEY = "k"; + private static final String VALUE = "v"; + + @Override public KeyWithStringValue read(JsonReader reader) throws IOException { + KeyWithStringValue.Builder builder = KeyWithStringValue.newBuilder(); + + reader.beginObject(); + while (reader.hasNext()) { + switch (reader.nextName()) { + case KEY: + builder.setKey(reader.nextString()); + break; + case VALUE: + builder.setValue(reader.nextString()); + break; + default: + reader.skipValue(); + break; + } + } + reader.endObject(); + + return builder.build(); + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/LogJsonReader.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/LogJsonReader.java new file mode 100644 index 0000000000000000000000000000000000000000..78b8a79eef37b70f2570c1fd193238240cd4ab6e --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/LogJsonReader.java @@ -0,0 +1,37 @@ +package org.skywalking.apm.collector.agentstream.jetty.handler.reader; + +import com.google.gson.stream.JsonReader; +import java.io.IOException; +import org.skywalking.apm.network.proto.LogMessage; + +/** + * @author pengys5 + */ +public class LogJsonReader implements StreamJsonReader { + + private KeyWithStringValueJsonReader keyWithStringValueJsonReader = new KeyWithStringValueJsonReader(); + + private static final String TIME = "ti"; + private static final String LOG_DATA = "ld"; + + @Override public LogMessage read(JsonReader reader) throws IOException { + LogMessage.Builder builder = LogMessage.newBuilder(); + + while (reader.hasNext()) { + switch (reader.nextName()) { + case TIME: + builder.setTime(reader.nextLong()); + case LOG_DATA: + reader.beginArray(); + while (reader.hasNext()) { + builder.addData(keyWithStringValueJsonReader.read(reader)); + } + reader.endArray(); + default: + reader.skipValue(); + } + } + + return builder.build(); + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/ReferenceJsonReader.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/ReferenceJsonReader.java new file mode 100644 index 0000000000000000000000000000000000000000..113edd19914cf5ac0be9c8ff538d7576c1fb7961 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/ReferenceJsonReader.java @@ -0,0 +1,74 @@ +package org.skywalking.apm.collector.agentstream.jetty.handler.reader; + +import com.google.gson.stream.JsonReader; +import java.io.IOException; +import org.skywalking.apm.network.proto.TraceSegmentReference; + +/** + * @author pengys5 + */ +public class ReferenceJsonReader implements StreamJsonReader { + + private UniqueIdJsonReader uniqueIdJsonReader = new UniqueIdJsonReader(); + + private static final String PARENT_TRACE_SEGMENT_ID = "ts"; + private static final String PARENT_APPLICATION_ID = "ai"; + private static final String PARENT_SPAN_ID = "si"; + private static final String PARENT_SERVICE_ID = "vi"; + private static final String PARENT_SERVICE_NAME = "vn"; + private static final String NETWORK_ADDRESS_ID = "ni"; + private static final String NETWORK_ADDRESS = "nn"; + private static final String ENTRY_APPLICATION_INSTANCE_ID = "ea"; + private static final String ENTRY_SERVICE_ID = "ei"; + private static final String ENTRY_SERVICE_NAME = "en"; + private static final String REF_TYPE_VALUE = "rv"; + + @Override public TraceSegmentReference read(JsonReader reader) throws IOException { + TraceSegmentReference.Builder builder = TraceSegmentReference.newBuilder(); + + reader.beginObject(); + while (reader.hasNext()) { + switch (reader.nextName()) { + case PARENT_TRACE_SEGMENT_ID: + builder.setParentTraceSegmentId(uniqueIdJsonReader.read(reader)); + break; + case PARENT_APPLICATION_ID: + builder.setParentApplicationInstanceId(reader.nextInt()); + break; + case PARENT_SPAN_ID: + builder.setParentSpanId(reader.nextInt()); + break; + case PARENT_SERVICE_ID: + builder.setParentServiceId(reader.nextInt()); + break; + case PARENT_SERVICE_NAME: + builder.setParentServiceName(reader.nextString()); + break; + case NETWORK_ADDRESS_ID: + builder.setNetworkAddressId(reader.nextInt()); + break; + case NETWORK_ADDRESS: + builder.setNetworkAddress(reader.nextString()); + break; + case ENTRY_APPLICATION_INSTANCE_ID: + builder.setEntryApplicationInstanceId(reader.nextInt()); + break; + case ENTRY_SERVICE_ID: + builder.setEntryServiceId(reader.nextInt()); + break; + case ENTRY_SERVICE_NAME: + builder.setEntryServiceName(reader.nextString()); + break; + case REF_TYPE_VALUE: + builder.setRefTypeValue(reader.nextInt()); + break; + default: + reader.skipValue(); + break; + } + } + reader.endObject(); + + return builder.build(); + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/SegmentJsonReader.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/SegmentJsonReader.java new file mode 100644 index 0000000000000000000000000000000000000000..a2daa965cfb799c55028e99432fc2fa805fc4b49 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/SegmentJsonReader.java @@ -0,0 +1,69 @@ +package org.skywalking.apm.collector.agentstream.jetty.handler.reader; + +import com.google.gson.stream.JsonReader; +import java.io.IOException; +import org.skywalking.apm.network.proto.TraceSegmentObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public class SegmentJsonReader implements StreamJsonReader { + + private final Logger logger = LoggerFactory.getLogger(SegmentJsonReader.class); + + private UniqueIdJsonReader uniqueIdJsonReader = new UniqueIdJsonReader(); + private ReferenceJsonReader referenceJsonReader = new ReferenceJsonReader(); + private SpanJsonReader spanJsonReader = new SpanJsonReader(); + + private static final String TRACE_SEGMENT_ID = "ts"; + private static final String APPLICATION_ID = "ai"; + private static final String APPLICATION_INSTANCE_ID = "ii"; + private static final String TRACE_SEGMENT_REFERENCE = "rs"; + private static final String SPANS = "ss"; + + @Override public TraceSegmentObject read(JsonReader reader) throws IOException { + TraceSegmentObject.Builder builder = TraceSegmentObject.newBuilder(); + + reader.beginObject(); + while (reader.hasNext()) { + switch (reader.nextName()) { + case TRACE_SEGMENT_ID: + builder.setTraceSegmentId(uniqueIdJsonReader.read(reader)); + if (logger.isDebugEnabled()) { + StringBuilder segmentId = new StringBuilder(); + builder.getTraceSegmentId().getIdPartsList().forEach(idPart -> segmentId.append(idPart)); + logger.debug("segment id: {}", segmentId); + } + break; + case APPLICATION_ID: + builder.setApplicationId(reader.nextInt()); + break; + case APPLICATION_INSTANCE_ID: + builder.setApplicationInstanceId(reader.nextInt()); + break; + case TRACE_SEGMENT_REFERENCE: + reader.beginArray(); + while (reader.hasNext()) { + builder.addRefs(referenceJsonReader.read(reader)); + } + reader.endArray(); + break; + case SPANS: + reader.beginArray(); + while (reader.hasNext()) { + builder.addSpans(spanJsonReader.read(reader)); + } + reader.endArray(); + break; + default: + reader.skipValue(); + break; + } + } + reader.endObject(); + + return builder.build(); + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/SpanJsonReader.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/SpanJsonReader.java new file mode 100644 index 0000000000000000000000000000000000000000..8de743512fd5ebf6761500adaf1c3f3007ab9425 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/SpanJsonReader.java @@ -0,0 +1,99 @@ +package org.skywalking.apm.collector.agentstream.jetty.handler.reader; + +import com.google.gson.stream.JsonReader; +import java.io.IOException; +import org.skywalking.apm.network.proto.SpanObject; + +/** + * @author pengys5 + */ +public class SpanJsonReader implements StreamJsonReader { + + private KeyWithStringValueJsonReader keyWithStringValueJsonReader = new KeyWithStringValueJsonReader(); + private LogJsonReader logJsonReader = new LogJsonReader(); + + private static final String SPAN_ID = "si"; + private static final String SPAN_TYPE_VALUE = "tv"; + private static final String SPAN_LAYER_VALUE = "lv"; + private static final String PARENT_SPAN_ID = "ps"; + private static final String START_TIME = "st"; + private static final String END_TIME = "et"; + private static final String COMPONENT_ID = "ci"; + private static final String COMPONENT_NAME = "cn"; + private static final String OPERATION_NAME_ID = "oi"; + private static final String OPERATION_NAME = "on"; + private static final String PEER_ID = "pi"; + private static final String PEER = "pn"; + private static final String IS_ERROR = "ie"; + private static final String TAGS = "to"; + private static final String LOGS = "lo"; + + @Override public SpanObject read(JsonReader reader) throws IOException { + SpanObject.Builder builder = SpanObject.newBuilder(); + + reader.beginObject(); + while (reader.hasNext()) { + switch (reader.nextName()) { + case SPAN_ID: + builder.setSpanId(reader.nextInt()); + break; + case SPAN_TYPE_VALUE: + builder.setSpanTypeValue(reader.nextInt()); + break; + case SPAN_LAYER_VALUE: + builder.setSpanLayerValue(reader.nextInt()); + break; + case PARENT_SPAN_ID: + builder.setParentSpanId(reader.nextInt()); + break; + case START_TIME: + builder.setStartTime(reader.nextLong()); + break; + case END_TIME: + builder.setEndTime(reader.nextLong()); + break; + case COMPONENT_ID: + builder.setComponentId(reader.nextInt()); + break; + case COMPONENT_NAME: + builder.setComponent(reader.nextString()); + break; + case OPERATION_NAME_ID: + builder.setOperationNameId(reader.nextInt()); + break; + case OPERATION_NAME: + builder.setOperationName(reader.nextString()); + break; + case PEER_ID: + builder.setPeerId(reader.nextInt()); + break; + case PEER: + builder.setPeer(reader.nextString()); + break; + case IS_ERROR: + builder.setIsError(reader.nextBoolean()); + break; + case TAGS: + reader.beginArray(); + while (reader.hasNext()) { + builder.addTags(keyWithStringValueJsonReader.read(reader)); + } + reader.endArray(); + break; + case LOGS: + reader.beginArray(); + while (reader.hasNext()) { + builder.addLogs(logJsonReader.read(reader)); + } + reader.endArray(); + break; + default: + reader.skipValue(); + break; + } + } + reader.endObject(); + + return builder.build(); + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/StreamJsonReader.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/StreamJsonReader.java new file mode 100644 index 0000000000000000000000000000000000000000..95095a731b66875fedf488c37b0af563abae7df3 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/StreamJsonReader.java @@ -0,0 +1,11 @@ +package org.skywalking.apm.collector.agentstream.jetty.handler.reader; + +import com.google.gson.stream.JsonReader; +import java.io.IOException; + +/** + * @author pengys5 + */ +public interface StreamJsonReader { + T read(JsonReader reader) throws IOException; +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/TraceSegment.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/TraceSegment.java new file mode 100644 index 0000000000000000000000000000000000000000..fa56b0f04738a0d89b4722229eed8705d94f7a07 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/TraceSegment.java @@ -0,0 +1,35 @@ +package org.skywalking.apm.collector.agentstream.jetty.handler.reader; + +import java.util.ArrayList; +import java.util.List; +import org.skywalking.apm.network.proto.TraceSegmentObject; +import org.skywalking.apm.network.proto.UniqueId; + +/** + * @author pengys5 + */ +public class TraceSegment { + + private List uniqueIds; + private TraceSegmentObject traceSegmentObject; + + public TraceSegment() { + uniqueIds = new ArrayList<>(); + } + + public List getGlobalTraceIds() { + return uniqueIds; + } + + public void addGlobalTraceId(UniqueId globalTraceId) { + uniqueIds.add(globalTraceId); + } + + public TraceSegmentObject getTraceSegmentObject() { + return traceSegmentObject; + } + + public void setTraceSegmentObject(TraceSegmentObject traceSegmentObject) { + this.traceSegmentObject = traceSegmentObject; + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/TraceSegmentJsonReader.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/TraceSegmentJsonReader.java new file mode 100644 index 0000000000000000000000000000000000000000..d00883b5339b36707ced684fafdb392248de6f81 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/TraceSegmentJsonReader.java @@ -0,0 +1,54 @@ +package org.skywalking.apm.collector.agentstream.jetty.handler.reader; + +import com.google.gson.stream.JsonReader; +import java.io.IOException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public class TraceSegmentJsonReader implements StreamJsonReader { + + private final Logger logger = LoggerFactory.getLogger(TraceSegmentJsonReader.class); + + private UniqueIdJsonReader uniqueIdJsonReader = new UniqueIdJsonReader(); + private SegmentJsonReader segmentJsonReader = new SegmentJsonReader(); + + private static final String GLOBAL_TRACE_IDS = "gt"; + private static final String SEGMENT = "sg"; + + @Override public TraceSegment read(JsonReader reader) throws IOException { + TraceSegment traceSegment = new TraceSegment(); + + reader.beginObject(); + while (reader.hasNext()) { + switch (reader.nextName()) { + case GLOBAL_TRACE_IDS: + reader.beginArray(); + while (reader.hasNext()) { + traceSegment.addGlobalTraceId(uniqueIdJsonReader.read(reader)); + } + reader.endArray(); + + if (logger.isDebugEnabled()) { + traceSegment.getGlobalTraceIds().forEach(uniqueId -> { + StringBuilder globalTraceId = new StringBuilder(); + uniqueId.getIdPartsList().forEach(idPart -> globalTraceId.append(idPart)); + logger.debug("global trace id: {}", globalTraceId.toString()); + }); + } + break; + case SEGMENT: + traceSegment.setTraceSegmentObject(segmentJsonReader.read(reader)); + break; + default: + reader.skipValue(); + break; + } + } + reader.endObject(); + + return traceSegment; + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/UniqueIdJsonReader.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/UniqueIdJsonReader.java new file mode 100644 index 0000000000000000000000000000000000000000..8df8e1e0bf0f8f6371c86b52f70ef91bdaf99be7 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/UniqueIdJsonReader.java @@ -0,0 +1,22 @@ +package org.skywalking.apm.collector.agentstream.jetty.handler.reader; + +import com.google.gson.stream.JsonReader; +import java.io.IOException; +import org.skywalking.apm.network.proto.UniqueId; + +/** + * @author pengys5 + */ +public class UniqueIdJsonReader implements StreamJsonReader { + + @Override public UniqueId read(JsonReader reader) throws IOException { + UniqueId.Builder builder = UniqueId.newBuilder(); + + reader.beginArray(); + while (reader.hasNext()) { + builder.addIdParts(reader.nextLong()); + } + reader.endArray(); + return builder.build(); + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/CommonTable.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/CommonTable.java index 9ddc733b81fd313b9febf0251b928a78585a92f4..c66f627adf10beda0f3031e360c7ecefa3800eee 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/CommonTable.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/CommonTable.java @@ -7,6 +7,5 @@ public class CommonTable { public static final String TABLE_TYPE = "type"; public static final String COLUMN_ID = "id"; public static final String COLUMN_AGG = "agg"; - public static final String COLUMN_EXCHANGE_TIMES = "exchange_times"; public static final String COLUMN_TIME_BUCKET = "time_bucket"; } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/Const.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/Const.java index b0d8ec4159d7e24229a8f15a7475706053a81832..df48bd8374a6959ff5e4efbe3dc432f868c3efa7 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/Const.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/Const.java @@ -8,6 +8,7 @@ public class Const { public static final String IDS_SPLIT = "\\.\\.-\\.\\."; public static final String PEERS_FRONT_SPLIT = "["; public static final String PEERS_BEHIND_SPLIT = "]"; + public static final int USER_ID = 1; public static final String USER_CODE = "User"; public static final String SEGMENT_SPAN_SPLIT = "S"; } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/cache/ApplicationCache.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/cache/ApplicationCache.java new file mode 100644 index 0000000000000000000000000000000000000000..ff557402962b68cb401e1a1f28b3630a061c6f25 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/cache/ApplicationCache.java @@ -0,0 +1,25 @@ +package org.skywalking.apm.collector.agentstream.worker.cache; + +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import org.skywalking.apm.collector.agentstream.worker.register.application.dao.IApplicationDAO; +import org.skywalking.apm.collector.storage.dao.DAOContainer; + +/** + * @author pengys5 + */ +public class ApplicationCache { + + private static Cache CACHE = CacheBuilder.newBuilder().maximumSize(1000).build(); + + public static int get(String applicationCode) { + try { + return CACHE.get(applicationCode, () -> { + IApplicationDAO dao = (IApplicationDAO)DAOContainer.INSTANCE.get(IApplicationDAO.class.getName()); + return dao.getApplicationId(applicationCode); + }); + } catch (Throwable e) { + return 0; + } + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/cache/ComponentCache.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/cache/ComponentCache.java deleted file mode 100644 index fe472ce6ed44799d6ebe15e4cfe4dfd71a39654c..0000000000000000000000000000000000000000 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/cache/ComponentCache.java +++ /dev/null @@ -1,26 +0,0 @@ -package org.skywalking.apm.collector.agentstream.worker.cache; - -import com.google.common.cache.Cache; -import com.google.common.cache.CacheBuilder; -import org.skywalking.apm.collector.agentstream.worker.Const; -import org.skywalking.apm.collector.agentstream.worker.node.component.dao.INodeComponentDAO; -import org.skywalking.apm.collector.storage.dao.DAOContainer; - -/** - * @author pengys5 - */ -public class ComponentCache { - - private static Cache CACHE = CacheBuilder.newBuilder().maximumSize(1000).build(); - - public static int get(int applicationId, String componentName) { - try { - return CACHE.get(applicationId + Const.ID_SPLIT + componentName, () -> { - INodeComponentDAO dao = (INodeComponentDAO)DAOContainer.INSTANCE.get(INodeComponentDAO.class.getName()); - return dao.getComponentId(applicationId, componentName); - }); - } catch (Throwable e) { - return 0; - } - } -} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/cache/InstanceCache.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/cache/InstanceCache.java new file mode 100644 index 0000000000000000000000000000000000000000..54ac23020acd51489e9784ecdabf69ac14cdca22 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/cache/InstanceCache.java @@ -0,0 +1,25 @@ +package org.skywalking.apm.collector.agentstream.worker.cache; + +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import org.skywalking.apm.collector.agentstream.worker.register.instance.dao.IInstanceDAO; +import org.skywalking.apm.collector.storage.dao.DAOContainer; + +/** + * @author pengys5 + */ +public class InstanceCache { + + private static Cache CACHE = CacheBuilder.newBuilder().maximumSize(1000).build(); + + public static int get(int applicationInstanceId) { + try { + return CACHE.get(applicationInstanceId, () -> { + IInstanceDAO dao = (IInstanceDAO)DAOContainer.INSTANCE.get(IInstanceDAO.class.getName()); + return dao.getApplicationId(applicationInstanceId); + }); + } catch (Throwable e) { + return 0; + } + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/cache/ServiceNameCache.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/cache/ServiceNameCache.java new file mode 100644 index 0000000000000000000000000000000000000000..f6a88b9807a1cce4942bdb338f2b9dea00ba93ae --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/cache/ServiceNameCache.java @@ -0,0 +1,26 @@ +package org.skywalking.apm.collector.agentstream.worker.cache; + +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import org.skywalking.apm.collector.agentstream.worker.Const; +import org.skywalking.apm.collector.agentstream.worker.register.servicename.dao.IServiceNameDAO; +import org.skywalking.apm.collector.storage.dao.DAOContainer; + +/** + * @author pengys5 + */ +public class ServiceNameCache { + + private static Cache CACHE = CacheBuilder.newBuilder().maximumSize(2000).build(); + + public static int get(int applicationId, String serviceName) { + try { + return CACHE.get(applicationId + Const.ID_SPLIT + serviceName, () -> { + IServiceNameDAO dao = (IServiceNameDAO)DAOContainer.INSTANCE.get(IServiceNameDAO.class.getName()); + return dao.getServiceId(applicationId, serviceName); + }); + } catch (Throwable e) { + return 0; + } + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/global/GlobalTracePersistenceWorker.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/global/GlobalTracePersistenceWorker.java index ba58718b64b613e06e17428301d84187e951837b..8998c71eeb2f7ad60fa57f6b596f04832bd504d4 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/global/GlobalTracePersistenceWorker.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/global/GlobalTracePersistenceWorker.java @@ -1,7 +1,5 @@ package org.skywalking.apm.collector.agentstream.worker.global; -import java.util.List; -import java.util.Map; import org.skywalking.apm.collector.agentstream.worker.global.dao.IGlobalTraceDAO; import org.skywalking.apm.collector.agentstream.worker.global.define.GlobalTraceDataDefine; import org.skywalking.apm.collector.storage.dao.DAOContainer; @@ -10,7 +8,7 @@ import org.skywalking.apm.collector.stream.worker.ClusterWorkerContext; import org.skywalking.apm.collector.stream.worker.ProviderNotFoundException; import org.skywalking.apm.collector.stream.worker.Role; import org.skywalking.apm.collector.stream.worker.impl.PersistenceWorker; -import org.skywalking.apm.collector.stream.worker.impl.data.Data; +import org.skywalking.apm.collector.stream.worker.impl.dao.IPersistenceDAO; import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; import org.skywalking.apm.collector.stream.worker.selector.RollingSelector; import org.skywalking.apm.collector.stream.worker.selector.WorkerSelector; @@ -28,9 +26,12 @@ public class GlobalTracePersistenceWorker extends PersistenceWorker { super.preStart(); } - @Override protected List prepareBatch(Map dataMap) { - IGlobalTraceDAO dao = (IGlobalTraceDAO)DAOContainer.INSTANCE.get(IGlobalTraceDAO.class.getName()); - return dao.prepareBatch(dataMap); + @Override protected boolean needMergeDBData() { + return false; + } + + @Override protected IPersistenceDAO persistenceDAO() { + return (IPersistenceDAO)DAOContainer.INSTANCE.get(IGlobalTraceDAO.class.getName()); } public static class Factory extends AbstractLocalAsyncWorkerProvider { diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/global/dao/GlobalTraceEsDAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/global/dao/GlobalTraceEsDAO.java index 4c1c49707facf0c8d6b7ad5d598119b02a831634..c36ca11085555b8c2caea1551baa97c67b16ffba 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/global/dao/GlobalTraceEsDAO.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/global/dao/GlobalTraceEsDAO.java @@ -1,35 +1,38 @@ package org.skywalking.apm.collector.agentstream.worker.global.dao; -import java.util.ArrayList; import java.util.HashMap; -import java.util.List; import java.util.Map; import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.action.update.UpdateRequestBuilder; import org.skywalking.apm.collector.agentstream.worker.global.define.GlobalTraceTable; import org.skywalking.apm.collector.storage.elasticsearch.dao.EsDAO; +import org.skywalking.apm.collector.stream.worker.impl.dao.IPersistenceDAO; import org.skywalking.apm.collector.stream.worker.impl.data.Data; +import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * @author pengys5 */ -public class GlobalTraceEsDAO extends EsDAO implements IGlobalTraceDAO { +public class GlobalTraceEsDAO extends EsDAO implements IGlobalTraceDAO, IPersistenceDAO { private final Logger logger = LoggerFactory.getLogger(GlobalTraceEsDAO.class); - @Override public List prepareBatch(Map dataMap) { - List indexRequestBuilders = new ArrayList<>(); - dataMap.forEach((id, data) -> { - logger.debug("global trace prepareBatch, id: {}", id); - Map source = new HashMap(); - source.put(GlobalTraceTable.COLUMN_SEGMENT_ID, data.getDataString(1)); - source.put(GlobalTraceTable.COLUMN_GLOBAL_TRACE_ID, data.getDataString(2)); - source.put(GlobalTraceTable.COLUMN_TIME_BUCKET, data.getDataLong(0)); - logger.debug("global trace source: {}", source.toString()); - IndexRequestBuilder builder = getClient().prepareIndex(GlobalTraceTable.TABLE, id).setSource(source); - indexRequestBuilders.add(builder); - }); - return indexRequestBuilders; + @Override public Data get(String id, DataDefine dataDefine) { + return null; + } + + @Override public UpdateRequestBuilder prepareBatchUpdate(Data data) { + return null; + } + + @Override public IndexRequestBuilder prepareBatchInsert(Data data) { + Map source = new HashMap<>(); + source.put(GlobalTraceTable.COLUMN_SEGMENT_ID, data.getDataString(1)); + source.put(GlobalTraceTable.COLUMN_GLOBAL_TRACE_ID, data.getDataString(2)); + source.put(GlobalTraceTable.COLUMN_TIME_BUCKET, data.getDataLong(0)); + logger.debug("global trace source: {}", source.toString()); + return getClient().prepareIndex(GlobalTraceTable.TABLE, data.getDataString(0)).setSource(source); } } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/global/dao/GlobalTraceH2DAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/global/dao/GlobalTraceH2DAO.java index 1a34c616a9e5d0b36280993f210deb87aa494898..ea48e612f3c99e055f47be76676f643ec0a86e3e 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/global/dao/GlobalTraceH2DAO.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/global/dao/GlobalTraceH2DAO.java @@ -1,15 +1,9 @@ package org.skywalking.apm.collector.agentstream.worker.global.dao; -import java.util.List; -import java.util.Map; import org.skywalking.apm.collector.storage.h2.dao.H2DAO; /** * @author pengys5 */ public class GlobalTraceH2DAO extends H2DAO implements IGlobalTraceDAO { - - @Override public List prepareBatch(Map map) { - return null; - } } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/global/dao/IGlobalTraceDAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/global/dao/IGlobalTraceDAO.java index 4138f423cbe1d34ad4a985b31b351dabf6361632..1b73837c8cfc461e46e827a8c363cb9ca472dac0 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/global/dao/IGlobalTraceDAO.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/global/dao/IGlobalTraceDAO.java @@ -1,12 +1,7 @@ package org.skywalking.apm.collector.agentstream.worker.global.dao; -import java.util.List; -import java.util.Map; -import org.skywalking.apm.collector.stream.worker.impl.data.Data; - /** * @author pengys5 */ public interface IGlobalTraceDAO { - List prepareBatch(Map dataMap); } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/global/define/GlobalTraceDataDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/global/define/GlobalTraceDataDefine.java index 16ba94c1ef2ec2b2172d143f22bd9fad3d7ef96c..9d851d5e5fe2eedf7c6e48304d10347c53f2d5dc 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/global/define/GlobalTraceDataDefine.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/global/define/GlobalTraceDataDefine.java @@ -14,10 +14,6 @@ import org.skywalking.apm.collector.stream.worker.impl.data.operate.NonOperation */ public class GlobalTraceDataDefine extends DataDefine { - @Override public int defineId() { - return 403; - } - @Override protected int initialCapacity() { return 4; } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentExchangeWorker.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentExchangeWorker.java deleted file mode 100644 index 0568474301057a49aeadbfa4e3ea8e50feee483b..0000000000000000000000000000000000000000 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentExchangeWorker.java +++ /dev/null @@ -1,83 +0,0 @@ -package org.skywalking.apm.collector.agentstream.worker.node.component; - -import org.skywalking.apm.collector.agentstream.worker.cache.ComponentCache; -import org.skywalking.apm.collector.agentstream.worker.node.component.define.NodeComponentDataDefine; -import org.skywalking.apm.collector.stream.worker.AbstractLocalAsyncWorkerProvider; -import org.skywalking.apm.collector.stream.worker.ClusterWorkerContext; -import org.skywalking.apm.collector.stream.worker.ProviderNotFoundException; -import org.skywalking.apm.collector.stream.worker.Role; -import org.skywalking.apm.collector.stream.worker.WorkerInvokeException; -import org.skywalking.apm.collector.stream.worker.WorkerNotFoundException; -import org.skywalking.apm.collector.stream.worker.impl.ExchangeWorker; -import org.skywalking.apm.collector.stream.worker.impl.data.Data; -import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; -import org.skywalking.apm.collector.stream.worker.selector.HashCodeSelector; -import org.skywalking.apm.collector.stream.worker.selector.WorkerSelector; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * @author pengys5 - */ -public class NodeComponentExchangeWorker extends ExchangeWorker { - - private final Logger logger = LoggerFactory.getLogger(NodeComponentExchangeWorker.class); - - public NodeComponentExchangeWorker(Role role, ClusterWorkerContext clusterContext) { - super(role, clusterContext); - } - - @Override public void preStart() throws ProviderNotFoundException { - super.preStart(); - } - - @Override protected void exchange(Data data) { - NodeComponentDataDefine.NodeComponent nodeComponent = new NodeComponentDataDefine.NodeComponent(); - nodeComponent.toSelf(data); - - int componentId = ComponentCache.get(nodeComponent.getApplicationId(), nodeComponent.getComponentName()); - if (componentId == 0 && nodeComponent.getTimes() < 10) { - try { - nodeComponent.increase(); - getClusterContext().lookup(NodeComponentExchangeWorker.WorkerRole.INSTANCE).tell(nodeComponent.toData()); - } catch (WorkerNotFoundException | WorkerInvokeException e) { - logger.error(e.getMessage(), e); - } - } - } - - public static class Factory extends AbstractLocalAsyncWorkerProvider { - @Override - public Role role() { - return WorkerRole.INSTANCE; - } - - @Override - public NodeComponentExchangeWorker workerInstance(ClusterWorkerContext clusterContext) { - return new NodeComponentExchangeWorker(role(), clusterContext); - } - - @Override - public int queueSize() { - return 1024; - } - } - - public enum WorkerRole implements Role { - INSTANCE; - - @Override - public String roleName() { - return NodeComponentExchangeWorker.class.getSimpleName(); - } - - @Override - public WorkerSelector workerSelector() { - return new HashCodeSelector(); - } - - @Override public DataDefine dataDefine() { - return new NodeComponentDataDefine(); - } - } -} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentPersistenceWorker.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentPersistenceWorker.java index 2e7ccf0b92d766970d6b248e3644fc21777ec091..5ff722b8657983ba178c7aa7f90dc609c1333583 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentPersistenceWorker.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentPersistenceWorker.java @@ -1,7 +1,5 @@ package org.skywalking.apm.collector.agentstream.worker.node.component; -import java.util.List; -import java.util.Map; import org.skywalking.apm.collector.agentstream.worker.node.component.dao.INodeComponentDAO; import org.skywalking.apm.collector.agentstream.worker.node.component.define.NodeComponentDataDefine; import org.skywalking.apm.collector.storage.dao.DAOContainer; @@ -10,7 +8,7 @@ import org.skywalking.apm.collector.stream.worker.ClusterWorkerContext; import org.skywalking.apm.collector.stream.worker.ProviderNotFoundException; import org.skywalking.apm.collector.stream.worker.Role; import org.skywalking.apm.collector.stream.worker.impl.PersistenceWorker; -import org.skywalking.apm.collector.stream.worker.impl.data.Data; +import org.skywalking.apm.collector.stream.worker.impl.dao.IPersistenceDAO; import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; import org.skywalking.apm.collector.stream.worker.selector.HashCodeSelector; import org.skywalking.apm.collector.stream.worker.selector.WorkerSelector; @@ -28,9 +26,12 @@ public class NodeComponentPersistenceWorker extends PersistenceWorker { super.preStart(); } - @Override protected List prepareBatch(Map dataMap) { - INodeComponentDAO dao = (INodeComponentDAO)DAOContainer.INSTANCE.get(INodeComponentDAO.class.getName()); - return dao.prepareBatch(dataMap); + @Override protected boolean needMergeDBData() { + return true; + } + + @Override protected IPersistenceDAO persistenceDAO() { + return (IPersistenceDAO)DAOContainer.INSTANCE.get(INodeComponentDAO.class.getName()); } public static class Factory extends AbstractLocalAsyncWorkerProvider { diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentSpanListener.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentSpanListener.java index 3bc4be9249874858d97be86eb916bb3cad34de5e..6f83321d49bf77d56fb8530baeaa95aeae363220 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentSpanListener.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/NodeComponentSpanListener.java @@ -3,85 +3,89 @@ package org.skywalking.apm.collector.agentstream.worker.node.component; import java.util.ArrayList; import java.util.List; import org.skywalking.apm.collector.agentstream.worker.Const; -import org.skywalking.apm.collector.agentstream.worker.cache.ComponentCache; import org.skywalking.apm.collector.agentstream.worker.node.component.define.NodeComponentDataDefine; import org.skywalking.apm.collector.agentstream.worker.segment.EntrySpanListener; import org.skywalking.apm.collector.agentstream.worker.segment.ExitSpanListener; +import org.skywalking.apm.collector.agentstream.worker.segment.FirstSpanListener; import org.skywalking.apm.collector.agentstream.worker.segment.LocalSpanListener; +import org.skywalking.apm.collector.agentstream.worker.util.ExchangeMarkUtils; +import org.skywalking.apm.collector.agentstream.worker.util.TimeBucketUtils; import org.skywalking.apm.collector.core.framework.CollectorContextHelper; import org.skywalking.apm.collector.stream.StreamModuleContext; import org.skywalking.apm.collector.stream.StreamModuleGroupDefine; import org.skywalking.apm.collector.stream.worker.WorkerInvokeException; import org.skywalking.apm.collector.stream.worker.WorkerNotFoundException; import org.skywalking.apm.network.proto.SpanObject; -import org.skywalking.apm.network.trace.component.ComponentsDefine; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * @author pengys5 */ -public class NodeComponentSpanListener implements EntrySpanListener, ExitSpanListener, LocalSpanListener { +public class NodeComponentSpanListener implements EntrySpanListener, ExitSpanListener, FirstSpanListener, LocalSpanListener { private final Logger logger = LoggerFactory.getLogger(NodeComponentSpanListener.class); - private List nodeComponents = new ArrayList<>(); + private List nodeComponents = new ArrayList<>(); + private long timeBucket; @Override public void parseExit(SpanObject spanObject, int applicationId, int applicationInstanceId, String segmentId) { - String componentName = ComponentsDefine.getInstance().getComponentName(spanObject.getComponentId()); - createNodeComponent(spanObject, applicationId, componentName); + String componentName = ExchangeMarkUtils.INSTANCE.buildMarkedID(spanObject.getComponentId()); + if (spanObject.getComponentId() == 0) { + componentName = spanObject.getComponent(); + } + String peer = ExchangeMarkUtils.INSTANCE.buildMarkedID(spanObject.getPeerId()); + if (spanObject.getPeerId() == 0) { + peer = spanObject.getPeer(); + } + + String agg = peer + Const.ID_SPLIT + componentName; + nodeComponents.add(agg); } @Override public void parseEntry(SpanObject spanObject, int applicationId, int applicationInstanceId, String segmentId) { - String componentName = ComponentsDefine.getInstance().getComponentName(spanObject.getComponentId()); - createNodeComponent(spanObject, applicationId, componentName); + buildEntryOrLocal(spanObject, applicationId); } @Override public void parseLocal(SpanObject spanObject, int applicationId, int applicationInstanceId, String segmentId) { - int componentId = ComponentCache.get(applicationId, spanObject.getComponent()); - - NodeComponentDataDefine.NodeComponent nodeComponent = new NodeComponentDataDefine.NodeComponent(); - nodeComponent.setApplicationId(applicationId); - nodeComponent.setComponentId(componentId); - nodeComponent.setComponentName(spanObject.getComponent()); + buildEntryOrLocal(spanObject, applicationId); + } - if (componentId == 0) { - StreamModuleContext context = (StreamModuleContext)CollectorContextHelper.INSTANCE.getContext(StreamModuleGroupDefine.GROUP_NAME); + private void buildEntryOrLocal(SpanObject spanObject, int applicationId) { + String componentName = ExchangeMarkUtils.INSTANCE.buildMarkedID(spanObject.getComponentId()); - logger.debug("send to node component exchange worker, id: {}", nodeComponent.getId()); - nodeComponent.setId(applicationId + Const.ID_SPLIT + spanObject.getComponent()); - try { - context.getClusterWorkerContext().lookup(NodeComponentExchangeWorker.WorkerRole.INSTANCE).tell(nodeComponent.toData()); - } catch (WorkerInvokeException | WorkerNotFoundException e) { - logger.error(e.getMessage(), e); - } - } else { - nodeComponent.setId(applicationId + Const.ID_SPLIT + componentId); - nodeComponents.add(nodeComponent); + if (spanObject.getComponentId() == 0) { + componentName = spanObject.getComponent(); } + + String peer = ExchangeMarkUtils.INSTANCE.buildMarkedID(applicationId); + String agg = peer + Const.ID_SPLIT + componentName; + nodeComponents.add(agg); } - private void createNodeComponent(SpanObject spanObject, int applicationId, String componentName) { - NodeComponentDataDefine.NodeComponent nodeComponent = new NodeComponentDataDefine.NodeComponent(); - nodeComponent.setApplicationId(applicationId); - nodeComponent.setComponentId(spanObject.getComponentId()); - nodeComponent.setComponentName(componentName); - nodeComponent.setId(applicationId + Const.ID_SPLIT + spanObject.getComponentId()); - nodeComponents.add(nodeComponent); + @Override + public void parseFirst(SpanObject spanObject, int applicationId, int applicationInstanceId, String segmentId) { + timeBucket = TimeBucketUtils.INSTANCE.getMinuteTimeBucket(spanObject.getStartTime()); } @Override public void build() { StreamModuleContext context = (StreamModuleContext)CollectorContextHelper.INSTANCE.getContext(StreamModuleGroupDefine.GROUP_NAME); - for (NodeComponentDataDefine.NodeComponent nodeComponent : nodeComponents) { + + nodeComponents.forEach(agg -> { + NodeComponentDataDefine.NodeComponent nodeComponent = new NodeComponentDataDefine.NodeComponent(); + nodeComponent.setId(timeBucket + Const.ID_SPLIT + agg); + nodeComponent.setAgg(agg); + nodeComponent.setTimeBucket(timeBucket); + try { logger.debug("send to node component aggregation worker, id: {}", nodeComponent.getId()); context.getClusterWorkerContext().lookup(NodeComponentAggregationWorker.WorkerRole.INSTANCE).tell(nodeComponent.toData()); } catch (WorkerInvokeException | WorkerNotFoundException e) { logger.error(e.getMessage(), e); } - } + }); } } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/dao/INodeComponentDAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/dao/INodeComponentDAO.java index 3ae17bd74c695ee8fb8c82f7abfea6a9612a370a..79f44ccb5f394148556918d14cbe64fb47d876c7 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/dao/INodeComponentDAO.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/dao/INodeComponentDAO.java @@ -1,14 +1,7 @@ package org.skywalking.apm.collector.agentstream.worker.node.component.dao; -import java.util.List; -import java.util.Map; -import org.skywalking.apm.collector.stream.worker.impl.data.Data; - /** * @author pengys5 */ public interface INodeComponentDAO { - List prepareBatch(Map dataMap); - - int getComponentId(int applicationId, String componentName); } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/dao/NodeComponentEsDAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/dao/NodeComponentEsDAO.java index 54e2e1442fc1ccd8f3ad1be99a99a352efe582c8..a3e114f90a74100f11921ceceb37cac70fa25c31 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/dao/NodeComponentEsDAO.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/dao/NodeComponentEsDAO.java @@ -1,58 +1,47 @@ package org.skywalking.apm.collector.agentstream.worker.node.component.dao; -import java.util.ArrayList; import java.util.HashMap; -import java.util.List; import java.util.Map; +import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.index.IndexRequestBuilder; -import org.elasticsearch.action.search.SearchRequestBuilder; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.action.search.SearchType; -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.search.SearchHit; +import org.elasticsearch.action.update.UpdateRequestBuilder; import org.skywalking.apm.collector.agentstream.worker.node.component.define.NodeComponentTable; -import org.skywalking.apm.collector.client.elasticsearch.ElasticSearchClient; import org.skywalking.apm.collector.storage.elasticsearch.dao.EsDAO; +import org.skywalking.apm.collector.stream.worker.impl.dao.IPersistenceDAO; import org.skywalking.apm.collector.stream.worker.impl.data.Data; +import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; /** * @author pengys5 */ -public class NodeComponentEsDAO extends EsDAO implements INodeComponentDAO { +public class NodeComponentEsDAO extends EsDAO implements INodeComponentDAO, IPersistenceDAO { - @Override public List prepareBatch(Map dataMap) { - List indexRequestBuilders = new ArrayList<>(); - dataMap.forEach((id, data) -> { - Map source = new HashMap(); - source.put(NodeComponentTable.COLUMN_APPLICATION_ID, data.getDataInteger(0)); - source.put(NodeComponentTable.COLUMN_COMPONENT_NAME, data.getDataString(1)); - source.put(NodeComponentTable.COLUMN_COMPONENT_ID, data.getDataInteger(1)); - - IndexRequestBuilder builder = getClient().prepareIndex(NodeComponentTable.TABLE, id).setSource(source); - indexRequestBuilders.add(builder); - }); - return indexRequestBuilders; + @Override public Data get(String id, DataDefine dataDefine) { + GetResponse getResponse = getClient().prepareGet(NodeComponentTable.TABLE, id).get(); + if (getResponse.isExists()) { + Data data = dataDefine.build(id); + Map source = getResponse.getSource(); + data.setDataString(1, (String)source.get(NodeComponentTable.COLUMN_AGG)); + data.setDataLong(0, (Long)source.get(NodeComponentTable.COLUMN_TIME_BUCKET)); + return data; + } else { + return null; + } } - public int getComponentId(int applicationId, String componentName) { - ElasticSearchClient client = getClient(); + @Override public IndexRequestBuilder prepareBatchInsert(Data data) { + Map source = new HashMap<>(); + source.put(NodeComponentTable.COLUMN_AGG, data.getDataString(1)); + source.put(NodeComponentTable.COLUMN_TIME_BUCKET, data.getDataLong(0)); - SearchRequestBuilder searchRequestBuilder = client.prepareSearch(NodeComponentTable.TABLE); - searchRequestBuilder.setTypes("type"); - searchRequestBuilder.setSearchType(SearchType.QUERY_THEN_FETCH); - BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); - boolQueryBuilder.must(QueryBuilders.termQuery(NodeComponentTable.COLUMN_APPLICATION_ID, applicationId)); - boolQueryBuilder.must(QueryBuilders.termQuery(NodeComponentTable.COLUMN_COMPONENT_NAME, componentName)); - searchRequestBuilder.setQuery(boolQueryBuilder); - searchRequestBuilder.setSize(1); + return getClient().prepareIndex(NodeComponentTable.TABLE, data.getDataString(0)).setSource(source); + } - SearchResponse searchResponse = searchRequestBuilder.execute().actionGet(); - if (searchResponse.getHits().totalHits > 0) { - SearchHit searchHit = searchResponse.getHits().iterator().next(); - int componentId = (int)searchHit.getSource().get(NodeComponentTable.COLUMN_COMPONENT_ID); - return componentId; - } - return 0; + @Override public UpdateRequestBuilder prepareBatchUpdate(Data data) { + Map source = new HashMap<>(); + source.put(NodeComponentTable.COLUMN_AGG, data.getDataString(1)); + source.put(NodeComponentTable.COLUMN_TIME_BUCKET, data.getDataLong(0)); + + return getClient().prepareUpdate(NodeComponentTable.TABLE, data.getDataString(0)).setDoc(source); } } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/dao/NodeComponentH2DAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/dao/NodeComponentH2DAO.java index 3277e059d6c52feb44aee23880e1ce36b500b59f..5e08b344508fa1a1e09627c38b6c88fe541d54e8 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/dao/NodeComponentH2DAO.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/dao/NodeComponentH2DAO.java @@ -1,7 +1,5 @@ package org.skywalking.apm.collector.agentstream.worker.node.component.dao; -import java.util.List; -import java.util.Map; import org.skywalking.apm.collector.storage.h2.dao.H2DAO; /** @@ -9,11 +7,4 @@ import org.skywalking.apm.collector.storage.h2.dao.H2DAO; */ public class NodeComponentH2DAO extends H2DAO implements INodeComponentDAO { - @Override public List prepareBatch(Map map) { - return null; - } - - @Override public int getComponentId(int applicationId, String componentName) { - return 0; - } } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/define/NodeComponentDataDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/define/NodeComponentDataDefine.java index 7e2342d22b5b9f1c8ba58fa70dcb1e61c9f79e30..e617a9679641419ca1fe0c3b212284ab3eb9c05c 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/define/NodeComponentDataDefine.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/define/NodeComponentDataDefine.java @@ -5,7 +5,6 @@ import org.skywalking.apm.collector.stream.worker.impl.data.Attribute; import org.skywalking.apm.collector.stream.worker.impl.data.AttributeType; import org.skywalking.apm.collector.stream.worker.impl.data.Data; import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; -import org.skywalking.apm.collector.stream.worker.impl.data.Exchange; import org.skywalking.apm.collector.stream.worker.impl.data.Transform; import org.skywalking.apm.collector.stream.worker.impl.data.operate.CoverOperation; import org.skywalking.apm.collector.stream.worker.impl.data.operate.NonOperation; @@ -15,20 +14,14 @@ import org.skywalking.apm.collector.stream.worker.impl.data.operate.NonOperation */ public class NodeComponentDataDefine extends DataDefine { - @Override public int defineId() { - return 101; - } - @Override protected int initialCapacity() { - return 5; + return 3; } @Override protected void attributeDefine() { addAttribute(0, new Attribute(NodeComponentTable.COLUMN_ID, AttributeType.STRING, new NonOperation())); - addAttribute(1, new Attribute(NodeComponentTable.COLUMN_APPLICATION_ID, AttributeType.INTEGER, new CoverOperation())); - addAttribute(2, new Attribute(NodeComponentTable.COLUMN_COMPONENT_NAME, AttributeType.STRING, new CoverOperation())); - addAttribute(3, new Attribute(NodeComponentTable.COLUMN_COMPONENT_ID, AttributeType.INTEGER, new CoverOperation())); - addAttribute(4, new Attribute(NodeComponentTable.COLUMN_EXCHANGE_TIMES, AttributeType.INTEGER, new NonOperation())); + addAttribute(1, new Attribute(NodeComponentTable.COLUMN_AGG, AttributeType.STRING, new CoverOperation())); + addAttribute(2, new Attribute(NodeComponentTable.COLUMN_TIME_BUCKET, AttributeType.LONG, new CoverOperation())); } @Override public Object deserialize(RemoteData remoteData) { @@ -39,41 +32,33 @@ public class NodeComponentDataDefine extends DataDefine { return null; } - public static class NodeComponent extends Exchange implements Transform { + public static class NodeComponent implements Transform { private String id; - private int applicationId; - private String componentName; - private int componentId; + private String agg; + private long timeBucket; - public NodeComponent(String id, int applicationId, String componentName, int componentId) { - super(0); + NodeComponent(String id, String agg, long timeBucket) { this.id = id; - this.applicationId = applicationId; - this.componentName = componentName; - this.componentId = componentId; + this.agg = agg; + this.timeBucket = timeBucket; } public NodeComponent() { - super(0); } @Override public Data toData() { NodeComponentDataDefine define = new NodeComponentDataDefine(); Data data = define.build(id); data.setDataString(0, this.id); - data.setDataInteger(0, this.applicationId); - data.setDataString(1, this.componentName); - data.setDataInteger(1, this.componentId); - data.setDataInteger(2, this.getTimes()); + data.setDataString(1, this.agg); + data.setDataLong(0, this.timeBucket); return data; } @Override public NodeComponent toSelf(Data data) { this.id = data.getDataString(0); - this.applicationId = data.getDataInteger(0); - this.componentName = data.getDataString(1); - this.componentId = data.getDataInteger(1); - this.setTimes(data.getDataInteger(2)); + this.agg = data.getDataString(1); + this.timeBucket = data.getDataLong(0); return this; } @@ -81,32 +66,24 @@ public class NodeComponentDataDefine extends DataDefine { return id; } - public void setId(String id) { - this.id = id; - } - - public String getComponentName() { - return componentName; - } - - public void setComponentName(String componentName) { - this.componentName = componentName; + public String getAgg() { + return agg; } - public int getComponentId() { - return componentId; + public long getTimeBucket() { + return timeBucket; } - public void setComponentId(int componentId) { - this.componentId = componentId; + public void setId(String id) { + this.id = id; } - public int getApplicationId() { - return applicationId; + public void setAgg(String agg) { + this.agg = agg; } - public void setApplicationId(int applicationId) { - this.applicationId = applicationId; + public void setTimeBucket(long timeBucket) { + this.timeBucket = timeBucket; } } } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/define/NodeComponentEsTableDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/define/NodeComponentEsTableDefine.java index 9085e2b8321c2401264c2bdb374cb1a963c91513..e4e8e2809d03e4d3fca1158ab0309855bf36c3fd 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/define/NodeComponentEsTableDefine.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/define/NodeComponentEsTableDefine.java @@ -25,8 +25,7 @@ public class NodeComponentEsTableDefine extends ElasticSearchTableDefine { } @Override public void initialize() { - addColumn(new ElasticSearchColumnDefine(NodeComponentTable.COLUMN_APPLICATION_ID, ElasticSearchColumnDefine.Type.Integer.name())); - addColumn(new ElasticSearchColumnDefine(NodeComponentTable.COLUMN_COMPONENT_NAME, ElasticSearchColumnDefine.Type.Keyword.name())); - addColumn(new ElasticSearchColumnDefine(NodeComponentTable.COLUMN_COMPONENT_ID, ElasticSearchColumnDefine.Type.Integer.name())); + addColumn(new ElasticSearchColumnDefine(NodeComponentTable.COLUMN_AGG, ElasticSearchColumnDefine.Type.Keyword.name())); + addColumn(new ElasticSearchColumnDefine(NodeComponentTable.COLUMN_TIME_BUCKET, ElasticSearchColumnDefine.Type.Long.name())); } } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/define/NodeComponentH2TableDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/define/NodeComponentH2TableDefine.java index b85787ee77474566f169e723b7112a09e85abe47..62b77b04e33fbca91d030d86eb71453d0b31a386 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/define/NodeComponentH2TableDefine.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/define/NodeComponentH2TableDefine.java @@ -14,8 +14,7 @@ public class NodeComponentH2TableDefine extends H2TableDefine { @Override public void initialize() { addColumn(new H2ColumnDefine(NodeComponentTable.COLUMN_ID, H2ColumnDefine.Type.Varchar.name())); - addColumn(new H2ColumnDefine(NodeComponentTable.COLUMN_APPLICATION_ID, H2ColumnDefine.Type.Int.name())); - addColumn(new H2ColumnDefine(NodeComponentTable.COLUMN_COMPONENT_ID, H2ColumnDefine.Type.Int.name())); - addColumn(new H2ColumnDefine(NodeComponentTable.COLUMN_COMPONENT_NAME, H2ColumnDefine.Type.Varchar.name())); + addColumn(new H2ColumnDefine(NodeComponentTable.COLUMN_AGG, H2ColumnDefine.Type.Varchar.name())); + addColumn(new H2ColumnDefine(NodeComponentTable.COLUMN_TIME_BUCKET, H2ColumnDefine.Type.Bigint.name())); } } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/define/NodeComponentTable.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/define/NodeComponentTable.java index e9536528769a95855209aa28b894d387e9f338a8..1bce8df8347a2f43af9cff781429ba53872fd460 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/define/NodeComponentTable.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/component/define/NodeComponentTable.java @@ -7,7 +7,4 @@ import org.skywalking.apm.collector.agentstream.worker.CommonTable; */ public class NodeComponentTable extends CommonTable { public static final String TABLE = "node_component"; - public static final String COLUMN_APPLICATION_ID = "application_id"; - public static final String COLUMN_COMPONENT_NAME = "component_name"; - public static final String COLUMN_COMPONENT_ID = "component_id"; } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/NodeMappingPersistenceWorker.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/NodeMappingPersistenceWorker.java index 48a882d5a90c91d595619b475812b865348fb389..57edf24c52bf1c6176ff66785a3b425de25779bd 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/NodeMappingPersistenceWorker.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/NodeMappingPersistenceWorker.java @@ -1,7 +1,5 @@ package org.skywalking.apm.collector.agentstream.worker.node.mapping; -import java.util.List; -import java.util.Map; import org.skywalking.apm.collector.agentstream.worker.node.mapping.dao.INodeMappingDAO; import org.skywalking.apm.collector.agentstream.worker.node.mapping.define.NodeMappingDataDefine; import org.skywalking.apm.collector.storage.dao.DAOContainer; @@ -10,7 +8,7 @@ import org.skywalking.apm.collector.stream.worker.ClusterWorkerContext; import org.skywalking.apm.collector.stream.worker.ProviderNotFoundException; import org.skywalking.apm.collector.stream.worker.Role; import org.skywalking.apm.collector.stream.worker.impl.PersistenceWorker; -import org.skywalking.apm.collector.stream.worker.impl.data.Data; +import org.skywalking.apm.collector.stream.worker.impl.dao.IPersistenceDAO; import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; import org.skywalking.apm.collector.stream.worker.selector.HashCodeSelector; import org.skywalking.apm.collector.stream.worker.selector.WorkerSelector; @@ -28,9 +26,12 @@ public class NodeMappingPersistenceWorker extends PersistenceWorker { super.preStart(); } - @Override protected List prepareBatch(Map dataMap) { - INodeMappingDAO dao = (INodeMappingDAO)DAOContainer.INSTANCE.get(INodeMappingDAO.class.getName()); - return dao.prepareBatch(dataMap); + @Override protected boolean needMergeDBData() { + return true; + } + + @Override protected IPersistenceDAO persistenceDAO() { + return (IPersistenceDAO)DAOContainer.INSTANCE.get(INodeMappingDAO.class.getName()); } public static class Factory extends AbstractLocalAsyncWorkerProvider { diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/NodeMappingSpanListener.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/NodeMappingSpanListener.java index 4b8e4af7155b50cb24f64163b159715a6dd2cb77..38230c29b6138caa8a24cd4457a2d6f899c97ff5 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/NodeMappingSpanListener.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/NodeMappingSpanListener.java @@ -6,6 +6,7 @@ import org.skywalking.apm.collector.agentstream.worker.Const; import org.skywalking.apm.collector.agentstream.worker.node.mapping.define.NodeMappingDataDefine; import org.skywalking.apm.collector.agentstream.worker.segment.FirstSpanListener; import org.skywalking.apm.collector.agentstream.worker.segment.RefsListener; +import org.skywalking.apm.collector.agentstream.worker.util.ExchangeMarkUtils; import org.skywalking.apm.collector.agentstream.worker.util.TimeBucketUtils; import org.skywalking.apm.collector.core.framework.CollectorContextHelper; import org.skywalking.apm.collector.stream.StreamModuleContext; @@ -30,12 +31,12 @@ public class NodeMappingSpanListener implements RefsListener, FirstSpanListener @Override public void parseRef(TraceSegmentReference reference, int applicationId, int applicationInstanceId, String segmentId) { logger.debug("node mapping listener parse reference"); - String peers = Const.PEERS_FRONT_SPLIT + reference.getNetworkAddressId() + Const.PEERS_BEHIND_SPLIT; - if (reference.getNetworkAddressId() == 0) { - peers = Const.PEERS_FRONT_SPLIT + reference.getNetworkAddress() + Const.PEERS_BEHIND_SPLIT; + String peers = reference.getNetworkAddress(); + if (reference.getNetworkAddressId() != 0) { + peers = ExchangeMarkUtils.INSTANCE.buildMarkedID(reference.getNetworkAddressId()); } - String agg = applicationId + Const.ID_SPLIT + peers; + String agg = ExchangeMarkUtils.INSTANCE.buildMarkedID(applicationId) + Const.ID_SPLIT + peers; nodeMappings.add(agg); } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/dao/INodeMappingDAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/dao/INodeMappingDAO.java index f864617a2b158230a4312593ba5bf1d862888528..450cc54512a7906b3593b5b7ddd5a0feb6b49886 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/dao/INodeMappingDAO.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/dao/INodeMappingDAO.java @@ -1,12 +1,7 @@ package org.skywalking.apm.collector.agentstream.worker.node.mapping.dao; -import java.util.List; -import java.util.Map; -import org.skywalking.apm.collector.stream.worker.impl.data.Data; - /** * @author pengys5 */ public interface INodeMappingDAO { - List prepareBatch(Map dataMap); } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/dao/NodeMappingEsDAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/dao/NodeMappingEsDAO.java index 4a36d037831b22bb2fe1369a8b439ced6fbdfbbe..66a22211dbfa1e8078f63ffad1c5b7525e2b2be9 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/dao/NodeMappingEsDAO.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/dao/NodeMappingEsDAO.java @@ -1,29 +1,47 @@ package org.skywalking.apm.collector.agentstream.worker.node.mapping.dao; -import java.util.ArrayList; import java.util.HashMap; -import java.util.List; import java.util.Map; +import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.action.update.UpdateRequestBuilder; import org.skywalking.apm.collector.agentstream.worker.node.mapping.define.NodeMappingTable; import org.skywalking.apm.collector.storage.elasticsearch.dao.EsDAO; +import org.skywalking.apm.collector.stream.worker.impl.dao.IPersistenceDAO; import org.skywalking.apm.collector.stream.worker.impl.data.Data; +import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; /** * @author pengys5 */ -public class NodeMappingEsDAO extends EsDAO implements INodeMappingDAO { +public class NodeMappingEsDAO extends EsDAO implements INodeMappingDAO, IPersistenceDAO { - @Override public List prepareBatch(Map dataMap) { - List indexRequestBuilders = new ArrayList<>(); - dataMap.forEach((id, data) -> { - Map source = new HashMap(); - source.put(NodeMappingTable.COLUMN_AGG, data.getDataString(1)); - source.put(NodeMappingTable.COLUMN_TIME_BUCKET, data.getDataLong(0)); + @Override public Data get(String id, DataDefine dataDefine) { + GetResponse getResponse = getClient().prepareGet(NodeMappingTable.TABLE, id).get(); + if (getResponse.isExists()) { + Data data = dataDefine.build(id); + Map source = getResponse.getSource(); + data.setDataString(1, (String)source.get(NodeMappingTable.COLUMN_AGG)); + data.setDataLong(0, (Long)source.get(NodeMappingTable.COLUMN_TIME_BUCKET)); + return data; + } else { + return null; + } + } + + @Override public IndexRequestBuilder prepareBatchInsert(Data data) { + Map source = new HashMap<>(); + source.put(NodeMappingTable.COLUMN_AGG, data.getDataString(1)); + source.put(NodeMappingTable.COLUMN_TIME_BUCKET, data.getDataLong(0)); + + return getClient().prepareIndex(NodeMappingTable.TABLE, data.getDataString(0)).setSource(source); + } + + @Override public UpdateRequestBuilder prepareBatchUpdate(Data data) { + Map source = new HashMap<>(); + source.put(NodeMappingTable.COLUMN_AGG, data.getDataString(1)); + source.put(NodeMappingTable.COLUMN_TIME_BUCKET, data.getDataLong(0)); - IndexRequestBuilder builder = getClient().prepareIndex(NodeMappingTable.TABLE, id).setSource(source); - indexRequestBuilders.add(builder); - }); - return indexRequestBuilders; + return getClient().prepareUpdate(NodeMappingTable.TABLE, data.getDataString(0)).setDoc(source); } } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/dao/NodeMappingH2DAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/dao/NodeMappingH2DAO.java index 8ad24c091778ade401ebd132f55fabc871d36b49..045d0738e6f7391b8937505c20721d6716e14fa4 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/dao/NodeMappingH2DAO.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/dao/NodeMappingH2DAO.java @@ -1,15 +1,9 @@ package org.skywalking.apm.collector.agentstream.worker.node.mapping.dao; -import java.util.List; -import java.util.Map; import org.skywalking.apm.collector.storage.h2.dao.H2DAO; /** * @author pengys5 */ public class NodeMappingH2DAO extends H2DAO implements INodeMappingDAO { - - @Override public List prepareBatch(Map map) { - return null; - } } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/define/NodeMappingDataDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/define/NodeMappingDataDefine.java index 4c949355b1257c20c1f4293a5c75b24bfa234c87..881a0f163d6e672c861dd2506624d9c3047c2dad 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/define/NodeMappingDataDefine.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/node/mapping/define/NodeMappingDataDefine.java @@ -14,10 +14,6 @@ import org.skywalking.apm.collector.stream.worker.impl.data.operate.NonOperation */ public class NodeMappingDataDefine extends DataDefine { - @Override public int defineId() { - return 102; - } - @Override protected int initialCapacity() { return 3; } @@ -36,12 +32,12 @@ public class NodeMappingDataDefine extends DataDefine { return null; } - public static class NodeMapping implements Transform { + public static class NodeMapping implements Transform { private String id; private String agg; private long timeBucket; - public NodeMapping(String id, String agg, long timeBucket) { + NodeMapping(String id, String agg, long timeBucket) { this.id = id; this.agg = agg; this.timeBucket = timeBucket; @@ -59,8 +55,11 @@ public class NodeMappingDataDefine extends DataDefine { return data; } - @Override public Object toSelf(Data data) { - return null; + @Override public NodeMapping toSelf(Data data) { + this.id = data.getDataString(0); + this.agg = data.getDataString(1); + this.timeBucket = data.getDataLong(0); + return this; } public String getId() { diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/NodeRefPersistenceWorker.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/NodeRefPersistenceWorker.java index 06e363a1af7bf2810c26d8603d70ec5b1458f6ea..ec48c3a8c3dbd3d6926f8c85bc4510ffdad3d2ee 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/NodeRefPersistenceWorker.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/NodeRefPersistenceWorker.java @@ -1,7 +1,5 @@ package org.skywalking.apm.collector.agentstream.worker.noderef.reference; -import java.util.List; -import java.util.Map; import org.skywalking.apm.collector.agentstream.worker.noderef.reference.dao.INodeReferenceDAO; import org.skywalking.apm.collector.agentstream.worker.noderef.reference.define.NodeRefDataDefine; import org.skywalking.apm.collector.storage.dao.DAOContainer; @@ -10,7 +8,7 @@ import org.skywalking.apm.collector.stream.worker.ClusterWorkerContext; import org.skywalking.apm.collector.stream.worker.ProviderNotFoundException; import org.skywalking.apm.collector.stream.worker.Role; import org.skywalking.apm.collector.stream.worker.impl.PersistenceWorker; -import org.skywalking.apm.collector.stream.worker.impl.data.Data; +import org.skywalking.apm.collector.stream.worker.impl.dao.IPersistenceDAO; import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; import org.skywalking.apm.collector.stream.worker.selector.HashCodeSelector; import org.skywalking.apm.collector.stream.worker.selector.WorkerSelector; @@ -28,9 +26,12 @@ public class NodeRefPersistenceWorker extends PersistenceWorker { super.preStart(); } - @Override protected List prepareBatch(Map dataMap) { - INodeReferenceDAO dao = (INodeReferenceDAO)DAOContainer.INSTANCE.get(INodeReferenceDAO.class.getName()); - return dao.prepareBatch(dataMap); + @Override protected boolean needMergeDBData() { + return true; + } + + @Override protected IPersistenceDAO persistenceDAO() { + return (IPersistenceDAO)DAOContainer.INSTANCE.get(INodeReferenceDAO.class.getName()); } public static class Factory extends AbstractLocalAsyncWorkerProvider { diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/NodeRefSpanListener.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/NodeRefSpanListener.java index 0e5da04301c77f1595f4938010448d73d0c07158..ba89e24be6bc05b09245f9aa4064aceac1704a88 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/NodeRefSpanListener.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/NodeRefSpanListener.java @@ -3,11 +3,13 @@ package org.skywalking.apm.collector.agentstream.worker.noderef.reference; import java.util.ArrayList; import java.util.List; import org.skywalking.apm.collector.agentstream.worker.Const; +import org.skywalking.apm.collector.agentstream.worker.cache.InstanceCache; import org.skywalking.apm.collector.agentstream.worker.noderef.reference.define.NodeRefDataDefine; import org.skywalking.apm.collector.agentstream.worker.segment.EntrySpanListener; import org.skywalking.apm.collector.agentstream.worker.segment.ExitSpanListener; import org.skywalking.apm.collector.agentstream.worker.segment.FirstSpanListener; import org.skywalking.apm.collector.agentstream.worker.segment.RefsListener; +import org.skywalking.apm.collector.agentstream.worker.util.ExchangeMarkUtils; import org.skywalking.apm.collector.agentstream.worker.util.TimeBucketUtils; import org.skywalking.apm.collector.core.framework.CollectorContextHelper; import org.skywalking.apm.collector.stream.StreamModuleContext; @@ -26,28 +28,27 @@ public class NodeRefSpanListener implements EntrySpanListener, ExitSpanListener, private final Logger logger = LoggerFactory.getLogger(NodeRefSpanListener.class); - private List nodeExitReferences = new ArrayList<>(); + private List nodeReferences = new ArrayList<>(); private List nodeEntryReferences = new ArrayList<>(); private long timeBucket; private boolean hasReference = false; @Override public void parseExit(SpanObject spanObject, int applicationId, int applicationInstanceId, String segmentId) { - String front = String.valueOf(applicationId); - String behind = String.valueOf(spanObject.getPeerId()); - if (spanObject.getPeerId() == 0) { - behind = spanObject.getPeer(); + String front = ExchangeMarkUtils.INSTANCE.buildMarkedID(applicationId); + String behind = spanObject.getPeer(); + if (spanObject.getPeerId() != 0) { + behind = ExchangeMarkUtils.INSTANCE.buildMarkedID(spanObject.getPeerId()); } String agg = front + Const.ID_SPLIT + behind; - nodeExitReferences.add(agg); + nodeReferences.add(agg); } @Override public void parseEntry(SpanObject spanObject, int applicationId, int applicationInstanceId, String segmentId) { - String behind = String.valueOf(applicationId); - String front = Const.USER_CODE; - + String behind = ExchangeMarkUtils.INSTANCE.buildMarkedID(applicationId); + String front = ExchangeMarkUtils.INSTANCE.buildMarkedID(Const.USER_ID); String agg = front + Const.ID_SPLIT + behind; nodeEntryReferences.add(agg); } @@ -59,6 +60,14 @@ public class NodeRefSpanListener implements EntrySpanListener, ExitSpanListener, @Override public void parseRef(TraceSegmentReference reference, int applicationId, int applicationInstanceId, String segmentId) { + int parentApplicationId = InstanceCache.get(reference.getParentApplicationInstanceId()); + + String front = ExchangeMarkUtils.INSTANCE.buildMarkedID(parentApplicationId); + String behind = ExchangeMarkUtils.INSTANCE.buildMarkedID(applicationId); + + String agg = front + Const.ID_SPLIT + behind; + nodeReferences.add(agg); + hasReference = true; } @@ -66,10 +75,10 @@ public class NodeRefSpanListener implements EntrySpanListener, ExitSpanListener, logger.debug("node reference listener build"); StreamModuleContext context = (StreamModuleContext)CollectorContextHelper.INSTANCE.getContext(StreamModuleGroupDefine.GROUP_NAME); if (!hasReference) { - nodeExitReferences.addAll(nodeEntryReferences); + nodeReferences.addAll(nodeEntryReferences); } - for (String agg : nodeExitReferences) { + for (String agg : nodeReferences) { NodeRefDataDefine.NodeReference nodeReference = new NodeRefDataDefine.NodeReference(); nodeReference.setId(timeBucket + Const.ID_SPLIT + agg); nodeReference.setAgg(agg); diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/dao/INodeReferenceDAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/dao/INodeReferenceDAO.java index 122829dfb8fc1201f2e75cd482b62c4743bac489..7f2406d1e9bd3e0933940eeead250b458e18cde0 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/dao/INodeReferenceDAO.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/dao/INodeReferenceDAO.java @@ -1,12 +1,7 @@ package org.skywalking.apm.collector.agentstream.worker.noderef.reference.dao; -import java.util.List; -import java.util.Map; -import org.skywalking.apm.collector.stream.worker.impl.data.Data; - /** * @author pengys5 */ public interface INodeReferenceDAO { - List prepareBatch(Map dataMap); } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/dao/NodeReferenceEsDAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/dao/NodeReferenceEsDAO.java index d1334bec6a8b7854a8dafd75ab025daa53a3b6c2..84d572a703ba76e318b840e6106839aed112be23 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/dao/NodeReferenceEsDAO.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/dao/NodeReferenceEsDAO.java @@ -1,29 +1,47 @@ package org.skywalking.apm.collector.agentstream.worker.noderef.reference.dao; -import java.util.ArrayList; import java.util.HashMap; -import java.util.List; import java.util.Map; +import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.action.update.UpdateRequestBuilder; import org.skywalking.apm.collector.agentstream.worker.noderef.reference.define.NodeRefTable; import org.skywalking.apm.collector.storage.elasticsearch.dao.EsDAO; +import org.skywalking.apm.collector.stream.worker.impl.dao.IPersistenceDAO; import org.skywalking.apm.collector.stream.worker.impl.data.Data; +import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; /** * @author pengys5 */ -public class NodeReferenceEsDAO extends EsDAO implements INodeReferenceDAO { +public class NodeReferenceEsDAO extends EsDAO implements INodeReferenceDAO, IPersistenceDAO { - @Override public List prepareBatch(Map dataMap) { - List indexRequestBuilders = new ArrayList<>(); - dataMap.forEach((id, data) -> { - Map source = new HashMap(); - source.put(NodeRefTable.COLUMN_AGG, data.getDataString(1)); - source.put(NodeRefTable.COLUMN_TIME_BUCKET, data.getDataLong(0)); + @Override public Data get(String id, DataDefine dataDefine) { + GetResponse getResponse = getClient().prepareGet(NodeRefTable.TABLE, id).get(); + if (getResponse.isExists()) { + Data data = dataDefine.build(id); + Map source = getResponse.getSource(); + data.setDataString(1, (String)source.get(NodeRefTable.COLUMN_AGG)); + data.setDataLong(0, (Long)source.get(NodeRefTable.COLUMN_TIME_BUCKET)); + return data; + } else { + return null; + } + } + + @Override public IndexRequestBuilder prepareBatchInsert(Data data) { + Map source = new HashMap<>(); + source.put(NodeRefTable.COLUMN_AGG, data.getDataString(1)); + source.put(NodeRefTable.COLUMN_TIME_BUCKET, data.getDataLong(0)); + + return getClient().prepareIndex(NodeRefTable.TABLE, data.getDataString(0)).setSource(source); + } + + @Override public UpdateRequestBuilder prepareBatchUpdate(Data data) { + Map source = new HashMap<>(); + source.put(NodeRefTable.COLUMN_AGG, data.getDataString(1)); + source.put(NodeRefTable.COLUMN_TIME_BUCKET, data.getDataLong(0)); - IndexRequestBuilder builder = getClient().prepareIndex(NodeRefTable.TABLE, id).setSource(source); - indexRequestBuilders.add(builder); - }); - return indexRequestBuilders; + return getClient().prepareUpdate(NodeRefTable.TABLE, data.getDataString(0)).setDoc(source); } } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/dao/NodeReferenceH2DAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/dao/NodeReferenceH2DAO.java index bc2d7a0a1b47a96e5f756e7f3d6fa878a2fb3783..7820be366ea07a00b0819b3dd44a87bab4f9552f 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/dao/NodeReferenceH2DAO.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/dao/NodeReferenceH2DAO.java @@ -1,15 +1,24 @@ package org.skywalking.apm.collector.agentstream.worker.noderef.reference.dao; -import java.util.List; -import java.util.Map; import org.skywalking.apm.collector.storage.h2.dao.H2DAO; +import org.skywalking.apm.collector.stream.worker.impl.dao.IPersistenceDAO; +import org.skywalking.apm.collector.stream.worker.impl.data.Data; +import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; /** * @author pengys5 */ -public class NodeReferenceH2DAO extends H2DAO implements INodeReferenceDAO { +public class NodeReferenceH2DAO extends H2DAO implements INodeReferenceDAO, IPersistenceDAO { - @Override public List prepareBatch(Map map) { + @Override public Data get(String id, DataDefine dataDefine) { + return null; + } + + @Override public String prepareBatchInsert(Data data) { + return null; + } + + @Override public String prepareBatchUpdate(Data data) { return null; } } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/define/NodeRefDataDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/define/NodeRefDataDefine.java index 10c2c969fa47fab21eda3da550706cd0890aeb76..7ec0fb367a9e2face993b4ab20b03f43f620dacb 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/define/NodeRefDataDefine.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/reference/define/NodeRefDataDefine.java @@ -14,12 +14,6 @@ import org.skywalking.apm.collector.stream.worker.impl.data.operate.NonOperation */ public class NodeRefDataDefine extends DataDefine { - public static final int DEFINE_ID = 201; - - @Override public int defineId() { - return DEFINE_ID; - } - @Override protected int initialCapacity() { return 3; } @@ -51,7 +45,7 @@ public class NodeRefDataDefine extends DataDefine { private String agg; private long timeBucket; - public NodeReference(String id, String agg, long timeBucket) { + NodeReference(String id, String agg, long timeBucket) { this.id = id; this.agg = agg; this.timeBucket = timeBucket; @@ -70,7 +64,10 @@ public class NodeRefDataDefine extends DataDefine { } @Override public Object toSelf(Data data) { - return null; + this.id = data.getDataString(0); + this.agg = data.getDataString(1); + this.timeBucket = data.getDataLong(0); + return this; } public String getId() { diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/NodeRefSumPersistenceWorker.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/NodeRefSumPersistenceWorker.java index 96dead7e0c05293a0209f51036f3b1ce33046c33..6fcb36eb77f3dba47ace5275e84690203822ed38 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/NodeRefSumPersistenceWorker.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/NodeRefSumPersistenceWorker.java @@ -1,7 +1,5 @@ package org.skywalking.apm.collector.agentstream.worker.noderef.summary; -import java.util.List; -import java.util.Map; import org.skywalking.apm.collector.agentstream.worker.noderef.summary.dao.INodeRefSumDAO; import org.skywalking.apm.collector.agentstream.worker.noderef.summary.define.NodeRefSumDataDefine; import org.skywalking.apm.collector.storage.dao.DAOContainer; @@ -10,7 +8,7 @@ import org.skywalking.apm.collector.stream.worker.ClusterWorkerContext; import org.skywalking.apm.collector.stream.worker.ProviderNotFoundException; import org.skywalking.apm.collector.stream.worker.Role; import org.skywalking.apm.collector.stream.worker.impl.PersistenceWorker; -import org.skywalking.apm.collector.stream.worker.impl.data.Data; +import org.skywalking.apm.collector.stream.worker.impl.dao.IPersistenceDAO; import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; import org.skywalking.apm.collector.stream.worker.selector.HashCodeSelector; import org.skywalking.apm.collector.stream.worker.selector.WorkerSelector; @@ -28,9 +26,12 @@ public class NodeRefSumPersistenceWorker extends PersistenceWorker { super.preStart(); } - @Override protected List prepareBatch(Map dataMap) { - INodeRefSumDAO dao = (INodeRefSumDAO)DAOContainer.INSTANCE.get(INodeRefSumDAO.class.getName()); - return dao.prepareBatch(dataMap); + @Override protected boolean needMergeDBData() { + return true; + } + + @Override protected IPersistenceDAO persistenceDAO() { + return (IPersistenceDAO)DAOContainer.INSTANCE.get(INodeRefSumDAO.class.getName()); } public static class Factory extends AbstractLocalAsyncWorkerProvider { diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/NodeRefSumSpanListener.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/NodeRefSumSpanListener.java index 69bffe8ea08f9b6edfcd3f48925a09406d4de714..aedfdeceb857a3706ab316e4fa48b278d6718901 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/NodeRefSumSpanListener.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/NodeRefSumSpanListener.java @@ -3,11 +3,13 @@ package org.skywalking.apm.collector.agentstream.worker.noderef.summary; import java.util.ArrayList; import java.util.List; import org.skywalking.apm.collector.agentstream.worker.Const; +import org.skywalking.apm.collector.agentstream.worker.cache.InstanceCache; import org.skywalking.apm.collector.agentstream.worker.noderef.summary.define.NodeRefSumDataDefine; import org.skywalking.apm.collector.agentstream.worker.segment.EntrySpanListener; import org.skywalking.apm.collector.agentstream.worker.segment.ExitSpanListener; import org.skywalking.apm.collector.agentstream.worker.segment.FirstSpanListener; import org.skywalking.apm.collector.agentstream.worker.segment.RefsListener; +import org.skywalking.apm.collector.agentstream.worker.util.ExchangeMarkUtils; import org.skywalking.apm.collector.agentstream.worker.util.TimeBucketUtils; import org.skywalking.apm.collector.core.framework.CollectorContextHelper; import org.skywalking.apm.collector.stream.StreamModuleContext; @@ -28,15 +30,19 @@ public class NodeRefSumSpanListener implements EntrySpanListener, ExitSpanListen private List nodeExitReferences = new ArrayList<>(); private List nodeEntryReferences = new ArrayList<>(); + private List nodeReferences = new ArrayList<>(); private long timeBucket; private boolean hasReference = false; + private long startTime; + private long endTime; + private boolean isError; @Override public void parseExit(SpanObject spanObject, int applicationId, int applicationInstanceId, String segmentId) { - String front = String.valueOf(applicationId); - String behind = String.valueOf(spanObject.getPeerId()); - if (spanObject.getPeerId() == 0) { - behind = spanObject.getPeer(); + String front = ExchangeMarkUtils.INSTANCE.buildMarkedID(applicationId); + String behind = spanObject.getPeer(); + if (spanObject.getPeerId() != 0) { + behind = ExchangeMarkUtils.INSTANCE.buildMarkedID(spanObject.getPeerId()); } String agg = front + Const.ID_SPLIT + behind; @@ -45,9 +51,8 @@ public class NodeRefSumSpanListener implements EntrySpanListener, ExitSpanListen @Override public void parseEntry(SpanObject spanObject, int applicationId, int applicationInstanceId, String segmentId) { - String behind = String.valueOf(applicationId); - String front = Const.USER_CODE; - + String behind = ExchangeMarkUtils.INSTANCE.buildMarkedID(applicationId); + String front = ExchangeMarkUtils.INSTANCE.buildMarkedID(Const.USER_ID); String agg = front + Const.ID_SPLIT + behind; nodeEntryReferences.add(buildNodeRefSum(spanObject.getStartTime(), spanObject.getEndTime(), agg, spanObject.getIsError())); } @@ -76,11 +81,22 @@ public class NodeRefSumSpanListener implements EntrySpanListener, ExitSpanListen @Override public void parseFirst(SpanObject spanObject, int applicationId, int applicationInstanceId, String segmentId) { timeBucket = TimeBucketUtils.INSTANCE.getMinuteTimeBucket(spanObject.getStartTime()); + startTime = spanObject.getStartTime(); + endTime = spanObject.getEndTime(); + isError = spanObject.getIsError(); } @Override public void parseRef(TraceSegmentReference reference, int applicationId, int applicationInstanceId, String segmentId) { + int parentApplicationId = InstanceCache.get(reference.getParentApplicationInstanceId()); + + String front = ExchangeMarkUtils.INSTANCE.buildMarkedID(parentApplicationId); + String behind = ExchangeMarkUtils.INSTANCE.buildMarkedID(applicationId); + + String agg = front + Const.ID_SPLIT + behind; + hasReference = true; + nodeReferences.add(agg); } @Override public void build() { @@ -88,6 +104,10 @@ public class NodeRefSumSpanListener implements EntrySpanListener, ExitSpanListen StreamModuleContext context = (StreamModuleContext)CollectorContextHelper.INSTANCE.getContext(StreamModuleGroupDefine.GROUP_NAME); if (!hasReference) { nodeExitReferences.addAll(nodeEntryReferences); + } else { + nodeReferences.forEach(agg -> { + nodeExitReferences.add(buildNodeRefSum(startTime, endTime, agg, isError)); + }); } for (NodeRefSumDataDefine.NodeReferenceSum referenceSum : nodeExitReferences) { diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/dao/INodeRefSumDAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/dao/INodeRefSumDAO.java index ef4f69737fc8a51395ee68ddeaa3d210261fe3e9..6be97995717c37172fcaf72ed38d6ea41e2840aa 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/dao/INodeRefSumDAO.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/dao/INodeRefSumDAO.java @@ -1,12 +1,7 @@ package org.skywalking.apm.collector.agentstream.worker.noderef.summary.dao; -import java.util.List; -import java.util.Map; -import org.skywalking.apm.collector.stream.worker.impl.data.Data; - /** * @author pengys5 */ public interface INodeRefSumDAO { - List prepareBatch(Map dataMap); } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/dao/NodeRefSumEsDAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/dao/NodeRefSumEsDAO.java index a200997847d72cc3346de53c35c22bc0da43a687..1810412d369379d42f9072ad1bda5755284b1a1c 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/dao/NodeRefSumEsDAO.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/dao/NodeRefSumEsDAO.java @@ -1,35 +1,66 @@ package org.skywalking.apm.collector.agentstream.worker.noderef.summary.dao; -import java.util.ArrayList; import java.util.HashMap; -import java.util.List; import java.util.Map; +import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.action.update.UpdateRequestBuilder; +import org.skywalking.apm.collector.agentstream.worker.noderef.reference.define.NodeRefTable; import org.skywalking.apm.collector.agentstream.worker.noderef.summary.define.NodeRefSumTable; import org.skywalking.apm.collector.storage.elasticsearch.dao.EsDAO; +import org.skywalking.apm.collector.stream.worker.impl.dao.IPersistenceDAO; import org.skywalking.apm.collector.stream.worker.impl.data.Data; +import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; /** * @author pengys5 */ -public class NodeRefSumEsDAO extends EsDAO implements INodeRefSumDAO { +public class NodeRefSumEsDAO extends EsDAO implements INodeRefSumDAO, IPersistenceDAO { - @Override public List prepareBatch(Map dataMap) { - List indexRequestBuilders = new ArrayList<>(); - dataMap.forEach((id, data) -> { - Map source = new HashMap(); - source.put(NodeRefSumTable.COLUMN_ONE_SECOND_LESS, data.getDataLong(0)); - source.put(NodeRefSumTable.COLUMN_THREE_SECOND_LESS, data.getDataLong(1)); - source.put(NodeRefSumTable.COLUMN_FIVE_SECOND_LESS, data.getDataLong(2)); - source.put(NodeRefSumTable.COLUMN_FIVE_SECOND_GREATER, data.getDataLong(3)); - source.put(NodeRefSumTable.COLUMN_ERROR, data.getDataLong(4)); - source.put(NodeRefSumTable.COLUMN_SUMMARY, data.getDataLong(5)); - source.put(NodeRefSumTable.COLUMN_AGG, data.getDataString(1)); - source.put(NodeRefSumTable.COLUMN_TIME_BUCKET, data.getDataLong(6)); + @Override public Data get(String id, DataDefine dataDefine) { + GetResponse getResponse = getClient().prepareGet(NodeRefSumTable.TABLE, id).get(); + if (getResponse.isExists()) { + Data data = dataDefine.build(id); + Map source = getResponse.getSource(); + data.setDataLong(0, ((Number)source.get(NodeRefSumTable.COLUMN_ONE_SECOND_LESS)).longValue()); + data.setDataLong(1, ((Number)source.get(NodeRefSumTable.COLUMN_THREE_SECOND_LESS)).longValue()); + data.setDataLong(2, ((Number)source.get(NodeRefSumTable.COLUMN_FIVE_SECOND_LESS)).longValue()); + data.setDataLong(3, ((Number)source.get(NodeRefSumTable.COLUMN_FIVE_SECOND_GREATER)).longValue()); + data.setDataLong(4, ((Number)source.get(NodeRefSumTable.COLUMN_ERROR)).longValue()); + data.setDataLong(5, ((Number)source.get(NodeRefSumTable.COLUMN_SUMMARY)).longValue()); + data.setDataLong(6, ((Number)source.get(NodeRefSumTable.COLUMN_TIME_BUCKET)).longValue()); + data.setDataString(1, (String)source.get(NodeRefSumTable.COLUMN_AGG)); + return data; + } else { + return null; + } + } + + @Override public IndexRequestBuilder prepareBatchInsert(Data data) { + Map source = new HashMap<>(); + source.put(NodeRefSumTable.COLUMN_ONE_SECOND_LESS, data.getDataLong(0)); + source.put(NodeRefSumTable.COLUMN_THREE_SECOND_LESS, data.getDataLong(1)); + source.put(NodeRefSumTable.COLUMN_FIVE_SECOND_LESS, data.getDataLong(2)); + source.put(NodeRefSumTable.COLUMN_FIVE_SECOND_GREATER, data.getDataLong(3)); + source.put(NodeRefSumTable.COLUMN_ERROR, data.getDataLong(4)); + source.put(NodeRefSumTable.COLUMN_SUMMARY, data.getDataLong(5)); + source.put(NodeRefSumTable.COLUMN_AGG, data.getDataString(1)); + source.put(NodeRefSumTable.COLUMN_TIME_BUCKET, data.getDataLong(6)); + + return getClient().prepareIndex(NodeRefSumTable.TABLE, data.getDataString(0)).setSource(source); + } + + @Override public UpdateRequestBuilder prepareBatchUpdate(Data data) { + Map source = new HashMap<>(); + source.put(NodeRefSumTable.COLUMN_ONE_SECOND_LESS, data.getDataLong(0)); + source.put(NodeRefSumTable.COLUMN_THREE_SECOND_LESS, data.getDataLong(1)); + source.put(NodeRefSumTable.COLUMN_FIVE_SECOND_LESS, data.getDataLong(2)); + source.put(NodeRefSumTable.COLUMN_FIVE_SECOND_GREATER, data.getDataLong(3)); + source.put(NodeRefSumTable.COLUMN_ERROR, data.getDataLong(4)); + source.put(NodeRefSumTable.COLUMN_SUMMARY, data.getDataLong(5)); + source.put(NodeRefSumTable.COLUMN_AGG, data.getDataString(1)); + source.put(NodeRefSumTable.COLUMN_TIME_BUCKET, data.getDataLong(6)); - IndexRequestBuilder builder = getClient().prepareIndex(NodeRefSumTable.TABLE, id).setSource(source); - indexRequestBuilders.add(builder); - }); - return indexRequestBuilders; + return getClient().prepareUpdate(NodeRefTable.TABLE, data.getDataString(0)).setDoc(source); } } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/dao/NodeRefSumH2DAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/dao/NodeRefSumH2DAO.java index 2a3b835018fa63d8a58589f2acf7ec4edaa7be07..7483c8a82619339af1ded84c74acda032b0c3203 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/dao/NodeRefSumH2DAO.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/dao/NodeRefSumH2DAO.java @@ -1,15 +1,9 @@ package org.skywalking.apm.collector.agentstream.worker.noderef.summary.dao; -import java.util.List; -import java.util.Map; import org.skywalking.apm.collector.storage.h2.dao.H2DAO; /** * @author pengys5 */ public class NodeRefSumH2DAO extends H2DAO implements INodeRefSumDAO { - - @Override public List prepareBatch(Map map) { - return null; - } } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/define/NodeRefSumDataDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/define/NodeRefSumDataDefine.java index 39146ea630b536e58fee6dc0077be9b250ff2bcf..192c6630e20a9afc7995525c575aed23c4d1d32d 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/define/NodeRefSumDataDefine.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/noderef/summary/define/NodeRefSumDataDefine.java @@ -14,12 +14,6 @@ import org.skywalking.apm.collector.stream.worker.impl.data.operate.NonOperation */ public class NodeRefSumDataDefine extends DataDefine { - public static final int DEFINE_ID = 202; - - @Override public int defineId() { - return DEFINE_ID; - } - @Override protected int initialCapacity() { return 9; } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationDataDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationDataDefine.java index cb964a3a403c2a39c7438b08def04c133d5f7eb3..3aebfe46a823e032b00545fe2f6d4fbf51f63f5e 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationDataDefine.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationDataDefine.java @@ -12,18 +12,12 @@ import org.skywalking.apm.collector.stream.worker.impl.data.operate.NonOperation */ public class ApplicationDataDefine extends DataDefine { - public static final int DEFINE_ID = 101; - - @Override public int defineId() { - return DEFINE_ID; - } - @Override protected int initialCapacity() { return 3; } @Override protected void attributeDefine() { - addAttribute(0, new Attribute("id", AttributeType.STRING, new NonOperation())); + addAttribute(0, new Attribute(ApplicationTable.COLUMN_ID, AttributeType.STRING, new NonOperation())); addAttribute(1, new Attribute(ApplicationTable.COLUMN_APPLICATION_CODE, AttributeType.STRING, new CoverOperation())); addAttribute(2, new Attribute(ApplicationTable.COLUMN_APPLICATION_ID, AttributeType.INTEGER, new CoverOperation())); } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationRegisterSerialWorker.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationRegisterSerialWorker.java index 791b3768ebf5df01673ed1ca34df0ee30e06a179..ba2811a42f1c737f367e8ff85cf72910bce82e24 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationRegisterSerialWorker.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationRegisterSerialWorker.java @@ -1,5 +1,6 @@ package org.skywalking.apm.collector.agentstream.worker.register.application; +import org.skywalking.apm.collector.agentstream.worker.Const; import org.skywalking.apm.collector.agentstream.worker.register.IdAutoIncrement; import org.skywalking.apm.collector.agentstream.worker.register.application.dao.IApplicationDAO; import org.skywalking.apm.collector.storage.dao.DAOContainer; @@ -40,8 +41,11 @@ public class ApplicationRegisterSerialWorker extends AbstractLocalAsyncWorker { if (applicationId == 0) { int min = dao.getMinApplicationId(); if (min == 0) { - application.setApplicationId(1); - application.setId("1"); + ApplicationDataDefine.Application userApplication = new ApplicationDataDefine.Application(String.valueOf(Const.USER_ID), Const.USER_CODE, Const.USER_ID); + dao.save(userApplication); + + application.setApplicationId(-1); + application.setId("-1"); } else { int max = dao.getMaxApplicationId(); applicationId = IdAutoIncrement.INSTANCE.increment(min, max); diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationTable.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationTable.java index 797b8d80a4b14940a431a4cab06bde6232675975..6b067c2b11736c530624b12b7df95c846cf37faa 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationTable.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/application/ApplicationTable.java @@ -1,9 +1,11 @@ package org.skywalking.apm.collector.agentstream.worker.register.application; +import org.skywalking.apm.collector.agentstream.worker.CommonTable; + /** * @author pengys5 */ -public class ApplicationTable { +public class ApplicationTable extends CommonTable { public static final String TABLE = "application"; public static final String COLUMN_APPLICATION_CODE = "application_code"; public static final String COLUMN_APPLICATION_ID = "application_id"; diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceDataDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceDataDefine.java index e14afa7b2bc25df3607885c3a80524b173807d63..cdb98601373e355274883789a630e89fe1c8621f 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceDataDefine.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceDataDefine.java @@ -12,20 +12,14 @@ import org.skywalking.apm.collector.stream.worker.impl.data.operate.NonOperation */ public class InstanceDataDefine extends DataDefine { - public static final int DEFINE_ID = 102; - - @Override public int defineId() { - return DEFINE_ID; - } - @Override protected int initialCapacity() { return 6; } @Override protected void attributeDefine() { - addAttribute(0, new Attribute("id", AttributeType.STRING, new NonOperation())); + addAttribute(0, new Attribute(InstanceTable.COLUMN_ID, AttributeType.STRING, new NonOperation())); addAttribute(1, new Attribute(InstanceTable.COLUMN_APPLICATION_ID, AttributeType.INTEGER, new CoverOperation())); - addAttribute(2, new Attribute(InstanceTable.COLUMN_AGENTUUID, AttributeType.STRING, new CoverOperation())); + addAttribute(2, new Attribute(InstanceTable.COLUMN_AGENT_UUID, AttributeType.STRING, new CoverOperation())); addAttribute(3, new Attribute(InstanceTable.COLUMN_REGISTER_TIME, AttributeType.LONG, new CoverOperation())); addAttribute(4, new Attribute(InstanceTable.COLUMN_INSTANCE_ID, AttributeType.INTEGER, new CoverOperation())); addAttribute(5, new Attribute(InstanceTable.COLUMN_HEARTBEAT_TIME, AttributeType.LONG, new CoverOperation())); diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceEsTableDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceEsTableDefine.java index dffcfa9fb27e7c3af361eacff83e9e6a0c9af448..55417fbaf38c03e41fccc6cc9605d20f6de17a75 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceEsTableDefine.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceEsTableDefine.java @@ -26,7 +26,7 @@ public class InstanceEsTableDefine extends ElasticSearchTableDefine { @Override public void initialize() { addColumn(new ElasticSearchColumnDefine(InstanceTable.COLUMN_APPLICATION_ID, ElasticSearchColumnDefine.Type.Integer.name())); - addColumn(new ElasticSearchColumnDefine(InstanceTable.COLUMN_AGENTUUID, ElasticSearchColumnDefine.Type.Keyword.name())); + addColumn(new ElasticSearchColumnDefine(InstanceTable.COLUMN_AGENT_UUID, ElasticSearchColumnDefine.Type.Keyword.name())); addColumn(new ElasticSearchColumnDefine(InstanceTable.COLUMN_REGISTER_TIME, ElasticSearchColumnDefine.Type.Long.name())); addColumn(new ElasticSearchColumnDefine(InstanceTable.COLUMN_INSTANCE_ID, ElasticSearchColumnDefine.Type.Integer.name())); addColumn(new ElasticSearchColumnDefine(InstanceTable.COLUMN_HEARTBEAT_TIME, ElasticSearchColumnDefine.Type.Long.name())); diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceH2TableDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceH2TableDefine.java index 58ca3e626dc2f8bdce76e0c70d6fb34e4a4eaa3a..17afcea8ee3b12e94109420b71558246764bf793 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceH2TableDefine.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceH2TableDefine.java @@ -14,7 +14,7 @@ public class InstanceH2TableDefine extends H2TableDefine { @Override public void initialize() { addColumn(new H2ColumnDefine(InstanceTable.COLUMN_APPLICATION_ID, H2ColumnDefine.Type.Int.name())); - addColumn(new H2ColumnDefine(InstanceTable.COLUMN_AGENTUUID, H2ColumnDefine.Type.Varchar.name())); + addColumn(new H2ColumnDefine(InstanceTable.COLUMN_AGENT_UUID, H2ColumnDefine.Type.Varchar.name())); addColumn(new H2ColumnDefine(InstanceTable.COLUMN_REGISTER_TIME, H2ColumnDefine.Type.Bigint.name())); addColumn(new H2ColumnDefine(InstanceTable.COLUMN_INSTANCE_ID, H2ColumnDefine.Type.Int.name())); addColumn(new H2ColumnDefine(InstanceTable.COLUMN_HEARTBEAT_TIME, H2ColumnDefine.Type.Bigint.name())); diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceTable.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceTable.java index 5a26adb6eb6c8d7b33c7f7bb4e7480d52efd9fdf..fa59f82206a53f7b1a7ef484d625dab601e39259 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceTable.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/InstanceTable.java @@ -1,12 +1,14 @@ package org.skywalking.apm.collector.agentstream.worker.register.instance; +import org.skywalking.apm.collector.agentstream.worker.CommonTable; + /** * @author pengys5 */ -public class InstanceTable { +public class InstanceTable extends CommonTable { public static final String TABLE = "instance"; public static final String COLUMN_APPLICATION_ID = "application_id"; - public static final String COLUMN_AGENTUUID = "agent_uuid"; + public static final String COLUMN_AGENT_UUID = "agent_uuid"; public static final String COLUMN_REGISTER_TIME = "register_time"; public static final String COLUMN_INSTANCE_ID = "instance_id"; public static final String COLUMN_HEARTBEAT_TIME = "heartbeatTime"; diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/dao/IInstanceDAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/dao/IInstanceDAO.java index f0c31114669e9b9748d11f537e1e2a357b5f7dc8..79d22d27e67c292bc60d8ab27f32b5eba0ddc582 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/dao/IInstanceDAO.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/dao/IInstanceDAO.java @@ -15,4 +15,6 @@ public interface IInstanceDAO { void save(InstanceDataDefine.Instance instance); void updateHeartbeatTime(int instanceId, long heartbeatTime); + + int getApplicationId(int applicationInstanceId); } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/dao/InstanceEsDAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/dao/InstanceEsDAO.java index 237f50009f529e351f2701f542181b8ddfd55f22..bf872bca595bd0046b6bfafb09ee9b013a3915c8 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/dao/InstanceEsDAO.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/dao/InstanceEsDAO.java @@ -2,6 +2,7 @@ package org.skywalking.apm.collector.agentstream.worker.register.instance.dao; import java.util.HashMap; import java.util.Map; +import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; @@ -33,15 +34,14 @@ public class InstanceEsDAO extends EsDAO implements IInstanceDAO { searchRequestBuilder.setSearchType(SearchType.QUERY_THEN_FETCH); BoolQueryBuilder builder = QueryBuilders.boolQuery(); builder.must().add(QueryBuilders.termQuery(InstanceTable.COLUMN_APPLICATION_ID, applicationId)); - builder.must().add(QueryBuilders.termQuery(InstanceTable.COLUMN_AGENTUUID, agentUUID)); + builder.must().add(QueryBuilders.termQuery(InstanceTable.COLUMN_AGENT_UUID, agentUUID)); searchRequestBuilder.setQuery(builder); searchRequestBuilder.setSize(1); SearchResponse searchResponse = searchRequestBuilder.execute().actionGet(); if (searchResponse.getHits().totalHits > 0) { SearchHit searchHit = searchResponse.getHits().iterator().next(); - int instanceId = (int)searchHit.getSource().get(InstanceTable.COLUMN_INSTANCE_ID); - return instanceId; + return (int)searchHit.getSource().get(InstanceTable.COLUMN_INSTANCE_ID); } return 0; } @@ -57,10 +57,10 @@ public class InstanceEsDAO extends EsDAO implements IInstanceDAO { @Override public void save(InstanceDataDefine.Instance instance) { logger.debug("save instance register info, application id: {}, agentUUID: {}", instance.getApplicationId(), instance.getAgentUUID()); ElasticSearchClient client = getClient(); - Map source = new HashMap(); + Map source = new HashMap<>(); source.put(InstanceTable.COLUMN_INSTANCE_ID, instance.getInstanceId()); source.put(InstanceTable.COLUMN_APPLICATION_ID, instance.getApplicationId()); - source.put(InstanceTable.COLUMN_AGENTUUID, instance.getAgentUUID()); + source.put(InstanceTable.COLUMN_AGENT_UUID, instance.getAgentUUID()); source.put(InstanceTable.COLUMN_REGISTER_TIME, instance.getRegisterTime()); IndexResponse response = client.prepareIndex(InstanceTable.TABLE, instance.getId()).setSource(source).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); @@ -75,10 +75,19 @@ public class InstanceEsDAO extends EsDAO implements IInstanceDAO { updateRequest.id(String.valueOf(instanceId)); updateRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); - Map source = new HashMap(); + Map source = new HashMap<>(); source.put(InstanceTable.COLUMN_HEARTBEAT_TIME, heartbeatTime); updateRequest.doc(source); client.update(updateRequest); } + + @Override public int getApplicationId(int applicationInstanceId) { + GetResponse response = getClient().prepareGet(InstanceTable.TABLE, String.valueOf(applicationInstanceId)).get(); + if (response.isExists()) { + return (int)response.getSource().get(InstanceTable.COLUMN_APPLICATION_ID); + } else { + return 0; + } + } } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/dao/InstanceH2DAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/dao/InstanceH2DAO.java index 64070090ef0c719c3bfb8ff526a85509e5a8282d..d46aaaaecabe9c39d18c7914b2e051814b794810 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/dao/InstanceH2DAO.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/instance/dao/InstanceH2DAO.java @@ -26,4 +26,8 @@ public class InstanceH2DAO extends H2DAO implements IInstanceDAO { @Override public void updateHeartbeatTime(int instanceId, long heartbeatTime) { } + + @Override public int getApplicationId(int applicationInstanceId) { + return 0; + } } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/servicename/ServiceNameDataDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/servicename/ServiceNameDataDefine.java index d30a15240b2d57a93f9bfa0e7c3344dfdb1b3223..42463c27e1b7def922c855b1df847f44065e81b9 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/servicename/ServiceNameDataDefine.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/servicename/ServiceNameDataDefine.java @@ -12,18 +12,12 @@ import org.skywalking.apm.collector.stream.worker.impl.data.operate.NonOperation */ public class ServiceNameDataDefine extends DataDefine { - public static final int DEFINE_ID = 103; - - @Override public int defineId() { - return DEFINE_ID; - } - @Override protected int initialCapacity() { return 4; } @Override protected void attributeDefine() { - addAttribute(0, new Attribute("id", AttributeType.STRING, new NonOperation())); + addAttribute(0, new Attribute(ServiceNameTable.COLUMN_ID, AttributeType.STRING, new NonOperation())); addAttribute(1, new Attribute(ServiceNameTable.COLUMN_SERVICE_NAME, AttributeType.STRING, new CoverOperation())); addAttribute(2, new Attribute(ServiceNameTable.COLUMN_APPLICATION_ID, AttributeType.INTEGER, new CoverOperation())); addAttribute(3, new Attribute(ServiceNameTable.COLUMN_SERVICE_ID, AttributeType.INTEGER, new CoverOperation())); diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/servicename/ServiceNameTable.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/servicename/ServiceNameTable.java index f15fd6075c8f49432ce5433ea8aebcd65dd03a7c..f6c81e49166994e6e468e4e29a9679544c100900 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/servicename/ServiceNameTable.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/register/servicename/ServiceNameTable.java @@ -1,9 +1,11 @@ package org.skywalking.apm.collector.agentstream.worker.register.servicename; +import org.skywalking.apm.collector.agentstream.worker.CommonTable; + /** * @author pengys5 */ -public class ServiceNameTable { +public class ServiceNameTable extends CommonTable { public static final String TABLE = "service_name"; public static final String COLUMN_SERVICE_NAME = "service_name"; public static final String COLUMN_APPLICATION_ID = "application_id"; diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/SegmentParse.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/SegmentParse.java index 3d3738cd20a74685cd2964427204ddfe7bd26cff..4485db63936867d66289314b002e5d4328d07b5f 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/SegmentParse.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/SegmentParse.java @@ -10,6 +10,8 @@ import org.skywalking.apm.collector.agentstream.worker.noderef.summary.NodeRefSu import org.skywalking.apm.collector.agentstream.worker.segment.cost.SegmentCostSpanListener; import org.skywalking.apm.collector.agentstream.worker.segment.origin.SegmentPersistenceWorker; import org.skywalking.apm.collector.agentstream.worker.segment.origin.define.SegmentDataDefine; +import org.skywalking.apm.collector.agentstream.worker.service.entry.ServiceEntrySpanListener; +import org.skywalking.apm.collector.agentstream.worker.serviceref.reference.ServiceRefSpanListener; import org.skywalking.apm.collector.core.framework.CollectorContextHelper; import org.skywalking.apm.collector.core.util.CollectionUtils; import org.skywalking.apm.collector.stream.StreamModuleContext; @@ -38,10 +40,11 @@ public class SegmentParse { spanListeners.add(new NodeRefSpanListener()); spanListeners.add(new NodeComponentSpanListener()); spanListeners.add(new NodeMappingSpanListener()); - spanListeners.add(new NodeRefSpanListener()); spanListeners.add(new NodeRefSumSpanListener()); spanListeners.add(new SegmentCostSpanListener()); spanListeners.add(new GlobalTraceSpanListener()); + spanListeners.add(new ServiceEntrySpanListener()); + spanListeners.add(new ServiceRefSpanListener()); } public void parse(List traceIds, TraceSegmentObject segmentObject) { diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/SegmentCostPersistenceWorker.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/SegmentCostPersistenceWorker.java index d6b72f8a232125d5c3c59f95a6b356749125fe4b..036a291faab8f091d7944a38b773e1283365a406 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/SegmentCostPersistenceWorker.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/SegmentCostPersistenceWorker.java @@ -1,7 +1,5 @@ package org.skywalking.apm.collector.agentstream.worker.segment.cost; -import java.util.List; -import java.util.Map; import org.skywalking.apm.collector.agentstream.worker.segment.cost.dao.ISegmentCostDAO; import org.skywalking.apm.collector.agentstream.worker.segment.cost.define.SegmentCostDataDefine; import org.skywalking.apm.collector.storage.dao.DAOContainer; @@ -10,7 +8,7 @@ import org.skywalking.apm.collector.stream.worker.ClusterWorkerContext; import org.skywalking.apm.collector.stream.worker.ProviderNotFoundException; import org.skywalking.apm.collector.stream.worker.Role; import org.skywalking.apm.collector.stream.worker.impl.PersistenceWorker; -import org.skywalking.apm.collector.stream.worker.impl.data.Data; +import org.skywalking.apm.collector.stream.worker.impl.dao.IPersistenceDAO; import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; import org.skywalking.apm.collector.stream.worker.selector.RollingSelector; import org.skywalking.apm.collector.stream.worker.selector.WorkerSelector; @@ -28,9 +26,12 @@ public class SegmentCostPersistenceWorker extends PersistenceWorker { super.preStart(); } - @Override protected List prepareBatch(Map dataMap) { - ISegmentCostDAO dao = (ISegmentCostDAO)DAOContainer.INSTANCE.get(ISegmentCostDAO.class.getName()); - return dao.prepareBatch(dataMap); + @Override protected boolean needMergeDBData() { + return false; + } + + @Override protected IPersistenceDAO persistenceDAO() { + return (IPersistenceDAO)DAOContainer.INSTANCE.get(ISegmentCostDAO.class.getName()); } public static class Factory extends AbstractLocalAsyncWorkerProvider { diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/SegmentCostSpanListener.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/SegmentCostSpanListener.java index e5934792fb6360ec804d0f42c8c077dd0f139737..c49e86183da372fb0b7e62045f94ba2b31c15667 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/SegmentCostSpanListener.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/SegmentCostSpanListener.java @@ -7,6 +7,7 @@ import org.skywalking.apm.collector.agentstream.worker.segment.ExitSpanListener; import org.skywalking.apm.collector.agentstream.worker.segment.FirstSpanListener; import org.skywalking.apm.collector.agentstream.worker.segment.LocalSpanListener; import org.skywalking.apm.collector.agentstream.worker.segment.cost.define.SegmentCostDataDefine; +import org.skywalking.apm.collector.agentstream.worker.util.ExchangeMarkUtils; import org.skywalking.apm.collector.agentstream.worker.util.TimeBucketUtils; import org.skywalking.apm.collector.core.framework.CollectorContextHelper; import org.skywalking.apm.collector.stream.StreamModuleContext; @@ -40,10 +41,14 @@ public class SegmentCostSpanListener implements EntrySpanListener, ExitSpanListe segmentCost.setCost(spanObject.getEndTime() - spanObject.getStartTime()); segmentCost.setStartTime(spanObject.getStartTime()); segmentCost.setEndTime(spanObject.getEndTime()); - segmentCost.setOperationName(spanObject.getOperationName()); segmentCost.setId(segmentId); - segmentCosts.add(segmentCost); + if (spanObject.getOperationNameId() == 0) { + segmentCost.setServiceName(spanObject.getOperationName()); + } else { + segmentCost.setServiceName(ExchangeMarkUtils.INSTANCE.buildMarkedID(spanObject.getOperationNameId())); + } + segmentCosts.add(segmentCost); isError = isError || spanObject.getIsError(); } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/dao/ISegmentCostDAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/dao/ISegmentCostDAO.java index 03787ca00da990807dcc373c6ee846b3e72749b2..03a78dd9a4ae4b7bbca65102b9d6d99f1bd332f8 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/dao/ISegmentCostDAO.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/dao/ISegmentCostDAO.java @@ -1,12 +1,7 @@ package org.skywalking.apm.collector.agentstream.worker.segment.cost.dao; -import java.util.List; -import java.util.Map; -import org.skywalking.apm.collector.stream.worker.impl.data.Data; - /** * @author pengys5 */ public interface ISegmentCostDAO { - List prepareBatch(Map dataMap); } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/dao/SegmentCostEsDAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/dao/SegmentCostEsDAO.java index d1b9e19b28594195c562ca9204a1bcda7eb4f924..794b0d32a2b7afe15ed5ee419cd2a45ef8d5fe05 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/dao/SegmentCostEsDAO.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/dao/SegmentCostEsDAO.java @@ -1,39 +1,43 @@ package org.skywalking.apm.collector.agentstream.worker.segment.cost.dao; -import java.util.ArrayList; import java.util.HashMap; -import java.util.List; import java.util.Map; import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.action.update.UpdateRequestBuilder; import org.skywalking.apm.collector.agentstream.worker.segment.cost.define.SegmentCostTable; import org.skywalking.apm.collector.storage.elasticsearch.dao.EsDAO; +import org.skywalking.apm.collector.stream.worker.impl.dao.IPersistenceDAO; import org.skywalking.apm.collector.stream.worker.impl.data.Data; +import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * @author pengys5 */ -public class SegmentCostEsDAO extends EsDAO implements ISegmentCostDAO { +public class SegmentCostEsDAO extends EsDAO implements ISegmentCostDAO, IPersistenceDAO { private final Logger logger = LoggerFactory.getLogger(SegmentCostEsDAO.class); - @Override public List prepareBatch(Map dataMap) { - List indexRequestBuilders = new ArrayList<>(); - dataMap.forEach((id, data) -> { - logger.debug("segment cost prepareBatch, id: {}", id); - Map source = new HashMap(); - source.put(SegmentCostTable.COLUMN_SEGMENT_ID, data.getDataString(1)); - source.put(SegmentCostTable.COLUMN_OPERATION_NAME, data.getDataString(2)); - source.put(SegmentCostTable.COLUMN_COST, data.getDataLong(0)); - source.put(SegmentCostTable.COLUMN_START_TIME, data.getDataLong(1)); - source.put(SegmentCostTable.COLUMN_END_TIME, data.getDataLong(2)); - source.put(SegmentCostTable.COLUMN_IS_ERROR, data.getDataBoolean(0)); - source.put(SegmentCostTable.COLUMN_TIME_BUCKET, data.getDataLong(3)); - logger.debug("segment cost source: {}", source.toString()); - IndexRequestBuilder builder = getClient().prepareIndex(SegmentCostTable.TABLE, id).setSource(source); - indexRequestBuilders.add(builder); - }); - return indexRequestBuilders; + @Override public Data get(String id, DataDefine dataDefine) { + return null; + } + + @Override public UpdateRequestBuilder prepareBatchUpdate(Data data) { + return null; + } + + @Override public IndexRequestBuilder prepareBatchInsert(Data data) { + logger.debug("segment cost prepareBatchInsert, id: {}", data.getDataString(0)); + Map source = new HashMap<>(); + source.put(SegmentCostTable.COLUMN_SEGMENT_ID, data.getDataString(1)); + source.put(SegmentCostTable.COLUMN_SERVICE_NAME, data.getDataString(2)); + source.put(SegmentCostTable.COLUMN_COST, data.getDataLong(0)); + source.put(SegmentCostTable.COLUMN_START_TIME, data.getDataLong(1)); + source.put(SegmentCostTable.COLUMN_END_TIME, data.getDataLong(2)); + source.put(SegmentCostTable.COLUMN_IS_ERROR, data.getDataBoolean(0)); + source.put(SegmentCostTable.COLUMN_TIME_BUCKET, data.getDataLong(3)); + logger.debug("segment cost source: {}", source.toString()); + return getClient().prepareIndex(SegmentCostTable.TABLE, data.getDataString(0)).setSource(source); } } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/dao/SegmentCostH2DAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/dao/SegmentCostH2DAO.java index 90fd044b9e49134e00c56af647546845405229cd..63a6c78c54f1a047ee818b0d2ba5d0078bb91c58 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/dao/SegmentCostH2DAO.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/dao/SegmentCostH2DAO.java @@ -1,7 +1,5 @@ package org.skywalking.apm.collector.agentstream.worker.segment.cost.dao; -import java.util.List; -import java.util.Map; import org.skywalking.apm.collector.storage.h2.dao.H2DAO; /** @@ -9,7 +7,4 @@ import org.skywalking.apm.collector.storage.h2.dao.H2DAO; */ public class SegmentCostH2DAO extends H2DAO implements ISegmentCostDAO { - @Override public List prepareBatch(Map map) { - return null; - } } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/define/SegmentCostDataDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/define/SegmentCostDataDefine.java index c86de4fcd9f6988bc11d366f8e2cb045cdfd41d5..fd9f85c0364ea7333c6dcd583afc20b5c5d90b7e 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/define/SegmentCostDataDefine.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/define/SegmentCostDataDefine.java @@ -14,10 +14,6 @@ import org.skywalking.apm.collector.stream.worker.impl.data.operate.NonOperation */ public class SegmentCostDataDefine extends DataDefine { - @Override public int defineId() { - return 402; - } - @Override protected int initialCapacity() { return 8; } @@ -25,7 +21,7 @@ public class SegmentCostDataDefine extends DataDefine { @Override protected void attributeDefine() { addAttribute(0, new Attribute(SegmentCostTable.COLUMN_ID, AttributeType.STRING, new NonOperation())); addAttribute(1, new Attribute(SegmentCostTable.COLUMN_SEGMENT_ID, AttributeType.STRING, new CoverOperation())); - addAttribute(2, new Attribute(SegmentCostTable.COLUMN_OPERATION_NAME, AttributeType.STRING, new CoverOperation())); + addAttribute(2, new Attribute(SegmentCostTable.COLUMN_SERVICE_NAME, AttributeType.STRING, new CoverOperation())); addAttribute(3, new Attribute(SegmentCostTable.COLUMN_COST, AttributeType.LONG, new CoverOperation())); addAttribute(4, new Attribute(SegmentCostTable.COLUMN_START_TIME, AttributeType.LONG, new CoverOperation())); addAttribute(5, new Attribute(SegmentCostTable.COLUMN_END_TIME, AttributeType.LONG, new CoverOperation())); @@ -36,13 +32,13 @@ public class SegmentCostDataDefine extends DataDefine { @Override public Object deserialize(RemoteData remoteData) { String id = remoteData.getDataStrings(0); String segmentId = remoteData.getDataStrings(1); - String operationName = remoteData.getDataStrings(2); + String serviceName = remoteData.getDataStrings(2); Long cost = remoteData.getDataLongs(0); Long startTime = remoteData.getDataLongs(1); Long endTime = remoteData.getDataLongs(2); Boolean isError = remoteData.getDataBooleans(0); Long timeBucket = remoteData.getDataLongs(2); - return new SegmentCost(id, segmentId, operationName, cost, startTime, endTime, isError, timeBucket); + return new SegmentCost(id, segmentId, serviceName, cost, startTime, endTime, isError, timeBucket); } @Override public RemoteData serialize(Object object) { @@ -50,7 +46,7 @@ public class SegmentCostDataDefine extends DataDefine { RemoteData.Builder builder = RemoteData.newBuilder(); builder.addDataStrings(segmentCost.getId()); builder.addDataStrings(segmentCost.getSegmentId()); - builder.addDataStrings(segmentCost.getOperationName()); + builder.addDataStrings(segmentCost.getServiceName()); builder.addDataLongs(segmentCost.getCost()); builder.addDataLongs(segmentCost.getStartTime()); builder.addDataLongs(segmentCost.getEndTime()); @@ -62,18 +58,18 @@ public class SegmentCostDataDefine extends DataDefine { public static class SegmentCost implements Transform { private String id; private String segmentId; - private String operationName; + private String serviceName; private Long cost; private Long startTime; private Long endTime; private boolean isError; private long timeBucket; - SegmentCost(String id, String segmentId, String operationName, Long cost, + SegmentCost(String id, String segmentId, String serviceName, Long cost, Long startTime, Long endTime, boolean isError, long timeBucket) { this.id = id; this.segmentId = segmentId; - this.operationName = operationName; + this.serviceName = serviceName; this.cost = cost; this.startTime = startTime; this.endTime = endTime; @@ -89,7 +85,7 @@ public class SegmentCostDataDefine extends DataDefine { Data data = define.build(id); data.setDataString(0, this.id); data.setDataString(1, this.segmentId); - data.setDataString(2, this.operationName); + data.setDataString(2, this.serviceName); data.setDataLong(0, this.cost); data.setDataLong(1, this.startTime); data.setDataLong(2, this.endTime); @@ -118,12 +114,12 @@ public class SegmentCostDataDefine extends DataDefine { this.segmentId = segmentId; } - public String getOperationName() { - return operationName; + public String getServiceName() { + return serviceName; } - public void setOperationName(String operationName) { - this.operationName = operationName; + public void setServiceName(String serviceName) { + this.serviceName = serviceName; } public Long getCost() { diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/define/SegmentCostEsTableDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/define/SegmentCostEsTableDefine.java index 4f493da997ada28bef7b76e12a14133e68a74fec..e44e40829c337f03c47407478b09bed9d753c7d2 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/define/SegmentCostEsTableDefine.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/define/SegmentCostEsTableDefine.java @@ -26,7 +26,7 @@ public class SegmentCostEsTableDefine extends ElasticSearchTableDefine { @Override public void initialize() { addColumn(new ElasticSearchColumnDefine(SegmentCostTable.COLUMN_SEGMENT_ID, ElasticSearchColumnDefine.Type.Keyword.name())); - addColumn(new ElasticSearchColumnDefine(SegmentCostTable.COLUMN_OPERATION_NAME, ElasticSearchColumnDefine.Type.Keyword.name())); + addColumn(new ElasticSearchColumnDefine(SegmentCostTable.COLUMN_SERVICE_NAME, ElasticSearchColumnDefine.Type.Keyword.name())); addColumn(new ElasticSearchColumnDefine(SegmentCostTable.COLUMN_COST, ElasticSearchColumnDefine.Type.Long.name())); addColumn(new ElasticSearchColumnDefine(SegmentCostTable.COLUMN_START_TIME, ElasticSearchColumnDefine.Type.Long.name())); addColumn(new ElasticSearchColumnDefine(SegmentCostTable.COLUMN_END_TIME, ElasticSearchColumnDefine.Type.Long.name())); diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/define/SegmentCostH2TableDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/define/SegmentCostH2TableDefine.java index c4ca90226075f89b159b94c85e0badc18155d909..f1d896919a8c12b78a8639430f5fb120c4511c56 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/define/SegmentCostH2TableDefine.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/define/SegmentCostH2TableDefine.java @@ -15,7 +15,7 @@ public class SegmentCostH2TableDefine extends H2TableDefine { @Override public void initialize() { addColumn(new H2ColumnDefine(SegmentCostTable.COLUMN_ID, H2ColumnDefine.Type.Varchar.name())); addColumn(new H2ColumnDefine(SegmentCostTable.COLUMN_SEGMENT_ID, H2ColumnDefine.Type.Varchar.name())); - addColumn(new H2ColumnDefine(SegmentCostTable.COLUMN_OPERATION_NAME, H2ColumnDefine.Type.Varchar.name())); + addColumn(new H2ColumnDefine(SegmentCostTable.COLUMN_SERVICE_NAME, H2ColumnDefine.Type.Varchar.name())); addColumn(new H2ColumnDefine(SegmentCostTable.COLUMN_COST, H2ColumnDefine.Type.Bigint.name())); addColumn(new H2ColumnDefine(SegmentCostTable.COLUMN_START_TIME, H2ColumnDefine.Type.Bigint.name())); addColumn(new H2ColumnDefine(SegmentCostTable.COLUMN_END_TIME, H2ColumnDefine.Type.Bigint.name())); diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/define/SegmentCostTable.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/define/SegmentCostTable.java index 0bc0ee947dc497c5bbe1e80ae823ef52140f08ae..fb9c7f83dd7340c3da494f5209a5599eb22f434c 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/define/SegmentCostTable.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/cost/define/SegmentCostTable.java @@ -10,7 +10,7 @@ public class SegmentCostTable extends CommonTable { public static final String COLUMN_SEGMENT_ID = "segment_id"; public static final String COLUMN_START_TIME = "start_time"; public static final String COLUMN_END_TIME = "end_time"; - public static final String COLUMN_OPERATION_NAME = "operation_name"; + public static final String COLUMN_SERVICE_NAME = "service_name"; public static final String COLUMN_COST = "cost"; public static final String COLUMN_IS_ERROR = "is_error"; } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/origin/SegmentPersistenceWorker.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/origin/SegmentPersistenceWorker.java index a76fbd11db0e5875a36a1a57f520c33e1d8bc8fd..03bbf03517059f65aab107be9bcc3f755ba410f6 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/origin/SegmentPersistenceWorker.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/origin/SegmentPersistenceWorker.java @@ -1,7 +1,5 @@ package org.skywalking.apm.collector.agentstream.worker.segment.origin; -import java.util.List; -import java.util.Map; import org.skywalking.apm.collector.agentstream.worker.segment.origin.dao.ISegmentDAO; import org.skywalking.apm.collector.agentstream.worker.segment.origin.define.SegmentDataDefine; import org.skywalking.apm.collector.storage.dao.DAOContainer; @@ -10,7 +8,7 @@ import org.skywalking.apm.collector.stream.worker.ClusterWorkerContext; import org.skywalking.apm.collector.stream.worker.ProviderNotFoundException; import org.skywalking.apm.collector.stream.worker.Role; import org.skywalking.apm.collector.stream.worker.impl.PersistenceWorker; -import org.skywalking.apm.collector.stream.worker.impl.data.Data; +import org.skywalking.apm.collector.stream.worker.impl.dao.IPersistenceDAO; import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; import org.skywalking.apm.collector.stream.worker.selector.RollingSelector; import org.skywalking.apm.collector.stream.worker.selector.WorkerSelector; @@ -28,9 +26,12 @@ public class SegmentPersistenceWorker extends PersistenceWorker { super.preStart(); } - @Override protected List prepareBatch(Map dataMap) { - ISegmentDAO dao = (ISegmentDAO)DAOContainer.INSTANCE.get(ISegmentDAO.class.getName()); - return dao.prepareBatch(dataMap); + @Override protected boolean needMergeDBData() { + return false; + } + + @Override protected IPersistenceDAO persistenceDAO() { + return (IPersistenceDAO)DAOContainer.INSTANCE.get(ISegmentDAO.class.getName()); } public static class Factory extends AbstractLocalAsyncWorkerProvider { diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/origin/dao/ISegmentDAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/origin/dao/ISegmentDAO.java index 1c1d67e03cab5d359bdcc6c6b3882e9f817141ee..4e8bb03532b0b10d4531db8ee3d08b9d84d1247f 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/origin/dao/ISegmentDAO.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/origin/dao/ISegmentDAO.java @@ -1,12 +1,7 @@ package org.skywalking.apm.collector.agentstream.worker.segment.origin.dao; -import java.util.List; -import java.util.Map; -import org.skywalking.apm.collector.stream.worker.impl.data.Data; - /** * @author pengys5 */ public interface ISegmentDAO { - List prepareBatch(Map dataMap); } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/origin/dao/SegmentEsDAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/origin/dao/SegmentEsDAO.java index 11ced7431e7dca8cafa1ff67eb69a42078b90599..edfb0cd12334cae37ff2abd26a0761b9dea45720 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/origin/dao/SegmentEsDAO.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/origin/dao/SegmentEsDAO.java @@ -1,34 +1,37 @@ package org.skywalking.apm.collector.agentstream.worker.segment.origin.dao; -import java.util.ArrayList; import java.util.Base64; import java.util.HashMap; -import java.util.List; import java.util.Map; import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.action.update.UpdateRequestBuilder; import org.skywalking.apm.collector.agentstream.worker.segment.origin.define.SegmentTable; import org.skywalking.apm.collector.storage.elasticsearch.dao.EsDAO; +import org.skywalking.apm.collector.stream.worker.impl.dao.IPersistenceDAO; import org.skywalking.apm.collector.stream.worker.impl.data.Data; +import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * @author pengys5 */ -public class SegmentEsDAO extends EsDAO implements ISegmentDAO { +public class SegmentEsDAO extends EsDAO implements ISegmentDAO, IPersistenceDAO { private final Logger logger = LoggerFactory.getLogger(SegmentEsDAO.class); - @Override public List prepareBatch(Map dataMap) { - List indexRequestBuilders = new ArrayList<>(); - dataMap.forEach((id, data) -> { - logger.debug("segment prepareBatch, id: {}", id); - Map source = new HashMap(); - source.put(SegmentTable.COLUMN_DATA_BINARY, new String(Base64.getEncoder().encode(data.getDataBytes(0)))); - logger.debug("segment source: {}", source.toString()); - IndexRequestBuilder builder = getClient().prepareIndex(SegmentTable.TABLE, id).setSource(source); - indexRequestBuilders.add(builder); - }); - return indexRequestBuilders; + @Override public Data get(String id, DataDefine dataDefine) { + return null; + } + + @Override public UpdateRequestBuilder prepareBatchUpdate(Data data) { + return null; + } + + @Override public IndexRequestBuilder prepareBatchInsert(Data data) { + Map source = new HashMap<>(); + source.put(SegmentTable.COLUMN_DATA_BINARY, new String(Base64.getEncoder().encode(data.getDataBytes(0)))); + logger.debug("segment source: {}", source.toString()); + return getClient().prepareIndex(SegmentTable.TABLE, data.getDataString(0)).setSource(source); } } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/origin/dao/SegmentH2DAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/origin/dao/SegmentH2DAO.java index 5e9918fe8f46dda69b49a1dddab68bf84e8b5f5a..363088b87d633e01ae46fb038078f3534c65bd98 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/origin/dao/SegmentH2DAO.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/origin/dao/SegmentH2DAO.java @@ -1,15 +1,9 @@ package org.skywalking.apm.collector.agentstream.worker.segment.origin.dao; -import java.util.List; -import java.util.Map; import org.skywalking.apm.collector.storage.h2.dao.H2DAO; /** * @author pengys5 */ public class SegmentH2DAO extends H2DAO implements ISegmentDAO { - - @Override public List prepareBatch(Map map) { - return null; - } } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/origin/define/SegmentDataDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/origin/define/SegmentDataDefine.java index 6edf0322fdbe1130025c0833f6a1774d5341e662..7e2f75caa2a96c633c58b0db97c9329489458c3d 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/origin/define/SegmentDataDefine.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/segment/origin/define/SegmentDataDefine.java @@ -15,12 +15,6 @@ import org.skywalking.apm.collector.stream.worker.impl.data.operate.NonOperation */ public class SegmentDataDefine extends DataDefine { - public static final int DEFINE_ID = 401; - - @Override public int defineId() { - return DEFINE_ID; - } - @Override protected int initialCapacity() { return 2; } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/service/entry/ServiceEntryAggregationWorker.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/service/entry/ServiceEntryAggregationWorker.java new file mode 100644 index 0000000000000000000000000000000000000000..c74fce3f8584c62e65ccda801a90a8476abb94af --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/service/entry/ServiceEntryAggregationWorker.java @@ -0,0 +1,66 @@ +package org.skywalking.apm.collector.agentstream.worker.service.entry; + +import org.skywalking.apm.collector.agentstream.worker.service.entry.define.ServiceEntryDataDefine; +import org.skywalking.apm.collector.stream.worker.AbstractLocalAsyncWorkerProvider; +import org.skywalking.apm.collector.stream.worker.ClusterWorkerContext; +import org.skywalking.apm.collector.stream.worker.ProviderNotFoundException; +import org.skywalking.apm.collector.stream.worker.Role; +import org.skywalking.apm.collector.stream.worker.WorkerNotFoundException; +import org.skywalking.apm.collector.stream.worker.WorkerRefs; +import org.skywalking.apm.collector.stream.worker.impl.AggregationWorker; +import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; +import org.skywalking.apm.collector.stream.worker.selector.HashCodeSelector; +import org.skywalking.apm.collector.stream.worker.selector.WorkerSelector; + +/** + * @author pengys5 + */ +public class ServiceEntryAggregationWorker extends AggregationWorker { + + public ServiceEntryAggregationWorker(Role role, ClusterWorkerContext clusterContext) { + super(role, clusterContext); + } + + @Override public void preStart() throws ProviderNotFoundException { + super.preStart(); + } + + @Override protected WorkerRefs nextWorkRef(String id) throws WorkerNotFoundException { + return getClusterContext().lookup(ServiceEntryRemoteWorker.WorkerRole.INSTANCE); + } + + public static class Factory extends AbstractLocalAsyncWorkerProvider { + @Override + public Role role() { + return WorkerRole.INSTANCE; + } + + @Override + public ServiceEntryAggregationWorker workerInstance(ClusterWorkerContext clusterContext) { + return new ServiceEntryAggregationWorker(role(), clusterContext); + } + + @Override + public int queueSize() { + return 1024; + } + } + + public enum WorkerRole implements Role { + INSTANCE; + + @Override + public String roleName() { + return ServiceEntryAggregationWorker.class.getSimpleName(); + } + + @Override + public WorkerSelector workerSelector() { + return new HashCodeSelector(); + } + + @Override public DataDefine dataDefine() { + return new ServiceEntryDataDefine(); + } + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/service/entry/ServiceEntryPersistenceWorker.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/service/entry/ServiceEntryPersistenceWorker.java new file mode 100644 index 0000000000000000000000000000000000000000..a5caec87a2bc1a32a451f5e71045000acf16aeec --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/service/entry/ServiceEntryPersistenceWorker.java @@ -0,0 +1,71 @@ +package org.skywalking.apm.collector.agentstream.worker.service.entry; + +import org.skywalking.apm.collector.agentstream.worker.service.entry.dao.IServiceEntryDAO; +import org.skywalking.apm.collector.agentstream.worker.service.entry.define.ServiceEntryDataDefine; +import org.skywalking.apm.collector.storage.dao.DAOContainer; +import org.skywalking.apm.collector.stream.worker.AbstractLocalAsyncWorkerProvider; +import org.skywalking.apm.collector.stream.worker.ClusterWorkerContext; +import org.skywalking.apm.collector.stream.worker.ProviderNotFoundException; +import org.skywalking.apm.collector.stream.worker.Role; +import org.skywalking.apm.collector.stream.worker.impl.PersistenceWorker; +import org.skywalking.apm.collector.stream.worker.impl.dao.IPersistenceDAO; +import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; +import org.skywalking.apm.collector.stream.worker.selector.HashCodeSelector; +import org.skywalking.apm.collector.stream.worker.selector.WorkerSelector; + +/** + * @author pengys5 + */ +public class ServiceEntryPersistenceWorker extends PersistenceWorker { + + public ServiceEntryPersistenceWorker(Role role, ClusterWorkerContext clusterContext) { + super(role, clusterContext); + } + + @Override public void preStart() throws ProviderNotFoundException { + super.preStart(); + } + + @Override protected boolean needMergeDBData() { + return true; + } + + @Override protected IPersistenceDAO persistenceDAO() { + return (IPersistenceDAO)DAOContainer.INSTANCE.get(IServiceEntryDAO.class.getName()); + } + + public static class Factory extends AbstractLocalAsyncWorkerProvider { + @Override + public Role role() { + return WorkerRole.INSTANCE; + } + + @Override + public ServiceEntryPersistenceWorker workerInstance(ClusterWorkerContext clusterContext) { + return new ServiceEntryPersistenceWorker(role(), clusterContext); + } + + @Override + public int queueSize() { + return 1024; + } + } + + public enum WorkerRole implements Role { + INSTANCE; + + @Override + public String roleName() { + return ServiceEntryPersistenceWorker.class.getSimpleName(); + } + + @Override + public WorkerSelector workerSelector() { + return new HashCodeSelector(); + } + + @Override public DataDefine dataDefine() { + return new ServiceEntryDataDefine(); + } + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/service/entry/ServiceEntryRemoteWorker.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/service/entry/ServiceEntryRemoteWorker.java new file mode 100644 index 0000000000000000000000000000000000000000..3ca0a81956a61c58ce2bae1acf3db6d1d6447cd4 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/service/entry/ServiceEntryRemoteWorker.java @@ -0,0 +1,60 @@ +package org.skywalking.apm.collector.agentstream.worker.service.entry; + +import org.skywalking.apm.collector.agentstream.worker.service.entry.define.ServiceEntryDataDefine; +import org.skywalking.apm.collector.stream.worker.AbstractRemoteWorker; +import org.skywalking.apm.collector.stream.worker.AbstractRemoteWorkerProvider; +import org.skywalking.apm.collector.stream.worker.ClusterWorkerContext; +import org.skywalking.apm.collector.stream.worker.ProviderNotFoundException; +import org.skywalking.apm.collector.stream.worker.Role; +import org.skywalking.apm.collector.stream.worker.WorkerException; +import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; +import org.skywalking.apm.collector.stream.worker.selector.HashCodeSelector; +import org.skywalking.apm.collector.stream.worker.selector.WorkerSelector; + +/** + * @author pengys5 + */ +public class ServiceEntryRemoteWorker extends AbstractRemoteWorker { + + protected ServiceEntryRemoteWorker(Role role, ClusterWorkerContext clusterContext) { + super(role, clusterContext); + } + + @Override public void preStart() throws ProviderNotFoundException { + + } + + @Override protected void onWork(Object message) throws WorkerException { + getClusterContext().lookup(ServiceEntryPersistenceWorker.WorkerRole.INSTANCE).tell(message); + } + + public static class Factory extends AbstractRemoteWorkerProvider { + @Override + public Role role() { + return WorkerRole.INSTANCE; + } + + @Override + public ServiceEntryRemoteWorker workerInstance(ClusterWorkerContext clusterContext) { + return new ServiceEntryRemoteWorker(role(), clusterContext); + } + } + + public enum WorkerRole implements Role { + INSTANCE; + + @Override + public String roleName() { + return ServiceEntryRemoteWorker.class.getSimpleName(); + } + + @Override + public WorkerSelector workerSelector() { + return new HashCodeSelector(); + } + + @Override public DataDefine dataDefine() { + return new ServiceEntryDataDefine(); + } + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/service/entry/ServiceEntrySpanListener.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/service/entry/ServiceEntrySpanListener.java new file mode 100644 index 0000000000000000000000000000000000000000..aaeba441b9aaf6ce8fb12f4ff1d179d29492466b --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/service/entry/ServiceEntrySpanListener.java @@ -0,0 +1,70 @@ +package org.skywalking.apm.collector.agentstream.worker.service.entry; + +import org.skywalking.apm.collector.agentstream.worker.Const; +import org.skywalking.apm.collector.agentstream.worker.segment.EntrySpanListener; +import org.skywalking.apm.collector.agentstream.worker.segment.FirstSpanListener; +import org.skywalking.apm.collector.agentstream.worker.segment.RefsListener; +import org.skywalking.apm.collector.agentstream.worker.service.entry.define.ServiceEntryDataDefine; +import org.skywalking.apm.collector.agentstream.worker.util.ExchangeMarkUtils; +import org.skywalking.apm.collector.agentstream.worker.util.TimeBucketUtils; +import org.skywalking.apm.collector.core.framework.CollectorContextHelper; +import org.skywalking.apm.collector.stream.StreamModuleContext; +import org.skywalking.apm.collector.stream.StreamModuleGroupDefine; +import org.skywalking.apm.collector.stream.worker.WorkerInvokeException; +import org.skywalking.apm.collector.stream.worker.WorkerNotFoundException; +import org.skywalking.apm.network.proto.SpanObject; +import org.skywalking.apm.network.proto.TraceSegmentReference; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public class ServiceEntrySpanListener implements RefsListener, FirstSpanListener, EntrySpanListener { + + private final Logger logger = LoggerFactory.getLogger(ServiceEntrySpanListener.class); + + private long timeBucket; + private boolean hasReference = false; + private String agg; + private int applicationId; + + @Override + public void parseEntry(SpanObject spanObject, int applicationId, int applicationInstanceId, String segmentId) { + String entryServiceName = spanObject.getOperationName(); + if (spanObject.getOperationNameId() != 0) { + entryServiceName = ExchangeMarkUtils.INSTANCE.buildMarkedID(spanObject.getOperationNameId()); + } + this.agg = ExchangeMarkUtils.INSTANCE.buildMarkedID(applicationId) + Const.ID_SPLIT + entryServiceName; + this.applicationId = applicationId; + } + + @Override public void parseRef(TraceSegmentReference reference, int applicationId, int applicationInstanceId, + String segmentId) { + hasReference = true; + } + + @Override + public void parseFirst(SpanObject spanObject, int applicationId, int applicationInstanceId, String segmentId) { + timeBucket = TimeBucketUtils.INSTANCE.getMinuteTimeBucket(spanObject.getStartTime()); + } + + @Override public void build() { + logger.debug("entry service listener build"); + StreamModuleContext context = (StreamModuleContext)CollectorContextHelper.INSTANCE.getContext(StreamModuleGroupDefine.GROUP_NAME); + if (!hasReference) { + ServiceEntryDataDefine.ServiceEntry serviceEntry = new ServiceEntryDataDefine.ServiceEntry(); + serviceEntry.setId(timeBucket + Const.ID_SPLIT + agg); + serviceEntry.setApplicationId(applicationId); + serviceEntry.setAgg(agg); + serviceEntry.setTimeBucket(timeBucket); + + try { + logger.debug("send to service entry aggregation worker, id: {}", serviceEntry.getId()); + context.getClusterWorkerContext().lookup(ServiceEntryAggregationWorker.WorkerRole.INSTANCE).tell(serviceEntry.toData()); + } catch (WorkerInvokeException | WorkerNotFoundException e) { + logger.error(e.getMessage(), e); + } + } + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/service/entry/dao/IServiceEntryDAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/service/entry/dao/IServiceEntryDAO.java new file mode 100644 index 0000000000000000000000000000000000000000..fcfe8f572c1f7dd6f18e6159e882fb64fdea701b --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/service/entry/dao/IServiceEntryDAO.java @@ -0,0 +1,7 @@ +package org.skywalking.apm.collector.agentstream.worker.service.entry.dao; + +/** + * @author pengys5 + */ +public interface IServiceEntryDAO { +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/service/entry/dao/ServiceEntryEsDAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/service/entry/dao/ServiceEntryEsDAO.java new file mode 100644 index 0000000000000000000000000000000000000000..b35edd11543558f32de4f5fa413fa6ebace6fb1f --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/service/entry/dao/ServiceEntryEsDAO.java @@ -0,0 +1,50 @@ +package org.skywalking.apm.collector.agentstream.worker.service.entry.dao; + +import java.util.HashMap; +import java.util.Map; +import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.action.update.UpdateRequestBuilder; +import org.skywalking.apm.collector.agentstream.worker.service.entry.define.ServiceEntryTable; +import org.skywalking.apm.collector.storage.elasticsearch.dao.EsDAO; +import org.skywalking.apm.collector.stream.worker.impl.dao.IPersistenceDAO; +import org.skywalking.apm.collector.stream.worker.impl.data.Data; +import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; + +/** + * @author pengys5 + */ +public class ServiceEntryEsDAO extends EsDAO implements IServiceEntryDAO, IPersistenceDAO { + + @Override public Data get(String id, DataDefine dataDefine) { + GetResponse getResponse = getClient().prepareGet(ServiceEntryTable.TABLE, id).get(); + if (getResponse.isExists()) { + Data data = dataDefine.build(id); + Map source = getResponse.getSource(); + data.setDataInteger(0, (Integer)source.get(ServiceEntryTable.COLUMN_APPLICATION_ID)); + data.setDataString(1, (String)source.get(ServiceEntryTable.COLUMN_AGG)); + data.setDataLong(0, (Long)source.get(ServiceEntryTable.COLUMN_TIME_BUCKET)); + return data; + } else { + return null; + } + } + + @Override public IndexRequestBuilder prepareBatchInsert(Data data) { + Map source = new HashMap<>(); + source.put(ServiceEntryTable.COLUMN_APPLICATION_ID, data.getDataInteger(0)); + source.put(ServiceEntryTable.COLUMN_AGG, data.getDataString(1)); + source.put(ServiceEntryTable.COLUMN_TIME_BUCKET, data.getDataLong(0)); + + return getClient().prepareIndex(ServiceEntryTable.TABLE, data.getDataString(0)).setSource(source); + } + + @Override public UpdateRequestBuilder prepareBatchUpdate(Data data) { + Map source = new HashMap<>(); + source.put(ServiceEntryTable.COLUMN_APPLICATION_ID, data.getDataInteger(0)); + source.put(ServiceEntryTable.COLUMN_AGG, data.getDataString(1)); + source.put(ServiceEntryTable.COLUMN_TIME_BUCKET, data.getDataLong(0)); + + return getClient().prepareUpdate(ServiceEntryTable.TABLE, data.getDataString(0)).setDoc(source); + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/service/entry/dao/ServiceEntryH2DAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/service/entry/dao/ServiceEntryH2DAO.java new file mode 100644 index 0000000000000000000000000000000000000000..fb2cd96c7a3c93371d22b9239a3c3b73a4fd096c --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/service/entry/dao/ServiceEntryH2DAO.java @@ -0,0 +1,9 @@ +package org.skywalking.apm.collector.agentstream.worker.service.entry.dao; + +import org.skywalking.apm.collector.storage.h2.dao.H2DAO; + +/** + * @author pengys5 + */ +public class ServiceEntryH2DAO extends H2DAO implements IServiceEntryDAO { +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/service/entry/define/ServiceEntryDataDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/service/entry/define/ServiceEntryDataDefine.java new file mode 100644 index 0000000000000000000000000000000000000000..ea3a37d751ca5f09cc741a6aa7717f50e0c00a4e --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/service/entry/define/ServiceEntryDataDefine.java @@ -0,0 +1,102 @@ +package org.skywalking.apm.collector.agentstream.worker.service.entry.define; + +import org.skywalking.apm.collector.remote.grpc.proto.RemoteData; +import org.skywalking.apm.collector.stream.worker.impl.data.Attribute; +import org.skywalking.apm.collector.stream.worker.impl.data.AttributeType; +import org.skywalking.apm.collector.stream.worker.impl.data.Data; +import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; +import org.skywalking.apm.collector.stream.worker.impl.data.Transform; +import org.skywalking.apm.collector.stream.worker.impl.data.operate.CoverOperation; +import org.skywalking.apm.collector.stream.worker.impl.data.operate.NonOperation; + +/** + * @author pengys5 + */ +public class ServiceEntryDataDefine extends DataDefine { + + @Override protected int initialCapacity() { + return 4; + } + + @Override protected void attributeDefine() { + addAttribute(0, new Attribute(ServiceEntryTable.COLUMN_ID, AttributeType.STRING, new NonOperation())); + addAttribute(1, new Attribute(ServiceEntryTable.COLUMN_APPLICATION_ID, AttributeType.INTEGER, new NonOperation())); + addAttribute(2, new Attribute(ServiceEntryTable.COLUMN_AGG, AttributeType.STRING, new CoverOperation())); + addAttribute(3, new Attribute(ServiceEntryTable.COLUMN_TIME_BUCKET, AttributeType.LONG, new CoverOperation())); + } + + @Override public Object deserialize(RemoteData remoteData) { + return null; + } + + @Override public RemoteData serialize(Object object) { + return null; + } + + public static class ServiceEntry implements Transform { + private String id; + private int applicationId; + private String agg; + private long timeBucket; + + ServiceEntry(String id, int applicationId, String agg, long timeBucket) { + this.id = id; + this.applicationId = applicationId; + this.agg = agg; + this.timeBucket = timeBucket; + } + + public ServiceEntry() { + } + + @Override public Data toData() { + ServiceEntryDataDefine define = new ServiceEntryDataDefine(); + Data data = define.build(id); + data.setDataString(0, this.id); + data.setDataInteger(0, this.applicationId); + data.setDataString(1, this.agg); + data.setDataLong(0, this.timeBucket); + return data; + } + + @Override public ServiceEntry toSelf(Data data) { + this.id = data.getDataString(0); + this.applicationId = data.getDataInteger(0); + this.agg = data.getDataString(1); + this.timeBucket = data.getDataLong(0); + return this; + } + + public String getId() { + return id; + } + + public String getAgg() { + return agg; + } + + public long getTimeBucket() { + return timeBucket; + } + + public void setId(String id) { + this.id = id; + } + + public void setAgg(String agg) { + this.agg = agg; + } + + public void setTimeBucket(long timeBucket) { + this.timeBucket = timeBucket; + } + + public int getApplicationId() { + return applicationId; + } + + public void setApplicationId(int applicationId) { + this.applicationId = applicationId; + } + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/service/entry/define/ServiceEntryEsTableDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/service/entry/define/ServiceEntryEsTableDefine.java new file mode 100644 index 0000000000000000000000000000000000000000..54191227f6c0ad3938f0b12c59f5757d900a3021 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/service/entry/define/ServiceEntryEsTableDefine.java @@ -0,0 +1,31 @@ +package org.skywalking.apm.collector.agentstream.worker.service.entry.define; + +import org.skywalking.apm.collector.storage.elasticsearch.define.ElasticSearchColumnDefine; +import org.skywalking.apm.collector.storage.elasticsearch.define.ElasticSearchTableDefine; + +/** + * @author pengys5 + */ +public class ServiceEntryEsTableDefine extends ElasticSearchTableDefine { + + public ServiceEntryEsTableDefine() { + super(ServiceEntryTable.TABLE); + } + + @Override public int refreshInterval() { + return 2; + } + + @Override public int numberOfShards() { + return 2; + } + + @Override public int numberOfReplicas() { + return 0; + } + + @Override public void initialize() { + addColumn(new ElasticSearchColumnDefine(ServiceEntryTable.COLUMN_AGG, ElasticSearchColumnDefine.Type.Keyword.name())); + addColumn(new ElasticSearchColumnDefine(ServiceEntryTable.COLUMN_TIME_BUCKET, ElasticSearchColumnDefine.Type.Long.name())); + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/service/entry/define/ServiceEntryH2TableDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/service/entry/define/ServiceEntryH2TableDefine.java new file mode 100644 index 0000000000000000000000000000000000000000..e38d6e7260b3b2b876735eb2b32321c8f19a9563 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/service/entry/define/ServiceEntryH2TableDefine.java @@ -0,0 +1,20 @@ +package org.skywalking.apm.collector.agentstream.worker.service.entry.define; + +import org.skywalking.apm.collector.storage.h2.define.H2ColumnDefine; +import org.skywalking.apm.collector.storage.h2.define.H2TableDefine; + +/** + * @author pengys5 + */ +public class ServiceEntryH2TableDefine extends H2TableDefine { + + public ServiceEntryH2TableDefine() { + super(ServiceEntryTable.TABLE); + } + + @Override public void initialize() { + addColumn(new H2ColumnDefine(ServiceEntryTable.COLUMN_ID, H2ColumnDefine.Type.Varchar.name())); + addColumn(new H2ColumnDefine(ServiceEntryTable.COLUMN_AGG, H2ColumnDefine.Type.Varchar.name())); + addColumn(new H2ColumnDefine(ServiceEntryTable.COLUMN_TIME_BUCKET, H2ColumnDefine.Type.Bigint.name())); + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/service/entry/define/ServiceEntryTable.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/service/entry/define/ServiceEntryTable.java new file mode 100644 index 0000000000000000000000000000000000000000..5ec9957f8310b5333322bed523156d9dbcf12c59 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/service/entry/define/ServiceEntryTable.java @@ -0,0 +1,11 @@ +package org.skywalking.apm.collector.agentstream.worker.service.entry.define; + +import org.skywalking.apm.collector.agentstream.worker.CommonTable; + +/** + * @author pengys5 + */ +public class ServiceEntryTable extends CommonTable { + public static final String TABLE = "service_entry"; + public static final String COLUMN_APPLICATION_ID = "application_id"; +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/serviceref/reference/ServiceRefAggregationWorker.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/serviceref/reference/ServiceRefAggregationWorker.java new file mode 100644 index 0000000000000000000000000000000000000000..b8759d051bbabdc8162d93a1409777fdc76cfcc4 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/serviceref/reference/ServiceRefAggregationWorker.java @@ -0,0 +1,66 @@ +package org.skywalking.apm.collector.agentstream.worker.serviceref.reference; + +import org.skywalking.apm.collector.agentstream.worker.serviceref.reference.define.ServiceRefDataDefine; +import org.skywalking.apm.collector.stream.worker.AbstractLocalAsyncWorkerProvider; +import org.skywalking.apm.collector.stream.worker.ClusterWorkerContext; +import org.skywalking.apm.collector.stream.worker.ProviderNotFoundException; +import org.skywalking.apm.collector.stream.worker.Role; +import org.skywalking.apm.collector.stream.worker.WorkerNotFoundException; +import org.skywalking.apm.collector.stream.worker.WorkerRefs; +import org.skywalking.apm.collector.stream.worker.impl.AggregationWorker; +import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; +import org.skywalking.apm.collector.stream.worker.selector.HashCodeSelector; +import org.skywalking.apm.collector.stream.worker.selector.WorkerSelector; + +/** + * @author pengys5 + */ +public class ServiceRefAggregationWorker extends AggregationWorker { + + public ServiceRefAggregationWorker(Role role, ClusterWorkerContext clusterContext) { + super(role, clusterContext); + } + + @Override public void preStart() throws ProviderNotFoundException { + super.preStart(); + } + + @Override protected WorkerRefs nextWorkRef(String id) throws WorkerNotFoundException { + return getClusterContext().lookup(ServiceRefRemoteWorker.WorkerRole.INSTANCE); + } + + public static class Factory extends AbstractLocalAsyncWorkerProvider { + @Override + public Role role() { + return WorkerRole.INSTANCE; + } + + @Override + public ServiceRefAggregationWorker workerInstance(ClusterWorkerContext clusterContext) { + return new ServiceRefAggregationWorker(role(), clusterContext); + } + + @Override + public int queueSize() { + return 1024; + } + } + + public enum WorkerRole implements Role { + INSTANCE; + + @Override + public String roleName() { + return ServiceRefAggregationWorker.class.getSimpleName(); + } + + @Override + public WorkerSelector workerSelector() { + return new HashCodeSelector(); + } + + @Override public DataDefine dataDefine() { + return new ServiceRefDataDefine(); + } + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/serviceref/reference/ServiceRefPersistenceWorker.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/serviceref/reference/ServiceRefPersistenceWorker.java new file mode 100644 index 0000000000000000000000000000000000000000..64796b5208031c14e91b5afe76de50dd10a535cd --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/serviceref/reference/ServiceRefPersistenceWorker.java @@ -0,0 +1,71 @@ +package org.skywalking.apm.collector.agentstream.worker.serviceref.reference; + +import org.skywalking.apm.collector.agentstream.worker.serviceref.reference.dao.IServiceRefDAO; +import org.skywalking.apm.collector.agentstream.worker.serviceref.reference.define.ServiceRefDataDefine; +import org.skywalking.apm.collector.storage.dao.DAOContainer; +import org.skywalking.apm.collector.stream.worker.AbstractLocalAsyncWorkerProvider; +import org.skywalking.apm.collector.stream.worker.ClusterWorkerContext; +import org.skywalking.apm.collector.stream.worker.ProviderNotFoundException; +import org.skywalking.apm.collector.stream.worker.Role; +import org.skywalking.apm.collector.stream.worker.impl.PersistenceWorker; +import org.skywalking.apm.collector.stream.worker.impl.dao.IPersistenceDAO; +import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; +import org.skywalking.apm.collector.stream.worker.selector.HashCodeSelector; +import org.skywalking.apm.collector.stream.worker.selector.WorkerSelector; + +/** + * @author pengys5 + */ +public class ServiceRefPersistenceWorker extends PersistenceWorker { + + public ServiceRefPersistenceWorker(Role role, ClusterWorkerContext clusterContext) { + super(role, clusterContext); + } + + @Override public void preStart() throws ProviderNotFoundException { + super.preStart(); + } + + @Override protected boolean needMergeDBData() { + return true; + } + + @Override protected IPersistenceDAO persistenceDAO() { + return (IPersistenceDAO)DAOContainer.INSTANCE.get(IServiceRefDAO.class.getName()); + } + + public static class Factory extends AbstractLocalAsyncWorkerProvider { + @Override + public Role role() { + return WorkerRole.INSTANCE; + } + + @Override + public ServiceRefPersistenceWorker workerInstance(ClusterWorkerContext clusterContext) { + return new ServiceRefPersistenceWorker(role(), clusterContext); + } + + @Override + public int queueSize() { + return 1024; + } + } + + public enum WorkerRole implements Role { + INSTANCE; + + @Override + public String roleName() { + return ServiceRefPersistenceWorker.class.getSimpleName(); + } + + @Override + public WorkerSelector workerSelector() { + return new HashCodeSelector(); + } + + @Override public DataDefine dataDefine() { + return new ServiceRefDataDefine(); + } + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/serviceref/reference/ServiceRefRemoteWorker.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/serviceref/reference/ServiceRefRemoteWorker.java new file mode 100644 index 0000000000000000000000000000000000000000..07b3864434f2f59a62ec824cbf3ed8149638375f --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/serviceref/reference/ServiceRefRemoteWorker.java @@ -0,0 +1,60 @@ +package org.skywalking.apm.collector.agentstream.worker.serviceref.reference; + +import org.skywalking.apm.collector.agentstream.worker.serviceref.reference.define.ServiceRefDataDefine; +import org.skywalking.apm.collector.stream.worker.AbstractRemoteWorker; +import org.skywalking.apm.collector.stream.worker.AbstractRemoteWorkerProvider; +import org.skywalking.apm.collector.stream.worker.ClusterWorkerContext; +import org.skywalking.apm.collector.stream.worker.ProviderNotFoundException; +import org.skywalking.apm.collector.stream.worker.Role; +import org.skywalking.apm.collector.stream.worker.WorkerException; +import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; +import org.skywalking.apm.collector.stream.worker.selector.HashCodeSelector; +import org.skywalking.apm.collector.stream.worker.selector.WorkerSelector; + +/** + * @author pengys5 + */ +public class ServiceRefRemoteWorker extends AbstractRemoteWorker { + + protected ServiceRefRemoteWorker(Role role, ClusterWorkerContext clusterContext) { + super(role, clusterContext); + } + + @Override public void preStart() throws ProviderNotFoundException { + + } + + @Override protected void onWork(Object message) throws WorkerException { + getClusterContext().lookup(ServiceRefPersistenceWorker.WorkerRole.INSTANCE).tell(message); + } + + public static class Factory extends AbstractRemoteWorkerProvider { + @Override + public Role role() { + return WorkerRole.INSTANCE; + } + + @Override + public ServiceRefRemoteWorker workerInstance(ClusterWorkerContext clusterContext) { + return new ServiceRefRemoteWorker(role(), clusterContext); + } + } + + public enum WorkerRole implements Role { + INSTANCE; + + @Override + public String roleName() { + return ServiceRefRemoteWorker.class.getSimpleName(); + } + + @Override + public WorkerSelector workerSelector() { + return new HashCodeSelector(); + } + + @Override public DataDefine dataDefine() { + return new ServiceRefDataDefine(); + } + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/serviceref/reference/ServiceRefSpanListener.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/serviceref/reference/ServiceRefSpanListener.java index 12986db71f25a19549fd45b478305ba0d7d93402..b675498a5d52ec3a52a0f3f439d6885f0f509e33 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/serviceref/reference/ServiceRefSpanListener.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/serviceref/reference/ServiceRefSpanListener.java @@ -3,12 +3,19 @@ package org.skywalking.apm.collector.agentstream.worker.serviceref.reference; import java.util.ArrayList; import java.util.List; import org.skywalking.apm.collector.agentstream.worker.Const; +import org.skywalking.apm.collector.agentstream.worker.cache.InstanceCache; import org.skywalking.apm.collector.agentstream.worker.segment.EntrySpanListener; import org.skywalking.apm.collector.agentstream.worker.segment.ExitSpanListener; import org.skywalking.apm.collector.agentstream.worker.segment.FirstSpanListener; import org.skywalking.apm.collector.agentstream.worker.segment.RefsListener; import org.skywalking.apm.collector.agentstream.worker.serviceref.reference.define.ServiceRefDataDefine; +import org.skywalking.apm.collector.agentstream.worker.util.ExchangeMarkUtils; import org.skywalking.apm.collector.agentstream.worker.util.TimeBucketUtils; +import org.skywalking.apm.collector.core.framework.CollectorContextHelper; +import org.skywalking.apm.collector.stream.StreamModuleContext; +import org.skywalking.apm.collector.stream.StreamModuleGroupDefine; +import org.skywalking.apm.collector.stream.worker.WorkerInvokeException; +import org.skywalking.apm.collector.stream.worker.WorkerNotFoundException; import org.skywalking.apm.network.proto.SpanObject; import org.skywalking.apm.network.proto.TraceSegmentReference; import org.slf4j.Logger; @@ -21,9 +28,10 @@ public class ServiceRefSpanListener implements FirstSpanListener, EntrySpanListe private final Logger logger = LoggerFactory.getLogger(ServiceRefSpanListener.class); - private String front; - private List behinds = new ArrayList<>(); - private List fronts = new ArrayList<>(); + private List exitServiceNames = new ArrayList<>(); + private String currentServiceName; + private List referenceServices = new ArrayList<>(); + private boolean hasReference = false; private long timeBucket; @Override @@ -33,51 +41,96 @@ public class ServiceRefSpanListener implements FirstSpanListener, EntrySpanListe @Override public void parseRef(TraceSegmentReference reference, int applicationId, int applicationInstanceId, String segmentId) { - String entryService = String.valueOf(reference.getEntryServiceId()); - if (reference.getEntryServiceId() == 0) { - entryService = reference.getEntryServiceName(); + int entryApplicationId = InstanceCache.get(reference.getEntryApplicationInstanceId()); + String entryServiceName = reference.getEntryServiceName(); + if (reference.getEntryServiceId() != 0) { + entryServiceName = ExchangeMarkUtils.INSTANCE.buildMarkedID(reference.getEntryServiceId()); } - String parentService = String.valueOf(reference.getParentServiceId()); + entryServiceName = ExchangeMarkUtils.INSTANCE.buildMarkedID(entryApplicationId) + Const.ID_SPLIT + entryServiceName; + + int parentApplicationId = InstanceCache.get(reference.getParentApplicationInstanceId()); + String parentServiceName = ExchangeMarkUtils.INSTANCE.buildMarkedID(reference.getParentServiceId()); if (reference.getParentServiceId() == 0) { - parentService = reference.getParentServiceName(); + parentServiceName = reference.getParentServiceName(); } - fronts.add(new ServiceTemp(entryService, parentService)); + parentServiceName = ExchangeMarkUtils.INSTANCE.buildMarkedID(parentApplicationId) + Const.ID_SPLIT + parentServiceName; + + referenceServices.add(new ServiceTemp(entryServiceName, parentServiceName)); + hasReference = true; } @Override public void parseEntry(SpanObject spanObject, int applicationId, int applicationInstanceId, String segmentId) { - front = String.valueOf(spanObject.getOperationNameId()); - if (spanObject.getOperationNameId() == 0) { - front = spanObject.getOperationName(); + String serviceName = spanObject.getOperationName(); + if (spanObject.getOperationNameId() != 0) { + serviceName = ExchangeMarkUtils.INSTANCE.buildMarkedID(spanObject.getOperationNameId()); } + + serviceName = ExchangeMarkUtils.INSTANCE.buildMarkedID(applicationId) + Const.ID_SPLIT + serviceName; + currentServiceName = serviceName; } @Override public void parseExit(SpanObject spanObject, int applicationId, int applicationInstanceId, String segmentId) { - String behind = String.valueOf(spanObject.getOperationNameId()); + String serviceName = ExchangeMarkUtils.INSTANCE.buildMarkedID(spanObject.getOperationNameId()); if (spanObject.getOperationNameId() == 0) { - behind = spanObject.getOperationName(); + serviceName = spanObject.getOperationName(); } - behinds.add(behind); + serviceName = ExchangeMarkUtils.INSTANCE.buildMarkedID(applicationId) + Const.ID_SPLIT + serviceName; + exitServiceNames.add(serviceName); } @Override public void build() { - for (String behind : behinds) { - String agg = front + Const.ID_SPLIT + behind; + logger.debug("service reference listener build"); + StreamModuleContext context = (StreamModuleContext)CollectorContextHelper.INSTANCE.getContext(StreamModuleGroupDefine.GROUP_NAME); + + List serviceReferences = new ArrayList<>(); + for (ServiceTemp referenceService : referenceServices) { + String agg = referenceService.parentServiceName + Const.ID_SPLIT + currentServiceName; ServiceRefDataDefine.ServiceReference serviceReference = new ServiceRefDataDefine.ServiceReference(); - serviceReference.setId(timeBucket + Const.ID_SPLIT + agg); + serviceReference.setId(timeBucket + Const.ID_SPLIT + referenceService.entryServiceName + Const.ID_SPLIT + agg); + serviceReference.setEntryService(referenceService.entryServiceName); serviceReference.setAgg(agg); serviceReference.setTimeBucket(timeBucket); + + serviceReferences.add(serviceReference); + } + + for (String exitServiceName : exitServiceNames) { + String entryServiceName; + if (referenceServices.size() > 0) { + entryServiceName = referenceServices.get(0).entryServiceName; + } else { + entryServiceName = currentServiceName; + } + + String agg = currentServiceName + Const.ID_SPLIT + exitServiceName; + ServiceRefDataDefine.ServiceReference serviceReference = new ServiceRefDataDefine.ServiceReference(); + serviceReference.setId(timeBucket + Const.ID_SPLIT + entryServiceName + Const.ID_SPLIT + agg); + serviceReference.setEntryService(entryServiceName); + serviceReference.setAgg(agg); + serviceReference.setTimeBucket(timeBucket); + + serviceReferences.add(serviceReference); + } + + for (ServiceRefDataDefine.ServiceReference serviceReference : serviceReferences) { + try { + logger.debug("send to service reference aggregation worker, id: {}", serviceReference.getId()); + context.getClusterWorkerContext().lookup(ServiceRefAggregationWorker.WorkerRole.INSTANCE).tell(serviceReference.toData()); + } catch (WorkerInvokeException | WorkerNotFoundException e) { + logger.error(e.getMessage(), e); + } } } class ServiceTemp { - private final String entryService; - private final String parentService; + private final String entryServiceName; + private final String parentServiceName; - public ServiceTemp(String entryService, String parentService) { - this.entryService = entryService; - this.parentService = parentService; + public ServiceTemp(String entryServiceName, String parentServiceName) { + this.entryServiceName = entryServiceName; + this.parentServiceName = parentServiceName; } } } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/serviceref/reference/dao/IServiceRefDAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/serviceref/reference/dao/IServiceRefDAO.java new file mode 100644 index 0000000000000000000000000000000000000000..a3ae8b47aa617e7157e7d59f7f78883c09302fee --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/serviceref/reference/dao/IServiceRefDAO.java @@ -0,0 +1,7 @@ +package org.skywalking.apm.collector.agentstream.worker.serviceref.reference.dao; + +/** + * @author pengys5 + */ +public interface IServiceRefDAO { +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/serviceref/reference/dao/ServiceRefEsDAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/serviceref/reference/dao/ServiceRefEsDAO.java new file mode 100644 index 0000000000000000000000000000000000000000..58e849b28ab4dfe0f2de22977d1e4135195f24e3 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/serviceref/reference/dao/ServiceRefEsDAO.java @@ -0,0 +1,54 @@ +package org.skywalking.apm.collector.agentstream.worker.serviceref.reference.dao; + +import java.util.HashMap; +import java.util.Map; +import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.action.update.UpdateRequestBuilder; +import org.skywalking.apm.collector.agentstream.worker.serviceref.reference.define.ServiceRefTable; +import org.skywalking.apm.collector.storage.elasticsearch.dao.EsDAO; +import org.skywalking.apm.collector.stream.worker.impl.dao.IPersistenceDAO; +import org.skywalking.apm.collector.stream.worker.impl.data.Data; +import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public class ServiceRefEsDAO extends EsDAO implements IServiceRefDAO, IPersistenceDAO { + + private final Logger logger = LoggerFactory.getLogger(ServiceRefEsDAO.class); + + @Override public Data get(String id, DataDefine dataDefine) { + GetResponse getResponse = getClient().prepareGet(ServiceRefTable.TABLE, id).get(); + if (getResponse.isExists()) { + Data data = dataDefine.build(id); + Map source = getResponse.getSource(); + data.setDataString(1, (String)source.get(ServiceRefTable.COLUMN_ENTRY_SERVICE)); + data.setDataString(2, (String)source.get(ServiceRefTable.COLUMN_AGG)); + data.setDataLong(0, (Long)source.get(ServiceRefTable.COLUMN_TIME_BUCKET)); + return data; + } else { + return null; + } + } + + @Override public IndexRequestBuilder prepareBatchInsert(Data data) { + Map source = new HashMap<>(); + source.put(ServiceRefTable.COLUMN_ENTRY_SERVICE, data.getDataString(1)); + source.put(ServiceRefTable.COLUMN_AGG, data.getDataString(2)); + source.put(ServiceRefTable.COLUMN_TIME_BUCKET, data.getDataLong(0)); + + return getClient().prepareIndex(ServiceRefTable.TABLE, data.getDataString(0)).setSource(source); + } + + @Override public UpdateRequestBuilder prepareBatchUpdate(Data data) { + Map source = new HashMap<>(); + source.put(ServiceRefTable.COLUMN_ENTRY_SERVICE, data.getDataString(1)); + source.put(ServiceRefTable.COLUMN_AGG, data.getDataString(2)); + source.put(ServiceRefTable.COLUMN_TIME_BUCKET, data.getDataLong(0)); + + return getClient().prepareUpdate(ServiceRefTable.TABLE, data.getDataString(0)).setDoc(source); + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/serviceref/reference/dao/ServiceRefH2DAO.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/serviceref/reference/dao/ServiceRefH2DAO.java new file mode 100644 index 0000000000000000000000000000000000000000..f911e5841562fbe9c3581ff78d7b218d17359d24 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/serviceref/reference/dao/ServiceRefH2DAO.java @@ -0,0 +1,24 @@ +package org.skywalking.apm.collector.agentstream.worker.serviceref.reference.dao; + +import org.skywalking.apm.collector.storage.h2.dao.H2DAO; +import org.skywalking.apm.collector.stream.worker.impl.dao.IPersistenceDAO; +import org.skywalking.apm.collector.stream.worker.impl.data.Data; +import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; + +/** + * @author pengys5 + */ +public class ServiceRefH2DAO extends H2DAO implements IServiceRefDAO, IPersistenceDAO { + + @Override public Data get(String id, DataDefine dataDefine) { + return null; + } + + @Override public String prepareBatchInsert(Data data) { + return null; + } + + @Override public String prepareBatchUpdate(Data data) { + return null; + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/serviceref/reference/define/ServiceRefDataDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/serviceref/reference/define/ServiceRefDataDefine.java index 00ff6b140babaf4c8002dc4f314c3014f3960e06..55e4b723daf62def5b00fc657ad64500b29bfd0a 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/serviceref/reference/define/ServiceRefDataDefine.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/serviceref/reference/define/ServiceRefDataDefine.java @@ -3,7 +3,9 @@ package org.skywalking.apm.collector.agentstream.worker.serviceref.reference.def import org.skywalking.apm.collector.remote.grpc.proto.RemoteData; import org.skywalking.apm.collector.stream.worker.impl.data.Attribute; import org.skywalking.apm.collector.stream.worker.impl.data.AttributeType; +import org.skywalking.apm.collector.stream.worker.impl.data.Data; import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; +import org.skywalking.apm.collector.stream.worker.impl.data.Transform; import org.skywalking.apm.collector.stream.worker.impl.data.operate.CoverOperation; import org.skywalking.apm.collector.stream.worker.impl.data.operate.NonOperation; @@ -12,12 +14,6 @@ import org.skywalking.apm.collector.stream.worker.impl.data.operate.NonOperation */ public class ServiceRefDataDefine extends DataDefine { - public static final int DEFINE_ID = 501; - - @Override public int defineId() { - return DEFINE_ID; - } - @Override protected int initialCapacity() { return 4; } @@ -47,7 +43,7 @@ public class ServiceRefDataDefine extends DataDefine { return builder.build(); } - public static class ServiceReference { + public static class ServiceReference implements Transform { private String id; private String entryService; private String agg; @@ -63,6 +59,24 @@ public class ServiceRefDataDefine extends DataDefine { public ServiceReference() { } + @Override public Data toData() { + ServiceRefDataDefine define = new ServiceRefDataDefine(); + Data data = define.build(id); + data.setDataString(0, this.id); + data.setDataString(1, this.entryService); + data.setDataString(2, this.agg); + data.setDataLong(0, this.timeBucket); + return data; + } + + @Override public Object toSelf(Data data) { + this.id = data.getDataString(0); + this.entryService = data.getDataString(1); + this.agg = data.getDataString(2); + this.timeBucket = data.getDataLong(0); + return this; + } + public String getId() { return id; } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/serviceref/reference/define/ServiceRefEsTableDefine.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/serviceref/reference/define/ServiceRefEsTableDefine.java index 067624ed5c4d51b852b0c87d17f1e76ea3f58896..798b673a243ce1eff1e7d86fb24e3f490ec42c55 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/serviceref/reference/define/ServiceRefEsTableDefine.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/serviceref/reference/define/ServiceRefEsTableDefine.java @@ -13,7 +13,7 @@ public class ServiceRefEsTableDefine extends ElasticSearchTableDefine { } @Override public int refreshInterval() { - return 0; + return 2; } @Override public int numberOfShards() { diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/storage/IDNameExchangeTimer.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/storage/IDNameExchangeTimer.java deleted file mode 100644 index c1880a589338f723ac88a0e399a8d6fe8b8e432f..0000000000000000000000000000000000000000 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/storage/IDNameExchangeTimer.java +++ /dev/null @@ -1,41 +0,0 @@ -package org.skywalking.apm.collector.agentstream.worker.storage; - -import java.util.List; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; -import org.skywalking.apm.collector.core.framework.Starter; -import org.skywalking.apm.collector.stream.worker.WorkerException; -import org.skywalking.apm.collector.stream.worker.impl.ExchangeWorker; -import org.skywalking.apm.collector.stream.worker.impl.ExchangeWorkerContainer; -import org.skywalking.apm.collector.stream.worker.impl.FlushAndSwitch; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * @author pengys5 - */ -public class IDNameExchangeTimer implements Starter { - - private final Logger logger = LoggerFactory.getLogger(IDNameExchangeTimer.class); - - public void start() { - logger.info("id and name exchange timer start"); - //TODO timer value config -// final long timeInterval = EsConfig.Es.Persistence.Timer.VALUE * 1000; - final long timeInterval = 3; - - Executors.newSingleThreadScheduledExecutor().schedule(() -> exchangeLastData(), timeInterval, TimeUnit.SECONDS); - } - - private void exchangeLastData() { - List workers = ExchangeWorkerContainer.INSTANCE.getExchangeWorkers(); - workers.forEach((ExchangeWorker worker) -> { - try { - worker.allocateJob(new FlushAndSwitch()); - worker.exchangeLastData(); - } catch (WorkerException e) { - logger.error(e.getMessage(), e); - } - }); - } -} diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/storage/PersistenceTimer.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/storage/PersistenceTimer.java index ac02fee13b56003932bbc99da09db64d4864d912..a55f6e28576c190b798f43cedd9168c8056ec81e 100644 --- a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/storage/PersistenceTimer.java +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/storage/PersistenceTimer.java @@ -26,23 +26,31 @@ public class PersistenceTimer implements Starter { //TODO timer value config // final long timeInterval = EsConfig.Es.Persistence.Timer.VALUE * 1000; final long timeInterval = 3; - Executors.newSingleThreadScheduledExecutor().schedule(() -> extractDataAndSave(), timeInterval, TimeUnit.SECONDS); + Executors.newSingleThreadScheduledExecutor().scheduleAtFixedRate(() -> extractDataAndSave(), 1, timeInterval, TimeUnit.SECONDS); } private void extractDataAndSave() { - List workers = PersistenceWorkerContainer.INSTANCE.getPersistenceWorkers(); - List batchAllCollection = new ArrayList<>(); - workers.forEach((PersistenceWorker worker) -> { - try { - worker.allocateJob(new FlushAndSwitch()); - List batchCollection = worker.buildBatchCollection(); - batchAllCollection.addAll(batchCollection); - } catch (WorkerException e) { - logger.error(e.getMessage(), e); - } - }); + try { + List workers = PersistenceWorkerContainer.INSTANCE.getPersistenceWorkers(); + List batchAllCollection = new ArrayList<>(); + workers.forEach((PersistenceWorker worker) -> { + logger.debug("extract {} worker data and save", worker.getRole().roleName()); + try { + worker.allocateJob(new FlushAndSwitch()); + List batchCollection = worker.buildBatchCollection(); + logger.debug("extract {} worker data size: {}", worker.getRole().roleName(), batchCollection.size()); + batchAllCollection.addAll(batchCollection); + } catch (WorkerException e) { + logger.error(e.getMessage(), e); + } + }); - IBatchDAO dao = (IBatchDAO)DAOContainer.INSTANCE.get(IBatchDAO.class.getName()); - dao.batchPersistence(batchAllCollection); + IBatchDAO dao = (IBatchDAO)DAOContainer.INSTANCE.get(IBatchDAO.class.getName()); + dao.batchPersistence(batchAllCollection); + } catch (Throwable e) { + logger.error(e.getMessage(), e); + } finally { + logger.debug("persistence data save finish"); + } } } diff --git a/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/util/ExchangeMarkUtils.java b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/util/ExchangeMarkUtils.java new file mode 100644 index 0000000000000000000000000000000000000000..84faf171eff50f600e820ac17e1ecee228674429 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/main/java/org/skywalking/apm/collector/agentstream/worker/util/ExchangeMarkUtils.java @@ -0,0 +1,14 @@ +package org.skywalking.apm.collector.agentstream.worker.util; + +/** + * @author pengys5 + */ +public enum ExchangeMarkUtils { + INSTANCE; + + private static final String MARK_TAG = "M"; + + public String buildMarkedID(int id) { + return MARK_TAG + id; + } +} diff --git a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/data.define b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/data.define deleted file mode 100644 index c8abe9d72dc8d0bcd732f462f9cc06fa0ec161cf..0000000000000000000000000000000000000000 --- a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/data.define +++ /dev/null @@ -1,4 +0,0 @@ -org.skywalking.apm.collector.agentstream.worker.node.component.define.NodeComponentDataDefine -org.skywalking.apm.collector.agentstream.worker.register.application.ApplicationDataDefine -org.skywalking.apm.collector.agentstream.worker.register.instance.InstanceDataDefine -org.skywalking.apm.collector.agentstream.worker.register.servicename.ServiceNameDataDefine \ No newline at end of file diff --git a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/es_dao.define b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/es_dao.define index dd2e4e545ff219815cd6622b9d24351208e366bc..63656c1069fc7ac60bd5afa1cfbd9701fba513a0 100644 --- a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/es_dao.define +++ b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/es_dao.define @@ -7,4 +7,6 @@ org.skywalking.apm.collector.agentstream.worker.noderef.reference.dao.NodeRefere org.skywalking.apm.collector.agentstream.worker.segment.origin.dao.SegmentEsDAO org.skywalking.apm.collector.agentstream.worker.noderef.summary.dao.NodeRefSumEsDAO org.skywalking.apm.collector.agentstream.worker.segment.cost.dao.SegmentCostEsDAO -org.skywalking.apm.collector.agentstream.worker.global.dao.GlobalTraceEsDAO \ No newline at end of file +org.skywalking.apm.collector.agentstream.worker.global.dao.GlobalTraceEsDAO +org.skywalking.apm.collector.agentstream.worker.service.entry.dao.ServiceEntryEsDAO +org.skywalking.apm.collector.agentstream.worker.serviceref.reference.dao.ServiceRefEsDAO \ No newline at end of file diff --git a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/h2_dao.define b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/h2_dao.define index da2352e3744f298a037380a89062cfc54d67818b..3f18e0daac9674a5da78a7cb279dc514f45be581 100644 --- a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/h2_dao.define +++ b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/h2_dao.define @@ -7,4 +7,6 @@ org.skywalking.apm.collector.agentstream.worker.noderef.reference.dao.NodeRefere org.skywalking.apm.collector.agentstream.worker.segment.origin.dao.SegmentH2DAO org.skywalking.apm.collector.agentstream.worker.noderef.summary.dao.NodeRefSumH2DAO org.skywalking.apm.collector.agentstream.worker.segment.cost.dao.SegmentCostH2DAO -org.skywalking.apm.collector.agentstream.worker.global.dao.GlobalTraceH2DAO \ No newline at end of file +org.skywalking.apm.collector.agentstream.worker.global.dao.GlobalTraceH2DAO +org.skywalking.apm.collector.agentstream.worker.service.entry.dao.ServiceEntryH2DAO +org.skywalking.apm.collector.agentstream.worker.serviceref.reference.dao.ServiceRefH2DAO \ No newline at end of file diff --git a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/local_async_worker_provider.define b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/local_worker_provider.define similarity index 74% rename from apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/local_async_worker_provider.define rename to apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/local_worker_provider.define index a1744eee42c37870ee31e6257f5a46fc97d76fae..5ce5803b78c171d192e45163c995842dadeb1fb2 100644 --- a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/local_async_worker_provider.define +++ b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/local_worker_provider.define @@ -2,7 +2,6 @@ org.skywalking.apm.collector.agentstream.worker.node.component.NodeComponentAggr org.skywalking.apm.collector.agentstream.worker.node.component.NodeComponentPersistenceWorker$Factory org.skywalking.apm.collector.agentstream.worker.node.mapping.NodeMappingAggregationWorker$Factory -org.skywalking.apm.collector.agentstream.worker.node.mapping.NodeComponentExchangeWorker$Factory org.skywalking.apm.collector.agentstream.worker.node.mapping.NodeMappingPersistenceWorker$Factory org.skywalking.apm.collector.agentstream.worker.noderef.reference.NodeRefAggregationWorker$Factory @@ -11,6 +10,12 @@ org.skywalking.apm.collector.agentstream.worker.noderef.reference.NodeRefPersist org.skywalking.apm.collector.agentstream.worker.noderef.summary.NodeRefSumAggregationWorker$Factory org.skywalking.apm.collector.agentstream.worker.noderef.summary.NodeRefSumPersistenceWorker$Factory +org.skywalking.apm.collector.agentstream.worker.service.entry.ServiceEntryAggregationWorker$Factory +org.skywalking.apm.collector.agentstream.worker.service.entry.ServiceEntryPersistenceWorker$Factory + +org.skywalking.apm.collector.agentstream.worker.serviceref.reference.ServiceRefAggregationWorker$Factory +org.skywalking.apm.collector.agentstream.worker.serviceref.reference.ServiceRefPersistenceWorker$Factory + org.skywalking.apm.collector.agentstream.worker.segment.origin.SegmentPersistenceWorker$Factory org.skywalking.apm.collector.agentstream.worker.segment.cost.SegmentCostPersistenceWorker$Factory org.skywalking.apm.collector.agentstream.worker.global.GlobalTracePersistenceWorker$Factory diff --git a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/remote_worker_provider.define b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/remote_worker_provider.define index b82aef6bd0334dc72dfc7b51a456a2d22c842861..5d59347a7525915c7b7d30aa18db7f4cdc94d597 100644 --- a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/remote_worker_provider.define +++ b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/remote_worker_provider.define @@ -5,4 +5,7 @@ org.skywalking.apm.collector.agentstream.worker.register.servicename.ServiceName org.skywalking.apm.collector.agentstream.worker.node.component.NodeComponentRemoteWorker$Factory org.skywalking.apm.collector.agentstream.worker.node.mapping.NodeMappingRemoteWorker$Factory org.skywalking.apm.collector.agentstream.worker.noderef.reference.NodeRefRemoteWorker$Factory -org.skywalking.apm.collector.agentstream.worker.noderef.summary.NodeRefSumRemoteWorker$Factory \ No newline at end of file +org.skywalking.apm.collector.agentstream.worker.noderef.summary.NodeRefSumRemoteWorker$Factory + +org.skywalking.apm.collector.agentstream.worker.service.entry.ServiceEntryRemoteWorker$Factory +org.skywalking.apm.collector.agentstream.worker.serviceref.reference.ServiceRefRemoteWorker$Factory \ No newline at end of file diff --git a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/storage.define b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/storage.define index d046f461383e29ac276fde3badb6dce8d7e9c83d..c462b86ecdb29f8f9ba6af9b9bf69511b61bd3b2 100644 --- a/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/storage.define +++ b/apm-collector/apm-collector-agentstream/src/main/resources/META-INF/defines/storage.define @@ -26,4 +26,10 @@ org.skywalking.apm.collector.agentstream.worker.segment.cost.define.SegmentCostE org.skywalking.apm.collector.agentstream.worker.segment.cost.define.SegmentCostH2TableDefine org.skywalking.apm.collector.agentstream.worker.global.define.GlobalTraceEsTableDefine -org.skywalking.apm.collector.agentstream.worker.global.define.GlobalTraceH2TableDefine \ No newline at end of file +org.skywalking.apm.collector.agentstream.worker.global.define.GlobalTraceH2TableDefine + +org.skywalking.apm.collector.agentstream.worker.service.entry.define.ServiceEntryEsTableDefine +org.skywalking.apm.collector.agentstream.worker.service.entry.define.ServiceEntryH2TableDefine + +org.skywalking.apm.collector.agentstream.worker.serviceref.reference.define.ServiceRefEsTableDefine +org.skywalking.apm.collector.agentstream.worker.serviceref.reference.define.ServiceRefH2TableDefine \ No newline at end of file diff --git a/apm-collector/apm-collector-agentstream/src/test/java/org/skywalking/apm/collector/agentstream/HttpClientTools.java b/apm-collector/apm-collector-agentstream/src/test/java/org/skywalking/apm/collector/agentstream/HttpClientTools.java new file mode 100644 index 0000000000000000000000000000000000000000..ccaf9184c345a8f6326e6446af52223b955c4691 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/test/java/org/skywalking/apm/collector/agentstream/HttpClientTools.java @@ -0,0 +1,77 @@ +package org.skywalking.apm.collector.agentstream; + +import java.io.IOException; +import java.net.URI; +import java.util.List; +import org.apache.http.Consts; +import org.apache.http.HttpEntity; +import org.apache.http.NameValuePair; +import org.apache.http.client.entity.UrlEncodedFormEntity; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.util.EntityUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public enum HttpClientTools { + INSTANCE; + + private final Logger logger = LoggerFactory.getLogger(HttpClientTools.class); + + public String get(String url, List params) throws IOException { + CloseableHttpClient httpClient = HttpClients.createDefault(); + try { + HttpGet httpget = new HttpGet(url); + String paramStr = EntityUtils.toString(new UrlEncodedFormEntity(params)); + httpget.setURI(new URI(httpget.getURI().toString() + "?" + paramStr)); + logger.debug("executing get request {}", httpget.getURI()); + + try (CloseableHttpResponse response = httpClient.execute(httpget)) { + HttpEntity entity = response.getEntity(); + if (entity != null) { + return EntityUtils.toString(entity); + } + } + } catch (Exception e) { + logger.error(e.getMessage(), e); + } finally { + try { + httpClient.close(); + } catch (IOException e) { + logger.error(e.getMessage(), e); + } + } + return null; + } + + public String post(String url, String data) throws IOException { + CloseableHttpClient httpClient = HttpClients.createDefault(); + try { + HttpPost httppost = new HttpPost(url); + httppost.setEntity(new StringEntity(data, Consts.UTF_8)); + logger.debug("executing post request {}", httppost.getURI()); + try (CloseableHttpResponse response = httpClient.execute(httppost)) { + HttpEntity entity = response.getEntity(); + if (entity != null) { + return EntityUtils.toString(entity); + } + } + } catch (Exception e) { + logger.error(e.getMessage(), e); + } finally { + try { + httpClient.close(); + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + } + return null; + } +} diff --git a/apm-collector/apm-collector-agentstream/src/test/java/org/skywalking/apm/collector/agentstream/grpc/handler/TraceSegmentServiceHandlerTestCase.java b/apm-collector/apm-collector-agentstream/src/test/java/org/skywalking/apm/collector/agentstream/grpc/handler/TraceSegmentServiceHandlerTestCase.java index fc988f5215624483418524abb7102cf12b6e712d..3949d8a15a4c1b3fb4c8b00f03dad2368cf8a6e6 100644 --- a/apm-collector/apm-collector-agentstream/src/test/java/org/skywalking/apm/collector/agentstream/grpc/handler/TraceSegmentServiceHandlerTestCase.java +++ b/apm-collector/apm-collector-agentstream/src/test/java/org/skywalking/apm/collector/agentstream/grpc/handler/TraceSegmentServiceHandlerTestCase.java @@ -109,7 +109,7 @@ public class TraceSegmentServiceHandlerTestCase { ref_0.setParentServiceName(""); ref_0.setParentSpanId(2); ref_0.setParentTraceSegmentId(UniqueId.newBuilder().addIdParts(100).addIdParts(100).addIdParts(100).build()); - segmentBuilder.addRefs(ref_0); +// segmentBuilder.addRefs(ref_0); builder.setSegment(segmentBuilder.build().toByteString()); } diff --git a/apm-collector/apm-collector-agentstream/src/test/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/TraceSegmentJsonReaderTestCase.java b/apm-collector/apm-collector-agentstream/src/test/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/TraceSegmentJsonReaderTestCase.java new file mode 100644 index 0000000000000000000000000000000000000000..5e42bec96ae14b8f555039911632c481b3d5875b --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/test/java/org/skywalking/apm/collector/agentstream/jetty/handler/reader/TraceSegmentJsonReaderTestCase.java @@ -0,0 +1,28 @@ +package org.skywalking.apm.collector.agentstream.jetty.handler.reader; + +import com.google.gson.JsonElement; +import com.google.gson.stream.JsonReader; +import java.io.IOException; +import java.io.StringReader; +import org.junit.Test; +import org.skywalking.apm.collector.agentstream.mock.JsonFileReader; + +/** + * @author pengys5 + */ +public class TraceSegmentJsonReaderTestCase { + + @Test + public void testRead() throws IOException { + TraceSegmentJsonReader reader = new TraceSegmentJsonReader(); + JsonElement jsonElement = JsonFileReader.INSTANCE.read("json/segment/normal/dubbox-consumer.json"); + System.out.println(jsonElement.toString()); + + JsonReader jsonReader = new JsonReader(new StringReader(jsonElement.toString())); + jsonReader.beginArray(); + while (jsonReader.hasNext()) { + reader.read(jsonReader); + } + jsonReader.endArray(); + } +} diff --git a/apm-collector/apm-collector-agentstream/src/test/java/org/skywalking/apm/collector/agentstream/mock/JsonFileReader.java b/apm-collector/apm-collector-agentstream/src/test/java/org/skywalking/apm/collector/agentstream/mock/JsonFileReader.java new file mode 100644 index 0000000000000000000000000000000000000000..b22c8edfcb71d39cf9f996df2a2662c8f8777abc --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/test/java/org/skywalking/apm/collector/agentstream/mock/JsonFileReader.java @@ -0,0 +1,24 @@ +package org.skywalking.apm.collector.agentstream.mock; + +import com.google.gson.JsonElement; +import com.google.gson.JsonParser; +import java.io.FileNotFoundException; +import java.io.FileReader; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author pengys5 + */ +public enum JsonFileReader { + INSTANCE; + + private final Logger logger = LoggerFactory.getLogger(JsonFileReader.class); + + public JsonElement read(String fileName) throws FileNotFoundException { + String path = this.getClass().getClassLoader().getResource(fileName).getFile(); + logger.debug("path: {}", path); + JsonParser jsonParser = new JsonParser(); + return jsonParser.parse(new FileReader(path)); + } +} diff --git a/apm-collector/apm-collector-agentstream/src/test/java/org/skywalking/apm/collector/agentstream/mock/SegmentPost.java b/apm-collector/apm-collector-agentstream/src/test/java/org/skywalking/apm/collector/agentstream/mock/SegmentPost.java new file mode 100644 index 0000000000000000000000000000000000000000..0a16508ab0e5f09a5ce1dc7006eba19ade496c66 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/test/java/org/skywalking/apm/collector/agentstream/mock/SegmentPost.java @@ -0,0 +1,38 @@ +package org.skywalking.apm.collector.agentstream.mock; + +import com.google.gson.JsonElement; +import java.io.IOException; +import org.skywalking.apm.collector.agentstream.HttpClientTools; +import org.skywalking.apm.collector.agentstream.worker.register.instance.InstanceDataDefine; +import org.skywalking.apm.collector.agentstream.worker.register.instance.dao.InstanceEsDAO; +import org.skywalking.apm.collector.client.elasticsearch.ElasticSearchClient; +import org.skywalking.apm.collector.core.CollectorException; + +/** + * @author pengys5 + */ +public class SegmentPost { + + public static void main(String[] args) throws IOException, InterruptedException, CollectorException { + ElasticSearchClient client = new ElasticSearchClient("CollectorDBCluster", true, "127.0.0.1:9300"); + client.initialize(); + + InstanceEsDAO instanceEsDAO = new InstanceEsDAO(); + instanceEsDAO.setClient(client); + + InstanceDataDefine.Instance consumerInstance = new InstanceDataDefine.Instance("2", 2, "dubbox-consumer", 1501858094526L, 2); + instanceEsDAO.save(consumerInstance); + InstanceDataDefine.Instance providerInstance = new InstanceDataDefine.Instance("3", 3, "dubbox-provider", 1501858094526L, 3); + instanceEsDAO.save(providerInstance); + + JsonElement consumer = JsonFileReader.INSTANCE.read("json/segment/normal/dubbox-consumer.json"); + HttpClientTools.INSTANCE.post("http://localhost:12800/segments", consumer.toString()); + + Thread.sleep(5000); + + JsonElement provider = JsonFileReader.INSTANCE.read("json/segment/normal/dubbox-provider.json"); + HttpClientTools.INSTANCE.post("http://localhost:12800/segments", provider.toString()); + + Thread.sleep(5000); + } +} diff --git a/apm-collector/apm-collector-agentstream/src/test/resources/json/segment/normal/application-register.json b/apm-collector/apm-collector-agentstream/src/test/resources/json/segment/normal/application-register.json new file mode 100644 index 0000000000000000000000000000000000000000..eb829c2cb4c375b6ce163c4751b26b8dac061769 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/test/resources/json/segment/normal/application-register.json @@ -0,0 +1,4 @@ +[ + "dubbox-consumer", + "dubbox-provider" +] \ No newline at end of file diff --git a/apm-collector/apm-collector-agentstream/src/test/resources/json/segment/normal/dubbox-consumer.json b/apm-collector/apm-collector-agentstream/src/test/resources/json/segment/normal/dubbox-consumer.json new file mode 100644 index 0000000000000000000000000000000000000000..57c79c9c31249edd4abea9bdd960e4975452af6e --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/test/resources/json/segment/normal/dubbox-consumer.json @@ -0,0 +1,71 @@ +[ + { + "gt": [ + [ + 230150, + 185809, + 24040000 + ] + ], + "sg": { + "ts": [ + 230150, + 185809, + 24040000 + ], + "ai": 2, + "ii": 2, + "rs": [], + "ss": [ + { + "si": 1, + "tv": 1, + "lv": 1, + "ps": 0, + "st": 1501858094526, + "et": 1501858097004, + "ci": 3, + "cn": "", + "oi": 0, + "on": "org.skywaking.apm.testcase.dubbo.services.GreetService.doBusiness()", + "pi": 0, + "pn": "172.25.0.4:20880", + "ie": false, + "to": [ + { + "k": "url", + "v": "rest://172.25.0.4:20880/org.skywaking.apm.testcase.dubbo.services.GreetService.doBusiness()" + } + ], + "lo": [] + }, + { + "si": 0, + "tv": 0, + "lv": 2, + "ps": -1, + "st": 1501858092409, + "et": 1501858097033, + "ci": 1, + "cn": "", + "oi": 0, + "on": "/dubbox-case/case/dubbox-rest", + "pi": 0, + "pn": "", + "ie": false, + "to": [ + { + "k": "url", + "v": "http://localhost:18080/dubbox-case/case/dubbox-rest" + }, + { + "k": "http.method", + "v": "GET" + } + ], + "lo": [] + } + ] + } + } +] \ No newline at end of file diff --git a/apm-collector/apm-collector-agentstream/src/test/resources/json/segment/normal/dubbox-provider.json b/apm-collector/apm-collector-agentstream/src/test/resources/json/segment/normal/dubbox-provider.json new file mode 100644 index 0000000000000000000000000000000000000000..19b69ecaf52a276ba5ee052a1504410e12d51a2a --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/test/resources/json/segment/normal/dubbox-provider.json @@ -0,0 +1,67 @@ +[ + { + "gt": [ + [ + 230150, + 185809, + 24040000 + ] + ], + "sg": { + "ts": [ + 137150, + 185809, + 48780000 + ], + "ai": 3, + "ii": 3, + "rs": [ + { + "ts": [ + 230150, + 185809, + 24040000 + ], + "ai": 2, + "si": 1, + "vi": 0, + "vn": "org.skywaking.apm.testcase.dubbo.services.GreetService.doBusiness()", + "ni": 0, + "nn": "172.25.0.4:20880", + "ea": 2, + "ei": 0, + "en": "/dubbox-case/case/dubbox-rest", + "rn": 0 + } + ], + "ss": [ + { + "si": 0, + "tv": 0, + "lv": 1, + "ps": -1, + "st": 1501858094883, + "et": 1501858096950, + "ci": 3, + "cn": "", + "oi": 0, + "on": "org.skywaking.apm.testcase.dubbo.services.GreetService.doBusiness()", + "pi": 0, + "pn": "", + "ie": false, + "to": [ + { + "k": "url", + "v": "rest://172.25.0.4:20880/org.skywaking.apm.testcase.dubbo.services.GreetService.doBusiness()" + }, + { + "k": "http.method", + "v": "GET" + } + ], + "lo": [] + } + ] + } + } +] \ No newline at end of file diff --git a/apm-collector/apm-collector-agentstream/src/test/resources/json/segment/normal/instance-register.json b/apm-collector/apm-collector-agentstream/src/test/resources/json/segment/normal/instance-register.json new file mode 100644 index 0000000000000000000000000000000000000000..f29ab121d7f2594f53d4c2291ee55d4bdd971e39 --- /dev/null +++ b/apm-collector/apm-collector-agentstream/src/test/resources/json/segment/normal/instance-register.json @@ -0,0 +1,5 @@ +{ + "ai": 0, + "au": "dubbox-consumer", + "rt": 1501858094526 +} \ No newline at end of file diff --git a/apm-collector/apm-collector-client/src/main/java/org/skywalking/apm/collector/client/elasticsearch/ElasticSearchClient.java b/apm-collector/apm-collector-client/src/main/java/org/skywalking/apm/collector/client/elasticsearch/ElasticSearchClient.java index 58cf145ef78e1914e334cf8a4bcb5a39781c5cb1..c2362ff5e5477ca87cdd6db22026c889ba453622 100644 --- a/apm-collector/apm-collector-client/src/main/java/org/skywalking/apm/collector/client/elasticsearch/ElasticSearchClient.java +++ b/apm-collector/apm-collector-client/src/main/java/org/skywalking/apm/collector/client/elasticsearch/ElasticSearchClient.java @@ -13,6 +13,7 @@ import org.elasticsearch.action.get.GetRequestBuilder; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.update.UpdateRequest; +import org.elasticsearch.action.update.UpdateRequestBuilder; import org.elasticsearch.client.IndicesAdminClient; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.InetSocketTransportAddress; @@ -114,6 +115,10 @@ public class ElasticSearchClient implements Client { return client.prepareIndex(indexName, "type", id); } + public UpdateRequestBuilder prepareUpdate(String indexName, String id) { + return client.prepareUpdate(indexName, "type", id); + } + public GetRequestBuilder prepareGet(String indexName, String id) { return client.prepareGet(indexName, "type", id); } diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/cluster/ClusterDefinitionFile.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/cluster/ClusterDefinitionFile.java deleted file mode 100644 index aa1bb7b5428d3771a9e6f255b941e9aae5f30f08..0000000000000000000000000000000000000000 --- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/cluster/ClusterDefinitionFile.java +++ /dev/null @@ -1,13 +0,0 @@ -package org.skywalking.apm.collector.core.cluster; - -import org.skywalking.apm.collector.core.framework.DefinitionFile; - -/** - * @author pengys5 - */ -public class ClusterDefinitionFile extends DefinitionFile { - - @Override protected String fileName() { - return "cluster-configuration.define"; - } -} diff --git a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/storage/StorageInstaller.java b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/storage/StorageInstaller.java index 0ae1857945fa876f3597f73ce6d411cd836423db..3cd7559af148096959bae9ddc6f056ffb790c1bc 100644 --- a/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/storage/StorageInstaller.java +++ b/apm-collector/apm-collector-core/src/main/java/org/skywalking/apm/collector/core/storage/StorageInstaller.java @@ -23,8 +23,11 @@ public abstract class StorageInstaller { if (!isExists(client, tableDefine)) { logger.info("table: {} not exists", tableDefine.getName()); tableDefine.initialize(); - createTable(client, tableDefine); + } else { + logger.info("table: {} exists", tableDefine.getName()); + deleteTable(client, tableDefine); } + createTable(client, tableDefine); } } catch (DefineException e) { throw new StorageInstallException(e.getMessage(), e); @@ -35,7 +38,7 @@ public abstract class StorageInstaller { protected abstract boolean isExists(Client client, TableDefine tableDefine) throws StorageException; - protected abstract boolean deleteIndex(Client client, TableDefine tableDefine) throws StorageException; + protected abstract boolean deleteTable(Client client, TableDefine tableDefine) throws StorageException; protected abstract boolean createTable(Client client, TableDefine tableDefine) throws StorageException; } diff --git a/apm-collector/apm-collector-server/src/main/java/org/skywalking/apm/collector/server/jetty/JettyHandler.java b/apm-collector/apm-collector-server/src/main/java/org/skywalking/apm/collector/server/jetty/JettyHandler.java index 7b7c39bc595203430cccffee51bc197d3d27d931..22e21afbac3da4c5b7dff6fa38081ccec941cb5f 100644 --- a/apm-collector/apm-collector-server/src/main/java/org/skywalking/apm/collector/server/jetty/JettyHandler.java +++ b/apm-collector/apm-collector-server/src/main/java/org/skywalking/apm/collector/server/jetty/JettyHandler.java @@ -13,6 +13,7 @@ import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import org.skywalking.apm.collector.core.framework.Handler; +import org.skywalking.apm.collector.core.util.ObjectUtils; /** * @author pengys5 @@ -34,9 +35,15 @@ public abstract class JettyHandler extends HttpServlet implements Handler { @Override protected final void doPost(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { - super.doPost(req, resp); + try { + reply(resp, doPost(req)); + } catch (ArgumentsParseException e) { + replyError(resp, e.getMessage(), HttpServletResponse.SC_BAD_REQUEST); + } } + protected abstract JsonElement doPost(HttpServletRequest req) throws ArgumentsParseException; + @Override protected final void doHead(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { super.doHead(req, resp); @@ -129,7 +136,9 @@ public abstract class JettyHandler extends HttpServlet implements Handler { response.setStatus(HttpServletResponse.SC_OK); PrintWriter out = response.getWriter(); - out.print(resJson); + if (ObjectUtils.isNotEmpty(resJson)) { + out.print(resJson); + } out.flush(); out.close(); } @@ -138,10 +147,10 @@ public abstract class JettyHandler extends HttpServlet implements Handler { response.setContentType("text/plain"); response.setCharacterEncoding("utf-8"); response.setStatus(status); + response.setHeader("error-message", errorMessage); PrintWriter out = response.getWriter(); - out.print(errorMessage); out.flush(); out.close(); } -} +} \ No newline at end of file diff --git a/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/elasticsearch/dao/BatchEsDAO.java b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/elasticsearch/dao/BatchEsDAO.java index 74c06e634bc429af233cf8ef6cea49c4c4fac176..5a222145a115b9657956533de6ee7251c99b6545 100644 --- a/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/elasticsearch/dao/BatchEsDAO.java +++ b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/elasticsearch/dao/BatchEsDAO.java @@ -4,6 +4,7 @@ import java.util.List; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.action.update.UpdateRequestBuilder; import org.skywalking.apm.collector.core.util.CollectionUtils; import org.skywalking.apm.collector.storage.dao.IBatchDAO; import org.slf4j.Logger; @@ -19,11 +20,16 @@ public class BatchEsDAO extends EsDAO implements IBatchDAO { @Override public void batchPersistence(List batchCollection) { BulkRequestBuilder bulkRequest = getClient().prepareBulk(); - logger.info("bulk data size: {}", batchCollection.size()); + logger.debug("bulk data size: {}", batchCollection.size()); if (CollectionUtils.isNotEmpty(batchCollection)) { for (int i = 0; i < batchCollection.size(); i++) { - IndexRequestBuilder builder = (IndexRequestBuilder)batchCollection.get(i); - bulkRequest.add(builder); + Object builder = batchCollection.get(i); + if (builder instanceof IndexRequestBuilder) { + bulkRequest.add((IndexRequestBuilder)builder); + } + if (builder instanceof UpdateRequestBuilder) { + bulkRequest.add((UpdateRequestBuilder)builder); + } } BulkResponse bulkResponse = bulkRequest.execute().actionGet(); diff --git a/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/elasticsearch/define/ElasticSearchStorageInstaller.java b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/elasticsearch/define/ElasticSearchStorageInstaller.java index 25dd33d61b4ba4e7882dd2a3426f77c366e249f9..26c618f13ed3e7a49313f91d923845edab46d09c 100644 --- a/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/elasticsearch/define/ElasticSearchStorageInstaller.java +++ b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/elasticsearch/define/ElasticSearchStorageInstaller.java @@ -81,7 +81,7 @@ public class ElasticSearchStorageInstaller extends StorageInstaller { return mappingBuilder; } - @Override protected boolean deleteIndex(Client client, TableDefine tableDefine) { + @Override protected boolean deleteTable(Client client, TableDefine tableDefine) { ElasticSearchClient esClient = (ElasticSearchClient)client; try { return esClient.deleteIndex(tableDefine.getName()); diff --git a/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/h2/define/H2StorageInstaller.java b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/h2/define/H2StorageInstaller.java index d27a8ca740ac4a5bd7338eca1c33c0ba687b7ca0..3244808a5422b50f347dbbe215915d430ce52f68 100644 --- a/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/h2/define/H2StorageInstaller.java +++ b/apm-collector/apm-collector-storage/src/main/java/org/skywalking/apm/collector/storage/h2/define/H2StorageInstaller.java @@ -27,7 +27,7 @@ public class H2StorageInstaller extends StorageInstaller { return false; } - @Override protected boolean deleteIndex(Client client, TableDefine tableDefine) throws StorageException { + @Override protected boolean deleteTable(Client client, TableDefine tableDefine) throws StorageException { return false; } diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/StreamModuleInstaller.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/StreamModuleInstaller.java index 908eecbb0afe9dcdcd0aa32b616e02082c7f5cea..a1f22c2fa672e66bd5e82797246c49355c2b7545 100644 --- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/StreamModuleInstaller.java +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/StreamModuleInstaller.java @@ -16,8 +16,6 @@ import org.skywalking.apm.collector.stream.worker.ClusterWorkerContext; import org.skywalking.apm.collector.stream.worker.LocalAsyncWorkerProviderDefineLoader; import org.skywalking.apm.collector.stream.worker.ProviderNotFoundException; import org.skywalking.apm.collector.stream.worker.RemoteWorkerProviderDefineLoader; -import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; -import org.skywalking.apm.collector.stream.worker.impl.data.DataDefineLoader; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -34,10 +32,6 @@ public class StreamModuleInstaller implements ModuleInstaller { StreamModuleContext context = new StreamModuleContext(StreamModuleGroupDefine.GROUP_NAME); CollectorContextHelper.INSTANCE.putContext(context); - DataDefineLoader dataDefineLoader = new DataDefineLoader(); - Map dataDefineMap = dataDefineLoader.load(); - context.putAllDataDefine(dataDefineMap); - initializeWorker(context); logger.info("could not configure cluster module, use the default"); diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalAsyncWorker.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalAsyncWorker.java index e43561bee60047519668786e124ac91805e3f3d4..896a2ad510019b41bdff66f2d0e081c6c5d87f0f 100644 --- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalAsyncWorker.java +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalAsyncWorker.java @@ -9,7 +9,9 @@ import org.skywalking.apm.collector.core.queue.QueueExecutor; * @author pengys5 * @since v3.0-2017 */ -public abstract class AbstractLocalAsyncWorker extends AbstractLocalWorker implements QueueExecutor { +public abstract class AbstractLocalAsyncWorker extends AbstractWorker implements QueueExecutor { + + private LocalAsyncWorkerRef workerRef; /** * Construct an AbstractLocalAsyncWorker with the worker role and context. @@ -32,6 +34,14 @@ public abstract class AbstractLocalAsyncWorker extends AbstractLocalWorker imple public void preStart() throws ProviderNotFoundException { } + @Override protected final LocalAsyncWorkerRef getSelf() { + return workerRef; + } + + @Override protected final void putSelfRef(LocalAsyncWorkerRef workerRef) { + this.workerRef = workerRef; + } + /** * Receive message * diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalAsyncWorkerProvider.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalAsyncWorkerProvider.java index 7f00cb13cfd0399a3691557d5a2a1bbc11db423d..fe702b1b71715b81f328dcafecd3ee1a195dccb2 100644 --- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalAsyncWorkerProvider.java +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalAsyncWorkerProvider.java @@ -6,15 +6,13 @@ import org.skywalking.apm.collector.core.queue.QueueEventHandler; import org.skywalking.apm.collector.core.queue.QueueExecutor; import org.skywalking.apm.collector.queue.QueueModuleContext; import org.skywalking.apm.collector.queue.QueueModuleGroupDefine; -import org.skywalking.apm.collector.stream.worker.impl.ExchangeWorker; -import org.skywalking.apm.collector.stream.worker.impl.ExchangeWorkerContainer; import org.skywalking.apm.collector.stream.worker.impl.PersistenceWorker; import org.skywalking.apm.collector.stream.worker.impl.PersistenceWorkerContainer; /** * @author pengys5 */ -public abstract class AbstractLocalAsyncWorkerProvider extends AbstractLocalWorkerProvider { +public abstract class AbstractLocalAsyncWorkerProvider extends AbstractWorkerProvider { public abstract int queueSize(); @@ -24,8 +22,6 @@ public abstract class AbstractLocalAsyncWorkerProviderAbstractLocalSyncWorker defines workers who receive data from jvm inside call and response in real - * time. - * - * @author pengys5 - * @since v3.0-2017 - */ -public abstract class AbstractLocalSyncWorker extends AbstractLocalWorker { - public AbstractLocalSyncWorker(Role role, ClusterWorkerContext clusterContext) { - super(role, clusterContext); - } - - /** - * Called by the worker reference to execute the worker service. - * - * @param request {@link Object} is an input parameter - * @param response {@link Object} is an output parameter - */ - final public void allocateJob(Object request, Object response) throws WorkerInvokeException { - try { - onWork(request, response); - } catch (WorkerException e) { - throw new WorkerInvokeException(e.getMessage(), e.getCause()); - } - } - - /** - * Override this method to implement business logic. - * - * @param request {@link Object} is a in parameter - * @param response {@link Object} is a out parameter - */ - protected abstract void onWork(Object request, Object response) throws WorkerException; - - /** - * Prepare methods before this work starts to work. - *

Usually, create or find the workers reference should be call. - * - * @throws ProviderNotFoundException - */ - @Override - public void preStart() throws ProviderNotFoundException { - } -} diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalSyncWorkerProvider.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalSyncWorkerProvider.java deleted file mode 100644 index d3a542a61e75cb67c6c425b3c6bfe2373185da5f..0000000000000000000000000000000000000000 --- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalSyncWorkerProvider.java +++ /dev/null @@ -1,15 +0,0 @@ -package org.skywalking.apm.collector.stream.worker; - -/** - * @author pengys5 - */ -public abstract class AbstractLocalSyncWorkerProvider extends AbstractLocalWorkerProvider { - - @Override final public WorkerRef create() throws ProviderNotFoundException { - T localSyncWorker = workerInstance(getClusterContext()); - localSyncWorker.preStart(); - - LocalSyncWorkerRef workerRef = new LocalSyncWorkerRef(role(), localSyncWorker); - return workerRef; - } -} diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalWorker.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalWorker.java deleted file mode 100644 index c92a23fc7afebdef9740e60f39c6d6d9d82b28a9..0000000000000000000000000000000000000000 --- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalWorker.java +++ /dev/null @@ -1,10 +0,0 @@ -package org.skywalking.apm.collector.stream.worker; - -/** - * @author pengys5 - */ -public abstract class AbstractLocalWorker extends AbstractWorker { - public AbstractLocalWorker(Role role, ClusterWorkerContext clusterContext) { - super(role, clusterContext); - } -} diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalWorkerProvider.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalWorkerProvider.java deleted file mode 100644 index 8cc96b71f52c5516aba96cb78b5bc976a3dcd83a..0000000000000000000000000000000000000000 --- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractLocalWorkerProvider.java +++ /dev/null @@ -1,7 +0,0 @@ -package org.skywalking.apm.collector.stream.worker; - -/** - * @author pengys5 - */ -public abstract class AbstractLocalWorkerProvider extends AbstractWorkerProvider { -} diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractRemoteWorker.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractRemoteWorker.java index 8dab25277278174e5f628122d49f10b09d1528ee..bba64efd2edfa3e55f31f2d0df61fffc496ed22f 100644 --- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractRemoteWorker.java +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractRemoteWorker.java @@ -9,7 +9,9 @@ package org.skywalking.apm.collector.stream.worker; * @author pengys5 * @since v3.0-2017 */ -public abstract class AbstractRemoteWorker extends AbstractWorker { +public abstract class AbstractRemoteWorker extends AbstractWorker { + + private RemoteWorkerRef workerRef; /** * Construct an AbstractRemoteWorker with the worker role and context. @@ -35,4 +37,12 @@ public abstract class AbstractRemoteWorker extends AbstractWorker { throw new WorkerInvokeException(e.getMessage(), e.getCause()); } } + + @Override protected final RemoteWorkerRef getSelf() { + return workerRef; + } + + @Override protected final void putSelfRef(RemoteWorkerRef workerRef) { + this.workerRef = workerRef; + } } diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractWorker.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractWorker.java index 42d69415a500fe12e931448626534365673d8064..01918b5e40aeef437e4906795bd7d4b1c5b04ee7 100644 --- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractWorker.java +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/AbstractWorker.java @@ -7,7 +7,7 @@ import org.slf4j.LoggerFactory; /** * @author pengys5 */ -public abstract class AbstractWorker implements Executor { +public abstract class AbstractWorker implements Executor { private final Logger logger = LoggerFactory.getLogger(AbstractWorker.class); @@ -45,4 +45,8 @@ public abstract class AbstractWorker implements Executor { final public Role getRole() { return role; } + + protected abstract S getSelf(); + + protected abstract void putSelfRef(S workerRef); } diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/LocalAsyncWorkerProviderDefineLoader.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/LocalAsyncWorkerProviderDefineLoader.java index 27fd2f1df6e304e8ad7ac86800d4fc5a5bd9bc3d..77ed5d4a5716cfa9246b575ea393711a94033620 100644 --- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/LocalAsyncWorkerProviderDefineLoader.java +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/LocalAsyncWorkerProviderDefineLoader.java @@ -17,7 +17,7 @@ public class LocalAsyncWorkerProviderDefineLoader implements Loader load() throws DefineException { List providers = new ArrayList<>(); - LocalAsyncWorkerProviderDefinitionFile definitionFile = new LocalAsyncWorkerProviderDefinitionFile(); + LocalWorkerProviderDefinitionFile definitionFile = new LocalWorkerProviderDefinitionFile(); logger.info("local async worker provider definition file name: {}", definitionFile.fileName()); DefinitionLoader definitionLoader = DefinitionLoader.load(AbstractLocalAsyncWorkerProvider.class, definitionFile); diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/LocalSyncWorkerRef.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/LocalSyncWorkerRef.java deleted file mode 100644 index d24d07779775ae6cf0f7fe17534029dd60bd85d8..0000000000000000000000000000000000000000 --- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/LocalSyncWorkerRef.java +++ /dev/null @@ -1,23 +0,0 @@ -package org.skywalking.apm.collector.stream.worker; - -/** - * @author pengys5 - */ -public class LocalSyncWorkerRef extends WorkerRef { - - private AbstractLocalSyncWorker localSyncWorker; - - public LocalSyncWorkerRef(Role role, AbstractLocalSyncWorker localSyncWorker) { - super(role); - this.localSyncWorker = localSyncWorker; - } - - @Override - public void tell(Object message) throws WorkerInvokeException { - localSyncWorker.allocateJob(message, null); - } - - public void ask(Object request, Object response) throws WorkerInvokeException { - localSyncWorker.allocateJob(request, response); - } -} diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/LocalAsyncWorkerProviderDefinitionFile.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/LocalWorkerProviderDefinitionFile.java similarity index 60% rename from apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/LocalAsyncWorkerProviderDefinitionFile.java rename to apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/LocalWorkerProviderDefinitionFile.java index 356cf9ffbe8b4d9bd8e2090925ae5961bec88b82..5dbfa09f7eb4638c46e90d755a6884b63a7ddf9c 100644 --- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/LocalAsyncWorkerProviderDefinitionFile.java +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/LocalWorkerProviderDefinitionFile.java @@ -5,8 +5,8 @@ import org.skywalking.apm.collector.core.framework.DefinitionFile; /** * @author pengys5 */ -public class LocalAsyncWorkerProviderDefinitionFile extends DefinitionFile { +public class LocalWorkerProviderDefinitionFile extends DefinitionFile { @Override protected String fileName() { - return "local_async_worker_provider.define"; + return "local_worker_provider.define"; } } diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/ExchangeWorker.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/ExchangeWorker.java deleted file mode 100644 index 3a2e8a098efc0648d8ab9ba0435ddba6cb76da4a..0000000000000000000000000000000000000000 --- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/ExchangeWorker.java +++ /dev/null @@ -1,74 +0,0 @@ -package org.skywalking.apm.collector.stream.worker.impl; - -import org.skywalking.apm.collector.core.queue.EndOfBatchCommand; -import org.skywalking.apm.collector.stream.worker.AbstractLocalAsyncWorker; -import org.skywalking.apm.collector.stream.worker.ClusterWorkerContext; -import org.skywalking.apm.collector.stream.worker.ProviderNotFoundException; -import org.skywalking.apm.collector.stream.worker.Role; -import org.skywalking.apm.collector.stream.worker.WorkerException; -import org.skywalking.apm.collector.stream.worker.impl.data.Data; -import org.skywalking.apm.collector.stream.worker.impl.data.DataCache; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * @author pengys5 - */ -public abstract class ExchangeWorker extends AbstractLocalAsyncWorker { - - private final Logger logger = LoggerFactory.getLogger(ExchangeWorker.class); - - private DataCache dataCache; - - public ExchangeWorker(Role role, ClusterWorkerContext clusterContext) { - super(role, clusterContext); - dataCache = new DataCache(); - } - - @Override public void preStart() throws ProviderNotFoundException { - super.preStart(); - } - - @Override protected final void onWork(Object message) throws WorkerException { - if (message instanceof FlushAndSwitch) { - if (dataCache.trySwitchPointer()) { - dataCache.switchPointer(); - } - } else if (message instanceof EndOfBatchCommand) { - } else { - if (dataCache.currentCollectionSize() <= 1000) { - aggregate(message); - } - } - } - - protected abstract void exchange(Data data); - - public final void exchangeLastData() { - try { - while (dataCache.getLast().isHolding()) { - try { - Thread.sleep(10); - } catch (InterruptedException e) { - logger.warn("thread wake up"); - } - } - dataCache.getLast().asMap().values().forEach(data -> { - exchange(data); - }); - } finally { - dataCache.releaseLast(); - } - } - - protected final void aggregate(Object message) { - Data data = (Data)message; - dataCache.hold(); - if (dataCache.containsKey(data.id())) { - getRole().dataDefine().mergeData(data, dataCache.get(data.id())); - } else { - dataCache.put(data.id(), data); - } - dataCache.release(); - } -} diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/ExchangeWorkerContainer.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/ExchangeWorkerContainer.java deleted file mode 100644 index 38ffea545e174fcd03a2c2c2f2041f1e6da4e9b6..0000000000000000000000000000000000000000 --- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/ExchangeWorkerContainer.java +++ /dev/null @@ -1,21 +0,0 @@ -package org.skywalking.apm.collector.stream.worker.impl; - -import java.util.ArrayList; -import java.util.List; - -/** - * @author pengys5 - */ -public enum ExchangeWorkerContainer { - INSTANCE; - - private List exchangeWorkers = new ArrayList<>(); - - public void addWorker(ExchangeWorker worker) { - exchangeWorkers.add(worker); - } - - public List getExchangeWorkers() { - return exchangeWorkers; - } -} diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/PersistenceWorker.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/PersistenceWorker.java index f0f3bae3b5a235fab0caabdd89447c7bb8c2e50f..878c2c6e09e0408920f666de2f3364ebb73e80a8 100644 --- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/PersistenceWorker.java +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/PersistenceWorker.java @@ -1,13 +1,16 @@ package org.skywalking.apm.collector.stream.worker.impl; +import java.util.ArrayList; import java.util.List; import java.util.Map; import org.skywalking.apm.collector.core.queue.EndOfBatchCommand; +import org.skywalking.apm.collector.core.util.ObjectUtils; import org.skywalking.apm.collector.stream.worker.AbstractLocalAsyncWorker; import org.skywalking.apm.collector.stream.worker.ClusterWorkerContext; import org.skywalking.apm.collector.stream.worker.ProviderNotFoundException; import org.skywalking.apm.collector.stream.worker.Role; import org.skywalking.apm.collector.stream.worker.WorkerException; +import org.skywalking.apm.collector.stream.worker.impl.dao.IPersistenceDAO; import org.skywalking.apm.collector.stream.worker.impl.data.Data; import org.skywalking.apm.collector.stream.worker.impl.data.DataCache; import org.slf4j.Logger; @@ -46,7 +49,7 @@ public abstract class PersistenceWorker extends AbstractLocalAsyncWorker { } } - public List buildBatchCollection() throws WorkerException { + public final List buildBatchCollection() throws WorkerException { List batchCollection; try { while (dataCache.getLast().isHolding()) { @@ -56,6 +59,7 @@ public abstract class PersistenceWorker extends AbstractLocalAsyncWorker { logger.warn("thread wake up"); } } + batchCollection = prepareBatch(dataCache.getLast().asMap()); } finally { dataCache.releaseLast(); @@ -63,7 +67,26 @@ public abstract class PersistenceWorker extends AbstractLocalAsyncWorker { return batchCollection; } - protected abstract List prepareBatch(Map dataMap); + protected final List prepareBatch(Map dataMap) { + List insertBatchCollection = new ArrayList<>(); + List updateBatchCollection = new ArrayList<>(); + dataMap.forEach((id, data) -> { + if (needMergeDBData()) { + Data dbData = persistenceDAO().get(id, getRole().dataDefine()); + if (ObjectUtils.isNotEmpty(dbData)) { + getRole().dataDefine().mergeData(data, dbData); + updateBatchCollection.add(persistenceDAO().prepareBatchUpdate(data)); + } else { + insertBatchCollection.add(persistenceDAO().prepareBatchInsert(data)); + } + } else { + insertBatchCollection.add(persistenceDAO().prepareBatchInsert(data)); + } + }); + + insertBatchCollection.addAll(updateBatchCollection); + return insertBatchCollection; + } private void aggregate(Object message) { dataCache.hold(); @@ -79,4 +102,8 @@ public abstract class PersistenceWorker extends AbstractLocalAsyncWorker { dataCache.release(); } + + protected abstract IPersistenceDAO persistenceDAO(); + + protected abstract boolean needMergeDBData(); } diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/dao/IPersistenceDAO.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/dao/IPersistenceDAO.java new file mode 100644 index 0000000000000000000000000000000000000000..956586265462bb784c5fd41b52e476b5f7c3c544 --- /dev/null +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/dao/IPersistenceDAO.java @@ -0,0 +1,15 @@ +package org.skywalking.apm.collector.stream.worker.impl.dao; + +import org.skywalking.apm.collector.stream.worker.impl.data.Data; +import org.skywalking.apm.collector.stream.worker.impl.data.DataDefine; + +/** + * @author pengys5 + */ +public interface IPersistenceDAO { + Data get(String id, DataDefine dataDefine); + + I prepareBatchInsert(Data data); + + U prepareBatchUpdate(Data data); +} diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/Data.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/Data.java index 5d5e25af88eea8e898c591050b5810fdabb834c2..5a8832ba3b1e6d06ec590fa19dc318054c0aac0d 100644 --- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/Data.java +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/Data.java @@ -8,7 +8,6 @@ import org.skywalking.apm.collector.stream.worker.selector.AbstractHashMessage; * @author pengys5 */ public class Data extends AbstractHashMessage { - private int defineId; private final int stringCapacity; private final int longCapacity; private final int floatCapacity; @@ -22,10 +21,9 @@ public class Data extends AbstractHashMessage { private Boolean[] dataBooleans; private byte[][] dataBytes; - public Data(String id, int defineId, int stringCapacity, int longCapacity, int floatCapacity, int integerCapacity, + public Data(String id, int stringCapacity, int longCapacity, int floatCapacity, int integerCapacity, int booleanCapacity, int byteCapacity) { super(id); - this.defineId = defineId; this.dataStrings = new String[stringCapacity]; this.dataLongs = new Long[longCapacity]; this.dataFloats = new Float[floatCapacity]; @@ -92,10 +90,6 @@ public class Data extends AbstractHashMessage { return dataStrings[0]; } - public int getDefineId() { - return defineId; - } - public RemoteData serialize() { RemoteData.Builder builder = RemoteData.newBuilder(); builder.setIntegerCapacity(integerCapacity); diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/DataDefine.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/DataDefine.java index 254e742d6c9cd814ea5821d2fd8c02bccfd77bf6..ddbd9c2a367580f45b2079f8ff3696d0cf5bc477 100644 --- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/DataDefine.java +++ b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/DataDefine.java @@ -42,14 +42,12 @@ public abstract class DataDefine { attributes[position] = attribute; } - public abstract int defineId(); - protected abstract int initialCapacity(); protected abstract void attributeDefine(); public final Data build(String id) { - return new Data(id, defineId(), stringCapacity, longCapacity, floatCapacity, integerCapacity, booleanCapacity, byteCapacity); + return new Data(id, stringCapacity, longCapacity, floatCapacity, integerCapacity, booleanCapacity, byteCapacity); } public void mergeData(Data newData, Data oldData) { diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/DataDefineLoader.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/DataDefineLoader.java deleted file mode 100644 index 425fcb427a44bfd990891b2a399d029c0420d1f5..0000000000000000000000000000000000000000 --- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/DataDefineLoader.java +++ /dev/null @@ -1,29 +0,0 @@ -package org.skywalking.apm.collector.stream.worker.impl.data; - -import java.util.HashMap; -import java.util.Map; -import org.skywalking.apm.collector.core.framework.DefineException; -import org.skywalking.apm.collector.core.framework.Loader; -import org.skywalking.apm.collector.core.util.DefinitionLoader; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * @author pengys5 - */ -public class DataDefineLoader implements Loader> { - - private final Logger logger = LoggerFactory.getLogger(DataDefineLoader.class); - - @Override public Map load() throws DefineException { - Map dataDefineMap = new HashMap<>(); - - DataDefinitionFile definitionFile = new DataDefinitionFile(); - DefinitionLoader definitionLoader = DefinitionLoader.load(DataDefine.class, definitionFile); - for (DataDefine dataDefine : definitionLoader) { - logger.info("loaded data definition class: {}", dataDefine.getClass().getName()); - dataDefineMap.put(dataDefine.defineId(), dataDefine); - } - return dataDefineMap; - } -} diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/DataDefinitionFile.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/DataDefinitionFile.java deleted file mode 100644 index e7875d8b87b0abcee2b17492107cbd5f59c85b7c..0000000000000000000000000000000000000000 --- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/DataDefinitionFile.java +++ /dev/null @@ -1,13 +0,0 @@ -package org.skywalking.apm.collector.stream.worker.impl.data; - -import org.skywalking.apm.collector.core.framework.DefinitionFile; - -/** - * @author pengys5 - */ -public class DataDefinitionFile extends DefinitionFile { - - @Override protected String fileName() { - return "data.define"; - } -} diff --git a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/Exchange.java b/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/Exchange.java deleted file mode 100644 index 3d37d4500f760980dbcbf4b887f6d1c19db3d176..0000000000000000000000000000000000000000 --- a/apm-collector/apm-collector-stream/src/main/java/org/skywalking/apm/collector/stream/worker/impl/data/Exchange.java +++ /dev/null @@ -1,24 +0,0 @@ -package org.skywalking.apm.collector.stream.worker.impl.data; - -/** - * @author pengys5 - */ -public abstract class Exchange { - private int times; - - public Exchange(int times) { - this.times = times; - } - - public void increase() { - times++; - } - - public int getTimes() { - return times; - } - - public void setTimes(int times) { - this.times = times; - } -} diff --git a/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/dao/SegmentCostEsDAO.java b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/dao/SegmentCostEsDAO.java index c06df8079e57a70f414b732a400e09b4ff755ea1..4e1d05699431493984a857efcf4e5f543d516339 100644 --- a/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/dao/SegmentCostEsDAO.java +++ b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/dao/SegmentCostEsDAO.java @@ -45,7 +45,7 @@ public class SegmentCostEsDAO extends EsDAO implements ISegmentCostDAO { boolQueryBuilder.must().add(rangeQueryBuilder); } if (!StringUtils.isEmpty(operationName)) { - mustQueryList.add(QueryBuilders.matchQuery(SegmentCostTable.COLUMN_OPERATION_NAME, operationName)); + mustQueryList.add(QueryBuilders.matchQuery(SegmentCostTable.COLUMN_SERVICE_NAME, operationName)); } searchRequestBuilder.addSort(SegmentCostTable.COLUMN_COST, SortOrder.DESC); @@ -77,7 +77,7 @@ public class SegmentCostEsDAO extends EsDAO implements ISegmentCostDAO { topSegmentJson.addProperty(GlobalTraceTable.COLUMN_GLOBAL_TRACE_ID, globalTraces.get(0)); } - topSegmentJson.addProperty(SegmentCostTable.COLUMN_OPERATION_NAME, (String)searchHit.getSource().get(SegmentCostTable.COLUMN_OPERATION_NAME)); + topSegmentJson.addProperty(SegmentCostTable.COLUMN_SERVICE_NAME, (String)searchHit.getSource().get(SegmentCostTable.COLUMN_SERVICE_NAME)); topSegmentJson.addProperty(SegmentCostTable.COLUMN_COST, (Number)searchHit.getSource().get(SegmentCostTable.COLUMN_COST)); topSegmentJson.addProperty(SegmentCostTable.COLUMN_IS_ERROR, (Boolean)searchHit.getSource().get(SegmentCostTable.COLUMN_IS_ERROR)); diff --git a/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/handler/SegmentTopGetHandler.java b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/handler/SegmentTopGetHandler.java index 0feae9192030a82609bcc9f830d3c0278422abce..003d37e5501025d3b89ac8e5b3a336e10fc126bd 100644 --- a/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/handler/SegmentTopGetHandler.java +++ b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/handler/SegmentTopGetHandler.java @@ -79,4 +79,8 @@ public class SegmentTopGetHandler extends JettyHandler { return service.loadTop(startTime, endTime, minCost, maxCost, operationName, globalTraceId, limit, from); } + + @Override protected JsonElement doPost(HttpServletRequest req) throws ArgumentsParseException { + throw new UnsupportedOperationException(); + } } diff --git a/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/handler/SpanGetHandler.java b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/handler/SpanGetHandler.java index eda5239f18fc69a3d83ab1c02c1c009a53674caf..241e5bea5ab19e17967a659bd3e98df436429bb4 100644 --- a/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/handler/SpanGetHandler.java +++ b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/handler/SpanGetHandler.java @@ -35,4 +35,8 @@ public class SpanGetHandler extends JettyHandler { return service.load(segmentId, spanId); } + + @Override protected JsonElement doPost(HttpServletRequest req) throws ArgumentsParseException { + throw new UnsupportedOperationException(); + } } diff --git a/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/handler/TraceDagGetHandler.java b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/handler/TraceDagGetHandler.java index 71a8a2c40dbf844458ccf96bbd82531ddb0a31fa..a11540d833fb0f4b973de2424356e94f5b3fc506 100644 --- a/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/handler/TraceDagGetHandler.java +++ b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/handler/TraceDagGetHandler.java @@ -31,4 +31,8 @@ public class TraceDagGetHandler extends JettyHandler { long endTime = Long.valueOf(endTimeStr); return service.load(startTime, endTime, timeBucketType); } + + @Override protected JsonElement doPost(HttpServletRequest req) throws ArgumentsParseException { + throw new UnsupportedOperationException(); + } } diff --git a/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/handler/TraceStackGetHandler.java b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/handler/TraceStackGetHandler.java index 1792158b0c8c71e6c5d3d1c3313d0f65c25c7b7e..b497e2580b8116b03c35e6dbbba3d5fa2b7fa7f4 100644 --- a/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/handler/TraceStackGetHandler.java +++ b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/handler/TraceStackGetHandler.java @@ -27,4 +27,8 @@ public class TraceStackGetHandler extends JettyHandler { return service.load(globalTraceId); } + + @Override protected JsonElement doPost(HttpServletRequest req) throws ArgumentsParseException { + throw new UnsupportedOperationException(); + } } diff --git a/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/handler/UIJettyServerHandler.java b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/handler/UIJettyServerHandler.java index ca6f393408f0d56efb8f1ee4ad2fbc7a62501dff..78b49cd422f2374670d7f9f3535a30509b7a3379 100644 --- a/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/handler/UIJettyServerHandler.java +++ b/apm-collector/apm-collector-ui/src/main/java/org/skywalking/apm/collector/ui/jetty/handler/UIJettyServerHandler.java @@ -30,4 +30,8 @@ public class UIJettyServerHandler extends JettyHandler { }); return serverArray; } + + @Override protected JsonElement doPost(HttpServletRequest req) throws ArgumentsParseException { + throw new UnsupportedOperationException(); + } }