Merge pull request #165 from wu-sheng/feature/159

Fix a bug and improve performance

skywalking-collector/pom.xml

@@ -26,6 +26,11 @@
             <artifactId>akka-cluster_2.11</artifactId>
             <version>${akka.version}</version>
         </dependency>
+        <dependency>
+            <groupId>com.typesafe.akka</groupId>
+            <artifactId>akka-slf4j_2.11</artifactId>
+            <version>${akka.version}</version>
+        </dependency>
         <dependency>
             <groupId>org.apache.logging.log4j</groupId>
             <artifactId>log4j-core</artifactId>

skywalking-collector/skywalking-collector-cluster/src/main/java/com/a/eye/skywalking/collector/AkkaSystem.java

@@ -6,6 +6,7 @@ import com.a.eye.skywalking.collector.cluster.ClusterConfig;
 import com.a.eye.skywalking.collector.cluster.Const;
 import com.typesafe.config.Config;
 import com.typesafe.config.ConfigFactory;
+import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
@@ -17,10 +18,12 @@ public enum AkkaSystem {
     private Logger logger = LogManager.getFormatterLogger(AkkaSystem.class);
 
     public ActorSystem create() {
+        Level logLevel = logger.getLevel();
+
         final Config config = ConfigFactory.parseString("akka.remote.netty.tcp.HOSTNAME=" + ClusterConfig.Cluster.Current.HOSTNAME).
                 withFallback(ConfigFactory.parseString("akka.remote.netty.tcp.PORT=" + ClusterConfig.Cluster.Current.PORT)).
                 withFallback(ConfigFactory.parseString("akka.loggers=[\"akka.event.slf4j.Slf4jLogger\"]")).
-                withFallback(ConfigFactory.parseString("akka.loglevel=\"ERROR\"")).
+                withFallback(ConfigFactory.parseString("akka.loglevel=\"" + logLevel.name() + "\"")).
 
                 withFallback(ConfigFactory.load("application.conf"));
         if (!StringUtil.isEmpty(ClusterConfig.Cluster.SEED_NODES)) {

skywalking-collector/skywalking-collector-cluster/src/main/java/com/a/eye/skywalking/collector/actor/AbstractClusterWorker.java

@@ -7,7 +7,7 @@ import akka.cluster.Member;
 import akka.cluster.MemberStatus;
 import com.a.eye.skywalking.collector.cluster.WorkerListenerMessage;
 import com.a.eye.skywalking.collector.cluster.WorkersListener;
-import org.apache.logging.log4j.LogManager;
+import com.a.eye.skywalking.collector.log.LogManager;
 import org.apache.logging.log4j.Logger;
 
 /**
@@ -52,7 +52,7 @@ public abstract class AbstractClusterWorker extends AbstractWorker {
     protected abstract void onWork(Object message) throws Exception;
 
     static class WorkerWithAkka extends UntypedActor {
-        private Logger logger = LogManager.getFormatterLogger(WorkerWithAkka.class);
+        private Logger logger = LogManager.INSTANCE.getFormatterLogger(WorkerWithAkka.class);
 
         private Cluster cluster;
         private final AbstractClusterWorker ownerWorker;

skywalking-collector/skywalking-collector-cluster/src/main/java/com/a/eye/skywalking/collector/actor/AbstractWorker.java

 package com.a.eye.skywalking.collector.actor;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
 /**
  * @author pengys5
  */
 public abstract class AbstractWorker {
+
+    private final Logger logger;
+
     private final LocalWorkerContext selfContext;
 
     private final Role role;
@@ -14,6 +20,11 @@ public abstract class AbstractWorker {
         this.role = role;
         this.clusterContext = clusterContext;
         this.selfContext = selfContext;
+        this.logger = LogManager.getFormatterLogger(role.roleName());
+    }
+
+    final public Logger logger() {
+        return logger;
     }
 
     public abstract void preStart() throws ProviderNotFoundException;
@@ -35,6 +46,6 @@ public abstract class AbstractWorker {
     }
 
     final protected void saveException(Exception e) {
-//        e.printStackTrace();
+        logger().error(e);
     }
 }

skywalking-collector/skywalking-collector-cluster/src/main/java/com/a/eye/skywalking/collector/actor/AbstractWorkerProvider.java

@@ -24,9 +24,6 @@ public abstract class AbstractWorkerProvider<T extends AbstractWorker> implement
 
     final public WorkerRef create(
         AbstractWorker workerOwner) throws IllegalArgumentException, ProviderNotFoundException {
-        if (workerInstance(clusterContext) == null) {
-            throw new IllegalArgumentException("cannot get worker instance with nothing obtained from workerInstance()");
-        }
 
         if (workerOwner == null) {
             return onCreate(null);

skywalking-collector/skywalking-collector-cluster/src/main/java/com/a/eye/skywalking/collector/log/LogManager.java

+package com.a.eye.skywalking.collector.log;
+
+import org.apache.logging.log4j.Logger;
+
+/**
+ * @author pengys5
+ */
+public enum LogManager {
+    INSTANCE;
+
+    public Logger getFormatterLogger(final Class<?> clazz) {
+        return org.apache.logging.log4j.LogManager.getFormatterLogger(clazz);
+    }
+}

skywalking-collector/skywalking-collector-cluster/src/test/java/com/a/eye/skywalking/collector/actor/AbstractClusterWorkerProviderTestCase.java

+package com.a.eye.skywalking.collector.actor;
+
+import akka.actor.ActorSystem;
+import com.a.eye.skywalking.collector.actor.selector.RollingSelector;
+import com.a.eye.skywalking.collector.actor.selector.WorkerSelector;
+import com.a.eye.skywalking.collector.log.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.mockito.Mockito;
+import org.powermock.reflect.Whitebox;
+
+/**
+ * @author pengys5
+ */
+//@RunWith(PowerMockRunner.class)
+//@PrepareForTest({LogManager.class})
+public class AbstractClusterWorkerProviderTestCase {
+
+//    @Test
+    public void testOnCreate() throws ProviderNotFoundException {
+        LogManager logManager = Mockito.mock(LogManager.class);
+        Whitebox.setInternalState(LogManager.class, "INSTANCE", logManager);
+        Logger logger = Mockito.mock(Logger.class);
+        Mockito.when(logManager.getFormatterLogger(Mockito.any())).thenReturn(logger);
+
+        ActorSystem actorSystem = Mockito.mock(ActorSystem.class);
+        ClusterWorkerContext clusterWorkerContext = new ClusterWorkerContext(actorSystem);
+        Impl impl = new Impl();
+        impl.onCreate(null);
+    }
+
+    class Impl extends AbstractClusterWorkerProvider<AbstractClusterWorkerTestCase.Impl> {
+        @Override public Role role() {
+            return Role.INSTANCE;
+        }
+
+        @Override public AbstractClusterWorkerTestCase.Impl workerInstance(ClusterWorkerContext clusterContext) {
+            return new AbstractClusterWorkerTestCase.Impl(role(), clusterContext, new LocalWorkerContext());
+        }
+
+        @Override public int workerNum() {
+            return 0;
+        }
+    }
+
+    enum Role implements com.a.eye.skywalking.collector.actor.Role {
+        INSTANCE;
+
+        @Override
+        public String roleName() {
+            return AbstractClusterWorkerTestCase.Impl.class.getSimpleName();
+        }
+
+        @Override
+        public WorkerSelector workerSelector() {
+            return new RollingSelector();
+        }
+    }
+}

skywalking-collector/skywalking-collector-cluster/src/test/java/com/a/eye/skywalking/collector/actor/AbstractClusterWorkerTestCase.java

 package com.a.eye.skywalking.collector.actor;
 
+import akka.actor.Address;
+import akka.cluster.ClusterEvent;
+import akka.cluster.Member;
+import com.a.eye.skywalking.collector.actor.selector.RollingSelector;
+import com.a.eye.skywalking.collector.actor.selector.WorkerSelector;
+import org.apache.logging.log4j.Logger;
+import org.junit.Before;
 import org.junit.Test;
-import org.mockito.Mockito;
+import org.junit.runner.RunWith;
 import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+import org.powermock.reflect.Whitebox;
+
+import static org.mockito.Mockito.CALLS_REAL_METHODS;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
 
 /**
  * @author pengys5
  */
+@RunWith(PowerMockRunner.class)
+@PrepareForTest({ClusterEvent.MemberUp.class, Address.class})
 public class AbstractClusterWorkerTestCase {
 
+    private AbstractClusterWorker.WorkerWithAkka workerWithAkka = mock(AbstractClusterWorker.WorkerWithAkka.class, CALLS_REAL_METHODS);
+    private AbstractClusterWorker worker = PowerMockito.spy(new Impl(WorkerRole.INSTANCE, null, null));
+
+    @Before
+    public void init(){
+        Logger logger = mock(Logger.class);
+        Whitebox.setInternalState(workerWithAkka, "logger", logger);
+        Whitebox.setInternalState(workerWithAkka, "ownerWorker", worker);
+    }
+
     @Test
     public void testAllocateJob() throws Exception {
-        AbstractClusterWorker worker = PowerMockito.mock(AbstractClusterWorker.class);
 
         String jobStr = "TestJob";
         worker.allocateJob(jobStr);
 
-        Mockito.verify(worker).onWork(jobStr);
+        verify(worker).onWork(jobStr);
+    }
+
+    @Test
+    public void testMemberUp() throws Throwable {
+        ClusterEvent.MemberUp memberUp = mock(ClusterEvent.MemberUp.class);
+
+        Address address = mock(Address.class);
+        when(address.toString()).thenReturn("address");
+
+        Member member = mock(Member.class);
+        when(member.address()).thenReturn(address);
+
+        when(memberUp.member()).thenReturn(member);
+
+        workerWithAkka.onReceive(memberUp);
+
+        verify(workerWithAkka).register(member);
+    }
+
+    @Test
+    public void testMessage() throws Throwable {
+        String message = "test";
+        workerWithAkka.onReceive(message);
+
+        verify(worker).allocateJob(message);
+    }
+
+    static class Impl extends AbstractClusterWorker {
+        @Override public void preStart() throws ProviderNotFoundException {
+        }
+
+        public Impl(Role role, ClusterWorkerContext clusterContext, LocalWorkerContext selfContext) {
+            super(role, clusterContext, selfContext);
+        }
+
+        @Override protected void onWork(Object message) throws Exception {
+
+        }
+    }
+
+    public enum WorkerRole implements Role {
+        INSTANCE;
+
+        @Override
+        public String roleName() {
+            return Impl.class.getSimpleName();
+        }
+
+        @Override
+        public WorkerSelector workerSelector() {
+            return new RollingSelector();
+        }
     }
 }

skywalking-collector/skywalking-collector-cluster/src/test/java/com/a/eye/skywalking/collector/actor/selector/AbstractHashMessageTestCase.java

+package com.a.eye.skywalking.collector.actor.selector;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * @author pengys5
+ */
+public class AbstractHashMessageTestCase {
+
+    @Test
+    public void testGetHashCode() {
+        String key = "key";
+
+        Impl impl = new Impl(key);
+        Assert.assertEquals(key.hashCode(), impl.getHashCode());
+    }
+
+    class Impl extends AbstractHashMessage {
+        public Impl(String key) {
+            super(key);
+        }
+    }
+}

skywalking-collector/skywalking-collector-cluster/src/test/java/com/a/eye/skywalking/collector/actor/selector/HashCodeSelectorTestCase.java

@@ -46,4 +46,10 @@ public class HashCodeSelectorTestCase {
         WorkerRef select_3 = selector.select(members, message_3);
         Assert.assertEquals(workerRef_3.hashCode(), select_3.hashCode());
     }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void testSelectError() {
+        HashCodeSelector selector = new HashCodeSelector();
+        selector.select(null, new Object());
+    }
 }

skywalking-collector/skywalking-collector-cluster/src/test/java/com/a/eye/skywalking/collector/log/MockLog.java

+package com.a.eye.skywalking.collector.log;
+
+import org.apache.logging.log4j.Logger;
+import org.mockito.Mockito;
+import org.powermock.api.mockito.PowerMockito;
+
+/**
+ * @author pengys5
+ */
+public class MockLog {
+
+    public Logger mockito() {
+        LogManager logManager = PowerMockito.mock(LogManager.class);
+        Logger logger = Mockito.mock(Logger.class);
+        Mockito.when(logManager.getFormatterLogger(Mockito.any())).thenReturn(logger);
+        return logger;
+    }
+}

skywalking-collector/skywalking-collector-cluster/src/test/resources/application.conf

-akka {
-  actor {
-    provider = "akka.cluster.ClusterActorRefProvider"
-
-    serializers {
-      java = "akka.serialization.JavaSerializer"
-      proto = "akka.remote.serialization.ProtobufSerializer"
-//      data = "com.a.eye.skywalking.collector.worker.TraceSegmentSerializer"
-//      json = "com.a.eye.skywalking.collector.commons.serializer.JsonSerializer"
-    }
-
-    serialization-bindings {
-      "java.lang.String" = java
-      "com.google.protobuf.Message" = proto
-//      "com.a.eye.skywalking.messages.ISerializable" = data
-//      "com.google.gson.JsonObject" = json
-      //      "java.io.Serializable" = none
-    }
-
-    //    serialize-messages = on
-    warn-about-java-serializer-usage = on
-  }
-
-  remote {
-    log-remote-lifecycle-events = off
-
-    netty.tcp {
-      hostname = "127.0.0.1"
-      port = 1000
-    }
-  }
-
-  cluster {
-    auto-down-unreachable-after = off
-    metrics.enabled = off
-  }
-}
\ No newline at end of file

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/AnalysisMember.java

@@ -2,6 +2,7 @@ package com.a.eye.skywalking.collector.worker;
 
 import com.a.eye.skywalking.collector.actor.*;
 import com.a.eye.skywalking.collector.queue.EndOfBatchCommand;
+import com.a.eye.skywalking.collector.worker.config.CacheSizeConfig;
 
 /**
  * @author pengys5
@@ -12,6 +13,8 @@ public abstract class AnalysisMember extends AbstractLocalAsyncWorker {
         super(role, clusterContext, selfContext);
     }
 
+    private int messageNum;
+
     public abstract void analyse(Object message) throws Exception;
 
     @Override
@@ -23,11 +26,17 @@ public abstract class AnalysisMember extends AbstractLocalAsyncWorker {
         if (message instanceof EndOfBatchCommand) {
             aggregation();
         } else {
+            messageNum++;
             try {
                 analyse(message);
             } catch (Exception e) {
                 saveException(e);
             }
+
+            if (messageNum >= CacheSizeConfig.Cache.Analysis.SIZE) {
+                aggregation();
+                messageNum = 0;
+            }
         }
     }
 

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/CollectorBootStartUp.java

@@ -5,6 +5,7 @@ import com.a.eye.skywalking.collector.actor.ClusterWorkerContext;
 import com.a.eye.skywalking.collector.worker.httpserver.HttpServer;
 import com.a.eye.skywalking.collector.worker.storage.EsClient;
 import com.a.eye.skywalking.collector.worker.storage.IndexCreator;
+import com.a.eye.skywalking.collector.worker.storage.PersistenceTimer;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
@@ -20,6 +21,7 @@ public class CollectorBootStartUp {
         collectorSystem.boot();
         EsClient.INSTANCE.boot();
         IndexCreator.INSTANCE.create();
+        PersistenceTimer.INSTANCE.boot();
         HttpServer.INSTANCE.boot((ClusterWorkerContext)collectorSystem.getClusterContext());
     }
 }

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/JoinAndSplitAnalysisMember.java

+package com.a.eye.skywalking.collector.worker;
+
+import com.a.eye.skywalking.collector.actor.ClusterWorkerContext;
+import com.a.eye.skywalking.collector.actor.LocalWorkerContext;
+import com.a.eye.skywalking.collector.actor.Role;
+import com.a.eye.skywalking.collector.actor.WorkerRefs;
+import com.a.eye.skywalking.collector.worker.storage.JoinAndSplitAnalysisData;
+
+/**
+ * @author pengys5
+ */
+public abstract class JoinAndSplitAnalysisMember extends AnalysisMember {
+    private JoinAndSplitAnalysisData joinAndSplitAnalysisData;
+
+    public JoinAndSplitAnalysisMember(Role role, ClusterWorkerContext clusterContext, LocalWorkerContext selfContext) {
+        super(role, clusterContext, selfContext);
+        joinAndSplitAnalysisData = new JoinAndSplitAnalysisData();
+    }
+
+    private JoinAndSplitAnalysisData getJoinAndSplitAnalysisData() {
+        return joinAndSplitAnalysisData;
+    }
+
+    final protected void set(String id, String attributeName, String value) throws Exception {
+        getJoinAndSplitAnalysisData().getOrCreate(id).set(attributeName, value);
+    }
+
+    @Override
+    final protected void aggregation() throws Exception {
+        getJoinAndSplitAnalysisData().asMap().forEach((key, value) -> {
+            try {
+                aggWorkRefs().tell(value);
+            } catch (Exception e) {
+                logger().error(e);
+            }
+        });
+        getJoinAndSplitAnalysisData().asMap().clear();
+    }
+
+    protected abstract WorkerRefs aggWorkRefs();
+}

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/JoinAndSplitPersistenceMember.java

+package com.a.eye.skywalking.collector.worker;
+
+import com.a.eye.skywalking.collector.actor.ClusterWorkerContext;
+import com.a.eye.skywalking.collector.actor.LocalWorkerContext;
+import com.a.eye.skywalking.collector.actor.Role;
+import com.a.eye.skywalking.collector.worker.storage.EsClient;
+import com.a.eye.skywalking.collector.worker.storage.JoinAndSplitData;
+import com.a.eye.skywalking.collector.worker.storage.JoinAndSplitPersistenceData;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.index.IndexRequestBuilder;
+import org.elasticsearch.client.Client;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * @author pengys5
+ */
+public abstract class JoinAndSplitPersistenceMember extends PersistenceMember<JoinAndSplitPersistenceData, JoinAndSplitData> {
+
+    private Logger logger = LogManager.getFormatterLogger(JoinAndSplitPersistenceMember.class);
+
+    protected JoinAndSplitPersistenceMember(Role role, ClusterWorkerContext clusterContext, LocalWorkerContext selfContext) {
+        super(role, clusterContext, selfContext);
+    }
+
+    @Override
+    public JoinAndSplitPersistenceData initializeData() {
+        return new JoinAndSplitPersistenceData();
+    }
+
+    @Override
+    final public void analyse(Object message) throws Exception {
+        if (message instanceof JoinAndSplitData) {
+            JoinAndSplitData joinAndSplitData = (JoinAndSplitData) message;
+            JoinAndSplitPersistenceData data = getPersistenceData();
+            data.hold();
+            data.getOrCreate(joinAndSplitData.getId()).merge(joinAndSplitData);
+            data.release();
+        } else {
+            logger.error("unhandled message, message instance must JoinAndSplitData, but is %s", message.getClass().toString());
+        }
+    }
+
+    @Override
+    final protected void prepareIndex(List<IndexRequestBuilder> builderList) {
+        Map<String, JoinAndSplitData> lastData = getPersistenceData().getLast().asMap();
+        extractData(lastData);
+
+        Client client = EsClient.INSTANCE.getClient();
+        lastData.forEach((key, value) -> {
+            IndexRequestBuilder builder = client.prepareIndex(esIndex(), esType(), key).setSource(value.asMap());
+            builderList.add(builder);
+        });
+        lastData.clear();
+    }
+}

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/MergeAnalysisMember.java

-package com.a.eye.skywalking.collector.worker;
-
-import com.a.eye.skywalking.collector.actor.ClusterWorkerContext;
-import com.a.eye.skywalking.collector.actor.LocalWorkerContext;
-import com.a.eye.skywalking.collector.actor.Role;
-import com.a.eye.skywalking.collector.worker.config.CacheSizeConfig;
-import com.a.eye.skywalking.collector.worker.storage.MergeData;
-import com.a.eye.skywalking.collector.worker.storage.MergePersistenceData;
-
-/**
- * @author pengys5
- */
-public abstract class MergeAnalysisMember extends AnalysisMember {
-
-    private MergePersistenceData persistenceData;
-
-    protected MergeAnalysisMember(Role role, ClusterWorkerContext clusterContext, LocalWorkerContext selfContext) {
-        super(role, clusterContext, selfContext);
-        persistenceData = new MergePersistenceData();
-    }
-
-    private MergePersistenceData getPersistenceData() {
-        return persistenceData;
-    }
-
-    final protected void setMergeData(String id, String column, String value) throws Exception {
-        getPersistenceData().getElseCreate(id).setMergeData(column, value);
-        if (getPersistenceData().size() >= CacheSizeConfig.Cache.Analysis.SIZE) {
-            aggregation();
-        }
-    }
-
-    final public MergeData pushOne() {
-        if (getPersistenceData().iterator().hasNext()) {
-            return getPersistenceData().pushOne();
-        }
-        return null;
-    }
-}

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/MergePersistenceMember.java

-package com.a.eye.skywalking.collector.worker;
-
-import com.a.eye.skywalking.collector.actor.ClusterWorkerContext;
-import com.a.eye.skywalking.collector.actor.LocalWorkerContext;
-import com.a.eye.skywalking.collector.actor.Role;
-import com.a.eye.skywalking.collector.worker.config.CacheSizeConfig;
-import com.a.eye.skywalking.collector.worker.storage.EsClient;
-import com.a.eye.skywalking.collector.worker.storage.MergeData;
-import com.a.eye.skywalking.collector.worker.storage.MergePersistenceData;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.elasticsearch.action.bulk.BulkRequestBuilder;
-import org.elasticsearch.action.bulk.BulkResponse;
-import org.elasticsearch.action.get.GetResponse;
-import org.elasticsearch.action.get.MultiGetItemResponse;
-import org.elasticsearch.action.get.MultiGetRequestBuilder;
-import org.elasticsearch.action.get.MultiGetResponse;
-import org.elasticsearch.client.Client;
-
-import java.util.Iterator;
-import java.util.Map;
-
-/**
- * @author pengys5
- */
-public abstract class MergePersistenceMember extends PersistenceMember {
-
-    private Logger logger = LogManager.getFormatterLogger(MergePersistenceMember.class);
-
-    private MergePersistenceData persistenceData;
-
-    protected MergePersistenceMember(Role role, ClusterWorkerContext clusterContext, LocalWorkerContext selfContext) {
-        super(role, clusterContext, selfContext);
-        persistenceData = new MergePersistenceData();
-    }
-
-    private MergePersistenceData getPersistenceData() {
-        return persistenceData;
-    }
-
-    @Override final public void analyse(Object message) throws Exception {
-        if (message instanceof MergeData) {
-            MergeData mergeData = (MergeData)message;
-            getPersistenceData().getElseCreate(mergeData.getId()).merge(mergeData);
-            if (getPersistenceData().size() >= CacheSizeConfig.Cache.Persistence.SIZE) {
-                persistence();
-            }
-        } else {
-            logger.error("message unhandled");
-        }
-    }
-
-    final protected void persistence() {
-        MultiGetResponse multiGetResponse = searchFromEs();
-        for (MultiGetItemResponse itemResponse : multiGetResponse) {
-            GetResponse response = itemResponse.getResponse();
-            if (response != null && response.isExists()) {
-                getPersistenceData().getElseCreate(response.getId()).merge(response.getSource());
-            }
-        }
-
-        boolean success = saveToEs();
-        if (success) {
-            getPersistenceData().clear();
-        }
-    }
-
-    private MultiGetResponse searchFromEs() {
-        Client client = EsClient.INSTANCE.getClient();
-        MultiGetRequestBuilder multiGetRequestBuilder = client.prepareMultiGet();
-
-        Iterator<Map.Entry<String, MergeData>> iterator = getPersistenceData().iterator();
-
-        while (iterator.hasNext()) {
-            multiGetRequestBuilder.add(esIndex(), esType(), iterator.next().getKey());
-        }
-
-        return multiGetRequestBuilder.get();
-    }
-
-    private boolean saveToEs() {
-        Client client = EsClient.INSTANCE.getClient();
-        BulkRequestBuilder bulkRequest = client.prepareBulk();
-        logger.debug("persistenceData SIZE: %s", getPersistenceData().size());
-
-        Iterator<Map.Entry<String, MergeData>> iterator = getPersistenceData().iterator();
-        while (iterator.hasNext()) {
-            MergeData mergeData = iterator.next().getValue();
-            bulkRequest.add(client.prepareIndex(esIndex(), esType(), mergeData.getId()).setSource(mergeData.toMap()));
-        }
-
-        BulkResponse bulkResponse = bulkRequest.execute().actionGet();
-        return !bulkResponse.hasFailures();
-    }
-}

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/MetricAnalysisMember.java

@@ -3,32 +3,38 @@ package com.a.eye.skywalking.collector.worker;
 import com.a.eye.skywalking.collector.actor.ClusterWorkerContext;
 import com.a.eye.skywalking.collector.actor.LocalWorkerContext;
 import com.a.eye.skywalking.collector.actor.Role;
-import com.a.eye.skywalking.collector.worker.config.CacheSizeConfig;
-import com.a.eye.skywalking.collector.worker.storage.MetricData;
-import com.a.eye.skywalking.collector.worker.storage.MetricPersistenceData;
+import com.a.eye.skywalking.collector.actor.WorkerRefs;
+import com.a.eye.skywalking.collector.worker.storage.MetricAnalysisData;
 
 /**
  * @author pengys5
  */
 public abstract class MetricAnalysisMember extends AnalysisMember {
-
-    private MetricPersistenceData persistenceData = new MetricPersistenceData();
+    private MetricAnalysisData metricAnalysisData = new MetricAnalysisData();
 
     public MetricAnalysisMember(Role role, ClusterWorkerContext clusterContext, LocalWorkerContext selfContext) {
         super(role, clusterContext, selfContext);
     }
 
-    final protected void setMetric(String id, String column, Long value) throws Exception {
-        persistenceData.getElseCreate(id).setMetric(column, value);
-        if (persistenceData.size() >= CacheSizeConfig.Cache.Persistence.SIZE) {
-            aggregation();
-        }
+    final protected void set(String id, String metricName, Long value) throws Exception {
+        getMetricAnalysisData().getOrCreate(id).set(metricName, value);
+    }
+
+    private MetricAnalysisData getMetricAnalysisData() {
+        return metricAnalysisData;
     }
 
-    final public MetricData pushOne() {
-        if (persistenceData.iterator().hasNext()) {
-            return persistenceData.pushOne();
-        }
-        return null;
+    @Override
+    final protected void aggregation() throws Exception {
+        getMetricAnalysisData().asMap().forEach((key, value) -> {
+            try {
+                aggWorkRefs().tell(value);
+            } catch (Exception e) {
+                e.printStackTrace();
+            }
+        });
+        getMetricAnalysisData().asMap().clear();
     }
+
+    protected abstract WorkerRefs aggWorkRefs();
 }

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/MetricPersistenceMember.java

@@ -3,89 +3,55 @@ package com.a.eye.skywalking.collector.worker;
 import com.a.eye.skywalking.collector.actor.ClusterWorkerContext;
 import com.a.eye.skywalking.collector.actor.LocalWorkerContext;
 import com.a.eye.skywalking.collector.actor.Role;
-import com.a.eye.skywalking.collector.worker.config.CacheSizeConfig;
 import com.a.eye.skywalking.collector.worker.storage.EsClient;
 import com.a.eye.skywalking.collector.worker.storage.MetricData;
 import com.a.eye.skywalking.collector.worker.storage.MetricPersistenceData;
+import java.util.List;
+import java.util.Map;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.elasticsearch.action.bulk.BulkRequestBuilder;
-import org.elasticsearch.action.bulk.BulkResponse;
-import org.elasticsearch.action.get.GetResponse;
-import org.elasticsearch.action.get.MultiGetItemResponse;
-import org.elasticsearch.action.get.MultiGetRequestBuilder;
-import org.elasticsearch.action.get.MultiGetResponse;
+import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.client.Client;
 
-import java.util.Iterator;
-import java.util.Map;
-
 /**
  * @author pengys5
  */
-public abstract class MetricPersistenceMember extends PersistenceMember {
+public abstract class MetricPersistenceMember extends PersistenceMember<MetricPersistenceData, MetricData> {
 
     private Logger logger = LogManager.getFormatterLogger(MetricPersistenceMember.class);
 
-    private MetricPersistenceData persistenceData = new MetricPersistenceData();
-
     public MetricPersistenceMember(Role role, ClusterWorkerContext clusterContext, LocalWorkerContext selfContext) {
         super(role, clusterContext, selfContext);
     }
 
-    @Override final public void analyse(Object message) throws Exception {
-        if (message instanceof MetricData) {
-            MetricData metricData = (MetricData)message;
-            persistenceData.getElseCreate(metricData.getId()).merge(metricData);
-            if (persistenceData.size() >= CacheSizeConfig.Cache.Persistence.SIZE) {
-                persistence();
-            }
-        } else {
-            logger.error("message unhandled");
-        }
+    @Override
+    public MetricPersistenceData initializeData() {
+        return new MetricPersistenceData();
     }
 
-    final protected void persistence() {
-        MultiGetResponse multiGetResponse = searchFromEs();
-        for (MultiGetItemResponse itemResponse : multiGetResponse) {
-            GetResponse response = itemResponse.getResponse();
-            if (response != null && response.isExists()) {
-                persistenceData.getElseCreate(response.getId()).merge(response.getSource());
-            }
-        }
-
-        boolean success = saveToEs();
-        if (success) {
-            persistenceData.clear();
+    @Override
+    final public void analyse(Object message) throws Exception {
+        if (message instanceof MetricData) {
+            MetricData metricData = (MetricData) message;
+            MetricPersistenceData data = getPersistenceData();
+            data.hold();
+            data.getOrCreate(metricData.getId()).merge(metricData);
+            data.release();
+        } else {
+            logger.error("unhandled message, message instance must MetricData, but is %s", message.getClass().toString());
         }
     }
 
-    private MultiGetResponse searchFromEs() {
-        Client client = EsClient.INSTANCE.getClient();
-        MultiGetRequestBuilder multiGetRequestBuilder = client.prepareMultiGet();
-
-        Iterator<Map.Entry<String, MetricData>> iterator = persistenceData.iterator();
-
-        while (iterator.hasNext()) {
-            multiGetRequestBuilder.add(esIndex(), esType(), iterator.next().getKey());
-        }
-
-        MultiGetResponse multiGetResponse = multiGetRequestBuilder.get();
-        return multiGetResponse;
-    }
+    @Override
+    final protected void prepareIndex(List<IndexRequestBuilder> builderList) {
+        Map<String, MetricData> lastData = getPersistenceData().getLast().asMap();
+        extractData(lastData);
 
-    private boolean saveToEs() {
         Client client = EsClient.INSTANCE.getClient();
-        BulkRequestBuilder bulkRequest = client.prepareBulk();
-        logger.debug("persistenceData SIZE: %s", persistenceData.size());
-
-        Iterator<Map.Entry<String, MetricData>> iterator = persistenceData.iterator();
-        while (iterator.hasNext()) {
-            MetricData metricData = iterator.next().getValue();
-            bulkRequest.add(client.prepareIndex(esIndex(), esType(), metricData.getId()).setSource(metricData.toMap()));
-        }
-
-        BulkResponse bulkResponse = bulkRequest.execute().actionGet();
-        return !bulkResponse.hasFailures();
+        lastData.forEach((key, value) -> {
+            IndexRequestBuilder builder = client.prepareIndex(esIndex(), esType(), key).setSource(value.asMap());
+            builderList.add(builder);
+        });
+        lastData.clear();
     }
-}
+}
\ No newline at end of file

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/PersistenceMember.java

 package com.a.eye.skywalking.collector.worker;
 
 import com.a.eye.skywalking.collector.actor.*;
-import com.a.eye.skywalking.collector.queue.EndOfBatchCommand;
+import com.a.eye.skywalking.collector.worker.storage.*;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.get.GetResponse;
+import org.elasticsearch.action.get.MultiGetItemResponse;
+import org.elasticsearch.action.get.MultiGetRequestBuilder;
+import org.elasticsearch.action.get.MultiGetResponse;
+import org.elasticsearch.action.index.IndexRequestBuilder;
+import org.elasticsearch.client.Client;
+
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
 
 /**
  * @author pengys5
  */
-public abstract class PersistenceMember extends AbstractLocalAsyncWorker {
+public abstract class PersistenceMember<T extends Window & PersistenceData, D extends Data> extends AbstractLocalSyncWorker {
 
     private Logger logger = LogManager.getFormatterLogger(PersistenceMember.class);
 
     public PersistenceMember(Role role, ClusterWorkerContext clusterContext, LocalWorkerContext selfContext) {
         super(role, clusterContext, selfContext);
+        persistenceData = initializeData();
+    }
+
+    private T persistenceData;
+
+    public abstract T initializeData();
+
+    protected T getPersistenceData() {
+        return persistenceData;
     }
 
     public abstract String esIndex();
@@ -22,17 +41,76 @@ public abstract class PersistenceMember extends AbstractLocalAsyncWorker {
 
     public abstract void analyse(Object message) throws Exception;
 
-    @Override final public void preStart() throws ProviderNotFoundException {
+    @Override
+    final public void preStart() throws ProviderNotFoundException {
+
+    }
+
+    @Override
+    protected void onWork(Object request, Object response) throws Exception {
+        if (request instanceof FlushAndSwitch) {
+            persistenceData.switchPointer();
+            while (persistenceData.getLast().isHolding()) {
+                Thread.sleep(10);
+            }
+
+            if (response instanceof LinkedList) {
+                prepareIndex((LinkedList) response);
+            } else {
+                logger.error("unhandled response, response instance must LinkedList, but is %s", response.getClass().toString());
+            }
 
+        } else {
+            analyse(request);
+        }
     }
 
-    @Override final protected void onWork(Object message) throws Exception {
-        if (message instanceof EndOfBatchCommand) {
-            persistence();
+    private MultiGetResponse searchFromEs(Map<String, D> dataMap) {
+        Client client = EsClient.INSTANCE.getClient();
+        MultiGetRequestBuilder multiGetRequestBuilder = client.prepareMultiGet();
+
+        HasDataFlag flag = new HasDataFlag();
+        dataMap.forEach((key, value) -> {
+            multiGetRequestBuilder.add(esIndex(), esType(), value.getId());
+            flag.doTagHasData();
+        });
+
+        if (flag.isHasData()) {
+            return multiGetRequestBuilder.get();
         } else {
-            analyse(message);
+            return null;
+        }
+    }
+
+    final void extractData(Map<String, D> dataMap) {
+        MultiGetResponse multiGetResponse = searchFromEs(dataMap);
+        if (multiGetResponse != null) {
+            for (MultiGetItemResponse itemResponse : multiGetResponse) {
+                GetResponse response = itemResponse.getResponse();
+                if (response != null && response.isExists()) {
+                    if (dataMap.containsKey(response.getId())) {
+                        dataMap.get(response.getId()).merge(response.getSource());
+                    }
+                }
+            }
         }
     }
 
-    protected abstract void persistence();
+    protected abstract void prepareIndex(List<IndexRequestBuilder> builderList);
+
+    class HasDataFlag {
+        private boolean hasData;
+
+        HasDataFlag() {
+            hasData = false;
+        }
+
+        boolean isHasData() {
+            return hasData;
+        }
+
+        void doTagHasData() {
+            this.hasData = true;
+        }
+    }
 }

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/RecordAnalysisMember.java

@@ -3,33 +3,43 @@ package com.a.eye.skywalking.collector.worker;
 import com.a.eye.skywalking.collector.actor.ClusterWorkerContext;
 import com.a.eye.skywalking.collector.actor.LocalWorkerContext;
 import com.a.eye.skywalking.collector.actor.Role;
-import com.a.eye.skywalking.collector.worker.config.CacheSizeConfig;
-import com.a.eye.skywalking.collector.worker.storage.RecordData;
-import com.a.eye.skywalking.collector.worker.storage.RecordPersistenceData;
+import com.a.eye.skywalking.collector.actor.WorkerRefs;
+import com.a.eye.skywalking.collector.worker.storage.RecordAnalysisData;
 import com.google.gson.JsonObject;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 
 /**
  * @author pengys5
  */
 public abstract class RecordAnalysisMember extends AnalysisMember {
 
-    private RecordPersistenceData persistenceData = new RecordPersistenceData();
+    private Logger logger = LogManager.getFormatterLogger(RecordAnalysisMember.class);
+
+    private RecordAnalysisData recordAnalysisData = new RecordAnalysisData();
 
     public RecordAnalysisMember(Role role, ClusterWorkerContext clusterContext, LocalWorkerContext selfContext) {
         super(role, clusterContext, selfContext);
     }
 
-    final public void setRecord(String id, JsonObject record) throws Exception {
-        persistenceData.getElseCreate(id).setRecord(record);
-        if (persistenceData.size() >= CacheSizeConfig.Cache.Analysis.SIZE) {
-            aggregation();
-        }
+    final public void set(String id, JsonObject record) throws Exception {
+        getRecordAnalysisData().getOrCreate(id).set(record);
+    }
+
+    private RecordAnalysisData getRecordAnalysisData() {
+        return recordAnalysisData;
     }
 
-    final public RecordData pushOne() {
-        if (persistenceData.hasNext()) {
-            return persistenceData.pushOne();
-        }
-        return null;
+    @Override final protected void aggregation() throws Exception {
+        getRecordAnalysisData().asMap().forEach((key, value) -> {
+            try {
+                aggWorkRefs().tell(value);
+            } catch (Exception e) {
+                logger.error(e);
+            }
+        });
+        getRecordAnalysisData().asMap().clear();
     }
+
+    protected abstract WorkerRefs aggWorkRefs();
 }

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/RecordPersistenceMember.java

@@ -3,75 +3,52 @@ package com.a.eye.skywalking.collector.worker;
 import com.a.eye.skywalking.collector.actor.ClusterWorkerContext;
 import com.a.eye.skywalking.collector.actor.LocalWorkerContext;
 import com.a.eye.skywalking.collector.actor.Role;
-import com.a.eye.skywalking.collector.worker.config.CacheSizeConfig;
 import com.a.eye.skywalking.collector.worker.storage.EsClient;
 import com.a.eye.skywalking.collector.worker.storage.RecordData;
 import com.a.eye.skywalking.collector.worker.storage.RecordPersistenceData;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.elasticsearch.action.bulk.BulkRequestBuilder;
-import org.elasticsearch.action.bulk.BulkResponse;
-import org.elasticsearch.client.Client;
-
-import java.util.Iterator;
+import java.util.List;
 import java.util.Map;
+import org.elasticsearch.action.index.IndexRequestBuilder;
+import org.elasticsearch.client.Client;
 
 /**
  * @author pengys5
  */
-public abstract class RecordPersistenceMember extends PersistenceMember {
-
-    private Logger logger = LogManager.getFormatterLogger(RecordPersistenceMember.class);
-
-    private RecordPersistenceData persistenceData;
+public abstract class RecordPersistenceMember extends PersistenceMember<RecordPersistenceData, RecordData> {
 
     public RecordPersistenceMember(Role role, ClusterWorkerContext clusterContext, LocalWorkerContext selfContext) {
         super(role, clusterContext, selfContext);
-        persistenceData = new RecordPersistenceData();
     }
 
-    private RecordPersistenceData getPersistenceData() {
-        return this.persistenceData;
+    @Override
+    final public RecordPersistenceData initializeData() {
+        return new RecordPersistenceData();
     }
 
     @Override
     public void analyse(Object message) throws Exception {
         if (message instanceof RecordData) {
-            RecordData recordData = (RecordData)message;
-            logger.debug("setRecord: id: %s, data: %s", recordData.getId(), recordData.getRecord());
-            getPersistenceData().getElseCreate(recordData.getId()).setRecord(recordData.getRecord());
-            if (getPersistenceData().size() >= CacheSizeConfig.Cache.Persistence.SIZE) {
-                persistence();
-            }
+            RecordData recordData = (RecordData) message;
+            logger().debug("set: id: %s, data: %s", recordData.getId(), recordData.get());
+            RecordPersistenceData data = getPersistenceData();
+            data.hold();
+            data.getOrCreate(recordData.getId()).set(recordData.get());
+            data.release();
         } else {
-            logger.error("message unhandled");
+            logger().error("message unhandled");
         }
     }
 
-    protected void persistence() {
-        boolean success = saveToEs();
-        if (success) {
-            getPersistenceData().clear();
-        }
-    }
+    @Override
+    final protected void prepareIndex(List<IndexRequestBuilder> builderList) {
+        Map<String, RecordData> lastData = getPersistenceData().getLast().asMap();
+        extractData(lastData);
 
-    private boolean saveToEs() {
         Client client = EsClient.INSTANCE.getClient();
-        BulkRequestBuilder bulkRequest = client.prepareBulk();
-        logger.debug("persistenceData SIZE: %s", getPersistenceData().size());
-
-        Iterator<Map.Entry<String, RecordData>> iterator = getPersistenceData().iterator();
-
-        while (iterator.hasNext()) {
-            Map.Entry<String, RecordData> recordData = iterator.next();
-            logger.debug("saveToEs: key: %s, data: %s", recordData.getKey(), recordData.getValue().getRecord().toString());
-            bulkRequest.add(client.prepareIndex(esIndex(), esType(), recordData.getKey()).setSource(recordData.getValue().getRecord().toString()));
-        }
-
-        BulkResponse bulkResponse = bulkRequest.execute().actionGet();
-        if (bulkResponse.hasFailures()) {
-            logger.error(bulkResponse.buildFailureMessage());
-        }
-        return !bulkResponse.hasFailures();
+        lastData.forEach((key, value) -> {
+            IndexRequestBuilder builder = client.prepareIndex(esIndex(), esType(), key).setSource(value.get().toString());
+            builderList.add(builder);
+        });
+        lastData.clear();
     }
 }

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/config/CacheSizeConfig.java

@@ -7,11 +7,11 @@ public class CacheSizeConfig {
 
     public static class Cache {
         public static class Analysis {
-            public static int SIZE = 1000;
+            public static int SIZE = 1024;
         }
 
         public static class Persistence {
-            public static int SIZE = 1000;
+            public static int SIZE = 5000;
         }
     }
 }

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/config/EsConfig.java

@@ -15,6 +15,12 @@ public class EsConfig {
             }
         }
 
+        public static class Persistence {
+            public static class Timer {
+                public static Integer VALUE = 3;
+            }
+        }
+
         public static class Index {
 
             public static class Initialize {
@@ -28,6 +34,40 @@ public class EsConfig {
             public static class Replicas {
                 public static String NUMBER = "";
             }
+
+            public static class RefreshInterval {
+                public static class GlobalTraceIndex {
+                    public static Integer VALUE = 1;
+                }
+
+                public static class NodeCompIndex {
+                    public static Integer VALUE = 1;
+                }
+
+                public static class NodeMappingIndex {
+                    public static Integer VALUE = 1;
+                }
+
+                public static class NodeRefIndex {
+                    public static Integer VALUE = 1;
+                }
+
+                public static class NodeRefResSumIndex {
+                    public static Integer VALUE = 1;
+                }
+
+                public static class SegmentCostIndex {
+                    public static Integer VALUE = 10;
+                }
+
+                public static class SegmentExceptionIndex {
+                    public static Integer VALUE = 10;
+                }
+
+                public static class SegmentIndex {
+                    public static Integer VALUE = 10;
+                }
+            }
         }
     }
 

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/config/WorkerConfig.java

@@ -8,61 +8,57 @@ public class WorkerConfig {
     public static class WorkerNum {
         public static class Node {
             public static class NodeCompAgg {
-                public static int VALUE = 10;
+                public static int VALUE = 2;
             }
 
             public static class NodeMappingDayAgg {
-                public static int VALUE = 10;
+                public static int VALUE = 2;
             }
 
             public static class NodeMappingHourAgg {
-                public static int VALUE = 10;
+                public static int VALUE = 2;
             }
 
             public static class NodeMappingMinuteAgg {
-                public static int VALUE = 10;
+                public static int VALUE = 2;
             }
         }
 
         public static class NodeRef {
             public static class NodeRefDayAgg {
-                public static int VALUE = 10;
+                public static int VALUE = 2;
             }
 
             public static class NodeRefHourAgg {
-                public static int VALUE = 10;
+                public static int VALUE = 2;
             }
 
             public static class NodeRefMinuteAgg {
-                public static int VALUE = 10;
+                public static int VALUE = 2;
             }
 
             public static class NodeRefResSumDayAgg {
-                public static int VALUE = 10;
+                public static int VALUE = 2;
             }
 
             public static class NodeRefResSumHourAgg {
-                public static int VALUE = 10;
+                public static int VALUE = 2;
             }
 
             public static class NodeRefResSumMinuteAgg {
-                public static int VALUE = 10;
+                public static int VALUE = 2;
             }
         }
 
         public static class GlobalTrace {
             public static class GlobalTraceAgg {
-                public static int VALUE = 10;
+                public static int VALUE = 2;
             }
         }
     }
 
     public static class Queue {
         public static class GlobalTrace {
-            public static class GlobalTraceSave {
-                public static int SIZE = 1024;
-            }
-
             public static class GlobalTraceAnalysis {
                 public static int SIZE = 1024;
             }
@@ -70,19 +66,19 @@ public class WorkerConfig {
 
         public static class Segment {
             public static class SegmentPost {
-                public static int SIZE = 1024;
+                public static int SIZE = 4096;
             }
 
-            public static class SegmentCostSave {
+            public static class SegmentAnalysis {
                 public static int SIZE = 1024;
             }
 
-            public static class SegmentSave {
-                public static int SIZE = 1024;
+            public static class SegmentCostAnalysis {
+                public static int SIZE = 4096;
             }
 
-            public static class SegmentExceptionSave {
-                public static int SIZE = 1024;
+            public static class SegmentExceptionAnalysis {
+                public static int SIZE = 4096;
             }
         }
 
@@ -102,22 +98,6 @@ public class WorkerConfig {
             public static class NodeMappingMinuteAnalysis {
                 public static int SIZE = 1024;
             }
-
-            public static class NodeCompSave {
-                public static int SIZE = 1024;
-            }
-
-            public static class NodeMappingDaySave {
-                public static int SIZE = 1024;
-            }
-
-            public static class NodeMappingHourSave {
-                public static int SIZE = 1024;
-            }
-
-            public static class NodeMappingMinuteSave {
-                public static int SIZE = 1024;
-            }
         }
 
         public static class NodeRef {
@@ -133,30 +113,6 @@ public class WorkerConfig {
                 public static int SIZE = 1024;
             }
 
-            public static class NodeRefDaySave {
-                public static int SIZE = 1024;
-            }
-
-            public static class NodeRefHourSave {
-                public static int SIZE = 1024;
-            }
-
-            public static class NodeRefMinuteSave {
-                public static int SIZE = 1024;
-            }
-
-            public static class NodeRefResSumDaySave {
-                public static int SIZE = 1024;
-            }
-
-            public static class NodeRefResSumHourSave {
-                public static int SIZE = 1024;
-            }
-
-            public static class NodeRefResSumMinuteSave {
-                public static int SIZE = 1024;
-            }
-
             public static class NodeRefResSumDayAnalysis {
                 public static int SIZE = 1024;
             }

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/globaltrace/GlobalTraceGetWithGlobalId.java

@@ -46,8 +46,6 @@ public class GlobalTraceGetWithGlobalId extends AbstractGet {
     }
 
     public static class Factory extends AbstractGetProvider<GlobalTraceGetWithGlobalId> {
-        public static Factory INSTANCE = new Factory();
-
         @Override
         public Role role() {
             return WorkerRole.INSTANCE;

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/globaltrace/GlobalTraceIndex.java

 package com.a.eye.skywalking.collector.worker.globaltrace;
 
+import com.a.eye.skywalking.collector.worker.config.EsConfig;
 import com.a.eye.skywalking.collector.worker.storage.AbstractIndex;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
@@ -24,14 +25,18 @@ public class GlobalTraceIndex extends AbstractIndex {
         return true;
     }
 
+    @Override
+    public int refreshInterval() {
+        return EsConfig.Es.Index.RefreshInterval.GlobalTraceIndex.VALUE;
+    }
+
     @Override
     public XContentBuilder createMappingBuilder() throws IOException {
         XContentBuilder mappingBuilder = XContentFactory.jsonBuilder()
             .startObject()
             .startObject("properties")
             .startObject(SUB_SEG_IDS)
-            .field("type", "text")
-            .field("index", "not_analyzed")
+            .field("type", "keyword")
             .endObject()
             .endObject()
             .endObject();

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/globaltrace/analysis/GlobalTraceAnalysis.java

 package com.a.eye.skywalking.collector.worker.globaltrace.analysis;
 
-import com.a.eye.skywalking.collector.actor.AbstractLocalAsyncWorkerProvider;
-import com.a.eye.skywalking.collector.actor.ClusterWorkerContext;
-import com.a.eye.skywalking.collector.actor.LocalWorkerContext;
+import com.a.eye.skywalking.collector.actor.*;
 import com.a.eye.skywalking.collector.actor.selector.RollingSelector;
 import com.a.eye.skywalking.collector.actor.selector.WorkerSelector;
-import com.a.eye.skywalking.collector.worker.MergeAnalysisMember;
+import com.a.eye.skywalking.collector.worker.JoinAndSplitAnalysisMember;
 import com.a.eye.skywalking.collector.worker.config.WorkerConfig;
 import com.a.eye.skywalking.collector.worker.globaltrace.GlobalTraceIndex;
 import com.a.eye.skywalking.collector.worker.globaltrace.persistence.GlobalTraceAgg;
 import com.a.eye.skywalking.collector.worker.segment.SegmentPost;
 import com.a.eye.skywalking.collector.worker.segment.entity.GlobalTraceId;
 import com.a.eye.skywalking.collector.worker.segment.entity.Segment;
-import com.a.eye.skywalking.collector.worker.storage.MergeData;
 import com.a.eye.skywalking.collector.worker.tools.CollectionTools;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 
 import java.util.List;
 
 /**
  * @author pengys5
  */
-public class GlobalTraceAnalysis extends MergeAnalysisMember {
+public class GlobalTraceAnalysis extends JoinAndSplitAnalysisMember {
+
+    private Logger logger = LogManager.getFormatterLogger(GlobalTraceAnalysis.class);
 
     GlobalTraceAnalysis(Role role, ClusterWorkerContext clusterContext, LocalWorkerContext selfContext) {
         super(role, clusterContext, selfContext);
@@ -36,23 +37,25 @@ public class GlobalTraceAnalysis extends MergeAnalysisMember {
             if (CollectionTools.isNotEmpty(globalTraceIdList)) {
                 for (GlobalTraceId disTraceId : globalTraceIdList) {
                     String traceId = disTraceId.get();
-                    setMergeData(traceId, GlobalTraceIndex.SUB_SEG_IDS, subSegmentId);
+                    set(traceId, GlobalTraceIndex.SUB_SEG_IDS, subSegmentId);
                 }
             }
+        } else {
+            logger.error("unhandled message, message instance must SegmentPost.SegmentWithTimeSlice, but is %s", message.getClass().toString());
         }
     }
 
     @Override
-    protected void aggregation() throws Exception {
-        MergeData oneRecord;
-        while ((oneRecord = pushOne()) != null) {
-            getClusterContext().lookup(GlobalTraceAgg.Role.INSTANCE).tell(oneRecord);
+    protected WorkerRefs aggWorkRefs() {
+        try {
+            return getClusterContext().lookup(GlobalTraceAgg.Role.INSTANCE);
+        } catch (WorkerNotFoundException e) {
+            logger.error("The role of %s worker not found", GlobalTraceAgg.Role.INSTANCE.roleName());
         }
+        return null;
     }
 
     public static class Factory extends AbstractLocalAsyncWorkerProvider<GlobalTraceAnalysis> {
-        public static Factory INSTANCE = new Factory();
-
         @Override
         public Role role() {
             return Role.INSTANCE;

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/globaltrace/persistence/GlobalTraceAgg.java

@@ -4,7 +4,7 @@ import com.a.eye.skywalking.collector.actor.*;
 import com.a.eye.skywalking.collector.actor.selector.HashCodeSelector;
 import com.a.eye.skywalking.collector.actor.selector.WorkerSelector;
 import com.a.eye.skywalking.collector.worker.config.WorkerConfig;
-import com.a.eye.skywalking.collector.worker.storage.MergeData;
+import com.a.eye.skywalking.collector.worker.storage.JoinAndSplitData;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
@@ -27,16 +27,14 @@ public class GlobalTraceAgg extends AbstractClusterWorker {
 
     @Override
     protected void onWork(Object message) throws Exception {
-        if (message instanceof MergeData) {
+        if (message instanceof JoinAndSplitData) {
             getSelfContext().lookup(GlobalTraceSave.Role.INSTANCE).tell(message);
         } else {
-            logger.error("message unhandled");
+            logger.error("unhandled message, message instance must JoinAndSplitData, but is %s", message.getClass().toString());
         }
     }
 
     public static class Factory extends AbstractClusterWorkerProvider<GlobalTraceAgg> {
-        public static Factory INSTANCE = new Factory();
-
         @Override
         public Role role() {
             return Role.INSTANCE;

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/globaltrace/persistence/GlobalTraceSave.java

 package com.a.eye.skywalking.collector.worker.globaltrace.persistence;
 
-import com.a.eye.skywalking.collector.actor.AbstractLocalAsyncWorkerProvider;
+import com.a.eye.skywalking.collector.actor.AbstractLocalSyncWorkerProvider;
 import com.a.eye.skywalking.collector.actor.ClusterWorkerContext;
 import com.a.eye.skywalking.collector.actor.LocalWorkerContext;
 import com.a.eye.skywalking.collector.actor.selector.HashCodeSelector;
 import com.a.eye.skywalking.collector.actor.selector.WorkerSelector;
-import com.a.eye.skywalking.collector.worker.MergePersistenceMember;
-import com.a.eye.skywalking.collector.worker.config.WorkerConfig;
+import com.a.eye.skywalking.collector.worker.JoinAndSplitPersistenceMember;
 import com.a.eye.skywalking.collector.worker.globaltrace.GlobalTraceIndex;
+import com.a.eye.skywalking.collector.worker.storage.PersistenceWorkerListener;
 
 /**
  * @author pengys5
  */
-public class GlobalTraceSave extends MergePersistenceMember {
+public class GlobalTraceSave extends JoinAndSplitPersistenceMember {
 
     GlobalTraceSave(com.a.eye.skywalking.collector.actor.Role role, ClusterWorkerContext clusterContext,
-        LocalWorkerContext selfContext) {
+                    LocalWorkerContext selfContext) {
         super(role, clusterContext, selfContext);
     }
 
@@ -29,22 +29,17 @@ public class GlobalTraceSave extends MergePersistenceMember {
         return GlobalTraceIndex.TYPE_RECORD;
     }
 
-    public static class Factory extends AbstractLocalAsyncWorkerProvider<GlobalTraceSave> {
-        public static Factory INSTANCE = new Factory();
-
+    public static class Factory extends AbstractLocalSyncWorkerProvider<GlobalTraceSave> {
         @Override
         public Role role() {
             return Role.INSTANCE;
         }
 
-        @Override
-        public int queueSize() {
-            return WorkerConfig.Queue.GlobalTrace.GlobalTraceSave.SIZE;
-        }
-
         @Override
         public GlobalTraceSave workerInstance(ClusterWorkerContext clusterContext) {
-            return new GlobalTraceSave(role(), clusterContext, new LocalWorkerContext());
+            GlobalTraceSave worker = new GlobalTraceSave(role(), clusterContext, new LocalWorkerContext());
+            PersistenceWorkerListener.INSTANCE.register(worker);
+            return worker;
         }
     }
 

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/globaltrace/persistence/GlobalTraceSearchWithGlobalId.java

@@ -8,7 +8,7 @@ import com.a.eye.skywalking.collector.worker.globaltrace.GlobalTraceIndex;
 import com.a.eye.skywalking.collector.worker.segment.SegmentIndex;
 import com.a.eye.skywalking.collector.worker.segment.entity.*;
 import com.a.eye.skywalking.collector.worker.storage.GetResponseFromEs;
-import com.a.eye.skywalking.collector.worker.storage.MergeData;
+import com.a.eye.skywalking.collector.worker.storage.JoinAndSplitData;
 import com.a.eye.skywalking.collector.worker.tools.CollectionTools;
 import com.google.gson.Gson;
 import com.google.gson.JsonObject;
@@ -28,7 +28,7 @@ public class GlobalTraceSearchWithGlobalId extends AbstractLocalSyncWorker {
 
     private Gson gson = new Gson();
 
-    GlobalTraceSearchWithGlobalId(Role role, ClusterWorkerContext clusterContext, LocalWorkerContext selfContext) {
+    public GlobalTraceSearchWithGlobalId(Role role, ClusterWorkerContext clusterContext, LocalWorkerContext selfContext) {
         super(role, clusterContext, selfContext);
     }
 
@@ -41,7 +41,7 @@ public class GlobalTraceSearchWithGlobalId extends AbstractLocalSyncWorker {
             logger.debug("globalTraceObj: %s", globalTraceObj);
 
             String subSegIdsStr = globalTraceObj.get(GlobalTraceIndex.SUB_SEG_IDS).getAsString();
-            String[] subSegIds = subSegIdsStr.split(MergeData.SPLIT);
+            String[] subSegIds = subSegIdsStr.split(JoinAndSplitData.SPLIT);
 
             List<SpanView> spanViewList = new ArrayList<>();
             for (String subSegId : subSegIds) {
@@ -60,6 +60,8 @@ public class GlobalTraceSearchWithGlobalId extends AbstractLocalSyncWorker {
 
             JsonObject responseObj = (JsonObject)response;
             responseObj.addProperty("result", buildTree(spanViewList));
+        } else {
+            logger.error("unhandled message, message instance must String, but is %s", request.getClass().toString());
         }
     }
 
@@ -153,8 +155,6 @@ public class GlobalTraceSearchWithGlobalId extends AbstractLocalSyncWorker {
     }
 
     public static class Factory extends AbstractLocalSyncWorkerProvider<GlobalTraceSearchWithGlobalId> {
-        public static Factory INSTANCE = new Factory();
-
         @Override
         public Role role() {
             return WorkerRole.INSTANCE;

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/httpserver/AbstractPost.java

 package com.a.eye.skywalking.collector.worker.httpserver;
 
-import com.a.eye.skywalking.collector.actor.*;
+import com.a.eye.skywalking.collector.actor.AbstractLocalAsyncWorker;
+import com.a.eye.skywalking.collector.actor.ClusterWorkerContext;
+import com.a.eye.skywalking.collector.actor.LocalAsyncWorkerRef;
+import com.a.eye.skywalking.collector.actor.LocalWorkerContext;
+import com.a.eye.skywalking.collector.actor.Role;
 import com.a.eye.skywalking.collector.worker.segment.entity.Segment;
 import com.google.gson.JsonObject;
 import com.google.gson.stream.JsonReader;
-
+import java.io.BufferedReader;
+import java.io.IOException;
 import javax.servlet.ServletException;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
-import java.io.BufferedReader;
-import java.io.IOException;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 
 /**
  * @author pengys5
@@ -21,8 +26,7 @@ public abstract class AbstractPost extends AbstractLocalAsyncWorker {
         super(role, clusterContext, selfContext);
     }
 
-    @Override
-    final public void onWork(Object message) throws Exception {
+    @Override final public void onWork(Object message) throws Exception {
         onReceive(message);
     }
 
@@ -30,21 +34,23 @@ public abstract class AbstractPost extends AbstractLocalAsyncWorker {
 
     static class PostWithHttpServlet extends AbstractHttpServlet {
 
+        private Logger logger = LogManager.getFormatterLogger(PostWithHttpServlet.class);
+
         private final LocalAsyncWorkerRef ownerWorkerRef;
 
         PostWithHttpServlet(LocalAsyncWorkerRef ownerWorkerRef) {
             this.ownerWorkerRef = ownerWorkerRef;
         }
 
-        @Override
-        final protected void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
+        @Override        final protected void doPost(HttpServletRequest request,
+            HttpServletResponse response) throws ServletException, IOException {
             JsonObject resJson = new JsonObject();
             try {
                 BufferedReader bufferedReader = request.getReader();
                 streamReader(bufferedReader);
                 reply(response, resJson, HttpServletResponse.SC_OK);
             } catch (Exception e) {
-                e.printStackTrace();
+                logger.error(e);
                 resJson.addProperty("error", e.getMessage());
                 reply(response, resJson, HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
             }

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/node/NodeCompIndex.java

 package com.a.eye.skywalking.collector.worker.node;
 
+import com.a.eye.skywalking.collector.worker.config.EsConfig;
 import com.a.eye.skywalking.collector.worker.storage.AbstractIndex;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
@@ -25,25 +26,27 @@ public class NodeCompIndex extends AbstractIndex {
         return false;
     }
 
+    @Override
+    public int refreshInterval() {
+        return EsConfig.Es.Index.RefreshInterval.NodeCompIndex.VALUE;
+    }
+
     @Override
     public XContentBuilder createMappingBuilder() throws IOException {
         XContentBuilder mappingBuilder = XContentFactory.jsonBuilder()
-            .startObject()
-            .startObject("properties")
-            .startObject(NAME)
-            .field("type", "string")
-            .field("index", "not_analyzed")
-            .endObject()
-            .startObject(PEERS)
-            .field("type", "string")
-            .field("index", "not_analyzed")
-            .endObject()
-            .startObject(AGG_COLUMN)
-            .field("type", "string")
-            .field("index", "not_analyzed")
-            .endObject()
-            .endObject()
-            .endObject();
+                .startObject()
+                .startObject("properties")
+                .startObject(NAME)
+                .field("type", "keyword")
+                .endObject()
+                .startObject(PEERS)
+                .field("type", "keyword")
+                .endObject()
+                .startObject(AGG_COLUMN)
+                .field("type", "keyword")
+                .endObject()
+                .endObject()
+                .endObject();
         return mappingBuilder;
     }
 }

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/node/NodeMappingIndex.java

 package com.a.eye.skywalking.collector.worker.node;
 
+import com.a.eye.skywalking.collector.worker.config.EsConfig;
 import com.a.eye.skywalking.collector.worker.storage.AbstractIndex;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
@@ -25,29 +26,34 @@ public class NodeMappingIndex extends AbstractIndex {
         return false;
     }
 
+    @Override
+    public int refreshInterval() {
+        return EsConfig.Es.Index.RefreshInterval.NodeMappingIndex.VALUE;
+    }
+
     @Override
     public XContentBuilder createMappingBuilder() throws IOException {
         XContentBuilder mappingBuilder = XContentFactory.jsonBuilder()
-            .startObject()
-            .startObject("properties")
-            .startObject(CODE)
-            .field("type", "string")
-            .field("index", "not_analyzed")
-            .endObject()
-            .startObject(PEERS)
-            .field("type", "string")
-            .field("index", "not_analyzed")
-            .endObject()
-            .startObject(AGG_COLUMN)
-            .field("type", "string")
-            .field("index", "not_analyzed")
-            .endObject()
-            .startObject(TIME_SLICE)
-            .field("type", "long")
-            .field("index", "not_analyzed")
-            .endObject()
-            .endObject()
-            .endObject();
+                .startObject()
+                .startObject("properties")
+                .startObject(CODE)
+                .field("type", "string")
+                .field("index", "not_analyzed")
+                .endObject()
+                .startObject(PEERS)
+                .field("type", "string")
+                .field("index", "not_analyzed")
+                .endObject()
+                .startObject(AGG_COLUMN)
+                .field("type", "string")
+                .field("index", "not_analyzed")
+                .endObject()
+                .startObject(TIME_SLICE)
+                .field("type", "long")
+                .field("index", "not_analyzed")
+                .endObject()
+                .endObject()
+                .endObject();
         return mappingBuilder;
     }
 }

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/node/analysis/AbstractNodeCompAnalysis.java

@@ -28,7 +28,7 @@ abstract class AbstractNodeCompAnalysis extends RecordAnalysisMember {
         super(role, clusterContext, selfContext);
     }
 
-    void analyseSpans(Segment segment) throws Exception {
+    final void analyseSpans(Segment segment) throws Exception {
         List<Span> spanList = segment.getSpans();
         logger.debug("node analysis span isNotEmpty %s", CollectionTools.isNotEmpty(spanList));
 
@@ -43,7 +43,7 @@ abstract class AbstractNodeCompAnalysis extends RecordAnalysisMember {
                     compJsonObj.addProperty(NodeCompIndex.PEERS, peers);
                     compJsonObj.addProperty(NodeCompIndex.NAME, Tags.COMPONENT.get(span));
 
-                    setRecord(peers, compJsonObj);
+                    set(peers, compJsonObj);
                 } else if (Tags.SPAN_KIND_SERVER.equals(kind) && span.getParentSpanId() == -1) {
                     String peers = segment.getApplicationCode();
 
@@ -51,9 +51,7 @@ abstract class AbstractNodeCompAnalysis extends RecordAnalysisMember {
                     compJsonObj.addProperty(NodeCompIndex.PEERS, peers);
                     compJsonObj.addProperty(NodeCompIndex.NAME, Tags.COMPONENT.get(span));
 
-                    setRecord(peers, compJsonObj);
-                } else {
-                    logger.error("The span kind value is incorrect which segment record id is %s, the value must client or server", segment.getTraceSegmentId());
+                    set(peers, compJsonObj);
                 }
             }
         }

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/node/analysis/AbstractNodeMappingAnalysis.java

@@ -26,7 +26,7 @@ abstract class AbstractNodeMappingAnalysis extends RecordAnalysisMember {
         super(role, clusterContext, selfContext);
     }
 
-    void analyseRefs(Segment segment, long timeSlice) throws Exception {
+    final void analyseRefs(Segment segment, long timeSlice) throws Exception {
         List<TraceSegmentRef> segmentRefList = segment.getRefs();
         logger.debug("node mapping analysis refs isNotEmpty %s", CollectionTools.isNotEmpty(segmentRefList));
 
@@ -42,7 +42,7 @@ abstract class AbstractNodeMappingAnalysis extends RecordAnalysisMember {
                 nodeMappingJsonObj.addProperty(NodeMappingIndex.TIME_SLICE, timeSlice);
 
                 String id = timeSlice + Const.ID_SPLIT + code + Const.ID_SPLIT + peers;
-                setRecord(id, nodeMappingJsonObj);
+                set(id, nodeMappingJsonObj);
             }
         }
     }

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/node/analysis/NodeCompAnalysis.java

 package com.a.eye.skywalking.collector.worker.node.analysis;
 
-import com.a.eye.skywalking.collector.actor.AbstractLocalAsyncWorkerProvider;
-import com.a.eye.skywalking.collector.actor.ClusterWorkerContext;
-import com.a.eye.skywalking.collector.actor.LocalWorkerContext;
+import com.a.eye.skywalking.collector.actor.*;
 import com.a.eye.skywalking.collector.actor.selector.RollingSelector;
 import com.a.eye.skywalking.collector.actor.selector.WorkerSelector;
 import com.a.eye.skywalking.collector.worker.config.WorkerConfig;
 import com.a.eye.skywalking.collector.worker.node.persistence.NodeCompAgg;
 import com.a.eye.skywalking.collector.worker.segment.SegmentPost;
 import com.a.eye.skywalking.collector.worker.segment.entity.Segment;
-import com.a.eye.skywalking.collector.worker.storage.RecordData;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 
 /**
  * @author pengys5
  */
 public class NodeCompAnalysis extends AbstractNodeCompAnalysis {
 
+    private Logger logger = LogManager.getFormatterLogger(NodeCompAnalysis.class);
+
     NodeCompAnalysis(com.a.eye.skywalking.collector.actor.Role role, ClusterWorkerContext clusterContext,
                      LocalWorkerContext selfContext) {
         super(role, clusterContext, selfContext);
@@ -27,20 +28,22 @@ public class NodeCompAnalysis extends AbstractNodeCompAnalysis {
             SegmentPost.SegmentWithTimeSlice segmentWithTimeSlice = (SegmentPost.SegmentWithTimeSlice) message;
             Segment segment = segmentWithTimeSlice.getSegment();
             analyseSpans(segment);
+        } else {
+            logger.error("unhandled message, message instance must SegmentPost.SegmentWithTimeSlice, but is %s", message.getClass().toString());
         }
     }
 
     @Override
-    protected void aggregation() throws Exception {
-        RecordData oneRecord;
-        while ((oneRecord = pushOne()) != null) {
-            getClusterContext().lookup(NodeCompAgg.Role.INSTANCE).tell(oneRecord);
+    protected WorkerRefs aggWorkRefs() {
+        try {
+            return getClusterContext().lookup(NodeCompAgg.Role.INSTANCE);
+        } catch (WorkerNotFoundException e) {
+            logger.error("The role of %s worker not found", NodeCompAgg.Role.INSTANCE.roleName());
         }
+        return null;
     }
 
     public static class Factory extends AbstractLocalAsyncWorkerProvider<NodeCompAnalysis> {
-        public static Factory INSTANCE = new Factory();
-
         @Override
         public Role role() {
             return Role.INSTANCE;

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/node/analysis/NodeMappingDayAnalysis.java

 package com.a.eye.skywalking.collector.worker.node.analysis;
 
-import com.a.eye.skywalking.collector.actor.AbstractLocalAsyncWorkerProvider;
-import com.a.eye.skywalking.collector.actor.ClusterWorkerContext;
-import com.a.eye.skywalking.collector.actor.LocalWorkerContext;
+import com.a.eye.skywalking.collector.actor.*;
 import com.a.eye.skywalking.collector.actor.selector.RollingSelector;
 import com.a.eye.skywalking.collector.actor.selector.WorkerSelector;
 import com.a.eye.skywalking.collector.worker.config.WorkerConfig;
 import com.a.eye.skywalking.collector.worker.node.persistence.NodeMappingDayAgg;
 import com.a.eye.skywalking.collector.worker.segment.SegmentPost;
 import com.a.eye.skywalking.collector.worker.segment.entity.Segment;
-import com.a.eye.skywalking.collector.worker.storage.RecordData;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 
 /**
  * @author pengys5
  */
 public class NodeMappingDayAnalysis extends AbstractNodeMappingAnalysis {
 
+    private Logger logger = LogManager.getFormatterLogger(NodeMappingDayAnalysis.class);
+
     public NodeMappingDayAnalysis(com.a.eye.skywalking.collector.actor.Role role, ClusterWorkerContext clusterContext,
                                   LocalWorkerContext selfContext) {
         super(role, clusterContext, selfContext);
@@ -27,20 +28,22 @@ public class NodeMappingDayAnalysis extends AbstractNodeMappingAnalysis {
             SegmentPost.SegmentWithTimeSlice segmentWithTimeSlice = (SegmentPost.SegmentWithTimeSlice) message;
             Segment segment = segmentWithTimeSlice.getSegment();
             analyseRefs(segment, segmentWithTimeSlice.getDay());
+        } else {
+            logger.error("unhandled message, message instance must SegmentPost.SegmentWithTimeSlice, but is %s", message.getClass().toString());
         }
     }
 
     @Override
-    protected void aggregation() throws Exception {
-        RecordData oneRecord;
-        while ((oneRecord = pushOne()) != null) {
-            getClusterContext().lookup(NodeMappingDayAgg.Role.INSTANCE).tell(oneRecord);
+    protected WorkerRefs aggWorkRefs() {
+        try {
+            return getClusterContext().lookup(NodeMappingDayAgg.Role.INSTANCE);
+        } catch (WorkerNotFoundException e) {
+            logger.error("The role of %s worker not found", NodeMappingDayAgg.Role.INSTANCE.roleName());
         }
+        return null;
     }
 
     public static class Factory extends AbstractLocalAsyncWorkerProvider<NodeMappingDayAnalysis> {
-        public static Factory INSTANCE = new Factory();
-
         @Override
         public Role role() {
             return Role.INSTANCE;

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/node/analysis/NodeMappingHourAnalysis.java

 package com.a.eye.skywalking.collector.worker.node.analysis;
 
-import com.a.eye.skywalking.collector.actor.AbstractLocalAsyncWorkerProvider;
-import com.a.eye.skywalking.collector.actor.ClusterWorkerContext;
-import com.a.eye.skywalking.collector.actor.LocalWorkerContext;
+import com.a.eye.skywalking.collector.actor.*;
 import com.a.eye.skywalking.collector.actor.selector.RollingSelector;
 import com.a.eye.skywalking.collector.actor.selector.WorkerSelector;
 import com.a.eye.skywalking.collector.worker.config.WorkerConfig;
 import com.a.eye.skywalking.collector.worker.node.persistence.NodeMappingHourAgg;
 import com.a.eye.skywalking.collector.worker.segment.SegmentPost;
 import com.a.eye.skywalking.collector.worker.segment.entity.Segment;
-import com.a.eye.skywalking.collector.worker.storage.RecordData;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 
 /**
  * @author pengys5
  */
 public class NodeMappingHourAnalysis extends AbstractNodeMappingAnalysis {
 
+    private Logger logger = LogManager.getFormatterLogger(NodeMappingHourAnalysis.class);
+
     NodeMappingHourAnalysis(com.a.eye.skywalking.collector.actor.Role role, ClusterWorkerContext clusterContext,
                             LocalWorkerContext selfContext) {
         super(role, clusterContext, selfContext);
@@ -27,20 +28,22 @@ public class NodeMappingHourAnalysis extends AbstractNodeMappingAnalysis {
             SegmentPost.SegmentWithTimeSlice segmentWithTimeSlice = (SegmentPost.SegmentWithTimeSlice) message;
             Segment segment = segmentWithTimeSlice.getSegment();
             analyseRefs(segment, segmentWithTimeSlice.getHour());
+        } else {
+            logger.error("unhandled message, message instance must SegmentPost.SegmentWithTimeSlice, but is %s", message.getClass().toString());
         }
     }
 
     @Override
-    protected void aggregation() throws Exception {
-        RecordData oneRecord;
-        while ((oneRecord = pushOne()) != null) {
-            getClusterContext().lookup(NodeMappingHourAgg.Role.INSTANCE).tell(oneRecord);
+    protected WorkerRefs aggWorkRefs() {
+        try {
+            return getClusterContext().lookup(NodeMappingHourAgg.Role.INSTANCE);
+        } catch (WorkerNotFoundException e) {
+            logger.error("The role of %s worker not found", NodeMappingHourAgg.Role.INSTANCE.roleName());
         }
+        return null;
     }
-
+    
     public static class Factory extends AbstractLocalAsyncWorkerProvider<NodeMappingHourAnalysis> {
-        public static Factory INSTANCE = new Factory();
-
         @Override
         public Role role() {
             return Role.INSTANCE;

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/node/analysis/NodeMappingMinuteAnalysis.java

 package com.a.eye.skywalking.collector.worker.node.analysis;
 
-import com.a.eye.skywalking.collector.actor.AbstractLocalAsyncWorkerProvider;
-import com.a.eye.skywalking.collector.actor.ClusterWorkerContext;
-import com.a.eye.skywalking.collector.actor.LocalWorkerContext;
+import com.a.eye.skywalking.collector.actor.*;
 import com.a.eye.skywalking.collector.actor.selector.RollingSelector;
 import com.a.eye.skywalking.collector.actor.selector.WorkerSelector;
 import com.a.eye.skywalking.collector.worker.config.WorkerConfig;
 import com.a.eye.skywalking.collector.worker.node.persistence.NodeMappingMinuteAgg;
 import com.a.eye.skywalking.collector.worker.segment.SegmentPost;
 import com.a.eye.skywalking.collector.worker.segment.entity.Segment;
-import com.a.eye.skywalking.collector.worker.storage.RecordData;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 
 /**
  * @author pengys5
  */
 public class NodeMappingMinuteAnalysis extends AbstractNodeMappingAnalysis {
 
+    private Logger logger = LogManager.getFormatterLogger(NodeMappingMinuteAnalysis.class);
+
     NodeMappingMinuteAnalysis(com.a.eye.skywalking.collector.actor.Role role, ClusterWorkerContext clusterContext,
                               LocalWorkerContext selfContext) {
         super(role, clusterContext, selfContext);
@@ -27,20 +28,22 @@ public class NodeMappingMinuteAnalysis extends AbstractNodeMappingAnalysis {
             SegmentPost.SegmentWithTimeSlice segmentWithTimeSlice = (SegmentPost.SegmentWithTimeSlice) message;
             Segment segment = segmentWithTimeSlice.getSegment();
             analyseRefs(segment, segmentWithTimeSlice.getMinute());
+        } else {
+            logger.error("unhandled message, message instance must SegmentPost.SegmentWithTimeSlice, but is %s", message.getClass().toString());
         }
     }
 
     @Override
-    protected void aggregation() throws Exception {
-        RecordData oneRecord;
-        while ((oneRecord = pushOne()) != null) {
-            getClusterContext().lookup(NodeMappingMinuteAgg.Role.INSTANCE).tell(oneRecord);
+    protected WorkerRefs aggWorkRefs() {
+        try {
+            return getClusterContext().lookup(NodeMappingMinuteAgg.Role.INSTANCE);
+        } catch (WorkerNotFoundException e) {
+            logger.error("The role of %s worker not found", NodeMappingMinuteAgg.Role.INSTANCE.roleName());
         }
+        return null;
     }
-
+    
     public static class Factory extends AbstractLocalAsyncWorkerProvider<NodeMappingMinuteAnalysis> {
-        public static Factory INSTANCE = new Factory();
-
         @Override
         public Role role() {
             return Role.INSTANCE;

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/node/persistence/NodeCompAgg.java

@@ -5,12 +5,16 @@ import com.a.eye.skywalking.collector.actor.selector.HashCodeSelector;
 import com.a.eye.skywalking.collector.actor.selector.WorkerSelector;
 import com.a.eye.skywalking.collector.worker.config.WorkerConfig;
 import com.a.eye.skywalking.collector.worker.storage.RecordData;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 
 /**
  * @author pengys5
  */
 public class NodeCompAgg extends AbstractClusterWorker {
 
+    private Logger logger = LogManager.getFormatterLogger(NodeCompAgg.class);
+
     NodeCompAgg(com.a.eye.skywalking.collector.actor.Role role, ClusterWorkerContext clusterContext,
         LocalWorkerContext selfContext) {
         super(role, clusterContext, selfContext);
@@ -25,13 +29,12 @@ public class NodeCompAgg extends AbstractClusterWorker {
     protected void onWork(Object message) throws Exception {
         if (message instanceof RecordData) {
             getSelfContext().lookup(NodeCompSave.Role.INSTANCE).tell(message);
-        } else
-            throw new IllegalArgumentException("message instance must RecordData");
+        } else {
+            logger.error("unhandled message, message instance must RecordData, but is %s", message.getClass().toString());
+        }
     }
 
     public static class Factory extends AbstractClusterWorkerProvider<NodeCompAgg> {
-        public static Factory INSTANCE = new Factory();
-
         @Override
         public Role role() {
             return Role.INSTANCE;

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/node/persistence/NodeCompLoad.java

@@ -49,8 +49,6 @@ public class NodeCompLoad extends AbstractLocalSyncWorker {
     }
 
     public static class Factory extends AbstractLocalSyncWorkerProvider<NodeCompLoad> {
-        public static Factory INSTANCE = new Factory();
-
         @Override
         public Role role() {
             return WorkerRole.INSTANCE;

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/node/persistence/NodeCompSave.java

 package com.a.eye.skywalking.collector.worker.node.persistence;
 
-import com.a.eye.skywalking.collector.actor.AbstractLocalAsyncWorkerProvider;
+import com.a.eye.skywalking.collector.actor.AbstractLocalSyncWorkerProvider;
 import com.a.eye.skywalking.collector.actor.ClusterWorkerContext;
 import com.a.eye.skywalking.collector.actor.LocalWorkerContext;
 import com.a.eye.skywalking.collector.actor.selector.HashCodeSelector;
 import com.a.eye.skywalking.collector.actor.selector.WorkerSelector;
 import com.a.eye.skywalking.collector.worker.RecordPersistenceMember;
-import com.a.eye.skywalking.collector.worker.config.WorkerConfig;
 import com.a.eye.skywalking.collector.worker.node.NodeCompIndex;
+import com.a.eye.skywalking.collector.worker.storage.PersistenceWorkerListener;
 
 /**
  * @author pengys5
@@ -15,7 +15,7 @@ import com.a.eye.skywalking.collector.worker.node.NodeCompIndex;
 public class NodeCompSave extends RecordPersistenceMember {
 
     NodeCompSave(com.a.eye.skywalking.collector.actor.Role role, ClusterWorkerContext clusterContext,
-        LocalWorkerContext selfContext) {
+                 LocalWorkerContext selfContext) {
         super(role, clusterContext, selfContext);
     }
 
@@ -29,9 +29,7 @@ public class NodeCompSave extends RecordPersistenceMember {
         return NodeCompIndex.TYPE_RECORD;
     }
 
-    public static class Factory extends AbstractLocalAsyncWorkerProvider<NodeCompSave> {
-        public static Factory INSTANCE = new Factory();
-
+    public static class Factory extends AbstractLocalSyncWorkerProvider<NodeCompSave> {
         @Override
         public Role role() {
             return Role.INSTANCE;
@@ -39,12 +37,9 @@ public class NodeCompSave extends RecordPersistenceMember {
 
         @Override
         public NodeCompSave workerInstance(ClusterWorkerContext clusterContext) {
-            return new NodeCompSave(role(), clusterContext, new LocalWorkerContext());
-        }
-
-        @Override
-        public int queueSize() {
-            return WorkerConfig.Queue.Node.NodeCompSave.SIZE;
+            NodeCompSave worker = new NodeCompSave(role(), clusterContext, new LocalWorkerContext());
+            PersistenceWorkerListener.INSTANCE.register(worker);
+            return worker;
         }
     }
 

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/node/persistence/NodeMappingDayAgg.java

@@ -5,12 +5,16 @@ import com.a.eye.skywalking.collector.actor.selector.HashCodeSelector;
 import com.a.eye.skywalking.collector.actor.selector.WorkerSelector;
 import com.a.eye.skywalking.collector.worker.config.WorkerConfig;
 import com.a.eye.skywalking.collector.worker.storage.RecordData;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 
 /**
  * @author pengys5
  */
 public class NodeMappingDayAgg extends AbstractClusterWorker {
 
+    private Logger logger = LogManager.getFormatterLogger(NodeMappingDayAgg.class);
+
     NodeMappingDayAgg(com.a.eye.skywalking.collector.actor.Role role, ClusterWorkerContext clusterContext,
         LocalWorkerContext selfContext) {
         super(role, clusterContext, selfContext);
@@ -26,13 +30,11 @@ public class NodeMappingDayAgg extends AbstractClusterWorker {
         if (message instanceof RecordData) {
             getSelfContext().lookup(NodeMappingDaySave.Role.INSTANCE).tell(message);
         } else {
-            throw new IllegalArgumentException("message instance must RecordData");
+            logger.error("unhandled message, message instance must RecordData, but is %s", message.getClass().toString());
         }
     }
 
     public static class Factory extends AbstractClusterWorkerProvider<NodeMappingDayAgg> {
-        public static Factory INSTANCE = new Factory();
-
         @Override
         public Role role() {
             return Role.INSTANCE;

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/node/persistence/NodeMappingDaySave.java

 package com.a.eye.skywalking.collector.worker.node.persistence;
 
-import com.a.eye.skywalking.collector.actor.AbstractLocalAsyncWorkerProvider;
+import com.a.eye.skywalking.collector.actor.AbstractLocalSyncWorkerProvider;
 import com.a.eye.skywalking.collector.actor.ClusterWorkerContext;
 import com.a.eye.skywalking.collector.actor.LocalWorkerContext;
 import com.a.eye.skywalking.collector.actor.selector.HashCodeSelector;
 import com.a.eye.skywalking.collector.actor.selector.WorkerSelector;
 import com.a.eye.skywalking.collector.worker.RecordPersistenceMember;
-import com.a.eye.skywalking.collector.worker.config.WorkerConfig;
 import com.a.eye.skywalking.collector.worker.node.NodeMappingIndex;
+import com.a.eye.skywalking.collector.worker.storage.PersistenceWorkerListener;
 
 /**
  * @author pengys5
@@ -15,7 +15,7 @@ import com.a.eye.skywalking.collector.worker.node.NodeMappingIndex;
 public class NodeMappingDaySave extends RecordPersistenceMember {
 
     NodeMappingDaySave(com.a.eye.skywalking.collector.actor.Role role, ClusterWorkerContext clusterContext,
-        LocalWorkerContext selfContext) {
+                       LocalWorkerContext selfContext) {
         super(role, clusterContext, selfContext);
     }
 
@@ -29,9 +29,7 @@ public class NodeMappingDaySave extends RecordPersistenceMember {
         return NodeMappingIndex.TYPE_DAY;
     }
 
-    public static class Factory extends AbstractLocalAsyncWorkerProvider<NodeMappingDaySave> {
-        public static Factory INSTANCE = new Factory();
-
+    public static class Factory extends AbstractLocalSyncWorkerProvider<NodeMappingDaySave> {
         @Override
         public Role role() {
             return Role.INSTANCE;
@@ -39,12 +37,9 @@ public class NodeMappingDaySave extends RecordPersistenceMember {
 
         @Override
         public NodeMappingDaySave workerInstance(ClusterWorkerContext clusterContext) {
-            return new NodeMappingDaySave(role(), clusterContext, new LocalWorkerContext());
-        }
-
-        @Override
-        public int queueSize() {
-            return WorkerConfig.Queue.Node.NodeMappingDaySave.SIZE;
+            NodeMappingDaySave worker = new NodeMappingDaySave(role(), clusterContext, new LocalWorkerContext());
+            PersistenceWorkerListener.INSTANCE.register(worker);
+            return worker;
         }
     }
 

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/node/persistence/NodeMappingHourAgg.java

@@ -5,12 +5,16 @@ import com.a.eye.skywalking.collector.actor.selector.HashCodeSelector;
 import com.a.eye.skywalking.collector.actor.selector.WorkerSelector;
 import com.a.eye.skywalking.collector.worker.config.WorkerConfig;
 import com.a.eye.skywalking.collector.worker.storage.RecordData;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 
 /**
  * @author pengys5
  */
 public class NodeMappingHourAgg extends AbstractClusterWorker {
 
+    private Logger logger = LogManager.getFormatterLogger(NodeMappingHourAgg.class);
+
     NodeMappingHourAgg(com.a.eye.skywalking.collector.actor.Role role, ClusterWorkerContext clusterContext,
         LocalWorkerContext selfContext) {
         super(role, clusterContext, selfContext);
@@ -26,13 +30,11 @@ public class NodeMappingHourAgg extends AbstractClusterWorker {
         if (message instanceof RecordData) {
             getSelfContext().lookup(NodeMappingHourSave.Role.INSTANCE).tell(message);
         } else {
-            throw new IllegalArgumentException("message instance must RecordData");
+            logger.error("unhandled message, message instance must RecordData, but is %s", message.getClass().toString());
         }
     }
 
     public static class Factory extends AbstractClusterWorkerProvider<NodeMappingHourAgg> {
-        public static Factory INSTANCE = new Factory();
-
         @Override
         public Role role() {
             return Role.INSTANCE;

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/node/persistence/NodeMappingHourSave.java

 package com.a.eye.skywalking.collector.worker.node.persistence;
 
-import com.a.eye.skywalking.collector.actor.AbstractLocalAsyncWorkerProvider;
+import com.a.eye.skywalking.collector.actor.AbstractLocalSyncWorkerProvider;
 import com.a.eye.skywalking.collector.actor.ClusterWorkerContext;
 import com.a.eye.skywalking.collector.actor.LocalWorkerContext;
 import com.a.eye.skywalking.collector.actor.selector.HashCodeSelector;
 import com.a.eye.skywalking.collector.actor.selector.WorkerSelector;
 import com.a.eye.skywalking.collector.worker.RecordPersistenceMember;
-import com.a.eye.skywalking.collector.worker.config.WorkerConfig;
 import com.a.eye.skywalking.collector.worker.node.NodeMappingIndex;
+import com.a.eye.skywalking.collector.worker.storage.PersistenceWorkerListener;
 
 /**
  * @author pengys5
@@ -15,7 +15,7 @@ import com.a.eye.skywalking.collector.worker.node.NodeMappingIndex;
 public class NodeMappingHourSave extends RecordPersistenceMember {
 
     NodeMappingHourSave(com.a.eye.skywalking.collector.actor.Role role, ClusterWorkerContext clusterContext,
-        LocalWorkerContext selfContext) {
+                        LocalWorkerContext selfContext) {
         super(role, clusterContext, selfContext);
     }
 
@@ -29,9 +29,7 @@ public class NodeMappingHourSave extends RecordPersistenceMember {
         return NodeMappingIndex.TYPE_HOUR;
     }
 
-    public static class Factory extends AbstractLocalAsyncWorkerProvider<NodeMappingHourSave> {
-        public static Factory INSTANCE = new Factory();
-
+    public static class Factory extends AbstractLocalSyncWorkerProvider<NodeMappingHourSave> {
         @Override
         public Role role() {
             return Role.INSTANCE;
@@ -39,12 +37,9 @@ public class NodeMappingHourSave extends RecordPersistenceMember {
 
         @Override
         public NodeMappingHourSave workerInstance(ClusterWorkerContext clusterContext) {
-            return new NodeMappingHourSave(role(), clusterContext, new LocalWorkerContext());
-        }
-
-        @Override
-        public int queueSize() {
-            return WorkerConfig.Queue.Node.NodeMappingHourSave.SIZE;
+            NodeMappingHourSave worker = new NodeMappingHourSave(role(), clusterContext, new LocalWorkerContext());
+            PersistenceWorkerListener.INSTANCE.register(worker);
+            return worker;
         }
     }
 

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/node/persistence/NodeMappingMinuteAgg.java

@@ -5,12 +5,16 @@ import com.a.eye.skywalking.collector.actor.selector.HashCodeSelector;
 import com.a.eye.skywalking.collector.actor.selector.WorkerSelector;
 import com.a.eye.skywalking.collector.worker.config.WorkerConfig;
 import com.a.eye.skywalking.collector.worker.storage.RecordData;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 
 /**
  * @author pengys5
  */
 public class NodeMappingMinuteAgg extends AbstractClusterWorker {
 
+    private Logger logger = LogManager.getFormatterLogger(NodeMappingMinuteAgg.class);
+
     NodeMappingMinuteAgg(com.a.eye.skywalking.collector.actor.Role role, ClusterWorkerContext clusterContext,
         LocalWorkerContext selfContext) {
         super(role, clusterContext, selfContext);
@@ -26,13 +30,11 @@ public class NodeMappingMinuteAgg extends AbstractClusterWorker {
         if (message instanceof RecordData) {
             getSelfContext().lookup(NodeMappingMinuteSave.Role.INSTANCE).tell(message);
         } else {
-            throw new IllegalArgumentException("message instance must RecordData");
+            logger.error("unhandled message, message instance must RecordData, but is %s", message.getClass().toString());
         }
     }
 
     public static class Factory extends AbstractClusterWorkerProvider<NodeMappingMinuteAgg> {
-        public static Factory INSTANCE = new Factory();
-
         @Override
         public Role role() {
             return Role.INSTANCE;

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/node/persistence/NodeMappingMinuteSave.java

 package com.a.eye.skywalking.collector.worker.node.persistence;
 
-import com.a.eye.skywalking.collector.actor.AbstractLocalAsyncWorkerProvider;
+import com.a.eye.skywalking.collector.actor.AbstractLocalSyncWorkerProvider;
 import com.a.eye.skywalking.collector.actor.ClusterWorkerContext;
 import com.a.eye.skywalking.collector.actor.LocalWorkerContext;
 import com.a.eye.skywalking.collector.actor.selector.HashCodeSelector;
 import com.a.eye.skywalking.collector.actor.selector.WorkerSelector;
 import com.a.eye.skywalking.collector.worker.RecordPersistenceMember;
-import com.a.eye.skywalking.collector.worker.config.WorkerConfig;
 import com.a.eye.skywalking.collector.worker.node.NodeMappingIndex;
+import com.a.eye.skywalking.collector.worker.storage.PersistenceWorkerListener;
 
 /**
  * @author pengys5
@@ -15,7 +15,7 @@ import com.a.eye.skywalking.collector.worker.node.NodeMappingIndex;
 public class NodeMappingMinuteSave extends RecordPersistenceMember {
 
     NodeMappingMinuteSave(com.a.eye.skywalking.collector.actor.Role role, ClusterWorkerContext clusterContext,
-        LocalWorkerContext selfContext) {
+                          LocalWorkerContext selfContext) {
         super(role, clusterContext, selfContext);
     }
 
@@ -29,9 +29,7 @@ public class NodeMappingMinuteSave extends RecordPersistenceMember {
         return NodeMappingIndex.TYPE_MINUTE;
     }
 
-    public static class Factory extends AbstractLocalAsyncWorkerProvider<NodeMappingMinuteSave> {
-        public static Factory INSTANCE = new Factory();
-
+    public static class Factory extends AbstractLocalSyncWorkerProvider<NodeMappingMinuteSave> {
         @Override
         public Role role() {
             return Role.INSTANCE;
@@ -39,12 +37,9 @@ public class NodeMappingMinuteSave extends RecordPersistenceMember {
 
         @Override
         public NodeMappingMinuteSave workerInstance(ClusterWorkerContext clusterContext) {
-            return new NodeMappingMinuteSave(role(), clusterContext, new LocalWorkerContext());
-        }
-
-        @Override
-        public int queueSize() {
-            return WorkerConfig.Queue.Node.NodeMappingMinuteSave.SIZE;
+            NodeMappingMinuteSave worker = new NodeMappingMinuteSave(role(), clusterContext, new LocalWorkerContext());
+            PersistenceWorkerListener.INSTANCE.register(worker);
+            return worker;
         }
     }
 

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/node/persistence/NodeMappingSearchWithTimeSlice.java

@@ -62,7 +62,7 @@ public class NodeMappingSearchWithTimeSlice extends AbstractLocalSyncWorker {
             JsonObject resJsonObj = (JsonObject)response;
             resJsonObj.add(Const.RESULT, nodeMappingArray);
         } else {
-            throw new IllegalArgumentException("message instance must be RequestEntity");
+            logger.error("unhandled message, message instance must NodeMappingSearchWithTimeSlice.RequestEntity, but is %s", request.getClass().toString());
         }
     }
 
@@ -73,8 +73,6 @@ public class NodeMappingSearchWithTimeSlice extends AbstractLocalSyncWorker {
     }
 
     public static class Factory extends AbstractLocalSyncWorkerProvider<NodeMappingSearchWithTimeSlice> {
-        public static Factory INSTANCE = new Factory();
-
         @Override
         public Role role() {
             return WorkerRole.INSTANCE;

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/noderef/NodeRefIndex.java

 package com.a.eye.skywalking.collector.worker.noderef;
 
+import com.a.eye.skywalking.collector.worker.config.EsConfig;
 import com.a.eye.skywalking.collector.worker.storage.AbstractIndex;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
@@ -26,30 +27,32 @@ public class NodeRefIndex extends AbstractIndex {
         return false;
     }
 
+    @Override
+    public int refreshInterval() {
+        return EsConfig.Es.Index.RefreshInterval.NodeRefIndex.VALUE;
+    }
+
     @Override
     public XContentBuilder createMappingBuilder() throws IOException {
         XContentBuilder mappingBuilder = XContentFactory.jsonBuilder()
             .startObject()
             .startObject("properties")
             .startObject(FRONT)
-            .field("type", "string")
-            .field("index", "not_analyzed")
+            .field("type", "keyword")
             .endObject()
             .startObject(FRONT_IS_REAL_CODE)
             .field("type", "boolean")
             .field("index", "not_analyzed")
             .endObject()
             .startObject(BEHIND)
-            .field("type", "string")
-            .field("index", "not_analyzed")
+            .field("type", "keyword")
             .endObject()
             .startObject(BEHIND_IS_REAL_CODE)
             .field("type", "boolean")
             .field("index", "not_analyzed")
             .endObject()
             .startObject(AGG_COLUMN)
-            .field("type", "string")
-            .field("index", "not_analyzed")
+            .field("type", "keyword")
             .endObject()
             .startObject(TIME_SLICE)
             .field("type", "long")

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/noderef/NodeRefResSumGetGroupWithTimeSlice.java

@@ -61,8 +61,6 @@ public class NodeRefResSumGetGroupWithTimeSlice extends AbstractGet {
     }
 
     public static class Factory extends AbstractGetProvider<NodeRefResSumGetGroupWithTimeSlice> {
-        public static Factory INSTANCE = new Factory();
-
         @Override
         public Role role() {
             return WorkerRole.INSTANCE;

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/noderef/NodeRefResSumIndex.java

 package com.a.eye.skywalking.collector.worker.noderef;
 
+import com.a.eye.skywalking.collector.worker.config.EsConfig;
 import com.a.eye.skywalking.collector.worker.storage.AbstractIndex;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
@@ -28,45 +29,49 @@ public class NodeRefResSumIndex extends AbstractIndex {
         return false;
     }
 
+    @Override
+    public int refreshInterval() {
+        return EsConfig.Es.Index.RefreshInterval.NodeRefResSumIndex.VALUE;
+    }
+
     @Override
     public XContentBuilder createMappingBuilder() throws IOException {
         XContentBuilder mappingBuilder = XContentFactory.jsonBuilder()
-            .startObject()
-            .startObject("properties")
-            .startObject(ONE_SECOND_LESS)
-            .field("type", "long")
-            .field("index", "not_analyzed")
-            .endObject()
-            .startObject(THREE_SECOND_LESS)
-            .field("type", "long")
-            .field("index", "not_analyzed")
-            .endObject()
-            .startObject(FIVE_SECOND_LESS)
-            .field("type", "long")
-            .field("index", "not_analyzed")
-            .endObject()
-            .startObject(FIVE_SECOND_GREATER)
-            .field("type", "long")
-            .field("index", "not_analyzed")
-            .endObject()
-            .startObject(ERROR)
-            .field("type", "long")
-            .field("index", "not_analyzed")
-            .endObject()
-            .startObject(SUMMARY)
-            .field("type", "long")
-            .field("index", "not_analyzed")
-            .endObject()
-            .startObject(AGG_COLUMN)
-            .field("type", "string")
-            .field("index", "not_analyzed")
-            .endObject()
-            .startObject(TIME_SLICE)
-            .field("type", "long")
-            .field("index", "not_analyzed")
-            .endObject()
-            .endObject()
-            .endObject();
+                .startObject()
+                .startObject("properties")
+                .startObject(ONE_SECOND_LESS)
+                .field("type", "long")
+                .field("index", "not_analyzed")
+                .endObject()
+                .startObject(THREE_SECOND_LESS)
+                .field("type", "long")
+                .field("index", "not_analyzed")
+                .endObject()
+                .startObject(FIVE_SECOND_LESS)
+                .field("type", "long")
+                .field("index", "not_analyzed")
+                .endObject()
+                .startObject(FIVE_SECOND_GREATER)
+                .field("type", "long")
+                .field("index", "not_analyzed")
+                .endObject()
+                .startObject(ERROR)
+                .field("type", "long")
+                .field("index", "not_analyzed")
+                .endObject()
+                .startObject(SUMMARY)
+                .field("type", "long")
+                .field("index", "not_analyzed")
+                .endObject()
+                .startObject(AGG_COLUMN)
+                .field("type", "keyword")
+                .endObject()
+                .startObject(TIME_SLICE)
+                .field("type", "long")
+                .field("index", "not_analyzed")
+                .endObject()
+                .endObject()
+                .endObject();
         return mappingBuilder;
     }
 }

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/noderef/analysis/AbstractNodeRefAnalysis.java

@@ -49,7 +49,7 @@ abstract class AbstractNodeRefAnalysis extends RecordAnalysisMember {
 
                     String id = timeSlice + Const.ID_SPLIT + front + Const.ID_SPLIT + behind;
                     logger.debug("dag node ref: %s", dataJsonObj.toString());
-                    setRecord(id, dataJsonObj);
+                    set(id, dataJsonObj);
                     buildNodeRefResRecordData(id, span, minute, hour, day, second);
                 } else if (Tags.SPAN_KIND_SERVER.equals(Tags.SPAN_KIND.get(span))) {
                     if (span.getParentSpanId() == -1 && CollectionTools.isEmpty(segment.getRefs())) {
@@ -60,7 +60,7 @@ abstract class AbstractNodeRefAnalysis extends RecordAnalysisMember {
                         dataJsonObj.addProperty(NodeRefIndex.FRONT, front);
 
                         String id = timeSlice + Const.ID_SPLIT + front + Const.ID_SPLIT + behind;
-                        setRecord(id, dataJsonObj);
+                        set(id, dataJsonObj);
                         buildNodeRefResRecordData(id, span, minute, hour, day, second);
                     }
                 }

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/noderef/analysis/AbstractNodeRefResSumAnalysis.java

@@ -22,23 +22,23 @@ abstract class AbstractNodeRefResSumAnalysis extends MetricAnalysisMember {
         boolean isError = nodeRefRes.isError;
         long cost = endTime - startTime;
 
-        setMetric(nodeRefRes.nodeRefId, NodeRefResSumIndex.ONE_SECOND_LESS, 0L);
-        setMetric(nodeRefRes.nodeRefId, NodeRefResSumIndex.THREE_SECOND_LESS, 0L);
-        setMetric(nodeRefRes.nodeRefId, NodeRefResSumIndex.FIVE_SECOND_LESS, 0L);
-        setMetric(nodeRefRes.nodeRefId, NodeRefResSumIndex.FIVE_SECOND_GREATER, 0L);
-        setMetric(nodeRefRes.nodeRefId, NodeRefResSumIndex.ERROR, 0L);
+        set(nodeRefRes.nodeRefId, NodeRefResSumIndex.ONE_SECOND_LESS, 0L);
+        set(nodeRefRes.nodeRefId, NodeRefResSumIndex.THREE_SECOND_LESS, 0L);
+        set(nodeRefRes.nodeRefId, NodeRefResSumIndex.FIVE_SECOND_LESS, 0L);
+        set(nodeRefRes.nodeRefId, NodeRefResSumIndex.FIVE_SECOND_GREATER, 0L);
+        set(nodeRefRes.nodeRefId, NodeRefResSumIndex.ERROR, 0L);
         if (cost <= 1000 && !isError) {
-            setMetric(nodeRefRes.nodeRefId, NodeRefResSumIndex.ONE_SECOND_LESS, 1L);
+            set(nodeRefRes.nodeRefId, NodeRefResSumIndex.ONE_SECOND_LESS, 1L);
         } else if (1000 < cost && cost <= 3000 && !isError) {
-            setMetric(nodeRefRes.nodeRefId, NodeRefResSumIndex.THREE_SECOND_LESS, 1L);
+            set(nodeRefRes.nodeRefId, NodeRefResSumIndex.THREE_SECOND_LESS, 1L);
         } else if (3000 < cost && cost <= 5000 && !isError) {
-            setMetric(nodeRefRes.nodeRefId, NodeRefResSumIndex.FIVE_SECOND_LESS, 1L);
+            set(nodeRefRes.nodeRefId, NodeRefResSumIndex.FIVE_SECOND_LESS, 1L);
         } else if (5000 < cost && !isError) {
-            setMetric(nodeRefRes.nodeRefId, NodeRefResSumIndex.FIVE_SECOND_GREATER, 1L);
+            set(nodeRefRes.nodeRefId, NodeRefResSumIndex.FIVE_SECOND_GREATER, 1L);
         } else {
-            setMetric(nodeRefRes.nodeRefId, NodeRefResSumIndex.ERROR, 1L);
+            set(nodeRefRes.nodeRefId, NodeRefResSumIndex.ERROR, 1L);
         }
-        setMetric(nodeRefRes.nodeRefId, NodeRefResSumIndex.SUMMARY, 1L);
+        set(nodeRefRes.nodeRefId, NodeRefResSumIndex.SUMMARY, 1L);
     }
 
     public static class NodeRefResRecord extends AbstractTimeSlice {

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/noderef/analysis/NodeRefDayAnalysis.java

 package com.a.eye.skywalking.collector.worker.noderef.analysis;
 
-import com.a.eye.skywalking.collector.actor.AbstractLocalAsyncWorkerProvider;
-import com.a.eye.skywalking.collector.actor.ClusterWorkerContext;
-import com.a.eye.skywalking.collector.actor.LocalWorkerContext;
-import com.a.eye.skywalking.collector.actor.ProviderNotFoundException;
+import com.a.eye.skywalking.collector.actor.*;
 import com.a.eye.skywalking.collector.actor.selector.RollingSelector;
 import com.a.eye.skywalking.collector.actor.selector.WorkerSelector;
 import com.a.eye.skywalking.collector.worker.config.WorkerConfig;
 import com.a.eye.skywalking.collector.worker.noderef.persistence.NodeRefDayAgg;
 import com.a.eye.skywalking.collector.worker.segment.SegmentPost;
 import com.a.eye.skywalking.collector.worker.segment.entity.Segment;
-import com.a.eye.skywalking.collector.worker.storage.RecordData;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 
 /**
  * @author pengys5
  */
 public class NodeRefDayAnalysis extends AbstractNodeRefAnalysis {
 
+    private Logger logger = LogManager.getFormatterLogger(NodeRefDayAnalysis.class);
+
     protected NodeRefDayAnalysis(com.a.eye.skywalking.collector.actor.Role role, ClusterWorkerContext clusterContext,
                                  LocalWorkerContext selfContext) {
         super(role, clusterContext, selfContext);
@@ -39,6 +39,8 @@ public class NodeRefDayAnalysis extends AbstractNodeRefAnalysis {
             long day = segmentWithTimeSlice.getDay();
             int second = segmentWithTimeSlice.getSecond();
             analyseNodeRef(segment, segmentWithTimeSlice.getDay(), minute, hour, day, second);
+        } else {
+            logger.error("unhandled message, message instance must SegmentPost.SegmentWithTimeSlice, but is %s", message.getClass().toString());
         }
     }
 
@@ -48,17 +50,16 @@ public class NodeRefDayAnalysis extends AbstractNodeRefAnalysis {
     }
 
     @Override
-    protected void aggregation() throws Exception {
-        RecordData oneRecord;
-        while ((oneRecord = pushOne()) != null) {
-            getClusterContext().lookup(NodeRefDayAgg.Role.INSTANCE).tell(oneRecord);
+    protected WorkerRefs aggWorkRefs() {
+        try {
+            return getClusterContext().lookup(NodeRefDayAgg.Role.INSTANCE);
+        } catch (WorkerNotFoundException e) {
+            logger.error("The role of %s worker not found", NodeRefDayAgg.Role.INSTANCE.roleName());
         }
+        return null;
     }
 
     public static class Factory extends AbstractLocalAsyncWorkerProvider<NodeRefDayAnalysis> {
-
-        public static Factory INSTANCE = new Factory();
-
         @Override
         public Role role() {
             return Role.INSTANCE;

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/noderef/analysis/NodeRefHourAnalysis.java

 package com.a.eye.skywalking.collector.worker.noderef.analysis;
 
-import com.a.eye.skywalking.collector.actor.AbstractLocalAsyncWorkerProvider;
-import com.a.eye.skywalking.collector.actor.ClusterWorkerContext;
-import com.a.eye.skywalking.collector.actor.LocalWorkerContext;
-import com.a.eye.skywalking.collector.actor.ProviderNotFoundException;
+import com.a.eye.skywalking.collector.actor.*;
 import com.a.eye.skywalking.collector.actor.selector.RollingSelector;
 import com.a.eye.skywalking.collector.actor.selector.WorkerSelector;
 import com.a.eye.skywalking.collector.worker.config.WorkerConfig;
 import com.a.eye.skywalking.collector.worker.noderef.persistence.NodeRefHourAgg;
 import com.a.eye.skywalking.collector.worker.segment.SegmentPost;
 import com.a.eye.skywalking.collector.worker.segment.entity.Segment;
-import com.a.eye.skywalking.collector.worker.storage.RecordData;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 
 /**
  * @author pengys5
  */
 public class NodeRefHourAnalysis extends AbstractNodeRefAnalysis {
 
+    private Logger logger = LogManager.getFormatterLogger(NodeRefHourAnalysis.class);
+
     protected NodeRefHourAnalysis(com.a.eye.skywalking.collector.actor.Role role, ClusterWorkerContext clusterContext,
                                   LocalWorkerContext selfContext) {
         super(role, clusterContext, selfContext);
@@ -39,6 +39,8 @@ public class NodeRefHourAnalysis extends AbstractNodeRefAnalysis {
             long day = segmentWithTimeSlice.getDay();
             int second = segmentWithTimeSlice.getSecond();
             analyseNodeRef(segment, segmentWithTimeSlice.getHour(), minute, hour, day, second);
+        } else {
+            logger.error("unhandled message, message instance must SegmentPost.SegmentWithTimeSlice, but is %s", message.getClass().toString());
         }
     }
 
@@ -48,17 +50,16 @@ public class NodeRefHourAnalysis extends AbstractNodeRefAnalysis {
     }
 
     @Override
-    protected void aggregation() throws Exception {
-        RecordData oneRecord;
-        while ((oneRecord = pushOne()) != null) {
-            getClusterContext().lookup(NodeRefHourAgg.Role.INSTANCE).tell(oneRecord);
+    protected WorkerRefs aggWorkRefs() {
+        try {
+            return getClusterContext().lookup(NodeRefHourAgg.Role.INSTANCE);
+        } catch (WorkerNotFoundException e) {
+            logger.error("The role of %s worker not found", NodeRefHourAgg.Role.INSTANCE.roleName());
         }
+        return null;
     }
 
     public static class Factory extends AbstractLocalAsyncWorkerProvider<NodeRefHourAnalysis> {
-
-        public static Factory INSTANCE = new Factory();
-
         @Override
         public Role role() {
             return Role.INSTANCE;

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/noderef/analysis/NodeRefMinuteAnalysis.java

 package com.a.eye.skywalking.collector.worker.noderef.analysis;
 
-import com.a.eye.skywalking.collector.actor.AbstractLocalAsyncWorkerProvider;
-import com.a.eye.skywalking.collector.actor.ClusterWorkerContext;
-import com.a.eye.skywalking.collector.actor.LocalWorkerContext;
-import com.a.eye.skywalking.collector.actor.ProviderNotFoundException;
+import com.a.eye.skywalking.collector.actor.*;
 import com.a.eye.skywalking.collector.actor.selector.RollingSelector;
 import com.a.eye.skywalking.collector.actor.selector.WorkerSelector;
 import com.a.eye.skywalking.collector.worker.config.WorkerConfig;
 import com.a.eye.skywalking.collector.worker.noderef.persistence.NodeRefMinuteAgg;
 import com.a.eye.skywalking.collector.worker.segment.SegmentPost;
 import com.a.eye.skywalking.collector.worker.segment.entity.Segment;
-import com.a.eye.skywalking.collector.worker.storage.RecordData;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 
 /**
  * @author pengys5
  */
 public class NodeRefMinuteAnalysis extends AbstractNodeRefAnalysis {
 
+    private Logger logger = LogManager.getFormatterLogger(NodeRefMinuteAnalysis.class);
+
     protected NodeRefMinuteAnalysis(com.a.eye.skywalking.collector.actor.Role role, ClusterWorkerContext clusterContext,
                                     LocalWorkerContext selfContext) {
         super(role, clusterContext, selfContext);
@@ -38,6 +38,8 @@ public class NodeRefMinuteAnalysis extends AbstractNodeRefAnalysis {
             long day = segmentWithTimeSlice.getDay();
             int second = segmentWithTimeSlice.getSecond();
             analyseNodeRef(segment, segmentWithTimeSlice.getMinute(), minute, hour, day, second);
+        } else {
+            logger.error("unhandled message, message instance must SegmentPost.SegmentWithTimeSlice, but is %s", message.getClass().toString());
         }
     }
 
@@ -47,17 +49,16 @@ public class NodeRefMinuteAnalysis extends AbstractNodeRefAnalysis {
     }
 
     @Override
-    protected void aggregation() throws Exception {
-        RecordData oneRecord;
-        while ((oneRecord = pushOne()) != null) {
-            getClusterContext().lookup(NodeRefMinuteAgg.Role.INSTANCE).tell(oneRecord);
+    protected WorkerRefs aggWorkRefs() {
+        try {
+            return getClusterContext().lookup(NodeRefMinuteAgg.Role.INSTANCE);
+        } catch (WorkerNotFoundException e) {
+            logger.error("The role of %s worker not found", NodeRefMinuteAgg.Role.INSTANCE.roleName());
         }
+        return null;
     }
 
     public static class Factory extends AbstractLocalAsyncWorkerProvider<NodeRefMinuteAnalysis> {
-
-        public static Factory INSTANCE = new Factory();
-
         @Override
         public Role role() {
             return Role.INSTANCE;

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/noderef/analysis/NodeRefResSumDayAnalysis.java

 package com.a.eye.skywalking.collector.worker.noderef.analysis;
 
-import com.a.eye.skywalking.collector.actor.AbstractLocalAsyncWorkerProvider;
-import com.a.eye.skywalking.collector.actor.ClusterWorkerContext;
-import com.a.eye.skywalking.collector.actor.LocalWorkerContext;
+import com.a.eye.skywalking.collector.actor.*;
 import com.a.eye.skywalking.collector.actor.selector.RollingSelector;
 import com.a.eye.skywalking.collector.actor.selector.WorkerSelector;
 import com.a.eye.skywalking.collector.worker.config.WorkerConfig;
 import com.a.eye.skywalking.collector.worker.noderef.persistence.NodeRefResSumDayAgg;
-import com.a.eye.skywalking.collector.worker.storage.MetricData;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 
 /**
  * @author pengys5
  */
 public class NodeRefResSumDayAnalysis extends AbstractNodeRefResSumAnalysis {
 
+    private Logger logger = LogManager.getFormatterLogger(NodeRefResSumDayAnalysis.class);
+
     NodeRefResSumDayAnalysis(com.a.eye.skywalking.collector.actor.Role role, ClusterWorkerContext clusterContext,
-        LocalWorkerContext selfContext) {
+                             LocalWorkerContext selfContext) {
         super(role, clusterContext, selfContext);
     }
 
     @Override
     public void analyse(Object message) throws Exception {
         if (message instanceof NodeRefResRecord) {
-            NodeRefResRecord refResRecord = (NodeRefResRecord)message;
+            NodeRefResRecord refResRecord = (NodeRefResRecord) message;
             analyseResSum(refResRecord);
+        } else {
+            logger.error("unhandled message, message instance must NodeRefResRecord, but is %s", message.getClass().toString());
         }
     }
 
     @Override
-    protected void aggregation() throws Exception {
-        MetricData oneMetric;
-        while ((oneMetric = pushOne()) != null) {
-            getClusterContext().lookup(NodeRefResSumDayAgg.Role.INSTANCE).tell(oneMetric);
+    protected WorkerRefs aggWorkRefs() {
+        try {
+            return getClusterContext().lookup(NodeRefResSumDayAgg.Role.INSTANCE);
+        } catch (WorkerNotFoundException e) {
+            logger.error("The role of %s worker not found", NodeRefResSumDayAgg.Role.INSTANCE.roleName());
         }
+        return null;
     }
 
     public static class Factory extends AbstractLocalAsyncWorkerProvider<NodeRefResSumDayAnalysis> {
-        public static Factory INSTANCE = new Factory();
-
         @Override
         public Role role() {
             return Role.INSTANCE;

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/noderef/analysis/NodeRefResSumHourAnalysis.java

 package com.a.eye.skywalking.collector.worker.noderef.analysis;
 
-import com.a.eye.skywalking.collector.actor.AbstractLocalAsyncWorkerProvider;
-import com.a.eye.skywalking.collector.actor.ClusterWorkerContext;
-import com.a.eye.skywalking.collector.actor.LocalWorkerContext;
+import com.a.eye.skywalking.collector.actor.*;
 import com.a.eye.skywalking.collector.actor.selector.RollingSelector;
 import com.a.eye.skywalking.collector.actor.selector.WorkerSelector;
 import com.a.eye.skywalking.collector.worker.config.WorkerConfig;
 import com.a.eye.skywalking.collector.worker.noderef.persistence.NodeRefResSumHourAgg;
-import com.a.eye.skywalking.collector.worker.storage.MetricData;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 
 /**
  * @author pengys5
  */
 public class NodeRefResSumHourAnalysis extends AbstractNodeRefResSumAnalysis {
 
+    private Logger logger = LogManager.getFormatterLogger(NodeRefResSumHourAnalysis.class);
+
     NodeRefResSumHourAnalysis(com.a.eye.skywalking.collector.actor.Role role, ClusterWorkerContext clusterContext,
-        LocalWorkerContext selfContext) {
+                              LocalWorkerContext selfContext) {
         super(role, clusterContext, selfContext);
     }
 
     @Override
     public void analyse(Object message) throws Exception {
         if (message instanceof NodeRefResRecord) {
-            NodeRefResRecord refResRecord = (NodeRefResRecord)message;
+            NodeRefResRecord refResRecord = (NodeRefResRecord) message;
             analyseResSum(refResRecord);
+        } else {
+            logger.error("unhandled message, message instance must NodeRefResRecord, but is %s", message.getClass().toString());
         }
     }
 
     @Override
-    protected void aggregation() throws Exception {
-        MetricData oneMetric;
-        while ((oneMetric = pushOne()) != null) {
-            getClusterContext().lookup(NodeRefResSumHourAgg.Role.INSTANCE).tell(oneMetric);
+    protected WorkerRefs aggWorkRefs() {
+        try {
+            return getClusterContext().lookup(NodeRefResSumHourAgg.Role.INSTANCE);
+        } catch (WorkerNotFoundException e) {
+            logger.error("The role of %s worker not found", NodeRefResSumHourAgg.Role.INSTANCE.roleName());
         }
+        return null;
     }
 
     public static class Factory extends AbstractLocalAsyncWorkerProvider<NodeRefResSumHourAnalysis> {
-        public static Factory INSTANCE = new Factory();
-
         @Override
         public Role role() {
             return Role.INSTANCE;

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/noderef/analysis/NodeRefResSumMinuteAnalysis.java

 package com.a.eye.skywalking.collector.worker.noderef.analysis;
 
-import com.a.eye.skywalking.collector.actor.AbstractLocalAsyncWorkerProvider;
-import com.a.eye.skywalking.collector.actor.ClusterWorkerContext;
-import com.a.eye.skywalking.collector.actor.LocalWorkerContext;
+import com.a.eye.skywalking.collector.actor.*;
 import com.a.eye.skywalking.collector.actor.selector.RollingSelector;
 import com.a.eye.skywalking.collector.actor.selector.WorkerSelector;
 import com.a.eye.skywalking.collector.worker.config.WorkerConfig;
 import com.a.eye.skywalking.collector.worker.noderef.persistence.NodeRefResSumMinuteAgg;
-import com.a.eye.skywalking.collector.worker.storage.MetricData;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 
 /**
  * @author pengys5
  */
 public class NodeRefResSumMinuteAnalysis extends AbstractNodeRefResSumAnalysis {
 
+    private Logger logger = LogManager.getFormatterLogger(NodeRefResSumMinuteAnalysis.class);
+
     NodeRefResSumMinuteAnalysis(com.a.eye.skywalking.collector.actor.Role role, ClusterWorkerContext clusterContext,
-        LocalWorkerContext selfContext) {
+                                LocalWorkerContext selfContext) {
         super(role, clusterContext, selfContext);
     }
 
     @Override
     public void analyse(Object message) throws Exception {
         if (message instanceof NodeRefResRecord) {
-            NodeRefResRecord refResRecord = (NodeRefResRecord)message;
+            NodeRefResRecord refResRecord = (NodeRefResRecord) message;
             analyseResSum(refResRecord);
+        } else {
+            logger.error("unhandled message, message instance must NodeRefResRecord, but is %s", message.getClass().toString());
         }
     }
 
     @Override
-    protected void aggregation() throws Exception {
-        MetricData oneMetric;
-        while ((oneMetric = pushOne()) != null) {
-            getClusterContext().lookup(NodeRefResSumMinuteAgg.Role.INSTANCE).tell(oneMetric);
+    protected WorkerRefs aggWorkRefs() {
+        try {
+            return getClusterContext().lookup(NodeRefResSumMinuteAgg.Role.INSTANCE);
+        } catch (WorkerNotFoundException e) {
+            logger.error("The role of %s worker not found", NodeRefResSumMinuteAgg.Role.INSTANCE.roleName());
         }
+        return null;
     }
 
     public static class Factory extends AbstractLocalAsyncWorkerProvider<NodeRefResSumMinuteAnalysis> {
-        public static Factory INSTANCE = new Factory();
-
         @Override
         public Role role() {
             return Role.INSTANCE;

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/noderef/persistence/NodeRefDayAgg.java

@@ -30,13 +30,11 @@ public class NodeRefDayAgg extends AbstractClusterWorker {
         if (message instanceof RecordData) {
             getSelfContext().lookup(NodeRefDaySave.Role.INSTANCE).tell(message);
         } else {
-            logger.error("message unhandled");
+            logger.error("unhandled message, message instance must RecordData, but is %s", message.getClass().toString());
         }
     }
 
     public static class Factory extends AbstractClusterWorkerProvider<NodeRefDayAgg> {
-        public static Factory INSTANCE = new Factory();
-
         @Override
         public Role role() {
             return Role.INSTANCE;

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/noderef/persistence/NodeRefDaySave.java

 package com.a.eye.skywalking.collector.worker.noderef.persistence;
 
-import com.a.eye.skywalking.collector.actor.AbstractLocalAsyncWorkerProvider;
+import com.a.eye.skywalking.collector.actor.AbstractLocalSyncWorkerProvider;
 import com.a.eye.skywalking.collector.actor.ClusterWorkerContext;
 import com.a.eye.skywalking.collector.actor.LocalWorkerContext;
 import com.a.eye.skywalking.collector.actor.selector.HashCodeSelector;
 import com.a.eye.skywalking.collector.actor.selector.WorkerSelector;
 import com.a.eye.skywalking.collector.worker.RecordPersistenceMember;
-import com.a.eye.skywalking.collector.worker.config.WorkerConfig;
 import com.a.eye.skywalking.collector.worker.noderef.NodeRefIndex;
+import com.a.eye.skywalking.collector.worker.storage.PersistenceWorkerListener;
 
 /**
  * @author pengys5
@@ -15,7 +15,7 @@ import com.a.eye.skywalking.collector.worker.noderef.NodeRefIndex;
 public class NodeRefDaySave extends RecordPersistenceMember {
 
     NodeRefDaySave(com.a.eye.skywalking.collector.actor.Role role, ClusterWorkerContext clusterContext,
-        LocalWorkerContext selfContext) {
+                   LocalWorkerContext selfContext) {
         super(role, clusterContext, selfContext);
     }
 
@@ -29,10 +29,7 @@ public class NodeRefDaySave extends RecordPersistenceMember {
         return NodeRefIndex.TYPE_DAY;
     }
 
-    public static class Factory extends AbstractLocalAsyncWorkerProvider<NodeRefDaySave> {
-
-        public static Factory INSTANCE = new Factory();
-
+    public static class Factory extends AbstractLocalSyncWorkerProvider<NodeRefDaySave> {
         @Override
         public Role role() {
             return Role.INSTANCE;
@@ -40,12 +37,9 @@ public class NodeRefDaySave extends RecordPersistenceMember {
 
         @Override
         public NodeRefDaySave workerInstance(ClusterWorkerContext clusterContext) {
-            return new NodeRefDaySave(role(), clusterContext, new LocalWorkerContext());
-        }
-
-        @Override
-        public int queueSize() {
-            return WorkerConfig.Queue.NodeRef.NodeRefDaySave.SIZE;
+            NodeRefDaySave worker = new NodeRefDaySave(role(), clusterContext, new LocalWorkerContext());
+            PersistenceWorkerListener.INSTANCE.register(worker);
+            return worker;
         }
     }
 

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/noderef/persistence/NodeRefHourAgg.java

@@ -30,13 +30,11 @@ public class NodeRefHourAgg extends AbstractClusterWorker {
         if (message instanceof RecordData) {
             getSelfContext().lookup(NodeRefHourSave.Role.INSTANCE).tell(message);
         } else {
-            logger.error("message unhandled");
+            logger.error("unhandled message, message instance must RecordData, but is %s", message.getClass().toString());
         }
     }
 
     public static class Factory extends AbstractClusterWorkerProvider<NodeRefHourAgg> {
-        public static Factory INSTANCE = new Factory();
-
         @Override
         public Role role() {
             return Role.INSTANCE;

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/noderef/persistence/NodeRefHourSave.java

 package com.a.eye.skywalking.collector.worker.noderef.persistence;
 
-import com.a.eye.skywalking.collector.actor.AbstractLocalAsyncWorkerProvider;
+import com.a.eye.skywalking.collector.actor.AbstractLocalSyncWorkerProvider;
 import com.a.eye.skywalking.collector.actor.ClusterWorkerContext;
 import com.a.eye.skywalking.collector.actor.LocalWorkerContext;
 import com.a.eye.skywalking.collector.actor.selector.HashCodeSelector;
 import com.a.eye.skywalking.collector.actor.selector.WorkerSelector;
 import com.a.eye.skywalking.collector.worker.RecordPersistenceMember;
-import com.a.eye.skywalking.collector.worker.config.WorkerConfig;
 import com.a.eye.skywalking.collector.worker.noderef.NodeRefIndex;
+import com.a.eye.skywalking.collector.worker.storage.PersistenceWorkerListener;
 
 /**
  * @author pengys5
@@ -15,7 +15,7 @@ import com.a.eye.skywalking.collector.worker.noderef.NodeRefIndex;
 public class NodeRefHourSave extends RecordPersistenceMember {
 
     NodeRefHourSave(com.a.eye.skywalking.collector.actor.Role role, ClusterWorkerContext clusterContext,
-        LocalWorkerContext selfContext) {
+                    LocalWorkerContext selfContext) {
         super(role, clusterContext, selfContext);
     }
 
@@ -29,10 +29,7 @@ public class NodeRefHourSave extends RecordPersistenceMember {
         return NodeRefIndex.TYPE_HOUR;
     }
 
-    public static class Factory extends AbstractLocalAsyncWorkerProvider<NodeRefHourSave> {
-
-        public static Factory INSTANCE = new Factory();
-
+    public static class Factory extends AbstractLocalSyncWorkerProvider<NodeRefHourSave> {
         @Override
         public Role role() {
             return Role.INSTANCE;
@@ -40,12 +37,9 @@ public class NodeRefHourSave extends RecordPersistenceMember {
 
         @Override
         public NodeRefHourSave workerInstance(ClusterWorkerContext clusterContext) {
-            return new NodeRefHourSave(role(), clusterContext, new LocalWorkerContext());
-        }
-
-        @Override
-        public int queueSize() {
-            return WorkerConfig.Queue.NodeRef.NodeRefHourSave.SIZE;
+            NodeRefHourSave worker = new NodeRefHourSave(role(), clusterContext, new LocalWorkerContext());
+            PersistenceWorkerListener.INSTANCE.register(worker);
+            return worker;
         }
     }
 

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/noderef/persistence/NodeRefMinuteAgg.java

@@ -30,13 +30,11 @@ public class NodeRefMinuteAgg extends AbstractClusterWorker {
         if (message instanceof RecordData) {
             getSelfContext().lookup(NodeRefMinuteSave.Role.INSTANCE).tell(message);
         } else {
-            logger.error("message unhandled");
+            logger.error("unhandled message, message instance must RecordData, but is %s", message.getClass().toString());
         }
     }
 
     public static class Factory extends AbstractClusterWorkerProvider<NodeRefMinuteAgg> {
-        public static Factory INSTANCE = new Factory();
-
         @Override
         public Role role() {
             return Role.INSTANCE;

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/noderef/persistence/NodeRefMinuteSave.java

 package com.a.eye.skywalking.collector.worker.noderef.persistence;
 
-import com.a.eye.skywalking.collector.actor.AbstractLocalAsyncWorkerProvider;
+import com.a.eye.skywalking.collector.actor.AbstractLocalSyncWorkerProvider;
 import com.a.eye.skywalking.collector.actor.ClusterWorkerContext;
 import com.a.eye.skywalking.collector.actor.LocalWorkerContext;
 import com.a.eye.skywalking.collector.actor.selector.HashCodeSelector;
 import com.a.eye.skywalking.collector.actor.selector.WorkerSelector;
 import com.a.eye.skywalking.collector.worker.RecordPersistenceMember;
-import com.a.eye.skywalking.collector.worker.config.WorkerConfig;
 import com.a.eye.skywalking.collector.worker.noderef.NodeRefIndex;
+import com.a.eye.skywalking.collector.worker.storage.PersistenceWorkerListener;
 
 /**
  * @author pengys5
@@ -15,7 +15,7 @@ import com.a.eye.skywalking.collector.worker.noderef.NodeRefIndex;
 public class NodeRefMinuteSave extends RecordPersistenceMember {
 
     NodeRefMinuteSave(com.a.eye.skywalking.collector.actor.Role role, ClusterWorkerContext clusterContext,
-        LocalWorkerContext selfContext) {
+                      LocalWorkerContext selfContext) {
         super(role, clusterContext, selfContext);
     }
 
@@ -29,10 +29,7 @@ public class NodeRefMinuteSave extends RecordPersistenceMember {
         return NodeRefIndex.TYPE_MINUTE;
     }
 
-    public static class Factory extends AbstractLocalAsyncWorkerProvider<NodeRefMinuteSave> {
-
-        public static Factory INSTANCE = new Factory();
-
+    public static class Factory extends AbstractLocalSyncWorkerProvider<NodeRefMinuteSave> {
         @Override
         public Role role() {
             return Role.INSTANCE;
@@ -40,12 +37,9 @@ public class NodeRefMinuteSave extends RecordPersistenceMember {
 
         @Override
         public NodeRefMinuteSave workerInstance(ClusterWorkerContext clusterContext) {
-            return new NodeRefMinuteSave(role(), clusterContext, new LocalWorkerContext());
-        }
-
-        @Override
-        public int queueSize() {
-            return WorkerConfig.Queue.NodeRef.NodeRefMinuteSave.SIZE;
+            NodeRefMinuteSave worker = new NodeRefMinuteSave(role(), clusterContext, new LocalWorkerContext());
+            PersistenceWorkerListener.INSTANCE.register(worker);
+            return worker;
         }
     }
 

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/noderef/persistence/NodeRefResSumDayAgg.java

@@ -30,13 +30,11 @@ public class NodeRefResSumDayAgg extends AbstractClusterWorker {
         if (message instanceof MetricData) {
             getSelfContext().lookup(NodeRefResSumDaySave.Role.INSTANCE).tell(message);
         } else {
-            logger.error("message unhandled");
+            logger.error("unhandled message, message instance must MetricData, but is %s", message.getClass().toString());
         }
     }
 
     public static class Factory extends AbstractClusterWorkerProvider<NodeRefResSumDayAgg> {
-        public static Factory INSTANCE = new Factory();
-
         @Override
         public Role role() {
             return Role.INSTANCE;

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/noderef/persistence/NodeRefResSumDaySave.java

 package com.a.eye.skywalking.collector.worker.noderef.persistence;
 
-import com.a.eye.skywalking.collector.actor.AbstractLocalAsyncWorkerProvider;
+import com.a.eye.skywalking.collector.actor.AbstractLocalSyncWorkerProvider;
 import com.a.eye.skywalking.collector.actor.ClusterWorkerContext;
 import com.a.eye.skywalking.collector.actor.LocalWorkerContext;
 import com.a.eye.skywalking.collector.actor.selector.HashCodeSelector;
 import com.a.eye.skywalking.collector.actor.selector.WorkerSelector;
 import com.a.eye.skywalking.collector.worker.MetricPersistenceMember;
-import com.a.eye.skywalking.collector.worker.config.WorkerConfig;
 import com.a.eye.skywalking.collector.worker.noderef.NodeRefResSumIndex;
+import com.a.eye.skywalking.collector.worker.storage.PersistenceWorkerListener;
 
 /**
  * @author pengys5
@@ -15,7 +15,7 @@ import com.a.eye.skywalking.collector.worker.noderef.NodeRefResSumIndex;
 public class NodeRefResSumDaySave extends MetricPersistenceMember {
 
     NodeRefResSumDaySave(com.a.eye.skywalking.collector.actor.Role role, ClusterWorkerContext clusterContext,
-        LocalWorkerContext selfContext) {
+                         LocalWorkerContext selfContext) {
         super(role, clusterContext, selfContext);
     }
 
@@ -29,9 +29,7 @@ public class NodeRefResSumDaySave extends MetricPersistenceMember {
         return NodeRefResSumIndex.TYPE_DAY;
     }
 
-    public static class Factory extends AbstractLocalAsyncWorkerProvider<NodeRefResSumDaySave> {
-        public static Factory INSTANCE = new Factory();
-
+    public static class Factory extends AbstractLocalSyncWorkerProvider<NodeRefResSumDaySave> {
         @Override
         public Role role() {
             return Role.INSTANCE;
@@ -39,12 +37,9 @@ public class NodeRefResSumDaySave extends MetricPersistenceMember {
 
         @Override
         public NodeRefResSumDaySave workerInstance(ClusterWorkerContext clusterContext) {
-            return new NodeRefResSumDaySave(role(), clusterContext, new LocalWorkerContext());
-        }
-
-        @Override
-        public int queueSize() {
-            return WorkerConfig.Queue.NodeRef.NodeRefResSumDaySave.SIZE;
+            NodeRefResSumDaySave worker = new NodeRefResSumDaySave(role(), clusterContext, new LocalWorkerContext());
+            PersistenceWorkerListener.INSTANCE.register(worker);
+            return worker;
         }
     }
 

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/noderef/persistence/NodeRefResSumGroupWithTimeSlice.java

@@ -80,7 +80,7 @@ public class NodeRefResSumGroupWithTimeSlice extends AbstractLocalSyncWorker {
             JsonObject resJsonObj = (JsonObject)response;
             resJsonObj.add("result", nodeRefResSumArray);
         } else {
-            throw new IllegalArgumentException("message instance must be RequestEntity");
+            logger.error("unhandled message, message instance must NodeRefResSumGroupWithTimeSlice.RequestEntity, but is %s", request.getClass().toString());
         }
     }
 
@@ -91,8 +91,6 @@ public class NodeRefResSumGroupWithTimeSlice extends AbstractLocalSyncWorker {
     }
 
     public static class Factory extends AbstractLocalSyncWorkerProvider<NodeRefResSumGroupWithTimeSlice> {
-        public static Factory INSTANCE = new Factory();
-
         @Override
         public Role role() {
             return WorkerRole.INSTANCE;

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/noderef/persistence/NodeRefResSumHourAgg.java

@@ -30,13 +30,11 @@ public class NodeRefResSumHourAgg extends AbstractClusterWorker {
         if (message instanceof MetricData) {
             getSelfContext().lookup(NodeRefResSumHourSave.Role.INSTANCE).tell(message);
         } else {
-            logger.error("message unhandled");
+            logger.error("unhandled message, message instance must MetricData, but is %s", message.getClass().toString());
         }
     }
 
     public static class Factory extends AbstractClusterWorkerProvider<NodeRefResSumHourAgg> {
-        public static Factory INSTANCE = new Factory();
-
         @Override
         public Role role() {
             return Role.INSTANCE;

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/noderef/persistence/NodeRefResSumHourSave.java

 package com.a.eye.skywalking.collector.worker.noderef.persistence;
 
-import com.a.eye.skywalking.collector.actor.AbstractLocalAsyncWorkerProvider;
+import com.a.eye.skywalking.collector.actor.AbstractLocalSyncWorkerProvider;
 import com.a.eye.skywalking.collector.actor.ClusterWorkerContext;
 import com.a.eye.skywalking.collector.actor.LocalWorkerContext;
 import com.a.eye.skywalking.collector.actor.selector.HashCodeSelector;
 import com.a.eye.skywalking.collector.actor.selector.WorkerSelector;
 import com.a.eye.skywalking.collector.worker.MetricPersistenceMember;
-import com.a.eye.skywalking.collector.worker.config.WorkerConfig;
 import com.a.eye.skywalking.collector.worker.noderef.NodeRefResSumIndex;
+import com.a.eye.skywalking.collector.worker.storage.PersistenceWorkerListener;
 
 /**
  * @author pengys5
@@ -15,7 +15,7 @@ import com.a.eye.skywalking.collector.worker.noderef.NodeRefResSumIndex;
 public class NodeRefResSumHourSave extends MetricPersistenceMember {
 
     NodeRefResSumHourSave(com.a.eye.skywalking.collector.actor.Role role, ClusterWorkerContext clusterContext,
-        LocalWorkerContext selfContext) {
+                          LocalWorkerContext selfContext) {
         super(role, clusterContext, selfContext);
     }
 
@@ -29,9 +29,7 @@ public class NodeRefResSumHourSave extends MetricPersistenceMember {
         return NodeRefResSumIndex.TYPE_HOUR;
     }
 
-    public static class Factory extends AbstractLocalAsyncWorkerProvider<NodeRefResSumHourSave> {
-        public static Factory INSTANCE = new Factory();
-
+    public static class Factory extends AbstractLocalSyncWorkerProvider<NodeRefResSumHourSave> {
         @Override
         public Role role() {
             return Role.INSTANCE;
@@ -39,12 +37,9 @@ public class NodeRefResSumHourSave extends MetricPersistenceMember {
 
         @Override
         public NodeRefResSumHourSave workerInstance(ClusterWorkerContext clusterContext) {
-            return new NodeRefResSumHourSave(role(), clusterContext, new LocalWorkerContext());
-        }
-
-        @Override
-        public int queueSize() {
-            return WorkerConfig.Queue.NodeRef.NodeRefResSumHourSave.SIZE;
+            NodeRefResSumHourSave worker = new NodeRefResSumHourSave(role(), clusterContext, new LocalWorkerContext());
+            PersistenceWorkerListener.INSTANCE.register(worker);
+            return worker;
         }
     }
 

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/noderef/persistence/NodeRefResSumMinuteAgg.java

@@ -30,13 +30,11 @@ public class NodeRefResSumMinuteAgg extends AbstractClusterWorker {
         if (message instanceof MetricData) {
             getSelfContext().lookup(NodeRefResSumMinuteSave.Role.INSTANCE).tell(message);
         } else {
-            logger.error("message unhandled");
+            logger.error("unhandled message, message instance must MetricData, but is %s", message.getClass().toString());
         }
     }
 
     public static class Factory extends AbstractClusterWorkerProvider<NodeRefResSumMinuteAgg> {
-        public static Factory INSTANCE = new Factory();
-
         @Override
         public Role role() {
             return Role.INSTANCE;

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/noderef/persistence/NodeRefResSumMinuteSave.java

 package com.a.eye.skywalking.collector.worker.noderef.persistence;
 
-import com.a.eye.skywalking.collector.actor.AbstractLocalAsyncWorkerProvider;
+import com.a.eye.skywalking.collector.actor.AbstractLocalSyncWorkerProvider;
 import com.a.eye.skywalking.collector.actor.ClusterWorkerContext;
 import com.a.eye.skywalking.collector.actor.LocalWorkerContext;
 import com.a.eye.skywalking.collector.actor.selector.HashCodeSelector;
 import com.a.eye.skywalking.collector.actor.selector.WorkerSelector;
 import com.a.eye.skywalking.collector.worker.MetricPersistenceMember;
-import com.a.eye.skywalking.collector.worker.config.WorkerConfig;
 import com.a.eye.skywalking.collector.worker.noderef.NodeRefResSumIndex;
+import com.a.eye.skywalking.collector.worker.storage.PersistenceWorkerListener;
 
 /**
  * @author pengys5
@@ -15,7 +15,7 @@ import com.a.eye.skywalking.collector.worker.noderef.NodeRefResSumIndex;
 public class NodeRefResSumMinuteSave extends MetricPersistenceMember {
 
     NodeRefResSumMinuteSave(com.a.eye.skywalking.collector.actor.Role role, ClusterWorkerContext clusterContext,
-        LocalWorkerContext selfContext) {
+                            LocalWorkerContext selfContext) {
         super(role, clusterContext, selfContext);
     }
 
@@ -29,9 +29,7 @@ public class NodeRefResSumMinuteSave extends MetricPersistenceMember {
         return NodeRefResSumIndex.TYPE_MINUTE;
     }
 
-    public static class Factory extends AbstractLocalAsyncWorkerProvider<NodeRefResSumMinuteSave> {
-        public static Factory INSTANCE = new Factory();
-
+    public static class Factory extends AbstractLocalSyncWorkerProvider<NodeRefResSumMinuteSave> {
         @Override
         public Role role() {
             return Role.INSTANCE;
@@ -39,12 +37,9 @@ public class NodeRefResSumMinuteSave extends MetricPersistenceMember {
 
         @Override
         public NodeRefResSumMinuteSave workerInstance(ClusterWorkerContext clusterContext) {
-            return new NodeRefResSumMinuteSave(role(), clusterContext, new LocalWorkerContext());
-        }
-
-        @Override
-        public int queueSize() {
-            return WorkerConfig.Queue.NodeRef.NodeRefResSumMinuteSave.SIZE;
+            NodeRefResSumMinuteSave worker = new NodeRefResSumMinuteSave(role(), clusterContext, new LocalWorkerContext());
+            PersistenceWorkerListener.INSTANCE.register(worker);
+            return worker;
         }
     }
 

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/noderef/persistence/NodeRefResSumSearchWithTimeSlice.java

@@ -91,7 +91,7 @@ public class NodeRefResSumSearchWithTimeSlice extends AbstractLocalSyncWorker {
             JsonObject resJsonObj = (JsonObject)response;
             resJsonObj.add("result", nodeRefResSumArray);
         } else {
-            throw new IllegalArgumentException("message instance must be RequestEntity");
+            logger.error("unhandled message, message instance must NodeRefResSumSearchWithTimeSlice.RequestEntity, but is %s", request.getClass().toString());
         }
     }
 
@@ -102,8 +102,6 @@ public class NodeRefResSumSearchWithTimeSlice extends AbstractLocalSyncWorker {
     }
 
     public static class Factory extends AbstractLocalSyncWorkerProvider<NodeRefResSumSearchWithTimeSlice> {
-        public static Factory INSTANCE = new Factory();
-
         @Override
         public Role role() {
             return WorkerRole.INSTANCE;

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/noderef/persistence/NodeRefSearchWithTimeSlice.java

@@ -63,7 +63,7 @@ public class NodeRefSearchWithTimeSlice extends AbstractLocalSyncWorker {
             JsonObject resJsonObj = (JsonObject)response;
             resJsonObj.add("result", nodeRefArray);
         } else {
-            throw new IllegalArgumentException("message instance must be RequestEntity");
+            logger.error("unhandled message, message instance must NodeRefSearchWithTimeSlice.RequestEntity, but is %s", request.getClass().toString());
         }
     }
 
@@ -74,8 +74,6 @@ public class NodeRefSearchWithTimeSlice extends AbstractLocalSyncWorker {
     }
 
     public static class Factory extends AbstractLocalSyncWorkerProvider<NodeRefSearchWithTimeSlice> {
-        public static Factory INSTANCE = new Factory();
-
         @Override
         public Role role() {
             return WorkerRole.INSTANCE;

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/segment/SegmentCostIndex.java

 package com.a.eye.skywalking.collector.worker.segment;
 
+import com.a.eye.skywalking.collector.worker.config.EsConfig;
 import com.a.eye.skywalking.collector.worker.storage.AbstractIndex;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
@@ -28,32 +29,35 @@ public class SegmentCostIndex extends AbstractIndex {
         return true;
     }
 
+    @Override
+    public int refreshInterval() {
+        return EsConfig.Es.Index.RefreshInterval.SegmentCostIndex.VALUE;
+    }
+
     @Override
     public XContentBuilder createMappingBuilder() throws IOException {
         return XContentFactory.jsonBuilder()
-            .startObject()
-            .startObject("properties")
-            .startObject(SEG_ID)
-            .field("type", "string")
-            .field("index", "not_analyzed")
-            .endObject()
-            .startObject(START_TIME)
-            .field("type", "long")
-            .field("index", "not_analyzed")
-            .endObject()
-            .startObject(END_TIME)
-            .field("type", "long")
-            .field("index", "not_analyzed")
-            .endObject()
-            .startObject(OPERATION_NAME)
-            .field("type", "string")
-            .field("index", "not_analyzed")
-            .endObject()
-            .startObject(COST)
-            .field("type", "long")
-            .field("index", "not_analyzed")
-            .endObject()
-            .endObject()
-            .endObject();
+                .startObject()
+                .startObject("properties")
+                .startObject(SEG_ID)
+                .field("type", "keyword")
+                .endObject()
+                .startObject(START_TIME)
+                .field("type", "long")
+                .field("index", "not_analyzed")
+                .endObject()
+                .startObject(END_TIME)
+                .field("type", "long")
+                .field("index", "not_analyzed")
+                .endObject()
+                .startObject(OPERATION_NAME)
+                .field("type", "keyword")
+                .endObject()
+                .startObject(COST)
+                .field("type", "long")
+                .field("index", "not_analyzed")
+                .endObject()
+                .endObject()
+                .endObject();
     }
 }

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/segment/SegmentExceptionIndex.java

 package com.a.eye.skywalking.collector.worker.segment;
 
+import com.a.eye.skywalking.collector.worker.config.EsConfig;
 import com.a.eye.skywalking.collector.worker.storage.AbstractIndex;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
@@ -25,25 +26,28 @@ public class SegmentExceptionIndex extends AbstractIndex {
         return true;
     }
 
+    @Override
+    public int refreshInterval() {
+        return EsConfig.Es.Index.RefreshInterval.SegmentExceptionIndex.VALUE;
+    }
+
     @Override
     public XContentBuilder createMappingBuilder() throws IOException {
         XContentBuilder mappingBuilder = XContentFactory.jsonBuilder()
-            .startObject()
-            .startObject("properties")
-            .startObject(SEG_ID)
-            .field("type", "string")
-            .field("index", "not_analyzed")
-            .endObject()
-            .startObject(IS_ERROR)
-            .field("type", "boolean")
-            .field("index", "not_analyzed")
-            .endObject()
-            .startObject(ERROR_KIND)
-            .field("type", "string")
-            .field("index", "not_analyzed")
-            .endObject()
-            .endObject()
-            .endObject();
+                .startObject()
+                .startObject("properties")
+                .startObject(SEG_ID)
+                .field("type", "keyword")
+                .endObject()
+                .startObject(IS_ERROR)
+                .field("type", "boolean")
+                .field("index", "not_analyzed")
+                .endObject()
+                .startObject(ERROR_KIND)
+                .field("type", "keyword")
+                .endObject()
+                .endObject()
+                .endObject();
         return mappingBuilder;
     }
 }

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/segment/SegmentIndex.java

 package com.a.eye.skywalking.collector.worker.segment;
 
+import com.a.eye.skywalking.collector.worker.config.EsConfig;
 import com.a.eye.skywalking.collector.worker.storage.AbstractIndex;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
@@ -23,40 +24,43 @@ public class SegmentIndex extends AbstractIndex {
         return true;
     }
 
+    @Override
+    public int refreshInterval() {
+        return EsConfig.Es.Index.RefreshInterval.SegmentIndex.VALUE;
+    }
+
     @Override
     public XContentBuilder createMappingBuilder() throws IOException {
         return XContentFactory.jsonBuilder()
-            .startObject()
-            .startObject("properties")
-            .startObject("traceSegmentId")
-            .field("type", "string")
-            .field("index", "not_analyzed")
-            .endObject()
-            .startObject("startTime")
-            .field("type", "date")
-            .field("index", "not_analyzed")
-            .endObject()
-            .startObject("endTime")
-            .field("type", "date")
-            .field("index", "not_analyzed")
-            .endObject()
-            .startObject("applicationCode")
-            .field("type", "string")
-            .field("index", "not_analyzed")
-            .endObject()
-            .startObject("minute")
-            .field("type", "long")
-            .field("index", "not_analyzed")
-            .endObject()
-            .startObject("hour")
-            .field("type", "long")
-            .field("index", "not_analyzed")
-            .endObject()
-            .startObject("day")
-            .field("type", "long")
-            .field("index", "not_analyzed")
-            .endObject()
-            .endObject()
-            .endObject();
+                .startObject()
+                .startObject("properties")
+                .startObject("traceSegmentId")
+                .field("type", "keyword")
+                .endObject()
+                .startObject("startTime")
+                .field("type", "date")
+                .field("index", "not_analyzed")
+                .endObject()
+                .startObject("endTime")
+                .field("type", "date")
+                .field("index", "not_analyzed")
+                .endObject()
+                .startObject("applicationCode")
+                .field("type", "keyword")
+                .endObject()
+                .startObject("minute")
+                .field("type", "long")
+                .field("index", "not_analyzed")
+                .endObject()
+                .startObject("hour")
+                .field("type", "long")
+                .field("index", "not_analyzed")
+                .endObject()
+                .startObject("day")
+                .field("type", "long")
+                .field("index", "not_analyzed")
+                .endObject()
+                .endObject()
+                .endObject();
     }
 }

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/segment/SegmentPost.java

@@ -18,10 +18,10 @@ import com.a.eye.skywalking.collector.worker.node.analysis.NodeMappingMinuteAnal
 import com.a.eye.skywalking.collector.worker.noderef.analysis.NodeRefDayAnalysis;
 import com.a.eye.skywalking.collector.worker.noderef.analysis.NodeRefHourAnalysis;
 import com.a.eye.skywalking.collector.worker.noderef.analysis.NodeRefMinuteAnalysis;
+import com.a.eye.skywalking.collector.worker.segment.analysis.SegmentAnalysis;
+import com.a.eye.skywalking.collector.worker.segment.analysis.SegmentCostAnalysis;
+import com.a.eye.skywalking.collector.worker.segment.analysis.SegmentExceptionAnalysis;
 import com.a.eye.skywalking.collector.worker.segment.entity.Segment;
-import com.a.eye.skywalking.collector.worker.segment.persistence.SegmentCostSave;
-import com.a.eye.skywalking.collector.worker.segment.persistence.SegmentExceptionSave;
-import com.a.eye.skywalking.collector.worker.segment.persistence.SegmentSave;
 import com.a.eye.skywalking.collector.worker.storage.AbstractTimeSlice;
 import com.a.eye.skywalking.collector.worker.tools.DateTools;
 import org.apache.logging.log4j.LogManager;
@@ -41,16 +41,16 @@ public class SegmentPost extends AbstractPost {
     public void preStart() throws ProviderNotFoundException {
         getClusterContext().findProvider(GlobalTraceAnalysis.Role.INSTANCE).create(this);
 
-        getClusterContext().findProvider(NodeCompAnalysis.Role.INSTANCE).create(this);
-
-        getClusterContext().findProvider(SegmentSave.Role.INSTANCE).create(this);
-        getClusterContext().findProvider(SegmentCostSave.Role.INSTANCE).create(this);
-        getClusterContext().findProvider(SegmentExceptionSave.Role.INSTANCE).create(this);
+        getClusterContext().findProvider(SegmentAnalysis.Role.INSTANCE).create(this);
+        getClusterContext().findProvider(SegmentCostAnalysis.Role.INSTANCE).create(this);
+        getClusterContext().findProvider(SegmentExceptionAnalysis.Role.INSTANCE).create(this);
 
         getClusterContext().findProvider(NodeRefMinuteAnalysis.Role.INSTANCE).create(this);
         getClusterContext().findProvider(NodeRefHourAnalysis.Role.INSTANCE).create(this);
         getClusterContext().findProvider(NodeRefDayAnalysis.Role.INSTANCE).create(this);
 
+        getClusterContext().findProvider(NodeCompAnalysis.Role.INSTANCE).create(this);
+
         getClusterContext().findProvider(NodeMappingDayAnalysis.Role.INSTANCE).create(this);
         getClusterContext().findProvider(NodeMappingHourAnalysis.Role.INSTANCE).create(this);
         getClusterContext().findProvider(NodeMappingMinuteAnalysis.Role.INSTANCE).create(this);
@@ -75,11 +75,11 @@ public class SegmentPost extends AbstractPost {
             logger.debug("minuteSlice: %s, hourSlice: %s, daySlice: %s, second:%s", minuteSlice, hourSlice, daySlice, second);
 
             SegmentWithTimeSlice segmentWithTimeSlice = new SegmentWithTimeSlice(segment, minuteSlice, hourSlice, daySlice, second);
-            getSelfContext().lookup(SegmentSave.Role.INSTANCE).tell(segment);
+            getSelfContext().lookup(SegmentAnalysis.Role.INSTANCE).tell(segment);
 
-            getSelfContext().lookup(SegmentCostSave.Role.INSTANCE).tell(segmentWithTimeSlice);
+            getSelfContext().lookup(SegmentCostAnalysis.Role.INSTANCE).tell(segmentWithTimeSlice);
             getSelfContext().lookup(GlobalTraceAnalysis.Role.INSTANCE).tell(segmentWithTimeSlice);
-            getSelfContext().lookup(SegmentExceptionSave.Role.INSTANCE).tell(segmentWithTimeSlice);
+            getSelfContext().lookup(SegmentExceptionAnalysis.Role.INSTANCE).tell(segmentWithTimeSlice);
 
             getSelfContext().lookup(NodeCompAnalysis.Role.INSTANCE).tell(segmentWithTimeSlice);
 
@@ -110,8 +110,6 @@ public class SegmentPost extends AbstractPost {
     }
 
     public static class Factory extends AbstractPostProvider<SegmentPost> {
-        public static Factory INSTANCE = new Factory();
-
         @Override
         public String servletPath() {
             return "/segments";

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/segment/SegmentTopGetWithGlobalTraceId.java

@@ -62,8 +62,6 @@ public class SegmentTopGetWithGlobalTraceId extends AbstractGet {
     }
 
     public static class Factory extends AbstractGetProvider<SegmentTopGetWithGlobalTraceId> {
-        public static Factory INSTANCE = new Factory();
-
         @Override
         public Role role() {
             return WorkerRole.INSTANCE;

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/segment/SegmentTopGetWithTimeSlice.java

@@ -86,8 +86,6 @@ public class SegmentTopGetWithTimeSlice extends AbstractGet {
     }
 
     public static class Factory extends AbstractGetProvider<SegmentTopGetWithTimeSlice> {
-        public static Factory INSTANCE = new Factory();
-
         @Override
         public Role role() {
             return WorkerRole.INSTANCE;

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/segment/analysis/SegmentAnalysis.java

+package com.a.eye.skywalking.collector.worker.segment.analysis;
+
+import com.a.eye.skywalking.collector.actor.*;
+import com.a.eye.skywalking.collector.actor.selector.RollingSelector;
+import com.a.eye.skywalking.collector.actor.selector.WorkerSelector;
+import com.a.eye.skywalking.collector.worker.RecordAnalysisMember;
+import com.a.eye.skywalking.collector.worker.config.WorkerConfig;
+import com.a.eye.skywalking.collector.worker.segment.entity.Segment;
+import com.a.eye.skywalking.collector.worker.segment.persistence.SegmentSave;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+/**
+ * @author pengys5
+ */
+public class SegmentAnalysis extends RecordAnalysisMember {
+
+    private Logger logger = LogManager.getFormatterLogger(SegmentAnalysis.class);
+
+    SegmentAnalysis(Role role, ClusterWorkerContext clusterContext, LocalWorkerContext selfContext) {
+        super(role, clusterContext, selfContext);
+    }
+
+    @Override
+    public void preStart() throws ProviderNotFoundException {
+        getClusterContext().findProvider(SegmentSave.Role.INSTANCE).create(this);
+    }
+
+    @Override
+    public void analyse(Object message) throws Exception {
+        if (message instanceof Segment) {
+            Segment segment = (Segment) message;
+            getSelfContext().lookup(SegmentSave.Role.INSTANCE).tell(segment);
+        } else {
+            logger.error("unhandled message, message instance must Segment, but is %s", message.getClass().toString());
+        }
+    }
+
+    @Override
+    protected WorkerRefs aggWorkRefs() {
+        return null;
+    }
+
+    public static class Factory extends AbstractLocalAsyncWorkerProvider<SegmentAnalysis> {
+        @Override
+        public Role role() {
+            return SegmentAnalysis.Role.INSTANCE;
+        }
+
+        @Override
+        public SegmentAnalysis workerInstance(ClusterWorkerContext clusterContext) {
+            return new SegmentAnalysis(role(), clusterContext, new LocalWorkerContext());
+        }
+
+        @Override
+        public int queueSize() {
+            return WorkerConfig.Queue.Segment.SegmentAnalysis.SIZE;
+        }
+    }
+
+    public enum Role implements com.a.eye.skywalking.collector.actor.Role {
+        INSTANCE;
+
+        @Override
+        public String roleName() {
+            return SegmentAnalysis.class.getSimpleName();
+        }
+
+        @Override
+        public WorkerSelector workerSelector() {
+            return new RollingSelector();
+        }
+    }
+}

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/segment/analysis/SegmentCostAnalysis.java

+package com.a.eye.skywalking.collector.worker.segment.analysis;
+
+import com.a.eye.skywalking.collector.actor.*;
+import com.a.eye.skywalking.collector.actor.selector.RollingSelector;
+import com.a.eye.skywalking.collector.actor.selector.WorkerSelector;
+import com.a.eye.skywalking.collector.worker.RecordAnalysisMember;
+import com.a.eye.skywalking.collector.worker.config.WorkerConfig;
+import com.a.eye.skywalking.collector.worker.segment.SegmentCostIndex;
+import com.a.eye.skywalking.collector.worker.segment.SegmentPost;
+import com.a.eye.skywalking.collector.worker.segment.entity.Segment;
+import com.a.eye.skywalking.collector.worker.segment.entity.Span;
+import com.a.eye.skywalking.collector.worker.segment.persistence.SegmentCostSave;
+import com.a.eye.skywalking.collector.worker.tools.CollectionTools;
+import com.google.gson.JsonObject;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+/**
+ * @author pengys5
+ */
+public class SegmentCostAnalysis extends RecordAnalysisMember {
+
+    private Logger logger = LogManager.getFormatterLogger(SegmentCostAnalysis.class);
+
+    SegmentCostAnalysis(Role role, ClusterWorkerContext clusterContext, LocalWorkerContext selfContext) {
+        super(role, clusterContext, selfContext);
+    }
+
+    @Override
+    public void preStart() throws ProviderNotFoundException {
+        getClusterContext().findProvider(SegmentCostSave.Role.INSTANCE).create(this);
+    }
+
+    @Override
+    public void analyse(Object message) throws Exception {
+        if (message instanceof SegmentPost.SegmentWithTimeSlice) {
+            SegmentPost.SegmentWithTimeSlice segmentWithTimeSlice = (SegmentPost.SegmentWithTimeSlice) message;
+            Segment segment = segmentWithTimeSlice.getSegment();
+
+            if (CollectionTools.isNotEmpty(segment.getSpans())) {
+                for (Span span : segment.getSpans()) {
+                    if (span.getParentSpanId() == -1) {
+                        JsonObject dataJsonObj = new JsonObject();
+                        dataJsonObj.addProperty(SegmentCostIndex.SEG_ID, segment.getTraceSegmentId());
+                        dataJsonObj.addProperty(SegmentCostIndex.START_TIME, span.getStartTime());
+                        dataJsonObj.addProperty(SegmentCostIndex.END_TIME, span.getEndTime());
+                        dataJsonObj.addProperty(SegmentCostIndex.OPERATION_NAME, span.getOperationName());
+                        dataJsonObj.addProperty(SegmentCostIndex.TIME_SLICE, segmentWithTimeSlice.getMinute());
+
+                        long startTime = span.getStartTime();
+                        long endTime = span.getEndTime();
+                        long cost = endTime - startTime;
+                        if (cost == 0) {
+                            cost = 1;
+                        }
+                        dataJsonObj.addProperty(SegmentCostIndex.COST, cost);
+                        set(segment.getTraceSegmentId(), dataJsonObj);
+                    }
+                }
+            }
+        } else {
+            logger.error("unhandled message, message instance must SegmentPost.SegmentWithTimeSlice, but is %s", message.getClass().toString());
+        }
+    }
+
+    @Override
+    protected WorkerRefs aggWorkRefs() {
+        try {
+            return getSelfContext().lookup(SegmentCostSave.Role.INSTANCE);
+        } catch (WorkerNotFoundException e) {
+            logger.error("The role of %s worker not found", SegmentCostSave.Role.INSTANCE.roleName());
+        }
+        return null;
+    }
+
+    public static class Factory extends AbstractLocalAsyncWorkerProvider<SegmentCostAnalysis> {
+        @Override
+        public Role role() {
+            return SegmentCostAnalysis.Role.INSTANCE;
+        }
+
+        @Override
+        public SegmentCostAnalysis workerInstance(ClusterWorkerContext clusterContext) {
+            return new SegmentCostAnalysis(role(), clusterContext, new LocalWorkerContext());
+        }
+
+        @Override
+        public int queueSize() {
+            return WorkerConfig.Queue.Segment.SegmentCostAnalysis.SIZE;
+        }
+    }
+
+    public enum Role implements com.a.eye.skywalking.collector.actor.Role {
+        INSTANCE;
+
+        @Override
+        public String roleName() {
+            return SegmentCostAnalysis.class.getSimpleName();
+        }
+
+        @Override
+        public WorkerSelector workerSelector() {
+            return new RollingSelector();
+        }
+    }
+}

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/segment/analysis/SegmentExceptionAnalysis.java

+package com.a.eye.skywalking.collector.worker.segment.analysis;
+
+import com.a.eye.skywalking.collector.actor.*;
+import com.a.eye.skywalking.collector.actor.selector.RollingSelector;
+import com.a.eye.skywalking.collector.actor.selector.WorkerSelector;
+import com.a.eye.skywalking.collector.worker.RecordAnalysisMember;
+import com.a.eye.skywalking.collector.worker.config.WorkerConfig;
+import com.a.eye.skywalking.collector.worker.segment.SegmentExceptionIndex;
+import com.a.eye.skywalking.collector.worker.segment.SegmentPost;
+import com.a.eye.skywalking.collector.worker.segment.entity.LogData;
+import com.a.eye.skywalking.collector.worker.segment.entity.Segment;
+import com.a.eye.skywalking.collector.worker.segment.entity.Span;
+import com.a.eye.skywalking.collector.worker.segment.entity.tag.Tags;
+import com.a.eye.skywalking.collector.worker.segment.persistence.SegmentExceptionSave;
+import com.a.eye.skywalking.collector.worker.tools.CollectionTools;
+import com.google.gson.JsonArray;
+import com.google.gson.JsonObject;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.util.List;
+
+/**
+ * @author pengys5
+ */
+public class SegmentExceptionAnalysis extends RecordAnalysisMember {
+
+    private Logger logger = LogManager.getFormatterLogger(SegmentExceptionAnalysis.class);
+
+    SegmentExceptionAnalysis(Role role, ClusterWorkerContext clusterContext, LocalWorkerContext selfContext) {
+        super(role, clusterContext, selfContext);
+    }
+
+    @Override
+    public void preStart() throws ProviderNotFoundException {
+        getClusterContext().findProvider(SegmentExceptionSave.Role.INSTANCE).create(this);
+    }
+
+    @Override
+    public void analyse(Object message) throws Exception {
+        if (message instanceof SegmentPost.SegmentWithTimeSlice) {
+            SegmentPost.SegmentWithTimeSlice segmentWithTimeSlice = (SegmentPost.SegmentWithTimeSlice) message;
+            Segment segment = segmentWithTimeSlice.getSegment();
+
+            if (CollectionTools.isNotEmpty(segment.getSpans())) {
+                for (Span span : segment.getSpans()) {
+                    boolean isError = Tags.ERROR.get(span);
+
+                    JsonObject dataJsonObj = new JsonObject();
+                    dataJsonObj.addProperty(SegmentExceptionIndex.IS_ERROR, isError);
+                    dataJsonObj.addProperty(SegmentExceptionIndex.SEG_ID, segment.getTraceSegmentId());
+
+                    JsonArray errorKind = new JsonArray();
+                    if (isError) {
+                        List<LogData> logDataList = span.getLogs();
+                        for (LogData logData : logDataList) {
+                            if (logData.getFields().containsKey("error.kind")) {
+                                errorKind.add(String.valueOf(logData.getFields().get("error.kind")));
+                            }
+                        }
+                    }
+                    dataJsonObj.add(SegmentExceptionIndex.ERROR_KIND, errorKind);
+                    set(segment.getTraceSegmentId(), dataJsonObj);
+                }
+            }
+        } else {
+            logger.error("unhandled message, message instance must SegmentPost.SegmentWithTimeSlice, but is %s", message.getClass().toString());
+        }
+    }
+
+    @Override
+    protected WorkerRefs aggWorkRefs() {
+        try {
+            return getSelfContext().lookup(SegmentExceptionSave.Role.INSTANCE);
+        } catch (WorkerNotFoundException e) {
+            logger.error("The role of %s worker not found", SegmentExceptionSave.Role.INSTANCE.roleName());
+        }
+        return null;
+    }
+
+    public static class Factory extends AbstractLocalAsyncWorkerProvider<SegmentExceptionAnalysis> {
+        @Override
+        public Role role() {
+            return SegmentExceptionAnalysis.Role.INSTANCE;
+        }
+
+        @Override
+        public SegmentExceptionAnalysis workerInstance(ClusterWorkerContext clusterContext) {
+            return new SegmentExceptionAnalysis(role(), clusterContext, new LocalWorkerContext());
+        }
+
+        @Override
+        public int queueSize() {
+            return WorkerConfig.Queue.Segment.SegmentExceptionAnalysis.SIZE;
+        }
+    }
+
+    public enum Role implements com.a.eye.skywalking.collector.actor.Role {
+        INSTANCE;
+
+        @Override
+        public String roleName() {
+            return SegmentExceptionAnalysis.class.getSimpleName();
+        }
+
+        @Override
+        public WorkerSelector workerSelector() {
+            return new RollingSelector();
+        }
+    }
+}

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/segment/entity/TraceSegmentRef.java

@@ -40,7 +40,7 @@ public class TraceSegmentRef extends DeserializeObject {
         reader.beginObject();
         while (reader.hasNext()) {
             switch (reader.nextName()) {
-                case "rs":
+                case "ts":
                     String ts = reader.nextString();
                     this.traceSegmentId = ts;
                     JsonBuilder.INSTANCE.append(stringBuilder, "ts", ts, first);

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/segment/persistence/SegmentCostSave.java

 package com.a.eye.skywalking.collector.worker.segment.persistence;
 
-import com.a.eye.skywalking.collector.actor.AbstractLocalAsyncWorkerProvider;
+import com.a.eye.skywalking.collector.actor.AbstractLocalSyncWorkerProvider;
 import com.a.eye.skywalking.collector.actor.ClusterWorkerContext;
 import com.a.eye.skywalking.collector.actor.LocalWorkerContext;
 import com.a.eye.skywalking.collector.actor.selector.RollingSelector;
 import com.a.eye.skywalking.collector.actor.selector.WorkerSelector;
 import com.a.eye.skywalking.collector.worker.RecordPersistenceMember;
-import com.a.eye.skywalking.collector.worker.config.WorkerConfig;
 import com.a.eye.skywalking.collector.worker.segment.SegmentCostIndex;
-import com.a.eye.skywalking.collector.worker.segment.SegmentPost;
-import com.a.eye.skywalking.collector.worker.segment.entity.Segment;
-import com.a.eye.skywalking.collector.worker.segment.entity.Span;
-import com.a.eye.skywalking.collector.worker.storage.RecordData;
-import com.a.eye.skywalking.collector.worker.tools.CollectionTools;
-import com.google.gson.JsonObject;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
+import com.a.eye.skywalking.collector.worker.storage.PersistenceWorkerListener;
 
 /**
  * @author pengys5
  */
 public class SegmentCostSave extends RecordPersistenceMember {
-
-    private Logger logger = LogManager.getFormatterLogger(SegmentCostSave.class);
-
     @Override
     public String esIndex() {
         return SegmentCostIndex.INDEX;
@@ -35,61 +24,21 @@ public class SegmentCostSave extends RecordPersistenceMember {
     }
 
     protected SegmentCostSave(com.a.eye.skywalking.collector.actor.Role role, ClusterWorkerContext clusterContext,
-        LocalWorkerContext selfContext) {
+                              LocalWorkerContext selfContext) {
         super(role, clusterContext, selfContext);
     }
 
-    @Override
-    public void analyse(Object message) throws Exception {
-        if (message instanceof SegmentPost.SegmentWithTimeSlice) {
-            SegmentPost.SegmentWithTimeSlice segmentWithTimeSlice = (SegmentPost.SegmentWithTimeSlice)message;
-            Segment segment = segmentWithTimeSlice.getSegment();
-
-            if (CollectionTools.isNotEmpty(segment.getSpans())) {
-                for (Span span : segment.getSpans()) {
-                    if (span.getParentSpanId() == -1) {
-                        JsonObject dataJsonObj = new JsonObject();
-                        dataJsonObj.addProperty(SegmentCostIndex.SEG_ID, segment.getTraceSegmentId());
-                        dataJsonObj.addProperty(SegmentCostIndex.START_TIME, span.getStartTime());
-                        dataJsonObj.addProperty(SegmentCostIndex.END_TIME, span.getEndTime());
-                        dataJsonObj.addProperty(SegmentCostIndex.OPERATION_NAME, span.getOperationName());
-                        dataJsonObj.addProperty(SegmentCostIndex.TIME_SLICE, segmentWithTimeSlice.getMinute());
-
-                        long startTime = span.getStartTime();
-                        long endTime = span.getEndTime();
-                        long cost = endTime - startTime;
-                        if (cost == 0) {
-                            cost = 1;
-                        }
-                        dataJsonObj.addProperty(SegmentCostIndex.COST, cost);
-
-                        RecordData recordData = new RecordData(segment.getTraceSegmentId());
-                        recordData.setRecord(dataJsonObj);
-                        super.analyse(recordData);
-                    }
-                }
-            }
-        } else {
-            logger.error("unhandled message, message instance must JsonObject, but is %s", message.getClass().toString());
-        }
-    }
-
-    public static class Factory extends AbstractLocalAsyncWorkerProvider<SegmentCostSave> {
-        public static Factory INSTANCE = new Factory();
-
+    public static class Factory extends AbstractLocalSyncWorkerProvider<SegmentCostSave> {
         @Override
         public Role role() {
             return Role.INSTANCE;
         }
 
-        @Override
-        public int queueSize() {
-            return WorkerConfig.Queue.Segment.SegmentCostSave.SIZE;
-        }
-
         @Override
         public SegmentCostSave workerInstance(ClusterWorkerContext clusterContext) {
-            return new SegmentCostSave(role(), clusterContext, new LocalWorkerContext());
+            SegmentCostSave worker = new SegmentCostSave(role(), clusterContext, new LocalWorkerContext());
+            PersistenceWorkerListener.INSTANCE.register(worker);
+            return worker;
         }
     }
 

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/segment/persistence/SegmentExceptionSave.java

 package com.a.eye.skywalking.collector.worker.segment.persistence;
 
-import com.a.eye.skywalking.collector.actor.AbstractLocalAsyncWorkerProvider;
+import com.a.eye.skywalking.collector.actor.AbstractLocalSyncWorkerProvider;
 import com.a.eye.skywalking.collector.actor.ClusterWorkerContext;
 import com.a.eye.skywalking.collector.actor.LocalWorkerContext;
 import com.a.eye.skywalking.collector.actor.selector.RollingSelector;
 import com.a.eye.skywalking.collector.actor.selector.WorkerSelector;
 import com.a.eye.skywalking.collector.worker.RecordPersistenceMember;
-import com.a.eye.skywalking.collector.worker.config.WorkerConfig;
 import com.a.eye.skywalking.collector.worker.segment.SegmentExceptionIndex;
-import com.a.eye.skywalking.collector.worker.segment.SegmentPost;
-import com.a.eye.skywalking.collector.worker.segment.entity.LogData;
-import com.a.eye.skywalking.collector.worker.segment.entity.Segment;
-import com.a.eye.skywalking.collector.worker.segment.entity.Span;
-import com.a.eye.skywalking.collector.worker.segment.entity.tag.Tags;
 import com.a.eye.skywalking.collector.worker.storage.AbstractIndex;
-import com.a.eye.skywalking.collector.worker.storage.RecordData;
-import com.a.eye.skywalking.collector.worker.tools.CollectionTools;
-import com.google.gson.JsonArray;
-import com.google.gson.JsonObject;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-
-import java.util.List;
+import com.a.eye.skywalking.collector.worker.storage.PersistenceWorkerListener;
 
 /**
  * @author pengys5
  */
 public class SegmentExceptionSave extends RecordPersistenceMember {
-
-    private Logger logger = LogManager.getFormatterLogger(SegmentExceptionSave.class);
-
     @Override
     public String esIndex() {
         return SegmentExceptionIndex.INDEX;
@@ -41,61 +25,21 @@ public class SegmentExceptionSave extends RecordPersistenceMember {
     }
 
     protected SegmentExceptionSave(com.a.eye.skywalking.collector.actor.Role role, ClusterWorkerContext clusterContext,
-        LocalWorkerContext selfContext) {
+                                   LocalWorkerContext selfContext) {
         super(role, clusterContext, selfContext);
     }
 
-    @Override
-    public void analyse(Object message) throws Exception {
-        if (message instanceof SegmentPost.SegmentWithTimeSlice) {
-            SegmentPost.SegmentWithTimeSlice segmentWithTimeSlice = (SegmentPost.SegmentWithTimeSlice)message;
-            Segment segment = segmentWithTimeSlice.getSegment();
-
-            if (CollectionTools.isNotEmpty(segment.getSpans())) {
-                for (Span span : segment.getSpans()) {
-                    boolean isError = Tags.ERROR.get(span);
-
-                    JsonObject dataJsonObj = new JsonObject();
-                    dataJsonObj.addProperty(SegmentExceptionIndex.IS_ERROR, isError);
-                    dataJsonObj.addProperty(SegmentExceptionIndex.SEG_ID, segment.getTraceSegmentId());
-
-                    JsonArray errorKind = new JsonArray();
-                    if (isError) {
-                        List<LogData> logDataList = span.getLogs();
-                        for (LogData logData : logDataList) {
-                            if (logData.getFields().containsKey("error.kind")) {
-                                errorKind.add(String.valueOf(logData.getFields().get("error.kind")));
-                            }
-                        }
-                    }
-                    dataJsonObj.add(SegmentExceptionIndex.ERROR_KIND, errorKind);
-
-                    RecordData recordData = new RecordData(segment.getTraceSegmentId());
-                    recordData.setRecord(dataJsonObj);
-                    super.analyse(recordData);
-                }
-            }
-        } else {
-            logger.error("unhandled message, message instance must JsonObject, but is %s", message.getClass().toString());
-        }
-    }
-
-    public static class Factory extends AbstractLocalAsyncWorkerProvider<SegmentExceptionSave> {
-        public static Factory INSTANCE = new Factory();
-
+    public static class Factory extends AbstractLocalSyncWorkerProvider<SegmentExceptionSave> {
         @Override
         public Role role() {
             return Role.INSTANCE;
         }
 
-        @Override
-        public int queueSize() {
-            return WorkerConfig.Queue.Segment.SegmentExceptionSave.SIZE;
-        }
-
         @Override
         public SegmentExceptionSave workerInstance(ClusterWorkerContext clusterContext) {
-            return new SegmentExceptionSave(role(), clusterContext, new LocalWorkerContext());
+            SegmentExceptionSave worker = new SegmentExceptionSave(role(), clusterContext, new LocalWorkerContext());
+            PersistenceWorkerListener.INSTANCE.register(worker);
+            return worker;
         }
     }
 

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/segment/persistence/SegmentExceptionWithSegId.java

@@ -42,8 +42,6 @@ public class SegmentExceptionWithSegId extends AbstractLocalSyncWorker {
     }
 
     public static class Factory extends AbstractLocalSyncWorkerProvider<SegmentExceptionWithSegId> {
-        public static Factory INSTANCE = new Factory();
-
         @Override
         public Role role() {
             return WorkerRole.INSTANCE;

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/segment/persistence/SegmentSave.java

 package com.a.eye.skywalking.collector.worker.segment.persistence;
 
-import com.a.eye.skywalking.collector.actor.AbstractLocalAsyncWorkerProvider;
+import com.a.eye.skywalking.collector.actor.AbstractLocalSyncWorkerProvider;
 import com.a.eye.skywalking.collector.actor.ClusterWorkerContext;
 import com.a.eye.skywalking.collector.actor.LocalWorkerContext;
 import com.a.eye.skywalking.collector.actor.selector.RollingSelector;
 import com.a.eye.skywalking.collector.actor.selector.WorkerSelector;
-import com.a.eye.skywalking.collector.worker.RecordPersistenceMember;
+import com.a.eye.skywalking.collector.worker.PersistenceMember;
 import com.a.eye.skywalking.collector.worker.config.CacheSizeConfig;
-import com.a.eye.skywalking.collector.worker.config.WorkerConfig;
 import com.a.eye.skywalking.collector.worker.segment.SegmentIndex;
 import com.a.eye.skywalking.collector.worker.segment.entity.Segment;
 import com.a.eye.skywalking.collector.worker.storage.AbstractIndex;
 import com.a.eye.skywalking.collector.worker.storage.EsClient;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.elasticsearch.action.bulk.BulkRequestBuilder;
-import org.elasticsearch.action.bulk.BulkResponse;
-import org.elasticsearch.client.Client;
-
-import java.util.LinkedHashMap;
+import com.a.eye.skywalking.collector.worker.storage.PersistenceWorkerListener;
+import com.a.eye.skywalking.collector.worker.storage.SegmentData;
+import com.a.eye.skywalking.collector.worker.storage.SegmentPersistenceData;
+import java.util.LinkedList;
+import java.util.List;
 import java.util.Map;
+import org.elasticsearch.action.index.IndexRequestBuilder;
+import org.elasticsearch.client.Client;
 
 /**
  * @author pengys5
  */
-public class SegmentSave extends RecordPersistenceMember {
-
-    private Logger logger = LogManager.getFormatterLogger(SegmentSave.class);
-
-    private Map<String, String> persistenceData = new LinkedHashMap<>();
+public class SegmentSave extends PersistenceMember<SegmentPersistenceData, SegmentData> {
 
     @Override
     public String esIndex() {
@@ -46,56 +41,60 @@ public class SegmentSave extends RecordPersistenceMember {
     }
 
     @Override
-    public void analyse(Object message) throws Exception {
+    public SegmentPersistenceData initializeData() {
+        return new SegmentPersistenceData();
+    }
+
+    @Override
+    final public void analyse(Object message) throws Exception {
         if (message instanceof Segment) {
             Segment segment = (Segment) message;
-            persistenceData.put(segment.getTraceSegmentId(), segment.getJsonStr());
-            if (persistenceData.size() >= CacheSizeConfig.Cache.Persistence.SIZE) {
-                persistence();
+            SegmentPersistenceData data = getPersistenceData();
+            data.hold();
+            data.getOrCreate(segment.getTraceSegmentId()).setSegmentStr(segment.getJsonStr());
+            if (data.size() >= CacheSizeConfig.Cache.Persistence.SIZE) {
+                persistence(data.asMap());
             }
+            data.release();
         } else {
-            logger.error("unhandled message, message instance must JsonObject, but is %s", message.getClass().toString());
-        }
-    }
-
-    @Override
-    protected void persistence() {
-        boolean success = saveToEs();
-        if (success) {
-            persistenceData.clear();
+            logger().error("unhandled message, message instance must Segment, but is %s", message.getClass().toString());
         }
     }
 
-    private boolean saveToEs() {
+    private void persistence(Map<String, SegmentData> dataMap) {
+        List<IndexRequestBuilder> builderList = new LinkedList<>();
         Client client = EsClient.INSTANCE.getClient();
-        BulkRequestBuilder bulkRequest = client.prepareBulk();
-        logger.debug("persistenceData SIZE: %s", persistenceData.size());
+        dataMap.forEach((key, value) -> {
+            IndexRequestBuilder builder = client.prepareIndex(esIndex(), esType(), key).setSource(value.getSegmentStr());
+            builderList.add(builder);
+        });
+        EsClient.INSTANCE.bulk(builderList);
+        dataMap.clear();
+    }
 
-        persistenceData.forEach((key, value) -> bulkRequest.add(client.prepareIndex(esIndex(), esType(), key).setSource(value)));
+    @Override
+    final protected void prepareIndex(List<IndexRequestBuilder> builderList) {
+        Map<String, SegmentData> lastData = getPersistenceData().getLast().asMap();
 
-        BulkResponse bulkResponse = bulkRequest.execute().actionGet();
-        if (bulkResponse.hasFailures()) {
-            logger.error(bulkResponse.buildFailureMessage());
-        }
-        return !bulkResponse.hasFailures();
+        Client client = EsClient.INSTANCE.getClient();
+        lastData.forEach((key, value) -> {
+            IndexRequestBuilder builder = client.prepareIndex(esIndex(), esType(), key).setSource(value.getSegmentStr());
+            builderList.add(builder);
+        });
+        lastData.clear();
     }
 
-    public static class Factory extends AbstractLocalAsyncWorkerProvider<SegmentSave> {
-        public static Factory INSTANCE = new Factory();
-
+    public static class Factory extends AbstractLocalSyncWorkerProvider<SegmentSave> {
         @Override
         public Role role() {
             return Role.INSTANCE;
         }
 
-        @Override
-        public int queueSize() {
-            return WorkerConfig.Queue.Segment.SegmentSave.SIZE;
-        }
-
         @Override
         public SegmentSave workerInstance(ClusterWorkerContext clusterContext) {
-            return new SegmentSave(role(), clusterContext, new LocalWorkerContext());
+            SegmentSave worker = new SegmentSave(role(), clusterContext, new LocalWorkerContext());
+            PersistenceWorkerListener.INSTANCE.register(worker);
+            return worker;
         }
     }
 

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/segment/persistence/SegmentTopSearchWithGlobalTraceId.java

@@ -11,7 +11,7 @@ import com.a.eye.skywalking.collector.worker.segment.entity.GlobalTraceId;
 import com.a.eye.skywalking.collector.worker.segment.entity.Segment;
 import com.a.eye.skywalking.collector.worker.segment.entity.SegmentDeserialize;
 import com.a.eye.skywalking.collector.worker.storage.EsClient;
-import com.a.eye.skywalking.collector.worker.storage.MergeData;
+import com.a.eye.skywalking.collector.worker.storage.JoinAndSplitData;
 import com.a.eye.skywalking.collector.worker.tools.CollectionTools;
 import com.google.gson.Gson;
 import com.google.gson.JsonArray;
@@ -55,7 +55,7 @@ public class SegmentTopSearchWithGlobalTraceId extends AbstractLocalSyncWorker {
 
             if (globalTraceObj != null && globalTraceObj.has(GlobalTraceIndex.SUB_SEG_IDS)) {
                 String subSegIdsStr = globalTraceObj.get(GlobalTraceIndex.SUB_SEG_IDS).getAsString();
-                String[] subSegIds = subSegIdsStr.split(MergeData.SPLIT);
+                String[] subSegIds = subSegIdsStr.split(JoinAndSplitData.SPLIT);
 
                 topSegPaging.addProperty("recordsTotal", subSegIds.length);
 
@@ -138,8 +138,6 @@ public class SegmentTopSearchWithGlobalTraceId extends AbstractLocalSyncWorker {
     }
 
     public static class Factory extends AbstractLocalSyncWorkerProvider<SegmentTopSearchWithGlobalTraceId> {
-        public static Factory INSTANCE = new Factory();
-
         @Override
         public Role role() {
             return WorkerRole.INSTANCE;

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/segment/persistence/SegmentTopSearchWithTimeSlice.java

@@ -168,8 +168,6 @@ public class SegmentTopSearchWithTimeSlice extends AbstractLocalSyncWorker {
     }
 
     public static class Factory extends AbstractLocalSyncWorkerProvider<SegmentTopSearchWithTimeSlice> {
-        public static Factory INSTANCE = new Factory();
-
         @Override
         public Role role() {
             return WorkerRole.INSTANCE;

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/span/SpanGetWithId.java

@@ -48,8 +48,6 @@ public class SpanGetWithId extends AbstractGet {
     }
 
     public static class Factory extends AbstractGetProvider<SpanGetWithId> {
-        public static Factory INSTANCE = new Factory();
-
         @Override
         public Role role() {
             return WorkerRole.INSTANCE;

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/span/persistence/SpanSearchWithId.java

@@ -68,8 +68,6 @@ public class SpanSearchWithId extends AbstractLocalSyncWorker {
     }
 
     public static class Factory extends AbstractLocalSyncWorkerProvider<SpanSearchWithId> {
-        public static Factory INSTANCE = new Factory();
-
         @Override
         public Role role() {
             return WorkerRole.INSTANCE;

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/storage/AbstractIndex.java

@@ -30,12 +30,15 @@ public abstract class AbstractIndex {
 
     final XContentBuilder createSettingBuilder() throws IOException {
         return XContentFactory.jsonBuilder()
-            .startObject()
-            .field("index.number_of_shards", EsConfig.Es.Index.Shards.NUMBER)
-            .field("index.number_of_replicas", EsConfig.Es.Index.Replicas.NUMBER)
-            .endObject();
+                .startObject()
+                .field("index.number_of_shards", EsConfig.Es.Index.Shards.NUMBER)
+                .field("index.number_of_replicas", EsConfig.Es.Index.Replicas.NUMBER)
+                .field("index.refresh_interval", String.valueOf(refreshInterval()) + "s")
+                .endObject();
     }
 
+    public abstract int refreshInterval();
+
     public abstract boolean isRecord();
 
     public abstract XContentBuilder createMappingBuilder() throws IOException;

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/storage/Data.java

+package com.a.eye.skywalking.collector.worker.storage;
+
+import java.util.Map;
+
+/**
+ * @author pengys5
+ */
+public interface Data {
+    String getId();
+
+    void merge(Map<String, ?> dbData);
+}

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/storage/EsClient.java

@@ -5,6 +5,9 @@ import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
 import org.elasticsearch.action.admin.indices.refresh.RefreshResponse;
+import org.elasticsearch.action.bulk.BulkRequestBuilder;
+import org.elasticsearch.action.bulk.BulkResponse;
+import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.transport.InetSocketTransportAddress;
@@ -77,4 +80,21 @@ public enum EsClient {
             this.port = port;
         }
     }
+
+    public void bulk(List<IndexRequestBuilder> dataList) {
+        Client client = EsClient.INSTANCE.getClient();
+        BulkRequestBuilder bulkRequest = client.prepareBulk();
+
+        logger.info("bulk data size: %s", dataList.size());
+        if (dataList.size() > 0) {
+            for (IndexRequestBuilder builder : dataList) {
+                bulkRequest.add(builder);
+            }
+
+            BulkResponse bulkResponse = bulkRequest.execute().actionGet();
+            if (bulkResponse.hasFailures()) {
+                logger.error(bulkResponse.buildFailureMessage());
+            }
+        }
+    }
 }

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/storage/FlushAndSwitch.java

+package com.a.eye.skywalking.collector.worker.storage;
+
+/**
+ * @author pengys5
+ */
+public class FlushAndSwitch {
+}

skywalking-collector/skywalking-collector-worker/src/main/java/com/a/eye/skywalking/collector/worker/storage/JoinAndSplitAnalysisData.java

+package com.a.eye.skywalking.collector.worker.storage;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+/**
+ * @author pengys5
+ */
+public class JoinAndSplitAnalysisData {
+
+    private WindowData<JoinAndSplitData> windowData = new WindowData(new LinkedHashMap<String, JoinAndSplitData>());
+
+    public JoinAndSplitData getOrCreate(String id) {
+        if (!windowData.containsKey(id)) {
+            windowData.put(id, new JoinAndSplitData(id));
+        }
+        return windowData.get(id);
+    }
+
+    public Map<String, JoinAndSplitData> asMap() {
+        return windowData.asMap();
+    }
+}