新增health-report工程，修改日志输出，添加配置

pom.xml

@@ -12,10 +12,9 @@
         <module>skywalking-webui</module>
         <module>skywalking-sniffer</module>
         <module>skywalking-storage-center</module>
-        <module>skywalking-registry</module>
         <module>samples/skywalking-auth</module>
         <module>samples/skywalking-example</module>
-        <module>skywalking-logging</module>
+        <module>skywalking-commons</module>
     </modules>
     <packaging>pom</packaging>
 

skywalking-commons/pom.xml

+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>skywalking</artifactId>
+        <groupId>com.a.eye</groupId>
+        <version>2.0-2016</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>skywalking-commons</artifactId>
+    <packaging>pom</packaging>
+
+    <modules>
+        <module>skywalking-logging</module>
+        <module>skywalking-registry</module>
+        <module>skywalking-health-report</module>
+    </modules>
+
+    <name>skywalking-commons</name>
+    <url>http://maven.apache.org</url>
+
+    <properties>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+        <project.version>2.0-2016</project.version>
+    </properties>
+</project>

skywalking-commons/skywalking-health-report/pom.xml

+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>skywalking-commons</artifactId>
+        <groupId>com.a.eye</groupId>
+        <version>2.0-2016</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>skywalking-health-report</artifactId>
+    <packaging>jar</packaging>
+
+    <name>skywalking-health-report</name>
+    <url>http://maven.apache.org</url>
+
+    <dependencies>
+        <dependency>
+            <groupId>com.a.eye</groupId>
+            <artifactId>skywalking-logging-api</artifactId>
+            <version>2.0-2016</version>
+        </dependency>
+    </dependencies>
+</project>

skywalking-commons/skywalking-health-report/src/main/java/com/a/eye/skywalking/health/report/HealthCollector.java

+package com.a.eye.skywalking.health.report;
+
+
+import com.a.eye.skywalking.health.report.util.MachineUtil;
+import com.a.eye.skywalking.logging.api.ILog;
+import com.a.eye.skywalking.logging.api.LogManager;
+
+import java.util.Arrays;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+public class HealthCollector extends Thread {
+    private static       ILog                      logger                  =
+            LogManager.getLogger(HealthCollector.class);
+    private static       Map<String, HeathReading> heathReadings           =
+            new ConcurrentHashMap<String, HeathReading>();
+    private static final long                      DEFAULT_REPORT_INTERVAL = 60 * 1000;
+    private final long reportInterval;
+
+    private HealthCollector() {
+        this(DEFAULT_REPORT_INTERVAL);
+    }
+
+    private HealthCollector(long reportInterval) {
+        super("HealthCollector");
+        this.setDaemon(true);
+        this.reportInterval = reportInterval;
+    }
+
+    public static void init() {
+        new HealthCollector().start();
+    }
+
+    public static HeathReading getCurrentHeathReading(String extraId) {
+        String id = getId(extraId);
+        if (!heathReadings.containsKey(id)) {
+            synchronized (heathReadings) {
+                if (!heathReadings.containsKey(id)) {
+                    if (heathReadings.keySet().size() > 5000) {
+                        throw new RuntimeException(
+                                "use HealthCollector illegal. There is an overflow trend of Server Health Collector Report Data.");
+                    }
+                    heathReadings.put(id, new HeathReading(id));
+                }
+            }
+        }
+        return heathReadings.get(id);
+    }
+
+    private static String getId(String extraId) {
+        return "SkyWalking Health Report,P:" + MachineUtil.getProcessNo() + ",T:" + Thread.currentThread().getName()
+                + "(" + Thread.currentThread().getId() + ")" + (extraId == null ? "" : ",extra:" + extraId);
+    }
+
+    @Override
+    public void run() {
+        while (true) {
+            try {
+                Map<String, HeathReading> heathReadingsSnapshot = heathReadings;
+                heathReadings = new ConcurrentHashMap<String, HeathReading>();
+                String[] keyList = heathReadingsSnapshot.keySet().toArray(new String[0]);
+                Arrays.sort(keyList);
+                StringBuilder log = new StringBuilder();
+                log.append("\n---------Server Health Collector Report---------\n");
+                for (String key : keyList) {
+                    log.append(heathReadingsSnapshot.get(key)).append("\n");
+                }
+                log.append("------------------------------------------------\n");
+
+                logger.info(log.toString());
+
+                try {
+                    Thread.sleep(reportInterval);
+                } catch (InterruptedException e) {
+                    logger.warn("sleep error.", e);
+                }
+            } catch (Throwable t) {
+                logger.error("HealthCollector report error.", t);
+            }
+        }
+    }
+}

skywalking-commons/skywalking-health-report/src/main/java/com/a/eye/skywalking/health/report/HeathReading.java

+package com.a.eye.skywalking.health.report;
+
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class HeathReading {
+    public static final String ERROR   = "[ERROR]";
+    public static final String WARNING = "[WARNING]";
+    public static final String INFO    = "[INFO]";
+
+    private String id;
+
+    private Map<String, HeathDetailData> datas = new HashMap<String, HeathDetailData>();
+
+    /**
+     * 健康读数，只应该在工作线程中创建
+     */
+    public HeathReading(String id) {
+        this.id = id;
+    }
+
+    public void updateData(String key, String message) {
+        updateData(key, message, new Object[0]);
+    }
+
+    public void updateData(String key, String newData, Object... arguments) {
+        if (datas.containsKey(key)) {
+            datas.get(key).updateData(newData, arguments);
+        } else {
+            datas.put(key, new HeathDetailData(newData));
+        }
+    }
+
+    @Override
+    public String toString() {
+        StringBuilder sb = new StringBuilder();
+        sb.append("id<").append(this.id).append(">\n");
+        for (Map.Entry<String, HeathDetailData> data : datas.entrySet()) {
+            sb.append(data.getKey()).append(data.getValue().toString()).append("\n");
+        }
+
+        datas = new HashMap<String, HeathReading.HeathDetailData>();
+        return sb.toString();
+    }
+
+    class HeathDetailData {
+        private String data;
+
+        private long statusTime;
+
+        HeathDetailData(String initialData) {
+            data = initialData;
+            statusTime = System.currentTimeMillis();
+        }
+
+        void updateData(String newData, Object... arguments) {
+            data = newData;
+            if (arguments.length > 0)
+                data = String.format(newData, arguments);
+            statusTime = System.currentTimeMillis();
+        }
+
+        String getData() {
+            return data;
+        }
+
+        long getStatusTime() {
+            return statusTime;
+        }
+
+        @Override
+        public String toString() {
+            return data + "(t:" + statusTime + ")";
+        }
+    }
+}

skywalking-commons/skywalking-health-report/src/main/java/com/a/eye/skywalking/health/report/util/MachineUtil.java

+package com.a.eye.skywalking.health.report.util;
+
+import java.lang.management.ManagementFactory;
+
+/**
+ * Created by xin on 2016/11/14.
+ */
+
+public class MachineUtil {
+    private static String processNo;
+
+    static {
+        processNo = getProcessNo();
+    }
+
+    public static String getProcessNo() {
+        if (processNo == null) {
+            String name = ManagementFactory.getRuntimeMXBean().getName();
+            processNo = name.split("@")[0];
+        }
+        return processNo;
+    }
+
+    private MachineUtil() {
+        // Non
+    }
+}

skywalking-logging/pom.xml → skywalking-commons/skywalking-logging/pom.xml

 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <parent>
-        <artifactId>skywalking</artifactId>
+        <artifactId>skywalking-commons</artifactId>
         <groupId>com.a.eye</groupId>
         <version>2.0-2016</version>
     </parent>

skywalking-logging/skywalking-logging-api/src/main/java/com/a/eye/skywalking/logging/api/ILog.java → skywalking-commons/skywalking-logging/skywalking-logging-api/src/main/java/com/a/eye/skywalking/logging/api/ILog.java

 package com.a.eye.skywalking.logging.api;
 
+import java.util.Objects;
+
 /**
  * Created by xin on 2016/11/10.
  */
@@ -16,5 +18,9 @@ public interface ILog {
 
     void error(String format, Object argument, Throwable e);
 
+    boolean isDebugEnable();
+
+    void debug(String format);
 
+    void debug(String format, Object... arguments);
 }

skywalking-logging/skywalking-logging-api/src/main/java/com/a/eye/skywalking/logging/api/NoopLogger.java → skywalking-commons/skywalking-logging/skywalking-logging-api/src/main/java/com/a/eye/skywalking/logging/api/NoopLogger.java

@@ -35,4 +35,19 @@ public class NoopLogger implements ILog{
     @Override
     public void error(String format, Object argument, Throwable e) {
     }
+
+    @Override
+    public boolean isDebugEnable() {
+        return false;
+    }
+
+    @Override
+    public void debug(String format) {
+
+    }
+
+    @Override
+    public void debug(String format, Object... arguments) {
+
+    }
 }

skywalking-logging/skywalking-logging-impl-log4j2/src/main/java/com/a/eye/skywalking/logging/impl/log4j2/Log4j2Logger.java → skywalking-commons/skywalking-logging/skywalking-logging-impl-log4j2/src/main/java/com/a/eye/skywalking/logging/impl/log4j2/Log4j2Logger.java

@@ -42,4 +42,19 @@ public class Log4j2Logger implements ILog {
     public void error(String message, Object argument, Throwable e) {
         logger.error(message, argument, e);
     }
+
+    @Override
+    public boolean isDebugEnable() {
+        return logger.isDebugEnabled();
+    }
+
+    @Override
+    public void debug(String format) {
+        logger.debug(format);
+    }
+
+    @Override
+    public void debug(String format, Object... arguments) {
+        logger.debug(format, arguments);
+    }
 }

skywalking-registry/pom.xml → skywalking-commons/skywalking-registry/pom.xml

 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <parent>
-        <artifactId>skywalking</artifactId>
+        <artifactId>skywalking-commons</artifactId>
         <groupId>com.a.eye</groupId>
         <version>2.0-2016</version>
     </parent>
@@ -26,12 +26,12 @@
         <dependency>
             <groupId>com.a.eye</groupId>
             <artifactId>skywalking-logging-api</artifactId>
-            <version>${project.version}</version>
+            <version>2.0-2016</version>
         </dependency>
         <dependency>
             <groupId>com.a.eye</groupId>
             <artifactId>skywalking-logging-impl-log4j2</artifactId>
-            <version>${project.version}</version>
+            <version>2.0-2016</version>
         </dependency>
     </dependencies>
 </project>

skywalking-network/pom.xml

@@ -120,7 +120,7 @@
                     </execution>
                 </executions>
             </plugin>
-            <!-- add this plugin to fix shade-plugin bug: not rename services file name. -->
+            <!-- add this plugin to fix shade-plugin bug: not rename provider file name. -->
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-antrun-plugin</artifactId>

skywalking-network/src/main/java/com/a/eye/skywalking/network/TransferService.java → skywalking-network/src/main/java/com/a/eye/skywalking/network/ServiceProvider.java

 package com.a.eye.skywalking.network;
 
-import com.a.eye.skywalking.network.grpc.services.SpanStorageService;
-import com.a.eye.skywalking.network.grpc.services.TraceSearchService;
-import com.a.eye.skywalking.network.listener.SpanStorageNotifier;
-import com.a.eye.skywalking.network.listener.TraceSearchNotifier;
+import com.a.eye.skywalking.network.grpc.provider.SpanStorageService;
+import com.a.eye.skywalking.network.grpc.provider.TraceSearchService;
+import com.a.eye.skywalking.network.listener.SpanStorageListener;
+import com.a.eye.skywalking.network.listener.TraceSearchListener;
 import io.grpc.Server;
 import io.grpc.ServerBuilder;
 
 import java.io.IOException;
 
-public class TransferService {
+public class ServiceProvider {
     private Server server;
 
-    private TransferService(Server server) {
+    private ServiceProvider(Server server) {
         this.server = server;
     }
 
@@ -21,7 +21,7 @@ public class TransferService {
         // 当JVM停止之后，Server也需要停止
         Runtime.getRuntime().addShutdownHook(new Thread() {
             public void run() {
-                TransferService.this.stop();
+                ServiceProvider.this.stop();
             }
         });
     }
@@ -32,10 +32,8 @@ public class TransferService {
         }
     }
 
-    private void blockUntilShutdown() throws InterruptedException {
-        if (server != null) {
-            server.awaitTermination();
-        }
+    public static TransferServiceBuilder newBuilder(int port) {
+        return new TransferServiceBuilder(port);
     }
 
     public static class TransferServiceBuilder {
@@ -45,21 +43,17 @@ public class TransferService {
 
         private ServerBuilder serverBuilder;
 
-        public static TransferServiceBuilder newBuilder(int port) {
-            return new TransferServiceBuilder(port);
-        }
-
-        public TransferService build() {
-            return new TransferService(serverBuilder.build());
+        public ServiceProvider build() {
+            return new ServiceProvider(serverBuilder.build());
         }
 
-        public TransferServiceBuilder startSpanStorageService(SpanStorageNotifier spanStorageListener) {
+        public TransferServiceBuilder addSpanStorageService(SpanStorageListener spanStorageListener) {
             serverBuilder.addService(new SpanStorageService(spanStorageListener));
             return this;
         }
 
-        public TransferServiceBuilder startTraceSearchService(TraceSearchNotifier traceSearchNotifier) {
-            serverBuilder.addService(new TraceSearchService(traceSearchNotifier));
+        public TransferServiceBuilder addTraceSearchService(TraceSearchListener traceSearchListener) {
+            serverBuilder.addService(new TraceSearchService(traceSearchListener));
             return this;
         }
     }

skywalking-network/src/main/java/com/a/eye/skywalking/network/grpc/services/SpanStorageService.java → skywalking-network/src/main/java/com/a/eye/skywalking/network/grpc/provider/SpanStorageService.java

-package com.a.eye.skywalking.network.grpc.services;
+package com.a.eye.skywalking.network.grpc.provider;
 
 import com.a.eye.skywalking.network.grpc.AckSpan;
 import com.a.eye.skywalking.network.grpc.RequestSpan;
 import com.a.eye.skywalking.network.grpc.SendResult;
 import com.a.eye.skywalking.network.grpc.SpanStorageServiceGrpc;
-import com.a.eye.skywalking.network.listener.SpanStorageNotifier;
+import com.a.eye.skywalking.network.listener.SpanStorageListener;
 import io.grpc.stub.StreamObserver;
 
 public class SpanStorageService extends SpanStorageServiceGrpc.SpanStorageServiceImplBase {
 
-    private SpanStorageNotifier listener;
-
-    public SpanStorageService(SpanStorageNotifier listener) {
+    private SpanStorageListener listener;
+    public SpanStorageService(SpanStorageListener listener) {
         this.listener = listener;
     }
 
@@ -20,12 +19,12 @@ public class SpanStorageService extends SpanStorageServiceGrpc.SpanStorageServic
         return new StreamObserver<AckSpan>() {
             @Override
             public void onNext(AckSpan value) {
+
                 listener.storage(value);
             }
 
             @Override
             public void onError(Throwable t) {
-
             }
 
             @Override
@@ -47,7 +46,6 @@ public class SpanStorageService extends SpanStorageServiceGrpc.SpanStorageServic
 
             @Override
             public void onError(Throwable t) {
-
             }
 
             @Override

skywalking-network/src/main/java/com/a/eye/skywalking/network/grpc/services/TraceSearchService.java → skywalking-network/src/main/java/com/a/eye/skywalking/network/grpc/provider/TraceSearchService.java

-package com.a.eye.skywalking.network.grpc.services;
+package com.a.eye.skywalking.network.grpc.provider;
 
 import com.a.eye.skywalking.network.grpc.SearchResult;
 import com.a.eye.skywalking.network.grpc.Span;
 import com.a.eye.skywalking.network.grpc.TraceSearchCondition;
 import com.a.eye.skywalking.network.grpc.TraceSearchServiceGrpc;
-import com.a.eye.skywalking.network.listener.TraceSearchNotifier;
+import com.a.eye.skywalking.network.listener.TraceSearchListener;
 import io.grpc.stub.StreamObserver;
 
 import java.util.List;
@@ -14,15 +14,15 @@ import java.util.List;
  */
 public class TraceSearchService extends TraceSearchServiceGrpc.TraceSearchServiceImplBase {
 
-    private TraceSearchNotifier traceSearchNotifier;
+    private TraceSearchListener traceSearchListener;
 
-    public TraceSearchService(TraceSearchNotifier traceSearchNotifier) {
-        this.traceSearchNotifier = traceSearchNotifier;
+    public TraceSearchService(TraceSearchListener traceSearchListener) {
+        this.traceSearchListener = traceSearchListener;
     }
 
     @Override
     public void search(TraceSearchCondition request, StreamObserver<SearchResult> responseObserver) {
-        List<Span> spans = traceSearchNotifier.search(request.getTraceid());
+        List<Span> spans = traceSearchListener.search(request.getTraceid());
         responseObserver.onNext(SearchResult.newBuilder().addAllSpans(spans).build());
         responseObserver.onCompleted();
     }

skywalking-network/src/main/java/com/a/eye/skywalking/network/listener/SpanStorageNotifier.java → skywalking-network/src/main/java/com/a/eye/skywalking/network/listener/SpanStorageListener.java

@@ -3,7 +3,7 @@ package com.a.eye.skywalking.network.listener;
 import com.a.eye.skywalking.network.grpc.AckSpan;
 import com.a.eye.skywalking.network.grpc.RequestSpan;
 
-public interface SpanStorageNotifier {
+public interface SpanStorageListener{
     boolean storage(RequestSpan requestSpan);
 
     boolean storage(AckSpan ackSpan);

skywalking-network/src/main/java/com/a/eye/skywalking/network/listener/TraceSearchNotifier.java → skywalking-network/src/main/java/com/a/eye/skywalking/network/listener/TraceSearchListener.java

@@ -7,6 +7,6 @@ import java.util.List;
 /**
  * Created by xin on 2016/11/12.
  */
-public interface TraceSearchNotifier {
+public interface TraceSearchListener{
     List<Span> search(String traceId);
 }

skywalking-storage-center/skywalking-storage/pom.xml

@@ -23,7 +23,7 @@
         <dependency>
             <groupId>com.a.eye</groupId>
             <artifactId>skywalking-registry</artifactId>
-            <version>${project.version}</version>
+            <version>2.0-2016</version>
         </dependency>
         <dependency>
             <groupId>com.a.eye</groupId>
@@ -51,6 +51,11 @@
             <artifactId>HikariCP</artifactId>
             <version>2.4.3</version>
         </dependency>
+        <dependency>
+            <groupId>com.a.eye</groupId>
+            <artifactId>skywalking-health-report</artifactId>
+            <version>2.0-2016</version>
+        </dependency>
     </dependencies>
     <build>
         <plugins>

skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/Main.java

 package com.a.eye.skywalking.storage;
 
+import com.a.eye.skywalking.health.report.HealthCollector;
 import com.a.eye.skywalking.logging.api.ILog;
 import com.a.eye.skywalking.logging.api.LogManager;
 import com.a.eye.skywalking.logging.impl.log4j2.Log4j2Resolver;
-import com.a.eye.skywalking.network.TransferService;
-import com.a.eye.skywalking.network.TransferService.TransferServiceBuilder;
+import com.a.eye.skywalking.network.ServiceProvider;
 import com.a.eye.skywalking.registry.RegistryCenterFactory;
 import com.a.eye.skywalking.registry.api.CenterType;
 import com.a.eye.skywalking.registry.api.RegistryCenter;
@@ -13,14 +13,16 @@ import com.a.eye.skywalking.storage.block.index.BlockIndexEngine;
 import com.a.eye.skywalking.storage.config.Config;
 import com.a.eye.skywalking.storage.config.ConfigInitializer;
 import com.a.eye.skywalking.storage.data.IndexDataCapacityMonitor;
-import com.a.eye.skywalking.storage.notifier.SearchNotifier;
-import com.a.eye.skywalking.storage.notifier.StorageNotifier;
+import com.a.eye.skywalking.storage.data.file.DataFilesManager;
+import com.a.eye.skywalking.storage.listener.SearchListener;
+import com.a.eye.skywalking.storage.listener.StorageListener;
 import com.a.eye.skywalking.storage.util.NetUtils;
 
 import java.io.IOException;
+import java.util.Map;
 import java.util.Properties;
 
-import static com.a.eye.skywalking.storage.config.Config.RegistryCenter.REGISTRY_PATH_PREFIX;
+import static com.a.eye.skywalking.storage.config.Config.RegistryCenter.PATH_PREFIX;
 
 /**
  * Created by xin on 2016/11/12.
@@ -33,28 +35,36 @@ public class Main {
         LogManager.setLogResolver(new Log4j2Resolver());
     }
 
-    private static TransferService transferService;
+    private static ServiceProvider provider;
 
     public static void main(String[] args) {
         try {
             initializeParam();
 
+            HealthCollector.init();
+
+            DataFilesManager.init();
+
             BlockIndexEngine.start();
+
             IndexDataCapacityMonitor.start();
-            transferService =
-                    TransferServiceBuilder.newBuilder(Config.Server.PORT).startSpanStorageService(new StorageNotifier())
-                            .startTraceSearchService(new SearchNotifier()).build();
-            transferService.start();
-            logger.info("transfer service started successfully.");
+
+            provider = ServiceProvider.newBuilder(Config.Server.PORT).addSpanStorageService(new StorageListener())
+                    .addTraceSearchService(new SearchListener()).build();
+            provider.start();
+
+            if (logger.isDebugEnable()) {
+                logger.debug("Service provider started successfully.");
+            }
 
             registryNode();
-            logger.info("storage service started successfully.");
+
+            logger.info("Storage service started successfully.");
             Thread.currentThread().join();
         } catch (Throwable e) {
-            e.printStackTrace();
             logger.error("Failed to start service.", e);
         } finally {
-            transferService.stop();
+            provider.stop();
             IndexDataCapacityMonitor.stop();
         }
     }
@@ -68,13 +78,14 @@ public class Main {
         registerConfig.setProperty(ZookeeperConfig.AUTH_INFO, Config.RegistryCenter.AUTH_INFO);
         registryCenter.start(registerConfig);
         registryCenter.register(
-                REGISTRY_PATH_PREFIX + NetUtils.getLocalAddress().getHostAddress() + ":" + Config.Server.PORT);
+                PATH_PREFIX + NetUtils.getLocalAddress().getHostAddress() + ":" + Config.Server.PORT);
     }
 
     private static void initializeParam() throws IllegalAccessException, IOException {
         Properties properties = new Properties();
         try {
             properties.load(Main.class.getResourceAsStream("/config.properties"));
+            printStorageConfig(properties);
             ConfigInitializer.initialize(properties, Config.class);
         } catch (IllegalAccessException e) {
             logger.error("Initialize the collect server configuration failed", e);
@@ -84,4 +95,11 @@ public class Main {
             throw e;
         }
     }
+
+
+    private static void printStorageConfig(Properties config) {
+        for (Map.Entry<Object, Object> entry : config.entrySet()) {
+            logger.info("{} = {}", entry.getKey(), entry.getValue());
+        }
+    }
 }

skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/block/index/BlockFinder.java

 package com.a.eye.skywalking.storage.block.index;
 
+import com.a.eye.skywalking.logging.api.ILog;
+import com.a.eye.skywalking.logging.api.LogManager;
+
 /**
  * Created by xin on 2016/11/2.
  */
 public class BlockFinder {
 
+    private static ILog logger = LogManager.getLogger(BlockFinder.class);
+
     private L1Cache l1Cache;
     private L2Cache l2Cache;
 
@@ -19,11 +24,15 @@ public class BlockFinder {
             index = l2Cache.find(timestamp);
         }
 
+        if (logger.isDebugEnable()) {
+            logger.debug("Time stamp : {} is mapping with block Index {}.", timestamp, index);
+        }
+
         return index;
     }
 
 
-    public long findLastBlockIndex(){
+    public long findLastBlockIndex() {
         return l2Cache.getLastBlockIndex();
     }
 

skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/block/index/BlockIndexUpdator.java

@@ -9,8 +9,9 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 
-import static com.a.eye.skywalking.storage.config.Config.BlockIndex.DATA_FILE_INDEX_FILE_NAME;
-import static com.a.eye.skywalking.storage.config.Config.BlockIndex.STORAGE_BASE_PATH;
+import static com.a.eye.skywalking.storage.config.Config.BlockIndex.FILE_NAME;
+import static com.a.eye.skywalking.storage.config.Config.BlockIndex.PATH;
+import static com.a.eye.skywalking.storage.util.PathResolver.getAbsolutePath;
 
 public class BlockIndexUpdator {
 
@@ -24,12 +25,12 @@ public class BlockIndexUpdator {
     }
 
     public void addRecord(long timestamp) {
-        logger.info("Updating index. timestamp:{}", timestamp);
+        logger.info("Updating block index. index key:{}", timestamp);
         try {
             updateFile(timestamp);
             updateCache(timestamp);
         } catch (Exception e) {
-            logger.error("Failed to add index record", e);
+            logger.error("Failed to add block index record", e);
         }
     }
 
@@ -83,19 +84,30 @@ public class BlockIndexUpdator {
             }
         }
 
+
         if (indexData.size() == 0) {
+            if (logger.isDebugEnable()) {
+                logger.debug("Any block index was not founded. will add new block index.", indexData.size());
+            }
             //如果此前没有记录，则取之前五分钟到目前的数据
             addRecord(System.currentTimeMillis() - 5 * 60 * 1000);
             return;
         }
 
+        if (logger.isDebugEnable()) {
+            logger.debug("There are {} block index was founded. Begin to init L1Cache and L2Cache", indexData.size());
+        }
+
         Collections.reverse(indexData);
         l1Cache.init(indexData);
         l2Cache.init(indexData);
     }
 
     public File getOrCreateBlockIndexFile() throws IOException {
-        File blockIndexFile = new File(STORAGE_BASE_PATH, DATA_FILE_INDEX_FILE_NAME);
+        if (logger.isDebugEnable()) {
+
+        }
+        File blockIndexFile = new File(getAbsolutePath(PATH), FILE_NAME);
 
         if (!blockIndexFile.getParentFile().exists()) {
             blockIndexFile.getParentFile().mkdirs();

skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/block/index/L1Cache.java

@@ -9,6 +9,8 @@ import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReadWriteLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 
+import static com.a.eye.skywalking.storage.config.Config.BlockIndexEngine.L1_CACHE_SIZE;
+
 /**
  * 块索引的一级缓存
  * <p>
@@ -16,15 +18,20 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
  */
 public class L1Cache {
 
-    private static final int           MAX_DATA_KEEP_SIZE = 30;
-    private static       ILog          logger             = LogManager.getLogger(L1Cache.class);
-    private              TreeSet<Long> cacheData          = new TreeSet<Long>();
-    private final        ReadWriteLock updateLock         = new ReentrantReadWriteLock();
+    private static ILog          logger     = LogManager.getLogger(L1Cache.class);
+    private        TreeSet<Long> cacheData  = new TreeSet<Long>();
+    private final  ReadWriteLock updateLock = new ReentrantReadWriteLock();
 
     void init(List<Long> data) {
-        int size = data.size() > MAX_DATA_KEEP_SIZE ? MAX_DATA_KEEP_SIZE : data.size();
+        StringBuilder initData = new StringBuilder();
+        int size = data.size() > L1_CACHE_SIZE ? L1_CACHE_SIZE : data.size();
         for (int i = 0; i < size; i++) {
             this.cacheData.add(data.get(i));
+            initData.append(data.get(i)).append(",");
+        }
+
+        if (logger.isDebugEnable()) {
+            logger.info("L1 cache init data : [{}]", initData.deleteCharAt(initData.length() - 1));
         }
     }
 
@@ -42,7 +49,7 @@ public class L1Cache {
         TreeSet<Long> newCacheData = new TreeSet<>(cacheData);
         newCacheData.add(timestamp);
 
-        if (newCacheData.size() >= MAX_DATA_KEEP_SIZE + 1) {
+        if (newCacheData.size() >= L1_CACHE_SIZE + 1) {
             long removedData = newCacheData.pollFirst();
             logger.info("Add cache data : {}, removed cache Data:{}", timestamp, removedData);
         }

skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/block/index/L2Cache.java

 package com.a.eye.skywalking.storage.block.index;
 
+import com.a.eye.skywalking.logging.api.ILog;
+import com.a.eye.skywalking.logging.api.LogManager;
+
 import java.util.List;
 import java.util.TreeSet;
 import java.util.concurrent.locks.Lock;
@@ -11,11 +14,16 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
  */
 public class L2Cache {
 
+    private ILog logger = LogManager.getLogger(L2Cache.class);
+
     private TreeSet<Long> cacheData  = new TreeSet<Long>();
     private ReadWriteLock updateLock = new ReentrantReadWriteLock();
 
     void init(List<Long> data) {
         this.cacheData.addAll(data);
+        if (logger.isDebugEnable()) {
+            logger.info("L2 cache init data : {}", data);
+        }
     }
 
     public Long find(long timestamp) {

skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/config/Config.java

@@ -6,34 +6,44 @@ package com.a.eye.skywalking.storage.config;
 public class Config {
     public static class Server {
         public static int PORT = 34000;
+
+        public static int CHANNEL_SIZE = 10;
+
+        public static int BUFFER_SIZE = 1000;
     }
 
 
     public static class BlockIndex {
+        public static String PATH = "/block_index";
 
-        public static String STORAGE_BASE_PATH = "/tmp/skywalking/block_index";
-
-        public static String DATA_FILE_INDEX_FILE_NAME = "data_file.index";
+        public static String FILE_NAME = "data_file.index";
     }
 
 
     public static class DataFile {
-        public static String BASE_PATH = "/tmp/skywalking/data/file";
+        public static String PATH = "/data/file";
 
-        public static long MAX_LENGTH = 3 * 1024 * 1024 * 1024;
+        public static long SIZE = 3 * 1024 * 1024 * 1024L;
     }
 
 
     public static class DataIndex {
 
-        public static String TABLE_NAME = "data_index";
+        public static String PATH = "/data/index";
+
+        public static String FILE_NAME = "dataIndex";
 
-        public static String BASE_PATH = "/tmp/skywalking/data/index";
+        public static long SIZE = 1000 * 1000 * 1000;
 
-        public static String STORAGE_INDEX_FILE_NAME = "dataIndex";
 
-        public static long MAX_CAPACITY_PER_INDEX = 1000 * 1000 * 1000 * 1000;
+        public static class Operator {
+            public static int CACHE_SIZE = 5;
+        }
+    }
+
 
+    public static class BlockIndexEngine {
+        public static int L1_CACHE_SIZE = 10;
     }
 
 
@@ -45,11 +55,17 @@ public class Config {
 
         public static String CONNECT_URL = "127.0.0.1:2181";
 
-        public static String REGISTRY_PATH_PREFIX = "/storage_list/";
+        public static String PATH_PREFIX = "/skywalking/storage_list/";
     }
 
 
-    public static class SpanFinder {
-        public static int MAX_CACHE_SIZE = 10;
+    public static class Finder {
+        public static int CACHED_SIZE = 10;
+
+
+        public static class DataSource {
+            public static int MAX_POOL_SIZE = 20;
+            public static int MIN_IDLE      = 5;
+        }
     }
 }

skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/config/Constants.java

 package com.a.eye.skywalking.storage.config;
 
-import static com.a.eye.skywalking.storage.config.Config.DataIndex.TABLE_NAME;
-
 public class Constants {
 
+    public final static String TABLE_NAME = "data_index";
+
+    public static int MAX_BATCH_SIZE = 50;
+
     public static class SQL {
         public static final String CREATE_TABLE = "CREATE TABLE " + TABLE_NAME + "\n" + "(\n"
                 + "    id INT PRIMARY KEY NOT NULL IDENTITY,\n"
                 + "    trace_id VARCHAR(64) NOT NULL,\n"
-                + "    levelId VARCHAR(1024) NOT NULL,\n"
                 + "    span_type INT NOT NULL, \n"
                 + "    file_name VARCHAR(32) NOT NULL,\n"
                 + "    offset BIGINT NOT NULL,\n"
@@ -16,8 +17,8 @@ public class Constants {
 
         public static final String CREATE_INDEX = "CREATE INDEX \"index_data_trace_id_index\" ON " + TABLE_NAME + " (trace_id);";
 
-        public static final String INSERT_INDEX = "INSERT INTO " +TABLE_NAME + "(trace_id,levelId,span_type"
-                + ",file_name,offset,length) VALUES(?,?,?,?,?,?)";
+        public static final String INSERT_INDEX = "INSERT INTO " +TABLE_NAME + "(trace_id,span_type"
+                + ",file_name,offset,length) VALUES(?,?,?,?,?)";
 
         public static final String QUERY_TABLES = "SELECT count(1) AS TABLE_COUNT FROM   INFORMATION_SCHEMA.TABLES "
                 + "WHERE TABLE_NAME= '" + TABLE_NAME.toUpperCase() + "';";

skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/IndexDataCapacityMonitor.java

 package com.a.eye.skywalking.storage.data;
 
+import com.a.eye.skywalking.health.report.HealthCollector;
+import com.a.eye.skywalking.health.report.HeathReading;
 import com.a.eye.skywalking.logging.api.ILog;
 import com.a.eye.skywalking.logging.api.LogManager;
 import com.a.eye.skywalking.storage.block.index.BlockIndexEngine;
@@ -11,7 +13,7 @@ import java.util.TimerTask;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
 
-import static com.a.eye.skywalking.storage.config.Config.DataIndex.MAX_CAPACITY_PER_INDEX;
+import static com.a.eye.skywalking.storage.config.Config.DataIndex.SIZE;
 
 /**
  * Created by xin on 2016/11/6.
@@ -63,9 +65,12 @@ public class IndexDataCapacityMonitor {
 
         @Override
         public void run() {
-            if (currentSize.get() > MAX_CAPACITY_PER_INDEX * 0.8) {
+            if (currentSize.get() > SIZE * 0.8) {
                 notificationAddNewBlockIndexAndCreateNewIndexDB();
                 stopTimer();
+                HealthCollector.getCurrentHeathReading("Index Data Capacity Detector").updateData(HeathReading.INFO,
+                        "Detector is detecting the index [%d]. and the capacity of  {}  is %d", timestamp,
+                        currentSize.get());
             }
         }
     }
@@ -73,6 +78,7 @@ public class IndexDataCapacityMonitor {
     private static void notificationAddNewBlockIndexAndCreateNewIndexDB() {
         long timestamp = System.currentTimeMillis() + 5 * 60 * 1000;
         BlockIndexEngine.newUpdator().addRecord(timestamp);
+        logger.info("Create a new Index DB [{}]", timestamp);
         createNewIndexDB(timestamp);
         detector = new Detector(timestamp);
     }

skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/SpanDataConsumer.java

 package com.a.eye.skywalking.storage.data;
 
 import com.a.eye.datacarrier.consumer.IConsumer;
+import com.a.eye.skywalking.health.report.HealthCollector;
+import com.a.eye.skywalking.health.report.HeathReading;
 import com.a.eye.skywalking.storage.block.index.BlockIndexEngine;
 import com.a.eye.skywalking.storage.data.file.DataFileWriter;
 import com.a.eye.skywalking.storage.data.index.*;
@@ -34,6 +36,8 @@ public class SpanDataConsumer implements IConsumer<SpanData> {
             IndexMetaGroup<Long> metaGroup = iterator.next();
             IndexOperator indexOperator = IndexOperator.newOperator(getDBConnector(metaGroup));
             indexOperator.batchUpdate(metaGroup);
+            HealthCollector.getCurrentHeathReading("SpanDataConsumer")
+                    .updateData(HeathReading.INFO, "%s messages were successful consumed .", data.size());
         }
 
     }

skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/SpanDataFinder.java

@@ -15,10 +15,12 @@ import java.util.*;
 
 import static com.a.eye.skywalking.storage.config.Constants.SQL.DEFAULT_PASSWORD;
 import static com.a.eye.skywalking.storage.config.Constants.SQL.DEFAULT_USER;
+import static com.a.eye.skywalking.storage.util.PathResolver.getAbsolutePath;
 
 public class SpanDataFinder {
-    private static ILog                 logger          = LogManager.getLogger(SpanDataFinder.class);
-    private static IndexDataSourceCache datasourceCache = new IndexDataSourceCache(Config.SpanFinder.MAX_CACHE_SIZE);
+    private static      ILog                 logger          = LogManager.getLogger(SpanDataFinder.class);
+    private static      IndexDataSourceCache datasourceCache =
+            new IndexDataSourceCache(Config.Finder.CACHED_SIZE);
 
     public static List<SpanData> find(String traceId) {
         long blockIndex = BlockIndexEngine.newFinder().find(fetchStartTimeFromTraceId(traceId));
@@ -71,13 +73,13 @@ public class SpanDataFinder {
 
     private static HikariConfig generateDatasourceConfig(long blockIndex) {
         HikariConfig config = new HikariConfig();
-        config.setJdbcUrl(new ConnectURLGenerator(Config.DataIndex.BASE_PATH, Config.DataIndex.STORAGE_INDEX_FILE_NAME)
-                .generate(blockIndex));
+        config.setJdbcUrl(new ConnectURLGenerator(getAbsolutePath(Config.DataIndex.PATH),
+                Config.DataIndex.FILE_NAME).generate(blockIndex));
         config.setDriverClassName("org.hsqldb.jdbc.JDBCDriver");
         config.setUsername(DEFAULT_USER);
         config.setPassword(DEFAULT_PASSWORD);
-        config.setMaximumPoolSize(20);
-        config.setMinimumIdle(5);
+        config.setMaximumPoolSize(Config.Finder.DataSource.MAX_POOL_SIZE);
+        config.setMinimumIdle(Config.Finder.DataSource.MIN_IDLE);
         return config;
     }
 

skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/exception/SpanConvertFailedException.java

+package com.a.eye.skywalking.storage.data.exception;
+
+/**
+ * Created by xin on 2016/11/14.
+ */
+public class SpanConvertFailedException extends RuntimeException {
+    public SpanConvertFailedException(String message, Exception e) {
+        super(message, e);
+    }
+}

skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/file/DataFile.java

@@ -14,6 +14,8 @@ import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.io.IOException;
 
+import static com.a.eye.skywalking.storage.util.PathResolver.getAbsolutePath;
+
 /**
  * 数据文件
  */
@@ -25,7 +27,7 @@ public class DataFile {
     private DataFileOperator operator;
 
     static {
-        File dataFileDir = new File(Config.DataFile.BASE_PATH);
+        File dataFileDir = new File(getAbsolutePath(Config.DataFile.PATH));
         if (!dataFileDir.exists()) {
             dataFileDir.mkdirs();
         }
@@ -52,10 +54,13 @@ public class DataFile {
     }
 
     private void createFile() {
-        File dataFile = new File(Config.DataFile.BASE_PATH, fileName);
+        File dataFile = getDataFile();
         if (!dataFile.exists()) {
             try {
                 dataFile.createNewFile();
+                if (logger.isDebugEnable()) {
+                    logger.debug("Create an new data file[{}].", fileName);
+                }
             } catch (IOException e) {
                 logger.error("Failed to create data file.", e);
                 throw new DataFileOperatorCreateFailedException("Failed to create data file", e);
@@ -64,7 +69,11 @@ public class DataFile {
     }
 
     public boolean overLimitLength() {
-        return currentOffset >= Config.DataFile.MAX_LENGTH;
+        boolean isOverLimitLength = currentOffset >= Config.DataFile.SIZE;
+        if (isOverLimitLength) {
+            logger.info("Data File[{}] is over limit length.", fileName);
+        }
+        return isOverLimitLength;
     }
 
     public IndexMetaInfo write(SpanData data) {
@@ -107,7 +116,7 @@ public class DataFile {
 
             if (writer == null) {
                 try {
-                    writer = new FileOutputStream(new File(Config.DataFile.BASE_PATH, fileName), true);
+                    writer = new FileOutputStream(getDataFile(), true);
                 } catch (IOException e) {
                     throw new DataFileOperatorCreateFailedException("Failed to create datafile output stream", e);
                 }
@@ -119,7 +128,7 @@ public class DataFile {
         public FileInputStream getReader() {
             if (reader == null) {
                 try {
-                    reader = new FileInputStream(new File(Config.DataFile.BASE_PATH, fileName));
+                    reader = new FileInputStream(getDataFile());
                 } catch (IOException e) {
                     throw new DataFileOperatorCreateFailedException("Failed to create datafile input stream", e);
                 }
@@ -128,4 +137,13 @@ public class DataFile {
             return reader;
         }
     }
+
+    private File getDataFile() {
+        return new File(getAbsolutePath(Config.DataFile.PATH), fileName);
+    }
+
+    @Override
+    public String toString() {
+        return "DataFile{" + "fileName='" + fileName + '\'' + '}';
+    }
 }

skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/file/DataFileLoader.java

@@ -14,6 +14,11 @@ public class DataFileLoader {
 
     public List<DataFile> load() {
         File dataFileDir = new File(basePath);
+
+        if (!dataFileDir.exists()) {
+            dataFileDir.mkdirs();
+        }
+
         List<DataFile> allDataFile = new ArrayList<DataFile>();
         for (File fileEntry : dataFileDir.listFiles()) {
             allDataFile.add(new DataFile(fileEntry));

skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/file/DataFileReader.java

 package com.a.eye.skywalking.storage.data.file;
 
+import com.a.eye.skywalking.health.report.HealthCollector;
+import com.a.eye.skywalking.health.report.HeathReading;
 import com.a.eye.skywalking.logging.api.ILog;
 import com.a.eye.skywalking.logging.api.LogManager;
 import com.a.eye.skywalking.network.grpc.AckSpan;
@@ -13,6 +15,9 @@ import com.a.eye.skywalking.storage.data.spandata.SpanType;
 import java.util.ArrayList;
 import java.util.List;
 
+import static com.a.eye.skywalking.storage.data.spandata.SpanDataBuilder.buildAckSpan;
+import static com.a.eye.skywalking.storage.data.spandata.SpanDataBuilder.buildRequestSpan;
+
 /**
  * Created by xin on 2016/11/6.
  */
@@ -31,12 +36,12 @@ public class DataFileReader {
             byte[] dataByte = dataFile.read(indexMetaInfo.getOffset(), indexMetaInfo.getLength());
             try {
                 if (indexMetaInfo.getSpanType() == SpanType.RequestSpan) {
-                    metaData.add(new RequestSpanData(RequestSpan.parseFrom(dataByte)));
+                    metaData.add(new RequestSpanData(buildRequestSpan(dataByte)));
                 } else {
-                    metaData.add(new AckSpanData(AckSpan.parseFrom(dataByte)));
+                    metaData.add(new AckSpanData(buildAckSpan(dataByte)));
                 }
             } catch (Exception e) {
-                logger.error("Failed to conver to data", e);
+                logger.error("Failed to convert to data", e);
             }
         }
 

skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/file/DataFilesManager.java

@@ -5,6 +5,8 @@ import com.a.eye.skywalking.storage.config.Config;
 import java.util.List;
 import java.util.concurrent.ConcurrentLinkedDeque;
 
+import static com.a.eye.skywalking.storage.util.PathResolver.getAbsolutePath;
+
 /**
  * 管理数据文件，目前主要是用来加载所有未满的数据文件，以及创建数据文件
  */
@@ -13,7 +15,7 @@ public class DataFilesManager {
     private static ConcurrentLinkedDeque<DataFile> unFinishedDataFiles = new ConcurrentLinkedDeque<>();
 
     public static void init() {
-        List<DataFile> allDataFile = new DataFileLoader(Config.DataFile.BASE_PATH).load();
+        List<DataFile> allDataFile = new DataFileLoader(getAbsolutePath(Config.DataFile.PATH)).load();
         unFinishedDataFiles = new ConcurrentLinkedDeque<>(new UnFinishedDataFilePicker(allDataFile).pickUp());
     }
 

skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/file/UnFinishedDataFilePicker.java

 package com.a.eye.skywalking.storage.data.file;
 
+import com.a.eye.skywalking.logging.api.ILog;
+import com.a.eye.skywalking.logging.api.LogManager;
+
 import java.util.ArrayList;
 import java.util.List;
 
@@ -8,6 +11,8 @@ import java.util.List;
  */
 public class UnFinishedDataFilePicker {
 
+    private static ILog logger = LogManager.getLogger(UnFinishedDataFilePicker.class);
+
     private List<DataFile> dataFiles;
 
     public UnFinishedDataFilePicker(List<DataFile> dataFiles) {
@@ -17,10 +22,11 @@ public class UnFinishedDataFilePicker {
     public List<DataFile> pickUp() {
         List<DataFile> result = new ArrayList<DataFile>();
         for (DataFile file : dataFiles) {
-            if (file.overLimitLength()){
+            if (!file.overLimitLength()) {
                 result.add(file);
             }
         }
+        logger.info("Unfinished files: [{}].", result);
         return result;
     }
 }

skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/index/IndexDBConnector.java

@@ -3,6 +3,7 @@ package com.a.eye.skywalking.storage.data.index;
 import com.a.eye.skywalking.logging.api.ILog;
 import com.a.eye.skywalking.logging.api.LogManager;
 import com.a.eye.skywalking.storage.config.Config;
+import com.a.eye.skywalking.storage.config.Constants;
 import com.a.eye.skywalking.storage.data.exception.ConnectorInitializeFailedException;
 import com.a.eye.skywalking.storage.data.spandata.AckSpanData;
 import com.a.eye.skywalking.storage.data.spandata.RequestSpanData;
@@ -12,14 +13,13 @@ import com.a.eye.skywalking.storage.data.spandata.SpanType;
 import java.sql.*;
 
 import static com.a.eye.skywalking.storage.config.Constants.SQL.*;
+import static com.a.eye.skywalking.storage.util.PathResolver.getAbsolutePath;
 
 /**
  * Created by xin on 2016/11/4.
  */
 public class IndexDBConnector {
 
-    private static final int MAX_BATCH_SIZE = 20;
-
     private static ILog logger = LogManager.getLogger(IndexDBConnector.class);
 
     static {
@@ -32,8 +32,8 @@ public class IndexDBConnector {
 
     private long       timestamp;
     private Connection connection;
-    private ConnectURLGenerator generator =
-            new ConnectURLGenerator(Config.DataIndex.BASE_PATH, Config.DataIndex.STORAGE_INDEX_FILE_NAME);
+    private ConnectURLGenerator generator = new ConnectURLGenerator(getAbsolutePath(Config.DataIndex.PATH),
+            Config.DataIndex.FILE_NAME);
 
     public IndexDBConnector(long timestamp) {
         this.timestamp = timestamp;
@@ -41,7 +41,7 @@ public class IndexDBConnector {
         createTableAndIndexIfNecessary();
     }
 
-    public IndexDBConnector(Connection connection){
+    public IndexDBConnector(Connection connection) {
         this.connection = connection;
         createTableAndIndexIfNecessary();
     }
@@ -100,13 +100,12 @@ public class IndexDBConnector {
         PreparedStatement ps = connection.prepareStatement(INSERT_INDEX);
         for (IndexMetaInfo metaInfo : metaGroup.getMetaInfo()) {
             ps.setString(1, metaInfo.getTraceId());
-            ps.setString(2, metaInfo.getLevelId());
-            ps.setInt(3, metaInfo.getSpanType().getValue());
-            ps.setString(4, metaInfo.getFileName());
-            ps.setLong(5, metaInfo.getOffset());
-            ps.setInt(6, metaInfo.getLength());
+            ps.setInt(2, metaInfo.getSpanType().getValue());
+            ps.setString(3, metaInfo.getFileName());
+            ps.setLong(4, metaInfo.getOffset());
+            ps.setInt(5, metaInfo.getLength());
             ps.addBatch();
-            if (++currentIndex > MAX_BATCH_SIZE) {
+            if (++currentIndex > Constants.MAX_BATCH_SIZE) {
                 ps.executeBatch();
             }
         }
@@ -126,7 +125,7 @@ public class IndexDBConnector {
         return indexSize;
     }
 
-    public IndexMetaCollection queryByTraceId(String traceId){
+    public IndexMetaCollection queryByTraceId(String traceId) {
         try {
             PreparedStatement ps = connection.prepareStatement(QUERY_TRACE_ID);
             ps.setString(1, traceId);
@@ -143,10 +142,12 @@ public class IndexDBConnector {
                     spanData = new RequestSpanData();
                 }
 
-                collection.add(new IndexMetaInfo(spanData, rs.getString("file_name"), rs.getLong("offset"), rs.getInt("length")));
+                collection.add(new IndexMetaInfo(spanData, rs.getString("file_name"), rs.getLong("offset"),
+                        rs.getInt("length")));
             }
             return collection;
-        }catch(SQLException e){
+        } catch (SQLException e) {
+            logger.error("Failed to query trace Id [{}]", traceId, e);
             return new IndexMetaCollection();
         }
     }

skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/index/IndexDBConnectorCache.java

 package com.a.eye.skywalking.storage.data.index;
 
+import com.a.eye.skywalking.storage.config.Config;
+
 import java.util.LinkedHashMap;
 import java.util.Map;
 
 public class IndexDBConnectorCache {
 
-    private static final int MAX_CACHE_SIZE = 5;
-
     private LRUCache cachedOperators;
 
     public IndexDBConnectorCache() {
-        cachedOperators = new LRUCache(MAX_CACHE_SIZE);
+        cachedOperators = new LRUCache(Config.DataIndex.Operator.CACHE_SIZE);
     }
 
     public IndexDBConnector get(long timestamp) {
@@ -33,7 +33,7 @@ public class IndexDBConnectorCache {
     private class LRUCache extends LinkedHashMap<Long, IndexDBConnector> {
         @Override
         protected boolean removeEldestEntry(Map.Entry<Long, IndexDBConnector> eldest) {
-            boolean isNeedRemove = size() > MAX_CACHE_SIZE;
+            boolean isNeedRemove = size() > Config.DataIndex.Operator.CACHE_SIZE;
             if (isNeedRemove) {
                 removedCache(eldest.getValue());
             }

skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/index/IndexOperator.java

@@ -16,11 +16,6 @@ public class IndexOperator {
         timestamp = connector.getTimestamp();
     }
 
-    public List<IndexMetaInfo> find(String taceId) {
-        return new ArrayList<IndexMetaInfo>();
-    }
-
-
     public void batchUpdate(IndexMetaGroup metaGroup) {
         try {
             connector.batchUpdate(metaGroup);

skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/spandata/SpanDataBuilder.java

@@ -5,9 +5,7 @@ import com.a.eye.skywalking.logging.api.LogManager;
 import com.a.eye.skywalking.network.dependencies.com.google.protobuf.InvalidProtocolBufferException;
 import com.a.eye.skywalking.network.grpc.AckSpan;
 import com.a.eye.skywalking.network.grpc.RequestSpan;
-import com.a.eye.skywalking.storage.data.spandata.AckSpanData;
-import com.a.eye.skywalking.storage.data.spandata.RequestSpanData;
-import com.a.eye.skywalking.storage.data.spandata.SpanData;
+import com.a.eye.skywalking.storage.data.exception.SpanConvertFailedException;
 
 /**
  * Created by xin on 2016/11/12.
@@ -29,7 +27,7 @@ public class SpanDataBuilder {
             return AckSpan.parseFrom(data);
         } catch (InvalidProtocolBufferException e) {
             logger.error("Failed to convert data to ack span.", e);
-            return null;
+            throw new SpanConvertFailedException("Failed to convert byte data to ack span", e);
         }
     }
 
@@ -38,7 +36,7 @@ public class SpanDataBuilder {
             return RequestSpan.parseFrom(data);
         } catch (InvalidProtocolBufferException e) {
             logger.error("Failed to convert data to request span.", e);
-            return null;
+            throw new SpanConvertFailedException("Failed to convert byte data to request span", e);
         }
     }
 }

skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/notifier/SpanDataHelper.java → skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/data/spandata/SpanDataHelper.java

-package com.a.eye.skywalking.storage.notifier;
+package com.a.eye.skywalking.storage.data.spandata;
 
 import com.a.eye.skywalking.network.grpc.Span;
-import com.a.eye.skywalking.storage.data.spandata.AckSpanData;
-import com.a.eye.skywalking.storage.data.spandata.RequestSpanData;
-import com.a.eye.skywalking.storage.data.spandata.SpanData;
 
 import java.util.ArrayList;
 import java.util.HashMap;

skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/notifier/SearchNotifier.java → skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/listener/SearchListener.java

-package com.a.eye.skywalking.storage.notifier;
+package com.a.eye.skywalking.storage.listener;
 
 import com.a.eye.skywalking.network.grpc.Span;
-import com.a.eye.skywalking.network.listener.TraceSearchNotifier;
+import com.a.eye.skywalking.network.listener.TraceSearchListener;
 import com.a.eye.skywalking.storage.data.SpanDataFinder;
 import com.a.eye.skywalking.storage.data.spandata.SpanData;
+import com.a.eye.skywalking.storage.data.spandata.SpanDataHelper;
 
 import java.util.List;
 
-public class SearchNotifier implements TraceSearchNotifier {
+public class SearchListener implements TraceSearchListener {
 
     @Override
     public List<Span> search(String s) {

skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/notifier/StorageNotifier.java → skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/listener/StorageListener.java

-package com.a.eye.skywalking.storage.notifier;
+package com.a.eye.skywalking.storage.listener;
 
 import com.a.eye.datacarrier.DataCarrier;
 import com.a.eye.skywalking.logging.api.ILog;
 import com.a.eye.skywalking.logging.api.LogManager;
 import com.a.eye.skywalking.network.grpc.AckSpan;
 import com.a.eye.skywalking.network.grpc.RequestSpan;
-import com.a.eye.skywalking.network.listener.SpanStorageNotifier;
+import com.a.eye.skywalking.network.listener.SpanStorageListener;
+import com.a.eye.skywalking.storage.config.Config;
+import com.a.eye.skywalking.storage.data.SpanDataConsumer;
 import com.a.eye.skywalking.storage.data.spandata.SpanData;
 import com.a.eye.skywalking.storage.data.spandata.SpanDataBuilder;
-import com.a.eye.skywalking.storage.data.SpanDataConsumer;
 
-public class StorageNotifier implements SpanStorageNotifier {
+public class StorageListener implements SpanStorageListener {
 
-    private ILog logger = LogManager.getLogger(StorageNotifier.class);
+    private ILog logger = LogManager.getLogger(StorageListener.class);
 
     private DataCarrier<SpanData> spanDataDataCarrier;
 
-    public StorageNotifier() {
-        spanDataDataCarrier = new DataCarrier<>(10, 1000);
+    public StorageListener() {
+        spanDataDataCarrier = new DataCarrier<>(Config.Server.CHANNEL_SIZE, Config.Server.BUFFER_SIZE);
         spanDataDataCarrier.consume(new SpanDataConsumer(), 5, true);
     }
 

skywalking-storage-center/skywalking-storage/src/main/java/com/a/eye/skywalking/storage/util/PathResolver.java

+package com.a.eye.skywalking.storage.util;
+
+import java.io.File;
+
+public class PathResolver {
+
+    private final static String STORAGE_HOME;
+
+    static {
+        STORAGE_HOME = System.getProperty("user.dir") + File.separator + "..";
+    }
+
+    public static String getAbsolutePath(String path) {
+        if (path.charAt(0) != '/') {
+            path = '/' + path;
+        }
+        return STORAGE_HOME + path;
+    }
+
+}

skywalking-storage-center/skywalking-storage/src/main/resources/config.properties

-#
+# the port which storage server listening
 server.port=34000
 #
+# the path that storage block index
+#blockindex.path=/block-index
 #
+# the name of file which storage block index
+#blockindex.file_name=data_file.index
 #
-blockindex.storage_base_path=/tmp/skywalking/block-index
-#
-blockindex.data_file_index_file_name=data_file.index
-#
-#
+# the path that storage data file
+#datafile.path=/data/file
 #
-datafile.base_path=/tmp/skywalking/data/file
+# the max size of data file (byte)
+#datafile.size=3221225472
 #
-datafile.max_length=3221225472
+# the path that storage data index
+#dataindex.path=/data/index
 #
+# the name of the file which storage data index
+#dataindex.file_name=dataIndex
 #
-#存放数据文件索引表名
-dataindex.table_name=data_index
-#数据文件索引存储位置
-dataindex.base_path=/tmp/skywalking/data/index
+# the max size of data index
+#dataindex.size=100000000
 #
-dataindex.storage_index_file_name=dataIndex
+# the cached size of data index operator
+#dataindex.operator.cache_size=10
 #
-dataindex.max_capacity_per_index=1000000000
+# the auth info which registry center
+#registrycenter.auth_info=
 #
+# the auth schema which registry center
+#registrycenter.auth_schema=
 #
+# the registry center connect url (Default: zookeeper)
+registrycenter.connect_url=127.0.0.1:2181
 #
-registrycenter.auth_info=
+# the prefix of path that each storage node register
+#registrycenter.path_prefix=/skywalking/storage_list/
 #
-registrycenter.auth_schema=
+# the level1 size of block index engine
+#blockindexengine.l1_cache_size=10
 #
-registrycenter.connect_url= 127.0.0.1:2181
+# the cached size of finder
+#finder.cached_size=10
 #
-registrycenter.registry_path_prefix=
+# the max pool size of data source that finder operate
+#finder.datasource.max_pool_size=20
 #
+# the min idle of data source that finder operate
+#finder.datasource.min_idle=5

skywalking-storage-center/skywalking-storage/src/main/resources/log4j.properties

-# log4j properties #
-
 # logger #
-log4j.rootLogger=DEBUG,CONSOLE
-log4j.logger.org=ON
-#log4j.logger.org.systemgo.devframework=DEBUG
+log4j.rootLogger=CONSOLE,Rolling_File
+log4j.logger.org.apache=OFF
+log4j.logger.io.netty=OFF
+log4j.logger.com.a.eye.skywalking.network.dependencies.io.netty=OFF
 
 # Console Appender #
 log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
@@ -11,3 +10,12 @@ log4j.appender.CONSOLE.Target=System.out
 log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
 log4j.appender.CONSOLE.layout.ConversionPattern=%d %-5p %c{1}:%L - %m%n
 
+log4j.appender.Rolling_File=org.apache.log4j.RollingFileAppender
+log4j.appender.Rolling_File.Threshold=WARN
+log4j.appender.Rolling_File.File=logs/storage-server-log4j.log
+log4j.appender.Rolling_File.Append=true
+log4j.appender.Rolling_File.MaxFileSize=100MB
+log4j.appender.Rolling_File.MaxBackupIndex=5
+log4j.appender.Rolling_File.layout=org.apache.log4j.PatternLayout
+log4j.appender.Rolling_File.layout.ConversionPattern=%d %-5p %c{1}:%L - %m%n
+

skywalking-storage-center/skywalking-storage/src/main/resources/log4j2.xml

 <?xml version="1.0" encoding="UTF-8"?>
-<Configuration status="debug">
-	<Appenders>
-		<Console name="Console" target="SYSTEM_OUT">
-			<PatternLayout pattern="%d - %c -%-4r [%t] %-5p %x - %m%n"/>
-		</Console>
-	</Appenders>
-	<Loggers>
-		<Root level="debug">
-			<AppenderRef ref="Console"/>
-		</Root>
-	</Loggers>
-</Configuration>
\ No newline at end of file
+<Configuration status="WARN" name="storage-server" packages="">
+    <Appenders>
+        <Console name="Console" target="SYSTEM_OUT">
+            <PatternLayout pattern="%d %-5p %c{1}:%L - %m%n"/>
+        </Console>
+        <RollingFile name="RollingFile" fileName="logs/storage-server.log"
+                     filePattern="logs/storage-server-%d{MM-dd-yyyy}.log.gz"
+                     ignoreExceptions="false">
+            <PatternLayout>
+                <Pattern>%d %-5p %c{1}:%L - %m%n</Pattern>
+            </PatternLayout>
+            <SizeBasedTriggeringPolicy size="200MB"/>
+            <DefaultRolloverStrategy max="5"/>
+        </RollingFile>
+    </Appenders>
+    <Loggers>
+        <Root level="DEBUG">
+            <AppenderRef ref="RollingFile" level="INFO"/>
+            <appender-ref ref="Console" level="DEBUG"/>
+        </Root>
+    </Loggers>
+</Configuration>