add memory tool shell script

iotdb/iotdb/bin/momory-tool.bat → iotdb/iotdb/bin/memory-tool.bat

@@ -17,32 +17,41 @@
 @REM under the License.
 @REM
 
+
+@echo off
 if "%OS%" == "Windows_NT" setlocal
 
 pushd %~dp0..
 if NOT DEFINED IOTDB_HOME set IOTDB_HOME=%CD%
 popd
 
-if NOT DEFINED MAIN_CLASS set MAIN_CLASS=org.apache.iotdb.cluster.service.nodetool.NodeTool
+set IOTDB_CONF=%IOTDB_HOME%\conf
+
+if NOT DEFINED MAIN_CLASS set MAIN_CLASS=org.apache.iotdb.db.tools.MemEst.MemEstTool
 if NOT DEFINED JAVA_HOME goto :err
 
 @REM -----------------------------------------------------------------------------
 @REM JVM Opts we'll use in legacy run or installation
 set JAVA_OPTS=-ea^
+ -Dlogback.configurationFile="%IOTDB_CONF%\logback-tool.xml"^
  -DIOTDB_HOME=%IOTDB_HOME%
 
-REM For each jar in the IOTDB_HOME lib directory call append to build the CLASSPATH variable.
-for %%i in ("%IOTDB_HOME%\lib_cluster\*.jar") do call :append "%%i"
+@REM ***** CLASSPATH library setting *****
+@REM Ensure that any user defined CLASSPATH variables are not used on startup
+set CLASSPATH="%IOTDB_HOME%\lib"
+
+@REM For each jar in the IOTDB_HOME lib directory call append to build the CLASSPATH variable.
+for %%i in ("%IOTDB_HOME%\lib\*.jar") do call :append "%%i"
 goto okClasspath
 
 :append
 set CLASSPATH=%CLASSPATH%;%1
 goto :eof
 
-REM -----------------------------------------------------------------------------
+@REM -----------------------------------------------------------------------------
 :okClasspath
 
-"%JAVA_HOME%\bin\java" %JAVA_OPTS% -cp "%CLASSPATH%" %MAIN_CLASS% %*
+"%JAVA_HOME%\bin\java" %JAVA_OPTS% %JAVA_OPTS% -cp "%CLASSPATH%" %MAIN_CLASS% %*
 
 goto finally
 

iotdb/iotdb/bin/memory-tool.sh

@@ -22,15 +22,15 @@ if [ -z "${IOTDB_HOME}" ]; then
   export IOTDB_HOME="$(cd "`dirname "$0"`"/..; pwd)"
 fi
 
-
-MAIN_CLASS=org.apache.iotdb.cluster.service.nodetool.NodeTool
-
+IOTDB_CONF=${IOTDB_HOME}/conf
+# IOTDB_LOGS=${IOTDB_HOME}/logs
 
 CLASSPATH=""
-for f in ${IOTDB_HOME}/lib_cluster/*.jar; do
+for f in ${IOTDB_HOME}/lib/*.jar; do
   CLASSPATH=${CLASSPATH}":"$f
 done
 
+MAIN_CLASS=org.apache.iotdb.db.tools.MemEst.MemEstTool
 
 if [ -n "$JAVA_HOME" ]; then
     for java in "$JAVA_HOME"/bin/amd64/java "$JAVA_HOME"/bin/java; do
@@ -43,6 +43,6 @@ else
     JAVA=java
 fi
 
-exec "$JAVA" -cp "$CLASSPATH" "$MAIN_CLASS" "$@"
+iotdb_parms="-Dlogback.configurationFile=${IOTDB_CONF}/logback-tool.xml"
 
-exit $?
+exec "$JAVA" $iotdb_parms -cp "$CLASSPATH" "$MAIN_CLASS" "$@"
\ No newline at end of file

iotdb/iotdb/bin/start-server.bat

@@ -78,7 +78,7 @@ set JAVA_OPTS=-ea^
 @REM Ensure that any user defined CLASSPATH variables are not used on startup
 set CLASSPATH="%IOTDB_HOME%\lib"
 
-REM For each jar in the IOTDB_HOME lib directory call append to build the CLASSPATH variable.
+@REM For each jar in the IOTDB_HOME lib directory call append to build the CLASSPATH variable.
 for %%i in ("%IOTDB_HOME%\lib\*.jar") do call :append "%%i"
 set CLASSPATH=%CLASSPATH%;iotdb.IoTDB
 goto okClasspath
@@ -87,7 +87,7 @@ goto okClasspath
 set CLASSPATH=%CLASSPATH%;%1
 goto :eof
 
-REM -----------------------------------------------------------------------------
+@REM -----------------------------------------------------------------------------
 :okClasspath
 
 rem echo CLASSPATH: %CLASSPATH%

iotdb/iotdb/bin/start-sync-client.bat

@@ -44,7 +44,7 @@ set JAVA_OPTS=-ea^
 @REM Ensure that any user defined CLASSPATH variables are not used on startup
 set CLASSPATH="%IOTDB_HOME%\lib"
 
-REM For each jar in the IOTDB_HOME lib directory call append to build the CLASSPATH variable.
+@REM For each jar in the IOTDB_HOME lib directory call append to build the CLASSPATH variable.
 for %%i in ("%IOTDB_HOME%\lib\*.jar") do call :append "%%i"
 set CLASSPATH=%CLASSPATH%;SyncClient
 goto okClasspath
@@ -53,10 +53,10 @@ goto okClasspath
 set CLASSPATH=%CLASSPATH%;%1
 goto :eof
 
-REM -----------------------------------------------------------------------------
+@REM -----------------------------------------------------------------------------
 :okClasspath
 
-rem echo CLASSPATH: %CLASSPATH%
+@rem echo CLASSPATH: %CLASSPATH%
 
 "%JAVA_HOME%\bin\java" %JAVA_OPTS% %IOTDB_HEAP_OPTS% -cp %CLASSPATH% %IOTDB_JMX_OPTS% %MAIN_CLASS%
 goto finally

iotdb/iotdb/conf/logback-tool.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+    Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+        http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+-->
+<configuration/>

iotdb/src/main/java/org/apache/iotdb/db/conf/adapter/CompressionRatio.java

@@ -68,11 +68,7 @@ public class CompressionRatio {
   private CompressionRatio() {
     directory = new File(
         FilePathUtils.regularizePath(CONFIG.getSystemDir()) + COMPRESSION_RATIO_DIR);
-    try {
-      restore();
-    } catch (IOException e) {
-      LOGGER.error("Can not restore CompressionRatio", e);
-    }
+    restore();
   }
 
   /**
@@ -123,8 +119,10 @@ public class CompressionRatio {
   /**
    * Restore compression ratio statistics from disk when system restart
    */
-  void restore() throws IOException {
-    checkDirectoryExist();
+  void restore() {
+    if (!directory.exists()) {
+      return;
+    }
     File[] ratioFiles = directory.listFiles((dir, name) -> name.startsWith(FILE_PREFIX));
     if (ratioFiles != null && ratioFiles.length > 0) {
       long maxTimes = 0;

iotdb/src/main/java/org/apache/iotdb/db/tools/MemEst/MemEstTool.java

+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.db.tools.MemEst;
+
+import com.google.common.base.Throwables;
+import com.google.common.collect.Lists;
+import io.airlift.airline.Cli;
+import io.airlift.airline.Help;
+import io.airlift.airline.ParseArgumentsMissingException;
+import io.airlift.airline.ParseArgumentsUnexpectedException;
+import io.airlift.airline.ParseCommandMissingException;
+import io.airlift.airline.ParseCommandUnrecognizedException;
+import io.airlift.airline.ParseOptionConversionException;
+import io.airlift.airline.ParseOptionMissingException;
+import io.airlift.airline.ParseOptionMissingValueException;
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+import org.apache.commons.io.FileUtils;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+
+public class MemEstTool {
+
+  public static void main(String... args) throws IOException {
+    List<Class<? extends Runnable>> commands = Lists.newArrayList(
+        Help.class,
+        MemEstToolCmd.class
+    );
+    Cli.CliBuilder<Runnable> builder = Cli.builder("memory-tool");
+
+    builder.withDescription("Estimate memory for writing")
+        .withDefaultCommand(Help.class)
+        .withCommands(commands);
+
+    Cli<Runnable> parser = builder.build();
+
+    int status = 0;
+    try {
+      Runnable parse = parser.parse(args);
+      parse.run();
+    } catch (IllegalArgumentException |
+        IllegalStateException |
+        ParseArgumentsMissingException |
+        ParseArgumentsUnexpectedException |
+        ParseOptionConversionException |
+        ParseOptionMissingException |
+        ParseOptionMissingValueException |
+        ParseCommandMissingException |
+        ParseCommandUnrecognizedException e) {
+      badUse(e);
+      status = 1;
+    } catch (Exception e) {
+      err(Throwables.getRootCause(e));
+      status = 2;
+    }
+    FileUtils.deleteDirectory(new File(IoTDBDescriptor.getInstance().getConfig().getBaseDir()));
+    System.exit(status);
+  }
+
+  private static void badUse(Exception e) {
+    System.out.println("memory-tool: " + e.getMessage());
+    System.out.println("See 'memory-tool help' or 'memory-tool help <command>'.");
+  }
+
+  private static void err(Throwable e) {
+    System.err.println("error: " + e.getMessage());
+    System.err.println("-- StackTrace --");
+    System.err.println(Throwables.getStackTraceAsString(e));
+  }
+}

iotdb/src/main/java/org/apache/iotdb/db/tools/MemEst/MemEstToolCmd.java

+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.db.tools.MemEst;
+
+import io.airlift.airline.Command;
+import io.airlift.airline.Option;
+import io.airlift.airline.OptionType;
+import org.apache.iotdb.db.conf.IoTDBConfig;
+import org.apache.iotdb.db.conf.IoTDBConstant;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.conf.adapter.IoTDBConfigDynamicAdapter;
+import org.apache.iotdb.db.exception.ConfigAdjusterException;
+import org.apache.iotdb.db.metadata.MManager;
+
+@Command(name = "calmem", description = "calculate minimum memory required for writing based on the number of storage groups and timeseries")
+public class MemEstToolCmd implements Runnable {
+
+  @Option(type = OptionType.GLOBAL, title = "storage group number", name = {"-sg",
+      "--storagegroup"}, description = "Storage group number")
+  private String sgNumString = "10";
+
+  @Option(type = OptionType.GLOBAL, title = "total timeseries number", name = {"-ts",
+      "--tsNum"}, description = "Total timeseries number")
+  private String tsNumString = "1000";
+
+  @Option(title = "max timeseries", name = {"-mts",
+      "--mtsNum"}, description = "Maximum timeseries number among storage groups, make sure that it's smaller than total timeseries number")
+  private String maxTsNumString = "0";
+
+  @Override
+  public void run() {
+    IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
+    long memTableSize = config.getMemtableSizeThreshold();
+    int maxMemtableNumber = config.getMaxMemtableNumber();
+    long tsFileSize = config.getTsFileSizeThreshold();
+    long memory = IoTDBConstant.GB;
+    int sgNum = Integer.parseInt(sgNumString);
+    int tsNum = Integer.parseInt(tsNumString);
+    int maxTsNum = Integer.parseInt(maxTsNumString);
+    while (true) {
+      // init parameter
+      config.setAllocateMemoryForWrite(memory);
+      config.setMemtableSizeThreshold(memTableSize);
+      config.setMaxMemtableNumber(maxMemtableNumber);
+      config.setTsFileSizeThreshold(tsFileSize);
+      IoTDBConfigDynamicAdapter.getInstance().reset();
+      IoTDBConfigDynamicAdapter.getInstance().setInitialized(true);
+      MManager.getInstance().clear();
+
+      int sgCnt = 1;
+      int tsCnt = 1;
+      try {
+        for (; sgCnt <= sgNum; sgCnt++) {
+          IoTDBConfigDynamicAdapter.getInstance().addOrDeleteStorageGroup(1);
+        }
+        for (; tsCnt <= tsNum; tsCnt++) {
+          IoTDBConfigDynamicAdapter.getInstance().addOrDeleteTimeSeries(1);
+          MManager.getInstance().setMaxSeriesNumberAmongStorageGroup(
+              maxTsNum == 0 ? tsCnt / sgNum + 1 : Math.min(tsCnt, maxTsNum));
+        }
+
+      } catch (ConfigAdjusterException e) {
+        if(sgCnt > sgNum) {
+          System.out
+              .print(String.format("Memory estimation progress : %d%%\r", tsCnt * 100 / tsNum));
+        }
+        memory += IoTDBConstant.GB;
+        continue;
+      }
+      break;
+    }
+    System.out.println(String.format("SG: %d, TS: %d, MTS: %d, Memory for writing: %dGB", sgNum,
+        tsNum, maxTsNum == 0 ? tsNum / sgNum + 1 : maxTsNum, memory / IoTDBConstant.GB));
+  }
+}

iotdb/src/main/java/org/apache/iotdb/db/tools/WalChecker.java

@@ -16,7 +16,6 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-
 package org.apache.iotdb.db.tools;
 
 import static org.apache.iotdb.db.writelog.node.ExclusiveWriteLogNode.WAL_FILE_NAME;

iotdb/src/test/java/org/apache/iotdb/db/conf/adapter/IoTDBConfigDynamicAdapterTest.java

@@ -146,44 +146,4 @@ public class IoTDBConfigDynamicAdapterTest {
       assertEquals("The IoTDB system load is too large to add timeseries.", e.getMessage());
     }
   }
-
-  @Test
-  public void addOrDeleteTimeSeriesSyso2() {
-    int sgNum = 1;
-    IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
-    long memTableSize = config.getMemtableSizeThreshold();
-    int maxMemtableNumber = config.getMaxMemtableNumber();
-    long tsFileSize = config.getTsFileSizeThreshold();
-    long memory = 1024 * 1024 * 1024L;
-    while (true) {
-      config.setAllocateMemoryForWrite(memory);
-      config.setMemtableSizeThreshold(memTableSize);
-      config.setMaxMemtableNumber(maxMemtableNumber);
-      config.setTsFileSizeThreshold(tsFileSize);
-      IoTDBConfigDynamicAdapter.getInstance().reset();
-      IoTDBConfigDynamicAdapter.getInstance().setInitialized(true);
-      MManager.getInstance().clear();
-      for (int i = 1; i <= 50 ; i++) {
-        try {
-          IoTDBConfigDynamicAdapter.getInstance().addOrDeleteStorageGroup(sgNum);
-        } catch (ConfigAdjusterException e) {
-          e.printStackTrace();
-        }
-      }
-      int i = 1;
-      try {
-        for (; i <= 10000000; i++) {
-          IoTDBConfigDynamicAdapter.getInstance().addOrDeleteTimeSeries(1);
-          MManager.getInstance().setMaxSeriesNumberAmongStorageGroup(i / 50 + 1);
-        }
-      } catch (ConfigAdjusterException e) {
-//        System.out.println(i);
-        memory += 1024 * 1024 * 1024L;
-//        System.out.println("Memory for writing: " + memory / 1024 / 1024 / 1024 + "GB");
-        continue;
-      }
-      break;
-    }
-    System.out.println("Memory for writing: " + memory / 1024 / 1024 / 1024 + "GB");
-  }
 }
\ No newline at end of file