chore: move few tests items out (#16637)

* chore: move few tests items out

* fix: add taos as dep to tmq_demo

CMakeLists.txt

@@ -34,7 +34,8 @@ endif(${BUILD_TEST})
 
 add_subdirectory(source)
 add_subdirectory(tools)
-add_subdirectory(tests)
+#add_subdirectory(tests)
+add_subdirectory(utils)
 add_subdirectory(examples/c)
 
 # docs 

tests/test/c/CMakeLists.txt

 add_executable(tmq_demo tmqDemo.c)
+add_dependencies(tmq_demo taos)
 add_executable(tmq_sim tmqSim.c)
 add_executable(create_table createTable.c)
 add_executable(tmq_taosx_ci tmq_taosx_ci.c)

utils/CMakeLists.txt

+#ADD_SUBDIRECTORY(examples/c)
+ADD_SUBDIRECTORY(tsim)
+ADD_SUBDIRECTORY(test/c)
+#ADD_SUBDIRECTORY(comparisonTest/tdengine)

utils/test/c/CMakeLists.txt

+add_executable(tmq_demo tmqDemo.c)
+add_dependencies(tmq_demo taos)
+add_executable(tmq_sim tmqSim.c)
+add_executable(create_table createTable.c)
+add_executable(tmq_taosx_ci tmq_taosx_ci.c)
+add_executable(sml_test sml_test.c)
+target_link_libraries(
+    create_table
+    PUBLIC taos_static
+    PUBLIC util
+    PUBLIC common
+    PUBLIC os
+)
+target_link_libraries(
+    tmq_demo
+    PUBLIC taos_static
+    PUBLIC util
+    PUBLIC common
+    PUBLIC os
+)
+target_link_libraries(
+    tmq_sim
+    PUBLIC taos_static
+    PUBLIC util
+    PUBLIC common
+    PUBLIC os
+)
+target_link_libraries(
+    tmq_taosx_ci
+    PUBLIC taos_static
+    PUBLIC util
+    PUBLIC common
+    PUBLIC os
+)
+
+target_link_libraries(
+    sml_test
+    PUBLIC taos_static
+    PUBLIC util
+    PUBLIC common
+    PUBLIC os
+)
+
+add_executable(sdbDump sdbDump.c)
+target_link_libraries(
+    sdbDump
+    PUBLIC dnode
+    PUBLIC mnode
+    PUBLIC stream
+    PUBLIC sdb
+    PUBLIC os
+)
+target_include_directories(
+    sdbDump
+    PUBLIC "${TD_SOURCE_DIR}/include/dnode/mnode"
+    PRIVATE "${TD_SOURCE_DIR}/source/dnode/mnode/impl/inc"
+    PRIVATE "${TD_SOURCE_DIR}/source/dnode/mnode/sdb/inc"
+    PRIVATE "${TD_SOURCE_DIR}/source/dnode/mgmt/node_mgmt/inc"
+)

utils/test/c/createTable.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "taos.h"
+#include "taoserror.h"
+#include "tlog.h"
+
+#define GREEN "\033[1;32m"
+#define NC "\033[0m"
+
+char    dbName[32] = "db";
+char    stbName[64] = "st";
+int32_t numOfThreads = 1;
+int64_t numOfTables = 200000;
+int64_t startOffset = 0;
+int32_t createTable = 1;
+int32_t insertData = 0;
+int32_t batchNumOfTbl = 100;
+int32_t batchNumOfRow = 1;
+int32_t totalRowsOfPerTbl = 1;
+int32_t numOfVgroups = 2;
+int32_t showTablesFlag = 0;
+int32_t queryFlag = 0;
+
+int64_t startTimestamp = 1640966400000;  // 2020-01-01 00:00:00.000
+
+typedef struct {
+  int64_t   tableBeginIndex;
+  int64_t   tableEndIndex;
+  int32_t   threadIndex;
+  char      dbName[32];
+  char      stbName[64];
+  float     createTableSpeed;
+  float     insertDataSpeed;
+  int64_t   startMs;
+  int64_t   maxDelay;
+  int64_t   minDelay;
+  TdThread thread;
+} SThreadInfo;
+
+// void  parseArgument(int32_t argc, char *argv[]);
+// void *threadFunc(void *param);
+// void  createDbAndStb();
+
+void createDbAndStb() {
+  pPrint("start to create db and stable");
+  char qstr[64000];
+
+  TAOS *con = taos_connect(NULL, "root", "taosdata", NULL, 0);
+  if (con == NULL) {
+    pError("failed to connect to DB, reason:%s", taos_errstr(NULL));
+    exit(1);
+  }
+
+  sprintf(qstr, "create database if not exists %s vgroups %d", dbName, numOfVgroups);
+  TAOS_RES *pRes = taos_query(con, qstr);
+  int32_t   code = taos_errno(pRes);
+  if (code != 0) {
+    pError("failed to create database:%s, sql:%s, code:%d reason:%s", dbName, qstr, taos_errno(pRes),
+           taos_errstr(pRes));
+    exit(0);
+  }
+  taos_free_result(pRes);
+
+  sprintf(qstr, "use %s", dbName);
+  pRes = taos_query(con, qstr);
+  code = taos_errno(pRes);
+  if (code != 0) {
+    pError("failed to use db, code:%d reason:%s", taos_errno(pRes), taos_errstr(pRes));
+    exit(0);
+  }
+  taos_free_result(pRes);
+
+  sprintf(qstr, "create table if not exists %s (ts timestamp, i int) tags (j bigint)", stbName);
+  pRes = taos_query(con, qstr);
+  code = taos_errno(pRes);
+  if (code != 0) {
+    pError("failed to create stable, code:%d reason:%s", taos_errno(pRes), taos_errstr(pRes));
+    exit(0);
+  }
+  taos_free_result(pRes);
+
+  taos_close(con);
+}
+
+void printCreateProgress(SThreadInfo *pInfo, int64_t t) {
+  int64_t endMs = taosGetTimestampMs();
+  int64_t totalTables = t - pInfo->tableBeginIndex;
+  float   seconds = (endMs - pInfo->startMs) / 1000.0;
+  float   speed = totalTables / seconds;
+  pInfo->createTableSpeed = speed;
+  pPrint("thread:%d, %" PRId64 " tables created, time:%.2f sec, speed:%.1f tables/second, ", pInfo->threadIndex,
+         totalTables, seconds, speed);
+}
+
+void printInsertProgress(SThreadInfo *pInfo, int64_t insertTotalRows) {
+  int64_t endMs = taosGetTimestampMs();
+  //int64_t totalTables = t - pInfo->tableBeginIndex;
+  float   seconds = (endMs - pInfo->startMs) / 1000.0;
+  float   speed = insertTotalRows / seconds;
+  pInfo->insertDataSpeed = speed;
+  pPrint("thread:%d, %" PRId64 " rows inserted, time:%.2f sec, speed:%.1f rows/second, ", pInfo->threadIndex,
+         insertTotalRows, seconds, speed);
+}
+
+static int64_t getResult(TAOS_RES *tres) {
+  TAOS_ROW row = taos_fetch_row(tres);
+  if (row == NULL) {
+    return 0;
+  }
+
+  int         num_fields = taos_num_fields(tres);
+  TAOS_FIELD *fields = taos_fetch_fields(tres);
+  int         precision = taos_result_precision(tres);
+
+  int64_t numOfRows = 0;
+  do {
+    numOfRows++;
+    row = taos_fetch_row(tres);
+  } while (row != NULL);
+
+  return numOfRows;
+}
+
+void showTables() {
+  pPrint("start to show tables");
+  char qstr[128];
+
+  TAOS *con = taos_connect(NULL, "root", "taosdata", NULL, 0);
+  if (con == NULL) {
+    pError("failed to connect to DB, reason:%s", taos_errstr(NULL));
+    exit(1);
+  }
+
+  snprintf(qstr, 128, "use %s", dbName);
+  TAOS_RES *pRes = taos_query(con, qstr);
+  int       code = taos_errno(pRes);
+  if (code != 0) {
+    pError("failed to use db, code:%d reason:%s", taos_errno(pRes), taos_errstr(pRes));
+    exit(1);
+  }
+  taos_free_result(pRes);
+
+  sprintf(qstr, "show tables");
+  pRes = taos_query(con, qstr);
+  code = taos_errno(pRes);
+  if (code != 0) {
+    pError("failed to show tables, code:%d reason:%s", taos_errno(pRes), taos_errstr(pRes));
+    exit(0);
+  }
+
+  int64_t totalTableNum = getResult(pRes);
+  taos_free_result(pRes);
+
+  pPrint("%s database: %s, total %" PRId64 " tables %s", GREEN, dbName, totalTableNum, NC);
+
+  taos_close(con);
+}
+
+void *threadFunc(void *param) {
+  SThreadInfo *pInfo = (SThreadInfo *)param;
+  char        *qstr = taosMemoryMalloc(batchNumOfTbl * batchNumOfRow * 128);
+  int32_t      code = 0;
+
+  TAOS *con = taos_connect(NULL, "root", "taosdata", NULL, 0);
+  if (con == NULL) {
+    pError("index:%d, failed to connect to DB, reason:%s", pInfo->threadIndex, taos_errstr(NULL));
+    exit(1);
+  }
+
+  //pPrint("====before thread:%d, table range: %" PRId64 " - %" PRId64 "\n", pInfo->threadIndex, pInfo->tableBeginIndex,
+  //       pInfo->tableEndIndex);
+
+  pInfo->tableBeginIndex += startOffset;
+  pInfo->tableEndIndex += startOffset;
+
+  pPrint("====thread:%d, table range: %" PRId64 " - %" PRId64 "\n", pInfo->threadIndex, pInfo->tableBeginIndex, pInfo->tableEndIndex);
+
+  sprintf(qstr, "use %s", pInfo->dbName);
+  TAOS_RES *pRes = taos_query(con, qstr);
+  taos_free_result(pRes);
+
+  if (createTable) {
+    int64_t curMs = 0;
+    int64_t beginMs = taosGetTimestampMs();
+    pInfo->startMs = beginMs;
+    int64_t t = pInfo->tableBeginIndex;
+    for (; t <= pInfo->tableEndIndex;) {
+      // int64_t batch = (pInfo->tableEndIndex - t);
+      // batch = MIN(batch, batchNum);
+
+      int32_t len = sprintf(qstr, "create table");
+      for (int32_t i = 0; i < batchNumOfTbl;) {
+        len += sprintf(qstr + len, " %s_t%" PRId64 " using %s tags(%" PRId64 ")", stbName, t, stbName, t);
+        t++;
+        i++;
+        if (t > pInfo->tableEndIndex) {
+          break;
+        }
+      }
+
+      int64_t   startTs = taosGetTimestampUs();
+      TAOS_RES *pRes = taos_query(con, qstr);
+      code = taos_errno(pRes);
+      if (code != 0) {
+        pError("failed to create table reason:%s, sql: %s", tstrerror(code), qstr);
+      }
+      taos_free_result(pRes);
+      int64_t endTs = taosGetTimestampUs();
+      int64_t delay = endTs - startTs;
+      // printf("==== %"PRId64" -  %"PRId64", %"PRId64"\n", startTs, endTs, delay);
+      if (delay > pInfo->maxDelay) pInfo->maxDelay = delay;
+      if (delay < pInfo->minDelay) pInfo->minDelay = delay;
+
+      curMs = taosGetTimestampMs();
+      if (curMs - beginMs > 10000) {
+        beginMs = curMs;
+        // printf("==== tableBeginIndex: %"PRId64", t: %"PRId64"\n", pInfo->tableBeginIndex, t);
+        printCreateProgress(pInfo, t);
+      }
+    }
+    printCreateProgress(pInfo, t);
+  }
+
+  if (insertData) {
+  	int64_t insertTotalRows = 0;
+    int64_t curMs = 0;
+    int64_t beginMs = taosGetTimestampMs();
+    pInfo->startMs = beginMs;
+    int64_t t = pInfo->tableBeginIndex;
+    for (; t <= pInfo->tableEndIndex; t++) {
+      //printf("table name: %"PRId64"\n", t);
+      int64_t ts = startTimestamp;
+      for (int32_t i = 0; i < totalRowsOfPerTbl;) {
+        int32_t len = sprintf(qstr, "insert into ");
+        len += sprintf(qstr + len, "%s_t%" PRId64 " values ", stbName, t);
+        for (int32_t j = 0; j < batchNumOfRow; j++) {
+          len += sprintf(qstr + len, "(%" PRId64 ", 6666) ", ts++);
+		  i++;
+		  insertTotalRows++;
+		  if (i >= totalRowsOfPerTbl) {
+		  	break;
+		  }
+        }
+
+        #if 1
+        int64_t	startTs = taosGetTimestampUs();
+        TAOS_RES *pRes = taos_query(con, qstr);
+        code = taos_errno(pRes);
+        if (code != 0) {
+          pError("failed to insert %s_t%" PRId64 ", reason:%s", stbName, t, tstrerror(code));
+        }
+        taos_free_result(pRes);
+        int64_t endTs = taosGetTimestampUs();
+        int64_t delay = endTs - startTs;
+        // printf("==== %"PRId64" -  %"PRId64", %"PRId64"\n", startTs, endTs, delay);
+        if (delay > pInfo->maxDelay) pInfo->maxDelay = delay;
+        if (delay < pInfo->minDelay) pInfo->minDelay = delay;
+        
+        curMs = taosGetTimestampMs();
+        if (curMs - beginMs > 10000) {
+          beginMs = curMs;
+          // printf("==== tableBeginIndex: %"PRId64", t: %"PRId64"\n", pInfo->tableBeginIndex, t);
+          printInsertProgress(pInfo, insertTotalRows);
+        }
+        #endif		
+      }
+    }	
+    printInsertProgress(pInfo, insertTotalRows);    
+  }
+
+  taos_close(con);
+  taosMemoryFree(qstr);
+  return 0;
+}
+
+void printHelp() {
+  char indent[10] = "        ";
+  printf("Used to test the performance while create table\n");
+
+  printf("%s%s\n", indent, "-c");
+  printf("%s%s%s%s\n", indent, indent, "Configuration directory, default is ", configDir);
+  printf("%s%s\n", indent, "-d");
+  printf("%s%s%s%s\n", indent, indent, "The name of the database to be created, default is ", dbName);
+  printf("%s%s\n", indent, "-s");
+  printf("%s%s%s%s\n", indent, indent, "The name of the super table to be created, default is ", stbName);
+  printf("%s%s\n", indent, "-t");
+  printf("%s%s%s%d\n", indent, indent, "numOfThreads, default is ", numOfThreads);
+  printf("%s%s\n", indent, "-n");
+  printf("%s%s%s%" PRId64 "\n", indent, indent, "numOfTables, default is ", numOfTables);
+  printf("%s%s\n", indent, "-g");
+  printf("%s%s%s%" PRId64 "\n", indent, indent, "startOffset, default is ", startOffset);
+  printf("%s%s\n", indent, "-v");
+  printf("%s%s%s%d\n", indent, indent, "numOfVgroups, default is ", numOfVgroups);
+  printf("%s%s\n", indent, "-a");
+  printf("%s%s%s%d\n", indent, indent, "createTable, default is ", createTable);
+  printf("%s%s\n", indent, "-i");
+  printf("%s%s%s%d\n", indent, indent, "insertData, default is ", insertData);
+  printf("%s%s\n", indent, "-b");
+  printf("%s%s%s%d\n", indent, indent, "batchNumOfTbl, default is ", batchNumOfTbl);
+  printf("%s%s\n", indent, "-w");
+  printf("%s%s%s%d\n", indent, indent, "showTablesFlag, default is ", showTablesFlag);
+  printf("%s%s\n", indent, "-q");
+  printf("%s%s%s%d\n", indent, indent, "queryFlag, default is ", queryFlag);
+  printf("%s%s\n", indent, "-l");
+  printf("%s%s%s%d\n", indent, indent, "batchNumOfRow, default is ", batchNumOfRow);
+  printf("%s%s\n", indent, "-r");
+  printf("%s%s%s%d\n", indent, indent, "totalRowsOfPerTbl, default is ", totalRowsOfPerTbl);
+
+  exit(EXIT_SUCCESS);
+}
+
+void parseArgument(int32_t argc, char *argv[]) {
+  for (int32_t i = 1; i < argc; i++) {
+    if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0) {
+      printHelp();
+      exit(0);
+    } else if (strcmp(argv[i], "-d") == 0) {
+      strcpy(dbName, argv[++i]);
+    } else if (strcmp(argv[i], "-c") == 0) {
+      strcpy(configDir, argv[++i]);
+    } else if (strcmp(argv[i], "-s") == 0) {
+      strcpy(stbName, argv[++i]);
+    } else if (strcmp(argv[i], "-t") == 0) {
+      numOfThreads = atoi(argv[++i]);
+    } else if (strcmp(argv[i], "-n") == 0) {
+      numOfTables = atoll(argv[++i]);
+    } else if (strcmp(argv[i], "-g") == 0) {
+      startOffset = atoll(argv[++i]);
+    } else if (strcmp(argv[i], "-v") == 0) {
+      numOfVgroups = atoi(argv[++i]);
+    } else if (strcmp(argv[i], "-a") == 0) {
+      createTable = atoi(argv[++i]);
+    } else if (strcmp(argv[i], "-i") == 0) {
+      insertData = atoi(argv[++i]);
+    } else if (strcmp(argv[i], "-b") == 0) {
+      batchNumOfTbl = atoi(argv[++i]);
+    } else if (strcmp(argv[i], "-l") == 0) {
+      batchNumOfRow = atoi(argv[++i]);
+    } else if (strcmp(argv[i], "-r") == 0) {
+      totalRowsOfPerTbl = atoi(argv[++i]);
+    } else if (strcmp(argv[i], "-w") == 0) {
+      showTablesFlag = atoi(argv[++i]);
+    } else if (strcmp(argv[i], "-q") == 0) {
+      queryFlag = atoi(argv[++i]);
+    } else {
+      pPrint("%s unknow para: %s %s", GREEN, argv[++i], NC);
+    }
+  }
+
+  pPrint("%s dbName:%s %s", GREEN, dbName, NC);
+  pPrint("%s stbName:%s %s", GREEN, stbName, NC);
+  pPrint("%s configDir:%s %s", GREEN, configDir, NC);
+  pPrint("%s numOfTables:%" PRId64 " %s", GREEN, numOfTables, NC);
+  pPrint("%s startOffset:%" PRId64 " %s", GREEN, startOffset, NC);
+  pPrint("%s numOfThreads:%d %s", GREEN, numOfThreads, NC);
+  pPrint("%s numOfVgroups:%d %s", GREEN, numOfVgroups, NC);
+  pPrint("%s createTable:%d %s", GREEN, createTable, NC);
+  pPrint("%s insertData:%d %s", GREEN, insertData, NC);
+  pPrint("%s batchNumOfTbl:%d %s", GREEN, batchNumOfTbl, NC);
+  pPrint("%s batchNumOfRow:%d %s", GREEN, batchNumOfRow, NC);
+  pPrint("%s totalRowsOfPerTbl:%d %s", GREEN, totalRowsOfPerTbl, NC);
+  pPrint("%s showTablesFlag:%d %s", GREEN, showTablesFlag, NC);
+  pPrint("%s queryFlag:%d %s", GREEN, queryFlag, NC);
+
+  pPrint("%s start create table performace test %s", GREEN, NC);
+}
+
+int32_t main(int32_t argc, char *argv[]) {
+  parseArgument(argc, argv);
+
+  if (showTablesFlag) {
+    showTables();
+    return 0;
+  }
+
+  if (queryFlag) {
+    // selectRowsFromTable();
+    return 0;
+  }
+
+  if (createTable) {
+    createDbAndStb();
+  }
+
+  pPrint("%d threads are spawned to create %" PRId64 " tables, offset is %" PRId64 " ", numOfThreads, numOfTables,
+         startOffset);
+
+  TdThreadAttr thattr;
+  taosThreadAttrInit(&thattr);
+  taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE);
+  SThreadInfo *pInfo = (SThreadInfo *)taosMemoryCalloc(numOfThreads, sizeof(SThreadInfo));
+
+  // int64_t numOfTablesPerThread = numOfTables / numOfThreads;
+  // numOfTables = numOfTablesPerThread * numOfThreads;
+
+  if (numOfThreads < 1) {
+    numOfThreads = 1;
+  }
+
+  int64_t a = numOfTables / numOfThreads;
+  if (a < 1) {
+    numOfThreads = numOfTables;
+    a = 1;
+  }
+
+  int64_t b = 0;
+  b = numOfTables % numOfThreads;
+
+  int64_t tableFrom = 0;
+  for (int32_t i = 0; i < numOfThreads; ++i) {
+    pInfo[i].tableBeginIndex = tableFrom;
+    pInfo[i].tableEndIndex = (i < b ? tableFrom + a : tableFrom + a - 1);
+    tableFrom = pInfo[i].tableEndIndex + 1;
+    pInfo[i].threadIndex = i;
+    pInfo[i].minDelay = INT64_MAX;
+    strcpy(pInfo[i].dbName, dbName);
+    strcpy(pInfo[i].stbName, stbName);
+    taosThreadCreate(&(pInfo[i].thread), &thattr, threadFunc, (void *)(pInfo + i));
+  }
+
+  taosMsleep(300);
+  for (int32_t i = 0; i < numOfThreads; i++) {
+    taosThreadJoin(pInfo[i].thread, NULL);
+    taosThreadClear(&pInfo[i].thread);
+  }
+
+  int64_t maxDelay = 0;
+  int64_t minDelay = INT64_MAX;
+
+  float createTableSpeed = 0;
+  for (int32_t i = 0; i < numOfThreads; ++i) {
+    createTableSpeed += pInfo[i].createTableSpeed;
+
+    if (pInfo[i].maxDelay > maxDelay) maxDelay = pInfo[i].maxDelay;
+    if (pInfo[i].minDelay < minDelay) minDelay = pInfo[i].minDelay;
+  }
+
+  float insertDataSpeed = 0;
+  for (int32_t i = 0; i < numOfThreads; ++i) {
+    insertDataSpeed += pInfo[i].insertDataSpeed;
+  }
+
+  if (createTable) {
+    pPrint("%s total %" PRId64 " tables, %.1f tables/second, threads:%d, maxDelay: %" PRId64 "us, minDelay: %" PRId64
+           "us %s",
+           GREEN, numOfTables, createTableSpeed, numOfThreads, maxDelay, minDelay, NC);
+  }
+
+  if (insertData) {
+    pPrint("%s total %" PRId64 " tables, %.1f rows/second, threads:%d %s", GREEN, numOfTables, insertDataSpeed,
+           numOfThreads, NC);
+  }
+
+  taosThreadAttrDestroy(&thattr);
+  taosMemoryFree(pInfo);
+}

utils/test/c/sdbDump.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "dmMgmt.h"
+#include "mndInt.h"
+#include "sdb.h"
+#include "tconfig.h"
+#include "tjson.h"
+
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wunused-result"
+
+#define TMP_DNODE_DIR           TD_TMP_DIR_PATH "dumpsdb"
+#define TMP_MNODE_DIR           TD_TMP_DIR_PATH "dumpsdb" TD_DIRSEP "mnode"
+#define TMP_SDB_DATA_DIR        TD_TMP_DIR_PATH "dumpsdb" TD_DIRSEP "mnode" TD_DIRSEP "data"
+#define TMP_SDB_SYNC_DIR        TD_TMP_DIR_PATH "dumpsdb" TD_DIRSEP "mnode" TD_DIRSEP "sync"
+#define TMP_SDB_MNODE_JSON      TD_TMP_DIR_PATH "dumpsdb" TD_DIRSEP "mnode" TD_DIRSEP "mnode.json"
+#define TMP_SDB_DATA_FILE       TD_TMP_DIR_PATH "dumpsdb" TD_DIRSEP "mnode" TD_DIRSEP "data" TD_DIRSEP "sdb.data"
+#define TMP_SDB_RAFT_CFG_FILE   TD_TMP_DIR_PATH "dumpsdb" TD_DIRSEP "mnode" TD_DIRSEP "sync" TD_DIRSEP "raft_config.json"
+#define TMP_SDB_RAFT_STORE_FILE TD_TMP_DIR_PATH "dumpsdb" TD_DIRSEP "mnode" TD_DIRSEP "sync" TD_DIRSEP "raft_store.json"
+
+void reportStartup(const char *name, const char *desc) {}
+
+void sendRsp(SRpcMsg *pMsg) { rpcFreeCont(pMsg->pCont); }
+
+int32_t sendReq(const SEpSet *pEpSet, SRpcMsg *pMsg) {
+  terrno = TSDB_CODE_INVALID_PTR;
+  return -1;
+}
+
+char *i642str(int64_t val) {
+  static char str[24] = {0};
+  snprintf(str, sizeof(str), "%" PRId64, val);
+  return str;
+}
+
+void dumpFunc(SSdb *pSdb, SJson *json) {}
+
+void dumpDb(SSdb *pSdb, SJson *json) {
+  void  *pIter = NULL;
+  SJson *items = tjsonCreateObject();
+  tjsonAddItemToObject(json, "dbs", items);
+
+  while (1) {
+    SDbObj *pObj = NULL;
+    pIter = sdbFetch(pSdb, SDB_DB, pIter, (void **)&pObj);
+    if (pIter == NULL) break;
+
+    SJson *item = tjsonCreateObject();
+    tjsonAddItemToObject(items, "db", item);
+
+    tjsonAddStringToObject(item, "name", pObj->name);
+    tjsonAddStringToObject(item, "acct", pObj->acct);
+    tjsonAddStringToObject(item, "createUser", pObj->createUser);
+    tjsonAddStringToObject(item, "createdTime", i642str(pObj->createdTime));
+    tjsonAddStringToObject(item, "updateTime", i642str(pObj->updateTime));
+    tjsonAddStringToObject(item, "uid", i642str(pObj->uid));
+    tjsonAddIntegerToObject(item, "cfgVersion", pObj->cfgVersion);
+    tjsonAddIntegerToObject(item, "vgVersion", pObj->vgVersion);
+    tjsonAddIntegerToObject(item, "numOfVgroups", pObj->cfg.numOfVgroups);
+    tjsonAddIntegerToObject(item, "numOfStables", pObj->cfg.numOfStables);
+    tjsonAddIntegerToObject(item, "buffer", pObj->cfg.buffer);
+    tjsonAddIntegerToObject(item, "pageSize", pObj->cfg.pageSize);
+    tjsonAddIntegerToObject(item, "pages", pObj->cfg.pages);
+    tjsonAddIntegerToObject(item, "cacheLastSize", pObj->cfg.cacheLastSize);
+    tjsonAddIntegerToObject(item, "daysPerFile", pObj->cfg.daysPerFile);
+    tjsonAddIntegerToObject(item, "daysToKeep0", pObj->cfg.daysToKeep0);
+    tjsonAddIntegerToObject(item, "daysToKeep1", pObj->cfg.daysToKeep1);
+    tjsonAddIntegerToObject(item, "daysToKeep2", pObj->cfg.daysToKeep2);
+    tjsonAddIntegerToObject(item, "minRows", pObj->cfg.minRows);
+    tjsonAddIntegerToObject(item, "maxRows", pObj->cfg.maxRows);
+    tjsonAddIntegerToObject(item, "precision", pObj->cfg.precision);
+    tjsonAddIntegerToObject(item, "compression", pObj->cfg.compression);
+    tjsonAddIntegerToObject(item, "replications", pObj->cfg.replications);
+    tjsonAddIntegerToObject(item, "strict", pObj->cfg.strict);
+    tjsonAddIntegerToObject(item, "cacheLast", pObj->cfg.cacheLast);
+    tjsonAddIntegerToObject(item, "hashMethod", pObj->cfg.hashMethod);
+    tjsonAddIntegerToObject(item, "numOfRetensions", pObj->cfg.numOfRetensions);
+    tjsonAddIntegerToObject(item, "schemaless", pObj->cfg.schemaless);
+    tjsonAddIntegerToObject(item, "walLevel", pObj->cfg.walLevel);
+    tjsonAddIntegerToObject(item, "walFsyncPeriod", pObj->cfg.walFsyncPeriod);
+    tjsonAddIntegerToObject(item, "walRetentionPeriod", pObj->cfg.walRetentionPeriod);
+    tjsonAddIntegerToObject(item, "walRetentionSize", pObj->cfg.walRetentionSize);
+    tjsonAddIntegerToObject(item, "walRollPeriod", pObj->cfg.walRollPeriod);
+    tjsonAddIntegerToObject(item, "walSegmentSize", pObj->cfg.walSegmentSize);
+    
+    sdbRelease(pSdb, pObj);
+  }
+}
+
+void dumpStb(SSdb *pSdb, SJson *json) {
+  void  *pIter = NULL;
+  SJson *items = tjsonCreateObject();
+  tjsonAddItemToObject(json, "stbs", items);
+
+  while (1) {
+    SStbObj *pObj = NULL;
+    pIter = sdbFetch(pSdb, SDB_STB, pIter, (void **)&pObj);
+    if (pIter == NULL) break;
+
+    SJson *item = tjsonCreateObject();
+    tjsonAddItemToObject(items, "stb", item);
+
+    tjsonAddStringToObject(item, "name", pObj->name);
+    tjsonAddStringToObject(item, "db", pObj->db);
+    tjsonAddStringToObject(item, "createdTime", i642str(pObj->createdTime));
+    tjsonAddStringToObject(item, "updateTime", i642str(pObj->updateTime));
+    tjsonAddStringToObject(item, "uid", i642str(pObj->uid));
+    tjsonAddStringToObject(item, "dbUid", i642str(pObj->dbUid));
+    tjsonAddIntegerToObject(item, "tagVer", pObj->tagVer);
+    tjsonAddIntegerToObject(item, "colVer", pObj->colVer);
+    tjsonAddIntegerToObject(item, "nextColId", pObj->nextColId);
+    tjsonAddIntegerToObject(item, "watermark1", pObj->watermark[0]);
+    tjsonAddIntegerToObject(item, "watermark2", pObj->watermark[1]);
+    tjsonAddIntegerToObject(item, "maxdelay1", pObj->maxdelay[0]);
+    tjsonAddIntegerToObject(item, "maxdelay2", pObj->maxdelay[1]);
+    tjsonAddIntegerToObject(item, "ttl", pObj->ttl);
+    tjsonAddIntegerToObject(item, "numOfColumns", pObj->numOfColumns);
+    tjsonAddIntegerToObject(item, "numOfTags", pObj->numOfTags);
+    tjsonAddIntegerToObject(item, "commentLen", pObj->commentLen);
+    tjsonAddIntegerToObject(item, "ast1Len", pObj->ast1Len);
+    tjsonAddIntegerToObject(item, "ast2Len", pObj->ast2Len);
+
+    sdbRelease(pSdb, pObj);
+  }
+}
+
+void dumpSma(SSdb *pSdb, SJson *json) {}
+
+void dumpVgroup(SSdb *pSdb, SJson *json) {}
+
+void dumpTopic(SSdb *pSdb, SJson *json) {}
+
+void dumpConsumber(SSdb *pSdb, SJson *json) {}
+
+void dumpSubscribe(SSdb *pSdb, SJson *json) {}
+
+void dumpOffset(SSdb *pSdb, SJson *json) {}
+
+void dumpStream(SSdb *pSdb, SJson *json) {}
+
+void dumpAcct(SSdb *pSdb, SJson *json) {
+  void  *pIter = NULL;
+  SJson *items = tjsonCreateObject();
+  tjsonAddItemToObject(json, "accts", items);
+
+  while (1) {
+    SAcctObj *pObj = NULL;
+    pIter = sdbFetch(pSdb, SDB_ACCT, pIter, (void **)&pObj);
+    if (pIter == NULL) break;
+
+    SJson *item = tjsonCreateObject();
+    tjsonAddItemToObject(items, "acct", item);
+
+    tjsonAddStringToObject(item, "acct", pObj->acct);
+    tjsonAddStringToObject(item, "createdTime", i642str(pObj->createdTime));
+    tjsonAddStringToObject(item, "updateTime", i642str(pObj->updateTime));
+    tjsonAddIntegerToObject(item, "acctId", pObj->acctId);
+
+    sdbRelease(pSdb, pObj);
+  }
+}
+
+void dumpAuth(SSdb *pSdb, SJson *json) {}
+
+void dumpUser(SSdb *pSdb, SJson *json) {
+  void  *pIter = NULL;
+  SJson *items = tjsonCreateObject();
+  tjsonAddItemToObject(json, "users", items);
+
+  while (1) {
+    SUserObj *pObj = NULL;
+    pIter = sdbFetch(pSdb, SDB_USER, pIter, (void **)&pObj);
+    if (pIter == NULL) break;
+
+    SJson *item = tjsonCreateObject();
+    tjsonAddItemToObject(items, "user", item);
+
+    tjsonAddStringToObject(item, "name", pObj->user);
+    tjsonAddStringToObject(item, "pass", pObj->pass);
+    tjsonAddStringToObject(item, "acct", pObj->acct);
+    tjsonAddStringToObject(item, "createdTime", i642str(pObj->createdTime));
+    tjsonAddStringToObject(item, "updateTime", i642str(pObj->updateTime));
+    tjsonAddIntegerToObject(item, "superUser", pObj->superUser);
+    tjsonAddIntegerToObject(item, "authVersion", pObj->authVersion);
+    tjsonAddIntegerToObject(item, "numOfReadDbs", taosHashGetSize(pObj->readDbs));
+    tjsonAddIntegerToObject(item, "numOfWriteDbs", taosHashGetSize(pObj->writeDbs));
+
+    sdbRelease(pSdb, pObj);
+  }
+}
+
+void dumpDnode(SSdb *pSdb, SJson *json) {
+  void  *pIter = NULL;
+  SJson *items = tjsonCreateObject();
+  tjsonAddItemToObject(json, "dnodes", items);
+
+  while (1) {
+    SDnodeObj *pObj = NULL;
+    pIter = sdbFetch(pSdb, SDB_DNODE, pIter, (void **)&pObj);
+    if (pIter == NULL) break;
+
+    SJson *item = tjsonCreateObject();
+    tjsonAddItemToObject(items, "dnode", item);
+
+    tjsonAddIntegerToObject(item, "id", pObj->id);
+    tjsonAddStringToObject(item, "createdTime", i642str(pObj->createdTime));
+    tjsonAddStringToObject(item, "updateTime", i642str(pObj->updateTime));
+    tjsonAddIntegerToObject(item, "port", pObj->port);
+    tjsonAddStringToObject(item, "fqdn", pObj->fqdn);
+
+    sdbRelease(pSdb, pObj);
+  }
+}
+
+void dumpBnode(SSdb *pSdb, SJson *json) {}
+
+void dumpSnode(SSdb *pSdb, SJson *json) {}
+
+void dumpQnode(SSdb *pSdb, SJson *json) {}
+
+void dumpMnode(SSdb *pSdb, SJson *json) {
+  void  *pIter = NULL;
+  SJson *items = tjsonCreateObject();
+  tjsonAddItemToObject(json, "mnodes", items);
+
+  while (1) {
+    SMnodeObj *pObj = NULL;
+    pIter = sdbFetch(pSdb, SDB_MNODE, pIter, (void **)&pObj);
+    if (pIter == NULL) break;
+
+    SJson *item = tjsonCreateObject();
+    tjsonAddItemToObject(items, "mnode", item);
+
+    tjsonAddIntegerToObject(item, "id", pObj->id);
+    tjsonAddStringToObject(item, "createdTime", i642str(pObj->createdTime));
+    tjsonAddStringToObject(item, "updateTime", i642str(pObj->updateTime));
+
+    sdbRelease(pSdb, pObj);
+  }
+}
+
+void dumpCluster(SSdb *pSdb, SJson *json) {
+  void  *pIter = NULL;
+  SJson *items = tjsonCreateObject();
+  tjsonAddItemToObject(json, "clusters", items);
+
+  while (1) {
+    SClusterObj *pObj = NULL;
+    pIter = sdbFetch(pSdb, SDB_CLUSTER, pIter, (void **)&pObj);
+    if (pIter == NULL) break;
+
+    SJson *item = tjsonCreateObject();
+    tjsonAddItemToObject(items, "cluster", item);
+
+    tjsonAddStringToObject(item, "id", i642str(pObj->id));
+    tjsonAddStringToObject(item, "createdTime", i642str(pObj->createdTime));
+    tjsonAddStringToObject(item, "updateTime", i642str(pObj->updateTime));
+    tjsonAddStringToObject(item, "name", pObj->name);
+
+    sdbRelease(pSdb, pObj);
+  }
+}
+
+void dumpTrans(SSdb *pSdb, SJson *json) {
+  void  *pIter = NULL;
+  SJson *items = tjsonCreateObject();
+  tjsonAddItemToObject(json, "transactions", items);
+
+  while (1) {
+    STrans *pObj = NULL;
+    pIter = sdbFetch(pSdb, SDB_TRANS, pIter, (void **)&pObj);
+    if (pIter == NULL) break;
+
+    SJson *item = tjsonCreateObject();
+    tjsonAddItemToObject(items, "trans", item);
+
+    tjsonAddIntegerToObject(item, "id", pObj->id);
+    tjsonAddIntegerToObject(item, "stage", pObj->stage);
+    tjsonAddIntegerToObject(item, "policy", pObj->policy);
+    tjsonAddIntegerToObject(item, "conflict", pObj->conflict);
+    tjsonAddIntegerToObject(item, "exec", pObj->exec);
+    tjsonAddStringToObject(item, "createdTime", i642str(pObj->createdTime));
+    tjsonAddStringToObject(item, "dbname1", pObj->dbname1);
+    tjsonAddStringToObject(item, "dbname2", pObj->dbname2);
+    tjsonAddIntegerToObject(item, "commitLogNum", taosArrayGetSize(pObj->commitActions));
+    tjsonAddIntegerToObject(item, "redoActionNum", taosArrayGetSize(pObj->redoActions));
+    tjsonAddIntegerToObject(item, "undoActionNum", taosArrayGetSize(pObj->undoActions));
+
+    sdbRelease(pSdb, pObj);
+  }
+}
+
+void dumpHeader(SSdb *pSdb, SJson *json) {
+  tjsonAddIntegerToObject(json, "sver", 1);
+  tjsonAddStringToObject(json, "applyIndex", i642str(pSdb->applyIndex));
+  tjsonAddStringToObject(json, "applyTerm", i642str(pSdb->applyTerm));
+  tjsonAddStringToObject(json, "applyConfig", i642str(pSdb->applyConfig));
+
+  SJson *maxIdsJson = tjsonCreateObject();
+  tjsonAddItemToObject(json, "maxIds", maxIdsJson);
+  for (int32_t i = 0; i < SDB_MAX; ++i) {
+    int64_t maxId = 0;
+    if (i < SDB_MAX) {
+      maxId = pSdb->maxId[i];
+    }
+    tjsonAddStringToObject(maxIdsJson, sdbTableName(i), i642str(maxId));
+  }
+
+  SJson *tableVersJson = tjsonCreateObject();
+  tjsonAddItemToObject(json, "tableVers", tableVersJson);
+  for (int32_t i = 0; i < SDB_MAX; ++i) {
+    int64_t tableVer = 0;
+    if (i < SDB_MAX) {
+      tableVer = pSdb->tableVer[i];
+    }
+    tjsonAddStringToObject(tableVersJson, sdbTableName(i), i642str(tableVer));
+  }
+}
+
+int32_t dumpSdb() {
+  wDebugFlag = 0;
+  mDebugFlag = 0;
+  sDebugFlag = 0;
+
+  SMsgCb msgCb = {0};
+  msgCb.reportStartupFp = reportStartup;
+  msgCb.sendReqFp = sendReq;
+  msgCb.sendRspFp = sendRsp;
+  msgCb.mgmt = (SMgmtWrapper *)(&msgCb);  // hack
+  tmsgSetDefault(&msgCb);
+  walInit();
+  syncInit();
+
+  SMnodeOpt opt = {.msgCb = msgCb};
+  SMnode   *pMnode = mndOpen(TMP_MNODE_DIR, &opt);
+  if (pMnode == NULL) return -1;
+
+  SSdb  *pSdb = pMnode->pSdb;
+  SJson *json = tjsonCreateObject();
+  dumpHeader(pSdb, json);
+  dumpFunc(pSdb, json);
+  dumpDb(pSdb, json);
+  dumpStb(pSdb, json);
+  dumpSma(pSdb, json);
+  dumpVgroup(pSdb, json);
+  dumpTopic(pSdb, json);
+  dumpConsumber(pSdb, json);
+  dumpSubscribe(pSdb, json);
+  dumpOffset(pSdb, json);
+  dumpStream(pSdb, json);
+  dumpAcct(pSdb, json);
+  dumpAuth(pSdb, json);
+  dumpUser(pSdb, json);
+  dumpDnode(pSdb, json);
+  dumpBnode(pSdb, json);
+  dumpSnode(pSdb, json);
+  dumpQnode(pSdb, json);
+  dumpMnode(pSdb, json);
+  dumpCluster(pSdb, json);
+  dumpTrans(pSdb, json);
+
+  char     *pCont = tjsonToString(json);
+  int32_t   contLen = strlen(pCont);
+  char      file[] = "sdb.json";
+  TdFilePtr pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
+  if (pFile == NULL) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    dError("failed to write %s since %s", file, terrstr());
+    return -1;
+  }
+  taosWriteFile(pFile, pCont, contLen);
+  taosWriteFile(pFile, "\n", 1);
+  taosFsyncFile(pFile);
+  taosCloseFile(&pFile);
+  tjsonDelete(json);
+  taosMemoryFree(pCont);
+  taosRemoveDir(TMP_DNODE_DIR);
+  return 0;
+}
+
+int32_t parseArgs(int32_t argc, char *argv[]) {
+  for (int32_t i = 1; i < argc; ++i) {
+    if (strcmp(argv[i], "-c") == 0) {
+      if (i < argc - 1) {
+        if (strlen(argv[++i]) >= PATH_MAX) {
+          printf("config file path overflow");
+          return -1;
+        }
+        tstrncpy(configDir, argv[i], PATH_MAX);
+      } else {
+        printf("'-c' requires a parameter, default is %s\n", configDir);
+        return -1;
+      }
+    } else {
+      printf("-c Configuration directory. \n");
+      return -1;
+    }
+  }
+
+  if (taosCreateLog("dumplog", 1, configDir, NULL, NULL, NULL, NULL, 1) != 0) {
+    printf("failed to dump since init log error\n");
+    return -1;
+  }
+
+  if (taosInitCfg(configDir, NULL, NULL, NULL, NULL, 0) != 0) {
+    printf("failed to dump since read config error\n");
+    return -1;
+  }
+
+  char mnodeJson[PATH_MAX] = {0};
+  char dataFile[PATH_MAX] = {0};
+  char raftCfgFile[PATH_MAX] = {0};
+  char raftStoreFile[PATH_MAX] = {0};
+  snprintf(mnodeJson, PATH_MAX, "%s" TD_DIRSEP "mnode" TD_DIRSEP "mnode.json", tsDataDir);
+  snprintf(dataFile, PATH_MAX, "%s" TD_DIRSEP "mnode" TD_DIRSEP "data" TD_DIRSEP "sdb.data", tsDataDir);
+  snprintf(raftCfgFile, PATH_MAX, "%s" TD_DIRSEP "mnode" TD_DIRSEP "sync" TD_DIRSEP "raft_config.json", tsDataDir);
+  snprintf(raftStoreFile, PATH_MAX, "%s" TD_DIRSEP "mnode" TD_DIRSEP "sync" TD_DIRSEP "raft_store.json", tsDataDir);
+
+  char cmd[PATH_MAX * 2] = {0};
+  snprintf(cmd, sizeof(cmd), "rm -rf %s", TMP_DNODE_DIR);
+  
+  system(cmd);
+#ifdef WINDOWS
+  taosMulMkDir(TMP_SDB_DATA_DIR);
+  taosMulMkDir(TMP_SDB_SYNC_DIR);
+  snprintf(cmd, sizeof(cmd), "cp %s %s 2>nul", mnodeJson, TMP_SDB_MNODE_JSON);
+  system(cmd);
+  snprintf(cmd, sizeof(cmd), "cp %s %s 2>nul", dataFile, TMP_SDB_DATA_FILE);
+  system(cmd);
+  snprintf(cmd, sizeof(cmd), "cp %s %s 2>nul", raftCfgFile, TMP_SDB_RAFT_CFG_FILE);
+  system(cmd);
+  snprintf(cmd, sizeof(cmd), "cp %s %s 2>nul", raftStoreFile, TMP_SDB_RAFT_STORE_FILE);
+  system(cmd);
+#else
+  snprintf(cmd, sizeof(cmd), "mkdir -p %s", TMP_SDB_DATA_DIR);
+  system(cmd);
+  snprintf(cmd, sizeof(cmd), "mkdir -p %s", TMP_SDB_SYNC_DIR);
+  system(cmd);
+  snprintf(cmd, sizeof(cmd), "cp %s %s 2>/dev/null", mnodeJson, TMP_SDB_MNODE_JSON);
+  system(cmd);
+  snprintf(cmd, sizeof(cmd), "cp %s %s 2>/dev/null", dataFile, TMP_SDB_DATA_FILE);
+  system(cmd);
+  snprintf(cmd, sizeof(cmd), "cp %s %s 2>/dev/null", raftCfgFile, TMP_SDB_RAFT_CFG_FILE);
+  system(cmd);
+  snprintf(cmd, sizeof(cmd), "cp %s %s 2>/dev/null", raftStoreFile, TMP_SDB_RAFT_STORE_FILE);
+  system(cmd);
+#endif
+
+  strcpy(tsDataDir, TMP_DNODE_DIR);
+  return 0;
+}
+
+int32_t main(int32_t argc, char *argv[]) {
+  if (parseArgs(argc, argv) != 0) {
+    return -1;
+  }
+
+  return dumpSdb();
+}
+
+#pragma GCC diagnostic pop
\ No newline at end of file

utils/tsim/CMakeLists.txt

+aux_source_directory(src TSIM_SRC)
+add_executable(tsim ${TSIM_SRC})
+target_link_libraries(
+    tsim
+    PUBLIC taos_static
+    PUBLIC util
+    PUBLIC common
+    PUBLIC os
+    PUBLIC cjson
+)
+target_include_directories(
+    tsim
+    PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
+)

utils/tsim/inc/simInt.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_SIM_INT_H_
+#define _TD_SIM_INT_H_
+
+#include "os.h"
+
+#include "cJSON.h"
+#include "tconfig.h"
+#include "taos.h"
+#include "taoserror.h"
+#include "tidpool.h"
+#include "tlog.h"
+#include "ttimer.h"
+#include "ttypes.h"
+#include "tutil.h"
+#include "tglobal.h"
+
+#define MAX_MAIN_SCRIPT_NUM       10
+#define MAX_BACKGROUND_SCRIPT_NUM 10
+#define MAX_FILE_NAME_LEN         256
+#define MAX_ERROR_LEN             1024
+#define MAX_QUERY_VALUE_LEN       1024
+#define MAX_QUERY_COL_NUM         100
+#define MAX_QUERY_ROW_NUM         100
+#define MAX_SYSTEM_RESULT_LEN     2048
+#define MAX_VAR_LEN               100
+#define MAX_VAR_NAME_LEN          32
+#define MAX_VAR_VAL_LEN           80
+#define MAX_OPT_NAME_LEN          32
+#define MAX_SIM_CMD_NAME_LEN      40
+
+#ifdef LINUX
+#define SUCCESS_PREFIX  "\033[44;32;1m"
+#define SUCCESS_POSTFIX "\033[0m"
+#define FAILED_PREFIX   "\033[44;31;1m"
+#define FAILED_POSTFIX  "\033[0m"
+#else
+#define SUCCESS_PREFIX  ""
+#define SUCCESS_POSTFIX ""
+#define FAILED_PREFIX   ""
+#define FAILED_POSTFIX  ""
+#endif
+
+#define simFatal(...) { if (simDebugFlag & DEBUG_FATAL) { taosPrintLog("SIM FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }}
+#define simError(...) { if (simDebugFlag & DEBUG_ERROR) { taosPrintLog("SIM ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }}
+#define simWarn(...)  { if (simDebugFlag & DEBUG_WARN)  { taosPrintLog("SIM WARN ", DEBUG_WARN, 255, __VA_ARGS__); }}
+#define simInfo(...)  { if (simDebugFlag & DEBUG_INFO)  { taosPrintLog("SIM ", DEBUG_INFO, 255, __VA_ARGS__); }}
+#define simDebug(...) { if (simDebugFlag & DEBUG_DEBUG) { taosPrintLog("SIM ", DEBUG_DEBUG, simDebugFlag, __VA_ARGS__); }}
+#define simTrace(...) { if (simDebugFlag & DEBUG_TRACE) { taosPrintLog("SIM ", DEBUG_TRACE, simDebugFlag, __VA_ARGS__); }}
+
+enum { SIM_SCRIPT_TYPE_MAIN, SIM_SCRIPT_TYPE_BACKGROUND };
+
+enum {
+  SIM_CMD_EXP,
+  SIM_CMD_IF,
+  SIM_CMD_ELIF,
+  SIM_CMD_ELSE,
+  SIM_CMD_ENDI,
+  SIM_CMD_WHILE,
+  SIM_CMD_ENDW,
+  SIM_CMD_SWITCH,
+  SIM_CMD_CASE,
+  SIM_CMD_DEFAULT,
+  SIM_CMD_CONTINUE,
+  SIM_CMD_BREAK,
+  SIM_CMD_ENDS,
+  SIM_CMD_SLEEP,
+  SIM_CMD_GOTO,
+  SIM_CMD_RUN,
+  SIM_CMD_RUN_BACK,
+  SIM_CMD_PRINT,
+  SIM_CMD_SYSTEM,
+  SIM_CMD_SYSTEM_CONTENT,
+  SIM_CMD_SQL,
+  SIM_CMD_SQL_ERROR,
+  SIM_CMD_SQL_SLOW,
+  SIM_CMD_RESTFUL,
+  SIM_CMD_TEST,
+  SIM_CMD_RETURN,
+  SIM_CMD_LINE_INSERT,
+  SIM_CMD_LINE_INSERT_ERROR,
+  SIM_CMD_END
+};
+
+enum {
+  SQL_JUMP_FALSE,
+  SQL_JUMP_TRUE,
+};
+
+struct _script_t;
+typedef struct _cmd_t {
+  int16_t cmdno;
+  int16_t nlen;
+  char    name[MAX_SIM_CMD_NAME_LEN];
+  bool (*parseCmd)(char *, struct _cmd_t *, int32_t);
+  bool (*executeCmd)(struct _script_t *script, char *option);
+  struct _cmd_t *next;
+} SCommand;
+
+typedef struct {
+  int16_t cmdno;
+  int16_t jump;          // jump position
+  int16_t errorJump;     // sql jump flag, while '-x' exist in sql cmd, this flag
+                         // will be SQL_JUMP_TRUE, otherwise is SQL_JUMP_FALSE */
+  int16_t lineNum;       // correspodning line number in original file
+  int32_t optionOffset;  // relative option offset
+} SCmdLine;
+
+typedef struct _var_t {
+  char varName[MAX_VAR_NAME_LEN];
+  char varValue[MAX_VAR_VAL_LEN];
+  char varNameLen;
+} SVariable;
+
+typedef struct _script_t {
+  int32_t           type;
+  bool              killed;
+  void             *taos;
+  char              rows[12];                                                         // number of rows data retrieved
+  char              data[MAX_QUERY_ROW_NUM][MAX_QUERY_COL_NUM][MAX_QUERY_VALUE_LEN];  // query results
+  char              system_exit_code[12];
+  char              system_ret_content[MAX_SYSTEM_RESULT_LEN];
+  int32_t           varLen;
+  int32_t           linePos;     // current cmd position
+  int32_t           numOfLines;  // number of lines in the script
+  int32_t           bgScriptLen;
+  char              fileName[MAX_FILE_NAME_LEN];  // script file name
+  char              error[MAX_ERROR_LEN];
+  char             *optionBuffer;
+  SCmdLine         *lines;  // command list
+  SVariable         variables[MAX_VAR_LEN];
+  TdThread         bgPid;
+  char              auth[128];
+  struct _script_t *bgScripts[MAX_BACKGROUND_SCRIPT_NUM];
+} SScript;
+
+extern SScript *simScriptList[MAX_MAIN_SCRIPT_NUM];
+extern SCommand simCmdList[];
+extern int32_t  simScriptPos;
+extern int32_t  simScriptSucced;
+extern int32_t  simDebugFlag;
+extern char     simScriptDir[];
+extern bool     abortExecution;
+extern bool     useMultiProcess;
+extern bool     useValgrind;
+
+SScript *simParseScript(char *fileName);
+SScript *simProcessCallOver(SScript *script);
+void    *simExecuteScript(void *script);
+void     simInitsimCmdList();
+bool     simSystemInit();
+void     simSystemCleanUp();
+char    *simGetVariable(SScript *script, char *varName, int32_t varLen);
+bool     simExecuteExpCmd(SScript *script, char *option);
+bool     simExecuteTestCmd(SScript *script, char *option);
+bool     simExecuteGotoCmd(SScript *script, char *option);
+bool     simExecuteRunCmd(SScript *script, char *option);
+bool     simExecuteRunBackCmd(SScript *script, char *option);
+bool     simExecuteSystemCmd(SScript *script, char *option);
+bool     simExecuteSystemContentCmd(SScript *script, char *option);
+bool     simExecutePrintCmd(SScript *script, char *option);
+bool     simExecuteSleepCmd(SScript *script, char *option);
+bool     simExecuteReturnCmd(SScript *script, char *option);
+bool     simExecuteSqlCmd(SScript *script, char *option);
+bool     simExecuteSqlErrorCmd(SScript *script, char *rest);
+bool     simExecuteSqlSlowCmd(SScript *script, char *option);
+bool     simExecuteRestfulCmd(SScript *script, char *rest);
+bool     simExecuteLineInsertCmd(SScript *script, char *option);
+bool     simExecuteLineInsertErrorCmd(SScript *script, char *option);
+void     simVisuallizeOption(SScript *script, char *src, char *dst);
+
+#endif /*_TD_SIM_INT_H_*/
\ No newline at end of file

utils/tsim/inc/simParse.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_SIM_PARSE_H_
+#define _TD_SIM_PARSE_H_
+
+#include "simInt.h"
+
+#define MAX_NUM_CMD       64
+#define MAX_NUM_LABLES    100
+#define MAX_LABEL_LEN     40
+#define MAX_NUM_BLOCK     100
+#define MAX_NUM_JUMP      100
+#define MAX_LINE_LEN      3000
+#define MAX_CMD_LINES     2048
+#define MAX_OPTION_BUFFER 64000
+
+enum {
+  BLOCK_IF,
+  BLOCK_WHILE,
+  BLOCK_SWITCH,
+};
+
+/* label stack */
+typedef struct {
+  int8_t  top;                                  /* number of labels */
+  int16_t pos[MAX_NUM_LABLES];                  /* the position of the label */
+  char    label[MAX_NUM_LABLES][MAX_LABEL_LEN]; /* name of the label */
+} SLabel;
+
+/* block definition */
+typedef struct {
+  int8_t   top;                 /* the number of blocks stacked */
+  char     type[MAX_NUM_BLOCK]; /* the block type */
+  int16_t *pos[MAX_NUM_BLOCK];  /* position of the jump for if/elif/case */
+  int16_t  back[MAX_NUM_BLOCK]; /* go back, endw and continue */
+  char     numJump[MAX_NUM_BLOCK];
+  int16_t *jump[MAX_NUM_BLOCK][MAX_NUM_JUMP]; /* break or elif */
+  char     sexp[MAX_NUM_BLOCK][40];           /*switch expression */
+  char     sexpLen[MAX_NUM_BLOCK];            /*switch expression length */
+} SBlock;
+
+bool simParseExpression(char *token, int32_t lineNum);
+
+#endif /*_TD_SIM_PARSE_H_*/
\ No newline at end of file

utils/tsim/src/simMain.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "simInt.h"
+
+bool simExecSuccess = false;
+bool abortExecution = false;
+bool useMultiProcess = false;
+bool useValgrind = false;
+
+void simHandleSignal(int32_t signo, void *sigInfo, void *context) {
+  simSystemCleanUp();
+  abortExecution = true;
+}
+
+int32_t main(int32_t argc, char *argv[]) {
+  char scriptFile[MAX_FILE_NAME_LEN] = "sim_main_test.sim";
+
+  for (int32_t i = 1; i < argc; ++i) {
+    if (strcmp(argv[i], "-c") == 0 && i < argc - 1) {
+      tstrncpy(configDir, argv[++i], 128);
+    } else if (strcmp(argv[i], "-f") == 0 && i < argc - 1) {
+      strcpy(scriptFile, argv[++i]);
+    } else if (strcmp(argv[i], "-m") == 0) {
+      useMultiProcess = true;
+    } else if (strcmp(argv[i], "-v") == 0) {
+      useValgrind = true;
+    } else {
+      printf("usage: %s [options] \n", argv[0]);
+      printf("       [-c config]: config directory, default is: %s\n", configDir);
+      printf("       [-f script]: script filename\n");
+      return 0;
+    }
+  }
+
+  if (!simSystemInit()) {
+    simError("failed to initialize the system");
+    simSystemCleanUp();
+    return -1;
+  }
+
+  simInfo("simulator is running ...");
+  taosSetSignal(SIGINT, simHandleSignal);
+
+  SScript *script = simParseScript(scriptFile);
+  if (script == NULL) {
+    simError("parse script file:%s failed", scriptFile);
+    return -1;
+  }
+
+  if (abortExecution) {
+    simError("execute abort");
+    return -1;
+  }
+
+  simScriptList[++simScriptPos] = script;
+  simExecuteScript(script);
+
+  int32_t ret = simExecSuccess ? 0 : -1;
+  simInfo("execute result %d", ret);
+
+  return ret;
+}

utils/tsim/src/simSystem.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "simInt.h"
+#include "tconfig.h"
+
+SScript *simScriptList[MAX_MAIN_SCRIPT_NUM];
+SCommand simCmdList[SIM_CMD_END];
+int32_t  simScriptPos = -1;
+int32_t  simScriptSucced = 0;
+int32_t  simDebugFlag = 143;
+void     simCloseTaosdConnect(SScript *script);
+char     simScriptDir[PATH_MAX] = {0};
+
+extern bool simExecSuccess;
+
+int32_t simInitCfg() {
+  taosCreateLog("simlog", 1, configDir, NULL, NULL, NULL, NULL, 1);
+  taosInitCfg(configDir, NULL, NULL, NULL, NULL, 1);
+
+  SConfig *pCfg = taosGetCfg();
+  simDebugFlag = cfgGetItem(pCfg, "simDebugFlag")->i32;
+  tstrncpy(simScriptDir, cfgGetItem(pCfg, "scriptDir")->str, PATH_MAX);
+  return 0;
+}
+
+bool simSystemInit() {
+  simInitCfg();
+  simInitsimCmdList();
+  memset(simScriptList, 0, sizeof(SScript *) * MAX_MAIN_SCRIPT_NUM);
+  return true;
+}
+
+void simSystemCleanUp() {}
+
+void simFreeScript(SScript *script) {
+  if (script->type == SIM_SCRIPT_TYPE_MAIN) {
+    simInfo("script:%s, background script num:%d, stop them", script->fileName, script->bgScriptLen);
+
+    for (int32_t i = 0; i < script->bgScriptLen; ++i) {
+      SScript *bgScript = script->bgScripts[i];
+      simDebug("script:%s, is background script, set stop flag", bgScript->fileName);
+      bgScript->killed = true;
+      if (taosCheckPthreadValid(bgScript->bgPid)) {
+        taosThreadJoin(bgScript->bgPid, NULL);
+        taosThreadClear(&bgScript->bgPid);
+      }
+
+      simDebug("script:%s, background thread joined", bgScript->fileName);
+      taos_close(bgScript->taos);
+      taosMemoryFreeClear(bgScript->lines);
+      taosMemoryFreeClear(bgScript->optionBuffer);
+      taosMemoryFreeClear(bgScript);
+    }
+
+    simDebug("script:%s, is cleaned", script->fileName);
+    taos_close(script->taos);
+    taosMemoryFreeClear(script->lines);
+    taosMemoryFreeClear(script->optionBuffer);
+    taosMemoryFreeClear(script);
+  }
+}
+
+SScript *simProcessCallOver(SScript *script) {
+  if (script->type == SIM_SCRIPT_TYPE_MAIN) {
+    simDebug("script:%s, is main script, set stop flag", script->fileName);
+    if (script->killed) {
+      simExecSuccess = false;
+      simInfo("script:" FAILED_PREFIX "%s" FAILED_POSTFIX ", " FAILED_PREFIX "failed" FAILED_POSTFIX ", error:%s",
+              script->fileName, script->error);
+    } else {
+      simExecSuccess = true;
+      simInfo("script:" SUCCESS_PREFIX "%s" SUCCESS_POSTFIX ", " SUCCESS_PREFIX "success" SUCCESS_POSTFIX,
+              script->fileName);
+    }
+
+    simCloseTaosdConnect(script);
+    simScriptSucced++;
+    simScriptPos--;
+    simFreeScript(script);
+
+    if (simScriptPos == -1 && simExecSuccess) {
+      simInfo("----------------------------------------------------------------------");
+      simInfo("Simulation Test Done, " SUCCESS_PREFIX "%d" SUCCESS_POSTFIX " Passed:\n", simScriptSucced);
+      return NULL;
+    }
+
+    if (simScriptPos == -1) return NULL;
+    if (!simExecSuccess) return NULL;
+
+    return simScriptList[simScriptPos];
+  } else {
+    simDebug("script:%s,  is stopped", script->fileName);
+    simFreeScript(script);
+    return NULL;
+  }
+}
+
+void *simExecuteScript(void *inputScript) {
+  SScript *script = (SScript *)inputScript;
+
+  while (1) {
+    if (script->type == SIM_SCRIPT_TYPE_MAIN) {
+      script = simScriptList[simScriptPos];
+    }
+
+    if (abortExecution) {
+      script->killed = true;
+    }
+
+    if (script->killed || script->linePos >= script->numOfLines) {
+      script = simProcessCallOver(script);
+      if (script == NULL) {
+        simDebug("sim test abort now!");
+        break;
+      }
+    } else {
+      SCmdLine *line = &script->lines[script->linePos];
+      char     *option = script->optionBuffer + line->optionOffset;
+      simDebug("script:%s, line:%d with option \"%s\"", script->fileName, line->lineNum, option);
+
+      SCommand *cmd = &simCmdList[line->cmdno];
+      int32_t   ret = (*(cmd->executeCmd))(script, option);
+      if (!ret) {
+        script->killed = true;
+      }
+    }
+  }
+
+  simInfo("thread is stopped");
+  return NULL;
+}