Merge pull request #2655 from taosdata/feature/vnode

Feature/vnode

packaging/cfg/taos.cfg

@@ -51,18 +51,15 @@
 # number of threads per CPU core
 # numOfThreadsPerCore   1.0
 
-# number of vnodes per core in DNode
-# numOfVnodesPerCore    8
+# number of vgroups per db
+# maxVgroupsPerDb       0
+
+# max number of tables per vnode
+# maxTablesPerVnode     1000000
 
 # the ratio of threads responsible for querying in the total thread
 # ratioOfQueryThreads   0.5
 
-# number of total vnodes in DNode
-# numOfTotalVnodes      0
-
-# max number of tables per vnode
-# maxtablesPerVnode     1000
-
 # interval of check load balance when the management node is in normal operation
 # balanceInterval       300
 

src/common/inc/tglobal.h

@@ -68,7 +68,9 @@ extern int64_t tsMaxRetentWindow;
 // db parameters in client
 extern int32_t tsCacheBlockSize;
 extern int32_t tsBlocksPerVnode;
+extern int32_t tsMinTablePerVnode;
 extern int32_t tsMaxTablePerVnode;
+extern int32_t tsTableIncStepPerVnode;
 extern int32_t tsMaxVgroupsPerDb;
 extern int16_t tsDaysPerFile;
 extern int32_t tsDaysToKeep;

src/common/src/tglobal.c

@@ -38,7 +38,7 @@ uint16_t tsDnodeShellPort = 6030;  // udp[6035-6039] tcp[6035]
 uint16_t tsDnodeDnodePort = 6035;  // udp/tcp
 uint16_t tsSyncPort = 6040;
 int32_t  tsStatusInterval = 1;  // second
-int16_t  tsNumOfVnodesPerCore = 8;
+int16_t  tsNumOfVnodesPerCore = 32;
 int16_t  tsNumOfTotalVnodes = TSDB_INVALID_VNODE_NUM;
 int32_t  tsNumOfMnodes = 3;
 int32_t  tsEnableVnodeBak = 1;
@@ -112,7 +112,10 @@ int16_t tsCompression   = TSDB_DEFAULT_COMP_LEVEL;
 int16_t tsWAL           = TSDB_DEFAULT_WAL_LEVEL;
 int32_t tsReplications  = TSDB_DEFAULT_DB_REPLICA_OPTION;
 int32_t tsMaxVgroupsPerDb  = 0;
+int32_t tsMinTablePerVnode = 100;
 int32_t tsMaxTablePerVnode = TSDB_DEFAULT_TABLES;
+int32_t tsTableIncStepPerVnode = TSDB_TABLES_STEP;
+
 // balance
 int32_t tsEnableBalance = 1;
 int32_t tsAlternativeRole = 0;
@@ -389,16 +392,6 @@ static void doInitGlobalConfig() {
   cfg.unitType = TAOS_CFG_UTYPE_NONE;
   taosInitConfigOption(cfg);
 
-  cfg.option = "numOfVnodesPerCore";
-  cfg.ptr = &tsNumOfVnodesPerCore;
-  cfg.valType = TAOS_CFG_VTYPE_INT16;
-  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
-  cfg.minValue = 1;
-  cfg.maxValue = 64;
-  cfg.ptrLength = 0;
-  cfg.unitType = TAOS_CFG_UTYPE_NONE;
-  taosInitConfigOption(cfg);
-
   cfg.option = "numOfTotalVnodes";
   cfg.ptr = &tsNumOfTotalVnodes;
   cfg.valType = TAOS_CFG_VTYPE_INT16;
@@ -622,6 +615,26 @@ static void doInitGlobalConfig() {
   cfg.unitType = TAOS_CFG_UTYPE_NONE;
   taosInitConfigOption(cfg);
 
+  cfg.option = "minTablesPerVnode";
+  cfg.ptr = &tsMinTablePerVnode;
+  cfg.valType = TAOS_CFG_VTYPE_INT32;
+  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
+  cfg.minValue = TSDB_MIN_TABLES;
+  cfg.maxValue = TSDB_MAX_TABLES;
+  cfg.ptrLength = 0;
+  cfg.unitType = TAOS_CFG_UTYPE_NONE;
+  taosInitConfigOption(cfg);
+
+  cfg.option = "tableIncStepPerVnode";
+  cfg.ptr = &tsTableIncStepPerVnode;
+  cfg.valType = TAOS_CFG_VTYPE_INT32;
+  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
+  cfg.minValue = TSDB_MIN_TABLES;
+  cfg.maxValue = TSDB_MAX_TABLES;
+  cfg.ptrLength = 0;
+  cfg.unitType = TAOS_CFG_UTYPE_NONE;
+  taosInitConfigOption(cfg);
+
   cfg.option = "cache";
   cfg.ptr = &tsCacheBlockSize;
   cfg.valType = TAOS_CFG_VTYPE_INT32;

src/inc/taosdef.h

@@ -274,8 +274,8 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size);
 #define TSDB_DEFAULT_PAYLOAD_SIZE 5120   // default payload size, greater than PATH_MAX value
 #define TSDB_EXTRA_PAYLOAD_SIZE   128    // extra bytes for auth
 #define TSDB_CQ_SQL_SIZE          1024
-#define TSDB_MAX_VNODES           256
-#define TSDB_MIN_VNODES           50
+#define TSDB_MAX_VNODES           2048
+#define TSDB_MIN_VNODES           256
 #define TSDB_INVALID_VNODE_NUM    0
 
 #define TSDB_DNODE_ROLE_ANY       0
@@ -296,9 +296,9 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size);
 #define TSDB_DEFAULT_TOTAL_BLOCKS       4
 
 #define TSDB_MIN_TABLES                 4
-#define TSDB_MAX_TABLES                 5000000
-#define TSDB_DEFAULT_TABLES             200000
-#define TSDB_TABLES_STEP                10000
+#define TSDB_MAX_TABLES                 10000000
+#define TSDB_DEFAULT_TABLES             1000000
+#define TSDB_TABLES_STEP                1000
 
 #define TSDB_MIN_DAYS_PER_FILE          1
 #define TSDB_MAX_DAYS_PER_FILE          3650 

src/mnode/src/mnodeVgroup.c

@@ -323,7 +323,7 @@ static int32_t mnodeAllocVgroupIdPool(SVgObj *pInputVgroup) {
   if (pDb == NULL) return TSDB_CODE_MND_APP_ERROR;
 
   int32_t minIdPoolSize = TSDB_MAX_TABLES;
-  int32_t maxIdPoolSize = TSDB_MIN_TABLES;
+  int32_t maxIdPoolSize = tsMinTablePerVnode;
   for (int32_t v = 0; v < pDb->numOfVgroups; ++v) {
     SVgObj *pVgroup = pDb->vgList[v];
     if (pVgroup == NULL) continue;
@@ -347,10 +347,10 @@ static int32_t mnodeAllocVgroupIdPool(SVgObj *pInputVgroup) {
 
   // realloc all vgroups in db
   int32_t newIdPoolSize;
-  if (minIdPoolSize * 4 < TSDB_TABLES_STEP) {
+  if (minIdPoolSize * 4 < tsTableIncStepPerVnode) {
     newIdPoolSize = minIdPoolSize * 4;
   } else {
-    newIdPoolSize = ((minIdPoolSize / TSDB_TABLES_STEP) + 1) * TSDB_TABLES_STEP;
+    newIdPoolSize = ((minIdPoolSize / tsTableIncStepPerVnode) + 1) * tsTableIncStepPerVnode;
   }
 
   if (newIdPoolSize > tsMaxTablePerVnode) {
@@ -471,6 +471,8 @@ static int32_t mnodeCreateVgroupCb(SMnodeMsg *pMsg, int32_t code) {
   }
 
   pMsg->expected = pVgroup->numOfVnodes;
+  pMsg->successed = 0;
+  pMsg->received = 0;
   mnodeSendCreateVgroupMsg(pVgroup, pMsg);
 
   return TSDB_CODE_MND_ACTION_IN_PROGRESS;
@@ -836,6 +838,8 @@ static void mnodeProcessCreateVnodeRsp(SRpcMsg *rpcMsg) {
          pVgroup->vgId, tstrerror(rpcMsg->code), mnodeMsg->received, mnodeMsg->successed, mnodeMsg->expected,
          mnodeMsg->rpcMsg.handle, rpcMsg->ahandle);
 
+  assert(mnodeMsg->received <= mnodeMsg->expected);
+
   if (mnodeMsg->received != mnodeMsg->expected) return;
 
   if (mnodeMsg->received == mnodeMsg->successed) {

src/vnode/src/vnodeMain.c

@@ -59,7 +59,7 @@ int32_t vnodeInitResources() {
   vnodeInitWriteFp();
   vnodeInitReadFp();
 
-  tsDnodeVnodesHash = taosHashInit(TSDB_MAX_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true);
+  tsDnodeVnodesHash = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true);
   if (tsDnodeVnodesHash == NULL) {
     vError("failed to init vnode list");
     return TSDB_CODE_VND_OUT_OF_MEMORY;

tests/CMakeLists.txt

@@ -12,3 +12,4 @@ SET(CMAKE_VERBOSE_MAKEFILE ON)
 ADD_SUBDIRECTORY(examples/c)
 ADD_SUBDIRECTORY(tsim)
 ADD_SUBDIRECTORY(test/c)
+ADD_SUBDIRECTORY(comparisonTest/tdengine)

tests/comparisonTest/tdengine/CMakeLists.txt

+CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
+PROJECT(TDengine)
+
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
+INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
+
+IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
+  add_executable(tdengineTest tdengineTest.c)
+  target_link_libraries(tdengineTest taos_static tutil common pthread)
+ENDIF()

tests/comparisonTest/tdengine/tdengineTest.c

@@ -7,6 +7,7 @@
 #include <time.h>
 #include <pthread.h>
 #include <sys/time.h>
+#include <inttypes.h>
 
 typedef struct {
   char sql[256];
@@ -123,19 +124,21 @@ void writeDataImp(void *param) {
   if (taos == NULL)
     taos_error(taos);
 
-  int code = taos_query(taos, "use db");
+  TAOS_RES* result = taos_query(taos, "use db");
+  int32_t code = taos_errno(result);
   if (code != 0) {
     taos_error(taos);
   }
+  taos_free_result(result);
 
-  char sql[65000];
+  char *sql = calloc(1, 8*1024*1024);
   int sqlLen = 0;
   int lastMachineid = 0;
   int counter = 0;
   int totalRecords = 0;
 
   for (int j = pThread->sID; j <= pThread->eID; j++) {
-    char fileName[256];
+    char fileName[300];
     sprintf(fileName, "%s/testdata%d.csv", arguments.dataDir, j);
 
     FILE *fp = fopen(fileName, "r");
@@ -162,7 +165,7 @@ void writeDataImp(void *param) {
       int64_t timestamp;
       int temperature;
       float humidity;
-      sscanf(line, "%d%s%d%lld%d%f", &machineid, machinename, &machinegroup, &timestamp, &temperature, &humidity);
+      sscanf(line, "%d%s%d%" PRId64 "%d%f", &machineid, machinename, &machinegroup, &timestamp, &temperature, &humidity);
 
       if (counter == 0) {
         sqlLen = sprintf(sql, "insert into");
@@ -174,14 +177,16 @@ void writeDataImp(void *param) {
                           machineid, machineid, machinename, machinegroup);
       }
 
-      sqlLen += sprintf(sql + sqlLen, "(%lld,%d,%f)", timestamp, temperature, humidity);
+      sqlLen += sprintf(sql + sqlLen, "(%" PRId64 ",%d,%f)", timestamp, temperature, humidity);
       counter++;
 
       if (counter >= arguments.rowsPerRequest) {
-        int code = taos_query(taos, sql);
+        TAOS_RES *result = taos_query(taos, sql);
+        int32_t   code = taos_errno(result);
         if (code != 0) {
-          printf("thread:%d error:%d reason:%s\n", pThread->pid, code, taos_errstr(taos));
+          printf("thread:%d error:%d reason:%s\n", pThread->threadId, code, taos_errstr(taos));
         }
+        taos_free_result(result);
 
         totalRecords += counter;
         counter = 0;
@@ -194,15 +199,18 @@ void writeDataImp(void *param) {
   }
 
   if (counter > 0) {
-    int code = taos_query(taos, sql);
+    TAOS_RES *result = taos_query(taos, sql);
+    int32_t   code = taos_errno(result);
     if (code != 0) {
-      printf("thread:%d error:%d reason:%s\n", pThread->pid, code, taos_errstr(taos));
+      printf("thread:%d error:%d reason:%s\n", pThread->threadId, code, taos_errstr(taos));
     }
+    taos_free_result(result);
 
     totalRecords += counter;
   }
 
   __sync_fetch_and_add(&statis.totalRows, totalRecords);
+  free(sql);
 }
 
 void writeData() {
@@ -215,19 +223,23 @@ void writeData() {
   taos_init();
 
   void *taos = taos_connect("127.0.0.1", "root", "taosdata", NULL, 0);
-  if (taos == NULL)
-    taos_error(taos);
+  if (taos == NULL) taos_error(taos);
 
-  int code = taos_query(taos, "create database if not exists db");
+  TAOS_RES *result = taos_query(taos, "create database if not exists db");
+  int32_t   code = taos_errno(result);
   if (code != 0) {
     taos_error(taos);
   }
+  taos_free_result(result);
 
-  code = taos_query(taos, "create table if not exists db.devices(ts timestamp, temperature int, humidity float) "
-    "tags(devid int, devname binary(16), devgroup int)");
+  result = taos_query(taos,
+                      "create table if not exists db.devices(ts timestamp, temperature int, humidity float) "
+                      "tags(devid int, devname binary(16), devgroup int)");
+  code = taos_errno(result);
   if (code != 0) {
     taos_error(taos);
   }
+  taos_free_result(result);
 
   int64_t st = getTimeStampMs();
 
@@ -292,17 +304,12 @@ void readData() {
 
     int64_t st = getTimeStampMs();
 
-    int code = taos_query(taos, line);
+    TAOS_RES *result = taos_query(taos, line);
+    int32_t   code = taos_errno(result);
     if (code != 0) {
       taos_error(taos);
     }
 
-    void *result = taos_use_result(taos);
-    if (result == NULL) {
-      printf("failed to get result, reason:%s\n", taos_errstr(taos));
-      exit(1);
-    }
-
     TAOS_ROW row;
     int rows = 0;
     //int num_fields = taos_field_count(taos);

tests/pytest/util/dnodes.py

@@ -28,7 +28,9 @@ class TDSimClient:
             "locale": "en_US.UTF-8",
             "charset": "UTF-8",
             "asyncLog": "0",
-            "maxTablesPerVnode": "4",
+            "minTablesPerVnode": "4",
+            "maxTablesPerVnode": "1000",
+            "tableIncStepPerVnode": "10000",
             "maxVgroupsPerDb": "1000",
             "sdbDebugFlag": "143",
             "rpcDebugFlag": "135",

tests/script/sh/deploy.sh

@@ -139,7 +139,9 @@ echo "clog                2"              >> $TAOS_CFG
 echo "statusInterval      1"              >> $TAOS_CFG
 echo "numOfTotalVnodes    4"              >> $TAOS_CFG
 echo "maxVgroupsPerDb     4"              >> $TAOS_CFG
+echo "minTablesPerVnode   4"              >> $TAOS_CFG
 echo "maxTablesPerVnode   1000"           >> $TAOS_CFG
+echo "tableIncStepPerVnode 10000"         >> $TAOS_CFG
 echo "asyncLog            0"              >> $TAOS_CFG
 echo "numOfMnodes         1"              >> $TAOS_CFG
 echo "locale    en_US.UTF-8"              >> $TAOS_CFG

tests/script/tmp/prepare.sim

@@ -25,10 +25,15 @@ system sh/cfg.sh -n dnode2 -c numOfTotalVnodes -v 20
 system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 20
 system sh/cfg.sh -n dnode4 -c numOfTotalVnodes -v 20
 
-system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 20
-system sh/cfg.sh -n dnode2 -c maxVgroupsPerDb -v 20
-system sh/cfg.sh -n dnode3 -c maxVgroupsPerDb -v 20
-system sh/cfg.sh -n dnode4 -c maxVgroupsPerDb -v 20
+system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 2
+system sh/cfg.sh -n dnode2 -c maxVgroupsPerDb -v 2
+system sh/cfg.sh -n dnode3 -c maxVgroupsPerDb -v 2
+system sh/cfg.sh -n dnode4 -c maxVgroupsPerDb -v 2
+
+system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 100000
+system sh/cfg.sh -n dnode2 -c maxTablesPerVnode -v 100000
+system sh/cfg.sh -n dnode3 -c maxTablesPerVnode -v 100000
+system sh/cfg.sh -n dnode4 -c maxTablesPerVnode -v 100000
 
 system sh/cfg.sh -n dnode1 -c http -v 1
 system sh/cfg.sh -n dnode2 -c http -v 1