Merge pull request #1878 from taosdata/feature/tsim

Feature/tsim

src/mnode/inc/mgmtDef.h

@@ -85,8 +85,7 @@ typedef struct SSuperTableObj {
   int32_t    numOfTables;
   int16_t    nextColId;
   SSchema *  schema;
-  int32_t    vgLen;
-  int32_t *  vgList;
+  void *     vgHash;
 } SSuperTableObj;
 
 typedef struct {

src/mnode/src/mgmtTable.c

@@ -24,6 +24,7 @@
 #include "tname.h"
 #include "tidpool.h"
 #include "tglobal.h"
+#include "hash.h"
 #include "dnode.h"
 #include "mgmtDef.h"
 #include "mgmtInt.h"
@@ -363,39 +364,35 @@ static void mgmtCleanUpChildTables() {
 }
 
 static void mgmtAddTableIntoStable(SSuperTableObj *pStable, SChildTableObj *pCtable) {
-  if (pStable->vgLen == 0) {
-    pStable->vgLen = 8;
-    pStable->vgList = calloc(pStable->vgLen, sizeof(int32_t));
-  }
-  
-  bool find = false;
-  int32_t pos = 0;
-  for (pos = 0; pos < pStable->vgLen; ++pos) {
-    if (pStable->vgList[pos] == 0) break;
-    if (pStable->vgList[pos] == pCtable->vgId) {
-      find = true;
-      break;
-    }
-  }
+  pStable->numOfTables++;
 
-  if (!find) {
-    if (pos >= pStable->vgLen) {
-      pStable->vgLen *= 2;
-      pStable->vgList = realloc(pStable->vgList, pStable->vgLen * sizeof(int32_t));
-    }
-    pStable->vgList[pos] = pCtable->vgId;
+  if (pStable->vgHash == NULL) {
+    pStable->vgHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false);
   }
 
-  pStable->numOfTables++;
+  if (pStable->vgHash != NULL) {
+    taosHashPut(pStable->vgHash, (char *)&pCtable->vgId, sizeof(pCtable->vgId), &pCtable->vgId, sizeof(pCtable->vgId));
+  }
 }
 
 static void mgmtRemoveTableFromStable(SSuperTableObj *pStable, SChildTableObj *pCtable) {
   pStable->numOfTables--;
+
+  if (pStable->vgHash == NULL) return;
+
+  SVgObj *pVgroup = mgmtGetVgroup(pCtable->vgId);
+  if (pVgroup != NULL) {
+    taosHashRemove(pStable->vgHash, (char *)&pCtable->vgId, sizeof(pCtable->vgId));
+  }
+  mgmtDecVgroupRef(pVgroup);
 }
 
 static void mgmtDestroySuperTable(SSuperTableObj *pStable) {
+  if (pStable->vgHash != NULL) {
+    taosHashCleanup(pStable->vgHash);
+    pStable->vgHash = NULL;
+  }
   tfree(pStable->schema);
-  tfree(pStable->vgList)
   tfree(pStable);
 }
 
@@ -434,7 +431,7 @@ static int32_t mgmtSuperTableActionUpdate(SSdbOper *pOper) {
     void *oldSchema = pTable->schema;
     memcpy(pTable, pNew, pOper->rowSize);
     pTable->schema = pNew->schema;
-    free(pNew->vgList);
+    free(pNew->vgHash);
     free(pNew);
     free(oldSchema);
   }
@@ -797,26 +794,26 @@ static void mgmtProcessCreateSuperTableMsg(SQueuedMsg *pMsg) {
 static void mgmtProcessDropSuperTableMsg(SQueuedMsg *pMsg) {
   SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable;
   if (pStable->numOfTables != 0) {
-    mgmtDropAllChildTablesInStable(pStable);
-    for (int32_t vg = 0; vg < pStable->vgLen; ++vg) {
-      int32_t vgId = pStable->vgList[vg];
-      if (vgId == 0) break;
-
-      SVgObj *pVgroup = mgmtGetVgroup(vgId);
+    SHashMutableIterator *pIter = taosHashCreateIter(pStable->vgHash);
+    while (taosHashIterNext(pIter)) {
+      int32_t *pVgId = taosHashIterGet(pIter);
+      SVgObj *pVgroup = mgmtGetVgroup(*pVgId);
       if (pVgroup == NULL) break;
-      
+
       SMDDropSTableMsg *pDrop = rpcMallocCont(sizeof(SMDDropSTableMsg));
       pDrop->contLen = htonl(sizeof(SMDDropSTableMsg));
-      pDrop->vgId = htonl(vgId);
+      pDrop->vgId = htonl(pVgroup->vgId);
       pDrop->uid = htobe64(pStable->uid);
       mgmtExtractTableName(pStable->info.tableId, pDrop->tableId);
         
-      mPrint("stable:%s, send drop stable msg to vgId:%d", pStable->info.tableId, vgId);
+      mPrint("stable:%s, send drop stable msg to vgId:%d", pStable->info.tableId, pVgroup->vgId);
       SRpcIpSet ipSet = mgmtGetIpSetFromVgroup(pVgroup);
       SRpcMsg rpcMsg = {.pCont = pDrop, .contLen = sizeof(SMDDropSTableMsg), .msgType = TSDB_MSG_TYPE_MD_DROP_STABLE};
       dnodeSendMsgToDnode(&ipSet, &rpcMsg);
       mgmtDecVgroupRef(pVgroup);
     }
+
+    mgmtDropAllChildTablesInStable(pStable);
   } 
   
   SSdbOper oper = {
@@ -1243,59 +1240,58 @@ static void mgmtGetSuperTableMeta(SQueuedMsg *pMsg) {
 static void mgmtProcessSuperTableVgroupMsg(SQueuedMsg *pMsg) {
   SCMSTableVgroupMsg *pInfo = pMsg->pCont;
   int32_t numOfTable = htonl(pInfo->numOfTables);
-  
-  char* name = (char*) pInfo + sizeof(struct SCMSTableVgroupMsg);
-  SCMSTableVgroupRspMsg *pRsp = NULL;
-  
-  // todo set the initial size to be 10, fix me
-  int32_t contLen = sizeof(SCMSTableVgroupRspMsg) + (sizeof(SCMVgroupInfo) * 10 + sizeof(SVgroupsInfo))*numOfTable;
-  
-  pRsp = rpcMallocCont(contLen);
+
+  // reserve space
+  int32_t contLen = sizeof(SCMSTableVgroupRspMsg) + 32 * sizeof(SCMVgroupInfo) + sizeof(SVgroupsInfo); 
+  for (int32_t i = 0; i < numOfTable; ++i) {
+    char *stableName = (char*)pInfo + sizeof(SCMSTableVgroupMsg) + (TSDB_TABLE_ID_LEN) * i;
+    SSuperTableObj *pTable = mgmtGetSuperTable(stableName);
+    if (pTable->vgHash != NULL) {
+      contLen += (taosHashGetSize(pTable->vgHash) * sizeof(SCMVgroupInfo) + sizeof(SVgroupsInfo));
+    }
+    mgmtDecTableRef(pTable);
+  }
+
+  SCMSTableVgroupRspMsg *pRsp = rpcMallocCont(contLen);
   if (pRsp == NULL) {
     mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_SERV_OUT_OF_MEMORY);
     return;
   }
-  
+
   pRsp->numOfTables = htonl(numOfTable);
   char* msg = (char*) pRsp + sizeof(SCMSTableVgroupRspMsg);
-  
-  for(int32_t i = 0; i < numOfTable; ++i) {
-    SSuperTableObj *pTable = mgmtGetSuperTable(name);
-  
-    pMsg->pTable = (STableObj *)pTable;
-    if (pMsg->pTable == NULL) {
-      mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_TABLE);
-      return;
-    }
-    
-    SVgroupsInfo* pVgroup = (SVgroupsInfo*) msg;
-    
-    int32_t vg = 0;
-    for (; vg < pTable->vgLen; ++vg) {
-      int32_t vgId = pTable->vgList[vg];
-      if (vgId == 0) break;
-    
-      SVgObj *vgItem = mgmtGetVgroup(vgId);
-      if (vgItem == NULL) break;
-  
-      pVgroup->vgroups[vg].vgId = htonl(vgId);
-      for (int32_t vn = 0; vn < vgItem->numOfVnodes; ++vn) {
-        SDnodeObj *pDnode = vgItem->vnodeGid[vn].pDnode;
+
+  for (int32_t i = 0; i < numOfTable; ++i) {
+    char *stableName = (char*)pInfo + sizeof(SCMSTableVgroupMsg) + (TSDB_TABLE_ID_LEN) * i;
+    SSuperTableObj *pTable = mgmtGetSuperTable(stableName);
+    SVgroupsInfo *pVgroupInfo = (SVgroupsInfo *)msg;
+
+    SHashMutableIterator *pIter = taosHashCreateIter(pTable->vgHash);
+    int32_t vgSize = 0;
+    while (taosHashIterNext(pIter)) {
+      int32_t *pVgId = taosHashIterGet(pIter);
+      SVgObj * pVgroup = mgmtGetVgroup(*pVgId);
+      if (pVgroup == NULL) continue;
+
+      pVgroupInfo->vgroups[vgSize].vgId = htonl(pVgroup->vgId);
+      for (int32_t vn = 0; vn < pVgroup->numOfVnodes; ++vn) {
+        SDnodeObj *pDnode = pVgroup->vnodeGid[vn].pDnode;
         if (pDnode == NULL) break;
-  
-        strncpy(pVgroup->vgroups[vg].ipAddr[vn].fqdn, pDnode->dnodeFqdn, tListLen(pDnode->dnodeFqdn));
-        pVgroup->vgroups[vg].ipAddr[vn].port = htons(tsDnodeShellPort);
-        
-        pVgroup->vgroups[vg].numOfIps++;
+
+        strncpy(pVgroupInfo->vgroups[vgSize].ipAddr[vn].fqdn, pDnode->dnodeFqdn, tListLen(pDnode->dnodeFqdn));
+        pVgroupInfo->vgroups[vgSize].ipAddr[vn].port = htons(tsDnodeShellPort);
+
+        pVgroupInfo->vgroups[vgSize].numOfIps++;
       }
-    
-      mgmtDecVgroupRef(vgItem);
+
+      vgSize++;
+      mgmtDecVgroupRef(pVgroup);
     }
-  
-    pVgroup->numOfVgroups = htonl(vg);
-    
+
+    pVgroupInfo->numOfVgroups = htonl(vgSize);
+
     // one table is done, try the next table
-    msg += sizeof(SVgroupsInfo) + vg * sizeof(SCMVgroupInfo);
+    msg += sizeof(SVgroupsInfo) + vgSize * sizeof(SCMVgroupInfo);
   }
 
   SRpcMsg rpcRsp = {0};

tests/script/general/db/vnodes.sim

 system sh/stop_dnodes.sh
 
-$totalVnodes = 100
+$totalVnodes = 10
 $maxTables = 4
 $totalRows = $totalVnodes * $maxTables
 

tests/script/general/parser/auto_create_tb.sim

@@ -96,25 +96,25 @@ $ts1 = $ts0 + 1000
 $ts2 = $ts0 + 2000
 sql insert into tb_1 using $stb tags (-1) values ( $ts1 , 1,1,1,1,'bin',1,1,1,'涛思数据') ( $ts2 , 2,2,2,2,'binar', 1,1,1,'nchar')
 sql select * from $stb
-if $rows != 3 then
+if $rows != 5 then
   return -1
 endi
-if $data19 != 涛思数据 then
+if $data09 != 涛思数据 then
   return -1
 endi
-if $data11 != 1 then
+if $data01 != 1 then
   return -1
 endi
-if $data22 != 2 then
+if $data42 != 2 then
   return -2
 endi
-if $data23 != 2.00000 then
+if $data43 != 2.00000 then
   return -1
 endi
-if $data25 != binar then
+if $data45 != binar then
   return -1
 endi
-if $data29 != nchar then
+if $data49 != nchar then
   return -1
 endi
 sql drop table tb_1
@@ -127,22 +127,22 @@ sql select * from $stb
 if $rows != 5 then
   return -1
 endi
-if $data19 != 涛思数据 then
+if $data09 != 涛思数据 then
   return -1
 endi
-if $data11 != 1 then
+if $data01 != 1 then
   return -1
 endi
-if $data22 != 2 then
+if $data42 != 2 then
   return -2
 endi
-if $data23 != 2.00000 then
+if $data43 != 2.00000 then
   return -1
 endi
-if $data25 != binar then
+if $data45 != binar then
   return -1
 endi
-if $data29 != nchar then
+if $data49 != nchar then
   return -1
 endi
 
@@ -154,13 +154,13 @@ sql show tables
 if $rows != 3 then
   return -1
 endi
-if $data00 != tb3 then
+if $data00 != tb1 then
   return -1
 endi
 if $data10 != tb2 then
   return -1
 endi
-if $data20 != tb1 then
+if $data20 != tb3 then
   return -1
 endi
 

tests/script/general/table/fill.sim

@@ -42,7 +42,7 @@ sql select count(*), last(ts), min(k), max(k), avg(k) from db.mt where a=0 and t
 
 print =================== step2
 system sh/exec.sh -n dnode1 -s stop -x SIGINT
-sleep 10000
+sleep 5000
 system sh/exec.sh -n dnode1 -s start
 sleep 3000
 

tests/script/jenkins/basic.txt

 cd ../../debug; cmake ..
-#cd ../../debug; make clean
 cd ../../debug; make
-
 cd ../../../debug; cmake ..
-#cd ../../../debug; make clean
 cd ../../../debug; make
 
 #./test.sh -f general/alter/cached_schema_after_alter.sim
@@ -159,9 +156,7 @@ cd ../../../debug; make
 #./test.sh -f general/stable/disk.sim
 #./test.sh -f general/stable/metrics.sim
 #./test.sh -f general/stable/values.sim
-#./test.sh -f general/stable/vnode3.sim
-
-#stream
+./test.sh -f general/stable/vnode3.sim
 
 ./test.sh -f general/table/autocreate.sim
 ./test.sh -f general/table/basic1.sim
@@ -176,12 +171,12 @@ cd ../../../debug; make
 ./test.sh -f general/table/column2.sim
 ./test.sh -f general/table/date.sim
 ./test.sh -f general/table/db.table.sim
-#./test.sh -f general/table/delete_reuse1.sim
-#./test.sh -f general/table/delete_reuse2.sim
-#./test.sh -f general/table/delete_writing.sim
-#./test.sh -f general/table/describe.sim
+./test.sh -f general/table/delete_reuse1.sim
+./test.sh -f general/table/delete_reuse2.sim
+#hongze ./test.sh -f general/table/delete_writing.sim
+./test.sh -f general/table/describe.sim
 ./test.sh -f general/table/double.sim
-#./test.sh -f general/table/fill.sim
+./test.sh -f general/table/fill.sim
 ./test.sh -f general/table/float.sim
 ./test.sh -f general/table/int.sim
 ./test.sh -f general/table/limit.sim
@@ -236,8 +231,6 @@ cd ../../../debug; make
 ./test.sh -f general/vector/table_query.sim
 ./test.sh -f general/vector/table_time.sim
 
-#################################  
-
 ./test.sh -u -f unique/account/account_create.sim
 ./test.sh -u -f unique/account/account_delete.sim
 ./test.sh -u -f unique/account/account_len.sim
@@ -251,7 +244,7 @@ cd ../../../debug; make
 ./test.sh -u -f unique/account/user_len.sim
 
 #./test.sh -u -f unique/big/balance.sim
-#./test.sh -u -f unique/big/maxvnodes.sim
+#slguan ./test.sh -u -f unique/big/maxvnodes.sim
 ./test.sh -u -f unique/big/tcp.sim
 
 ##./test.sh -u -f unique/cluster/balance1.sim
@@ -307,8 +300,6 @@ cd ../../../debug; make
 #./test.sh -u -f unique/mnode/mgmtr2.sim
 #./test.sh -u -f unique/mnode/secondIp.sim
 
-#stream
-
 ##./test.sh -u -f unique/table/delete_part.sim
 
 #./test.sh -u -f unique/vnode/commit.sim