Merge branch 'develop' into feature/2.0tsdb

src/client/src/tscSystem.c

@@ -93,7 +93,7 @@ int32_t tscInitRpc(const char *user, const char *secret) {
     rpcInit.sessions = tsMaxMgmtConnections;
     rpcInit.connType = TAOS_CONN_CLIENT;
     rpcInit.idleTime = 2000;
-    rpcInit.user = "root";
+    rpcInit.user = (char*)user;
     rpcInit.ckey = "key";
     rpcInit.secret = secretEncrypt;
 

src/dnode/src/dnodeMClient.c

@@ -128,7 +128,9 @@ static void dnodeProcessStatusRsp(SRpcMsg *pMsg) {
 }
 
 void dnodeSendMsgToMnode(SRpcMsg *rpcMsg) {
-  rpcSendRequest(tsDnodeMClientRpc, &tsDnodeMnodeIpList, rpcMsg);
+  if (tsDnodeMClientRpc) {
+    rpcSendRequest(tsDnodeMClientRpc, &tsDnodeMnodeIpList, rpcMsg);
+  }
 }
 
 static bool dnodeReadMnodeIpList() {

src/dnode/src/dnodeMain.c

@@ -37,7 +37,6 @@ static void dnodeSetRunStatus(SDnodeRunStatus status);
 static void signal_handler(int32_t signum, siginfo_t *sigInfo, void *context);
 static void dnodeCheckDataDirOpenned(char *dir);
 static SDnodeRunStatus tsDnodeRunStatus = TSDB_DNODE_RUN_STATUS_STOPPED;
-void (*dnodeParseParameterKFp)() = NULL;
 
 int32_t main(int32_t argc, char *argv[]) {
   // Set global configuration file
@@ -57,25 +56,24 @@ int32_t main(int32_t argc, char *argv[]) {
       printf("buildinfo: %s\n", buildinfo);
       exit(EXIT_SUCCESS);
     } else if (strcmp(argv[i], "-k") == 0) {
-      if (dnodeParseParameterKFp) {
-        dnodeParseParameterKFp();
-        exit(EXIT_SUCCESS);
-      }
+      grantParseParameter();
+      exit(EXIT_SUCCESS);
+    }
 #ifdef TAOS_MEM_CHECK
-    } else if (strcmp(argv[i], "--alloc-random-fail") == 0) {
-      if ((i < argc - 1) && (argv[i+1][0] != '-')) {
+    else if (strcmp(argv[i], "--alloc-random-fail") == 0) {
+      if ((i < argc - 1) && (argv[i + 1][0] != '-')) {
         taosSetAllocMode(TAOS_ALLOC_MODE_RANDOM_FAIL, argv[++i], true);
       } else {
         taosSetAllocMode(TAOS_ALLOC_MODE_RANDOM_FAIL, NULL, true);
       }
     } else if (strcmp(argv[i], "--detect-mem-leak") == 0) {
-      if ((i < argc - 1) && (argv[i+1][0] != '-')) {
+      if ((i < argc - 1) && (argv[i + 1][0] != '-')) {
         taosSetAllocMode(TAOS_ALLOC_MODE_DETECT_LEAK, argv[++i], true);
       } else {
         taosSetAllocMode(TAOS_ALLOC_MODE_DETECT_LEAK, NULL, true);
       }
-#endif
     }
+#endif
   }
 
   /* Set termination handler. */
@@ -180,9 +178,9 @@ static void dnodeCleanUpSystem() {
     tclearModuleStatus(TSDB_MOD_MGMT);
     dnodeSetRunStatus(TSDB_DNODE_RUN_STATUS_STOPPED);
     dnodeCleanupShell();
-    dnodeCleanupMClient();
     dnodeCleanupMnode();
     dnodeCleanupMgmt();
+    dnodeCleanupMClient();
     dnodeCleanupWrite();
     dnodeCleanupRead();
     dnodeCleanUpModules();

src/dnode/src/dnodeMgmt.c

@@ -29,6 +29,8 @@
 #include "dnodeRead.h"
 #include "dnodeWrite.h"
 
+typedef enum { CLOSE_TSDB, DROP_TSDB } ECloseTsdbFlag;
+
 typedef struct {
   int32_t      vgId;      // global vnode group ID
   int32_t      refCount;  // reference count
@@ -47,10 +49,9 @@ static int32_t  dnodeOpenVnodes();
 static void     dnodeCleanupVnodes();
 static int32_t  dnodeOpenVnode(int32_t vnode, char *rootDir);
 static void     dnodeCleanupVnode(SVnodeObj *pVnode);
-static void     dnodeDoCleanupVnode(SVnodeObj *pVnode);
+static void     dnodeDoCleanupVnode(SVnodeObj *pVnode, ECloseTsdbFlag dropFlag);
 static int32_t  dnodeCreateVnode(SMDCreateVnodeMsg *cfg);
 static void     dnodeDropVnode(SVnodeObj *pVnode);
-static void     dnodeDoDropVnode(SVnodeObj *pVnode);
 static void     dnodeProcessCreateVnodeMsg(SRpcMsg *pMsg);
 static void     dnodeProcessDropVnodeMsg(SRpcMsg *pMsg);
 static void     dnodeProcessAlterVnodeMsg(SRpcMsg *pMsg);
@@ -178,14 +179,14 @@ void dnodeReleaseVnode(void *pVnodeRaw) {
   if (pVnode->status == TSDB_VN_STATUS_DELETING) {
     if (refCount <= 0) {
       dPrint("pVnode:%p, vgroup:%d, drop vnode, refCount:%d", pVnode, pVnode->vgId, refCount);
-      dnodeDoDropVnode(pVnode);
+      dnodeDoCleanupVnode(pVnode, DROP_TSDB);
     } else {
       dTrace("pVnode:%p, vgroup:%d, vnode will be dropped until refCount:%d is 0", pVnode, pVnode->vgId, refCount);
     }
   } else if (pVnode->status == TSDB_VN_STATUS_CLOSING) {
     if (refCount <= 0) {
       dPrint("pVnode:%p, vgroup:%d, cleanup vnode, refCount:%d", pVnode, pVnode->vgId, refCount);
-      dnodeDoCleanupVnode(pVnode);
+      dnodeDoCleanupVnode(pVnode, CLOSE_TSDB);
     } else {
       dTrace("pVnode:%p, vgroup:%d, vnode will cleanup until refCount:%d is 0", pVnode, pVnode->vgId, refCount);
     }
@@ -264,7 +265,7 @@ static int32_t dnodeOpenVnode(int32_t vnode, char *rootDir) {
   return TSDB_CODE_SUCCESS;
 }
 
-static void dnodeDoCleanupVnode(SVnodeObj *pVnode) {
+static void dnodeDoCleanupVnode(SVnodeObj *pVnode, ECloseTsdbFlag closeFlag) {
   dTrace("pVnode:%p, vgroup:%d, cleanup vnode", pVnode, pVnode->vgId);
   
   // remove replica
@@ -281,7 +282,12 @@ static void dnodeDoCleanupVnode(SVnodeObj *pVnode) {
 
   // remove tsdb
   if (pVnode->tsdb) {
-    tsdbCloseRepo(pVnode->tsdb);
+    if (closeFlag == DROP_TSDB) {
+      tsdbDropRepo(pVnode->tsdb);
+    } else if (closeFlag == CLOSE_TSDB) {
+      tsdbCloseRepo(pVnode->tsdb);
+    }
+    taosDeleteIntHash(tsDnodeVnodesHash, pVnode->vgId);
     pVnode->tsdb = NULL;
   }
 }
@@ -351,20 +357,10 @@ static int32_t dnodeCreateVnode(SMDCreateVnodeMsg *pVnodeCfg) {
     pVnode->status = TSDB_VN_STATUS_MASTER;
   }
 
-  dPrint("vgroup:%d, vnode:%d is created", pVnode->vgId, pVnode->vgId);
+  dPrint("pVnode:%p, vgroup:%d, vnode:%d is created", pVnode, pVnode->vgId, pVnode->vgId);
   return TSDB_CODE_SUCCESS;
 }
 
-static void dnodeDoDropVnode(SVnodeObj *pVnode) {
-  dnodeDoCleanupVnode(pVnode);
-  taosDeleteIntHash(tsDnodeVnodesHash, pVnode->vgId);
-
-  char rootDir[TSDB_FILENAME_LEN] = {0};
-  sprintf(rootDir, "%s/vnode%d", tsVnodeDir, pVnode->vgId);
-  dPrint("pVnode:%p, vgroup:%d, drop file:%s from disk", pVnode, pVnode->vgId, rootDir);
-  // rmdir(rootDir);
-}
-
 static void dnodeDropVnode(SVnodeObj *pVnode) {
   pVnode->status = TSDB_VN_STATUS_DELETING;
   dnodeReleaseVnode(pVnode);
@@ -383,7 +379,7 @@ static void dnodeProcessCreateVnodeMsg(SRpcMsg *rpcMsg) {
   SVnodeObj *pVnodeObj = (SVnodeObj *) taosGetIntHashData(tsDnodeVnodesHash, pCreate->cfg.vgId);
   if (pVnodeObj != NULL) {
     rpcRsp.code = TSDB_CODE_SUCCESS;
-    dPrint("vgroup:%d, vnode is already exist", pCreate->cfg.vgId);
+    dPrint("pVnode:%p, vgroup:%d, vnode is already exist", pVnodeObj, pCreate->cfg.vgId);
   } else {
     rpcRsp.code = dnodeCreateVnode(pCreate);
   }
@@ -453,7 +449,6 @@ static void dnodeProcessConfigDnodeMsg(SRpcMsg *pMsg) {
 
 static void dnodeBuildVloadMsg(char *pNode, void * param) {
   SVnodeObj *pVnode = (SVnodeObj *) pNode;
-  dPrint("===> pVnode:%p, vgroup:%d status:%s", pVnode, pVnode->vgId, taosGetVnodeStatusStr(pVnode->status));
   if (pVnode->status == TSDB_VN_STATUS_DELETING) return;
   
   SDMStatusMsg *pStatus = param;

src/mnode/inc/mgmtDnode.h

@@ -24,6 +24,7 @@ extern "C" {
 int32_t mgmtInitDnodes();
 void    mgmtCleanUpDnodes();
 int32_t mgmtGetDnodesNum();
+void *  mgmtGetNextDnode(void *pNode, SDnodeObj **pDnode);
 SDnodeObj* mgmtGetDnode(int32_t dnodeId);
 SDnodeObj* mgmtGetDnodeByIp(uint32_t ip);
 

src/mnode/src/mgmtBalance.c

@@ -18,47 +18,37 @@
 #include "mgmtBalance.h"
 #include "mgmtDnode.h"
 
-int32_t (*mgmtInitBalanceFp)() = NULL;
-void    (*mgmtCleanupBalanceFp)() = NULL;
-void    (*mgmtStartBalanceTimerFp)(int32_t afterMs) = NULL;
-int32_t (*mgmtAllocVnodesFp)(SVgObj *pVgroup) = NULL;
-
-int32_t mgmtInitBalance() {
-  if (mgmtInitBalanceFp) {
-    return (*mgmtInitBalanceFp)();
-  } else {
-    return 0;
-  }
-}
-
-void mgmtCleanupBalance() {
-  if (mgmtCleanupBalanceFp) {
-    (*mgmtCleanupBalanceFp)();
-  }
-}
-
-void mgmtStartBalanceTimer(int32_t afterMs) {
-  if (mgmtStartBalanceTimerFp) {
-    (*mgmtStartBalanceTimerFp)(afterMs);
-  }
-}
+int32_t mgmtInitBalance() { return 0; }
+void    mgmtCleanupBalance() {}
+void    mgmtStartBalanceTimer(int32_t afterMs) {}
 
 int32_t mgmtAllocVnodes(SVgObj *pVgroup) {
-  if (mgmtAllocVnodesFp) {
-    return (*mgmtAllocVnodesFp)(pVgroup);
+  void *     pNode = NULL;
+  SDnodeObj *pDnode = NULL;
+  SDnodeObj *pSelDnode = NULL;
+  float      vnodeUsage = 1.0;
+
+  while (1) {
+    pNode = mgmtGetNextDnode(pNode, &pDnode);
+    if (pDnode == NULL) break;
+    if (pDnode->numOfTotalVnodes <= 0) continue;
+    if (pDnode->openVnodes == pDnode->numOfTotalVnodes) continue;
+
+    float usage = (float)pDnode->openVnodes / pDnode->numOfTotalVnodes;
+    if (usage <= vnodeUsage) {
+      pSelDnode = pDnode;
+      vnodeUsage = usage;
+    }
   }
 
-  SDnodeObj *pDnode = mgmtGetDnode(1);
-  if (pDnode == NULL) return TSDB_CODE_OTHERS;
-
-  if (pDnode->openVnodes < pDnode->numOfTotalVnodes) {
-    pVgroup->vnodeGid[0].dnodeId   = pDnode->dnodeId;
-    pVgroup->vnodeGid[0].privateIp = pDnode->privateIp;
-    pVgroup->vnodeGid[0].publicIp  = pDnode->publicIp;
-    mTrace("dnode:%d, alloc one vnode to vgroup", pDnode->dnodeId);
-    return TSDB_CODE_SUCCESS;
-  } else {
-    mError("dnode:%d, failed to alloc vnode to vgroup", pDnode->dnodeId);
+  if (pSelDnode == NULL) {
+    mError("failed to alloc vnode to vgroup", pDnode->dnodeId);
     return TSDB_CODE_NO_ENOUGH_DNODES;
   }
+
+  pVgroup->vnodeGid[0].dnodeId = pSelDnode->dnodeId;
+  pVgroup->vnodeGid[0].privateIp = pSelDnode->privateIp;
+  pVgroup->vnodeGid[0].publicIp = pSelDnode->publicIp;
+  mTrace("dnode:%d, alloc one vnode to vgroup", pSelDnode->dnodeId);
+  return TSDB_CODE_SUCCESS;
 }

src/mnode/src/mgmtDnode.c

@@ -32,6 +32,7 @@ static void    mgmtProcessDnodeStatusMsg(SRpcMsg *rpcMsg);
 extern int32_t clusterInit();
 extern void    clusterCleanUp();
 extern int32_t clusterGetDnodesNum();
+extern void *  clusterGetNextDnode(void *pNode, SDnodeObj **pDnode);
 extern SDnodeObj* clusterGetDnode(int32_t dnodeId);
 extern SDnodeObj* clusterGetDnodeByIp(uint32_t ip);
 static SDnodeObj  tsDnodeObj = {0};
@@ -98,6 +99,19 @@ int32_t mgmtGetDnodesNum() {
 #endif
 }
 
+void *  mgmtGetNextDnode(void *pNode, SDnodeObj **pDnode) {
+#ifdef _CLUSTER
+  return (*clusterGetNextDnode)(pNode, pDnode);
+#else
+  if (*pDnode == NULL) {
+    *pDnode = &tsDnodeObj;
+  } else {
+    *pDnode = NULL;
+  }
+  return *pDnode;
+#endif
+}
+
 void mgmtProcessCfgDnodeMsg(SQueuedMsg *pMsg) {
   SRpcMsg rpcRsp = {.handle = pMsg->thandle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0};
   if (mgmtCheckRedirect(pMsg->thandle)) return;

src/mnode/src/mgmtMain.c

@@ -126,6 +126,8 @@ int32_t mgmtStartSystem() {
     mError("failed to init dnode balance")
   }
 
+  grantReset(TSDB_GRANT_ALL, 0);
+
   mPrint("TDengine mgmt is initialized successfully");
 
   return 0;

tests/script/sh/deploy.sh

@@ -90,9 +90,9 @@ echo "logDir              $LOG_DIR"       >> $TAOS_CFG
 echo "publicIp            $NODE_IP"       >> $TAOS_CFG
 echo "internalIp          $NODE_IP"       >> $TAOS_CFG
 echo "privateIp           $NODE_IP"       >> $TAOS_CFG
-echo "dDebugFlag          199"            >> $TAOS_CFG
+echo "dDebugFlag          135"            >> $TAOS_CFG
 echo "mDebugFlag          199"            >> $TAOS_CFG
-echo "sdbDebugFlag        199"            >> $TAOS_CFG
+echo "sdbDebugFlag        135"            >> $TAOS_CFG
 echo "rpcDebugFlag        135"            >> $TAOS_CFG
 echo "tmrDebugFlag        131"            >> $TAOS_CFG
 echo "cDebugFlag          135"            >> $TAOS_CFG

tests/script/tmp/prepare.sim

 system sh/stop_dnodes.sh
-system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1
\ No newline at end of file
+system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1
+system sh/deploy.sh -n dnode2 -m 192.168.0.1 -i 192.168.0.2
\ No newline at end of file

tests/script/unique/account/paras.sim

+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1
+system sh/exec_up.sh -n dnode1 -s start
+sql connect
+
+print =============== show accounts
+sql show accounts
+if $rows != 1 then 
+  return -1
+endi
+
+print $data00 $data01 $data02 $data03 $data04 $data05 $data06
+if $data00 != root then 
+  return -1
+endi
+if $data02 != 3/10 then 
+  return -1
+endi
+if $data03 != 0/64 then 
+  return -1
+endi
+if $data04 != 0/2147483647 then 
+  return -1
+endi
+if $data05 != 0/1000 then 
+  return -1
+endi
+if $data06 != 0.000/unlimited then 
+  return -1
+endi
+
+print =============== create account
+sql create account hou pass "hou" tseries 80000 storage 10737418240 streams 10 qtime 3600 dbs 3 users 3 conns 10 
+sql show accounts
+if $rows != 2 then 
+  return -1
+endi
+
+print $data10 $data11 $data12 $data13 $data14 $data15 $data16
+if $data10 != hou then 
+  return -1
+endi
+if $data12 != 2/3 then 
+  return -1
+endi
+if $data13 != 0/3 then 
+  return -1
+endi
+if $data14 != 0/80000 then 
+  return -1
+endi
+if $data15 != 0/10 then 
+  return -1
+endi
+if $data16 != 0.000/10.000 then 
+  return -1
+endi
+  
+print =============== alter account
+sql alter account hou pass "hou" tseries 8000 streams 10 dbs 5 users 5  
+sql show accounts
+if $rows != 2 then 
+  return -1
+endi
+
+print $data10 $data11 $data12 $data13 $data14 $data15 $data16
+if $data10 != hou then 
+  return -1
+endi
+if $data12 != 2/5 then 
+  return -1
+endi
+if $data13 != 0/5 then 
+  return -1
+endi
+if $data14 != 0/8000 then 
+  return -1
+endi
+if $data15 != 0/10 then 
+  return -1
+endi
+if $data16 != 0.000/10.000 then 
+  return -1
+endi  
+
+print =============== alter account
+sql create account hou pass "hou" tseries 8000 streams 10 dbs 5 users 6
+sql show accounts
+if $rows != 2 then 
+  return -1
+endi
+
+print $data10 $data11 $data12 $data13 $data14 $data15 $data16
+if $data10 != hou then 
+  return -1
+endi
+if $data12 != 2/6 then 
+  return -1
+endi
+if $data13 != 0/5 then 
+  return -1
+endi
+if $data14 != 0/8000 then 
+  return -1
+endi
+if $data15 != 0/10 then 
+  return -1
+endi
+if $data16 != 0.000/10.000 then 
+  return -1
+endi  

tests/script/unique/account/usage.sim

+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1
+system sh/exec_up.sh -n dnode1 -s start
+sql connect
+
+print =============== show accounts
+
+print =============== create account
+sql alter account root pass "taosdata" tseries 8000 streams 10 dbs 5 users 5  
+sql show accounts
+print $data00 $data01 $data02 $data03 $data04 $data05 $data06
+if $data00 != root then 
+  return -1
+endi
+if $data02 != 3/5 then 
+  return -1
+endi
+if $data03 != 0/5 then 
+  return -1
+endi
+if $data04 != 0/8000 then 
+  return -1
+endi
+if $data05 != 0/10 then 
+  return -1
+endi
+  
+print =============== check usage account
+sql create database d1
+sql create database d2
+sql create database d3
+sql create database d4
+sql create database d5
+
+sql create table d1.t1 (ts timestamp, i int);
+sql create user u1 pass "u1"
+
+sql show accounts
+print $data10 $data11 $data12 $data13 $data14 $data15 $data16
+if $data00 != root then 
+  return -1
+endi
+if $data02 != 4/5 then 
+  return -1
+endi
+if $data03 != 5/5 then 
+  return -1
+endi
+if $data04 != 1/8000 then 
+  return -1
+endi
+if $data05 != 0/10 then 
+  return -1
+endi
+
+print =============== check grant
+sql_error create database d6
\ No newline at end of file

tests/script/unique/testSuite.sim

 ################################# 
 run unique/account/basic.sim
+run unique/account/paras.sim
+run unique/account/usage.sim
 ##################################