Merge branch 'feature/dnode3' into feature/vnode

Jenkinsfile

@@ -105,15 +105,17 @@ pipeline {
               abort_previous()
               abortPreviousBuilds()
             }
-          pre_test()
-          sh'''
-          cd ${WKC}/tests
-          ./test-all.sh b1fq
-          '''
-          sh'''
-          cd ${WKC}/debug
-          ctest
-          '''
+            timeout(time: 45, unit: 'MINUTES'){
+              pre_test()
+              sh'''
+              cd ${WKC}/tests
+              ./test-all.sh b1fq
+              '''
+              sh'''
+              cd ${WKC}/debug
+              ctest
+              '''
+            }
           }
       }
   }

source/dnode/mgmt/impl/src/dndVnodes.c

@@ -165,6 +165,8 @@ static void dndCloseVnode(SDnode *pDnode, SVnodeObj *pVnode) {
   vnodeClose(pVnode->pImpl);
   pVnode->pImpl = NULL;
 
+  dDebug("vgId:%d, vnode is closed", pVnode->vgId);
+
   free(pVnode->path);
   free(pVnode->db);
   free(pVnode);
@@ -238,59 +240,57 @@ static int32_t dndGetVnodesFromFile(SDnode *pDnode, SWrapperCfg **ppCfgs, int32_
   }
 
   int32_t vnodesNum = cJSON_GetArraySize(vnodes);
-  if (vnodesNum <= 0) {
-    dError("failed to read %s since vnodes size:%d invalid", file, vnodesNum);
-    goto PRASE_VNODE_OVER;
-  }
+  if (vnodesNum > 0) {
+    pCfgs = calloc(vnodesNum, sizeof(SWrapperCfg));
+    if (pCfgs == NULL) {
+      dError("failed to read %s since out of memory", file);
+      goto PRASE_VNODE_OVER;
+    }
 
-  pCfgs = calloc(vnodesNum, sizeof(SWrapperCfg));
-  if (pCfgs == NULL) {
-    dError("failed to read %s since out of memory", file);
-    goto PRASE_VNODE_OVER;
-  }
+    for (int32_t i = 0; i < vnodesNum; ++i) {
+      cJSON       *vnode = cJSON_GetArrayItem(vnodes, i);
+      SWrapperCfg *pCfg = &pCfgs[i];
 
-  for (int32_t i = 0; i < vnodesNum; ++i) {
-    cJSON *      vnode = cJSON_GetArrayItem(vnodes, i);
-    SWrapperCfg *pCfg = &pCfgs[i];
+      cJSON *vgId = cJSON_GetObjectItem(vnode, "vgId");
+      if (!vgId || vgId->type != cJSON_Number) {
+        dError("failed to read %s since vgId not found", file);
+        goto PRASE_VNODE_OVER;
+      }
+      pCfg->vgId = vgId->valueint;
+      snprintf(pCfg->path, sizeof(pCfg->path), "%s/vnode%d", pDnode->dir.vnodes, pCfg->vgId);
 
-    cJSON *vgId = cJSON_GetObjectItem(vnode, "vgId");
-    if (!vgId || vgId->type != cJSON_Number) {
-      dError("failed to read %s since vgId not found", file);
-      goto PRASE_VNODE_OVER;
-    }
-    pCfg->vgId = vgId->valueint;
-    snprintf(pCfg->path, sizeof(pCfg->path), "%s/vnode%d", pDnode->dir.vnodes, pCfg->vgId);
+      cJSON *dropped = cJSON_GetObjectItem(vnode, "dropped");
+      if (!dropped || dropped->type != cJSON_Number) {
+        dError("failed to read %s since dropped not found", file);
+        goto PRASE_VNODE_OVER;
+      }
+      pCfg->dropped = dropped->valueint;
 
-    cJSON *dropped = cJSON_GetObjectItem(vnode, "dropped");
-    if (!dropped || dropped->type != cJSON_Number) {
-      dError("failed to read %s since dropped not found", file);
-      goto PRASE_VNODE_OVER;
-    }
-    pCfg->dropped = dropped->valueint;
+      cJSON *vgVersion = cJSON_GetObjectItem(vnode, "vgVersion");
+      if (!vgVersion || vgVersion->type != cJSON_Number) {
+        dError("failed to read %s since vgVersion not found", file);
+        goto PRASE_VNODE_OVER;
+      }
+      pCfg->vgVersion = vgVersion->valueint;
 
-    cJSON *vgVersion = cJSON_GetObjectItem(vnode, "vgVersion");
-    if (!vgVersion || vgVersion->type != cJSON_Number) {
-      dError("failed to read %s since vgVersion not found", file);
-      goto PRASE_VNODE_OVER;
-    }
-    pCfg->vgVersion = vgVersion->valueint;
+      cJSON *dbUid = cJSON_GetObjectItem(vnode, "dbUid");
+      if (!dbUid || dbUid->type != cJSON_String) {
+        dError("failed to read %s since dbUid not found", file);
+        goto PRASE_VNODE_OVER;
+      }
+      pCfg->dbUid = atoll(dbUid->valuestring);
 
-    cJSON *dbUid = cJSON_GetObjectItem(vnode, "dbUid");
-    if (!dbUid || dbUid->type != cJSON_String) {
-      dError("failed to read %s since dbUid not found", file);
-      goto PRASE_VNODE_OVER;
+      cJSON *db = cJSON_GetObjectItem(vnode, "db");
+      if (!db || db->type != cJSON_String) {
+        dError("failed to read %s since db not found", file);
+        goto PRASE_VNODE_OVER;
+      }
+      tstrncpy(pCfg->db, db->valuestring, TSDB_DB_FNAME_LEN);
     }
-    pCfg->dbUid = atoll(dbUid->valuestring);
 
-    cJSON *db = cJSON_GetObjectItem(vnode, "db");
-    if (!db || db->type != cJSON_String) {
-      dError("failed to read %s since db not found", file);
-      goto PRASE_VNODE_OVER;
-    }
-    tstrncpy(pCfg->db, db->valuestring, TSDB_DB_FNAME_LEN);
+    *ppCfgs = pCfgs;
   }
 
-  *ppCfgs = pCfgs;
   *numOfVnodes = vnodesNum;
   code = 0;
   dInfo("succcessed to read file %s", file);
@@ -545,13 +545,13 @@ static void dndGenerateWrapperCfg(SDnode *pDnode, SCreateVnodeReq *pCreate, SWra
   pCfg->vgVersion = pCreate->vgVersion;
 }
 
-static SDropVnodeReq *vnodeParseDropVnodeReq(SRpcMsg *pReq) {
+static SDropVnodeReq *dndParseDropVnodeReq(SRpcMsg *pReq) {
   SDropVnodeReq *pDrop = pReq->pCont;
   pDrop->vgId = htonl(pDrop->vgId);
   return pDrop;
 }
 
-static SAuthVnodeReq *vnodeParseAuthVnodeReq(SRpcMsg *pReq) {
+static SAuthVnodeReq *dndParseAuthVnodeReq(SRpcMsg *pReq) {
   SAuthVnodeReq *pAuth = pReq->pCont;
   pAuth->vgId = htonl(pAuth->vgId);
   return pAuth;
@@ -569,10 +569,10 @@ int32_t dndProcessCreateVnodeReq(SDnode *pDnode, SRpcMsg *pReq) {
 
   SVnodeObj *pVnode = dndAcquireVnode(pDnode, pCreate->vgId);
   if (pVnode != NULL) {
-    dDebug("vgId:%d, already exist, return success", pCreate->vgId);
+    dDebug("vgId:%d, already exist", pCreate->vgId);
     dndReleaseVnode(pDnode, pVnode);
     terrno = TSDB_CODE_DND_VNODE_ALREADY_DEPLOYED;
-    return 0;
+    return -1;
   }
 
   SVnode *pImpl = vnodeOpen(wrapperCfg.path, NULL /*pCfg*/, pCreate->vgId);
@@ -638,7 +638,7 @@ int32_t dndProcessAlterVnodeReq(SDnode *pDnode, SRpcMsg *pReq) {
 }
 
 int32_t dndProcessDropVnodeReq(SDnode *pDnode, SRpcMsg *pReq) {
-  SDropVnodeReq *pDrop = vnodeParseDropVnodeReq(pReq);
+  SDropVnodeReq *pDrop = dndParseDropVnodeReq(pReq);
 
   int32_t vgId = pDrop->vgId;
   dDebug("vgId:%d, drop vnode req is received", vgId);
@@ -646,7 +646,8 @@ int32_t dndProcessDropVnodeReq(SDnode *pDnode, SRpcMsg *pReq) {
   SVnodeObj *pVnode = dndAcquireVnode(pDnode, vgId);
   if (pVnode == NULL) {
     dDebug("vgId:%d, failed to drop since %s", vgId, terrstr());
-    return 0;
+    terrno = TSDB_CODE_DND_VNODE_NOT_DEPLOYED;
+    return -1;
   }
 
   pVnode->dropped = 1;
@@ -665,7 +666,7 @@ int32_t dndProcessDropVnodeReq(SDnode *pDnode, SRpcMsg *pReq) {
 }
 
 int32_t dndProcessAuthVnodeReq(SDnode *pDnode, SRpcMsg *pReq) {
-  SAuthVnodeReq *pAuth = (SAuthVnodeReq *)vnodeParseAuthVnodeReq(pReq);
+  SAuthVnodeReq *pAuth = (SAuthVnodeReq *)dndParseAuthVnodeReq(pReq);
 
   int32_t vgId = pAuth->vgId;
   dDebug("vgId:%d, auth vnode req is received", vgId);
@@ -682,7 +683,7 @@ int32_t dndProcessAuthVnodeReq(SDnode *pDnode, SRpcMsg *pReq) {
 }
 
 int32_t dndProcessSyncVnodeReq(SDnode *pDnode, SRpcMsg *pReq) {
-  SSyncVnodeReq *pSync = (SSyncVnodeReq *)vnodeParseDropVnodeReq(pReq);
+  SSyncVnodeReq *pSync = (SSyncVnodeReq *)dndParseDropVnodeReq(pReq);
 
   int32_t vgId = pSync->vgId;
   dDebug("vgId:%d, sync vnode req is received", vgId);
@@ -704,7 +705,7 @@ int32_t dndProcessSyncVnodeReq(SDnode *pDnode, SRpcMsg *pReq) {
 }
 
 int32_t dndProcessCompactVnodeReq(SDnode *pDnode, SRpcMsg *pReq) {
-  SCompactVnodeReq *pCompact = (SCompactVnodeReq *)vnodeParseDropVnodeReq(pReq);
+  SCompactVnodeReq *pCompact = (SCompactVnodeReq *)dndParseDropVnodeReq(pReq);
 
   int32_t vgId = pCompact->vgId;
   dDebug("vgId:%d, compact vnode req is received", vgId);

source/dnode/mgmt/impl/test/mnode/dmnode.cpp

@@ -13,7 +13,7 @@
 
 class DndTestMnode : public ::testing::Test {
  protected:
-  static void SetUpTestSuite() { test.Init("/tmp/dnode_test_mnode", 9113); }
+  static void SetUpTestSuite() { test.Init("/tmp/dnode_test_mnode", 9114); }
   static void TearDownTestSuite() { test.Cleanup(); }
 
   static Testbase   test;

source/dnode/mgmt/impl/test/snode/dsnode.cpp

@@ -13,7 +13,7 @@
 
 class DndTestSnode : public ::testing::Test {
  protected:
-  static void SetUpTestSuite() { test.Init("/tmp/dnode_test_snode", 9112); }
+  static void SetUpTestSuite() { test.Init("/tmp/dnode_test_snode", 9113); }
   static void TearDownTestSuite() { test.Cleanup(); }
 
   static Testbase test;

source/dnode/mgmt/impl/test/vnode/vnode.cpp

 /**
  * @file db.cpp
  * @author slguan (slguan@taosdata.com)
- * @brief DNODE module vgroup-msg tests
+ * @brief DNODE module vnode tests
  * @version 0.1
  * @date 2021-12-20
  *
@@ -13,7 +13,7 @@
 
 class DndTestVnode : public ::testing::Test {
  protected:
-  static void SetUpTestSuite() { test.Init("/tmp/dnode_test_vnode", 9150); }
+  static void SetUpTestSuite() { test.Init("/tmp/dnode_test_vnode", 9115); }
   static void TearDownTestSuite() { test.Cleanup(); }
 
   static Testbase test;
@@ -57,12 +57,17 @@ TEST_F(DndTestVnode, 01_Create_Restart_Drop_Vnode) {
       for (int r = 0; r < pReq->replica; ++r) {
         SReplica* pReplica = &pReq->replicas[r];
         pReplica->id = htonl(1);
-        pReplica->port = htons(9150);
+        pReplica->port = htons(9527);
       }
 
       SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_VNODE, pReq, contLen);
       ASSERT_NE(pRsp, nullptr);
-      ASSERT_EQ(pRsp->code, 0);
+      if (i == 0) {
+        ASSERT_EQ(pRsp->code, 0);
+        test.Restart();
+      } else {
+        ASSERT_EQ(pRsp->code, TSDB_CODE_DND_VNODE_ALREADY_DEPLOYED);
+      }
     }
   }
 
@@ -97,7 +102,7 @@ TEST_F(DndTestVnode, 01_Create_Restart_Drop_Vnode) {
       for (int r = 0; r < pReq->replica; ++r) {
         SReplica* pReplica = &pReq->replicas[r];
         pReplica->id = htonl(1);
-        pReplica->port = htons(9150);
+        pReplica->port = htons(9527);
       }
 
       SRpcMsg* pRsp = test.SendReq(TDMT_DND_ALTER_VNODE, pReq, contLen);
@@ -123,7 +128,12 @@ TEST_F(DndTestVnode, 01_Create_Restart_Drop_Vnode) {
 
       SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_VNODE, pReq, contLen);
       ASSERT_NE(pRsp, nullptr);
-      ASSERT_EQ(pRsp->code, 0);
+      if (i == 0) {
+        ASSERT_EQ(pRsp->code, 0);
+        test.Restart();
+      } else {
+        ASSERT_EQ(pRsp->code, TSDB_CODE_DND_VNODE_NOT_DEPLOYED);
+      }
     }
   }
 }

source/dnode/mnode/impl/src/mndDb.c

@@ -29,14 +29,14 @@ static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw);
 static int32_t  mndDbActionInsert(SSdb *pSdb, SDbObj *pDb);
 static int32_t  mndDbActionDelete(SSdb *pSdb, SDbObj *pDb);
 static int32_t  mndDbActionUpdate(SSdb *pSdb, SDbObj *pOldDb, SDbObj *pNewDb);
-static int32_t  mndProcessCreateDbMsg(SMnodeMsg *pMsg);
-static int32_t  mndProcessAlterDbMsg(SMnodeMsg *pMsg);
-static int32_t  mndProcessDropDbMsg(SMnodeMsg *pMsg);
-static int32_t  mndProcessUseDbMsg(SMnodeMsg *pMsg);
-static int32_t  mndProcessSyncDbMsg(SMnodeMsg *pMsg);
-static int32_t  mndProcessCompactDbMsg(SMnodeMsg *pMsg);
-static int32_t  mndGetDbMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta);
-static int32_t  mndRetrieveDbs(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows);
+static int32_t  mndProcessCreateDbReq(SMnodeMsg *pReq);
+static int32_t  mndProcessAlterDbReq(SMnodeMsg *pReq);
+static int32_t  mndProcessDropDbReq(SMnodeMsg *pReq);
+static int32_t  mndProcessUseDbReq(SMnodeMsg *pReq);
+static int32_t  mndProcessSyncDbReq(SMnodeMsg *pReq);
+static int32_t  mndProcessCompactDbReq(SMnodeMsg *pReq);
+static int32_t  mndGetDbMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaMsg *pMeta);
+static int32_t  mndRetrieveDbs(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows);
 static void     mndCancelGetNextDb(SMnode *pMnode, void *pIter);
 
 int32_t mndInitDb(SMnode *pMnode) {
@@ -48,12 +48,12 @@ int32_t mndInitDb(SMnode *pMnode) {
                      .updateFp = (SdbUpdateFp)mndDbActionUpdate,
                      .deleteFp = (SdbDeleteFp)mndDbActionDelete};
 
-  mndSetMsgHandle(pMnode, TDMT_MND_CREATE_DB, mndProcessCreateDbMsg);
-  mndSetMsgHandle(pMnode, TDMT_MND_ALTER_DB, mndProcessAlterDbMsg);
-  mndSetMsgHandle(pMnode, TDMT_MND_DROP_DB, mndProcessDropDbMsg);
-  mndSetMsgHandle(pMnode, TDMT_MND_USE_DB, mndProcessUseDbMsg);
-  mndSetMsgHandle(pMnode, TDMT_MND_SYNC_DB, mndProcessSyncDbMsg);
-  mndSetMsgHandle(pMnode, TDMT_MND_COMPACT_DB, mndProcessCompactDbMsg);
+  mndSetMsgHandle(pMnode, TDMT_MND_CREATE_DB, mndProcessCreateDbReq);
+  mndSetMsgHandle(pMnode, TDMT_MND_ALTER_DB, mndProcessAlterDbReq);
+  mndSetMsgHandle(pMnode, TDMT_MND_DROP_DB, mndProcessDropDbReq);
+  mndSetMsgHandle(pMnode, TDMT_MND_USE_DB, mndProcessUseDbReq);
+  mndSetMsgHandle(pMnode, TDMT_MND_SYNC_DB, mndProcessSyncDbReq);
+  mndSetMsgHandle(pMnode, TDMT_MND_COMPACT_DB, mndProcessCompactDbReq);
 
   mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_DB, mndGetDbMeta);
   mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_DB, mndRetrieveDbs);
@@ -471,7 +471,7 @@ CREATE_DB_OVER:
   return code;
 }
 
-static int32_t mndProcessCreateDbMsg(SMnodeMsg *pMsg) {
+static int32_t mndProcessCreateDbReq(SMnodeMsg *pMsg) {
   SMnode       *pMnode = pMsg->pMnode;
   SCreateDbMsg *pCreate = pMsg->rpcMsg.pCont;
 
@@ -668,7 +668,7 @@ UPDATE_DB_OVER:
   return code;
 }
 
-static int32_t mndProcessAlterDbMsg(SMnodeMsg *pMsg) {
+static int32_t mndProcessAlterDbReq(SMnodeMsg *pMsg) {
   SMnode      *pMnode = pMsg->pMnode;
   SAlterDbMsg *pAlter = pMsg->rpcMsg.pCont;
   pAlter->totalBlocks = htonl(pAlter->totalBlocks);
@@ -832,7 +832,7 @@ DROP_DB_OVER:
   return code;
 }
 
-static int32_t mndProcessDropDbMsg(SMnodeMsg *pMsg) {
+static int32_t mndProcessDropDbReq(SMnodeMsg *pMsg) {
   SMnode     *pMnode = pMsg->pMnode;
   SDropDbMsg *pDrop = pMsg->rpcMsg.pCont;
 
@@ -861,7 +861,7 @@ static int32_t mndProcessDropDbMsg(SMnodeMsg *pMsg) {
   return TSDB_CODE_MND_ACTION_IN_PROGRESS;
 }
 
-static int32_t mndProcessUseDbMsg(SMnodeMsg *pMsg) {
+static int32_t mndProcessUseDbReq(SMnodeMsg *pMsg) {
   SMnode    *pMnode = pMsg->pMnode;
   SSdb      *pSdb = pMnode->pSdb;
   SUseDbMsg *pUse = pMsg->rpcMsg.pCont;
@@ -929,7 +929,7 @@ static int32_t mndProcessUseDbMsg(SMnodeMsg *pMsg) {
   return 0;
 }
 
-static int32_t mndProcessSyncDbMsg(SMnodeMsg *pMsg) {
+static int32_t mndProcessSyncDbReq(SMnodeMsg *pMsg) {
   SMnode     *pMnode = pMsg->pMnode;
   SSyncDbMsg *pSync = pMsg->rpcMsg.pCont;
   SDbObj     *pDb = mndAcquireDb(pMnode, pSync->db);
@@ -942,7 +942,7 @@ static int32_t mndProcessSyncDbMsg(SMnodeMsg *pMsg) {
   return 0;
 }
 
-static int32_t mndProcessCompactDbMsg(SMnodeMsg *pMsg) {
+static int32_t mndProcessCompactDbReq(SMnodeMsg *pMsg) {
   SMnode        *pMnode = pMsg->pMnode;
   SCompactDbMsg *pCompact = pMsg->rpcMsg.pCont;
   SDbObj        *pDb = mndAcquireDb(pMnode, pCompact->db);
@@ -1096,8 +1096,8 @@ char *mnGetDbStr(char *src) {
   return pos;
 }
 
-static int32_t mndRetrieveDbs(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows) {
-  SMnode *pMnode = pMsg->pMnode;
+static int32_t mndRetrieveDbs(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) {
+  SMnode *pMnode = pReq->pMnode;
   SSdb   *pSdb = pMnode->pSdb;
   int32_t numOfRows = 0;
   SDbObj *pDb = NULL;

source/dnode/mnode/impl/test/db/db.cpp

@@ -13,17 +13,28 @@
 
 class MndTestDb : public ::testing::Test {
  protected:
-  static void SetUpTestSuite() { test.Init("/tmp/mnode_test_db", 9030); }
-  static void TearDownTestSuite() { test.Cleanup(); }
+  static void SetUpTestSuite() {
+    test.Init("/tmp/mnode_test_db", 9030);
+    const char* fqdn = "localhost";
+    const char* firstEp = "localhost:9030";
 
-  static Testbase test;
+    server2.Start("/tmp/mnode_test_db2", fqdn, 9031, firstEp);
+  }
+  static void TearDownTestSuite() {
+    server2.Stop();
+    test.Cleanup();
+  }
+
+  static Testbase   test;
+  static TestServer server2;
 
  public:
   void SetUp() override {}
   void TearDown() override {}
 };
 
-Testbase MndTestDb::test;
+Testbase   MndTestDb::test;
+TestServer MndTestDb::server2;
 
 TEST_F(MndTestDb, 01_ShowDb) {
   test.SendShowMetaReq(TSDB_MGMT_TABLE_DB, "");
@@ -149,7 +160,7 @@ TEST_F(MndTestDb, 02_Create_Alter_Drop_Db) {
   CheckBinary("d1", TSDB_DB_NAME_LEN - 1);
   CheckTimestamp();
   CheckInt16(2);                   // vgroups
-  CheckInt32(0);
+  CheckInt32(0);                   // tables
   CheckInt16(1);                   // replica
   CheckInt16(2);                   // quorum
   CheckInt16(10);                  // days
@@ -177,7 +188,7 @@ TEST_F(MndTestDb, 02_Create_Alter_Drop_Db) {
   CheckBinary("d1", TSDB_DB_NAME_LEN - 1);
   CheckTimestamp();
   CheckInt16(2);                   // vgroups
-  CheckInt32(0);
+  CheckInt32(0);                   // tables
   CheckInt16(1);                   // replica
   CheckInt16(2);                   // quorum
   CheckInt16(10);                  // days
@@ -301,4 +312,15 @@ TEST_F(MndTestDb, 03_Create_Use_Restart_Use_Db) {
       EXPECT_STREQ(pAddr->fqdn, "localhost");
     }
   }
+
+  {
+    int32_t contLen = sizeof(SDropDbMsg);
+
+    SDropDbMsg* pReq = (SDropDbMsg*)rpcMallocCont(contLen);
+    strcpy(pReq->db, "1.d2");
+
+    SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_DB, pReq, contLen);
+    ASSERT_NE(pRsp, nullptr);
+    ASSERT_EQ(pRsp->code, 0);
+  }
 }