Merge branch '3.0' of github.com:taosdata/TDengine into test/chr/TD-14699

include/util/taoserror.h

@@ -615,6 +615,7 @@ int32_t* taosGetErrno();
 //rsma
 #define TSDB_CODE_RSMA_INVALID_ENV               TAOS_DEF_ERROR_CODE(0, 0x3150)
 #define TSDB_CODE_RSMA_INVALID_STAT              TAOS_DEF_ERROR_CODE(0, 0x3151)
+#define TSDB_CODE_RSMA_QTASKINFO_CREATE          TAOS_DEF_ERROR_CODE(0, 0x3152)
 
 //index
 #define TSDB_CODE_INDEX_REBUILDING               TAOS_DEF_ERROR_CODE(0, 0x3200)

source/dnode/vnode/src/sma/smaRollup.c

@@ -40,6 +40,10 @@ static int32_t tdRSmaQTaskInfoIterNextBlock(SRSmaQTaskInfoIter *pIter, bool *isF
 static int32_t tdRSmaQTaskInfoRestore(SSma *pSma, SRSmaQTaskInfoIter *pIter);
 static int32_t tdRSmaQTaskInfoItemRestore(SSma *pSma, const SRSmaQTaskInfoItem *infoItem);
 
+static int32_t tdRSmaRestoreQTaskInfoInit(SSma *pSma);
+static int32_t tdRSmaRestoreQTaskInfoReload(SSma *pSma);
+static int32_t tdRSmaRestoreTSDataReload(SSma *pSma);
+
 struct SRSmaInfoItem {
   SRSmaInfo *pRsmaInfo;
   void      *taskInfo;  // qTaskInfo_t
@@ -246,6 +250,7 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaInfo
     pItem->pRsmaInfo = pRSmaInfo;
     pItem->taskInfo = qCreateStreamExecTaskInfo(param->qmsg[idx], pReadHandle);
     if (!pItem->taskInfo) {
+      terrno = TSDB_CODE_RSMA_QTASKINFO_CREATE;
       goto _err;
     }
     pItem->triggerStat = TASK_TRIGGER_STAT_INACTIVE;
@@ -696,10 +701,9 @@ int32_t tdProcessRSmaSubmit(SSma *pSma, void *pMsg, int32_t inputType) {
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t tdProcessRSmaRestoreImpl(SSma *pSma) {
+static int32_t tdRSmaRestoreQTaskInfoInit(SSma *pSma) {
   SVnode *pVnode = pSma->pVnode;
 
-  // step 1: iterate all stables to restore the rsma env
   SArray *suidList = taosArrayInit(1, sizeof(tb_uid_t));
   if (tsdbGetStbIdList(SMA_META(pSma), 0, suidList) < 0) {
     taosArrayDestroy(suidList);
@@ -741,42 +745,88 @@ int32_t tdProcessRSmaRestoreImpl(SSma *pSma) {
     }
   }
 
-  // step 2: retrieve qtaskinfo items from the persistence file(rsma/qtaskinfo) and restore
-  STFile tFile = {0};
-  char   qTaskInfoFName[TSDB_FILENAME_LEN];
+  metaReaderClear(&mr);
+  taosArrayDestroy(suidList);
+
+  return TSDB_CODE_SUCCESS;
+_err:
+  metaReaderClear(&mr);
+  taosArrayDestroy(suidList);
+
+  return TSDB_CODE_FAILED;
+}
+
+static int32_t tdRSmaRestoreQTaskInfoReload(SSma *pSma) {
+  SVnode *pVnode = pSma->pVnode;
+  STFile  tFile = {0};
+  char    qTaskInfoFName[TSDB_FILENAME_LEN];
 
   tdRSmaQTaskInfoGetFName(TD_VID(pVnode), TD_QTASK_CUR_FILE, qTaskInfoFName);
   if (tdInitTFile(&tFile, pVnode->pTfs, qTaskInfoFName) < 0) {
     goto _err;
   }
-  
-  if(!taosCheckExistFile(TD_TFILE_FULL_NAME(&tFile))) {
-    metaReaderClear(&mr);
-    taosArrayDestroy(suidList);
+
+  if (!taosCheckExistFile(TD_TFILE_FULL_NAME(&tFile))) {
     return TSDB_CODE_SUCCESS;
   }
 
   if (tdOpenTFile(&tFile, TD_FILE_READ) < 0) {
     goto _err;
   }
-  
+
   SRSmaQTaskInfoIter fIter = {0};
   if (tdRSmaQTaskInfoIterInit(&fIter, &tFile) < 0) {
+    tdRSmaQTaskInfoIterDestroy(&fIter);
+    tdCloseTFile(&tFile);
     goto _err;
   }
-  SRSmaQTaskInfoItem infoItem = {0};
+
   if (tdRSmaQTaskInfoRestore(pSma, &fIter) < 0) {
     tdRSmaQTaskInfoIterDestroy(&fIter);
+    tdCloseTFile(&tFile);
     goto _err;
   }
 
   tdRSmaQTaskInfoIterDestroy(&fIter);
-  metaReaderClear(&mr);
-  taosArrayDestroy(suidList);
+  tdCloseTFile(&tFile);
+  return TSDB_CODE_SUCCESS;
+_err:
+  smaError("rsma restore, qtaskinfo reload failed since %s", terrstr());
+  return TSDB_CODE_FAILED;
+}
+
+/**
+ * @brief reload ts data from checkpoint
+ * 
+ * @param pSma 
+ * @return int32_t 
+ */
+static int32_t tdRSmaRestoreTSDataReload(SSma *pSma) {
+  // TODO
+  return TSDB_CODE_SUCCESS;
+_err:
+  smaError("rsma restore, ts data reload failed since %s", terrstr());
+  return TSDB_CODE_FAILED;
+}
+
+int32_t tdProcessRSmaRestoreImpl(SSma *pSma) {
+  // step 1: iterate all stables to restore the rsma env
+  if (tdRSmaRestoreQTaskInfoInit(pSma) < 0) {
+    goto _err;
+  }
+
+  // step 2: retrieve qtaskinfo items from the persistence file(rsma/qtaskinfo) and restore
+  if (tdRSmaRestoreQTaskInfoReload(pSma) < 0) {
+    goto _err;
+  }
+
+  // step 3: reload ts data from checkpoint
+  if (tdRSmaRestoreTSDataReload(pSma) < 0) {
+    goto _err;
+  }
+
   return TSDB_CODE_SUCCESS;
 _err:
-  metaReaderClear(&mr);
-  taosArrayDestroy(suidList);
   smaError("failed to restore rsma task since %s", terrstr());
   return TSDB_CODE_FAILED;
 }

source/util/src/terror.c

@@ -581,10 +581,10 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TSMA_INVALID_PTR,               "Invalid tsma pointe
 TAOS_DEFINE_ERROR(TSDB_CODE_TSMA_INVALID_PARA,              "Invalid tsma parameters")
 TAOS_DEFINE_ERROR(TSDB_CODE_TSMA_NO_INDEX_IN_CACHE,         "No tsma index in cache")
 
-
 //rsma
 TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_INVALID_ENV,               "Invalid rsma env")
 TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_INVALID_STAT,              "Invalid rsma state")
+TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_QTASKINFO_CREATE,          "Rsma qtaskinfo creation error")
 
 //tq
 TAOS_DEFINE_ERROR(TSDB_CODE_TQ_NO_COMMITTED_OFFSET,         "No committed offset")

tests/script/jenkins/basic.txt

@@ -8,7 +8,6 @@
 ./test.sh -f tsim/user/privilege_sysinfo.sim
 
 ## ---- db
-./test.sh -f tsim/db/create_all_options.sim
 ./test.sh -f tsim/db/alter_option.sim
 ./test.sh -f tsim/db/alter_replica_13.sim
 ./test.sh -f tsim/db/alter_replica_31.sim
@@ -17,6 +16,10 @@
 ./test.sh -f tsim/db/basic3.sim
 ./test.sh -f tsim/db/basic6.sim
 ./test.sh -f tsim/db/basic7.sim
+#./test.sh -f tsim/db/commit.sim
+./test.sh -f tsim/db/create_all_options.sim
+#./test.sh -f tsim/db/delete_part.sim
+#./test.sh -f tsim/db/delete.sim
 ./test.sh -f tsim/db/error1.sim
 ./test.sh -f tsim/db/taosdlog.sim
 
@@ -27,6 +30,13 @@
 ./test.sh -f tsim/dnode/balance2.sim
 ./test.sh -f tsim/dnode/balance3.sim
 ./test.sh -f tsim/dnode/balancex.sim
+#./test.sh -f tsim/dnode/cluster_alter.sim
+#./test.sh -f tsim/dnode/cluster_balance1.sim
+#./test.sh -f tsim/dnode/cluster_balance2.sim
+#./test.sh -f tsim/dnode/cluster_balance3.sim
+#./test.sh -f tsim/dnode/cluster_cache.sim
+#./test.sh -f tsim/dnode/cluster_flowctrl.sim
+#./test.sh -f tsim/dnode/cluster_vgroup100.sim
 ./test.sh -f tsim/dnode/create_dnode.sim
 ./test.sh -f tsim/dnode/drop_dnode_has_mnode.sim
 ./test.sh -f tsim/dnode/drop_dnode_has_qnode_snode.sim
@@ -78,7 +88,7 @@
 ./test.sh -f tsim/mnode/basic2.sim
 ./test.sh -f tsim/mnode/basic3.sim
 ./test.sh -f tsim/mnode/basic4.sim
-#./test.sh -f tsim/mnode/basic5.sim
+./test.sh -f tsim/mnode/basic5.sim
 
 # ---- show
 ./test.sh -f tsim/show/basic.sim
@@ -103,7 +113,6 @@
 ./test.sh -f tsim/stream/schedSnode.sim
 ./test.sh -f tsim/stream/windowClose.sim
 
-
 # ---- transaction
  ./test.sh -f tsim/trans/lossdata1.sim
  ./test.sh -f tsim/trans/create_db.sim
@@ -168,6 +177,12 @@
 ./test.sh -f tsim/vnode/replica3_vgroup.sim
 #./test.sh -f tsim/vnode/replica3_many.sim
 #./test.sh -f tsim/vnode/replica3_import.sim
+./test.sh -f tsim/vnode/stable_balance_replica1.sim
+./test.sh -f tsim/vnode/stable_dnode2_stop.sim
+./test.sh -f tsim/vnode/stable_dnode2.sim
+./test.sh -f tsim/vnode/stable_dnode3.sim
+./test.sh -f tsim/vnode/stable_replica3_dnode6.sim
+./test.sh -f tsim/vnode/stable_replica3_vnode3.sim
 
 # --- sync
 ./test.sh -f tsim/sync/3Replica1VgElect.sim