Merge remote-tracking branch 'origin/3.0' into fix/TD-20458_g

cmake/taosws_CMakeLists.txt.in

@@ -2,7 +2,7 @@
 # taosws-rs
 ExternalProject_Add(taosws-rs
         GIT_REPOSITORY https://github.com/taosdata/taos-connector-rust.git
-        GIT_TAG 7664c41
+        GIT_TAG 9843872
         SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosws-rs"
         BINARY_DIR ""
         #BUILD_IN_SOURCE TRUE

source/libs/stream/src/stream.c

@@ -51,6 +51,7 @@ void streamSchedByTimer(void* param, void* tmrId) {
   SStreamTask* pTask = (void*)param;
 
   if (atomic_load_8(&pTask->taskStatus) == TASK_STATUS__DROPPING) {
+    streamMetaReleaseTask(NULL, pTask);
     return;
   }
 
@@ -80,6 +81,8 @@ void streamSchedByTimer(void* param, void* tmrId) {
 
 int32_t streamSetupTrigger(SStreamTask* pTask) {
   if (pTask->triggerParam != 0) {
+    int32_t ref = atomic_add_fetch_32(&pTask->refCnt, 1);
+    ASSERT(ref == 2);
     pTask->timer = taosTmrStart(streamSchedByTimer, (int32_t)pTask->triggerParam, pTask, streamEnv.timer);
     pTask->triggerStatus = TASK_TRIGGER_STATUS__INACTIVE;
   }

source/libs/stream/src/streamMeta.c

@@ -80,7 +80,12 @@ void streamMetaClose(SStreamMeta* pMeta) {
     pIter = taosHashIterate(pMeta->pTasks, pIter);
     if (pIter == NULL) break;
     SStreamTask* pTask = *(SStreamTask**)pIter;
+    if (pTask->timer) {
+      taosTmrStop(pTask->timer);
+      pTask->timer = NULL;
+    }
     tFreeSStreamTask(pTask);
+    /*streamMetaReleaseTask(pMeta, pTask);*/
   }
   taosHashCleanup(pMeta->pTasks);
   taosMemoryFree(pMeta->path);
@@ -202,6 +207,10 @@ void streamMetaRemoveTask1(SStreamMeta* pMeta, int32_t taskId) {
   if (ppTask) {
     SStreamTask* pTask = *ppTask;
     taosHashRemove(pMeta->pTasks, &taskId, sizeof(int32_t));
+    /*if (pTask->timer) {
+     * taosTmrStop(pTask->timer);*/
+    /*pTask->timer = NULL;*/
+    /*}*/
     atomic_store_8(&pTask->taskStatus, TASK_STATUS__DROPPING);
 
     taosWLockLatch(&pMeta->lock);

source/libs/sync/src/syncAppendEntriesReply.c

@@ -83,10 +83,14 @@ int32_t syncNodeOnAppendEntriesReply(SSyncNode* ths, const SRpcMsg* pRpcMsg) {
     ASSERT(pState != NULL);
 
     if (pMsg->lastSendIndex == pState->lastSendIndex) {
+      int64_t timeNow = taosGetTimestampMs();
+      int64_t elapsed = timeNow - pState->lastSendTime;
+      sNTrace(ths, "sync-append-entries rtt elapsed:%" PRId64 ", index:%" PRId64, elapsed, pState->lastSendIndex);
+
       syncNodeReplicateOne(ths, &(pMsg->srcId), true);
     }
   }
 
   syncLogRecvAppendEntriesReply(ths, pMsg, "process");
   return 0;
-}
\ No newline at end of file
+}

source/libs/sync/src/syncSnapshot.c

@@ -192,7 +192,9 @@ int32_t snapshotSend(SSyncSnapshotSender *pSender) {
 
   // pMsg->privateTerm = pSender->privateTerm;
 
-  memcpy(pMsg->data, pSender->pCurrentBlock, pSender->blockLen);
+  if (pSender->pCurrentBlock != NULL) {
+    memcpy(pMsg->data, pSender->pCurrentBlock, pSender->blockLen);
+  }
 
   // send msg
   syncNodeSendMsgById(&pMsg->destId, pSender->pSyncNode, &rpcMsg);

source/libs/wal/src/walWrite.c

@@ -102,6 +102,7 @@ int32_t walCommit(SWal *pWal, int64_t ver) {
 
 int32_t walRollback(SWal *pWal, int64_t ver) {
   taosThreadMutexLock(&pWal->mutex);
+  wInfo("vgId:%d, wal rollback for version %" PRId64, pWal->cfg.vgId, ver);
   int64_t code;
   char    fnameStr[WAL_FILE_LEN];
   if (ver > pWal->vers.lastVer || ver < pWal->vers.commitVer || ver <= pWal->vers.snapshotVer) {
@@ -123,8 +124,10 @@ int32_t walRollback(SWal *pWal, int64_t ver) {
     int fileSetSize = taosArrayGetSize(pWal->fileInfoSet);
     for (int i = pWal->writeCur + 1; i < fileSetSize; i++) {
       walBuildLogName(pWal, ((SWalFileInfo *)taosArrayGet(pWal->fileInfoSet, i))->firstVer, fnameStr);
+      wDebug("vgId:%d, wal remove file %s for rollback", pWal->cfg.vgId, fnameStr);
       taosRemoveFile(fnameStr);
       walBuildIdxName(pWal, ((SWalFileInfo *)taosArrayGet(pWal->fileInfoSet, i))->firstVer, fnameStr);
+      wDebug("vgId:%d, wal remove file %s for rollback", pWal->cfg.vgId, fnameStr);
       taosRemoveFile(fnameStr);
     }
     // pop from fileInfoSet
@@ -157,6 +160,7 @@ int32_t walRollback(SWal *pWal, int64_t ver) {
 
   walBuildLogName(pWal, walGetCurFileFirstVer(pWal), fnameStr);
   TdFilePtr pLogFile = taosOpenFile(fnameStr, TD_FILE_WRITE | TD_FILE_READ | TD_FILE_APPEND);
+  wDebug("vgId:%d, wal truncate file %s", pWal->cfg.vgId, fnameStr);
   if (pLogFile == NULL) {
     // TODO
     terrno = TAOS_SYSTEM_ERROR(errno);
@@ -324,9 +328,9 @@ int32_t walEndSnapshot(SWal *pWal) {
       pInfo++;
     }
     if (POINTER_DISTANCE(pInfo, pWal->fileInfoSet->pData) > 0) {
-      wDebug("vgId:%d, begin remove from %" PRId64, pWal->cfg.vgId, pInfo->firstVer);
+      wDebug("vgId:%d, wal end remove from %" PRId64, pWal->cfg.vgId, pInfo->firstVer);
     } else {
-      wDebug("vgId:%d, no remove", pWal->cfg.vgId);
+      wDebug("vgId:%d, wal no remove", pWal->cfg.vgId);
     }
     // iterate files, until the searched result
     for (SWalFileInfo *iter = pWal->fileInfoSet->pData; iter < pInfo; iter++) {
@@ -343,12 +347,12 @@ int32_t walEndSnapshot(SWal *pWal) {
     for (int i = 0; i < deleteCnt; i++) {
       pInfo = taosArrayGet(pWal->fileInfoSet, i);
       walBuildLogName(pWal, pInfo->firstVer, fnameStr);
-      wDebug("vgId:%d, remove file %s", pWal->cfg.vgId, fnameStr);
+      wDebug("vgId:%d, wal remove file %s", pWal->cfg.vgId, fnameStr);
       if (taosRemoveFile(fnameStr) < 0) {
         goto UPDATE_META;
       }
       walBuildIdxName(pWal, pInfo->firstVer, fnameStr);
-      wDebug("vgId:%d, remove file %s", pWal->cfg.vgId, fnameStr);
+      wDebug("vgId:%d, wal remove file %s", pWal->cfg.vgId, fnameStr);
       if (taosRemoveFile(fnameStr) < 0) {
         ASSERT(0);
       }

tests/parallel_test/cases.task

@@ -37,10 +37,10 @@
 ,,y,script,./test.sh -f tsim/db/taosdlog.sim
 ,,,script,./test.sh -f tsim/dnode/balance_replica1.sim
 ,,,script,./test.sh -f tsim/dnode/balance_replica3.sim
-,,,script,./test.sh -f tsim/dnode/balance1.sim
-,,,script,./test.sh -f tsim/dnode/balance2.sim
-,,,script,./test.sh -f tsim/dnode/balance3.sim
-,,,script,./test.sh -f tsim/dnode/balancex.sim
+#,,,script,./test.sh -f tsim/dnode/balance1.sim
+#,,,script,./test.sh -f tsim/dnode/balance2.sim
+#,,,script,./test.sh -f tsim/dnode/balance3.sim
+#,,,script,./test.sh -f tsim/dnode/balancex.sim
 ,,y,script,./test.sh -f tsim/dnode/create_dnode.sim
 ,,y,script,./test.sh -f tsim/dnode/drop_dnode_has_mnode.sim
 ,,y,script,./test.sh -f tsim/dnode/drop_dnode_has_qnode_snode.sim
@@ -55,7 +55,7 @@
 ,,,script,./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim
 ,,,script,./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v2.sim
 ,,,script,./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v3.sim
-,,,script,./test.sh -f tsim/dnode/vnode_clean.sim
+#,,,script,./test.sh -f tsim/dnode/vnode_clean.sim
 ,,,script,./test.sh -f tsim/dnode/use_dropped_dnode.sim 
 ,,,script,./test.sh -f tsim/dnode/split_vgroup_replica1.sim 
 ,,,script,./test.sh -f tsim/dnode/split_vgroup_replica3.sim 
@@ -115,7 +115,7 @@
 ,,y,script,./test.sh -f tsim/parser/interp.sim
 #,,y,script,./test.sh -f tsim/parser/limit2.sim
 ,,y,script,./test.sh -f tsim/parser/fourArithmetic-basic.sim
-,,,script,./test.sh -f tsim/parser/function.sim
+,,y,script,./test.sh -f tsim/parser/function.sim
 ,,y,script,./test.sh -f tsim/parser/groupby-basic.sim
 ,,,script,./test.sh -f tsim/parser/groupby.sim
 ,,y,script,./test.sh -f tsim/parser/having_child.sim
@@ -140,7 +140,7 @@
 ,,,script,./test.sh -f tsim/parser/limit1.sim
 ,,y,script,./test.sh -f tsim/parser/mixed_blocks.sim
 ,,y,script,./test.sh -f tsim/parser/nchar.sim
-,,,script,./test.sh -f tsim/parser/nestquery.sim
+,,y,script,./test.sh -f tsim/parser/nestquery.sim
 ,,y,script,./test.sh -f tsim/parser/null_char.sim
 ,,y,script,./test.sh -f tsim/parser/precision_ns.sim
 ,,y,script,./test.sh -f tsim/parser/projection_limit_offset.sim
@@ -296,7 +296,7 @@
 ,,y,script,./test.sh -f tsim/vnode/replica3_vgroup.sim
 ,,y,script,./test.sh -f tsim/vnode/replica3_many.sim
 ,,y,script,./test.sh -f tsim/vnode/replica3_import.sim
-,,,script,./test.sh -f tsim/vnode/stable_balance_replica1.sim
+#,,,script,./test.sh -f tsim/vnode/stable_balance_replica1.sim
 ,,y,script,./test.sh -f tsim/vnode/stable_dnode2_stop.sim
 ,,y,script,./test.sh -f tsim/vnode/stable_dnode2.sim
 ,,y,script,./test.sh -f tsim/vnode/stable_dnode3.sim

tests/script/test.sh

@@ -7,6 +7,7 @@
 ##################################################
 
 set +e
+#set -x
 
 FILE_NAME=
 RELEASE=0
@@ -137,8 +138,16 @@ if [ -n "$FILE_NAME" ]; then
     $PROGRAM -c $CFG_DIR -f $FILE_NAME -v
   else
     echo "ExcuteCmd:" $PROGRAM -c $CFG_DIR -f $FILE_NAME
-    $PROGRAM -c $CFG_DIR -f $FILE_NAME 2> $ASAN_DIR/tsim.asan 
-    $CODE_DIR/sh/checkAsan.sh
+    echo "AsanDir:" $ASAN_DIR/tsim.asan
+    eval $PROGRAM -c $CFG_DIR -f $FILE_NAME 2> $ASAN_DIR/tsim.asan
+    result=$?
+    echo "Execute result: " $result
+
+    if [ $result -eq 0 ]; then
+      $CODE_DIR/sh/checkAsan.sh
+    else
+      exit 1
+    fi
   fi
 else
   echo "ExcuteCmd:" $PROGRAM -c $CFG_DIR -f basicSuite.sim

tests/script/tsim/compute/diff2.sim

@@ -83,7 +83,7 @@ sql select diff(c1), diff(c2) from $tb
 sql select 2+diff(c1) from $tb
 sql select diff(c1+2) from $tb
 sql_error select diff(c1) from $tb where ts > 0 and ts < now + 100m interval(10m)
-sql select diff(c1) from $mt
+#sql select diff(c1) from $mt
 sql_error select diff(diff(c1)) from $tb
 sql_error select diff(c1) from m_di_tb1 where c2 like '2%'
 

tests/script/tsim/parser/function.sim

@@ -932,10 +932,10 @@ sql insert into t0 values('2020-1-1 1:4:10', 10);
 
 sql insert into t1 values('2020-1-1 1:1:2', 2);
 print ===========================>td-4739
-sql select diff(val) from (select derivative(k, 1s, 0) val from t1);
-if $rows != 0 then
-  return -1
-endi
+#sql select diff(val) from (select derivative(k, 1s, 0) val from t1);
+#if $rows != 0 then
+#  return -1
+#endi
 
 sql insert into t1 values('2020-1-1 1:1:4', 20);
 sql insert into t1 values('2020-1-1 1:1:6', 200);

tests/script/tsim/parser/nestquery.sim

@@ -338,13 +338,13 @@ if $data03 != @20-09-15 00:00:00.000@ then
   return -1
 endi
 
-sql select diff(val) from (select c1 val from nest_tb0);
-if $rows != 9999 then
-  return -1
-endi
-if $data00 != 1 then
-  return -1
-endi
+#sql select diff(val) from (select c1 val from nest_tb0);
+#if $rows != 9999 then
+#  return -1
+#endi
+#if $data00 != 1 then
+#  return -1
+#endi
 
 sql_error select last_row(*) from (select * from nest_tb0) having c1 > 0
 

tests/script/tsim/valgrind/checkError6.sim

@@ -92,7 +92,7 @@ sql select avg(tbcol) as b from stb where ts <= 1601481840000 partition by tgcol
 sql show table distributed stb
 sql select count(1) from stb
 sql select count(tbcol) as b from stb where ts <= 1601481840000 partition by tgcol interval(1m)
-sql select diff(tbcol) from stb where ts <= 1601481840000 
+#sql select diff(tbcol) from stb where ts <= 1601481840000 
 sql select first(tbcol), last(tbcol) as c from stb group by tgcol
 sql select first(tbcol), last(tbcol) as b from stb where ts <= 1601481840000 and tbcol2 is null partition by tgcol interval(1m)
 sql select first(tbcol), last(tbcol) as b from stb where ts <= 1601481840000 partition by tgcol interval(1m)
@@ -176,7 +176,7 @@ sql select avg(tbcol) as c from stb group by tgcol
 sql select avg(tbcol) as b from stb where ts <= 1601481840000 partition by tgcol interval(1m)
 sql show table distributed stb
 sql select count(tbcol) as b from stb where ts <= 1601481840000 partition by tgcol interval(1m)
-sql select diff(tbcol) from stb where ts <= 1601481840000 
+#sql select diff(tbcol) from stb where ts <= 1601481840000 
 sql select first(tbcol), last(tbcol) as c from stb group by tgcol
 sql select first(tbcol), last(tbcol) as b from stb where ts <= 1601481840000 and tbcol2 is null partition by tgcol interval(1m)
 sql select first(tbcol), last(tbcol) as b from stb where ts <= 1601481840000 partition by tgcol interval(1m)