Merge branch 'main' into FIX/TD-23613-main

cmake/taosadapter_CMakeLists.txt.in

@@ -2,7 +2,7 @@
 # taosadapter
 ExternalProject_Add(taosadapter
         GIT_REPOSITORY https://github.com/taosdata/taosadapter.git
-        GIT_TAG cb1e89c
+        GIT_TAG e02ddb2
         SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosadapter"
         BINARY_DIR ""
         #BUILD_IN_SOURCE TRUE

cmake/taostools_CMakeLists.txt.in

@@ -2,7 +2,7 @@
 # taos-tools
 ExternalProject_Add(taos-tools
         GIT_REPOSITORY https://github.com/taosdata/taos-tools.git
-        GIT_TAG 149ac34
+        GIT_TAG 0681d8b
         SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools"
         BINARY_DIR ""
         #BUILD_IN_SOURCE TRUE

source/dnode/mnode/impl/src/mndCluster.c

@@ -67,7 +67,7 @@ int32_t mndGetClusterName(SMnode *pMnode, char *clusterName, int32_t len) {
   return 0;
 }
 
-static SClusterObj *mndAcquireCluster(SMnode *pMnode) {
+static SClusterObj *mndAcquireCluster(SMnode *pMnode, void **ppIter) {
   SSdb *pSdb = pMnode->pSdb;
   void *pIter = NULL;
 
@@ -76,23 +76,27 @@ static SClusterObj *mndAcquireCluster(SMnode *pMnode) {
     pIter = sdbFetch(pSdb, SDB_CLUSTER, pIter, (void **)&pCluster);
     if (pIter == NULL) break;
 
+    *ppIter = pIter;
+
     return pCluster;
   }
 
   return NULL;
 }
 
-static void mndReleaseCluster(SMnode *pMnode, SClusterObj *pCluster) {
+static void mndReleaseCluster(SMnode *pMnode, SClusterObj *pCluster, void *pIter) {
   SSdb *pSdb = pMnode->pSdb;
+  sdbCancelFetch(pSdb, pIter);
   sdbRelease(pSdb, pCluster);
 }
 
 int64_t mndGetClusterId(SMnode *pMnode) {
   int64_t      clusterId = 0;
-  SClusterObj *pCluster = mndAcquireCluster(pMnode);
+  void        *pIter = NULL;
+  SClusterObj *pCluster = mndAcquireCluster(pMnode, &pIter);
   if (pCluster != NULL) {
     clusterId = pCluster->id;
-    mndReleaseCluster(pMnode, pCluster);
+    mndReleaseCluster(pMnode, pCluster, pIter);
   }
 
   return clusterId;
@@ -100,10 +104,11 @@ int64_t mndGetClusterId(SMnode *pMnode) {
 
 int64_t mndGetClusterCreateTime(SMnode *pMnode) {
   int64_t      createTime = 0;
-  SClusterObj *pCluster = mndAcquireCluster(pMnode);
+  void        *pIter = NULL;
+  SClusterObj *pCluster = mndAcquireCluster(pMnode, &pIter);
   if (pCluster != NULL) {
     createTime = pCluster->createdTime;
-    mndReleaseCluster(pMnode, pCluster);
+    mndReleaseCluster(pMnode, pCluster, pIter);
   }
 
   return createTime;
@@ -121,10 +126,11 @@ static int32_t mndGetClusterUpTimeImp(SClusterObj *pCluster) {
 
 float mndGetClusterUpTime(SMnode *pMnode) {
   int64_t      upTime = 0;
-  SClusterObj *pCluster = mndAcquireCluster(pMnode);
+  void        *pIter = NULL;
+  SClusterObj *pCluster = mndAcquireCluster(pMnode, &pIter);
   if (pCluster != NULL) {
     upTime = mndGetClusterUpTimeImp(pCluster);
-    mndReleaseCluster(pMnode, pCluster);
+    mndReleaseCluster(pMnode, pCluster, pIter);
   }
 
   return upTime / 86400.0f;
@@ -321,11 +327,12 @@ static void mndCancelGetNextCluster(SMnode *pMnode, void *pIter) {
 static int32_t mndProcessUptimeTimer(SRpcMsg *pReq) {
   SMnode      *pMnode = pReq->info.node;
   SClusterObj  clusterObj = {0};
-  SClusterObj *pCluster = mndAcquireCluster(pMnode);
+  void        *pIter = NULL;
+  SClusterObj *pCluster = mndAcquireCluster(pMnode, &pIter);
   if (pCluster != NULL) {
     memcpy(&clusterObj, pCluster, sizeof(SClusterObj));
     clusterObj.upTime += tsUptimeInterval;
-    mndReleaseCluster(pMnode, pCluster);
+    mndReleaseCluster(pMnode, pCluster, pIter);
   }
 
   if (clusterObj.id <= 0) {

source/dnode/vnode/src/tsdb/tsdbCache.c

@@ -35,7 +35,11 @@ _err:
 static void tsdbCloseBICache(STsdb *pTsdb) {
   SLRUCache *pCache = pTsdb->biCache;
   if (pCache) {
+    int32_t elems = taosLRUCacheGetElems(pCache);
+    tsdbTrace("vgId:%d, elems: %d", TD_VID(pTsdb->pVnode), elems);
     taosLRUCacheEraseUnrefEntries(pCache);
+    elems = taosLRUCacheGetElems(pCache);
+    tsdbTrace("vgId:%d, elems: %d", TD_VID(pTsdb->pVnode), elems);
 
     taosLRUCacheCleanup(pCache);
 
@@ -819,7 +823,12 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie
        * &state->blockIdx);
        */
       state->pBlockIdx = taosArraySearch(state->aBlockIdx, state->pBlockIdxExp, tCmprBlockIdx, TD_EQ);
-      if (!state->pBlockIdx) { /*
+      if (!state->pBlockIdx) {
+        tsdbBICacheRelease(state->pTsdb->biCache, state->aBlockIdxHandle);
+
+        state->aBlockIdxHandle = NULL;
+        state->aBlockIdx = NULL;
+        /*
          tsdbDataFReaderClose(state->pDataFReader);
          *state->pDataFReader = NULL;
          resetLastBlockLoadInfo(state->pLoadInfo);*/
@@ -1936,6 +1945,7 @@ int32_t tsdbCacheGetBlockIdx(SLRUCache *pCache, SDataFReader *pFileReader, LRUHa
     taosThreadMutexUnlock(&pTsdb->biMutex);
   }
 
+  tsdbTrace("bi cache:%p, ref", pCache);
   *handle = h;
 
   return code;
@@ -1945,6 +1955,7 @@ int32_t tsdbBICacheRelease(SLRUCache *pCache, LRUHandle *h) {
   int32_t code = 0;
 
   taosLRUCacheRelease(pCache, h, false);
+  tsdbTrace("bi cache:%p, release", pCache);
 
   return code;
 }

source/dnode/vnode/src/tsdb/tsdbRead.c

@@ -2320,7 +2320,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo*
 
         tsdbRowMergerAdd(&merge, pRow, pSchema);
       } else {
-        STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid);
+        // STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid);
         code = tsdbRowMergerInit(&merge, NULL, pRow, pSchema);
         if (code != TSDB_CODE_SUCCESS) {
           return code;
@@ -2352,7 +2352,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo*
         tsdbRowMergerAdd(&merge, piRow, piSchema);
       } else {
         init = true;
-        STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid);
+        // STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid);
         code = tsdbRowMergerInit(&merge, pSchema, piRow, piSchema);
         if (code != TSDB_CODE_SUCCESS) {
           return code;

source/libs/executor/src/scanoperator.c

@@ -329,7 +329,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca
   } else if (*status == FUNC_DATA_REQUIRED_NOT_LOAD) {
     qDebug("%s data block skipped, brange:%" PRId64 "-%" PRId64 ", rows:%" PRId64 ", uid:%" PRIu64, GET_TASKID(pTaskInfo),
            pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows, pBlockInfo->id.uid);
-    doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo, 1);
+    doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo, pBlock->info.rows);
     pCost->skipBlocks += 1;
     tsdbReleaseDataBlock(pTableScanInfo->dataReader);
     return TSDB_CODE_SUCCESS;
@@ -340,7 +340,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca
     if (success) {  // failed to load the block sma data, data block statistics does not exist, load data block instead
       qDebug("%s data block SMA loaded, brange:%" PRId64 "-%" PRId64 ", rows:%" PRId64 , GET_TASKID(pTaskInfo),
              pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows);
-      doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo, 1);
+      doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo, pBlock->info.rows);
       tsdbReleaseDataBlock(pTableScanInfo->dataReader);
       return TSDB_CODE_SUCCESS;
     } else {

source/libs/parser/src/parTranslater.c

@@ -645,6 +645,10 @@ static bool isSelectStmt(SNode* pCurrStmt) {
   return NULL != pCurrStmt && QUERY_NODE_SELECT_STMT == nodeType(pCurrStmt);
 }
 
+static bool isDeleteStmt(SNode* pCurrStmt) {
+  return NULL != pCurrStmt && QUERY_NODE_DELETE_STMT == nodeType(pCurrStmt);
+}
+
 static bool isSetOperator(SNode* pCurrStmt) {
   return NULL != pCurrStmt && QUERY_NODE_SET_OPERATOR == nodeType(pCurrStmt);
 }
@@ -669,6 +673,9 @@ static uint8_t getPrecisionFromCurrStmt(SNode* pCurrStmt, uint8_t defaultVal) {
   if (NULL != pCurrStmt && QUERY_NODE_CREATE_STREAM_STMT == nodeType(pCurrStmt)) {
     return getPrecisionFromCurrStmt(((SCreateStreamStmt*)pCurrStmt)->pQuery, defaultVal);
   }
+  if (isDeleteStmt(pCurrStmt)) {
+    return ((SDeleteStmt*)pCurrStmt)->precision;
+  }
   return defaultVal;
 }
 
@@ -1235,6 +1242,10 @@ static int32_t calcTypeBytes(SDataType dt) {
 }
 
 static EDealRes translateValue(STranslateContext* pCxt, SValueNode* pVal) {
+  if (pVal->translate) {
+    return TSDB_CODE_SUCCESS;
+  }
+
   SDataType dt = pVal->node.resType;
   dt.bytes = calcTypeBytes(dt);
   return translateValueImpl(pCxt, pVal, dt, false);
@@ -1692,7 +1703,8 @@ static void setFuncClassification(SNode* pCurrStmt, SFunctionNode* pFunc) {
     pSelect->hasUniqueFunc = pSelect->hasUniqueFunc ? true : (FUNCTION_TYPE_UNIQUE == pFunc->funcType);
     pSelect->hasTailFunc = pSelect->hasTailFunc ? true : (FUNCTION_TYPE_TAIL == pFunc->funcType);
     pSelect->hasInterpFunc = pSelect->hasInterpFunc ? true : (FUNCTION_TYPE_INTERP == pFunc->funcType);
-    pSelect->hasInterpPseudoColFunc = pSelect->hasInterpPseudoColFunc ? true : fmIsInterpPseudoColumnFunc(pFunc->funcId);
+    pSelect->hasInterpPseudoColFunc =
+        pSelect->hasInterpPseudoColFunc ? true : fmIsInterpPseudoColumnFunc(pFunc->funcId);
     pSelect->hasLastRowFunc = pSelect->hasLastRowFunc ? true : (FUNCTION_TYPE_LAST_ROW == pFunc->funcType);
     pSelect->hasLastFunc = pSelect->hasLastFunc ? true : (FUNCTION_TYPE_LAST == pFunc->funcType);
     pSelect->hasTimeLineFunc = pSelect->hasTimeLineFunc ? true : fmIsTimelineFunc(pFunc->funcId);
@@ -3372,7 +3384,8 @@ static int32_t translateInterp(STranslateContext* pCxt, SSelectStmt* pSelect) {
       return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_INTERP_CLAUSE);
     }
     if (pSelect->hasInterpPseudoColFunc) {
-      return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC, "Has Interp pseudo column(s) but missing interp function");
+      return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC,
+                                     "Has Interp pseudo column(s) but missing interp function");
     }
     return TSDB_CODE_SUCCESS;
   }
@@ -3741,6 +3754,7 @@ static int32_t translateDelete(STranslateContext* pCxt, SDeleteStmt* pDelete) {
   pCxt->pCurrStmt = (SNode*)pDelete;
   int32_t code = translateFrom(pCxt, pDelete->pFromTable);
   if (TSDB_CODE_SUCCESS == code) {
+    pDelete->precision = ((STableNode*)pDelete->pFromTable)->precision;
     code = translateDeleteWhere(pCxt, pDelete);
   }
   pCxt->currClause = SQL_CLAUSE_SELECT;

tests/parallel_test/cases.task

@@ -89,7 +89,7 @@
 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqUpdateWithConsume.py -N 3 -n 3
 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqUpdate-multiCtb-snapshot0.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqUpdate-multiCtb-snapshot1.py
-# ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDelete-1ctb.py
+,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDelete-1ctb.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDelete-multiCtb.py -N 3 -n 3
 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDropStbCtb.py
 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDropNtb-snapshot0.py
@@ -783,6 +783,7 @@
 ,,y,script,./test.sh -f tsim/insert/query_multi_file.sim
 ,,y,script,./test.sh -f tsim/insert/tcp.sim
 ,,y,script,./test.sh -f tsim/insert/update0.sim
+,,y,script,./test.sh -f tsim/insert/delete0.sim
 ,,y,script,./test.sh -f tsim/insert/update1_sort_merge.sim
 ,,y,script,./test.sh -f tsim/insert/update2.sim
 ,,y,script,./test.sh -f tsim/parser/alter__for_community_version.sim

tests/script/tsim/insert/delete0.sim

+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/exec.sh -n dnode1 -s start
+sql connect
+
+print =============== create database with different precision
+sql create database d0 keep 365
+sql create database d1 keep 365 precision 'ms'
+sql create database d2 keep 365 precision 'us'
+sql create database d3 keep 365 precision 'ns'
+
+sql select * from information_schema.ins_databases
+if $rows != 6 then
+  return -1
+endi
+
+print $data00 $data01 $data02
+
+
+sql create table if not exists d0.stb (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int unsigned)
+sql create table if not exists d1.stb (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int unsigned)
+sql create table if not exists d2.stb (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int unsigned)
+sql create table if not exists d3.stb (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int unsigned)
+sql create table if not exists d0.ntb (ts timestamp, c1 int, c2 float, c3 double)
+sql create table if not exists d1.ntb (ts timestamp, c1 int, c2 float, c3 double)
+sql create table if not exists d2.ntb (ts timestamp, c1 int, c2 float, c3 double)
+sql create table if not exists d3.ntb (ts timestamp, c1 int, c2 float, c3 double)
+
+sql create table d0.ct1 using d0.stb tags(1000)
+sql create table d1.ct1 using d1.stb tags(1000)
+sql create table d2.ct1 using d2.stb tags(1000)
+sql create table d3.ct1 using d3.stb tags(1000)
+sql create table d0.ct2 using d0.stb tags(1000)
+sql create table d1.ct2 using d1.stb tags(1000)
+sql create table d2.ct2 using d2.stb tags(1000)
+sql create table d3.ct2 using d3.stb tags(1000)
+
+
+sql insert into d0.ct1 values(now+0s, 10, 2.0, 3.0)
+sql insert into d1.ct1 values(now+0s, 10, 2.0, 3.0)
+sql insert into d2.ct1 values(now+0s, 10, 2.0, 3.0)
+sql insert into d3.ct1 values(now+0s, 10, 2.0, 3.0)
+sql insert into d0.ct2 values(now+0s, 10, 2.0, 3.0)
+sql insert into d1.ct2 values(now+0s, 10, 2.0, 3.0)
+sql insert into d2.ct2 values(now+0s, 10, 2.0, 3.0)
+sql insert into d3.ct2 values(now+0s, 10, 2.0, 3.0)
+sql insert into d0.ntb values(now+0s, 10, 2.0, 3.0)
+sql insert into d1.ntb values(now+0s, 10, 2.0, 3.0)
+sql insert into d2.ntb values(now+0s, 10, 2.0, 3.0)
+sql insert into d3.ntb values(now+0s, 10, 2.0, 3.0)
+
+
+print =============== query data from super table
+sql select count(*) from d0.stb
+if $data00 != 2 then
+  return -1
+endi
+sql select count(*) from d1.stb
+if $data00 != 2 then
+  return -1
+endi
+sql select count(*) from d2.stb
+if $data00 != 2 then
+  return -1
+endi
+sql select count(*) from d3.stb
+if $data00 != 2 then
+  return -1
+endi
+
+print =============== delete from child table
+sql delete from d0.ct1 where ts < now()
+sql delete from d1.ct1 where ts < now()
+sql delete from d2.ct1 where ts < now()
+sql delete from d3.ct1 where ts < now()
+
+
+print =============== query data from super table
+sql select count(*) from d0.stb
+if $data00 != 1 then
+  return -1
+endi
+sql select count(*) from d1.stb
+if $data00 != 1 then
+  return -1
+endi
+sql select count(*) from d2.stb
+if $data00 != 1 then
+  return -1
+endi
+sql select count(*) from d3.stb
+if $data00 != 1 then
+  return -1
+endi
+print =============== query data from normal table
+sql select count(*) from d0.ntb
+if $data00 != 1 then
+  return -1
+endi
+sql select count(*) from d1.ntb
+if $data00 != 1 then
+  return -1
+endi
+sql select count(*) from d2.ntb
+if $data00 != 1 then
+  return -1
+endi
+sql select count(*) from d3.ntb
+if $data00 != 1 then
+  return -1
+endi
+
+print =============== delete from super table
+sql delete from d0.stb where ts < now()
+sql delete from d1.stb where ts < now()
+sql delete from d2.stb where ts < now()
+sql delete from d3.stb where ts < now()
+
+print =============== query data from super table
+sql select count(*) from d0.stb
+if $data00 != 0 then
+  return -1
+endi
+sql select count(*) from d1.stb
+if $data00 != 0 then
+  return -1
+endi
+sql select count(*) from d2.stb
+if $data00 != 0 then
+  return -1
+endi
+sql select count(*) from d3.stb
+if $data00 != 0 then
+  return -1
+endi
+
+print =============== delete from normal table
+sql delete from d0.ntb where ts < now()
+sql delete from d1.ntb where ts < now()
+sql delete from d2.ntb where ts < now()
+sql delete from d3.ntb where ts < now()
+
+print =============== query data from normal table
+sql select count(*) from d0.ntb
+if $data00 != 0 then
+  return -1
+endi
+sql select count(*) from d1.ntb
+if $data00 != 0 then
+  return -1
+endi
+sql select count(*) from d2.ntb
+if $data00 != 0 then
+  return -1
+endi
+sql select count(*) from d3.ntb
+if $data00 != 0 then
+  return -1
+endi
+
+system sh/exec.sh -n dnode1 -s stop -x SIGINT

tests/script/tsim/testsuit.sim

@@ -114,6 +114,7 @@ run tsim/insert/basic1.sim
 run tsim/insert/commit-merge0.sim
 run tsim/insert/basic0.sim
 run tsim/insert/update0.sim
+run tsim/insert/delete0.sim
 run tsim/insert/backquote.sim
 run tsim/insert/null.sim
 run tsim/catalog/alterInCurrent.sim

tests/system-test/2-query/max.py

@@ -20,8 +20,8 @@ class TDTestCase:
         intData = []
         floatData = []
         tdSql.execute(f'''create table {dbname}.stb(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 tinyint unsigned, col6 smallint unsigned,
-                    col7 int unsigned, col8 bigint unsigned, col9 float, col10 double, col11 bool, col12 binary(20), col13 nchar(20)) tags(loc nchar(20))''')
-        tdSql.execute(f"create table {dbname}.stb_1 using {dbname}.stb tags('beijing')")
+                    col7 int unsigned, col8 bigint unsigned, col9 float, col10 double, col11 bool, col12 binary(20), col13 nchar(20)) tags(t0 tinyint, t1 float, loc nchar(20))''')
+        tdSql.execute(f"create table {dbname}.stb_1 using {dbname}.stb tags(5, 5.5, 'beijing')")
         for i in range(self.rowNum):
             tdSql.execute(f"insert into {dbname}.stb_1 values(%d, %d, %d, %d, %d, %d, %d, %d, %d, %f, %f, %d, '{self.binary_str}%d', '{self.nchar_str}%d')"
                           % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1))
@@ -55,13 +55,20 @@ class TDTestCase:
         tdSql.checkData(0, 1, np.max(intData))
 
         tdSql.query(f"select ts, min(col9) from {dbname}.stb")
-        tdSql.checkRows(1)        
+        tdSql.checkRows(1)
         tdSql.checkData(0, 1, np.min(floatData))
 
         tdSql.query(f"select ts, min(col9) from {dbname}.stb_1")
-        tdSql.checkRows(1)        
+        tdSql.checkRows(1)
         tdSql.checkData(0, 1, np.min(floatData))
 
+        # check tags
+        tdSql.query(f"select max(t0) from {dbname}.stb")
+        tdSql.checkData(0,0,5)
+
+        tdSql.query(f"select max(t1) from {dbname}.stb")
+        tdSql.checkData(0,0,5.5)
+
     def max_check_ntb_base(self, dbname="db"):
         tdSql.prepare()
         intData = []