Merge branch 'main' into fix/TS-2891-M

cmake/taostools_CMakeLists.txt.in

@@ -2,7 +2,7 @@
 # taos-tools
 ExternalProject_Add(taos-tools
         GIT_REPOSITORY https://github.com/taosdata/taos-tools.git
-        GIT_TAG ea02029
+        GIT_TAG d11f210
         SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools"
         BINARY_DIR ""
         #BUILD_IN_SOURCE TRUE

include/libs/qworker/qworker.h

@@ -106,6 +106,8 @@ int32_t qWorkerProcessLocalQuery(void *pMgmt, uint64_t sId, uint64_t qId, uint64
 int32_t qWorkerProcessLocalFetch(void *pMgmt, uint64_t sId, uint64_t qId, uint64_t tId, int64_t rId, int32_t eId,
                                  void **pRsp, SArray *explainRes);
 
+int32_t qWorkerDbgEnableDebug(char *option);
+
 #ifdef __cplusplus
 }
 #endif

include/util/tdef.h

@@ -316,10 +316,10 @@ typedef enum ELogicConditionType {
 #define TSDB_MAX_KEEP_NS                (365 * 292 * 1440)  // data in db to be reserved.
 #define TSDB_DEFAULT_KEEP               (3650 * 1440)       // ten years
 #define TSDB_MIN_MINROWS_FBLOCK         10
-#define TSDB_MAX_MINROWS_FBLOCK         1000
+#define TSDB_MAX_MINROWS_FBLOCK         1000000
 #define TSDB_DEFAULT_MINROWS_FBLOCK     100
 #define TSDB_MIN_MAXROWS_FBLOCK         200
-#define TSDB_MAX_MAXROWS_FBLOCK         10000
+#define TSDB_MAX_MAXROWS_FBLOCK         10000000
 #define TSDB_DEFAULT_MAXROWS_FBLOCK     4096
 #define TSDB_MIN_FSYNC_PERIOD           0
 #define TSDB_MAX_FSYNC_PERIOD           180000  // millisecond

source/client/src/clientJniConnector.c

@@ -581,8 +581,8 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_fetchBlockImp(JNI
     return JNI_RESULT_SET_NULL;
   }
 
-  void   *data;
-  int32_t numOfRows;
+  void   *data = NULL;
+  int32_t numOfRows = 0;
   int     error_code = taos_fetch_raw_block(tres, &numOfRows, &data);
   if (numOfRows == 0) {
     if (error_code == JNI_SUCCESS) {

source/client/src/clientMain.c

@@ -611,6 +611,9 @@ int taos_fetch_block_s(TAOS_RES *res, int *numOfRows, TAOS_ROW *rows) {
 }
 
 int taos_fetch_raw_block(TAOS_RES *res, int *numOfRows, void **pData) {
+  *numOfRows = 0;
+  *pData = NULL;
+
   if (res == NULL || TD_RES_TMQ_META(res)) {
     return 0;
   }

source/client/src/clientTmqConnector.c

@@ -361,8 +361,8 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_fetchRawBlockImp(
 
   TAOS_RES *tres = (TAOS_RES *)res;
 
-  void   *data;
-  int32_t numOfRows;
+  void   *data = NULL;
+  int32_t numOfRows = 0;
   int     error_code = taos_fetch_raw_block(tres, &numOfRows, &data);
   if (numOfRows == 0) {
     if (error_code == JNI_SUCCESS) {

source/dnode/vnode/src/tsdb/tsdbCache.c

@@ -600,6 +600,7 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow, bool *pIgnoreEa
                                     int nCols) {
   SFSLastNextRowIter *state = (SFSLastNextRowIter *)iter;
   int32_t             code = 0;
+  bool                checkRemainingRow = true;
 
   switch (state->state) {
     case SFSLASTNEXTROW_FS:
@@ -632,6 +633,8 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow, bool *pIgnoreEa
         if (code) goto _err;
       }
 
+      state->pLoadInfo->colIds = aCols;
+      state->pLoadInfo->numOfCols = nCols;
       tMergeTreeOpen(&state->mergeTree, 1, *state->pDataFReader, state->suid, state->uid,
                      &(STimeWindow){.skey = state->lastTs, .ekey = TSKEY_MAX},
                      &(SVersionRange){.minVer = 0, .maxVer = UINT64_MAX}, state->pLoadInfo, false, NULL, true);
@@ -647,22 +650,71 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow, bool *pIgnoreEa
         goto _next_fileset;
       }
       state->state = SFSLASTNEXTROW_BLOCKROW;
+      checkRemainingRow = false;
     }
     case SFSLASTNEXTROW_BLOCKROW: {
-      bool hasVal = false;
-      state->row = tMergeTreeGetRow(&state->mergeTree);
-      *ppRow = &state->row;
-      hasVal = tMergeTreeNext(&state->mergeTree);
-      if (TSDBROW_TS(&state->row) <= state->lastTs) {
-        *pIgnoreEarlierTs = true;
-        *ppRow = NULL;
-        return code;
-      }
+      bool skipRow = false;
+      do {
+        bool hasVal = false;
+        state->row = tMergeTreeGetRow(&state->mergeTree);
+        *ppRow = &state->row;
+        hasVal = tMergeTreeNext(&state->mergeTree);
+        if (TSDBROW_TS(&state->row) <= state->lastTs) {
+          *pIgnoreEarlierTs = true;
+          *ppRow = NULL;
+          return code;
+        }
 
-      *pIgnoreEarlierTs = false;
-      if (!hasVal) {
-        state->state = SFSLASTNEXTROW_FILESET;
-      }
+        *pIgnoreEarlierTs = false;
+        if (!hasVal) {
+          state->state = SFSLASTNEXTROW_FILESET;
+          break;
+        }
+
+        if (checkRemainingRow) {
+          bool skipBlock = true;
+
+          SBlockData *pBlockData = state->row.pBlockData;
+
+          for (int inputColIndex = 0; inputColIndex < nCols; ++inputColIndex) {
+            for (int colIndex = 0; colIndex < pBlockData->nColData; ++colIndex) {
+              SColData *pColData = &pBlockData->aColData[colIndex];
+              int16_t   cid = pColData->cid;
+
+              if (cid == aCols[inputColIndex]) {
+                if (isLast && (pColData->flag & HAS_VALUE)) {
+                  skipBlock = false;
+                  break;
+                } else if (pColData->flag & (HAS_VALUE | HAS_NULL)) {
+                  skipBlock = false;
+                  break;
+                }
+              }
+            }
+          }
+          /*
+          for (int colIndex = 0; colIndex < pBlockData->nColData; ++colIndex) {
+            SColData *pColData = &pBlockData->aColData[colIndex];
+            int16_t   cid = pColData->cid;
+
+            if (inputColIndex < nCols && cid == aCols[inputColIndex]) {
+              if (isLast && (pColData->flag & HAS_VALUE)) {
+                skipBlock = false;
+                break;
+              } else if (pColData->flag & (HAS_VALUE | HAS_NULL)) {
+                skipBlock = false;
+                break;
+              }
+
+              ++inputColIndex;
+            }
+          }
+          */
+          if (skipBlock) {
+            skipRow = true;
+          }
+        }
+      } while (skipRow);
 
       return code;
     }
@@ -908,10 +960,10 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie
           int16_t   cid = pColData->cid;
 
           if (inputColIndex < nCols && cid == aCols[inputColIndex]) {
-            if (isLast && pColData->numOfValue != 0) {
+            if (isLast && (pColData->flag & HAS_VALUE)) {
               skipBlock = false;
               break;
-            } else if (pColData->numOfNone != pColData->nVal) {
+            } else if (pColData->flag & (HAS_VALUE | HAS_NULL)) {
               skipBlock = false;
               break;
             }

source/dnode/vnode/src/tsdb/tsdbCacheRead.c

@@ -118,7 +118,6 @@ static int32_t setTableSchema(SCacheRowsReader* p, uint64_t suid, const char* id
   if (suid != 0) {
     p->pSchema = metaGetTbTSchema(p->pVnode->pMeta, suid, -1, 1);
     if (p->pSchema == NULL) {
-      taosMemoryFree(p);
       tsdbWarn("stable:%" PRIu64 " has been dropped, failed to retrieve cached rows, %s", suid, idstr);
       return TSDB_CODE_PAR_TABLE_NOT_EXIST;
     }

source/dnode/vnode/src/tsdb/tsdbRead.c

@@ -390,8 +390,10 @@ static SHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf
 
   pUidList->tableUidList = taosMemoryMalloc(numOfTables * sizeof(uint64_t));
   if (pUidList->tableUidList == NULL) {
+    taosHashCleanup(pTableMap);
     return NULL;
   }
+
   pUidList->currentIndex = 0;
 
   for (int32_t j = 0; j < numOfTables; ++j) {
@@ -4763,7 +4765,7 @@ int32_t tsdbGetFileBlocksDistInfo(STsdbReader* pReader, STableBlockDistInfo* pTa
   pTableBlockInfo->defMinRows = pc->minRows;
   pTableBlockInfo->defMaxRows = pc->maxRows;
 
-  int32_t bucketRange = ceil((pc->maxRows - pc->minRows) / numOfBucket);
+  int32_t bucketRange = ceil(((double)(pc->maxRows - pc->minRows)) / numOfBucket);
 
   pTableBlockInfo->numOfFiles += 1;
 

source/libs/catalog/src/ctgDbg.c

@@ -21,6 +21,8 @@
 extern SCatalogMgmt gCtgMgmt;
 SCtgDebug           gCTGDebug = {0};
 
+#if 0
+
 void ctgdUserCallback(SMetaData *pResult, void *param, int32_t code) {
   taosMemoryFree(param);
 
@@ -224,6 +226,7 @@ _return:
 
   CTG_RET(code);
 }
+#endif
 
 int32_t ctgdEnableDebug(char *option, bool enable) {
   if (0 == strcasecmp(option, "lock")) {

source/libs/catalog/src/ctgUtil.c

@@ -1330,6 +1330,7 @@ static void* ctgCloneDnodeList(void* pSrc) { return taosArrayDup((const SArray*)
 
 static void ctgFreeDnodeList(void* p) { taosArrayDestroy((SArray*)((SMetaRes*)p)->pRes); }
 
+#if 0
 static int32_t ctgCloneMetaDataArray(SArray* pSrc, __array_item_dup_fn_t copyFunc, SArray** pDst) {
   if (NULL == pSrc) {
     return TSDB_CODE_SUCCESS;
@@ -1421,3 +1422,5 @@ void catalogFreeMetaData(SMetaData* pData) {
   taosMemoryFreeClear(pData->pSvrVer);
   taosMemoryFree(pData);
 }
+#endif
+

source/libs/executor/src/executil.c

@@ -820,6 +820,7 @@ static SSDataBlock* createTagValBlockForFilter(SArray* pColList, int32_t numOfTa
   int32_t code = blockDataEnsureCapacity(pResBlock, numOfTables);
   if (code != TSDB_CODE_SUCCESS) {
     terrno = code;
+    taosMemoryFree(pResBlock);
     return NULL;
   }
 

source/libs/executor/src/executorimpl.c

@@ -2344,6 +2344,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
     pOptr = createEventwindowOperatorInfo(ops[0], pPhyNode, pTaskInfo);
   } else {
     terrno = TSDB_CODE_INVALID_PARA;
+    taosMemoryFree(ops);
     return NULL;
   }
 

source/libs/executor/src/groupoperator.c

@@ -173,9 +173,14 @@ static void recordNewGroupKeys(SArray* pGroupCols, SArray* pGroupColVals, SSData
   size_t numOfGroupCols = taosArrayGetSize(pGroupCols);
 
   for (int32_t i = 0; i < numOfGroupCols; ++i) {
-    SColumn*         pCol = taosArrayGet(pGroupCols, i);
+    SColumn*         pCol = (SColumn*) taosArrayGet(pGroupCols, i);
     SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pCol->slotId);
 
+    // valid range check. todo: return error code.
+    if (pCol->slotId > taosArrayGetSize(pBlock->pDataBlock)) {
+      continue;
+    }
+
     if (pBlock->pBlockAgg != NULL) {
       pColAgg = pBlock->pBlockAgg[pCol->slotId];  // TODO is agg data matched?
     }

source/libs/executor/src/tsort.c

@@ -214,7 +214,6 @@ static int32_t doAddToBuf(SSDataBlock* pDataBlock, SSortHandle* pHandle) {
     if (pPage == NULL) {
       taosArrayDestroy(pPageIdList);
       blockDataDestroy(p);
-      taosArrayDestroy(pPageIdList);
       return terrno;
     }
 

source/libs/qworker/src/qwDbg.c

@@ -271,7 +271,7 @@ void qwDbgSimulateDead(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *rsped) {
   }
 }
 
-int32_t qwDbgEnableDebug(char *option) {
+int32_t qWorkerDbgEnableDebug(char *option) {
   if (0 == strcasecmp(option, "lock")) {
     gQWDebug.lockEnable = true;
     qError("qw lock debug enabled");

tests/script/tsim/db/alter_option.sim

@@ -285,12 +285,12 @@ sql_error alter database db keep -1
 
 print ============== modify minrows
 sql_error alter database db minrows 8
-sql_error alter database db minrows 8000
-sql_error alter database db minrows 8001
+sql_error alter database db minrows 8000000
+sql_error alter database db minrows 8001000
 
 print ============== modify maxrows
-sql_error alter database db maxrows 1000
-sql_error alter database db maxrows 2000
+sql_error alter database db maxrows 10000001
+sql_error alter database db maxrows 20000000
 sql_error alter database db maxrows 11  # equal minrows
 sql_error alter database db maxrows 10  # little than minrows
 

tests/script/tsim/db/create_all_options.sim

@@ -298,11 +298,11 @@ sql drop database db
 sql_error create database db MAXROWS -1
 sql_error create database db MAXROWS 0
 sql_error create database db MAXROWS 199
-sql_error create database db MAXROWS 10001
+sql_error create database db MAXROWS 10000001
 sql_error create database db MINROWS -1
 sql_error create database db MINROWS 0
 sql_error create database db MINROWS 9
-sql_error create database db MINROWS 1001
+sql_error create database db MINROWS 1000001
 sql_error create database db MAXROWS 500 MINROWS 1000
 
 print ====> PRECISION ['ms' | 'us' | 'ns', default: ms]

tests/script/tsim/query/explain.sim

@@ -4,7 +4,7 @@ system sh/exec.sh -n dnode1 -s start
 sql connect
 
 print ======== step1
-sql create database db1 vgroups 3;
+sql create database db1 vgroups 3 cachesize 10 cachemodel 'both';
 sql use db1;
 sql select * from information_schema.ins_databases;
 sql create stable st1 (ts timestamp, f1 int, f2 binary(200)) tags(t1 int);
@@ -60,6 +60,15 @@ sql explain verbose true select * from information_schema.ins_stables where db_n
 sql explain verbose true select st1.f1 from st1 join st2 on st1.ts=st2.ts and st1.f1 > 0;
 sql explain verbose true insert into t1(ts) select st1.f1 from st1 join st2 on st1.ts=st2.ts and st1.f1 > 0;
 sql explain verbose true insert into t1(ts, t1) select _wstart, count(*) from st1 interval(10s);
+sql explain verbose true select distinct tbname, table_name from information_schema.ins_tables;
+sql explain verbose true select diff(f1) as f11 from tb1 order by f11;
+sql explain verbose true select count(*) from st1 where ts > now - 3m and ts < now interval(10s) fill(linear);
+sql explain verbose true select count(*) from st1 partition by tbname;
+sql explain verbose true select count(*) from information_schema.ins_tables group by stable_name;
+sql explain verbose true select last(*) from st1;
+sql explain verbose true select last_row(*) from st1;
+sql explain verbose true select interp(f1) from tb1 where ts > now - 3m and ts < now range(now-3m,now) every(1m) fill(prev);
+sql explain verbose true select _wstart, _wend, count(*) from tb1 EVENT_WINDOW start with f1 > 0 end with f1 < 10;
 
 print ======== step4
 sql explain analyze select ts from st1 where -2;
@@ -96,6 +105,13 @@ sql explain analyze verbose true select * from information_schema.ins_stables wh
 sql explain analyze verbose true select * from (select min(f1),count(*) a from st1 where f1 > 0) where a < 0;
 sql explain analyze verbose true select count(f1) from st1 group by tbname;
 sql explain analyze verbose true select st1.f1 from st1 join st2 on st1.ts=st2.ts and st1.f1 > 0;
+sql explain analyze verbose true select diff(f1) as f11 from tb1 order by f11;
+sql explain analyze verbose true select count(*) from st1 where ts > now - 3m and ts < now interval(10s) fill(linear);
+sql explain analyze verbose true select count(*) from information_schema.ins_tables group by stable_name;
+sql explain analyze verbose true select last(*) from st1;
+sql explain analyze verbose true select last_row(*) from st1;
+sql explain analyze verbose true select interp(f1) from tb1 where ts > now - 3m and ts < now range(now-3m,now) every(1m) fill(prev);
+sql explain analyze verbose true select _wstart, _wend, count(*) from tb1 EVENT_WINDOW start with f1 > 0 end with f1 < 10;
 
 #not pass case
 #sql explain verbose true select count(*),sum(f1) as aa from tb1 where (f1 > 0 or f1 < -1) and ts > '2020-10-31 00:00:00' and ts < '2021-10-31 00:00:00' order by aa;

tests/script/tsim/show/basic.sim

@@ -244,5 +244,15 @@ if $rows <= 0 then
   return -1
 endi
 
+sql show cluster alive;
+if $rows <= 0 then
+  return -1
+endi
+
+sql show db.alive;
+if $rows <= 0 then
+  return -1
+endi
+
 system sh/exec.sh -n dnode1 -s stop -x SIGINT
 system sh/exec.sh -n dnode2 -s stop -x SIGINT