Merge branch 'master' into hotfix/sangshuduo/TD-5844-cmdline-parameters-align-for-master

src/client/inc/tscSubquery.h

@@ -50,6 +50,12 @@ void tscUnlockByThread(int64_t *lockedBy);
 
 int tsInsertInitialCheck(SSqlObj *pSql);
 
+void doCleanupSubqueries(SSqlObj *pSql, int32_t numOfSubs);
+
+void tscFreeRetrieveSup(SSqlObj *pSql);
+
+
+
 #ifdef __cplusplus
 }
 #endif

src/client/inc/tsclient.h

@@ -123,17 +123,15 @@ typedef struct {
   int32_t kvLen;    // len of SKVRow
 } SMemRowInfo;
 typedef struct {
-  uint8_t      memRowType;
-  uint8_t      compareStat;  // 0 unknown, 1 need compare, 2 no need
-  TDRowTLenT   dataRowInitLen;
+  uint8_t      memRowType;   // default is 0, that is SDataRow 
+  uint8_t      compareStat;  // 0 no need, 1 need compare
   TDRowTLenT   kvRowInitLen;
   SMemRowInfo *rowInfo;
 } SMemRowBuilder;
 
 typedef enum {
-  ROW_COMPARE_UNKNOWN = 0,
+  ROW_COMPARE_NO_NEED = 0,
   ROW_COMPARE_NEED = 1,
-  ROW_COMPARE_NO_NEED = 2,
 } ERowCompareStat;
 
 int tsParseTime(SStrToken *pToken, int64_t *time, char **next, char *error, int16_t timePrec);

src/client/src/tscParseInsert.c

@@ -51,20 +51,18 @@ int            initMemRowBuilder(SMemRowBuilder *pBuilder, uint32_t nRows, uint3
     }
   }
 
+  // default compareStat is  ROW_COMPARE_NO_NEED
   if (nBoundCols == 0) {  // file input
     pBuilder->memRowType = SMEM_ROW_DATA;
-    pBuilder->compareStat = ROW_COMPARE_NO_NEED;
     return TSDB_CODE_SUCCESS;
   } else {
     float boundRatio = ((float)nBoundCols / (float)nCols);
 
     if (boundRatio < KVRatioKV) {
       pBuilder->memRowType = SMEM_ROW_KV;
-      pBuilder->compareStat = ROW_COMPARE_NO_NEED;
       return TSDB_CODE_SUCCESS;
     } else if (boundRatio > KVRatioData) {
       pBuilder->memRowType = SMEM_ROW_DATA;
-      pBuilder->compareStat = ROW_COMPARE_NO_NEED;
       return TSDB_CODE_SUCCESS;
     }
     pBuilder->compareStat = ROW_COMPARE_NEED;
@@ -76,7 +74,6 @@ int            initMemRowBuilder(SMemRowBuilder *pBuilder, uint32_t nRows, uint3
     }
   }
 
-  pBuilder->dataRowInitLen = TD_MEM_ROW_DATA_HEAD_SIZE + allNullLen;
   pBuilder->kvRowInitLen = TD_MEM_ROW_KV_HEAD_SIZE + nBoundCols * sizeof(SColIdx);
 
   if (nRows > 0) {
@@ -86,7 +83,7 @@ int            initMemRowBuilder(SMemRowBuilder *pBuilder, uint32_t nRows, uint3
     }
 
     for (int i = 0; i < nRows; ++i) {
-      (pBuilder->rowInfo + i)->dataLen = pBuilder->dataRowInitLen;
+      (pBuilder->rowInfo + i)->dataLen = TD_MEM_ROW_DATA_HEAD_SIZE + allNullLen;
       (pBuilder->rowInfo + i)->kvLen = pBuilder->kvRowInitLen;
     }
   }
@@ -460,7 +457,7 @@ int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, int16_t timePrec, i
   STableMeta *        pTableMeta = pDataBlocks->pTableMeta;
   SSchema *           schema = tscGetTableSchema(pTableMeta);
   SMemRowBuilder *    pBuilder = &pDataBlocks->rowBuilder;
-  int32_t             dataLen = pBuilder->dataRowInitLen;
+  int32_t             dataLen = spd->allNullLen + TD_MEM_ROW_DATA_HEAD_SIZE;
   int32_t             kvLen = pBuilder->kvRowInitLen;
   bool                isParseBindParam = false;
 
@@ -809,13 +806,12 @@ int tscSortRemoveDataBlockDupRows(STableDataBlocks *dataBuf, SBlockKeyInfo *pBlk
   // allocate memory
   size_t nAlloc = nRows * sizeof(SBlockKeyTuple);
   if (pBlkKeyInfo->pKeyTuple == NULL || pBlkKeyInfo->maxBytesAlloc < nAlloc) {
-    size_t nRealAlloc = nAlloc + 10 * sizeof(SBlockKeyTuple);
-    char * tmp = trealloc(pBlkKeyInfo->pKeyTuple, nRealAlloc);
+    char *tmp = trealloc(pBlkKeyInfo->pKeyTuple, nAlloc);
     if (tmp == NULL) {
       return TSDB_CODE_TSC_OUT_OF_MEMORY;
     }
     pBlkKeyInfo->pKeyTuple = (SBlockKeyTuple *)tmp;
-    pBlkKeyInfo->maxBytesAlloc = (int32_t)nRealAlloc;
+    pBlkKeyInfo->maxBytesAlloc = (int32_t)nAlloc;
   }
   memset(pBlkKeyInfo->pKeyTuple, 0, nAlloc);
 
@@ -1697,7 +1693,7 @@ static void parseFileSendDataBlock(void *param, TAOS_RES *tres, int32_t numOfRow
   STableMeta *    pTableMeta = pTableMetaInfo->pTableMeta;
   STableComInfo   tinfo = tscGetTableInfo(pTableMeta);
 
-  SInsertStatementParam* pInsertParam = &pCmd->insertParam;
+  SInsertStatementParam *pInsertParam = &pCmd->insertParam;
   destroyTableNameList(pInsertParam);
 
   pInsertParam->pDataBlocks = tscDestroyBlockArrayList(pInsertParam->pDataBlocks);
@@ -1726,12 +1722,6 @@ static void parseFileSendDataBlock(void *param, TAOS_RES *tres, int32_t numOfRow
     goto _error;
   }
 
-  if (TSDB_CODE_SUCCESS !=
-      (ret = initMemRowBuilder(&pTableDataBlock->rowBuilder, 0, tinfo.numOfColumns, pTableDataBlock->numOfParams,
-                               pTableDataBlock->boundColumnInfo.allNullLen))) {
-    goto _error;
-  }
-
   while ((readLen = tgetline(&line, &n, fp)) != -1) {
     if (('\r' == line[readLen - 1]) || ('\n' == line[readLen - 1])) {
       line[--readLen] = 0;

src/client/src/tscSubquery.c

@@ -2034,17 +2034,14 @@ void tscHandleMasterJoinQuery(SSqlObj* pSql) {
   tscAsyncResultOnError(pSql);
 }
 
-static void doCleanupSubqueries(SSqlObj *pSql, int32_t numOfSubs) {
+void doCleanupSubqueries(SSqlObj *pSql, int32_t numOfSubs) {
   assert(numOfSubs <= pSql->subState.numOfSub && numOfSubs >= 0);
   
   for(int32_t i = 0; i < numOfSubs; ++i) {
     SSqlObj* pSub = pSql->pSubs[i];
     assert(pSub != NULL);
-    
-    SRetrieveSupport* pSupport = pSub->param;
-    
-    tfree(pSupport->localBuffer);
-    tfree(pSupport);
+
+    tscFreeRetrieveSup(pSub);
     
     taos_free_result(pSub);
   }
@@ -2562,7 +2559,7 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) {
   return TSDB_CODE_SUCCESS;
 }
 
-static void tscFreeRetrieveSup(SSqlObj *pSql) {
+void tscFreeRetrieveSup(SSqlObj *pSql) {
   SRetrieveSupport *trsupport = pSql->param;
 
   void* p = atomic_val_compare_exchange_ptr(&pSql->param, trsupport, 0);
@@ -2720,33 +2717,43 @@ void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numO
   if (!TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_JOIN_SEC_STAGE)) {
 
     int32_t code = pParentSql->res.code;
-    if ((code == TSDB_CODE_TDB_INVALID_TABLE_ID || code == TSDB_CODE_VND_INVALID_VGROUP_ID) && pParentSql->retry < pParentSql->maxRetry) {
-      // remove the cached tableMeta and vgroup id list, and then parse the sql again
-      SSqlCmd* pParentCmd = &pParentSql->cmd;
-      STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pParentCmd, 0);
-      tscRemoveTableMetaBuf(pTableMetaInfo, pParentSql->self);
+    SSqlObj *userSql = NULL;
+    if (pParentSql->param) {
+      userSql = ((SRetrieveSupport*)pParentSql->param)->pParentSql;
+    }
 
-      pParentCmd->pTableMetaMap = tscCleanupTableMetaMap(pParentCmd->pTableMetaMap);
-      pParentCmd->pTableMetaMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
+    if (userSql == NULL) {
+      userSql = pParentSql;
+    }
+
+    if ((code == TSDB_CODE_TDB_INVALID_TABLE_ID || code == TSDB_CODE_VND_INVALID_VGROUP_ID) && userSql->retry < userSql->maxRetry) {
+      if (userSql != pParentSql) {
+        tscFreeRetrieveSup(pParentSql);
+      }
 
-      pParentSql->res.code = TSDB_CODE_SUCCESS;
-      pParentSql->retry++;
+      tscFreeSubobj(userSql);      
+      tfree(userSql->pSubs);
 
-      tscDebug("0x%"PRIx64" retry parse sql and send query, prev error: %s, retry:%d", pParentSql->self,
-          tstrerror(code), pParentSql->retry);
+      userSql->res.code = TSDB_CODE_SUCCESS;
+      userSql->retry++;
 
-      code = tsParseSql(pParentSql, true);
+      tscDebug("0x%"PRIx64" retry parse sql and send query, prev error: %s, retry:%d", userSql->self,
+          tstrerror(code), userSql->retry);
+
+      tscResetSqlCmd(&userSql->cmd, true);
+      code = tsParseSql(userSql, true);
       if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
         return;
       }
 
       if (code != TSDB_CODE_SUCCESS) {
-        pParentSql->res.code = code;
-        tscAsyncResultOnError(pParentSql);
+        userSql->res.code = code;
+        tscAsyncResultOnError(userSql);
         return;
       }
 
-      executeQuery(pParentSql, pQueryInfo);
+      pQueryInfo = tscGetQueryInfo(&userSql->cmd);
+      executeQuery(userSql, pQueryInfo);
     } else {
       (*pParentSql->fp)(pParentSql->param, pParentSql, pParentSql->res.code);
     }

src/client/src/tscUtil.c

@@ -3767,8 +3767,7 @@ static void tscSubqueryCompleteCallback(void* param, TAOS_RES* tres, int code) {
     int32_t index = ps->subqueryIndex;
     bool ret = subAndCheckDone(pSql, pParentSql, index);
 
-    tfree(ps);
-    pSql->param = NULL;
+    tscFreeRetrieveSup(pSql);
 
     if (!ret) {
       tscDebug("0x%"PRIx64" sub:0x%"PRIx64" orderOfSub:%d completed, not all subquery finished", pParentSql->self, pSql->self, index);
@@ -3778,12 +3777,13 @@ static void tscSubqueryCompleteCallback(void* param, TAOS_RES* tres, int code) {
     // todo refactor
     tscDebug("0x%"PRIx64" all subquery response received, retry", pParentSql->self);
 
-    SSqlCmd* pParentCmd = &pParentSql->cmd;
-    STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pParentCmd, 0);
-    tscRemoveTableMetaBuf(pTableMetaInfo, pParentSql->self);
+    if (code && !((code == TSDB_CODE_TDB_INVALID_TABLE_ID || code == TSDB_CODE_VND_INVALID_VGROUP_ID) && pParentSql->retry < pParentSql->maxRetry)) {
+      tscAsyncResultOnError(pParentSql);
+      return;
+    }
 
-    pParentCmd->pTableMetaMap = tscCleanupTableMetaMap(pParentCmd->pTableMetaMap);
-    pParentCmd->pTableMetaMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
+    tscFreeSubobj(pParentSql);    
+    tfree(pParentSql->pSubs);
 
     pParentSql->res.code = TSDB_CODE_SUCCESS;
     pParentSql->retry++;
@@ -3791,6 +3791,9 @@ static void tscSubqueryCompleteCallback(void* param, TAOS_RES* tres, int code) {
     tscDebug("0x%"PRIx64" retry parse sql and send query, prev error: %s, retry:%d", pParentSql->self,
              tstrerror(code), pParentSql->retry);
 
+    
+    tscResetSqlCmd(&pParentSql->cmd, true);
+    
     code = tsParseSql(pParentSql, true);
     if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
       return;
@@ -3802,7 +3805,8 @@ static void tscSubqueryCompleteCallback(void* param, TAOS_RES* tres, int code) {
       return;
     }
 
-    SQueryInfo *pQueryInfo = tscGetQueryInfo(pParentCmd);
+    SQueryInfo *pQueryInfo = tscGetQueryInfo(&pParentSql->cmd);
+
     executeQuery(pParentSql, pQueryInfo);
     return;
   }
@@ -3825,9 +3829,11 @@ void executeQuery(SSqlObj* pSql, SQueryInfo* pQueryInfo) {
   }
 
   if (taosArrayGetSize(pQueryInfo->pUpstream) > 0) {  // nest query. do execute it firstly
+    assert(pSql->subState.numOfSub == 0);
     pSql->subState.numOfSub = (int32_t) taosArrayGetSize(pQueryInfo->pUpstream);
-
+    assert(pSql->pSubs == NULL);
     pSql->pSubs = calloc(pSql->subState.numOfSub, POINTER_BYTES);
+    assert(pSql->subState.states == NULL);
     pSql->subState.states = calloc(pSql->subState.numOfSub, sizeof(int8_t));
     code = pthread_mutex_init(&pSql->subState.mutex, NULL);
 

src/common/inc/tglobal.h

@@ -158,6 +158,7 @@ extern char    tsDataDir[];
 extern char    tsLogDir[];
 extern char    tsScriptDir[];
 extern int64_t tsTickPerDay[3];
+extern int32_t tsTopicBianryLen;
 
 // system info
 extern char    tsOsName[];

src/common/src/tglobal.c

@@ -199,6 +199,7 @@ char   tsScriptDir[PATH_MAX] = {0};
 char   tsTempDir[PATH_MAX] = "/tmp/";
 
 int32_t  tsDiskCfgNum = 0;
+int32_t tsTopicBianryLen = 16000;
 
 #ifndef _STORAGE
 SDiskCfg tsDiskCfg[1];
@@ -1216,6 +1217,16 @@ static void doInitGlobalConfig(void) {
   cfg.unitType = TAOS_CFG_UTYPE_NONE;
   taosInitConfigOption(cfg);
 
+  cfg.option = "topicBianryLen";
+  cfg.ptr = &tsTopicBianryLen;
+  cfg.valType = TAOS_CFG_VTYPE_INT32;
+  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG;
+  cfg.minValue = 16;
+  cfg.maxValue = 16000;
+  cfg.ptrLength = 0;
+  cfg.unitType = TAOS_CFG_UTYPE_NONE;
+  taosInitConfigOption(cfg);
+
   cfg.option = "httpEnableRecordSql";
   cfg.ptr = &tsHttpEnableRecordSql;
   cfg.valType = TAOS_CFG_VTYPE_INT8;

src/common/src/ttypes.c

@@ -38,11 +38,7 @@ const int32_t TYPE_BYTES[15] = {
 
 #define DO_STATICS(__sum, __min, __max, __minIndex, __maxIndex, _list, _index) \
   do {                                                                         \
-    if (_list[(_index)] >= (INT64_MAX - (__sum))) {                            \
-      __sum = INT64_MAX;                                                       \
-    } else {                                                                   \
-      (__sum) += (_list)[(_index)];                                            \
-    }                                                                          \
+    (__sum) += (_list)[(_index)];                                              \
     if ((__min) > (_list)[(_index)]) {                                         \
       (__min) = (_list)[(_index)];                                             \
       (__minIndex) = (_index);                                                 \

src/kit/taosdemo/taosdemo.c

@@ -752,12 +752,13 @@ static void printHelp() {
             "Query mode -- 0: SYNC, 1: ASYNC. Default is SYNC.");
     printf("%s%s%s%s\n", indent, "-b", indent,
             "The data_type of columns, default: FLOAT, INT, FLOAT.");
-    printf("%s%s%s%s\n", indent, "-w", indent,
-            "The length of data_type 'BINARY' or 'NCHAR'. Default is 16");
+    printf("%s%s%s%s%d\n", indent, "-w", indent,
+            "The length of data_type 'BINARY' or 'NCHAR'. Default is ",
+            g_args.len_of_binary);
     printf("%s%s%s%s%d%s%d\n", indent, "-l", indent,
-            "The number of columns per record. Default is ",
+            "The number of columns per record. Demo mode by default is ",
             DEFAULT_DATATYPE_NUM,
-            ". Max values is ",
+            " (float, int, float). Max values is ",
             MAX_NUM_COLUMNS);
     printf("%s%s%s%s\n", indent, indent, indent,
             "All of the new column(s) type is INT. If use -b to specify column type, -l will be ignored.");

src/tsdb/src/tsdbMeta.c

@@ -1250,8 +1250,14 @@ static int tsdbEncodeTable(void **buf, STable *pTable) {
     tlen += taosEncodeFixedU64(buf, TABLE_SUID(pTable));
     tlen += tdEncodeKVRow(buf, pTable->tagVal);
   } else {
-    tlen += taosEncodeFixedU8(buf, (uint8_t)taosArrayGetSize(pTable->schema));
-    for (int i = 0; i < taosArrayGetSize(pTable->schema); i++) {
+    uint32_t arraySize = (uint32_t)taosArrayGetSize(pTable->schema); 
+    if(arraySize > UINT8_MAX) {
+      tlen += taosEncodeFixedU8(buf, 0);
+      tlen += taosEncodeFixedU32(buf, arraySize);
+    } else {
+      tlen += taosEncodeFixedU8(buf, (uint8_t)arraySize);
+    }
+    for (uint32_t i = 0; i < arraySize; i++) {
       STSchema *pSchema = taosArrayGetP(pTable->schema, i);
       tlen += tdEncodeSchema(buf, pSchema);
     }
@@ -1284,8 +1290,11 @@ static void *tsdbDecodeTable(void *buf, STable **pRTable) {
     buf = taosDecodeFixedU64(buf, &TABLE_SUID(pTable));
     buf = tdDecodeKVRow(buf, &(pTable->tagVal));
   } else {
-    uint8_t nSchemas;
-    buf = taosDecodeFixedU8(buf, &nSchemas);
+    uint32_t nSchemas = 0;
+    buf = taosDecodeFixedU8(buf, (uint8_t *)&nSchemas);
+    if(nSchemas == 0) {
+      buf = taosDecodeFixedU32(buf, &nSchemas);
+    }
     for (int i = 0; i < nSchemas; i++) {
       STSchema *pSchema;
       buf = tdDecodeSchema(buf, &pSchema);
@@ -1485,4 +1494,4 @@ static void tsdbFreeTableSchema(STable *pTable) {
 
     taosArrayDestroy(pTable->schema);
   }
-}
\ No newline at end of file
+}

tests/script/general/parser/function.sim

@@ -313,6 +313,12 @@ if $rows != 6 then
   return -1
 endi
 
+print =============================> TD-6086
+sql create stable td6086st(ts timestamp, d double) tags(t nchar(50));
+sql create table td6086ct1 using td6086st tags("ct1");
+sql create table td6086ct2 using td6086st tags("ct2");
+sql SELECT LAST(d),t FROM td6086st WHERE tbname in ('td6086ct1', 'td6086ct2') and ts>="2019-07-30 00:00:00" and ts<="2021-08-31 00:00:00" interval(1800s) fill(prev) GROUP BY tbname;
+
 print ==================> td-2624
 sql create table tm2(ts timestamp, k int, b binary(12));
 sql insert into tm2 values('2011-01-02 18:42:45.326',      -1,'abc');