Merge remote-tracking branch 'origin/3.0' into enh/TD-19460

docs/en/12-taos-sql/21-node.md

@@ -40,25 +40,25 @@ ALTER ALL DNODES dnode_option
 
 dnode_option: {
     'resetLog'
-  | 'balance' value
-  | 'monitor' value
-  | 'debugFlag' value
-  | 'monDebugFlag' value
-  | 'vDebugFlag' value
-  | 'mDebugFlag' value
-  | 'cDebugFlag' value
-  | 'httpDebugFlag' value
-  | 'qDebugflag' value
-  | 'sdbDebugFlag' value
-  | 'uDebugFlag' value
-  | 'tsdbDebugFlag' value
-  | 'sDebugflag' value
-  | 'rpcDebugFlag' value
-  | 'dDebugFlag' value
-  | 'mqttDebugFlag' value
-  | 'wDebugFlag' value
-  | 'tmrDebugFlag' value
-  | 'cqDebugFlag' value
+  | 'balance' 'value'
+  | 'monitor' 'value'
+  | 'debugFlag' 'value'
+  | 'monDebugFlag' 'value'
+  | 'vDebugFlag' 'value'
+  | 'mDebugFlag' 'value'
+  | 'cDebugFlag' 'value'
+  | 'httpDebugFlag' 'value'
+  | 'qDebugflag' 'value'
+  | 'sdbDebugFlag' 'value'
+  | 'uDebugFlag' 'value'
+  | 'tsdbDebugFlag' 'value'
+  | 'sDebugflag' 'value'
+  | 'rpcDebugFlag' 'value'
+  | 'dDebugFlag' 'value'
+  | 'mqttDebugFlag' 'value'
+  | 'wDebugFlag' 'value'
+  | 'tmrDebugFlag' 'value'
+  | 'cqDebugFlag' 'value'
 }
 ```
 
@@ -127,11 +127,11 @@ ALTER LOCAL local_option
 
 local_option: {
     'resetLog'
-  | 'rpcDebugFlag' value
-  | 'tmrDebugFlag' value
-  | 'cDebugFlag' value
-  | 'uDebugFlag' value
-  | 'debugFlag' value
+  | 'rpcDebugFlag' 'value'
+  | 'tmrDebugFlag' 'value'
+  | 'cDebugFlag' 'value'
+  | 'uDebugFlag' 'value'
+  | 'debugFlag' 'value'
 }
 ```
 

docs/en/14-reference/02-rest-api/02-rest-api.mdx

@@ -67,7 +67,7 @@ The following return value results indicate that the verification passed.
 ## HTTP request URL format
 
 ```text
-http://<fqdn>:<port>/rest/sql/[db_name]
+http://<fqdn>:<port>/rest/sql/[db_name][?tz=timezone]
 ```
 
 Parameter Description:
@@ -75,6 +75,7 @@ Parameter Description:
 - fqnd: FQDN or IP address of any host in the cluster.
 - port: httpPort configuration item in the configuration file, default is 6041.
 - db_name: Optional parameter that specifies the default database name for the executed SQL command.
+- tz: Optional parameter that specifies the timezone of the returned time, following the IANA Time Zone rules, e.g. `America/New_York`.
 
 For example, `http://h1.taos.com:6041/rest/sql/test` is a URL to `h1.taos.com:6041` and sets the default database name to `test`.
 
@@ -97,13 +98,13 @@ The HTTP request's BODY is a complete SQL command, and the data table in the SQL
 Use `curl` to initiate an HTTP request with a custom authentication method, with the following syntax.
 
 ```bash
-curl -L -H "Authorization: Basic <TOKEN>" -d "<SQL>" <ip>:<PORT>/rest/sql/[db_name]
+curl -L -H "Authorization: Basic <TOKEN>" -d "<SQL>" <ip>:<PORT>/rest/sql/[db_name][?tz=timezone]
 ```
 
 or
 
 ```bash
-curl -L -u username:password -d "<SQL>" <ip>:<PORT>/rest/sql/[db_name]
+curl -L -u username:password -d "<SQL>" <ip>:<PORT>/rest/sql/[db_name][?tz=timezone]
 ```
 
 where `TOKEN` is the string after Base64 encoding of `{username}:{password}`, e.g. `root:taosdata` is encoded as `cm9vdDp0YW9zZGF0YQ==`..

docs/en/27-train-faq/01-faq.md

@@ -106,11 +106,11 @@ ALTER LOCAL local_option
  
 local_option: {
     'resetLog'
-  | 'rpcDebugFlag' value
-  | 'tmrDebugFlag' value
-  | 'cDebugFlag' value
-  | 'uDebugFlag' value
-  | 'debugFlag' value
+  | 'rpcDebugFlag' 'value'
+  | 'tmrDebugFlag' 'value'
+  | 'cDebugFlag' 'value'
+  | 'uDebugFlag' 'value'
+  | 'debugFlag' 'value'
 }
 ```
 

docs/zh/08-connector/02-rest-api.mdx

@@ -69,7 +69,7 @@ curl -L -H "Authorization: Basic cm9vdDp0YW9zZGF0YQ==" \
 ## HTTP 请求格式
 
 ```text
-http://<fqdn>:<port>/rest/sql/[db_name]
+http://<fqdn>:<port>/rest/sql/[db_name][?tz=timezone]
 ```
 
 参数说明：
@@ -77,6 +77,7 @@ http://<fqdn>:<port>/rest/sql/[db_name]
 - fqdn: 集群中的任一台主机 FQDN 或 IP 地址。
 - port: 配置文件中 httpPort 配置项，缺省为 6041。
 - db_name: 可选参数，指定本次所执行的 SQL 语句的默认数据库库名。
+- tz: 可选参数，指定返回时间的时区，遵照 IANA Time Zone 规则，如 `America/New_York`。
 
 例如：`http://h1.taos.com:6041/rest/sql/test` 是指向地址为 `h1.taos.com:6041` 的 URL，并将默认使用的数据库库名设置为 `test`。
 
@@ -99,13 +100,13 @@ HTTP 请求的 BODY 里就是一个完整的 SQL 语句，SQL 语句中的数据
 使用 `curl` 通过自定义身份认证方式来发起一个 HTTP Request，语法如下：
 
 ```bash
-curl -L -H "Authorization: Basic <TOKEN>" -d "<SQL>" <ip>:<PORT>/rest/sql/[db_name]
+curl -L -H "Authorization: Basic <TOKEN>" -d "<SQL>" <ip>:<PORT>/rest/sql/[db_name][?tz=timezone]
 ```
 
 或者，
 
 ```bash
-curl -L -u username:password -d "<SQL>" <ip>:<PORT>/rest/sql/[db_name]
+curl -L -u username:password -d "<SQL>" <ip>:<PORT>/rest/sql/[db_name][?tz=timezone]
 ```
 
 其中，`TOKEN` 为 `{username}:{password}` 经过 Base64 编码之后的字符串，例如 `root:taosdata` 编码后为 `cm9vdDp0YW9zZGF0YQ==`。

docs/zh/12-taos-sql/21-node.md

@@ -41,25 +41,25 @@ ALTER ALL DNODES dnode_option
 
 dnode_option: {
     'resetLog'
-  | 'balance' value
-  | 'monitor' value
-  | 'debugFlag' value
-  | 'monDebugFlag' value
-  | 'vDebugFlag' value
-  | 'mDebugFlag' value
-  | 'cDebugFlag' value
-  | 'httpDebugFlag' value
-  | 'qDebugflag' value
-  | 'sdbDebugFlag' value
-  | 'uDebugFlag' value
-  | 'tsdbDebugFlag' value
-  | 'sDebugflag' value
-  | 'rpcDebugFlag' value
-  | 'dDebugFlag' value
-  | 'mqttDebugFlag' value
-  | 'wDebugFlag' value
-  | 'tmrDebugFlag' value
-  | 'cqDebugFlag' value
+  | 'balance' 'value'
+  | 'monitor' 'value'
+  | 'debugFlag' 'value'
+  | 'monDebugFlag' 'value'
+  | 'vDebugFlag' 'value'
+  | 'mDebugFlag' 'value'
+  | 'cDebugFlag' 'value'
+  | 'httpDebugFlag' 'value'
+  | 'qDebugflag' 'value'
+  | 'sdbDebugFlag' 'value'
+  | 'uDebugFlag' 'value'
+  | 'tsdbDebugFlag' 'value'
+  | 'sDebugflag' 'value'
+  | 'rpcDebugFlag' 'value'
+  | 'dDebugFlag' 'value'
+  | 'mqttDebugFlag' 'value'
+  | 'wDebugFlag' 'value'
+  | 'tmrDebugFlag' 'value'
+  | 'cqDebugFlag' 'value'
 }
 ```
 
@@ -128,11 +128,11 @@ ALTER LOCAL local_option
 
 local_option: {
     'resetLog'
-  | 'rpcDebugFlag' value
-  | 'tmrDebugFlag' value
-  | 'cDebugFlag' value
-  | 'uDebugFlag' value
-  | 'debugFlag' value
+  | 'rpcDebugFlag' 'value'
+  | 'tmrDebugFlag' 'value'
+  | 'cDebugFlag' 'value'
+  | 'uDebugFlag' 'value'
+  | 'debugFlag' 'value'
 }
 ```
 

docs/zh/27-train-faq/01-faq.md

@@ -135,11 +135,11 @@ ALTER LOCAL local_option
  
 local_option: {
     'resetLog'
-  | 'rpcDebugFlag' value
-  | 'tmrDebugFlag' value
-  | 'cDebugFlag' value
-  | 'uDebugFlag' value
-  | 'debugFlag' value
+  | 'rpcDebugFlag' 'value'
+  | 'tmrDebugFlag' 'value'
+  | 'cDebugFlag' 'value'
+  | 'uDebugFlag' 'value'
+  | 'debugFlag' 'value'
 }
 ```
 

include/common/trow.h

@@ -169,7 +169,7 @@ typedef struct {
 // #define TD_ROW_VER(r)      ((r)->ver)
 #define TD_ROW_KEY_ADDR(r) (r)
 
-// N.B. If without STSchema, getExtendedRowSize() is used to get the rowMaxBytes and
+// N.B. If without STSchema, insGetExtendedRowSize() is used to get the rowMaxBytes and
 // (int32_t)ceil((double)nCols/TD_VTYPE_PARTS) should be added if TD_SUPPORT_BITMAP defined.
 #define TD_ROW_MAX_BYTES_FROM_SCHEMA(s) ((s)->tlen + TD_ROW_HEAD_LEN)
 

source/client/inc/clientInt.h

@@ -46,7 +46,6 @@ extern "C" {
 
 #define ERROR_MSG_BUF_DEFAULT_SIZE 512
 #define HEARTBEAT_INTERVAL         1500  // ms
-#define SYNC_ON_TOP_OF_ASYNC       1
 
 enum {
   RES_TYPE__QUERY = 1,
@@ -271,9 +270,8 @@ void    doFreeReqResultInfo(SReqResultInfo* pResInfo);
 int32_t transferTableNameList(const char* tbList, int32_t acctId, char* dbName, SArray** pReq);
 void    syncCatalogFn(SMetaData* pResult, void* param, int32_t code);
 
-SRequestObj* execQuery(uint64_t connId, const char* sql, int sqlLen, bool validateOnly);
-TAOS_RES*    taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly);
-void         taosAsyncQueryImpl(uint64_t connId, const char* sql, __taos_async_fn_t fp, void* param, bool validateOnly);
+TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly);
+void      taosAsyncQueryImpl(uint64_t connId, const char* sql, __taos_async_fn_t fp, void* param, bool validateOnly);
 
 int32_t getVersion1BlockMetaSize(const char* p, int32_t numOfCols);
 
@@ -349,8 +347,6 @@ void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet);
 STscObj* taos_connect_internal(const char* ip, const char* user, const char* pass, const char* auth, const char* db,
                                uint16_t port, int connType);
 
-SRequestObj* launchQuery(uint64_t connId, const char* sql, int sqlLen, bool validateOnly, bool inRetry);
-
 int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery, SStmtCallback* pStmtCb);
 
 int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArray* pNodeList);

source/client/src/clientImpl.c

@@ -1013,28 +1013,6 @@ SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, bool keepQue
   return pRequest;
 }
 
-SRequestObj* launchQuery(uint64_t connId, const char* sql, int sqlLen, bool validateOnly, bool inRetry) {
-  SRequestObj* pRequest = NULL;
-  SQuery*      pQuery = NULL;
-
-  int32_t code = buildRequest(connId, sql, sqlLen, NULL, validateOnly, &pRequest);
-  if (code != TSDB_CODE_SUCCESS) {
-    terrno = code;
-    return NULL;
-  }
-
-  code = parseSql(pRequest, false, &pQuery, NULL);
-  if (code != TSDB_CODE_SUCCESS) {
-    pRequest->code = code;
-    return pRequest;
-  }
-
-  pRequest->inRetry = inRetry;
-  pRequest->stableQuery = pQuery->stableQuery;
-
-  return launchQueryImpl(pRequest, pQuery, false, NULL);
-}
-
 static int32_t asyncExecSchQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultMeta,
                                  SSqlCallbackWrapper* pWrapper) {
   pRequest->type = pQuery->msgType;
@@ -1197,32 +1175,6 @@ int32_t removeMeta(STscObj* pTscObj, SArray* tbList) {
   return TSDB_CODE_SUCCESS;
 }
 
-//  todo remove it soon
-SRequestObj* execQuery(uint64_t connId, const char* sql, int sqlLen, bool validateOnly) {
-  SRequestObj* pRequest = NULL;
-  int32_t      retryNum = 0;
-  int32_t      code = 0;
-  bool         inRetry = false;
-
-  do {
-    destroyRequest(pRequest);
-    pRequest = launchQuery(connId, sql, sqlLen, validateOnly, inRetry);
-    if (pRequest == NULL || TSDB_CODE_SUCCESS == pRequest->code || !NEED_CLIENT_HANDLE_ERROR(pRequest->code)) {
-      break;
-    }
-
-    code = refreshMeta(pRequest->pTscObj, pRequest);
-    if (code) {
-      pRequest->code = code;
-      break;
-    }
-
-    inRetry = true;
-  } while (retryNum++ < REQUEST_TOTAL_EXEC_TIMES);
-
-  return pRequest;
-}
-
 int initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* pEpSet) {
   pEpSet->version = 0;
 
@@ -2291,7 +2243,6 @@ TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly) {
     return NULL;
   }
 
-#if SYNC_ON_TOP_OF_ASYNC
   SSyncQueryParam* param = taosMemoryCalloc(1, sizeof(SSyncQueryParam));
   tsem_init(&param->sem, 0, 0);
 
@@ -2301,15 +2252,4 @@ TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly) {
     param->pRequest->syncQuery = true;
   }
   return param->pRequest;
-#else
-  size_t sqlLen = strlen(sql);
-  if (sqlLen > (size_t)TSDB_MAX_ALLOWED_SQL_LEN) {
-    tscError("sql string exceeds max length:%d", TSDB_MAX_ALLOWED_SQL_LEN);
-    terrno = TSDB_CODE_TSC_EXCEED_SQL_LIMIT;
-    return NULL;
-  }
-
-  TAOS_RES* pRes = execQuery(*(int64_t*)taos, sql, sqlLen, validateOnly);
-  return pRes;
-#endif
 }

source/client/src/clientMain.c

@@ -261,12 +261,7 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) {
       return NULL;
     }
 
-#if SYNC_ON_TOP_OF_ASYNC
     return doAsyncFetchRows(pRequest, true, true);
-#else
-    return doFetchRows(pRequest, true, true);
-#endif
-
   } else if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res)) {
     SMqRspObj      *msg = ((SMqRspObj *)res);
     SReqResultInfo *pResultInfo;
@@ -549,11 +544,7 @@ int taos_fetch_block_s(TAOS_RES *res, int *numOfRows, TAOS_ROW *rows) {
       return 0;
     }
 
-#if SYNC_ON_TOP_OF_ASYNC
     doAsyncFetchRows(pRequest, false, true);
-#else
-    doFetchRows(pRequest, true, true);
-#endif
 
     // TODO refactor
     SReqResultInfo *pResultInfo = &pRequest->body.resInfo;
@@ -601,11 +592,7 @@ int taos_fetch_raw_block(TAOS_RES *res, int *numOfRows, void **pData) {
     return 0;
   }
 
-#if SYNC_ON_TOP_OF_ASYNC
   doAsyncFetchRows(pRequest, false, false);
-#else
-  doFetchRows(pRequest, false, false);
-#endif
 
   SReqResultInfo *pResultInfo = &pRequest->body.resInfo;
 
@@ -989,7 +976,7 @@ const void *taos_get_raw_block(TAOS_RES *res) {
   return pRequest->body.resInfo.pData;
 }
 
-int taos_get_db_route_info(TAOS* taos, const char* db, TAOS_DB_ROUTE_INFO* dbInfo) {
+int taos_get_db_route_info(TAOS *taos, const char *db, TAOS_DB_ROUTE_INFO *dbInfo) {
   if (NULL == taos) {
     terrno = TSDB_CODE_TSC_DISCONNECTED;
     return terrno;
@@ -1001,16 +988,16 @@ int taos_get_db_route_info(TAOS* taos, const char* db, TAOS_DB_ROUTE_INFO* dbInf
     return terrno;
   }
 
-  int64_t       connId = *(int64_t *)taos;
-  SRequestObj  *pRequest = NULL;
-  char *sql = "taos_get_db_route_info";
-  int32_t code = buildRequest(connId, sql, strlen(sql), NULL, false, &pRequest);
+  int64_t      connId = *(int64_t *)taos;
+  SRequestObj *pRequest = NULL;
+  char        *sql = "taos_get_db_route_info";
+  int32_t      code = buildRequest(connId, sql, strlen(sql), NULL, false, &pRequest);
   if (code != TSDB_CODE_SUCCESS) {
     terrno = code;
     return terrno;
   }
 
-  STscObj *pTscObj = pRequest->pTscObj;
+  STscObj  *pTscObj = pRequest->pTscObj;
   SCatalog *pCtg = NULL;
   code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCtg);
   if (code != TSDB_CODE_SUCCESS) {
@@ -1024,7 +1011,7 @@ int taos_get_db_route_info(TAOS* taos, const char* db, TAOS_DB_ROUTE_INFO* dbInf
 
   char dbFName[TSDB_DB_FNAME_LEN] = {0};
   snprintf(dbFName, sizeof(dbFName), "%d.%s", pTscObj->acctId, db);
-  
+
   code = catalogGetDBVgInfo(pCtg, &conn, dbFName, dbInfo);
   if (code) {
     goto _return;
@@ -1038,7 +1025,7 @@ _return:
   return code;
 }
 
-int taos_get_table_vgId(TAOS* taos, const char* db, const char* table, int* vgId) {
+int taos_get_table_vgId(TAOS *taos, const char *db, const char *table, int *vgId) {
   if (NULL == taos) {
     terrno = TSDB_CODE_TSC_DISCONNECTED;
     return terrno;
@@ -1050,15 +1037,15 @@ int taos_get_table_vgId(TAOS* taos, const char* db, const char* table, int* vgId
     return terrno;
   }
 
-  int64_t       connId = *(int64_t *)taos;
-  SRequestObj  *pRequest = NULL;
-  char *sql = "taos_get_table_vgId";
-  int32_t code = buildRequest(connId, sql, strlen(sql), NULL, false, &pRequest);
+  int64_t      connId = *(int64_t *)taos;
+  SRequestObj *pRequest = NULL;
+  char        *sql = "taos_get_table_vgId";
+  int32_t      code = buildRequest(connId, sql, strlen(sql), NULL, false, &pRequest);
   if (code != TSDB_CODE_SUCCESS) {
     return terrno;
   }
 
-  STscObj *pTscObj = pRequest->pTscObj;
+  STscObj  *pTscObj = pRequest->pTscObj;
   SCatalog *pCtg = NULL;
   code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCtg);
   if (code != TSDB_CODE_SUCCESS) {

source/dnode/mnode/impl/src/mndGrant.c

@@ -21,10 +21,8 @@
 
 static int32_t mndRetrieveGrant(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) {
   int32_t numOfRows = 0;
-  char   *pWrite;
   int32_t cols = 0;
   char    tmp[32];
-  char    tmp1[32];
 
   if (pShow->numOfRows < 1) {
     cols = 0;

source/dnode/vnode/src/tsdb/tsdbRead.c

@@ -1910,6 +1910,10 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo*
 
     if (minKey == k.ts) {
       if (init) {
+        if (merge.pTSchema == NULL) {
+          return code;
+        }
+
         tRowMerge(&merge, pRow);
       } else {
         STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid);
@@ -1964,7 +1968,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo*
         tRowMerge(&merge, &fRow1);
       } else {
         init = true;
-        int32_t code = tRowMergerInit(&merge, &fRow1, pReader->pSchema);
+        code = tRowMergerInit(&merge, &fRow1, pReader->pSchema);
         if (code != TSDB_CODE_SUCCESS) {
           return code;
         }
@@ -1975,17 +1979,24 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo*
     if (minKey == key) {
       TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex);
       if (!init) {
-        int32_t code = tRowMergerInit(&merge, &fRow, pReader->pSchema);
+        code = tRowMergerInit(&merge, &fRow, pReader->pSchema);
         if (code != TSDB_CODE_SUCCESS) {
           return code;
         }
       } else {
+        if (merge.pTSchema == NULL) {
+          return code;
+        }
         tRowMerge(&merge, &fRow);
       }
       doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge);
     }
   }
 
+  if (merge.pTSchema == NULL)  {
+    return code;
+  }
+
   code = tRowMergerGetRow(&merge, &pTSRow);
   if (code != TSDB_CODE_SUCCESS) {
     return code;
@@ -3232,7 +3243,7 @@ int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* p
     STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid);
 
     int32_t code = tRowMergerInit(&merge, pRow, pSchema);
-    if (code != TSDB_CODE_SUCCESS) {
+    if (code != TSDB_CODE_SUCCESS || merge.pTSchema == NULL) {
       return code;
     }
 

source/libs/executor/inc/executil.h

@@ -74,7 +74,6 @@ typedef struct SResultRowPosition {
 typedef struct SResKeyPos {
   SResultRowPosition pos;
   uint64_t           groupId;
-  // char               parTbName[TSDB_TABLE_NAME_LEN];
   char key[];
 } SResKeyPos;
 
@@ -84,6 +83,18 @@ typedef struct SResultRowInfo {
   SList*             openWindow;
 } SResultRowInfo;
 
+typedef struct SColMatchItem {
+  int32_t colId;
+  int32_t srcSlotId;
+  int32_t dstSlotId;
+  bool    needOutput;
+} SColMatchItem;
+
+typedef struct SColMatchInfo {
+  SArray* pList;      // SArray<SColMatchItem>
+  int32_t matchType;  // determinate the source according to col id or slot id
+} SColMatchInfo;
+
 struct SqlFunctionCtx;
 
 size_t getResultRowSize(struct SqlFunctionCtx* pCtx, int32_t numOfOutput);
@@ -121,8 +132,8 @@ size_t   getTableTagsBufLen(const SNodeList* pGroups);
 
 SArray* createSortInfo(SNodeList* pNodeList);
 SArray* extractPartitionColInfo(SNodeList* pNodeList);
-SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNodeList, int32_t* numOfOutputCols,
-                            int32_t type);
+int32_t extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNodeList, int32_t* numOfOutputCols,
+                            int32_t type, SColMatchInfo* pMatchInfo);
 
 void       createExprFromOneNode(SExprInfo* pExp, SNode* pNode, int16_t slotId);
 void       createExprFromTargetNode(SExprInfo* pExp, STargetNode* pTargetNode);

source/libs/executor/inc/executorimpl.h

@@ -47,12 +47,7 @@ extern "C" {
 
 typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int32_t order);
 
-#define IS_QUERY_KILLED(_q)   ((_q)->code == TSDB_CODE_TSC_QUERY_CANCELLED)
 #define Q_STATUS_EQUAL(p, s)  (((p) & (s)) != 0u)
-#define QUERY_IS_ASC_QUERY(q) (GET_FORWARD_DIRECTION_FACTOR((q)->order.order) == QUERY_ASC_FORWARD_STEP)
-
-#define NEEDTO_COMPRESS_QUERY(size) ((size) > tsCompressColData ? 1 : 0)
-
 #define IS_VALID_SESSION_WIN(winInfo)        ((winInfo).sessionWin.win.skey > 0)
 #define SET_SESSION_WIN_INVALID(winInfo)     ((winInfo).sessionWin.win.skey = INT64_MIN)
 #define IS_INVALID_SESSION_WIN_KEY(winKey)   ((winKey).win.skey <= 0)
@@ -240,7 +235,7 @@ typedef struct SOperatorInfo {
 
 typedef enum {
   EX_SOURCE_DATA_NOT_READY = 0x1,
-  EX_SOURCE_DATA_READY = 0x2,
+  EX_SOURCE_DATA_READY     = 0x2,
   EX_SOURCE_DATA_EXHAUSTED = 0x3,
 } EX_SOURCE_STATUS;
 
@@ -289,15 +284,6 @@ typedef struct SExchangeInfo {
   SLimitInfo          limitInfo;
 } SExchangeInfo;
 
-typedef struct SColMatchInfo {
-  int32_t srcSlotId;  // source slot id
-  int32_t colId;
-  int32_t targetSlotId;
-  bool    output;  // todo remove this?
-  bool    reserved;
-  int32_t matchType;  // determinate the source according to col id or slot id
-} SColMatchInfo;
-
 typedef struct SScanInfo {
   int32_t numOfAsc;
   int32_t numOfDesc;
@@ -339,7 +325,7 @@ typedef struct STableScanInfo {
   SNode*                 pFilterNode;  // filter info, which is push down by optimizer
 
   SSDataBlock*         pResBlock;
-  SArray*              pColMatchInfo;
+  SColMatchInfo        matchInfo;
   SExprSupp            pseudoSup;
   SQueryTableDataCond  cond;
   int32_t              scanFlag;  // table scan flag to denote if it is a repeat/reverse/main scan
@@ -365,10 +351,9 @@ typedef struct STableMergeScanInfo {
   uint32_t        sortBufSize;  // max buffer size for in-memory sort
   SArray*         pSortInfo;
   SSortHandle*    pSortHandle;
-
-  SSDataBlock* pSortInputBlock;
-  int64_t      startTs;  // sort start time
-  SArray*      sortSourceParams;
+  SSDataBlock*    pSortInputBlock;
+  int64_t         startTs;  // sort start time
+  SArray*         sortSourceParams;
 
   SFileBlockLoadRecorder readRecorder;
   int64_t                numOfRows;
@@ -380,43 +365,40 @@ typedef struct STableMergeScanInfo {
   int32_t*               rowEntryInfoOffset;
   SExprInfo*             pExpr;
   SSDataBlock*           pResBlock;
-  SArray*                pColMatchInfo;
+  SColMatchInfo          matchInfo;
   int32_t                numOfOutput;
-
-  SExprSupp pseudoSup;
-
-  SQueryTableDataCond cond;
-  int32_t             scanFlag;  // table scan flag to denote if it is a repeat/reverse/main scan
-  int32_t             dataBlockLoadFlag;
+  SExprSupp              pseudoSup;
+  SQueryTableDataCond    cond;
+  int32_t                scanFlag;  // table scan flag to denote if it is a repeat/reverse/main scan
+  int32_t                dataBlockLoadFlag;
   // if the upstream is an interval operator, the interval info is also kept here to get the time
   // window to check if current data block needs to be loaded.
-  SInterval       interval;
-  SSampleExecInfo sample;  // sample execution info
-
-  SSortExecInfo sortExecInfo;
+  SInterval              interval;
+  SSampleExecInfo        sample;  // sample execution info
+  SSortExecInfo          sortExecInfo;
 } STableMergeScanInfo;
 
 typedef struct STagScanInfo {
   SColumnInfo*    pCols;
   SSDataBlock*    pRes;
-  SArray*         pColMatchInfo;
+  SColMatchInfo   matchInfo;
   int32_t         curPos;
   SReadHandle     readHandle;
   STableListInfo* pTableList;
 } STagScanInfo;
 
 typedef struct SLastrowScanInfo {
-  SSDataBlock* pRes;
-  SReadHandle  readHandle;
-  void*        pLastrowReader;
-  SArray*      pColMatchInfo;
-  int32_t*     pSlotIds;
-  SExprSupp    pseudoExprSup;
-  int32_t      retrieveType;
-  int32_t      currentGroupIndex;
-  SSDataBlock* pBufferredRes;
-  SArray*      pUidList;
-  int32_t      indexOfBufferedRes;
+  SSDataBlock*   pRes;
+  SReadHandle    readHandle;
+  void*          pLastrowReader;
+  SColMatchInfo  matchInfo;
+  int32_t*       pSlotIds;
+  SExprSupp      pseudoExprSup;
+  int32_t        retrieveType;
+  int32_t        currentGroupIndex;
+  SSDataBlock*   pBufferredRes;
+  SArray*        pUidList;
+  int32_t        indexOfBufferedRes;
 } SLastrowScanInfo;
 
 typedef enum EStreamScanMode {
@@ -483,28 +465,28 @@ typedef struct STimeWindowAggSupp {
 } STimeWindowAggSupp;
 
 typedef struct SStreamScanInfo {
-  uint64_t    tableUid;  // queried super table uid
-  SExprInfo*  pPseudoExpr;
-  int32_t     numOfPseudoExpr;
-  SExprSupp   tbnameCalSup;
-  SExprSupp   tagCalSup;
-  int32_t     primaryTsIndex;  // primary time stamp slot id
-  SReadHandle readHandle;
-  SInterval   interval;       // if the upstream is an interval operator, the interval info is also kept here.
-  SArray*     pColMatchInfo;  //
-  SNode*      pCondition;
-
-  SArray*      pBlockLists;  // multiple SSDatablock.
-  SSDataBlock* pRes;         // result SSDataBlock
-  SSDataBlock* pUpdateRes;   // update SSDataBlock
-  int32_t      updateResIndex;
-  int32_t      blockType;        // current block type
-  int32_t      validBlockIndex;  // Is current data has returned?
-  uint64_t     numOfExec;        // execution times
-  STqReader*   tqReader;
-
-  uint64_t     groupId;
-  SUpdateInfo* pUpdateInfo;
+  uint64_t              tableUid;  // queried super table uid
+  SExprInfo*            pPseudoExpr;
+  int32_t               numOfPseudoExpr;
+  SExprSupp             tbnameCalSup;
+  SExprSupp             tagCalSup;
+  int32_t               primaryTsIndex;  // primary time stamp slot id
+  SReadHandle           readHandle;
+  SInterval             interval;       // if the upstream is an interval operator, the interval info is also kept here.
+  SColMatchInfo         matchInfo;
+  SNode*                pCondition;
+                        
+  SArray*               pBlockLists;  // multiple SSDatablock.
+  SSDataBlock*          pRes;         // result SSDataBlock
+  SSDataBlock*          pUpdateRes;   // update SSDataBlock
+  int32_t               updateResIndex;
+  int32_t               blockType;        // current block type
+  int32_t               validBlockIndex;  // Is current data has returned?
+  uint64_t              numOfExec;        // execution times
+  STqReader*            tqReader;
+
+  uint64_t              groupId;
+  SUpdateInfo*          pUpdateInfo;
 
   EStreamScanMode       scanMode;
   SOperatorInfo*        pStreamScanOp;
@@ -559,8 +541,8 @@ typedef struct SSysTableScanInfo {
   bool                   showRewrite;
   SNode*                 pCondition;  // db_name filter condition, to discard data that are not in current database
   SMTbCursor*            pCur;        // cursor for iterate the local table meta store.
-  SSysTableIndex*          pIdx;         // idx for local table meta 
-  SArray*                scanCols;    // SArray<int16_t> scan column id list
+  SSysTableIndex*        pIdx;         // idx for local table meta
+  SColMatchInfo          matchInfo;
   SName                  name;
   SSDataBlock*           pRes;
   int64_t                numOfBlocks;  // extract basic running information.
@@ -680,7 +662,7 @@ typedef struct SFillOperatorInfo {
   SSDataBlock*      existNewGroupBlock;
   STimeWindow       win;
   SNode*            pCondition;
-  SArray*           pColMatchColInfo;
+  SColMatchInfo     matchInfo;
   int32_t           primaryTsCol;
   int32_t           primarySrcSlotId;
   uint64_t          curGroupId;  // current handled group id
@@ -819,7 +801,7 @@ typedef struct SStreamFillOperatorInfo {
   int32_t           srcDelRowIndex;
   SSDataBlock*      pDelRes;
   SNode*            pCondition;
-  SArray*           pColMatchColInfo;
+  SColMatchInfo     matchInfo;
   int32_t           primaryTsCol;
   int32_t           primarySrcSlotId;
   SStreamFillInfo*  pFillInfo;
@@ -863,7 +845,7 @@ typedef struct SSortOperatorInfo {
   uint32_t       sortBufSize;  // max buffer size for in-memory sort
   SArray*        pSortInfo;
   SSortHandle*   pSortHandle;
-  SArray*        pColMatchInfo;  // for index map from table scan output
+  SColMatchInfo  matchInfo;
   int32_t        bufPageSize;
   int64_t        startTs;      // sort start time
   uint64_t       sortElapsed;  // sort elapsed time, time to flush to disk not included.
@@ -932,7 +914,7 @@ int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scan
 int32_t getBufferPgSize(int32_t rowSize, uint32_t* defaultPgsz, uint32_t* defaultBufsz);
 
 void    doSetOperatorCompleted(SOperatorInfo* pOperator);
-void    doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, const SArray* pColMatchInfo, SFilterInfo* pFilterInfo);
+void    doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, SColMatchInfo* pColMatchInfo, SFilterInfo* pFilterInfo);
 int32_t addTagPseudoColumnData(SReadHandle* pHandle, SExprInfo* pPseudoExpr, int32_t numOfPseudoExpr,
                                SSDataBlock* pBlock, const char* idStr);
 

source/libs/executor/src/cachescanoperator.c

@@ -28,7 +28,7 @@
 static SSDataBlock* doScanCache(SOperatorInfo* pOperator);
 static void         destroyLastrowScanOperator(void* param);
 static int32_t      extractCacheScanSlotId(const SArray* pColMatchInfo, SExecTaskInfo* pTaskInfo, int32_t** pSlotIds);
-static SArray*      removeRedundantTsCol(SLastRowScanPhysiNode* pScanNode, SArray* pColMatchInfo);
+static int32_t      removeRedundantTsCol(SLastRowScanPhysiNode* pScanNode, SColMatchInfo* pColMatchInfo);
 
 SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SReadHandle* readHandle,
                                            SExecTaskInfo* pTaskInfo) {
@@ -46,10 +46,10 @@ SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SRe
   pInfo->pRes = createResDataBlock(pDescNode);
 
   int32_t numOfCols = 0;
-  SArray* pColMatchInfo = extractColMatchInfo(pScanNode->scan.pScanCols, pDescNode, &numOfCols, COL_MATCH_FROM_COL_ID);
-  pInfo->pColMatchInfo = removeRedundantTsCol(pScanNode, pColMatchInfo);
+  code = extractColMatchInfo(pScanNode->scan.pScanCols, pDescNode, &numOfCols, COL_MATCH_FROM_COL_ID, &pInfo->matchInfo);
+  removeRedundantTsCol(pScanNode, &pInfo->matchInfo);
 
-  code = extractCacheScanSlotId(pInfo->pColMatchInfo, pTaskInfo, &pInfo->pSlotIds);
+  code = extractCacheScanSlotId(pInfo->matchInfo.pList, pTaskInfo, &pInfo->pSlotIds);
   if (code != TSDB_CODE_SUCCESS) {
     goto _error;
   }
@@ -64,7 +64,7 @@ SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SRe
   if (taosArrayGetSize(pTableList->pGroupList) == taosArrayGetSize(pTableList->pTableList)) {
     pInfo->retrieveType = CACHESCAN_RETRIEVE_TYPE_ALL|(pScanNode->ignoreNull? CACHESCAN_RETRIEVE_LAST:CACHESCAN_RETRIEVE_LAST_ROW);
     code = tsdbCacherowsReaderOpen(pInfo->readHandle.vnode, pInfo->retrieveType, pTableList->pTableList,
-                                   taosArrayGetSize(pInfo->pColMatchInfo), &pInfo->pLastrowReader);
+                                   taosArrayGetSize(pInfo->matchInfo.pList), &pInfo->pLastrowReader);
     if (code != TSDB_CODE_SUCCESS) {
       goto _error;
     }
@@ -139,9 +139,9 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) {
     }
 
     if (pInfo->indexOfBufferedRes < pInfo->pBufferredRes->info.rows) {
-      for (int32_t i = 0; i < taosArrayGetSize(pInfo->pColMatchInfo); ++i) {
-        SColMatchInfo* pMatchInfo = taosArrayGet(pInfo->pColMatchInfo, i);
-        int32_t        slotId = pMatchInfo->targetSlotId;
+      for (int32_t i = 0; i < taosArrayGetSize(pInfo->matchInfo.pList); ++i) {
+        SColMatchItem* pMatchInfo = taosArrayGet(pInfo->matchInfo.pList, i);
+        int32_t        slotId = pMatchInfo->dstSlotId;
 
         SColumnInfoData* pSrc = taosArrayGet(pInfo->pBufferredRes->pDataBlock, slotId);
         SColumnInfoData* pDst = taosArrayGet(pInfo->pRes->pDataBlock, slotId);
@@ -188,7 +188,7 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) {
       SArray* pGroupTableList = taosArrayGetP(pTableList->pGroupList, pInfo->currentGroupIndex);
 
       tsdbCacherowsReaderOpen(pInfo->readHandle.vnode, pInfo->retrieveType, pGroupTableList,
-                              taosArrayGetSize(pInfo->pColMatchInfo), &pInfo->pLastrowReader);
+                              taosArrayGetSize(pInfo->matchInfo.pList), &pInfo->pLastrowReader);
       taosArrayClear(pInfo->pUidList);
 
       int32_t code = tsdbRetrieveCacheRows(pInfo->pLastrowReader, pInfo->pRes, pInfo->pSlotIds, pInfo->pUidList);
@@ -235,7 +235,7 @@ void destroyLastrowScanOperator(void* param) {
   blockDataDestroy(pInfo->pBufferredRes);
   taosMemoryFree(pInfo->pSlotIds);
   taosArrayDestroy(pInfo->pUidList);
-  taosArrayDestroy(pInfo->pColMatchInfo);
+  taosArrayDestroy(pInfo->matchInfo.pList);
 
   if (pInfo->pLastrowReader != NULL) {
     pInfo->pLastrowReader = tsdbCacherowsReaderClose(pInfo->pLastrowReader);
@@ -255,16 +255,16 @@ int32_t extractCacheScanSlotId(const SArray* pColMatchInfo, SExecTaskInfo* pTask
   SSchemaWrapper* pWrapper = pTaskInfo->schemaInfo.sw;
 
   for (int32_t i = 0; i < numOfCols; ++i) {
-    SColMatchInfo* pColMatch = taosArrayGet(pColMatchInfo, i);
+    SColMatchItem* pColMatch = taosArrayGet(pColMatchInfo, i);
     for (int32_t j = 0; j < pWrapper->nCols; ++j) {
       if (pColMatch->colId == pWrapper->pSchema[j].colId &&
           pColMatch->colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
-        (*pSlotIds)[pColMatch->targetSlotId] = -1;
+        (*pSlotIds)[pColMatch->dstSlotId] = -1;
         break;
       }
 
       if (pColMatch->colId == pWrapper->pSchema[j].colId) {
-        (*pSlotIds)[pColMatch->targetSlotId] = j;
+        (*pSlotIds)[pColMatch->dstSlotId] = j;
         break;
       }
     }
@@ -273,17 +273,18 @@ int32_t extractCacheScanSlotId(const SArray* pColMatchInfo, SExecTaskInfo* pTask
   return TSDB_CODE_SUCCESS;
 }
 
-SArray* removeRedundantTsCol(SLastRowScanPhysiNode* pScanNode, SArray* pColMatchInfo) {
+int32_t removeRedundantTsCol(SLastRowScanPhysiNode* pScanNode, SColMatchInfo* pColMatchInfo) {
   if (!pScanNode->ignoreNull) {  // retrieve cached last value
-    return pColMatchInfo;
+    return TSDB_CODE_SUCCESS;
   }
 
-  SArray* pMatchInfo = taosArrayInit(taosArrayGetSize(pColMatchInfo), sizeof(SColMatchInfo));
+  size_t size = taosArrayGetSize(pColMatchInfo->pList);
+  SArray* pMatchInfo = taosArrayInit(size, sizeof(SColMatchInfo));
 
-  for (int32_t i = 0; i < taosArrayGetSize(pColMatchInfo); ++i) {
-    SColMatchInfo* pColInfo = taosArrayGet(pColMatchInfo, i);
+  for (int32_t i = 0; i < size; ++i) {
+    SColMatchItem* pColInfo = taosArrayGet(pColMatchInfo->pList, i);
 
-    int32_t    slotId = pColInfo->targetSlotId;
+    int32_t    slotId = pColInfo->dstSlotId;
     SNodeList* pList = pScanNode->scan.node.pOutputDataBlockDesc->pSlots;
 
     SSlotDescNode* pDesc = (SSlotDescNode*)nodesListGetNode(pList, slotId);
@@ -292,6 +293,7 @@ SArray* removeRedundantTsCol(SLastRowScanPhysiNode* pScanNode, SArray* pColMatch
     }
   }
 
-  taosArrayDestroy(pColMatchInfo);
-  return pMatchInfo;
+  taosArrayDestroy(pColMatchInfo->pList);
+  pColMatchInfo->pList = pMatchInfo;
+  return TSDB_CODE_SUCCESS;
 }
\ No newline at end of file

source/libs/executor/src/dataDeleter.c

@@ -53,22 +53,6 @@ typedef struct SDataDeleterHandle {
   TdThreadMutex       mutex;
 } SDataDeleterHandle;
 
-static bool needCompress(const SSDataBlock* pData, int32_t numOfCols) {
-  if (tsCompressColData < 0 || 0 == pData->info.rows) {
-    return false;
-  }
-
-  for (int32_t col = 0; col < numOfCols; ++col) {
-    SColumnInfoData* pColRes = taosArrayGet(pData->pDataBlock, col);
-    int32_t          colSize = pColRes->info.bytes * pData->info.rows;
-    if (NEEDTO_COMPRESS_QUERY(colSize)) {
-      return true;
-    }
-  }
-
-  return false;
-}
-
 static void toDataCacheEntry(SDataDeleterHandle* pHandle, const SInputData* pInput, SDataDeleterBuf* pBuf) {
   int32_t numOfCols = LIST_LENGTH(pHandle->pSchema->pSlots);
 

source/libs/executor/src/dataDispatcher.c

@@ -51,22 +51,6 @@ typedef struct SDataDispatchHandle {
   TdThreadMutex       mutex;
 } SDataDispatchHandle;
 
-static bool needCompress(const SSDataBlock* pData, int32_t numOfCols) {
-  if (tsCompressColData < 0 || 0 == pData->info.rows) {
-    return false;
-  }
-
-  for (int32_t col = 0; col < numOfCols; ++col) {
-    SColumnInfoData* pColRes = taosArrayGet(pData->pDataBlock, col);
-    int32_t          colSize = pColRes->info.bytes * pData->info.rows;
-    if (NEEDTO_COMPRESS_QUERY(colSize)) {
-      return true;
-    }
-  }
-
-  return false;
-}
-
 // clang-format off
 // data format:
 // +----------------+------------------+--------------+--------------+------------------+--------------------------------------------+------------------------------------+-------------+-----------+-------------+-----------+
@@ -86,7 +70,7 @@ static void toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* pIn
     }
   }
   SDataCacheEntry* pEntry = (SDataCacheEntry*)pBuf->pData;
-  pEntry->compressed = (int8_t)needCompress(pInput->pData, numOfCols);
+  pEntry->compressed = 0;
   pEntry->numOfRows = pInput->pData->info.rows;
   pEntry->numOfCols = numOfCols;
   pEntry->dataLen = 0;

source/libs/executor/src/executil.c

@@ -1066,13 +1066,17 @@ SArray* extractPartitionColInfo(SNodeList* pNodeList) {
   return pList;
 }
 
-SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNodeList, int32_t* numOfOutputCols,
-                            int32_t type) {
-  size_t  numOfCols = LIST_LENGTH(pNodeList);
+int32_t extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNodeList, int32_t* numOfOutputCols,
+                            int32_t type, SColMatchInfo* pMatchInfo) {
+  size_t numOfCols = LIST_LENGTH(pNodeList);
+  int32_t code = 0;
+
+  pMatchInfo->matchType = type;
+
   SArray* pList = taosArrayInit(numOfCols, sizeof(SColMatchInfo));
   if (pList == NULL) {
-    terrno = TSDB_CODE_OUT_OF_MEMORY;
-    return NULL;
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    return code;
   }
 
   for (int32_t i = 0; i < numOfCols; ++i) {
@@ -1080,12 +1084,10 @@ SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNod
     if (nodeType(pNode->pExpr) == QUERY_NODE_COLUMN) {
       SColumnNode* pColNode = (SColumnNode*)pNode->pExpr;
 
-      SColMatchInfo c = {0};
-      c.output = true;
+      SColMatchItem c = {.needOutput = true};
       c.colId = pColNode->colId;
       c.srcSlotId = pColNode->slotId;
-      c.matchType = type;
-      c.targetSlotId = pNode->slotId;
+      c.dstSlotId = pNode->slotId;
       taosArrayPush(pList, &c);
     }
   }
@@ -1102,10 +1104,10 @@ SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNod
       continue;
     }
 
-    SColMatchInfo* info = NULL;
+    SColMatchItem* info = NULL;
     for (int32_t j = 0; j < taosArrayGetSize(pList); ++j) {
       info = taosArrayGet(pList, j);
-      if (info->targetSlotId == pNode->slotId) {
+      if (info->dstSlotId == pNode->slotId) {
         break;
       }
     }
@@ -1114,11 +1116,12 @@ SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNod
       (*numOfOutputCols) += 1;
     } else if (info != NULL) {
       // select distinct tbname from stb where tbname='abc';
-      info->output = false;
+      info->needOutput = false;
     }
   }
 
-  return pList;
+  pMatchInfo->pList = pList;
+  return code;
 }
 
 static SResSchema createResSchema(int32_t type, int32_t bytes, int32_t slotId, int32_t scale, int32_t precision,
@@ -1407,14 +1410,14 @@ void relocateColumnData(SSDataBlock* pBlock, const SArray* pColMatchInfo, SArray
   int32_t i = 0, j = 0;
   while (i < numOfSrcCols && j < taosArrayGetSize(pColMatchInfo)) {
     SColumnInfoData* p = taosArrayGet(pCols, i);
-    SColMatchInfo*   pmInfo = taosArrayGet(pColMatchInfo, j);
-    if (!outputEveryColumn && pmInfo->reserved) {
+    SColMatchItem*   pmInfo = taosArrayGet(pColMatchInfo, j);
+/*    if (!outputEveryColumn && pmInfo->reserved) {
       j++;
       continue;
-    }
+    }*/
 
     if (p->info.colId == pmInfo->colId) {
-      SColumnInfoData* pDst = taosArrayGet(pBlock->pDataBlock, pmInfo->targetSlotId);
+      SColumnInfoData* pDst = taosArrayGet(pBlock->pDataBlock, pmInfo->dstSlotId);
       colDataAssign(pDst, p, pBlock->info.rows, &pBlock->info);
       i++;
       j++;

source/libs/executor/src/executor.c

@@ -355,7 +355,7 @@ int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId,
     goto _error;
   }
 
-  SDataSinkMgtCfg cfg = {.maxDataBlockNum = 10000, .maxDataBlockNumPerQuery = 5000};
+  SDataSinkMgtCfg cfg = {.maxDataBlockNum = 500, .maxDataBlockNumPerQuery = 50};
   code = dsDataSinkMgtInit(&cfg);
   if (code != TSDB_CODE_SUCCESS) {
     qError("failed to dsDataSinkMgtInit, code:%s, %s", tstrerror(code), (*pTask)->id.str);

source/libs/executor/src/executorimpl.c

@@ -1086,7 +1086,7 @@ void setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numO
 static void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const SColumnInfoData* p, bool keep,
                                                 int32_t status);
 
-void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, const SArray* pColMatchInfo, SFilterInfo* pFilterInfo) {
+void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, SColMatchInfo* pColMatchInfo, SFilterInfo* pFilterInfo) {
   if (pFilterNode == NULL || pBlock->info.rows == 0) {
     return;
   }
@@ -1120,12 +1120,12 @@ void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, const SArray* pColM
   extractQualifiedTupleByFilterResult(pBlock, p, keep, status);
 
   if (pColMatchInfo != NULL) {
-    for (int32_t i = 0; i < taosArrayGetSize(pColMatchInfo); ++i) {
-      SColMatchInfo* pInfo = taosArrayGet(pColMatchInfo, i);
+    for (int32_t i = 0; i < taosArrayGetSize(pColMatchInfo->pList); ++i) {
+      SColMatchItem* pInfo = taosArrayGet(pColMatchInfo->pList, i);
       if (pInfo->colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
-        SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, pInfo->targetSlotId);
+        SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, pInfo->dstSlotId);
         if (pColData->info.type == TSDB_DATA_TYPE_TIMESTAMP) {
-          blockDataUpdateTsWindow(pBlock, pInfo->targetSlotId);
+          blockDataUpdateTsWindow(pBlock, pInfo->dstSlotId);
           break;
         }
       }
@@ -2560,57 +2560,6 @@ int32_t aggEncodeResultRow(SOperatorInfo* pOperator, char** result, int32_t* len
   return TDB_CODE_SUCCESS;
 }
 
-int32_t aggDecodeResultRow(SOperatorInfo* pOperator, char* result) {
-  if (result == NULL) {
-    return TSDB_CODE_TSC_INVALID_INPUT;
-  }
-  SOptrBasicInfo* pInfo = (SOptrBasicInfo*)(pOperator->info);
-  SAggSupporter*  pSup = (SAggSupporter*)POINTER_SHIFT(pOperator->info, sizeof(SOptrBasicInfo));
-
-  //  int32_t size = taosHashGetSize(pSup->pResultRowHashTable);
-  int32_t length = *(int32_t*)(result);
-  int32_t offset = sizeof(int32_t);
-
-  int32_t count = *(int32_t*)(result + offset);
-  offset += sizeof(int32_t);
-
-  while (count-- > 0 && length > offset) {
-    int32_t keyLen = *(int32_t*)(result + offset);
-    offset += sizeof(int32_t);
-
-    uint64_t    tableGroupId = *(uint64_t*)(result + offset);
-    SResultRow* resultRow = getNewResultRow(pSup->pResultBuf, &pSup->currentPageId, pSup->resultRowSize);
-    if (!resultRow) {
-      return TSDB_CODE_TSC_INVALID_INPUT;
-    }
-
-    // add a new result set for a new group
-    SResultRowPosition pos = {.pageId = resultRow->pageId, .offset = resultRow->offset};
-    tSimpleHashPut(pSup->pResultRowHashTable, result + offset, keyLen, &pos, sizeof(SResultRowPosition));
-
-    offset += keyLen;
-    int32_t valueLen = *(int32_t*)(result + offset);
-    if (valueLen != pSup->resultRowSize) {
-      return TSDB_CODE_TSC_INVALID_INPUT;
-    }
-    offset += sizeof(int32_t);
-    int32_t pageId = resultRow->pageId;
-    int32_t pOffset = resultRow->offset;
-    memcpy(resultRow, result + offset, valueLen);
-    resultRow->pageId = pageId;
-    resultRow->offset = pOffset;
-    offset += valueLen;
-
-    pInfo->resultRowInfo.cur = (SResultRowPosition){.pageId = resultRow->pageId, .offset = resultRow->offset};
-    // releaseBufPage(pSup->pResultBuf, getBufPage(pSup->pResultBuf, pageId));
-  }
-
-  if (offset != length) {
-    return TSDB_CODE_TSC_INVALID_INPUT;
-  }
-  return TDB_CODE_SUCCESS;
-}
-
 int32_t handleLimitOffset(SOperatorInfo* pOperator, SLimitInfo* pLimitInfo, SSDataBlock* pBlock, bool holdDataInBuf) {
   if (pLimitInfo->remainGroupOffset > 0) {
     if (pLimitInfo->currentGroupId == 0) {  // it is the first group
@@ -2850,7 +2799,7 @@ static SSDataBlock* doFill(SOperatorInfo* pOperator) {
       break;
     }
 
-    doFilter(pInfo->pCondition, fillResult, pInfo->pColMatchColInfo, NULL);
+    doFilter(pInfo->pCondition, fillResult, &pInfo->matchInfo, NULL);
     if (fillResult->info.rows > 0) {
       break;
     }
@@ -3098,9 +3047,11 @@ _error:
     destroyAggOperatorInfo(pInfo);
   }
 
-  cleanupExprSupp(&pOperator->exprSupp);
-  taosMemoryFreeClear(pOperator);
+  if (pOperator != NULL) {
+    cleanupExprSupp(&pOperator->exprSupp);
+  }
 
+  taosMemoryFreeClear(pOperator);
   pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY;
   return NULL;
 }
@@ -3136,7 +3087,7 @@ void destroyFillOperatorInfo(void* param) {
   cleanupExprSupp(&pInfo->noFillExprSupp);
 
   taosMemoryFreeClear(pInfo->p);
-  taosArrayDestroy(pInfo->pColMatchColInfo);
+  taosArrayDestroy(pInfo->matchInfo.pList);
   taosMemoryFreeClear(param);
 }
 
@@ -3280,8 +3231,8 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode*
   pInfo->primarySrcSlotId = ((SColumnNode*)((STargetNode*)pPhyFillNode->pWStartTs)->pExpr)->slotId;
 
   int32_t numOfOutputCols = 0;
-  pInfo->pColMatchColInfo = extractColMatchInfo(pPhyFillNode->pFillExprs, pPhyFillNode->node.pOutputDataBlockDesc,
-                                                &numOfOutputCols, COL_MATCH_FROM_SLOT_ID);
+  code = extractColMatchInfo(pPhyFillNode->pFillExprs, pPhyFillNode->node.pOutputDataBlockDesc, &numOfOutputCols,
+                             COL_MATCH_FROM_SLOT_ID, &pInfo->matchInfo);
 
   code = initFillInfo(pInfo, pExprInfo, pInfo->numOfExpr, pNoFillSupp->pExprInfo, pNoFillSupp->numOfExprs,
                       (SNodeListNode*)pPhyFillNode->pValues, pPhyFillNode->timeRange, pResultInfo->capacity,

source/libs/executor/src/scanoperator.c

@@ -331,12 +331,13 @@ static bool doLoadBlockSMA(STableScanInfo* pTableScanInfo, SSDataBlock* pBlock,
     }
   }
 
-  for (int32_t i = 0; i < taosArrayGetSize(pTableScanInfo->pColMatchInfo); ++i) {
-    SColMatchInfo* pColMatchInfo = taosArrayGet(pTableScanInfo->pColMatchInfo, i);
-    if (!pColMatchInfo->output) {
+  for (int32_t i = 0; i < taosArrayGetSize(pTableScanInfo->matchInfo.pList); ++i) {
+    SColMatchItem* pColMatchInfo = taosArrayGet(pTableScanInfo->matchInfo.pList, i);
+    if (!pColMatchInfo->needOutput) {
       continue;
     }
-    pBlock->pBlockAgg[pColMatchInfo->targetSlotId] = pColAgg[i];
+
+    pBlock->pBlockAgg[pColMatchInfo->dstSlotId] = pColAgg[i];
   }
 
   return true;
@@ -442,12 +443,12 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca
     return terrno;
   }
 
-  relocateColumnData(pBlock, pTableScanInfo->pColMatchInfo, pCols, true);
+  relocateColumnData(pBlock, pTableScanInfo->matchInfo.pList, pCols, true);
   doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo);
 
   if (pTableScanInfo->pFilterNode != NULL) {
     int64_t st = taosGetTimestampUs();
-    doFilter(pTableScanInfo->pFilterNode, pBlock, pTableScanInfo->pColMatchInfo, pOperator->exprSupp.pFilterInfo);
+    doFilter(pTableScanInfo->pFilterNode, pBlock, &pTableScanInfo->matchInfo, pOperator->exprSupp.pFilterInfo);
 
     double el = (taosGetTimestampUs() - st) / 1000.0;
     pTableScanInfo->readRecorder.filterTime += el;
@@ -780,8 +781,8 @@ static void destroyTableScanOperatorInfo(void* param) {
   tsdbReaderClose(pTableScanInfo->dataReader);
   pTableScanInfo->dataReader = NULL;
 
-  if (pTableScanInfo->pColMatchInfo != NULL) {
-    taosArrayDestroy(pTableScanInfo->pColMatchInfo);
+  if (pTableScanInfo->matchInfo.pList != NULL) {
+    taosArrayDestroy(pTableScanInfo->matchInfo.pList);
   }
 
   cleanupExprSupp(&pTableScanInfo->pseudoSup);
@@ -798,10 +799,10 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode,
 
   SDataBlockDescNode* pDescNode = pTableScanNode->scan.node.pOutputDataBlockDesc;
   int32_t             numOfCols = 0;
-  pInfo->pColMatchInfo =
-      extractColMatchInfo(pTableScanNode->scan.pScanCols, pDescNode, &numOfCols, COL_MATCH_FROM_COL_ID);
+  int32_t code = extractColMatchInfo(pTableScanNode->scan.pScanCols, pDescNode, &numOfCols, COL_MATCH_FROM_COL_ID,
+                                     &pInfo->matchInfo);
 
-  int32_t code = initQueryTableDataCond(&pInfo->cond, pTableScanNode);
+  code = initQueryTableDataCond(&pInfo->cond, pTableScanNode);
   if (code != TSDB_CODE_SUCCESS) {
     goto _error;
   }
@@ -1512,9 +1513,9 @@ static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock
   }
 
   // todo extract method
-  for (int32_t i = 0; i < taosArrayGetSize(pInfo->pColMatchInfo); ++i) {
-    SColMatchInfo* pColMatchInfo = taosArrayGet(pInfo->pColMatchInfo, i);
-    if (!pColMatchInfo->output) {
+  for (int32_t i = 0; i < taosArrayGetSize(pInfo->matchInfo.pList); ++i) {
+    SColMatchItem* pColMatchInfo = taosArrayGet(pInfo->matchInfo.pList, i);
+    if (!pColMatchInfo->needOutput) {
       continue;
     }
 
@@ -1522,7 +1523,7 @@ static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock
     for (int32_t j = 0; j < blockDataGetNumOfCols(pBlock); ++j) {
       SColumnInfoData* pResCol = bdGetColumnInfoData(pBlock, j);
       if (pResCol->info.colId == pColMatchInfo->colId) {
-        SColumnInfoData* pDst = taosArrayGet(pInfo->pRes->pDataBlock, pColMatchInfo->targetSlotId);
+        SColumnInfoData* pDst = taosArrayGet(pInfo->pRes->pDataBlock, pColMatchInfo->dstSlotId);
         colDataAssign(pDst, pResCol, pBlock->info.rows, &pInfo->pRes->info);
         colExists = true;
         break;
@@ -1531,7 +1532,7 @@ static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock
 
     // the required column does not exists in submit block, let's set it to be all null value
     if (!colExists) {
-      SColumnInfoData* pDst = taosArrayGet(pInfo->pRes->pDataBlock, pColMatchInfo->targetSlotId);
+      SColumnInfoData* pDst = taosArrayGet(pInfo->pRes->pDataBlock, pColMatchInfo->dstSlotId);
       colDataAppendNNULL(pDst, 0, pBlockInfo->rows);
     }
   }
@@ -2193,8 +2194,8 @@ static void destroyStreamScanOperatorInfo(void* param) {
   if (pStreamScan->tqReader) {
     tqCloseReader(pStreamScan->tqReader);
   }
-  if (pStreamScan->pColMatchInfo) {
-    taosArrayDestroy(pStreamScan->pColMatchInfo);
+  if (pStreamScan->matchInfo.pList) {
+    taosArrayDestroy(pStreamScan->matchInfo.pList);
   }
   if (pStreamScan->pPseudoExpr) {
     destroyExprInfo(pStreamScan->pPseudoExpr, pStreamScan->numOfPseudoExpr);
@@ -2228,17 +2229,17 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys
   pInfo->pGroupTags = pTableScanNode->pGroupTags;
 
   int32_t numOfCols = 0;
-  pInfo->pColMatchInfo = extractColMatchInfo(pScanPhyNode->pScanCols, pDescNode, &numOfCols, COL_MATCH_FROM_COL_ID);
+  int32_t code = extractColMatchInfo(pScanPhyNode->pScanCols, pDescNode, &numOfCols, COL_MATCH_FROM_COL_ID, &pInfo->matchInfo);
 
-  int32_t numOfOutput = taosArrayGetSize(pInfo->pColMatchInfo);
+  int32_t numOfOutput = taosArrayGetSize(pInfo->matchInfo.pList);
   SArray* pColIds = taosArrayInit(numOfOutput, sizeof(int16_t));
   for (int32_t i = 0; i < numOfOutput; ++i) {
-    SColMatchInfo* id = taosArrayGet(pInfo->pColMatchInfo, i);
+    SColMatchItem* id = taosArrayGet(pInfo->matchInfo.pList, i);
 
     int16_t colId = id->colId;
     taosArrayPush(pColIds, &colId);
     if (id->colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
-      pInfo->primaryTsIndex = id->targetSlotId;
+      pInfo->primaryTsIndex = id->dstSlotId;
     }
   }
 
@@ -2387,7 +2388,7 @@ static void destroySysScanOperator(void* param) {
     pInfo->pIdx = NULL;
   }
 
-  taosArrayDestroy(pInfo->scanCols);
+  taosArrayDestroy(pInfo->matchInfo.pList);
   taosMemoryFreeClear(pInfo->pUser);
 
   taosMemoryFreeClear(param);
@@ -2752,7 +2753,7 @@ static void relocateAndFilterSysTagsScanResult(SSysTableScanInfo* pInfo, int32_t
   dataBlock->info.rows = numOfRows;
   pInfo->pRes->info.rows = numOfRows;
 
-  relocateColumnData(pInfo->pRes, pInfo->scanCols, dataBlock->pDataBlock, false);
+  relocateColumnData(pInfo->pRes, pInfo->matchInfo.pList, dataBlock->pDataBlock, false);
   doFilterResult(pInfo);
 
   blockDataCleanup(dataBlock);
@@ -3441,7 +3442,7 @@ static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) {
       p->info.rows = numOfRows;
       pInfo->pRes->info.rows = numOfRows;
 
-      relocateColumnData(pInfo->pRes, pInfo->scanCols, p->pDataBlock, false);
+      relocateColumnData(pInfo->pRes, pInfo->matchInfo.pList, p->pDataBlock, false);
       doFilterResult(pInfo);
 
       blockDataCleanup(p);
@@ -3457,7 +3458,7 @@ static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) {
     p->info.rows = numOfRows;
     pInfo->pRes->info.rows = numOfRows;
 
-    relocateColumnData(pInfo->pRes, pInfo->scanCols, p->pDataBlock, false);
+    relocateColumnData(pInfo->pRes, pInfo->matchInfo.pList, p->pDataBlock, false);
     doFilterResult(pInfo);
 
     blockDataCleanup(p);
@@ -3618,7 +3619,7 @@ static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) {
       p->info.rows = numOfRows;
       pInfo->pRes->info.rows = numOfRows;
 
-      relocateColumnData(pInfo->pRes, pInfo->scanCols, p->pDataBlock, false);
+      relocateColumnData(pInfo->pRes, pInfo->matchInfo.pList, p->pDataBlock, false);
       doFilterResult(pInfo);
 
       blockDataCleanup(p);
@@ -3634,7 +3635,7 @@ static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) {
     p->info.rows = numOfRows;
     pInfo->pRes->info.rows = numOfRows;
 
-    relocateColumnData(pInfo->pRes, pInfo->scanCols, p->pDataBlock, false);
+    relocateColumnData(pInfo->pRes, pInfo->matchInfo.pList, p->pDataBlock, false);
     doFilterResult(pInfo);
 
     blockDataCleanup(p);
@@ -3798,7 +3799,7 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) {
       }
 
       char* pStart = pRsp->data;
-      extractDataBlockFromFetchRsp(pInfo->pRes, pRsp->data, pInfo->scanCols, &pStart);
+      extractDataBlockFromFetchRsp(pInfo->pRes, pRsp->data, pInfo->matchInfo.pList, &pStart);
       updateLoadRemoteInfo(&pInfo->loadInfo, pRsp->numOfRows, pRsp->compLen, startTs, pOperator);
 
       // todo log the filter info
@@ -3827,7 +3828,7 @@ int32_t buildSysDbTableInfo(const SSysTableScanInfo* pInfo, int32_t capacity) {
   p->info.rows = buildDbTableInfoBlock(pInfo->sysInfo, p, pSysDbTableMeta, size, TSDB_PERFORMANCE_SCHEMA_DB);
 
   pInfo->pRes->info.rows = p->info.rows;
-  relocateColumnData(pInfo->pRes, pInfo->scanCols, p->pDataBlock, false);
+  relocateColumnData(pInfo->pRes, pInfo->matchInfo.pList, p->pDataBlock, false);
   blockDataDestroy(p);
 
   return pInfo->pRes->info.rows;
@@ -3889,18 +3890,16 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScan
   SScanPhysiNode* pScanNode = &pScanPhyNode->scan;
 
   SDataBlockDescNode* pDescNode = pScanNode->node.pOutputDataBlockDesc;
-  SSDataBlock*        pResBlock = createResDataBlock(pDescNode);
 
   int32_t num = 0;
-  SArray* colList = extractColMatchInfo(pScanNode->pScanCols, pDescNode, &num, COL_MATCH_FROM_COL_ID);
+  int32_t code = extractColMatchInfo(pScanNode->pScanCols, pDescNode, &num, COL_MATCH_FROM_COL_ID, &pInfo->matchInfo);
 
   pInfo->accountId = pScanPhyNode->accountId;
   pInfo->pUser = taosMemoryStrDup((void*)pUser);
   pInfo->sysInfo = pScanPhyNode->sysInfo;
   pInfo->showRewrite = pScanPhyNode->showRewrite;
-  pInfo->pRes = pResBlock;
+  pInfo->pRes = createResDataBlock(pDescNode);
   pInfo->pCondition = pScanNode->node.pConditions;
-  pInfo->scanCols = colList;
 
   initResultSizeInfo(&pOperator->resultInfo, 4096);
 
@@ -3922,7 +3921,7 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScan
   pOperator->blocking = false;
   pOperator->status = OP_NOT_OPENED;
   pOperator->info = pInfo;
-  pOperator->exprSupp.numOfExprs = taosArrayGetSize(pResBlock->pDataBlock);
+  pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pRes->pDataBlock);
   pOperator->pTaskInfo = pTaskInfo;
 
   pOperator->fpSet =
@@ -4021,7 +4020,7 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) {
 static void destroyTagScanOperatorInfo(void* param) {
   STagScanInfo* pInfo = (STagScanInfo*)param;
   pInfo->pRes = blockDataDestroy(pInfo->pRes);
-  taosArrayDestroy(pInfo->pColMatchInfo);
+  taosArrayDestroy(pInfo->matchInfo.pList);
   taosMemoryFreeClear(param);
 }
 
@@ -4038,15 +4037,14 @@ SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysi
   int32_t    num = 0;
   int32_t    numOfExprs = 0;
   SExprInfo* pExprInfo = createExprInfo(pPhyNode->pScanPseudoCols, NULL, &numOfExprs);
-  SArray*    colList = extractColMatchInfo(pPhyNode->pScanPseudoCols, pDescNode, &num, COL_MATCH_FROM_COL_ID);
+  int32_t code = extractColMatchInfo(pPhyNode->pScanPseudoCols, pDescNode, &num, COL_MATCH_FROM_COL_ID, &pInfo->matchInfo);
 
-  int32_t code = initExprSupp(&pOperator->exprSupp, pExprInfo, numOfExprs);
+  code = initExprSupp(&pOperator->exprSupp, pExprInfo, numOfExprs);
   if (code != TSDB_CODE_SUCCESS) {
     goto _error;
   }
 
   pInfo->pTableList = pTableListInfo;
-  pInfo->pColMatchInfo = colList;
   pInfo->pRes = createResDataBlock(pDescNode);
   pInfo->readHandle = *pReadHandle;
   pInfo->curPos = 0;
@@ -4180,11 +4178,11 @@ static int32_t loadDataBlockFromOneTable(SOperatorInfo* pOperator, STableMergeSc
       }
 
       for (int32_t i = 0; i < numOfCols; ++i) {
-        SColMatchInfo* pColMatchInfo = taosArrayGet(pTableScanInfo->pColMatchInfo, i);
-        if (!pColMatchInfo->output) {
+        SColMatchItem* pColMatchInfo = taosArrayGet(pTableScanInfo->matchInfo.pList, i);
+        if (!pColMatchInfo->needOutput) {
           continue;
         }
-        pBlock->pBlockAgg[pColMatchInfo->targetSlotId] = pColAgg[i];
+        pBlock->pBlockAgg[pColMatchInfo->dstSlotId] = pColAgg[i];
       }
 
       return TSDB_CODE_SUCCESS;
@@ -4215,7 +4213,7 @@ static int32_t loadDataBlockFromOneTable(SOperatorInfo* pOperator, STableMergeSc
     return terrno;
   }
 
-  relocateColumnData(pBlock, pTableScanInfo->pColMatchInfo, pCols, true);
+  relocateColumnData(pBlock, pTableScanInfo->matchInfo.pList, pCols, true);
 
   // currently only the tbname pseudo column
   if (pTableScanInfo->pseudoSup.numOfExprs > 0) {
@@ -4228,7 +4226,7 @@ static int32_t loadDataBlockFromOneTable(SOperatorInfo* pOperator, STableMergeSc
 
   if (pTableScanInfo->pFilterNode != NULL) {
     int64_t st = taosGetTimestampMs();
-    doFilter(pTableScanInfo->pFilterNode, pBlock, pTableScanInfo->pColMatchInfo, NULL);
+    doFilter(pTableScanInfo->pFilterNode, pBlock, &pTableScanInfo->matchInfo, NULL);
 
     double el = (taosGetTimestampUs() - st) / 1000.0;
     pTableScanInfo->readRecorder.filterTime += el;
@@ -4312,9 +4310,9 @@ static SSDataBlock* getTableDataBlock(void* param) {
 SArray* generateSortByTsInfo(SArray* colMatchInfo, int32_t order) {
   int32_t tsTargetSlotId = 0;
   for (int32_t i = 0; i < taosArrayGetSize(colMatchInfo); ++i) {
-    SColMatchInfo* colInfo = taosArrayGet(colMatchInfo, i);
+    SColMatchItem* colInfo = taosArrayGet(colMatchInfo, i);
     if (colInfo->colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
-      tsTargetSlotId = colInfo->targetSlotId;
+      tsTargetSlotId = colInfo->dstSlotId;
     }
   }
 
@@ -4500,8 +4498,8 @@ void destroyTableMergeScanOperatorInfo(void* param) {
   }
   taosArrayDestroy(pTableScanInfo->dataReaders);
 
-  if (pTableScanInfo->pColMatchInfo != NULL) {
-    taosArrayDestroy(pTableScanInfo->pColMatchInfo);
+  if (pTableScanInfo->matchInfo.pList != NULL) {
+    taosArrayDestroy(pTableScanInfo->matchInfo.pList);
   }
 
   pTableScanInfo->pResBlock = blockDataDestroy(pTableScanInfo->pResBlock);
@@ -4559,11 +4557,11 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN
   SDataBlockDescNode* pDescNode = pTableScanNode->scan.node.pOutputDataBlockDesc;
 
   int32_t numOfCols = 0;
-  SArray* pColList = extractColMatchInfo(pTableScanNode->scan.pScanCols, pDescNode, &numOfCols, COL_MATCH_FROM_COL_ID);
+  int32_t code = extractColMatchInfo(pTableScanNode->scan.pScanCols, pDescNode, &numOfCols, COL_MATCH_FROM_COL_ID, &pInfo->matchInfo);
 
-  int32_t code = initQueryTableDataCond(&pInfo->cond, pTableScanNode);
+  code = initQueryTableDataCond(&pInfo->cond, pTableScanNode);
   if (code != TSDB_CODE_SUCCESS) {
-    taosArrayDestroy(pColList);
+    taosArrayDestroy(pInfo->matchInfo.pList);
     goto _error;
   }
 
@@ -4583,12 +4581,11 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN
   pInfo->pFilterNode = pTableScanNode->scan.node.pConditions;
   pInfo->tableListInfo = pTableListInfo;
   pInfo->scanFlag = MAIN_SCAN;
-  pInfo->pColMatchInfo = pColList;
 
   pInfo->pResBlock = createResDataBlock(pDescNode);
   pInfo->sortSourceParams = taosArrayInit(64, sizeof(STableMergeScanSortSourceParam));
 
-  pInfo->pSortInfo = generateSortByTsInfo(pInfo->pColMatchInfo, pInfo->cond.order);
+  pInfo->pSortInfo = generateSortByTsInfo(pInfo->matchInfo.pList, pInfo->cond.order);
   pInfo->pSortInputBlock = createOneDataBlock(pInfo->pResBlock, false);
 
   int32_t rowSize = pInfo->pResBlock->info.rowSize;

source/libs/executor/src/sortoperator.c

@@ -38,8 +38,7 @@ SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode*
   SExprInfo*   pExprInfo = createExprInfo(pSortNode->pExprs, NULL, &numOfCols);
 
   int32_t numOfOutputCols = 0;
-  SArray* pColMatchColInfo =
-      extractColMatchInfo(pSortNode->pTargets, pDescNode, &numOfOutputCols, COL_MATCH_FROM_SLOT_ID);
+  int32_t code = extractColMatchInfo(pSortNode->pTargets, pDescNode, &numOfOutputCols, COL_MATCH_FROM_SLOT_ID, &pInfo->matchInfo);
 
   pOperator->exprSupp.pCtx = createSqlFunctionCtx(pExprInfo, numOfCols, &pOperator->exprSupp.rowEntryInfoOffset);
 
@@ -48,7 +47,6 @@ SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode*
   pInfo->binfo.pRes = pResBlock;
   pInfo->pSortInfo = createSortInfo(pSortNode->pSortKeys);
   pInfo->pCondition = pSortNode->node.pConditions;
-  pInfo->pColMatchInfo = pColMatchColInfo;
   initLimitInfo(pSortNode->node.pLimit, pSortNode->node.pSlimit, &pInfo->limitInfo);
 
   pOperator->name = "SortOperator";
@@ -67,7 +65,7 @@ SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode*
   pOperator->fpSet = createOperatorFpSet(doOpenSortOperator, doSort, NULL, NULL, destroyOrderOperatorInfo,
                                          getExplainExecInfo);
 
-  int32_t code = appendDownstream(pOperator, &downstream, 1);
+  code = appendDownstream(pOperator, &downstream, 1);
   if (code != TSDB_CODE_SUCCESS) {
     goto _error;
   }
@@ -127,11 +125,11 @@ SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, i
     // todo extract function to handle this
     int32_t numOfCols = taosArrayGetSize(pColMatchInfo);
     for (int32_t i = 0; i < numOfCols; ++i) {
-      SColMatchInfo* pmInfo = taosArrayGet(pColMatchInfo, i);
-      ASSERT(pmInfo->matchType == COL_MATCH_FROM_SLOT_ID);
+      SColMatchItem* pmInfo = taosArrayGet(pColMatchInfo, i);
+//      ASSERT(pmInfo->matchType == COL_MATCH_FROM_SLOT_ID);
 
       SColumnInfoData* pSrc = taosArrayGet(p->pDataBlock, pmInfo->srcSlotId);
-      SColumnInfoData* pDst = taosArrayGet(pDataBlock->pDataBlock, pmInfo->targetSlotId);
+      SColumnInfoData* pDst = taosArrayGet(pDataBlock->pDataBlock, pmInfo->dstSlotId);
       colDataAssign(pDst, pSrc, p->info.rows, &pDataBlock->info);
     }
 
@@ -210,13 +208,13 @@ SSDataBlock* doSort(SOperatorInfo* pOperator) {
   SSDataBlock* pBlock = NULL;
   while (1) {
     pBlock = getSortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity,
-                                pInfo->pColMatchInfo, pInfo);
+                                pInfo->matchInfo.pList, pInfo);
     if (pBlock == NULL) {
       doSetOperatorCompleted(pOperator);
       return NULL;
     }
 
-    doFilter(pInfo->pCondition, pBlock, pInfo->pColMatchInfo, NULL);
+    doFilter(pInfo->pCondition, pBlock, &pInfo->matchInfo, NULL);
     if (blockDataGetNumOfRows(pBlock) == 0) {
       continue;
     }
@@ -256,7 +254,7 @@ void destroyOrderOperatorInfo(void* param) {
 
   tsortDestroySortHandle(pInfo->pSortHandle);
   taosArrayDestroy(pInfo->pSortInfo);
-  taosArrayDestroy(pInfo->pColMatchInfo);
+  taosArrayDestroy(pInfo->matchInfo.pList);
   taosMemoryFreeClear(param);
 }
 
@@ -277,20 +275,17 @@ int32_t getExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExplain, uint32_t*
 typedef enum EChildOperatorStatus { CHILD_OP_NEW_GROUP, CHILD_OP_SAME_GROUP, CHILD_OP_FINISHED } EChildOperatorStatus;
 
 typedef struct SGroupSortOperatorInfo {
-  SOptrBasicInfo binfo;
-  SArray*        pSortInfo;
-  SArray*        pColMatchInfo;
-
-  int64_t  startTs;
-  uint64_t sortElapsed;
-  bool     hasGroupId;
-  uint64_t currGroupId;
-
+  SOptrBasicInfo       binfo;
+  SArray*              pSortInfo;
+  SColMatchInfo        matchInfo;
+  int64_t              startTs;
+  uint64_t             sortElapsed;
+  bool                 hasGroupId;
+  uint64_t             currGroupId;
   SSDataBlock*         prefetchedSortInput;
   SSortHandle*         pCurrSortHandle;
   EChildOperatorStatus childOpStatus;
-
-  SSortExecInfo sortExecInfo;
+  SSortExecInfo        sortExecInfo;
 } SGroupSortOperatorInfo;
 
 SSDataBlock* getGroupSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity,
@@ -320,11 +315,11 @@ SSDataBlock* getGroupSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlo
   if (p->info.rows > 0) {
     int32_t numOfCols = taosArrayGetSize(pColMatchInfo);
     for (int32_t i = 0; i < numOfCols; ++i) {
-      SColMatchInfo* pmInfo = taosArrayGet(pColMatchInfo, i);
-      ASSERT(pmInfo->matchType == COL_MATCH_FROM_SLOT_ID);
+      SColMatchItem* pmInfo = taosArrayGet(pColMatchInfo, i);
+//      ASSERT(pmInfo->matchType == COL_MATCH_FROM_SLOT_ID);
 
       SColumnInfoData* pSrc = taosArrayGet(p->pDataBlock, pmInfo->srcSlotId);
-      SColumnInfoData* pDst = taosArrayGet(pDataBlock->pDataBlock, pmInfo->targetSlotId);
+      SColumnInfoData* pDst = taosArrayGet(pDataBlock->pDataBlock, pmInfo->dstSlotId);
       colDataAssign(pDst, pSrc, p->info.rows, &pDataBlock->info);
     }
 
@@ -442,7 +437,7 @@ SSDataBlock* doGroupSort(SOperatorInfo* pOperator) {
     // beginSortGroup would fetch all child blocks of pInfo->currGroupId;
     ASSERT(pInfo->childOpStatus != CHILD_OP_SAME_GROUP);
     pBlock = getGroupSortedBlockData(pInfo->pCurrSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity,
-                                     pInfo->pColMatchInfo, pInfo);
+                                     pInfo->matchInfo.pList, pInfo);
     if (pBlock != NULL) {
       pBlock->info.groupId = pInfo->currGroupId;
       pOperator->resultInfo.totalRows += pBlock->info.rows;
@@ -474,7 +469,7 @@ void destroyGroupSortOperatorInfo(void* param) {
   pInfo->binfo.pRes = blockDataDestroy(pInfo->binfo.pRes);
 
   taosArrayDestroy(pInfo->pSortInfo);
-  taosArrayDestroy(pInfo->pColMatchInfo);
+  taosArrayDestroy(pInfo->matchInfo.pList);
 
   taosMemoryFreeClear(param);
 }
@@ -494,8 +489,8 @@ SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SGroupSort
   SExprInfo*   pExprInfo = createExprInfo(pSortPhyNode->pExprs, NULL, &numOfCols);
 
   int32_t numOfOutputCols = 0;
-  SArray* pColMatchColInfo =
-      extractColMatchInfo(pSortPhyNode->pTargets, pDescNode, &numOfOutputCols, COL_MATCH_FROM_SLOT_ID);
+  int32_t code = extractColMatchInfo(pSortPhyNode->pTargets, pDescNode, &numOfOutputCols, COL_MATCH_FROM_SLOT_ID,
+                                     &pInfo->matchInfo);
 
   pOperator->exprSupp.pCtx = createSqlFunctionCtx(pExprInfo, numOfCols, &pOperator->exprSupp.rowEntryInfoOffset);
   pInfo->binfo.pRes = pResBlock;
@@ -503,8 +498,7 @@ SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SGroupSort
   initResultSizeInfo(&pOperator->resultInfo, 1024);
 
   pInfo->pSortInfo = createSortInfo(pSortPhyNode->pSortKeys);
-  ;
-  pInfo->pColMatchInfo = pColMatchColInfo;
+
   pOperator->name = "GroupSortOperator";
   pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT;
   pOperator->blocking = false;
@@ -517,7 +511,7 @@ SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SGroupSort
   pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doGroupSort, NULL, NULL, destroyGroupSortOperatorInfo,
                                          getGroupSortExplainExecInfo);
 
-  int32_t code = appendDownstream(pOperator, &downstream, 1);
+  code = appendDownstream(pOperator, &downstream, 1);
   if (code != TSDB_CODE_SUCCESS) {
     goto _error;
   }
@@ -535,20 +529,18 @@ _error:
 // Multiway Sort Merge operator
 typedef struct SMultiwayMergeOperatorInfo {
   SOptrBasicInfo binfo;
+  int32_t        bufPageSize;
+  uint32_t       sortBufSize;  // max buffer size for in-memory sort
 
-  int32_t  bufPageSize;
-  uint32_t sortBufSize;  // max buffer size for in-memory sort
-
-  SArray*      pSortInfo;
-  SSortHandle* pSortHandle;
-  SArray*      pColMatchInfo;  // for index map from table scan output
-
-  SSDataBlock*  pInputBlock;
-  int64_t       startTs;  // sort start time
-  bool          groupSort;
-  bool          hasGroupId;
-  uint64_t      groupId;
-  STupleHandle* prefetchedTuple;
+  SArray*        pSortInfo;
+  SSortHandle*   pSortHandle;
+  SColMatchInfo  matchInfo;
+  SSDataBlock*   pInputBlock;
+  int64_t        startTs;  // sort start time
+  bool           groupSort;
+  bool           hasGroupId;
+  uint64_t       groupId;
+  STupleHandle*  prefetchedTuple;
 } SMultiwayMergeOperatorInfo;
 
 int32_t doOpenMultiwayMergeOperator(SOperatorInfo* pOperator) {
@@ -645,11 +637,11 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData
     blockDataEnsureCapacity(pDataBlock, p->info.rows);
     int32_t numOfCols = taosArrayGetSize(pColMatchInfo);
     for (int32_t i = 0; i < numOfCols; ++i) {
-      SColMatchInfo* pmInfo = taosArrayGet(pColMatchInfo, i);
-      ASSERT(pmInfo->matchType == COL_MATCH_FROM_SLOT_ID);
+      SColMatchItem* pmInfo = taosArrayGet(pColMatchInfo, i);
+//      ASSERT(pColMatchInfo-> == COL_MATCH_FROM_SLOT_ID);
 
       SColumnInfoData* pSrc = taosArrayGet(p->pDataBlock, pmInfo->srcSlotId);
-      SColumnInfoData* pDst = taosArrayGet(pDataBlock->pDataBlock, pmInfo->targetSlotId);
+      SColumnInfoData* pDst = taosArrayGet(pDataBlock->pDataBlock, pmInfo->dstSlotId);
       colDataAssign(pDst, pSrc, p->info.rows, &pDataBlock->info);
     }
 
@@ -678,7 +670,7 @@ SSDataBlock* doMultiwayMerge(SOperatorInfo* pOperator) {
   }
 
   SSDataBlock* pBlock = getMultiwaySortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes,
-                                                   pOperator->resultInfo.capacity, pInfo->pColMatchInfo, pOperator);
+                                                   pOperator->resultInfo.capacity, pInfo->matchInfo.pList, pOperator);
   if (pBlock != NULL) {
     pOperator->resultInfo.totalRows += pBlock->info.rows;
   } else {
@@ -694,7 +686,7 @@ void destroyMultiwayMergeOperatorInfo(void* param) {
 
   tsortDestroySortHandle(pInfo->pSortHandle);
   taosArrayDestroy(pInfo->pSortInfo);
-  taosArrayDestroy(pInfo->pColMatchInfo);
+  taosArrayDestroy(pInfo->matchInfo.pList);
 
   taosMemoryFreeClear(param);
 }
@@ -731,15 +723,14 @@ SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** downStreams, size
 
   SArray* pSortInfo = createSortInfo(pMergePhyNode->pMergeKeys);
   int32_t numOfOutputCols = 0;
-  SArray* pColMatchColInfo =
-      extractColMatchInfo(pMergePhyNode->pTargets, pDescNode, &numOfOutputCols, COL_MATCH_FROM_SLOT_ID);
+
+  code = extractColMatchInfo(pMergePhyNode->pTargets, pDescNode, &numOfOutputCols, COL_MATCH_FROM_SLOT_ID, &pInfo->matchInfo);
   SPhysiNode*  pChildNode = (SPhysiNode*)nodesListGetNode(pPhyNode->pChildren, 0);
   SSDataBlock* pInputBlock = createResDataBlock(pChildNode->pOutputDataBlockDesc);
   initResultSizeInfo(&pOperator->resultInfo, 1024);
 
   pInfo->groupSort = pMergePhyNode->groupSort;
   pInfo->pSortInfo = pSortInfo;
-  pInfo->pColMatchInfo = pColMatchColInfo;
   pInfo->pInputBlock = pInputBlock;
   pInfo->bufPageSize = getProperSortPageSize(rowSize);
   pInfo->sortBufSize = pInfo->bufPageSize * (numStreams + 1);  // one additional is reserved for merged result.

source/libs/executor/src/tfill.c

@@ -722,7 +722,7 @@ void destroyStreamFillOperatorInfo(void* param) {
   pInfo->pSrcBlock = blockDataDestroy(pInfo->pSrcBlock);
   pInfo->pPrevSrcBlock = blockDataDestroy(pInfo->pPrevSrcBlock);
   pInfo->pDelRes = blockDataDestroy(pInfo->pDelRes);
-  pInfo->pColMatchColInfo = taosArrayDestroy(pInfo->pColMatchColInfo);
+  pInfo->matchInfo.pList = taosArrayDestroy(pInfo->matchInfo.pList);
   taosMemoryFree(pInfo);
 }
 
@@ -1499,7 +1499,7 @@ static SSDataBlock* doStreamFill(SOperatorInfo* pOperator) {
     }
 
     doStreamFillImpl(pOperator);
-    doFilter(pInfo->pCondition, pInfo->pRes, pInfo->pColMatchColInfo, NULL);
+    doFilter(pInfo->pCondition, pInfo->pRes, &pInfo->matchInfo, NULL);
     memcpy(pInfo->pRes->info.parTbName, pInfo->pSrcBlock->info.parTbName, TSDB_TABLE_NAME_LEN);
     pOperator->resultInfo.totalRows += pInfo->pRes->info.rows;
     if (pInfo->pRes->info.rows > 0) {
@@ -1675,11 +1675,10 @@ SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFi
   pInfo->primarySrcSlotId = ((SColumnNode*)((STargetNode*)pPhyFillNode->pWStartTs)->pExpr)->slotId;
 
   int32_t numOfOutputCols = 0;
-  SArray* pColMatchColInfo = extractColMatchInfo(pPhyFillNode->pFillExprs, pPhyFillNode->node.pOutputDataBlockDesc,
-                                                 &numOfOutputCols, COL_MATCH_FROM_SLOT_ID);
+  int32_t code = extractColMatchInfo(pPhyFillNode->pFillExprs, pPhyFillNode->node.pOutputDataBlockDesc,
+                                     &numOfOutputCols, COL_MATCH_FROM_SLOT_ID, &pInfo->matchInfo);
   pInfo->pCondition = pPhyFillNode->node.pConditions;
-  pInfo->pColMatchColInfo = pColMatchColInfo;
-  int32_t code = initExprSupp(&pOperator->exprSupp, pFillExprInfo, numOfFillCols);
+  code = initExprSupp(&pOperator->exprSupp, pFillExprInfo, numOfFillCols);
   if (code != TSDB_CODE_SUCCESS) {
     goto _error;
   }

source/libs/parser/inc/parInsertData.h → source/libs/parser/inc/parInsertUtil.h

@@ -13,17 +13,30 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef TDENGINE_DATABLOCKMGT_H
-#define TDENGINE_DATABLOCKMGT_H
+#ifndef TDENGINE_PAR_INSERT_UTIL_H
+#define TDENGINE_PAR_INSERT_UTIL_H
 
-#include "catalog.h"
-#include "os.h"
-#include "query.h"
-#include "tname.h"
-#include "ttypes.h"
+#include "parUtil.h"
+
+struct SToken;
 
 #define IS_DATA_COL_ORDERED(spd) ((spd->orderStatus) == (int8_t)ORDER_STATUS_ORDERED)
 
+#define NEXT_TOKEN(pSql, sToken)                \
+  do {                                          \
+    int32_t index = 0;                          \
+    sToken = tStrGetToken(pSql, &index, false); \
+    pSql += index;                              \
+  } while (0)
+
+#define CHECK_CODE(expr)             \
+  do {                               \
+    int32_t code = expr;             \
+    if (TSDB_CODE_SUCCESS != code) { \
+      return code;                   \
+    }                                \
+  } while (0)
+
 typedef enum EOrderStatus {
   ORDER_STATUS_UNKNOWN = 0,
   ORDER_STATUS_ORDERED = 1,
@@ -60,6 +73,49 @@ typedef struct SParsedDataColInfo {
   int8_t         orderStatus;  // bound columns
 } SParsedDataColInfo;
 
+typedef struct SInsertParseBaseContext {
+  SParseContext *pComCxt;
+  char          *pSql;
+  SMsgBuf        msg;
+} SInsertParseBaseContext;
+
+typedef struct SInsertParseContext {
+  SParseContext     *pComCxt;             // input
+  char              *pSql;                // input
+  SMsgBuf            msg;                 // input
+  STableMeta        *pTableMeta;          // each table
+  SParsedDataColInfo tags;                // each table
+  SVCreateTbReq      createTblReq;        // each table
+  SHashObj          *pVgroupsHashObj;     // global
+  SHashObj          *pTableBlockHashObj;  // global
+  SHashObj          *pSubTableHashObj;    // global
+  SArray            *pVgDataBlocks;       // global
+  SHashObj          *pTableNameHashObj;   // global
+  SHashObj          *pDbFNameHashObj;     // global
+  int32_t            totalNum;
+  SVnodeModifOpStmt *pOutput;
+  SStmtCallback     *pStmtCb;
+  SParseMetaCache   *pMetaCache;
+  char               sTableName[TSDB_TABLE_NAME_LEN];
+  char               tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW];
+  int64_t            memElapsed;
+  int64_t            parRowElapsed;
+} SInsertParseContext;
+
+typedef struct SInsertParseSyntaxCxt {
+  SParseContext   *pComCxt;
+  char            *pSql;
+  SMsgBuf          msg;
+  SParseMetaCache *pMetaCache;
+} SInsertParseSyntaxCxt;
+
+typedef struct SMemParam {
+  SRowBuilder *rb;
+  SSchema     *schema;
+  int32_t      toffset;
+  col_id_t     colIdx;
+} SMemParam;
+
 typedef struct {
   uint8_t rowType;  // default is 0, that is SDataRow
   int32_t rowSize;
@@ -84,65 +140,28 @@ typedef struct STableDataBlocks {
   SRowBuilder        rowBuilder;
 } STableDataBlocks;
 
-static FORCE_INLINE int32_t getExtendedRowSize(STableDataBlocks *pBlock) {
-  STableComInfo *pTableInfo = &pBlock->pTableMeta->tableInfo;
-  ASSERT(pBlock->rowSize == pTableInfo->rowSize);
-  return pBlock->rowSize + TD_ROW_HEAD_LEN - sizeof(TSKEY) + pBlock->boundColumnInfo.extendedVarLen +
-         (int32_t)TD_BITMAP_BYTES(pTableInfo->numOfColumns - 1);
-}
-
-static FORCE_INLINE void getSTSRowAppendInfo(uint8_t rowType, SParsedDataColInfo *spd, col_id_t idx, int32_t *toffset,
-                                             col_id_t *colIdx) {
-  col_id_t schemaIdx = 0;
-  if (IS_DATA_COL_ORDERED(spd)) {
-    schemaIdx = spd->boundColumns[idx];
-    if (TD_IS_TP_ROW_T(rowType)) {
-      *toffset = (spd->cols + schemaIdx)->toffset;  // the offset of firstPart
-      *colIdx = schemaIdx;
-    } else {
-      *toffset = idx * sizeof(SKvRowIdx);  // the offset of SKvRowIdx
-      *colIdx = idx;
-    }
-  } else {
-    ASSERT(idx == (spd->colIdxInfo + idx)->boundIdx);
-    schemaIdx = (spd->colIdxInfo + idx)->schemaColIdx;
-    if (TD_IS_TP_ROW_T(rowType)) {
-      *toffset = (spd->cols + schemaIdx)->toffset;
-      *colIdx = schemaIdx;
-    } else {
-      *toffset = ((spd->colIdxInfo + idx)->finalIdx) * sizeof(SKvRowIdx);
-      *colIdx = (spd->colIdxInfo + idx)->finalIdx;
-    }
-  }
-}
-
-static FORCE_INLINE int32_t setBlockInfo(SSubmitBlk *pBlocks, STableDataBlocks *dataBuf, int32_t numOfRows) {
-  pBlocks->suid = (TSDB_NORMAL_TABLE == dataBuf->pTableMeta->tableType ? 0 : dataBuf->pTableMeta->suid);
-  pBlocks->uid = dataBuf->pTableMeta->uid;
-  pBlocks->sversion = dataBuf->pTableMeta->sversion;
-  pBlocks->schemaLen = dataBuf->createTbReqLen;
-
-  if (pBlocks->numOfRows + numOfRows >= INT32_MAX) {
-    return TSDB_CODE_TSC_INVALID_OPERATION;
-  } else {
-    pBlocks->numOfRows += numOfRows;
-    return TSDB_CODE_SUCCESS;
-  }
-}
-
-int32_t schemaIdxCompar(const void *lhs, const void *rhs);
-int32_t boundIdxCompar(const void *lhs, const void *rhs);
-void    setBoundColumnInfo(SParsedDataColInfo *pColList, SSchema *pSchema, col_id_t numOfCols);
-void    destroyBlockArrayList(SArray *pDataBlockList);
-void    destroyBlockHashmap(SHashObj *pDataBlockHash);
-int     initRowBuilder(SRowBuilder *pBuilder, int16_t schemaVer, SParsedDataColInfo *pColInfo);
-int32_t allocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int32_t *numOfRows);
-int32_t getDataBlockFromList(SHashObj *pHashList, void *id, int32_t idLen, int32_t size, int32_t startOffset,
-                             int32_t rowSize, STableMeta *pTableMeta, STableDataBlocks **dataBlocks, SArray *pBlockList,
-                             SVCreateTbReq *pCreateTbReq);
-int32_t mergeTableDataBlocks(SHashObj *pHashObj, uint8_t payloadType, SArray **pVgDataBlocks);
-int32_t buildCreateTbMsg(STableDataBlocks *pBlocks, SVCreateTbReq *pCreateTbReq);
-
-int32_t allocateMemForSize(STableDataBlocks *pDataBlock, int32_t allSize);
-
-#endif  // TDENGINE_DATABLOCKMGT_H
+int32_t insGetExtendedRowSize(STableDataBlocks *pBlock);
+void insGetSTSRowAppendInfo(uint8_t rowType, SParsedDataColInfo *spd, col_id_t idx, int32_t *toffset, col_id_t *colIdx);
+int32_t insSetBlockInfo(SSubmitBlk *pBlocks, STableDataBlocks *dataBuf, int32_t numOfRows);
+int32_t insSchemaIdxCompar(const void *lhs, const void *rhs);
+int32_t insBoundIdxCompar(const void *lhs, const void *rhs);
+void    insSetBoundColumnInfo(SParsedDataColInfo *pColList, SSchema *pSchema, col_id_t numOfCols);
+void    insDestroyBlockArrayList(SArray *pDataBlockList);
+void    insDestroyBlockHashmap(SHashObj *pDataBlockHash);
+int32_t insInitRowBuilder(SRowBuilder *pBuilder, int16_t schemaVer, SParsedDataColInfo *pColInfo);
+int32_t insGetDataBlockFromList(SHashObj *pHashList, void *id, int32_t idLen, int32_t size, int32_t startOffset,
+                                int32_t rowSize, STableMeta *pTableMeta, STableDataBlocks **dataBlocks,
+                                SArray *pBlockList, SVCreateTbReq *pCreateTbReq);
+int32_t insMergeTableDataBlocks(SHashObj *pHashObj, uint8_t payloadType, SArray **pVgDataBlocks);
+int32_t insBuildCreateTbMsg(STableDataBlocks *pBlocks, SVCreateTbReq *pCreateTbReq);
+int32_t insAllocateMemForSize(STableDataBlocks *pDataBlock, int32_t allSize);
+int32_t insCreateSName(SName *pName, struct SToken *pTableName, int32_t acctId, const char *dbName, SMsgBuf *pMsgBuf);
+int32_t insFindCol(struct SToken *pColname, int32_t start, int32_t end, SSchema *pSchema);
+void    insBuildCreateTbReq(SVCreateTbReq *pTbReq, const char *tname, STag *pTag, int64_t suid, const char *sname,
+                            SArray *tagName, uint8_t tagNum);
+int32_t insMemRowAppend(SMsgBuf *pMsgBuf, const void *value, int32_t len, void *param);
+int32_t insCheckTimestamp(STableDataBlocks *pDataBlocks, const char *start);
+int32_t insBuildOutput(SInsertParseContext *pCxt);
+void    insDestroyDataBlock(STableDataBlocks *pDataBlock);
+
+#endif  // TDENGINE_PAR_INSERT_UTIL_H

source/libs/parser/src/parInsertSml.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "parInsertUtil.h"
+#include "parInt.h"
+#include "parToken.h"
+#include "ttime.h"
+
+int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char* dbName, char* msgBuf,
+                     int32_t msgBufLen) {
+  SMsgBuf msg = {.buf = msgBuf, .len = msgBufLen};
+  SToken  sToken;
+  int32_t code = 0;
+  char*   tbName = NULL;
+
+  NEXT_TOKEN(pTableName, sToken);
+
+  if (sToken.n == 0) {
+    return buildInvalidOperationMsg(&msg, "empty table name");
+  }
+
+  code = insCreateSName(pName, &sToken, acctId, dbName, &msg);
+  if (code) {
+    return code;
+  }
+
+  NEXT_TOKEN(pTableName, sToken);
+
+  if (sToken.n > 0) {
+    return buildInvalidOperationMsg(&msg, "table name format is wrong");
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+typedef struct SmlExecTableHandle {
+  SParsedDataColInfo tags;          // each table
+  SVCreateTbReq      createTblReq;  // each table
+} SmlExecTableHandle;
+
+typedef struct SmlExecHandle {
+  SHashObj*          pBlockHash;
+  SmlExecTableHandle tableExecHandle;
+  SQuery*            pQuery;
+} SSmlExecHandle;
+
+static void smlDestroyTableHandle(void* pHandle) {
+  SmlExecTableHandle* handle = (SmlExecTableHandle*)pHandle;
+  destroyBoundColumnInfo(&handle->tags);
+  tdDestroySVCreateTbReq(&handle->createTblReq);
+}
+
+static int32_t smlBoundColumnData(SArray* cols, SParsedDataColInfo* pColList, SSchema* pSchema, bool isTag) {
+  col_id_t nCols = pColList->numOfCols;
+
+  pColList->numOfBound = 0;
+  pColList->boundNullLen = 0;
+  memset(pColList->boundColumns, 0, sizeof(col_id_t) * nCols);
+  for (col_id_t i = 0; i < nCols; ++i) {
+    pColList->cols[i].valStat = VAL_STAT_NONE;
+  }
+
+  bool     isOrdered = true;
+  col_id_t lastColIdx = -1;  // last column found
+  for (int i = 0; i < taosArrayGetSize(cols); ++i) {
+    SSmlKv*  kv = taosArrayGetP(cols, i);
+    SToken   sToken = {.n = kv->keyLen, .z = (char*)kv->key};
+    col_id_t t = lastColIdx + 1;
+    col_id_t index = ((t == 0 && !isTag) ? 0 : insFindCol(&sToken, t, nCols, pSchema));
+    uDebug("SML, index:%d, t:%d, ncols:%d", index, t, nCols);
+    if (index < 0 && t > 0) {
+      index = insFindCol(&sToken, 0, t, pSchema);
+      isOrdered = false;
+    }
+    if (index < 0) {
+      uError("smlBoundColumnData. index:%d", index);
+      return TSDB_CODE_SML_INVALID_DATA;
+    }
+    if (pColList->cols[index].valStat == VAL_STAT_HAS) {
+      uError("smlBoundColumnData. already set. index:%d", index);
+      return TSDB_CODE_SML_INVALID_DATA;
+    }
+    lastColIdx = index;
+    pColList->cols[index].valStat = VAL_STAT_HAS;
+    pColList->boundColumns[pColList->numOfBound] = index;
+    ++pColList->numOfBound;
+    switch (pSchema[t].type) {
+      case TSDB_DATA_TYPE_BINARY:
+        pColList->boundNullLen += (sizeof(VarDataOffsetT) + VARSTR_HEADER_SIZE + CHAR_BYTES);
+        break;
+      case TSDB_DATA_TYPE_NCHAR:
+        pColList->boundNullLen += (sizeof(VarDataOffsetT) + VARSTR_HEADER_SIZE + TSDB_NCHAR_SIZE);
+        break;
+      default:
+        pColList->boundNullLen += TYPE_BYTES[pSchema[t].type];
+        break;
+    }
+  }
+
+  pColList->orderStatus = isOrdered ? ORDER_STATUS_ORDERED : ORDER_STATUS_DISORDERED;
+
+  if (!isOrdered) {
+    pColList->colIdxInfo = taosMemoryCalloc(pColList->numOfBound, sizeof(SBoundIdxInfo));
+    if (NULL == pColList->colIdxInfo) {
+      return TSDB_CODE_TSC_OUT_OF_MEMORY;
+    }
+    SBoundIdxInfo* pColIdx = pColList->colIdxInfo;
+    for (col_id_t i = 0; i < pColList->numOfBound; ++i) {
+      pColIdx[i].schemaColIdx = pColList->boundColumns[i];
+      pColIdx[i].boundIdx = i;
+    }
+    taosSort(pColIdx, pColList->numOfBound, sizeof(SBoundIdxInfo), insSchemaIdxCompar);
+    for (col_id_t i = 0; i < pColList->numOfBound; ++i) {
+      pColIdx[i].finalIdx = i;
+    }
+    taosSort(pColIdx, pColList->numOfBound, sizeof(SBoundIdxInfo), insBoundIdxCompar);
+  }
+
+  if (pColList->numOfCols > pColList->numOfBound) {
+    memset(&pColList->boundColumns[pColList->numOfBound], 0,
+           sizeof(col_id_t) * (pColList->numOfCols - pColList->numOfBound));
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+/**
+ * @brief No json tag for schemaless
+ *
+ * @param cols
+ * @param tags
+ * @param pSchema
+ * @param ppTag
+ * @param msg
+ * @return int32_t
+ */
+static int32_t smlBuildTagRow(SArray* cols, SParsedDataColInfo* tags, SSchema* pSchema, STag** ppTag, SArray** tagName,
+                              SMsgBuf* msg) {
+  SArray* pTagArray = taosArrayInit(tags->numOfBound, sizeof(STagVal));
+  if (!pTagArray) {
+    return TSDB_CODE_TSC_OUT_OF_MEMORY;
+  }
+  *tagName = taosArrayInit(8, TSDB_COL_NAME_LEN);
+  if (!*tagName) {
+    return TSDB_CODE_TSC_OUT_OF_MEMORY;
+  }
+
+  int32_t code = TSDB_CODE_SUCCESS;
+  for (int i = 0; i < tags->numOfBound; ++i) {
+    SSchema* pTagSchema = &pSchema[tags->boundColumns[i]];
+    SSmlKv*  kv = taosArrayGetP(cols, i);
+
+    taosArrayPush(*tagName, pTagSchema->name);
+    STagVal val = {.cid = pTagSchema->colId, .type = pTagSchema->type};
+    //    strcpy(val.colName, pTagSchema->name);
+    if (pTagSchema->type == TSDB_DATA_TYPE_BINARY) {
+      val.pData = (uint8_t*)kv->value;
+      val.nData = kv->length;
+    } else if (pTagSchema->type == TSDB_DATA_TYPE_NCHAR) {
+      int32_t output = 0;
+      void*   p = taosMemoryCalloc(1, kv->length * TSDB_NCHAR_SIZE);
+      if (p == NULL) {
+        code = TSDB_CODE_OUT_OF_MEMORY;
+        goto end;
+      }
+      if (!taosMbsToUcs4(kv->value, kv->length, (TdUcs4*)(p), kv->length * TSDB_NCHAR_SIZE, &output)) {
+        if (errno == E2BIG) {
+          taosMemoryFree(p);
+          code = generateSyntaxErrMsg(msg, TSDB_CODE_PAR_VALUE_TOO_LONG, pTagSchema->name);
+          goto end;
+        }
+        char buf[512] = {0};
+        snprintf(buf, tListLen(buf), " taosMbsToUcs4 error:%s", strerror(errno));
+        taosMemoryFree(p);
+        code = buildSyntaxErrMsg(msg, buf, kv->value);
+        goto end;
+      }
+      val.pData = p;
+      val.nData = output;
+    } else {
+      memcpy(&val.i64, &(kv->value), kv->length);
+    }
+    taosArrayPush(pTagArray, &val);
+  }
+
+  code = tTagNew(pTagArray, 1, false, ppTag);
+end:
+  for (int i = 0; i < taosArrayGetSize(pTagArray); ++i) {
+    STagVal* p = (STagVal*)taosArrayGet(pTagArray, i);
+    if (p->type == TSDB_DATA_TYPE_NCHAR) {
+      taosMemoryFree(p->pData);
+    }
+  }
+  taosArrayDestroy(pTagArray);
+  return code;
+}
+
+int32_t smlBindData(void* handle, SArray* tags, SArray* colsSchema, SArray* cols, bool format, STableMeta* pTableMeta,
+                    char* tableName, const char* sTableName, int32_t sTableNameLen, char* msgBuf, int16_t msgBufLen) {
+  SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen};
+
+  SSmlExecHandle* smlHandle = (SSmlExecHandle*)handle;
+  smlDestroyTableHandle(&smlHandle->tableExecHandle);  // free for each table
+  SSchema* pTagsSchema = getTableTagSchema(pTableMeta);
+  insSetBoundColumnInfo(&smlHandle->tableExecHandle.tags, pTagsSchema, getNumOfTags(pTableMeta));
+  int ret = smlBoundColumnData(tags, &smlHandle->tableExecHandle.tags, pTagsSchema, true);
+  if (ret != TSDB_CODE_SUCCESS) {
+    buildInvalidOperationMsg(&pBuf, "bound tags error");
+    return ret;
+  }
+  STag*   pTag = NULL;
+  SArray* tagName = NULL;
+  ret = smlBuildTagRow(tags, &smlHandle->tableExecHandle.tags, pTagsSchema, &pTag, &tagName, &pBuf);
+  if (ret != TSDB_CODE_SUCCESS) {
+    taosArrayDestroy(tagName);
+    return ret;
+  }
+
+  insBuildCreateTbReq(&smlHandle->tableExecHandle.createTblReq, tableName, pTag, pTableMeta->suid, NULL, tagName,
+                      pTableMeta->tableInfo.numOfTags);
+  taosArrayDestroy(tagName);
+
+  smlHandle->tableExecHandle.createTblReq.ctb.stbName = taosMemoryMalloc(sTableNameLen + 1);
+  memcpy(smlHandle->tableExecHandle.createTblReq.ctb.stbName, sTableName, sTableNameLen);
+  smlHandle->tableExecHandle.createTblReq.ctb.stbName[sTableNameLen] = 0;
+
+  STableDataBlocks* pDataBlock = NULL;
+  ret = insGetDataBlockFromList(smlHandle->pBlockHash, &pTableMeta->uid, sizeof(pTableMeta->uid),
+                                TSDB_DEFAULT_PAYLOAD_SIZE, sizeof(SSubmitBlk), getTableInfo(pTableMeta).rowSize,
+                                pTableMeta, &pDataBlock, NULL, &smlHandle->tableExecHandle.createTblReq);
+  if (ret != TSDB_CODE_SUCCESS) {
+    buildInvalidOperationMsg(&pBuf, "create data block error");
+    return ret;
+  }
+
+  SSchema* pSchema = getTableColumnSchema(pTableMeta);
+
+  ret = smlBoundColumnData(colsSchema, &pDataBlock->boundColumnInfo, pSchema, false);
+  if (ret != TSDB_CODE_SUCCESS) {
+    buildInvalidOperationMsg(&pBuf, "bound cols error");
+    return ret;
+  }
+  int32_t             extendedRowSize = insGetExtendedRowSize(pDataBlock);
+  SParsedDataColInfo* spd = &pDataBlock->boundColumnInfo;
+  SRowBuilder*        pBuilder = &pDataBlock->rowBuilder;
+  SMemParam           param = {.rb = pBuilder};
+
+  insInitRowBuilder(&pDataBlock->rowBuilder, pDataBlock->pTableMeta->sversion, &pDataBlock->boundColumnInfo);
+
+  int32_t rowNum = taosArrayGetSize(cols);
+  if (rowNum <= 0) {
+    return buildInvalidOperationMsg(&pBuf, "cols size <= 0");
+  }
+  ret = insAllocateMemForSize(pDataBlock, extendedRowSize * rowNum);
+  if (ret != TSDB_CODE_SUCCESS) {
+    buildInvalidOperationMsg(&pBuf, "allocate memory error");
+    return ret;
+  }
+  for (int32_t r = 0; r < rowNum; ++r) {
+    STSRow* row = (STSRow*)(pDataBlock->pData + pDataBlock->size);  // skip the SSubmitBlk header
+    tdSRowResetBuf(pBuilder, row);
+    void*  rowData = taosArrayGetP(cols, r);
+    size_t rowDataSize = 0;
+    if (format) {
+      rowDataSize = taosArrayGetSize(rowData);
+    }
+
+    // 1. set the parsed value from sql string
+    for (int c = 0, j = 0; c < spd->numOfBound; ++c) {
+      SSchema* pColSchema = &pSchema[spd->boundColumns[c]];
+
+      param.schema = pColSchema;
+      insGetSTSRowAppendInfo(pBuilder->rowType, spd, c, &param.toffset, &param.colIdx);
+
+      SSmlKv* kv = NULL;
+      if (format) {
+        if (j < rowDataSize) {
+          kv = taosArrayGetP(rowData, j);
+          if (rowDataSize != spd->numOfBound && j != 0 &&
+              (kv->keyLen != strlen(pColSchema->name) || strncmp(kv->key, pColSchema->name, kv->keyLen) != 0)) {
+            kv = NULL;
+          } else {
+            j++;
+          }
+        }
+      } else {
+        void** p = taosHashGet(rowData, pColSchema->name, strlen(pColSchema->name));
+        if (p) kv = *p;
+      }
+
+      if (kv) {
+        int32_t colLen = kv->length;
+        if (pColSchema->type == TSDB_DATA_TYPE_TIMESTAMP) {
+          //          uError("SML:data before:%" PRId64 ", precision:%d", kv->i, pTableMeta->tableInfo.precision);
+          kv->i = convertTimePrecision(kv->i, TSDB_TIME_PRECISION_NANO, pTableMeta->tableInfo.precision);
+          //          uError("SML:data after:%" PRId64 ", precision:%d", kv->i, pTableMeta->tableInfo.precision);
+        }
+
+        if (IS_VAR_DATA_TYPE(kv->type)) {
+          insMemRowAppend(&pBuf, kv->value, colLen, &param);
+        } else {
+          insMemRowAppend(&pBuf, &(kv->value), colLen, &param);
+        }
+      } else {
+        pBuilder->hasNone = true;
+      }
+
+      if (PRIMARYKEY_TIMESTAMP_COL_ID == pColSchema->colId) {
+        TSKEY tsKey = TD_ROW_KEY(row);
+        insCheckTimestamp(pDataBlock, (const char*)&tsKey);
+      }
+    }
+
+    // set the null value for the columns that do not assign values
+    if ((spd->numOfBound < spd->numOfCols) && TD_IS_TP_ROW(row)) {
+      pBuilder->hasNone = true;
+    }
+
+    tdSRowEnd(pBuilder);
+    pDataBlock->size += extendedRowSize;
+  }
+
+  SSubmitBlk* pBlocks = (SSubmitBlk*)(pDataBlock->pData);
+  if (TSDB_CODE_SUCCESS != insSetBlockInfo(pBlocks, pDataBlock, rowNum)) {
+    return buildInvalidOperationMsg(&pBuf, "too many rows in sql, total number of rows should be less than INT32_MAX");
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+void* smlInitHandle(SQuery* pQuery) {
+  SSmlExecHandle* handle = taosMemoryCalloc(1, sizeof(SSmlExecHandle));
+  if (!handle) return NULL;
+  handle->pBlockHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false);
+  handle->pQuery = pQuery;
+
+  return handle;
+}
+
+void smlDestroyHandle(void* pHandle) {
+  if (!pHandle) return;
+  SSmlExecHandle* handle = (SSmlExecHandle*)pHandle;
+  insDestroyBlockHashmap(handle->pBlockHash);
+  smlDestroyTableHandle(&handle->tableExecHandle);
+  taosMemoryFree(handle);
+}
+
+int32_t smlBuildOutput(void* handle, SHashObj* pVgHash) {
+  SSmlExecHandle* smlHandle = (SSmlExecHandle*)handle;
+  return qBuildStmtOutput(smlHandle->pQuery, pVgHash, smlHandle->pBlockHash);
+}

source/libs/parser/src/parInsertStmt.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "os.h"
+#include "parInsertUtil.h"
+#include "parInt.h"
+#include "parToken.h"
+#include "query.h"
+#include "tglobal.h"
+#include "ttime.h"
+#include "ttypes.h"
+
+typedef struct SKvParam {
+  int16_t  pos;
+  SArray*  pTagVals;
+  SSchema* schema;
+  char     buf[TSDB_MAX_TAGS_LEN];
+} SKvParam;
+
+int32_t qBuildStmtOutput(SQuery* pQuery, SHashObj* pVgHash, SHashObj* pBlockHash) {
+  SVnodeModifOpStmt*  modifyNode = (SVnodeModifOpStmt*)pQuery->pRoot;
+  int32_t             code = 0;
+  SInsertParseContext insertCtx = {
+      .pVgroupsHashObj = pVgHash,
+      .pTableBlockHashObj = pBlockHash,
+      .pOutput = (SVnodeModifOpStmt*)pQuery->pRoot,
+  };
+
+  // merge according to vgId
+  if (taosHashGetSize(insertCtx.pTableBlockHashObj) > 0) {
+    CHECK_CODE(
+        insMergeTableDataBlocks(insertCtx.pTableBlockHashObj, modifyNode->payloadType, &insertCtx.pVgDataBlocks));
+  }
+
+  CHECK_CODE(insBuildOutput(&insertCtx));
+
+  insDestroyBlockArrayList(insertCtx.pVgDataBlocks);
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, const char* sTableName, char* tName,
+                           TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen) {
+  STableDataBlocks*   pDataBlock = (STableDataBlocks*)pBlock;
+  SMsgBuf             pBuf = {.buf = msgBuf, .len = msgBufLen};
+  SParsedDataColInfo* tags = (SParsedDataColInfo*)boundTags;
+  if (NULL == tags) {
+    return TSDB_CODE_QRY_APP_ERROR;
+  }
+
+  SArray* pTagArray = taosArrayInit(tags->numOfBound, sizeof(STagVal));
+  if (!pTagArray) {
+    return buildInvalidOperationMsg(&pBuf, "out of memory");
+  }
+
+  SArray* tagName = taosArrayInit(8, TSDB_COL_NAME_LEN);
+  if (!tagName) {
+    return buildInvalidOperationMsg(&pBuf, "out of memory");
+  }
+
+  int32_t  code = TSDB_CODE_SUCCESS;
+  SSchema* pSchema = getTableTagSchema(pDataBlock->pTableMeta);
+
+  bool  isJson = false;
+  STag* pTag = NULL;
+
+  for (int c = 0; c < tags->numOfBound; ++c) {
+    if (bind[c].is_null && bind[c].is_null[0]) {
+      continue;
+    }
+
+    SSchema* pTagSchema = &pSchema[tags->boundColumns[c]];
+    int32_t  colLen = pTagSchema->bytes;
+    if (IS_VAR_DATA_TYPE(pTagSchema->type)) {
+      colLen = bind[c].length[0];
+    }
+    taosArrayPush(tagName, pTagSchema->name);
+    if (pTagSchema->type == TSDB_DATA_TYPE_JSON) {
+      if (colLen > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) {
+        code = buildSyntaxErrMsg(&pBuf, "json string too long than 4095", bind[c].buffer);
+        goto end;
+      }
+
+      isJson = true;
+      char* tmp = taosMemoryCalloc(1, colLen + 1);
+      memcpy(tmp, bind[c].buffer, colLen);
+      code = parseJsontoTagData(tmp, pTagArray, &pTag, &pBuf);
+      taosMemoryFree(tmp);
+      if (code != TSDB_CODE_SUCCESS) {
+        goto end;
+      }
+    } else {
+      STagVal val = {.cid = pTagSchema->colId, .type = pTagSchema->type};
+      //      strcpy(val.colName, pTagSchema->name);
+      if (pTagSchema->type == TSDB_DATA_TYPE_BINARY) {
+        val.pData = (uint8_t*)bind[c].buffer;
+        val.nData = colLen;
+      } else if (pTagSchema->type == TSDB_DATA_TYPE_NCHAR) {
+        int32_t output = 0;
+        void*   p = taosMemoryCalloc(1, colLen * TSDB_NCHAR_SIZE);
+        if (p == NULL) {
+          code = TSDB_CODE_OUT_OF_MEMORY;
+          goto end;
+        }
+        if (!taosMbsToUcs4(bind[c].buffer, colLen, (TdUcs4*)(p), colLen * TSDB_NCHAR_SIZE, &output)) {
+          if (errno == E2BIG) {
+            taosMemoryFree(p);
+            code = generateSyntaxErrMsg(&pBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pTagSchema->name);
+            goto end;
+          }
+          char buf[512] = {0};
+          snprintf(buf, tListLen(buf), " taosMbsToUcs4 error:%s", strerror(errno));
+          taosMemoryFree(p);
+          code = buildSyntaxErrMsg(&pBuf, buf, bind[c].buffer);
+          goto end;
+        }
+        val.pData = p;
+        val.nData = output;
+      } else {
+        memcpy(&val.i64, bind[c].buffer, colLen);
+      }
+      taosArrayPush(pTagArray, &val);
+    }
+  }
+
+  if (!isJson && (code = tTagNew(pTagArray, 1, false, &pTag)) != TSDB_CODE_SUCCESS) {
+    goto end;
+  }
+
+  SVCreateTbReq tbReq = {0};
+  insBuildCreateTbReq(&tbReq, tName, pTag, suid, sTableName, tagName, pDataBlock->pTableMeta->tableInfo.numOfTags);
+  code = insBuildCreateTbMsg(pDataBlock, &tbReq);
+  tdDestroySVCreateTbReq(&tbReq);
+
+end:
+  for (int i = 0; i < taosArrayGetSize(pTagArray); ++i) {
+    STagVal* p = (STagVal*)taosArrayGet(pTagArray, i);
+    if (p->type == TSDB_DATA_TYPE_NCHAR) {
+      taosMemoryFreeClear(p->pData);
+    }
+  }
+  taosArrayDestroy(pTagArray);
+  taosArrayDestroy(tagName);
+
+  return code;
+}
+
+int32_t qBindStmtColsValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen) {
+  STableDataBlocks*   pDataBlock = (STableDataBlocks*)pBlock;
+  SSchema*            pSchema = getTableColumnSchema(pDataBlock->pTableMeta);
+  int32_t             extendedRowSize = insGetExtendedRowSize(pDataBlock);
+  SParsedDataColInfo* spd = &pDataBlock->boundColumnInfo;
+  SRowBuilder*        pBuilder = &pDataBlock->rowBuilder;
+  SMemParam           param = {.rb = pBuilder};
+  SMsgBuf             pBuf = {.buf = msgBuf, .len = msgBufLen};
+  int32_t             rowNum = bind->num;
+
+  CHECK_CODE(
+      insInitRowBuilder(&pDataBlock->rowBuilder, pDataBlock->pTableMeta->sversion, &pDataBlock->boundColumnInfo));
+
+  CHECK_CODE(insAllocateMemForSize(pDataBlock, extendedRowSize * bind->num));
+
+  for (int32_t r = 0; r < bind->num; ++r) {
+    STSRow* row = (STSRow*)(pDataBlock->pData + pDataBlock->size);  // skip the SSubmitBlk header
+    tdSRowResetBuf(pBuilder, row);
+
+    for (int c = 0; c < spd->numOfBound; ++c) {
+      SSchema* pColSchema = &pSchema[spd->boundColumns[c]];
+
+      if (bind[c].num != rowNum) {
+        return buildInvalidOperationMsg(&pBuf, "row number in each bind param should be the same");
+      }
+
+      param.schema = pColSchema;
+      insGetSTSRowAppendInfo(pBuilder->rowType, spd, c, &param.toffset, &param.colIdx);
+
+      if (bind[c].is_null && bind[c].is_null[r]) {
+        if (pColSchema->colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
+          return buildInvalidOperationMsg(&pBuf, "primary timestamp should not be NULL");
+        }
+
+        CHECK_CODE(insMemRowAppend(&pBuf, NULL, 0, &param));
+      } else {
+        if (bind[c].buffer_type != pColSchema->type) {
+          return buildInvalidOperationMsg(&pBuf, "column type mis-match with buffer type");
+        }
+
+        int32_t colLen = pColSchema->bytes;
+        if (IS_VAR_DATA_TYPE(pColSchema->type)) {
+          colLen = bind[c].length[r];
+        }
+
+        CHECK_CODE(insMemRowAppend(&pBuf, (char*)bind[c].buffer + bind[c].buffer_length * r, colLen, &param));
+      }
+
+      if (PRIMARYKEY_TIMESTAMP_COL_ID == pColSchema->colId) {
+        TSKEY tsKey = TD_ROW_KEY(row);
+        insCheckTimestamp(pDataBlock, (const char*)&tsKey);
+      }
+    }
+    // set the null value for the columns that do not assign values
+    if ((spd->numOfBound < spd->numOfCols) && TD_IS_TP_ROW(row)) {
+      pBuilder->hasNone = true;
+    }
+    tdSRowEnd(pBuilder);
+#ifdef TD_DEBUG_PRINT_ROW
+    STSchema* pSTSchema = tdGetSTSChemaFromSSChema(pSchema, spd->numOfCols, 1);
+    tdSRowPrint(row, pSTSchema, __func__);
+    taosMemoryFree(pSTSchema);
+#endif
+    pDataBlock->size += extendedRowSize;
+  }
+
+  SSubmitBlk* pBlocks = (SSubmitBlk*)(pDataBlock->pData);
+  if (TSDB_CODE_SUCCESS != insSetBlockInfo(pBlocks, pDataBlock, bind->num)) {
+    return buildInvalidOperationMsg(&pBuf, "too many rows in sql, total number of rows should be less than INT32_MAX");
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t qBindStmtSingleColValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen, int32_t colIdx,
+                                int32_t rowNum) {
+  STableDataBlocks*   pDataBlock = (STableDataBlocks*)pBlock;
+  SSchema*            pSchema = getTableColumnSchema(pDataBlock->pTableMeta);
+  int32_t             extendedRowSize = insGetExtendedRowSize(pDataBlock);
+  SParsedDataColInfo* spd = &pDataBlock->boundColumnInfo;
+  SRowBuilder*        pBuilder = &pDataBlock->rowBuilder;
+  SMemParam           param = {.rb = pBuilder};
+  SMsgBuf             pBuf = {.buf = msgBuf, .len = msgBufLen};
+  bool                rowStart = (0 == colIdx);
+  bool                rowEnd = ((colIdx + 1) == spd->numOfBound);
+
+  if (rowStart) {
+    CHECK_CODE(
+        insInitRowBuilder(&pDataBlock->rowBuilder, pDataBlock->pTableMeta->sversion, &pDataBlock->boundColumnInfo));
+    CHECK_CODE(insAllocateMemForSize(pDataBlock, extendedRowSize * bind->num));
+  }
+
+  for (int32_t r = 0; r < bind->num; ++r) {
+    STSRow* row = (STSRow*)(pDataBlock->pData + pDataBlock->size + extendedRowSize * r);  // skip the SSubmitBlk header
+    if (rowStart) {
+      tdSRowResetBuf(pBuilder, row);
+    } else {
+      tdSRowGetBuf(pBuilder, row);
+    }
+
+    SSchema* pColSchema = &pSchema[spd->boundColumns[colIdx]];
+
+    if (bind->num != rowNum) {
+      return buildInvalidOperationMsg(&pBuf, "row number in each bind param should be the same");
+    }
+
+    param.schema = pColSchema;
+    insGetSTSRowAppendInfo(pBuilder->rowType, spd, colIdx, &param.toffset, &param.colIdx);
+
+    if (bind->is_null && bind->is_null[r]) {
+      if (pColSchema->colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
+        return buildInvalidOperationMsg(&pBuf, "primary timestamp should not be NULL");
+      }
+
+      CHECK_CODE(insMemRowAppend(&pBuf, NULL, 0, &param));
+    } else {
+      if (bind->buffer_type != pColSchema->type) {
+        return buildInvalidOperationMsg(&pBuf, "column type mis-match with buffer type");
+      }
+
+      int32_t colLen = pColSchema->bytes;
+      if (IS_VAR_DATA_TYPE(pColSchema->type)) {
+        colLen = bind->length[r];
+      }
+
+      CHECK_CODE(insMemRowAppend(&pBuf, (char*)bind->buffer + bind->buffer_length * r, colLen, &param));
+    }
+
+    if (PRIMARYKEY_TIMESTAMP_COL_ID == pColSchema->colId) {
+      TSKEY tsKey = TD_ROW_KEY(row);
+      insCheckTimestamp(pDataBlock, (const char*)&tsKey);
+    }
+
+    // set the null value for the columns that do not assign values
+    if (rowEnd && (spd->numOfBound < spd->numOfCols) && TD_IS_TP_ROW(row)) {
+      pBuilder->hasNone = true;
+    }
+    if (rowEnd) {
+      tdSRowEnd(pBuilder);
+    }
+#ifdef TD_DEBUG_PRINT_ROW
+    if (rowEnd) {
+      STSchema* pSTSchema = tdGetSTSChemaFromSSChema(pSchema, spd->numOfCols, 1);
+      tdSRowPrint(row, pSTSchema, __func__);
+      taosMemoryFree(pSTSchema);
+    }
+#endif
+  }
+
+  if (rowEnd) {
+    pDataBlock->size += extendedRowSize * bind->num;
+
+    SSubmitBlk* pBlocks = (SSubmitBlk*)(pDataBlock->pData);
+    if (TSDB_CODE_SUCCESS != insSetBlockInfo(pBlocks, pDataBlock, bind->num)) {
+      return buildInvalidOperationMsg(&pBuf,
+                                      "too many rows in sql, total number of rows should be less than INT32_MAX");
+    }
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t buildBoundFields(SParsedDataColInfo* boundInfo, SSchema* pSchema, int32_t* fieldNum, TAOS_FIELD_E** fields,
+                         uint8_t timePrec) {
+  if (fields) {
+    *fields = taosMemoryCalloc(boundInfo->numOfBound, sizeof(TAOS_FIELD));
+    if (NULL == *fields) {
+      return TSDB_CODE_OUT_OF_MEMORY;
+    }
+
+    SSchema* schema = &pSchema[boundInfo->boundColumns[0]];
+    if (TSDB_DATA_TYPE_TIMESTAMP == schema->type) {
+      (*fields)[0].precision = timePrec;
+    }
+
+    for (int32_t i = 0; i < boundInfo->numOfBound; ++i) {
+      schema = &pSchema[boundInfo->boundColumns[i]];
+      strcpy((*fields)[i].name, schema->name);
+      (*fields)[i].type = schema->type;
+      (*fields)[i].bytes = schema->bytes;
+    }
+  }
+
+  *fieldNum = boundInfo->numOfBound;
+
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t qBuildStmtTagFields(void* pBlock, void* boundTags, int32_t* fieldNum, TAOS_FIELD_E** fields) {
+  STableDataBlocks*   pDataBlock = (STableDataBlocks*)pBlock;
+  SParsedDataColInfo* tags = (SParsedDataColInfo*)boundTags;
+  if (NULL == tags) {
+    return TSDB_CODE_QRY_APP_ERROR;
+  }
+
+  if (pDataBlock->pTableMeta->tableType != TSDB_SUPER_TABLE && pDataBlock->pTableMeta->tableType != TSDB_CHILD_TABLE) {
+    return TSDB_CODE_TSC_STMT_API_ERROR;
+  }
+
+  SSchema* pSchema = getTableTagSchema(pDataBlock->pTableMeta);
+  if (tags->numOfBound <= 0) {
+    *fieldNum = 0;
+    *fields = NULL;
+
+    return TSDB_CODE_SUCCESS;
+  }
+
+  CHECK_CODE(buildBoundFields(tags, pSchema, fieldNum, fields, 0));
+
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t qBuildStmtColFields(void* pBlock, int32_t* fieldNum, TAOS_FIELD_E** fields) {
+  STableDataBlocks* pDataBlock = (STableDataBlocks*)pBlock;
+  SSchema*          pSchema = getTableColumnSchema(pDataBlock->pTableMeta);
+  if (pDataBlock->boundColumnInfo.numOfBound <= 0) {
+    *fieldNum = 0;
+    if (fields) {
+      *fields = NULL;
+    }
+
+    return TSDB_CODE_SUCCESS;
+  }
+
+  CHECK_CODE(buildBoundFields(&pDataBlock->boundColumnInfo, pSchema, fieldNum, fields,
+                              pDataBlock->pTableMeta->tableInfo.precision));
+
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t qResetStmtDataBlock(void* block, bool keepBuf) {
+  STableDataBlocks* pBlock = (STableDataBlocks*)block;
+
+  if (keepBuf) {
+    taosMemoryFreeClear(pBlock->pData);
+    pBlock->pData = taosMemoryMalloc(TSDB_PAYLOAD_SIZE);
+    if (NULL == pBlock->pData) {
+      return TSDB_CODE_OUT_OF_MEMORY;
+    }
+    memset(pBlock->pData, 0, sizeof(SSubmitBlk));
+  } else {
+    pBlock->pData = NULL;
+  }
+
+  pBlock->ordered = true;
+  pBlock->prevTS = INT64_MIN;
+  pBlock->size = sizeof(SSubmitBlk);
+  pBlock->tsSource = -1;
+  pBlock->numOfTables = 1;
+  pBlock->nAllocSize = TSDB_PAYLOAD_SIZE;
+  pBlock->headerSize = pBlock->size;
+  pBlock->createTbReqLen = 0;
+
+  memset(&pBlock->rowBuilder, 0, sizeof(pBlock->rowBuilder));
+
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t qCloneStmtDataBlock(void** pDst, void* pSrc) {
+  *pDst = taosMemoryMalloc(sizeof(STableDataBlocks));
+  if (NULL == *pDst) {
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+
+  memcpy(*pDst, pSrc, sizeof(STableDataBlocks));
+  ((STableDataBlocks*)(*pDst))->cloned = true;
+
+  STableDataBlocks* pBlock = (STableDataBlocks*)(*pDst);
+  if (pBlock->pTableMeta) {
+    void* pNewMeta = taosMemoryMalloc(TABLE_META_SIZE(pBlock->pTableMeta));
+    if (NULL == pNewMeta) {
+      taosMemoryFreeClear(*pDst);
+      return TSDB_CODE_OUT_OF_MEMORY;
+    }
+    memcpy(pNewMeta, pBlock->pTableMeta, TABLE_META_SIZE(pBlock->pTableMeta));
+    pBlock->pTableMeta = pNewMeta;
+  }
+
+  return qResetStmtDataBlock(*pDst, false);
+}
+
+int32_t qRebuildStmtDataBlock(void** pDst, void* pSrc, uint64_t uid, int32_t vgId) {
+  int32_t code = qCloneStmtDataBlock(pDst, pSrc);
+  if (code) {
+    return code;
+  }
+
+  STableDataBlocks* pBlock = (STableDataBlocks*)*pDst;
+  pBlock->pData = taosMemoryMalloc(pBlock->nAllocSize);
+  if (NULL == pBlock->pData) {
+    qFreeStmtDataBlock(pBlock);
+    return TSDB_CODE_OUT_OF_MEMORY;
+  }
+
+  pBlock->vgId = vgId;
+
+  if (pBlock->pTableMeta) {
+    pBlock->pTableMeta->uid = uid;
+    pBlock->pTableMeta->vgId = vgId;
+  }
+
+  memset(pBlock->pData, 0, sizeof(SSubmitBlk));
+
+  return TSDB_CODE_SUCCESS;
+}
+
+STableMeta* qGetTableMetaInDataBlock(void* pDataBlock) { return ((STableDataBlocks*)pDataBlock)->pTableMeta; }
+
+void qFreeStmtDataBlock(void* pDataBlock) {
+  if (pDataBlock == NULL) {
+    return;
+  }
+
+  taosMemoryFreeClear(((STableDataBlocks*)pDataBlock)->pTableMeta);
+  taosMemoryFreeClear(((STableDataBlocks*)pDataBlock)->pData);
+  taosMemoryFreeClear(pDataBlock);
+}
+
+void qDestroyStmtDataBlock(void* pBlock) {
+  if (pBlock == NULL) {
+    return;
+  }
+
+  STableDataBlocks* pDataBlock = (STableDataBlocks*)pBlock;
+
+  pDataBlock->cloned = false;
+  insDestroyDataBlock(pDataBlock);
+}

source/libs/sync/src/syncAppendEntries.c

@@ -383,6 +383,8 @@ int32_t syncNodeOnAppendEntries(SSyncNode* ths, SyncAppendEntries* pMsg) {
           snprintf(logBuf, sizeof(logBuf), "ignore, truncate error, append-index:%" PRId64, appendIndex);
           syncLogRecvAppendEntries(ths, pMsg, logBuf);
 
+          syncEntryDestory(pLocalEntry);
+          syncEntryDestory(pAppendEntry);
           goto _IGNORE;
         }
 
@@ -393,6 +395,8 @@ int32_t syncNodeOnAppendEntries(SSyncNode* ths, SyncAppendEntries* pMsg) {
           snprintf(logBuf, sizeof(logBuf), "ignore, append error, append-index:%" PRId64, appendIndex);
           syncLogRecvAppendEntries(ths, pMsg, logBuf);
 
+          syncEntryDestory(pLocalEntry);
+          syncEntryDestory(pAppendEntry);
           goto _IGNORE;
         }
       }
@@ -408,6 +412,8 @@ int32_t syncNodeOnAppendEntries(SSyncNode* ths, SyncAppendEntries* pMsg) {
           snprintf(logBuf, sizeof(logBuf), "ignore, log not exist, truncate error, append-index:%" PRId64, appendIndex);
           syncLogRecvAppendEntries(ths, pMsg, logBuf);
 
+          syncEntryDestory(pLocalEntry);
+          syncEntryDestory(pAppendEntry);
           goto _IGNORE;
         }
 
@@ -418,6 +424,8 @@ int32_t syncNodeOnAppendEntries(SSyncNode* ths, SyncAppendEntries* pMsg) {
           snprintf(logBuf, sizeof(logBuf), "ignore, log not exist, append error, append-index:%" PRId64, appendIndex);
           syncLogRecvAppendEntries(ths, pMsg, logBuf);
 
+          syncEntryDestory(pLocalEntry);
+          syncEntryDestory(pAppendEntry);
           goto _IGNORE;
         }
 
@@ -427,33 +435,12 @@ int32_t syncNodeOnAppendEntries(SSyncNode* ths, SyncAppendEntries* pMsg) {
         snprintf(logBuf, sizeof(logBuf), "ignore, get local entry error, append-index:%" PRId64, appendIndex);
         syncLogRecvAppendEntries(ths, pMsg, logBuf);
 
+        syncEntryDestory(pLocalEntry);
+        syncEntryDestory(pAppendEntry);
         goto _IGNORE;
       }
     }
 
-#if 0
-    if (code != 0 && terrno == TSDB_CODE_WAL_LOG_NOT_EXIST) {
-      code = ths->pLogStore->syncLogTruncate(ths->pLogStore, appendIndex);
-      ASSERT(code == 0);
-
-      code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pAppendEntry);
-      ASSERT(code == 0);
-
-    } else {
-      ASSERT(code == 0);
-
-      if (pLocalEntry->term == pAppendEntry->term) {
-        // do nothing
-      } else {
-        code = ths->pLogStore->syncLogTruncate(ths->pLogStore, appendIndex);
-        ASSERT(code == 0);
-
-        code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pAppendEntry);
-        ASSERT(code == 0);
-      }
-    }
-#endif
-
     // update match index
     pReply->matchIndex = pAppendEntry->index;
 

source/libs/sync/src/syncMain.c

@@ -336,7 +336,10 @@ char* syncNodePeerState2Str(const SSyncNode* pSyncNode) {
 
   for (int32_t i = 0; i < pSyncNode->replicaNum; ++i) {
     SPeerState* pState = syncNodeGetPeerState((SSyncNode*)pSyncNode, &(pSyncNode->replicasId[i]));
-    ASSERT(pState != NULL);
+    if (pState == NULL) {
+      sError("vgId:%d, replica maybe dropped", pSyncNode->vgId);
+      break;
+    }
 
     p = pStr + useLen;
     use = snprintf(p, leftLen, "%d:%" PRId64 " ,%" PRId64, i, pState->lastSendIndex, pState->lastSendTime);
@@ -3495,6 +3498,7 @@ SPeerState* syncNodeGetPeerState(SSyncNode* ths, const SRaftId* pDestId) {
 bool syncNodeNeedSendAppendEntries(SSyncNode* ths, const SRaftId* pDestId, const SyncAppendEntries* pMsg) {
   SPeerState* pState = syncNodeGetPeerState(ths, pDestId);
   if (pState == NULL) {
+    sError("vgId:%d, replica maybe dropped", ths->vgId);
     return false;
   }
 

source/libs/sync/src/syncReplication.c

@@ -163,7 +163,10 @@ int32_t syncNodeSendAppendEntries(SSyncNode* pSyncNode, const SRaftId* destRaftI
   syncNodeSendMsgById(destRaftId, pSyncNode, &rpcMsg);
 
   SPeerState* pState = syncNodeGetPeerState(pSyncNode, destRaftId);
-  ASSERT(pState != NULL);
+  if (pState == NULL) {
+    sError("vgId:%d, replica maybe dropped", pSyncNode->vgId);
+    return 0;
+  }
 
   if (pMsg->dataLen > 0) {
     pState->lastSendIndex = pMsg->prevLogIndex + 1;

source/libs/transport/src/thttp.c

@@ -148,14 +148,22 @@ static FORCE_INLINE void clientRecvCb(uv_stream_t* handle, ssize_t nread, const
   } else {
     uTrace("http-report succ to recv %d bytes", (int32_t)nread);
   }
-  uv_close((uv_handle_t*)&cli->tcp, clientCloseCb);
+  if (!uv_is_closing((uv_handle_t*)&cli->tcp)) {
+    uv_close((uv_handle_t*)&cli->tcp, clientCloseCb);
+  } else {
+    destroyHttpClient(cli);
+  }
 }
 static void clientSentCb(uv_write_t* req, int32_t status) {
   SHttpClient* cli = req->data;
   if (status != 0) {
     terrno = TAOS_SYSTEM_ERROR(status);
     uError("http-report failed to send data %s", uv_strerror(status));
-    uv_close((uv_handle_t*)&cli->tcp, clientCloseCb);
+    if (!uv_is_closing((uv_handle_t*)&cli->tcp)) {
+      uv_close((uv_handle_t*)&cli->tcp, clientCloseCb);
+    } else {
+      destroyHttpClient(cli);
+    }
     return;
   } else {
     uTrace("http-report succ to send data");
@@ -176,7 +184,11 @@ static void clientConnCb(uv_connect_t* req, int32_t status) {
   if (status != 0) {
     terrno = TAOS_SYSTEM_ERROR(status);
     uError("http-report failed to conn to server, reason:%s, dst:%s:%d", uv_strerror(status), cli->addr, cli->port);
-    uv_close((uv_handle_t*)&cli->tcp, clientCloseCb);
+    if (!uv_is_closing((uv_handle_t*)&cli->tcp)) {
+      uv_close((uv_handle_t*)&cli->tcp, clientCloseCb);
+    } else {
+      destroyHttpClient(cli);
+    }
     return;
   }
   status = uv_write(&cli->req, (uv_stream_t*)&cli->tcp, cli->wbuf, 2, clientSentCb);

source/os/src/osFile.c

@@ -191,7 +191,7 @@ int32_t taosRenameFile(const char *oldName, const char *newName) {
     printf("failed to rename file %s to %s, reason:%s\n", oldName, newName, strerror(errno));
   }
 
-  return !code;
+  return code ? 0 : -1;
 #else
   int32_t code = rename(oldName, newName);
   if (code < 0) {

source/util/src/tcompression.c

@@ -2274,6 +2274,9 @@ int32_t tsCompressBool(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t
     return tsCompressBoolImp(pIn, nEle, pOut);
   } else if (cmprAlg == TWO_STAGE_COMP) {
     int32_t len = tsCompressBoolImp(pIn, nEle, pBuf);
+    if (len < 0) {
+      return -1;
+    }
     return tsCompressStringImp(pBuf, len, pOut, nOut);
   } else {
     assert(0);

source/util/src/tdigest.c

@@ -263,7 +263,7 @@ double tdigestQuantile(TDigest *t, double q) {
     b = c;
     right = (b->weight * a->mean + a->weight * b->mean) / (a->weight + b->weight);
     if (idx < weight_so_far + a->weight) {
-      double p = (idx - weight_so_far) / a->weight;
+      double p = (idx - weight_so_far) / ((a->weight == 0) ? 1 : a->weight);
       return left * (1 - p) + right * p;
     }
     weight_so_far += a->weight;

source/util/src/tencode.c

@@ -109,7 +109,7 @@ void tEndEncode(SEncoder* pCoder) {
     pCoder->size = pNode->size;
     pCoder->pos = pNode->pos;
 
-    tEncodeI32(pCoder, len);
+    (void)tEncodeI32(pCoder, len);
 
     TD_CODER_MOVE_POS(pCoder, len);
   }

source/util/src/tenv.c

@@ -54,7 +54,7 @@ int32_t taosEnvNameToCfgName(const char *envNameStr, char *cfgNameStr, int32_t c
 }
 
 int32_t taosEnvToCfg(const char *envStr, char *cfgStr) {
-  if (envStr == NULL && cfgStr == NULL) {
+  if (envStr == NULL || cfgStr == NULL) {
     return -1;
   }
   if (cfgStr != envStr) strcpy(cfgStr, envStr);

source/util/src/thash.c

@@ -639,7 +639,7 @@ void taosHashTableResize(SHashObj *pHashObj) {
   }
 
   int64_t st = taosGetTimestampUs();
-  void   *pNewEntryList = taosMemoryRealloc(pHashObj->hashList, sizeof(void *) * newCapacity);
+  SHashEntry **pNewEntryList = taosMemoryRealloc(pHashObj->hashList, sizeof(SHashEntry *) * newCapacity);
   if (pNewEntryList == NULL) {
     //    uDebug("cache resize failed due to out of memory, capacity remain:%zu", pHashObj->capacity);
     return;

source/util/src/tlog.c

@@ -207,7 +207,8 @@ static void taosKeepOldLog(char *oldName) {
   char    fileName[LOG_FILE_NAME_LEN + 20];
   snprintf(fileName, LOG_FILE_NAME_LEN + 20, "%s.%" PRId64, tsLogObj.logName, fileSec);
 
-  taosRenameFile(oldName, fileName);
+  (void)taosRenameFile(oldName, fileName);
+
   if (tsLogKeepDays < 0) {
     char compressFileName[LOG_FILE_NAME_LEN + 20];
     snprintf(compressFileName, LOG_FILE_NAME_LEN + 20, "%s.%" PRId64 ".gz", tsLogObj.logName, fileSec);

source/util/src/tlrucache.c

@@ -337,6 +337,7 @@ static int taosLRUCacheShardInit(SLRUCacheShard *shard, size_t capacity, bool st
 
   taosThreadMutexInit(&shard->mutex, NULL);
 
+  taosThreadMutexLock(&shard->mutex);
   shard->capacity = 0;
   shard->highPriPoolUsage = 0;
   shard->strictCapacity = strict;
@@ -349,6 +350,7 @@ static int taosLRUCacheShardInit(SLRUCacheShard *shard, size_t capacity, bool st
   shard->lru.next = &shard->lru;
   shard->lru.prev = &shard->lru;
   shard->lruLowPri = &shard->lru;
+  taosThreadMutexUnlock(&shard->mutex);
 
   taosLRUCacheShardSetCapacity(shard, capacity);
 

source/util/src/tpagedbuf.c

@@ -534,7 +534,9 @@ void destroyDiskbasedBuf(SDiskbasedBuf* pBuf) {
     }
   }
 
-  taosRemoveFile(pBuf->path);
+  if (taosRemoveFile(pBuf->path) < 0) {
+    uDebug("WARNING tPage remove file failed. path=%s", pBuf->path);
+  }
   taosMemoryFreeClear(pBuf->path);
 
   size_t n = taosArrayGetSize(pBuf->pIdList);

tests/docs-examples-test/taosx.sh

-#!/bin/bash
-
-set -e
-
-taosd >>/dev/null 2>&1 &
-taosadapter >>/dev/null 2>&1 &
-sleep 5
-
-taosBenchmark -y -n 10000 -t 1000
-
-taosx run -f 'tmq:///test' -t 'taos:///test1' -y
-
-taos -s 'select count(*) from test1.meters' |grep 10000000 || (echo database sync failed!; exit 1)
-
-taosx run -f 'tmq:///test' -t 'local:./test-backup' -y
-taosx run -f 'local:./test-backup' -t 'taos:///test2' -y
-
-rm -rf ./test-backup

tests/parallel_test/container_build.sh

@@ -52,7 +52,7 @@ fi
 
 docker run \
     -v $REP_MOUNT_PARAM \
-    --rm --ulimit core=-1 taos_test:v1.0 sh -c "cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DWEBSOCKET=true -DBUILD_TAOSX=true;make -j $THREAD_COUNT"
+    --rm --ulimit core=-1 taos_test:v1.0 sh -c "cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DWEBSOCKET=true;make -j $THREAD_COUNT"
 
 ret=$?
 exit $ret

tests/system-test/1-insert/keep_expired.py

+# -*- coding: utf-8 -*-
+
+from util.log import *
+from util.cases import *
+from util.sql import *
+import time
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor(), logSql)
+        self.dbname = "test"
+        self.stbname = "stb"
+        self.ctbname = "ctb"
+        self.keep_value = "2d,2d,2d"
+        self.duration_value = "1d"
+        self.offset_time = 5
+        self.sleep_time = self.offset_time*2
+
+    def run(self):
+        tdSql.execute(f'create database if not exists {self.dbname} duration {self.duration_value} keep {self.keep_value};')
+        tdSql.execute(f'create table {self.dbname}.{self.stbname} (ts timestamp, c11 int) TAGS(t11 int, t12 int );')
+        tdSql.execute(f'create table {self.dbname}.{self.ctbname} using {self.dbname}.{self.stbname} TAGS (1, 1);')
+        expired_row_ts = f'now-{int(self.keep_value.split(",")[0].replace("d", "")) * 86400 - self.offset_time}s'
+        tdSql.execute(f'insert into {self.dbname}.{self.ctbname} values ({expired_row_ts}, 1);')
+        tdSql.query(f'select * from {self.dbname}.{self.ctbname}')
+        tdSql.checkEqual(tdSql.queryRows, 1)
+        time.sleep(self.offset_time * 2)
+        tdSql.query(f'select * from {self.dbname}.{self.ctbname}')
+        tdSql.checkEqual(tdSql.queryRows, 0)
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())

tests/system-test/fulltest.sh

@@ -209,6 +209,7 @@ python3 ./test.py -f 2-query/varchar.py -R
 python3 ./test.py -f 1-insert/update_data.py
 
 python3 ./test.py -f 1-insert/delete_data.py
+python3 ./test.py -f 1-insert/keep_expired.py
 
 python3 ./test.py -f 2-query/join2.py
 python3 ./test.py -f 2-query/union1.py

tools/shell/inc/shellTire.h

@@ -78,7 +78,7 @@ bool insertWord(STire* tire, char* word);
 bool deleteWord(STire* tire, char* word);
 
 // match prefix words, if match is not NULL , put all item to match and return match
-SMatch* matchPrefix(STire* tire, char* prefix, SMatch* match);
+void matchPrefix(STire* tire, char* prefix, SMatch* match);
 
 // get all items from tires tree
 SMatch* enumAll(STire* tire);

tools/shell/src/shellAuto.c

@@ -564,6 +564,7 @@ void parseCommand(SWords* command, bool pattern) {
 // free SShellCmd
 void freeCommand(SWords* command) {
   SWord* item = command->head;
+  command->head = NULL;
   // loop
   while (item) {
     SWord* tmp = item;
@@ -815,7 +816,9 @@ char* matchNextPrefix(STire* tire, char* pre) {
       match = enumAll(tire);
     } else {
       // NOT EMPTY
-      match = matchPrefix(tire, pre, NULL);
+      match = (SMatch*)taosMemoryMalloc(sizeof(SMatch));
+      memset(match, 0, sizeof(SMatch));
+      matchPrefix(tire, pre, match);
     }
 
     // save to lastMatch
@@ -828,7 +831,7 @@ char* matchNextPrefix(STire* tire, char* pre) {
   // check valid
   if (match == NULL || match->head == NULL) {
     // no one matched
-    return false;
+    return NULL;
   }
 
   if (cursorVar == -1) {

tools/shell/src/shellTire.c

@@ -308,25 +308,21 @@ void matchPrefixFromTree(STire* tire, char* prefix, SMatch* match) {
   }
 }
 
-SMatch* matchPrefix(STire* tire, char* prefix, SMatch* match) {
-  SMatch* rMatch = match; // define return match
-  if (rMatch == NULL) {
-    rMatch = (SMatch*)taosMemoryMalloc(sizeof(SMatch));
-    memset(rMatch, 0, sizeof(SMatch));
+void matchPrefix(STire* tire, char* prefix, SMatch* match) {
+  if (match == NULL) {
+    return;
   }
 
   switch (tire->type) {
     case TIRE_TREE:
-      matchPrefixFromTree(tire, prefix, rMatch);
+      matchPrefixFromTree(tire, prefix, match);
       break;
     case TIRE_LIST:
-      matchPrefixFromList(tire, prefix, rMatch);
+      matchPrefixFromList(tire, prefix, match);
       break;
     default:
       break;
   }
-
-  return rMatch;
 }
 
 // get all items from tires tree
@@ -360,10 +356,11 @@ void enumFromTree(STire* tire, SMatch* match) {
     }
 
     // this branch have data
-    if (c->end)
+    if (c->end) {
       addWordToMatch(match, pre);
-    else
+    } else {
       matchPrefix(tire, pre, match);
+    }
   }
 }