Merge branch 'develop' into feature/query

documentation20/cn/02.getting-started/docs.md

@@ -126,7 +126,7 @@ taos> source <filename>;
 $ taosdemo
 ```
 
-该命令将在数据库 test 下面自动创建一张超级表 meters，该超级表下有 1 万张表，表名为 "t0" 到 "t9999"，每张表有 1 万条记录，每条记录有 (ts, current, voltage, phase) 四个字段，时间戳从 "2017-07-14 10:40:00 000" 到 "2017-07-14 10:40:09 999"，每张表带有标签 location 和 groupdId，groupdId 被设置为 1 到 10， location 被设置为 "beijing" 或者 "shanghai"。
+该命令将在数据库 test 下面自动创建一张超级表 meters，该超级表下有 1 万张表，表名为 "d0" 到 "d9999"，每张表有 1 万条记录，每条记录有 (ts, current, voltage, phase) 四个字段，时间戳从 "2017-07-14 10:40:00 000" 到 "2017-07-14 10:40:09 999"，每张表带有标签 location 和 groupdId，groupdId 被设置为 1 到 10， location 被设置为 "beijing" 或者 "shanghai"。
 
 执行这条命令大概需要几分钟，最后共插入 1 亿条记录。
 
@@ -156,10 +156,10 @@ taos> select count(*) from test.meters where location="beijing";
 taos> select avg(current), max(voltage), min(phase) from test.meters where groupdId=10;
 ```
 
-- 对表 t10 按 10s 进行平均值、最大值和最小值聚合统计：
+- 对表 d10 按 10s 进行平均值、最大值和最小值聚合统计：
 
 ```mysql
-taos> select avg(current), max(voltage), min(phase) from test.t10 interval(10s);
+taos> select avg(current), max(voltage), min(phase) from test.d10 interval(10s);
 ```
 
 **Note:** taosdemo 命令本身带有很多选项，配置表的数目、记录条数等等，请执行 `taosdemo --help` 详细列出。您可以设置不同参数进行体验。

packaging/tools/install_client.sh

@@ -182,7 +182,13 @@ function install_jemalloc() {
             ${csudo} /usr/bin/install -c -d /usr/local/share/man/man3
             ${csudo} /usr/bin/install -c -m 644 ${jemalloc_dir}/share/man/man3/jemalloc.3 /usr/local/share/man/man3
         fi
-        ${csudo} ldconfig
+
+        if [ -d /etc/ld.so.conf.d ]; then
+            ${csudo} echo "/usr/local/lib" > /etc/ld.so.conf.d/jemalloc.conf
+            ${csudo} ldconfig
+        else
+            echo "/etc/ld.so.conf.d not found!"
+        fi
     fi
 }
 

packaging/tools/makeclient.sh

@@ -41,10 +41,10 @@ fi
 
 if [ "$osType" != "Darwin" ]; then
   if [ "$pagMode" == "lite" ]; then
-    #strip ${build_dir}/bin/taosd 
+    #strip ${build_dir}/bin/taosd
     strip ${build_dir}/bin/taos
     bin_files="${build_dir}/bin/taos ${script_dir}/remove_client.sh"
-  else  
+  else
     bin_files="${build_dir}/bin/taos ${build_dir}/bin/taosdump ${build_dir}/bin/taosdemo \
                ${script_dir}/remove_client.sh ${script_dir}/set_core.sh ${script_dir}/get_client.sh ${script_dir}/taosd-dump-cfg.gdb"
   fi
@@ -139,7 +139,7 @@ if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then
   cp -r ${examples_dir}/C#     ${install_dir}/examples
 fi
 # Copy driver
-mkdir -p ${install_dir}/driver 
+mkdir -p ${install_dir}/driver
 cp ${lib_files} ${install_dir}/driver
 
 # Copy connector
@@ -168,7 +168,7 @@ fi
 
 # exit 1
 
-cd ${release_dir} 
+cd ${release_dir}
 
 #  install_dir has been distinguishes  cluster from  edege, so comments this code
 pkg_name=${install_dir}-${osType}-${cpuType}
@@ -195,6 +195,15 @@ if [ "$pagMode" == "lite" ]; then
   pkg_name=${pkg_name}-Lite
 fi
 
+if [ "$verType" == "beta" ]; then
+  pkg_name=${pkg_name}-${verType}
+elif [ "$verType" == "stable" ]; then
+  pkg_name=${pkg_name}
+else
+  echo "unknow verType, nor stable or beta"
+  exit 1
+fi
+
 if [ "$osType" != "Darwin" ]; then
     tar -zcv -f "$(basename ${pkg_name}).tar.gz" $(basename ${install_dir}) --remove-files || :
 else

packaging/tools/makeclient_power.sh

@@ -41,10 +41,10 @@ fi
 
 if [ "$osType" != "Darwin" ]; then
 #  if [ "$pagMode" == "lite" ]; then
-#    strip ${build_dir}/bin/powerd 
+#    strip ${build_dir}/bin/powerd
 #    strip ${build_dir}/bin/power
 #    bin_files="${build_dir}/bin/power ${script_dir}/remove_client_power.sh"
-#  else  
+#  else
 #    bin_files="${build_dir}/bin/power ${build_dir}/bin/powerdemo ${script_dir}/remove_client_power.sh ${script_dir}/set_core.sh"
 #  fi
   lib_files="${build_dir}/lib/libtaos.so.${version}"
@@ -67,6 +67,39 @@ mkdir -p ${install_dir}
 mkdir -p ${install_dir}/inc && cp ${header_files} ${install_dir}/inc
 mkdir -p ${install_dir}/cfg && cp ${cfg_dir}/taos.cfg ${install_dir}/cfg/taos.cfg
 
+if [ -f ${build_dir}/bin/jemalloc-config ]; then
+    mkdir -p ${install_dir}/jemalloc/{bin,lib,lib/pkgconfig,include/jemalloc,share/doc/jemalloc,share/man/man3}
+    cp ${build_dir}/bin/jemalloc-config ${install_dir}/jemalloc/bin
+    if [ -f ${build_dir}/bin/jemalloc.sh ]; then
+        cp ${build_dir}/bin/jemalloc.sh ${install_dir}/jemalloc/bin
+    fi
+    if [ -f ${build_dir}/bin/jeprof ]; then
+        cp ${build_dir}/bin/jeprof ${install_dir}/jemalloc/bin
+    fi
+    if [ -f ${build_dir}/include/jemalloc/jemalloc.h ]; then
+        cp ${build_dir}/include/jemalloc/jemalloc.h ${install_dir}/jemalloc/include/jemalloc
+    fi
+    if [ -f ${build_dir}/lib/libjemalloc.so.2 ]; then
+        cp ${build_dir}/lib/libjemalloc.so.2 ${install_dir}/jemalloc/lib
+        ln -sf libjemalloc.so.2 ${install_dir}/jemalloc/lib/libjemalloc.so
+    fi
+    if [ -f ${build_dir}/lib/libjemalloc.a ]; then
+        cp ${build_dir}/lib/libjemalloc.a ${install_dir}/jemalloc/lib
+    fi
+    if [ -f ${build_dir}/lib/libjemalloc_pic.a ]; then
+        cp ${build_dir}/lib/libjemalloc_pic.a ${install_dir}/jemalloc/lib
+    fi
+    if [ -f ${build_dir}/lib/pkgconfig/jemalloc.pc ]; then
+        cp ${build_dir}/lib/pkgconfig/jemalloc.pc ${install_dir}/jemalloc/lib/pkgconfig
+    fi
+    if [ -f ${build_dir}/share/doc/jemalloc/jemalloc.html ]; then
+        cp ${build_dir}/share/doc/jemalloc/jemalloc.html ${install_dir}/jemalloc/share/doc/jemalloc
+    fi
+    if [ -f ${build_dir}/share/man/man3/jemalloc.3 ]; then
+        cp ${build_dir}/share/man/man3/jemalloc.3 ${install_dir}/jemalloc/share/man/man3
+    fi
+fi
+
 sed -i '/dataDir/ {s/taos/power/g}'  ${install_dir}/cfg/taos.cfg
 sed -i '/logDir/  {s/taos/power/g}'  ${install_dir}/cfg/taos.cfg
 sed -i "s/TDengine/PowerDB/g"        ${install_dir}/cfg/taos.cfg
@@ -77,11 +110,11 @@ if [ "$osType" != "Darwin" ]; then
     strip ${build_dir}/bin/taos
     cp ${build_dir}/bin/taos          ${install_dir}/bin/power
     cp ${script_dir}/remove_power.sh  ${install_dir}/bin
-  else 
+  else
     cp ${build_dir}/bin/taos          ${install_dir}/bin/power
     cp ${script_dir}/remove_power.sh  ${install_dir}/bin
-    cp ${build_dir}/bin/taosdemo      ${install_dir}/bin/powerdemo 
-    cp ${build_dir}/bin/taosdump      ${install_dir}/bin/powerdump  
+    cp ${build_dir}/bin/taosdemo      ${install_dir}/bin/powerdemo
+    cp ${build_dir}/bin/taosdump      ${install_dir}/bin/powerdump
     cp ${script_dir}/set_core.sh      ${install_dir}/bin
     cp ${script_dir}/get_client.sh    ${install_dir}/bin
     cp ${script_dir}/taosd-dump-cfg.gdb    ${install_dir}/bin
@@ -158,15 +191,15 @@ if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then
   cp -r ${examples_dir}/JDBC   ${install_dir}/examples
   cp -r ${examples_dir}/matlab ${install_dir}/examples
   sed -i '/password/ {s/taosdata/powerdb/g}'  ${install_dir}/examples/matlab/TDengineDemo.m
-  cp -r ${examples_dir}/python ${install_dir}/examples  
+  cp -r ${examples_dir}/python ${install_dir}/examples
   sed -i '/password/ {s/taosdata/powerdb/g}'  ${install_dir}/examples/python/read_example.py
   cp -r ${examples_dir}/R      ${install_dir}/examples
   sed -i '/password/ {s/taosdata/powerdb/g}'  ${install_dir}/examples/R/command.txt
-  cp -r ${examples_dir}/go     ${install_dir}/examples  
+  cp -r ${examples_dir}/go     ${install_dir}/examples
   sed -i '/root/ {s/taosdata/powerdb/g}'  ${install_dir}/examples/go/taosdemo.go
 fi
 # Copy driver
-mkdir -p ${install_dir}/driver 
+mkdir -p ${install_dir}/driver
 cp ${lib_files} ${install_dir}/driver
 
 # Copy connector
@@ -188,11 +221,11 @@ if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then
     echo "WARNING: go connector not found, please check if want to use it!"
   fi
   cp -r ${connector_dir}/python  ${install_dir}/connector
-  
+
   sed -i '/password/ {s/taosdata/powerdb/g}'  ${install_dir}/connector/python/taos/cinterface.py
-  
+
   sed -i '/password/ {s/taosdata/powerdb/g}'  ${install_dir}/connector/python/taos/subscription.py
-  
+
   sed -i '/self._password/ {s/taosdata/powerdb/g}'  ${install_dir}/connector/python/taos/connection.py
 fi
 # Copy release note
@@ -200,7 +233,7 @@ fi
 
 # exit 1
 
-cd ${release_dir} 
+cd ${release_dir}
 
 if [ "$verMode" == "cluster" ]; then
   pkg_name=${install_dir}-${osType}-${cpuType}
@@ -217,8 +250,8 @@ fi
 
 if [ "$verType" == "beta" ]; then
   pkg_name=${pkg_name}-${verType}
-elif [ "$verType" == "stable" ]; then  
-  pkg_name=${pkg_name} 
+elif [ "$verType" == "stable" ]; then
+  pkg_name=${pkg_name}
 else
   echo "unknow verType, nor stable or beta"
   exit 1

src/client/inc/tscUtil.h

@@ -94,11 +94,22 @@ typedef struct SVgroupTableInfo {
   SArray     *itemList;   // SArray<STableIdInfo>
 } SVgroupTableInfo;
 
+typedef struct SBlockKeyTuple {
+  TSKEY skey;
+  void* payloadAddr;
+} SBlockKeyTuple;
+
+typedef struct SBlockKeyInfo {
+  int32_t         maxBytesAlloc;
+  SBlockKeyTuple* pKeyTuple;
+} SBlockKeyInfo;
+
 int32_t converToStr(char *str, int type, void *buf, int32_t bufSize, int32_t *len);
 
 int32_t tscCreateDataBlock(size_t initialSize, int32_t rowSize, int32_t startOffset, SName* name, STableMeta* pTableMeta, STableDataBlocks** dataBlocks);
 void tscDestroyDataBlock(STableDataBlocks* pDataBlock, bool removeMeta);
-void tscSortRemoveDataBlockDupRows(STableDataBlocks* dataBuf);
+void    tscSortRemoveDataBlockDupRowsRaw(STableDataBlocks* dataBuf);
+int     tscSortRemoveDataBlockDupRows(STableDataBlocks* dataBuf, SBlockKeyInfo* pBlkKeyInfo);
 
 void tscDestroyBoundColumnInfo(SParsedDataColInfo* pColInfo);
 void doRetrieveSubqueryData(SSchedMsg *pMsg);

src/client/inc/tsclient.h

@@ -88,13 +88,43 @@ typedef struct SBoundColumn {
   int32_t offset;  // all column offset value
 } SBoundColumn;
 
+typedef struct {
+  uint16_t schemaColIdx;
+  uint16_t boundIdx;
+  uint16_t finalIdx;
+} SBoundIdxInfo;
+
+typedef enum _COL_ORDER_STATUS {
+  ORDER_STATUS_UNKNOWN = 0,
+  ORDER_STATUS_ORDERED = 1,
+  ORDER_STATUS_DISORDERED = 2,
+} EOrderStatus;
+
 typedef struct SParsedDataColInfo {
-  int16_t         numOfCols;
-  int16_t         numOfBound;
-  int32_t        *boundedColumns;
-  SBoundColumn   *cols;
+  int16_t        numOfCols;
+  int16_t        numOfBound;
+  int32_t *      boundedColumns;  // bounded column idx according to schema
+  SBoundColumn * cols;
+  SBoundIdxInfo *colIdxInfo;
+  int8_t         orderStatus;  // bounded columns: 
 } SParsedDataColInfo;
 
+#define IS_DATA_COL_ORDERED(s) ((s) == (int8_t)ORDER_STATUS_ORDERED)
+
+typedef struct {
+  SSchema *   pSchema;
+  int16_t     sversion;
+  int32_t     flen;
+  uint16_t    nCols;
+  void *      buf;
+  void *      pDataBlock;
+  SSubmitBlk *pSubmitBlk;
+} SMemRowBuilder;
+
+typedef struct {
+  TDRowLenT allNullLen;
+} SMemRowHelper;
+
 typedef struct STableDataBlocks {
   SName       tableName;
   int8_t      tsSource;     // where does the UNIX timestamp come from, server or client
@@ -109,12 +139,13 @@ typedef struct STableDataBlocks {
   STableMeta *pTableMeta;   // the tableMeta of current table, the table meta will be used during submit, keep a ref to avoid to be removed from cache
   char       *pData;
 
-  SParsedDataColInfo  boundColumnInfo;
+  SParsedDataColInfo boundColumnInfo;
 
   // for parameter ('?') binding
-  uint32_t    numOfAllocedParams;
-  uint32_t    numOfParams;
-  SParamInfo *params;
+  uint32_t       numOfAllocedParams;
+  uint32_t       numOfParams;
+  SParamInfo *   params;
+  SMemRowHelper  rowHelper;
 } STableDataBlocks;
 
 typedef struct {
@@ -128,6 +159,7 @@ typedef struct SInsertStatementParam {
   SHashObj    *pTableBlockHashList;     // data block for each table
   SArray      *pDataBlocks;             // SArray<STableDataBlocks*>. Merged submit block for each vgroup
   int8_t       schemaAttached;          // denote if submit block is built with table schema or not
+  uint8_t      payloadType;             // EPayloadType. 0: K-V payload for non-prepare insert, 1: rawPayload for prepare insert
   STagData     tagData;                 // NOTE: pTagData->data is used as a variant length array
 
   int32_t      batchSize;               // for parameter ('?') binding and batch processing
@@ -139,6 +171,14 @@ typedef struct SInsertStatementParam {
   char        *sql;                     // current sql statement position
 } SInsertStatementParam;
 
+typedef enum {
+  PAYLOAD_TYPE_KV = 0,
+  PAYLOAD_TYPE_RAW = 1,
+} EPayloadType;
+
+#define IS_RAW_PAYLOAD(t) \
+  (((int)(t)) == PAYLOAD_TYPE_RAW)  // 0: K-V payload for non-prepare insert, 1: rawPayload for prepare insert
+
 // TODO extract sql parser supporter
 typedef struct {
   int     command;
@@ -243,6 +283,7 @@ typedef struct SSqlObj {
   void *           pStream;
   void *           pSubscription;
   char *           sqlstr;
+  void *           pBuf;  // table meta buffer
   char             parseRetry;
   char             retry;
   char             maxRetry;
@@ -382,12 +423,17 @@ extern int   tscRefId;
 extern int   tscNumOfObj;     // number of existed sqlObj in current process.
 
 extern int (*tscBuildMsg[TSDB_SQL_MAX])(SSqlObj *pSql, SSqlInfo *pInfo);
-
+ 
 void tscBuildVgroupTableInfo(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, SArray* tables);
 int16_t getNewResColId(SSqlCmd* pCmd);
 
+int32_t schemaIdxCompar(const void *lhs, const void *rhs);
+int32_t boundIdxCompar(const void *lhs, const void *rhs);
+int     initSMemRowHelper(SMemRowHelper *pHelper, SSchema *pSSchema, uint16_t nCols, uint16_t allNullColsLen);
+int32_t getExtendedRowSize(STableComInfo *tinfo);
+
 #ifdef __cplusplus
 }
 #endif
 
-#endif
+#endif
\ No newline at end of file

src/client/src/taos.def

@@ -7,11 +7,16 @@ taos_connect_auth
 taos_close
 taos_stmt_init
 taos_stmt_prepare
+taos_stmt_set_tbname_tags
+taos_stmt_set_tbname
+taos_stmt_is_insert
+taos_stmt_num_params
 taos_stmt_bind_param
 taos_stmt_add_batch
 taos_stmt_execute
 taos_stmt_use_result
 taos_stmt_close
+taos_stmt_errstr
 taos_query
 taos_fetch_row
 taos_result_precision
@@ -37,6 +42,4 @@ taos_consume
 taos_unsubscribe
 taos_open_stream
 taos_close_stream
-taos_fetch_block
 taos_load_table_info
-

src/client/src/tscPrepare.c

@@ -1806,6 +1806,7 @@ int taos_stmt_execute(TAOS_STMT* stmt) {
 
     pStmt->last = STMT_EXECUTE;
 
+    pStmt->pSql->cmd.insertParam.payloadType = PAYLOAD_TYPE_RAW;
     if (pStmt->multiTbInsert) {
       ret = insertBatchStmtExecute(pStmt);
     } else {

src/client/src/tscSQLParser.c

@@ -1396,12 +1396,16 @@ static bool validateTableColumnInfo(SArray* pFieldList, SSqlCmd* pCmd) {
   const char* msg4 = "invalid data type";
   const char* msg5 = "invalid binary/nchar column length";
   const char* msg6 = "invalid column name";
+  const char* msg7 = "too many columns";
 
   // number of fields no less than 2
   size_t numOfCols = taosArrayGetSize(pFieldList);
-  if (numOfCols <= 1 || numOfCols > TSDB_MAX_COLUMNS) {
+  if (numOfCols <= 1 ) {
     invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg);
     return false;
+  } else if (numOfCols > TSDB_MAX_COLUMNS) {
+    invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg7);
+    return false;
   }
 
   // first column must be timestamp
@@ -1536,13 +1540,20 @@ bool validateOneTags(SSqlCmd* pCmd, TAOS_FIELD* pTagField) {
   const char* msg4 = "invalid tag name";
   const char* msg5 = "invalid binary/nchar tag length";
   const char* msg6 = "invalid data type in tags";
+  const char* msg7 = "too many columns";
 
   STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd,  0);
   STableMeta*     pTableMeta = pTableMetaInfo->pTableMeta;
 
   int32_t numOfTags = tscGetNumOfTags(pTableMeta);
   int32_t numOfCols = tscGetNumOfColumns(pTableMeta);
-  
+
+  // no more max columns
+  if (numOfTags + numOfCols >= TSDB_MAX_COLUMNS) {
+    invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg7);
+    return false;
+  }
+
   // no more than 6 tags
   if (numOfTags == TSDB_MAX_TAGS) {
     char msg[128] = {0};
@@ -5999,6 +6010,16 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
       return invalidOperationMsg(pMsg, msg22);
     }
 
+    SSchema* pSchema = (SSchema*) pTableMetaInfo->pTableMeta->schema;
+    int16_t numOfColumns = pTableMetaInfo->pTableMeta->tableInfo.numOfColumns;
+    int16_t i;
+    uint32_t nLen = 0;
+    for (i = 0; i < numOfColumns; ++i) {
+      nLen += pSchema[i].colId != columnIndex.columnIndex ? pSchema[i].bytes : pItem->bytes;
+    }
+    if (nLen >= TSDB_MAX_BYTES_PER_ROW) {
+      return invalidOperationMsg(pMsg, msg24);
+    }
     TAOS_FIELD f = tscCreateField(pColSchema->type, name.z, pItem->bytes);
     tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f);
   }else if (pAlterSQL->type == TSDB_ALTER_TABLE_MODIFY_TAG_COLUMN) {
@@ -6040,6 +6061,17 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
       return invalidOperationMsg(pMsg, msg22);
     }
 
+    SSchema* pSchema = (SSchema*) pTableMetaInfo->pTableMeta->schema;
+    int16_t numOfColumns = pTableMetaInfo->pTableMeta->tableInfo.numOfColumns;
+    int16_t i;
+    uint32_t nLen = 0;
+    for (i = 0; i < numOfColumns; ++i) {
+      nLen += pSchema[i].colId != columnIndex.columnIndex ? pSchema[i].bytes : pItem->bytes;
+    }
+    if (nLen >= TSDB_MAX_BYTES_PER_ROW) {
+      return invalidOperationMsg(pMsg, msg24);
+    }
+
     TAOS_FIELD f = tscCreateField(pColSchema->type, name.z, pItem->bytes);
     tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f);
   }
@@ -8070,8 +8102,12 @@ int32_t loadAllTableMeta(SSqlObj* pSql, struct SSqlInfo* pInfo) {
   uint32_t maxSize = tscGetTableMetaMaxSize();
   char     name[TSDB_TABLE_FNAME_LEN] = {0};
 
-  char buf[80 * 1024] = {0};
-  assert(maxSize < 80 * 1024);
+  assert(maxSize < 80 * TSDB_MAX_COLUMNS);
+  if (!pSql->pBuf) {
+    if (NULL == (pSql->pBuf = tcalloc(1, 80 * TSDB_MAX_COLUMNS))) {
+      return TSDB_CODE_TSC_OUT_OF_MEMORY;
+    }
+  }
   pTableMeta = calloc(1, maxSize);
 
   plist = taosArrayInit(4, POINTER_BYTES);
@@ -8091,7 +8127,7 @@ int32_t loadAllTableMeta(SSqlObj* pSql, struct SSqlInfo* pInfo) {
 
     if (pTableMeta->id.uid > 0) {
       if (pTableMeta->tableType == TSDB_CHILD_TABLE) {
-        code = tscCreateTableMetaFromSTableMeta(pTableMeta, name, buf);
+        code = tscCreateTableMetaFromSTableMeta(pTableMeta, name, pSql->pBuf);
 
         // create the child table meta from super table failed, try load it from mnode
         if (code != TSDB_CODE_SUCCESS) {
@@ -8287,7 +8323,9 @@ static int32_t doValidateSubquery(SSqlNode* pSqlNode, int32_t index, SSqlObj* pS
 
   // union all is not support currently
   SSqlNode* p = taosArrayGetP(subInfo->pSubquery, 0);
-
+  if (taosArrayGetSize(subInfo->pSubquery) >= 2) {
+    return invalidOperationMsg(msgBuf, "not support union in subquery");
+  }
   SQueryInfo* pSub = calloc(1, sizeof(SQueryInfo));
   tscInitQueryInfo(pSub);
 

src/client/src/tscServer.c

@@ -2804,22 +2804,24 @@ int32_t tscGetTableMetaImpl(SSqlObj* pSql, STableMetaInfo *pTableMetaInfo, bool
   taosHashGetClone(tscTableMetaInfo, name, len, NULL, pTableMetaInfo->pTableMeta, -1);
 
   // TODO resize the tableMeta
-  char buf[80*1024] = {0};
-  assert(size < 80*1024);
+  assert(size < 80 * TSDB_MAX_COLUMNS);
+  if (!pSql->pBuf) {
+    if (NULL == (pSql->pBuf = tcalloc(1, 80 * TSDB_MAX_COLUMNS))) {
+      return TSDB_CODE_TSC_OUT_OF_MEMORY;
+    }
+  }
 
   STableMeta* pMeta = pTableMetaInfo->pTableMeta;
   if (pMeta->id.uid > 0) {
     // in case of child table, here only get the
     if (pMeta->tableType == TSDB_CHILD_TABLE) {
-      int32_t code = tscCreateTableMetaFromSTableMeta(pTableMetaInfo->pTableMeta, name, buf);
+      int32_t code = tscCreateTableMetaFromSTableMeta(pTableMetaInfo->pTableMeta, name, pSql->pBuf);
       if (code != TSDB_CODE_SUCCESS) {
         return getTableMetaFromMnode(pSql, pTableMetaInfo, autocreate);
       }
     }
-
     return TSDB_CODE_SUCCESS;
   }
-
   return getTableMetaFromMnode(pSql, pTableMetaInfo, autocreate);
 }
 
@@ -2934,7 +2936,6 @@ int tscGetSTableVgroupInfo(SSqlObj *pSql, SQueryInfo* pQueryInfo) {
   if (allVgroupInfoRetrieved(pQueryInfo)) {
     return TSDB_CODE_SUCCESS;
   }
-
   SSqlObj *pNew = calloc(1, sizeof(SSqlObj));
   pNew->pTscObj = pSql->pTscObj;
   pNew->signature = pNew;

src/client/src/tscSubquery.c

@@ -712,6 +712,15 @@ static void updateQueryTimeRange(SQueryInfo* pQueryInfo, STimeWindow* win) {
   pQueryInfo->window = *win;
 }
 
+int32_t tagValCompar(const void* p1, const void* p2) {
+  const STidTags* t1 = (const STidTags*) varDataVal(p1);
+  const STidTags* t2 = (const STidTags*) varDataVal(p2);
+
+  __compar_fn_t func = getComparFunc(t1->padding, 0);
+
+  return func(t1->tag, t2->tag);
+}
+
 int32_t tidTagsCompar(const void* p1, const void* p2) {
   const STidTags* t1 = (const STidTags*) (p1);
   const STidTags* t2 = (const STidTags*) (p2);
@@ -720,28 +729,7 @@ int32_t tidTagsCompar(const void* p1, const void* p2) {
     return (t1->vgId > t2->vgId) ? 1 : -1;
   }
 
-  tstr* tag1 = (tstr*) t1->tag;
-  tstr* tag2 = (tstr*) t2->tag;
-
-  if (tag1->len != tag2->len) {
-    return (tag1->len > tag2->len)? 1: -1;
-  }
-
-  return strncmp(tag1->data, tag2->data, tag1->len);
-}
-
-int32_t tagValCompar(const void* p1, const void* p2) {
-  const STidTags* t1 = (const STidTags*) varDataVal(p1);
-  const STidTags* t2 = (const STidTags*) varDataVal(p2);
-
-  tstr* tag1 = (tstr*) t1->tag;
-  tstr* tag2 = (tstr*) t2->tag;
-
-  if (tag1->len != tag2->len) {
-    return (tag1->len > tag2->len)? 1: -1;
-  }
-
-  return memcmp(tag1->data, tag2->data, tag1->len);
+  return tagValCompar(p1, p2);
 }
 
 void tscBuildVgroupTableInfo(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, SArray* tables) {
@@ -871,6 +859,12 @@ static bool checkForDuplicateTagVal(SSchema* pColSchema, SJoinSupporter* p1, SSq
   return true;
 }
 
+static void setTidTagType(SJoinSupporter* p, uint8_t type) {
+  for (int32_t i = 0; i < p->num; ++i) {
+    STidTags * tag = (STidTags*) varDataVal(p->pIdTagList + i * p->tagSize);
+    tag->padding = type;
+  }
+}
 
 static int32_t getIntersectionOfTableTuple(SQueryInfo* pQueryInfo, SSqlObj* pParentSql, SArray* resList) {
   int16_t joinNum = pParentSql->subState.numOfSub;
@@ -890,6 +884,8 @@ static int32_t getIntersectionOfTableTuple(SQueryInfo* pQueryInfo, SSqlObj* pPar
   for (int32_t i = 0; i < joinNum; i++) {
     SJoinSupporter* p = pParentSql->pSubs[i]->param;
 
+    setTidTagType(p, pColSchema->type);
+
     ctxlist[i].p = p;
     ctxlist[i].res = taosArrayInit(p->num, size);
 
@@ -2440,9 +2436,9 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) {
   tOrderDescriptor *pDesc  = NULL;
 
   pRes->qId = 0x1;  // hack the qhandle check
-  
-  const uint32_t nBufferSize = (1u << 16u);  // 64KB
-  
+
+  const uint32_t nBufferSize = (1u << 18u);  // 256KB
+
   SQueryInfo     *pQueryInfo = tscGetQueryInfo(pCmd);
   STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
   SSubqueryState *pState = &pSql->subState;

src/client/src/tscUtil.c

@@ -1487,6 +1487,7 @@ void tscFreeSqlObj(SSqlObj* pSql) {
   pSql->signature = NULL;
   pSql->fp = NULL;
   tfree(pSql->sqlstr);
+  tfree(pSql->pBuf);
 
   tfree(pSql->pSubs);
   pSql->subState.numOfSub = 0;
@@ -1498,7 +1499,7 @@ void tscFreeSqlObj(SSqlObj* pSql) {
   memset(pCmd->payload, 0, (size_t)pCmd->allocSize);
   tfree(pCmd->payload);
   pCmd->allocSize = 0;
-  
+
   tsem_destroy(&pSql->rspSem);
   memset(pSql, 0, sizeof(*pSql));
   free(pSql);
@@ -1507,6 +1508,7 @@ void tscFreeSqlObj(SSqlObj* pSql) {
 void tscDestroyBoundColumnInfo(SParsedDataColInfo* pColInfo) {
   tfree(pColInfo->boundedColumns);
   tfree(pColInfo->cols);
+  tfree(pColInfo->colIdxInfo);
 }
 
 void tscDestroyDataBlock(STableDataBlocks* pDataBlock, bool removeMeta) {
@@ -1761,11 +1763,108 @@ int32_t tscGetDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, i
   return TSDB_CODE_SUCCESS;
 }
 
-static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, bool includeSchema) {
+static SMemRow tdGenMemRowFromBuilder(SMemRowBuilder* pBuilder) {
+  SSchema* pSchema = pBuilder->pSchema;
+  char*    p = (char*)pBuilder->buf;
+  int      toffset = 0;
+  uint16_t nCols = pBuilder->nCols;
+
+  uint8_t  memRowType = payloadType(p);
+  uint16_t nColsBound = payloadNCols(p);
+  if (pBuilder->nCols <= 0 || nColsBound <= 0) {
+    return NULL;
+  }
+  char*    pVals = POINTER_SHIFT(p, payloadValuesOffset(p));
+  SMemRow* memRow = (SMemRow)pBuilder->pDataBlock;
+  memRowSetType(memRow, memRowType);
+
+  // ----------------- Raw payload structure for row:
+  /* |<------------ Head ------------->|<----------- body of column data tuple ------------------->|
+   * |                                 |<----------------- flen ------------->|<--- value part --->|
+   * |SMemRowType| dataTLen |  nCols   |  colId  | colType | offset   |  ...  | value |...|...|... |
+   * +-----------+----------+----------+--------------------------------------|--------------------|
+   * | uint8_t   | uint32_t | uint16_t | int16_t | uint8_t | uint16_t |  ...  |.......|...|...|... |
+   * +-----------+----------+----------+--------------------------------------+--------------------|
+   *  1. offset in column data tuple starts from the value part in case of uint16_t overflow.
+   *  2. dataTLen: total length including the header and body.
+   */
+
+  if (memRowType == SMEM_ROW_DATA) {
+    SDataRow trow = (SDataRow)memRowDataBody(memRow);
+    dataRowSetLen(trow, (TDRowLenT)(TD_DATA_ROW_HEAD_SIZE + pBuilder->flen));
+    dataRowSetVersion(trow, pBuilder->sversion);
+
+    p = (char*)payloadBody(pBuilder->buf);
+    uint16_t i = 0, j = 0;
+    while (j < nCols) {
+      if (i >= nColsBound) {
+        break;
+      }
+      int16_t colId = payloadColId(p);
+      if (colId == pSchema[j].colId) {
+        // ASSERT(payloadColType(p) == pSchema[j].type);
+        tdAppendColVal(trow, POINTER_SHIFT(pVals, payloadColOffset(p)), pSchema[j].type, toffset);
+        toffset += TYPE_BYTES[pSchema[j].type];
+        p = payloadNextCol(p);
+        ++i;
+        ++j;
+      } else if (colId < pSchema[j].colId) {
+        p = payloadNextCol(p);
+        ++i;
+      } else {
+        tdAppendColVal(trow, getNullValue(pSchema[j].type), pSchema[j].type, toffset);
+        toffset += TYPE_BYTES[pSchema[j].type];
+        ++j;
+      }
+    }
+
+    while (j < nCols) {
+      tdAppendColVal(trow, getNullValue(pSchema[j].type), pSchema[j].type, toffset);
+      toffset += TYPE_BYTES[pSchema[j].type];
+      ++j;
+    }
+    
+    #if 0 // no need anymore
+    while (i < nColsBound) {
+      p = payloadNextCol(p);
+      ++i;
+    }
+    #endif
+
+  } else if (memRowType == SMEM_ROW_KV) {
+    SKVRow   kvRow = (SKVRow)memRowKvBody(memRow);
+    kvRowSetLen(kvRow, (TDRowLenT)(TD_KV_ROW_HEAD_SIZE + sizeof(SColIdx) * nColsBound));
+    kvRowSetNCols(kvRow, nColsBound);
+    memRowSetKvVersion(memRow, pBuilder->sversion);
+
+    p = (char*)payloadBody(pBuilder->buf);
+    int i = 0;
+    while (i < nColsBound) {
+      int16_t colId = payloadColId(p);
+      uint8_t colType = payloadColType(p);
+      tdAppendKvColVal(kvRow, POINTER_SHIFT(pVals,payloadColOffset(p)), colId, colType, toffset);
+      toffset += sizeof(SColIdx);
+      p = payloadNextCol(p);
+      ++i;
+    }
+
+  } else {
+    ASSERT(0);
+  }
+  int32_t rowTLen = memRowTLen(memRow);
+  pBuilder->pDataBlock = (char*)pBuilder->pDataBlock + rowTLen;  // next row
+  pBuilder->pSubmitBlk->dataLen += rowTLen;
+
+  return memRow;
+}
+
+// Erase the empty space reserved for binary data
+static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, SInsertStatementParam* insertParam,
+                         SBlockKeyTuple* blkKeyTuple) {
   // TODO: optimize this function, handle the case while binary is not presented
-  STableMeta*   pTableMeta = pTableDataBlock->pTableMeta;
-  STableComInfo tinfo = tscGetTableInfo(pTableMeta);
-  SSchema*      pSchema = tscGetTableSchema(pTableMeta);
+  STableMeta*     pTableMeta = pTableDataBlock->pTableMeta;
+  STableComInfo   tinfo = tscGetTableInfo(pTableMeta);
+  SSchema*        pSchema = tscGetTableSchema(pTableMeta);
 
   SSubmitBlk* pBlock = pDataBlock;
   memcpy(pDataBlock, pTableDataBlock->pData, sizeof(SSubmitBlk));
@@ -1774,7 +1873,7 @@ static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, bo
   int32_t flen = 0;  // original total length of row
 
   // schema needs to be included into the submit data block
-  if (includeSchema) {
+  if (insertParam->schemaAttached) {
     int32_t numOfCols = tscGetNumOfColumns(pTableDataBlock->pTableMeta);
     for(int32_t j = 0; j < numOfCols; ++j) {
       STColumn* pCol = (STColumn*) pDataBlock;
@@ -1800,21 +1899,39 @@ static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, bo
   char* p = pTableDataBlock->pData + sizeof(SSubmitBlk);
   pBlock->dataLen = 0;
   int32_t numOfRows = htons(pBlock->numOfRows);
-  
-  for (int32_t i = 0; i < numOfRows; ++i) {
-    SDataRow trow = (SDataRow) pDataBlock;
-    dataRowSetLen(trow, (uint16_t)(TD_DATA_ROW_HEAD_SIZE + flen));
-    dataRowSetVersion(trow, pTableMeta->sversion);
-
-    int toffset = 0;
-    for (int32_t j = 0; j < tinfo.numOfColumns; j++) {
-      tdAppendColVal(trow, p, pSchema[j].type, pSchema[j].bytes, toffset);
-      toffset += TYPE_BYTES[pSchema[j].type];
-      p += pSchema[j].bytes;
+
+  if (IS_RAW_PAYLOAD(insertParam->payloadType)) {
+    for (int32_t i = 0; i < numOfRows; ++i) {
+      SMemRow memRow = (SMemRow)pDataBlock;
+      memRowSetType(memRow, SMEM_ROW_DATA);
+      SDataRow trow = memRowDataBody(memRow);
+      dataRowSetLen(trow, (uint16_t)(TD_DATA_ROW_HEAD_SIZE + flen));
+      dataRowSetVersion(trow, pTableMeta->sversion);
+
+      int toffset = 0;
+      for (int32_t j = 0; j < tinfo.numOfColumns; j++) {
+        tdAppendColVal(trow, p, pSchema[j].type, toffset);
+        toffset += TYPE_BYTES[pSchema[j].type];
+        p += pSchema[j].bytes;
+      }
+
+      pDataBlock = (char*)pDataBlock + memRowTLen(memRow);
+      pBlock->dataLen += memRowTLen(memRow);
     }
+  } else {
+    SMemRowBuilder rowBuilder;
+    rowBuilder.pSchema = pSchema;
+    rowBuilder.sversion = pTableMeta->sversion;
+    rowBuilder.flen = flen;
+    rowBuilder.nCols = tinfo.numOfColumns;
+    rowBuilder.pDataBlock = pDataBlock;
+    rowBuilder.pSubmitBlk = pBlock;
+    rowBuilder.buf = p;
 
-    pDataBlock = (char*)pDataBlock + dataRowLen(trow);
-    pBlock->dataLen += dataRowLen(trow);
+    for (int32_t i = 0; i < numOfRows; ++i) {
+      rowBuilder.buf = (blkKeyTuple + i)->payloadAddr;
+      tdGenMemRowFromBuilder(&rowBuilder);
+    }
   }
 
   int32_t len = pBlock->dataLen + pBlock->schemaLen;
@@ -1825,7 +1942,7 @@ static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, bo
 }
 
 static int32_t getRowExpandSize(STableMeta* pTableMeta) {
-  int32_t result = TD_DATA_ROW_HEAD_SIZE;
+  int32_t  result = TD_MEM_ROW_DATA_HEAD_SIZE;
   int32_t columns = tscGetNumOfColumns(pTableMeta);
   SSchema* pSchema = tscGetTableSchema(pTableMeta);
   for(int32_t i = 0; i < columns; i++) {
@@ -1861,13 +1978,17 @@ static void extractTableNameList(SInsertStatementParam *pInsertParam, bool freeB
 
 int32_t tscMergeTableDataBlocks(SInsertStatementParam *pInsertParam, bool freeBlockMap) {
   const int INSERT_HEAD_SIZE = sizeof(SMsgDesc) + sizeof(SSubmitMsg);
-
-  void* pVnodeDataBlockHashList = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false);
-  SArray* pVnodeDataBlockList = taosArrayInit(8, POINTER_BYTES);
+  int       code = 0;
+  bool      isRawPayload = IS_RAW_PAYLOAD(pInsertParam->payloadType);
+  void*     pVnodeDataBlockHashList = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false);
+  SArray*   pVnodeDataBlockList = taosArrayInit(8, POINTER_BYTES);
 
   STableDataBlocks** p = taosHashIterate(pInsertParam->pTableBlockHashList, NULL);
 
   STableDataBlocks* pOneTableBlock = *p;
+
+  SBlockKeyInfo blkKeyInfo = {0};  // share by pOneTableBlock
+
   while(pOneTableBlock) {
     SSubmitBlk* pBlocks = (SSubmitBlk*) pOneTableBlock->pData;
     if (pBlocks->numOfRows > 0) {
@@ -1881,6 +2002,7 @@ int32_t tscMergeTableDataBlocks(SInsertStatementParam *pInsertParam, bool freeBl
         tscError("0x%"PRIx64" failed to prepare the data block buffer for merging table data, code:%d", pInsertParam->objectId, ret);
         taosHashCleanup(pVnodeDataBlockHashList);
         tscDestroyBlockArrayList(pVnodeDataBlockList);
+        tfree(blkKeyInfo.pKeyTuple);
         return ret;
       }
 
@@ -1901,17 +2023,35 @@ int32_t tscMergeTableDataBlocks(SInsertStatementParam *pInsertParam, bool freeBl
           taosHashCleanup(pVnodeDataBlockHashList);
           tscDestroyBlockArrayList(pVnodeDataBlockList);
           tfree(dataBuf->pData);
+          tfree(blkKeyInfo.pKeyTuple);
 
           return TSDB_CODE_TSC_OUT_OF_MEMORY;
         }
       }
 
-      tscSortRemoveDataBlockDupRows(pOneTableBlock);
-      char* ekey = (char*)pBlocks->data + pOneTableBlock->rowSize*(pBlocks->numOfRows-1);
+      if (isRawPayload) {
+        tscSortRemoveDataBlockDupRowsRaw(pOneTableBlock);
+        char* ekey = (char*)pBlocks->data + pOneTableBlock->rowSize * (pBlocks->numOfRows - 1);
 
-      tscDebug("0x%"PRIx64" name:%s, tid:%d rows:%d sversion:%d skey:%" PRId64 ", ekey:%" PRId64, pInsertParam->objectId, tNameGetTableName(&pOneTableBlock->tableName),
-          pBlocks->tid, pBlocks->numOfRows, pBlocks->sversion, GET_INT64_VAL(pBlocks->data), GET_INT64_VAL(ekey));
+        tscDebug("0x%" PRIx64 " name:%s, tid:%d rows:%d sversion:%d skey:%" PRId64 ", ekey:%" PRId64,
+                 pInsertParam->objectId, tNameGetTableName(&pOneTableBlock->tableName), pBlocks->tid,
+                 pBlocks->numOfRows, pBlocks->sversion, GET_INT64_VAL(pBlocks->data), GET_INT64_VAL(ekey));
+      } else {
+        if ((code = tscSortRemoveDataBlockDupRows(pOneTableBlock, &blkKeyInfo)) != 0) {
+          taosHashCleanup(pVnodeDataBlockHashList);
+          tscDestroyBlockArrayList(pVnodeDataBlockList);
+          tfree(dataBuf->pData);
+          tfree(blkKeyInfo.pKeyTuple);
+          return code;
+        }
+        ASSERT(blkKeyInfo.pKeyTuple != NULL && pBlocks->numOfRows > 0);
 
+        SBlockKeyTuple* pLastKeyTuple = blkKeyInfo.pKeyTuple + pBlocks->numOfRows - 1;
+        tscDebug("0x%" PRIx64 " name:%s, tid:%d rows:%d sversion:%d skey:%" PRId64 ", ekey:%" PRId64,
+                 pInsertParam->objectId, tNameGetTableName(&pOneTableBlock->tableName), pBlocks->tid,
+                 pBlocks->numOfRows, pBlocks->sversion, blkKeyInfo.pKeyTuple->skey, pLastKeyTuple->skey);
+      }
+      
       int32_t len = pBlocks->numOfRows * (pOneTableBlock->rowSize + expandSize) + sizeof(STColumn) * tscGetNumOfColumns(pOneTableBlock->pTableMeta);
 
       pBlocks->tid = htonl(pBlocks->tid);
@@ -1921,7 +2061,7 @@ int32_t tscMergeTableDataBlocks(SInsertStatementParam *pInsertParam, bool freeBl
       pBlocks->schemaLen = 0;
 
       // erase the empty space reserved for binary data
-      int32_t finalLen = trimDataBlock(dataBuf->pData + dataBuf->size, pOneTableBlock, pInsertParam->schemaAttached);
+      int32_t finalLen = trimDataBlock(dataBuf->pData + dataBuf->size, pOneTableBlock, pInsertParam, blkKeyInfo.pKeyTuple);
       assert(finalLen <= len);
 
       dataBuf->size += (finalLen + sizeof(SSubmitBlk));
@@ -1949,6 +2089,7 @@ int32_t tscMergeTableDataBlocks(SInsertStatementParam *pInsertParam, bool freeBl
   // free the table data blocks;
   pInsertParam->pDataBlocks = pVnodeDataBlockList;
   taosHashCleanup(pVnodeDataBlockHashList);
+  tfree(blkKeyInfo.pKeyTuple);
 
   return TSDB_CODE_SUCCESS;
 }

src/common/inc/tdataformat.h

@@ -53,10 +53,10 @@ extern "C" {
 
 // ----------------- TSDB COLUMN DEFINITION
 typedef struct {
-  int8_t  type;    // Column type
-  int16_t colId;   // column ID
-  int16_t bytes;   // column bytes
-  int16_t offset;  // point offset in SDataRow after the header part
+  int8_t   type;    // Column type
+  int16_t  colId;   // column ID
+  uint16_t bytes;   // column bytes
+  uint16_t offset;  // point offset in SDataRow after the header part.
 } STColumn;
 
 #define colType(col) ((col)->type)
@@ -167,10 +167,11 @@ static FORCE_INLINE int tkeyComparFn(const void *tkey1, const void *tkey2) {
     return 0;
   }
 }
+
 // ----------------- Data row structure
 
 /* A data row, the format is like below:
- * |<--------------------+--------------------------- len ---------------------------------->|
+ * |<------------------------------------------------ len ---------------------------------->|
  * |<--     Head      -->|<---------   flen -------------->|                                 |
  * +---------------------+---------------------------------+---------------------------------+
  * | uint16_t |  int16_t |                                 |                                 |
@@ -184,8 +185,8 @@ typedef void *SDataRow;
 
 #define TD_DATA_ROW_HEAD_SIZE (sizeof(uint16_t) + sizeof(int16_t))
 
-#define dataRowLen(r) (*(uint16_t *)(r))
-#define dataRowVersion(r) *(int16_t *)POINTER_SHIFT(r, sizeof(int16_t))
+#define dataRowLen(r) (*(TDRowLenT *)(r))  // 0~65535
+#define dataRowVersion(r) (*(int16_t *)POINTER_SHIFT(r, sizeof(int16_t)))
 #define dataRowTuple(r) POINTER_SHIFT(r, TD_DATA_ROW_HEAD_SIZE)
 #define dataRowTKey(r) (*(TKEY *)(dataRowTuple(r)))
 #define dataRowKey(r) tdGetKey(dataRowTKey(r))
@@ -201,20 +202,19 @@ void     tdInitDataRow(SDataRow row, STSchema *pSchema);
 SDataRow tdDataRowDup(SDataRow row);
 
 // offset here not include dataRow header length
-static FORCE_INLINE int tdAppendColVal(SDataRow row, void *value, int8_t type, int32_t bytes, int32_t offset) {
+static FORCE_INLINE int tdAppendColVal(SDataRow row, const void *value, int8_t type, int32_t offset) {
   ASSERT(value != NULL);
   int32_t toffset = offset + TD_DATA_ROW_HEAD_SIZE;
-  char *  ptr = (char *)POINTER_SHIFT(row, dataRowLen(row));
 
   if (IS_VAR_DATA_TYPE(type)) {
     *(VarDataOffsetT *)POINTER_SHIFT(row, toffset) = dataRowLen(row);
-    memcpy(ptr, value, varDataTLen(value));
+    memcpy(POINTER_SHIFT(row, dataRowLen(row)), value, varDataTLen(value));
     dataRowLen(row) += varDataTLen(value);
   } else {
     if (offset == 0) {
       ASSERT(type == TSDB_DATA_TYPE_TIMESTAMP);
       TKEY tvalue = tdGetTKEY(*(TSKEY *)value);
-      memcpy(POINTER_SHIFT(row, toffset), (void *)(&tvalue), TYPE_BYTES[type]);
+      memcpy(POINTER_SHIFT(row, toffset), (const void *)(&tvalue), TYPE_BYTES[type]);
     } else {
       memcpy(POINTER_SHIFT(row, toffset), value, TYPE_BYTES[type]);
     }
@@ -245,17 +245,21 @@ typedef struct SDataCol {
   TSKEY           ts;         // only used in last NULL column
 } SDataCol;
 
+#define isAllRowsNull(pCol) ((pCol)->len == 0)
 static FORCE_INLINE void dataColReset(SDataCol *pDataCol) { pDataCol->len = 0; }
 
 void dataColInit(SDataCol *pDataCol, STColumn *pCol, void **pBuf, int maxPoints);
-void dataColAppendVal(SDataCol *pCol, void *value, int numOfRows, int maxPoints);
+void dataColAppendVal(SDataCol *pCol, const void *value, int numOfRows, int maxPoints);
 void dataColSetOffset(SDataCol *pCol, int nEle);
 
 bool isNEleNull(SDataCol *pCol, int nEle);
 void dataColSetNEleNull(SDataCol *pCol, int nEle, int maxPoints);
 
 // Get the data pointer from a column-wised data
-static FORCE_INLINE void *tdGetColDataOfRow(SDataCol *pCol, int row) {
+static FORCE_INLINE const void *tdGetColDataOfRow(SDataCol *pCol, int row) {
+  if (isAllRowsNull(pCol)) {
+    return getNullValue(pCol->type);
+  }
   if (IS_VAR_DATA_TYPE(pCol->type)) {
     return POINTER_SHIFT(pCol->pData, pCol->dataOff[row]);
   } else {
@@ -287,7 +291,7 @@ typedef struct {
 } SDataCols;
 
 #define keyCol(pCols) (&((pCols)->cols[0]))  // Key column
-#define dataColsTKeyAt(pCols, idx) ((TKEY *)(keyCol(pCols)->pData))[(idx)]
+#define dataColsTKeyAt(pCols, idx) ((TKEY *)(keyCol(pCols)->pData))[(idx)]  // the idx row of column-wised data
 #define dataColsKeyAt(pCols, idx) tdGetKey(dataColsTKeyAt(pCols, idx))
 static FORCE_INLINE TKEY dataColsTKeyFirst(SDataCols *pCols) {
   if (pCols->numOfRows) {
@@ -331,13 +335,13 @@ void       tdResetDataCols(SDataCols *pCols);
 int        tdInitDataCols(SDataCols *pCols, STSchema *pSchema);
 SDataCols *tdDupDataCols(SDataCols *pCols, bool keepData);
 SDataCols *tdFreeDataCols(SDataCols *pCols);
-void       tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols *pCols);
 int        tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge, int *pOffset);
 
 // ----------------- K-V data row structure
-/*
+/* |<-------------------------------------- len -------------------------------------------->|
+ * |<----- header  ----->|<--------------------------- body -------------------------------->|
  * +----------+----------+---------------------------------+---------------------------------+
- * |  int16_t |  int16_t |                                 |                                 |
+ * | uint16_t |  int16_t |                                 |                                 |
  * +----------+----------+---------------------------------+---------------------------------+
  * |    len   |   ncols  |           cols index            |             data part           |
  * +----------+----------+---------------------------------+---------------------------------+
@@ -345,14 +349,14 @@ int        tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge
 typedef void *SKVRow;
 
 typedef struct {
-  int16_t colId;
-  int16_t offset;
+  int16_t  colId;
+  uint16_t offset;
 } SColIdx;
 
-#define TD_KV_ROW_HEAD_SIZE (2 * sizeof(int16_t))
+#define TD_KV_ROW_HEAD_SIZE (sizeof(uint16_t) + sizeof(int16_t))
 
-#define kvRowLen(r) (*(int16_t *)(r))
-#define kvRowNCols(r) (*(int16_t *)POINTER_SHIFT(r, sizeof(int16_t)))
+#define kvRowLen(r) (*(TDRowLenT *)(r))
+#define kvRowNCols(r) (*(int16_t *)POINTER_SHIFT(r, sizeof(uint16_t)))
 #define kvRowSetLen(r, len) kvRowLen(r) = (len)
 #define kvRowSetNCols(r, n) kvRowNCols(r) = (n)
 #define kvRowColIdx(r) (SColIdx *)POINTER_SHIFT(r, TD_KV_ROW_HEAD_SIZE)
@@ -362,6 +366,9 @@ typedef struct {
 #define kvRowColIdxAt(r, i) (kvRowColIdx(r) + (i))
 #define kvRowFree(r) tfree(r)
 #define kvRowEnd(r) POINTER_SHIFT(r, kvRowLen(r))
+#define kvRowTKey(r) (*(TKEY *)(kvRowValues(r)))
+#define kvRowKey(r) tdGetKey(kvRowTKey(r))
+#define kvRowDeleted(r) TKEY_IS_DELETED(kvRowTKey(r))
 
 SKVRow tdKVRowDup(SKVRow row);
 int    tdSetKVRowDataOfCol(SKVRow *orow, int16_t colId, int8_t type, void *value);
@@ -385,13 +392,44 @@ static FORCE_INLINE void *tdGetKVRowValOfCol(SKVRow row, int16_t colId) {
   return kvRowColVal(row, (SColIdx *)ret);
 }
 
+static FORCE_INLINE void *tdGetKVRowIdxOfCol(SKVRow row, int16_t colId) {
+  return taosbsearch(&colId, kvRowColIdx(row), kvRowNCols(row), sizeof(SColIdx), comparTagId, TD_EQ);
+}
+
+// offset here not include kvRow header length
+static FORCE_INLINE int tdAppendKvColVal(SKVRow row, const void *value, int16_t colId, int8_t type, int32_t offset) {
+  ASSERT(value != NULL);
+  int32_t  toffset = offset + TD_KV_ROW_HEAD_SIZE;
+  SColIdx *pColIdx = (SColIdx *)POINTER_SHIFT(row, toffset);
+  char *   ptr = (char *)POINTER_SHIFT(row, kvRowLen(row));
+
+  pColIdx->colId = colId;
+  pColIdx->offset = kvRowLen(row);  // offset of pColIdx including the TD_KV_ROW_HEAD_SIZE
+
+  if (IS_VAR_DATA_TYPE(type)) {
+    memcpy(ptr, value, varDataTLen(value));
+    kvRowLen(row) += varDataTLen(value);
+  } else {
+    if (offset == 0) {
+      ASSERT(type == TSDB_DATA_TYPE_TIMESTAMP);
+      TKEY tvalue = tdGetTKEY(*(TSKEY *)value);
+      memcpy(ptr, (void *)(&tvalue), TYPE_BYTES[type]);
+    } else {
+      memcpy(ptr, value, TYPE_BYTES[type]);
+    }
+    kvRowLen(row) += TYPE_BYTES[type];
+  }
+
+  return 0;
+}
+
 // ----------------- K-V data row builder
 typedef struct {
   int16_t  tCols;
   int16_t  nCols;
   SColIdx *pColIdx;
-  int16_t  alloc;
-  int16_t  size;
+  uint16_t alloc;
+  uint16_t size;
   void *   buf;
 } SKVRowBuilder;
 
@@ -427,8 +465,146 @@ static FORCE_INLINE int tdAddColToKVRow(SKVRowBuilder *pBuilder, int16_t colId,
   return 0;
 }
 
+// ----------------- SMemRow appended with sequential data row structure
+/*
+ * |---------|------------------------------------------------- len ---------------------------------->|
+ * |<--------     Head      ------>|<---------   flen -------------->|                                 |
+ * |---------+---------------------+---------------------------------+---------------------------------+
+ * | uint8_t | uint16_t |  int16_t |                                 |                                 |
+ * |---------+----------+----------+---------------------------------+---------------------------------+
+ * |  flag   |   len    | sversion |           First part            |             Second part         |
+ * +---------+----------+----------+---------------------------------+---------------------------------+
+ *
+ * NOTE: timestamp in this row structure is TKEY instead of TSKEY
+ */
+
+// ----------------- SMemRow appended with extended K-V data row structure
+/* |--------------------|------------------------------------------------  len ---------------------------------->|
+ * |<-------------     Head      ------------>|<---------   flen -------------->|                                 |
+ * |--------------------+----------+--------------------------------------------+---------------------------------+
+ * | uint8_t | int16_t  | uint16_t |  int16_t |                                 |                                 |
+ * |---------+----------+----------+----------+---------------------------------+---------------------------------+
+ * |   flag  | sversion |   len    |   ncols  |           cols index            |             data part           |
+ * |---------+----------+----------+----------+---------------------------------+---------------------------------+
+ */
+
+typedef void *SMemRow;
+
+#define TD_MEM_ROW_TYPE_SIZE sizeof(uint8_t)
+#define TD_MEM_ROW_KV_VER_SIZE sizeof(int16_t)
+#define TD_MEM_ROW_KV_TYPE_VER_SIZE (TD_MEM_ROW_TYPE_SIZE + TD_MEM_ROW_KV_VER_SIZE)
+#define TD_MEM_ROW_DATA_HEAD_SIZE (TD_MEM_ROW_TYPE_SIZE + TD_DATA_ROW_HEAD_SIZE)
+// #define TD_MEM_ROW_KV_HEAD_SIZE (TD_MEM_ROW_TYPE_SIZE + TD_MEM_ROW_KV_VER_SIZE + TD_KV_ROW_HEAD_SIZE)
+
+#define SMEM_ROW_DATA 0U  // SDataRow
+#define SMEM_ROW_KV 1U    // SKVRow
+
+#define memRowType(r) (*(uint8_t *)(r))
+#define isDataRow(r) (SMEM_ROW_DATA == memRowType(r))
+#define isKvRow(r) (SMEM_ROW_KV == memRowType(r))
+
+#define memRowDataBody(r) POINTER_SHIFT(r, TD_MEM_ROW_TYPE_SIZE)  // section after flag
+#define memRowKvBody(r) \
+  POINTER_SHIFT(r, TD_MEM_ROW_KV_TYPE_VER_SIZE)  // section after flag + sversion as to reuse SKVRow
+
+#define memRowDataLen(r) (*(TDRowLenT *)memRowDataBody(r))  //  0~65535
+#define memRowKvLen(r) (*(TDRowLenT *)memRowKvBody(r))      //  0~65535
+
+#define memRowDataTLen(r) \
+  ((TDRowTLenT)(memRowDataLen(r) + TD_MEM_ROW_TYPE_SIZE))  // using uint32_t/int32_t to store the TLen
+
+#define memRowKvTLen(r) ((TDRowTLenT)(memRowKvLen(r) + TD_MEM_ROW_KV_TYPE_VER_SIZE))
+
+#define memRowLen(r) (isDataRow(r) ? memRowDataLen(r) : memRowKvLen(r))
+#define memRowTLen(r) (isDataRow(r) ? memRowDataTLen(r) : memRowKvTLen(r))  // using uint32_t/int32_t to store the TLen
+
+#define memRowDataVersion(r) dataRowVersion(memRowDataBody(r))
+#define memRowKvVersion(r) (*(int16_t *)POINTER_SHIFT(r, TD_MEM_ROW_TYPE_SIZE))
+#define memRowVersion(r) (isDataRow(r) ? memRowDataVersion(r) : memRowKvVersion(r))  // schema version
+#define memRowSetKvVersion(r, v) (memRowKvVersion(r) = (v))
+#define memRowTuple(r) (isDataRow(r) ? dataRowTuple(memRowDataBody(r)) : kvRowValues(memRowKvBody(r)))
+
+#define memRowTKey(r) (isDataRow(r) ? dataRowTKey(memRowDataBody(r)) : kvRowTKey(memRowKvBody(r)))
+#define memRowKey(r) (isDataRow(r) ? dataRowKey(memRowDataBody(r)) : kvRowKey(memRowKvBody(r)))
+#define memRowSetTKey(r, k)                 \
+  do {                                      \
+    if (isDataRow(r)) {                     \
+      dataRowTKey(memRowDataBody(r)) = (k); \
+    } else {                                \
+      kvRowTKey(memRowKvBody(r)) = (k);     \
+    }                                       \
+  } while (0)
+
+#define memRowSetType(r, t) (memRowType(r) = (t))
+#define memRowSetLen(r, l) (isDataRow(r) ? memRowDataLen(r) = (l) : memRowKvLen(r) = (l))
+#define memRowSetVersion(r, v) (isDataRow(r) ? dataRowSetVersion(memRowDataBody(r), v) : memRowKvSetVersion(r, v))
+#define memRowCpy(dst, r) memcpy((dst), (r), memRowTLen(r))
+#define memRowMaxBytesFromSchema(s) (schemaTLen(s) + TD_MEM_ROW_DATA_HEAD_SIZE)
+#define memRowDeleted(r) TKEY_IS_DELETED(memRowTKey(r))
+
+SMemRow tdMemRowDup(SMemRow row);
+void    tdAppendMemRowToDataCol(SMemRow row, STSchema *pSchema, SDataCols *pCols);
+// NOTE: offset here including the header size
+static FORCE_INLINE void *tdGetKvRowDataOfCol(void *row, int32_t offset) { return POINTER_SHIFT(row, offset); }
+// NOTE: offset here including the header size
+static FORCE_INLINE void *tdGetMemRowDataOfCol(void *row, int8_t type, int32_t offset) {
+  if (isDataRow(row)) {
+    return tdGetRowDataOfCol(row, type, offset);
+  } else if (isKvRow(row)) {
+    return tdGetKvRowDataOfCol(row, offset);
+  } else {
+    ASSERT(0);
+  }
+  return NULL;
+}
+
+// ----------------- Raw payload structure for row:
+/* |<------------ Head ------------->|<----------- body of column data tuple ------------------->|
+ * |                                 |<----------------- flen ------------->|<--- value part --->|
+ * |SMemRowType| dataTLen |  nCols   |  colId  | colType | offset   |  ...  | value |...|...|... |
+ * +-----------+----------+----------+--------------------------------------|--------------------|
+ * | uint8_t   | uint32_t | uint16_t | int16_t | uint8_t | uint16_t |  ...  |.......|...|...|... |
+ * +-----------+----------+----------+--------------------------------------+--------------------|
+ *  1. offset in column data tuple starts from the value part in case of uint16_t overflow.
+ *  2. dataTLen: total length including the header and body.
+ */
+
+#define PAYLOAD_NCOLS_LEN sizeof(uint16_t)
+#define PAYLOAD_NCOLS_OFFSET (sizeof(uint8_t) + sizeof(TDRowTLenT))
+#define PAYLOAD_HEADER_LEN (PAYLOAD_NCOLS_OFFSET + PAYLOAD_NCOLS_LEN)
+#define PAYLOAD_ID_LEN sizeof(int16_t)
+#define PAYLOAD_ID_TYPE_LEN (sizeof(int16_t) + sizeof(uint8_t))
+#define PAYLOAD_COL_HEAD_LEN (PAYLOAD_ID_TYPE_LEN + sizeof(uint16_t))
+#define PAYLOAD_PRIMARY_COL_LEN (PAYLOAD_ID_TYPE_LEN + sizeof(TSKEY))
+
+#define payloadBody(r) POINTER_SHIFT(r, PAYLOAD_HEADER_LEN)
+#define payloadType(r) (*(uint8_t *)(r))
+#define payloadSetType(r, t) (payloadType(r) = (t))
+#define payloadTLen(r) (*(TDRowTLenT *)POINTER_SHIFT(r, TD_MEM_ROW_TYPE_SIZE))  // including total header
+#define payloadSetTLen(r, l) (payloadTLen(r) = (l))
+#define payloadNCols(r) (*(TDRowLenT *)POINTER_SHIFT(r, PAYLOAD_NCOLS_OFFSET))
+#define payloadSetNCols(r, n) (payloadNCols(r) = (n))
+#define payloadValuesOffset(r) \
+  (PAYLOAD_HEADER_LEN + payloadNCols(r) * PAYLOAD_COL_HEAD_LEN)    // avoid using the macro in loop
+#define payloadValues(r) POINTER_SHIFT(r, payloadValuesOffset(r))  // avoid using the macro in loop
+#define payloadColId(c) (*(int16_t *)(c))
+#define payloadColType(c) (*(uint8_t *)POINTER_SHIFT(c, PAYLOAD_ID_LEN))
+#define payloadColOffset(c) (*(uint16_t *)POINTER_SHIFT(c, PAYLOAD_ID_TYPE_LEN))
+#define payloadColValue(c) POINTER_SHIFT(c, payloadColOffset(c))
+
+#define payloadColSetId(c, i) (payloadColId(c) = (i))
+#define payloadColSetType(c, t) (payloadColType(c) = (t))
+#define payloadColSetOffset(c, o) (payloadColOffset(c) = (o))
+
+#define payloadTSKey(r) (*(TSKEY *)POINTER_SHIFT(r, payloadValuesOffset(r)))
+#define payloadTKey(r) (*(TKEY *)POINTER_SHIFT(r, payloadValuesOffset(r)))
+#define payloadKey(r) tdGetKey(payloadTKey(r))
+
+
+static FORCE_INLINE char *payloadNextCol(char *pCol) { return (char *)POINTER_SHIFT(pCol, PAYLOAD_COL_HEAD_LEN); }
+
 #ifdef __cplusplus
 }
 #endif
 
-#endif  // _TD_DATA_FORMAT_H_
+#endif  // _TD_DATA_FORMAT_H_
\ No newline at end of file

src/common/src/tdataformat.c

@@ -198,6 +198,14 @@ SDataRow tdDataRowDup(SDataRow row) {
   return trow;
 }
 
+SMemRow tdMemRowDup(SMemRow row) {
+  SMemRow trow = malloc(memRowTLen(row));
+  if (trow == NULL) return NULL;
+
+  memRowCpy(trow, row);
+  return trow;
+}
+
 void dataColInit(SDataCol *pDataCol, STColumn *pCol, void **pBuf, int maxPoints) {
   pDataCol->type = colType(pCol);
   pDataCol->colId = colColId(pCol);
@@ -217,11 +225,22 @@ void dataColInit(SDataCol *pDataCol, STColumn *pCol, void **pBuf, int maxPoints)
     *pBuf = POINTER_SHIFT(*pBuf, pDataCol->spaceSize);
   }
 }
-
 // value from timestamp should be TKEY here instead of TSKEY
-void dataColAppendVal(SDataCol *pCol, void *value, int numOfRows, int maxPoints) {
+void dataColAppendVal(SDataCol *pCol, const void *value, int numOfRows, int maxPoints) {
   ASSERT(pCol != NULL && value != NULL);
 
+  if (isAllRowsNull(pCol)) {
+    if (isNull(value, pCol->type)) {
+      // all null value yet, just return
+      return;
+    }
+
+    if (numOfRows > 0) {
+      // Find the first not null value, fill all previouse values as NULL
+      dataColSetNEleNull(pCol, numOfRows, maxPoints);
+    }
+  }
+
   if (IS_VAR_DATA_TYPE(pCol->type)) {
     // set offset
     pCol->dataOff[numOfRows] = pCol->len;
@@ -243,7 +262,7 @@ bool isNEleNull(SDataCol *pCol, int nEle) {
   return true;
 }
 
-void dataColSetNullAt(SDataCol *pCol, int index) {
+FORCE_INLINE void dataColSetNullAt(SDataCol *pCol, int index) {
   if (IS_VAR_DATA_TYPE(pCol->type)) {
     pCol->dataOff[index] = pCol->len;
     char *ptr = POINTER_SHIFT(pCol->pData, pCol->len);
@@ -399,8 +418,7 @@ void tdResetDataCols(SDataCols *pCols) {
     }
   }
 }
-
-void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols *pCols) {
+static void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols *pCols) {
   ASSERT(pCols->numOfRows == 0 || dataColsKeyLast(pCols) < dataRowKey(row));
 
   int rcol = 0;
@@ -419,7 +437,8 @@ void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols *pCols)
     while (dcol < pCols->numOfCols) {
       SDataCol *pDataCol = &(pCols->cols[dcol]);
       if (rcol >= schemaNCols(pSchema)) {
-        dataColSetNullAt(pDataCol, pCols->numOfRows);
+        // dataColSetNullAt(pDataCol, pCols->numOfRows);
+        dataColAppendVal(pDataCol, getNullValue(pDataCol->type), pCols->numOfRows, pCols->maxPoints);
         dcol++;
         continue;
       }
@@ -433,7 +452,8 @@ void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols *pCols)
       } else if (pRowCol->colId < pDataCol->colId) {
         rcol++;
       } else {
-        dataColSetNullAt(pDataCol, pCols->numOfRows);
+        // dataColSetNullAt(pDataCol, pCols->numOfRows);
+        dataColAppendVal(pDataCol, getNullValue(pDataCol->type), pCols->numOfRows, pCols->maxPoints);
         dcol++;
       }
     }
@@ -441,6 +461,62 @@ void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols *pCols)
   pCols->numOfRows++;
 }
 
+static void tdAppendKvRowToDataCol(SKVRow row, STSchema *pSchema, SDataCols *pCols) {
+  ASSERT(pCols->numOfRows == 0 || dataColsKeyLast(pCols) < kvRowKey(row));
+
+  int rcol = 0;
+  int dcol = 0;
+
+  if (kvRowDeleted(row)) {
+    for (; dcol < pCols->numOfCols; dcol++) {
+      SDataCol *pDataCol = &(pCols->cols[dcol]);
+      if (dcol == 0) {
+        dataColAppendVal(pDataCol, kvRowValues(row), pCols->numOfRows, pCols->maxPoints);
+      } else {
+        dataColSetNullAt(pDataCol, pCols->numOfRows);
+      }
+    }
+  } else {
+    int nRowCols = kvRowNCols(row);
+
+    while (dcol < pCols->numOfCols) {
+      SDataCol *pDataCol = &(pCols->cols[dcol]);
+      if (rcol >= nRowCols || rcol >= schemaNCols(pSchema)) {
+        // dataColSetNullAt(pDataCol, pCols->numOfRows);
+        dataColAppendVal(pDataCol, getNullValue(pDataCol->type), pCols->numOfRows, pCols->maxPoints);
+        ++dcol;
+        continue;
+      }
+
+      SColIdx *colIdx = kvRowColIdxAt(row, rcol);
+
+      if (colIdx->colId == pDataCol->colId) {
+        void *value = tdGetKvRowDataOfCol(row, colIdx->offset);
+        dataColAppendVal(pDataCol, value, pCols->numOfRows, pCols->maxPoints);
+        ++dcol;
+        ++rcol;
+      } else if (colIdx->colId < pDataCol->colId) {
+        ++rcol;
+      } else {
+        // dataColSetNullAt(pDataCol, pCols->numOfRows);
+        dataColAppendVal(pDataCol, getNullValue(pDataCol->type), pCols->numOfRows, pCols->maxPoints);
+        ++dcol;
+      }
+    }
+  }
+  pCols->numOfRows++;
+}
+
+void tdAppendMemRowToDataCol(SMemRow row, STSchema *pSchema, SDataCols *pCols) {
+  if (isDataRow(row)) {
+    tdAppendDataRowToDataCol(memRowDataBody(row), pSchema, pCols);
+  } else if (isKvRow(row)) {
+    tdAppendKvRowToDataCol(memRowKvBody(row), pSchema, pCols);
+  } else {
+    ASSERT(0);
+  }
+}
+
 int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge, int *pOffset) {
   ASSERT(rowsToMerge > 0 && rowsToMerge <= source->numOfRows);
   ASSERT(target->numOfCols == source->numOfCols);
@@ -563,7 +639,7 @@ int tdSetKVRowDataOfCol(SKVRow *orow, int16_t colId, int8_t type, void *value) {
     nrow = malloc(kvRowLen(row) + sizeof(SColIdx) + diff);
     if (nrow == NULL) return -1;
 
-    kvRowSetLen(nrow, kvRowLen(row) + (int16_t)sizeof(SColIdx) + diff);
+    kvRowSetLen(nrow, kvRowLen(row) + (uint16_t)sizeof(SColIdx) + diff);
     kvRowSetNCols(nrow, kvRowNCols(row) + 1);
 
     if (ptr == NULL) {
@@ -605,8 +681,8 @@ int tdSetKVRowDataOfCol(SKVRow *orow, int16_t colId, int8_t type, void *value) {
       if (varDataTLen(value) == varDataTLen(pOldVal)) { // just update the column value in place
         memcpy(pOldVal, value, varDataTLen(value));
       } else { // need to reallocate the memory
-        int16_t diff = varDataTLen(value) - varDataTLen(pOldVal);
-        int16_t nlen = kvRowLen(row) + diff;
+        uint16_t diff = varDataTLen(value) - varDataTLen(pOldVal);
+        uint16_t nlen = kvRowLen(row) + diff;
         ASSERT(nlen > 0);
         nrow = malloc(nlen);
         if (nrow == NULL) return -1;
@@ -708,4 +784,4 @@ SKVRow tdGetKVRowFromBuilder(SKVRowBuilder *pBuilder) {
   memcpy(kvRowValues(row), pBuilder->buf, pBuilder->size);
 
   return row;
-}
+}
\ No newline at end of file

src/common/src/ttypes.c

@@ -492,30 +492,32 @@ void setNullN(char *val, int32_t type, int32_t bytes, int32_t numOfElems) {
   }
 }
 
-static uint8_t   nullBool     = TSDB_DATA_BOOL_NULL;
-static uint8_t   nullTinyInt  = TSDB_DATA_TINYINT_NULL;
-static uint16_t  nullSmallInt = TSDB_DATA_SMALLINT_NULL;
-static uint32_t  nullInt      = TSDB_DATA_INT_NULL;
-static uint64_t  nullBigInt   = TSDB_DATA_BIGINT_NULL;
-static uint32_t  nullFloat    = TSDB_DATA_FLOAT_NULL;
-static uint64_t  nullDouble   = TSDB_DATA_DOUBLE_NULL;
-static uint8_t   nullTinyIntu = TSDB_DATA_UTINYINT_NULL;
-static uint16_t  nullSmallIntu = TSDB_DATA_USMALLINT_NULL;
-static uint32_t  nullIntu     = TSDB_DATA_UINT_NULL;
-static uint64_t  nullBigIntu  = TSDB_DATA_UBIGINT_NULL;
-
-static union {
-  tstr str;
-  char pad[sizeof(tstr) + 4];
-} nullBinary = {.str = {.len = 1}}, nullNchar = {.str = {.len = 4}};
-
-static void *nullValues[] = {
+static uint8_t      nullBool = TSDB_DATA_BOOL_NULL;
+static uint8_t      nullTinyInt = TSDB_DATA_TINYINT_NULL;
+static uint16_t     nullSmallInt = TSDB_DATA_SMALLINT_NULL;
+static uint32_t     nullInt = TSDB_DATA_INT_NULL;
+static uint64_t     nullBigInt = TSDB_DATA_BIGINT_NULL;
+static uint32_t     nullFloat = TSDB_DATA_FLOAT_NULL;
+static uint64_t     nullDouble = TSDB_DATA_DOUBLE_NULL;
+static uint8_t      nullTinyIntu = TSDB_DATA_UTINYINT_NULL;
+static uint16_t     nullSmallIntu = TSDB_DATA_USMALLINT_NULL;
+static uint32_t     nullIntu = TSDB_DATA_UINT_NULL;
+static uint64_t     nullBigIntu = TSDB_DATA_UBIGINT_NULL;
+static SBinaryNullT nullBinary = {1, TSDB_DATA_BINARY_NULL};
+static SNCharNullT  nullNchar = {4, TSDB_DATA_NCHAR_NULL};
+
+// static union {
+//   tstr str;
+//   char pad[sizeof(tstr) + 4];
+// } nullBinary = {.str = {.len = 1}}, nullNchar = {.str = {.len = 4}};
+
+static const void *nullValues[] = {
     &nullBool,     &nullTinyInt,   &nullSmallInt, &nullInt,    &nullBigInt,
     &nullFloat,    &nullDouble,    &nullBinary,   &nullBigInt, &nullNchar,
     &nullTinyIntu, &nullSmallIntu, &nullIntu,     &nullBigIntu,
 };
 
-void *getNullValue(int32_t type) {
+const void *getNullValue(int32_t type) {
   assert(type >= TSDB_DATA_TYPE_BOOL && type <= TSDB_DATA_TYPE_UBIGINT);
   return nullValues[type - 1];
 }

src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetRowData.java

@@ -19,6 +19,7 @@ import com.taosdata.jdbc.utils.NullType;
 import java.math.BigDecimal;
 import java.sql.SQLException;
 import java.sql.Timestamp;
+import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Collections;
 
@@ -463,6 +464,25 @@ public class TSDBResultSetRowData {
         data.set(col, tsObj);
     }
 
+    /**
+     * this implementation is used for TDengine old version
+     */
+    public void setTimestamp(int col, long ts) {
+        //TODO: this implementation contains logical error
+        // when precision is us the (long ts) is 16 digital number
+        // when precision is ms, the (long ts) is 13 digital number
+        // we need a JNI function like this:
+        //      public void setTimestamp(int col, long epochSecond, long nanoAdjustment)
+        if (ts < 1_0000_0000_0000_0L) {
+            data.set(col, new Timestamp(ts));
+        } else {
+            long epochSec = ts / 1000_000L;
+            long nanoAdjustment = ts % 1000_000L * 1000L;
+            Timestamp timestamp = Timestamp.from(Instant.ofEpochSecond(epochSec, nanoAdjustment));
+            data.set(col, timestamp);
+        }
+    }
+
     public Timestamp getTimestamp(int col, int nativeType) {
         Object obj = data.get(col - 1);
         if (obj == null)

src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/Utils.java

@@ -5,7 +5,6 @@ import com.google.common.collect.RangeSet;
 import com.google.common.collect.TreeRangeSet;
 import com.taosdata.jdbc.enums.TimestampPrecision;
 
-import java.nio.charset.Charset;
 import java.nio.charset.StandardCharsets;
 import java.sql.Date;
 import java.sql.Time;
@@ -110,15 +109,27 @@ public class Utils {
             return rawSql;
         // toLowerCase
         String preparedSql = rawSql.trim().toLowerCase();
-        String[] clause = new String[]{"values\\s*\\([\\s\\S]*?\\)", "tags\\s*\\([\\s\\S]*?\\)", "where[\\s\\S]*"};
+        String[] clause = new String[]{"tags\\s*\\([\\s\\S]*?\\)", "where[\\s\\S]*"};
         Map<Integer, Integer> placeholderPositions = new HashMap<>();
         RangeSet<Integer> clauseRangeSet = TreeRangeSet.create();
         findPlaceholderPosition(preparedSql, placeholderPositions);
+        // find tags and where clause's position
         findClauseRangeSet(preparedSql, clause, clauseRangeSet);
+        // find values clause's position
+        findValuesClauseRangeSet(preparedSql, clauseRangeSet);
 
         return transformSql(rawSql, parameters, placeholderPositions, clauseRangeSet);
     }
 
+    private static void findValuesClauseRangeSet(String preparedSql, RangeSet<Integer> clauseRangeSet) {
+        Matcher matcher = Pattern.compile("(values|,)\\s*(\\([^)]*\\))").matcher(preparedSql);
+        while (matcher.find()) {
+            int start = matcher.start(2);
+            int end = matcher.end(2);
+            clauseRangeSet.add(Range.closedOpen(start, end));
+        }
+    }
+
     private static void findClauseRangeSet(String preparedSql, String[] regexArr, RangeSet<Integer> clauseRangeSet) {
         clauseRangeSet.clear();
         for (String regex : regexArr) {
@@ -126,7 +137,7 @@ public class Utils {
             while (matcher.find()) {
                 int start = matcher.start();
                 int end = matcher.end();
-                clauseRangeSet.add(Range.closed(start, end));
+                clauseRangeSet.add(Range.closedOpen(start, end));
             }
         }
     }

src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/UtilsTest.java

@@ -3,6 +3,8 @@ package com.taosdata.jdbc.utils;
 import org.junit.Assert;
 import org.junit.Test;
 
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 import java.util.stream.Stream;
 
 public class UtilsTest {
@@ -32,7 +34,7 @@ public class UtilsTest {
     }
 
     @Test
-    public void getNativeSqlReplaceQuestionMarks() {
+    public void lowerCase() {
         // given
         String nativeSql = "insert into ?.? (ts, temperature, humidity) using ?.? tags(?,?) values(now, ?, ?)";
         Object[] parameters = Stream.of("test", "t1", "test", "weather", "beijing", 1, 12.2, 4).toArray();
@@ -46,7 +48,7 @@ public class UtilsTest {
     }
 
     @Test
-    public void getNativeSqlReplaceQuestionMarks2() {
+    public void upperCase() {
         // given
         String nativeSql = "INSERT INTO ? (TS,CURRENT,VOLTAGE,PHASE) USING METERS TAGS (?)  VALUES (?,?,?,?)";
         Object[] parameters = Stream.of("d1", 1, 123, 3.14, 220, 4).toArray();
@@ -59,9 +61,49 @@ public class UtilsTest {
         Assert.assertEquals(expected, actual);
     }
 
+    @Test
+    public void multiValues() {
+        // given
+        String nativeSql = "INSERT INTO ? (TS,CURRENT,VOLTAGE,PHASE) USING METERS TAGS (?)  VALUES (?,?,?,?),(?,?,?,?)";
+        Object[] parameters = Stream.of("d1", 1, 100, 3.14, "abc", 4, 200, 3.1415, "xyz", 5).toArray();
+
+        // when
+        String actual = Utils.getNativeSql(nativeSql, parameters);
+
+        // then
+        String expected = "INSERT INTO d1 (TS,CURRENT,VOLTAGE,PHASE) USING METERS TAGS (1)  VALUES (100,3.14,'abc',4),(200,3.1415,'xyz',5)";
+        Assert.assertEquals(expected, actual);
+    }
+
+    @Test
+    public void lineTerminator() {
+        // given
+        String nativeSql = "INSERT INTO ? (TS,CURRENT,VOLTAGE,PHASE) USING METERS TAGS (?)  VALUES (?,?,\r\n?,?),(?,?,?,?)";
+        Object[] parameters = Stream.of("d1", 1, 100, 3.14, "abc", 4, 200, 3.1415, "xyz", 5).toArray();
+
+        // when
+        String actual = Utils.getNativeSql(nativeSql, parameters);
+
+        // then
+        String expected = "INSERT INTO d1 (TS,CURRENT,VOLTAGE,PHASE) USING METERS TAGS (1)  VALUES (100,3.14,\r\n'abc',4),(200,3.1415,'xyz',5)";
+        Assert.assertEquals(expected, actual);
+    }
+
+    @Test
+    public void lineTerminatorAndMultiValues() {
+        String nativeSql = "INSERT Into ? TAGS(?) VALUES(?,?,\r\n?,?),(?,? ,\r\n?,?) t? tags (?) Values (?,?,?\r\n,?),(?,?,?,?) t? Tags(?) values  (?,?,?,?) , (?,?,?,?)";
+        Object[] parameters = Stream.of("t1", "abc", 100, 1.1, "xxx", "xxx", 200, 2.2, "xxx", "xxx", 2, "bcd", 300, 3.3, "xxx", "xxx", 400, 4.4, "xxx", "xxx", 3, "cde", 500, 5.5, "xxx", "xxx", 600, 6.6, "xxx", "xxx").toArray();
+
+        // when
+        String actual = Utils.getNativeSql(nativeSql, parameters);
+
+        // then
+        String expected = "INSERT Into t1 TAGS('abc') VALUES(100,1.1,\r\n'xxx','xxx'),(200,2.2 ,\r\n'xxx','xxx') t2 tags ('bcd') Values (300,3.3,'xxx'\r\n,'xxx'),(400,4.4,'xxx','xxx') t3 Tags('cde') values  (500,5.5,'xxx','xxx') , (600,6.6,'xxx','xxx')";
+        Assert.assertEquals(expected, actual);
+    }
 
     @Test
-    public void getNativeSqlReplaceNothing() {
+    public void replaceNothing() {
         // given
         String nativeSql = "insert into test.t1 (ts, temperature, humidity) using test.weather tags('beijing',1) values(now, 12.2, 4)";
 
@@ -73,7 +115,7 @@ public class UtilsTest {
     }
 
     @Test
-    public void getNativeSqlReplaceNothing2() {
+    public void replaceNothing2() {
         // given
         String nativeSql = "insert into test.t1 (ts, temperature, humidity) using test.weather tags('beijing',1) values(now, 12.2, 4)";
         Object[] parameters = Stream.of("test", "t1", "test", "weather", "beijing", 1, 12.2, 4).toArray();
@@ -86,7 +128,7 @@ public class UtilsTest {
     }
 
     @Test
-    public void getNativeSqlReplaceNothing3() {
+    public void replaceNothing3() {
         // given
         String nativeSql = "insert into ?.? (ts, temperature, humidity) using ?.? tags(?,?) values(now, ?, ?)";
 

src/cq/src/cqMain.c

@@ -476,21 +476,23 @@ static void cqProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row) {
   
   cDebug("vgId:%d, id:%d CQ:%s stream result is ready", pContext->vgId, pObj->tid, pObj->sqlStr);
 
-  int32_t size = sizeof(SWalHead) + sizeof(SSubmitMsg) + sizeof(SSubmitBlk) + TD_DATA_ROW_HEAD_SIZE + pObj->rowSize;
+  int32_t size = sizeof(SWalHead) + sizeof(SSubmitMsg) + sizeof(SSubmitBlk) + TD_MEM_ROW_DATA_HEAD_SIZE + pObj->rowSize;
   char *buffer = calloc(size, 1);
 
   SWalHead   *pHead = (SWalHead *)buffer;
   SSubmitMsg *pMsg = (SSubmitMsg *) (buffer + sizeof(SWalHead));
   SSubmitBlk *pBlk = (SSubmitBlk *) (buffer + sizeof(SWalHead) + sizeof(SSubmitMsg));
 
-  SDataRow trow = (SDataRow)pBlk->data;
-  tdInitDataRow(trow, pSchema);
+  SMemRow trow = (SMemRow)pBlk->data;
+  SDataRow dataRow = (SDataRow)memRowDataBody(trow);
+  memRowSetType(trow, SMEM_ROW_DATA);
+  tdInitDataRow(dataRow, pSchema);
 
   for (int32_t i = 0; i < pSchema->numOfCols; i++) {
     STColumn *c = pSchema->columns + i;
-    void* val = row[i];
+    void *val = row[i];
     if (val == NULL) {
-      val = getNullValue(c->type);
+      val = (void *)getNullValue(c->type);
     } else if (c->type == TSDB_DATA_TYPE_BINARY) {
       val = ((char*)val) - sizeof(VarDataLenT);
     } else if (c->type == TSDB_DATA_TYPE_NCHAR) {
@@ -500,9 +502,9 @@ static void cqProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row) {
       memcpy((char *)val + sizeof(VarDataLenT), buf, len);
       varDataLen(val) = len;
     }
-    tdAppendColVal(trow, val, c->type, c->bytes, c->offset);
+    tdAppendColVal(dataRow, val, c->type, c->offset);
   }
-  pBlk->dataLen = htonl(dataRowLen(trow));
+  pBlk->dataLen = htonl(memRowDataTLen(trow));
   pBlk->schemaLen = 0;
 
   pBlk->uid = htobe64(pObj->uid);
@@ -511,7 +513,7 @@ static void cqProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row) {
   pBlk->sversion = htonl(pSchema->version);
   pBlk->padding = 0;
 
-  pHead->len = sizeof(SSubmitMsg) + sizeof(SSubmitBlk) + dataRowLen(trow);
+  pHead->len = sizeof(SSubmitMsg) + sizeof(SSubmitBlk) + memRowDataTLen(trow);
 
   pMsg->header.vgId = htonl(pContext->vgId);
   pMsg->header.contLen = htonl(pHead->len);

src/inc/taos.h

@@ -72,6 +72,7 @@ DLL_EXPORT int  taos_init();
 DLL_EXPORT void  taos_cleanup(void);
 DLL_EXPORT int   taos_options(TSDB_OPTION option, const void *arg, ...);
 DLL_EXPORT TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port);
+DLL_EXPORT TAOS *taos_connect_auth(const char *ip, const char *user, const char *auth, const char *db, uint16_t port);
 DLL_EXPORT void  taos_close(TAOS *taos);
 
 const char *taos_data_type(int type);
@@ -110,21 +111,21 @@ typedef struct TAOS_MULTI_BIND {
 } TAOS_MULTI_BIND;
 
 
-TAOS_STMT *taos_stmt_init(TAOS *taos);
-int        taos_stmt_prepare(TAOS_STMT *stmt, const char *sql, unsigned long length);
-int        taos_stmt_set_tbname_tags(TAOS_STMT* stmt, const char* name, TAOS_BIND* tags);
-int        taos_stmt_set_tbname(TAOS_STMT* stmt, const char* name);
-int        taos_stmt_is_insert(TAOS_STMT *stmt, int *insert);
-int        taos_stmt_num_params(TAOS_STMT *stmt, int *nums);
+DLL_EXPORT TAOS_STMT *taos_stmt_init(TAOS *taos);
+DLL_EXPORT int        taos_stmt_prepare(TAOS_STMT *stmt, const char *sql, unsigned long length);
+DLL_EXPORT int        taos_stmt_set_tbname_tags(TAOS_STMT* stmt, const char* name, TAOS_BIND* tags);
+DLL_EXPORT int        taos_stmt_set_tbname(TAOS_STMT* stmt, const char* name);
+DLL_EXPORT int        taos_stmt_is_insert(TAOS_STMT *stmt, int *insert);
+DLL_EXPORT int        taos_stmt_num_params(TAOS_STMT *stmt, int *nums);
 int        taos_stmt_get_param(TAOS_STMT *stmt, int idx, int *type, int *bytes);
-int        taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_BIND *bind);
+DLL_EXPORT int        taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_BIND *bind);
 int        taos_stmt_bind_param_batch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind);
 int        taos_stmt_bind_single_param_batch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, int colIdx);
-int        taos_stmt_add_batch(TAOS_STMT *stmt);
-int        taos_stmt_execute(TAOS_STMT *stmt);
-TAOS_RES * taos_stmt_use_result(TAOS_STMT *stmt);
-int        taos_stmt_close(TAOS_STMT *stmt);
-char *     taos_stmt_errstr(TAOS_STMT *stmt);
+DLL_EXPORT int        taos_stmt_add_batch(TAOS_STMT *stmt);
+DLL_EXPORT int        taos_stmt_execute(TAOS_STMT *stmt);
+DLL_EXPORT TAOS_RES * taos_stmt_use_result(TAOS_STMT *stmt);
+DLL_EXPORT int        taos_stmt_close(TAOS_STMT *stmt);
+DLL_EXPORT char *     taos_stmt_errstr(TAOS_STMT *stmt);
 
 DLL_EXPORT TAOS_RES *taos_query(TAOS *taos, const char *sql);
 DLL_EXPORT TAOS_ROW taos_fetch_row(TAOS_RES *res);
@@ -139,10 +140,10 @@ DLL_EXPORT int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int n
 DLL_EXPORT void taos_stop_query(TAOS_RES *res);
 DLL_EXPORT bool taos_is_null(TAOS_RES *res, int32_t row, int32_t col);
 
-int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows);
-int taos_validate_sql(TAOS *taos, const char *sql);
+DLL_EXPORT int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows);
+DLL_EXPORT int taos_validate_sql(TAOS *taos, const char *sql);
 
-int* taos_fetch_lengths(TAOS_RES *res);
+DLL_EXPORT int* taos_fetch_lengths(TAOS_RES *res);
 
 // TAOS_RES   *taos_list_tables(TAOS *mysql, const char *wild);
 // TAOS_RES   *taos_list_dbs(TAOS *mysql, const char *wild);

src/inc/taosdef.h

@@ -180,7 +180,7 @@ do { \
 // this is the length of its string representation, including the terminator zero
 #define TSDB_ACCT_ID_LEN          11
 
-#define TSDB_MAX_COLUMNS          1024
+#define TSDB_MAX_COLUMNS          4096
 #define TSDB_MIN_COLUMNS          2       //PRIMARY COLUMN(timestamp) + other columns
 
 #define TSDB_NODE_NAME_LEN        64
@@ -199,7 +199,13 @@ do { \
 
 #define TSDB_APPNAME_LEN          TSDB_UNI_LEN
 
-#define TSDB_MAX_BYTES_PER_ROW    16384
+  /**
+   *  In some scenarios uint16_t (0~65535) is used to store the row len.
+   *  - Firstly, we use 65531(65535 - 4), as the SDataRow/SKVRow contains 4 bits header.
+   *  - Secondly, if all cols are VarDataT type except primary key, we need 4 bits to store the offset, thus
+   *    the final value is 65531-(4096-1)*4 = 49151.
+   */
+#define TSDB_MAX_BYTES_PER_ROW    49151
 #define TSDB_MAX_TAGS_LEN         16384
 #define TSDB_MAX_TAGS             128
 #define TSDB_MAX_TAG_CONDITIONS   1024
@@ -327,8 +333,9 @@ do { \
 #define TSDB_MAX_JOIN_TABLE_NUM         10
 #define TSDB_MAX_UNION_CLAUSE           5
 
-#define TSDB_MAX_BINARY_LEN            (TSDB_MAX_BYTES_PER_ROW-TSDB_KEYSIZE)
-#define TSDB_MAX_NCHAR_LEN             (TSDB_MAX_BYTES_PER_ROW-TSDB_KEYSIZE)
+#define TSDB_MAX_FIELD_LEN              16384
+#define TSDB_MAX_BINARY_LEN            (TSDB_MAX_FIELD_LEN-TSDB_KEYSIZE) // keep 16384
+#define TSDB_MAX_NCHAR_LEN             (TSDB_MAX_FIELD_LEN-TSDB_KEYSIZE) // keep 16384
 #define PRIMARYKEY_TIMESTAMP_COL_INDEX  0
 
 #define TSDB_MAX_RPC_THREADS            5

src/inc/taoserror.h

@@ -174,6 +174,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_MND_FIELD_NOT_EXIST           TAOS_DEF_ERROR_CODE(0, 0x036C)  //"Field does not exist")
 #define TSDB_CODE_MND_INVALID_STABLE_NAME       TAOS_DEF_ERROR_CODE(0, 0x036D)  //"Super table does not exist")
 #define TSDB_CODE_MND_INVALID_CREATE_TABLE_MSG  TAOS_DEF_ERROR_CODE(0, 0x036E)  //"Invalid create table message")
+#define TSDB_CODE_MND_EXCEED_MAX_ROW_BYTES      TAOS_DEF_ERROR_CODE(0, 0x036F)  //"Exceed max row bytes")
 
 #define TSDB_CODE_MND_INVALID_FUNC_NAME         TAOS_DEF_ERROR_CODE(0, 0x0370)  //"Invalid func name")
 #define TSDB_CODE_MND_INVALID_FUNC_LEN          TAOS_DEF_ERROR_CODE(0, 0x0371)  //"Invalid func length")

src/inc/ttype.h

@@ -10,14 +10,28 @@ extern "C" {
 #include "taosdef.h"
 
 // ----------------- For variable data types such as TSDB_DATA_TYPE_BINARY and TSDB_DATA_TYPE_NCHAR
-typedef int32_t VarDataOffsetT;
-typedef int16_t VarDataLenT;
+typedef int32_t  VarDataOffsetT;
+typedef int16_t  VarDataLenT;  // maxVarDataLen: 32767
+typedef uint16_t TDRowLenT;    // not including overhead: 0 ~ 65535
+typedef uint32_t TDRowTLenT;   // total length, including overhead
 
 typedef struct tstr {
   VarDataLenT len;
   char        data[];
 } tstr;
 
+#pragma pack(push, 1)
+typedef struct {
+  VarDataLenT len;
+  uint8_t     data;
+} SBinaryNullT;
+
+typedef struct {
+  VarDataLenT len;
+  uint32_t    data;
+} SNCharNullT;
+#pragma pack(pop)
+
 #define VARSTR_HEADER_SIZE  sizeof(VarDataLenT)
 
 #define varDataLen(v)       ((VarDataLenT *)(v))[0]
@@ -180,7 +194,7 @@ bool isValidDataType(int32_t type);
 void  setVardataNull(char* val, int32_t type);
 void  setNull(char *val, int32_t type, int32_t bytes);
 void  setNullN(char *val, int32_t type, int32_t bytes, int32_t numOfElems);
-void *getNullValue(int32_t type);
+const void *getNullValue(int32_t type);
 
 void assignVal(char *val, const char *src, int32_t len, int32_t type);
 void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf);

src/mnode/src/mnodeTable.c

@@ -1503,6 +1503,18 @@ static int32_t mnodeChangeSuperTableColumn(SMnodeMsg *pMsg) {
     return TSDB_CODE_MND_FIELD_NOT_EXIST;
   }
 
+  // check exceed max row bytes
+  int32_t i;
+  uint32_t nLen = 0;
+  for (i = 0; i < pStable->numOfColumns; ++i) {
+    nLen += (pStable->schema[i].colId == col) ? pAlter->schema[0].bytes : pStable->schema[i].bytes;
+  }
+  if (nLen > TSDB_MAX_BYTES_PER_ROW) {
+    mError("msg:%p, app:%p stable:%s, change column, name:%s exceed max row bytes", pMsg, pMsg->rpcMsg.ahandle,
+           pStable->info.tableId, name);
+    return TSDB_CODE_MND_EXCEED_MAX_ROW_BYTES;
+  }
+
   // update
   SSchema *schema = (SSchema *) (pStable->schema + col);
   ASSERT(schema->type == TSDB_DATA_TYPE_BINARY || schema->type == TSDB_DATA_TYPE_NCHAR);

src/plugins/http/inc/httpParser.h

@@ -100,6 +100,7 @@ typedef enum HTTP_PARSER_STATE {
   HTTP_PARSER_CHUNK,
   HTTP_PARSER_END,
   HTTP_PARSER_ERROR,
+  HTTP_PARSER_OPTIONAL_SP
 } HTTP_PARSER_STATE;
 
 typedef enum HTTP_AUTH_TYPE {

src/plugins/http/src/httpParser.c

@@ -744,6 +744,15 @@ static int32_t httpParserOnSp(HttpParser *parser, HTTP_PARSER_STATE state, const
   return ok;
 }
 
+static int32_t httpParserOnOptionalSp(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) {
+  int32_t ok = 0;
+  if (c != ' ') {
+    *again = 1;
+    httpPopStack(parser);
+  }
+  return ok;
+}
+
 static int32_t httpParserOnStatusCode(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) {
   HttpContext *pContext = parser->pContext;
   int32_t      ok = 0;
@@ -867,7 +876,7 @@ static int32_t httpParserOnHeader(HttpParser *parser, HTTP_PARSER_STATE state, c
       }
       httpPushStack(parser, HTTP_PARSER_CRLF);
       httpPushStack(parser, HTTP_PARSER_HEADER_VAL);
-      httpPushStack(parser, HTTP_PARSER_SP);
+      httpPushStack(parser, HTTP_PARSER_OPTIONAL_SP);
       httpPushStack(parser, HTTP_PARSER_HEADER_KEY);
       break;
     }
@@ -1061,6 +1070,10 @@ static int32_t httpParseChar(HttpParser *parser, const char c, int32_t *again) {
       ok = httpParserOnSp(parser, state, c, again);
       break;
     }
+    if (state == HTTP_PARSER_OPTIONAL_SP) {
+      ok = httpParserOnOptionalSp(parser, state, c, again);
+      break;
+    }
     if (state == HTTP_PARSER_STATUS_CODE) {
       ok = httpParserOnStatusCode(parser, state, c, again);
       break;

src/query/inc/qExtbuffer.h

@@ -77,13 +77,13 @@ typedef struct tFilePagesItem {
 
 typedef struct SSchemaEx {
   struct SSchema field;
-  int16_t        offset;
+  int32_t        offset;
 } SSchemaEx;
 
 typedef struct SColumnModel {
   int32_t    capacity;
   int32_t    numOfCols;
-  int16_t    rowSize;
+  int32_t    rowSize;
   SSchemaEx *pFields;
 } SColumnModel;
 

src/query/inc/qUtil.h

@@ -66,7 +66,8 @@ static FORCE_INLINE SResultRow *getResultRow(SResultRowInfo *pResultRowInfo, int
   return pResultRowInfo->pResult[slot];
 }
 
-static FORCE_INLINE char *getPosInResultPage(SQueryAttr *pQueryAttr, tFilePage* page, int32_t rowOffset, int16_t offset) {
+static FORCE_INLINE char* getPosInResultPage(SQueryAttr* pQueryAttr, tFilePage* page, int32_t rowOffset,
+                                             int32_t offset) {
   assert(rowOffset >= 0 && pQueryAttr != NULL);
 
   int32_t numOfRows = (int32_t)GET_ROW_PARAM_FOR_MULTIOUTPUT(pQueryAttr, pQueryAttr->topBotQuery, pQueryAttr->stableQuery);

src/query/src/qAggMain.c

@@ -3982,7 +3982,7 @@ void blockInfo_func(SQLFunctionCtx* pCtx) {
 
   int32_t len = *(int32_t*) pCtx->pInput;
   blockDistInfoFromBinary((char*)pCtx->pInput + sizeof(int32_t), len, pDist);
-  pDist->rowSize = (int16_t) pCtx->param[0].i64;
+  pDist->rowSize = (uint16_t)pCtx->param[0].i64;
 
   memcpy(pCtx->pOutput, pCtx->pInput, sizeof(int32_t) + len);
 
@@ -4129,7 +4129,7 @@ void blockinfo_func_finalizer(SQLFunctionCtx* pCtx) {
   SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
   STableBlockDist* pDist = (STableBlockDist*) GET_ROWCELL_INTERBUF(pResInfo);
 
-  pDist->rowSize = (int16_t)pCtx->param[0].i64;
+  pDist->rowSize = (uint16_t)pCtx->param[0].i64;
   generateBlockDistResult(pDist, pCtx->pOutput);
 
   // cannot set the numOfIteratedElems again since it is set during previous iteration

src/query/src/qExecutor.c

@@ -3583,7 +3583,7 @@ void setResultRowOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pRe
   // Note: pResult->pos[i]->num == 0, there is only fixed number of results for each group
   tFilePage* bufPage = getResBufPage(pRuntimeEnv->pResultBuf, pResult->pageId);
 
-  int16_t offset = 0;
+  int32_t offset = 0;
   for (int32_t i = 0; i < numOfOutput; ++i) {
     pCtx[i].resultInfo = getResultCell(pResult, i, rowCellInfoOffset);
 
@@ -3897,7 +3897,7 @@ static int32_t doCopyToSDataBlock(SQueryRuntimeEnv* pRuntimeEnv, SGroupResInfo*
 
     tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pRow->pageId);
 
-    int16_t offset = 0;
+    int32_t offset = 0;
     for (int32_t j = 0; j < pBlock->info.numOfCols; ++j) {
       SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, j);
       int32_t bytes = pColInfoData->info.bytes;

src/query/src/qTsbuf.c

@@ -267,6 +267,10 @@ static void writeDataToDisk(STSBuf* pTSBuf) {
   if (pBlock->tag.nType == TSDB_DATA_TYPE_BINARY || pBlock->tag.nType == TSDB_DATA_TYPE_NCHAR) {
     metaLen += (int32_t)fwrite(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f);
     metaLen += (int32_t)fwrite(pBlock->tag.pz, 1, (size_t)pBlock->tag.nLen, pTSBuf->f);
+  } else if (pBlock->tag.nType == TSDB_DATA_TYPE_FLOAT) {
+    metaLen += (int32_t)fwrite(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f);
+    float tfloat = (float)pBlock->tag.dKey;
+    metaLen += (int32_t)fwrite(&tfloat, 1, (size_t) pBlock->tag.nLen, pTSBuf->f);  
   } else if (pBlock->tag.nType != TSDB_DATA_TYPE_NULL) {
     metaLen += (int32_t)fwrite(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f);
     metaLen += (int32_t)fwrite(&pBlock->tag.i64, 1, (size_t) pBlock->tag.nLen, pTSBuf->f);
@@ -351,6 +355,11 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) {
 
     sz = fread(pBlock->tag.pz, (size_t)pBlock->tag.nLen, 1, pTSBuf->f);
     UNUSED(sz);
+  } else if (pBlock->tag.nType == TSDB_DATA_TYPE_FLOAT) {
+    float tfloat = 0;
+    sz = fread(&tfloat, (size_t) pBlock->tag.nLen, 1, pTSBuf->f);
+    pBlock->tag.dKey = (double)tfloat;
+    UNUSED(sz);
   } else if (pBlock->tag.nType != TSDB_DATA_TYPE_NULL) { //TODO check the return value
     sz = fread(&pBlock->tag.i64, (size_t) pBlock->tag.nLen, 1, pTSBuf->f);
     UNUSED(sz);

src/tsdb/inc/tsdbMemTable.h

@@ -71,27 +71,27 @@ int   tsdbLoadDataFromCache(STable* pTable, SSkipListIterator* pIter, TSKEY maxK
                             TKEY* filterKeys, int nFilterKeys, bool keepDup, SMergeInfo* pMergeInfo);
 void* tsdbCommitData(STsdbRepo* pRepo);
 
-static FORCE_INLINE SDataRow tsdbNextIterRow(SSkipListIterator* pIter) {
+static FORCE_INLINE SMemRow tsdbNextIterRow(SSkipListIterator* pIter) {
   if (pIter == NULL) return NULL;
 
   SSkipListNode* node = tSkipListIterGet(pIter);
   if (node == NULL) return NULL;
 
-  return (SDataRow)SL_GET_NODE_DATA(node);
+  return (SMemRow)SL_GET_NODE_DATA(node);
 }
 
 static FORCE_INLINE TSKEY tsdbNextIterKey(SSkipListIterator* pIter) {
-  SDataRow row = tsdbNextIterRow(pIter);
+  SMemRow row = tsdbNextIterRow(pIter);
   if (row == NULL) return TSDB_DATA_TIMESTAMP_NULL;
 
-  return dataRowKey(row);
+  return memRowKey(row);
 }
 
 static FORCE_INLINE TKEY tsdbNextIterTKey(SSkipListIterator* pIter) {
-  SDataRow row = tsdbNextIterRow(pIter);
+  SMemRow row = tsdbNextIterRow(pIter);
   if (row == NULL) return TKEY_NULL;
 
-  return dataRowTKey(row);
+  return memRowTKey(row);
 }
 
 #endif /* _TD_TSDB_MEMTABLE_H_ */
\ No newline at end of file

src/tsdb/inc/tsdbMeta.h

@@ -32,7 +32,7 @@ typedef struct STable {
   void*          eventHandler;   // TODO
   void*          streamHandler;  // TODO
   TSKEY          lastKey;
-  SDataRow       lastRow;
+  SMemRow        lastRow;
   char*          sql;
   void*          cqhandle;
   SRWLatch       latch;  // TODO: implementa latch functions
@@ -148,7 +148,7 @@ static FORCE_INLINE STSchema *tsdbGetTableTagSchema(STable *pTable) {
 }
 
 static FORCE_INLINE TSKEY tsdbGetTableLastKeyImpl(STable* pTable) {
-  ASSERT(pTable->lastRow == NULL || pTable->lastKey == dataRowKey(pTable->lastRow));
+  ASSERT((pTable->lastRow == NULL) || (pTable->lastKey == memRowKey(pTable->lastRow)));
   return pTable->lastKey;
 }
 

src/tsdb/src/tsdbCommit.c

@@ -920,7 +920,8 @@ int tsdbWriteBlockImpl(STsdbRepo *pRepo, STable *pTable, SDFile *pDFile, SDataCo
     SDataCol * pDataCol = pDataCols->cols + ncol;
     SBlockCol *pBlockCol = pBlockData->cols + nColsNotAllNull;
 
-    if (isNEleNull(pDataCol, rowsToWrite)) {  // all data to commit are NULL, just ignore it
+    // if (isNEleNull(pDataCol, rowsToWrite)) {  // all data to commit are NULL, just ignore it
+    if (isAllRowsNull(pDataCol)) {  // all data to commit are NULL, just ignore it
       continue;
     }
 
@@ -1264,12 +1265,12 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt
   while (true) {
     key1 = (*iter >= pDataCols->numOfRows) ? INT64_MAX : dataColsKeyAt(pDataCols, *iter);
     bool isRowDel = false;
-    SDataRow row = tsdbNextIterRow(pCommitIter->pIter);
-    if (row == NULL || dataRowKey(row) > maxKey) {
+    SMemRow row = tsdbNextIterRow(pCommitIter->pIter);
+    if (row == NULL || memRowKey(row) > maxKey) {
       key2 = INT64_MAX;
     } else {
-      key2 = dataRowKey(row);
-      isRowDel = dataRowDeleted(row);
+      key2 = memRowKey(row);
+      isRowDel = memRowDeleted(row);
     }
 
     if (key1 == INT64_MAX && key2 == INT64_MAX) break;
@@ -1284,24 +1285,24 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt
       (*iter)++;
     } else if (key1 > key2) {
       if (!isRowDel) {
-        if (pSchema == NULL || schemaVersion(pSchema) != dataRowVersion(row)) {
-          pSchema = tsdbGetTableSchemaImpl(pCommitIter->pTable, false, false, dataRowVersion(row));
+        if (pSchema == NULL || schemaVersion(pSchema) != memRowVersion(row)) {
+          pSchema = tsdbGetTableSchemaImpl(pCommitIter->pTable, false, false, memRowVersion(row));
           ASSERT(pSchema != NULL);
         }
 
-        tdAppendDataRowToDataCol(row, pSchema, pTarget);
+        tdAppendMemRowToDataCol(row, pSchema, pTarget);
       }
 
       tSkipListIterNext(pCommitIter->pIter);
     } else {
       if (update) {
         if (!isRowDel) {
-          if (pSchema == NULL || schemaVersion(pSchema) != dataRowVersion(row)) {
-            pSchema = tsdbGetTableSchemaImpl(pCommitIter->pTable, false, false, dataRowVersion(row));
+          if (pSchema == NULL || schemaVersion(pSchema) != memRowVersion(row)) {
+            pSchema = tsdbGetTableSchemaImpl(pCommitIter->pTable, false, false, memRowVersion(row));
             ASSERT(pSchema != NULL);
           }
 
-          tdAppendDataRowToDataCol(row, pSchema, pTarget);
+          tdAppendMemRowToDataCol(row, pSchema, pTarget);
         }
       } else {
         ASSERT(!isRowDel);

src/tsdb/src/tsdbMain.c

@@ -641,7 +641,7 @@ static int tsdbRestoreLastColumns(STsdbRepo *pRepo, STable *pTable, SReadH* pRea
   int numColumns;
   int32_t blockIdx;
   SDataStatis* pBlockStatis = NULL;
-  SDataRow row = NULL;
+  SMemRow      row = NULL;
   // restore last column data with last schema
   
   int err = 0;
@@ -657,13 +657,15 @@ static int tsdbRestoreLastColumns(STsdbRepo *pRepo, STable *pTable, SReadH* pRea
     }
   }
 
-  row = taosTMalloc(dataRowMaxBytesFromSchema(pSchema));
+  row = taosTMalloc(memRowMaxBytesFromSchema(pSchema));
   if (row == NULL) {
     terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
     err = -1;
     goto out;
   }
-  tdInitDataRow(row, pSchema);
+
+  memRowSetType(row, SMEM_ROW_DATA);
+  tdInitDataRow(memRowDataBody(row), pSchema);
 
   // first load block index info
   if (tsdbLoadBlockInfo(pReadh, NULL) < 0) {
@@ -720,9 +722,9 @@ static int tsdbRestoreLastColumns(STsdbRepo *pRepo, STable *pTable, SReadH* pRea
       // OK,let's load row from backward to get not-null column
       for (int32_t rowId = pBlock->numOfRows - 1; rowId >= 0; rowId--) {
         SDataCol *pDataCol = pReadh->pDCols[0]->cols + i;
-        tdAppendColVal(row, tdGetColDataOfRow(pDataCol, rowId), pCol->type, pCol->bytes, pCol->offset);
+        tdAppendColVal(memRowDataBody(row), tdGetColDataOfRow(pDataCol, rowId), pCol->type, pCol->offset);
         //SDataCol *pDataCol = readh.pDCols[0]->cols + j;
-        void* value = tdGetRowDataOfCol(row, (int8_t)pCol->type, TD_DATA_ROW_HEAD_SIZE + pCol->offset);
+        void *value = tdGetRowDataOfCol(memRowDataBody(row), (int8_t)pCol->type, TD_DATA_ROW_HEAD_SIZE + pCol->offset);
         if (isNull(value, pCol->type)) {
           continue;
         }
@@ -742,8 +744,8 @@ static int tsdbRestoreLastColumns(STsdbRepo *pRepo, STable *pTable, SReadH* pRea
         // save row ts(in column 0)
         pDataCol = pReadh->pDCols[0]->cols + 0;
         pCol = schemaColAt(pSchema, 0);
-        tdAppendColVal(row, tdGetColDataOfRow(pDataCol, rowId), pCol->type, pCol->bytes, pCol->offset);
-        pLastCol->ts = dataRowKey(row);
+        tdAppendColVal(memRowDataBody(row), tdGetColDataOfRow(pDataCol, rowId), pCol->type, pCol->offset);
+        pLastCol->ts = memRowKey(row);
 
         pTable->restoreColumnNum += 1;
 
@@ -779,18 +781,18 @@ static int tsdbRestoreLastRow(STsdbRepo *pRepo, STable *pTable, SReadH* pReadh,
   // Get the data in row
   
   STSchema *pSchema = tsdbGetTableSchema(pTable);
-  pTable->lastRow = taosTMalloc(dataRowMaxBytesFromSchema(pSchema));
+  pTable->lastRow = taosTMalloc(memRowMaxBytesFromSchema(pSchema));
   if (pTable->lastRow == NULL) {
     terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
     return -1;
   }
-
-  tdInitDataRow(pTable->lastRow, pSchema);
+  memRowSetType(pTable->lastRow, SMEM_ROW_DATA);
+  tdInitDataRow(memRowDataBody(pTable->lastRow), pSchema);
   for (int icol = 0; icol < schemaNCols(pSchema); icol++) {
     STColumn *pCol = schemaColAt(pSchema, icol);
     SDataCol *pDataCol = pReadh->pDCols[0]->cols + icol;
-    tdAppendColVal(pTable->lastRow, tdGetColDataOfRow(pDataCol, pBlock->numOfRows - 1), pCol->type, pCol->bytes,
-                    pCol->offset);
+    tdAppendColVal(memRowDataBody(pTable->lastRow), tdGetColDataOfRow(pDataCol, pBlock->numOfRows - 1), pCol->type,
+                   pCol->offset);
   }
 
   return 0;

src/tsdb/src/tsdbMemTable.c

@@ -21,7 +21,7 @@
 typedef struct {
   int32_t  totalLen;
   int32_t  len;
-  SDataRow row;
+  SMemRow  row;
 } SSubmitBlkIter;
 
 typedef struct {
@@ -36,20 +36,19 @@ static STableData *tsdbNewTableData(STsdbCfg *pCfg, STable *pTable);
 static void        tsdbFreeTableData(STableData *pTableData);
 static char *      tsdbGetTsTupleKey(const void *data);
 static int          tsdbAdjustMemMaxTables(SMemTable *pMemTable, int maxTables);
-static int          tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema **ppSchema, SDataRow row);
+static int              tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema **ppSchema, SMemRow row);
 static int          tsdbInitSubmitBlkIter(SSubmitBlk *pBlock, SSubmitBlkIter *pIter);
-static SDataRow     tsdbGetSubmitBlkNext(SSubmitBlkIter *pIter);
+static SMemRow          tsdbGetSubmitBlkNext(SSubmitBlkIter *pIter);
 static int          tsdbScanAndConvertSubmitMsg(STsdbRepo *pRepo, SSubmitMsg *pMsg);
 static int          tsdbInsertDataToTable(STsdbRepo *pRepo, SSubmitBlk *pBlock, int32_t *affectedrows);
-static int          tsdbCopyRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable, void **ppRow);
+static int              tsdbCopyRowToMem(STsdbRepo *pRepo, SMemRow row, STable *pTable, void **ppRow);
 static int          tsdbInitSubmitMsgIter(SSubmitMsg *pMsg, SSubmitMsgIter *pIter);
 static int          tsdbGetSubmitMsgNext(SSubmitMsgIter *pIter, SSubmitBlk **pPBlock);
 static int          tsdbCheckTableSchema(STsdbRepo *pRepo, SSubmitBlk *pBlock, STable *pTable);
 static int          tsdbInsertDataToTableImpl(STsdbRepo *pRepo, STable *pTable, void **rows, int rowCounter);
 static void         tsdbFreeRows(STsdbRepo *pRepo, void **rows, int rowCounter);
-static int          tsdbUpdateTableLatestInfo(STsdbRepo *pRepo, STable *pTable, SDataRow row);
-
-static FORCE_INLINE int tsdbCheckRowRange(STsdbRepo *pRepo, STable *pTable, SDataRow row, TSKEY minKey, TSKEY maxKey,
+static int              tsdbUpdateTableLatestInfo(STsdbRepo *pRepo, STable *pTable, SMemRow row);
+static FORCE_INLINE int tsdbCheckRowRange(STsdbRepo *pRepo, STable *pTable, SMemRow row, TSKEY minKey, TSKEY maxKey,
                                           TSKEY now);
 
 int32_t tsdbInsertData(STsdbRepo *repo, SSubmitMsg *pMsg, SShellSubmitRspMsg *pRsp) {
@@ -354,7 +353,7 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey
   TSKEY      fKey = 0;
   bool       isRowDel = false;
   int        filterIter = 0;
-  SDataRow   row = NULL;
+  SMemRow    row = NULL;
   SMergeInfo mInfo;
 
   if (pMergeInfo == NULL) pMergeInfo = &mInfo;
@@ -365,12 +364,12 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey
   if (pCols) tdResetDataCols(pCols);
 
   row = tsdbNextIterRow(pIter);
-  if (row == NULL || dataRowKey(row) > maxKey) {
+  if (row == NULL || memRowKey(row) > maxKey) {
     rowKey = INT64_MAX;
     isRowDel = false;
   } else {
-    rowKey = dataRowKey(row);
-    isRowDel = dataRowDeleted(row);
+    rowKey = memRowKey(row);
+    isRowDel = memRowDeleted(row);
   }
 
   if (filterIter >= nFilterKeys) {
@@ -407,12 +406,12 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey
 
       tSkipListIterNext(pIter);
       row = tsdbNextIterRow(pIter);
-      if (row == NULL || dataRowKey(row) > maxKey) {
+      if (row == NULL || memRowKey(row) > maxKey) {
         rowKey = INT64_MAX;
         isRowDel = false;
       } else {
-        rowKey = dataRowKey(row);
-        isRowDel = dataRowDeleted(row);
+        rowKey = memRowKey(row);
+        isRowDel = memRowDeleted(row);
       }
     } else {
       if (isRowDel) {
@@ -437,12 +436,12 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey
 
       tSkipListIterNext(pIter);
       row = tsdbNextIterRow(pIter);
-      if (row == NULL || dataRowKey(row) > maxKey) {
+      if (row == NULL || memRowKey(row) > maxKey) {
         rowKey = INT64_MAX;
         isRowDel = false;
       } else {
-        rowKey = dataRowKey(row);
-        isRowDel = dataRowDeleted(row);
+        rowKey = memRowKey(row);
+        isRowDel = memRowDeleted(row);
       }
 
       filterIter++;
@@ -548,7 +547,7 @@ static void tsdbFreeTableData(STableData *pTableData) {
   }
 }
 
-static char *tsdbGetTsTupleKey(const void *data) { return dataRowTuple((SDataRow)data); }
+static char *tsdbGetTsTupleKey(const void *data) { return memRowTuple((SMemRow)data); }
 
 static int tsdbAdjustMemMaxTables(SMemTable *pMemTable, int maxTables) {
   ASSERT(pMemTable->maxTables < maxTables);
@@ -572,17 +571,17 @@ static int tsdbAdjustMemMaxTables(SMemTable *pMemTable, int maxTables) {
   return 0;
 }
 
-static int tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema **ppSchema, SDataRow row) {
+static int tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema **ppSchema, SMemRow row) {
   if (pCols) {
-    if (*ppSchema == NULL || schemaVersion(*ppSchema) != dataRowVersion(row)) {
-      *ppSchema = tsdbGetTableSchemaImpl(pTable, false, false, dataRowVersion(row));
+    if (*ppSchema == NULL || schemaVersion(*ppSchema) != memRowVersion(row)) {
+      *ppSchema = tsdbGetTableSchemaImpl(pTable, false, false, memRowVersion(row));
       if (*ppSchema == NULL) {
         ASSERT(false);
         return -1;
       }
     }
 
-    tdAppendDataRowToDataCol(row, *ppSchema, pCols);
+    tdAppendMemRowToDataCol(row, *ppSchema, pCols);
   }
 
   return 0;
@@ -592,31 +591,32 @@ static int tsdbInitSubmitBlkIter(SSubmitBlk *pBlock, SSubmitBlkIter *pIter) {
   if (pBlock->dataLen <= 0) return -1;
   pIter->totalLen = pBlock->dataLen;
   pIter->len = 0;
-  pIter->row = (SDataRow)(pBlock->data+pBlock->schemaLen);
+  pIter->row = (SMemRow)(pBlock->data + pBlock->schemaLen);
   return 0;
 }
 
-static SDataRow tsdbGetSubmitBlkNext(SSubmitBlkIter *pIter) {
-  SDataRow row = pIter->row;
+static SMemRow tsdbGetSubmitBlkNext(SSubmitBlkIter *pIter) {
+  SMemRow row = pIter->row;  // firstly, get current row
   if (row == NULL) return NULL;
 
-  pIter->len += dataRowLen(row);
-  if (pIter->len >= pIter->totalLen) {
+  pIter->len += memRowTLen(row);
+  if (pIter->len >= pIter->totalLen) {  // reach the end
     pIter->row = NULL;
   } else {
-    pIter->row = (char *)row + dataRowLen(row);
+    pIter->row = (char *)row + memRowTLen(row);  // secondly, move to next row
   }
 
   return row;
 }
 
-static FORCE_INLINE int tsdbCheckRowRange(STsdbRepo *pRepo, STable *pTable, SDataRow row, TSKEY minKey, TSKEY maxKey,
+static FORCE_INLINE int tsdbCheckRowRange(STsdbRepo *pRepo, STable *pTable, SMemRow row, TSKEY minKey, TSKEY maxKey,
                                           TSKEY now) {
-  if (dataRowKey(row) < minKey || dataRowKey(row) > maxKey) {
+  TSKEY rowKey = memRowKey(row);
+  if (rowKey < minKey || rowKey > maxKey) {
     tsdbError("vgId:%d table %s tid %d uid %" PRIu64 " timestamp is out of range! now %" PRId64 " minKey %" PRId64
               " maxKey %" PRId64 " row key %" PRId64,
               REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable), now, minKey, maxKey,
-              dataRowKey(row));
+              rowKey);
     terrno = TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE;
     return -1;
   }
@@ -630,7 +630,7 @@ static int tsdbScanAndConvertSubmitMsg(STsdbRepo *pRepo, SSubmitMsg *pMsg) {
   SSubmitMsgIter msgIter = {0};
   SSubmitBlk *   pBlock = NULL;
   SSubmitBlkIter blkIter = {0};
-  SDataRow       row = NULL;
+  SMemRow        row = NULL;
   TSKEY          now = taosGetTimestamp(pRepo->config.precision);
   TSKEY          minKey = now - tsTickPerDay[pRepo->config.precision] * pRepo->config.keep;
   TSKEY          maxKey = now + tsTickPerDay[pRepo->config.precision] * pRepo->config.daysPerFile;
@@ -698,7 +698,7 @@ static int tsdbInsertDataToTable(STsdbRepo *pRepo, SSubmitBlk *pBlock, int32_t *
   int64_t        points = 0;
   STable *       pTable = NULL;
   SSubmitBlkIter blkIter = {0};
-  SDataRow       row = NULL;
+  SMemRow        row = NULL;
   void *         rows[TSDB_MAX_INSERT_BATCH] = {0};
   int            rowCounter = 0;
 
@@ -744,10 +744,10 @@ _err:
   return -1;
 }
 
-static int tsdbCopyRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable, void **ppRow) {
+static int tsdbCopyRowToMem(STsdbRepo *pRepo, SMemRow row, STable *pTable, void **ppRow) {
   STsdbCfg *  pCfg = &pRepo->config;
-  TKEY        tkey = dataRowTKey(row);
-  TSKEY       key = dataRowKey(row);
+  TKEY        tkey = memRowTKey(row);
+  TSKEY       key = memRowKey(row);
   bool        isRowDelete = TKEY_IS_DELETED(tkey);
 
   if (isRowDelete) {
@@ -765,15 +765,15 @@ static int tsdbCopyRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable, void
     }
   }
 
-  void *pRow = tsdbAllocBytes(pRepo, dataRowLen(row));
+  void *pRow = tsdbAllocBytes(pRepo, memRowTLen(row));
   if (pRow == NULL) {
-    tsdbError("vgId:%d failed to insert row with key %" PRId64 " to table %s while allocate %d bytes since %s",
-              REPO_ID(pRepo), key, TABLE_CHAR_NAME(pTable), dataRowLen(row), tstrerror(terrno));
+    tsdbError("vgId:%d failed to insert row with key %" PRId64 " to table %s while allocate %" PRIu32 " bytes since %s",
+              REPO_ID(pRepo), key, TABLE_CHAR_NAME(pTable), memRowTLen(row), tstrerror(terrno));
     return -1;
   }
 
-  dataRowCpy(pRow, row);
-  ppRow[0] = pRow;
+  memRowCpy(pRow, row);
+  ppRow[0] = pRow;  // save the memory address of data rows
 
   tsdbTrace("vgId:%d a row is %s table %s tid %d uid %" PRIu64 " key %" PRIu64, REPO_ID(pRepo),
             isRowDelete ? "deleted from" : "updated in", TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable),
@@ -932,13 +932,15 @@ static int tsdbInsertDataToTableImpl(STsdbRepo *pRepo, STable *pTable, void **ro
   int64_t osize = SL_SIZE(pTableData->pData);
   tSkipListPutBatch(pTableData->pData, rows, rowCounter);
   int64_t dsize = SL_SIZE(pTableData->pData) - osize;
+  TSKEY   keyFirstRow = memRowKey(rows[0]);
+  TSKEY   keyLastRow = memRowKey(rows[rowCounter - 1]);
 
-  if (pMemTable->keyFirst > dataRowKey(rows[0])) pMemTable->keyFirst = dataRowKey(rows[0]);
-  if (pMemTable->keyLast < dataRowKey(rows[rowCounter - 1])) pMemTable->keyLast = dataRowKey(rows[rowCounter - 1]);
+  if (pMemTable->keyFirst > keyFirstRow) pMemTable->keyFirst = keyFirstRow;
+  if (pMemTable->keyLast < keyLastRow) pMemTable->keyLast = keyLastRow;
   pMemTable->numOfRows += dsize;
 
-  if (pTableData->keyFirst > dataRowKey(rows[0])) pTableData->keyFirst = dataRowKey(rows[0]);
-  if (pTableData->keyLast < dataRowKey(rows[rowCounter - 1])) pTableData->keyLast = dataRowKey(rows[rowCounter - 1]);
+  if (pTableData->keyFirst > keyFirstRow) pTableData->keyFirst = keyFirstRow;
+  if (pTableData->keyLast < keyLastRow) pTableData->keyLast = keyLastRow;
   pTableData->numOfRows += dsize;
 
   // update table latest info
@@ -954,8 +956,8 @@ static void tsdbFreeRows(STsdbRepo *pRepo, void **rows, int rowCounter) {
   STsdbBufPool *pBufPool = pRepo->pPool;
 
   for (int i = rowCounter - 1; i >= 0; --i) {
-    SDataRow row = (SDataRow)rows[i];
-    int      bytes = (int)dataRowLen(row);
+    SMemRow row = (SMemRow)rows[i];
+    int     bytes = (int)memRowTLen(row);
 
     if (pRepo->mem->extraBuffList == NULL) {
       STsdbBufBlock *pBufBlock = tsdbGetCurrBufBlock(pRepo);
@@ -988,21 +990,23 @@ static void tsdbFreeRows(STsdbRepo *pRepo, void **rows, int rowCounter) {
   }
 }
 
-static void updateTableLatestColumn(STsdbRepo *pRepo, STable *pTable, SDataRow row) {
-  tsdbDebug("vgId:%d updateTableLatestColumn, %s row version:%d", REPO_ID(pRepo), pTable->name->data, dataRowVersion(row));
+static void updateTableLatestColumn(STsdbRepo *pRepo, STable *pTable, SMemRow row) {
+  tsdbDebug("vgId:%d updateTableLatestColumn, %s row version:%d", REPO_ID(pRepo), pTable->name->data,
+            memRowVersion(row));
 
   STSchema* pSchema = tsdbGetTableLatestSchema(pTable);
   if (tsdbUpdateLastColSchema(pTable, pSchema) < 0) {
     return;
   }
 
-  pSchema = tsdbGetTableSchemaByVersion(pTable, dataRowVersion(row));
+  pSchema = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row));
   if (pSchema == NULL) {
     return;
   }
 
   SDataCol *pLatestCols = pTable->lastCols;
 
+  bool isDataRow = isDataRow(row);
   for (int16_t j = 0; j < schemaNCols(pSchema); j++) {
     STColumn *pTCol = schemaColAt(pSchema, j);
     // ignore not exist colId
@@ -1010,9 +1014,21 @@ static void updateTableLatestColumn(STsdbRepo *pRepo, STable *pTable, SDataRow r
     if (idx == -1) {
       continue;
     }
-    
-    void* value = tdGetRowDataOfCol(row, (int8_t)pTCol->type, TD_DATA_ROW_HEAD_SIZE + pSchema->columns[j].offset);
-    if (isNull(value, pTCol->type)) {
+
+    void *value = NULL;
+
+    if (isDataRow) {
+      value = tdGetRowDataOfCol(memRowDataBody(row), (int8_t)pTCol->type,
+                                TD_DATA_ROW_HEAD_SIZE + pSchema->columns[j].offset);
+    } else {
+      // SKVRow
+      SColIdx *pColIdx = tdGetKVRowIdxOfCol(memRowKvBody(row), pTCol->colId);
+      if (pColIdx) {
+        value = tdGetKvRowDataOfCol(memRowKvBody(row), pColIdx->offset);
+      }
+    }
+
+    if ((value == NULL) || isNull(value, pTCol->type)) {
       continue;
     }
 
@@ -1027,11 +1043,11 @@ static void updateTableLatestColumn(STsdbRepo *pRepo, STable *pTable, SDataRow r
 
     memcpy(pDataCol->pData, value, pDataCol->bytes);
     //tsdbInfo("updateTableLatestColumn vgId:%d cache column %d for %d,%s", REPO_ID(pRepo), j, pDataCol->bytes, (char*)pDataCol->pData);
-    pDataCol->ts = dataRowKey(row);
+    pDataCol->ts = memRowKey(row);
   }
 }
 
-static int tsdbUpdateTableLatestInfo(STsdbRepo *pRepo, STable *pTable, SDataRow row) {
+static int tsdbUpdateTableLatestInfo(STsdbRepo *pRepo, STable *pTable, SMemRow row) {
   STsdbCfg *pCfg = &pRepo->config;
 
   // if cacheLastRow config has been reset, free the lastRow
@@ -1042,31 +1058,31 @@ static int tsdbUpdateTableLatestInfo(STsdbRepo *pRepo, STable *pTable, SDataRow
     TSDB_WUNLOCK_TABLE(pTable);
   }
 
-  if (tsdbGetTableLastKeyImpl(pTable) < dataRowKey(row)) {
+  if (tsdbGetTableLastKeyImpl(pTable) < memRowKey(row)) {
     if (CACHE_LAST_ROW(pCfg) || pTable->lastRow != NULL) {
-      SDataRow nrow = pTable->lastRow;
-      if (taosTSizeof(nrow) < dataRowLen(row)) {
-        SDataRow orow = nrow;
-        nrow = taosTMalloc(dataRowLen(row));
+      SMemRow nrow = pTable->lastRow;
+      if (taosTSizeof(nrow) < memRowTLen(row)) {
+        SMemRow orow = nrow;
+        nrow = taosTMalloc(memRowTLen(row));
         if (nrow == NULL) {
           terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
           return -1;
         }
 
-        dataRowCpy(nrow, row);
+        memRowCpy(nrow, row);
         TSDB_WLOCK_TABLE(pTable);
-        pTable->lastKey = dataRowKey(row);
+        pTable->lastKey = memRowKey(row);
         pTable->lastRow = nrow;
         TSDB_WUNLOCK_TABLE(pTable);
         taosTZfree(orow);
       } else {
         TSDB_WLOCK_TABLE(pTable);
-        pTable->lastKey = dataRowKey(row);
-        dataRowCpy(nrow, row);
+        pTable->lastKey = memRowKey(row);
+        memRowCpy(nrow, row);
         TSDB_WUNLOCK_TABLE(pTable);
       }
     } else {
-      pTable->lastKey = dataRowKey(row);
+      pTable->lastKey = memRowKey(row);
     }
 
     if (CACHE_LAST_NULL_COLUMN(pCfg)) {

src/tsdb/src/tsdbMeta.c

@@ -787,7 +787,7 @@ static char *getTagIndexKey(const void *pData) {
   void *    res = tdGetKVRowValOfCol(pTable->tagVal, pCol->colId);
   if (res == NULL) {
     // treat the column as NULL if we cannot find it
-    res = getNullValue(pCol->type);
+    res = (char*)getNullValue(pCol->type);
   }
   return res;
 }

src/tsdb/src/tsdbRead.c

@@ -147,7 +147,7 @@ typedef struct STableGroupSupporter {
 static STimeWindow updateLastrowForEachGroup(STableGroupInfo *groupList);
 static int32_t checkForCachedLastRow(STsdbQueryHandle* pQueryHandle, STableGroupInfo *groupList);
 static int32_t checkForCachedLast(STsdbQueryHandle* pQueryHandle);
-static int32_t tsdbGetCachedLastRow(STable* pTable, SDataRow* pRes, TSKEY* lastKey);
+static int32_t tsdbGetCachedLastRow(STable* pTable, SMemRow* pRes, TSKEY* lastKey);
 
 static void    changeQueryHandleForInterpQuery(TsdbQueryHandleT pHandle);
 static void    doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SBlock* pBlock);
@@ -734,8 +734,8 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh
     SSkipListNode* node = tSkipListIterGet(pCheckInfo->iter);
     assert(node != NULL);
 
-    SDataRow row = (SDataRow)SL_GET_NODE_DATA(node);
-    TSKEY key = dataRowKey(row);  // first timestamp in buffer
+    SMemRow row = (SMemRow)SL_GET_NODE_DATA(node);
+    TSKEY   key = memRowKey(row);  // first timestamp in buffer
     tsdbDebug("%p uid:%" PRId64 ", tid:%d check data in mem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64
               "-%" PRId64 ", lastKey:%" PRId64 ", numOfRows:%"PRId64", 0x%"PRIx64,
               pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, key, order, pMem->keyFirst, pMem->keyLast,
@@ -756,8 +756,8 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh
     SSkipListNode* node = tSkipListIterGet(pCheckInfo->iiter);
     assert(node != NULL);
 
-    SDataRow row = (SDataRow)SL_GET_NODE_DATA(node);
-    TSKEY key = dataRowKey(row);  // first timestamp in buffer
+    SMemRow row = (SMemRow)SL_GET_NODE_DATA(node);
+    TSKEY   key = memRowKey(row);  // first timestamp in buffer
     tsdbDebug("%p uid:%" PRId64 ", tid:%d check data in imem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64
               "-%" PRId64 ", lastKey:%" PRId64 ", numOfRows:%"PRId64", 0x%"PRIx64,
               pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, key, order, pIMem->keyFirst, pIMem->keyLast,
@@ -781,19 +781,19 @@ static void destroyTableMemIterator(STableCheckInfo* pCheckInfo) {
   tSkipListDestroyIter(pCheckInfo->iiter);
 }
 
-static SDataRow getSDataRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order, int32_t update) {
-  SDataRow rmem = NULL, rimem = NULL;
+static SMemRow getSDataRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order, int32_t update) {
+  SMemRow rmem = NULL, rimem = NULL;
   if (pCheckInfo->iter) {
     SSkipListNode* node = tSkipListIterGet(pCheckInfo->iter);
     if (node != NULL) {
-      rmem = (SDataRow)SL_GET_NODE_DATA(node);
+      rmem = (SMemRow)SL_GET_NODE_DATA(node);
     }
   }
 
   if (pCheckInfo->iiter) {
     SSkipListNode* node = tSkipListIterGet(pCheckInfo->iiter);
     if (node != NULL) {
-      rimem = (SDataRow)SL_GET_NODE_DATA(node);
+      rimem = (SMemRow)SL_GET_NODE_DATA(node);
     }
   }
 
@@ -811,8 +811,8 @@ static SDataRow getSDataRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order
     return rimem;
   }
 
-  TSKEY r1 = dataRowKey(rmem);
-  TSKEY r2 = dataRowKey(rimem);
+  TSKEY r1 = memRowKey(rmem);
+  TSKEY r2 = memRowKey(rimem);
 
   if (r1 == r2) { // data ts are duplicated, ignore the data in mem
     if (!update) {
@@ -891,12 +891,12 @@ static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) {
     initTableMemIterator(pHandle, pCheckInfo);
   }
 
-  SDataRow row = getSDataRowInTableMem(pCheckInfo, pHandle->order, pCfg->update);
+  SMemRow row = getSDataRowInTableMem(pCheckInfo, pHandle->order, pCfg->update);
   if (row == NULL) {
     return false;
   }
 
-  pCheckInfo->lastKey = dataRowKey(row);  // first timestamp in buffer
+  pCheckInfo->lastKey = memRowKey(row);  // first timestamp in buffer
   tsdbDebug("%p uid:%" PRId64", tid:%d check data in buffer from skey:%" PRId64 ", order:%d, 0x%"PRIx64, pHandle,
       pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, pCheckInfo->lastKey, pHandle->order, pHandle->qId);
 
@@ -1155,11 +1155,11 @@ static int32_t handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SBlock* p
   int32_t code = TSDB_CODE_SUCCESS;
 
   /*bool hasData = */ initTableMemIterator(pQueryHandle, pCheckInfo);
-  SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order, pCfg->update);
+  SMemRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order, pCfg->update);
 
   assert(cur->pos >= 0 && cur->pos <= binfo.rows);
 
-  TSKEY key = (row != NULL)? dataRowKey(row):TSKEY_INITIAL_VAL;
+  TSKEY key = (row != NULL) ? memRowKey(row) : TSKEY_INITIAL_VAL;
   if (key != TSKEY_INITIAL_VAL) {
     tsdbDebug("%p key in mem:%"PRId64", 0x%"PRIx64, pQueryHandle, key, pQueryHandle->qId);
   } else {
@@ -1401,7 +1401,7 @@ int32_t doCopyRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t capacity
 
         // todo refactor, only copy one-by-one
         for (int32_t k = start; k < num + start; ++k) {
-          char* p = tdGetColDataOfRow(src, k);
+          const char* p = tdGetColDataOfRow(src, k);
           memcpy(dst, p, varDataTLen(p));
           dst += bytes;
         }
@@ -1452,88 +1452,170 @@ int32_t doCopyRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t capacity
   return numOfRows + num;
 }
 
-static void copyOneRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity, int32_t numOfRows, SDataRow row,
+static void copyOneRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity, int32_t numOfRows, SMemRow row,
                               int32_t numOfCols, STable* pTable, STSchema* pSchema) {
   char* pData = NULL;
 
-  // the schema version info is embeded in SDataRow
+  // the schema version info is embedded in SDataRow, and use latest schema version for SKVRow
   int32_t numOfRowCols = 0;
   if (pSchema == NULL) {
-    pSchema = tsdbGetTableSchemaByVersion(pTable, dataRowVersion(row));
+    pSchema = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row));
     numOfRowCols = schemaNCols(pSchema);
   } else {
     numOfRowCols = schemaNCols(pSchema);
   }
-  
-  int32_t i = 0, j = 0;
-  while(i < numOfCols && j < numOfRowCols) {
-    SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i);
-    if (pSchema->columns[j].colId < pColInfo->info.colId) {
-      j++;
-      continue;
-    }
 
-    if (ASCENDING_TRAVERSE(pQueryHandle->order)) {
-      pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes;
-    } else {
-      pData = (char*)pColInfo->pData + (capacity - numOfRows - 1) * pColInfo->info.bytes;
+  int32_t i = 0;
+
+  if (isDataRow(row)) {
+    SDataRow dataRow = memRowDataBody(row);
+    int32_t j = 0;
+    while (i < numOfCols && j < numOfRowCols) {
+      SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i);
+      if (pSchema->columns[j].colId < pColInfo->info.colId) {
+        j++;
+        continue;
+      }
+
+      if (ASCENDING_TRAVERSE(pQueryHandle->order)) {
+        pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes;
+      } else {
+        pData = (char*)pColInfo->pData + (capacity - numOfRows - 1) * pColInfo->info.bytes;
+      }
+
+      if (pSchema->columns[j].colId == pColInfo->info.colId) {
+        void* value =
+            tdGetRowDataOfCol(dataRow, (int8_t)pColInfo->info.type, TD_DATA_ROW_HEAD_SIZE + pSchema->columns[j].offset);
+        switch (pColInfo->info.type) {
+          case TSDB_DATA_TYPE_BINARY:
+          case TSDB_DATA_TYPE_NCHAR:
+            memcpy(pData, value, varDataTLen(value));
+            break;
+          case TSDB_DATA_TYPE_NULL:
+          case TSDB_DATA_TYPE_BOOL:
+          case TSDB_DATA_TYPE_TINYINT:
+          case TSDB_DATA_TYPE_UTINYINT:
+            *(uint8_t*)pData = *(uint8_t*)value;
+            break;
+          case TSDB_DATA_TYPE_SMALLINT:
+          case TSDB_DATA_TYPE_USMALLINT:
+            *(uint16_t*)pData = *(uint16_t*)value;
+            break;
+          case TSDB_DATA_TYPE_INT:
+          case TSDB_DATA_TYPE_UINT:
+            *(uint32_t*)pData = *(uint32_t*)value;
+            break;
+          case TSDB_DATA_TYPE_BIGINT:
+          case TSDB_DATA_TYPE_UBIGINT:
+            *(uint64_t*)pData = *(uint64_t*)value;
+            break;
+          case TSDB_DATA_TYPE_FLOAT:
+            SET_FLOAT_PTR(pData, value);
+            break;
+          case TSDB_DATA_TYPE_DOUBLE:
+            SET_DOUBLE_PTR(pData, value);
+            break;
+          case TSDB_DATA_TYPE_TIMESTAMP:
+            if (pColInfo->info.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) {
+              *(TSKEY*)pData = tdGetKey(*(TKEY*)value);
+            } else {
+              *(TSKEY*)pData = *(TSKEY*)value;
+            }
+            break;
+          default:
+            memcpy(pData, value, pColInfo->info.bytes);
+        }
+
+        j++;
+        i++;
+      } else {  // pColInfo->info.colId < pSchema->columns[j].colId, it is a NULL data
+        if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) {
+          setVardataNull(pData, pColInfo->info.type);
+        } else {
+          setNull(pData, pColInfo->info.type, pColInfo->info.bytes);
+        }
+        i++;
+      }
     }
+  } else if (isKvRow(row)) {
+    SKVRow  kvRow = memRowKvBody(row);
+    int32_t k = 0;
+    int32_t nKvRowCols = kvRowNCols(kvRow);
 
-    if (pSchema->columns[j].colId == pColInfo->info.colId) {
-      void* value = tdGetRowDataOfCol(row, (int8_t)pColInfo->info.type, TD_DATA_ROW_HEAD_SIZE + pSchema->columns[j].offset);
-      switch (pColInfo->info.type) {
-        case TSDB_DATA_TYPE_BINARY:
-        case TSDB_DATA_TYPE_NCHAR:
-          memcpy(pData, value, varDataTLen(value));
-          break;
-        case TSDB_DATA_TYPE_NULL:
-        case TSDB_DATA_TYPE_BOOL:
-        case TSDB_DATA_TYPE_TINYINT:
-        case TSDB_DATA_TYPE_UTINYINT:  
-          *(uint8_t *)pData = *(uint8_t *)value;
-          break;
-        case TSDB_DATA_TYPE_SMALLINT:
-        case TSDB_DATA_TYPE_USMALLINT:
-          *(uint16_t *)pData = *(uint16_t *)value;
-          break;
-        case TSDB_DATA_TYPE_INT:
-        case TSDB_DATA_TYPE_UINT:
-          *(uint32_t *)pData = *(uint32_t *)value;
-          break;
-        case TSDB_DATA_TYPE_BIGINT:
-        case TSDB_DATA_TYPE_UBIGINT:
-          *(uint64_t *)pData = *(uint64_t *)value;
-          break;
-        case TSDB_DATA_TYPE_FLOAT:
-          SET_FLOAT_PTR(pData, value);
-          break;
-        case TSDB_DATA_TYPE_DOUBLE:
-          SET_DOUBLE_PTR(pData, value);
-          break;
-        case TSDB_DATA_TYPE_TIMESTAMP:
-          if (pColInfo->info.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) {
-            *(TSKEY *)pData = tdGetKey(*(TKEY *)value);
-          } else {
-            *(TSKEY *)pData = *(TSKEY *)value;
-          }
-          break;
-        default:
-          memcpy(pData, value, pColInfo->info.bytes);
+    while (i < numOfCols && k < nKvRowCols) {
+      SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i);
+      SColIdx*         pColIdx = kvRowColIdxAt(kvRow, k);
+
+      if (pColIdx->colId < pColInfo->info.colId) {
+        ++k;
+        continue;
       }
 
-      j++;
-      i++;
-    } else { // pColInfo->info.colId < pSchema->columns[j].colId, it is a NULL data
+      if (ASCENDING_TRAVERSE(pQueryHandle->order)) {
+        pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes;
+      } else {
+        pData = (char*)pColInfo->pData + (capacity - numOfRows - 1) * pColInfo->info.bytes;
+      }
+
+      if (pColIdx->colId == pColInfo->info.colId) {
+        // offset of pColIdx for SKVRow including the TD_KV_ROW_HEAD_SIZE
+        void* value = tdGetKvRowDataOfCol(kvRow, pColIdx->offset);
+        switch (pColInfo->info.type) {
+          case TSDB_DATA_TYPE_BINARY:
+          case TSDB_DATA_TYPE_NCHAR:
+            memcpy(pData, value, varDataTLen(value));
+            break;
+          case TSDB_DATA_TYPE_NULL:
+          case TSDB_DATA_TYPE_BOOL:
+          case TSDB_DATA_TYPE_TINYINT:
+          case TSDB_DATA_TYPE_UTINYINT:
+            *(uint8_t*)pData = *(uint8_t*)value;
+            break;
+          case TSDB_DATA_TYPE_SMALLINT:
+          case TSDB_DATA_TYPE_USMALLINT:
+            *(uint16_t*)pData = *(uint16_t*)value;
+            break;
+          case TSDB_DATA_TYPE_INT:
+          case TSDB_DATA_TYPE_UINT:
+            *(uint32_t*)pData = *(uint32_t*)value;
+            break;
+          case TSDB_DATA_TYPE_BIGINT:
+          case TSDB_DATA_TYPE_UBIGINT:
+            *(uint64_t*)pData = *(uint64_t*)value;
+            break;
+          case TSDB_DATA_TYPE_FLOAT:
+            SET_FLOAT_PTR(pData, value);
+            break;
+          case TSDB_DATA_TYPE_DOUBLE:
+            SET_DOUBLE_PTR(pData, value);
+            break;
+          case TSDB_DATA_TYPE_TIMESTAMP:
+            if (pColInfo->info.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) {
+              *(TSKEY*)pData = tdGetKey(*(TKEY*)value);
+            } else {
+              *(TSKEY*)pData = *(TSKEY*)value;
+            }
+            break;
+          default:
+            memcpy(pData, value, pColInfo->info.bytes);
+        }
+        ++k;
+        ++i;
+        continue;
+      }
+      // If (pColInfo->info.colId < pColIdx->colId), it is NULL data
       if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) {
         setVardataNull(pData, pColInfo->info.type);
       } else {
         setNull(pData, pColInfo->info.type, pColInfo->info.bytes);
       }
-      i++;
+      ++i;
     }
+  } else {
+    ASSERT(0);
   }
 
-  while (i < numOfCols) { // the remain columns are all null data
+  while (i < numOfCols) {  // the remain columns are all null data
     SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i);
     if (ASCENDING_TRAVERSE(pQueryHandle->order)) {
       pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes;
@@ -1550,7 +1632,6 @@ static void copyOneRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity,
     i++;
   }
 }
-
 static void moveDataToFront(STsdbQueryHandle* pQueryHandle, int32_t numOfRows, int32_t numOfCols) {
   if (numOfRows == 0 || ASCENDING_TRAVERSE(pQueryHandle->order)) {
     return;
@@ -1730,12 +1811,12 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
   } else if (pCheckInfo->iter != NULL || pCheckInfo->iiter != NULL) {
     SSkipListNode* node = NULL;
     do {
-      SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order, pCfg->update);
+      SMemRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order, pCfg->update);
       if (row == NULL) {
         break;
       }
 
-      TSKEY key = dataRowKey(row);
+      TSKEY key = memRowKey(row);
       if ((key > pQueryHandle->window.ekey && ASCENDING_TRAVERSE(pQueryHandle->order)) ||
           (key < pQueryHandle->window.ekey && !ASCENDING_TRAVERSE(pQueryHandle->order))) {
         break;
@@ -1748,11 +1829,11 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
 
       if ((key < tsArray[pos] && ASCENDING_TRAVERSE(pQueryHandle->order)) ||
           (key > tsArray[pos] && !ASCENDING_TRAVERSE(pQueryHandle->order))) {
-        if (rv != dataRowVersion(row)) {
-          pSchema = tsdbGetTableSchemaByVersion(pTable, dataRowVersion(row));
-          rv = dataRowVersion(row);
-        }  
-        
+        if (rv != memRowVersion(row)) {
+          pSchema = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row));
+          rv = memRowVersion(row);
+        }
+
         copyOneRowFromMem(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, row, numOfCols, pTable, pSchema);
         numOfRows += 1;
         if (cur->win.skey == TSKEY_INITIAL_VAL) {
@@ -1766,11 +1847,11 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
         moveToNextRowInMem(pCheckInfo);
       } else if (key == tsArray[pos]) {  // data in buffer has the same timestamp of data in file block, ignore it
         if (pCfg->update) {
-          if (rv != dataRowVersion(row)) {
-            pSchema = tsdbGetTableSchemaByVersion(pTable, dataRowVersion(row));
-            rv = dataRowVersion(row);
+          if (rv != memRowVersion(row)) {
+            pSchema = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row));
+            rv = memRowVersion(row);
           }
-          
+
           copyOneRowFromMem(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, row, numOfCols, pTable, pSchema);
           numOfRows += 1;
           if (cur->win.skey == TSKEY_INITIAL_VAL) {
@@ -1820,8 +1901,10 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
        * copy them all to result buffer, since it may be overlapped with file data block.
        */
       if (node == NULL ||
-          ((dataRowKey((SDataRow)SL_GET_NODE_DATA(node)) > pQueryHandle->window.ekey) && ASCENDING_TRAVERSE(pQueryHandle->order)) ||
-          ((dataRowKey((SDataRow)SL_GET_NODE_DATA(node)) < pQueryHandle->window.ekey) && !ASCENDING_TRAVERSE(pQueryHandle->order))) {
+          ((memRowKey((SMemRow)SL_GET_NODE_DATA(node)) > pQueryHandle->window.ekey) &&
+           ASCENDING_TRAVERSE(pQueryHandle->order)) ||
+          ((memRowKey((SMemRow)SL_GET_NODE_DATA(node)) < pQueryHandle->window.ekey) &&
+           !ASCENDING_TRAVERSE(pQueryHandle->order))) {
         // no data in cache or data in cache is greater than the ekey of time window, load data from file block
         if (cur->win.skey == TSKEY_INITIAL_VAL) {
           cur->win.skey = tsArray[pos];
@@ -2407,12 +2490,12 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int
   STSchema* pSchema = NULL;
 
   do {
-    SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order, pCfg->update);
+    SMemRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order, pCfg->update);
     if (row == NULL) {
       break;
     }
 
-    TSKEY key = dataRowKey(row);
+    TSKEY key = memRowKey(row);
     if ((key > maxKey && ASCENDING_TRAVERSE(pQueryHandle->order)) || (key < maxKey && !ASCENDING_TRAVERSE(pQueryHandle->order))) {
       tsdbDebug("%p key:%"PRIu64" beyond qrange:%"PRId64" - %"PRId64", no more data in buffer", pQueryHandle, key, pQueryHandle->window.skey,
                 pQueryHandle->window.ekey);
@@ -2425,9 +2508,9 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int
     }
 
     win->ekey = key;
-    if (rv != dataRowVersion(row)) {
-      pSchema = tsdbGetTableSchemaByVersion(pTable, dataRowVersion(row));
-      rv = dataRowVersion(row);
+    if (rv != memRowVersion(row)) {
+      pSchema = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row));
+      rv = memRowVersion(row);
     }
     copyOneRowFromMem(pQueryHandle, maxRowsToRead, numOfRows, row, numOfCols, pTable, pSchema);
 
@@ -2544,7 +2627,7 @@ static bool loadCachedLastRow(STsdbQueryHandle* pQueryHandle) {
 
   SQueryFilePos* cur = &pQueryHandle->cur;
 
-  SDataRow pRow = NULL;
+  SMemRow  pRow = NULL;
   TSKEY    key  = TSKEY_INITIAL_VAL;
   int32_t  step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1:-1;
 
@@ -2954,7 +3037,7 @@ bool tsdbGetExternalRow(TsdbQueryHandleT pHandle) {
  * if lastRow == NULL, return TSDB_CODE_TDB_NO_CACHE_LAST_ROW
  * else set pRes and return TSDB_CODE_SUCCESS and save lastKey
  */
-int32_t tsdbGetCachedLastRow(STable* pTable, SDataRow* pRes, TSKEY* lastKey) {
+int32_t tsdbGetCachedLastRow(STable* pTable, SMemRow* pRes, TSKEY* lastKey) {
   int32_t code = TSDB_CODE_SUCCESS;
 
   TSDB_RLOCK_TABLE(pTable);
@@ -2965,7 +3048,7 @@ int32_t tsdbGetCachedLastRow(STable* pTable, SDataRow* pRes, TSKEY* lastKey) {
   }
 
   if (pRes) {
-    *pRes = tdDataRowDup(pTable->lastRow);
+    *pRes = tdMemRowDup(pTable->lastRow);
     if (*pRes == NULL) {
       code = TSDB_CODE_TDB_OUT_OF_MEMORY;
     }
@@ -3702,6 +3785,10 @@ static void* doFreeColumnInfoData(SArray* pColumnInfoData) {
 }
 
 static void* destroyTableCheckInfo(SArray* pTableCheckInfo) {
+  if (pTableCheckInfo == NULL) {
+    return NULL;
+  }
+  
   size_t size = taosArrayGetSize(pTableCheckInfo);
   for (int32_t i = 0; i < size; ++i) {
     STableCheckInfo* p = taosArrayGet(pTableCheckInfo, i);

src/tsdb/tests/tsdbTests.cpp

@@ -55,10 +55,10 @@ static int insertData(SInsertInfo *pInfo) {
       for (int j = 0; j < schemaNCols(pInfo->pSchema); j++) {
         STColumn *pTCol = schemaColAt(pInfo->pSchema, j);
         if (j == 0) {  // Just for timestamp
-          tdAppendColVal(row, (void *)(&start_time), pTCol->type, pTCol->bytes, pTCol->offset);
+          tdAppendColVal(row, (void *)(&start_time), pTCol->type, pTCol->offset);
         } else {  // For int
           int val = 10;
-          tdAppendColVal(row, (void *)(&val), pTCol->type, pTCol->bytes, pTCol->offset);
+          tdAppendColVal(row, (void *)(&val), pTCol->type, pTCol->offset);
         }
       }
       pBlock->dataLen += dataRowLen(row);

src/util/src/terror.c

@@ -183,6 +183,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_FIELD_ALREAY_EXIST,       "Field already exists"
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_FIELD_NOT_EXIST,          "Field does not exist")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_STABLE_NAME,      "Super table does not exist")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_CREATE_TABLE_MSG, "Invalid create table message")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_EXCEED_MAX_ROW_BYTES,     "Exceed max row bytes")
 
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_NAME,        "Invalid func name")
 TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_LEN,         "Invalid func length")

tests/perftest-scripts/perftest-query.sh

 #!/bin/bash
 
+branch=
+if [ x$1 != x ];then
+        branch=$1
+        echo "Testing branch: $branch"
+else
+        echo "Please enter branch name as a parameter"
+        exit 1
+fi
+jemalloc=
+if [ x$2 != x ];then
+		jemalloc=jemalloc
+        echo "Building TDengine using jemalloc"
+else
+        echo "Building TDengine using glibc"
+fi
+
 today=`date +"%Y%m%d"`
 WORK_DIR=/home/ubuntu/pxiao
 PERFORMANCE_TEST_REPORT=$WORK_DIR/TDengine/tests/performance-test-report-$today.log
@@ -40,8 +56,8 @@ function buildTDengine {
 
 	git remote update > /dev/null
 	git reset --hard HEAD
-	git checkout master
-	REMOTE_COMMIT=`git rev-parse --short remotes/origin/master`
+	git checkout $branch
+	REMOTE_COMMIT=`git rev-parse --short remotes/origin/$branch`
 	LOCAL_COMMIT=`git rev-parse --short @`
 
 	echo " LOCAL: $LOCAL_COMMIT"
@@ -53,9 +69,20 @@ function buildTDengine {
 		git pull > /dev/null 2>&1
 
 		LOCAL_COMMIT=`git rev-parse --short @`
+		if [ $jemalloc = "jemalloc" ];then
+			echo "git submodule update --init --recursive"
+			git submodule update --init --recursive
+		fi
+		
 		cd debug
 		rm -rf *
-		cmake .. > /dev/null
+
+		if [ $jemalloc = "jemalloc" ];then
+			echo "cmake .. -DJEMALLOC_ENABLED=true > /dev/null"
+			cmake .. -DJEMALLOC_ENABLED=true > /dev/null
+		else
+			cmake .. > /dev/null
+		fi
 		make && make install > /dev/null		
 	fi
 }
@@ -95,7 +122,8 @@ function sendReport {
 
 	sed -i 's/\x1b\[[0-9;]*m//g' $PERFORMANCE_TEST_REPORT
 	BODY_CONTENT=`cat $PERFORMANCE_TEST_REPORT`
-	echo -e "From: <support@taosdata.com>\nto: ${receiver}\nsubject: Query Performace Report ${today}, commit ID: ${LOCAL_COMMIT}\n\n${today}:\n${BODY_CONTENT}" | \
+	
+	echo -e "From: <support@taosdata.com>\nto: ${receiver}\nsubject: Query Performace Report ${branch} ${jemalloc} ${today}, commit ID: ${LOCAL_COMMIT}\n\n${today}:\n${BODY_CONTENT}" | \
 	(cat - && uuencode $PERFORMANCE_TEST_REPORT performance-test-report-$today.log) | \
 	/usr/sbin/ssmtp "${receiver}" && echo "Report Sent!"
 }

tests/pytest/fulltest.sh

@@ -241,7 +241,7 @@ python3 ./test.py -f query/queryStateWindow.py
 python3 ./test.py -f query/nestedQuery/queryWithOrderLimit.py
 python3 ./test.py -f query/nestquery_last_row.py
 python3 ./test.py -f query/queryCnameDisplay.py
-
+python3 test.py -f query/nestedQuery/queryWithSpread.py
 
 #stream
 python3 ./test.py -f stream/metric_1.py
@@ -352,7 +352,7 @@ python3 ./test.py -f alter/alter_debugFlag.py
 python3 ./test.py -f query/queryBetweenAnd.py
 python3 ./test.py -f tag_lite/alter_tag.py
 
-python3 test.py -f tools/taosdemoAllTest/taosdemoTestInsertWithJson.py 
+# python3 test.py -f tools/taosdemoAllTest/taosdemoTestInsertWithJson.py 
 python3 test.py -f tools/taosdemoAllTest/taosdemoTestQueryWithJson.py
 python3 test.py -f tools/taosdemoAllTest/TD-4985/query-limit-offset.py
 python3 ./test.py -f tag_lite/drop_auto_create.py

tests/pytest/query/nestedQuery/queryWithSpread.py

+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+from util.log import *
+from util.cases import *
+from util.sql import *
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor(), logSql)
+
+    def run(self):
+        tdSql.prepare()
+
+        tdSql.execute('create table stb (ts timestamp, speed int) tags(t1 int);')
+
+        for i in range(10):
+            tdSql.execute(f'create table tb_{i} using stb tags({i});')
+            tdSql.execute(f'insert into tb_{i} values(now, 0)')
+            tdSql.execute(f'insert into tb_{i} values(now+10s, {i})')
+
+        tdSql.query('select max(col3) from (select spread(speed) as col3 from stb group by tbname);')
+        tdSql.checkData(0,0,'9.0')
+        tdSql.error('select max(col3) from (select spread(col3) as col3 from stb group by tbname);')
+
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())

tests/pytest/query/nestquery_last_row.py

@@ -160,22 +160,23 @@ class TDTestCase:
         tdSql.checkData(0,9,9.000000000)
         tdSql.checkData(0,10,'2020-09-13 20:26:40.009')
 
+        # incorrect result, not support nest > 2
         sql = '''select last_row(*) from 
                 ((select * from table_0) union all
                  (select * from table_1) union all
                  (select * from table_2));'''
-        tdSql.query(sql)
-        tdSql.checkRows(1)
-        tdSql.checkData(0,1,self.num-1)
-        tdSql.checkData(0,2,self.num-1)
-        tdSql.checkData(0,3,self.num-1)
-        tdSql.checkData(0,4,self.num-1)
-        tdSql.checkData(0,5,'False')
-        tdSql.checkData(0,6,'binary.9')
-        tdSql.checkData(0,7,'nchar.9')
-        tdSql.checkData(0,8,9.00000)
-        tdSql.checkData(0,9,9.000000000)
-        tdSql.checkData(0,10,'2020-09-13 20:26:40.009')
+        tdSql.error(sql)
+        #tdSql.checkRows(1)
+        #tdSql.checkData(0,1,self.num-1)
+        #tdSql.checkData(0,2,self.num-1)
+        #tdSql.checkData(0,3,self.num-1)
+        #tdSql.checkData(0,4,self.num-1)
+        #tdSql.checkData(0,5,'False')
+        #tdSql.checkData(0,6,'binary.9')
+        #tdSql.checkData(0,7,'nchar.9')
+        #tdSql.checkData(0,8,9.00000)
+        #tdSql.checkData(0,9,9.000000000)
+        #tdSql.checkData(0,10,'2020-09-13 20:26:40.009')
 
         # bug 5055
         # sql = '''select last_row(*) from 
@@ -189,18 +190,18 @@ class TDTestCase:
                 ((select last_row(*) from table_0) union all
                  (select last_row(*) from table_1) union all
                  (select last_row(*) from table_2));'''
-        tdSql.query(sql)
-        tdSql.checkRows(1)
-        tdSql.checkData(0,1,self.num-1)
-        tdSql.checkData(0,2,self.num-1)
-        tdSql.checkData(0,3,self.num-1)
-        tdSql.checkData(0,4,self.num-1)
-        tdSql.checkData(0,5,'False')
-        tdSql.checkData(0,6,'binary.9')
-        tdSql.checkData(0,7,'nchar.9')
-        tdSql.checkData(0,8,9.00000)
-        tdSql.checkData(0,9,9.000000000)
-        tdSql.checkData(0,10,'2020-09-13 20:26:40.009')
+        tdSql.error(sql)
+        #tdSql.checkRows(1)
+        #tdSql.checkData(0,1,self.num-1)
+        #tdSql.checkData(0,2,self.num-1)
+        #tdSql.checkData(0,3,self.num-1)
+        #tdSql.checkData(0,4,self.num-1)
+        #tdSql.checkData(0,5,'False')
+        #tdSql.checkData(0,6,'binary.9')
+        #tdSql.checkData(0,7,'nchar.9')
+        #tdSql.checkData(0,8,9.00000)
+        #tdSql.checkData(0,9,9.000000000)
+        #tdSql.checkData(0,10,'2020-09-13 20:26:40.009')
 
         # bug 5055
         # sql = '''select last_row(*) from 
@@ -214,18 +215,18 @@ class TDTestCase:
                 ((select * from table_0 limit 5 offset 5) union all
                  (select * from table_1 limit 5 offset 5) union all
                  (select * from regular_table_1 limit 5 offset 5));'''
-        tdSql.query(sql)
-        tdSql.checkRows(1)
-        tdSql.checkData(0,1,self.num-1)
-        tdSql.checkData(0,2,self.num-1)
-        tdSql.checkData(0,3,self.num-1)
-        tdSql.checkData(0,4,self.num-1)
-        tdSql.checkData(0,5,'False')
-        tdSql.checkData(0,6,'binary.9')
-        tdSql.checkData(0,7,'nchar.9')
-        tdSql.checkData(0,8,9.00000)
-        tdSql.checkData(0,9,9.000000000)
-        tdSql.checkData(0,10,'2020-09-13 20:26:40.009')
+        tdSql.error(sql)
+        #tdSql.checkRows(1)
+        #tdSql.checkData(0,1,self.num-1)
+        #tdSql.checkData(0,2,self.num-1)
+        #tdSql.checkData(0,3,self.num-1)
+        #tdSql.checkData(0,4,self.num-1)
+        #tdSql.checkData(0,5,'False')
+        #tdSql.checkData(0,6,'binary.9')
+        #tdSql.checkData(0,7,'nchar.9')
+        #tdSql.checkData(0,8,9.00000)
+        #tdSql.checkData(0,9,9.000000000)
+        #tdSql.checkData(0,10,'2020-09-13 20:26:40.009')
 
 
         sql = '''select last_row(*)  from 
@@ -260,4 +261,4 @@ class TDTestCase:
 
 
 tdCases.addWindows(__file__, TDTestCase())
-tdCases.addLinux(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
\ No newline at end of file

tests/pytest/query/query1970YearsAf.py

@@ -187,19 +187,19 @@ class TDTestCase:
             "select * from t9 where t9.ts > '1969-12-31 22:00:00.000' and t9.ts <'1970-01-01 02:00:00.000' "
         )
         tdSql.checkRows(719)
-        
+
         tdSql.query(
             "select * from t0,t1 where t0.ts=t1.ts and t1.ts >= '1970-01-01 00:00:00.000' "
         )
         tdSql.checkRows(680)
-        
+
         tdSql.query(
-            "select diff(col1) from t0 where t0.ts >= '1970-01-01 00:00:00.000' "
+            "select diff(c1) from t0 where t0.ts >= '1970-01-01 00:00:00.000' "
         )
         tdSql.checkRows(679)
 
         tdSql.query(
-            "select t0,col1 from stb2 where stb2.ts < '1970-01-01 00:00:00.000' order by ts"
+            "select t0,c1 from stb2 where stb2.ts < '1970-01-01 00:00:00.000' order by ts"
         )
         tdSql.checkRows(43200)
 

tests/pytest/query/queryPerformance.py

@@ -73,10 +73,11 @@ class taosdemoQueryPerformace:
         sql = "select avg(f1), max(f2), min(f3) from meters where ts <= '2017-07-15 10:40:01.000' and ts <= '2017-07-15 14:00:40.000'"
         tableid = 8
         cursor.execute("create table if not exists %s%d using %s tags(%d, \"%s\")" % (self.tbPerfix, tableid, self.stbName, tableid, sql))
-        
+ 
         sql = "select last(*) from meters"
         tableid = 9
         cursor.execute("create table if not exists %s%d using %s tags(%d, '%s')" % (self.tbPerfix, tableid, self.stbName, tableid, sql))
+
         cursor.close()
 
     def query(self): 

tests/pytest/table/max_table_length.py

@@ -42,7 +42,7 @@ class TDTestCase:
         
         print("==============step3")
         tdLog.info("check int & binary")
-        tdSql.error("create table anal2 (ts timestamp ,i binary(16371),j int)")
+        # tdSql.error("create table anal2 (ts timestamp ,i binary(16371),j int)")
         tdSql.execute("create table anal2 (ts timestamp ,i binary(16370),j int)")
         tdSql.execute("create table anal3 (ts timestamp ,i binary(16366), j int, k int)")
         

tests/pytest/tools/taosdemoAllTest/TD-4985/query-limit-offset.py

@@ -23,7 +23,7 @@ class TDTestCase:
     def init(self, conn, logSql):
         tdLog.debug("start to execute %s" % __file__)
         tdSql.init(conn.cursor(), logSql)
-        
+
     def getBuildPath(self):
         selfPath = os.path.dirname(os.path.realpath(__file__))
 
@@ -39,7 +39,7 @@ class TDTestCase:
                     buildPath = root[:len(root)-len("/build/bin")]
                     break
         return buildPath
-        
+
     def run(self):
         buildPath = self.getBuildPath()
         if (buildPath == ""):
@@ -48,7 +48,7 @@ class TDTestCase:
             tdLog.info("taosd found in %s" % buildPath)
         binPath = buildPath+ "/build/bin/"
 
-        # insert: create one  or mutiple tables per sql and insert multiple rows per sql 
+        # insert: create one  or mutiple tables per sql and insert multiple rows per sql
         # test case for https://jira.taosdata.com:18080/browse/TD-4985
         os.system("%staosdemo -f tools/taosdemoAllTest/TD-4985/query-limit-offset.json -y " % binPath)
         tdSql.execute("use db")
@@ -56,27 +56,27 @@ class TDTestCase:
         tdSql.checkData(0, 0, 10000)
 
         for i in range(1000):
-            tdSql.execute('''insert into stb00_9999 values(%d, %d, %d,'test99.%s')''' 
+            tdSql.execute('''insert into stb00_9999 values(%d, %d, %d,'test99.%s')'''
                             % (1600000000000 + i, i, -10000+i, i))
-            tdSql.execute('''insert into stb00_8888 values(%d, %d, %d,'test98.%s')''' 
+            tdSql.execute('''insert into stb00_8888 values(%d, %d, %d,'test98.%s')'''
                             % (1600000000000 + i, i, -10000+i, i))
-            tdSql.execute('''insert into stb00_7777 values(%d, %d, %d,'test97.%s')''' 
+            tdSql.execute('''insert into stb00_7777 values(%d, %d, %d,'test97.%s')'''
                             % (1600000000000 + i, i, -10000+i, i))
-            tdSql.execute('''insert into stb00_6666 values(%d, %d, %d,'test96.%s')''' 
+            tdSql.execute('''insert into stb00_6666 values(%d, %d, %d,'test96.%s')'''
                             % (1600000000000 + i, i, -10000+i, i))
-            tdSql.execute('''insert into stb00_5555 values(%d, %d, %d,'test95.%s')''' 
+            tdSql.execute('''insert into stb00_5555 values(%d, %d, %d,'test95.%s')'''
                             % (1600000000000 + i, i, -10000+i, i))
-            tdSql.execute('''insert into stb00_4444 values(%d, %d, %d,'test94.%s')''' 
+            tdSql.execute('''insert into stb00_4444 values(%d, %d, %d,'test94.%s')'''
                             % (1600000000000 + i, i, -10000+i, i))
-            tdSql.execute('''insert into stb00_3333 values(%d, %d, %d,'test93.%s')''' 
+            tdSql.execute('''insert into stb00_3333 values(%d, %d, %d,'test93.%s')'''
                             % (1600000000000 + i, i, -10000+i, i))
-            tdSql.execute('''insert into stb00_2222 values(%d, %d, %d,'test92.%s')''' 
+            tdSql.execute('''insert into stb00_2222 values(%d, %d, %d,'test92.%s')'''
                             % (1600000000000 + i, i, -10000+i, i))
-            tdSql.execute('''insert into stb00_1111 values(%d, %d, %d,'test91.%s')''' 
+            tdSql.execute('''insert into stb00_1111 values(%d, %d, %d,'test91.%s')'''
                             % (1600000000000 + i, i, -10000+i, i))
-            tdSql.execute('''insert into stb00_100 values(%d, %d, %d,'test90.%s')''' 
+            tdSql.execute('''insert into stb00_100 values(%d, %d, %d,'test90.%s')'''
                             % (1600000000000 + i, i, -10000+i, i))
-        tdSql.query("select * from stb0 where col2 like 'test99%' ")
+        tdSql.query("select * from stb0 where c2 like 'test99%' ")
         tdSql.checkRows(1000)
         tdSql.query("select * from stb0 where  tbname like 'stb00_9999'  limit 10" )
         tdSql.checkData(0, 1, 0)
@@ -86,7 +86,7 @@ class TDTestCase:
         tdSql.checkData(0, 1, 5)
         tdSql.checkData(1, 1, 6)
         tdSql.checkData(2, 1, 7)
-        tdSql.query("select *  from stb0 where col2 like 'test98%' ")
+        tdSql.query("select *  from stb0 where c2 like 'test98%' ")
         tdSql.checkRows(1000)
         tdSql.query("select  *  from stb0 where tbname like 'stb00_8888' limit 10" )
         tdSql.checkData(0, 1, 0)
@@ -96,7 +96,7 @@ class TDTestCase:
         tdSql.checkData(0, 1, 5)
         tdSql.checkData(1, 1, 6)
         tdSql.checkData(2, 1, 7)
-        tdSql.query("select *  from stb0 where col2 like 'test97%' ")
+        tdSql.query("select *  from stb0 where c2 like 'test97%' ")
         tdSql.checkRows(1000)
         tdSql.query("select  *  from stb0 where tbname like 'stb00_7777' limit 10" )
         tdSql.checkData(0, 1, 0)
@@ -106,7 +106,7 @@ class TDTestCase:
         tdSql.checkData(0, 1, 5)
         tdSql.checkData(1, 1, 6)
         tdSql.checkData(2, 1, 7)
-        tdSql.query("select *  from stb0 where col2 like 'test96%' ")
+        tdSql.query("select *  from stb0 where c2 like 'test96%' ")
         tdSql.checkRows(1000)
         tdSql.query("select  *  from stb0 where tbname like 'stb00_6666' limit 10" )
         tdSql.checkData(0, 1, 0)
@@ -116,7 +116,7 @@ class TDTestCase:
         tdSql.checkData(0, 1, 5)
         tdSql.checkData(1, 1, 6)
         tdSql.checkData(2, 1, 7)
-        tdSql.query("select *  from stb0 where col2 like 'test95%' ")
+        tdSql.query("select *  from stb0 where c2 like 'test95%' ")
         tdSql.checkRows(1000)
         tdSql.query("select  *  from stb0 where tbname like 'stb00_5555' limit 10" )
         tdSql.checkData(0, 1, 0)
@@ -126,7 +126,7 @@ class TDTestCase:
         tdSql.checkData(0, 1, 5)
         tdSql.checkData(1, 1, 6)
         tdSql.checkData(2, 1, 7)
-        tdSql.query("select * from stb0 where col2 like 'test94%' ")
+        tdSql.query("select * from stb0 where c2 like 'test94%' ")
         tdSql.checkRows(1000)
         tdSql.query("select * from stb0 where tbname like 'stb00_4444' limit 10" )
         tdSql.checkData(0, 1, 0)
@@ -136,7 +136,7 @@ class TDTestCase:
         tdSql.checkData(0, 1, 5)
         tdSql.checkData(1, 1, 6)
         tdSql.checkData(2, 1, 7)
-        tdSql.query("select *  from stb0 where col2 like 'test93%' ")
+        tdSql.query("select *  from stb0 where c2 like 'test93%' ")
         tdSql.checkRows(1000)
         tdSql.query("select  *  from stb0 where tbname like 'stb00_3333' limit 100" )
         tdSql.checkData(0, 1, 0)
@@ -146,7 +146,7 @@ class TDTestCase:
         tdSql.checkData(0, 1, 5)
         tdSql.checkData(1, 1, 6)
         tdSql.checkData(2, 1, 7)
-        tdSql.query("select *  from stb0 where col2 like 'test92%' ")
+        tdSql.query("select *  from stb0 where c2 like 'test92%' ")
         tdSql.checkRows(1000)
         tdSql.query("select  *  from stb0 where tbname like 'stb00_2222' limit 100" )
         tdSql.checkData(0, 1, 0)
@@ -156,7 +156,7 @@ class TDTestCase:
         tdSql.checkData(0, 1, 5)
         tdSql.checkData(1, 1, 6)
         tdSql.checkData(2, 1, 7)
-        tdSql.query("select *  from stb0 where col2 like 'test91%' ")
+        tdSql.query("select *  from stb0 where c2 like 'test91%' ")
         tdSql.checkRows(1000)
         tdSql.query("select  *  from stb0 where tbname like 'stb00_1111' limit 100" )
         tdSql.checkData(0, 1, 0)
@@ -166,7 +166,7 @@ class TDTestCase:
         tdSql.checkData(0, 1, 5)
         tdSql.checkData(1, 1, 6)
         tdSql.checkData(2, 1, 7)
-        tdSql.query("select *  from stb0 where col2 like 'test90%' ")
+        tdSql.query("select *  from stb0 where c2 like 'test90%' ")
         tdSql.checkRows(1000)
         tdSql.query("select  *  from stb0 where tbname like 'stb00_100' limit 100" )
         tdSql.checkData(0, 1, 0)
@@ -178,10 +178,10 @@ class TDTestCase:
         tdSql.checkData(2, 1, 7)
 
 
-        os.system("rm -rf tools/taosdemoAllTest/TD-4985/query-limit-offset.py.sql")        
-        
-        
-        
+        os.system("rm -rf tools/taosdemoAllTest/TD-4985/query-limit-offset.py.sql")
+
+
+
     def stop(self):
         tdSql.close()
         tdLog.success("%s successfully executed" % __file__)

tests/pytest/tools/taosdemoAllTest/TD-5213/insertSigcolumnsNum4096.csv

+0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+1,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+2,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+3,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+4,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+5,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+6,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+7,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+8,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+9,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
\ No newline at end of file

tests/pytest/tools/taosdemoAllTest/TD-5213/insertSigcolumnsNum4096.json

+{
+    "filetype": "insert",
+    "cfgdir": "/etc/taos",
+    "host": "127.0.0.1",
+    "port": 6030,
+    "user": "root",
+    "password": "taosdata",
+    "thread_count": 10,
+    "thread_count_create_tbl": 10,
+    "result_file": "./insert_res.txt",
+    "confirm_parameter_prompt": "no",
+    "insert_interval": 0,
+    "interlace_rows": 10,
+    "num_of_records_per_req": 1,
+    "max_sql_len": 102400000,
+    "databases": [{
+        "dbinfo": {
+            "name": "db",
+            "drop": "yes",
+            "replica": 1,
+            "days": 10,
+            "cache": 50,
+            "blocks": 8,
+            "precision": "ms",
+            "keep": 365,
+            "minRows": 100,
+            "maxRows": 4096,
+            "comp":2,
+            "walLevel":1,
+            "cachelast":0,
+            "quorum":1,
+            "fsync":3000,
+            "update": 0
+        },
+        "super_tables": [{
+            "name": "stb_old",
+            "child_table_exists":"no",
+            "childtable_count": 10,
+            "childtable_prefix": "stb_old_",
+            "auto_create_table": "no",
+            "batch_create_tbl_num": 5,
+            "data_source": "sample",
+            "insert_mode": "taosc",
+            "insert_rows": 100,
+            "childtable_limit": 0,
+            "childtable_offset":0,
+            "multi_thread_write_one_tbl": "no",
+            "interlace_rows": 0,
+            "insert_interval":0,
+            "max_sql_len": 1024000,
+            "disorder_ratio": 0,
+            "disorder_range": 1000,
+            "timestamp_step": 1,
+            "start_timestamp": "2020-10-01 00:00:00.000",
+            "sample_format": "csv",
+            "sample_file": "./tools/taosdemoAllTest/TD-5213/insertSigcolumnsNum4096.csv",
+            "tags_file": "",
+            "columns": [{"type": "INT","count":4000}, {"type": "BINARY", "len": 16, "count":1}],
+            "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":1}]
+        },{
+            "name": "stb_new",
+            "child_table_exists":"no",
+            "childtable_count": 10,
+            "childtable_prefix": "stb_new_",
+            "auto_create_table": "no",
+            "batch_create_tbl_num": 5,
+            "data_source": "rand",
+            "insert_mode": "taosc",
+            "insert_rows": 100,
+            "childtable_limit": 0,
+            "childtable_offset":0,
+            "multi_thread_write_one_tbl": "no",
+            "interlace_rows": 0,
+            "insert_interval":0,
+            "max_sql_len": 1024000,
+            "disorder_ratio": 0,
+            "disorder_range": 1000,
+            "timestamp_step": 1,
+            "start_timestamp": "2020-10-01 00:00:00.000",
+            "sample_format": "csv",
+            "sample_file": "./tools/taosdemoAllTest/sample.csv",
+            "tags_file": "",
+            "columns": [{"type": "DOUBLE","count":1020}],
+            "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":1}]
+        },{
+            "name": "stb_int",
+            "child_table_exists":"no",
+            "childtable_count": 10,
+            "childtable_prefix": "stb_int_",
+            "auto_create_table": "no",
+            "batch_create_tbl_num": 5,
+            "data_source": "rand",
+            "insert_mode": "taosc",
+            "insert_rows": 100,
+            "childtable_limit": 0,
+            "childtable_offset":0,
+            "multi_thread_write_one_tbl": "no",
+            "interlace_rows": 0,
+            "insert_interval":0,
+            "max_sql_len": 1024000,
+            "disorder_ratio": 0,
+            "disorder_range": 1000,
+            "timestamp_step": 1,
+            "start_timestamp": "2020-10-01 00:00:00.000",
+            "sample_format": "csv",
+            "sample_file": "./tools/taosdemoAllTest/sample.csv",
+            "tags_file": "",
+            "columns": [{"type": "int","count":1020}],
+            "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":1}]
+        }]
+    }]
+}

tests/pytest/tools/taosdemoAllTest/TD-5213/insertSigcolumnsNum4096.py

+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import os
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.dnodes import *
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor(), logSql)
+
+    def getBuildPath(self):
+        selfPath = os.path.dirname(os.path.realpath(__file__))
+
+        if ("community" in selfPath):
+            projPath = selfPath[:selfPath.find("community")]
+        else:
+            projPath = selfPath[:selfPath.find("tests")]
+
+        for root, dirs, files in os.walk(projPath):
+            if ("taosd" in files):
+                rootRealPath = os.path.dirname(os.path.realpath(root))
+                if ("packaging" not in rootRealPath):
+                    buildPath = root[:len(root)-len("/build/bin")]
+                    break
+        return buildPath
+
+    def run(self):
+        buildPath = self.getBuildPath()
+        if (buildPath == ""):
+            tdLog.exit("taosd not found!")
+        else:
+            tdLog.info("taosd found in %s" % buildPath)
+        binPath = buildPath+ "/build/bin/"
+
+        # insert: create one  or mutiple tables per sql and insert multiple rows per sql
+        # test case for https://jira.taosdata.com:18080/browse/TD-5213
+        os.system("%staosdemo -f tools/taosdemoAllTest/TD-5213/insertSigcolumnsNum4096.json -y " % binPath)
+        tdSql.execute("use db")
+        tdSql.query("select count (tbname) from stb_old")
+        tdSql.checkData(0, 0, 10)
+
+        # tdSql.query("select * from stb_old")
+        # tdSql.checkRows(10)
+        # tdSql.checkCols(1024)
+
+        # tdSql.query("select count (tbname) from stb_new")
+        # tdSql.checkData(0, 0, 10)
+
+        # tdSql.query("select * from stb_new")
+        # tdSql.checkRows(10)
+        # tdSql.checkCols(4096)
+
+        # tdLog.info("stop dnode to commit data to disk")
+        # tdDnodes.stop(1)
+        # tdDnodes.start(1)
+
+        #regular table
+        sql = "create table tb(ts timestamp, "
+        for i in range(1022):
+            sql += "c%d binary(14), " % (i + 1)
+        sql += "c1023 binary(22))"
+        tdSql.execute(sql)
+
+        for i in range(4):
+            sql = "insert into tb values(%d, "
+            for j in range(1022):
+                str = "'%s', " % self.get_random_string(14)
+                sql += str
+            sql += "'%s')" % self.get_random_string(22)
+            tdSql.execute(sql % (self.ts + i))
+
+        time.sleep(10)
+        tdSql.query("select count(*) from tb")
+        tdSql.checkData(0, 0, 4)
+
+        tdDnodes.stop(1)
+        tdDnodes.start(1)
+
+        time.sleep(1)
+        tdSql.query("select count(*) from tb")
+        tdSql.checkData(0, 0, 4)
+
+
+        sql = "create table tb1(ts timestamp, "
+        for i in range(4094):
+            sql += "c%d binary(14), " % (i + 1)
+        sql += "c4095 binary(22))"
+        tdSql.execute(sql)
+
+        for i in range(4):
+            sql = "insert into tb1 values(%d, "
+            for j in range(4094):
+                str = "'%s', " % self.get_random_string(14)
+                sql += str
+            sql += "'%s')" % self.get_random_string(22)
+            tdSql.execute(sql % (self.ts + i))
+
+        time.sleep(10)
+        tdSql.query("select count(*) from tb1")
+        tdSql.checkData(0, 0, 4)
+
+        tdDnodes.stop(1)
+        tdDnodes.start(1)
+
+        time.sleep(1)
+        tdSql.query("select count(*) from tb1")
+        tdSql.checkData(0, 0, 4)
+
+
+
+        #os.system("rm -rf tools/taosdemoAllTest/TD-5213/insertSigcolumnsNum4096.py.sql")
+
+
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())

tests/pytest/tools/taosdemoAllTest/taosdemoTestInsertWithJson.py

@@ -23,7 +23,7 @@ class TDTestCase:
     def init(self, conn, logSql):
         tdLog.debug("start to execute %s" % __file__)
         tdSql.init(conn.cursor(), logSql)
-        
+
     def getBuildPath(self):
         selfPath = os.path.dirname(os.path.realpath(__file__))
 
@@ -39,7 +39,7 @@ class TDTestCase:
                     buildPath = root[:len(root)-len("/build/bin")]
                     break
         return buildPath
-        
+
     def run(self):
         buildPath = self.getBuildPath()
         if (buildPath == ""):
@@ -48,7 +48,7 @@ class TDTestCase:
             tdLog.info("taosd found in %s" % buildPath)
         binPath = buildPath+ "/build/bin/"
 
-        # insert: create one  or mutiple tables per sql and insert multiple rows per sql 
+        # insert: create one  or mutiple tables per sql and insert multiple rows per sql
         os.system("%staosdemo -f tools/taosdemoAllTest/insert-1s1tnt1r.json -y " % binPath)
         tdSql.execute("use db")
         tdSql.query("select count (tbname) from stb0")
@@ -62,7 +62,7 @@ class TDTestCase:
         tdSql.query("select count(*) from stb01_1")
         tdSql.checkData(0, 0, 200)
         tdSql.query("select count(*) from stb1")
-        tdSql.checkData(0, 0, 200000)        
+        tdSql.checkData(0, 0, 200000)
 
         # restful connector insert data
         os.system("%staosdemo -f tools/taosdemoAllTest/insertRestful.json -y " % binPath)
@@ -81,7 +81,7 @@ class TDTestCase:
         tdSql.checkData(0, 0, 200)
 
 
-        # insert: create  mutiple tables per sql and insert one rows per sql . 
+        # insert: create  mutiple tables per sql and insert one rows per sql .
         os.system("%staosdemo -f tools/taosdemoAllTest/insert-1s1tntmr.json -y " % binPath)
         tdSql.execute("use db")
         tdSql.query("select count (tbname) from stb0")
@@ -89,34 +89,34 @@ class TDTestCase:
         tdSql.query("select count (tbname) from stb1")
         tdSql.checkData(0, 0, 20)
         tdSql.query("select count(*) from stb00_0")
-        tdSql.checkData(0, 0, 10000)   
+        tdSql.checkData(0, 0, 10000)
         tdSql.query("select count(*) from stb0")
-        tdSql.checkData(0, 0, 100000) 
+        tdSql.checkData(0, 0, 100000)
         tdSql.query("select count(*) from stb01_0")
-        tdSql.checkData(0, 0, 20000)   
+        tdSql.checkData(0, 0, 20000)
         tdSql.query("select count(*) from stb1")
-        tdSql.checkData(0, 0, 400000) 
+        tdSql.checkData(0, 0, 400000)
 
-        # insert: using parament "insert_interval to controls spped  of insert. 
+        # insert: using parament "insert_interval to controls spped  of insert.
         # but We need to have accurate methods to control the speed, such as getting the speed value, checking the count and so on。
         os.system("%staosdemo -f tools/taosdemoAllTest/insert-interval-speed.json -y" % binPath)
         tdSql.execute("use db")
         tdSql.query("show stables")
         tdSql.checkData(0, 4, 100)
         tdSql.query("select count(*) from stb00_0")
-        tdSql.checkData(0, 0, 20000)   
+        tdSql.checkData(0, 0, 20000)
         tdSql.query("select count(*) from stb0")
-        tdSql.checkData(0, 0, 2000000) 
+        tdSql.checkData(0, 0, 2000000)
         tdSql.query("show stables")
         tdSql.checkData(1, 4, 100)
         tdSql.query("select count(*) from stb01_0")
-        tdSql.checkData(0, 0, 20000)   
+        tdSql.checkData(0, 0, 20000)
         tdSql.query("select count(*) from stb1")
-        tdSql.checkData(0, 0, 2000000)           
-        
+        tdSql.checkData(0, 0, 2000000)
+
         # spend 2min30s for 3 testcases.
         # insert: drop and child_table_exists combination test
-        # insert: using parament "childtable_offset and childtable_limit" to control  table'offset point and offset 
+        # insert: using parament "childtable_offset and childtable_limit" to control  table'offset point and offset
         os.system("%staosdemo -f tools/taosdemoAllTest/insert-nodbnodrop.json -y" % binPath)
         tdSql.error("show dbno.stables")
         os.system("%staosdemo -f tools/taosdemoAllTest/insert-newdb.json -y" % binPath)
@@ -128,41 +128,41 @@ class TDTestCase:
         tdSql.query("select count (tbname) from stb2")
         tdSql.checkData(0, 0, 7)
         tdSql.query("select count (tbname) from stb3")
-        tdSql.checkData(0, 0, 8)        
+        tdSql.checkData(0, 0, 8)
         tdSql.query("select count (tbname) from stb4")
-        tdSql.checkData(0, 0, 8)  
+        tdSql.checkData(0, 0, 8)
         os.system("%staosdemo -f tools/taosdemoAllTest/insert-offset.json -y" % binPath)
-        tdSql.execute("use db")             
+        tdSql.execute("use db")
         tdSql.query("select count(*) from stb0")
-        tdSql.checkData(0, 0, 50) 
+        tdSql.checkData(0, 0, 50)
         tdSql.query("select count(*) from stb1")
-        tdSql.checkData(0, 0, 240) 
+        tdSql.checkData(0, 0, 240)
         tdSql.query("select count(*) from stb2")
-        tdSql.checkData(0, 0, 220) 
+        tdSql.checkData(0, 0, 220)
         tdSql.query("select count(*) from stb3")
         tdSql.checkData(0, 0, 180)
         tdSql.query("select count(*) from stb4")
         tdSql.checkData(0, 0, 160)
         os.system("%staosdemo -f tools/taosdemoAllTest/insert-newtable.json -y" % binPath)
-        tdSql.execute("use db")             
+        tdSql.execute("use db")
         tdSql.query("select count(*) from stb0")
-        tdSql.checkData(0, 0, 150) 
+        tdSql.checkData(0, 0, 150)
         tdSql.query("select count(*) from stb1")
-        tdSql.checkData(0, 0, 360) 
+        tdSql.checkData(0, 0, 360)
         tdSql.query("select count(*) from stb2")
-        tdSql.checkData(0, 0, 360) 
+        tdSql.checkData(0, 0, 360)
         tdSql.query("select count(*) from stb3")
         tdSql.checkData(0, 0, 340)
         tdSql.query("select count(*) from stb4")
         tdSql.checkData(0, 0, 400)
         os.system("%staosdemo -f tools/taosdemoAllTest/insert-renewdb.json -y" % binPath)
-        tdSql.execute("use db")             
+        tdSql.execute("use db")
         tdSql.query("select count(*) from stb0")
-        tdSql.checkData(0, 0, 50) 
+        tdSql.checkData(0, 0, 50)
         tdSql.query("select count(*) from stb1")
-        tdSql.checkData(0, 0, 120) 
+        tdSql.checkData(0, 0, 120)
         tdSql.query("select count(*) from stb2")
-        tdSql.checkData(0, 0, 140) 
+        tdSql.checkData(0, 0, 140)
         tdSql.query("select count(*) from stb3")
         tdSql.checkData(0, 0, 160)
         tdSql.query("select count(*) from stb4")
@@ -170,59 +170,59 @@ class TDTestCase:
 
 
         # insert:  let parament in json file  is illegal, it'll expect error.
-        tdSql.execute("drop database if exists db") 
+        tdSql.execute("drop database if exists db")
         os.system("%staosdemo -f tools/taosdemoAllTest/insertColumnsAndTagNumLarge1024.json -y " % binPath)
         tdSql.error("use db")
-        tdSql.execute("drop database if exists db") 
+        tdSql.execute("drop database if exists db")
         os.system("%staosdemo -f tools/taosdemoAllTest/insertSigcolumnsNum1024.json -y " % binPath)
         tdSql.error("select * from db.stb0")
-        tdSql.execute("drop database if exists db") 
+        tdSql.execute("drop database if exists db")
         os.system("%staosdemo -f tools/taosdemoAllTest/insertColumnsAndTagNum1024.json -y " % binPath)
         tdSql.query("select count(*) from db.stb0")
-        tdSql.checkData(0, 0, 10000) 
-        tdSql.execute("drop database if exists db") 
+        tdSql.checkData(0, 0, 10000)
+        tdSql.execute("drop database if exists db")
         os.system("%staosdemo -f tools/taosdemoAllTest/insertInterlaceRowsLarge1M.json -y " % binPath)
         tdSql.query("select count(*) from db.stb0")
         tdSql.checkRows(0)
-        tdSql.execute("drop database if exists db") 
+        tdSql.execute("drop database if exists db")
         os.system("%staosdemo -f tools/taosdemoAllTest/insertColumnsNum0.json -y " % binPath)
-        tdSql.execute("use db") 
+        tdSql.execute("use db")
         tdSql.query("show stables like 'stb0%' ")
         tdSql.checkData(0, 2, 11)
-        tdSql.execute("drop database if exists db") 
-        os.system("%staosdemo -f tools/taosdemoAllTest/insertTagsNumLarge128.json -y " % binPath)   
-        tdSql.error("use db1") 
-        tdSql.execute("drop database if exists db") 
-        os.system("%staosdemo -f tools/taosdemoAllTest/insertBinaryLenLarge16374AllcolLar16384.json -y " % binPath)   
-        tdSql.query("select count(*) from db.stb0") 
+        tdSql.execute("drop database if exists db")
+        os.system("%staosdemo -f tools/taosdemoAllTest/insertTagsNumLarge128.json -y " % binPath)
+        tdSql.error("use db1")
+        tdSql.execute("drop database if exists db")
+        os.system("%staosdemo -f tools/taosdemoAllTest/insertBinaryLenLarge16374AllcolLar16384.json -y " % binPath)
+        tdSql.query("select count(*) from db.stb0")
         tdSql.checkRows(1)
-        tdSql.query("select count(*) from db.stb1") 
+        tdSql.query("select count(*) from db.stb1")
         tdSql.checkRows(1)
         tdSql.error("select * from db.stb3")
         tdSql.error("select * from db.stb2")
-        tdSql.execute("drop database if exists db") 
-        os.system("%staosdemo -f tools/taosdemoAllTest/insertNumOfrecordPerReq0.json -y " % binPath)   
-        tdSql.error("select count(*) from db.stb0") 
-        tdSql.execute("drop database if exists db") 
-        os.system("%staosdemo -f tools/taosdemoAllTest/insertNumOfrecordPerReqless0.json -y " % binPath)   
-        tdSql.error("use db") 
-        tdSql.execute("drop database if exists db") 
-        os.system("%staosdemo -f tools/taosdemoAllTest/insertChildTab0.json -y " % binPath)   
-        tdSql.error("use db") 
-        tdSql.execute("drop database if exists db") 
-        os.system("%staosdemo -f tools/taosdemoAllTest/insertChildTabLess0.json -y " % binPath)   
-        tdSql.error("use db") 
-        tdSql.execute("drop database if exists blf") 
-        os.system("%staosdemo -f tools/taosdemoAllTest/insertTimestepMulRowsLargeint16.json -y " % binPath)   
-        tdSql.execute("use blf") 
-        tdSql.query("select ts from blf.p_0_topics_7 limit 262800,1") 
+        tdSql.execute("drop database if exists db")
+        os.system("%staosdemo -f tools/taosdemoAllTest/insertNumOfrecordPerReq0.json -y " % binPath)
+        tdSql.error("select count(*) from db.stb0")
+        tdSql.execute("drop database if exists db")
+        os.system("%staosdemo -f tools/taosdemoAllTest/insertNumOfrecordPerReqless0.json -y " % binPath)
+        tdSql.error("use db")
+        tdSql.execute("drop database if exists db")
+        os.system("%staosdemo -f tools/taosdemoAllTest/insertChildTab0.json -y " % binPath)
+        tdSql.error("use db")
+        tdSql.execute("drop database if exists db")
+        os.system("%staosdemo -f tools/taosdemoAllTest/insertChildTabLess0.json -y " % binPath)
+        tdSql.error("use db")
+        tdSql.execute("drop database if exists blf")
+        os.system("%staosdemo -f tools/taosdemoAllTest/insertTimestepMulRowsLargeint16.json -y " % binPath)
+        tdSql.execute("use blf")
+        tdSql.query("select ts from blf.p_0_topics_7 limit 262800,1")
         tdSql.checkData(0, 0, "2020-03-31 12:00:00.000")
         tdSql.query("select first(ts) from blf.p_0_topics_2")
         tdSql.checkData(0, 0, "2019-10-01 00:00:00")
-        tdSql.query("select last(ts) from blf.p_0_topics_6 ")        
+        tdSql.query("select last(ts) from blf.p_0_topics_6 ")
         tdSql.checkData(0, 0, "2020-09-29 23:59:00")
-        os.system("%staosdemo -f tools/taosdemoAllTest/insertMaxNumPerReq.json -y " % binPath)   
-        tdSql.execute("use db") 
+        os.system("%staosdemo -f tools/taosdemoAllTest/insertMaxNumPerReq.json -y " % binPath)
+        tdSql.execute("use db")
         tdSql.query("select count(*) from stb0")
         tdSql.checkData(0, 0, 5000000)
         tdSql.query("select count(*) from stb1")
@@ -230,7 +230,7 @@ class TDTestCase:
 
 
 
-        # insert: timestamp and step 
+        # insert: timestamp and step
         os.system("%staosdemo -f tools/taosdemoAllTest/insert-timestep.json -y " % binPath)
         tdSql.execute("use db")
         tdSql.query("show stables")
@@ -239,13 +239,13 @@ class TDTestCase:
         tdSql.query("select count (tbname) from stb1")
         tdSql.checkData(0, 0, 20)
         tdSql.query("select last(ts) from db.stb00_0")
-        tdSql.checkData(0, 0, "2020-10-01 00:00:00.019000")   
+        tdSql.checkData(0, 0, "2020-10-01 00:00:00.019000")
         tdSql.query("select count(*) from stb0")
-        tdSql.checkData(0, 0, 200) 
+        tdSql.checkData(0, 0, 200)
         tdSql.query("select last(ts) from db.stb01_0")
-        tdSql.checkData(0, 0, "2020-11-01 00:00:00.190000")   
+        tdSql.checkData(0, 0, "2020-11-01 00:00:00.190000")
         tdSql.query("select count(*) from stb1")
-        tdSql.checkData(0, 0, 400) 
+        tdSql.checkData(0, 0, 400)
 
         # # insert:  disorder_ratio
         os.system("%staosdemo -f tools/taosdemoAllTest/insert-disorder.json -g 2>&1  -y " % binPath)
@@ -255,14 +255,14 @@ class TDTestCase:
         tdSql.query("select count (tbname) from stb1")
         tdSql.checkData(0, 0, 1)
         tdSql.query("select count(*) from stb0")
-        tdSql.checkData(0, 0, 10) 
+        tdSql.checkData(0, 0, 10)
         tdSql.query("select count(*) from stb1")
-        tdSql.checkData(0, 0, 10) 
+        tdSql.checkData(0, 0, 10)
 
         # insert:  sample json
         os.system("%staosdemo -f tools/taosdemoAllTest/insert-sample.json -y " % binPath)
         tdSql.execute("use dbtest123")
-        tdSql.query("select col2 from stb0")
+        tdSql.query("select c2 from stb0")
         tdSql.checkData(0, 0, 2147483647)
         tdSql.query("select * from stb1 where t1=-127")
         tdSql.checkRows(20)
@@ -271,13 +271,13 @@ class TDTestCase:
         tdSql.query("select * from stb1 where t2=126")
         tdSql.checkRows(10)
 
-        # insert: test interlace parament 
+        # insert: test interlace parament
         os.system("%staosdemo -f tools/taosdemoAllTest/insert-interlace-row.json -y " % binPath)
         tdSql.execute("use db")
         tdSql.query("select count (tbname) from stb0")
         tdSql.checkData(0, 0, 100)
         tdSql.query("select count (*) from stb0")
-        tdSql.checkData(0, 0, 15000)        
+        tdSql.checkData(0, 0, 15000)
 
 
         # # insert: auto_create
@@ -317,10 +317,10 @@ class TDTestCase:
         tdSql.checkRows(20)
 
         os.system("rm -rf ./insert_res.txt")
-        os.system("rm -rf tools/taosdemoAllTest/taosdemoTestInsertWithJson.py.sql")        
-        
-        
-        
+        os.system("rm -rf tools/taosdemoAllTest/taosdemoTestInsertWithJson.py.sql")
+
+
+
     def stop(self):
         tdSql.close()
         tdLog.success("%s successfully executed" % __file__)

tests/pytest/tools/taosdemoPerformance.py

@@ -19,7 +19,6 @@ import json
 from util.log import tdLog
 from util.sql import tdSql
 
-
 class taosdemoPerformace:
     def __init__(self, commitID, dbName):
         self.commitID = commitID
@@ -123,7 +122,7 @@ class taosdemoPerformace:
         return buildPath
 
     def insertData(self):
-        tdSql.prepare()
+
         buildPath = self.getBuildPath()
         if (buildPath == ""):
             tdLog.exit("taosdemo not found!")

tests/pytest/tools/taosdemoTest.py

@@ -59,11 +59,11 @@ class TDTestCase:
         tdSql.checkData(0, 0, self.numberOfTables * self.numberOfRecords)
 
         tdSql.query(
-            "select sum(col1) from test.meters interval(1h) sliding(30m)")
+            "select sum(c1) from test.meters interval(1h) sliding(30m)")
         tdSql.checkRows(2)
 
         tdSql.query(
-            "select apercentile(col1, 1) from test.meters interval(100s)")
+            "select apercentile(c1, 1) from test.meters interval(100s)")
         tdSql.checkRows(1)
 
         tdSql.error("select loc, count(loc) from test.meters")

tests/pytest/tools/taosdumpTest.py

@@ -105,7 +105,6 @@ class TDTestCase:
         # 6--days,7--keep0,keep1,keep, 12--block,
 
         isCommunity = self.checkCommunity()
-
         print("iscommunity: %d" % isCommunity)
         for i in range(len(dbresult)):
             if dbresult[i][0] == 'db':

tests/pytest/wal/sdbComp.py

@@ -26,11 +26,11 @@ class TDTestCase:
     def init(self, conn, logSql):
         tdLog.debug("start to execute %s" % __file__)
         tdSql.init(conn.cursor(), logSql)
-        
+
     def getBuildPath(self):
         global selfPath
         selfPath = os.path.dirname(os.path.realpath(__file__))
-        
+
         if ("community" in selfPath):
             projPath = selfPath[:selfPath.find("community")]
         else:
@@ -43,7 +43,7 @@ class TDTestCase:
                     buildPath = root[:len(root)-len("/build/bin")]
                     break
         return buildPath
-        
+
     def run(self):
 
         # set path para
@@ -62,9 +62,9 @@ class TDTestCase:
         os.system("rm -rf  %s/sim/dnode1/data/mnode_bak/" % testPath)
         tdSql.execute("drop database if exists db2")
         os.system("%staosdemo -f wal/insertDataDb1.json -y " % binPath)
-        tdSql.execute("drop database if exists db1") 
+        tdSql.execute("drop database if exists db1")
         os.system("%staosdemo -f wal/insertDataDb2.json -y " % binPath)
-        tdSql.execute("drop table if exists db2.stb0") 
+        tdSql.execute("drop table if exists db2.stb0")
         os.system("%staosdemo -f wal/insertDataDb2Newstab.json -y " % binPath)
         query_pid1 = int(subprocess.getstatusoutput('ps aux|grep taosd |grep -v "grep"|awk \'{print $2}\'')[1])
         print(query_pid1)
@@ -72,14 +72,14 @@ class TDTestCase:
         tdSql.execute("drop table if exists stb1_0")
         tdSql.execute("drop table if exists stb1_1")
         tdSql.execute("insert into stb0_0 values(1614218412000,8637,78.861045,'R','bf3')(1614218422000,8637,98.861045,'R','bf3')")
-        tdSql.execute("alter table db2.stb0 add column col4 int")
-        tdSql.execute("alter table db2.stb0 drop column col2")
-        tdSql.execute("alter table db2.stb0  add tag t3 int;")        
+        tdSql.execute("alter table db2.stb0 add column c4 int")
+        tdSql.execute("alter table db2.stb0 drop column c2")
+        tdSql.execute("alter table db2.stb0  add tag t3 int;")
         tdSql.execute("alter table db2.stb0 drop tag t1")
-        tdSql.execute("create table  if not exists stb2_0 (ts timestamp, col0 int, col1 float)  ")
+        tdSql.execute("create table  if not exists stb2_0 (ts timestamp, c0 int, c1 float)  ")
         tdSql.execute("insert into stb2_0 values(1614218412000,8637,78.861045)")
-        tdSql.execute("alter table stb2_0 add column col2 binary(4)")
-        tdSql.execute("alter table stb2_0 drop column col1")
+        tdSql.execute("alter table stb2_0 add column c2 binary(4)")
+        tdSql.execute("alter table stb2_0 drop column c1")
         tdSql.execute("insert into stb2_0 values(1614218422000,8638,'R')")
 
         # stop taosd and compact wal file
@@ -87,9 +87,9 @@ class TDTestCase:
         sleep(10)
         os.system("nohup %s/taosd  --compact-mnode-wal  -c %s/sim/dnode1/cfg/ & " %(binPath,testPath) )
         sleep(5)
-        assert os.path.exists(walFilePath) , "%s is not generated, compact didn't  take effect " % walFilePath    
+        assert os.path.exists(walFilePath) , "%s is not generated, compact didn't  take effect " % walFilePath
 
-        # use new wal file to start  taosd 
+        # use new wal file to start  taosd
         tdDnodes.start(1)
         sleep(5)
         tdSql.execute("reset query cache")
@@ -108,14 +108,14 @@ class TDTestCase:
         tdSql.checkData(0, 0, 2)
         tdSql.query("select count(*) from stb2_0")
         tdSql.checkData(0, 0, 2)
-   
+
         # delete useless file
         testcaseFilename = os.path.split(__file__)[-1]
         os.system("rm -rf ./insert_res.txt")
-        os.system("rm -rf wal/%s.sql" % testcaseFilename )       
-        
-        
-        
+        os.system("rm -rf wal/%s.sql" % testcaseFilename)
+
+
+
     def stop(self):
         tdSql.close()
         tdLog.success("%s successfully executed" % __file__)

tests/script/sh/cfg.sh

 #!/bin/bash
 
-if [ $# != 6 ]; then 
+if [ $# != 6 ]; then
   echo "argument list need input : "
   echo "  -n nodeName"
   echo "  -c configName"
@@ -8,10 +8,12 @@ if [ $# != 6 ]; then
   exit 1
 fi
 
+UNAME_BIN=`which uname`
+OS_TYPE=`$UNAME_BIN`
 NODE_NAME=
 CONFIG_NAME=
 CONFIG_VALUE=
-while getopts "n:v:c:" arg 
+while getopts "n:v:c:" arg
 do
   case $arg in
     n)
@@ -43,10 +45,16 @@ fi
 TAOS_DIR=`pwd`
 TAOSD_DIR=`find . -name "taosd"|grep bin|head -n1`
 
+if [[ "$OS_TYPE" != "Darwin" ]]; then
+  cut_opt="--field="
+else
+  cut_opt="-f "
+fi
+
 if [[ "$TAOSD_DIR" == *"$IN_TDINTERNAL"* ]]; then
-  BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' --fields=2,3`
+  BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' ${cut_opt}2,3`
 else
-  BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' --fields=2`
+  BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' ${cut_opt}2`
 fi
 
 BUILD_DIR=$TAOS_DIR/$BIN_DIR/build
@@ -56,7 +64,7 @@ SIM_DIR=$TAOS_DIR/sim
 NODE_DIR=$SIM_DIR/$NODE_NAME
 TAOS_CFG=$NODE_DIR/cfg/taos.cfg
 TAOS_FLAG=$SIM_DIR/tsim/flag
-if [ -f "$TAOS_FLAG" ] ; then 
+if [ -f "$TAOS_FLAG" ] ; then
   TAOS_CFG=/etc/taos/taos.cfg
 fi
 

tests/script/sh/clear.sh

@@ -2,7 +2,7 @@
 
 echo "Executing clear.sh"
 
-if [ $# != 6 ]; then 
+if [ $# != 6 ]; then
   echo "argument list need input : "
   echo "  -n nodeName"
   echo "  -i nodeIp"
@@ -10,10 +10,12 @@ if [ $# != 6 ]; then
   exit 1
 fi
 
+UNAME_BIN=`which uname`
+OS_TYPE=`$UNAME_BIN`
 NODE_NAME=
 NODE_IP=
 MSATER_IP=
-while getopts "n:i:m:" arg 
+while getopts "n:i:m:" arg
 do
   case $arg in
     n)
@@ -34,7 +36,7 @@ done
 SCRIPT_DIR=`dirname $0`
 cd $SCRIPT_DIR/../
 SCRIPT_DIR=`pwd`
-echo "SCRIPT_DIR: $SCRIPT_DIR" 
+echo "SCRIPT_DIR: $SCRIPT_DIR"
 
 IN_TDINTERNAL="community"
 if [[ "$SCRIPT_DIR" == *"$IN_TDINTERNAL"* ]]; then
@@ -46,10 +48,16 @@ fi
 TAOS_DIR=`pwd`
 TAOSD_DIR=`find . -name "taosd"|grep bin|head -n1`
 
+if [[ "$OS_TYPE" != "Darwin" ]]; then
+  cut_opt="--field="
+else
+  cut_opt="-f "
+fi
+
 if [[ "$TAOSD_DIR" == *"$IN_TDINTERNAL"* ]]; then
-  BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' --fields=2,3`
+  BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' ${cut_opt}2,3`
 else
-  BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' --fields=2`
+  BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' ${cut_opt}2`
 fi
 
 BUILD_DIR=$TAOS_DIR/$BIN_DIR/build
@@ -88,7 +96,7 @@ TAOS_CFG=$NODE_DIR/cfg/taos.cfg
 touch -f $TAOS_CFG
 
 TAOS_FLAG=$SIM_DIR/tsim/flag
-if [ -f "$TAOS_FLAG" ] ; then 
+if [ -f "$TAOS_FLAG" ] ; then
   TAOS_CFG=/etc/taos/taos.cfg
   DATA_DIR=/var/lib/taos
   LOG_DIR=/var/log/taos
@@ -98,7 +106,7 @@ if [ -f "$TAOS_FLAG" ] ; then
   sudo rm -rf $LOG_DIR
 fi
 
-echo " "                                  >> $TAOS_CFG   
+echo " "                                  >> $TAOS_CFG
 echo "masterIp            $MASTER_IP"     >> $TAOS_CFG
 echo "dataDir             $DATA_DIR"      >> $TAOS_CFG
 echo "logDir              $LOG_DIR"       >> $TAOS_CFG

tests/script/sh/deploy.sh

@@ -2,16 +2,18 @@
 
 echo "Executing deploy.sh"
 
-if [ $# != 4 ]; then 
+if [ $# != 4 ]; then
   echo "argument list need input : "
   echo "  -n nodeName"
   echo "  -i nodePort"
   exit 1
 fi
 
+UNAME_BIN=`which uname`
+OS_TYPE=`$UNAME_BIN`
 NODE_NAME=
 NODE=
-while getopts "n:i:" arg 
+while getopts "n:i:" arg
 do
   case $arg in
     n)
@@ -29,7 +31,7 @@ done
 SCRIPT_DIR=`dirname $0`
 cd $SCRIPT_DIR/../
 SCRIPT_DIR=`pwd`
-echo "SCRIPT_DIR: $SCRIPT_DIR" 
+echo "SCRIPT_DIR: $SCRIPT_DIR"
 
 IN_TDINTERNAL="community"
 if [[ "$SCRIPT_DIR" == *"$IN_TDINTERNAL"* ]]; then
@@ -41,10 +43,16 @@ fi
 TAOS_DIR=`pwd`
 TAOSD_DIR=`find . -name "taosd"|grep bin|head -n1`
 
+if [[ "$OS_TYPE" != "Darwin" ]]; then
+  cut_opt="--field="
+else
+  cut_opt="-f "
+fi
+
 if [[ "$TAOSD_DIR" == *"$IN_TDINTERNAL"* ]]; then
-  BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' --fields=2,3`
+  BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' ${cut_opt}2,3`
 else
-  BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' --fields=2`
+  BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' ${cut_opt}2`
 fi
 
 BUILD_DIR=$TAOS_DIR/$BIN_DIR/build

tests/script/sh/exec-default.sh

 #!/bin/bash
 
-# if [ $# != 4 || $# != 5 ]; then 
+# if [ $# != 4 || $# != 5 ]; then
   # echo "argument list need input : "
   # echo "  -n nodeName"
   # echo "  -s start/stop"
@@ -8,10 +8,12 @@
   # exit 1
 # fi
 
+UNAME_BIN=`which uname`
+OS_TYPE=`$UNAME_BIN`
 NODE_NAME=
 EXEC_OPTON=
 CLEAR_OPTION="false"
-while getopts "n:s:u:x:ct" arg 
+while getopts "n:s:u:x:ct" arg
 do
   case $arg in
     n)
@@ -52,10 +54,16 @@ fi
 TAOS_DIR=`pwd`
 TAOSD_DIR=`find . -name "taosd"|grep bin|head -n1`
 
+if [[ "$OS_TYPE" != "Darwin" ]]; then
+  cut_opt="--field="
+else
+  cut_opt="-f "
+fi
+
 if [[ "$TAOSD_DIR" == *"$IN_TDINTERNAL"* ]]; then
-  BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' --fields=2,3`
+  BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' ${cut_opt}2,3`
 else
-  BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' --fields=2`
+  BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' ${cut_opt}2`
 fi
 
 BUILD_DIR=$TAOS_DIR/$BIN_DIR/build