Merge branch 'develop' into xiaoping/add_test_case2

cmake/version.inc

@@ -4,7 +4,7 @@ PROJECT(TDengine)
 IF (DEFINED VERNUMBER)
   SET(TD_VER_NUMBER ${VERNUMBER})
 ELSE ()
-  SET(TD_VER_NUMBER "2.0.6.0")
+  SET(TD_VER_NUMBER "2.0.7.0")
 ENDIF ()
 
 IF (DEFINED VERCOMPATIBLE)

packaging/deb/makedeb.sh

@@ -48,7 +48,7 @@ cp ${compile_dir}/../packaging/deb/taosd            ${pkg_dir}${install_home_pat
 cp ${compile_dir}/../packaging/tools/post.sh        ${pkg_dir}${install_home_path}/script
 cp ${compile_dir}/../packaging/tools/preun.sh       ${pkg_dir}${install_home_path}/script
 cp ${compile_dir}/build/bin/taosdemo                ${pkg_dir}${install_home_path}/bin
-#cp ${compile_dir}/build/bin/taosdump                ${pkg_dir}${install_home_path}/bin
+cp ${compile_dir}/build/bin/taosdump                ${pkg_dir}${install_home_path}/bin
 cp ${compile_dir}/build/bin/taosd                   ${pkg_dir}${install_home_path}/bin
 cp ${compile_dir}/build/bin/taos                    ${pkg_dir}${install_home_path}/bin
 cp ${compile_dir}/build/lib/${libfile}              ${pkg_dir}${install_home_path}/driver 

packaging/rpm/tdengine.spec

@@ -58,7 +58,7 @@ cp %{_compiledir}/../packaging/tools/preun.sh       %{buildroot}%{homepath}/scri
 cp %{_compiledir}/build/bin/taos                    %{buildroot}%{homepath}/bin
 cp %{_compiledir}/build/bin/taosd                   %{buildroot}%{homepath}/bin
 cp %{_compiledir}/build/bin/taosdemo                %{buildroot}%{homepath}/bin
-#cp %{_compiledir}/build/bin/taosdump                %{buildroot}%{homepath}/bin
+cp %{_compiledir}/build/bin/taosdump                %{buildroot}%{homepath}/bin
 cp %{_compiledir}/build/lib/${libfile}              %{buildroot}%{homepath}/driver
 cp %{_compiledir}/../src/inc/taos.h                 %{buildroot}%{homepath}/include
 cp %{_compiledir}/../src/inc/taoserror.h            %{buildroot}%{homepath}/include

packaging/tools/get_client.sh

+#!/bin/bash
+#
+
+log_dir=$1
+result_file=$2
+
+if [ ! -n "$1" ];then
+  echo "Pleas input the director of taosdlog."
+  echo "usage: ./get_client.sh <taosdlog directory> <result file>"
+  exit 1
+else
+  log_dir=$1
+fi
+
+if [ ! -n "$2" ];then
+  result_file=clientInfo.txt
+else
+  result_file=$2
+fi
+
+grep "new TCP connection" ${log_dir}/taosdlog.* | sed -e "s/0x.* from / /"|sed -e "s/,.*$//"|sed -e "s/:[0-9]*$//"|sort -r|uniq -f 2|sort -k 3 -r|uniq -f 2 > ${result_file}

packaging/tools/makeclient.sh

@@ -45,8 +45,7 @@ if [ "$osType" != "Darwin" ]; then
     strip ${build_dir}/bin/taos
     bin_files="${build_dir}/bin/taos ${script_dir}/remove_client.sh"
   else  
-    #bin_files="${build_dir}/bin/taos ${build_dir}/bin/taosdump ${build_dir}/bin/taosdemo ${script_dir}/remove_client.sh ${script_dir}/set_core.sh"
-    bin_files="${build_dir}/bin/taos ${build_dir}/bin/taosdemo ${script_dir}/remove_client.sh ${script_dir}/set_core.sh"
+    bin_files="${build_dir}/bin/taos ${build_dir}/bin/taosdump ${build_dir}/bin/taosdemo ${script_dir}/remove_client.sh ${script_dir}/set_core.sh ${script_dir}/get_client.sh"
   fi
   lib_files="${build_dir}/lib/libtaos.so.${version}"
 else

packaging/tools/makeclient_power.sh

@@ -77,7 +77,9 @@ if [ "$osType" != "Darwin" ]; then
     cp ${build_dir}/bin/taos          ${install_dir}/bin/power
     cp ${script_dir}/remove_power.sh  ${install_dir}/bin
     cp ${build_dir}/bin/taosdemo      ${install_dir}/bin/powerdemo 
+    cp ${build_dir}/bin/taosdump      ${install_dir}/bin/powerdump  
     cp ${script_dir}/set_core.sh      ${install_dir}/bin
+    cp ${script_dir}/get_client.sh    ${install_dir}/bin
   fi
 else
   cp ${bin_files} ${install_dir}/bin

packaging/tools/makepkg.sh

@@ -36,8 +36,7 @@ if [ "$pagMode" == "lite" ]; then
   strip ${build_dir}/bin/taos
   bin_files="${build_dir}/bin/taosd ${build_dir}/bin/taos ${script_dir}/remove.sh"
 else 
-  #bin_files="${build_dir}/bin/taosd ${build_dir}/bin/taos ${build_dir}/bin/taosdump ${build_dir}/bin/taosdemo ${build_dir}/bin/tarbitrator ${script_dir}/remove.sh ${script_dir}/set_core.sh"
-  bin_files="${build_dir}/bin/taosd ${build_dir}/bin/taos ${build_dir}/bin/taosdemo ${build_dir}/bin/tarbitrator ${script_dir}/remove.sh ${script_dir}/set_core.sh"
+  bin_files="${build_dir}/bin/taosd ${build_dir}/bin/taos ${build_dir}/bin/taosdump ${build_dir}/bin/taosdemo ${build_dir}/bin/tarbitrator ${script_dir}/remove.sh ${script_dir}/set_core.sh ${script_dir}/get_client.sh"
 fi
 
 lib_files="${build_dir}/lib/libtaos.so.${version}"

packaging/tools/makepkg_power.sh

@@ -77,8 +77,10 @@ else
   cp ${build_dir}/bin/taosd         ${install_dir}/bin/powerd
   cp ${script_dir}/remove_power.sh  ${install_dir}/bin
   cp ${build_dir}/bin/taosdemo      ${install_dir}/bin/powerdemo   
+  cp ${build_dir}/bin/taosdump      ${install_dir}/bin/powerdump  
   cp ${build_dir}/bin/tarbitrator   ${install_dir}/bin
   cp ${script_dir}/set_core.sh      ${install_dir}/bin
+  cp ${script_dir}/get_client.sh    ${install_dir}/bin
 fi
 chmod a+x ${install_dir}/bin/* || :
 

snap/snapcraft.yaml

 name: tdengine
 base: core18
-version: '2.0.6.0'
+version: '2.0.7.0'
 icon: snap/gui/t-dengine.svg
 summary: an open-source big data platform designed and optimized for IoT. 
 description: |

src/client/src/tscFunctionImpl.c

@@ -28,6 +28,7 @@
 #include "tscompression.h"
 #include "tsqlfunction.h"
 #include "tutil.h"
+#include "ttype.h"
 
 #define GET_INPUT_CHAR(x) (((char *)((x)->aInputElemBuf)) + ((x)->startOffset) * ((x)->inputBytes))
 #define GET_INPUT_CHAR_INDEX(x, y) (GET_INPUT_CHAR(x) + (y) * (x)->inputBytes)
@@ -2479,28 +2480,8 @@ static void percentile_function(SQLFunctionCtx *pCtx) {
           continue;
         }
 
-        // TODO extract functions
         double v = 0;
-        switch (pCtx->inputType) {
-          case TSDB_DATA_TYPE_TINYINT:
-            v = GET_INT8_VAL(data);
-            break;
-          case TSDB_DATA_TYPE_SMALLINT:
-            v = GET_INT16_VAL(data);
-            break;
-          case TSDB_DATA_TYPE_BIGINT:
-            v = (double)(GET_INT64_VAL(data));
-            break;
-          case TSDB_DATA_TYPE_FLOAT:
-            v = GET_FLOAT_VAL(data);
-            break;
-          case TSDB_DATA_TYPE_DOUBLE:
-            v = GET_DOUBLE_VAL(data);
-            break;
-          default:
-            v = GET_INT32_VAL(data);
-            break;
-        }
+        GET_TYPED_DATA(v, double, pCtx->inputType, data);
 
         if (v < GET_DOUBLE_VAL(&pInfo->minval)) {
           SET_DOUBLE_VAL(&pInfo->minval, v);
@@ -2541,30 +2522,10 @@ static void percentile_function_f(SQLFunctionCtx *pCtx, int32_t index) {
   SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
 
   SPercentileInfo *pInfo = (SPercentileInfo *)GET_ROWCELL_INTERBUF(pResInfo);
-
   if (pInfo->stage == 0) {
-    // TODO extract functions
+
     double v = 0;
-    switch (pCtx->inputType) {
-      case TSDB_DATA_TYPE_TINYINT:
-        v = GET_INT8_VAL(pData);
-        break;
-      case TSDB_DATA_TYPE_SMALLINT:
-        v = GET_INT16_VAL(pData);
-        break;
-      case TSDB_DATA_TYPE_BIGINT:
-        v = (double)(GET_INT64_VAL(pData));
-        break;
-      case TSDB_DATA_TYPE_FLOAT:
-        v = GET_FLOAT_VAL(pData);
-        break;
-      case TSDB_DATA_TYPE_DOUBLE:
-        v = GET_DOUBLE_VAL(pData);
-        break;
-      default:
-        v = GET_INT32_VAL(pData);
-        break;
-    }
+    GET_TYPED_DATA(v, double, pCtx->inputType, pData);
 
     if (v < GET_DOUBLE_VAL(&pInfo->minval)) {
       SET_DOUBLE_VAL(&pInfo->minval, v);
@@ -2653,29 +2614,9 @@ static void apercentile_function(SQLFunctionCtx *pCtx) {
     }
     
     notNullElems += 1;
-    double v = 0;
-    
-    switch (pCtx->inputType) {
-      case TSDB_DATA_TYPE_TINYINT:
-        v = GET_INT8_VAL(data);
-        break;
-      case TSDB_DATA_TYPE_SMALLINT:
-        v = GET_INT16_VAL(data);
-        break;
-      case TSDB_DATA_TYPE_BIGINT:
-        v = (double)(GET_INT64_VAL(data));
-        break;
-      case TSDB_DATA_TYPE_FLOAT:
-        v = GET_FLOAT_VAL(data);
-        break;
-      case TSDB_DATA_TYPE_DOUBLE:
-        v = GET_DOUBLE_VAL(data);
-        break;
-      default:
-        v = GET_INT32_VAL(data);
-        break;
-    }
 
+    double v = 0;
+    GET_TYPED_DATA(v, double, pCtx->inputType, data);
     tHistogramAdd(&pInfo->pHisto, v);
   }
   
@@ -2700,26 +2641,7 @@ static void apercentile_function_f(SQLFunctionCtx *pCtx, int32_t index) {
   SAPercentileInfo *pInfo = getAPerctInfo(pCtx);
   
   double v = 0;
-  switch (pCtx->inputType) {
-    case TSDB_DATA_TYPE_TINYINT:
-      v = GET_INT8_VAL(pData);
-      break;
-    case TSDB_DATA_TYPE_SMALLINT:
-      v = GET_INT16_VAL(pData);
-      break;
-    case TSDB_DATA_TYPE_BIGINT:
-      v = (double)(GET_INT64_VAL(pData));
-      break;
-    case TSDB_DATA_TYPE_FLOAT:
-      v = GET_FLOAT_VAL(pData);
-      break;
-    case TSDB_DATA_TYPE_DOUBLE:
-      v = GET_DOUBLE_VAL(pData);
-      break;
-    default:
-      v = GET_INT32_VAL(pData);
-      break;
-  }
+  GET_TYPED_DATA(v, double, pCtx->inputType, pData);
   
   tHistogramAdd(&pInfo->pHisto, v);
   
@@ -4142,22 +4064,7 @@ static void rate_function(SQLFunctionCtx *pCtx) {
     notNullElems++;
     
     int64_t v = 0;
-    switch (pCtx->inputType) {
-      case TSDB_DATA_TYPE_TINYINT:
-        v = (int64_t)GET_INT8_VAL(pData);
-        break;
-      case TSDB_DATA_TYPE_SMALLINT:
-        v = (int64_t)GET_INT16_VAL(pData);
-        break;
-      case TSDB_DATA_TYPE_INT:
-        v = (int64_t)GET_INT32_VAL(pData);
-        break;
-      case TSDB_DATA_TYPE_BIGINT:
-        v = (int64_t)GET_INT64_VAL(pData);
-        break;
-      default:
-        assert(0);
-    }
+    GET_TYPED_DATA(v, int64_t, pCtx->inputType, pData);
     
     if ((INT64_MIN == pRateInfo->firstValue) || (INT64_MIN == pRateInfo->firstKey)) {
       pRateInfo->firstValue = v;
@@ -4207,22 +4114,7 @@ static void rate_function_f(SQLFunctionCtx *pCtx, int32_t index) {
   TSKEY       *primaryKey = pCtx->ptsList;
   
   int64_t v = 0;
-  switch (pCtx->inputType) {
-    case TSDB_DATA_TYPE_TINYINT:
-      v = (int64_t)GET_INT8_VAL(pData);
-      break;
-    case TSDB_DATA_TYPE_SMALLINT:
-      v = (int64_t)GET_INT16_VAL(pData);
-      break;
-    case TSDB_DATA_TYPE_INT:
-      v = (int64_t)GET_INT32_VAL(pData);
-      break;
-    case TSDB_DATA_TYPE_BIGINT:
-      v = (int64_t)GET_INT64_VAL(pData);
-      break;
-    default:
-      assert(0);
-  }
+  GET_TYPED_DATA(v, int64_t, pCtx->inputType, pData);
   
   if ((INT64_MIN == pRateInfo->firstValue) || (INT64_MIN == pRateInfo->firstKey)) {
     pRateInfo->firstValue = v;
@@ -4349,22 +4241,7 @@ static void irate_function(SQLFunctionCtx *pCtx) {
     notNullElems++;
     
     int64_t v = 0;
-    switch (pCtx->inputType) {
-      case TSDB_DATA_TYPE_TINYINT:
-        v = (int64_t)GET_INT8_VAL(pData);
-        break;
-      case TSDB_DATA_TYPE_SMALLINT:
-        v = (int64_t)GET_INT16_VAL(pData);
-        break;
-      case TSDB_DATA_TYPE_INT:
-        v = (int64_t)GET_INT32_VAL(pData);
-        break;
-      case TSDB_DATA_TYPE_BIGINT:
-        v = (int64_t)GET_INT64_VAL(pData);
-        break;
-      default:
-        assert(0);
-    }
+    GET_TYPED_DATA(v, int64_t, pCtx->inputType, pData);
     
     // TODO: calc once if only call this function once ????
     if ((INT64_MIN == pRateInfo->lastKey) || (INT64_MIN == pRateInfo->lastValue)) {
@@ -4409,22 +4286,7 @@ static void irate_function_f(SQLFunctionCtx *pCtx, int32_t index) {
   TSKEY        *primaryKey = pCtx->ptsList;
   
   int64_t v = 0;
-  switch (pCtx->inputType) {
-    case TSDB_DATA_TYPE_TINYINT:
-      v = (int64_t)GET_INT8_VAL(pData);
-      break;
-    case TSDB_DATA_TYPE_SMALLINT:
-      v = (int64_t)GET_INT16_VAL(pData);
-      break;
-    case TSDB_DATA_TYPE_INT:
-      v = (int64_t)GET_INT32_VAL(pData);
-      break;
-    case TSDB_DATA_TYPE_BIGINT:
-      v = (int64_t)GET_INT64_VAL(pData);
-      break;
-    default:
-      assert(0);
-  }
+  GET_TYPED_DATA(v, int64_t, pCtx->inputType, pData);
 
   pRateInfo->firstKey   = pRateInfo->lastKey;
   pRateInfo->firstValue = pRateInfo->lastValue;

src/client/src/tscLocalMerge.c

@@ -373,7 +373,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd
     SFillColInfo* pFillCol = createFillColInfo(pQueryInfo);
     pReducer->pFillInfo = taosInitFillInfo(pQueryInfo->order.order, revisedSTime, pQueryInfo->groupbyExpr.numOfGroupCols,
                                            4096, (int32_t)numOfCols, pQueryInfo->interval.sliding, pQueryInfo->interval.slidingUnit,
-                                           tinfo.precision, pQueryInfo->fillType, pFillCol);
+                                           tinfo.precision, pQueryInfo->fillType, pFillCol, pSql);
   }
 }
 
@@ -916,7 +916,7 @@ static void doFillResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool doneO
   }
 
   while (1) {
-    int64_t newRows = taosGenerateDataBlock(pFillInfo, pResPages, pLocalReducer->resColModel->capacity);
+    int64_t newRows = taosFillResultDataBlock(pFillInfo, pResPages, pLocalReducer->resColModel->capacity);
 
     if (pQueryInfo->limit.offset < newRows) {
       newRows -= pQueryInfo->limit.offset;
@@ -945,7 +945,7 @@ static void doFillResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool doneO
         }
 
         // all output in current group are completed
-        int32_t totalRemainRows = (int32_t)getFilledNumOfRes(pFillInfo, actualETime, pLocalReducer->resColModel->capacity);
+        int32_t totalRemainRows = (int32_t)getNumOfResWithFill(pFillInfo, actualETime, pLocalReducer->resColModel->capacity);
         if (totalRemainRows <= 0) {
           break;
         }
@@ -1294,7 +1294,7 @@ static bool doBuildFilledResultForGroup(SSqlObj *pSql) {
     int64_t etime = *(int64_t *)(pFinalDataBuf->data + TSDB_KEYSIZE * (pFillInfo->numOfRows - 1));
 
     // the first column must be the timestamp column
-    int32_t rows = (int32_t)getFilledNumOfRes(pFillInfo, etime, pLocalReducer->resColModel->capacity);
+    int32_t rows = (int32_t) getNumOfResWithFill(pFillInfo, etime, pLocalReducer->resColModel->capacity);
     if (rows > 0) {  // do fill gap
       doFillResult(pSql, pLocalReducer, false);
     }
@@ -1323,7 +1323,7 @@ static bool doHandleLastRemainData(SSqlObj *pSql) {
       ((pRes->numOfRowsGroup < pQueryInfo->limit.limit && pQueryInfo->limit.limit > 0) || (pQueryInfo->limit.limit < 0))) {
       int64_t etime = (pQueryInfo->order.order == TSDB_ORDER_ASC)? pQueryInfo->window.ekey : pQueryInfo->window.skey;
 
-      int32_t rows = (int32_t)getFilledNumOfRes(pFillInfo, etime, pLocalReducer->resColModel->capacity);
+      int32_t rows = (int32_t)getNumOfResWithFill(pFillInfo, etime, pLocalReducer->resColModel->capacity);
       if (rows > 0) {
         doFillResult(pSql, pLocalReducer, true);
       }

src/client/src/tscSQLParser.c

@@ -57,11 +57,11 @@ static SSqlExpr* doAddProjectCol(SQueryInfo* pQueryInfo, int32_t outputIndex, in
 static int32_t setShowInfo(SSqlObj* pSql, SSqlInfo* pInfo);
 static char*   getAccountId(SSqlObj* pSql);
 
-static bool has(tFieldList* pFieldList, int32_t startIdx, const char* name);
+static bool has(SArray* pFieldList, int32_t startIdx, const char* name);
 static void getCurrentDBName(SSqlObj* pSql, SStrToken* pDBToken);
 static bool hasSpecifyDB(SStrToken* pTableName);
-static bool validateTableColumnInfo(tFieldList* pFieldList, SSqlCmd* pCmd);
-static bool validateTagParams(tFieldList* pTagsList, tFieldList* pFieldList, SSqlCmd* pCmd);
+static bool validateTableColumnInfo(SArray* pFieldList, SSqlCmd* pCmd);
+static bool validateTagParams(SArray* pTagsList, SArray* pFieldList, SSqlCmd* pCmd);
 
 static int32_t setObjFullName(char* fullName, const char* account, SStrToken* pDB, SStrToken* tableName, int32_t* len);
 
@@ -70,7 +70,10 @@ static void getColumnName(tSQLExprItem* pItem, char* resultFieldName, int32_t na
 static int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExprItem* pItem, bool finalResult);
 static int32_t insertResultField(SQueryInfo* pQueryInfo, int32_t outputIndex, SColumnList* pIdList, int16_t bytes,
                                  int8_t type, char* fieldName, SSqlExpr* pSqlExpr);
-static int32_t changeFunctionID(int32_t optr, int16_t* functionId);
+
+static int32_t convertFunctionId(int32_t optr, int16_t* functionId);
+static uint8_t convertOptr(SStrToken *pToken);
+
 static int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSelection, bool isSTable, bool joinQuery);
 
 static bool validateIpAddress(const char* ip, size_t size);
@@ -78,7 +81,7 @@ static bool hasUnsupportFunctionsForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQuer
 static bool functionCompatibleCheck(SQueryInfo* pQueryInfo, bool joinQuery);
 static void setColumnOffsetValueInResultset(SQueryInfo* pQueryInfo);
 
-static int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd* pCmd);
+static int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, SArray* pList, SSqlCmd* pCmd);
 
 static int32_t parseIntervalClause(SSqlObj* pSql, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql);
 static int32_t parseOffsetClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql);
@@ -124,6 +127,45 @@ static int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo);
 static int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index);
 static int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSQLExpr* pSqlExpr, SQueryInfo* pQueryInfo, SArray* pCols, int64_t *uid);
 
+static uint8_t convertOptr(SStrToken *pToken) {
+  switch (pToken->type) {
+    case TK_LT:
+      return TSDB_RELATION_LESS;
+    case TK_LE:
+      return TSDB_RELATION_LESS_EQUAL;
+    case TK_GT:
+      return TSDB_RELATION_GREATER;
+    case TK_GE:
+      return TSDB_RELATION_GREATER_EQUAL;
+    case TK_NE:
+      return TSDB_RELATION_NOT_EQUAL;
+    case TK_AND:
+      return TSDB_RELATION_AND;
+    case TK_OR:
+      return TSDB_RELATION_OR;
+    case TK_EQ:
+      return TSDB_RELATION_EQUAL;
+    case TK_PLUS:
+      return TSDB_BINARY_OP_ADD;
+    case TK_MINUS:
+      return TSDB_BINARY_OP_SUBTRACT;
+    case TK_STAR:
+      return TSDB_BINARY_OP_MULTIPLY;
+    case TK_SLASH:
+    case TK_DIVIDE:
+      return TSDB_BINARY_OP_DIVIDE;
+    case TK_REM:
+      return TSDB_BINARY_OP_REMAINDER;
+    case TK_LIKE:
+      return TSDB_RELATION_LIKE;
+    case TK_ISNULL:
+      return TSDB_RELATION_ISNULL;
+    case TK_NOTNULL:
+      return TSDB_RELATION_NOTNULL;
+    default: { return 0; }
+  }
+}
+
 /*
  * Used during parsing query sql. Since the query sql usually small in length, error position
  * is not needed in the final error message.
@@ -778,7 +820,7 @@ int32_t parseSlidingClause(SSqlObj* pSql, SQueryInfo* pQueryInfo, SQuerySQL* pQu
   const char* msg1 = "sliding value no larger than the interval value";
   const char* msg2 = "sliding value can not less than 1% of interval value";
   const char* msg3 = "does not support sliding when interval is natural month/year";
-  const char* msg4 = "sliding not support yet in ordinary query";
+//  const char* msg4 = "sliding not support yet in ordinary query";
 
   const static int32_t INTERVAL_SLIDING_FACTOR = 100;
   SSqlCmd* pCmd = &pSql->cmd;
@@ -814,9 +856,9 @@ int32_t parseSlidingClause(SSqlObj* pSql, SQueryInfo* pQueryInfo, SQuerySQL* pQu
     return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2);
   }
 
-  if (pQueryInfo->interval.sliding != pQueryInfo->interval.interval && pSql->pStream == NULL) {
-    return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4);
-  }
+//  if (pQueryInfo->interval.sliding != pQueryInfo->interval.interval && pSql->pStream == NULL) {
+//    return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4);
+//  }
 
   return TSDB_CODE_SUCCESS;
 }
@@ -864,7 +906,7 @@ int32_t tscSetTableFullName(STableMetaInfo* pTableMetaInfo, SStrToken* pzTableNa
   return TSDB_CODE_SUCCESS;
 }
 
-static bool validateTableColumnInfo(tFieldList* pFieldList, SSqlCmd* pCmd) {
+static bool validateTableColumnInfo(SArray* pFieldList, SSqlCmd* pCmd) {
   assert(pFieldList != NULL);
 
   const char* msg = "illegal number of columns";
@@ -876,20 +918,22 @@ static bool validateTableColumnInfo(tFieldList* pFieldList, SSqlCmd* pCmd) {
   const char* msg6 = "invalid column name";
 
   // number of fields no less than 2
-  if (pFieldList->nField <= 1 || pFieldList->nField > TSDB_MAX_COLUMNS) {
+  size_t numOfCols = taosArrayGetSize(pFieldList);
+  if (numOfCols <= 1 || numOfCols > TSDB_MAX_COLUMNS) {
     invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg);
     return false;
   }
 
   // first column must be timestamp
-  if (pFieldList->p[0].type != TSDB_DATA_TYPE_TIMESTAMP) {
+  TAOS_FIELD* pField = taosArrayGet(pFieldList, 0);
+  if (pField->type != TSDB_DATA_TYPE_TIMESTAMP) {
     invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1);
     return false;
   }
 
   int32_t nLen = 0;
-  for (int32_t i = 0; i < pFieldList->nField; ++i) {
-    TAOS_FIELD* pField = &pFieldList->p[i];
+  for (int32_t i = 0; i < numOfCols; ++i) {
+    pField = taosArrayGet(pFieldList, i);
 
     if (pField->bytes == 0) {
       invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5);
@@ -913,7 +957,7 @@ static bool validateTableColumnInfo(tFieldList* pFieldList, SSqlCmd* pCmd) {
     }
 
     // field name must be unique
-    if (has(pFieldList, i + 1, pFieldList->p[i].name) == true) {
+    if (has(pFieldList, i + 1, pField->name) == true) {
       invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3);
       return false;
     }
@@ -930,7 +974,7 @@ static bool validateTableColumnInfo(tFieldList* pFieldList, SSqlCmd* pCmd) {
   return true;
 }
 
-static bool validateTagParams(tFieldList* pTagsList, tFieldList* pFieldList, SSqlCmd* pCmd) {
+static bool validateTagParams(SArray* pTagsList, SArray* pFieldList, SSqlCmd* pCmd) {
   assert(pTagsList != NULL);
 
   const char* msg1 = "invalid number of tag columns";
@@ -942,18 +986,21 @@ static bool validateTagParams(tFieldList* pTagsList, tFieldList* pFieldList, SSq
   const char* msg7 = "invalid binary/nchar tag length";
 
   // number of fields at least 1
-  if (pTagsList->nField < 1 || pTagsList->nField > TSDB_MAX_TAGS) {
+  size_t numOfTags = taosArrayGetSize(pTagsList);
+  if (numOfTags < 1 || numOfTags > TSDB_MAX_TAGS) {
     invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1);
     return false;
   }
 
   int32_t nLen = 0;
-  for (int32_t i = 0; i < pTagsList->nField; ++i) {
-    if (pTagsList->p[i].bytes == 0) {
+  for (int32_t i = 0; i < numOfTags; ++i) {
+    TAOS_FIELD* p = taosArrayGet(pTagsList, i);
+    if (p->bytes == 0) {
       invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg7);
       return false;
     }
-    nLen += pTagsList->p[i].bytes;
+
+    nLen += p->bytes;
   }
 
   // max tag row length must be less than TSDB_MAX_TAGS_LEN
@@ -963,37 +1010,41 @@ static bool validateTagParams(tFieldList* pTagsList, tFieldList* pFieldList, SSq
   }
 
   // field name must be unique
-  for (int32_t i = 0; i < pTagsList->nField; ++i) {
-    if (has(pFieldList, 0, pTagsList->p[i].name) == true) {
+  for (int32_t i = 0; i < numOfTags; ++i) {
+    TAOS_FIELD* p = taosArrayGet(pTagsList, i);
+
+    if (has(pFieldList, 0, p->name) == true) {
       invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3);
       return false;
     }
   }
 
   /* timestamp in tag is not allowed */
-  for (int32_t i = 0; i < pTagsList->nField; ++i) {
-    if (pTagsList->p[i].type == TSDB_DATA_TYPE_TIMESTAMP) {
+  for (int32_t i = 0; i < numOfTags; ++i) {
+    TAOS_FIELD* p = taosArrayGet(pTagsList, i);
+
+    if (p->type == TSDB_DATA_TYPE_TIMESTAMP) {
       invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4);
       return false;
     }
 
-    if (pTagsList->p[i].type < TSDB_DATA_TYPE_BOOL || pTagsList->p[i].type > TSDB_DATA_TYPE_NCHAR) {
+    if (p->type < TSDB_DATA_TYPE_BOOL || p->type > TSDB_DATA_TYPE_NCHAR) {
       invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5);
       return false;
     }
 
-    if ((pTagsList->p[i].type == TSDB_DATA_TYPE_BINARY && pTagsList->p[i].bytes <= 0) ||
-        (pTagsList->p[i].type == TSDB_DATA_TYPE_NCHAR && pTagsList->p[i].bytes <= 0)) {
+    if ((p->type == TSDB_DATA_TYPE_BINARY && p->bytes <= 0) ||
+        (p->type == TSDB_DATA_TYPE_NCHAR && p->bytes <= 0)) {
       invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg7);
       return false;
     }
 
-    if (validateColumnName(pTagsList->p[i].name) != TSDB_CODE_SUCCESS) {
+    if (validateColumnName(p->name) != TSDB_CODE_SUCCESS) {
       invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6);
       return false;
     }
 
-    if (has(pTagsList, i + 1, pTagsList->p[i].name) == true) {
+    if (has(pTagsList, i + 1, p->name) == true) {
       invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3);
       return false;
     }
@@ -1140,9 +1191,10 @@ bool validateOneColumn(SSqlCmd* pCmd, TAOS_FIELD* pColField) {
 }
 
 /* is contained in pFieldList or not */
-static bool has(tFieldList* pFieldList, int32_t startIdx, const char* name) {
-  for (int32_t j = startIdx; j < pFieldList->nField; ++j) {
-    TAOS_FIELD* field = pFieldList->p + j;
+static bool has(SArray* pFieldList, int32_t startIdx, const char* name) {
+  size_t numOfCols = taosArrayGetSize(pFieldList);
+  for (int32_t j = startIdx; j < numOfCols; ++j) {
+    TAOS_FIELD* field = taosArrayGet(pFieldList, j);
     if (strncasecmp(name, field->name, sizeof(field->name) - 1) == 0) return true;
   }
 
@@ -1726,7 +1778,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
       }
 
       int16_t functionID = 0;
-      if (changeFunctionID(optr, &functionID) != TSDB_CODE_SUCCESS) {
+      if (convertFunctionId(optr, &functionID) != TSDB_CODE_SUCCESS) {
         return TSDB_CODE_TSC_INVALID_SQL;
       }
 
@@ -1862,7 +1914,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
       int32_t intermediateResSize = 0;
 
       int16_t functionID = 0;
-      if (changeFunctionID(optr, &functionID) != TSDB_CODE_SUCCESS) {
+      if (convertFunctionId(optr, &functionID) != TSDB_CODE_SUCCESS) {
         return TSDB_CODE_TSC_INVALID_SQL;
       }
 
@@ -1933,7 +1985,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
       bool requireAllFields = (pItem->pNode->pParam == NULL);
 
       int16_t functionID = 0;
-      if (changeFunctionID(optr, &functionID) != TSDB_CODE_SUCCESS) {
+      if (convertFunctionId(optr, &functionID) != TSDB_CODE_SUCCESS) {
         return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg9);
       }
 
@@ -2122,7 +2174,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
          * for dp = 100, it is max,
          */
         int16_t functionId = 0;
-        if (changeFunctionID(optr, &functionId) != TSDB_CODE_SUCCESS) {
+        if (convertFunctionId(optr, &functionId) != TSDB_CODE_SUCCESS) {
           return TSDB_CODE_TSC_INVALID_SQL;
         }
         tscInsertPrimaryTSSourceColumn(pQueryInfo, &index);
@@ -2139,7 +2191,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
         }
 
         int16_t functionId = 0;
-        if (changeFunctionID(optr, &functionId) != TSDB_CODE_SUCCESS) {
+        if (convertFunctionId(optr, &functionId) != TSDB_CODE_SUCCESS) {
           return TSDB_CODE_TSC_INVALID_SQL;
         }
 
@@ -2398,7 +2450,7 @@ int32_t getColumnIndexByName(SSqlCmd* pCmd, const SStrToken* pToken, SQueryInfo*
   return doGetColumnIndexByName(pCmd, &tmpToken, pQueryInfo, pIndex);
 }
 
-int32_t changeFunctionID(int32_t optr, int16_t* functionId) {
+int32_t convertFunctionId(int32_t optr, int16_t* functionId) {
   switch (optr) {
     case TK_COUNT:
       *functionId = TSDB_FUNC_COUNT;
@@ -2717,6 +2769,9 @@ bool hasUnsupportFunctionsForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo)
 static bool functionCompatibleCheck(SQueryInfo* pQueryInfo, bool joinQuery) {
   int32_t startIdx = 0;
 
+  size_t numOfExpr = tscSqlExprNumOfExprs(pQueryInfo);
+  assert(numOfExpr > 0);
+
   SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, startIdx);
   int32_t functionID = pExpr->functionId;
 
@@ -2755,7 +2810,7 @@ static bool functionCompatibleCheck(SQueryInfo* pQueryInfo, bool joinQuery) {
   return true;
 }
 
-int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd* pCmd) {
+int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, SArray* pList, SSqlCmd* pCmd) {
   const char* msg1 = "too many columns in group by clause";
   const char* msg2 = "invalid column name in group by clause";
   const char* msg3 = "columns from one table allowed as group by columns";
@@ -2775,8 +2830,8 @@ int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd*
     pQueryInfo->colList = taosArrayInit(4, POINTER_BYTES);
   }
   
-  pQueryInfo->groupbyExpr.numOfGroupCols = pList->nExpr;
-  if (pList->nExpr > TSDB_MAX_TAGS) {
+  pQueryInfo->groupbyExpr.numOfGroupCols = (int16_t)taosArrayGetSize(pList);
+  if (pQueryInfo->groupbyExpr.numOfGroupCols > TSDB_MAX_TAGS) {
     return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1);
   }
 
@@ -2790,8 +2845,11 @@ int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd*
 
   int32_t tableIndex = COLUMN_INDEX_INITIAL_VAL;
 
-  for (int32_t i = 0; i < pList->nExpr; ++i) {
-    tVariant* pVar = &pList->a[i].pVar;
+  size_t num = taosArrayGetSize(pList);
+  for (int32_t i = 0; i < num; ++i) {
+    tVariantListItem * pItem = taosArrayGet(pList, i);
+    tVariant* pVar = &pItem->pVar;
+
     SStrToken token = {pVar->nLen, pVar->nType, pVar->pz};
 
     SColumnIndex index = COLUMN_INDEX_INITIALIZER;
@@ -2852,7 +2910,7 @@ int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd*
       taosArrayPush(pGroupExpr->columnInfo, &colIndex);
       pQueryInfo->groupbyExpr.orderType = TSDB_ORDER_ASC;
 
-      if (i == 0 && pList->nExpr > 1) {
+      if (i == 0 && num > 1) {
         return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg7);
       }
     }
@@ -4366,8 +4424,8 @@ int32_t tsRewriteFieldNameIfNecessary(SSqlCmd* pCmd, SQueryInfo* pQueryInfo) {
 }
 
 int32_t parseFillClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySQL) {
-  tVariantList*     pFillToken = pQuerySQL->fillType;
-  tVariantListItem* pItem = &pFillToken->a[0];
+  SArray*     pFillToken = pQuerySQL->fillType;
+  tVariantListItem* pItem = taosArrayGet(pFillToken, 0);
 
   const int32_t START_INTERPO_COL_IDX = 1;
 
@@ -4407,12 +4465,13 @@ int32_t parseFillClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuery
   } else if (strncasecmp(pItem->pVar.pz, "value", 5) == 0 && pItem->pVar.nLen == 5) {
     pQueryInfo->fillType = TSDB_FILL_SET_VALUE;
 
-    if (pFillToken->nExpr == 1) {
+    size_t num = taosArrayGetSize(pFillToken);
+    if (num == 1) {
       return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1);
     }
 
     int32_t startPos = 1;
-    int32_t numOfFillVal = pFillToken->nExpr - 1;
+    int32_t numOfFillVal = (int32_t)(num - 1);
 
     /* for point interpolation query, we do not have the timestamp column */
     if (tscIsPointInterpQuery(pQueryInfo)) {
@@ -4422,7 +4481,7 @@ int32_t parseFillClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuery
         numOfFillVal = (int32_t)size;
       }
     } else {
-      numOfFillVal = (pFillToken->nExpr >  (int32_t)size) ? (int32_t)size : pFillToken->nExpr;
+      numOfFillVal = (int16_t)((num >  (int32_t)size) ? (int32_t)size : num);
     }
 
     int32_t j = 1;
@@ -4435,14 +4494,15 @@ int32_t parseFillClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuery
         continue;
       }
 
-      int32_t ret = tVariantDump(&pFillToken->a[j].pVar, (char*)&pQueryInfo->fillVal[i], pFields->type, true);
+      tVariant* p = taosArrayGet(pFillToken, j);
+      int32_t ret = tVariantDump(p, (char*)&pQueryInfo->fillVal[i], pFields->type, true);
       if (ret != TSDB_CODE_SUCCESS) {
         return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg);
       }
     }
     
-    if ((pFillToken->nExpr < size) || ((pFillToken->nExpr - 1 < size) && (tscIsPointInterpQuery(pQueryInfo)))) {
-      tVariantListItem* lastItem = &pFillToken->a[pFillToken->nExpr - 1];
+    if ((num < size) || ((num - 1 < size) && (tscIsPointInterpQuery(pQueryInfo)))) {
+      tVariantListItem* lastItem = taosArrayGetLast(pFillToken);
 
       for (int32_t i = numOfFillVal; i < size; ++i) {
         TAOS_FIELD* pFields = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i);
@@ -4491,7 +4551,7 @@ int32_t parseOrderbyClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQu
     return TSDB_CODE_SUCCESS;
   }
 
-  tVariantList* pSortorder = pQuerySql->pSortOrder;
+  SArray* pSortorder = pQuerySql->pSortOrder;
 
   /*
    * for table query, there is only one or none order option is allowed, which is the
@@ -4499,18 +4559,19 @@ int32_t parseOrderbyClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQu
    *
    * for super table query, the order option must be less than 3.
    */
+  size_t size = taosArrayGetSize(pSortorder);
   if (UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo)) {
-    if (pSortorder->nExpr > 1) {
+    if (size > 1) {
       return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg0);
     }
   } else {
-    if (pSortorder->nExpr > 2) {
+    if (size > 2) {
       return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3);
     }
   }
 
   // handle the first part of order by
-  tVariant* pVar = &pSortorder->a[0].pVar;
+  tVariant* pVar = taosArrayGet(pSortorder, 0);
 
   // e.g., order by 1 asc, return directly with out further check.
   if (pVar->nType >= TSDB_DATA_TYPE_TINYINT && pVar->nType <= TSDB_DATA_TYPE_BIGINT) {
@@ -4553,10 +4614,13 @@ int32_t parseOrderbyClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQu
       assert(!(orderByTags && orderByTS));
     }
 
-    if (pSortorder->nExpr == 1) {
+    size_t s = taosArrayGetSize(pSortorder);
+    if (s == 1) {
       if (orderByTags) {
         pQueryInfo->groupbyExpr.orderIndex = index.columnIndex - tscGetNumOfColumns(pTableMetaInfo->pTableMeta);
-        pQueryInfo->groupbyExpr.orderType = pQuerySql->pSortOrder->a[0].sortOrder;
+
+        tVariantListItem* p1 = taosArrayGet(pQuerySql->pSortOrder, 0);
+        pQueryInfo->groupbyExpr.orderType = p1->sortOrder;
       } else if (isTopBottomQuery(pQueryInfo)) {
         /* order of top/bottom query in interval is not valid  */
         SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, 0);
@@ -4567,11 +4631,14 @@ int32_t parseOrderbyClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQu
           return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2);
         }
 
-        pQueryInfo->order.order = pQuerySql->pSortOrder->a[0].sortOrder;
+        tVariantListItem* p1 = taosArrayGet(pQuerySql->pSortOrder, 0);
+        pQueryInfo->order.order = p1->sortOrder;
         pQueryInfo->order.orderColId = pSchema[index.columnIndex].colId;
         return TSDB_CODE_SUCCESS;
       } else {
-        pQueryInfo->order.order = pSortorder->a[0].sortOrder;
+        tVariantListItem* p1 = taosArrayGet(pQuerySql->pSortOrder, 0);
+
+        pQueryInfo->order.order = p1->sortOrder;
         pQueryInfo->order.orderColId = PRIMARYKEY_TIMESTAMP_COL_INDEX;
 
         // orderby ts query on super table
@@ -4581,16 +4648,18 @@ int32_t parseOrderbyClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQu
       }
     }
 
-    if (pSortorder->nExpr == 2) {
+    if (s == 2) {
+      tVariantListItem *pItem = taosArrayGet(pQuerySql->pSortOrder, 0);
       if (orderByTags) {
         pQueryInfo->groupbyExpr.orderIndex = index.columnIndex - tscGetNumOfColumns(pTableMetaInfo->pTableMeta);
-        pQueryInfo->groupbyExpr.orderType = pQuerySql->pSortOrder->a[0].sortOrder;
+        pQueryInfo->groupbyExpr.orderType = pItem->sortOrder;
       } else {
-        pQueryInfo->order.order = pSortorder->a[0].sortOrder;
+        pQueryInfo->order.order = pItem->sortOrder;
         pQueryInfo->order.orderColId = PRIMARYKEY_TIMESTAMP_COL_INDEX;
       }
 
-      tVariant* pVar2 = &pSortorder->a[1].pVar;
+      pItem = taosArrayGet(pQuerySql->pSortOrder, 1);
+      tVariant* pVar2 = &pItem->pVar;
       SStrToken cname = {pVar2->nLen, pVar2->nType, pVar2->pz};
       if (getColumnIndexByName(pCmd, &cname, pQueryInfo, &index) != TSDB_CODE_SUCCESS) {
         return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1);
@@ -4599,7 +4668,8 @@ int32_t parseOrderbyClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQu
       if (index.columnIndex != PRIMARYKEY_TIMESTAMP_COL_INDEX) {
         return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2);
       } else {
-        pQueryInfo->order.order = pSortorder->a[1].sortOrder;
+        tVariantListItem* p1 = taosArrayGet(pSortorder, 1);
+        pQueryInfo->order.order = p1->sortOrder;
         pQueryInfo->order.orderColId = PRIMARYKEY_TIMESTAMP_COL_INDEX;
       }
     }
@@ -4623,12 +4693,14 @@ int32_t parseOrderbyClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQu
         return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2);
       }
 
-      pQueryInfo->order.order = pQuerySql->pSortOrder->a[0].sortOrder;
+      tVariantListItem* pItem = taosArrayGet(pQuerySql->pSortOrder, 0);
+      pQueryInfo->order.order = pItem->sortOrder;
       pQueryInfo->order.orderColId = pSchema[index.columnIndex].colId;
       return TSDB_CODE_SUCCESS;
     }
 
-    pQueryInfo->order.order = pQuerySql->pSortOrder->a[0].sortOrder;
+    tVariantListItem* pItem = taosArrayGet(pQuerySql->pSortOrder, 0);
+    pQueryInfo->order.order = pItem->sortOrder;
   }
 
   return TSDB_CODE_SUCCESS;
@@ -4696,27 +4768,28 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
   }
 
   if (pAlterSQL->type == TSDB_ALTER_TABLE_ADD_TAG_COLUMN) {
-    tFieldList* pFieldList = pAlterSQL->pAddColumns;
-    if (pFieldList->nField > 1) {
+    SArray* pFieldList = pAlterSQL->pAddColumns;
+    if (taosArrayGetSize(pFieldList) > 1) {
       return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5);
     }
 
-    if (!validateOneTags(pCmd, &pFieldList->p[0])) {
+    TAOS_FIELD* p = taosArrayGet(pFieldList, 0);
+    if (!validateOneTags(pCmd, p)) {
       return TSDB_CODE_TSC_INVALID_SQL;
     }
   
-    tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &pFieldList->p[0]);
+    tscFieldInfoAppend(&pQueryInfo->fieldsInfo, p);
   } else if (pAlterSQL->type == TSDB_ALTER_TABLE_DROP_TAG_COLUMN) {
     if (tscGetNumOfTags(pTableMeta) == 1) {
       return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg7);
     }
 
     // numOfTags == 1
-    if (pAlterSQL->varList->nExpr > 1) {
+    if (taosArrayGetSize(pAlterSQL->varList) > 1) {
       return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg8);
     }
 
-    tVariantListItem* pItem = &pAlterSQL->varList->a[0];
+    tVariantListItem* pItem = taosArrayGet(pAlterSQL->varList, 0);
     if (pItem->pVar.nLen >= TSDB_COL_NAME_LEN) {
       return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg9);
     }
@@ -4741,13 +4814,13 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
     TAOS_FIELD f = tscCreateField(TSDB_DATA_TYPE_INT, name1, tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize);
     tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f);
   } else if (pAlterSQL->type == TSDB_ALTER_TABLE_CHANGE_TAG_COLUMN) {
-    tVariantList* pVarList = pAlterSQL->varList;
-    if (pVarList->nExpr > 2) {
+    SArray* pVarList = pAlterSQL->varList;
+    if (taosArrayGetSize(pVarList) > 2) {
       return TSDB_CODE_TSC_INVALID_SQL;
     }
 
-    tVariantListItem* pSrcItem = &pAlterSQL->varList->a[0];
-    tVariantListItem* pDstItem = &pAlterSQL->varList->a[1];
+    tVariantListItem* pSrcItem = taosArrayGet(pAlterSQL->varList, 0);
+    tVariantListItem* pDstItem = taosArrayGet(pAlterSQL->varList, 1);
 
     if (pSrcItem->pVar.nLen >= TSDB_COL_NAME_LEN || pDstItem->pVar.nLen >= TSDB_COL_NAME_LEN) {
       return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg9);
@@ -4770,13 +4843,17 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
       return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg19);
     }
 
+    tVariantListItem* pItem = taosArrayGet(pVarList, 0);
+
     char name[TSDB_COL_NAME_LEN] = {0};
-    strncpy(name, pVarList->a[0].pVar.pz, pVarList->a[0].pVar.nLen);
+    strncpy(name, pItem->pVar.pz, pItem->pVar.nLen);
     TAOS_FIELD f = tscCreateField(TSDB_DATA_TYPE_INT, name, tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize);
     tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f);
 
+    pItem = taosArrayGet(pVarList, 1);
     memset(name, 0, tListLen(name));
-    strncpy(name, pVarList->a[1].pVar.pz, pVarList->a[1].pVar.nLen);
+
+    strncpy(name, pItem->pVar.pz, pItem->pVar.nLen);
     f = tscCreateField(TSDB_DATA_TYPE_INT, name, tDataTypeDesc[TSDB_DATA_TYPE_INT].nSize);
     tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f);
   } else if (pAlterSQL->type == TSDB_ALTER_TABLE_UPDATE_TAG_VAL) {
@@ -4784,12 +4861,12 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
     // the following is used to handle tags value for table created according to super table
     pCmd->command = TSDB_SQL_UPDATE_TAGS_VAL;
     
-    tVariantList* pVarList = pAlterSQL->varList;
-    tVariant*     pTagName = &pVarList->a[0].pVar;
+    SArray* pVarList = pAlterSQL->varList;
+    tVariantListItem* item = taosArrayGet(pVarList, 0);
     int16_t       numOfTags = tscGetNumOfTags(pTableMeta);
 
     SColumnIndex columnIndex = COLUMN_INDEX_INITIALIZER;
-    SStrToken    name = {.type = TK_STRING, .z = pTagName->pz, .n = pTagName->nLen};
+    SStrToken    name = {.type = TK_STRING, .z = item->pVar.pz, .n = item->pVar.nLen};
     if (getColumnIndexByName(pCmd, &name, pQueryInfo, &columnIndex) != TSDB_CODE_SUCCESS) {
       return TSDB_CODE_TSC_INVALID_SQL;
     }
@@ -4798,8 +4875,9 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
       return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg12);
     }
 
+    tVariantListItem* pItem = taosArrayGet(pVarList, 1);
     SSchema* pTagsSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, columnIndex.columnIndex);
-    if (tVariantDump(&pVarList->a[1].pVar, pAlterSQL->tagData.data, pTagsSchema->type, true) != TSDB_CODE_SUCCESS) {
+    if (tVariantDump(&pItem->pVar, pAlterSQL->tagData.data, pTagsSchema->type, true) != TSDB_CODE_SUCCESS) {
       return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg13);
     }
     
@@ -4844,7 +4922,8 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
     }
 
     // copy the tag value to msg body
-    tVariantDump(&pVarList->a[1].pVar, pUpdateMsg->data + schemaLen, pTagsSchema->type, true);
+    pItem = taosArrayGet(pVarList, 1);
+    tVariantDump(&pItem->pVar, pUpdateMsg->data + schemaLen, pTagsSchema->type, true);
     
     int32_t len = 0;
     if (pTagsSchema->type != TSDB_DATA_TYPE_BINARY && pTagsSchema->type != TSDB_DATA_TYPE_NCHAR) {
@@ -4859,27 +4938,29 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
     pUpdateMsg->head.contLen = htonl(total);
     
   } else if (pAlterSQL->type == TSDB_ALTER_TABLE_ADD_COLUMN) {
-    tFieldList* pFieldList = pAlterSQL->pAddColumns;
-    if (pFieldList->nField > 1) {
+    SArray* pFieldList = pAlterSQL->pAddColumns;
+    if (taosArrayGetSize(pFieldList) > 1) {
       const char* msg = "only support add one column";
       return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg);
     }
 
-    if (!validateOneColumn(pCmd, &pFieldList->p[0])) {
+    TAOS_FIELD* p = taosArrayGet(pFieldList, 0);
+    if (!validateOneColumn(pCmd, p)) {
       return TSDB_CODE_TSC_INVALID_SQL;
     }
   
-    tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &pFieldList->p[0]);
+    tscFieldInfoAppend(&pQueryInfo->fieldsInfo, p);
   } else if (pAlterSQL->type == TSDB_ALTER_TABLE_DROP_COLUMN) {
     if (tscGetNumOfColumns(pTableMeta) == TSDB_MIN_COLUMNS) {  //
       return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg15);
     }
 
-    if (pAlterSQL->varList->nExpr > 1) {
+    size_t size = taosArrayGetSize(pAlterSQL->varList);
+    if (size > 1) {
       return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg16);
     }
 
-    tVariantListItem* pItem = &pAlterSQL->varList->a[0];
+    tVariantListItem* pItem = taosArrayGet(pAlterSQL->varList, 0);
 
     SColumnIndex columnIndex = COLUMN_INDEX_INITIALIZER;
     SStrToken    name = {.type = TK_STRING, .z = pItem->pVar.pz, .n = pItem->pVar.nLen};
@@ -5250,21 +5331,28 @@ static int32_t setKeepOption(SSqlCmd* pCmd, SCreateDbMsg* pMsg, SCreateDBInfo* p
   pMsg->daysToKeep1 = htonl(-1);
   pMsg->daysToKeep2 = htonl(-1);
 
-  tVariantList* pKeep = pCreateDb->keep;
+  SArray* pKeep = pCreateDb->keep;
   if (pKeep != NULL) {
-    switch (pKeep->nExpr) {
-      case 1:
-        pMsg->daysToKeep = htonl((int32_t)pKeep->a[0].pVar.i64Key);
+    size_t s = taosArrayGetSize(pKeep);
+    tVariantListItem* p0 = taosArrayGet(pKeep, 0);
+    switch (s) {
+      case 1: {
+        pMsg->daysToKeep = htonl((int32_t)p0->pVar.i64Key);
+      }
         break;
       case 2: {
-        pMsg->daysToKeep = htonl((int32_t)pKeep->a[0].pVar.i64Key);
-        pMsg->daysToKeep1 = htonl((int32_t)pKeep->a[1].pVar.i64Key);
+        tVariantListItem* p1 = taosArrayGet(pKeep, 1);
+        pMsg->daysToKeep = htonl((int32_t)p0->pVar.i64Key);
+        pMsg->daysToKeep1 = htonl((int32_t)p1->pVar.i64Key);
         break;
       }
       case 3: {
-        pMsg->daysToKeep = htonl((int32_t)pKeep->a[0].pVar.i64Key);
-        pMsg->daysToKeep1 = htonl((int32_t)pKeep->a[1].pVar.i64Key);
-        pMsg->daysToKeep2 = htonl((int32_t)pKeep->a[2].pVar.i64Key);
+        tVariantListItem* p1 = taosArrayGet(pKeep, 1);
+        tVariantListItem* p2 = taosArrayGet(pKeep, 2);
+
+        pMsg->daysToKeep = htonl((int32_t)p0->pVar.i64Key);
+        pMsg->daysToKeep1 = htonl((int32_t)p1->pVar.i64Key);
+        pMsg->daysToKeep2 = htonl((int32_t)p2->pVar.i64Key);
         break;
       }
       default: { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); }
@@ -5951,8 +6039,8 @@ int32_t doCheckForCreateTable(SSqlObj* pSql, int32_t subClauseIndex, SSqlInfo* p
 
   SCreateTableSQL* pCreateTable = pInfo->pCreateTableInfo;
 
-  tFieldList* pFieldList = pCreateTable->colInfo.pColumns;
-  tFieldList* pTagList = pCreateTable->colInfo.pTagColumns;
+  SArray* pFieldList = pCreateTable->colInfo.pColumns;
+  SArray* pTagList = pCreateTable->colInfo.pTagColumns;
 
   assert(pFieldList != NULL);
 
@@ -5974,18 +6062,23 @@ int32_t doCheckForCreateTable(SSqlObj* pSql, int32_t subClauseIndex, SSqlInfo* p
   }
 
   int32_t col = 0;
-  for (; col < pFieldList->nField; ++col) {
-    tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &pFieldList->p[col]);
+  size_t numOfFields = taosArrayGetSize(pFieldList);
+
+  for (; col < numOfFields; ++col) {
+    TAOS_FIELD* p = taosArrayGet(pFieldList, col);
+    tscFieldInfoAppend(&pQueryInfo->fieldsInfo, p);
   }
 
-  pCmd->numOfCols = (int16_t)pFieldList->nField;
+  pCmd->numOfCols = (int16_t)numOfFields;
 
   if (pTagList != NULL) {  // create super table[optional]
-    for (int32_t i = 0; i < pTagList->nField; ++i) {
-      tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &pTagList->p[i]);
+    size_t numOfTags = taosArrayGetSize(pTagList);
+    for (int32_t i = 0; i < numOfTags; ++i) {
+      TAOS_FIELD* p = taosArrayGet(pTagList, i);
+      tscFieldInfoAppend(&pQueryInfo->fieldsInfo, p);
     }
 
-    pCmd->count = pTagList->nField;
+    pCmd->count =(int32_t) numOfTags;
   }
 
   return TSDB_CODE_SUCCESS;
@@ -6026,14 +6119,15 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) {
 
   // get meter meta from mnode
   tstrncpy(pCreateTable->usingInfo.tagdata.name, pStableMeterMetaInfo->name, sizeof(pCreateTable->usingInfo.tagdata.name));
-  tVariantList* pList = pInfo->pCreateTableInfo->usingInfo.pTagVals;
+  SArray* pList = pInfo->pCreateTableInfo->usingInfo.pTagVals;
 
   code = tscGetTableMeta(pSql, pStableMeterMetaInfo);
   if (code != TSDB_CODE_SUCCESS) {
     return code;
   }
 
-  if (tscGetNumOfTags(pStableMeterMetaInfo->pTableMeta) != pList->nExpr) {
+  size_t size = taosArrayGetSize(pList);
+  if (tscGetNumOfTags(pStableMeterMetaInfo->pTableMeta) != size) {
     return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5);
   }
 
@@ -6047,18 +6141,19 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) {
   }
 
   int32_t ret = TSDB_CODE_SUCCESS;
-  for (int32_t i = 0; i < pList->nExpr; ++i) {
+  for (int32_t i = 0; i < size; ++i) {
     SSchema* pSchema = &pTagSchema[i];
+    tVariantListItem* pItem = taosArrayGet(pList, i);
 
     char tagVal[TSDB_MAX_TAGS_LEN];
     if (pSchema->type == TSDB_DATA_TYPE_BINARY || pSchema->type == TSDB_DATA_TYPE_NCHAR) {
-      if (pList->a[i].pVar.nLen > pSchema->bytes) {
+      if (pItem->pVar.nLen > pSchema->bytes) {
         tdDestroyKVRowBuilder(&kvRowBuilder);
         return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3);
       }
     }
 
-    ret = tVariantDump(&(pList->a[i].pVar), tagVal, pSchema->type, true);
+    ret = tVariantDump(&(pItem->pVar), tagVal, pSchema->type, true);
 
     // check again after the convert since it may be converted from binary to nchar.
     if (pSchema->type == TSDB_DATA_TYPE_BINARY || pSchema->type == TSDB_DATA_TYPE_NCHAR) {
@@ -6125,13 +6220,13 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) {
     return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1);
   }
   
-  tVariantList* pSrcMeterName = pInfo->pCreateTableInfo->pSelect->from;
-  if (pSrcMeterName == NULL || pSrcMeterName->nExpr == 0) {
+  SArray* pSrcMeterName = pInfo->pCreateTableInfo->pSelect->from;
+  if (pSrcMeterName == NULL || taosArrayGetSize(pSrcMeterName) == 0) {
     return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6);
   }
   
-  tVariant* pVar = &pSrcMeterName->a[0].pVar;
-  SStrToken srcToken = {.z = pVar->pz, .n = pVar->nLen, .type = TK_STRING};
+  tVariantListItem* p1 = taosArrayGet(pSrcMeterName, 0);
+  SStrToken srcToken = {.z = p1->pVar.pz, .n = p1->pVar.nLen, .type = TK_STRING};
   if (tscValidateName(&srcToken) != TSDB_CODE_SUCCESS) {
     return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1);
   }
@@ -6196,7 +6291,7 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) {
       return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3);
     }
 
-    tVariantListItem* pItem = &pQuerySql->fillType->a[0];
+    tVariantListItem* pItem = taosArrayGet(pQuerySql->fillType, 0);
     if (pItem->pVar.nType == TSDB_DATA_TYPE_BINARY) {
       if (!((strncmp(pItem->pVar.pz, "none", 4) == 0 && pItem->pVar.nLen == 4) ||
             (strncmp(pItem->pVar.pz, "null", 4) == 0 && pItem->pVar.nLen == 4))) {
@@ -6211,7 +6306,7 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) {
 }
 
 int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) {
-  assert(pQuerySql != NULL && (pQuerySql->from == NULL || pQuerySql->from->nExpr > 0));
+  assert(pQuerySql != NULL && (pQuerySql->from == NULL || taosArrayGetSize(pQuerySql->from) > 0));
 
   const char* msg0  = "invalid table name";
   const char* msg2  = "point interpolation query needs timestamp";
@@ -6253,19 +6348,21 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) {
     return doLocalQueryProcess(pCmd, pQueryInfo, pQuerySql);
   }
 
-  if (pQuerySql->from->nExpr > TSDB_MAX_JOIN_TABLE_NUM * 2) {
+  size_t fromSize = taosArrayGetSize(pQuerySql->from);
+  if (fromSize > TSDB_MAX_JOIN_TABLE_NUM * 2) {
     return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg7);
   }
 
   pQueryInfo->command = TSDB_SQL_SELECT;
 
-  if (pQuerySql->from->nExpr > 4) {
+  if (fromSize > 4) {
     return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg10);
   }
 
   // set all query tables, which are maybe more than one.
-  for (int32_t i = 0; i < pQuerySql->from->nExpr; ) {
-    tVariant* pTableItem = &pQuerySql->from->a[i].pVar;
+  for (int32_t i = 0; i < fromSize; ) {
+    tVariantListItem* item = taosArrayGet(pQuerySql->from, i);
+    tVariant* pTableItem = &item->pVar;
 
     if (pTableItem->nType != TSDB_DATA_TYPE_BINARY) {
       return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg0);
@@ -6290,21 +6387,21 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) {
       return code;
     }
 
-    tVariant* pTableItem1 = &pQuerySql->from->a[i + 1].pVar;
-    if (pTableItem1->nType != TSDB_DATA_TYPE_BINARY) {
+    tVariantListItem* p1 = taosArrayGet(pQuerySql->from, i);
+    if (p1->pVar.nType != TSDB_DATA_TYPE_BINARY) {
       return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg11);
     }
 
-    SStrToken aliasName = {.z = pTableItem1->pz, .n = pTableItem1->nLen, .type = TK_STRING};
+    SStrToken aliasName = {.z = p1->pVar.pz, .n = p1->pVar.nLen, .type = TK_STRING};
     if (tscValidateName(&aliasName) != TSDB_CODE_SUCCESS) {
       return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg11);
     }
 
     // has no table alias name
-    if (memcmp(pTableItem->pz, pTableItem1->pz, pTableItem1->nLen) == 0) {
+    if (memcmp(pTableItem->pz, p1->pVar.pz, p1->pVar.nLen) == 0) {
       extractTableName(pTableMetaInfo1->name, pTableMetaInfo1->aliasName);
     } else {
-      tstrncpy(pTableMetaInfo1->aliasName, pTableItem1->pz, sizeof(pTableMetaInfo1->aliasName));
+      tstrncpy(pTableMetaInfo1->aliasName, p1->pVar.pz, sizeof(pTableMetaInfo1->aliasName));
     }
 
     code = tscGetTableMeta(pSql, pTableMetaInfo1);
@@ -6315,7 +6412,7 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) {
     i += 2;
   }
 
-  assert(pQueryInfo->numOfTables == pQuerySql->from->nExpr / 2);
+  assert(pQueryInfo->numOfTables == taosArrayGetSize(pQuerySql->from) / 2);
   bool isSTable = false;
   
   if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
@@ -6349,12 +6446,12 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) {
       pQueryInfo->window.ekey = pQueryInfo->window.ekey / 1000;
     }
   } else {  // set the time rang
-    if (pQuerySql->from->nExpr > 2) { // it is a join query, no wher clause is not allowed.
+    if (taosArrayGetSize(pQuerySql->from) > 2) { // it is a join query, no wher clause is not allowed.
       return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), "condition missing for join query ");
     }
   }
 
-  int32_t joinQuery = (pQuerySql->from != NULL && pQuerySql->from->nExpr > 2);
+  int32_t joinQuery = (pQuerySql->from != NULL && taosArrayGetSize(pQuerySql->from) > 2);
 
   if (parseSelectClause(pCmd, index, pQuerySql->pSelection, isSTable, joinQuery) != TSDB_CODE_SUCCESS) {
     return TSDB_CODE_TSC_INVALID_SQL;
@@ -6537,7 +6634,7 @@ int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSQLExpr* pS
     (*pExpr)->_node.pRight = pRight;
     
     SStrToken t = {.type = pSqlExpr->nSQLOptr};
-    (*pExpr)->_node.optr = getBinaryExprOptr(&t);
+    (*pExpr)->_node.optr = convertOptr(&t);
     
     assert((*pExpr)->_node.optr != 0);
 

src/client/src/tscSchemaUtil.c

@@ -130,16 +130,6 @@ SSchema* tscGetColumnSchemaById(STableMeta* pTableMeta, int16_t colId) {
   return NULL;
 }
 
-struct SSchema tscGetTbnameColumnSchema() {
-  struct SSchema s = {
-      .colId = TSDB_TBNAME_COLUMN_INDEX,
-      .type  = TSDB_DATA_TYPE_BINARY,
-      .bytes = TSDB_TABLE_NAME_LEN
-  };
-  
-  strcpy(s.name, TSQL_TBNAME_L);
-  return s;
-}
 static void tscInitCorVgroupInfo(SCorVgroupInfo *corVgroupInfo, SVgroupInfo *vgroupInfo) {
   corVgroupInfo->version = 0;
   corVgroupInfo->inUse = 0;

src/client/src/tscServer.c

@@ -241,11 +241,7 @@ int tscSendMsgToServer(SSqlObj *pSql) {
       .code    = 0
   };
 
-  // NOTE: the rpc context should be acquired before sending data to server.
-  // Otherwise, the pSql object may have been released already during the response function, which is
-  // processMsgFromServer function. In the meanwhile, the assignment of the rpc context to sql object will absolutely
-  // cause crash.
-  pSql->rpcRid = rpcSendRequest(pObj->pDnodeConn, &pSql->epSet, &rpcMsg);
+  rpcSendRequest(pObj->pDnodeConn, &pSql->epSet, &rpcMsg, &pSql->rpcRid);
   return TSDB_CODE_SUCCESS;
 }
 
@@ -361,7 +357,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) {
         memcpy(pRes->pRsp, rpcMsg->pCont, pRes->rspLen);
       }
     } else {
-      pRes->pRsp = NULL;
+      tfree(pRes->pRsp);
     }
 
     /*
@@ -892,7 +888,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
 
   int32_t msgLen = (int32_t)(pMsg - pCmd->payload);
 
-  tscDebug("%p msg built success,len:%d bytes", pSql, msgLen);
+  tscDebug("%p msg built success, len:%d bytes", pSql, msgLen);
   pCmd->payloadLen = msgLen;
   pSql->cmd.msgType = TSDB_MSG_TYPE_QUERY;
   

src/client/src/tscSystem.c

@@ -105,6 +105,7 @@ void taos_init_imp(void) {
     taosReadGlobalCfg();
     taosCheckGlobalCfg();
 
+    rpcInit();
     tscDebug("starting to initialize TAOS client ...");
     tscDebug("Local End Point is:%s", tsLocalEp);
   }
@@ -179,6 +180,7 @@ void taos_cleanup(void) {
   taosCloseRef(tscRefId);
   taosCleanupKeywordsTable();
   taosCloseLog();
+  if (tscEmbedded == 0) rpcCleanup();
 
   m = tscTmr;
   if (m != NULL && atomic_val_compare_exchange_ptr(&tscTmr, m, 0) == m) {

src/client/src/tscUtil.c

@@ -71,7 +71,8 @@ void tsSetSTableQueryCond(STagCond* pTagCond, uint64_t uid, SBufferWriter* bw) {
 }
 
 bool tscQueryTags(SQueryInfo* pQueryInfo) {
-  for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) {
+  int32_t numOfCols = (int32_t) tscSqlExprNumOfExprs(pQueryInfo);
+  for (int32_t i = 0; i < numOfCols; ++i) {
     SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i);
     int32_t functId = pExpr->functionId;
 
@@ -201,13 +202,9 @@ bool tscIsProjectionQuery(SQueryInfo* pQueryInfo) {
 
 bool tscIsPointInterpQuery(SQueryInfo* pQueryInfo) {
   size_t size = tscSqlExprNumOfExprs(pQueryInfo);
-  
   for (int32_t i = 0; i < size; ++i) {
     SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i);
     assert(pExpr != NULL);
-//    if (pExpr == NULL) {
-//      return false;
-//    }
 
     int32_t functionId = pExpr->functionId;
     if (functionId == TSDB_FUNC_TAG) {

src/common/inc/tname.h

@@ -35,6 +35,6 @@ bool tscValidateTableNameLength(size_t len);
 
 SColumnFilterInfo* tscFilterInfoClone(const SColumnFilterInfo* src, int32_t numOfFilters);
 
-// int64_t taosGetIntervalStartTimestamp(int64_t startTime, int64_t slidingTime, int64_t intervalTime, char timeUnit, int16_t precision);
+SSchema tscGetTbnameColumnSchema();
 
 #endif  // TDENGINE_NAME_H

src/common/src/tname.c

@@ -188,3 +188,14 @@ void extractTableNameFromToken(SStrToken* pToken, SStrToken* pTable) {
     pToken->z = r;
   }
 }
+
+SSchema tscGetTbnameColumnSchema() {
+  struct SSchema s = {
+      .colId = TSDB_TBNAME_COLUMN_INDEX,
+      .type  = TSDB_DATA_TYPE_BINARY,
+      .bytes = TSDB_TABLE_NAME_LEN
+  };
+
+  strcpy(s.name, TSQL_TBNAME_L);
+  return s;
+}
\ No newline at end of file

src/common/src/ttypes.c

@@ -355,32 +355,6 @@ bool isValidDataType(int32_t type) {
   return type >= TSDB_DATA_TYPE_NULL && type <= TSDB_DATA_TYPE_NCHAR;
 }
 
-//bool isNull(const char *val, int32_t type) {
-//  switch (type) {
-//    case TSDB_DATA_TYPE_BOOL:
-//      return *(uint8_t *)val == TSDB_DATA_BOOL_NULL;
-//    case TSDB_DATA_TYPE_TINYINT:
-//      return *(uint8_t *)val == TSDB_DATA_TINYINT_NULL;
-//    case TSDB_DATA_TYPE_SMALLINT:
-//      return *(uint16_t *)val == TSDB_DATA_SMALLINT_NULL;
-//    case TSDB_DATA_TYPE_INT:
-//      return *(uint32_t *)val == TSDB_DATA_INT_NULL;
-//    case TSDB_DATA_TYPE_BIGINT:
-//    case TSDB_DATA_TYPE_TIMESTAMP:
-//      return *(uint64_t *)val == TSDB_DATA_BIGINT_NULL;
-//    case TSDB_DATA_TYPE_FLOAT:
-//      return *(uint32_t *)val == TSDB_DATA_FLOAT_NULL;
-//    case TSDB_DATA_TYPE_DOUBLE:
-//      return *(uint64_t *)val == TSDB_DATA_DOUBLE_NULL;
-//    case TSDB_DATA_TYPE_NCHAR:
-//      return *(uint32_t*) varDataVal(val) == TSDB_DATA_NCHAR_NULL;
-//    case TSDB_DATA_TYPE_BINARY:
-//      return *(uint8_t *) varDataVal(val) == TSDB_DATA_BINARY_NULL;
-//    default:
-//      return false;
-//  };
-//}
-
 void setVardataNull(char* val, int32_t type) {
   if (type == TSDB_DATA_TYPE_BINARY) {
     varDataSetLen(val, sizeof(int8_t));
@@ -433,14 +407,10 @@ void setNullN(char *val, int32_t type, int32_t bytes, int32_t numOfElems) {
         *(uint64_t *)(val + i * tDataTypeDesc[type].nSize) = TSDB_DATA_DOUBLE_NULL;
       }
       break;
-    case TSDB_DATA_TYPE_NCHAR: // todo : without length?
-      for (int32_t i = 0; i < numOfElems; ++i) {
-        *(uint32_t *)(val + i * bytes) = TSDB_DATA_NCHAR_NULL;
-      }
-      break;
+    case TSDB_DATA_TYPE_NCHAR:
     case TSDB_DATA_TYPE_BINARY:
       for (int32_t i = 0; i < numOfElems; ++i) {
-        *(uint8_t *)(val + i * bytes) = TSDB_DATA_BINARY_NULL;
+        setVardataNull(val + i * bytes, type);
       }
       break;
     default: {

src/dnode/src/dnodeMain.c

@@ -20,6 +20,7 @@
 #include "tconfig.h"
 #include "tglobal.h"
 #include "twal.h"
+#include "trpc.h"
 #include "dnode.h"
 #include "dnodeInt.h"
 #include "dnodeMgmt.h"
@@ -54,6 +55,7 @@ typedef struct {
 } SDnodeComponent;
 
 static const SDnodeComponent tsDnodeComponents[] = {
+  {"rpc",       rpcInit,             rpcCleanup},
   {"storage",   dnodeInitStorage,    dnodeCleanupStorage},
   {"dnodecfg",  dnodeInitCfg,        dnodeCleanupCfg},
   {"dnodeeps",  dnodeInitEps,        dnodeCleanupEps},

src/dnode/src/dnodePeer.c

@@ -169,7 +169,7 @@ void dnodeAddClientRspHandle(uint8_t msgType, void (*fp)(SRpcMsg *rpcMsg)) {
 }
 
 void dnodeSendMsgToDnode(SRpcEpSet *epSet, SRpcMsg *rpcMsg) {
-  rpcSendRequest(tsClientRpc, epSet, rpcMsg);
+  rpcSendRequest(tsClientRpc, epSet, rpcMsg, NULL);
 }
 
 void dnodeSendMsgToMnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp) {

src/dnode/src/dnodeVWrite.c

@@ -212,7 +212,7 @@ static void *dnodeProcessVWriteQueue(void *param) {
 
       pWrite->code = vnodeProcessWrite(pVnode, pWrite->pHead, qtype, &pWrite->rspRet);
       if (pWrite->code <= 0) pWrite->processedCount = 1;
-      if (pWrite->pHead->msgType != TSDB_MSG_TYPE_SUBMIT) forceFsync = true;
+      if (pWrite->code == 0 && pWrite->pHead->msgType != TSDB_MSG_TYPE_SUBMIT) forceFsync = true;
 
       dTrace("msg:%p is processed in vwrite queue, result:%s", pWrite, tstrerror(pWrite->code));
     }
@@ -225,11 +225,13 @@ static void *dnodeProcessVWriteQueue(void *param) {
       taosGetQitem(pWorker->qall, &qtype, (void **)&pWrite);
       if (qtype == TAOS_QTYPE_RPC) {
         dnodeSendRpcVWriteRsp(pVnode, pWrite, pWrite->code);
-      } else if (qtype == TAOS_QTYPE_FWD) {
-        vnodeConfirmForward(pVnode, pWrite->pHead->version, 0);
-        taosFreeQitem(pWrite);
-        vnodeRelease(pVnode);
       } else {
+        if (qtype == TAOS_QTYPE_FWD) {
+          vnodeConfirmForward(pVnode, pWrite->pHead->version, 0);
+        }
+        if (pWrite->rspRet.rsp) {
+          rpcFreeCont(pWrite->rspRet.rsp);
+        }
         taosFreeQitem(pWrite);
         vnodeRelease(pVnode);
       }

src/inc/taosdef.h

@@ -61,6 +61,7 @@ typedef struct tstr {
 
 // Bytes for each type.
 extern const int32_t TYPE_BYTES[11];
+
 // TODO: replace and remove code below
 #define CHAR_BYTES    sizeof(char)
 #define SHORT_BYTES   sizeof(int16_t)
@@ -68,6 +69,15 @@ extern const int32_t TYPE_BYTES[11];
 #define LONG_BYTES    sizeof(int64_t)
 #define FLOAT_BYTES   sizeof(float)
 #define DOUBLE_BYTES  sizeof(double)
+#define POINTER_BYTES sizeof(void *)  // 8 by default  assert(sizeof(ptrdiff_t) == sizseof(void*)
+
+#define TSDB_KEYSIZE            sizeof(TSKEY)
+
+#if LINUX
+#define TSDB_NCHAR_SIZE         sizeof(wchar_t)
+#else
+#define TSDB_NCHAR_SIZE         sizeof(int32_t)
+#endif
 
 // NULL definition
 #define TSDB_DATA_BOOL_NULL             0x02
@@ -102,10 +112,12 @@ extern const int32_t TYPE_BYTES[11];
 #define TSDB_TIME_PRECISION_MILLI 0
 #define TSDB_TIME_PRECISION_MICRO 1
 #define TSDB_TIME_PRECISION_NANO  2
-#define TSDB_TICK_PER_SECOND(precision) ((precision)==TSDB_TIME_PRECISION_MILLI ? 1e3L : ((precision)==TSDB_TIME_PRECISION_MICRO ? 1e6L : 1e9L))
 
 #define TSDB_TIME_PRECISION_MILLI_STR "ms"
 #define TSDB_TIME_PRECISION_MICRO_STR "us"
+#define TSDB_TIME_PRECISION_NANO_STR  "ns"
+
+#define TSDB_TICK_PER_SECOND(precision) ((precision)==TSDB_TIME_PRECISION_MILLI ? 1e3L : ((precision)==TSDB_TIME_PRECISION_MICRO ? 1e6L : 1e9L))
 
 #define T_MEMBER_SIZE(type, member) sizeof(((type *)0)->member)
 #define T_APPEND_MEMBER(dst, ptr, type, member) \
@@ -119,15 +131,6 @@ do { \
   (src) = (void *)((char *)src + sizeof(type));\
 } while(0)
 
-#define TSDB_KEYSIZE              sizeof(TSKEY)
-
-#if LINUX
-  #define TSDB_NCHAR_SIZE         sizeof(wchar_t)
-#else
-  #define TSDB_NCHAR_SIZE         4
-#endif
-//#define TSDB_CHAR_TERMINATED_SPACE 1
-
 #define GET_INT8_VAL(x)   (*(int8_t *)(x))
 #define GET_INT16_VAL(x)  (*(int16_t *)(x))
 #define GET_INT32_VAL(x)  (*(int32_t *)(x))
@@ -173,7 +176,6 @@ typedef struct tDataTypeDescriptor {
 } tDataTypeDescriptor;
 
 extern tDataTypeDescriptor tDataTypeDesc[11];
-#define POINTER_BYTES sizeof(void *)  // 8 by default  assert(sizeof(ptrdiff_t) == sizseof(void*)
 
 bool isValidDataType(int32_t type);
 //bool isNull(const char *val, int32_t type);
@@ -267,10 +269,6 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf
 #define TSDB_AUTH_LEN             16
 #define TSDB_KEY_LEN              16
 #define TSDB_VERSION_LEN          12
-#define TSDB_STREET_LEN           64
-#define TSDB_CITY_LEN             20
-#define TSDB_STATE_LEN            20
-#define TSDB_COUNTRY_LEN          20
 #define TSDB_LOCALE_LEN           64
 #define TSDB_TIMEZONE_LEN         96
 #define TSDB_LABEL_LEN            8 
@@ -425,6 +423,11 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf
 #define TSDB_DEFAULT_STABLES_HASH_SIZE  100
 #define TSDB_DEFAULT_CTABLES_HASH_SIZE  20000
 
+#define TSDB_PORT_DNODESHELL            0
+#define TSDB_PORT_DNODEDNODE            5
+#define TSDB_PORT_SYNC                  10
+#define TSDB_PORT_HTTP                  11
+#define TSDB_PORT_ARBITRATOR            12
 #define TSDB_PORT_DNODESHELL 0
 #define TSDB_PORT_DNODEDNODE 5
 #define TSDB_PORT_SYNC       10

src/inc/trpc.h

@@ -78,12 +78,14 @@ typedef struct SRpcInit {
   int  (*afp)(char *tableId, char *spi, char *encrypt, char *secret, char *ckey);
 } SRpcInit;
 
+int32_t rpcInit();
+void  rpcCleanup();
 void *rpcOpen(const SRpcInit *pRpc);
 void  rpcClose(void *);
 void *rpcMallocCont(int contLen);
 void  rpcFreeCont(void *pCont);
 void *rpcReallocCont(void *ptr, int contLen);
-int64_t rpcSendRequest(void *thandle, const SRpcEpSet *pEpSet, SRpcMsg *pMsg);
+void  rpcSendRequest(void *thandle, const SRpcEpSet *pEpSet, SRpcMsg *pMsg, int64_t *rid);
 void  rpcSendResponse(const SRpcMsg *pMsg);
 void  rpcSendRedirectRsp(void *pConn, const SRpcEpSet *pEpSet); 
 int   rpcGetConnInfo(void *thandle, SRpcConnInfo *pInfo);

src/inc/tsdb.h

@@ -46,7 +46,7 @@ extern "C" {
 typedef struct {
   void *appH;
   void *cqH;
-  int (*notifyStatus)(void *, int status);
+  int (*notifyStatus)(void *, int status, int eno);
   int (*eventCallBack)(void *);
   void *(*cqCreateFunc)(void *handle, uint64_t uid, int sid, char *sqlStr, STSchema *pSchema);
   void (*cqDropFunc)(void *handle);
@@ -126,7 +126,7 @@ uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_
 // the TSDB repository info
 typedef struct STsdbRepoInfo {
   STsdbCfg tsdbCfg;
-  int64_t  version;            // version of the repository
+  uint64_t version;            // version of the repository
   int64_t  tsdbTotalDataSize;  // the original inserted data size
   int64_t  tsdbTotalDiskSize;  // the total disk size taken by this TSDB repository
   // TODO: Other informations to add
@@ -136,7 +136,7 @@ STsdbRepoInfo *tsdbGetStatus(TSDB_REPO_T *pRepo);
 // the meter information report structure
 typedef struct {
   STableCfg tableCfg;
-  int64_t   version;
+  uint64_t  version;
   int64_t   tableTotalDataSize;  // In bytes
   int64_t   tableTotalDiskSize;  // In bytes
 } STableInfo;

src/inc/ttokendef.h

@@ -16,7 +16,6 @@
 #ifndef TDENGINE_TTOKENDEF_H
 #define TDENGINE_TTOKENDEF_H
 
-
 #define TK_ID                               1
 #define TK_BOOL                             2
 #define TK_TINYINT                          3
@@ -224,6 +223,7 @@
 #define TK_INTO                           205
 #define TK_VALUES                         206
 
+
 #define TK_SPACE                          300
 #define TK_COMMENT                        301
 #define TK_ILLEGAL                        302

src/inc/ttype.h

+#ifndef TDENGINE_TTYPE_H
+#define TDENGINE_TTYPE_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "taosdef.h"
+
+#define GET_TYPED_DATA(_v, _finalType, _type, _data) \
+  switch (_type) {                       \
+    case TSDB_DATA_TYPE_TINYINT:         \
+      (_v) = (_finalType)GET_INT8_VAL(_data);        \
+      break;                             \
+    case TSDB_DATA_TYPE_SMALLINT:        \
+      (_v) = (_finalType)GET_INT16_VAL(_data);       \
+      break;                             \
+    case TSDB_DATA_TYPE_BIGINT:          \
+      (_v) = (_finalType)(GET_INT64_VAL(_data));     \
+      break;                             \
+    case TSDB_DATA_TYPE_FLOAT:           \
+      (_v) = (_finalType)GET_FLOAT_VAL(_data);       \
+      break;                             \
+    case TSDB_DATA_TYPE_DOUBLE:          \
+      (_v) = (_finalType)GET_DOUBLE_VAL(_data);      \
+      break;                             \
+    default:                             \
+      (_v) = (_finalType)GET_INT32_VAL(_data);       \
+      break;                             \
+  };
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TDENGINE_TTYPE_H

src/kit/CMakeLists.txt

@@ -3,3 +3,4 @@ PROJECT(TDengine)
 
 ADD_SUBDIRECTORY(shell)
 ADD_SUBDIRECTORY(taosdemo)
+ADD_SUBDIRECTORY(taosdump)

src/kit/shell/src/shellLinux.c

@@ -46,7 +46,7 @@ static struct argp_option options[] = {
   {"thread",     'T', "THREADNUM",  0,                   "Number of threads when using multi-thread to import data."},
   {"database",   'd', "DATABASE",   0,                   "Database to use when connecting to the server."},
   {"timezone",   't', "TIMEZONE",   0,                   "Time zone of the shell, default is local."},
-  {"netrole",    'n', "NETROLE",    0,                   "Net role when network connectivity test, default is NULL, valid option: client | server."},
+  {"netrole",    'n', "NETROLE",    0,                   "Net role when network connectivity test, default is NULL, options: client|clients|server."},
   {"endport",    'e', "ENDPORT",    0,                   "Net test end port, default is 6042."},
   {"pktlen",     'l', "PKTLEN",     0,                   "Packet length used for net test, default is 1000 bytes."},
   {0}};

src/kit/taosdump/CMakeLists.txt

+CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
+PROJECT(TDengine)
+
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc)
+INCLUDE_DIRECTORIES(inc)
+AUX_SOURCE_DIRECTORY(. SRC)
+
+IF (TD_LINUX)
+  ADD_EXECUTABLE(taosdump ${SRC})
+  IF (TD_SOMODE_STATIC)
+    TARGET_LINK_LIBRARIES(taosdump taos_static)
+  ELSE ()
+    TARGET_LINK_LIBRARIES(taosdump taos)
+  ENDIF ()  
+ENDIF ()

src/kit/taosdump/taosdump.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <iconv.h>
+#include "os.h"
+#include "taos.h"
+#include "taosdef.h"
+#include "taosmsg.h"
+#include "tglobal.h"
+#include "tsclient.h"
+#include "tsdb.h"
+#include "tutil.h"
+
+#define COMMAND_SIZE 65536
+//#define DEFAULT_DUMP_FILE "taosdump.sql"
+
+int  converStringToReadable(char *str, int size, char *buf, int bufsize);
+int  convertNCharToReadable(char *str, int size, char *buf, int bufsize);
+void taosDumpCharset(FILE *fp);
+void taosLoadFileCharset(FILE *fp, char *fcharset);
+
+typedef struct {
+  short bytes;
+  int8_t type;
+} SOColInfo;
+
+// -------------------------- SHOW DATABASE INTERFACE-----------------------
+enum _show_db_index {
+  TSDB_SHOW_DB_NAME_INDEX,
+  TSDB_SHOW_DB_CREATED_TIME_INDEX,
+  TSDB_SHOW_DB_VGROUPS_INDEX,
+  TSDB_SHOW_DB_NTABLES_INDEX,
+  TSDB_SHOW_DB_REPLICA_INDEX,
+  TSDB_SHOW_DB_DAYS_INDEX,
+  TSDB_SHOW_DB_KEEP_INDEX,
+  TSDB_SHOW_DB_TABLES_INDEX,
+  TSDB_SHOW_DB_ROWS_INDEX,
+  TSDB_SHOW_DB_CACHE_INDEX,
+  TSDB_SHOW_DB_ABLOCKS_INDEX,
+  TSDB_SHOW_DB_TBLOCKS_INDEX,
+  TSDB_SHOW_DB_CTIME_INDEX,
+  TSDB_SHOW_DB_CLOG_INDEX,
+  TSDB_SHOW_DB_COMP_INDEX,
+  TSDB_MAX_SHOW_DB
+};
+
+// -----------------------------------------SHOW TABLES CONFIGURE -------------------------------------
+enum _show_tables_index {
+  TSDB_SHOW_TABLES_NAME_INDEX,
+  TSDB_SHOW_TABLES_CREATED_TIME_INDEX,
+  TSDB_SHOW_TABLES_COLUMNS_INDEX,
+  TSDB_SHOW_TABLES_METRIC_INDEX,
+  TSDB_MAX_SHOW_TABLES
+};
+
+// ---------------------------------- DESCRIBE METRIC CONFIGURE ------------------------------
+enum _describe_table_index {
+  TSDB_DESCRIBE_METRIC_FIELD_INDEX,
+  TSDB_DESCRIBE_METRIC_TYPE_INDEX,
+  TSDB_DESCRIBE_METRIC_LENGTH_INDEX,
+  TSDB_DESCRIBE_METRIC_NOTE_INDEX,
+  TSDB_MAX_DESCRIBE_METRIC
+};
+
+typedef struct {
+  char field[TSDB_COL_NAME_LEN + 1];
+  char type[16];
+  int length;
+  char note[128];
+} SColDes;
+
+typedef struct {
+  char name[TSDB_COL_NAME_LEN + 1];
+  SColDes cols[];
+} STableDef;
+
+extern char version[];
+
+typedef struct {
+  char name[TSDB_DB_NAME_LEN + 1];
+  int32_t replica;
+  int32_t days;
+  int32_t keep;
+  int32_t tables;
+  int32_t rows;
+  int32_t cache;
+  int32_t ablocks;
+  int32_t tblocks;
+  int32_t ctime;
+  int32_t clog;
+  int32_t comp;
+} SDbInfo;
+
+typedef struct {
+  char name[TSDB_TABLE_NAME_LEN + 1];
+  char metric[TSDB_TABLE_NAME_LEN + 1];
+} STableRecord;
+
+typedef struct {
+  bool isMetric;
+  STableRecord tableRecord;
+} STableRecordInfo;
+
+typedef struct {
+  pthread_t threadID;
+  int32_t   threadIndex;
+  int32_t   totalThreads;
+  char      dbName[TSDB_TABLE_NAME_LEN + 1];
+  void     *taosCon;
+} SThreadParaObj;
+
+static int64_t totalDumpOutRows = 0;
+
+SDbInfo **dbInfos = NULL;
+
+const char *argp_program_version = version;
+const char *argp_program_bug_address = "<support@taosdata.com>";
+
+/* Program documentation. */
+static char doc[] = "";
+/* "Argp example #4 -- a program with somewhat more complicated\ */
+/*         options\ */
+/*         \vThis part of the documentation comes *after* the options;\ */
+/*         note that the text is automatically filled, but it's possible\ */
+/*         to force a line-break, e.g.\n<-- here."; */
+
+/* A description of the arguments we accept. */
+static char args_doc[] = "dbname [tbname ...]\n--databases dbname ...\n--all-databases\n-i inpath\n-o outpath";
+
+/* Keys for options without short-options. */
+#define OPT_ABORT 1 /* –abort */
+
+/* The options we understand. */
+static struct argp_option options[] = {
+  // connection option
+  {"host",          'h', "HOST",        0,  "Server host dumping data from. Default is localhost.",        0},
+  {"user",          'u', "USER",        0,  "User name used to connect to server. Default is root.",       0},
+  #ifdef _TD_POWER_
+  {"password",      'p', "PASSWORD",    0,  "User password to connect to server. Default is powerdb.",     0},
+  #else
+  {"password",      'p', "PASSWORD",    0,  "User password to connect to server. Default is taosdata.",    0},
+  #endif
+  {"port",          'P', "PORT",        0,  "Port to connect",                                             0},
+  {"cversion",      'v', "CVERION",     0,  "client version",                                              0},
+  {"mysqlFlag",     'q', "MYSQLFLAG",   0,  "mysqlFlag, Default is 0",                                     0},
+  // input/output file
+  {"outpath",       'o', "OUTPATH",     0,  "Output file path.",                                          1},
+  {"inpath",        'i', "INPATH",      0,  "Input file path.",                                           1},
+  #ifdef _TD_POWER_
+  {"config",        'c', "CONFIG_DIR",  0,  "Configure directory. Default is /etc/power/taos.cfg.",       1},
+  #else
+  {"config",        'c', "CONFIG_DIR",  0,  "Configure directory. Default is /etc/taos/taos.cfg.",        1},
+  #endif
+  {"encode",        'e', "ENCODE",      0,  "Input file encoding.",                                       1},
+  // dump unit options
+  {"all-databases", 'A', 0,             0,  "Dump all databases.",                                         2},
+  {"databases",     'B', 0,             0,  "Dump assigned databases",                                     2},
+  // dump format options
+  {"schemaonly",    's', 0,             0,  "Only dump schema.",                                           3},
+  {"with-property", 'M', 0,             0,  "Dump schema with properties.",                                3},
+  {"start-time",    'S', "START_TIME",  0,  "Start time to dump.",                                         3},
+  {"end-time",      'E', "END_TIME",    0,  "End time to dump.",                                           3},
+  {"data-batch",    'N', "DATA_BATCH",  0,  "Number of data point per insert statement. Default is 1.",    3},
+  {"table-batch",   't', "TABLE_BATCH", 0,  "Number of table dumpout into one output file. Default is 1.", 3},
+  {"thread_num",    'T', "THREAD_NUM",  0,  "Number of thread for dump in file. Default is 5.",            3},  
+  {"allow-sys",     'a', 0,             0,  "Allow to dump sys database",                                  3},
+  {0}};
+
+/* Used by main to communicate with parse_opt. */
+struct arguments {
+  // connection option
+  char    *host;
+  char    *user;
+  char    *password;
+  uint16_t port;  
+  char     cversion[12];
+  uint16_t mysqlFlag;
+  // output file
+  char     outpath[TSDB_FILENAME_LEN+1];
+  char     inpath[TSDB_FILENAME_LEN+1];
+  char    *encode;
+  // dump unit option
+  bool     all_databases;
+  bool     databases;
+  // dump format option
+  bool     schemaonly;
+  bool     with_property;
+  int64_t  start_time;
+  int64_t  end_time;
+  int32_t  data_batch;
+  int32_t  table_batch; // num of table which will be dump into one output file.
+  bool     allow_sys;
+  // other options
+  int32_t  thread_num;
+  int      abort;
+  char   **arg_list;
+  int      arg_list_len;
+  bool     isDumpIn;
+};
+
+/* Parse a single option. */
+static error_t parse_opt(int key, char *arg, struct argp_state *state) {
+  /* Get the input argument from argp_parse, which we
+     know is a pointer to our arguments structure. */
+  struct arguments *arguments = state->input;
+  wordexp_t full_path;
+
+  switch (key) {
+    // connection option
+    case 'a':
+      arguments->allow_sys = true;
+      break;
+    case 'h':
+      arguments->host = arg;
+      break;
+    case 'u':
+      arguments->user = arg;
+      break;
+    case 'p':
+      arguments->password = arg;
+      break;
+    case 'P':
+      arguments->port = atoi(arg);
+      break;
+    case 'q':
+      arguments->mysqlFlag = atoi(arg);
+      break;
+    case 'v':
+      if (wordexp(arg, &full_path, 0) != 0) {
+        fprintf(stderr, "Invalid client vesion %s\n", arg);
+        return -1;
+      }
+      tstrncpy(arguments->cversion, full_path.we_wordv[0], 11);
+      wordfree(&full_path);
+      break;
+    // output file path
+    case 'o':
+      if (wordexp(arg, &full_path, 0) != 0) {
+        fprintf(stderr, "Invalid path %s\n", arg);
+        return -1;
+      }
+      tstrncpy(arguments->outpath, full_path.we_wordv[0], TSDB_FILENAME_LEN);
+      wordfree(&full_path);
+      break;
+    case 'i':
+      arguments->isDumpIn = true;
+      if (wordexp(arg, &full_path, 0) != 0) {
+        fprintf(stderr, "Invalid path %s\n", arg);
+        return -1;
+      }
+      tstrncpy(arguments->inpath, full_path.we_wordv[0], TSDB_FILENAME_LEN);
+      wordfree(&full_path);
+      break;
+    case 'c':
+      if (wordexp(arg, &full_path, 0) != 0) {
+        fprintf(stderr, "Invalid path %s\n", arg);
+        return -1;
+      }
+      tstrncpy(configDir, full_path.we_wordv[0], TSDB_FILENAME_LEN);
+      wordfree(&full_path);
+      break;
+    case 'e':
+      arguments->encode = arg;
+      break;
+    // dump unit option
+    case 'A':
+      arguments->all_databases = true;
+      break;
+    case 'B':
+      arguments->databases = true;
+      break;
+    // dump format option
+    case 's':
+      arguments->schemaonly = true;
+      break;
+    case 'M':
+      arguments->with_property = true;
+      break;
+    case 'S':
+      // parse time here.
+      arguments->start_time = atol(arg);
+      break;
+    case 'E':
+      arguments->end_time = atol(arg);
+      break;
+    case 'N':
+      arguments->data_batch = atoi(arg);
+      break;
+    case 't':
+      arguments->table_batch = atoi(arg);
+      break;
+    case 'T':
+      arguments->thread_num = atoi(arg);
+      break;
+    case OPT_ABORT:
+      arguments->abort = 1;
+      break;
+    case ARGP_KEY_ARG:
+      arguments->arg_list     = &state->argv[state->next - 1];
+      arguments->arg_list_len = state->argc - state->next + 1;
+      state->next             = state->argc;
+      break;
+
+    default:
+      return ARGP_ERR_UNKNOWN;
+  }
+  return 0;
+}
+
+/* Our argp parser. */
+static struct argp argp = {options, parse_opt, args_doc, doc};
+
+int taosDumpOut(struct arguments *arguments);
+int taosDumpIn(struct arguments *arguments);
+void taosDumpCreateDbClause(SDbInfo *dbInfo, bool isDumpProperty, FILE *fp);
+int taosDumpDb(SDbInfo *dbInfo, struct arguments *arguments, FILE *fp, TAOS *taosCon);
+int32_t taosDumpStable(char *table, FILE *fp, TAOS* taosCon);
+void taosDumpCreateTableClause(STableDef *tableDes, int numOfCols, FILE *fp);
+void taosDumpCreateMTableClause(STableDef *tableDes, char *metric, int numOfCols, FILE *fp);
+int32_t taosDumpTable(char *table, char *metric, struct arguments *arguments, FILE *fp, TAOS* taosCon);
+int taosDumpTableData(FILE *fp, char *tbname, struct arguments *arguments, TAOS* taosCon);
+int taosCheckParam(struct arguments *arguments);
+void taosFreeDbInfos();
+static void taosStartDumpOutWorkThreads(struct arguments* args, int32_t  numOfThread, char *dbName);
+
+struct arguments tsArguments = {
+  // connection option
+  NULL, 
+  "root", 
+  #ifdef _TD_POWER_
+  "powerdb", 
+  #else
+  "taosdata", 
+  #endif
+  0,
+  "",
+  0,
+  // outpath and inpath
+  "", 
+  "", 
+  NULL,
+  // dump unit option
+  false, 
+  false,
+  // dump format option
+  false, 
+  false, 
+  0, 
+  INT64_MAX, 
+  1,
+  1,
+  false,
+  // other options
+  5,
+  0,
+  NULL, 
+  0, 
+  false
+};
+
+int queryDB(TAOS *taos, char *command) {
+  TAOS_RES *pSql = NULL;
+  int32_t   code = -1;
+    
+  pSql = taos_query(taos, command);
+  code = taos_errno(pSql);
+  if (code) {
+    fprintf(stderr, "sql error: %s, reason:%s\n", command, taos_errstr(pSql));
+  } 
+  taos_free_result(pSql);
+  return code;
+}
+
+int main(int argc, char *argv[]) {
+
+  /* Parse our arguments; every option seen by parse_opt will be
+     reflected in arguments. */
+  argp_parse(&argp, argc, argv, 0, 0, &tsArguments);
+
+  if (tsArguments.abort) {
+    #ifndef _ALPINE
+      error(10, 0, "ABORTED");
+    #else
+      abort();
+    #endif
+  }
+
+  printf("====== arguments config ======\n");
+  {
+    printf("host: %s\n", tsArguments.host);
+    printf("user: %s\n", tsArguments.user);
+    printf("password: %s\n", tsArguments.password);
+    printf("port: %u\n", tsArguments.port);
+    printf("cversion: %s\n", tsArguments.cversion);    
+    printf("mysqlFlag: %d\n", tsArguments.mysqlFlag);    
+    printf("outpath: %s\n", tsArguments.outpath);
+    printf("inpath: %s\n", tsArguments.inpath);
+    printf("encode: %s\n", tsArguments.encode);
+    printf("all_databases: %d\n", tsArguments.all_databases);
+    printf("databases: %d\n", tsArguments.databases);
+    printf("schemaonly: %d\n", tsArguments.schemaonly);
+    printf("with_property: %d\n", tsArguments.with_property);
+    printf("start_time: %" PRId64 "\n", tsArguments.start_time);
+    printf("end_time: %" PRId64 "\n", tsArguments.end_time);
+    printf("data_batch: %d\n", tsArguments.data_batch);
+    printf("table_batch: %d\n", tsArguments.table_batch);
+    printf("allow_sys: %d\n", tsArguments.allow_sys);
+    printf("abort: %d\n", tsArguments.abort);
+    printf("isDumpIn: %d\n", tsArguments.isDumpIn);
+    printf("arg_list_len: %d\n", tsArguments.arg_list_len);
+
+    for (int32_t i = 0; i < tsArguments.arg_list_len; i++) {
+      printf("arg_list[%d]: %s\n", i, tsArguments.arg_list[i]);
+    }
+  }  
+  printf("==============================\n");
+
+  if (tsArguments.cversion[0] != 0){
+    tstrncpy(version, tsArguments.cversion, 11);
+  }
+
+  if (taosCheckParam(&tsArguments) < 0) {
+    exit(EXIT_FAILURE);
+  }
+
+  if (tsArguments.isDumpIn) {
+    if (taosDumpIn(&tsArguments) < 0) return -1;
+  } else {
+    if (taosDumpOut(&tsArguments) < 0) return -1;
+  }
+
+  return 0;
+}
+
+void taosFreeDbInfos() {
+  if (dbInfos == NULL) return;
+  for (int i = 0; i < 128; i++) tfree(dbInfos[i]);
+  tfree(dbInfos);
+}
+
+// check table is normal table or super table
+int taosGetTableRecordInfo(char *table, STableRecordInfo *pTableRecordInfo, TAOS *taosCon) {
+  TAOS_ROW row = NULL;
+  bool isSet = false;
+  TAOS_RES *result     = NULL;
+
+  memset(pTableRecordInfo, 0, sizeof(STableRecordInfo));
+
+  char* tempCommand = (char *)malloc(COMMAND_SIZE);
+  if (tempCommand == NULL) {
+    fprintf(stderr, "failed to allocate memory\n");
+    return -1;
+  }
+
+  sprintf(tempCommand, "show tables like %s", table);
+  
+  result = taos_query(taosCon, tempCommand);  
+  int32_t code = taos_errno(result);
+  
+  if (code != 0) {
+    fprintf(stderr, "failed to run command %s\n", tempCommand);
+    free(tempCommand);
+    taos_free_result(result);
+    return -1;
+  }
+
+  TAOS_FIELD *fields = taos_fetch_fields(result);
+
+  while ((row = taos_fetch_row(result)) != NULL) {
+    isSet = true;
+    pTableRecordInfo->isMetric = false;
+    strncpy(pTableRecordInfo->tableRecord.name, (char *)row[TSDB_SHOW_TABLES_NAME_INDEX],
+            fields[TSDB_SHOW_TABLES_NAME_INDEX].bytes);
+    strncpy(pTableRecordInfo->tableRecord.metric, (char *)row[TSDB_SHOW_TABLES_METRIC_INDEX],
+            fields[TSDB_SHOW_TABLES_METRIC_INDEX].bytes);
+    break;
+  }
+
+  taos_free_result(result);
+  result = NULL;
+
+  if (isSet) {
+    free(tempCommand);
+    return 0;
+  }
+  
+  sprintf(tempCommand, "show stables like %s", table);
+  
+  result = taos_query(taosCon, tempCommand);  
+  code = taos_errno(result);
+  
+  if (code != 0) {
+    fprintf(stderr, "failed to run command %s\n", tempCommand);
+    free(tempCommand);
+    taos_free_result(result);
+    return -1;
+  }
+
+  while ((row = taos_fetch_row(result)) != NULL) {
+    isSet = true;
+    pTableRecordInfo->isMetric = true;
+    tstrncpy(pTableRecordInfo->tableRecord.metric, table, TSDB_TABLE_NAME_LEN);
+    break;
+  }
+
+  taos_free_result(result);
+  result = NULL;
+
+  if (isSet) {
+    free(tempCommand);
+    return 0;
+  }
+  fprintf(stderr, "invalid table/metric %s\n", table);
+  free(tempCommand);
+  return -1;
+}
+
+
+int32_t taosSaveAllNormalTableToTempFile(TAOS *taosCon, char*meter, char* metric, int* fd) {
+  STableRecord tableRecord;
+
+  if (-1 == *fd) {
+    *fd = open(".tables.tmp.0", O_RDWR | O_CREAT, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH);
+    if (*fd == -1) {
+      fprintf(stderr, "failed to open temp file: .tables.tmp.0\n");
+      return -1;
+    }
+  }
+  
+  memset(&tableRecord, 0, sizeof(STableRecord));
+  tstrncpy(tableRecord.name, meter, TSDB_TABLE_NAME_LEN);
+  tstrncpy(tableRecord.metric, metric, TSDB_TABLE_NAME_LEN);
+
+  taosWrite(*fd, &tableRecord, sizeof(STableRecord));
+  return 0;
+}
+
+
+int32_t taosSaveTableOfMetricToTempFile(TAOS *taosCon, char* metric, struct arguments *arguments, int32_t*  totalNumOfThread) {
+  TAOS_ROW row;
+  int fd = -1;
+  STableRecord tableRecord;
+
+  char* tmpCommand = (char *)malloc(COMMAND_SIZE);
+  if (tmpCommand == NULL) {
+    fprintf(stderr, "failed to allocate memory\n");
+    return -1;
+  }
+
+  sprintf(tmpCommand, "select tbname from %s", metric);
+  
+  TAOS_RES *result = taos_query(taosCon, tmpCommand);  
+  int32_t code = taos_errno(result);
+  if (code != 0) {
+    fprintf(stderr, "failed to run command %s\n", tmpCommand);
+    free(tmpCommand);
+    taos_free_result(result);
+    return -1;
+  }
+
+  TAOS_FIELD *fields = taos_fetch_fields(result);
+
+  int32_t  numOfTable  = 0;
+  int32_t  numOfThread = *totalNumOfThread;
+  char     tmpFileName[TSDB_FILENAME_LEN + 1];
+  while ((row = taos_fetch_row(result)) != NULL) {
+    if (0 == numOfTable) {
+      memset(tmpFileName, 0, TSDB_FILENAME_LEN);
+      sprintf(tmpFileName, ".tables.tmp.%d", numOfThread);
+      fd = open(tmpFileName, O_RDWR | O_CREAT, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH);
+      if (fd == -1) {
+        fprintf(stderr, "failed to open temp file: %s\n", tmpFileName);
+        taos_free_result(result);
+        for (int32_t loopCnt = 0; loopCnt < numOfThread; loopCnt++) {
+          sprintf(tmpFileName, ".tables.tmp.%d", loopCnt);
+          (void)remove(tmpFileName);
+        }
+        free(tmpCommand);
+        return -1;
+      }
+      
+      numOfThread++;
+    }
+  
+    memset(&tableRecord, 0, sizeof(STableRecord));
+    tstrncpy(tableRecord.name, (char *)row[0], fields[0].bytes);
+    tstrncpy(tableRecord.metric, metric, TSDB_TABLE_NAME_LEN);
+
+    taosWrite(fd, &tableRecord, sizeof(STableRecord));
+
+    numOfTable++;
+
+    if (numOfTable >= arguments->table_batch) {
+      numOfTable = 0;
+      close(fd);
+      fd = -1;
+    }
+  }
+  
+  if (fd >= 0) {
+    close(fd);
+    fd = -1;
+  }
+  
+  taos_free_result(result);
+
+  *totalNumOfThread = numOfThread;
+ 
+  free(tmpCommand);
+
+  return 0;
+}
+
+int taosDumpOut(struct arguments *arguments) {
+  TAOS     *taos       = NULL;
+  TAOS_RES *result     = NULL;
+  char     *command    = NULL;
+
+  TAOS_ROW row;
+  FILE *fp = NULL;
+  int32_t count = 0;
+  STableRecordInfo tableRecordInfo;
+
+  char tmpBuf[TSDB_FILENAME_LEN+9] = {0};
+  if (arguments->outpath[0] != 0) {
+      sprintf(tmpBuf, "%s/dbs.sql", arguments->outpath);
+  } else {
+    sprintf(tmpBuf, "dbs.sql");
+  }
+  
+  fp = fopen(tmpBuf, "w");
+  if (fp == NULL) {
+    fprintf(stderr, "failed to open file %s\n", tmpBuf);
+    return -1;
+  }
+
+  dbInfos = (SDbInfo **)calloc(128, sizeof(SDbInfo *));
+  if (dbInfos == NULL) {
+    fprintf(stderr, "failed to allocate memory\n");
+    goto _exit_failure;
+  }
+
+  command = (char *)malloc(COMMAND_SIZE);
+  if (command == NULL) {
+    fprintf(stderr, "failed to allocate memory\n");
+    goto _exit_failure;
+  }
+
+  /* Connect to server */
+  taos = taos_connect(arguments->host, arguments->user, arguments->password, NULL, arguments->port);
+  if (taos == NULL) {
+    fprintf(stderr, "failed to connect to TDengine server\n");
+    goto _exit_failure;
+  }
+
+  /* --------------------------------- Main Code -------------------------------- */
+  /* if (arguments->databases || arguments->all_databases) { // dump part of databases or all databases */
+  /*  */
+  taosDumpCharset(fp);
+
+  sprintf(command, "show databases");
+  result = taos_query(taos, command);  
+  int32_t code = taos_errno(result);
+  
+  if (code != 0) {
+    fprintf(stderr, "failed to run command: %s, reason: %s\n", command, taos_errstr(taos));
+    goto _exit_failure;
+  }
+
+  TAOS_FIELD *fields = taos_fetch_fields(result);
+
+  while ((row = taos_fetch_row(result)) != NULL) {
+    // sys database name : 'monitor', but subsequent version changed to 'log'
+    if (strncasecmp(row[TSDB_SHOW_DB_NAME_INDEX], "monitor", fields[TSDB_SHOW_DB_NAME_INDEX].bytes) == 0 &&
+        (!arguments->allow_sys))
+      continue;
+
+    if (arguments->databases) {  // input multi dbs
+      for (int i = 0; arguments->arg_list[i]; i++) {
+        if (strncasecmp(arguments->arg_list[i], (char *)row[TSDB_SHOW_DB_NAME_INDEX],
+                        fields[TSDB_SHOW_DB_NAME_INDEX].bytes) == 0)
+          goto _dump_db_point;
+      }
+      continue;
+    } else if (!arguments->all_databases) {  // only input one db
+      if (strncasecmp(arguments->arg_list[0], (char *)row[TSDB_SHOW_DB_NAME_INDEX],
+                      fields[TSDB_SHOW_DB_NAME_INDEX].bytes) == 0)
+        goto _dump_db_point;
+      else
+        continue;
+    }
+
+  _dump_db_point:
+
+    dbInfos[count] = (SDbInfo *)calloc(1, sizeof(SDbInfo));
+    if (dbInfos[count] == NULL) {
+      fprintf(stderr, "failed to allocate memory\n");
+      goto _exit_failure;
+    }
+
+    strncpy(dbInfos[count]->name, (char *)row[TSDB_SHOW_DB_NAME_INDEX], fields[TSDB_SHOW_DB_NAME_INDEX].bytes);
+ #if 0   
+    dbInfos[count]->replica = (int)(*((int16_t *)row[TSDB_SHOW_DB_REPLICA_INDEX]));
+    dbInfos[count]->days = (int)(*((int16_t *)row[TSDB_SHOW_DB_DAYS_INDEX]));
+    dbInfos[count]->keep = *((int *)row[TSDB_SHOW_DB_KEEP_INDEX]);
+    dbInfos[count]->tables = *((int *)row[TSDB_SHOW_DB_TABLES_INDEX]);
+    dbInfos[count]->rows = *((int *)row[TSDB_SHOW_DB_ROWS_INDEX]);
+    dbInfos[count]->cache = *((int *)row[TSDB_SHOW_DB_CACHE_INDEX]);
+    dbInfos[count]->ablocks = *((int *)row[TSDB_SHOW_DB_ABLOCKS_INDEX]);
+    dbInfos[count]->tblocks = (int)(*((int16_t *)row[TSDB_SHOW_DB_TBLOCKS_INDEX]));
+    dbInfos[count]->ctime = *((int *)row[TSDB_SHOW_DB_CTIME_INDEX]);
+    dbInfos[count]->clog = (int)(*((int8_t *)row[TSDB_SHOW_DB_CLOG_INDEX]));
+    dbInfos[count]->comp = (int)(*((int8_t *)row[TSDB_SHOW_DB_COMP_INDEX]));
+#endif
+
+    count++;
+
+    if (arguments->databases) {
+      if (count > arguments->arg_list_len) break;
+
+    } else if (!arguments->all_databases) {
+      if (count >= 1) break;
+    }
+  }
+
+  if (count == 0) {
+    fprintf(stderr, "No databases valid to dump\n");
+    goto _exit_failure;
+  }
+
+  if (arguments->databases || arguments->all_databases) { // case: taosdump --databases dbx dby ...   OR  taosdump --all-databases
+    for (int i = 0; i < count; i++) {
+      taosDumpDb(dbInfos[i], arguments, fp, taos);
+    }
+  } else {
+    if (arguments->arg_list_len == 1) {             // case: taosdump <db>
+      taosDumpDb(dbInfos[0], arguments, fp, taos);
+    } else {                                        // case: taosdump <db> tablex tabley ...
+      taosDumpCreateDbClause(dbInfos[0], arguments->with_property, fp);
+
+      sprintf(command, "use %s", dbInfos[0]->name);
+      
+      result = taos_query(taos, command);  
+      int32_t code = taos_errno(result);
+      if (code != 0) {
+        fprintf(stderr, "invalid database %s\n", dbInfos[0]->name);
+        goto _exit_failure;
+      }
+
+      fprintf(fp, "USE %s;\n\n", dbInfos[0]->name);
+
+      int32_t totalNumOfThread = 1;  // 0: all normal talbe into .tables.tmp.0
+      int  normalTblFd = -1;
+      int32_t retCode;
+      for (int i = 1; arguments->arg_list[i]; i++) {
+        if (taosGetTableRecordInfo(arguments->arg_list[i], &tableRecordInfo, taos) < 0) {
+          fprintf(stderr, "input the invalide table %s\n", arguments->arg_list[i]);
+          continue;
+        }
+
+        if (tableRecordInfo.isMetric) {  // dump all table of this metric
+          (void)taosDumpStable(tableRecordInfo.tableRecord.metric, fp, taos);
+          retCode = taosSaveTableOfMetricToTempFile(taos, tableRecordInfo.tableRecord.metric, arguments, &totalNumOfThread);
+        } else {
+          if (tableRecordInfo.tableRecord.metric[0] != '\0') {  // dump this sub table and it's metric
+            (void)taosDumpStable(tableRecordInfo.tableRecord.metric, fp, taos);          
+          }
+          retCode = taosSaveAllNormalTableToTempFile(taos, tableRecordInfo.tableRecord.name, tableRecordInfo.tableRecord.metric, &normalTblFd);
+        }
+
+        if (retCode < 0) {
+          if (-1 != normalTblFd){
+            taosClose(normalTblFd);
+          }
+          goto _clean_tmp_file;
+        }
+      }
+
+      if (-1 != normalTblFd){
+        taosClose(normalTblFd);
+      }
+
+      // start multi threads to dumpout
+      taosStartDumpOutWorkThreads(arguments, totalNumOfThread, dbInfos[0]->name);
+
+      char tmpFileName[TSDB_FILENAME_LEN + 1];
+      _clean_tmp_file:
+      for (int loopCnt = 0; loopCnt < totalNumOfThread; loopCnt++) {
+        sprintf(tmpFileName, ".tables.tmp.%d", loopCnt);
+        remove(tmpFileName);
+      }
+    }
+  }
+
+  /* Close the handle and return */
+  fclose(fp);
+  taos_close(taos);
+  taos_free_result(result);
+  tfree(command);
+  taosFreeDbInfos();  
+  fprintf(stderr, "dump out rows: %" PRId64 "\n", totalDumpOutRows);
+  return 0;
+
+_exit_failure:
+  fclose(fp);
+  taos_close(taos);
+  taos_free_result(result);
+  tfree(command);
+  taosFreeDbInfos();
+  fprintf(stderr, "dump out rows: %" PRId64 "\n", totalDumpOutRows);
+  return -1;
+}
+
+int taosGetTableDes(char *table, STableDef *tableDes, TAOS* taosCon, bool isSuperTable) {
+  TAOS_ROW row = NULL;
+  TAOS_RES *tmpResult = NULL;
+  int count = 0;
+
+  char* tempCommand = (char *)malloc(COMMAND_SIZE);
+  if (tempCommand == NULL) {
+    fprintf(stderr, "failed to allocate memory\n");
+    return -1;
+  }
+
+  char* tbuf = (char *)malloc(COMMAND_SIZE);
+  if (tbuf == NULL) {
+    fprintf(stderr, "failed to allocate memory\n");
+    free(tempCommand);
+    return -1;
+  }
+
+  sprintf(tempCommand, "describe %s", table);
+  
+  tmpResult = taos_query(taosCon, tempCommand);  
+  int32_t code = taos_errno(tmpResult);
+  if (code != 0) {
+    fprintf(stderr, "failed to run command %s\n", tempCommand);
+    free(tempCommand);
+    free(tbuf);
+    taos_free_result(tmpResult);
+    return -1;
+  }
+
+  TAOS_FIELD *fields = taos_fetch_fields(tmpResult);
+
+  tstrncpy(tableDes->name, table, TSDB_COL_NAME_LEN);
+
+  while ((row = taos_fetch_row(tmpResult)) != NULL) {
+    strncpy(tableDes->cols[count].field, (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX],
+            fields[TSDB_DESCRIBE_METRIC_FIELD_INDEX].bytes);
+    strncpy(tableDes->cols[count].type, (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX],
+            fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes);
+    tableDes->cols[count].length = *((int *)row[TSDB_DESCRIBE_METRIC_LENGTH_INDEX]);
+    strncpy(tableDes->cols[count].note, (char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX],
+            fields[TSDB_DESCRIBE_METRIC_NOTE_INDEX].bytes);
+
+    count++;
+  }
+
+  taos_free_result(tmpResult);
+  tmpResult = NULL;
+
+  if (isSuperTable) {
+    free(tempCommand);
+    free(tbuf);
+    return count;
+  }
+  
+  // if chidl-table have tag, using  select tagName from table to get tagValue
+  for (int i = 0 ; i < count; i++) {
+    if (strcmp(tableDes->cols[i].note, "TAG") != 0) continue;
+
+
+    sprintf(tempCommand, "select %s from %s", tableDes->cols[i].field, table);
+    
+    tmpResult = taos_query(taosCon, tempCommand);  
+    code = taos_errno(tmpResult);
+    if (code != 0) {
+      fprintf(stderr, "failed to run command %s\n", tempCommand);
+      free(tempCommand);
+      free(tbuf);
+      taos_free_result(tmpResult);
+      return -1;
+    }
+    
+    fields = taos_fetch_fields(tmpResult); 
+
+    row = taos_fetch_row(tmpResult);
+    if (NULL == row) {
+      fprintf(stderr, " fetch failed to run command %s\n", tempCommand);
+      free(tempCommand);
+      free(tbuf);
+      taos_free_result(tmpResult);
+      return -1;
+    }
+
+    if (row[0] == NULL) {
+      sprintf(tableDes->cols[i].note, "%s", "NULL");
+      taos_free_result(tmpResult);
+      tmpResult = NULL;
+      continue;
+    }
+    
+    int32_t* length = taos_fetch_lengths(tmpResult);
+
+    //int32_t* length = taos_fetch_lengths(tmpResult);
+    switch (fields[0].type) {
+      case TSDB_DATA_TYPE_BOOL:
+        sprintf(tableDes->cols[i].note, "%d", ((((int32_t)(*((char *)row[0]))) == 1) ? 1 : 0));
+        break;
+      case TSDB_DATA_TYPE_TINYINT:
+        sprintf(tableDes->cols[i].note, "%d", *((int8_t *)row[0]));
+        break;
+      case TSDB_DATA_TYPE_SMALLINT:
+        sprintf(tableDes->cols[i].note, "%d", *((int16_t *)row[0]));
+        break;
+      case TSDB_DATA_TYPE_INT:
+        sprintf(tableDes->cols[i].note, "%d", *((int32_t *)row[0]));
+        break;
+      case TSDB_DATA_TYPE_BIGINT:
+        sprintf(tableDes->cols[i].note, "%" PRId64 "", *((int64_t *)row[0]));
+        break;
+      case TSDB_DATA_TYPE_FLOAT:
+        sprintf(tableDes->cols[i].note, "%f", GET_FLOAT_VAL(row[0]));
+        break;
+      case TSDB_DATA_TYPE_DOUBLE:
+        sprintf(tableDes->cols[i].note, "%f", GET_DOUBLE_VAL(row[0]));
+        break;
+      case TSDB_DATA_TYPE_BINARY:
+        memset(tableDes->cols[i].note, 0, sizeof(tableDes->cols[i].note));
+        tableDes->cols[i].note[0] = '\'';
+        converStringToReadable((char *)row[0], length[0], tbuf, COMMAND_SIZE);
+        char* pstr = stpcpy(&(tableDes->cols[i].note[1]), tbuf);
+        *(pstr++) = '\'';
+        break;
+      case TSDB_DATA_TYPE_NCHAR:
+        memset(tableDes->cols[i].note, 0, sizeof(tableDes->cols[i].note));
+        convertNCharToReadable((char *)row[0], length[0], tbuf, COMMAND_SIZE);
+        sprintf(tableDes->cols[i].note, "\'%s\'", tbuf);
+        break;
+      case TSDB_DATA_TYPE_TIMESTAMP:
+        sprintf(tableDes->cols[i].note, "%" PRId64 "", *(int64_t *)row[0]);
+        #if 0
+        if (!arguments->mysqlFlag) {
+          sprintf(tableDes->cols[i].note, "%" PRId64 "", *(int64_t *)row[0]);
+        } else {
+          char buf[64] = "\0";
+          int64_t ts = *((int64_t *)row[0]);
+          time_t tt = (time_t)(ts / 1000);
+          struct tm *ptm = localtime(&tt);
+          strftime(buf, 64, "%y-%m-%d %H:%M:%S", ptm);
+          sprintf(tableDes->cols[i].note, "\'%s.%03d\'", buf, (int)(ts % 1000));
+        }
+        #endif
+        break;
+      default:
+        break;
+    }
+  
+    taos_free_result(tmpResult);
+    tmpResult = NULL;    
+  }
+
+  free(tempCommand);
+  free(tbuf);
+
+  return count;
+}
+
+int32_t taosDumpTable(char *table, char *metric, struct arguments *arguments, FILE *fp, TAOS* taosCon) {
+  int count = 0;
+
+  STableDef *tableDes = (STableDef *)calloc(1, sizeof(STableDef) + sizeof(SColDes) * TSDB_MAX_COLUMNS);
+
+  if (metric != NULL && metric[0] != '\0') {  // dump table schema which is created by using super table
+    /*
+    count = taosGetTableDes(metric, tableDes, taosCon);
+
+    if (count < 0) {
+      free(tableDes);
+      return -1;
+    }
+
+    taosDumpCreateTableClause(tableDes, count, fp);
+
+    memset(tableDes, 0, sizeof(STableDef) + sizeof(SColDes) * TSDB_MAX_COLUMNS);
+    */
+
+    count = taosGetTableDes(table, tableDes, taosCon, false);
+
+    if (count < 0) {
+      free(tableDes);
+      return -1;
+    }
+
+    // create child-table using super-table
+    taosDumpCreateMTableClause(tableDes, metric, count, fp);
+
+  } else {  // dump table definition
+    count = taosGetTableDes(table, tableDes, taosCon, false);
+
+    if (count < 0) {
+      free(tableDes);
+      return -1;
+    }
+
+    // create normal-table or super-table
+    taosDumpCreateTableClause(tableDes, count, fp);
+  }
+
+  free(tableDes);
+
+  return taosDumpTableData(fp, table, arguments, taosCon);
+}
+
+void taosDumpCreateDbClause(SDbInfo *dbInfo, bool isDumpProperty, FILE *fp) {
+
+  char* tmpCommand = (char *)malloc(COMMAND_SIZE);
+  if (tmpCommand == NULL) {
+    fprintf(stderr, "failed to allocate memory\n");
+    return;
+  }
+
+  char *pstr = tmpCommand;
+
+  pstr += sprintf(pstr, "CREATE DATABASE IF NOT EXISTS %s", dbInfo->name);
+  if (isDumpProperty) {
+    pstr += sprintf(pstr,
+        " REPLICA %d DAYS %d KEEP %d TABLES %d ROWS %d CACHE %d ABLOCKS %d TBLOCKS %d CTIME %d CLOG %d COMP %d",
+        dbInfo->replica, dbInfo->days, dbInfo->keep, dbInfo->tables, dbInfo->rows, dbInfo->cache,
+        dbInfo->ablocks, dbInfo->tblocks, dbInfo->ctime, dbInfo->clog, dbInfo->comp);
+  }
+
+  pstr += sprintf(pstr, ";");
+
+  fprintf(fp, "%s\n\n", tmpCommand);
+  free(tmpCommand);
+}
+
+void* taosDumpOutWorkThreadFp(void *arg)
+{
+  SThreadParaObj *pThread = (SThreadParaObj*)arg;
+  STableRecord    tableRecord;
+  int fd;  
+  
+  char tmpFileName[TSDB_FILENAME_LEN*4] = {0};
+  sprintf(tmpFileName, ".tables.tmp.%d", pThread->threadIndex);
+  fd = open(tmpFileName, O_RDWR | O_CREAT, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH);
+  if (fd == -1) {
+    fprintf(stderr, "taosDumpTableFp() failed to open temp file: %s\n", tmpFileName);
+    return NULL;
+  }
+
+  FILE *fp = NULL;
+  memset(tmpFileName, 0, TSDB_FILENAME_LEN + 128);
+  
+  if (tsArguments.outpath[0] != 0) {
+    sprintf(tmpFileName, "%s/%s.tables.%d.sql", tsArguments.outpath, pThread->dbName, pThread->threadIndex);
+  } else {
+    sprintf(tmpFileName, "%s.tables.%d.sql", pThread->dbName, pThread->threadIndex);
+  }
+  
+  fp = fopen(tmpFileName, "w");
+  if (fp == NULL) {
+    fprintf(stderr, "failed to open file %s\n", tmpFileName);
+    close(fd);
+    return NULL;
+  }
+
+  memset(tmpFileName, 0, TSDB_FILENAME_LEN);
+  sprintf(tmpFileName, "use %s", pThread->dbName);
+  
+  TAOS_RES* tmpResult = taos_query(pThread->taosCon, tmpFileName);  
+  int32_t code = taos_errno(tmpResult);
+  if (code != 0) {
+    fprintf(stderr, "invalid database %s\n", pThread->dbName);
+    taos_free_result(tmpResult);
+    fclose(fp);  
+    close(fd);
+    return NULL;
+  }
+
+  fprintf(fp, "USE %s;\n\n", pThread->dbName);
+  while (1) {
+    ssize_t readLen = read(fd, &tableRecord, sizeof(STableRecord));
+    if (readLen <= 0) break;
+    taosDumpTable(tableRecord.name, tableRecord.metric, &tsArguments, fp, pThread->taosCon);
+  }
+
+  taos_free_result(tmpResult);
+  close(fd);
+  fclose(fp);  
+
+  return NULL;
+}
+
+static void taosStartDumpOutWorkThreads(struct arguments* args, int32_t  numOfThread, char *dbName)
+{
+  pthread_attr_t thattr;
+  SThreadParaObj *threadObj = (SThreadParaObj *)calloc(numOfThread, sizeof(SThreadParaObj));
+  for (int t = 0; t < numOfThread; ++t) {
+    SThreadParaObj *pThread = threadObj + t;
+    pThread->threadIndex = t;
+    pThread->totalThreads = numOfThread;
+    tstrncpy(pThread->dbName, dbName, TSDB_TABLE_NAME_LEN);
+    pThread->taosCon = taos_connect(args->host, args->user, args->password, NULL, args->port);
+                                
+    if (pThread->taosCon == NULL) {
+      fprintf(stderr, "ERROR: thread:%d failed connect to TDengine, error:%s\n", pThread->threadIndex, taos_errstr(pThread->taosCon));
+      exit(0);
+    }
+
+    pthread_attr_init(&thattr);
+    pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
+
+    if (pthread_create(&(pThread->threadID), &thattr, taosDumpOutWorkThreadFp, (void*)pThread) != 0) {
+      fprintf(stderr, "ERROR: thread:%d failed to start\n", pThread->threadIndex);
+      exit(0);
+    }
+  }
+
+  for (int32_t t = 0; t < numOfThread; ++t) {
+    pthread_join(threadObj[t].threadID, NULL);
+  }
+
+  for (int32_t t = 0; t < numOfThread; ++t) {
+    taos_close(threadObj[t].taosCon);
+  }
+  free(threadObj);
+}
+
+
+
+int32_t taosDumpStable(char *table, FILE *fp, TAOS* taosCon) {
+  int count = 0;
+
+  STableDef *tableDes = (STableDef *)calloc(1, sizeof(STableDef) + sizeof(SColDes) * TSDB_MAX_COLUMNS);
+  if (NULL == tableDes) {
+    fprintf(stderr, "failed to allocate memory\n");
+    exit(-1);
+  }
+
+  count = taosGetTableDes(table, tableDes, taosCon, true);
+
+  if (count < 0) {
+    free(tableDes);
+    fprintf(stderr, "failed to get stable schema\n");
+    exit(-1);
+  }
+
+  taosDumpCreateTableClause(tableDes, count, fp);
+
+  free(tableDes);
+  return 0;
+}
+
+
+int32_t taosDumpCreateSuperTableClause(TAOS* taosCon, char* dbName, FILE *fp) 
+{
+  TAOS_ROW row;
+  int fd = -1;
+  STableRecord tableRecord;
+
+  char* tmpCommand = (char *)malloc(COMMAND_SIZE);
+  if (tmpCommand == NULL) {
+    fprintf(stderr, "failed to allocate memory\n");
+    exit(-1);
+  }
+
+  sprintf(tmpCommand, "use %s", dbName);
+  
+  TAOS_RES* tmpResult = taos_query(taosCon, tmpCommand);  
+  int32_t code = taos_errno(tmpResult);
+  if (code != 0) {
+    fprintf(stderr, "invalid database %s, error: %s\n", dbName, taos_errstr(taosCon));
+    free(tmpCommand);
+    taos_free_result(tmpResult);
+    exit(-1);
+  }
+  
+  taos_free_result(tmpResult);
+
+  sprintf(tmpCommand, "show stables");
+  
+  tmpResult = taos_query(taosCon, tmpCommand);  
+  code = taos_errno(tmpResult);
+  if (code != 0) {
+    fprintf(stderr, "failed to run command %s, error: %s\n", tmpCommand, taos_errstr(taosCon));
+    free(tmpCommand);
+    taos_free_result(tmpResult);
+    exit(-1);
+  }
+
+  TAOS_FIELD *fields = taos_fetch_fields(tmpResult);
+
+  char     tmpFileName[TSDB_FILENAME_LEN + 1];
+  memset(tmpFileName, 0, TSDB_FILENAME_LEN);
+  sprintf(tmpFileName, ".stables.tmp");
+  fd = open(tmpFileName, O_RDWR | O_CREAT, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH);
+  if (fd == -1) {
+    fprintf(stderr, "failed to open temp file: %s\n", tmpFileName);
+    taos_free_result(tmpResult);
+    free(tmpCommand);
+    (void)remove(".stables.tmp");
+    exit(-1);
+  }
+  
+  while ((row = taos_fetch_row(tmpResult)) != NULL) {  
+    memset(&tableRecord, 0, sizeof(STableRecord));
+    strncpy(tableRecord.name, (char *)row[TSDB_SHOW_TABLES_NAME_INDEX], fields[TSDB_SHOW_TABLES_NAME_INDEX].bytes);
+    taosWrite(fd, &tableRecord, sizeof(STableRecord));
+  }  
+  
+  taos_free_result(tmpResult);
+  (void)lseek(fd, 0, SEEK_SET);
+
+  while (1) {
+    ssize_t readLen = read(fd, &tableRecord, sizeof(STableRecord));
+    if (readLen <= 0) break;
+    
+    (void)taosDumpStable(tableRecord.name, fp, taosCon);
+  }
+
+  close(fd);
+  (void)remove(".stables.tmp");
+  
+  free(tmpCommand);
+  return 0;  
+}
+
+
+int taosDumpDb(SDbInfo *dbInfo, struct arguments *arguments, FILE *fp, TAOS *taosCon) {
+  TAOS_ROW row;
+  int fd = -1;
+  STableRecord tableRecord;
+
+  taosDumpCreateDbClause(dbInfo, arguments->with_property, fp);
+
+  char* tmpCommand = (char *)malloc(COMMAND_SIZE);
+  if (tmpCommand == NULL) {
+    fprintf(stderr, "failed to allocate memory\n");
+    return -1;
+  }
+
+  sprintf(tmpCommand, "use %s", dbInfo->name);
+  
+  TAOS_RES* tmpResult = taos_query(taosCon, tmpCommand);  
+  int32_t code = taos_errno(tmpResult);
+  if (code != 0) {
+    fprintf(stderr, "invalid database %s\n", dbInfo->name);
+    free(tmpCommand);
+    taos_free_result(tmpResult);
+    return -1;
+  }
+  taos_free_result(tmpResult);
+
+  fprintf(fp, "USE %s;\n\n", dbInfo->name);
+  
+  (void)taosDumpCreateSuperTableClause(taosCon, dbInfo->name, fp);
+
+  sprintf(tmpCommand, "show tables");
+  
+  tmpResult = taos_query(taosCon, tmpCommand);  
+  code = taos_errno(tmpResult);
+  if (code != 0) {
+    fprintf(stderr, "failed to run command %s\n", tmpCommand);
+    free(tmpCommand);
+    taos_free_result(tmpResult);
+    return -1;
+  }
+
+  TAOS_FIELD *fields = taos_fetch_fields(tmpResult);
+
+  int32_t  numOfTable  = 0;
+  int32_t  numOfThread = 0;
+  char     tmpFileName[TSDB_FILENAME_LEN + 1];
+  while ((row = taos_fetch_row(tmpResult)) != NULL) {
+    if (0 == numOfTable) {
+      memset(tmpFileName, 0, TSDB_FILENAME_LEN);
+      sprintf(tmpFileName, ".tables.tmp.%d", numOfThread);
+      fd = open(tmpFileName, O_RDWR | O_CREAT, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH);
+      if (fd == -1) {
+        fprintf(stderr, "failed to open temp file: %s\n", tmpFileName);
+        taos_free_result(tmpResult);
+        for (int32_t loopCnt = 0; loopCnt < numOfThread; loopCnt++) {
+          sprintf(tmpFileName, ".tables.tmp.%d", loopCnt);
+          (void)remove(tmpFileName);
+        }
+        free(tmpCommand);
+        return -1;
+      }
+
+      numOfThread++;
+    }
+  
+    memset(&tableRecord, 0, sizeof(STableRecord));
+    tstrncpy(tableRecord.name, (char *)row[TSDB_SHOW_TABLES_NAME_INDEX], fields[TSDB_SHOW_TABLES_NAME_INDEX].bytes);
+    tstrncpy(tableRecord.metric, (char *)row[TSDB_SHOW_TABLES_METRIC_INDEX], fields[TSDB_SHOW_TABLES_METRIC_INDEX].bytes);
+
+    taosWrite(fd, &tableRecord, sizeof(STableRecord));
+
+    numOfTable++;
+
+    if (numOfTable >= arguments->table_batch) {
+      numOfTable = 0;
+      close(fd);
+      fd = -1;
+    }
+  }
+
+  if (fd >= 0) {
+    close(fd);
+    fd = -1;
+  }
+  
+  taos_free_result(tmpResult);
+
+  // start multi threads to dumpout
+  taosStartDumpOutWorkThreads(arguments, numOfThread, dbInfo->name);
+  for (int loopCnt = 0; loopCnt < numOfThread; loopCnt++) {
+    sprintf(tmpFileName, ".tables.tmp.%d", loopCnt);
+    (void)remove(tmpFileName);
+  }
+  
+  free(tmpCommand);
+
+  return 0;
+}
+
+void taosDumpCreateTableClause(STableDef *tableDes, int numOfCols, FILE *fp) {
+  int counter = 0;
+  int count_temp = 0;
+
+  char* tmpBuf = (char *)malloc(COMMAND_SIZE);
+  if (tmpBuf == NULL) {
+    fprintf(stderr, "failed to allocate memory\n");
+    return;
+  }
+
+  char* pstr = tmpBuf;
+
+  pstr += sprintf(tmpBuf, "CREATE TABLE IF NOT EXISTS %s", tableDes->name);
+
+  for (; counter < numOfCols; counter++) {
+    if (tableDes->cols[counter].note[0] != '\0') break;
+
+    if (counter == 0) {
+      pstr += sprintf(pstr, " (%s %s", tableDes->cols[counter].field, tableDes->cols[counter].type);
+    } else {
+      pstr += sprintf(pstr, ", %s %s", tableDes->cols[counter].field, tableDes->cols[counter].type);
+    }
+
+    if (strcasecmp(tableDes->cols[counter].type, "binary") == 0 ||
+        strcasecmp(tableDes->cols[counter].type, "nchar") == 0) {
+      pstr += sprintf(pstr, "(%d)", tableDes->cols[counter].length);
+    }
+  }
+
+  count_temp = counter;
+
+  for (; counter < numOfCols; counter++) {
+    if (counter == count_temp) {
+      pstr += sprintf(pstr, ") TAGS (%s %s", tableDes->cols[counter].field, tableDes->cols[counter].type);
+    } else {
+      pstr += sprintf(pstr, ", %s %s", tableDes->cols[counter].field, tableDes->cols[counter].type);
+    }
+
+    if (strcasecmp(tableDes->cols[counter].type, "binary") == 0 ||
+        strcasecmp(tableDes->cols[counter].type, "nchar") == 0) {
+      pstr += sprintf(pstr, "(%d)", tableDes->cols[counter].length);
+    }
+  }
+
+  pstr += sprintf(pstr, ");");
+
+  fprintf(fp, "%s\n", tmpBuf);
+
+  free(tmpBuf);
+}
+
+void taosDumpCreateMTableClause(STableDef *tableDes, char *metric, int numOfCols, FILE *fp) {
+  int counter = 0;
+  int count_temp = 0;
+
+  char* tmpBuf = (char *)malloc(COMMAND_SIZE);
+  if (tmpBuf == NULL) {
+    fprintf(stderr, "failed to allocate memory\n");
+    return;
+  }
+
+  char *pstr = NULL;
+  pstr = tmpBuf;
+
+  pstr += sprintf(tmpBuf, "CREATE TABLE IF NOT EXISTS %s USING %s TAGS (", tableDes->name, metric);
+
+  for (; counter < numOfCols; counter++) {
+    if (tableDes->cols[counter].note[0] != '\0') break;
+  }
+
+  assert(counter < numOfCols);
+  count_temp = counter;
+
+  for (; counter < numOfCols; counter++) {
+    if (counter != count_temp) {
+      if (strcasecmp(tableDes->cols[counter].type, "binary") == 0 ||
+          strcasecmp(tableDes->cols[counter].type, "nchar") == 0) {
+        //pstr += sprintf(pstr, ", \'%s\'", tableDes->cols[counter].note);
+        pstr += sprintf(pstr, ", %s", tableDes->cols[counter].note);
+      } else {
+        pstr += sprintf(pstr, ", %s", tableDes->cols[counter].note);
+      }
+    } else {
+      if (strcasecmp(tableDes->cols[counter].type, "binary") == 0 ||
+          strcasecmp(tableDes->cols[counter].type, "nchar") == 0) {
+        //pstr += sprintf(pstr, "\'%s\'", tableDes->cols[counter].note);
+        pstr += sprintf(pstr, "%s", tableDes->cols[counter].note);
+      } else {
+        pstr += sprintf(pstr, "%s", tableDes->cols[counter].note);
+      }
+      /* pstr += sprintf(pstr, "%s", tableDes->cols[counter].note); */
+    }
+
+    /* if (strcasecmp(tableDes->cols[counter].type, "binary") == 0 || strcasecmp(tableDes->cols[counter].type, "nchar")
+     * == 0) { */
+    /*     pstr += sprintf(pstr, "(%d)", tableDes->cols[counter].length); */
+    /* } */
+  }
+
+  pstr += sprintf(pstr, ");");
+
+  fprintf(fp, "%s\n", tmpBuf);
+  free(tmpBuf);
+}
+
+int taosDumpTableData(FILE *fp, char *tbname, struct arguments *arguments, TAOS* taosCon) {
+  /* char       temp[MAX_COMMAND_SIZE] = "\0"; */
+  int64_t totalRows = 0;
+  int count = 0;
+  char *pstr = NULL;
+  TAOS_ROW row = NULL;
+  int numFields = 0;
+  char *tbuf = NULL;
+
+  char* tmpCommand = (char *)calloc(1, COMMAND_SIZE);
+  if (tmpCommand == NULL) {
+    fprintf(stderr, "failed to allocate memory\n");
+    return -1;
+  }
+
+  char* tmpBuffer = (char *)calloc(1, COMMAND_SIZE);
+  if (tmpBuffer == NULL) {
+    fprintf(stderr, "failed to allocate memory\n");
+    free(tmpCommand);
+    return -1;
+  }
+
+  pstr = tmpBuffer;
+
+  if (arguments->schemaonly) {
+    free(tmpCommand);
+    free(tmpBuffer);
+    return 0;
+  }
+  
+  sprintf(tmpCommand, 
+          "select * from %s where _c0 >= %" PRId64 " and _c0 <= %" PRId64 " order by _c0 asc", 
+          tbname, 
+          arguments->start_time,
+          arguments->end_time);
+  
+  TAOS_RES* tmpResult = taos_query(taosCon, tmpCommand);  
+  int32_t code = taos_errno(tmpResult);
+  if (code != 0) {
+    fprintf(stderr, "failed to run command %s, reason: %s\n", tmpCommand, taos_errstr(taosCon));
+    free(tmpCommand);
+    free(tmpBuffer);
+    taos_free_result(tmpResult);
+    return -1;
+  }
+
+  numFields = taos_field_count(tmpResult);
+  assert(numFields > 0);
+  TAOS_FIELD *fields = taos_fetch_fields(tmpResult);
+  tbuf = (char *)malloc(COMMAND_SIZE);
+  if (tbuf == NULL) {
+    fprintf(stderr, "No enough memory\n");
+    free(tmpCommand);
+    free(tmpBuffer);
+    taos_free_result(tmpResult);
+    return -1;
+  }
+
+  char sqlStr[8] = "\0";
+  if (arguments->mysqlFlag) {
+    sprintf(sqlStr, "INSERT");
+  } else {
+    sprintf(sqlStr, "IMPORT");
+  }
+
+  int rowFlag = 0;
+  count = 0;
+  while ((row = taos_fetch_row(tmpResult)) != NULL) {
+    pstr = tmpBuffer;
+    
+    int32_t* length = taos_fetch_lengths(tmpResult);   // act len
+
+    if (count == 0) {
+      pstr += sprintf(pstr, "%s INTO %s VALUES (", sqlStr, tbname);
+    } else {      
+      if (arguments->mysqlFlag) {
+        if (0 == rowFlag) {
+          pstr += sprintf(pstr, "(");
+          rowFlag++;
+        } else {
+          pstr += sprintf(pstr, ", (");
+        }
+      } else {
+        pstr += sprintf(pstr, "(");
+      }
+    }
+
+    for (int col = 0; col < numFields; col++) {
+      if (col != 0) pstr += sprintf(pstr, ", ");
+
+      if (row[col] == NULL) {
+        pstr += sprintf(pstr, "NULL");
+        continue;
+      }
+
+      switch (fields[col].type) {
+        case TSDB_DATA_TYPE_BOOL:
+          pstr += sprintf(pstr, "%d", ((((int32_t)(*((char *)row[col]))) == 1) ? 1 : 0));
+          break;
+        case TSDB_DATA_TYPE_TINYINT:
+          pstr += sprintf(pstr, "%d", *((int8_t *)row[col]));
+          break;
+        case TSDB_DATA_TYPE_SMALLINT:
+          pstr += sprintf(pstr, "%d", *((int16_t *)row[col]));
+          break;
+        case TSDB_DATA_TYPE_INT:
+          pstr += sprintf(pstr, "%d", *((int32_t *)row[col]));
+          break;
+        case TSDB_DATA_TYPE_BIGINT:
+          pstr += sprintf(pstr, "%" PRId64 "", *((int64_t *)row[col]));
+          break;
+        case TSDB_DATA_TYPE_FLOAT:
+          pstr += sprintf(pstr, "%f", GET_FLOAT_VAL(row[col]));
+          break;
+        case TSDB_DATA_TYPE_DOUBLE:
+          pstr += sprintf(pstr, "%f", GET_DOUBLE_VAL(row[col]));
+          break;
+        case TSDB_DATA_TYPE_BINARY:
+          *(pstr++) = '\'';
+          converStringToReadable((char *)row[col], length[col], tbuf, COMMAND_SIZE);
+          pstr = stpcpy(pstr, tbuf);
+          *(pstr++) = '\'';
+          break;
+        case TSDB_DATA_TYPE_NCHAR:
+          convertNCharToReadable((char *)row[col], length[col], tbuf, COMMAND_SIZE);
+          pstr += sprintf(pstr, "\'%s\'", tbuf);
+          break;
+        case TSDB_DATA_TYPE_TIMESTAMP:
+          if (!arguments->mysqlFlag) {
+            pstr += sprintf(pstr, "%" PRId64 "", *(int64_t *)row[col]);
+          } else {
+            char buf[64] = "\0";
+            int64_t ts = *((int64_t *)row[col]);
+            time_t tt = (time_t)(ts / 1000);
+            struct tm *ptm = localtime(&tt);
+            strftime(buf, 64, "%y-%m-%d %H:%M:%S", ptm);
+            pstr += sprintf(pstr, "\'%s.%03d\'", buf, (int)(ts % 1000));
+          }
+          break;
+        default:
+          break;
+      }
+    }
+
+    pstr += sprintf(pstr, ") ");
+
+    totalRows++;
+    count++;
+    fprintf(fp, "%s", tmpBuffer);
+
+    if (count >= arguments->data_batch) {
+      fprintf(fp, ";\n");
+      count = 0;
+    } //else {
+      //fprintf(fp, "\\\n");
+    //}
+  }
+
+  atomic_add_fetch_64(&totalDumpOutRows, totalRows);
+
+  fprintf(fp, "\n");
+
+  if (tbuf) {
+    free(tbuf);
+  }
+  
+  taos_free_result(tmpResult);
+  tmpResult = NULL;
+  free(tmpCommand);
+  free(tmpBuffer);
+  return 0;
+}
+
+int taosCheckParam(struct arguments *arguments) {
+  if (arguments->all_databases && arguments->databases) {
+    fprintf(stderr, "conflict option --all-databases and --databases\n");
+    return -1;
+  }
+
+  if (arguments->start_time > arguments->end_time) {
+    fprintf(stderr, "start time is larger than end time\n");
+    return -1;
+  }
+  
+  if (arguments->arg_list_len == 0) {
+    if ((!arguments->all_databases) && (!arguments->isDumpIn)) {
+      fprintf(stderr, "taosdump requires parameters\n");
+      return -1;
+    }
+  }
+/*
+  if (arguments->isDumpIn && (strcmp(arguments->outpath, DEFAULT_DUMP_FILE) != 0)) {
+    fprintf(stderr, "duplicate parameter input and output file path\n");
+    return -1;
+  }
+*/
+  if (!arguments->isDumpIn && arguments->encode != NULL) {
+    fprintf(stderr, "invalid option in dump out\n");
+    return -1;
+  }
+
+  if (arguments->table_batch <= 0) {
+    fprintf(stderr, "invalid option in dump out\n");
+    return -1;
+  }
+
+  return 0;
+}
+
+bool isEmptyCommand(char *cmd) {
+  char *pchar = cmd;
+
+  while (*pchar != '\0') {
+    if (*pchar != ' ') return false;
+    pchar++;
+  }
+
+  return true;
+}
+
+void taosReplaceCtrlChar(char *str) {
+  _Bool ctrlOn = false;
+  char *pstr = NULL;
+
+  for (pstr = str; *str != '\0'; ++str) {
+    if (ctrlOn) {
+      switch (*str) {
+        case 'n':
+          *pstr = '\n';
+          pstr++;
+          break;
+        case 'r':
+          *pstr = '\r';
+          pstr++;
+          break;
+        case 't':
+          *pstr = '\t';
+          pstr++;
+          break;
+        case '\\':
+          *pstr = '\\';
+          pstr++;
+          break;
+        case '\'':
+          *pstr = '\'';
+          pstr++;
+          break;
+        default:
+          break;
+      }
+      ctrlOn = false;
+    } else {
+      if (*str == '\\') {
+        ctrlOn = true;
+      } else {
+        *pstr = *str;
+        pstr++;
+      }
+    }
+  }
+
+  *pstr = '\0';
+}
+
+char *ascii_literal_list[] = {
+    "\\x00", "\\x01", "\\x02", "\\x03", "\\x04", "\\x05", "\\x06", "\\x07", "\\x08", "\\t",   "\\n",   "\\x0b", "\\x0c",
+    "\\r",   "\\x0e", "\\x0f", "\\x10", "\\x11", "\\x12", "\\x13", "\\x14", "\\x15", "\\x16", "\\x17", "\\x18", "\\x19",
+    "\\x1a", "\\x1b", "\\x1c", "\\x1d", "\\x1e", "\\x1f", " ",     "!",     "\\\"",  "#",     "$",     "%",     "&",
+    "\\'",   "(",     ")",     "*",     "+",     ",",     "-",     ".",     "/",     "0",     "1",     "2",     "3",
+    "4",     "5",     "6",     "7",     "8",     "9",     ":",     ";",     "<",     "=",     ">",     "?",     "@",
+    "A",     "B",     "C",     "D",     "E",     "F",     "G",     "H",     "I",     "J",     "K",     "L",     "M",
+    "N",     "O",     "P",     "Q",     "R",     "S",     "T",     "U",     "V",     "W",     "X",     "Y",     "Z",
+    "[",     "\\\\",  "]",     "^",     "_",     "`",     "a",     "b",     "c",     "d",     "e",     "f",     "g",
+    "h",     "i",     "j",     "k",     "l",     "m",     "n",     "o",     "p",     "q",     "r",     "s",     "t",
+    "u",     "v",     "w",     "x",     "y",     "z",     "{",     "|",     "}",     "~",     "\\x7f", "\\x80", "\\x81",
+    "\\x82", "\\x83", "\\x84", "\\x85", "\\x86", "\\x87", "\\x88", "\\x89", "\\x8a", "\\x8b", "\\x8c", "\\x8d", "\\x8e",
+    "\\x8f", "\\x90", "\\x91", "\\x92", "\\x93", "\\x94", "\\x95", "\\x96", "\\x97", "\\x98", "\\x99", "\\x9a", "\\x9b",
+    "\\x9c", "\\x9d", "\\x9e", "\\x9f", "\\xa0", "\\xa1", "\\xa2", "\\xa3", "\\xa4", "\\xa5", "\\xa6", "\\xa7", "\\xa8",
+    "\\xa9", "\\xaa", "\\xab", "\\xac", "\\xad", "\\xae", "\\xaf", "\\xb0", "\\xb1", "\\xb2", "\\xb3", "\\xb4", "\\xb5",
+    "\\xb6", "\\xb7", "\\xb8", "\\xb9", "\\xba", "\\xbb", "\\xbc", "\\xbd", "\\xbe", "\\xbf", "\\xc0", "\\xc1", "\\xc2",
+    "\\xc3", "\\xc4", "\\xc5", "\\xc6", "\\xc7", "\\xc8", "\\xc9", "\\xca", "\\xcb", "\\xcc", "\\xcd", "\\xce", "\\xcf",
+    "\\xd0", "\\xd1", "\\xd2", "\\xd3", "\\xd4", "\\xd5", "\\xd6", "\\xd7", "\\xd8", "\\xd9", "\\xda", "\\xdb", "\\xdc",
+    "\\xdd", "\\xde", "\\xdf", "\\xe0", "\\xe1", "\\xe2", "\\xe3", "\\xe4", "\\xe5", "\\xe6", "\\xe7", "\\xe8", "\\xe9",
+    "\\xea", "\\xeb", "\\xec", "\\xed", "\\xee", "\\xef", "\\xf0", "\\xf1", "\\xf2", "\\xf3", "\\xf4", "\\xf5", "\\xf6",
+    "\\xf7", "\\xf8", "\\xf9", "\\xfa", "\\xfb", "\\xfc", "\\xfd", "\\xfe", "\\xff"};
+
+int converStringToReadable(char *str, int size, char *buf, int bufsize) {
+  char *pstr = str;
+  char *pbuf = buf;
+  while (size > 0) {
+    if (*pstr == '\0') break;
+    pbuf = stpcpy(pbuf, ascii_literal_list[((uint8_t)(*pstr))]);
+    pstr++;
+    size--;
+  }
+  *pbuf = '\0';
+  return 0;
+}
+
+int convertNCharToReadable(char *str, int size, char *buf, int bufsize) {
+  char *pstr = str;
+  char *pbuf = buf;
+  // TODO
+  wchar_t wc;
+  while (size > 0) {
+    if (*pstr == '\0') break;
+    int byte_width = mbtowc(&wc, pstr, MB_CUR_MAX);
+    if (byte_width < 0) {
+      fprintf(stderr, "mbtowc() return fail.\n");
+      exit(-1);
+    }
+
+    if ((int)wc < 256) {
+      pbuf = stpcpy(pbuf, ascii_literal_list[(int)wc]);
+    } else {
+      memcpy(pbuf, pstr, byte_width);
+      pbuf += byte_width;
+    }
+    pstr += byte_width;
+  }
+
+  *pbuf = '\0';
+
+  return 0;
+}
+
+void taosDumpCharset(FILE *fp) {
+  char charsetline[256];
+
+  (void)fseek(fp, 0, SEEK_SET);
+  sprintf(charsetline, "#!%s\n", tsCharset);
+  (void)fwrite(charsetline, strlen(charsetline), 1, fp);
+}
+
+void taosLoadFileCharset(FILE *fp, char *fcharset) {
+  char * line = NULL;
+  size_t line_size = 0;
+
+  (void)fseek(fp, 0, SEEK_SET);
+  ssize_t size = getline(&line, &line_size, fp);
+  if (size <= 2) {
+    goto _exit_no_charset;
+  }
+
+  if (strncmp(line, "#!", 2) != 0) {
+    goto _exit_no_charset;
+  }
+  if (line[size - 1] == '\n') {
+    line[size - 1] = '\0';
+    size--;
+  }
+  strcpy(fcharset, line + 2);
+
+  tfree(line);
+  return;
+
+_exit_no_charset:
+  (void)fseek(fp, 0, SEEK_SET);
+  *fcharset = '\0';
+  tfree(line);
+  return;
+}
+
+// ========  dumpIn support multi threads functions ================================//
+
+static char    **tsDumpInSqlFiles   = NULL;
+static int32_t   tsSqlFileNum = 0;
+static char      tsDbSqlFile[TSDB_FILENAME_LEN] = {0};
+static char      tsfCharset[64] = {0};
+static int taosGetFilesNum(const char *directoryName, const char *prefix)
+{
+  char cmd[1024] = { 0 };
+  sprintf(cmd, "ls %s/*.%s | wc -l ", directoryName, prefix);
+
+  FILE *fp = popen(cmd, "r");
+  if (fp == NULL) {
+    fprintf(stderr, "ERROR: failed to execute:%s, error:%s\n", cmd, strerror(errno));
+    exit(0);
+  }
+
+  int fileNum = 0;
+  if (fscanf(fp, "%d", &fileNum) != 1) {
+    fprintf(stderr, "ERROR: failed to execute:%s, parse result error\n", cmd);
+    exit(0);
+  }
+
+  if (fileNum <= 0) {
+    fprintf(stderr, "ERROR: directory:%s is empry\n", directoryName);
+    exit(0);
+  }
+
+  pclose(fp);
+  return fileNum;
+}
+
+static void taosParseDirectory(const char *directoryName, const char *prefix, char **fileArray, int totalFiles)
+{
+  char cmd[1024] = { 0 };
+  sprintf(cmd, "ls %s/*.%s | sort", directoryName, prefix);
+
+  FILE *fp = popen(cmd, "r");
+  if (fp == NULL) {
+    fprintf(stderr, "ERROR: failed to execute:%s, error:%s\n", cmd, strerror(errno));
+    exit(0);
+  }
+
+  int fileNum = 0;
+  while (fscanf(fp, "%128s", fileArray[fileNum++])) {
+    if (strcmp(fileArray[fileNum-1], tsDbSqlFile) == 0) {
+      fileNum--;
+    }
+    if (fileNum >= totalFiles) {
+      break;
+    }
+  }
+
+  if (fileNum != totalFiles) {
+    fprintf(stderr, "ERROR: directory:%s changed while read\n", directoryName);
+    pclose(fp);
+    exit(0);
+  }
+
+  pclose(fp);
+}
+
+static void taosCheckTablesSQLFile(const char *directoryName)
+{
+  char cmd[1024] = { 0 };
+  sprintf(cmd, "ls %s/dbs.sql", directoryName);
+
+  FILE *fp = popen(cmd, "r");
+  if (fp == NULL) {
+    fprintf(stderr, "ERROR: failed to execute:%s, error:%s\n", cmd, strerror(errno));
+    exit(0);
+  }
+
+  while (fscanf(fp, "%128s", tsDbSqlFile)) {
+    break;
+  }
+
+  pclose(fp);
+}
+
+static void taosMallocSQLFiles()
+{
+  tsDumpInSqlFiles = (char**)calloc(tsSqlFileNum, sizeof(char*));
+  for (int i = 0; i < tsSqlFileNum; i++) {
+    tsDumpInSqlFiles[i] = calloc(1, TSDB_FILENAME_LEN);
+  }
+}
+
+static void taosFreeSQLFiles()
+{
+  for (int i = 0; i < tsSqlFileNum; i++) {
+    tfree(tsDumpInSqlFiles[i]);
+  }
+  tfree(tsDumpInSqlFiles);
+}
+
+static void taosGetDirectoryFileList(char *inputDir)
+{
+  struct stat fileStat;
+  if (stat(inputDir, &fileStat) < 0) {
+    fprintf(stderr, "ERROR: %s not exist\n", inputDir);
+    exit(0);
+  }
+
+  if (fileStat.st_mode & S_IFDIR) {
+    taosCheckTablesSQLFile(inputDir);
+    tsSqlFileNum = taosGetFilesNum(inputDir, "sql");
+    int totalSQLFileNum = tsSqlFileNum;
+    if (tsDbSqlFile[0] != 0) {
+      tsSqlFileNum--;
+    }
+    taosMallocSQLFiles();
+    taosParseDirectory(inputDir, "sql", tsDumpInSqlFiles, tsSqlFileNum);
+    fprintf(stdout, "\nstart to dispose %d files in %s\n", totalSQLFileNum, inputDir);
+  }
+  else {
+    fprintf(stderr, "ERROR: %s is not a directory\n", inputDir);
+    exit(0);
+  }
+}
+
+static FILE*  taosOpenDumpInFile(char *fptr) {
+  wordexp_t full_path;
+
+  if (wordexp(fptr, &full_path, 0) != 0) {
+    fprintf(stderr, "ERROR: illegal file name: %s\n", fptr);
+    return NULL;
+  }
+
+  char *fname = full_path.we_wordv[0];
+  
+  FILE *f = fopen(fname, "r");
+  if (f == NULL) {
+    fprintf(stderr, "ERROR: failed to open file %s\n", fname);
+    wordfree(&full_path);
+    return NULL;
+  }
+
+  wordfree(&full_path);
+
+  return f;
+}
+
+int taosDumpInOneFile_old(TAOS     * taos, FILE* fp, char* fcharset, char* encode) {
+  char     *command       = NULL;
+  char     *lcommand      = NULL;
+  int       tsize         = 0;
+  char     *line          = NULL;
+  _Bool     isRun         = true;
+  size_t    line_size     = 0;
+  char     *pstr          = NULL;
+  char     *lstr          = NULL;
+  size_t    inbytesleft   = 0;
+  size_t    outbytesleft  = COMMAND_SIZE;
+  char     *tcommand      = NULL;
+  char     *charsetOfFile = NULL;
+  iconv_t   cd            = (iconv_t)(-1);
+
+  command  = (char *)malloc(COMMAND_SIZE);
+  lcommand = (char *)malloc(COMMAND_SIZE);
+  if (command == NULL || lcommand == NULL) {
+    fprintf(stderr, "failed to connect to allocate memory\n");
+    goto _dumpin_exit_failure;
+  }
+
+  // Resolve locale
+  if (*fcharset != '\0') {
+    charsetOfFile = fcharset;
+  } else {
+    charsetOfFile = encode;
+  }
+
+  if (charsetOfFile != NULL && strcasecmp(tsCharset, charsetOfFile) != 0) {
+    cd = iconv_open(tsCharset, charsetOfFile);
+    if (cd == ((iconv_t)(-1))) {
+      fprintf(stderr, "Failed to open iconv handle\n");
+      goto _dumpin_exit_failure;
+    }
+  }
+
+  pstr = command;
+  int64_t linenu = 0;
+  while (1) {
+    ssize_t size = getline(&line, &line_size, fp);
+    linenu++;
+    if (size <= 0) break;
+    if (size == 1) {
+      if (pstr != command) {
+        inbytesleft = pstr - command;
+        memset(lcommand, 0, COMMAND_SIZE);
+        pstr = command;
+        lstr = lcommand;
+        outbytesleft = COMMAND_SIZE;
+        if (cd != ((iconv_t)(-1))) {
+          iconv(cd, &pstr, &inbytesleft, &lstr, &outbytesleft);
+          tcommand = lcommand;
+        } else {
+          tcommand = command;
+        }
+        
+        taosReplaceCtrlChar(tcommand);
+        
+        if (queryDB(taos, tcommand) != 0) {
+          fprintf(stderr, "error sql: linenu: %" PRId64 " failed\n", linenu);
+          exit(0);
+        }
+        
+        pstr = command;
+        pstr[0] = '\0';
+        tsize = 0;
+        isRun = true;
+      }
+
+      continue;
+    }
+
+    /* if (line[0] == '-' && line[1] == '-') continue; */
+
+    line[size - 1] = 0;
+
+    if (tsize + size - 1 > COMMAND_SIZE) {
+      fprintf(stderr, "command is too long\n");
+      goto _dumpin_exit_failure;
+    }
+
+    if (line[size - 2] == '\\') {
+      line[size - 2] = ' ';
+      isRun = false;
+    } else {
+      isRun = true;
+    }
+
+    memcpy(pstr, line, size - 1);
+    pstr += (size - 1);
+    *pstr = '\0';
+
+    if (!isRun) continue;
+
+    if (command != pstr && !isEmptyCommand(command)) {
+      inbytesleft = pstr - command;
+      memset(lcommand, 0, COMMAND_SIZE);
+      pstr = command;
+      lstr = lcommand;
+      outbytesleft = COMMAND_SIZE;
+      if (cd != ((iconv_t)(-1))) {
+        iconv(cd, &pstr, &inbytesleft, &lstr, &outbytesleft);
+        tcommand = lcommand;
+      } else {
+        tcommand = command;
+      }
+      taosReplaceCtrlChar(tcommand);
+      if (queryDB(taos, tcommand) != 0) {
+        fprintf(stderr, "error sql: linenu:%" PRId64 " failed\n", linenu);
+        exit(0); 
+      }
+    }
+
+    pstr = command;
+    *pstr = '\0';
+    tsize = 0;
+  }
+
+  if (pstr != command) {
+    inbytesleft = pstr - command;
+    memset(lcommand, 0, COMMAND_SIZE);
+    pstr = command;
+    lstr = lcommand;
+    outbytesleft = COMMAND_SIZE;
+    if (cd != ((iconv_t)(-1))) {
+      iconv(cd, &pstr, &inbytesleft, &lstr, &outbytesleft);
+      tcommand = lcommand;
+    } else {
+      tcommand = command;
+    }
+    taosReplaceCtrlChar(lcommand);
+    if (queryDB(taos, tcommand) != 0)
+      fprintf(stderr, "error sql: linenu:%" PRId64 " failed \n", linenu);
+  }
+
+  if (cd != ((iconv_t)(-1))) iconv_close(cd);
+  tfree(line);
+  tfree(command);
+  tfree(lcommand);
+  taos_close(taos);
+  fclose(fp);
+  return 0;
+
+_dumpin_exit_failure:
+  if (cd != ((iconv_t)(-1))) iconv_close(cd);
+  tfree(command);
+  tfree(lcommand);
+  taos_close(taos);
+  fclose(fp);
+  return -1;
+}
+
+int taosDumpInOneFile(TAOS     * taos, FILE* fp, char* fcharset, char* encode, char* fileName) {
+  int       read_len = 0;
+  char *    cmd      = NULL;
+  size_t    cmd_len  = 0;
+  char *    line     = NULL;
+  size_t    line_len = 0;
+
+  cmd  = (char *)malloc(COMMAND_SIZE);
+  if (cmd == NULL) {
+    fprintf(stderr, "failed to allocate memory\n");
+    return -1;
+  }
+
+  int lineNo = 0;
+  while ((read_len = getline(&line, &line_len, fp)) != -1) {
+    ++lineNo;
+    if (read_len >= COMMAND_SIZE) continue;
+    line[--read_len] = '\0';
+
+    //if (read_len == 0 || isCommentLine(line)) {  // line starts with #
+    if (read_len == 0 ) { 
+      continue;
+    }
+
+    if (line[read_len - 1] == '\\') {
+      line[read_len - 1] = ' ';
+      memcpy(cmd + cmd_len, line, read_len);
+      cmd_len += read_len;
+      continue;
+    }
+
+    memcpy(cmd + cmd_len, line, read_len);
+    cmd[read_len + cmd_len]= '\0';
+    if (queryDB(taos, cmd)) {
+      fprintf(stderr, "error sql: linenu:%d, file:%s\n", lineNo, fileName);
+    }
+
+    memset(cmd, 0, COMMAND_SIZE);
+    cmd_len = 0;
+  }
+
+  tfree(cmd);
+  tfree(line);
+  fclose(fp);
+  return 0;
+}
+
+void* taosDumpInWorkThreadFp(void *arg)
+{
+  SThreadParaObj *pThread = (SThreadParaObj*)arg;
+  for (int32_t f = 0; f < tsSqlFileNum; ++f) {
+    if (f % pThread->totalThreads == pThread->threadIndex) {
+      char *SQLFileName = tsDumpInSqlFiles[f];
+      FILE* fp = taosOpenDumpInFile(SQLFileName);
+      if (NULL == fp) {
+        continue;
+      }
+      fprintf(stderr, "Success Open input file: %s\n", SQLFileName);
+      taosDumpInOneFile(pThread->taosCon, fp, tsfCharset, tsArguments.encode, SQLFileName);
+    }
+  }
+
+  return NULL;
+}
+
+static void taosStartDumpInWorkThreads(struct arguments *args)
+{
+  pthread_attr_t  thattr;
+  SThreadParaObj *pThread;
+  int32_t         totalThreads = args->thread_num;
+
+  if (totalThreads > tsSqlFileNum) {
+    totalThreads = tsSqlFileNum;
+  }
+  
+  SThreadParaObj *threadObj = (SThreadParaObj *)calloc(totalThreads, sizeof(SThreadParaObj));
+  for (int32_t t = 0; t < totalThreads; ++t) {
+    pThread = threadObj + t;
+    pThread->threadIndex = t;
+    pThread->totalThreads = totalThreads;
+    pThread->taosCon = taos_connect(args->host, args->user, args->password, NULL, args->port);
+    if (pThread->taosCon == NULL) {
+      fprintf(stderr, "ERROR: thread:%d failed connect to TDengine, error:%s\n", pThread->threadIndex, taos_errstr(pThread->taosCon));
+      exit(0);
+    }
+
+    pthread_attr_init(&thattr);
+    pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
+
+    if (pthread_create(&(pThread->threadID), &thattr, taosDumpInWorkThreadFp, (void*)pThread) != 0) {
+      fprintf(stderr, "ERROR: thread:%d failed to start\n", pThread->threadIndex);
+      exit(0);
+    }
+  }
+
+  for (int t = 0; t < totalThreads; ++t) {
+    pthread_join(threadObj[t].threadID, NULL);
+  }
+
+  for (int t = 0; t < totalThreads; ++t) {
+    taos_close(threadObj[t].taosCon);
+  }
+  free(threadObj);
+}
+
+
+int taosDumpIn(struct arguments *arguments) {
+  assert(arguments->isDumpIn);
+  
+  TAOS     *taos    = NULL;
+  FILE     *fp      = NULL;
+
+  taos = taos_connect(arguments->host, arguments->user, arguments->password, NULL, arguments->port);
+  if (taos == NULL) {
+    fprintf(stderr, "failed to connect to TDengine server\n");
+    return -1;
+  }
+
+  taosGetDirectoryFileList(arguments->inpath);
+
+  if (tsDbSqlFile[0] != 0) {
+    fp = taosOpenDumpInFile(tsDbSqlFile);
+    if (NULL == fp) {
+      fprintf(stderr, "failed to open input file %s\n", tsDbSqlFile);
+      return -1;
+    }
+    fprintf(stderr, "Success Open input file: %s\n", tsDbSqlFile);
+    
+    taosLoadFileCharset(fp, tsfCharset);
+    
+    taosDumpInOneFile(taos, fp, tsfCharset, arguments->encode, tsDbSqlFile);
+  }
+
+  taosStartDumpInWorkThreads(arguments);
+
+  taos_close(taos);
+  taosFreeSQLFiles();
+  return 0;
+}
+
+

src/kit/taosdump/taosdump.sh

+taos1_6="/root/mnt/work/test/td1.6/build/bin/taos"
+taosdump1_6="/root/mnt/work/test/td1.6/build/bin/taosdump"
+taoscfg1_6="/root/mnt/work/test/td1.6/test/cfg"
+
+taos2_0="/root/mnt/work/test/td2.0/build/bin/taos"
+taosdump2_0="/root/mnt/work/test/td2.0/build/bin/taosdump"
+taoscfg2_0="/root/mnt/work/test/td2.0/test/cfg"
+
+data_dir="/root/mnt/work/test/td1.6/output"
+table_list="/root/mnt/work/test/td1.6/tables"
+
+DBNAME="test"
+NTABLES=$(wc -l ${table_list} | awk '{print $1;}')
+NTABLES_PER_DUMP=101
+
+mkdir -p ${data_dir}
+i=0
+round=0
+command="${taosdump1_6} -c ${taoscfg1_6} -o ${data_dir} -N 100 -T 20 ${DBNAME}"
+while IFS= read -r line
+do
+  i=$((i+1))
+
+  command="${command} ${line}"
+
+  if [[ "$i" -eq ${NTABLES_PER_DUMP} ]]; then
+    round=$((round+1))
+    echo "Starting round ${round} dump out..."
+    rm -f ${data_dir}/*
+    ${command}
+    echo "Starting round ${round} dump in..."
+    ${taosdump2_0} -c ${taoscfg2_0} -i ${data_dir}
+
+    # Reset variables
+    # command="${taosdump1_6} -c ${taoscfg1_6} -o ${data_dir} -N 100 ${DBNAME}"
+    command="${taosdump1_6} -c ${taoscfg1_6} -o ${data_dir} -N 100 -T 20 ${DBNAME}"
+    i=0
+  fi
+done < "${table_list}"
+
+if [[ ${i} -ne "0" ]]; then
+  round=$((round+1))
+  echo "Starting round ${round} dump out..."
+  rm -f ${data_dir}/*
+  ${command}
+  echo "Starting round ${round} dump in..."
+  ${taosdump2_0} -c ${taoscfg2_0} -i ${data_dir}
+fi

src/mnode/src/mnodeSdb.c

@@ -71,7 +71,7 @@ typedef struct _SSdbTable {
 typedef struct {
   ESyncRole  role;
   ESdbStatus status;
-  int64_t    version;
+  uint64_t   version;
   int64_t    sync;
   void *     wal;
   SSyncCfg   cfg;

src/query/inc/qAst.h

@@ -74,20 +74,18 @@ typedef struct tExprNode {
   };
 } tExprNode;
 
-void tExprTreeDestroy(tExprNode **pExprs, void (*fp)(void*));
-
 void tExprTreeTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, SExprTraverseSupp *param);
 
 void tExprTreeCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, void *param, int32_t order,
                                 char *(*cb)(void *, const char*, int32_t));
 
-uint8_t getBinaryExprOptr(SStrToken *pToken);
+tExprNode* exprTreeFromBinary(const void* data, size_t size);
+tExprNode* exprTreeFromTableName(const char* tbnameCond);
 
-void tExprNodeDestroy(tExprNode *pNode, void (*fp)(void *));
 void exprTreeToBinary(SBufferWriter* bw, tExprNode* pExprTree);
 
-tExprNode* exprTreeFromBinary(const void* data, size_t size);
-tExprNode* exprTreeFromTableName(const char* tbnameCond);
+void tExprNodeDestroy(tExprNode *pNode, void (*fp)(void *));
+void tExprTreeDestroy(tExprNode **pExprs, void (*fp)(void*));
 
 #ifdef __cplusplus
 }

src/query/inc/qFill.h

@@ -28,6 +28,7 @@ typedef struct {
   STColumn col;             // column info
   int16_t  functionId;      // sql function id
   int16_t  flag;            // column flag: TAG COLUMN|NORMAL COLUMN
+  int16_t  tagIndex;        // index of current tag in SFillTagColInfo array list
   union {int64_t i; double d;} fillVal;
 } SFillColInfo;
 
@@ -38,25 +39,28 @@ typedef struct {
   
 typedef struct SFillInfo {
   TSKEY     start;                // start timestamp
-  TSKEY   endKey;               // endKey for fill
+  TSKEY     end;                  // endKey for fill
+  TSKEY     currentKey;           // current active timestamp, the value may be changed during the fill procedure.
   int32_t   order;                // order [TSDB_ORDER_ASC|TSDB_ORDER_DESC]
-  int32_t fillType;             // fill type
+  int32_t   type;                 // fill type
   int32_t   numOfRows;            // number of rows in the input data block
-  int32_t rowIdx;               // rowIdx
+  int32_t   index;                // active row index
   int32_t   numOfTotal;           // number of filled rows in one round
   int32_t   numOfCurrent;         // number of filled rows in current results
 
   int32_t   numOfTags;            // number of tags
   int32_t   numOfCols;            // number of columns, including the tags columns
   int32_t   rowSize;              // size of each row
-  SFillTagColInfo* pTags;       // tags value for filling gap
   SInterval interval;
   char *    prevValues;           // previous row of data, to generate the interpolation results
   char *    nextValues;           // next row of data
   char**    pData;                // original result data block involved in filling data
-  int32_t capacityInRows;       // data buffer size in rows
+  int32_t   alloc;                // data buffer size in rows
   int8_t    precision;            // time resoluation
+
   SFillColInfo* pFillCol;         // column info for fill operations
+  SFillTagColInfo* pTags;         // tags value for filling gap
+  void*     handle;               // for dubug purpose
 } SFillInfo;
 
 typedef struct SPoint {
@@ -66,7 +70,7 @@ typedef struct SPoint {
 
 SFillInfo* taosInitFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_t capacity, int32_t numOfCols,
                             int64_t slidingTime, int8_t slidingUnit, int8_t precision, int32_t fillType,
-                            SFillColInfo* pFillCol);
+                            SFillColInfo* pFillCol, void* handle);
 
 void taosResetFillInfo(SFillInfo* pFillInfo, TSKEY startTimestamp);
 
@@ -74,17 +78,17 @@ void* taosDestroyFillInfo(SFillInfo *pFillInfo);
 
 void taosFillSetStartInfo(SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey);
 
-void taosFillCopyInputDataFromFilePage(SFillInfo* pFillInfo, tFilePage** pInput);
+void taosFillCopyInputDataFromFilePage(SFillInfo* pFillInfo, const tFilePage** pInput);
 
-void taosFillCopyInputDataFromOneFilePage(SFillInfo* pFillInfo, tFilePage* pInput);
+void taosFillCopyInputDataFromOneFilePage(SFillInfo* pFillInfo, const tFilePage* pInput);
 
-int64_t getFilledNumOfRes(SFillInfo* pFillInfo, int64_t ekey, int32_t maxNumOfRows);
+int64_t getNumOfResWithFill(SFillInfo* pFillInfo, int64_t ekey, int32_t maxNumOfRows);
 
 int32_t taosNumOfRemainRows(SFillInfo *pFillInfo);
 
 int32_t taosGetLinearInterpolationVal(int32_t type, SPoint *point1, SPoint *point2, SPoint *point);
 
-int64_t taosGenerateDataBlock(SFillInfo* pFillInfo, tFilePage** output, int32_t capacity);
+int64_t taosFillResultDataBlock(SFillInfo* pFillInfo, tFilePage** output, int32_t capacity);
 
 #ifdef __cplusplus
 }

src/query/inc/qSqlparser.h

@@ -20,10 +20,10 @@
 extern "C" {
 #endif
 
-#include <tstrbuild.h>
 #include "taos.h"
 #include "taosmsg.h"
 #include "tstoken.h"
+#include "tstrbuild.h"
 #include "tvariant.h"
 
 #define ParseTOKENTYPE SStrToken
@@ -38,12 +38,6 @@ extern char tTokenTypeSwitcher[13];
     }                                          \
   } while (0)
 
-typedef struct tFieldList {
-  int32_t     nField;
-  int32_t     nAlloc;
-  TAOS_FIELD *p;
-} tFieldList;
-
 typedef struct SLimitVal {
   int64_t limit;
   int64_t offset;
@@ -59,12 +53,6 @@ typedef struct tVariantListItem {
   uint8_t  sortOrder;
 } tVariantListItem;
 
-typedef struct tVariantList {
-  int32_t           nExpr;  /* Number of expressions on the list */
-  int32_t           nAlloc; /* Number of entries allocated below */
-  tVariantListItem *a;      /* One entry for each expression */
-} tVariantList;
-
 typedef struct SIntervalVal {
   SStrToken interval;
   SStrToken offset;
@@ -72,16 +60,16 @@ typedef struct SIntervalVal {
 
 typedef struct SQuerySQL {
   struct tSQLExprList *pSelection;   // select clause
-  tVariantList *       from;         // from clause
+  SArray *             from;         // from clause  SArray<tVariantListItem>
   struct tSQLExpr *    pWhere;       // where clause [optional]
-  tVariantList *       pGroupby;     // groupby clause, only for tags[optional]
-  tVariantList *       pSortOrder;   // orderby [optional]
+  SArray *             pGroupby;     // groupby clause, only for tags[optional], SArray<tVariantListItem>
+  SArray *             pSortOrder;   // orderby [optional], SArray<tVariantListItem>
   SStrToken            interval;     // interval [optional]
   SStrToken            offset;       // offset window [optional]
   SStrToken            sliding;      // sliding window [optional]
   SLimitVal            limit;        // limit offset [optional]
   SLimitVal            slimit;       // group limit offset [optional]
-  tVariantList *       fillType;     // fill type[optional]
+  SArray *             fillType;     // fill type[optional], SArray<tVariantListItem>
   SStrToken            selectToken;  // sql string
 } SQuerySQL;
 
@@ -91,13 +79,13 @@ typedef struct SCreateTableSQL {
   
   int8_t           type; // create normal table/from super table/ stream
   struct {
-    tFieldList *pTagColumns;  // for normal table, pTagColumns = NULL;
-    tFieldList *pColumns;
+    SArray        *pTagColumns; // SArray<TAOS_FIELD>
+    SArray        *pColumns;    // SArray<TAOS_FIELD>
   } colInfo;
   
   struct {
     SStrToken      stableName;  // super table name, for using clause
-    tVariantList *pTagVals;    // create by using metric, tag value
+    SArray        *pTagVals;    // create by using metric, tag value
     STagData       tagdata;
   } usingInfo;
   
@@ -108,9 +96,8 @@ typedef struct SAlterTableSQL {
   SStrToken     name;
   int16_t       type;
   STagData      tagData;
-  
-  tFieldList *  pAddColumns;
-  tVariantList *varList;  // set t=val or: change src dst
+  SArray       *pAddColumns; // SArray<TAOS_FIELD>
+  SArray       *varList;  // set t=val or: change src dst, SArray<tVariantListItem>
 } SAlterTableSQL;
 
 typedef struct SCreateDBInfo {
@@ -131,7 +118,7 @@ typedef struct SCreateDBInfo {
   bool      ignoreExists;
   int8_t    update; 
   
-  tVariantList *keep;
+  SArray *keep;
 } SCreateDBInfo;
 
 typedef struct SCreateAcctSQL {
@@ -187,8 +174,8 @@ typedef struct SSqlInfo {
   
   union {
     SCreateTableSQL *pCreateTableInfo;
-    SAlterTableSQL * pAlterInfo;
-    tDCLSQL *        pDCLInfo;
+    SAlterTableSQL  *pAlterInfo;
+    tDCLSQL         *pDCLInfo;
   };
   
   SSubclauseInfo     subclauseInfo;
@@ -196,19 +183,18 @@ typedef struct SSqlInfo {
 } SSqlInfo;
 
 typedef struct tSQLExpr {
-  // TK_FUNCTION: sql function, TK_LE: less than(binary expr)
-  uint32_t nSQLOptr;
+  uint32_t         nSQLOptr;    // TK_FUNCTION: sql function, TK_LE: less than(binary expr)
   
   // the full sql string of function(col, param), which is actually the raw
   // field name, since the function name is kept in nSQLOptr already
   SStrToken        operand;
   SStrToken        colInfo;     // field id
   tVariant         val;         // value only for string, float, int
+  SStrToken        token;       // original sql expr string
 
   struct tSQLExpr *pLeft;       // left child
   struct tSQLExpr *pRight;      // right child
   struct tSQLExprList *pParam;  // function parameters
-  SStrToken token;              // original sql expr string
 } tSQLExpr;
 
 // used in select clause. select <tSQLExprList> from xxx
@@ -239,16 +225,9 @@ void Parse(void *yyp, int yymajor, ParseTOKENTYPE yyminor, SSqlInfo *);
  */
 void ParseFree(void *p, void (*freeProc)(void *));
 
-tVariantList *tVariantListAppend(tVariantList *pList, tVariant *pVar, uint8_t sortOrder);
-
-tVariantList *tVariantListInsert(tVariantList *pList, tVariant *pVar, uint8_t sortOrder, int32_t index);
-
-tVariantList *tVariantListAppendToken(tVariantList *pList, SStrToken *pAliasToken, uint8_t sortOrder);
-void          tVariantListDestroy(tVariantList *pList);
-
-tFieldList *tFieldListAppend(tFieldList *pList, TAOS_FIELD *pField);
-
-void tFieldListDestroy(tFieldList *pList);
+SArray *tVariantListAppend(SArray *pList, tVariant *pVar, uint8_t sortOrder);
+SArray *tVariantListInsert(SArray *pList, tVariant *pVar, uint8_t sortOrder, int32_t index);
+SArray *tVariantListAppendToken(SArray *pList, SStrToken *pAliasToken, uint8_t sortOrder);
 
 tSQLExpr *tSQLExprCreate(tSQLExpr *pLeft, tSQLExpr *pRight, int32_t optType);
 
@@ -258,17 +237,16 @@ tSQLExprList *tSQLExprListAppend(tSQLExprList *pList, tSQLExpr *pNode, SStrToken
 
 void tSQLExprListDestroy(tSQLExprList *pList);
 
-SQuerySQL *tSetQuerySQLElems(SStrToken *pSelectToken, tSQLExprList *pSelection, tVariantList *pFrom, tSQLExpr *pWhere,
-                             tVariantList *pGroupby, tVariantList *pSortOrder, SIntervalVal *pInterval,
-                             SStrToken *pSliding, tVariantList *pFill, SLimitVal *pLimit, SLimitVal *pGLimit);
+SQuerySQL *tSetQuerySQLElems(SStrToken *pSelectToken, tSQLExprList *pSelection, SArray *pFrom, tSQLExpr *pWhere,
+                             SArray *pGroupby, SArray *pSortOrder, SIntervalVal *pInterval,
+                             SStrToken *pSliding, SArray *pFill, SLimitVal *pLimit, SLimitVal *pGLimit);
 
-SCreateTableSQL *tSetCreateSQLElems(tFieldList *pCols, tFieldList *pTags, SStrToken *pMetricName,
-                                    tVariantList *pTagVals, SQuerySQL *pSelect, int32_t type);
+SCreateTableSQL *tSetCreateSQLElems(SArray *pCols, SArray *pTags, SStrToken *pMetricName,
+                                    SArray *pTagVals, SQuerySQL *pSelect, int32_t type);
 
 void tSQLExprNodeDestroy(tSQLExpr *pExpr);
-tSQLExpr *tSQLExprNodeClone(tSQLExpr *pExpr);
 
-SAlterTableSQL *tAlterTableSQLElems(SStrToken *pMeterName, tFieldList *pCols, tVariantList *pVals, int32_t type);
+SAlterTableSQL *tAlterTableSQLElems(SStrToken *pMeterName, SArray *pCols, SArray *pVals, int32_t type);
 
 void destroyAllSelectClause(SSubclauseInfo *pSql);
 void doDestroyQuerySql(SQuerySQL *pSql);
@@ -310,9 +288,6 @@ void tSQLSetColumnType(TAOS_FIELD *pField, SStrToken *pToken);
 
 void *ParseAlloc(void *(*mallocProc)(size_t));
 
-// convert the sql filter expression into binary data
-int32_t tSQLExprToBinary(tSQLExpr* pExpr, SStringBuilder* sb);
-
 enum {
   TSQL_NODE_TYPE_EXPR  = 0x1,
   TSQL_NODE_TYPE_ID    = 0x2,

src/query/inc/sql.y

@@ -223,8 +223,8 @@ acct_optr(Y) ::= pps(C) tseries(D) storage(P) streams(F) qtime(Q) dbs(E) users(K
     Y.stat    = M;
 }
 
-%type keep {tVariantList*}
-%destructor keep {tVariantListDestroy($$);}
+%type keep {SArray*}
+%destructor keep {taosArrayDestroy($$);}
 keep(Y)    ::= KEEP tagitemlist(X).           { Y = X; }
 
 cache(Y)   ::= CACHE INTEGER(X).              { Y = X; }
@@ -327,10 +327,10 @@ create_table_args(A) ::= AS select(S). {
 }
 
 %type column{TAOS_FIELD}
-%type columnlist{tFieldList*}
-%destructor columnlist {tFieldListDestroy($$);}
-columnlist(A) ::= columnlist(X) COMMA column(Y).  {A = tFieldListAppend(X, &Y);   }
-columnlist(A) ::= column(X).                      {A = tFieldListAppend(NULL, &X);}
+%type columnlist{SArray*}
+%destructor columnlist {taosArrayDestroy($$);}
+columnlist(A) ::= columnlist(X) COMMA column(Y).  {taosArrayPush(X, &Y); A = X;  }
+columnlist(A) ::= column(X).                      {A = taosArrayInit(4, sizeof(TAOS_FIELD)); taosArrayPush(A, &X);}
 
 // The information used for a column is the name and type of column:
 // tinyint smallint int bigint float double bool timestamp binary(x) nchar(x)
@@ -338,8 +338,8 @@ column(A) ::= ids(X) typename(Y).          {
     tSQLSetColumnInfo(&A, &X, &Y);
 }
 
-%type tagitemlist {tVariantList*}
-%destructor tagitemlist {tVariantListDestroy($$);}
+%type tagitemlist {SArray*}
+%destructor tagitemlist {taosArrayDestroy($$);}
 
 %type tagitem {tVariant}
 tagitemlist(A) ::= tagitemlist(X) COMMA tagitem(Y). { A = tVariantListAppend(X, &Y, -1);    }
@@ -432,11 +432,11 @@ as(X) ::= ids(Y).       { X = Y;    }
 as(X) ::= .             { X.n = 0;  }
 
 // A complete FROM clause.
-%type from {tVariantList*}
+%type from {SArray*}
 // current not support query from no-table
 from(A) ::= FROM tablelist(X).                 {A = X;}
 
-%type tablelist {tVariantList*}
+%type tablelist {SArray*}
 tablelist(A) ::= ids(X) cpxName(Y).                     {
   toTSDBType(X.type);
   X.n += Y.n;
@@ -476,8 +476,8 @@ interval_opt(N) ::= INTERVAL LP tmvar(E) RP.    {N.interval = E; N.offset.n = 0;
 interval_opt(N) ::= INTERVAL LP tmvar(E) COMMA tmvar(O) RP.    {N.interval = E; N.offset = O;}
 interval_opt(N) ::= .                           {memset(&N, 0, sizeof(N));}
 
-%type fill_opt {tVariantList*}
-%destructor fill_opt {tVariantListDestroy($$);}
+%type fill_opt {SArray*}
+%destructor fill_opt {taosArrayDestroy($$);}
 fill_opt(N) ::= .                               {N = 0;     }
 fill_opt(N) ::= FILL LP ID(Y) COMMA tagitemlist(X) RP.      {
     tVariant A = {0};
@@ -497,11 +497,11 @@ fill_opt(N) ::= FILL LP ID(Y) RP.               {
 sliding_opt(K) ::= SLIDING LP tmvar(E) RP.      {K = E;     }
 sliding_opt(K) ::= .                            {K.n = 0; K.z = NULL; K.type = 0;   }
 
-%type orderby_opt {tVariantList*}
-%destructor orderby_opt {tVariantListDestroy($$);}
+%type orderby_opt {SArray*}
+%destructor orderby_opt {taosArrayDestroy($$);}
 
-%type sortlist {tVariantList*}
-%destructor sortlist {tVariantListDestroy($$);}
+%type sortlist {SArray*}
+%destructor sortlist {taosArrayDestroy($$);}
 
 %type sortitem {tVariant}
 %destructor sortitem {tVariantDestroy(&$$);}
@@ -531,10 +531,10 @@ sortorder(A) ::= DESC.          {A = TSDB_ORDER_DESC;}
 sortorder(A) ::= .              {A = TSDB_ORDER_ASC;}  //default is descend order
 
 //group by clause
-%type groupby_opt {tVariantList*}
-%destructor groupby_opt {tVariantListDestroy($$);}
-%type grouplist {tVariantList*}
-%destructor grouplist {tVariantListDestroy($$);}
+%type groupby_opt {SArray*}
+%destructor groupby_opt {taosArrayDestroy($$);}
+%type grouplist {SArray*}
+%destructor grouplist {taosArrayDestroy($$);}
 
 groupby_opt(A) ::= .                       {A = 0;}
 groupby_opt(A) ::= GROUP BY grouplist(X).  {A = X;}
@@ -556,11 +556,11 @@ having_opt(A) ::= HAVING expr(X).   {A = X;}
 //limit-offset subclause
 %type limit_opt {SLimitVal}
 limit_opt(A) ::= .                     {A.limit = -1; A.offset = 0;}
-limit_opt(A) ::= LIMIT signed(X).      {A.limit = X;  A.offset = 0;}
+limit_opt(A) ::= LIMIT signed(X).      {printf("aa1, %d\n", X); A.limit = X;  A.offset = 0;}
 limit_opt(A) ::= LIMIT signed(X) OFFSET signed(Y).
-                                       {A.limit = X;  A.offset = Y;}
+                                       {printf("aa2\n, %d\n", X); A.limit = X;  A.offset = Y;}
 limit_opt(A) ::= LIMIT signed(X) COMMA signed(Y).
-                                       {A.limit = Y;  A.offset = X;}
+                                       {printf("aa3\n, %d\n", X); A.limit = Y;  A.offset = X;}
 
 %type slimit_opt {SLimitVal}
 slimit_opt(A) ::= .                    {A.limit = -1; A.offset = 0;}
@@ -657,7 +657,7 @@ cmd ::= ALTER TABLE ids(X) cpxName(F) DROP COLUMN ids(A).     {
     X.n += F.n;
 
     toTSDBType(A.type);
-    tVariantList* K = tVariantListAppendToken(NULL, &A, -1);
+    SArray* K = tVariantListAppendToken(NULL, &A, -1);
 
     SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&X, NULL, K, TSDB_ALTER_TABLE_DROP_COLUMN);
     setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE);
@@ -673,7 +673,7 @@ cmd ::= ALTER TABLE ids(X) cpxName(Z) DROP TAG ids(Y).          {
     X.n += Z.n;
 
     toTSDBType(Y.type);
-    tVariantList* A = tVariantListAppendToken(NULL, &Y, -1);
+    SArray* A = tVariantListAppendToken(NULL, &Y, -1);
 
     SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&X, NULL, A, TSDB_ALTER_TABLE_DROP_TAG_COLUMN);
     setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE);
@@ -683,7 +683,7 @@ cmd ::= ALTER TABLE ids(X) cpxName(F) CHANGE TAG ids(Y) ids(Z). {
     X.n += F.n;
 
     toTSDBType(Y.type);
-    tVariantList* A = tVariantListAppendToken(NULL, &Y, -1);
+    SArray* A = tVariantListAppendToken(NULL, &Y, -1);
 
     toTSDBType(Z.type);
     A = tVariantListAppendToken(A, &Z, -1);
@@ -696,7 +696,7 @@ cmd ::= ALTER TABLE ids(X) cpxName(F) SET TAG ids(Y) EQ tagitem(Z).     {
     X.n += F.n;
 
     toTSDBType(Y.type);
-    tVariantList* A = tVariantListAppendToken(NULL, &Y, -1);
+    SArray* A = tVariantListAppendToken(NULL, &Y, -1);
     A = tVariantListAppend(A, &Z, -1);
 
     SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&X, NULL, A, TSDB_ALTER_TABLE_UPDATE_TAG_VAL);

src/query/src/qAst.c

@@ -13,12 +13,10 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-
 #include "os.h"
 
 #include "exception.h"
 #include "qAst.h"
-#include "qSqlparser.h"
 #include "qSyntaxtreefunction.h"
 #include "taosdef.h"
 #include "taosmsg.h"
@@ -30,200 +28,19 @@
 #include "tskiplist.h"
 #include "tsqlfunction.h"
 #include "tstoken.h"
-#include "ttokendef.h"
-#include "tulog.h"
-
-/*
- *
- * @date 2018-2-15
- * @version 0.2  operation for column filter
- *
- * @Description parse tag query expression to build ast
- * ver 0.2, filter the result on first column with high priority to limit the candidate set
- * ver 0.3, pipeline filter in the form of: (a+2)/9 > 14
- *
- */
-static tExprNode *tExprNodeCreate(SSchema *pSchema, int32_t numOfCols, SStrToken *pToken);
-
-static tExprNode *createSyntaxTree(SSchema *pSchema, int32_t numOfCols, char *str, int32_t *i);
-static void       destroySyntaxTree(tExprNode *);
-
-static uint8_t isQueryOnPrimaryKey(const char *primaryColumnName, const tExprNode *pLeft, const tExprNode *pRight);
-
-/*
- * Check the filter value type on the right hand side based on the column id on the left hand side,
- * the filter value type must be identical to field type for relational operation
- * As for binary arithmetic operation, it is not necessary to do so.
- */
-static void reviseBinaryExprIfNecessary(tExprNode **pLeft, tExprNode **pRight, uint8_t *optr) {
-  if (*optr >= TSDB_RELATION_LESS && *optr <= TSDB_RELATION_LIKE) {
-    // make sure that the type of data on both sides of relational comparision are identical
-    if ((*pLeft)->nodeType == TSQL_NODE_VALUE) {
-      tVariantTypeSetType((*pLeft)->pVal, (*pRight)->pSchema->type);
-    } else if ((*pRight)->nodeType == TSQL_NODE_VALUE) {
-      tVariantTypeSetType((*pRight)->pVal, (*pLeft)->pSchema->type);
-    }
-
-  } else if (*optr >= TSDB_BINARY_OP_ADD && *optr <= TSDB_BINARY_OP_REMAINDER) {
-    if ((*pLeft)->nodeType == TSQL_NODE_VALUE) {
-      /* convert to int/bigint may cause the precision loss */
-      tVariantTypeSetType((*pLeft)->pVal, TSDB_DATA_TYPE_DOUBLE);
-    } else if ((*pRight)->nodeType == TSQL_NODE_VALUE) {
-      /* convert to int/bigint may cause the precision loss */
-      tVariantTypeSetType((*pRight)->pVal, TSDB_DATA_TYPE_DOUBLE);
-    }
-  }
-
-  /*
-   * for expressions that are suitable for switch principle,
-   * switch left and left and right hand side in expr if possible
-   */
-  if ((*pLeft)->nodeType == TSQL_NODE_VALUE && (*pRight)->nodeType == TSQL_NODE_COL) {
-    if (*optr >= TSDB_RELATION_GREATER && *optr <= TSDB_RELATION_GREATER_EQUAL && *optr != TSDB_RELATION_EQUAL) {
-      SWAP(*pLeft, *pRight, tExprNode *);
-    }
-
-    switch (*optr) {
-      case TSDB_RELATION_GREATER:
-        (*optr) = TSDB_RELATION_LESS;
-        break;
-      case TSDB_RELATION_LESS:
-        (*optr) = TSDB_RELATION_GREATER;
-        break;
-      case TSDB_RELATION_GREATER_EQUAL:
-        (*optr) = TSDB_RELATION_LESS_EQUAL;
-        break;
-      case TSDB_RELATION_LESS_EQUAL:
-        (*optr) = TSDB_RELATION_GREATER_EQUAL;
-        break;
-      default:;
-        // for other type of operations, do nothing
-    }
-  }
-}
-
-static tExprNode *tExprNodeCreate(SSchema *pSchema, int32_t numOfCols, SStrToken *pToken) {
-  /* if the token is not a value, return false */
-  if (pToken->type == TK_RP || (pToken->type != TK_INTEGER && pToken->type != TK_FLOAT && pToken->type != TK_ID &&
-                                pToken->type != TK_TBNAME && pToken->type != TK_STRING && pToken->type != TK_BOOL)) {
-    return NULL;
-  }
-
-  size_t nodeSize = sizeof(tExprNode);
-  tExprNode *pNode = NULL;
-
-  if (pToken->type == TK_ID || pToken->type == TK_TBNAME) {
-    int32_t i = 0;
-    if (pToken->type == TK_ID) {
-      do {
-        SStrToken tableToken = {0};
-        extractTableNameFromToken(pToken, &tableToken);
-
-        size_t len = strlen(pSchema[i].name);
-        if (strncmp(pToken->z, pSchema[i].name, pToken->n) == 0 && pToken->n == len) break;
-      } while (++i < numOfCols);
-
-      if (i == numOfCols) {  // column name is not valid, parse the expression failed
-        return NULL;
-      }
-    }
-
-    nodeSize += sizeof(SSchema);
-
-    pNode = calloc(1, nodeSize);
-    pNode->pSchema = (struct SSchema *)((char *)pNode + sizeof(tExprNode));
-    pNode->nodeType = TSQL_NODE_COL;
-
-    if (pToken->type == TK_ID) {
-      memcpy(pNode->pSchema, &pSchema[i], sizeof(SSchema));
-    } else {
-      pNode->pSchema->type = TSDB_DATA_TYPE_BINARY;
-      pNode->pSchema->bytes = TSDB_TABLE_NAME_LEN - 1;
-      strcpy(pNode->pSchema->name, TSQL_TBNAME_L);
-      pNode->pSchema->colId = -1;
-    }
-
-  } else {
-    nodeSize += sizeof(tVariant);
-    pNode = calloc(1, nodeSize);
-    pNode->pVal = (tVariant *)((char *)pNode + sizeof(tExprNode));
-
-    toTSDBType(pToken->type);
-    tVariantCreate(pNode->pVal, pToken);
-    pNode->nodeType = TSQL_NODE_VALUE;
-  }
-
-  return pNode;
-}
-
-uint8_t getBinaryExprOptr(SStrToken *pToken) {
-  switch (pToken->type) {
-    case TK_LT:
-      return TSDB_RELATION_LESS;
-    case TK_LE:
-      return TSDB_RELATION_LESS_EQUAL;
-    case TK_GT:
-      return TSDB_RELATION_GREATER;
-    case TK_GE:
-      return TSDB_RELATION_GREATER_EQUAL;
-    case TK_NE:
-      return TSDB_RELATION_NOT_EQUAL;
-    case TK_AND:
-      return TSDB_RELATION_AND;
-    case TK_OR:
-      return TSDB_RELATION_OR;
-    case TK_EQ:
-      return TSDB_RELATION_EQUAL;
-    case TK_PLUS:
-      return TSDB_BINARY_OP_ADD;
-    case TK_MINUS:
-      return TSDB_BINARY_OP_SUBTRACT;
-    case TK_STAR:
-      return TSDB_BINARY_OP_MULTIPLY;
-    case TK_SLASH:
-    case TK_DIVIDE:
-      return TSDB_BINARY_OP_DIVIDE;
-    case TK_REM:
-      return TSDB_BINARY_OP_REMAINDER;
-    case TK_LIKE:
-      return TSDB_RELATION_LIKE;
-    case TK_ISNULL:
-      return TSDB_RELATION_ISNULL;
-    case TK_NOTNULL:
-      return TSDB_RELATION_NOTNULL;
-    default: { return 0; }
-  }
-}
-
-// previous generated expr is reduced as the left child
-static tExprNode *parseRemainStr(char *pstr, tExprNode *pExpr, SSchema *pSchema, int32_t optr,
-                                      int32_t numOfCols, int32_t *i) {
-  // set the previous generated node as the left child of new root
-  pExpr->nodeType = TSQL_NODE_EXPR;
-
-  // remain is the right child
-  tExprNode *pRight = createSyntaxTree(pSchema, numOfCols, pstr, i);
-  if (pRight == NULL || (pRight->nodeType == TSQL_NODE_COL && pExpr->nodeType != TSQL_NODE_VALUE) ||
-      (pExpr->nodeType == TSQL_NODE_VALUE && pRight->nodeType != TSQL_NODE_COL)) {
-    tExprNodeDestroy(pExpr, NULL);
-    tExprNodeDestroy(pRight, NULL);
-    return NULL;
-  }
-
-  tExprNode *pNewExpr = (tExprNode *)calloc(1, sizeof(tExprNode));
-  uint8_t         k = optr;
-  reviseBinaryExprIfNecessary(&pExpr, &pRight, &k);
-  pNewExpr->_node.pLeft = pExpr;
-  pNewExpr->_node.pRight = pRight;
-  pNewExpr->_node.optr = k;
+#include "tschemautil.h"
 
-  pNewExpr->_node.hasPK = isQueryOnPrimaryKey(pSchema[0].name, pExpr, pRight);
-  pNewExpr->nodeType = TSQL_NODE_EXPR;
+typedef struct {
+  char*    v;
+  int32_t  optr;
+} SEndPoint;
 
-  return pNewExpr;
-}
+typedef struct {
+  SEndPoint* start;
+  SEndPoint* end;
+} SQueryCond;
 
-uint8_t isQueryOnPrimaryKey(const char *primaryColumnName, const tExprNode *pLeft, const tExprNode *pRight) {
+static uint8_t UNUSED_FUNC isQueryOnPrimaryKey(const char *primaryColumnName, const tExprNode *pLeft, const tExprNode *pRight) {
   if (pLeft->nodeType == TSQL_NODE_COL) {
     // if left node is the primary column,return true
     return (strcmp(primaryColumnName, pLeft->pSchema->name) == 0) ? 1 : 0;
@@ -236,103 +53,6 @@ uint8_t isQueryOnPrimaryKey(const char *primaryColumnName, const tExprNode *pLef
   }
 }
 
-static tExprNode *createSyntaxTree(SSchema *pSchema, int32_t numOfCols, char *str, int32_t *i) {
-  SStrToken t0 = tStrGetToken(str, i, false, 0, NULL);
-  if (t0.n == 0) {
-    return NULL;
-  }
-
-  tExprNode *pLeft = NULL;
-  if (t0.type == TK_LP) {  // start new left child branch
-    pLeft = createSyntaxTree(pSchema, numOfCols, str, i);
-  } else {
-    if (t0.type == TK_RP) {
-      return NULL;
-    }
-    
-    pLeft = tExprNodeCreate(pSchema, numOfCols, &t0);
-  }
-
-  if (pLeft == NULL) {
-    return NULL;
-  }
-
-  t0 = tStrGetToken(str, i, false, 0, NULL);
-  if (t0.n == 0 || t0.type == TK_RP) {
-    if (pLeft->nodeType != TSQL_NODE_EXPR) {  // if left is not the expr, it is not a legal expr
-      tExprNodeDestroy(pLeft, NULL);
-      return NULL;
-    }
-
-    return pLeft;
-  }
-
-  // get the operator of expr
-  uint8_t optr = getBinaryExprOptr(&t0);
-  if (optr == 0) {
-    uError("not support binary operator:%d", t0.type);
-    tExprNodeDestroy(pLeft, NULL);
-    return NULL;
-  }
-
-  assert(pLeft != NULL);
-  tExprNode *pRight = NULL;
-
-  if (t0.type == TK_AND || t0.type == TK_OR || t0.type == TK_LP) {
-    pRight = createSyntaxTree(pSchema, numOfCols, str, i);
-  } else {
-    /*
-     * In case that pLeft is a field identification,
-     * we parse the value in expression according to queried field type,
-     * if we do not get the information, in case of value of field presented first,
-     * we revised the value after the binary expression is completed.
-     */
-    t0 = tStrGetToken(str, i, true, 0, NULL);
-    if (t0.n == 0) {
-      tExprNodeDestroy(pLeft, NULL);  // illegal expression
-      return NULL;
-    }
-
-    if (t0.type == TK_LP) {
-      pRight = createSyntaxTree(pSchema, numOfCols, str, i);
-    } else {
-      pRight = tExprNodeCreate(pSchema, numOfCols, &t0);
-    }
-  }
-
-  if (pRight == NULL) {
-    tExprNodeDestroy(pLeft, NULL);
-    return NULL;
-  }
-
-  /* create binary expr as the child of new parent node */
-  tExprNode *pExpr = (tExprNode *)calloc(1, sizeof(tExprNode));
-  reviseBinaryExprIfNecessary(&pLeft, &pRight, &optr);
-
-  pExpr->_node.hasPK = isQueryOnPrimaryKey(pSchema[0].name, pLeft, pRight);
-  pExpr->_node.pLeft = pLeft;
-  pExpr->_node.pRight = pRight;
-  pExpr->_node.optr = optr;
-
-  t0 = tStrGetToken(str, i, true, 0, NULL);
-
-  if (t0.n == 0 || t0.type == TK_RP) {
-    pExpr->nodeType = TSQL_NODE_EXPR;
-    return pExpr;
-  } else {
-    uint8_t localOptr = getBinaryExprOptr(&t0);
-    if (localOptr == 0) {
-      uError("not support binary operator:%d", t0.type);
-      free(pExpr);
-      return NULL;
-    }
-
-    return parseRemainStr(str, pExpr, pSchema, localOptr, numOfCols, i);
-  }
-}
-
-static void UNUSED_FUNC destroySyntaxTree(tExprNode *pNode) { tExprNodeDestroy(pNode, NULL); }
-
 void tExprNodeDestroy(tExprNode *pNode, void (*fp)(void *)) {
   if (pNode == NULL) {
     return;
@@ -372,16 +92,6 @@ void tExprTreeDestroy(tExprNode **pExpr, void (*fp)(void *)) {
   *pExpr = NULL;
 }
 
-typedef struct {
-  char*    v;
-  int32_t  optr;
-} SEndPoint;
-
-typedef struct {
-  SEndPoint* start;
-  SEndPoint* end;
-} SQueryCond;
-
 // todo check for malloc failure
 static int32_t setQueryCond(tQueryInfo *queryColInfo, SQueryCond* pCond) {
   int32_t optr = queryColInfo->optr;
@@ -395,13 +105,10 @@ static int32_t setQueryCond(tQueryInfo *queryColInfo, SQueryCond* pCond) {
     pCond->end = calloc(1, sizeof(SEndPoint));
     pCond->end->optr = queryColInfo->optr;
     pCond->end->v = queryColInfo->q;
-  } else if (optr == TSDB_RELATION_IN) {
-    printf("relation is in\n");
-    assert(0);
-  } else if (optr == TSDB_RELATION_LIKE) {
-    printf("relation is like\n");
+  } else if (optr == TSDB_RELATION_IN || optr == TSDB_RELATION_LIKE) {
     assert(0);
   }
+
   return TSDB_CODE_SUCCESS;
 }
 
@@ -529,99 +236,6 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr
   tSkipListDestroyIter(iter);
 }
 
-int32_t merge(SArray *pLeft, SArray *pRight, SArray *pFinalRes) {
-//  assert(pFinalRes->pRes == 0);
-//
-//  pFinalRes->pRes = calloc((size_t)(pLeft->num + pRight->num), POINTER_BYTES);
-//  pFinalRes->num = 0;
-//
-//  // sort according to address
-//  tSkipListNode **pLeftNodes = (tSkipListNode **)pLeft->pRes;
-//  qsort(pLeftNodes, pLeft->num, sizeof(pLeft->pRes[0]), compareByAddr);
-//
-//  tSkipListNode **pRightNodes = (tSkipListNode **)pRight->pRes;
-//  qsort(pRightNodes, pRight->num, sizeof(pRight->pRes[0]), compareByAddr);
-//
-//  int32_t i = 0, j = 0;
-//
-//  // merge two sorted arrays in O(n) time
-//  while (i < pLeft->num && j < pRight->num) {
-//    int64_t ret = (int64_t)pLeftNodes[i] - (int64_t)pRightNodes[j];
-//
-//    if (ret < 0) {
-//      pFinalRes->pRes[pFinalRes->num++] = pLeftNodes[i++];
-//    } else if (ret > 0) {
-//      pFinalRes->pRes[pFinalRes->num++] = pRightNodes[j++];
-//    } else {  // pNode->key > pkey[i]
-//      pFinalRes->pRes[pFinalRes->num++] = pRightNodes[j++];
-//      i++;
-//    }
-//  }
-//
-//  while (i < pLeft->num) {
-//    pFinalRes->pRes[pFinalRes->num++] = pLeftNodes[i++];
-//  }
-//
-//  while (j < pRight->num) {
-//    pFinalRes->pRes[pFinalRes->num++] = pRightNodes[j++];
-//  }
-//
-//  return pFinalRes->num;
-  return 0;
-}
-
-int32_t intersect(SArray *pLeft, SArray *pRight, SArray *pFinalRes) {
-//  int64_t num = MIN(pLeft->num, pRight->num);
-//
-//  assert(pFinalRes->pRes == 0);
-//
-//  pFinalRes->pRes = calloc(num, POINTER_BYTES);
-//  pFinalRes->num = 0;
-//
-//  // sort according to address
-//  tSkipListNode **pLeftNodes = (tSkipListNode **)pLeft->pRes;
-//  qsort(pLeftNodes, pLeft->num, sizeof(pLeft->pRes[0]), compareByAddr);
-//
-//  tSkipListNode **pRightNodes = (tSkipListNode **)pRight->pRes;
-//  qsort(pRightNodes, pRight->num, sizeof(pRight->pRes[0]), compareByAddr);
-//
-//  int32_t i = 0, j = 0;
-//  // merge two sorted arrays in O(n) time
-//  while (i < pLeft->num && j < pRight->num) {
-//    int64_t ret = (int64_t)pLeftNodes[i] - (int64_t)pRightNodes[j];
-//
-//    if (ret < 0) {
-//      i++;
-//    } else if (ret > 0) {
-//      j++;
-//    } else {  // pNode->key > pkey[i]
-//      pFinalRes->pRes[pFinalRes->num++] = pRightNodes[j];
-//      i++;
-//      j++;
-//    }
-//  }
-//
-//  return pFinalRes->num;
- return 0;
-}
-
-/*
- * traverse the result and apply the function to each item to check if the item is qualified or not
- */
-static UNUSED_FUNC void tArrayTraverse(tExprNode *pExpr, __result_filter_fn_t fp, SArray *pResult) {
-  assert(pExpr->_node.pLeft->nodeType == TSQL_NODE_COL && pExpr->_node.pRight->nodeType == TSQL_NODE_VALUE && fp != NULL);
-
-  //  scan the result array list and check for each item in the list
-  for (int32_t i = 0; i < taosArrayGetSize(pResult); ++i) {
-    void* item = taosArrayGet(pResult, i);
-    if (fp(item, pExpr->_node.info)) {
-      i++;
-    } else {
-      taosArrayRemove(pResult, i);
-    }
-  }
-}
-
 static bool filterItem(tExprNode *pExpr, const void *pItem, SExprTraverseSupp *param) {
   tExprNode *pLeft = pExpr->_node.pLeft;
   tExprNode *pRight = pExpr->_node.pRight;
@@ -649,32 +263,6 @@ static bool filterItem(tExprNode *pExpr, const void *pItem, SExprTraverseSupp *p
   return param->nodeFilterFn(pItem, pExpr->_node.info);
 }
 
-/**
- * Apply the filter expression on non-indexed tag columns to each element in the result list, which is generated
- * by filtering on indexed tag column. So the whole result set only needs to be iterated once to generate
- * result that is satisfied to the filter expression, no matter how the filter expression consisting of.
- *
- * @param pExpr     filter expression on non-indexed tag columns.
- * @param pResult   results from filter on the indexed tag column, which is usually the first tag column
- * @param pSchema   tag schemas
- * @param fp        filter callback function
- */
-static UNUSED_FUNC void exprTreeTraverseImpl(tExprNode *pExpr, SArray *pResult, SExprTraverseSupp *param) {
-  size_t size = taosArrayGetSize(pResult);
-  
-  SArray* array = taosArrayInit(size, POINTER_BYTES);
-  for (int32_t i = 0; i < size; ++i) {
-    void *pItem = taosArrayGetP(pResult, i);
-
-    if (filterItem(pExpr, pItem, param)) {
-      taosArrayPush(array, &pItem);
-    }
-  }
-  
-  taosArrayCopy(pResult, array);
-  taosArrayDestroy(array);
-}
-
 static void tSQLBinaryTraverseOnSkipList(tExprNode *pExpr, SArray *pResult, SSkipList *pSkipList, SExprTraverseSupp *param ) {
   SSkipListIterator* iter = tSkipListCreateIter(pSkipList);
 
@@ -750,7 +338,7 @@ void tExprTreeTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, S
   //apply the hierarchical expression to every node in skiplist for find the qualified nodes
   tSQLBinaryTraverseOnSkipList(pExpr, result, pSkipList, param);
 
-  #if 0
+#if 0
   /*
     * (weight == 1 && pExpr->nSQLBinaryOptr == TSDB_RELATION_AND) is handled here
     *
@@ -972,6 +560,7 @@ tExprNode* exprTreeFromBinary(const void* data, size_t size) {
   if (size == 0) {
     return NULL;
   }
+
   SBufferReader br = tbufInitReader(data, size, false);
   return exprTreeFromBinaryImpl(&br);
 }
@@ -995,10 +584,7 @@ tExprNode* exprTreeFromTableName(const char* tbnameCond) {
   SSchema* pSchema = exception_calloc(1, sizeof(SSchema));
   left->pSchema = pSchema;
 
-  pSchema->type = TSDB_DATA_TYPE_BINARY;
-  pSchema->bytes = TSDB_TABLE_NAME_LEN - 1;
-  strcpy(pSchema->name, TSQL_TBNAME_L);
-  pSchema->colId = -1;
+  *pSchema = tscGetTbnameColumnSchema();
 
   tExprNode* right = exception_calloc(1, sizeof(tExprNode));
   expr->_node.pRight = right;

src/query/src/qExecutor.c

@@ -170,8 +170,6 @@ static void getNextTimeWindow(SQuery* pQuery, STimeWindow* tw) {
   tw->ekey -= 1;
 }
 
-#define GET_NEXT_TIMEWINDOW(_q, tw) getNextTimeWindow((_q), (tw))
-
 #define SET_STABLE_QUERY_OVER(_q) ((_q)->tableIndex = (int32_t)((_q)->tableqinfoGroupInfo.numOfTables))
 #define IS_STASBLE_QUERY_OVER(_q) ((_q)->tableIndex >= (int32_t)((_q)->tableqinfoGroupInfo.numOfTables))
 
@@ -827,7 +825,7 @@ static int32_t getNextQualifiedWindow(SQueryRuntimeEnv *pRuntimeEnv, STimeWindow
     TSKEY *primaryKeys, __block_search_fn_t searchFn, int32_t prevPosition) {
   SQuery *pQuery = pRuntimeEnv->pQuery;
 
-  GET_NEXT_TIMEWINDOW(pQuery, pNext);
+  getNextTimeWindow(pQuery, pNext);
 
   // next time window is not in current block
   if ((pNext->skey > pDataBlockInfo->window.ekey && QUERY_IS_ASC_QUERY(pQuery)) ||
@@ -1342,9 +1340,9 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS
       int32_t     index = pWindowResInfo->curIndex;
 
       while (1) {
-        GET_NEXT_TIMEWINDOW(pQuery, &nextWin);
+        getNextTimeWindow(pQuery, &nextWin);
         if ((nextWin.skey > pQuery->window.ekey && QUERY_IS_ASC_QUERY(pQuery)) ||
-            (nextWin.skey < pQuery->window.ekey && !QUERY_IS_ASC_QUERY(pQuery))) {
+            (nextWin.ekey < pQuery->window.ekey && !QUERY_IS_ASC_QUERY(pQuery))) {
           break;
         }
 
@@ -2202,7 +2200,7 @@ static bool overlapWithTimeWindow(SQuery* pQuery, SDataBlockInfo* pBlockInfo) {
     }
 
     while(1) {
-      GET_NEXT_TIMEWINDOW(pQuery, &w);
+      getNextTimeWindow(pQuery, &w);
       if (w.skey > pBlockInfo->window.ekey) {
         break;
       }
@@ -2221,7 +2219,7 @@ static bool overlapWithTimeWindow(SQuery* pQuery, SDataBlockInfo* pBlockInfo) {
     }
 
     while(1) {
-      GET_NEXT_TIMEWINDOW(pQuery, &w);
+      getNextTimeWindow(pQuery, &w);
       if (w.ekey < pBlockInfo->window.skey) {
         break;
       }
@@ -2536,7 +2534,7 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) {
     setQueryStatus(pQuery, QUERY_COMPLETED);
   }
 
-  if (QUERY_IS_INTERVAL_QUERY(pQuery) && IS_MASTER_SCAN(pRuntimeEnv)) {
+  if (QUERY_IS_INTERVAL_QUERY(pQuery) && (IS_MASTER_SCAN(pRuntimeEnv)|| pRuntimeEnv->scanFlag == REPEAT_SCAN)) {
     if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) {
       closeAllTimeWindow(&pRuntimeEnv->windowResInfo);
       pRuntimeEnv->windowResInfo.curIndex = pRuntimeEnv->windowResInfo.size - 1;  // point to the last time window
@@ -4116,7 +4114,7 @@ bool queryHasRemainResForTableQuery(SQueryRuntimeEnv* pRuntimeEnv) {
      * first result row in the actual result set will fill nothing.
      */
     if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) {
-      int32_t numOfTotal = (int32_t)getFilledNumOfRes(pFillInfo, pQuery->window.ekey, (int32_t)pQuery->rec.capacity);
+      int32_t numOfTotal = (int32_t)getNumOfResWithFill(pFillInfo, pQuery->window.ekey, (int32_t)pQuery->rec.capacity);
       return numOfTotal > 0;
     }
 
@@ -4174,7 +4172,7 @@ int32_t doFillGapsInResults(SQueryRuntimeEnv* pRuntimeEnv, tFilePage **pDst, int
   SFillInfo* pFillInfo = pRuntimeEnv->pFillInfo;
 
   while (1) {
-    int32_t ret = (int32_t)taosGenerateDataBlock(pFillInfo, (tFilePage**)pQuery->sdata, (int32_t)pQuery->rec.capacity);
+    int32_t ret = (int32_t)taosFillResultDataBlock(pFillInfo, (tFilePage**)pQuery->sdata, (int32_t)pQuery->rec.capacity);
 
     // todo apply limit output function
     /* reached the start position of according to offset value, return immediately */
@@ -4354,14 +4352,17 @@ static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv, TSKEY* start) {
     STimeWindow win = getActiveTimeWindow(pWindowResInfo, pWindowResInfo->prevSKey, pQuery);
 
     while (pQuery->limit.offset > 0) {
+      STimeWindow tw = win;
+
       if ((win.ekey <= blockInfo.window.ekey && QUERY_IS_ASC_QUERY(pQuery)) ||
           (win.ekey >= blockInfo.window.skey && !QUERY_IS_ASC_QUERY(pQuery))) {
         pQuery->limit.offset -= 1;
         pWindowResInfo->prevSKey = win.skey;
-      }
 
-      STimeWindow tw = win;
-      GET_NEXT_TIMEWINDOW(pQuery, &tw);
+        getNextTimeWindow(pQuery, &tw);
+      } else { // current window does not ended in current data block, try next data block
+        getNextTimeWindow(pQuery, &tw);
+      }
 
       if (pQuery->limit.offset == 0) {
         if ((tw.skey <= blockInfo.window.ekey && QUERY_IS_ASC_QUERY(pQuery)) ||
@@ -4414,6 +4415,49 @@ static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv, TSKEY* start) {
         SArray *         pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL);
         SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0);
 
+        if ((win.ekey > blockInfo.window.ekey && QUERY_IS_ASC_QUERY(pQuery)) ||
+            (win.ekey < blockInfo.window.skey && !QUERY_IS_ASC_QUERY(pQuery))) {
+          pQuery->limit.offset -= 1;
+        }
+
+        if (pQuery->limit.offset == 0) {
+          if ((tw.skey <= blockInfo.window.ekey && QUERY_IS_ASC_QUERY(pQuery)) ||
+              (tw.ekey >= blockInfo.window.skey && !QUERY_IS_ASC_QUERY(pQuery))) {
+            // load the data block and check data remaining in current data block
+            // TODO optimize performance
+            SArray *         pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL);
+            SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0);
+
+            tw = win;
+            int32_t startPos =
+                getNextQualifiedWindow(pRuntimeEnv, &tw, &blockInfo, pColInfoData->pData, binarySearchForKey, -1);
+            assert(startPos >= 0);
+
+            // set the abort info
+            pQuery->pos = startPos;
+
+            // reset the query start timestamp
+            pTableQueryInfo->win.skey = ((TSKEY *)pColInfoData->pData)[startPos];
+            pQuery->window.skey = pTableQueryInfo->win.skey;
+            *start = pTableQueryInfo->win.skey;
+
+            pWindowResInfo->prevSKey = tw.skey;
+            int32_t index = pRuntimeEnv->windowResInfo.curIndex;
+
+            int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, &blockInfo, NULL, binarySearchForKey, pDataBlock);
+            pRuntimeEnv->windowResInfo.curIndex = index;  // restore the window index
+
+            qDebug("QInfo:%p check data block, brange:%" PRId64 "-%" PRId64 ", numOfRows:%d, numOfRes:%d, lastKey:%"PRId64,
+                   GET_QINFO_ADDR(pRuntimeEnv), blockInfo.window.skey, blockInfo.window.ekey, blockInfo.rows, numOfRes, pQuery->current->lastKey);
+
+            return true;
+          } else { // do nothing
+            *start = tw.skey;
+            pQuery->window.skey = tw.skey;
+            pWindowResInfo->prevSKey = tw.skey;
+            return true;
+          }
+        } else {
           tw = win;
           int32_t startPos =
               getNextQualifiedWindow(pRuntimeEnv, &tw, &blockInfo, pColInfoData->pData, binarySearchForKey, -1);
@@ -4424,6 +4468,7 @@ static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv, TSKEY* start) {
           pTableQueryInfo->lastKey = ((TSKEY *)pColInfoData->pData)[startPos];
           pWindowResInfo->prevSKey = tw.skey;
           win = tw;
+        }
       } else {
         break;  // offset is not 0, and next time window begins or ends in the next block.
       }
@@ -4519,6 +4564,7 @@ static SFillColInfo* taosCreateFillColInfo(SQuery* pQuery) {
     pFillCol[i].col.bytes  = pExprInfo->bytes;
     pFillCol[i].col.type   = (int8_t)pExprInfo->type;
     pFillCol[i].col.offset = offset;
+    pFillCol[i].tagIndex   = -2;
     pFillCol[i].flag       = TSDB_COL_NORMAL;    // always be ta normal column for table query
     pFillCol[i].functionId = pExprInfo->base.functionId;
     pFillCol[i].fillVal.i = pQuery->fillVal[i];
@@ -4532,7 +4578,6 @@ static SFillColInfo* taosCreateFillColInfo(SQuery* pQuery) {
 int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bool isSTableQuery) {
   SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv;
 
-  int32_t code = TSDB_CODE_SUCCESS;
   SQuery *pQuery = pQInfo->runtimeEnv.pQuery;
 
   pRuntimeEnv->topBotQuery = isTopBottomQuery(pQuery);
@@ -4540,7 +4585,7 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo
 
   setScanLimitationByResultBuffer(pQuery);
 
-  code = setupQueryHandle(tsdb, pQInfo, isSTableQuery);
+  int32_t code = setupQueryHandle(tsdb, pQInfo, isSTableQuery);
   if (code != TSDB_CODE_SUCCESS) {
     return code;
   }
@@ -4628,7 +4673,7 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo
 
     pRuntimeEnv->pFillInfo = taosInitFillInfo(pQuery->order.order, w.skey, 0, (int32_t)pQuery->rec.capacity, pQuery->numOfOutput,
                                               pQuery->interval.sliding, pQuery->interval.slidingUnit, (int8_t)pQuery->precision,
-                                              pQuery->fillType, pColInfo);
+                                              pQuery->fillType, pColInfo, pQInfo);
   }
 
   setQueryStatus(pQuery, QUERY_NOT_COMPLETED);
@@ -5430,7 +5475,7 @@ static void tableIntervalProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) {
       break;
     } else {
       taosFillSetStartInfo(pRuntimeEnv->pFillInfo, (int32_t)pQuery->rec.rows, pQuery->window.ekey);
-      taosFillCopyInputDataFromFilePage(pRuntimeEnv->pFillInfo, (tFilePage**) pQuery->sdata);
+      taosFillCopyInputDataFromFilePage(pRuntimeEnv->pFillInfo, (const tFilePage**) pQuery->sdata);
       numOfFilled = 0;
 
       pQuery->rec.rows = doFillGapsInResults(pRuntimeEnv, (tFilePage **)pQuery->sdata, &numOfFilled);
@@ -6926,7 +6971,6 @@ int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *co
 
   if (IS_QUERY_KILLED(pQInfo) || Q_STATUS_EQUAL(pQuery->status, QUERY_OVER)) {
     // here current thread hold the refcount, so it is safe to free tsdbQueryHandle.
-    doFreeQueryHandle(pQInfo);
     *continueExec = false;
     (*pRsp)->completed = 1;  // notify no more result to client
   } else {

src/query/src/qFill.c

@@ -13,17 +13,64 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "qFill.h"
 #include "os.h"
-#include "qExtbuffer.h"
+
 #include "taosdef.h"
 #include "taosmsg.h"
 #include "tsqlfunction.h"
+#include "ttype.h"
+
+#include "qFill.h"
+#include "qExtbuffer.h"
+#include "queryLog.h"
 
 #define FILL_IS_ASC_FILL(_f) ((_f)->order == TSDB_ORDER_ASC)
 
+// there are no duplicated tags in the SFillTagColInfo list
+static int32_t setTagColumnInfo(SFillInfo* pFillInfo, int32_t numOfCols, int32_t capacity) {
+  int32_t rowsize = 0;
+
+  int32_t k = 0;
+  for (int32_t i = 0; i < numOfCols; ++i) {
+    SFillColInfo* pColInfo = &pFillInfo->pFillCol[i];
+    pFillInfo->pData[i] = calloc(1, pColInfo->col.bytes * capacity);
+
+    if (TSDB_COL_IS_TAG(pColInfo->flag)) {
+      bool exists = false;
+      int32_t index = -1;
+      for (int32_t j = 0; j < k; ++j) {
+        if (pFillInfo->pTags[j].col.colId == pColInfo->col.colId) {
+          exists = true;
+          index = j;
+          break;
+        }
+      }
+
+      if (!exists) {
+        SSchema* pSchema = &pFillInfo->pTags[k].col;
+        pSchema->colId = pColInfo->col.colId;
+        pSchema->type  = pColInfo->col.type;
+        pSchema->bytes = pColInfo->col.bytes;
+
+        pFillInfo->pTags[k].tagVal = calloc(1, pColInfo->col.bytes);
+        pColInfo->tagIndex = k;
+
+        k += 1;
+      } else {
+        pColInfo->tagIndex = index;
+      }
+    }
+
+    rowsize += pColInfo->col.bytes;
+  }
+
+  assert(k <= pFillInfo->numOfTags);
+  return rowsize;
+}
+
 SFillInfo* taosInitFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_t capacity, int32_t numOfCols,
-    int64_t slidingTime, int8_t slidingUnit, int8_t precision, int32_t fillType, SFillColInfo* pFillCol) {
+                            int64_t slidingTime, int8_t slidingUnit, int8_t precision, int32_t fillType,
+                            SFillColInfo* pCol, void* handle) {
   if (fillType == TSDB_FILL_NONE) {
     return NULL;
   }
@@ -33,11 +80,13 @@ SFillInfo* taosInitFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_
   taosResetFillInfo(pFillInfo, skey);
 
   pFillInfo->order    = order;
-  pFillInfo->fillType  = fillType;
-  pFillInfo->pFillCol  = pFillCol;
+  pFillInfo->type     = fillType;
+  pFillInfo->pFillCol = pCol;
   pFillInfo->numOfTags = numOfTags;
   pFillInfo->numOfCols = numOfCols;
   pFillInfo->precision = precision;
+  pFillInfo->alloc     = capacity;
+  pFillInfo->handle    = handle;
 
   pFillInfo->interval.interval = slidingTime;
   pFillInfo->interval.intervalUnit = slidingUnit;
@@ -47,45 +96,21 @@ SFillInfo* taosInitFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_
   pFillInfo->pData = malloc(POINTER_BYTES * numOfCols);
   if (numOfTags > 0) {
     pFillInfo->pTags = calloc(pFillInfo->numOfTags, sizeof(SFillTagColInfo));
-    for(int32_t i = 0; i < numOfTags; ++i) {
-      pFillInfo->pTags[i].col.colId = -2;
+    for (int32_t i = 0; i < numOfTags; ++i) {
+      pFillInfo->pTags[i].col.colId = -2;  // TODO
     }
   }
 
-  int32_t rowsize = 0;
-  int32_t k = 0;
-  for (int32_t i = 0; i < numOfCols; ++i) {
-    SFillColInfo* pColInfo = &pFillInfo->pFillCol[i];
-    pFillInfo->pData[i] = calloc(1, pColInfo->col.bytes * capacity);
-
-    if (TSDB_COL_IS_TAG(pColInfo->flag)) {
-      bool exists = false;
-      for(int32_t j = 0; j < k; ++j) {
-        if (pFillInfo->pTags[j].col.colId == pColInfo->col.colId) {
-          exists = true;
-          break;
-        }
-      }
-
-      if (!exists) {
-        pFillInfo->pTags[k].col.colId = pColInfo->col.colId;
-        pFillInfo->pTags[k].tagVal = calloc(1, pColInfo->col.bytes);
-
-        k += 1;
-      }
-    }
-    rowsize += pColInfo->col.bytes;
-  }
-
-  pFillInfo->rowSize = rowsize;
-  pFillInfo->capacityInRows = capacity;
+  pFillInfo->rowSize = setTagColumnInfo(pFillInfo, pFillInfo->numOfCols, pFillInfo->alloc);
+  assert(pFillInfo->rowSize > 0);
 
   return pFillInfo;
 }
 
 void taosResetFillInfo(SFillInfo* pFillInfo, TSKEY startTimestamp) {
   pFillInfo->start        = startTimestamp;
-  pFillInfo->rowIdx       = -1;
+  pFillInfo->currentKey   = startTimestamp;
+  pFillInfo->index        = -1;
   pFillInfo->numOfRows    = 0;
   pFillInfo->numOfCurrent = 0;
   pFillInfo->numOfTotal   = 0;
@@ -112,20 +137,20 @@ void* taosDestroyFillInfo(SFillInfo* pFillInfo) {
 }
 
 void taosFillSetStartInfo(SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey) {
-  if (pFillInfo->fillType == TSDB_FILL_NONE) {
+  if (pFillInfo->type == TSDB_FILL_NONE) {
     return;
   }
 
-  pFillInfo->endKey = endKey;
-  if (pFillInfo->order != TSDB_ORDER_ASC) {
-    pFillInfo->endKey = taosTimeTruncate(endKey, &pFillInfo->interval, pFillInfo->precision);
+  pFillInfo->end = endKey;
+  if (!FILL_IS_ASC_FILL(pFillInfo)) {
+    pFillInfo->end = taosTimeTruncate(endKey, &pFillInfo->interval, pFillInfo->precision);
   }
 
-  pFillInfo->rowIdx    = 0;
+  pFillInfo->index     = 0;
   pFillInfo->numOfRows = numOfRows;
   
   // ensure the space
-  if (pFillInfo->capacityInRows < numOfRows) {
+  if (pFillInfo->alloc < numOfRows) {
     for(int32_t i = 0; i < pFillInfo->numOfCols; ++i) {
       char* tmp = realloc(pFillInfo->pData[i], numOfRows*pFillInfo->pFillCol[i].col.bytes);
       assert(tmp != NULL); // todo handle error
@@ -136,42 +161,38 @@ void taosFillSetStartInfo(SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey)
   }
 }
 
-void taosFillCopyInputDataFromFilePage(SFillInfo* pFillInfo, tFilePage** pInput) {
-  // copy the data into source data buffer
+// copy the data into source data buffer
+void taosFillCopyInputDataFromFilePage(SFillInfo* pFillInfo, const tFilePage** pInput) {
   for (int32_t i = 0; i < pFillInfo->numOfCols; ++i) {
     memcpy(pFillInfo->pData[i], pInput[i]->data, pFillInfo->numOfRows * pFillInfo->pFillCol[i].col.bytes);
   }
 }
 
-void taosFillCopyInputDataFromOneFilePage(SFillInfo* pFillInfo, tFilePage* pInput) {
+void taosFillCopyInputDataFromOneFilePage(SFillInfo* pFillInfo, const tFilePage* pInput) {
   assert(pFillInfo->numOfRows == pInput->num);
 
   for(int32_t i = 0; i < pFillInfo->numOfCols; ++i) {
     SFillColInfo* pCol = &pFillInfo->pFillCol[i];
 
-    char* data = pInput->data + pCol->col.offset * pInput->num;
+    const char* data = pInput->data + pCol->col.offset * pInput->num;
     memcpy(pFillInfo->pData[i], data, (size_t)(pInput->num * pCol->col.bytes));
 
     if (TSDB_COL_IS_TAG(pCol->flag)) {  // copy the tag value to tag value buffer
-      for (int32_t j = 0; j < pFillInfo->numOfTags; ++j) {
-        SFillTagColInfo* pTag = &pFillInfo->pTags[j];
-        if (pTag->col.colId == pCol->col.colId) {
+      SFillTagColInfo* pTag = &pFillInfo->pTags[pCol->tagIndex];
+      assert (pTag->col.colId == pCol->col.colId);
       memcpy(pTag->tagVal, data, pCol->col.bytes);
-          break;
-        }
-      }
     }
   }
 }
 
-int64_t getFilledNumOfRes(SFillInfo* pFillInfo, TSKEY ekey, int32_t maxNumOfRows) {
+int64_t getNumOfResWithFill(SFillInfo* pFillInfo, TSKEY ekey, int32_t maxNumOfRows) {
   int64_t* tsList = (int64_t*) pFillInfo->pData[0];
 
   int32_t numOfRows = taosNumOfRemainRows(pFillInfo);
 
   TSKEY ekey1 = ekey;
   if (!FILL_IS_ASC_FILL(pFillInfo)) {
-    pFillInfo->endKey = taosTimeTruncate(ekey, &pFillInfo->interval, pFillInfo->precision);
+    pFillInfo->end = taosTimeTruncate(ekey, &pFillInfo->interval, pFillInfo->precision);
   }
 
   int64_t numOfRes = -1;
@@ -179,20 +200,20 @@ int64_t getFilledNumOfRes(SFillInfo* pFillInfo, TSKEY ekey, int32_t maxNumOfRows
     TSKEY lastKey = tsList[pFillInfo->numOfRows - 1];
     numOfRes = taosTimeCountInterval(
       lastKey,
-      pFillInfo->start,
+      pFillInfo->currentKey,
       pFillInfo->interval.sliding,
       pFillInfo->interval.slidingUnit,
       pFillInfo->precision);
     numOfRes += 1;
     assert(numOfRes >= numOfRows);
   } else { // reach the end of data
-    if ((ekey1 < pFillInfo->start && FILL_IS_ASC_FILL(pFillInfo)) ||
-        (ekey1 > pFillInfo->start && !FILL_IS_ASC_FILL(pFillInfo))) {
+    if ((ekey1 < pFillInfo->currentKey && FILL_IS_ASC_FILL(pFillInfo)) ||
+        (ekey1 > pFillInfo->currentKey && !FILL_IS_ASC_FILL(pFillInfo))) {
       return 0;
     }
     numOfRes = taosTimeCountInterval(
       ekey1,
-      pFillInfo->start,
+      pFillInfo->currentKey,
       pFillInfo->interval.sliding,
       pFillInfo->interval.slidingUnit,
       pFillInfo->precision);
@@ -203,254 +224,200 @@ int64_t getFilledNumOfRes(SFillInfo* pFillInfo, TSKEY ekey, int32_t maxNumOfRows
 }
 
 int32_t taosNumOfRemainRows(SFillInfo* pFillInfo) {
-  if (pFillInfo->rowIdx == -1 || pFillInfo->numOfRows == 0) {
+  if (pFillInfo->numOfRows == 0 || (pFillInfo->numOfRows > 0 && pFillInfo->index >= pFillInfo->numOfRows)) {
     return 0;
   }
 
-  return pFillInfo->numOfRows - pFillInfo->rowIdx;
+  return pFillInfo->numOfRows - pFillInfo->index;
 }
 
-// todo: refactor
-static double linearInterpolationImpl(double v1, double v2, double k1, double k2, double k) {
-  return v1 + (v2 - v1) * (k - k1) / (k2 - k1);
-}
+#define DO_INTERPOLATION(_v1, _v2, _k1, _k2, _k) ((_v1) + ((_v2) - (_v1)) * (((double)(_k)) - ((double)(_k1))) / (((double)(_k2)) - ((double)(_k1))))
 
 int32_t taosGetLinearInterpolationVal(int32_t type, SPoint* point1, SPoint* point2, SPoint* point) {
-  switch (type) {
-    case TSDB_DATA_TYPE_INT: {
-      *(int32_t*)point->val = (int32_t)linearInterpolationImpl(*(int32_t*)point1->val, *(int32_t*)point2->val, (double)point1->key,
-                                                              (double)point2->key, (double)point->key);
-      break;
-    }
-    case TSDB_DATA_TYPE_FLOAT: {
-      *(float*)point->val = (float)
-        linearInterpolationImpl(*(float*)point1->val, *(float*)point2->val, (double)point1->key, (double)point2->key, (double)point->key);
-      break;
-    };
-    case TSDB_DATA_TYPE_DOUBLE: {
-      *(double*)point->val =
-        linearInterpolationImpl(*(double*)point1->val, *(double*)point2->val, (double)point1->key, (double)point2->key, (double)point->key);
-      break;
-    };
-    case TSDB_DATA_TYPE_TIMESTAMP:
-    case TSDB_DATA_TYPE_BIGINT: {
-      *(int64_t*)point->val = (int64_t)linearInterpolationImpl((double)(*(int64_t*)point1->val), (double)(*(int64_t*)point2->val), (double)point1->key,
-        (double)point2->key, (double)point->key);
-      break;
-    };
-    case TSDB_DATA_TYPE_SMALLINT: {
-      *(int16_t*)point->val = (int16_t)linearInterpolationImpl(*(int16_t*)point1->val, *(int16_t*)point2->val, (double)point1->key,
-        (double)point2->key, (double)point->key);
-      break;
-    };
-    case TSDB_DATA_TYPE_TINYINT: {
-      *(int8_t*) point->val = (int8_t)
-        linearInterpolationImpl(*(int8_t*)point1->val, *(int8_t*)point2->val, (double)point1->key, (double)point2->key, (double)point->key);
-      break;
-    };
-    default: {
-      // TODO: Deal with interpolation with bool and strings and timestamp
-      return -1;
-    }
+  double v1 = -1;
+  double v2 = -1;
+
+  GET_TYPED_DATA(v1, double, type, point1->val);
+  GET_TYPED_DATA(v2, double, type, point2->val);
+
+  double r = DO_INTERPOLATION(v1, v2, point1->key, point2->key, point->key);
+
+  switch(type) {
+    case TSDB_DATA_TYPE_TINYINT:  *(int8_t*) point->val  = (int8_t) r;break;
+    case TSDB_DATA_TYPE_SMALLINT: *(int16_t*) point->val = (int16_t) r;break;
+    case TSDB_DATA_TYPE_INT:      *(int32_t*) point->val = (int32_t) r;break;
+    case TSDB_DATA_TYPE_BIGINT:   *(int64_t*) point->val = (int64_t) r;break;
+    case TSDB_DATA_TYPE_DOUBLE:   *(double*) point->val  = (double) r;break;
+    case TSDB_DATA_TYPE_FLOAT:    *(float*) point->val   = (float) r;break;
+    default:
+      assert(0);
   }
 
-  return 0;
+  return TSDB_CODE_SUCCESS;
 }
 
-static void setTagsValue(SFillInfo* pFillInfo, tFilePage** data, int32_t num) {
+static void setTagsValue(SFillInfo* pFillInfo, tFilePage** data, int32_t genRows) {
   for(int32_t j = 0; j < pFillInfo->numOfCols; ++j) {
     SFillColInfo* pCol = &pFillInfo->pFillCol[j];
     if (TSDB_COL_IS_NORMAL_COL(pCol->flag)) {
       continue;
     }
 
-    char* val1 = elePtrAt(data[j]->data, pCol->col.bytes, num);
+    char* val1 = elePtrAt(data[j]->data, pCol->col.bytes, genRows);
 
-    for(int32_t i = 0; i < pFillInfo->numOfTags; ++i) {
-      SFillTagColInfo* pTag = &pFillInfo->pTags[i];
-      if (pTag->col.colId == pCol->col.colId) {
+    assert(pCol->tagIndex >= 0 && pCol->tagIndex < pFillInfo->numOfTags);
+    SFillTagColInfo* pTag = &pFillInfo->pTags[pCol->tagIndex];
+
+    assert (pTag->col.colId == pCol->col.colId);
     assignVal(val1, pTag->tagVal, pCol->col.bytes, pCol->col.type);
-        break;
-      }
   }
+}
+
+static void setNullValueForRow(SFillInfo* pFillInfo, tFilePage** data, int32_t numOfCol, int32_t rowIndex) {
+  // the first are always the timestamp column, so start from the second column.
+  for (int32_t i = 1; i < numOfCol; ++i) {
+    SFillColInfo* pCol = &pFillInfo->pFillCol[i];
+
+    char* output = elePtrAt(data[i]->data, pCol->col.bytes, rowIndex);
+    setNull(output, pCol->col.type, pCol->col.bytes);
   }
 }
 
-static void doFillResultImpl(SFillInfo* pFillInfo, tFilePage** data, int32_t* num, char** srcData, int64_t ts,
-                             bool outOfBound) {
-  char* prevValues = pFillInfo->prevValues;
-  char* nextValues = pFillInfo->nextValues;
+static void doFillOneRowResult(SFillInfo* pFillInfo, tFilePage** data, char** srcData, int64_t ts, bool outOfBound) {
+  char* prev = pFillInfo->prevValues;
+  char* next = pFillInfo->nextValues;
 
   SPoint point1, point2, point;
   int32_t step = GET_FORWARD_DIRECTION_FACTOR(pFillInfo->order);
 
-  char* val = elePtrAt(data[0]->data, TSDB_KEYSIZE, *num);
-  *(TSKEY*) val = pFillInfo->start;
-
-  int32_t numOfValCols = pFillInfo->numOfCols - pFillInfo->numOfTags;
+  // set the primary timestamp column value
+  int32_t index = pFillInfo->numOfCurrent;
+  char* val = elePtrAt(data[0]->data, TSDB_KEYSIZE, index);
+  *(TSKEY*) val = pFillInfo->currentKey;
 
   // set the other values
-  if (pFillInfo->fillType == TSDB_FILL_PREV) {
-    char* p = FILL_IS_ASC_FILL(pFillInfo) ? prevValues : nextValues;
+  if (pFillInfo->type == TSDB_FILL_PREV) {
+    char* p = FILL_IS_ASC_FILL(pFillInfo) ? prev : next;
     
     if (p != NULL) {
-      for (int32_t i = 1; i < numOfValCols; ++i) {
+      for (int32_t i = 1; i < pFillInfo->numOfCols; ++i) {
         SFillColInfo* pCol = &pFillInfo->pFillCol[i];
-        
-        char* val1 = elePtrAt(data[i]->data, pCol->col.bytes, *num);
-        if (isNull(p + pCol->col.offset, pCol->col.type)) {
-          if (pCol->col.type == TSDB_DATA_TYPE_BINARY || pCol->col.type == TSDB_DATA_TYPE_NCHAR) {
-            setVardataNull(val1, pCol->col.type);
-          } else {
-            setNull(val1, pCol->col.type, pCol->col.bytes);
-          }
-        } else {
-          assignVal(val1, p + pCol->col.offset, pCol->col.bytes, pCol->col.type);
-        }
+        if (TSDB_COL_IS_TAG(pCol->flag)) {
+          continue;
         }
-    } else { // no prev value yet, set the value for NULL
-      for (int32_t i = 1; i < numOfValCols; ++i) {
-        SFillColInfo* pCol = &pFillInfo->pFillCol[i];
 
-        char* val1 = elePtrAt(data[i]->data, pCol->col.bytes, *num);
-        if (pCol->col.type == TSDB_DATA_TYPE_BINARY||pCol->col.type == TSDB_DATA_TYPE_NCHAR) {
-          setVardataNull(val1, pCol->col.type);
-        } else {
-          setNull(val1, pCol->col.type, pCol->col.bytes);
-        }
+        char* output = elePtrAt(data[i]->data, pCol->col.bytes, index);
+        assignVal(output, p + pCol->col.offset, pCol->col.bytes, pCol->col.type);
       }
+    } else { // no prev value yet, set the value for NULL
+      setNullValueForRow(pFillInfo, data, pFillInfo->numOfCols, index);
     }
-
-    setTagsValue(pFillInfo, data, *num);
-  } else if (pFillInfo->fillType == TSDB_FILL_LINEAR) {
+  } else if (pFillInfo->type == TSDB_FILL_LINEAR) {
     // TODO : linear interpolation supports NULL value
-    if (prevValues != NULL && !outOfBound) {
-      for (int32_t i = 1; i < numOfValCols; ++i) {
+    if (prev != NULL && !outOfBound) {
+      for (int32_t i = 1; i < pFillInfo->numOfCols; ++i) {
         SFillColInfo* pCol = &pFillInfo->pFillCol[i];
+        if (TSDB_COL_IS_TAG(pCol->flag)) {
+          continue;
+        }
 
         int16_t type  = pCol->col.type;
         int16_t bytes = pCol->col.bytes;
         
-        char *val1 = elePtrAt(data[i]->data, pCol->col.bytes, *num);
-        if (type == TSDB_DATA_TYPE_BINARY|| type == TSDB_DATA_TYPE_NCHAR) {
-          setVardataNull(val1, pCol->col.type);
-          continue;
-        } else if (type == TSDB_DATA_TYPE_BOOL) {
+        char *val1 = elePtrAt(data[i]->data, pCol->col.bytes, index);
+        if (type == TSDB_DATA_TYPE_BINARY|| type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_BOOL) {
           setNull(val1, pCol->col.type, bytes);
           continue;
         }
 
-        point1 = (SPoint){.key = *(TSKEY*)(prevValues), .val = prevValues + pCol->col.offset};
-        point2 = (SPoint){.key = ts, .val = srcData[i] + pFillInfo->rowIdx * bytes};
+        point1 = (SPoint){.key = *(TSKEY*)(prev), .val = prev + pCol->col.offset};
+        point2 = (SPoint){.key = ts, .val = srcData[i] + pFillInfo->index * bytes};
         point  = (SPoint){.key = pFillInfo->start, .val = val1};
         taosGetLinearInterpolationVal(type, &point1, &point2, &point);
       }
-
-      setTagsValue(pFillInfo, data, *num);
-
     } else {
-      for (int32_t i = 1; i < numOfValCols; ++i) {
-        SFillColInfo* pCol = &pFillInfo->pFillCol[i];
-  
-        char* val1 = elePtrAt(data[i]->data, pCol->col.bytes, *num);
-  
-        if (pCol->col.type == TSDB_DATA_TYPE_BINARY || pCol->col.type == TSDB_DATA_TYPE_NCHAR) {
-          setVardataNull(val1, pCol->col.type);
-        } else {
-          setNull(val1, pCol->col.type, pCol->col.bytes);
-        }
-      }
-
-      setTagsValue(pFillInfo, data, *num);
-  
+      setNullValueForRow(pFillInfo, data, pFillInfo->numOfCols, index);
     }
   } else { /* fill the default value */
-    for (int32_t i = 1; i < numOfValCols; ++i) {
+    for (int32_t i = 1; i < pFillInfo->numOfCols; ++i) {
       SFillColInfo* pCol = &pFillInfo->pFillCol[i];
+      if (TSDB_COL_IS_TAG(pCol->flag)) {
+        continue;
+      }
 
-      char* val1 = elePtrAt(data[i]->data, pCol->col.bytes, *num);
+      char* val1 = elePtrAt(data[i]->data, pCol->col.bytes, index);
       assignVal(val1, (char*)&pCol->fillVal.i, pCol->col.bytes, pCol->col.type);
     }
-
-    setTagsValue(pFillInfo, data, *num);
   }
 
-  pFillInfo->start = taosTimeAdd(pFillInfo->start, pFillInfo->interval.sliding * step, pFillInfo->interval.slidingUnit, pFillInfo->precision);
+  setTagsValue(pFillInfo, data, index);
+  pFillInfo->currentKey = taosTimeAdd(pFillInfo->currentKey, pFillInfo->interval.sliding * step, pFillInfo->interval.slidingUnit, pFillInfo->precision);
   pFillInfo->numOfCurrent++;
-
-  (*num) += 1;
 }
 
-static void initBeforeAfterDataBuf(SFillInfo* pFillInfo, char** nextValues) {
-  if (*nextValues != NULL) {
+static void initBeforeAfterDataBuf(SFillInfo* pFillInfo, char** next) {
+  if (*next != NULL) {
     return;
   }
   
-  *nextValues = calloc(1, pFillInfo->rowSize);
+  *next = calloc(1, pFillInfo->rowSize);
   for (int i = 1; i < pFillInfo->numOfCols; i++) {
     SFillColInfo* pCol = &pFillInfo->pFillCol[i];
-  
-    if (pCol->col.type == TSDB_DATA_TYPE_BINARY||pCol->col.type == TSDB_DATA_TYPE_NCHAR) {
-      setVardataNull(*nextValues + pCol->col.offset, pCol->col.type);
-    } else {
-      setNull(*nextValues + pCol->col.offset, pCol->col.type, pCol->col.bytes);
+    setNull(*next + pCol->col.offset, pCol->col.type, pCol->col.bytes);
   }
+}
+
+static void copyCurrentRowIntoBuf(SFillInfo* pFillInfo, char** srcData, char* buf) {
+  int32_t rowIndex = pFillInfo->index;
+  for (int32_t i = 0; i < pFillInfo->numOfCols; ++i) {
+    SFillColInfo* pCol = &pFillInfo->pFillCol[i];
+    memcpy(buf + pCol->col.offset, srcData[i] + rowIndex * pCol->col.bytes, pCol->col.bytes);
   }
 }
 
-int32_t generateDataBlockImpl(SFillInfo* pFillInfo, tFilePage** data, int32_t numOfRows, int32_t outputRows, char** srcData) {
-  int32_t num = 0;
+static int32_t fillResultImpl(SFillInfo* pFillInfo, tFilePage** data, int32_t outputRows) {
   pFillInfo->numOfCurrent = 0;
 
-  char** prevValues = &pFillInfo->prevValues;
-  char** nextValues = &pFillInfo->nextValues;
+  char** srcData = pFillInfo->pData;
+  char** prev = &pFillInfo->prevValues;
+  char** next = &pFillInfo->nextValues;
 
-  int32_t numOfTags = pFillInfo->numOfTags;
   int32_t step = GET_FORWARD_DIRECTION_FACTOR(pFillInfo->order);
-  if (numOfRows == 0) {
-    /*
-     * These data are generated according to fill strategy, since the current timestamp is out of time window of
-     * real result set. Note that we need to keep the direct previous result rows, to generated the filled data.
-     */
-    while (num < outputRows) {
-      doFillResultImpl(pFillInfo, data, &num, srcData, pFillInfo->start, true);
-    }
-    
-    pFillInfo->numOfTotal += pFillInfo->numOfCurrent;
-    return outputRows;
 
+  if (FILL_IS_ASC_FILL(pFillInfo)) {
+    assert(pFillInfo->currentKey >= pFillInfo->start);
   } else {
-    while (1) {
-      int64_t ts = ((int64_t*)pFillInfo->pData[0])[pFillInfo->rowIdx];
+    assert(pFillInfo->currentKey <= pFillInfo->start);
+  }
 
-      if ((pFillInfo->start < ts && FILL_IS_ASC_FILL(pFillInfo)) ||
-          (pFillInfo->start > ts && !FILL_IS_ASC_FILL(pFillInfo))) {
-        /* set the next value for interpolation */
-        initBeforeAfterDataBuf(pFillInfo, nextValues);
+  while (pFillInfo->numOfCurrent < outputRows) {
+    int64_t ts = ((int64_t*)pFillInfo->pData[0])[pFillInfo->index];
 
-        int32_t offset = pFillInfo->rowIdx;
-        for (int32_t i = 0; i < pFillInfo->numOfCols - numOfTags; ++i) {
-          SFillColInfo* pCol = &pFillInfo->pFillCol[i];
-          memcpy(*nextValues + pCol->col.offset, srcData[i] + offset * pCol->col.bytes, pCol->col.bytes);
-        }
+    if ((pFillInfo->currentKey < ts && FILL_IS_ASC_FILL(pFillInfo)) ||
+        (pFillInfo->currentKey > ts && !FILL_IS_ASC_FILL(pFillInfo))) {
+      /* set the next value for interpolation */
+      initBeforeAfterDataBuf(pFillInfo, next);
+      copyCurrentRowIntoBuf(pFillInfo, srcData, *next);
     }
 
-      if (((pFillInfo->start < ts && FILL_IS_ASC_FILL(pFillInfo)) ||
-           (pFillInfo->start > ts && !FILL_IS_ASC_FILL(pFillInfo))) && num < outputRows) {
+    if (((pFillInfo->currentKey < ts && FILL_IS_ASC_FILL(pFillInfo)) || (pFillInfo->currentKey > ts && !FILL_IS_ASC_FILL(pFillInfo))) &&
+        pFillInfo->numOfCurrent < outputRows) {
 
-        while (((pFillInfo->start < ts && FILL_IS_ASC_FILL(pFillInfo)) ||
-                (pFillInfo->start > ts && !FILL_IS_ASC_FILL(pFillInfo))) && num < outputRows) {
-          doFillResultImpl(pFillInfo, data, &num, srcData, ts, false);
+      // fill the gap between two actual input rows
+      while (((pFillInfo->currentKey < ts && FILL_IS_ASC_FILL(pFillInfo)) ||
+              (pFillInfo->currentKey > ts && !FILL_IS_ASC_FILL(pFillInfo))) &&
+             pFillInfo->numOfCurrent < outputRows) {
+        doFillOneRowResult(pFillInfo, data, srcData, ts, false);
       }
 
-        /* output buffer is full, abort */
-        if ((num == outputRows && FILL_IS_ASC_FILL(pFillInfo)) || (num < 0 && !FILL_IS_ASC_FILL(pFillInfo))) {
+      // output buffer is full, abort
+      if (pFillInfo->numOfCurrent == outputRows) {
         pFillInfo->numOfTotal += pFillInfo->numOfCurrent;
         return outputRows;
       }
     } else {
-        assert(pFillInfo->start == ts);
-        initBeforeAfterDataBuf(pFillInfo, prevValues);
+      assert(pFillInfo->currentKey == ts);
+      initBeforeAfterDataBuf(pFillInfo, prev);
 
       // assign rows to dst buffer
       for (int32_t i = 0; i < pFillInfo->numOfCols; ++i) {
@@ -459,59 +426,81 @@ int32_t generateDataBlockImpl(SFillInfo* pFillInfo, tFilePage** data, int32_t nu
           continue;
         }
 
-          char* val1 = elePtrAt(data[i]->data, pCol->col.bytes, num);
-          char* src  = elePtrAt(srcData[i], pCol->col.bytes, pFillInfo->rowIdx);
+        char* output = elePtrAt(data[i]->data, pCol->col.bytes, pFillInfo->numOfCurrent);
+        char* src = elePtrAt(srcData[i], pCol->col.bytes, pFillInfo->index);
 
-          if (i == 0 ||
-              (pCol->functionId != TSDB_FUNC_COUNT && !isNull(src, pCol->col.type)) ||
+        if (i == 0 || (pCol->functionId != TSDB_FUNC_COUNT && !isNull(src, pCol->col.type)) ||
             (pCol->functionId == TSDB_FUNC_COUNT && GET_INT64_VAL(src) != 0)) {
-            assignVal(val1, src, pCol->col.bytes, pCol->col.type);
-            memcpy(*prevValues + pCol->col.offset, src, pCol->col.bytes);
+          assignVal(output, src, pCol->col.bytes, pCol->col.type);
+          memcpy(*prev + pCol->col.offset, src, pCol->col.bytes);
         } else {  // i > 0 and data is null , do interpolation
-            if (pFillInfo->fillType == TSDB_FILL_PREV) {
-              assignVal(val1, *prevValues + pCol->col.offset, pCol->col.bytes, pCol->col.type);
-            } else if (pFillInfo->fillType == TSDB_FILL_LINEAR) {
-              assignVal(val1, src, pCol->col.bytes, pCol->col.type);
-              memcpy(*prevValues + pCol->col.offset, src, pCol->col.bytes);
+          if (pFillInfo->type == TSDB_FILL_PREV) {
+            assignVal(output, *prev + pCol->col.offset, pCol->col.bytes, pCol->col.type);
+          } else if (pFillInfo->type == TSDB_FILL_LINEAR) {
+            assignVal(output, src, pCol->col.bytes, pCol->col.type);
+            memcpy(*prev + pCol->col.offset, src, pCol->col.bytes);
           } else {
-              assignVal(val1, (char*) &pCol->fillVal.i, pCol->col.bytes, pCol->col.type);
+            assignVal(output, (char*)&pCol->fillVal.i, pCol->col.bytes, pCol->col.type);
           }
         }
       }
 
       // set the tag value for final result
-        setTagsValue(pFillInfo, data, num);
-
-        pFillInfo->start = taosTimeAdd(pFillInfo->start, pFillInfo->interval.sliding*step, pFillInfo->interval.slidingUnit, pFillInfo->precision);
-        pFillInfo->rowIdx += 1;
+      setTagsValue(pFillInfo, data, pFillInfo->numOfCurrent);
 
-        pFillInfo->numOfCurrent +=1;
-        num += 1;
+      pFillInfo->currentKey = taosTimeAdd(pFillInfo->currentKey, pFillInfo->interval.sliding * step,
+                                     pFillInfo->interval.slidingUnit, pFillInfo->precision);
+      pFillInfo->index += 1;
+      pFillInfo->numOfCurrent += 1;
     }
 
-      if ((pFillInfo->rowIdx >= pFillInfo->numOfRows && FILL_IS_ASC_FILL(pFillInfo)) ||
-          (pFillInfo->rowIdx < 0 && !FILL_IS_ASC_FILL(pFillInfo)) || num >= outputRows) {
-        if (pFillInfo->rowIdx >= pFillInfo->numOfRows || pFillInfo->rowIdx < 0) {
-          pFillInfo->rowIdx = -1;
-          pFillInfo->numOfRows = 0;
-
+    if (pFillInfo->index >= pFillInfo->numOfRows || pFillInfo->numOfCurrent >= outputRows) {
       /* the raw data block is exhausted, next value does not exists */
-          tfree(*nextValues);
+      if (pFillInfo->index >= pFillInfo->numOfRows) {
+        tfree(*next);
       }
 
       pFillInfo->numOfTotal += pFillInfo->numOfCurrent;
-        return num;
+      return pFillInfo->numOfCurrent;
     }
   }
+
+  return pFillInfo->numOfCurrent;
+}
+
+static int64_t fillExternalResults(SFillInfo* pFillInfo, tFilePage** output, int64_t resultCapacity) {
+  /*
+   * These data are generated according to fill strategy, since the current timestamp is out of the time window of
+   * real result set. Note that we need to keep the direct previous result rows, to generated the filled data.
+   */
+  pFillInfo->numOfCurrent = 0;
+  while (pFillInfo->numOfCurrent < resultCapacity) {
+    doFillOneRowResult(pFillInfo, output, pFillInfo->pData, pFillInfo->start, true);
   }
+
+  pFillInfo->numOfTotal += pFillInfo->numOfCurrent;
+
+  assert(pFillInfo->numOfCurrent == resultCapacity);
+  return resultCapacity;
 }
 
-int64_t taosGenerateDataBlock(SFillInfo* pFillInfo, tFilePage** output, int32_t capacity) {
-  int32_t remain = taosNumOfRemainRows(pFillInfo);  // todo use iterator?
+int64_t taosFillResultDataBlock(SFillInfo* pFillInfo, tFilePage** output, int32_t capacity) {
+  int32_t remain = taosNumOfRemainRows(pFillInfo);
+
+  int64_t numOfRes = getNumOfResWithFill(pFillInfo, pFillInfo->end, capacity);
+  assert(numOfRes <= capacity);
+
+  // no data existed for fill operation now, append result according to the fill strategy
+  if (remain == 0) {
+    fillExternalResults(pFillInfo, output, numOfRes);
+  } else {
+    fillResultImpl(pFillInfo, output, (int32_t) numOfRes);
+    assert(numOfRes == pFillInfo->numOfCurrent);
+  }
 
-  int32_t rows = (int32_t)getFilledNumOfRes(pFillInfo, pFillInfo->endKey, capacity);
-  int32_t numOfRes = generateDataBlockImpl(pFillInfo, output, remain, rows, pFillInfo->pData);
-  assert(numOfRes == rows);
+  qDebug("fill:%p, generated fill result, src block:%d, index:%d, brange:%"PRId64"-%"PRId64", currentKey:%"PRId64", current:%d, total:%d, %p",
+      pFillInfo, pFillInfo->numOfRows, pFillInfo->index, pFillInfo->start, pFillInfo->end, pFillInfo->currentKey, pFillInfo->numOfCurrent,
+         pFillInfo->numOfTotal, pFillInfo->handle);
 
   return numOfRes;
 }

src/query/src/qParserImpl.c

@@ -15,11 +15,9 @@
 
 #include "os.h"
 #include "qSqlparser.h"
-#include "queryLog.h"
 #include "taosdef.h"
 #include "taosmsg.h"
 #include "tcmdtype.h"
-#include "tglobal.h"
 #include "tstoken.h"
 #include "tstrbuild.h"
 #include "ttokendef.h"
@@ -227,13 +225,12 @@ tSQLExpr *tSQLExprCreate(tSQLExpr *pLeft, tSQLExpr *pRight, int32_t optrType) {
       tSQLExprDestroy(pLeft);
       tSQLExprDestroy(pRight);
 
-    } else if ((pLeft->val.nType == TSDB_DATA_TYPE_DOUBLE && pRight->val.nType == TSDB_DATA_TYPE_BIGINT) ||
-               (pRight->val.nType == TSDB_DATA_TYPE_DOUBLE && pLeft->val.nType == TSDB_DATA_TYPE_BIGINT)) {
+    } else if (pLeft->nSQLOptr == TK_FLOAT || pRight->nSQLOptr == TK_FLOAT) {
       pExpr->val.nType = TSDB_DATA_TYPE_DOUBLE;
       pExpr->nSQLOptr  = TK_FLOAT;
 
-      double left = pLeft->val.nType == TSDB_DATA_TYPE_DOUBLE ? pLeft->val.dKey : pLeft->val.i64Key;
-      double right = pRight->val.nType == TSDB_DATA_TYPE_DOUBLE ? pRight->val.dKey : pRight->val.i64Key;
+      double left  = (pLeft->val.nType == TSDB_DATA_TYPE_DOUBLE) ? pLeft->val.dKey : pLeft->val.i64Key;
+      double right = (pRight->val.nType == TSDB_DATA_TYPE_DOUBLE) ? pRight->val.dKey : pRight->val.i64Key;
 
       switch (optrType) {
         case TK_PLUS: {
@@ -314,132 +311,59 @@ void tSQLExprDestroy(tSQLExpr *pExpr) {
   tSQLExprNodeDestroy(pExpr);
 }
 
-static void *tVariantListExpand(tVariantList *pList) {
-  if (pList->nAlloc <= pList->nExpr) {  //
-    int32_t newSize = (pList->nAlloc << 1) + 4;
-
-    void *ptr = realloc(pList->a, newSize * sizeof(pList->a[0]));
-    if (ptr == 0) {
-      return NULL;
-    }
-
-    pList->nAlloc = newSize;
-    pList->a = ptr;
-  }
-
-  assert(pList->a != 0);
-  return pList;
-}
-
-tVariantList *tVariantListAppend(tVariantList *pList, tVariant *pVar, uint8_t sortOrder) {
+SArray *tVariantListAppendToken(SArray *pList, SStrToken *pToken, uint8_t order) {
   if (pList == NULL) {
-    pList = calloc(1, sizeof(tVariantList));
+    pList = taosArrayInit(4, sizeof(tVariantListItem));
   }
 
-  if (tVariantListExpand(pList) == NULL) {
-    return pList;
-  }
+  if (pToken) {
+    tVariantListItem item;
+    tVariantCreate(&item.pVar, pToken);
+    item.sortOrder = order;
 
-  if (pVar) {
-    tVariantListItem *pItem = &pList->a[pList->nExpr++];
-    /*
-     * Here we do not employ the assign function, since we need the pz attribute of structure
-     * , which is the point to char string, to free it!
-     *
-     * Otherwise, the original pointer may be lost, which causes memory leak.
-     */
-    memcpy(pItem, pVar, sizeof(tVariant));
-    pItem->sortOrder = sortOrder;
+    taosArrayPush(pList, &item);
   }
+
   return pList;
 }
 
-tVariantList *tVariantListInsert(tVariantList *pList, tVariant *pVar, uint8_t sortOrder, int32_t index) {
-  if (pList == NULL || index >= pList->nExpr) {
-    return tVariantListAppend(NULL, pVar, sortOrder);
+SArray *tVariantListAppend(SArray *pList, tVariant *pVar, uint8_t sortOrder) {
+  if (pList == NULL) {
+    pList = taosArrayInit(4, sizeof(tVariantListItem));
   }
 
-  if (tVariantListExpand(pList) == NULL) {
+  if (pVar == NULL) {
     return pList;
   }
 
-  if (pVar) {
-    memmove(&pList->a[index + 1], &pList->a[index], sizeof(tVariantListItem) * (pList->nExpr - index));
-
-    tVariantListItem *pItem = &pList->a[index];
   /*
    * Here we do not employ the assign function, since we need the pz attribute of structure
    * , which is the point to char string, to free it!
    *
    * Otherwise, the original pointer may be lost, which causes memory leak.
    */
-    memcpy(pItem, pVar, sizeof(tVariant));
-    pItem->sortOrder = sortOrder;
-
-    pList->nExpr++;
-  }
+  tVariantListItem item;
+  item.pVar = *pVar;
+  item.sortOrder = sortOrder;
 
+  taosArrayPush(pList, &item);
   return pList;
 }
 
-void tVariantListDestroy(tVariantList *pList) {
-  if (pList == NULL) return;
-
-  for (int32_t i = 0; i < pList->nExpr; ++i) {
-    tVariantDestroy(&pList->a[i].pVar);
-  }
-
-  free(pList->a);
-  free(pList);
-}
-
-tVariantList *tVariantListAppendToken(tVariantList *pList, SStrToken *pToken, uint8_t sortOrder) {
-  if (pList == NULL) {
-    pList = calloc(1, sizeof(tVariantList));
-  }
-
-  if (tVariantListExpand(pList) == NULL) {
-    return pList;
-  }
-
-  if (pToken) {
-    tVariant t = {0};
-    tVariantCreate(&t, pToken);
-
-    tVariantListItem *pItem = &pList->a[pList->nExpr++];
-    memcpy(pItem, &t, sizeof(tVariant));
-    pItem->sortOrder = sortOrder;
+SArray *tVariantListInsert(SArray *pList, tVariant *pVar, uint8_t sortOrder, int32_t index) {
+  if (pList == NULL || pVar == NULL || index >= taosArrayGetSize(pList)) {
+    return tVariantListAppend(NULL, pVar, sortOrder);
   }
-  return pList;
-}
 
-tFieldList *tFieldListAppend(tFieldList *pList, TAOS_FIELD *pField) {
-  if (pList == NULL) pList = calloc(1, sizeof(tFieldList));
+  tVariantListItem item;
 
-  if (pList->nAlloc <= pList->nField) {  //
-    pList->nAlloc = (pList->nAlloc << 1) + 4;
-    pList->p = realloc(pList->p, pList->nAlloc * sizeof(pList->p[0]));
-    if (pList->p == 0) {
-      pList->nField = pList->nAlloc = 0;
-      return pList;
-    }
-  }
-  assert(pList->p != 0);
+  item.pVar = *pVar;
+  item.sortOrder = sortOrder;
 
-  if (pField) {
-    struct TAOS_FIELD *pItem = (struct TAOS_FIELD *)&pList->p[pList->nField++];
-    memcpy(pItem, pField, sizeof(TAOS_FIELD));
-  }
+  taosArrayInsert(pList, index, &item);
   return pList;
 }
 
-void tFieldListDestroy(tFieldList *pList) {
-  if (pList == NULL) return;
-
-  free(pList->p);
-  free(pList);
-}
-
 void setDBName(SStrToken *pCpxName, SStrToken *pDB) {
   pCpxName->type = pDB->type;
   pCpxName->z = pDB->z;
@@ -464,8 +388,6 @@ void tSQLSetColumnInfo(TAOS_FIELD *pField, SStrToken *pName, TAOS_FIELD *pType)
 void tSQLSetColumnType(TAOS_FIELD *pField, SStrToken *type) {
   pField->type = -1;
 
-  int32_t LENGTH_SIZE_OF_STR = 2;  // in case of nchar and binary, there two bytes to keep the length of binary|nchar.
-  
   for (int8_t i = 0; i < tListLen(tDataTypeDesc); ++i) {
     if ((strncasecmp(type->z, tDataTypeDesc[i].aName, tDataTypeDesc[i].nameLen) == 0) &&
         (type->n == tDataTypeDesc[i].nameLen)) {
@@ -481,14 +403,14 @@ void tSQLSetColumnType(TAOS_FIELD *pField, SStrToken *type) {
         if (type->type == 0) {
           pField->bytes = 0;
         } else {
-          pField->bytes = -(int32_t)type->type * TSDB_NCHAR_SIZE + LENGTH_SIZE_OF_STR;
+          pField->bytes = (int16_t)(-(int32_t)type->type * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE);
         }
       } else if (i == TSDB_DATA_TYPE_BINARY) {
         /* for binary, the TOKENTYPE is the length of binary */
         if (type->type == 0) {
           pField->bytes = 0;
         } else {
-          pField->bytes = -(int32_t) type->type + LENGTH_SIZE_OF_STR;
+          pField->bytes = (int16_t) (-(int32_t) type->type + VARSTR_HEADER_SIZE);
         } 
       }
       break;
@@ -499,9 +421,9 @@ void tSQLSetColumnType(TAOS_FIELD *pField, SStrToken *type) {
 /*
  * extract the select info out of sql string
  */
-SQuerySQL *tSetQuerySQLElems(SStrToken *pSelectToken, tSQLExprList *pSelection, tVariantList *pFrom, tSQLExpr *pWhere,
-                             tVariantList *pGroupby, tVariantList *pSortOrder, SIntervalVal *pInterval,
-                             SStrToken *pSliding, tVariantList *pFill, SLimitVal *pLimit, SLimitVal *pGLimit) {
+SQuerySQL *tSetQuerySQLElems(SStrToken *pSelectToken, tSQLExprList *pSelection, SArray *pFrom, tSQLExpr *pWhere,
+                             SArray *pGroupby, SArray *pSortOrder, SIntervalVal *pInterval,
+                             SStrToken *pSliding, SArray *pFill, SLimitVal *pLimit, SLimitVal *pGLimit) {
   assert(pSelection != NULL);
 
   SQuerySQL *pQuery = calloc(1, sizeof(SQuerySQL));
@@ -535,6 +457,11 @@ SQuerySQL *tSetQuerySQLElems(SStrToken *pSelectToken, tSQLExprList *pSelection,
   return pQuery;
 }
 
+void freeVariant(void *pItem) {
+  tVariantListItem* p = (tVariantListItem*) pItem;
+  tVariantDestroy(&p->pVar);
+}
+
 void doDestroyQuerySql(SQuerySQL *pQuerySql) {
   if (pQuerySql == NULL) {
     return;
@@ -547,16 +474,17 @@ void doDestroyQuerySql(SQuerySQL *pQuerySql) {
   tSQLExprDestroy(pQuerySql->pWhere);
   pQuerySql->pWhere = NULL;
   
-  tVariantListDestroy(pQuerySql->pSortOrder);
+  taosArrayDestroyEx(pQuerySql->pSortOrder, freeVariant);
   pQuerySql->pSortOrder = NULL;
 
-  tVariantListDestroy(pQuerySql->pGroupby);
+  taosArrayDestroyEx(pQuerySql->pGroupby, freeVariant);
   pQuerySql->pGroupby = NULL;
 
-  tVariantListDestroy(pQuerySql->from);
+  taosArrayDestroyEx(pQuerySql->from, freeVariant);
   pQuerySql->from = NULL;
 
-  tVariantListDestroy(pQuerySql->fillType);
+  taosArrayDestroyEx(pQuerySql->fillType, freeVariant);
+  pQuerySql->fillType = NULL;
 
   free(pQuerySql);
 }
@@ -574,8 +502,8 @@ void destroyAllSelectClause(SSubclauseInfo *pClause) {
   tfree(pClause->pClause);
 }
 
-SCreateTableSQL *tSetCreateSQLElems(tFieldList *pCols, tFieldList *pTags, SStrToken *pStableName,
-                                    tVariantList *pTagVals, SQuerySQL *pSelect, int32_t type) {
+SCreateTableSQL *tSetCreateSQLElems(SArray *pCols, SArray *pTags, SStrToken *pStableName,
+                                    SArray *pTagVals, SQuerySQL *pSelect, int32_t type) {
   SCreateTableSQL *pCreate = calloc(1, sizeof(SCreateTableSQL));
 
   switch (type) {
@@ -607,7 +535,7 @@ SCreateTableSQL *tSetCreateSQLElems(tFieldList *pCols, tFieldList *pTags, SStrTo
   return pCreate;
 }
 
-SAlterTableSQL *tAlterTableSQLElems(SStrToken *pMeterName, tFieldList *pCols, tVariantList *pVals, int32_t type) {
+SAlterTableSQL *tAlterTableSQLElems(SStrToken *pMeterName, SArray *pCols, SArray *pVals, int32_t type) {
   SAlterTableSQL *pAlterTable = calloc(1, sizeof(SAlterTableSQL));
   
   pAlterTable->name = *pMeterName;
@@ -637,14 +565,14 @@ void SQLInfoDestroy(SSqlInfo *pInfo) {
     SCreateTableSQL *pCreateTableInfo = pInfo->pCreateTableInfo;
     doDestroyQuerySql(pCreateTableInfo->pSelect);
 
-    tFieldListDestroy(pCreateTableInfo->colInfo.pColumns);
-    tFieldListDestroy(pCreateTableInfo->colInfo.pTagColumns);
+    taosArrayDestroy(pCreateTableInfo->colInfo.pColumns);
+    taosArrayDestroy(pCreateTableInfo->colInfo.pTagColumns);
 
-    tVariantListDestroy(pCreateTableInfo->usingInfo.pTagVals);
+    taosArrayDestroyEx(pCreateTableInfo->usingInfo.pTagVals, freeVariant);
     tfree(pInfo->pCreateTableInfo);
   } else if (pInfo->type == TSDB_SQL_ALTER_TABLE) {
-    tVariantListDestroy(pInfo->pAlterInfo->varList);
-    tFieldListDestroy(pInfo->pAlterInfo->pAddColumns);
+    taosArrayDestroyEx(pInfo->pAlterInfo->varList, freeVariant);
+    taosArrayDestroy(pInfo->pAlterInfo->pAddColumns);
     
     tfree(pInfo->pAlterInfo);
   } else {
@@ -653,7 +581,7 @@ void SQLInfoDestroy(SSqlInfo *pInfo) {
     }
 
     if (pInfo->pDCLInfo != NULL && pInfo->type == TSDB_SQL_CREATE_DB) {
-      tVariantListDestroy(pInfo->pDCLInfo->dbOpt.keep);
+      taosArrayDestroyEx(pInfo->pDCLInfo->dbOpt.keep, freeVariant);
     }
 
     tfree(pInfo->pDCLInfo);

src/query/src/sql.c

@@ -107,6 +107,7 @@ typedef union {
   tSQLExpr* yy64;
   tVariant yy134;
   SCreateAcctSQL yy149;
+  SArray* yy165;
   int64_t yy207;
   SLimitVal yy216;
   TAOS_FIELD yy223;
@@ -115,8 +116,6 @@ typedef union {
   tSQLExprList* yy290;
   SQuerySQL* yy414;
   SCreateTableSQL* yy470;
-  tVariantList* yy498;
-  tFieldList* yy523;
   SIntervalVal yy532;
 } YYMINORTYPE;
 #ifndef YYSTACKDEPTH
@@ -1368,18 +1367,14 @@ static void yy_destructor(
 /********* Begin destructor definitions ***************************************/
     case 227: /* keep */
     case 228: /* tagitemlist */
+    case 245: /* columnlist */
     case 253: /* fill_opt */
     case 255: /* groupby_opt */
     case 256: /* orderby_opt */
     case 266: /* sortlist */
     case 270: /* grouplist */
 {
-tVariantListDestroy((yypminor->yy498));
-}
-      break;
-    case 245: /* columnlist */
-{
-tFieldListDestroy((yypminor->yy523));
+taosArrayDestroy((yypminor->yy165));
 }
       break;
     case 246: /* select */
@@ -2238,7 +2233,7 @@ static void yy_reduce(
   yymsp[-8].minor.yy149 = yylhsminor.yy149;
         break;
       case 72: /* keep ::= KEEP tagitemlist */
-{ yymsp[-1].minor.yy498 = yymsp[0].minor.yy498; }
+{ yymsp[-1].minor.yy165 = yymsp[0].minor.yy165; }
         break;
       case 73: /* cache ::= CACHE INTEGER */
       case 74: /* replica ::= REPLICA INTEGER */ yytestcase(yyruleno==74);
@@ -2314,7 +2309,7 @@ static void yy_reduce(
         break;
       case 99: /* db_optr ::= db_optr keep */
       case 104: /* alter_db_optr ::= alter_db_optr keep */ yytestcase(yyruleno==104);
-{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.keep = yymsp[0].minor.yy498; }
+{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.keep = yymsp[0].minor.yy165; }
   yymsp[-1].minor.yy268 = yylhsminor.yy268;
         break;
       case 100: /* db_optr ::= db_optr update */
@@ -2362,20 +2357,20 @@ static void yy_reduce(
         break;
       case 116: /* create_table_args ::= LP columnlist RP */
 {
-    yymsp[-2].minor.yy470 = tSetCreateSQLElems(yymsp[-1].minor.yy523, NULL, NULL, NULL, NULL, TSQL_CREATE_TABLE);
+    yymsp[-2].minor.yy470 = tSetCreateSQLElems(yymsp[-1].minor.yy165, NULL, NULL, NULL, NULL, TSQL_CREATE_TABLE);
     setSQLInfo(pInfo, yymsp[-2].minor.yy470, NULL, TSDB_SQL_CREATE_TABLE);
 }
         break;
       case 117: /* create_table_args ::= LP columnlist RP TAGS LP columnlist RP */
 {
-    yymsp[-6].minor.yy470 = tSetCreateSQLElems(yymsp[-5].minor.yy523, yymsp[-1].minor.yy523, NULL, NULL, NULL, TSQL_CREATE_STABLE);
+    yymsp[-6].minor.yy470 = tSetCreateSQLElems(yymsp[-5].minor.yy165, yymsp[-1].minor.yy165, NULL, NULL, NULL, TSQL_CREATE_STABLE);
     setSQLInfo(pInfo, yymsp[-6].minor.yy470, NULL, TSDB_SQL_CREATE_TABLE);
 }
         break;
       case 118: /* create_table_args ::= USING ids cpxName TAGS LP tagitemlist RP */
 {
     yymsp[-5].minor.yy0.n += yymsp[-4].minor.yy0.n;
-    yymsp[-6].minor.yy470 = tSetCreateSQLElems(NULL, NULL, &yymsp[-5].minor.yy0, yymsp[-1].minor.yy498, NULL, TSQL_CREATE_TABLE_FROM_STABLE);
+    yymsp[-6].minor.yy470 = tSetCreateSQLElems(NULL, NULL, &yymsp[-5].minor.yy0, yymsp[-1].minor.yy165, NULL, TSQL_CREATE_TABLE_FROM_STABLE);
     setSQLInfo(pInfo, yymsp[-6].minor.yy470, NULL, TSDB_SQL_CREATE_TABLE);
 }
         break;
@@ -2386,12 +2381,12 @@ static void yy_reduce(
 }
         break;
       case 120: /* columnlist ::= columnlist COMMA column */
-{yylhsminor.yy523 = tFieldListAppend(yymsp[-2].minor.yy523, &yymsp[0].minor.yy223);   }
-  yymsp[-2].minor.yy523 = yylhsminor.yy523;
+{taosArrayPush(yymsp[-2].minor.yy165, &yymsp[0].minor.yy223); yylhsminor.yy165 = yymsp[-2].minor.yy165;  }
+  yymsp[-2].minor.yy165 = yylhsminor.yy165;
         break;
       case 121: /* columnlist ::= column */
-{yylhsminor.yy523 = tFieldListAppend(NULL, &yymsp[0].minor.yy223);}
-  yymsp[0].minor.yy523 = yylhsminor.yy523;
+{yylhsminor.yy165 = taosArrayInit(4, sizeof(TAOS_FIELD)); taosArrayPush(yylhsminor.yy165, &yymsp[0].minor.yy223);}
+  yymsp[0].minor.yy165 = yylhsminor.yy165;
         break;
       case 122: /* column ::= ids typename */
 {
@@ -2400,12 +2395,12 @@ static void yy_reduce(
   yymsp[-1].minor.yy223 = yylhsminor.yy223;
         break;
       case 123: /* tagitemlist ::= tagitemlist COMMA tagitem */
-{ yylhsminor.yy498 = tVariantListAppend(yymsp[-2].minor.yy498, &yymsp[0].minor.yy134, -1);    }
-  yymsp[-2].minor.yy498 = yylhsminor.yy498;
+{ yylhsminor.yy165 = tVariantListAppend(yymsp[-2].minor.yy165, &yymsp[0].minor.yy134, -1);    }
+  yymsp[-2].minor.yy165 = yylhsminor.yy165;
         break;
       case 124: /* tagitemlist ::= tagitem */
-{ yylhsminor.yy498 = tVariantListAppend(NULL, &yymsp[0].minor.yy134, -1); }
-  yymsp[0].minor.yy498 = yylhsminor.yy498;
+{ yylhsminor.yy165 = tVariantListAppend(NULL, &yymsp[0].minor.yy134, -1); }
+  yymsp[0].minor.yy165 = yylhsminor.yy165;
         break;
       case 125: /* tagitem ::= INTEGER */
       case 126: /* tagitem ::= FLOAT */ yytestcase(yyruleno==126);
@@ -2432,7 +2427,7 @@ static void yy_reduce(
         break;
       case 134: /* select ::= SELECT selcollist from where_opt interval_opt fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt */
 {
-  yylhsminor.yy414 = tSetQuerySQLElems(&yymsp[-11].minor.yy0, yymsp[-10].minor.yy290, yymsp[-9].minor.yy498, yymsp[-8].minor.yy64, yymsp[-4].minor.yy498, yymsp[-3].minor.yy498, &yymsp[-7].minor.yy532, &yymsp[-5].minor.yy0, yymsp[-6].minor.yy498, &yymsp[0].minor.yy216, &yymsp[-1].minor.yy216);
+  yylhsminor.yy414 = tSetQuerySQLElems(&yymsp[-11].minor.yy0, yymsp[-10].minor.yy290, yymsp[-9].minor.yy165, yymsp[-8].minor.yy64, yymsp[-4].minor.yy165, yymsp[-3].minor.yy165, &yymsp[-7].minor.yy532, &yymsp[-5].minor.yy0, yymsp[-6].minor.yy165, &yymsp[0].minor.yy216, &yymsp[-1].minor.yy216);
 }
   yymsp[-11].minor.yy414 = yylhsminor.yy414;
         break;
@@ -2491,45 +2486,45 @@ static void yy_reduce(
 { yymsp[1].minor.yy0.n = 0;  }
         break;
       case 148: /* from ::= FROM tablelist */
-{yymsp[-1].minor.yy498 = yymsp[0].minor.yy498;}
+{yymsp[-1].minor.yy165 = yymsp[0].minor.yy165;}
         break;
       case 149: /* tablelist ::= ids cpxName */
 {
   toTSDBType(yymsp[-1].minor.yy0.type);
   yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n;
-  yylhsminor.yy498 = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1);
-  yylhsminor.yy498 = tVariantListAppendToken(yylhsminor.yy498, &yymsp[-1].minor.yy0, -1);  // table alias name
+  yylhsminor.yy165 = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1);
+  yylhsminor.yy165 = tVariantListAppendToken(yylhsminor.yy165, &yymsp[-1].minor.yy0, -1);  // table alias name
 }
-  yymsp[-1].minor.yy498 = yylhsminor.yy498;
+  yymsp[-1].minor.yy165 = yylhsminor.yy165;
         break;
       case 150: /* tablelist ::= ids cpxName ids */
 {
    toTSDBType(yymsp[-2].minor.yy0.type);
    toTSDBType(yymsp[0].minor.yy0.type);
    yymsp[-2].minor.yy0.n += yymsp[-1].minor.yy0.n;
-   yylhsminor.yy498 = tVariantListAppendToken(NULL, &yymsp[-2].minor.yy0, -1);
-   yylhsminor.yy498 = tVariantListAppendToken(yylhsminor.yy498, &yymsp[0].minor.yy0, -1);
+   yylhsminor.yy165 = tVariantListAppendToken(NULL, &yymsp[-2].minor.yy0, -1);
+   yylhsminor.yy165 = tVariantListAppendToken(yylhsminor.yy165, &yymsp[0].minor.yy0, -1);
 }
-  yymsp[-2].minor.yy498 = yylhsminor.yy498;
+  yymsp[-2].minor.yy165 = yylhsminor.yy165;
         break;
       case 151: /* tablelist ::= tablelist COMMA ids cpxName */
 {
   toTSDBType(yymsp[-1].minor.yy0.type);
   yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n;
-  yylhsminor.yy498 = tVariantListAppendToken(yymsp[-3].minor.yy498, &yymsp[-1].minor.yy0, -1);
-  yylhsminor.yy498 = tVariantListAppendToken(yylhsminor.yy498, &yymsp[-1].minor.yy0, -1);
+  yylhsminor.yy165 = tVariantListAppendToken(yymsp[-3].minor.yy165, &yymsp[-1].minor.yy0, -1);
+  yylhsminor.yy165 = tVariantListAppendToken(yylhsminor.yy165, &yymsp[-1].minor.yy0, -1);
 }
-  yymsp[-3].minor.yy498 = yylhsminor.yy498;
+  yymsp[-3].minor.yy165 = yylhsminor.yy165;
         break;
       case 152: /* tablelist ::= tablelist COMMA ids cpxName ids */
 {
    toTSDBType(yymsp[-2].minor.yy0.type);
    toTSDBType(yymsp[0].minor.yy0.type);
    yymsp[-2].minor.yy0.n += yymsp[-1].minor.yy0.n;
-   yylhsminor.yy498 = tVariantListAppendToken(yymsp[-4].minor.yy498, &yymsp[-2].minor.yy0, -1);
-   yylhsminor.yy498 = tVariantListAppendToken(yylhsminor.yy498, &yymsp[0].minor.yy0, -1);
+   yylhsminor.yy165 = tVariantListAppendToken(yymsp[-4].minor.yy165, &yymsp[-2].minor.yy0, -1);
+   yylhsminor.yy165 = tVariantListAppendToken(yylhsminor.yy165, &yymsp[0].minor.yy0, -1);
 }
-  yymsp[-4].minor.yy498 = yylhsminor.yy498;
+  yymsp[-4].minor.yy165 = yylhsminor.yy165;
         break;
       case 153: /* tmvar ::= VARIABLE */
 {yylhsminor.yy0 = yymsp[0].minor.yy0;}
@@ -2545,7 +2540,7 @@ static void yy_reduce(
 {memset(&yymsp[1].minor.yy532, 0, sizeof(yymsp[1].minor.yy532));}
         break;
       case 157: /* fill_opt ::= */
-{yymsp[1].minor.yy498 = 0;     }
+{yymsp[1].minor.yy165 = 0;     }
         break;
       case 158: /* fill_opt ::= FILL LP ID COMMA tagitemlist RP */
 {
@@ -2553,14 +2548,14 @@ static void yy_reduce(
     toTSDBType(yymsp[-3].minor.yy0.type);
     tVariantCreate(&A, &yymsp[-3].minor.yy0);
 
-    tVariantListInsert(yymsp[-1].minor.yy498, &A, -1, 0);
-    yymsp[-5].minor.yy498 = yymsp[-1].minor.yy498;
+    tVariantListInsert(yymsp[-1].minor.yy165, &A, -1, 0);
+    yymsp[-5].minor.yy165 = yymsp[-1].minor.yy165;
 }
         break;
       case 159: /* fill_opt ::= FILL LP ID RP */
 {
     toTSDBType(yymsp[-1].minor.yy0.type);
-    yymsp[-3].minor.yy498 = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1);
+    yymsp[-3].minor.yy165 = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1);
 }
         break;
       case 160: /* sliding_opt ::= SLIDING LP tmvar RP */
@@ -2571,23 +2566,23 @@ static void yy_reduce(
         break;
       case 162: /* orderby_opt ::= */
       case 170: /* groupby_opt ::= */ yytestcase(yyruleno==170);
-{yymsp[1].minor.yy498 = 0;}
+{yymsp[1].minor.yy165 = 0;}
         break;
       case 163: /* orderby_opt ::= ORDER BY sortlist */
       case 171: /* groupby_opt ::= GROUP BY grouplist */ yytestcase(yyruleno==171);
-{yymsp[-2].minor.yy498 = yymsp[0].minor.yy498;}
+{yymsp[-2].minor.yy165 = yymsp[0].minor.yy165;}
         break;
       case 164: /* sortlist ::= sortlist COMMA item sortorder */
 {
-    yylhsminor.yy498 = tVariantListAppend(yymsp[-3].minor.yy498, &yymsp[-1].minor.yy134, yymsp[0].minor.yy46);
+    yylhsminor.yy165 = tVariantListAppend(yymsp[-3].minor.yy165, &yymsp[-1].minor.yy134, yymsp[0].minor.yy46);
 }
-  yymsp[-3].minor.yy498 = yylhsminor.yy498;
+  yymsp[-3].minor.yy165 = yylhsminor.yy165;
         break;
       case 165: /* sortlist ::= item sortorder */
 {
-  yylhsminor.yy498 = tVariantListAppend(NULL, &yymsp[-1].minor.yy134, yymsp[0].minor.yy46);
+  yylhsminor.yy165 = tVariantListAppend(NULL, &yymsp[-1].minor.yy134, yymsp[0].minor.yy46);
 }
-  yymsp[-1].minor.yy498 = yylhsminor.yy498;
+  yymsp[-1].minor.yy165 = yylhsminor.yy165;
         break;
       case 166: /* item ::= ids cpxName */
 {
@@ -2609,15 +2604,15 @@ static void yy_reduce(
         break;
       case 172: /* grouplist ::= grouplist COMMA item */
 {
-  yylhsminor.yy498 = tVariantListAppend(yymsp[-2].minor.yy498, &yymsp[0].minor.yy134, -1);
+  yylhsminor.yy165 = tVariantListAppend(yymsp[-2].minor.yy165, &yymsp[0].minor.yy134, -1);
 }
-  yymsp[-2].minor.yy498 = yylhsminor.yy498;
+  yymsp[-2].minor.yy165 = yylhsminor.yy165;
         break;
       case 173: /* grouplist ::= item */
 {
-  yylhsminor.yy498 = tVariantListAppend(NULL, &yymsp[0].minor.yy134, -1);
+  yylhsminor.yy165 = tVariantListAppend(NULL, &yymsp[0].minor.yy134, -1);
 }
-  yymsp[0].minor.yy498 = yylhsminor.yy498;
+  yymsp[0].minor.yy165 = yylhsminor.yy165;
         break;
       case 174: /* having_opt ::= */
       case 184: /* where_opt ::= */ yytestcase(yyruleno==184);
@@ -2633,15 +2628,21 @@ static void yy_reduce(
 {yymsp[1].minor.yy216.limit = -1; yymsp[1].minor.yy216.offset = 0;}
         break;
       case 177: /* limit_opt ::= LIMIT signed */
-      case 181: /* slimit_opt ::= SLIMIT signed */ yytestcase(yyruleno==181);
 {yymsp[-1].minor.yy216.limit = yymsp[0].minor.yy207;  yymsp[-1].minor.yy216.offset = 0;}
         break;
       case 178: /* limit_opt ::= LIMIT signed OFFSET signed */
-      case 182: /* slimit_opt ::= SLIMIT signed SOFFSET signed */ yytestcase(yyruleno==182);
 {yymsp[-3].minor.yy216.limit = yymsp[-2].minor.yy207;  yymsp[-3].minor.yy216.offset = yymsp[0].minor.yy207;}
         break;
       case 179: /* limit_opt ::= LIMIT signed COMMA signed */
-      case 183: /* slimit_opt ::= SLIMIT signed COMMA signed */ yytestcase(yyruleno==183);
+{yymsp[-3].minor.yy216.limit = yymsp[0].minor.yy207;  yymsp[-3].minor.yy216.offset = yymsp[-2].minor.yy207;}
+        break;
+      case 181: /* slimit_opt ::= SLIMIT signed */
+{yymsp[-1].minor.yy216.limit = yymsp[0].minor.yy207;  yymsp[-1].minor.yy216.offset = 0;}
+        break;
+      case 182: /* slimit_opt ::= SLIMIT signed SOFFSET signed */
+{yymsp[-3].minor.yy216.limit = yymsp[-2].minor.yy207;  yymsp[-3].minor.yy216.offset = yymsp[0].minor.yy207;}
+        break;
+      case 183: /* slimit_opt ::= SLIMIT signed COMMA signed */
 {yymsp[-3].minor.yy216.limit = yymsp[0].minor.yy207;  yymsp[-3].minor.yy216.offset = yymsp[-2].minor.yy207;}
         break;
       case 186: /* expr ::= LP expr RP */
@@ -2787,7 +2788,7 @@ static void yy_reduce(
       case 224: /* cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */
 {
     yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n;
-    SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&yymsp[-4].minor.yy0, yymsp[0].minor.yy523, NULL, TSDB_ALTER_TABLE_ADD_COLUMN);
+    SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&yymsp[-4].minor.yy0, yymsp[0].minor.yy165, NULL, TSDB_ALTER_TABLE_ADD_COLUMN);
     setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE);
 }
         break;
@@ -2796,7 +2797,7 @@ static void yy_reduce(
     yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n;
 
     toTSDBType(yymsp[0].minor.yy0.type);
-    tVariantList* K = tVariantListAppendToken(NULL, &yymsp[0].minor.yy0, -1);
+    SArray* K = tVariantListAppendToken(NULL, &yymsp[0].minor.yy0, -1);
 
     SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&yymsp[-4].minor.yy0, NULL, K, TSDB_ALTER_TABLE_DROP_COLUMN);
     setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE);
@@ -2805,7 +2806,7 @@ static void yy_reduce(
       case 226: /* cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */
 {
     yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n;
-    SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&yymsp[-4].minor.yy0, yymsp[0].minor.yy523, NULL, TSDB_ALTER_TABLE_ADD_TAG_COLUMN);
+    SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&yymsp[-4].minor.yy0, yymsp[0].minor.yy165, NULL, TSDB_ALTER_TABLE_ADD_TAG_COLUMN);
     setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE);
 }
         break;
@@ -2814,7 +2815,7 @@ static void yy_reduce(
     yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n;
 
     toTSDBType(yymsp[0].minor.yy0.type);
-    tVariantList* A = tVariantListAppendToken(NULL, &yymsp[0].minor.yy0, -1);
+    SArray* A = tVariantListAppendToken(NULL, &yymsp[0].minor.yy0, -1);
 
     SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&yymsp[-4].minor.yy0, NULL, A, TSDB_ALTER_TABLE_DROP_TAG_COLUMN);
     setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE);
@@ -2825,7 +2826,7 @@ static void yy_reduce(
     yymsp[-5].minor.yy0.n += yymsp[-4].minor.yy0.n;
 
     toTSDBType(yymsp[-1].minor.yy0.type);
-    tVariantList* A = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1);
+    SArray* A = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1);
 
     toTSDBType(yymsp[0].minor.yy0.type);
     A = tVariantListAppendToken(A, &yymsp[0].minor.yy0, -1);
@@ -2839,7 +2840,7 @@ static void yy_reduce(
     yymsp[-6].minor.yy0.n += yymsp[-5].minor.yy0.n;
 
     toTSDBType(yymsp[-2].minor.yy0.type);
-    tVariantList* A = tVariantListAppendToken(NULL, &yymsp[-2].minor.yy0, -1);
+    SArray* A = tVariantListAppendToken(NULL, &yymsp[-2].minor.yy0, -1);
     A = tVariantListAppend(A, &yymsp[0].minor.yy134, -1);
 
     SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&yymsp[-6].minor.yy0, NULL, A, TSDB_ALTER_TABLE_UPDATE_TAG_VAL);

src/rpc/src/rpcMain.c

@@ -135,7 +135,7 @@ int tsRpcOverhead;
 
 static int     tsRpcRefId = -1;
 static int32_t tsRpcNum = 0;
-static pthread_once_t tsRpcInit = PTHREAD_ONCE_INIT;
+//static pthread_once_t tsRpcInit = PTHREAD_ONCE_INIT;
 
 // server:0 client:1  tcp:2 udp:0
 #define RPC_CONN_UDPS   0
@@ -221,13 +221,15 @@ static void rpcFree(void *p) {
   free(p);
 }
 
-void rpcInit(void) {
+int32_t rpcInit(void) {
   tsProgressTimer = tsRpcTimer/2; 
   tsRpcMaxRetry = tsRpcMaxTime * 1000/tsProgressTimer;
   tsRpcHeadSize = RPC_MSG_OVERHEAD; 
   tsRpcOverhead = sizeof(SRpcReqContext);
 
   tsRpcRefId = taosOpenRef(200, rpcFree);
+
+  return 0;
 }
  
 void rpcCleanup(void) {
@@ -238,7 +240,7 @@ void rpcCleanup(void) {
 void *rpcOpen(const SRpcInit *pInit) {
   SRpcInfo *pRpc;
 
-  pthread_once(&tsRpcInit, rpcInit);
+  //pthread_once(&tsRpcInit, rpcInit);
 
   pRpc = (SRpcInfo *)calloc(1, sizeof(SRpcInfo));
   if (pRpc == NULL) return NULL;
@@ -379,7 +381,7 @@ void *rpcReallocCont(void *ptr, int contLen) {
   return start + sizeof(SRpcReqContext) + sizeof(SRpcHead);
 }
 
-int64_t rpcSendRequest(void *shandle, const SRpcEpSet *pEpSet, SRpcMsg *pMsg) {
+void rpcSendRequest(void *shandle, const SRpcEpSet *pEpSet, SRpcMsg *pMsg, int64_t *pRid) {
   SRpcInfo       *pRpc = (SRpcInfo *)shandle;
   SRpcReqContext *pContext;
 
@@ -405,14 +407,10 @@ int64_t rpcSendRequest(void *shandle, const SRpcEpSet *pEpSet, SRpcMsg *pMsg) {
     || type == TSDB_MSG_TYPE_CM_SHOW )
     pContext->connType = RPC_CONN_TCPC;
   
-  // set the handle to pContext, so app can cancel the request
-  if (pMsg->handle) *((void **)pMsg->handle) = pContext;
-
   pContext->rid = taosAddRef(tsRpcRefId, pContext);
+  if (pRid) *pRid = pContext->rid;
 
   rpcSendReqToServer(pRpc, pContext);
-
-  return pContext->rid;
 }
 
 void rpcSendResponse(const SRpcMsg *pRsp) {
@@ -528,7 +526,7 @@ void rpcSendRecv(void *shandle, SRpcEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp)
   pContext->pRsp = pRsp;
   pContext->pSet = pEpSet;
 
-  rpcSendRequest(shandle, pEpSet, pMsg);
+  rpcSendRequest(shandle, pEpSet, pMsg, NULL);
 
   tsem_wait(&sem);
   tsem_destroy(&sem);

src/rpc/src/rpcTcp.c

@@ -373,8 +373,8 @@ void *taosOpenTcpClientConnection(void *shandle, void *thandle, uint32_t ip, uin
     tDebug("%s %p TCP connection to 0x%x:%hu is created, localPort:%hu FD:%p numOfFds:%d", 
             pThreadObj->label, thandle, ip, port, localPort, pFdObj, pThreadObj->numOfFds);
   } else {
-    taosCloseSocket(fd);
     tError("%s failed to malloc client FdObj(%s)", pThreadObj->label, strerror(errno));
+    taosCloseSocket(fd);
   }
 
   return pFdObj;

src/rpc/test/rclient.c

@@ -57,7 +57,7 @@ static void *sendRequest(void *param) {
     rpcMsg.ahandle = pInfo;
     rpcMsg.msgType = 1;
     tDebug("thread:%d, send request, contLen:%d num:%d", pInfo->index, pInfo->msgSize, pInfo->num);
-    rpcSendRequest(pInfo->pRpc, &pInfo->epSet, &rpcMsg);
+    rpcSendRequest(pInfo->pRpc, &pInfo->epSet, &rpcMsg, NULL);
     if ( pInfo->num % 20000 == 0 ) 
       tInfo("thread:%d, %d requests have been sent", pInfo->index, pInfo->num);
     tsem_wait(&pInfo->rspSem);

src/sync/src/syncMain.c

@@ -498,7 +498,7 @@ static SSyncPeer *syncAddPeer(SSyncNode *pNode, const SNodeInfo *pInfo) {
   int32_t ret = strcmp(pPeer->fqdn, tsNodeFqdn);
   if (pPeer->nodeId == 0 || (ret > 0) || (ret == 0 && pPeer->port > tsSyncPort)) {
     int32_t checkMs = 100 + (pNode->vgId * 10) % 100;
-    if (pNode->vgId > 1) checkMs = tsStatusInterval * 2000 + checkMs;
+    if (pNode->vgId > 1) checkMs = tsStatusInterval * 1000 + checkMs;
     sDebug("%s, start to check peer connection after %d ms", pPeer->id, checkMs);
     taosTmrReset(syncCheckPeerConnection, checkMs, pPeer, tsSyncTmrCtrl, &pPeer->timer);
   }
@@ -575,6 +575,18 @@ static void syncChooseMaster(SSyncNode *pNode) {
     if (index == pNode->selfIndex) {
       sInfo("vgId:%d, start to work as master", pNode->vgId);
       nodeRole = TAOS_SYNC_ROLE_MASTER;
+
+#if 0
+      for (int32_t i = 0; i < pNode->replica; ++i) {
+        if (i == index) continue;
+        pPeer = pNode->peerInfo[i];
+        if (pPeer->version == nodeVersion) {
+          pPeer->role = TAOS_SYNC_ROLE_SLAVE;
+          pPeer->sstatus = TAOS_SYNC_STATUS_CACHE;
+          sInfo("%s, it shall work as slave", pPeer->id);
+        }
+      }
+#endif
       syncResetFlowCtrl(pNode);
       (*pNode->notifyRole)(pNode->ahandle, nodeRole);
     } else {
@@ -1097,7 +1109,7 @@ static void syncProcessBrokenLink(void *param) {
   SSyncPeer *pPeer = param;
   SSyncNode *pNode = pPeer->pSyncNode;
 
-  if (taosAcquireRef(tsSyncRefId, pNode->rid) < 0) return;
+  if (taosAcquireRef(tsSyncRefId, pNode->rid) == NULL) return;
   pthread_mutex_lock(&(pNode->mutex));
 
   sDebug("%s, TCP link is broken(%s)", pPeer->id, strerror(errno));
@@ -1209,13 +1221,17 @@ static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle
   int32_t    fwdLen;
   int32_t    code = 0;
 
-  if (nodeRole == TAOS_SYNC_ROLE_SLAVE && pWalHead->version != nodeVersion + 1) {
-    sError("vgId:%d, received ver:%" PRIu64 ", inconsistent with last ver:%" PRIu64 ", restart connection", pNode->vgId,
-           pWalHead->version, nodeVersion);
+
+  if (pWalHead->version > nodeVersion + 1) {
+    sError("vgId:%d, hver:%" PRIu64 ", inconsistent with ver:%" PRIu64, pNode->vgId, pWalHead->version, nodeVersion);
+    if (nodeRole == TAOS_SYNC_ROLE_SLAVE) {
+      sInfo("vgId:%d, restart connection", pNode->vgId);
       for (int32_t i = 0; i < pNode->replica; ++i) {
         pPeer = pNode->peerInfo[i];
         syncRestartConnection(pPeer);
       }
+    }
+
     return TSDB_CODE_SYN_INVALID_VERSION;
   }
 

src/sync/src/syncRetrieve.c

@@ -182,6 +182,8 @@ static int32_t syncReadOneWalRecord(int32_t sfd, SWalHead *pHead, uint32_t *pEve
     return 0;
   }
 
+  assert(pHead->len <= TSDB_MAX_WAL_SIZE);
+
   ret = read(sfd, pHead->cont, pHead->len);
   if (ret < 0) return -1;
 

src/sync/src/taosTcpPool.c

@@ -327,5 +327,5 @@ static void taosStopPoolThread(SThreadObj *pThread) {
   }
 
   pthread_join(thread, NULL);
-  if (fd >= 0) taosClose(fd);
+  taosClose(fd);
 }

src/sync/test/syncClient.c

@@ -57,7 +57,7 @@ void *sendRequest(void *param) {
     rpcMsg.ahandle = pInfo;
     rpcMsg.msgType = 1;
     uDebug("thread:%d, send request, contLen:%d num:%d", pInfo->index, pInfo->msgSize, pInfo->num);
-    rpcSendRequest(pInfo->pRpc, &pInfo->epSet, &rpcMsg);
+    rpcSendRequest(pInfo->pRpc, &pInfo->epSet, &rpcMsg, NULL);
     if (pInfo->num % 20000 == 0) {
       uInfo("thread:%d, %d requests have been sent", pInfo->index, pInfo->num);
     }

src/tsdb/inc/tsdbMain.h

@@ -208,6 +208,18 @@ typedef struct {
 } SFileGroupIter;
 
 // ------------------ tsdbMain.c
+typedef struct {
+  int32_t  totalLen;
+  int32_t  len;
+  SDataRow row;
+} SSubmitBlkIter;
+
+typedef struct {
+  int32_t totalLen;
+  int32_t len;
+  void *  pMsg;
+} SSubmitMsgIter;
+
 typedef struct {
   int8_t state;
 
@@ -430,7 +442,6 @@ void          tsdbCloseBufPool(STsdbRepo* pRepo);
 SListNode*    tsdbAllocBufBlockFromPool(STsdbRepo* pRepo);
 
 // ------------------ tsdbMemTable.c
-int   tsdbUpdateRowInMem(STsdbRepo* pRepo, SDataRow row, STable* pTable);
 int   tsdbRefMemTable(STsdbRepo* pRepo, SMemTable* pMemTable);
 int   tsdbUnRefMemTable(STsdbRepo* pRepo, SMemTable* pMemTable);
 int   tsdbTakeMemSnapshot(STsdbRepo* pRepo, SMemTable** pMem, SMemTable** pIMem);

src/tsdb/src/tsdbCommit.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+#include "tsdbMain.h"
+
+static int  tsdbCommitTSData(STsdbRepo *pRepo);
+static int  tsdbCommitMeta(STsdbRepo *pRepo);
+static void tsdbEndCommit(STsdbRepo *pRepo, int eno);
+static int  tsdbHasDataToCommit(SCommitIter *iters, int nIters, TSKEY minKey, TSKEY maxKey);
+static int  tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHelper *pHelper, SDataCols *pDataCols);
+static SCommitIter *tsdbCreateCommitIters(STsdbRepo *pRepo);
+static void         tsdbDestroyCommitIters(SCommitIter *iters, int maxTables);
+
+void *tsdbCommitData(STsdbRepo *pRepo) {
+  SMemTable *  pMem = pRepo->imem;
+
+  tsdbInfo("vgId:%d start to commit! keyFirst %" PRId64 " keyLast %" PRId64 " numOfRows %" PRId64 " meta rows: %d",
+           REPO_ID(pRepo), pMem->keyFirst, pMem->keyLast, pMem->numOfRows, listNEles(pMem->actList));
+
+  // Commit to update meta file
+  if (tsdbCommitMeta(pRepo) < 0) {
+    tsdbError("vgId:%d error occurs while committing META data since %s", REPO_ID(pRepo), tstrerror(terrno));
+    goto _err;
+  }
+
+  // Create the iterator to read from cache
+  if (tsdbCommitTSData(pRepo) < 0) {
+    tsdbError("vgId:%d error occurs while committing TS data since %s", REPO_ID(pRepo), tstrerror(terrno));
+    goto _err;
+  }
+
+  tsdbFitRetention(pRepo);
+
+  tsdbInfo("vgId:%d commit over, succeed", REPO_ID(pRepo));
+  tsdbEndCommit(pRepo, TSDB_CODE_SUCCESS);
+
+  return NULL;
+
+_err:
+  ASSERT(terrno != TSDB_CODE_SUCCESS);
+  tsdbInfo("vgId:%d commit over, failed", REPO_ID(pRepo));
+  tsdbEndCommit(pRepo, terrno);
+
+  return NULL;
+}
+
+static int tsdbCommitTSData(STsdbRepo *pRepo) {
+  SMemTable *  pMem = pRepo->imem;
+  SDataCols *  pDataCols = NULL;
+  STsdbMeta *  pMeta = pRepo->tsdbMeta;
+  SCommitIter *iters = NULL;
+  SRWHelper    whelper = {0};
+  STsdbCfg *   pCfg = &(pRepo->config);
+
+  if (pMem->numOfRows <= 0) return 0;
+
+  iters = tsdbCreateCommitIters(pRepo);
+  if (iters == NULL) {
+    tsdbError("vgId:%d failed to create commit iterator since %s", REPO_ID(pRepo), tstrerror(terrno));
+    goto _err;
+  }
+
+  if (tsdbInitWriteHelper(&whelper, pRepo) < 0) {
+    tsdbError("vgId:%d failed to init write helper since %s", REPO_ID(pRepo), tstrerror(terrno));
+    goto _err;
+  }
+
+  if ((pDataCols = tdNewDataCols(pMeta->maxRowBytes, pMeta->maxCols, pCfg->maxRowsPerFileBlock)) == NULL) {
+    terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+    tsdbError("vgId:%d failed to init data cols with maxRowBytes %d maxCols %d maxRowsPerFileBlock %d since %s",
+              REPO_ID(pRepo), pMeta->maxCols, pMeta->maxRowBytes, pCfg->maxRowsPerFileBlock, tstrerror(terrno));
+    goto _err;
+  }
+
+  int sfid = (int)(TSDB_KEY_FILEID(pMem->keyFirst, pCfg->daysPerFile, pCfg->precision));
+  int efid = (int)(TSDB_KEY_FILEID(pMem->keyLast, pCfg->daysPerFile, pCfg->precision));
+
+  // Loop to commit to each file
+  for (int fid = sfid; fid <= efid; fid++) {
+    if (tsdbCommitToFile(pRepo, fid, iters, &whelper, pDataCols) < 0) {
+      tsdbError("vgId:%d failed to commit to file %d since %s", REPO_ID(pRepo), fid, tstrerror(terrno));
+      goto _err;
+    }
+  }
+
+  tdFreeDataCols(pDataCols);
+  tsdbDestroyCommitIters(iters, pMem->maxTables);
+  tsdbDestroyHelper(&whelper);
+
+  return 0;
+
+_err:
+  tdFreeDataCols(pDataCols);
+  tsdbDestroyCommitIters(iters, pMem->maxTables);
+  tsdbDestroyHelper(&whelper);
+
+  return -1;
+}
+
+static int tsdbCommitMeta(STsdbRepo *pRepo) {
+  SMemTable *pMem = pRepo->imem;
+  STsdbMeta *pMeta = pRepo->tsdbMeta;
+  SActObj *  pAct = NULL;
+  SActCont * pCont = NULL;
+
+  if (listNEles(pMem->actList) <= 0) return 0;
+
+  if (tdKVStoreStartCommit(pMeta->pStore) < 0) {
+    tsdbError("vgId:%d failed to commit data while start commit meta since %s", REPO_ID(pRepo), tstrerror(terrno));
+    goto _err;
+  }
+
+  SListNode *pNode = NULL;
+
+  while ((pNode = tdListPopHead(pMem->actList)) != NULL) {
+    pAct = (SActObj *)pNode->data;
+    if (pAct->act == TSDB_UPDATE_META) {
+      pCont = (SActCont *)POINTER_SHIFT(pAct, sizeof(SActObj));
+      if (tdUpdateKVStoreRecord(pMeta->pStore, pAct->uid, (void *)(pCont->cont), pCont->len) < 0) {
+        tsdbError("vgId:%d failed to update meta with uid %" PRIu64 " since %s", REPO_ID(pRepo), pAct->uid,
+                  tstrerror(terrno));
+        tdKVStoreEndCommit(pMeta->pStore);
+        goto _err;
+      }
+    } else if (pAct->act == TSDB_DROP_META) {
+      if (tdDropKVStoreRecord(pMeta->pStore, pAct->uid) < 0) {
+        tsdbError("vgId:%d failed to drop meta with uid %" PRIu64 " since %s", REPO_ID(pRepo), pAct->uid,
+                  tstrerror(terrno));
+        tdKVStoreEndCommit(pMeta->pStore);
+        goto _err;
+      }
+    } else {
+      ASSERT(false);
+    }
+  }
+
+  if (tdKVStoreEndCommit(pMeta->pStore) < 0) {
+    tsdbError("vgId:%d failed to commit data while end commit meta since %s", REPO_ID(pRepo), tstrerror(terrno));
+    goto _err;
+  }
+
+  return 0;
+
+_err:
+  return -1;
+}
+
+static void tsdbEndCommit(STsdbRepo *pRepo, int eno) {
+  if (pRepo->appH.notifyStatus) pRepo->appH.notifyStatus(pRepo->appH.appH, TSDB_STATUS_COMMIT_OVER, eno);
+  sem_post(&(pRepo->readyToCommit));
+}
+
+static int tsdbHasDataToCommit(SCommitIter *iters, int nIters, TSKEY minKey, TSKEY maxKey) {
+  for (int i = 0; i < nIters; i++) {
+    TSKEY nextKey = tsdbNextIterKey((iters + i)->pIter);
+    if (nextKey != TSDB_DATA_TIMESTAMP_NULL && (nextKey >= minKey && nextKey <= maxKey)) return 1;
+  }
+  return 0;
+}
+
+static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHelper *pHelper, SDataCols *pDataCols) {
+  char *      dataDir = NULL;
+  STsdbCfg *  pCfg = &pRepo->config;
+  STsdbFileH *pFileH = pRepo->tsdbFileH;
+  SFileGroup *pGroup = NULL;
+  SMemTable * pMem = pRepo->imem;
+  bool        newLast = false;
+
+  TSKEY minKey = 0, maxKey = 0;
+  tsdbGetFidKeyRange(pCfg->daysPerFile, pCfg->precision, fid, &minKey, &maxKey);
+
+  // Check if there are data to commit to this file
+  int hasDataToCommit = tsdbHasDataToCommit(iters, pMem->maxTables, minKey, maxKey);
+  if (!hasDataToCommit) {
+    tsdbDebug("vgId:%d no data to commit to file %d", REPO_ID(pRepo), fid);
+    return 0;
+  }
+
+  // Create and open files for commit
+  dataDir = tsdbGetDataDirName(pRepo->rootDir);
+  if (dataDir == NULL) {
+    terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  if ((pGroup = tsdbCreateFGroupIfNeed(pRepo, dataDir, fid)) == NULL) {
+    tsdbError("vgId:%d failed to create file group %d since %s", REPO_ID(pRepo), fid, tstrerror(terrno));
+    goto _err;
+  }
+
+  // Open files for write/read
+  if (tsdbSetAndOpenHelperFile(pHelper, pGroup) < 0) {
+    tsdbError("vgId:%d failed to set helper file since %s", REPO_ID(pRepo), tstrerror(terrno));
+    goto _err;
+  }
+
+  newLast = TSDB_NLAST_FILE_OPENED(pHelper);
+
+  if (tsdbLoadCompIdx(pHelper, NULL) < 0) {
+    tsdbError("vgId:%d failed to load SCompIdx part since %s", REPO_ID(pRepo), tstrerror(terrno));
+    goto _err;
+  }
+
+  // Loop to commit data in each table
+  for (int tid = 1; tid < pMem->maxTables; tid++) {
+    SCommitIter *pIter = iters + tid;
+    if (pIter->pTable == NULL) continue;
+
+    taosRLockLatch(&(pIter->pTable->latch));
+
+    if (tsdbSetHelperTable(pHelper, pIter->pTable, pRepo) < 0) goto _err;
+
+    if (pIter->pIter != NULL) {
+      if (tdInitDataCols(pDataCols, tsdbGetTableSchemaImpl(pIter->pTable, false, false, -1)) < 0) {
+        terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+        goto _err;
+      }
+
+      if (tsdbCommitTableData(pHelper, pIter, pDataCols, maxKey) < 0) {
+        taosRUnLockLatch(&(pIter->pTable->latch));
+        tsdbError("vgId:%d failed to write data of table %s tid %d uid %" PRIu64 " since %s", REPO_ID(pRepo),
+                  TABLE_CHAR_NAME(pIter->pTable), TABLE_TID(pIter->pTable), TABLE_UID(pIter->pTable),
+                  tstrerror(terrno));
+        goto _err;
+      }
+    }
+
+    taosRUnLockLatch(&(pIter->pTable->latch));
+
+    // Move the last block to the new .l file if neccessary
+    if (tsdbMoveLastBlockIfNeccessary(pHelper) < 0) {
+      tsdbError("vgId:%d, failed to move last block, since %s", REPO_ID(pRepo), tstrerror(terrno));
+      goto _err;
+    }
+
+    // Write the SCompBlock part
+    if (tsdbWriteCompInfo(pHelper) < 0) {
+      tsdbError("vgId:%d, failed to write compInfo part since %s", REPO_ID(pRepo), tstrerror(terrno));
+      goto _err;
+    }
+  }
+
+  if (tsdbWriteCompIdx(pHelper) < 0) {
+    tsdbError("vgId:%d failed to write compIdx part to file %d since %s", REPO_ID(pRepo), fid, tstrerror(terrno));
+    goto _err;
+  }
+
+  tfree(dataDir);
+  tsdbCloseHelperFile(pHelper, 0, pGroup);
+
+  pthread_rwlock_wrlock(&(pFileH->fhlock));
+
+  (void)rename(helperNewHeadF(pHelper)->fname, helperHeadF(pHelper)->fname);
+  pGroup->files[TSDB_FILE_TYPE_HEAD].info = helperNewHeadF(pHelper)->info;
+
+  if (newLast) {
+    (void)rename(helperNewLastF(pHelper)->fname, helperLastF(pHelper)->fname);
+    pGroup->files[TSDB_FILE_TYPE_LAST].info = helperNewLastF(pHelper)->info;
+  } else {
+    pGroup->files[TSDB_FILE_TYPE_LAST].info = helperLastF(pHelper)->info;
+  }
+
+  pGroup->files[TSDB_FILE_TYPE_DATA].info = helperDataF(pHelper)->info;
+
+  pthread_rwlock_unlock(&(pFileH->fhlock));
+
+  return 0;
+
+_err:
+  tfree(dataDir);
+  tsdbCloseHelperFile(pHelper, 1, NULL);
+  return -1;
+}
+
+static SCommitIter *tsdbCreateCommitIters(STsdbRepo *pRepo) {
+  SMemTable *pMem = pRepo->imem;
+  STsdbMeta *pMeta = pRepo->tsdbMeta;
+
+  SCommitIter *iters = (SCommitIter *)calloc(pMem->maxTables, sizeof(SCommitIter));
+  if (iters == NULL) {
+    terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+    return NULL;
+  }
+
+  if (tsdbRLockRepoMeta(pRepo) < 0) goto _err;
+
+  // reference all tables
+  for (int i = 0; i < pMem->maxTables; i++) {
+    if (pMeta->tables[i] != NULL) {
+      tsdbRefTable(pMeta->tables[i]);
+      iters[i].pTable = pMeta->tables[i];
+    }
+  }
+
+  if (tsdbUnlockRepoMeta(pRepo) < 0) goto _err;
+
+  for (int i = 0; i < pMem->maxTables; i++) {
+    if ((iters[i].pTable != NULL) && (pMem->tData[i] != NULL) && (TABLE_UID(iters[i].pTable) == pMem->tData[i]->uid)) {
+      if ((iters[i].pIter = tSkipListCreateIter(pMem->tData[i]->pData)) == NULL) {
+        terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+        goto _err;
+      }
+
+      tSkipListIterNext(iters[i].pIter);
+    }
+  }
+
+  return iters;
+
+_err:
+  tsdbDestroyCommitIters(iters, pMem->maxTables);
+  return NULL;
+}
+
+static void tsdbDestroyCommitIters(SCommitIter *iters, int maxTables) {
+  if (iters == NULL) return;
+
+  for (int i = 1; i < maxTables; i++) {
+    if (iters[i].pTable != NULL) {
+      tsdbUnRefTable(iters[i].pTable);
+      tSkipListDestroyIter(iters[i].pIter);
+    }
+  }
+
+  free(iters);
+}

src/tsdb/src/tsdbFile.c

@@ -256,7 +256,8 @@ SFileGroup *tsdbCreateFGroupIfNeed(STsdbRepo *pRepo, char *dataDir, int fid) {
     pFileH->pFGroup[pFileH->nFGroups++] = fGroup;
     qsort((void *)(pFileH->pFGroup), pFileH->nFGroups, sizeof(SFileGroup), compFGroup);
     pthread_rwlock_unlock(&pFileH->fhlock);
-    return tsdbSearchFGroup(pFileH, fid, TD_EQ);
+    pGroup = tsdbSearchFGroup(pFileH, fid, TD_EQ);
+    ASSERT(pGroup != NULL);
   }
 
   return pGroup;
@@ -516,7 +517,7 @@ void tsdbGetFileInfoImpl(char *fname, uint32_t *magic, int64_t *size) {
   SFile         file;
   SFile *       pFile = &file;
 
-  strncpy(pFile->fname, fname, TSDB_FILENAME_LEN);
+  strncpy(pFile->fname, fname, TSDB_FILENAME_LEN - 1);
   pFile->fd = -1;
 
   if (tsdbOpenFile(pFile, O_RDONLY) < 0) goto _err;

src/tsdb/src/tsdbMain.c

@@ -32,18 +32,6 @@
 #define TSDB_DEFAULT_COMPRESSION TWO_STAGE_COMP
 #define IS_VALID_COMPRESSION(compression) (((compression) >= NO_COMPRESSION) && ((compression) <= TWO_STAGE_COMP))
 
-typedef struct {
-  int32_t  totalLen;
-  int32_t  len;
-  SDataRow row;
-} SSubmitBlkIter;
-
-typedef struct {
-  int32_t totalLen;
-  int32_t len;
-  void *  pMsg;
-} SSubmitMsgIter;
-
 static int32_t     tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg);
 static int32_t     tsdbSetRepoEnv(char *rootDir, STsdbCfg *pCfg);
 static int32_t     tsdbUnsetRepoEnv(char *rootDir);
@@ -52,20 +40,13 @@ static int         tsdbLoadConfig(char *rootDir, STsdbCfg *pCfg);
 static char *      tsdbGetCfgFname(char *rootDir);
 static STsdbRepo * tsdbNewRepo(char *rootDir, STsdbAppH *pAppH, STsdbCfg *pCfg);
 static void        tsdbFreeRepo(STsdbRepo *pRepo);
-static int         tsdbInitSubmitMsgIter(SSubmitMsg *pMsg, SSubmitMsgIter *pIter);
-static int32_t     tsdbInsertDataToTable(STsdbRepo *pRepo, SSubmitBlk *pBlock, TSKEY now, int32_t *affectedrows);
-static int         tsdbGetSubmitMsgNext(SSubmitMsgIter *pIter, SSubmitBlk **pPBlock);
-static SDataRow    tsdbGetSubmitBlkNext(SSubmitBlkIter *pIter);
 static int         tsdbRestoreInfo(STsdbRepo *pRepo);
-static int         tsdbInitSubmitBlkIter(SSubmitBlk *pBlock, SSubmitBlkIter *pIter);
 static void        tsdbAlterCompression(STsdbRepo *pRepo, int8_t compression);
 static int         tsdbAlterKeep(STsdbRepo *pRepo, int32_t keep);
 static int         tsdbAlterCacheTotalBlocks(STsdbRepo *pRepo, int totalBlocks);
 static int         keyFGroupCompFunc(const void *key, const void *fgroup);
 static int         tsdbEncodeCfg(void **buf, STsdbCfg *pCfg);
 static void *      tsdbDecodeCfg(void *buf, STsdbCfg *pCfg);
-static int         tsdbCheckTableSchema(STsdbRepo *pRepo, SSubmitBlk *pBlock, STable *pTable);
-static int         tsdbScanAndConvertSubmitMsg(STsdbRepo *pRepo, SSubmitMsg *pMsg);
 static void        tsdbStartStream(STsdbRepo *pRepo);
 static void        tsdbStopStream(STsdbRepo *pRepo);
 
@@ -177,40 +158,6 @@ void tsdbCloseRepo(TSDB_REPO_T *repo, int toCommit) {
   tsdbDebug("vgId:%d repository is closed", vgId);
 }
 
-int32_t tsdbInsertData(TSDB_REPO_T *repo, SSubmitMsg *pMsg, SShellSubmitRspMsg *pRsp) {
-  STsdbRepo *    pRepo = (STsdbRepo *)repo;
-  SSubmitMsgIter msgIter = {0};
-
-  if (tsdbScanAndConvertSubmitMsg(pRepo, pMsg) < 0) {
-    if (terrno != TSDB_CODE_TDB_TABLE_RECONFIGURE) {
-      tsdbError("vgId:%d failed to insert data since %s", REPO_ID(pRepo), tstrerror(terrno));
-    }
-    return -1;
-  }
-
-  if (tsdbInitSubmitMsgIter(pMsg, &msgIter) < 0) {
-    tsdbError("vgId:%d failed to insert data since %s", REPO_ID(pRepo), tstrerror(terrno));
-    return -1;
-  }
-
-  SSubmitBlk *pBlock = NULL;
-  int32_t     affectedrows = 0;
-
-  TSKEY now = taosGetTimestamp(pRepo->config.precision);
-  while (true) {
-    tsdbGetSubmitMsgNext(&msgIter, &pBlock);
-    if (pBlock == NULL) break;
-    if (tsdbInsertDataToTable(pRepo, pBlock, now, &affectedrows) < 0) {
-      return -1;
-    }
-  }
-
-  if (pRsp != NULL) pRsp->affectedRows = htonl(affectedrows);
-
-  if (tsdbCheckCommit(pRepo) < 0) return -1;
-  return 0;
-}
-
 uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_t eindex, int64_t *size) {
   STsdbRepo *pRepo = (STsdbRepo *)repo;
   // STsdbMeta *pMeta = pRepo->tsdbMeta;
@@ -294,6 +241,7 @@ STsdbCfg *tsdbGetCfg(const TSDB_REPO_T *repo) {
 int32_t tsdbConfigRepo(TSDB_REPO_T *repo, STsdbCfg *pCfg) {
   // TODO: think about multithread cases
   STsdbRepo *pRepo = (STsdbRepo *)repo;
+  STsdbCfg   config = pRepo->config;
   STsdbCfg * pRCfg = &pRepo->config;
 
   if (tsdbCheckAndSetDefaultCfg(pCfg) < 0) return -1;
@@ -308,22 +256,25 @@ int32_t tsdbConfigRepo(TSDB_REPO_T *repo, STsdbCfg *pCfg) {
   bool configChanged = false;
   if (pRCfg->compression != pCfg->compression) {
     tsdbAlterCompression(pRepo, pCfg->compression);
+    config.compression = pCfg->compression;
     configChanged = true;
   }
   if (pRCfg->keep != pCfg->keep) {
     if (tsdbAlterKeep(pRepo, pCfg->keep) < 0) {
       tsdbError("vgId:%d failed to configure repo when alter keep since %s", REPO_ID(pRepo), tstrerror(terrno));
+      config.keep = pCfg->keep;
       return -1;
     }
     configChanged = true;
   }
   if (pRCfg->totalBlocks != pCfg->totalBlocks) {
     tsdbAlterCacheTotalBlocks(pRepo, pCfg->totalBlocks);
+    config.totalBlocks = pCfg->totalBlocks;
     configChanged = true;
   }
 
   if (configChanged) {
-    if (tsdbSaveConfig(pRepo->rootDir, &pRepo->config) < 0) {
+    if (tsdbSaveConfig(pRepo->rootDir, &config) < 0) {
       tsdbError("vgId:%d failed to configure repository while save config since %s", REPO_ID(pRepo), tstrerror(terrno));
       return -1;
     }
@@ -731,93 +682,6 @@ static void tsdbFreeRepo(STsdbRepo *pRepo) {
   }
 }
 
-static int tsdbInitSubmitMsgIter(SSubmitMsg *pMsg, SSubmitMsgIter *pIter) {
-  if (pMsg == NULL) {
-    terrno = TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP;
-    return -1;
-  }
-
-  pIter->totalLen = pMsg->length;
-  pIter->len = 0;
-  pIter->pMsg = pMsg;
-  if (pMsg->length <= TSDB_SUBMIT_MSG_HEAD_SIZE) {
-    terrno = TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP;
-    return -1;
-  }
-
-  return 0;
-}
-
-static int32_t tsdbInsertDataToTable(STsdbRepo *pRepo, SSubmitBlk *pBlock, TSKEY now, int32_t *affectedrows) {
-  STsdbMeta *pMeta = pRepo->tsdbMeta;
-  int64_t    points = 0;
-
-  ASSERT(pBlock->tid < pMeta->maxTables);
-  STable *pTable = pMeta->tables[pBlock->tid];
-  ASSERT(pTable != NULL && TABLE_UID(pTable) == pBlock->uid);
-
-  SSubmitBlkIter blkIter = {0};
-  SDataRow       row = NULL;
-
-  TSKEY minKey = now - tsMsPerDay[pRepo->config.precision] * pRepo->config.keep;
-  TSKEY maxKey = now + tsMsPerDay[pRepo->config.precision] * pRepo->config.daysPerFile;
-
-  tsdbInitSubmitBlkIter(pBlock, &blkIter);
-  while ((row = tsdbGetSubmitBlkNext(&blkIter)) != NULL) {
-    if (dataRowKey(row) < minKey || dataRowKey(row) > maxKey) {
-      tsdbError("vgId:%d table %s tid %d uid %" PRIu64 " timestamp is out of range! now %" PRId64 " minKey %" PRId64
-                " maxKey %" PRId64,
-                REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable), now, minKey, maxKey);
-      terrno = TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE;
-      return -1;
-    }
-
-    if (tsdbUpdateRowInMem(pRepo, row, pTable) < 0) return -1;
-
-    (*affectedrows)++;
-    points++;
-  }
-
-  STSchema *pSchema = tsdbGetTableSchemaByVersion(pTable, pBlock->sversion);
-  pRepo->stat.pointsWritten += points * schemaNCols(pSchema);
-  pRepo->stat.totalStorage += points * schemaVLen(pSchema);
-
-  return 0;
-}
-
-static int tsdbGetSubmitMsgNext(SSubmitMsgIter *pIter, SSubmitBlk **pPBlock) {
-  if (pIter->len == 0) {
-    pIter->len += TSDB_SUBMIT_MSG_HEAD_SIZE;
-  } else {
-    SSubmitBlk *pSubmitBlk = (SSubmitBlk *)POINTER_SHIFT(pIter->pMsg, pIter->len);
-    pIter->len += (sizeof(SSubmitBlk) + pSubmitBlk->dataLen + pSubmitBlk->schemaLen);
-  }
-
-  if (pIter->len > pIter->totalLen) {
-    terrno = TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP;
-    *pPBlock = NULL;
-    return -1;
-  }
-
-  *pPBlock = (pIter->len == pIter->totalLen) ? NULL : (SSubmitBlk *)POINTER_SHIFT(pIter->pMsg, pIter->len);
-
-  return 0;
-}
-
-static SDataRow tsdbGetSubmitBlkNext(SSubmitBlkIter *pIter) {
-  SDataRow row = pIter->row;
-  if (row == NULL) return NULL;
-
-  pIter->len += dataRowLen(row);
-  if (pIter->len >= pIter->totalLen) {
-    pIter->row = NULL;
-  } else {
-    pIter->row = (char *)row + dataRowLen(row);
-  }
-
-  return row;
-}
-
 static int tsdbRestoreInfo(STsdbRepo *pRepo) {
   STsdbMeta * pMeta = pRepo->tsdbMeta;
   STsdbFileH *pFileH = pRepo->tsdbFileH;
@@ -851,14 +715,6 @@ _err:
   return -1;
 }
 
-static int tsdbInitSubmitBlkIter(SSubmitBlk *pBlock, SSubmitBlkIter *pIter) {
-  if (pBlock->dataLen <= 0) return -1;
-  pIter->totalLen = pBlock->dataLen;
-  pIter->len = 0;
-  pIter->row = (SDataRow)(pBlock->data+pBlock->schemaLen);
-  return 0;
-}
-
 static void tsdbAlterCompression(STsdbRepo *pRepo, int8_t compression) {
   int8_t ocompression = pRepo->config.compression;
   pRepo->config.compression = compression;
@@ -955,134 +811,6 @@ static void *tsdbDecodeCfg(void *buf, STsdbCfg *pCfg) {
   return buf;
 }
 
-static int tsdbCheckTableSchema(STsdbRepo *pRepo, SSubmitBlk *pBlock, STable *pTable) {
-  ASSERT(pTable != NULL);
-
-  STSchema *pSchema = tsdbGetTableSchemaImpl(pTable, false, false, -1);
-  int       sversion = schemaVersion(pSchema);
-
-  if (pBlock->sversion == sversion) {
-    return 0;
-  } else {
-    if (TABLE_TYPE(pTable) == TSDB_STREAM_TABLE) {  // stream table is not allowed to change schema
-      terrno = TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION;
-      return -1;
-    }
-  }
-
-  if (pBlock->sversion > sversion) {  // may need to update table schema
-    if (pBlock->schemaLen > 0) {
-      tsdbDebug(
-          "vgId:%d table %s tid %d uid %" PRIu64 " schema version %d is out of data, client version %d, update...",
-          REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable), sversion, pBlock->sversion);
-      ASSERT(pBlock->schemaLen % sizeof(STColumn) == 0);
-      int       numOfCols = pBlock->schemaLen / sizeof(STColumn);
-      STColumn *pTCol = (STColumn *)pBlock->data;
-
-      STSchemaBuilder schemaBuilder = {0};
-      if (tdInitTSchemaBuilder(&schemaBuilder, pBlock->sversion) < 0) {
-        terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
-        tsdbError("vgId:%d failed to update schema of table %s since %s", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable),
-                  tstrerror(terrno));
-        return -1;
-      }
-
-      for (int i = 0; i < numOfCols; i++) {
-        if (tdAddColToSchema(&schemaBuilder, pTCol[i].type, htons(pTCol[i].colId), htons(pTCol[i].bytes)) < 0) {
-          terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
-          tsdbError("vgId:%d failed to update schema of table %s since %s", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable),
-                    tstrerror(terrno));
-          tdDestroyTSchemaBuilder(&schemaBuilder);
-          return -1;
-        }
-      }
-
-      STSchema *pNSchema = tdGetSchemaFromBuilder(&schemaBuilder);
-      if (pNSchema == NULL) {
-        terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
-        tdDestroyTSchemaBuilder(&schemaBuilder);
-        return -1;
-      }
-
-      tdDestroyTSchemaBuilder(&schemaBuilder);
-      tsdbUpdateTableSchema(pRepo, pTable, pNSchema, true);
-    } else {
-      tsdbDebug(
-          "vgId:%d table %s tid %d uid %" PRIu64 " schema version %d is out of data, client version %d, reconfigure...",
-          REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable), sversion, pBlock->sversion);
-      terrno = TSDB_CODE_TDB_TABLE_RECONFIGURE;
-      return -1;
-    }
-  } else {
-    ASSERT(pBlock->sversion >= 0);
-    if (tsdbGetTableSchemaImpl(pTable, false, false, pBlock->sversion) == NULL) {
-      tsdbError("vgId:%d invalid submit schema version %d to table %s tid %d from client", REPO_ID(pRepo),
-                pBlock->sversion, TABLE_CHAR_NAME(pTable), TABLE_TID(pTable));
-    }
-    terrno = TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION;
-    return -1;
-  }
-
-  return 0;
-}
-
-static int tsdbScanAndConvertSubmitMsg(STsdbRepo *pRepo, SSubmitMsg *pMsg) {
-  ASSERT(pMsg != NULL);
-  STsdbMeta *    pMeta = pRepo->tsdbMeta;
-  SSubmitMsgIter msgIter = {0};
-  SSubmitBlk *   pBlock = NULL;
-
-  terrno = TSDB_CODE_SUCCESS;
-  pMsg->length = htonl(pMsg->length);
-  pMsg->numOfBlocks = htonl(pMsg->numOfBlocks);
-
-  if (tsdbInitSubmitMsgIter(pMsg, &msgIter) < 0) return -1;
-  while (true) {
-    if (tsdbGetSubmitMsgNext(&msgIter, &pBlock) < 0) return -1;
-    if (pBlock == NULL) break;
-
-    pBlock->uid = htobe64(pBlock->uid);
-    pBlock->tid = htonl(pBlock->tid);
-    pBlock->sversion = htonl(pBlock->sversion);
-    pBlock->dataLen = htonl(pBlock->dataLen);
-    pBlock->schemaLen = htonl(pBlock->schemaLen);
-    pBlock->numOfRows = htons(pBlock->numOfRows);
-
-    if (pBlock->tid <= 0 || pBlock->tid >= pMeta->maxTables) {
-      tsdbError("vgId:%d failed to get table to insert data, uid %" PRIu64 " tid %d", REPO_ID(pRepo), pBlock->uid,
-                pBlock->tid);
-      terrno = TSDB_CODE_TDB_INVALID_TABLE_ID;
-      return -1;
-    }
-
-    STable *pTable = pMeta->tables[pBlock->tid];
-    if (pTable == NULL || TABLE_UID(pTable) != pBlock->uid) {
-      tsdbError("vgId:%d failed to get table to insert data, uid %" PRIu64 " tid %d", REPO_ID(pRepo), pBlock->uid,
-                pBlock->tid);
-      terrno = TSDB_CODE_TDB_INVALID_TABLE_ID;
-      return -1;
-    }
-
-    if (TABLE_TYPE(pTable) == TSDB_SUPER_TABLE) {
-      tsdbError("vgId:%d invalid action trying to insert a super table %s", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable));
-      terrno = TSDB_CODE_TDB_INVALID_ACTION;
-      return -1;
-    }
-
-    // Check schema version and update schema if needed
-    if (tsdbCheckTableSchema(pRepo, pBlock, pTable) < 0) {
-      if (terrno == TSDB_CODE_TDB_TABLE_RECONFIGURE) {
-        continue;
-      } else {
-        return -1;
-      }
-    }
-  }
-
-  if (terrno != TSDB_CODE_SUCCESS) return -1;
-  return 0;
-}
-
 static int tsdbAlterCacheTotalBlocks(STsdbRepo *pRepo, int totalBlocks) {
   // TODO
   // STsdbCache *pCache = pRepo->tsdbCache;

src/tsdb/src/tsdbMemTable.c

@@ -18,117 +18,56 @@
 
 #define TSDB_DATA_SKIPLIST_LEVEL 5
 
-static void        tsdbFreeBytes(STsdbRepo *pRepo, void *ptr, int bytes);
 static SMemTable * tsdbNewMemTable(STsdbRepo *pRepo);
 static void        tsdbFreeMemTable(SMemTable *pMemTable);
 static STableData *tsdbNewTableData(STsdbCfg *pCfg, STable *pTable);
 static void        tsdbFreeTableData(STableData *pTableData);
 static char *      tsdbGetTsTupleKey(const void *data);
-static int         tsdbCommitMeta(STsdbRepo *pRepo);
-static void        tsdbEndCommit(STsdbRepo *pRepo);
-static int         tsdbHasDataToCommit(SCommitIter *iters, int nIters, TSKEY minKey, TSKEY maxKey);
-static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHelper *pHelper, SDataCols *pDataCols);
-static SCommitIter *tsdbCreateCommitIters(STsdbRepo *pRepo);
-static void         tsdbDestroyCommitIters(SCommitIter *iters, int maxTables);
 static int          tsdbAdjustMemMaxTables(SMemTable *pMemTable, int maxTables);
 static int          tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema **ppSchema, SDataRow row);
-
-// ---------------- INTERNAL FUNCTIONS ----------------
-int tsdbUpdateRowInMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) {
-  STsdbCfg *  pCfg = &pRepo->config;
-  STsdbMeta * pMeta = pRepo->tsdbMeta;
-  TKEY        tkey = dataRowTKey(row);
-  TSKEY       key = dataRowKey(row);
-  SMemTable * pMemTable = pRepo->mem;
-  STableData *pTableData = NULL;
-  bool        isRowDelete = TKEY_IS_DELETED(tkey);
-
-  if (isRowDelete) {
-    if (!pCfg->update) {
-      tsdbWarn("vgId:%d vnode is not allowed to update but try to delete a data row", REPO_ID(pRepo));
-      terrno = TSDB_CODE_TDB_INVALID_ACTION;
-      return -1;
+static int          tsdbInitSubmitBlkIter(SSubmitBlk *pBlock, SSubmitBlkIter *pIter);
+static SDataRow     tsdbGetSubmitBlkNext(SSubmitBlkIter *pIter);
+static int          tsdbScanAndConvertSubmitMsg(STsdbRepo *pRepo, SSubmitMsg *pMsg);
+static int          tsdbInsertDataToTable(STsdbRepo *pRepo, SSubmitBlk *pBlock, int32_t *affectedrows);
+static int          tsdbCopyRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable, void **ppRow);
+static int          tsdbInitSubmitMsgIter(SSubmitMsg *pMsg, SSubmitMsgIter *pIter);
+static int          tsdbGetSubmitMsgNext(SSubmitMsgIter *pIter, SSubmitBlk **pPBlock);
+static int          tsdbCheckTableSchema(STsdbRepo *pRepo, SSubmitBlk *pBlock, STable *pTable);
+static int          tsdbInsertDataToTableImpl(STsdbRepo *pRepo, STable *pTable, void **rows, int rowCounter);
+static void         tsdbFreeRows(STsdbRepo *pRepo, void **rows, int rowCounter);
+
+static FORCE_INLINE int tsdbCheckRowRange(STsdbRepo *pRepo, STable *pTable, SDataRow row, TSKEY minKey, TSKEY maxKey,
+                                          TSKEY now);
+
+int32_t tsdbInsertData(TSDB_REPO_T *repo, SSubmitMsg *pMsg, SShellSubmitRspMsg *pRsp) {
+  STsdbRepo *    pRepo = (STsdbRepo *)repo;
+  SSubmitMsgIter msgIter = {0};
+  SSubmitBlk *   pBlock = NULL;
+  int32_t        affectedrows = 0;
+
+  if (tsdbScanAndConvertSubmitMsg(pRepo, pMsg) < 0) {
+    if (terrno != TSDB_CODE_TDB_TABLE_RECONFIGURE) {
+      tsdbError("vgId:%d failed to insert data since %s", REPO_ID(pRepo), tstrerror(terrno));
     }
-
-    if (key > TABLE_LASTKEY(pTable)) {
-      tsdbTrace("vgId:%d skip to delete row key %" PRId64 " which is larger than table lastKey %" PRId64,
-                REPO_ID(pRepo), key, TABLE_LASTKEY(pTable));
-      return 0;
-    }
-  }
-
-  void *pRow = tsdbAllocBytes(pRepo, dataRowLen(row));
-  if (pRow == NULL) {
-    tsdbError("vgId:%d failed to insert row with key %" PRId64 " to table %s while allocate %d bytes since %s",
-              REPO_ID(pRepo), key, TABLE_CHAR_NAME(pTable), dataRowLen(row), tstrerror(terrno));
     return -1;
   }
 
-  dataRowCpy(pRow, row);
-
-  // Operations above may change pRepo->mem, retake those values
-  ASSERT(pRepo->mem != NULL);
-  pMemTable = pRepo->mem;
-
-  if (TABLE_TID(pTable) >= pMemTable->maxTables) {
-    if (tsdbAdjustMemMaxTables(pMemTable, pMeta->maxTables) < 0) {
-      tsdbFreeBytes(pRepo, pRow, dataRowLen(row));
-      return -1;
-    }
-  }
-  pTableData = pMemTable->tData[TABLE_TID(pTable)];
-
-  if (pTableData == NULL || pTableData->uid != TABLE_UID(pTable)) {
-    if (pTableData != NULL) {
-      taosWLockLatch(&(pMemTable->latch));
-      pMemTable->tData[TABLE_TID(pTable)] = NULL;
-      tsdbFreeTableData(pTableData);
-      taosWUnLockLatch(&(pMemTable->latch));
-    }
-
-    pTableData = tsdbNewTableData(pCfg, pTable);
-    if (pTableData == NULL) {
-      tsdbError("vgId:%d failed to insert row with key %" PRId64
-                " to table %s while create new table data object since %s",
-                REPO_ID(pRepo), key, TABLE_CHAR_NAME(pTable), tstrerror(terrno));
-      tsdbFreeBytes(pRepo, (void *)pRow, dataRowLen(row));
+  tsdbInitSubmitMsgIter(pMsg, &msgIter);
+  while (true) {
+    tsdbGetSubmitMsgNext(&msgIter, &pBlock);
+    if (pBlock == NULL) break;
+    if (tsdbInsertDataToTable(pRepo, pBlock, &affectedrows) < 0) {
       return -1;
     }
-
-    pRepo->mem->tData[TABLE_TID(pTable)] = pTableData;
   }
 
-  ASSERT((pTableData != NULL) && pTableData->uid == TABLE_UID(pTable));
-
-  int64_t oldSize = SL_SIZE(pTableData->pData);
-  if (tSkipListPut(pTableData->pData, pRow) == NULL) {
-    tsdbFreeBytes(pRepo, (void *)pRow, dataRowLen(row));
-  } else {
-    int64_t deltaSize = SL_SIZE(pTableData->pData) - oldSize;
-    if (isRowDelete) {
-      if (TABLE_LASTKEY(pTable) == key) {
-        // TODO: need to update table last key here (may from file)
-      }
-    } else {
-      if (TABLE_LASTKEY(pTable) < key) TABLE_LASTKEY(pTable) = key;
-    }
-
-    if (pMemTable->keyFirst > key) pMemTable->keyFirst = key;
-    if (pMemTable->keyLast < key) pMemTable->keyLast = key;
-    pMemTable->numOfRows += deltaSize;
-
-    if (pTableData->keyFirst > key) pTableData->keyFirst = key;
-    if (pTableData->keyLast < key) pTableData->keyLast = key;
-    pTableData->numOfRows += deltaSize;
-  }
-
-  tsdbTrace("vgId:%d a row is %s table %s tid %d uid %" PRIu64 " key %" PRIu64, REPO_ID(pRepo),
-            isRowDelete ? "deleted from" : "updated in", TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable),
-            key);
+  if (pRsp != NULL) pRsp->affectedRows = htonl(affectedrows);
 
+  if (tsdbCheckCommit(pRepo) < 0) return -1;
   return 0;
 }
 
+// ---------------- INTERNAL FUNCTIONS ----------------
 int tsdbRefMemTable(STsdbRepo *pRepo, SMemTable *pMemTable) {
   if (pMemTable == NULL) return 0;
   int ref = T_REF_INC(pMemTable);
@@ -152,7 +91,7 @@ int tsdbUnRefMemTable(STsdbRepo *pRepo, SMemTable *pMemTable) {
     }
     int code = pthread_cond_signal(&pBufPool->poolNotEmpty);
     if (code != 0) {
-      tsdbUnlockRepo(pRepo);
+      if (tsdbUnlockRepo(pRepo) < 0) return -1;
       tsdbError("vgId:%d failed to signal pool not empty since %s", REPO_ID(pRepo), strerror(code));
       terrno = TAOS_SYSTEM_ERROR(code);
       return -1;
@@ -189,6 +128,8 @@ int tsdbTakeMemSnapshot(STsdbRepo *pRepo, SMemTable **pMem, SMemTable **pIMem) {
 }
 
 void tsdbUnTakeMemSnapShot(STsdbRepo *pRepo, SMemTable *pMem, SMemTable *pIMem) {
+  tsdbDebug("vgId:%d untake memory snapshot, pMem %p pIMem %p", REPO_ID(pRepo), pMem, pIMem);
+
   if (pMem != NULL) {
     taosRUnLockLatch(&(pMem->latch));
     tsdbUnRefMemTable(pRepo, pMem);
@@ -197,8 +138,6 @@ void tsdbUnTakeMemSnapShot(STsdbRepo *pRepo, SMemTable *pMem, SMemTable *pIMem)
   if (pIMem != NULL) {
     tsdbUnRefMemTable(pRepo, pIMem);
   }
-
-  tsdbDebug("vgId:%d utake memory snapshot, pMem %p pIMem %p", REPO_ID(pRepo), pMem, pIMem);
 }
 
 void *tsdbAllocBytes(STsdbRepo *pRepo, int bytes) {
@@ -230,6 +169,10 @@ void *tsdbAllocBytes(STsdbRepo *pRepo, int bytes) {
     ASSERT(pRepo->mem->extraBuffList != NULL);
     SListNode *pNode = (SListNode *)malloc(sizeof(SListNode) + bytes);
     if (pNode == NULL) {
+      if (listNEles(pRepo->mem->extraBuffList) == 0) {
+        tdListFree(pRepo->mem->extraBuffList);
+        pRepo->mem->extraBuffList = NULL;
+      }
       terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
       return NULL;
     }
@@ -260,18 +203,18 @@ void *tsdbAllocBytes(STsdbRepo *pRepo, int bytes) {
 }
 
 int tsdbAsyncCommit(STsdbRepo *pRepo) {
+  if (pRepo->mem == NULL) return 0;
+
   SMemTable *pIMem = pRepo->imem;
 
-  if (pRepo->mem != NULL) {
   sem_wait(&(pRepo->readyToCommit));
 
-    if (pRepo->appH.notifyStatus) pRepo->appH.notifyStatus(pRepo->appH.appH, TSDB_STATUS_COMMIT_START);
+  if (pRepo->appH.notifyStatus) pRepo->appH.notifyStatus(pRepo->appH.appH, TSDB_STATUS_COMMIT_START, TSDB_CODE_SUCCESS);
   if (tsdbLockRepo(pRepo) < 0) return -1;
   pRepo->imem = pRepo->mem;
   pRepo->mem = NULL;
   tsdbScheduleCommit(pRepo);
   if (tsdbUnlockRepo(pRepo) < 0) return -1;
-  }
 
   if (tsdbUnRefMemTable(pRepo, pIMem) < 0) return -1;
 
@@ -406,88 +349,7 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey
   return 0;
 }
 
-void *tsdbCommitData(STsdbRepo *pRepo) {
-  SMemTable *  pMem = pRepo->imem;
-  STsdbCfg *   pCfg = &pRepo->config;
-  SDataCols *  pDataCols = NULL;
-  STsdbMeta *  pMeta = pRepo->tsdbMeta;
-  SCommitIter *iters = NULL;
-  SRWHelper    whelper = {0};
-  ASSERT(pMem != NULL);
-
-  tsdbInfo("vgId:%d start to commit! keyFirst %" PRId64 " keyLast %" PRId64 " numOfRows %" PRId64, REPO_ID(pRepo),
-            pMem->keyFirst, pMem->keyLast, pMem->numOfRows);
-
-  // Create the iterator to read from cache
-  if (pMem->numOfRows > 0) {
-    iters = tsdbCreateCommitIters(pRepo);
-    if (iters == NULL) {
-      tsdbError("vgId:%d failed to create commit iterator since %s", REPO_ID(pRepo), tstrerror(terrno));
-      goto _exit;
-    }
-
-    if (tsdbInitWriteHelper(&whelper, pRepo) < 0) {
-      tsdbError("vgId:%d failed to init write helper since %s", REPO_ID(pRepo), tstrerror(terrno));
-      goto _exit;
-    }
-
-    if ((pDataCols = tdNewDataCols(pMeta->maxRowBytes, pMeta->maxCols, pCfg->maxRowsPerFileBlock)) == NULL) {
-      terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
-      tsdbError("vgId:%d failed to init data cols with maxRowBytes %d maxCols %d maxRowsPerFileBlock %d since %s",
-                REPO_ID(pRepo), pMeta->maxCols, pMeta->maxRowBytes, pCfg->maxRowsPerFileBlock, tstrerror(terrno));
-      goto _exit;
-    }
-
-    int sfid = (int)(TSDB_KEY_FILEID(pMem->keyFirst, pCfg->daysPerFile, pCfg->precision));
-    int efid = (int)(TSDB_KEY_FILEID(pMem->keyLast, pCfg->daysPerFile, pCfg->precision));
-
-    // Loop to commit to each file
-    for (int fid = sfid; fid <= efid; fid++) {
-      if (tsdbCommitToFile(pRepo, fid, iters, &whelper, pDataCols) < 0) {
-        tsdbError("vgId:%d failed to commit to file %d since %s", REPO_ID(pRepo), fid, tstrerror(terrno));
-        goto _exit;
-      }
-    }
-  }
-
-  // Commit to update meta file
-  if (tsdbCommitMeta(pRepo) < 0) {
-    tsdbError("vgId:%d failed to commit data while committing meta data since %s", REPO_ID(pRepo), tstrerror(terrno));
-    goto _exit;
-  }
-
-  tsdbFitRetention(pRepo);
-
-_exit:
-  tdFreeDataCols(pDataCols);
-  tsdbDestroyCommitIters(iters, pMem->maxTables);
-  tsdbDestroyHelper(&whelper);
-  tsdbInfo("vgId:%d commit over", pRepo->config.tsdbId);
-  tsdbEndCommit(pRepo);
-
-  return NULL;
-}
-
 // ---------------- LOCAL FUNCTIONS ----------------
-static void tsdbFreeBytes(STsdbRepo *pRepo, void *ptr, int bytes) {
-  ASSERT(pRepo->mem != NULL);
-  if (pRepo->mem->extraBuffList == NULL) {
-    STsdbBufBlock *pBufBlock = tsdbGetCurrBufBlock(pRepo);
-    ASSERT(pBufBlock != NULL);
-    pBufBlock->offset -= bytes;
-    pBufBlock->remain += bytes;
-    ASSERT(ptr == POINTER_SHIFT(pBufBlock->data, pBufBlock->offset));
-    tsdbTrace("vgId:%d free %d bytes to TSDB buffer pool, nBlocks %d offset %d remain %d", REPO_ID(pRepo), bytes,
-              listNEles(pRepo->mem->bufBlockList), pBufBlock->offset, pBufBlock->remain);
-  } else {
-    SListNode *pNode = (SListNode *)POINTER_SHIFT(ptr, -(int)(sizeof(SListNode)));
-    ASSERT(listTail(pRepo->mem->extraBuffList) == pNode);
-    tdListPopNode(pRepo->mem->extraBuffList, pNode);
-    free(pNode);
-    tsdbTrace("vgId:%d free %d bytes to SYSTEM buffer pool", REPO_ID(pRepo), bytes);
-  }
-}
-
 static SMemTable* tsdbNewMemTable(STsdbRepo *pRepo) {
   STsdbMeta *pMeta = pRepo->tsdbMeta;
 
@@ -578,274 +440,441 @@ static void tsdbFreeTableData(STableData *pTableData) {
 
 static char *tsdbGetTsTupleKey(const void *data) { return dataRowTuple((SDataRow)data); }
 
+void tsdbGetFidKeyRange(int daysPerFile, int8_t precision, int fileId, TSKEY *minKey, TSKEY *maxKey) {
+  *minKey = fileId * daysPerFile * tsMsPerDay[precision];
+  *maxKey = *minKey + daysPerFile * tsMsPerDay[precision] - 1;
+}
 
-static int tsdbCommitMeta(STsdbRepo *pRepo) {
-  SMemTable *pMem = pRepo->imem;
-  STsdbMeta *pMeta = pRepo->tsdbMeta;
-  SActObj *  pAct = NULL;
-  SActCont * pCont = NULL;
+static int tsdbAdjustMemMaxTables(SMemTable *pMemTable, int maxTables) {
+  ASSERT(pMemTable->maxTables < maxTables);
 
-  if (listNEles(pMem->actList) > 0) {
-    if (tdKVStoreStartCommit(pMeta->pStore) < 0) {
-      tsdbError("vgId:%d failed to commit data while start commit meta since %s", REPO_ID(pRepo), tstrerror(terrno));
-      goto _err;
+  STableData **pTableData = (STableData **)calloc(maxTables, sizeof(STableData *));
+  if (pTableData == NULL) {
+    terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+    return -1;
   }
+  memcpy((void *)pTableData, (void *)pMemTable->tData, sizeof(STableData *) * pMemTable->maxTables);
 
-    SListNode *pNode = NULL;
+  STableData **tData = pMemTable->tData;
 
-    while ((pNode = tdListPopHead(pMem->actList)) != NULL) {
-      pAct = (SActObj *)pNode->data;
-      if (pAct->act == TSDB_UPDATE_META) {
-        pCont = (SActCont *)POINTER_SHIFT(pAct, sizeof(SActObj));
-        if (tdUpdateKVStoreRecord(pMeta->pStore, pAct->uid, (void *)(pCont->cont), pCont->len) < 0) {
-          tsdbError("vgId:%d failed to update meta with uid %" PRIu64 " since %s", REPO_ID(pRepo), pAct->uid,
-                    tstrerror(terrno));
-          tdKVStoreEndCommit(pMeta->pStore);
-          goto _err;
-        }
-      } else if (pAct->act == TSDB_DROP_META) {
-        if (tdDropKVStoreRecord(pMeta->pStore, pAct->uid) < 0) {
-          tsdbError("vgId:%d failed to drop meta with uid %" PRIu64 " since %s", REPO_ID(pRepo), pAct->uid,
-                    tstrerror(terrno));
-          tdKVStoreEndCommit(pMeta->pStore);
-          goto _err;
-        }
-      } else {
+  taosWLockLatch(&(pMemTable->latch));
+  pMemTable->maxTables = maxTables;
+  pMemTable->tData = pTableData;
+  taosWUnLockLatch(&(pMemTable->latch));
+
+  tfree(tData);
+
+  return 0;
+}
+
+static int tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema **ppSchema, SDataRow row) {
+  if (pCols) {
+    if (*ppSchema == NULL || schemaVersion(*ppSchema) != dataRowVersion(row)) {
+      *ppSchema = tsdbGetTableSchemaImpl(pTable, false, false, dataRowVersion(row));
+      if (*ppSchema == NULL) {
         ASSERT(false);
+        return -1;
       }
     }
 
-    if (tdKVStoreEndCommit(pMeta->pStore) < 0) {
-      tsdbError("vgId:%d failed to commit data while end commit meta since %s", REPO_ID(pRepo), tstrerror(terrno));
-      goto _err;
-    }
+    tdAppendDataRowToDataCol(row, *ppSchema, pCols);
   }
 
   return 0;
+}
 
-_err:
-  return -1;
+static int tsdbInitSubmitBlkIter(SSubmitBlk *pBlock, SSubmitBlkIter *pIter) {
+  if (pBlock->dataLen <= 0) return -1;
+  pIter->totalLen = pBlock->dataLen;
+  pIter->len = 0;
+  pIter->row = (SDataRow)(pBlock->data+pBlock->schemaLen);
+  return 0;
 }
 
-static void tsdbEndCommit(STsdbRepo *pRepo) {
-  if (pRepo->appH.notifyStatus) pRepo->appH.notifyStatus(pRepo->appH.appH, TSDB_STATUS_COMMIT_OVER);
-  sem_post(&(pRepo->readyToCommit));
+static SDataRow tsdbGetSubmitBlkNext(SSubmitBlkIter *pIter) {
+  SDataRow row = pIter->row;
+  if (row == NULL) return NULL;
+
+  pIter->len += dataRowLen(row);
+  if (pIter->len >= pIter->totalLen) {
+    pIter->row = NULL;
+  } else {
+    pIter->row = (char *)row + dataRowLen(row);
+  }
+
+  return row;
 }
 
-static int tsdbHasDataToCommit(SCommitIter *iters, int nIters, TSKEY minKey, TSKEY maxKey) {
-  for (int i = 0; i < nIters; i++) {
-    TSKEY nextKey = tsdbNextIterKey((iters + i)->pIter);
-    if (nextKey != TSDB_DATA_TIMESTAMP_NULL && (nextKey >= minKey && nextKey <= maxKey)) return 1;
+static FORCE_INLINE int tsdbCheckRowRange(STsdbRepo *pRepo, STable *pTable, SDataRow row, TSKEY minKey, TSKEY maxKey,
+                                          TSKEY now) {
+  if (dataRowKey(row) < minKey || dataRowKey(row) > maxKey) {
+    tsdbError("vgId:%d table %s tid %d uid %" PRIu64 " timestamp is out of range! now %" PRId64 " minKey %" PRId64
+              " maxKey %" PRId64 " row key %" PRId64,
+              REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable), now, minKey, maxKey,
+              dataRowKey(row));
+    terrno = TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE;
+    return -1;
   }
+
   return 0;
 }
 
-void tsdbGetFidKeyRange(int daysPerFile, int8_t precision, int fileId, TSKEY *minKey, TSKEY *maxKey) {
-  *minKey = fileId * daysPerFile * tsMsPerDay[precision];
-  *maxKey = *minKey + daysPerFile * tsMsPerDay[precision] - 1;
-}
+static int tsdbScanAndConvertSubmitMsg(STsdbRepo *pRepo, SSubmitMsg *pMsg) {
+  ASSERT(pMsg != NULL);
+  STsdbMeta *    pMeta = pRepo->tsdbMeta;
+  SSubmitMsgIter msgIter = {0};
+  SSubmitBlk *   pBlock = NULL;
+  SSubmitBlkIter blkIter = {0};
+  SDataRow       row = NULL;
+  TSKEY          now = taosGetTimestamp(pRepo->config.precision);
+  TSKEY          minKey = now - tsMsPerDay[pRepo->config.precision] * pRepo->config.keep;
+  TSKEY          maxKey = now + tsMsPerDay[pRepo->config.precision] * pRepo->config.daysPerFile;
 
-static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHelper *pHelper, SDataCols *pDataCols) {
-  char *      dataDir = NULL;
-  STsdbCfg *  pCfg = &pRepo->config;
-  STsdbFileH *pFileH = pRepo->tsdbFileH;
-  SFileGroup *pGroup = NULL;
-  SMemTable * pMem = pRepo->imem;
-  bool        newLast = false;
-
-  TSKEY minKey = 0, maxKey = 0;
-  tsdbGetFidKeyRange(pCfg->daysPerFile, pCfg->precision, fid, &minKey, &maxKey);
-
-  // Check if there are data to commit to this file
-  int hasDataToCommit = tsdbHasDataToCommit(iters, pMem->maxTables, minKey, maxKey);
-  if (!hasDataToCommit) {
-    tsdbDebug("vgId:%d no data to commit to file %d", REPO_ID(pRepo), fid);
-    return 0;
-  }
+  terrno = TSDB_CODE_SUCCESS;
+  pMsg->length = htonl(pMsg->length);
+  pMsg->numOfBlocks = htonl(pMsg->numOfBlocks);
 
-  // Create and open files for commit
-  dataDir = tsdbGetDataDirName(pRepo->rootDir);
-  if (dataDir == NULL) {
-    terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+  if (tsdbInitSubmitMsgIter(pMsg, &msgIter) < 0) return -1;
+  while (true) {
+    if (tsdbGetSubmitMsgNext(&msgIter, &pBlock) < 0) return -1;
+    if (pBlock == NULL) break;
+
+    pBlock->uid = htobe64(pBlock->uid);
+    pBlock->tid = htonl(pBlock->tid);
+    pBlock->sversion = htonl(pBlock->sversion);
+    pBlock->dataLen = htonl(pBlock->dataLen);
+    pBlock->schemaLen = htonl(pBlock->schemaLen);
+    pBlock->numOfRows = htons(pBlock->numOfRows);
+
+    if (pBlock->tid <= 0 || pBlock->tid >= pMeta->maxTables) {
+      tsdbError("vgId:%d failed to get table to insert data, uid %" PRIu64 " tid %d", REPO_ID(pRepo), pBlock->uid,
+                pBlock->tid);
+      terrno = TSDB_CODE_TDB_INVALID_TABLE_ID;
       return -1;
     }
 
-  if ((pGroup = tsdbCreateFGroupIfNeed(pRepo, dataDir, fid)) == NULL) {
-    tsdbError("vgId:%d failed to create file group %d since %s", REPO_ID(pRepo), fid, tstrerror(terrno));
-    goto _err;
+    STable *pTable = pMeta->tables[pBlock->tid];
+    if (pTable == NULL || TABLE_UID(pTable) != pBlock->uid) {
+      tsdbError("vgId:%d failed to get table to insert data, uid %" PRIu64 " tid %d", REPO_ID(pRepo), pBlock->uid,
+                pBlock->tid);
+      terrno = TSDB_CODE_TDB_INVALID_TABLE_ID;
+      return -1;
     }
 
-  // Open files for write/read
-  if (tsdbSetAndOpenHelperFile(pHelper, pGroup) < 0) {
-    tsdbError("vgId:%d failed to set helper file since %s", REPO_ID(pRepo), tstrerror(terrno));
-    goto _err;
+    if (TABLE_TYPE(pTable) == TSDB_SUPER_TABLE) {
+      tsdbError("vgId:%d invalid action trying to insert a super table %s", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable));
+      terrno = TSDB_CODE_TDB_INVALID_ACTION;
+      return -1;
     }
 
-  newLast = TSDB_NLAST_FILE_OPENED(pHelper);
+    // Check schema version and update schema if needed
+    if (tsdbCheckTableSchema(pRepo, pBlock, pTable) < 0) {
+      if (terrno == TSDB_CODE_TDB_TABLE_RECONFIGURE) {
+        continue;
+      } else {
+        return -1;
+      }
+    }
 
-  if (tsdbLoadCompIdx(pHelper, NULL) < 0) {
-    tsdbError("vgId:%d failed to load SCompIdx part since %s", REPO_ID(pRepo), tstrerror(terrno));
-    goto _err;
+    tsdbInitSubmitBlkIter(pBlock, &blkIter);
+    while ((row = tsdbGetSubmitBlkNext(&blkIter)) != NULL) {
+      if (tsdbCheckRowRange(pRepo, pTable, row, minKey, maxKey, now) < 0) {
+        return -1;
+      }
+    }
   }
 
-  // Loop to commit data in each table
-  for (int tid = 1; tid < pMem->maxTables; tid++) {
-    SCommitIter *pIter = iters + tid;
-    if (pIter->pTable == NULL) continue;
+  if (terrno != TSDB_CODE_SUCCESS) return -1;
+  return 0;
+}
 
-    taosRLockLatch(&(pIter->pTable->latch));
+static int tsdbInsertDataToTable(STsdbRepo *pRepo, SSubmitBlk *pBlock, int32_t *affectedrows) {
+  STsdbMeta *    pMeta = pRepo->tsdbMeta;
+  int64_t        points = 0;
+  STable *       pTable = NULL;
+  SSubmitBlkIter blkIter = {0};
+  SDataRow       row = NULL;
+  void **        rows = NULL;
+  int            rowCounter = 0;
 
-    if (tsdbSetHelperTable(pHelper, pIter->pTable, pRepo) < 0) goto _err;
+  ASSERT(pBlock->tid < pMeta->maxTables);
+  pTable = pMeta->tables[pBlock->tid];
+  ASSERT(pTable != NULL && TABLE_UID(pTable) == pBlock->uid);
 
-    if (pIter->pIter != NULL) {
-      if (tdInitDataCols(pDataCols, tsdbGetTableSchemaImpl(pIter->pTable, false, false, -1)) < 0) {
+  rows = (void **)calloc(pBlock->numOfRows, sizeof(void *));
+  if (rows == NULL) {
     terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
-        goto _err;
+    return -1;
   }
 
-      if (tsdbCommitTableData(pHelper, pIter, pDataCols, maxKey) < 0) {
-        taosRUnLockLatch(&(pIter->pTable->latch));
-        tsdbError("vgId:%d failed to write data of table %s tid %d uid %" PRIu64 " since %s", REPO_ID(pRepo),
-                  TABLE_CHAR_NAME(pIter->pTable), TABLE_TID(pIter->pTable), TABLE_UID(pIter->pTable),
-                  tstrerror(terrno));
+  tsdbInitSubmitBlkIter(pBlock, &blkIter);
+  while ((row = tsdbGetSubmitBlkNext(&blkIter)) != NULL) {
+    if (tsdbCopyRowToMem(pRepo, row, pTable, &(rows[rowCounter])) < 0) {
+      tsdbFreeRows(pRepo, rows, rowCounter);
       goto _err;
     }
-    }
-
-    taosRUnLockLatch(&(pIter->pTable->latch));
 
-    // Move the last block to the new .l file if neccessary
-    if (tsdbMoveLastBlockIfNeccessary(pHelper) < 0) {
-      tsdbError("vgId:%d, failed to move last block, since %s", REPO_ID(pRepo), tstrerror(terrno));
-      goto _err;
-    }
+    (*affectedrows)++;
+    points++;
 
-    // Write the SCompBlock part
-    if (tsdbWriteCompInfo(pHelper) < 0) {
-      tsdbError("vgId:%d, failed to write compInfo part since %s", REPO_ID(pRepo), tstrerror(terrno));
-      goto _err;
+    if (rows[rowCounter] != NULL) {
+      rowCounter++;
     }
   }
 
-  if (tsdbWriteCompIdx(pHelper) < 0) {
-    tsdbError("vgId:%d failed to write compIdx part to file %d since %s", REPO_ID(pRepo), fid, tstrerror(terrno));
+  if (tsdbInsertDataToTableImpl(pRepo, pTable, rows, rowCounter) < 0) {
     goto _err;
   }
 
-  tfree(dataDir);
-  tsdbCloseHelperFile(pHelper, 0, pGroup);
+  STSchema *pSchema = tsdbGetTableSchemaByVersion(pTable, pBlock->sversion);
+  pRepo->stat.pointsWritten += points * schemaNCols(pSchema);
+  pRepo->stat.totalStorage += points * schemaVLen(pSchema);
 
-  pthread_rwlock_wrlock(&(pFileH->fhlock));
+  free(rows);
+  return 0;
 
-  (void)rename(helperNewHeadF(pHelper)->fname, helperHeadF(pHelper)->fname);
-  pGroup->files[TSDB_FILE_TYPE_HEAD].info = helperNewHeadF(pHelper)->info;
+_err:
+  free(rows);
+  return -1;
+}
 
-  if (newLast) {
-    (void)rename(helperNewLastF(pHelper)->fname, helperLastF(pHelper)->fname);
-    pGroup->files[TSDB_FILE_TYPE_LAST].info = helperNewLastF(pHelper)->info;
-  } else {
-    pGroup->files[TSDB_FILE_TYPE_LAST].info = helperLastF(pHelper)->info;
+static int tsdbCopyRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable, void **ppRow) {
+  STsdbCfg *  pCfg = &pRepo->config;
+  TKEY        tkey = dataRowTKey(row);
+  TSKEY       key = dataRowKey(row);
+  bool        isRowDelete = TKEY_IS_DELETED(tkey);
+
+  if (isRowDelete) {
+    if (!pCfg->update) {
+      tsdbWarn("vgId:%d vnode is not allowed to update but try to delete a data row", REPO_ID(pRepo));
+      terrno = TSDB_CODE_TDB_INVALID_ACTION;
+      return -1;
     }
 
-  pGroup->files[TSDB_FILE_TYPE_DATA].info = helperDataF(pHelper)->info;
+    if (key > TABLE_LASTKEY(pTable)) {
+      tsdbTrace("vgId:%d skip to delete row key %" PRId64 " which is larger than table lastKey %" PRId64,
+                REPO_ID(pRepo), key, TABLE_LASTKEY(pTable));
+      return 0;
+    }
+  }
 
-  pthread_rwlock_unlock(&(pFileH->fhlock));
+  void *pRow = tsdbAllocBytes(pRepo, dataRowLen(row));
+  if (pRow == NULL) {
+    tsdbError("vgId:%d failed to insert row with key %" PRId64 " to table %s while allocate %d bytes since %s",
+              REPO_ID(pRepo), key, TABLE_CHAR_NAME(pTable), dataRowLen(row), tstrerror(terrno));
+    return -1;
+  }
+
+  dataRowCpy(pRow, row);
+  ppRow[0] = pRow;
+
+  tsdbTrace("vgId:%d a row is %s table %s tid %d uid %" PRIu64 " key %" PRIu64, REPO_ID(pRepo),
+            isRowDelete ? "deleted from" : "updated in", TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable),
+            key);
 
   return 0;
+}
 
-_err:
-  tfree(dataDir);
-  tsdbCloseHelperFile(pHelper, 1, NULL);
+static int tsdbInitSubmitMsgIter(SSubmitMsg *pMsg, SSubmitMsgIter *pIter) {
+  if (pMsg == NULL) {
+    terrno = TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP;
+    return -1;
+  }
+
+  pIter->totalLen = pMsg->length;
+  pIter->len = 0;
+  pIter->pMsg = pMsg;
+  if (pMsg->length <= TSDB_SUBMIT_MSG_HEAD_SIZE) {
+    terrno = TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP;
     return -1;
+  }
+
+  return 0;
 }
 
-static SCommitIter *tsdbCreateCommitIters(STsdbRepo *pRepo) {
-  SMemTable *pMem = pRepo->imem;
-  STsdbMeta *pMeta = pRepo->tsdbMeta;
+static int tsdbGetSubmitMsgNext(SSubmitMsgIter *pIter, SSubmitBlk **pPBlock) {
+  if (pIter->len == 0) {
+    pIter->len += TSDB_SUBMIT_MSG_HEAD_SIZE;
+  } else {
+    SSubmitBlk *pSubmitBlk = (SSubmitBlk *)POINTER_SHIFT(pIter->pMsg, pIter->len);
+    pIter->len += (sizeof(SSubmitBlk) + pSubmitBlk->dataLen + pSubmitBlk->schemaLen);
+  }
 
-  SCommitIter *iters = (SCommitIter *)calloc(pMem->maxTables, sizeof(SCommitIter));
-  if (iters == NULL) {
-    terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
-    return NULL;
+  if (pIter->len > pIter->totalLen) {
+    terrno = TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP;
+    *pPBlock = NULL;
+    return -1;
   }
 
-  if (tsdbRLockRepoMeta(pRepo) < 0) goto _err;
+  *pPBlock = (pIter->len == pIter->totalLen) ? NULL : (SSubmitBlk *)POINTER_SHIFT(pIter->pMsg, pIter->len);
 
-  // reference all tables
-  for (int i = 0; i < pMem->maxTables; i++) {
-    if (pMeta->tables[i] != NULL) {
-      tsdbRefTable(pMeta->tables[i]);
-      iters[i].pTable = pMeta->tables[i];
+  return 0;
+}
+
+static int tsdbCheckTableSchema(STsdbRepo *pRepo, SSubmitBlk *pBlock, STable *pTable) {
+  ASSERT(pTable != NULL);
+
+  STSchema *pSchema = tsdbGetTableSchemaImpl(pTable, false, false, -1);
+  int       sversion = schemaVersion(pSchema);
+
+  if (pBlock->sversion == sversion) {
+    return 0;
+  } else {
+    if (TABLE_TYPE(pTable) == TSDB_STREAM_TABLE) {  // stream table is not allowed to change schema
+      terrno = TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION;
+      return -1;
     }
   }
 
-  if (tsdbUnlockRepoMeta(pRepo) < 0) goto _err;
+  if (pBlock->sversion > sversion) {  // may need to update table schema
+    if (pBlock->schemaLen > 0) {
+      tsdbDebug(
+          "vgId:%d table %s tid %d uid %" PRIu64 " schema version %d is out of data, client version %d, update...",
+          REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable), sversion, pBlock->sversion);
+      ASSERT(pBlock->schemaLen % sizeof(STColumn) == 0);
+      int       numOfCols = pBlock->schemaLen / sizeof(STColumn);
+      STColumn *pTCol = (STColumn *)pBlock->data;
 
-  for (int i = 0; i < pMem->maxTables; i++) {
-    if ((iters[i].pTable != NULL) && (pMem->tData[i] != NULL) && (TABLE_UID(iters[i].pTable) == pMem->tData[i]->uid)) {
-      if ((iters[i].pIter = tSkipListCreateIter(pMem->tData[i]->pData)) == NULL) {
+      STSchemaBuilder schemaBuilder = {0};
+      if (tdInitTSchemaBuilder(&schemaBuilder, pBlock->sversion) < 0) {
         terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
-        goto _err;
+        tsdbError("vgId:%d failed to update schema of table %s since %s", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable),
+                  tstrerror(terrno));
+        return -1;
       }
 
-      tSkipListIterNext(iters[i].pIter);
+      for (int i = 0; i < numOfCols; i++) {
+        if (tdAddColToSchema(&schemaBuilder, pTCol[i].type, htons(pTCol[i].colId), htons(pTCol[i].bytes)) < 0) {
+          terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+          tsdbError("vgId:%d failed to update schema of table %s since %s", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable),
+                    tstrerror(terrno));
+          tdDestroyTSchemaBuilder(&schemaBuilder);
+          return -1;
         }
       }
 
-  return iters;
+      STSchema *pNSchema = tdGetSchemaFromBuilder(&schemaBuilder);
+      if (pNSchema == NULL) {
+        terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+        tdDestroyTSchemaBuilder(&schemaBuilder);
+        return -1;
+      }
 
-_err:
-  tsdbDestroyCommitIters(iters, pMem->maxTables);
-  return NULL;
+      tdDestroyTSchemaBuilder(&schemaBuilder);
+      tsdbUpdateTableSchema(pRepo, pTable, pNSchema, true);
+    } else {
+      tsdbDebug(
+          "vgId:%d table %s tid %d uid %" PRIu64 " schema version %d is out of data, client version %d, reconfigure...",
+          REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable), sversion, pBlock->sversion);
+      terrno = TSDB_CODE_TDB_TABLE_RECONFIGURE;
+      return -1;
+    }
+  } else {
+    ASSERT(pBlock->sversion >= 0);
+    if (tsdbGetTableSchemaImpl(pTable, false, false, pBlock->sversion) == NULL) {
+      tsdbError("vgId:%d invalid submit schema version %d to table %s tid %d from client", REPO_ID(pRepo),
+                pBlock->sversion, TABLE_CHAR_NAME(pTable), TABLE_TID(pTable));
+    }
+    terrno = TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION;
+    return -1;
+  }
+
+  return 0;
 }
 
-static void tsdbDestroyCommitIters(SCommitIter *iters, int maxTables) {
-  if (iters == NULL) return;
+static int tsdbInsertDataToTableImpl(STsdbRepo *pRepo, STable *pTable, void **rows, int rowCounter) {
+  if (rowCounter < 1) return 0;
+
+  SMemTable * pMemTable = NULL;
+  STableData *pTableData = NULL;
+  STsdbMeta * pMeta = pRepo->tsdbMeta;
+  STsdbCfg *  pCfg = &(pRepo->config);
+
+  ASSERT(pRepo->mem != NULL);
+  pMemTable = pRepo->mem;
 
-  for (int i = 1; i < maxTables; i++) {
-    if (iters[i].pTable != NULL) {
-      tsdbUnRefTable(iters[i].pTable);
-      tSkipListDestroyIter(iters[i].pIter);
+  if (TABLE_TID(pTable) >= pMemTable->maxTables) {
+    if (tsdbAdjustMemMaxTables(pMemTable, pMeta->maxTables) < 0) {
+      tsdbFreeRows(pRepo, rows, rowCounter);
+      return -1;
     }
   }
+  pTableData = pMemTable->tData[TABLE_TID(pTable)];
 
-  free(iters);
-}
-
-static int tsdbAdjustMemMaxTables(SMemTable *pMemTable, int maxTables) {
-  ASSERT(pMemTable->maxTables < maxTables);
+  if (pTableData == NULL || pTableData->uid != TABLE_UID(pTable)) {
+    if (pTableData != NULL) {
+      taosWLockLatch(&(pMemTable->latch));
+      pMemTable->tData[TABLE_TID(pTable)] = NULL;
+      tsdbFreeTableData(pTableData);
+      taosWUnLockLatch(&(pMemTable->latch));
+    }
 
-  STableData **pTableData = (STableData **)calloc(maxTables, sizeof(STableData *));
+    pTableData = tsdbNewTableData(pCfg, pTable);
     if (pTableData == NULL) {
-    terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+      tsdbError("vgId:%d failed to insert data to table %s uid %" PRId64 " tid %d since %s", REPO_ID(pRepo),
+                TABLE_CHAR_NAME(pTable), TABLE_UID(pTable), TABLE_TID(pTable), tstrerror(terrno));
+      tsdbFreeRows(pRepo, rows, rowCounter);
       return -1;
     }
-  memcpy((void *)pTableData, (void *)pMemTable->tData, sizeof(STableData *) * pMemTable->maxTables);
 
-  STableData **tData = pMemTable->tData;
+    pRepo->mem->tData[TABLE_TID(pTable)] = pTableData;
+  }
 
-  taosWLockLatch(&(pMemTable->latch));
-  pMemTable->maxTables = maxTables;
-  pMemTable->tData = pTableData;
-  taosWUnLockLatch(&(pMemTable->latch));
+  ASSERT((pTableData != NULL) && pTableData->uid == TABLE_UID(pTable));
 
-  tfree(tData);
+  int64_t osize = SL_SIZE(pTableData->pData);
+  tSkipListPutBatch(pTableData->pData, rows, rowCounter);
+  int64_t dsize = SL_SIZE(pTableData->pData) - osize;
+
+  if (pMemTable->keyFirst > dataRowKey(rows[0])) pMemTable->keyFirst = dataRowKey(rows[0]);
+  if (pMemTable->keyLast < dataRowKey(rows[rowCounter - 1])) pMemTable->keyLast = dataRowKey(rows[rowCounter - 1]);
+  pMemTable->numOfRows += dsize;
+
+  if (pTableData->keyFirst > dataRowKey(rows[0])) pTableData->keyFirst = dataRowKey(rows[0]);
+  if (pTableData->keyLast < dataRowKey(rows[rowCounter - 1])) pTableData->keyLast = dataRowKey(rows[rowCounter - 1]);
+  pTableData->numOfRows += dsize;
+
+  // TODO: impl delete row thing
+  if (TABLE_LASTKEY(pTable) < dataRowKey(rows[rowCounter-1])) TABLE_LASTKEY(pTable) = dataRowKey(rows[rowCounter-1]);
 
   return 0;
 }
 
-static int tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema **ppSchema, SDataRow row) {
-  if (pCols) {
-    if (*ppSchema == NULL || schemaVersion(*ppSchema) != dataRowVersion(row)) {
-      *ppSchema = tsdbGetTableSchemaImpl(pTable, false, false, dataRowVersion(row));
-      if (*ppSchema == NULL) {
-        ASSERT(false);
-        return -1;
-      }
-    }
+static void tsdbFreeRows(STsdbRepo *pRepo, void **rows, int rowCounter) {
+  ASSERT(pRepo->mem != NULL);
+  STsdbBufPool *pBufPool = pRepo->pPool;
 
-    tdAppendDataRowToDataCol(row, *ppSchema, pCols);
+  for (int i = rowCounter - 1; i >= 0; --i) {
+    SDataRow row = (SDataRow)rows[i];
+    int      bytes = (int)dataRowLen(row);
+
+    if (pRepo->mem->extraBuffList == NULL) {
+      STsdbBufBlock *pBufBlock = tsdbGetCurrBufBlock(pRepo);
+      ASSERT(pBufBlock != NULL && pBufBlock->offset >= bytes);
+
+      pBufBlock->offset -= bytes;
+      pBufBlock->remain += bytes;
+      ASSERT(row == POINTER_SHIFT(pBufBlock->data, pBufBlock->offset));
+      tsdbTrace("vgId:%d free %d bytes to TSDB buffer pool, nBlocks %d offset %d remain %d", REPO_ID(pRepo), bytes,
+                listNEles(pRepo->mem->bufBlockList), pBufBlock->offset, pBufBlock->remain);
+
+      if (pBufBlock->offset == 0) {  // return the block to buffer pool
+        tsdbLockRepo(pRepo);
+        SListNode *pNode = tdListPopTail(pRepo->mem->bufBlockList);
+        tdListPrependNode(pBufPool->bufBlockList, pNode);
+        tsdbUnlockRepo(pRepo);
       }
+    } else {
+      ASSERT(listNEles(pRepo->mem->extraBuffList) > 0);
+      SListNode *pNode = tdListPopTail(pRepo->mem->extraBuffList);
+      ASSERT(row == pNode->data);
+      free(pNode);
+      tsdbTrace("vgId:%d free %d bytes to SYSTEM buffer pool", REPO_ID(pRepo), bytes);
 
-  return 0;
+      if (listNEles(pRepo->mem->extraBuffList) == 0) {
+        tdListFree(pRepo->mem->extraBuffList);
+        pRepo->mem->extraBuffList = NULL;
+      }
+    }
+  }
 }
\ No newline at end of file

src/tsdb/src/tsdbRWHelper.c

@@ -1595,7 +1595,7 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter,
       tblkIdx++;
     } else if (oBlock.numOfRows + pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed == 0) {
       // Delete the block and do some stuff
-      ASSERT(pMergeInfo->keyFirst == INT64_MAX && pMergeInfo->keyFirst == INT64_MIN);
+      // ASSERT(pMergeInfo->keyFirst == INT64_MAX && pMergeInfo->keyFirst == INT64_MIN);
       if (tsdbDeleteSuperBlock(pHelper, tblkIdx) < 0) return -1;
       *pCommitIter->pIter = slIter;
       if (oBlock.last && pHelper->hasOldLastBlock) pHelper->hasOldLastBlock = false;

src/tsdb/src/tsdbRead.c

@@ -120,8 +120,6 @@ typedef struct STsdbQueryHandle {
   SDataCols     *pDataCols;        // in order to hold current file data block
   int32_t        allocSize;        // allocated data block size
   SMemRef       *pMemRef;
-//  SMemTable     *mem;              // mem-table
-//  SMemTable     *imem;             // imem-table, acquired from snapshot
   SArray        *defaultLoadColumn;// default load column
   SDataBlockLoadInfo dataBlockLoadInfo; /* record current block load information */
   SLoadCompBlockInfo compBlockLoadInfo; /* record current compblock information in SQuery */
@@ -194,9 +192,12 @@ static void tsdbMayTakeMemSnapshot(STsdbQueryHandle* pQueryHandle) {
 }
 
 static void tsdbMayUnTakeMemSnapshot(STsdbQueryHandle* pQueryHandle) {
-  assert(pQueryHandle != NULL && pQueryHandle->pMemRef != NULL);
-
+  assert(pQueryHandle != NULL);
   SMemRef* pMemRef = pQueryHandle->pMemRef;
+  if (pMemRef == NULL) { // it has been freed
+    return;
+  }
+
   if (--pMemRef->ref == 0) {
     tsdbUnTakeMemSnapShot(pQueryHandle->pTsdb, pMemRef->mem, pMemRef->imem);
     pMemRef->mem = NULL;
@@ -205,6 +206,7 @@ static void tsdbMayUnTakeMemSnapshot(STsdbQueryHandle* pQueryHandle) {
 
   pQueryHandle->pMemRef = NULL;
 }
+
 static SArray* createCheckInfoFromTableGroup(STsdbQueryHandle* pQueryHandle, STableGroupInfo* pGroupList, STsdbMeta* pMeta) {
   size_t sizeOfGroup = taosArrayGetSize(pGroupList->pGroupList);
   assert(sizeOfGroup >= 1 && pMeta != NULL);
@@ -1849,6 +1851,8 @@ static bool doHasDataInBuffer(STsdbQueryHandle* pQueryHandle) {
     pQueryHandle->activeIndex += 1;
   }
 
+  // no data in memtable or imemtable, decrease the memory reference.
+  tsdbMayUnTakeMemSnapshot(pQueryHandle);
   return false;
 }
 
@@ -1976,27 +1980,20 @@ static void destroyHelper(void* param) {
   free(param);
 }
 
-// handle data in cache situation
-bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) {
-  STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pHandle;
-
-  int64_t stime = taosGetTimestampUs();
-  int64_t elapsedTime = stime;
-
-  size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo);
-  assert(numOfTables > 0);
+static bool getNeighborRows(STsdbQueryHandle* pQueryHandle) {
+  assert (pQueryHandle->type == TSDB_QUERY_TYPE_EXTERNAL);
 
   SDataBlockInfo blockInfo = {{0}, 0};
-  if (pQueryHandle->type == TSDB_QUERY_TYPE_EXTERNAL) {
+
   pQueryHandle->type = TSDB_QUERY_TYPE_ALL;
   pQueryHandle->order = TSDB_ORDER_DESC;
 
-    if (!tsdbNextDataBlock(pHandle)) {
+  if (!tsdbNextDataBlock((void*) pQueryHandle)) {
     return false;
   }
 
-    tsdbRetrieveDataBlockInfo(pHandle, &blockInfo);
-    /*SArray *pDataBlock = */tsdbRetrieveDataBlock(pHandle, pQueryHandle->defaultLoadColumn);
+  tsdbRetrieveDataBlockInfo((void*) pQueryHandle, &blockInfo);
+  /*SArray *pDataBlock = */tsdbRetrieveDataBlock((void*) pQueryHandle, pQueryHandle->defaultLoadColumn);
   if (terrno != TSDB_CODE_SUCCESS) {
     return false;
   }
@@ -2086,6 +2083,27 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) {
   pQueryHandle->type = TSDB_QUERY_TYPE_EXTERNAL;
   pQueryHandle->checkFiles = false;
   return true;
+}
+
+// handle data in cache situation
+bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) {
+  STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pHandle;
+
+  int64_t stime = taosGetTimestampUs();
+  int64_t elapsedTime = stime;
+
+  size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo);
+  assert(numOfTables > 0);
+
+  if (pQueryHandle->type == TSDB_QUERY_TYPE_EXTERNAL) {
+    SMemRef* pMemRef = pQueryHandle->pMemRef;
+    tsdbMayTakeMemSnapshot(pQueryHandle);
+    bool ret = getNeighborRows(pQueryHandle);
+    tsdbMayUnTakeMemSnapshot(pQueryHandle);
+
+    // restore the pMemRef
+    pQueryHandle->pMemRef = pMemRef;
+    return ret;
   }
 
   if (pQueryHandle->checkFiles) {
@@ -2343,7 +2361,7 @@ void filterPrepare(void* expr, void* param) {
 
   if (pInfo->optr == TSDB_RELATION_IN) {
     pInfo->q = (char*) pCond->arr;
-  } else {
+  } else if (pCond != NULL) {
     uint32_t size = pCond->nLen * TSDB_NCHAR_SIZE;
     if (size < (uint32_t)pSchema->bytes) {
       size = pSchema->bytes;

src/util/inc/tarray.h

@@ -70,6 +70,13 @@ void* taosArrayGet(const SArray* pArray, size_t index);
  */
 void* taosArrayGetP(const SArray* pArray, size_t index);
 
+/**
+ * get the last element in the array list
+ * @param pArray
+ * @return
+ */
+void* taosArrayGetLast(const SArray* pArray);
+
 /**
  * return the size of array
  * @param pArray
@@ -117,6 +124,13 @@ void taosArrayClear(SArray* pArray);
  */
 void taosArrayDestroy(SArray* pArray);
 
+/**
+ *
+ * @param pArray
+ * @param fp
+ */
+void taosArrayDestroyEx(SArray* pArray, void (*fp)(void*));
+
 /**
  * sort the array
  * @param pArray

src/util/inc/tskiplist.h

@@ -131,6 +131,7 @@ SSkipList *tSkipListCreate(uint8_t maxLevel, uint8_t keyType, uint16_t keyLen, _
                            __sl_key_fn_t fn);
 void       tSkipListDestroy(SSkipList *pSkipList);
 SSkipListNode *    tSkipListPut(SSkipList *pSkipList, void *pData);
+void               tSkipListPutBatch(SSkipList *pSkipList, void **ppData, int ndata);
 SArray *           tSkipListGet(SSkipList *pSkipList, SSkipListKey pKey);
 void               tSkipListPrint(SSkipList *pSkipList, int16_t nlevel);
 SSkipListIterator *tSkipListCreateIter(SSkipList *pSkipList);

src/util/src/tarray.c

@@ -99,6 +99,10 @@ void* taosArrayGetP(const SArray* pArray, size_t index) {
   return *(void**)d;
 }
 
+void* taosArrayGetLast(const SArray* pArray) {
+  return TARRAY_GET_ELEM(pArray, pArray->size - 1);
+}
+
 size_t taosArrayGetSize(const SArray* pArray) { return pArray->size; }
 
 void* taosArrayInsert(SArray* pArray, size_t index, void* pData) {
@@ -189,6 +193,23 @@ void taosArrayDestroy(SArray* pArray) {
   free(pArray);
 }
 
+void taosArrayDestroyEx(SArray* pArray, void (*fp)(void*)) {
+  if (pArray == NULL) {
+    return;
+  }
+
+  if (fp == NULL) {
+    taosArrayDestroy(pArray);
+    return;
+  }
+
+  for(int32_t i = 0; i < pArray->size; ++i) {
+    fp(TARRAY_GET_ELEM(pArray, i));
+  }
+
+  taosArrayDestroy(pArray);
+}
+
 void taosArraySort(SArray* pArray, int (*compar)(const void*, const void*)) {
   assert(pArray != NULL);
   assert(compar != NULL);

src/util/src/tcache.c

@@ -658,7 +658,11 @@ void* taosCacheTimedRefresh(void *handle) {
 
   int64_t count = 0;
   while(1) {
+#if defined LINUX
+    usleep(500*1000);
+#else
     taosMsleep(500);
+#endif
 
     // check if current cache object will be deleted every 500ms.
     if (pCacheObj->deleting) {
@@ -677,6 +681,7 @@ void* taosCacheTimedRefresh(void *handle) {
       continue;
     }
 
+    uDebug("%s refresh thread timed scan", pCacheObj->name);
     pCacheObj->statistics.refreshCount++;
 
     // refresh data in hash table

src/util/src/tcompare.c

 #include "taosdef.h"
 #include "tcompare.h"
 #include "tarray.h"
-#include "tutil.h"
 
 int32_t compareInt32Val(const void *pLeft, const void *pRight) {
   int32_t left = GET_INT32_VAL(pLeft), right = GET_INT32_VAL(pRight);

src/util/src/tkvstore.c

@@ -236,6 +236,7 @@ int tdUpdateKVStoreRecord(SKVStore *pStore, uint64_t uid, void *cont, int contLe
   rInfo.offset = lseek(pStore->fd, 0, SEEK_CUR);
   if (rInfo.offset < 0) {
     uError("failed to lseek file %s since %s", pStore->fname, strerror(errno));
+    terrno = TAOS_SYSTEM_ERROR(errno);
     return -1;
   }
 
@@ -254,6 +255,7 @@ int tdUpdateKVStoreRecord(SKVStore *pStore, uint64_t uid, void *cont, int contLe
 
   if (taosWrite(pStore->fd, cont, contLen) < contLen) {
     uError("failed to write %d bytes to file %s since %s", contLen, pStore->fname, strerror(errno));
+    terrno = TAOS_SYSTEM_ERROR(errno);
     return -1;
   }
 

src/util/src/tlog.c

@@ -287,17 +287,17 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) {
   tsLogObj.fileNum = maxFileNum;
   taosGetLogFileName(fn);
 
+
   if (strlen(fn) < LOG_FILE_NAME_LEN + 50 - 2) {
     strcpy(name, fn);
     strcat(name, ".0");
   }
+  bool log0Exist = stat(name, &logstat0) >= 0;
 
   if (strlen(fn) < LOG_FILE_NAME_LEN + 50 - 2) {
     strcpy(name, fn);
     strcat(name, ".1");
   }
-
-  bool log0Exist = stat(name, &logstat0) >= 0;
   bool log1Exist = stat(name, &logstat1) >= 0;
   
   // if none of the log files exist, open 0, if both exists, open the old one

src/util/src/tref.c

@@ -426,11 +426,11 @@ static int taosDecRefCount(int rsetId, int64_t rid, int remove) {
 		
       (*pSet->fp)(pNode->p); 
 
-      uTrace("rsetId:%d p:%p rid:%" PRId64 "is removed, count:%d, free mem: %p", rsetId, pNode->p, rid, pSet->count, pNode);
+      uTrace("rsetId:%d p:%p rid:%" PRId64 " is removed, count:%d, free mem: %p", rsetId, pNode->p, rid, pSet->count, pNode);
       free(pNode);
       released = 1;
     } else {
-      uTrace("rsetId:%d p:%p rid:%" PRId64 "is released, count:%d", rsetId, pNode->p, rid, pNode->count);
+      uTrace("rsetId:%d p:%p rid:%" PRId64 " is released, count:%d", rsetId, pNode->p, rid, pNode->count);
     }
   } else {
     uTrace("rsetId:%d rid:%" PRId64 " is not there, failed to release/remove", rsetId, rid);

src/util/src/tskiplist.c

@@ -24,10 +24,12 @@ static SSkipListNode *    getPriorNode(SSkipList *pSkipList, const char *val, in
 static void               tSkipListRemoveNodeImpl(SSkipList *pSkipList, SSkipListNode *pNode);
 static void               tSkipListCorrectLevel(SSkipList *pSkipList);
 static SSkipListIterator *doCreateSkipListIterator(SSkipList *pSkipList, int32_t order);
-static void               tSkipListDoInsert(SSkipList *pSkipList, SSkipListNode **backward, SSkipListNode *pNode);
+static void tSkipListDoInsert(SSkipList *pSkipList, SSkipListNode **direction, SSkipListNode *pNode, bool isForward);
 static bool tSkipListGetPosToPut(SSkipList *pSkipList, SSkipListNode **backward, void *pData);
-static SSkipListNode *    tSkipListNewNode(uint8_t level);
+static SSkipListNode *tSkipListNewNode(uint8_t level);
 #define tSkipListFreeNode(n) tfree((n))
+static SSkipListNode *tSkipListPutImpl(SSkipList *pSkipList, void *pData, SSkipListNode **direction, bool isForward,
+                                       bool hasDup);
 
 static FORCE_INLINE int     tSkipListWLock(SSkipList *pSkipList);
 static FORCE_INLINE int     tSkipListRLock(SSkipList *pSkipList);
@@ -109,30 +111,85 @@ SSkipListNode *tSkipListPut(SSkipList *pSkipList, void *pData) {
   if (pSkipList == NULL || pData == NULL) return NULL;
 
   SSkipListNode *backward[MAX_SKIP_LIST_LEVEL] = {0};
-  uint8_t        dupMode = SL_DUP_MODE(pSkipList);
   SSkipListNode *pNode = NULL;
 
   tSkipListWLock(pSkipList);
 
   bool hasDup = tSkipListGetPosToPut(pSkipList, backward, pData);
+  pNode = tSkipListPutImpl(pSkipList, pData, backward, false, hasDup);
 
-  if (hasDup && (dupMode == SL_DISCARD_DUP_KEY || dupMode == SL_UPDATE_DUP_KEY)) {
-    if (dupMode == SL_UPDATE_DUP_KEY) {
-      pNode = SL_NODE_GET_BACKWARD_POINTER(backward[0], 0);
-      atomic_store_ptr(&(pNode->pData), pData);
+  tSkipListUnlock(pSkipList);
+
+  return pNode;
+}
+
+// Put a batch of data into skiplist. The batch of data must be in ascending order
+void tSkipListPutBatch(SSkipList *pSkipList, void **ppData, int ndata) {
+  SSkipListNode *backward[MAX_SKIP_LIST_LEVEL] = {0};
+  SSkipListNode *forward[MAX_SKIP_LIST_LEVEL] = {0};
+  bool           hasDup = false;
+  char *         pKey = NULL;
+  char *         pDataKey = NULL;
+  int            compare = 0;
+
+  tSkipListWLock(pSkipList);
+
+  // backward to put the first data
+  hasDup = tSkipListGetPosToPut(pSkipList, backward, ppData[0]);
+  tSkipListPutImpl(pSkipList, ppData[0], backward, false, hasDup);
+
+  for (int level = 0; level < pSkipList->maxLevel; level++) {
+    forward[level] = SL_NODE_GET_BACKWARD_POINTER(backward[level], level);
+  }
+
+  // forward to put the rest of data
+  for (int idata = 1; idata < ndata; idata++) {
+    pDataKey = pSkipList->keyFn(ppData[idata]);
+
+    // Compare max key
+    pKey = SL_GET_MAX_KEY(pSkipList);
+    compare = pSkipList->comparFn(pDataKey, pKey);
+    if (compare > 0) {
+      for (int i = 0; i < pSkipList->maxLevel; i++) {
+        forward[i] = SL_NODE_GET_BACKWARD_POINTER(pSkipList->pTail, i);
       }
+
+      hasDup = false;
     } else {
-    pNode = tSkipListNewNode(getSkipListRandLevel(pSkipList));
-    if (pNode != NULL) {
-      pNode->pData = pData;
+      SSkipListNode *px = pSkipList->pHead;
+      for (int i = pSkipList->maxLevel - 1; i >= 0; --i) {
+        if (i < pSkipList->level) {
+          // set new px
+          if (forward[i] != pSkipList->pHead) {
+            if (px == pSkipList->pHead ||
+                pSkipList->comparFn(SL_GET_NODE_KEY(pSkipList, forward[i]), SL_GET_NODE_KEY(pSkipList, px)) > 0) {
+              px = forward[i];
+            }
+          }
+
+          SSkipListNode *p = SL_NODE_GET_FORWARD_POINTER(px, i);
+          while (p != pSkipList->pTail) {
+            pKey = SL_GET_NODE_KEY(pSkipList, p);
+
+            compare = pSkipList->comparFn(pKey, pDataKey);
+            if (compare >= 0) {
+              if (compare == 0) hasDup = true;
+              break;
+            } else {
+              px = p;
+              p = SL_NODE_GET_FORWARD_POINTER(px, i);
+            }
+          }
+        }
 
-      tSkipListDoInsert(pSkipList, backward, pNode);
+        forward[i] = px;
       }
     }
 
-  tSkipListUnlock(pSkipList);
+    tSkipListPutImpl(pSkipList, ppData[idata], forward, true, hasDup);
+  }
 
-  return pNode;
+  tSkipListUnlock(pSkipList);
 }
 
 uint32_t tSkipListRemove(SSkipList *pSkipList, SSkipListKey key) {
@@ -310,22 +367,25 @@ void tSkipListPrint(SSkipList *pSkipList, int16_t nlevel) {
   }
 }
 
-static void tSkipListDoInsert(SSkipList *pSkipList, SSkipListNode **backward, SSkipListNode *pNode) {
+static void tSkipListDoInsert(SSkipList *pSkipList, SSkipListNode **direction, SSkipListNode *pNode, bool isForward) {
   for (int32_t i = 0; i < pNode->level; ++i) {
-    if (i >= pSkipList->level) {
-      SL_NODE_GET_FORWARD_POINTER(pNode, i) = pSkipList->pTail;
-      SL_NODE_GET_BACKWARD_POINTER(pNode, i) = pSkipList->pHead;
-      SL_NODE_GET_FORWARD_POINTER(pSkipList->pHead, i) = pNode;
-      SL_NODE_GET_BACKWARD_POINTER(pSkipList->pTail, i) = pNode;
+    SSkipListNode *x = direction[i];
+    if (isForward) {
+      SL_NODE_GET_BACKWARD_POINTER(pNode, i) = x;
+
+      SSkipListNode *next = SL_NODE_GET_FORWARD_POINTER(x, i);
+      SL_NODE_GET_BACKWARD_POINTER(next, i) = pNode;
+
+      SL_NODE_GET_FORWARD_POINTER(pNode, i) = next;
+      SL_NODE_GET_FORWARD_POINTER(x, i) = pNode;
     } else {
-      SSkipListNode *x = backward[i];
       SL_NODE_GET_FORWARD_POINTER(pNode, i) = x;
 
       SSkipListNode *prev = SL_NODE_GET_BACKWARD_POINTER(x, i);
       SL_NODE_GET_FORWARD_POINTER(prev, i) = pNode;
 
-      SL_NODE_GET_BACKWARD_POINTER(x, i) = pNode;
       SL_NODE_GET_BACKWARD_POINTER(pNode, i) = prev;
+      SL_NODE_GET_BACKWARD_POINTER(x, i) = pNode;
     }
   }
 
@@ -377,7 +437,7 @@ static bool tSkipListGetPosToPut(SSkipList *pSkipList, SSkipListNode **backward,
   char *  pDataKey = pSkipList->keyFn(pData);
 
   if (pSkipList->size == 0) {
-    for (int i = 0; i < pSkipList->level; i++) {
+    for (int i = 0; i < pSkipList->maxLevel; i++) {
       backward[i] = pSkipList->pTail;
     }
   } else {
@@ -387,7 +447,7 @@ static bool tSkipListGetPosToPut(SSkipList *pSkipList, SSkipListNode **backward,
     pKey = SL_GET_MAX_KEY(pSkipList);
     compare = pSkipList->comparFn(pDataKey, pKey);
     if (compare >= 0) {
-      for (int i = 0; i < pSkipList->level; i++) {
+      for (int i = 0; i < pSkipList->maxLevel; i++) {
         backward[i] = pSkipList->pTail;
       }
 
@@ -398,7 +458,7 @@ static bool tSkipListGetPosToPut(SSkipList *pSkipList, SSkipListNode **backward,
     pKey = SL_GET_MIN_KEY(pSkipList);
     compare = pSkipList->comparFn(pDataKey, pKey);
     if (compare < 0) {
-      for (int i = 0; i < pSkipList->level; i++) {
+      for (int i = 0; i < pSkipList->maxLevel; i++) {
         backward[i] = SL_NODE_GET_FORWARD_POINTER(pSkipList->pHead, i);
       }
 
@@ -406,7 +466,8 @@ static bool tSkipListGetPosToPut(SSkipList *pSkipList, SSkipListNode **backward,
     }
 
     SSkipListNode *px = pSkipList->pTail;
-    for (int i = pSkipList->level - 1; i >= 0; --i) {
+    for (int i = pSkipList->maxLevel - 1; i >= 0; --i) {
+      if (i < pSkipList->level) {
         SSkipListNode *p = SL_NODE_GET_BACKWARD_POINTER(px, i);
         while (p != pSkipList->pHead) {
           pKey = SL_GET_NODE_KEY(pSkipList, p);
@@ -420,6 +481,7 @@ static bool tSkipListGetPosToPut(SSkipList *pSkipList, SSkipListNode **backward,
             p = SL_NODE_GET_BACKWARD_POINTER(px, i);
           }
         }
+      }
 
       backward[i] = px;
     }
@@ -579,6 +641,32 @@ static SSkipListNode *tSkipListNewNode(uint8_t level) {
   return pNode;
 }
 
+static SSkipListNode *tSkipListPutImpl(SSkipList *pSkipList, void *pData, SSkipListNode **direction, bool isForward,
+                                       bool hasDup) {
+  uint8_t        dupMode = SL_DUP_MODE(pSkipList);
+  SSkipListNode *pNode = NULL;
+
+  if (hasDup && (dupMode == SL_DISCARD_DUP_KEY || dupMode == SL_UPDATE_DUP_KEY)) {
+    if (dupMode == SL_UPDATE_DUP_KEY) {
+      if (isForward) {
+        pNode = SL_NODE_GET_FORWARD_POINTER(direction[0], 0);
+      } else {
+        pNode = SL_NODE_GET_BACKWARD_POINTER(direction[0], 0);
+      }
+      atomic_store_ptr(&(pNode->pData), pData);
+    }
+  } else {
+    pNode = tSkipListNewNode(getSkipListRandLevel(pSkipList));
+    if (pNode != NULL) {
+      pNode->pData = pData;
+
+      tSkipListDoInsert(pSkipList, direction, pNode, isForward);
+    }
+  }
+
+  return pNode;
+}
+
 // static int32_t tSkipListEndParQuery(SSkipList *pSkipList, SSkipListNode *pStartNode, SSkipListKey *pEndKey,
 //                                    int32_t cond, SSkipListNode ***pRes) {
 //  pthread_rwlock_rdlock(&pSkipList->lock);

src/util/src/tsocket.c

@@ -107,7 +107,7 @@ int32_t taosWriteMsg(SOCKET fd, void *buf, int32_t nbytes) {
   while (nleft > 0) {
     nwritten = (int32_t)taosWriteSocket(fd, (char *)ptr, (size_t)nleft);
     if (nwritten <= 0) {
-      if (errno == EINTR)
+      if (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK)
         continue;
       else
         return -1;
@@ -133,7 +133,7 @@ int32_t taosReadMsg(SOCKET fd, void *buf, int32_t nbytes) {
     if (nread == 0) {
       break;
     } else if (nread < 0) {
-      if (errno == EINTR) {
+      if (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK) {
         continue;
       } else {
         return -1;

src/util/tests/trefTest.c

@@ -77,8 +77,8 @@ void *acquireRelease(void *param) {
     printf("a");
     
     id = random() % pSpace->refNum; 
-    code = taosAcquireRef(pSpace->rsetId, pSpace->p[id]);
-    if (code >= 0) {
+    void *p = taosAcquireRef(pSpace->rsetId, pSpace->p[id]);
+    if (p) {
       usleep(id % 5 + 1);
       taosReleaseRef(pSpace->rsetId, pSpace->p[id]);
     }

src/vnode/inc/vnodeInt.h

@@ -58,7 +58,7 @@ typedef struct {
   char        *rootDir;
   tsem_t       sem;
   int8_t       dropped;
-  char         db[TSDB_DB_NAME_LEN];
+  char         db[TSDB_ACCT_LEN + TSDB_DB_NAME_LEN];
 } SVnodeObj;
 
 void vnodeInitWriteFp(void);

src/vnode/src/vnodeCfg.c

@@ -25,7 +25,7 @@
 #include "vnodeCfg.h"
 
 static void vnodeLoadCfg(SVnodeObj *pVnode, SCreateVnodeMsg* vnodeMsg) {
-  strcpy(pVnode->db, vnodeMsg->db);
+  tstrncpy(pVnode->db, vnodeMsg->db, sizeof(pVnode->db));
   pVnode->cfgVersion = vnodeMsg->cfg.cfgVersion;
   pVnode->tsdbCfg.cacheBlockSize = vnodeMsg->cfg.cacheBlockSize;
   pVnode->tsdbCfg.totalBlocks = vnodeMsg->cfg.totalBlocks;
@@ -97,7 +97,7 @@ int32_t vnodeReadCfg(SVnodeObj *pVnode) {
     vError("vgId:%d, failed to read %s, db not found", pVnode->vgId, file);
     goto PARSE_VCFG_ERROR;
   }
-  strcpy(vnodeMsg.db, db->valuestring);
+  tstrncpy(vnodeMsg.db, db->valuestring, sizeof(vnodeMsg.db));
 
   cJSON *cfgVersion = cJSON_GetObjectItem(root, "cfgVersion");
   if (!cfgVersion || cfgVersion->type != cJSON_Number) {

src/vnode/src/vnodeMain.c

@@ -30,7 +30,7 @@
 
 static SHashObj*tsVnodesHash;
 static void     vnodeCleanUp(SVnodeObj *pVnode);
-static int      vnodeProcessTsdbStatus(void *arg, int status);
+static int      vnodeProcessTsdbStatus(void *arg, int status, int eno);
 static uint32_t vnodeGetFileInfo(void *ahandle, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fversion);
 static int      vnodeGetWalInfo(void *ahandle, char *fileName, int64_t *fileId);
 static void     vnodeNotifyRole(void *ahandle, int8_t role);
@@ -315,7 +315,13 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) {
     pVnode->version = walGetVersion(pVnode->wal);
   }
 
-  tsdbSyncCommit(pVnode->tsdb);
+  code = tsdbSyncCommit(pVnode->tsdb);
+  if (code != 0) {
+    vError("vgId:%d, failed to commit after restore from wal since %s", pVnode->vgId, tstrerror(code));
+    vnodeCleanUp(pVnode);
+    return code;
+  }
+
   walRemoveAllOldFiles(pVnode->wal);
   walRenew(pVnode->wal);
 
@@ -412,6 +418,7 @@ void vnodeRelease(void *pVnodeRaw) {
   }
 
   if (pVnode->wal) {
+    walRemoveAllOldFiles(pVnode->wal);
     walClose(pVnode->wal);
     pVnode->wal = NULL;
   }
@@ -583,14 +590,22 @@ static void vnodeCleanUp(SVnodeObj *pVnode) {
 }
 
 // TODO: this is a simple implement
-static int vnodeProcessTsdbStatus(void *arg, int status) {
+static int vnodeProcessTsdbStatus(void *arg, int status, int eno) {
   SVnodeObj *pVnode = arg;
 
+  if (eno != TSDB_CODE_SUCCESS) {
+    // TODO: deal with the error here
+  }
+
   if (status == TSDB_STATUS_COMMIT_START) {
     pVnode->fversion = pVnode->version;
     vDebug("vgId:%d, start commit, fver:%" PRIu64 " vver:%" PRIu64, pVnode->vgId, pVnode->fversion, pVnode->version);
+    if (pVnode->status == TAOS_VN_STATUS_INIT) {
+      return 0;
+    } else {
       return walRenew(pVnode->wal);
     }
+  }
 
   if (status == TSDB_STATUS_COMMIT_OVER) {
     vDebug("vgId:%d, commit over, fver:%" PRIu64 " vver:%" PRIu64, pVnode->vgId, pVnode->fversion, pVnode->version);

src/wal/inc/walInt.h

@@ -37,7 +37,7 @@ extern int32_t wDebugFlag;
 #define WAL_MAX_SIZE   (TSDB_MAX_WAL_SIZE + sizeof(SWalHead) + 16)
 #define WAL_SIGNATURE  ((uint32_t)(0xFAFBFDFE))
 #define WAL_PATH_LEN   (TSDB_FILENAME_LEN + 12)
-#define WAL_FILE_LEN   (TSDB_FILENAME_LEN + 32)
+#define WAL_FILE_LEN   (WAL_PATH_LEN + 32)
 #define WAL_FILE_NUM   3
 
 typedef struct {

src/wal/src/walMgmt.c

@@ -124,15 +124,7 @@ void walClose(void *handle) {
 
   SWal *pWal = handle;
   pthread_mutex_lock(&pWal->mutex);
-
   taosClose(pWal->fd);
-
-  if (pWal->keep != TAOS_WAL_KEEP) {
-    walRemoveAllOldFiles(pWal);
-  } else {
-    wDebug("vgId:%d, wal:%p file:%s, it is closed and kept", pWal->vgId, pWal, pWal->name);
-  }
-
   pthread_mutex_unlock(&pWal->mutex);
   taosRemoveRef(tsWal.refId, pWal->rid);
 }

src/wal/src/walWrite.c

@@ -67,6 +67,7 @@ void walRemoveOneOldFile(void *handle) {
   SWal *pWal = handle;
   if (pWal == NULL) return;
   if (pWal->keep == TAOS_WAL_KEEP) return;
+  if (pWal->fd <= 0) return;
 
   pthread_mutex_lock(&pWal->mutex);
 
@@ -91,6 +92,8 @@ void walRemoveAllOldFiles(void *handle) {
 
   SWal *  pWal = handle;
   int64_t fileId = -1;
+
+  pthread_mutex_lock(&pWal->mutex);
   while (walGetNextFile(pWal, &fileId) >= 0) {
     snprintf(pWal->name, sizeof(pWal->name), "%s/%s%" PRId64, pWal->path, WAL_PREFIX, fileId);
 
@@ -100,6 +103,7 @@ void walRemoveAllOldFiles(void *handle) {
       wInfo("vgId:%d, wal:%p file:%s, it is removed", pWal->vgId, pWal, pWal->name);
     }
   }
+  pthread_mutex_unlock(&pWal->mutex);
 }
 
 int32_t walWrite(void *handle, SWalHead *pHead) {

tests/pytest/fulltest.sh

@@ -155,6 +155,9 @@ python3 ./test.py -f query/queryCountCSVData.py
 python3 ./test.py -f query/natualInterval.py
 python3 ./test.py -f query/bug1471.py
 #python3 ./test.py -f query/dataLossTest.py
+python3 ./test.py -f query/bug1874.py
+python3 ./test.py -f query/bug1875.py
+python3 ./test.py -f query/bug1876.py
 
 #stream
 python3 ./test.py -f stream/metric_1.py

tests/script/general/parser/constCol.sim

@@ -347,6 +347,17 @@ if $rows != 3 then
   return -1
 endi
 
+sql select 0.1 + 0.2 from t1
+if $rows != 3 then
+  return -1
+endi
+
+print =============================> td-2036
+if $data00 != 0.3000000 then
+  print expect: 0.3000000, actual:$data00
+  return -1
+endi
+
 print ======================udc with normal column group by
 
 sql_error select from t1

tests/script/general/parser/sliding.sim

@@ -4,6 +4,7 @@ system sh/deploy.sh -n dnode1 -i 1
 system sh/cfg.sh -n dnode1 -c walLevel -v 1
 system sh/cfg.sh -n dnode1 -c debugFlag -v 135
 system sh/cfg.sh -n dnode1 -c rpcDebugFlag -v 135
+system sh/cfg.sh -n dnode1 -c maxtablespervnode -v 4
 system sh/exec.sh -n dnode1 -s start
 sleep 1000
 sql connect
@@ -28,7 +29,7 @@ $mt = $mtPrefix . $i
 
 sql drop database if exists $db -x step1
 step1:
-sql create database if not exists $db maxtables 4 keep 36500
+sql create database if not exists $db  keep 36500
 sql use $db
 sql create table $mt (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9)) TAGS(t1 int, t2 binary(12))
 
@@ -169,19 +170,20 @@ if $data94 != 90 then
 endi
 
 sql select count(c2),last(c4) from sliding_tb0 interval(30s) sliding(10s) order by ts asc;
-if $row != 30 then
+if $row != 32 then
   return -1
 endi
 
-if $data00 != @00-01-01 00:00:00.000@ then
+if $data00 != @99-12-31 23:59:40.000@ then
+  print expect 12-31 23:59:40.000, actual: $data00
   return -1
 endi
 
-if $data01 != 1000 then
+if $data01 != 334 then
   return -1
 endi
 
-if $data02 != 99 then
+if $data02 != 33 then
   return -1
 endi
 
@@ -304,11 +306,11 @@ if $data13 != 9.810708435 then
 endi
 
 sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by ts asc limit 100 offset 1;
-if $row != 14 then
+if $row != 15 then
  return -1
 endi
 
-if $data00 != @00-01-01 00:00:20.000@ then
+if $data00 != @00-01-01 00:00:00.000@ then
  return -1
 endi
 
@@ -316,7 +318,7 @@ if $data01 != 1000 then
  return -1
 endi
 
-if $data02 != 66 then
+if $data02 != 99 then
  return -1
 endi
 
@@ -324,7 +326,7 @@ if $data03 != 28.866070048 then
  return -1
 endi
 
-if $data90 != @00-01-01 00:03:20.000@ then
+if $data90 != @00-01-01 00:03:00.000@ then
  return -1
 endi
 
@@ -332,113 +334,158 @@ if $data91 != 1000 then
  return -1
 endi
 
-if $data92 != 66 then
+if $data92 != 99 then
  return -1
 endi
 
-sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by ts asc limit 100 offset 14;
-if $row != 1 then
+sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by ts asc limit 100 offset 5;
+if $row != 11 then
  return -1
 endi
 
-sql select count(c2),last(c4),stddev(c3),spread(c3) from sliding_tb0 where c2 = 0 interval(30s) order by ts desc;
+sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by ts asc limit 100 offset 6;
 if $row != 10 then
  return -1
 endi
 
-#00-01-01 00:04:30.000|                   10|       0|              0.000000000|              0.000000000|
-if $data00 != @00-01-01 00:04:30.000@ then
+sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by ts asc limit 100 offset 7;
+if $row != 9 then
  return -1
 endi
 
-if $data01 != 10 then
+sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by ts asc limit 100 offset 8;
+if $row != 8 then
  return -1
 endi
 
-if $data02 != 0 then
+sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by ts asc limit 100 offset 9;
+if $row != 7 then
  return -1
 endi
 
-if $data03 != 0.000000000 then
+sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by ts asc limit 100 offset 10;
+if $row != 6 then
  return -1
 endi
 
-sql select count(c2),last(c4),stddev(c3),spread(c3) from sliding_tb0 where c2 = 0 interval(30s) sliding(20s) order by ts desc limit 1 offset 15;
+sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by ts asc limit 100 offset 11;
+if $row != 5 then
+ return -1
+endi
+
+sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by ts asc limit 100 offset 12;
+if $row != 4 then
+ return -1
+endi
+
+sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by ts asc limit 100 offset 13;
+if $row != 3 then
+ return -1
+endi
+
+sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by ts asc limit 100 offset 14;
+if $row != 2 then
+ return -1
+endi
+
+sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by ts asc limit 100 offset 15;
+if $row != 1 then
+ return -1
+endi
+
+sql select count(c2),last(c4),stddev(c3) from sliding_tb0 interval(30s) sliding(20s) order by ts asc limit 100 offset 16;
 if $row != 0 then
  return -1
 endi
 
-sql select count(*),stddev(c1),count(c1),first(c2),last(c3) from sliding_tb0 where ts>'2000-1-1 00:00:00' and ts<'2000-1-1 00:00:01.002' and c2 >= 0 interval(30s) sliding(10a) order by ts asc limit 1000;
-if $row != 100 then
+sql select count(c2),last(c4),stddev(c3),spread(c3) from sliding_tb0 where c2 = 0 interval(30s) order by ts desc;
+if $row != 10 then
  return -1
 endi
 
-if $data00 != @00-01-01 00:00:00.000@ then
+#00-01-01 00:04:30.000|                   10|       0|              0.000000000|              0.000000000|
+if $data00 != @00-01-01 00:04:30.000@ then
  return -1
 endi
 
-if $data02 != 9.521904571 then
+if $data01 != 10 then
  return -1
 endi
 
-if $data05 != 33 then
+if $data02 != 0 then
  return -1
 endi
 
-if $data10 != @00-01-01 00:00:00.010@ then
+if $data03 != 0.000000000 then
  return -1
 endi
 
-if $data12 != 9.521904571 then
+sql select count(c2),last(c4),stddev(c3),spread(c3) from sliding_tb0 where c2 = 0 interval(30s) sliding(20s) order by ts desc limit 1 offset 15;
+if $row != 1 then
  return -1
 endi
 
-if $data15 != 33 then
+sql select count(c2),last(c4),stddev(c3),spread(c3) from sliding_tb0 where c2 = 0 interval(30s) sliding(20s) order by ts desc limit 1 offset 16;
+if $row != 0 then
  return -1
 endi
 
-if $data95 != 33 then
+sql select count(c2), first(c3),stddev(c4) from sliding_tb0 interval(10a) order by ts desc limit 10 offset 2;
+if $data00 != @00-01-01 00:04:59.910@ then
   return -1
 endi
 
-sql select count(*),stddev(c1),count(c1),first(c2),last(c3) from sliding_tb0 where ts>'2000-1-1 00:00:00' and ts<'2000-1-1 00:00:01.002' and c2 >= 0 interval(30s) sliding(10a) order by ts desc limit 1000;
-if $row != 100 then
+sql select count(*),stddev(c1),count(c1),first(c2),last(c3) from sliding_tb0 where ts>'2000-1-1 00:00:00' and ts<'2000-1-1 00:00:01.002' and c2 >= 0 interval(30s) sliding(10s) order by ts asc limit 1000;
+if $row != 3 then
  return -1
 endi
 
-if $data00 != @00-01-01 00:00:00.990@ then
+if $data00 != @99-12-31 23:59:40.000@ then
  return -1
 endi
 
-if $data01 != 1 then
+if $data02 != 9.521904571 then
+ return -1
+endi
+
+if $data05 != 33 then
+ return -1
+endi
+
+if $data10 != @99-12-31 23:59:50.000@ then
+ return -1
+endi
+
+if $data12 != 9.521904571 then
  return -1
 endi
 
-if $data02 != 0.000000000 then
+if $data15 != 33 then
  return -1
 endi
 
-if $data03 != 1 then
+if $data25 != 33 then
  return -1
 endi
 
-if $data90 != @00-01-01 00:00:00.900@ then
+sql select count(*),stddev(c1),count(c1),first(c2),last(c3) from sliding_tb0 where ts>'2000-1-1 00:00:00' and ts<'2000-1-1 00:00:01.002' and c2 >= 0 interval(30s) sliding(10s) order by ts desc limit 1000;
+if $row != 1 then
  return -1
 endi
 
-if $data91 != 4 then
+if $data00 != @99-12-31 23:59:40.000@ then
  return -1
 endi
 
-if $data92 != 1.118033989 then
+if $data01 != 33 then
  return -1
 endi
 
-if $data93 != 4 then
+if $data02 != 9.521904571 then
  return -1
 endi
 
-if $data94 != 30.00000 then
+if $data03 != 33 then
  return -1
 endi
 
@@ -457,5 +504,7 @@ sql_error select sum(c1) from sliding_tb0 interval(0)  sliding(0);
 sql_error select sum(c1) from sliding_tb0 interval(0m)  sliding(0m);
 sql_error select sum(c1) from sliding_tb0 interval(m)  sliding(m);
 sql_error select sum(c1) from sliding_tb0 sliding(4m);
+sql_error select count(*) from sliding_tb0 interval(1s) sliding(10s);
+sql_error select count(*) from sliding_tb0 interval(10s) sliding(10a);
 
 system sh/exec.sh -n dnode1 -s stop -x SIGINT