Merge branch 'develop' into hotfix/crash

.travis.yml

@@ -45,7 +45,9 @@ matrix:
           cd ${TRAVIS_BUILD_DIR}/debug
           make install > /dev/null || travis_terminate $?
 
+          pip install numpy
           pip install --user ${TRAVIS_BUILD_DIR}/src/connector/python/linux/python2/
+          pip3 install numpy
           pip3 install --user ${TRAVIS_BUILD_DIR}/src/connector/python/linux/python3/
 
           cd ${TRAVIS_BUILD_DIR}/tests
@@ -164,7 +166,9 @@ matrix:
           cd ${TRAVIS_BUILD_DIR}/debug
           make install > /dev/null || travis_terminate $?
 
+          pip install numpy
           pip install --user ${TRAVIS_BUILD_DIR}/src/connector/python/linux/python2/
+          pip3 install numpy
           pip3 install --user ${TRAVIS_BUILD_DIR}/src/connector/python/linux/python3/
 
           cd ${TRAVIS_BUILD_DIR}/tests

src/client/inc/tsclient.h

@@ -286,7 +286,7 @@ typedef struct STscObj {
   char               user[TSDB_USER_LEN];
   char               pass[TSDB_KEY_LEN];
   char               acctId[TSDB_ACCT_LEN];
-  char               db[TSDB_DB_NAME_LEN];
+  char               db[TSDB_ACCT_LEN + TSDB_DB_NAME_LEN];
   char               sversion[TSDB_VERSION_LEN];
   char               writeAuth : 1;
   char               superAuth : 1;

src/client/src/tscAsync.c

@@ -368,7 +368,9 @@ void tscProcessAsyncRes(SSchedMsg *pMsg) {
     pSql->fp = pSql->fetchFp;
   }
 
-  (*pSql->fp)(pSql->param, taosres, code);
+  if (pSql->fp) {
+    (*pSql->fp)(pSql->param, taosres, code);
+  }
 
   if (shouldFree) {
     tscTrace("%p sqlObj is automatically freed in async res", pSql);

src/client/src/tscFunctionImpl.c

@@ -699,7 +699,7 @@ static int32_t first_data_req_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end,
 }
 
 static int32_t last_data_req_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) {
-  if (pCtx->order == TSDB_ORDER_ASC) {
+  if (pCtx->order != pCtx->param[0].i64Key) {
     return BLK_DATA_NO_NEEDED;
   }
   
@@ -727,7 +727,7 @@ static int32_t first_dist_data_req_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY
 }
 
 static int32_t last_dist_data_req_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, int32_t colId) {
-  if (pCtx->order == TSDB_ORDER_ASC) {
+  if (pCtx->order != pCtx->param[0].i64Key) {
     return BLK_DATA_NO_NEEDED;
   }
   
@@ -1593,7 +1593,7 @@ static void first_dist_function_f(SQLFunctionCtx *pCtx, int32_t index) {
   if (pCtx->hasNull && isNull(pData, pCtx->inputType)) {
     return;
   }
-  
+
   if (pCtx->order == TSDB_ORDER_DESC) {
     return;
   }
@@ -1652,7 +1652,7 @@ static void first_dist_func_second_merge(SQLFunctionCtx *pCtx) {
  *    least one data in this block that is not null.(TODO opt for this case)
  */
 static void last_function(SQLFunctionCtx *pCtx) {
-  if (pCtx->order == TSDB_ORDER_ASC) {
+  if (pCtx->order != pCtx->param[0].i64Key) {
     return;
   }
   
@@ -1681,7 +1681,6 @@ static void last_function(SQLFunctionCtx *pCtx) {
 }
 
 static void last_function_f(SQLFunctionCtx *pCtx, int32_t index) {
-  assert(pCtx->order != TSDB_ORDER_ASC);
   void *pData = GET_INPUT_CHAR_INDEX(pCtx, index);
   if (pCtx->hasNull && isNull(pData, pCtx->inputType)) {
     return;
@@ -1725,7 +1724,7 @@ static void last_dist_function(SQLFunctionCtx *pCtx) {
    * 1. for scan data in asc order, no need to check data
    * 2. for data blocks that are not loaded, no need to check data
    */
-  if (pCtx->order == TSDB_ORDER_ASC) {
+  if (pCtx->order != pCtx->param[0].i64Key) {
     return;
   }
   
@@ -1763,7 +1762,7 @@ static void last_dist_function_f(SQLFunctionCtx *pCtx, int32_t index) {
    * 1. for scan data in asc order, no need to check data
    * 2. for data blocks that are not loaded, no need to check data
    */
-  if (pCtx->order == TSDB_ORDER_ASC) {
+  if (pCtx->order != pCtx->param[0].i64Key) {
     return;
   }
   

src/client/src/tscParseInsert.c

@@ -97,7 +97,7 @@ int tsParseTime(SSQLToken *pToken, int64_t *time, char **next, char *error, int1
     useconds = str2int64(pToken->z);
   } else {
     // strptime("2001-11-12 18:31:01", "%Y-%m-%d %H:%M:%S", &tm);
-    if (taosParseTime(pToken->z, time, pToken->n, timePrec) != TSDB_CODE_SUCCESS) {
+    if (taosParseTime(pToken->z, time, pToken->n, timePrec, tsDaylight) != TSDB_CODE_SUCCESS) {
       return tscInvalidSQLErrMsg(error, "invalid timestamp format", pToken->z);
     }
 

src/client/src/tscSQLParser.c

@@ -138,7 +138,7 @@ static int setColumnFilterInfoForTimestamp(SQueryInfo* pQueryInfo, tVariant* pVa
   STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta);
   
   if (seg != NULL) {
-    if (taosParseTime(pVar->pz, &time, pVar->nLen, tinfo.precision) != TSDB_CODE_SUCCESS) {
+    if (taosParseTime(pVar->pz, &time, pVar->nLen, tinfo.precision, tsDaylight) != TSDB_CODE_SUCCESS) {
       return invalidSqlErrMsg(pQueryInfo->msg, msg);
     }
   } else {
@@ -1042,7 +1042,7 @@ int32_t setObjFullName(char* fullName, const char* account, SSQLToken* pDB, SSQL
 
   /* db name is not specified, the tableName dose not include db name */
   if (pDB != NULL) {
-    if (pDB->n >= TSDB_DB_NAME_LEN) {
+    if (pDB->n >= TSDB_ACCT_LEN + TSDB_DB_NAME_LEN) {
       return TSDB_CODE_TSC_INVALID_SQL;
     }
 
@@ -1452,6 +1452,13 @@ static int32_t setExprInfoForFunctions(SQueryInfo* pQueryInfo, SSchema* pSchema,
   
   SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, functionID, pColIndex, type, bytes, bytes, false);
   tstrncpy(pExpr->aliasName, columnName, sizeof(pExpr->aliasName));
+
+  // set reverse order scan data blocks for last query
+  if (functionID == TSDB_FUNC_LAST) {
+    pExpr->numOfParams = 1;
+    pExpr->param[0].i64Key = TSDB_ORDER_DESC;
+    pExpr->param[0].nType = TSDB_DATA_TYPE_INT;
+  }
   
   // for all queries, the timestamp column needs to be loaded
   SColumnIndex index = {.tableIndex = pColIndex->tableIndex, .columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX};
@@ -1724,6 +1731,22 @@ int32_t addExprAndResultField(SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExpr
             if (setExprInfoForFunctions(pQueryInfo, pSchema, functionID, pItem->aliasName, colIndex + i, &index) != 0) {
               return TSDB_CODE_TSC_INVALID_SQL;
             }
+
+            if (optr == TK_LAST) {  // todo refactor
+              SSqlGroupbyExpr* pGroupBy = &pQueryInfo->groupbyExpr;
+              if (pGroupBy->numOfGroupCols > 0) {
+                for(int32_t k = 0; k < pGroupBy->numOfGroupCols; ++k) {
+                  SColIndex* pIndex = taosArrayGet(pGroupBy->columnInfo, k);
+                  if (!TSDB_COL_IS_TAG(pIndex->flag) && pIndex->colIndex < tscGetNumOfColumns(pTableMetaInfo->pTableMeta)) { // group by normal columns
+                    SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprList, colIndex + i);
+                    pExpr->numOfParams = 1;
+                    pExpr->param->i64Key = TSDB_ORDER_ASC;
+
+                    break;
+                  }
+                }
+              }
+            }
           }
         }
         
@@ -1990,6 +2013,7 @@ static int16_t doGetColumnIndex(SQueryInfo* pQueryInfo, int32_t index, SSQLToken
 
     if (strncasecmp(pSchema[i].name, pToken->z, pToken->n) == 0) {
       columnIndex = i;
+      break;
     }
   }
 
@@ -2586,9 +2610,7 @@ int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd*
 
       tscColumnListInsert(pQueryInfo->colList, &index);
       
-      SColIndex colIndex = {
-          .colIndex = index.columnIndex, .flag = TSDB_COL_NORMAL, .colId = pSchema->colId,
-      };
+      SColIndex colIndex = { .colIndex = index.columnIndex, .flag = TSDB_COL_NORMAL, .colId = pSchema->colId };
       taosArrayPush(pGroupExpr->columnInfo, &colIndex);
       pQueryInfo->groupbyExpr.orderType = TSDB_ORDER_ASC;
 
@@ -2886,7 +2908,8 @@ static int32_t extractColumnFilterInfo(SQueryInfo* pQueryInfo, SColumnIndex* pIn
   SSchema*    pSchema = tscGetTableColumnSchema(pTableMeta, pIndex->columnIndex);
 
   const char* msg1 = "non binary column not support like operator";
-  const char* msg2 = "binary column not support this operator";
+  const char* msg2 = "binary column not support this operator";  
+  const char* msg3 = "bool column not support this operator";
 
   SColumn* pColumn = tscColumnListInsert(pQueryInfo->colList, pIndex);
   SColumnFilterInfo* pColFilter = NULL;
@@ -2920,6 +2943,12 @@ static int32_t extractColumnFilterInfo(SQueryInfo* pQueryInfo, SColumnIndex* pIn
     if (pExpr->nSQLOptr == TK_LIKE) {
       return invalidSqlErrMsg(pQueryInfo->msg, msg1);
     }
+    
+    if (pSchema->type == TSDB_DATA_TYPE_BOOL) {
+      if (pExpr->nSQLOptr != TK_EQ && pExpr->nSQLOptr != TK_NE) {
+        return invalidSqlErrMsg(pQueryInfo->msg, msg3);
+      }
+    }
   }
 
   pColumn->colIndex = *pIndex;
@@ -3921,7 +3950,7 @@ int32_t getTimeRange(STimeWindow* win, tSQLExpr* pRight, int32_t optr, int16_t t
 
     char* seg = strnchr(pRight->val.pz, '-', pRight->val.nLen, false);
     if (seg != NULL) {
-      if (taosParseTime(pRight->val.pz, &val, pRight->val.nLen, TSDB_TIME_PRECISION_MICRO) == TSDB_CODE_SUCCESS) {
+      if (taosParseTime(pRight->val.pz, &val, pRight->val.nLen, TSDB_TIME_PRECISION_MICRO, tsDaylight) == TSDB_CODE_SUCCESS) {
         parsed = true;
       } else {
         return TSDB_CODE_TSC_INVALID_SQL;
@@ -4479,7 +4508,7 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
     if (pTagsSchema->type != TSDB_DATA_TYPE_BINARY && pTagsSchema->type != TSDB_DATA_TYPE_NCHAR) {
       len = tDataTypeDesc[pTagsSchema->type].nSize;
     } else {
-      len = varDataLen(pUpdateMsg->data);
+      len = varDataTLen(pUpdateMsg->data);
     }
     
     pUpdateMsg->tagValLen = htonl(len);  // length may be changed after dump data
@@ -6052,6 +6081,16 @@ int32_t exprTreeFromSqlExpr(tExprNode **pExpr, const tSQLExpr* pSqlExpr, SArray*
         }
       }
     }
+
+    if ((*pExpr)->_node.optr != TSDB_RELATION_EQUAL && (*pExpr)->_node.optr != TSDB_RELATION_NOT_EQUAL) {
+      if (pRight->nodeType == TSQL_NODE_VALUE) {
+        if (  pRight->pVal->nType == TSDB_DATA_TYPE_BOOL 
+           || pRight->pVal->nType == TSDB_DATA_TYPE_BINARY 
+           || pRight->pVal->nType == TSDB_DATA_TYPE_NCHAR) {
+          return TSDB_CODE_TSC_INVALID_SQL;
+        }
+      }
+    }
   }
   
   return TSDB_CODE_SUCCESS;

src/client/src/tscServer.c

@@ -430,7 +430,7 @@ void tscKillSTableQuery(SSqlObj *pSql) {
   /*
    * 1. if the subqueries are not launched or partially launched, we need to waiting the launched
    * query return to successfully free allocated resources.
-   * 2. if no any subqueries are launched yet, which means the metric query only in parse sql stage,
+   * 2. if no any subqueries are launched yet, which means the super table query only in parse sql stage,
    * set the res.code, and return.
    */
   const int64_t MAX_WAITING_TIME = 10000;  // 10 Sec.
@@ -2201,7 +2201,7 @@ int tscProcessDropTableRsp(SSqlObj *pSql) {
    * The cached information is expired, however, we may have lost the ref of original meter. So, clear whole cache
    * instead.
    */
-  tscTrace("%p force release metermeta after drop table:%s", pSql, pTableMetaInfo->name);
+  tscTrace("%p force release table meta after drop table:%s", pSql, pTableMetaInfo->name);
   taosCacheRelease(tscCacheHandle, (void **)&pTableMeta, true);
 
   if (pTableMetaInfo->pTableMeta) {

src/client/src/tscUtil.c

@@ -1451,8 +1451,6 @@ bool tscShouldBeFreed(SSqlObj* pSql) {
     return false;
   }
   
-  assert(pSql->fp != NULL);
-  
   STscObj* pTscObj = pSql->pTscObj;
   if (pSql->pStream != NULL || pTscObj->pHb == pSql || pSql->pSubscription != NULL) {
     return false;

src/common/inc/tglobal.h

@@ -170,6 +170,7 @@ extern char gitinfo[];
 extern char gitinfoOfInternal[];
 extern char buildinfo[];
 
+extern int8_t tsDaylight;
 extern char tsTimezone[64];
 extern char tsLocale[64];
 extern char tsCharset[64];  // default encode string

src/common/src/tglobal.c

@@ -198,6 +198,7 @@ char tsMonitorDbName[TSDB_DB_NAME_LEN] = "log";
 char tsInternalPass[] = "secretkey";
 int32_t tsMonitorInterval = 30;  // seconds
 
+int8_t tsDaylight = 0;
 char tsTimezone[64] = {0};
 char tsLocale[TSDB_LOCALE_LEN] = {0};
 char tsCharset[TSDB_LOCALE_LEN] = {0};  // default encode string

src/common/src/ttimezone.c

@@ -58,6 +58,7 @@ void tsSetTimeZone() {
   * (BST, +0100)
   */
   sprintf(tsTimezone, "(%s, %s%02d00)", tzname[daylight], tz >= 0 ? "+" : "-", abs(tz));
+  tsDaylight = daylight;
 
   uPrint("timezone format changed to %s", tsTimezone);
 }

src/inc/query.h

@@ -70,7 +70,8 @@ int32_t qRetrieveQueryResultInfo(qinfo_t qinfo);
 int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp** pRsp, int32_t* contLen);
 
 /**
- * Decide if more results will be produced or not
+ * Decide if more results will be produced or not, NOTE: this function will increase the ref count of QInfo,
+ * so it can be only called once for each retrieve
  *
  * @param qinfo
  * @return

src/inc/taosmsg.h

@@ -250,7 +250,7 @@ typedef struct {
 
 typedef struct {
   char    tableId[TSDB_TABLE_ID_LEN];
-  char    db[TSDB_DB_NAME_LEN];
+  char    db[TSDB_ACCT_LEN + TSDB_DB_NAME_LEN];
   int8_t  igExists;
   int8_t  getMeta;
   int16_t numOfTags;
@@ -268,7 +268,7 @@ typedef struct {
 
 typedef struct {
   char    tableId[TSDB_TABLE_ID_LEN];
-  char    db[TSDB_DB_NAME_LEN];
+  char    db[TSDB_ACCT_LEN + TSDB_DB_NAME_LEN];
   int16_t type; /* operation type   */
   int16_t numOfCols; /* number of schema */
   int32_t tagValLen;
@@ -670,7 +670,7 @@ typedef struct {
  */
 typedef struct {
   int8_t   type;
-  char     db[TSDB_DB_NAME_LEN];
+  char     db[TSDB_ACCT_LEN + TSDB_DB_NAME_LEN];
   uint16_t payloadLen;
   char     payload[];
 } SCMShowMsg;

src/inc/trpc.h

@@ -78,12 +78,13 @@ void  rpcClose(void *);
 void *rpcMallocCont(int contLen);
 void  rpcFreeCont(void *pCont);
 void *rpcReallocCont(void *ptr, int contLen);
-void  rpcSendRequest(void *thandle, const SRpcIpSet *pIpSet, const SRpcMsg *pMsg);
+void *rpcSendRequest(void *thandle, const SRpcIpSet *pIpSet, const SRpcMsg *pMsg);
 void  rpcSendResponse(const SRpcMsg *pMsg);
 void  rpcSendRedirectRsp(void *pConn, const SRpcIpSet *pIpSet); 
 int   rpcGetConnInfo(void *thandle, SRpcConnInfo *pInfo);
 void  rpcSendRecv(void *shandle, SRpcIpSet *pIpSet, const SRpcMsg *pReq, SRpcMsg *pRsp);
 int   rpcReportProgress(void *pConn, char *pCont, int contLen);
+void  rpcCanelRequest(void *pContext);
 
 #ifdef __cplusplus
 }

src/kit/shell/src/shellEngine.c

@@ -469,7 +469,6 @@ static int dumpResultToFile(const char* fname, TAOS_RES* result) {
   } while( row != NULL);
 
   fclose(fp);
-  taos_free_result(result);
   return numOfRows;
 }
 

src/kit/taosdemo/taosdemo.c

@@ -297,7 +297,7 @@ void *deleteTable();
 
 void *asyncWrite(void *sarg);
 
-void generateData(char *res, char **data_type, int num_of_cols, int64_t timestamp, int len_of_binary);
+int generateData(char *res, char **data_type, int num_of_cols, int64_t timestamp, int len_of_binary);
 
 void rand_string(char *str, int size);
 
@@ -817,7 +817,7 @@ void queryDB(TAOS *taos, char *command) {
     i--; 
   }
 
-  if (i == 0) {
+  if (code != 0) {
     fprintf(stderr, "Failed to run %s, reason: %s\n", command, taos_errstr(pSql));
     taos_free_result(pSql);
 
@@ -846,14 +846,19 @@ void *syncWrite(void *sarg) {
       int k;
       for (k = 0; k < winfo->nrecords_per_request;) {
         int rand_num = rand() % 100;
-        if (winfo->data_of_order ==1 && rand_num < winfo->data_of_rate)
-        {
+        int len = -1;
+        if (winfo->data_of_order ==1 && rand_num < winfo->data_of_rate) {
           long d = tmp_time - rand() % 1000000 + rand_num;
-          generateData(data, data_type, ncols_per_record, d, len_of_binary);
-        } else 
-        {
-          generateData(data, data_type, ncols_per_record, tmp_time += 1000, len_of_binary);
+          len = generateData(data, data_type, ncols_per_record, d, len_of_binary);
+        } else {
+          len = generateData(data, data_type, ncols_per_record, tmp_time += 1000, len_of_binary);
         }
+
+        //assert(len + pstr - buffer < BUFFER_SIZE);
+        if (len + pstr - buffer >= BUFFER_SIZE) { // too long
+          break;
+        }
+
         pstr += sprintf(pstr, " %s", data);
         inserted++;
         k++;
@@ -968,7 +973,7 @@ double getCurrentTime() {
   return tv.tv_sec + tv.tv_usec / 1E6;
 }
 
-void generateData(char *res, char **data_type, int num_of_cols, int64_t timestamp, int len_of_binary) {
+int32_t generateData(char *res, char **data_type, int num_of_cols, int64_t timestamp, int len_of_binary) {
   memset(res, 0, MAX_DATA_SIZE);
   char *pstr = res;
   pstr += sprintf(pstr, "(%" PRId64, timestamp);
@@ -1002,9 +1007,16 @@ void generateData(char *res, char **data_type, int num_of_cols, int64_t timestam
       rand_string(s, len_of_binary);
       pstr += sprintf(pstr, ", \"%s\"", s);
     }
+
+    if (pstr - res > MAX_DATA_SIZE) {
+      perror("column length too long, abort");
+      exit(-1);
+    }
   }
 
   pstr += sprintf(pstr, ")");
+
+  return pstr - res;
 }
 
 static const char charset[] = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJK1234567890";

src/mnode/src/mnodeDb.c

@@ -84,9 +84,12 @@ static int32_t mnodeDbActionDelete(SSdbOper *pOper) {
   mnodeDropAllChildTables(pDb);
   mnodeDropAllSuperTables(pDb);
   mnodeDropAllDbVgroups(pDb);
-  mnodeDropDbFromAcct(pAcct, pDb);
-  mnodeDecAcctRef(pAcct);
-  
+
+  if (pAcct) {
+    mnodeDropDbFromAcct(pAcct, pDb);
+    mnodeDecAcctRef(pAcct);
+  }
+
   return TSDB_CODE_SUCCESS;
 }
 

src/mnode/src/mnodeProfile.c

@@ -108,8 +108,8 @@ SConnObj *mnodeCreateConn(char *user, uint32_t ip, uint16_t port) {
 }
 
 void mnodeReleaseConn(SConnObj *pConn) {
-  if(pConn == NULL) return;
-  taosCacheRelease(tsMnodeConnCache, (void**)&pConn, false);
+  if (pConn == NULL) return;
+  taosCacheRelease(tsMnodeConnCache, (void **)&pConn, false);
 }
 
 SConnObj *mnodeAccquireConn(uint32_t connId, char *user, uint32_t ip, uint16_t port) {
@@ -138,7 +138,7 @@ SConnObj *mnodeAccquireConn(uint32_t connId, char *user, uint32_t ip, uint16_t p
 static void mnodeFreeConn(void *data) {
   SConnObj *pConn = data;
   tfree(pConn->pQueries);
-  tfree(pConn->pQueries);
+  tfree(pConn->pStreams);
 
   mTrace("connId:%d, is destroyed", pConn->connId);
 }

src/mnode/src/mnodeTable.c

@@ -157,10 +157,12 @@ static int32_t mnodeChildTableActionDelete(SSdbOper *pOper) {
   if (pDb != NULL) pAcct = mnodeGetAcct(pDb->acct);
 
   if (pTable->info.type == TSDB_CHILD_TABLE) {
-    grantRestore(TSDB_GRANT_TIMESERIES, pTable->superTable->numOfColumns - 1);
-    if (pAcct != NULL) pAcct->acctInfo.numOfTimeSeries -= (pTable->superTable->numOfColumns - 1);
-    mnodeRemoveTableFromStable(pTable->superTable, pTable);
-    mnodeDecTableRef(pTable->superTable);
+    if (pTable->superTable) {
+      grantRestore(TSDB_GRANT_TIMESERIES, pTable->superTable->numOfColumns - 1);
+      if (pAcct != NULL) pAcct->acctInfo.numOfTimeSeries -= (pTable->superTable->numOfColumns - 1);
+      mnodeRemoveTableFromStable(pTable->superTable, pTable);
+      mnodeDecTableRef(pTable->superTable);
+    }
   } else {
     grantRestore(TSDB_GRANT_TIMESERIES, pTable->numOfColumns - 1);
     if (pAcct != NULL) pAcct->acctInfo.numOfTimeSeries -= (pTable->numOfColumns - 1);
@@ -1015,7 +1017,7 @@ static int32_t mnodeAddSuperTableColumn(SMnodeMsg *pMsg, SSuperTableObj *pStable
           sizeof(SSchema) * pStable->numOfTags);
   memcpy(pStable->schema + pStable->numOfColumns, schema, sizeof(SSchema) * ncols);
 
-  SSchema *tschema = (SSchema *) (pStable->schema + sizeof(SSchema) * pStable->numOfColumns);
+  SSchema *tschema = (SSchema *) (pStable->schema + pStable->numOfColumns);
   for (int32_t i = 0; i < ncols; i++) {
     tschema[i].colId = pStable->nextColId++;
   }

src/plugins/http/inc/gcHandle.h

@@ -16,15 +16,11 @@
 #ifndef TDENGINE_GC_HANDLE_H
 #define TDENGINE_GC_HANDLE_H
 
-#include <stdbool.h>
-#include <stdint.h>
-#include <string.h>
-#include <unistd.h>
-
 #include "http.h"
-#include "httpCode.h"
-#include "httpHandle.h"
+#include "httpInt.h"
+#include "httpUtil.h"
 #include "httpResp.h"
+#include "httpSql.h"
 
 #define GC_ROOT_URL_POS   0
 #define GC_ACTION_URL_POS 1

src/plugins/http/inc/httpAuth.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_HTTP_TOKEN_H
+#define TDENGINE_HTTP_TOKEN_H
+
+bool httpParseBasicAuthToken(HttpContext *pContext, char *token, int len);
+bool httpParseTaosdAuthToken(HttpContext *pContext, char *token, int len);
+bool httpGenTaosdAuthToken(HttpContext *pContext, char *token, int maxLen);
+
+#endif
\ No newline at end of file

src/plugins/http/inc/httpContext.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_HTTP_CONTEXT_H
+#define TDENGINE_HTTP_CONTEXT_H
+
+#include "httpInt.h"
+
+bool        httpInitContexts();
+void        httpCleanupContexts();
+const char *httpContextStateStr(HttpContextState state);
+
+HttpContext *httpCreateContext(int32_t fd);
+bool         httpInitContext(HttpContext *pContext);
+HttpContext *httpGetContext(void * pContext);
+void         httpReleaseContext(HttpContext *pContext);
+void         httpCloseContextByServer(HttpContext *pContext);
+void         httpCloseContextByApp(HttpContext *pContext);
+void         httpNotifyContextClose(HttpContext *pContext);
+bool         httpAlterContextState(HttpContext *pContext, HttpContextState srcState, HttpContextState destState);
+
+#endif

src/plugins/http/inc/httpHandle.h

@@ -13,304 +13,11 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef TDENGINE_HTTP_SERVER_H
-#define TDENGINE_HTTP_SERVER_H
-
-#include <stdbool.h>
-#include "pthread.h"
-#include "semaphore.h"
-#include "tmempool.h"
-#include "taosdef.h"
-#include "tutil.h"
-#include "zlib.h"
-#include "http.h"
-#include "httpJson.h"
-
-#define HTTP_MAX_CMD_SIZE           1024
-#define HTTP_MAX_BUFFER_SIZE        1024*1024
-
-#define HTTP_LABEL_SIZE             8
-#define HTTP_MAX_EVENTS             10
-#define HTTP_BUFFER_SIZE            1024*65 //65k
-#define HTTP_DECOMPRESS_BUF_SIZE    1024*64
-#define HTTP_STEP_SIZE              1024    //http message get process step by step
-#define HTTP_MAX_URL                5       //http url stack size
-#define HTTP_METHOD_SCANNER_SIZE    7       //http method fp size
-#define HTTP_GC_TARGET_SIZE         512
-
-#define HTTP_VERSION_10             0
-#define HTTP_VERSION_11             1
-//#define HTTP_VERSION_12           2
-
-#define HTTP_UNCUNKED               0
-#define HTTP_CHUNKED                1
-
-#define HTTP_KEEPALIVE_NO_INPUT     0
-#define HTTP_KEEPALIVE_ENABLE       1
-#define HTTP_KEEPALIVE_DISABLE      2
-
-#define HTTP_REQTYPE_OTHERS         0
-#define HTTP_REQTYPE_LOGIN          1
-#define HTTP_REQTYPE_HEARTBEAT      2
-#define HTTP_REQTYPE_SINGLE_SQL     3
-#define HTTP_REQTYPE_MULTI_SQL      4
-
-#define HTTP_CHECK_BODY_ERROR      -1
-#define HTTP_CHECK_BODY_CONTINUE    0
-#define HTTP_CHECK_BODY_SUCCESS     1
-
-#define HTTP_WRITE_RETRY_TIMES      500
-#define HTTP_WRITE_WAIT_TIME_MS     5
-#define HTTP_EXPIRED_TIME           60000
-#define HTTP_DELAY_CLOSE_TIME_MS    500
-
-#define HTTP_COMPRESS_IDENTITY      0
-#define HTTP_COMPRESS_GZIP          2
-
-#define HTTP_SESSION_ID_LEN         (TSDB_USER_LEN + TSDB_PASSWORD_LEN)
-
-typedef enum {
-    HTTP_CONTEXT_STATE_READY,
-    HTTP_CONTEXT_STATE_HANDLING,
-    HTTP_CONTEXT_STATE_DROPPING,
-    HTTP_CONTEXT_STATE_CLOSED
-} HttpContextState;
-
-struct HttpContext;
-struct HttpThread;
-
-typedef struct {
-  void *signature;
-  int   expire;
-  int   access;
-  void *taos;
-  char  id[HTTP_SESSION_ID_LEN];
-} HttpSession;
-
-typedef enum {
-  HTTP_CMD_TYPE_UN_SPECIFIED,
-  HTTP_CMD_TYPE_CREATE_DB,
-  HTTP_CMD_TYPE_CREATE_STBALE,
-  HTTP_CMD_TYPE_INSERT
-} HttpSqlCmdType;
-
-typedef enum { HTTP_CMD_STATE_NOT_RUN_YET, HTTP_CMD_STATE_RUN_FINISHED } HttpSqlCmdState;
-
-typedef enum { HTTP_CMD_RETURN_TYPE_WITH_RETURN, HTTP_CMD_RETURN_TYPE_NO_RETURN } HttpSqlCmdReturnType;
-
-typedef struct {
-  // used by single cmd
-  char   *nativSql;
-  int32_t numOfRows;
-  int32_t code;
-
-  // these are the locations in the buffer
-  int32_t tagNames[TSDB_MAX_TAGS];
-  int32_t tagValues[TSDB_MAX_TAGS];
-  int32_t timestamp;
-  int32_t metric;
-  int32_t stable;
-  int32_t table;
-  int32_t values;
-  int32_t sql;
-
-  // used by multi-cmd
-  int8_t cmdType;
-  int8_t cmdReturnType;
-  int8_t cmdState;
-  int8_t tagNum;
-} HttpSqlCmd;
-
-typedef struct {
-  HttpSqlCmd *cmds;
-  int16_t     pos;
-  int16_t     size;
-  int16_t     maxSize;
-  int32_t     bufferPos;
-  int32_t     bufferSize;
-  char *      buffer;
-} HttpSqlCmds;
-
-typedef struct {
-  char *module;
-  bool (*decodeFp)(struct HttpContext *pContext);
-} HttpDecodeMethod;
-
-typedef struct {
-  void (*startJsonFp)(struct HttpContext *pContext, HttpSqlCmd *cmd, void *result);
-  void (*stopJsonFp)(struct HttpContext *pContext, HttpSqlCmd *cmd);
-  bool (*buildQueryJsonFp)(struct HttpContext *pContext, HttpSqlCmd *cmd, void *result, int numOfRows);
-  void (*buildAffectRowJsonFp)(struct HttpContext *pContext, HttpSqlCmd *cmd, int affectRows);
-  void (*initJsonFp)(struct HttpContext *pContext);
-  void (*cleanJsonFp)(struct HttpContext *pContext);
-  bool (*checkFinishedFp)(struct HttpContext *pContext, HttpSqlCmd *cmd, int code);
-  void (*setNextCmdFp)(struct HttpContext *pContext, HttpSqlCmd *cmd, int code);
-} HttpEncodeMethod;
-
-typedef struct {
-  char   *pos;
-  int32_t len;
-} HttpBuf;
-
-typedef struct {
-  char              buffer[HTTP_BUFFER_SIZE];
-  int               bufsize;
-  char             *pLast;
-  char             *pCur;
-  HttpBuf           method;
-  HttpBuf           path[HTTP_MAX_URL];  // url: dbname/meter/query
-  HttpBuf           data;                // body content
-  HttpBuf           token;               // auth token
-  HttpDecodeMethod *pMethod;
-} HttpParser;
-
-typedef struct HttpContext {
-  void *       signature;
-  int          fd;
-  uint32_t     accessTimes;
-  uint32_t     lastAccessTime;
-  uint8_t      httpVersion;
-  uint8_t      httpChunked;
-  uint8_t      httpKeepAlive;  // http1.0 and not keep-alive, close connection immediately
-  uint8_t      fromMemPool;
-  uint8_t      acceptEncoding;
-  uint8_t      contentEncoding;
-  uint8_t      reqType;
-  uint8_t      parsed;
-  int32_t      state;
-  char         ipstr[22];
-  char         user[TSDB_USER_LEN];  // parsed from auth token or login message
-  char         pass[TSDB_PASSWORD_LEN];
-  void        *taos;
-  HttpSession *session;
-  z_stream     gzipStream;
-  HttpEncodeMethod   *encodeMethod;
-  HttpSqlCmd          singleCmd;
-  HttpSqlCmds        *multiCmds;
-  JsonBuf            *jsonBuf;
-  HttpParser          parser;
-  void               *timer;
-  struct HttpThread  *pThread;
-  struct HttpContext *prev;
-  struct HttpContext *next;
-} HttpContext;
-
-typedef struct HttpThread {
-  pthread_t       thread;
-  HttpContext *   pHead;
-  pthread_mutex_t threadMutex;
-  bool            stop;
-  int             pollFd;
-  int             numOfFds;
-  int             threadId;
-  char            label[HTTP_LABEL_SIZE];
-  bool (*processData)(HttpContext *pContext);
-  struct HttpServer *pServer;  // handle passed by upper layer during pServer initialization
-} HttpThread;
-
-typedef struct HttpServer {
-  char              label[HTTP_LABEL_SIZE];
-  uint32_t          serverIp;
-  uint16_t          serverPort;
-  bool              online;
-  int               fd;
-  int               cacheContext;
-  int               sessionExpire;
-  int               numOfThreads;
-  HttpDecodeMethod *methodScanner[HTTP_METHOD_SCANNER_SIZE];
-  int               methodScannerLen;
-  pthread_mutex_t   serverMutex;
-  void             *pSessionHash;
-  void             *pContextPool;
-  void             *expireTimer;
-  HttpThread       *pThreads;
-  pthread_t         thread;
-  bool            (*processData)(HttpContext *pContext);
-  int               requestNum;
-  void             *timerHandle;
-} HttpServer;
-
-// http util method
-bool httpCheckUsedbSql(char *sql);
-void httpTimeToString(time_t t, char *buf, int buflen);
-
-// http init method
-void *httpInitServer(char *ip, uint16_t port, char *label, int numOfThreads, void *fp, void *shandle);
-void httpCleanUpServer(HttpServer *pServer);
-
-// http server connection
-void httpCleanUpConnect(HttpServer *pServer);
-bool httpInitConnect(HttpServer *pServer);
-
-// http context for each client connection
-HttpContext *httpCreateContext(HttpServer *pServer);
-bool httpInitContext(HttpContext *pContext);
-void httpCloseContextByApp(HttpContext *pContext);
-void httpCloseContextByServer(HttpThread *pThread, HttpContext *pContext);
-
-// http session method
-void httpCreateSession(HttpContext *pContext, void *taos);
-void httpAccessSession(HttpContext *pContext);
-void httpFetchSession(HttpContext *pContext);
-void httpRestoreSession(HttpContext *pContext);
-void httpRemoveExpireSessions(HttpServer *pServer);
-bool httpInitAllSessions(HttpServer *pServer);
-void httpRemoveAllSessions(HttpServer *pServer);
-void httpProcessSessionExpire(void *handle, void *tmrId);
-
-// http request parser
-void httpAddMethod(HttpServer *pServer, HttpDecodeMethod *pMethod);
-
-// http token method
-bool httpParseBasicAuthToken(HttpContext *pContext, char *token, int len);
-bool httpParseTaosdAuthToken(HttpContext *pContext, char *token, int len);
-bool httpGenTaosdAuthToken(HttpContext *pContext, char *token, int maxLen);
-
-// util
-bool httpUrlMatch(HttpContext *pContext, int pos, char *cmp);
-bool httpProcessData(HttpContext *pContext);
-bool httpReadDataImp(HttpContext *pContext);
-bool httpParseRequest(HttpContext* pContext);
-int  httpCheckReadCompleted(HttpContext* pContext);
-void httpReadDirtyData(HttpContext *pContext);
+#ifndef TDENGINE_HTTP_HANDLE_H
+#define TDENGINE_HTTP_HANDLE_H
 
 // http request handler
 void httpProcessRequest(HttpContext *pContext);
-
-// http json printer
-JsonBuf *httpMallocJsonBuf(HttpContext *pContext);
-void httpFreeJsonBuf(HttpContext *pContext);
-
-// http multicmds util
-
-int32_t httpAddToSqlCmdBuffer(HttpContext *pContext, const char *const format, ...);
-int32_t httpAddToSqlCmdBufferNoTerminal(HttpContext *pContext, const char *const format, ...);
-int32_t httpAddToSqlCmdBufferWithSize(HttpContext *pContext, int mallocSize);
-int32_t httpAddToSqlCmdBufferTerminal(HttpContext *pContext);
-
-bool httpMallocMultiCmds(HttpContext *pContext, int cmdSize, int bufferSize);
-bool httpReMallocMultiCmdsSize(HttpContext *pContext, int cmdSize);
-bool httpReMallocMultiCmdsBuffer(HttpContext *pContext, int bufferSize);
-void httpFreeMultiCmds(HttpContext *pContext);
-
-HttpSqlCmd *httpNewSqlCmd(HttpContext *pContext);
-HttpSqlCmd *httpCurrSqlCmd(HttpContext *pContext);
-int httpCurSqlCmdPos(HttpContext *pContext);
-
-void httpTrimTableName(char *name);
-int httpShrinkTableName(HttpContext *pContext, int pos, char *name);
-char *httpGetCmdsString(HttpContext *pContext, int pos);
-
-int httpGzipDeCompress(char *srcData, int32_t nSrcData, char *destData, int32_t *nDestData);
-int httpGzipCompressInit(HttpContext *pContext);
-int httpGzipCompress(HttpContext *pContext, char *inSrcData, int32_t inSrcDataLen,
-                     char *outDestData, int32_t *outDestDataLen, bool isTheLast);
-
-extern const char *httpKeepAliveStr[];
-extern const char *httpVersionStr[];
-const char* httpContextStateStr(HttpContextState state);
-
-bool httpAlterContextState(HttpContext *pContext, HttpContextState srcState, HttpContextState destState);
-void httpRemoveContextFromEpoll(HttpThread *pThread, HttpContext *pContext);
+bool httpProcessData(HttpContext *pContext);
 
 #endif

src/plugins/http/inc/httpInt.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_HTTP_INT_H
+#define TDENGINE_HTTP_INT_H
+
+#include <stdbool.h>
+#include "pthread.h"
+#include "semaphore.h"
+#include "tmempool.h"
+#include "taosdef.h"
+#include "tutil.h"
+#include "zlib.h"
+#include "http.h"
+#include "httpCode.h"
+#include "httpLog.h"
+#include "httpJson.h"
+
+#define HTTP_MAX_CMD_SIZE           1024
+#define HTTP_MAX_BUFFER_SIZE        1024*1024
+
+#define HTTP_LABEL_SIZE             8
+#define HTTP_MAX_EVENTS             10
+#define HTTP_BUFFER_SIZE            1024*65 //65k
+#define HTTP_DECOMPRESS_BUF_SIZE    1024*64
+#define HTTP_STEP_SIZE              1024    //http message get process step by step
+#define HTTP_MAX_URL                5       //http url stack size
+#define HTTP_METHOD_SCANNER_SIZE    7       //http method fp size
+#define HTTP_GC_TARGET_SIZE         512
+
+#define HTTP_VERSION_10             0
+#define HTTP_VERSION_11             1
+//#define HTTP_VERSION_12           2
+
+#define HTTP_UNCUNKED               0
+#define HTTP_CHUNKED                1
+
+#define HTTP_KEEPALIVE_NO_INPUT     0
+#define HTTP_KEEPALIVE_ENABLE       1
+#define HTTP_KEEPALIVE_DISABLE      2
+
+#define HTTP_REQTYPE_OTHERS         0
+#define HTTP_REQTYPE_LOGIN          1
+#define HTTP_REQTYPE_HEARTBEAT      2
+#define HTTP_REQTYPE_SINGLE_SQL     3
+#define HTTP_REQTYPE_MULTI_SQL      4
+
+#define HTTP_CHECK_BODY_ERROR      -1
+#define HTTP_CHECK_BODY_CONTINUE    0
+#define HTTP_CHECK_BODY_SUCCESS     1
+
+#define HTTP_WRITE_RETRY_TIMES      500
+#define HTTP_WRITE_WAIT_TIME_MS     5
+#define HTTP_EXPIRED_TIME           60000
+#define HTTP_DELAY_CLOSE_TIME_MS    500
+
+#define HTTP_COMPRESS_IDENTITY      0
+#define HTTP_COMPRESS_GZIP          2
+
+#define HTTP_SESSION_ID_LEN         (TSDB_USER_LEN + TSDB_PASSWORD_LEN)
+
+typedef enum {
+  HTTP_SERVER_INIT,
+  HTTP_SERVER_RUNNING,
+  HTTP_SERVER_CLOSING,
+  HTTP_SERVER_CLOSED
+} HttpServerStatus;
+
+typedef enum {
+    HTTP_CONTEXT_STATE_READY,
+    HTTP_CONTEXT_STATE_HANDLING,
+    HTTP_CONTEXT_STATE_DROPPING,
+    HTTP_CONTEXT_STATE_CLOSED
+} HttpContextState;
+
+struct HttpContext;
+struct HttpThread;
+
+typedef struct {
+  char  id[HTTP_SESSION_ID_LEN];
+  int   refCount;
+  void *taos;
+} HttpSession;
+
+typedef enum {
+  HTTP_CMD_TYPE_UN_SPECIFIED,
+  HTTP_CMD_TYPE_CREATE_DB,
+  HTTP_CMD_TYPE_CREATE_STBALE,
+  HTTP_CMD_TYPE_INSERT
+} HttpSqlCmdType;
+
+typedef enum { HTTP_CMD_STATE_NOT_RUN_YET, HTTP_CMD_STATE_RUN_FINISHED } HttpSqlCmdState;
+
+typedef enum { HTTP_CMD_RETURN_TYPE_WITH_RETURN, HTTP_CMD_RETURN_TYPE_NO_RETURN } HttpSqlCmdReturnType;
+
+typedef struct {
+  // used by single cmd
+  char   *nativSql;
+  int32_t numOfRows;
+  int32_t code;
+
+  // these are the locations in the buffer
+  int32_t tagNames[TSDB_MAX_TAGS];
+  int32_t tagValues[TSDB_MAX_TAGS];
+  int32_t timestamp;
+  int32_t metric;
+  int32_t stable;
+  int32_t table;
+  int32_t values;
+  int32_t sql;
+
+  // used by multi-cmd
+  int8_t cmdType;
+  int8_t cmdReturnType;
+  int8_t cmdState;
+  int8_t tagNum;
+} HttpSqlCmd;
+
+typedef struct {
+  HttpSqlCmd *cmds;
+  int16_t     pos;
+  int16_t     size;
+  int16_t     maxSize;
+  int32_t     bufferPos;
+  int32_t     bufferSize;
+  char *      buffer;
+} HttpSqlCmds;
+
+typedef struct {
+  char *module;
+  bool (*decodeFp)(struct HttpContext *pContext);
+} HttpDecodeMethod;
+
+typedef struct {
+  void (*startJsonFp)(struct HttpContext *pContext, HttpSqlCmd *cmd, void *result);
+  void (*stopJsonFp)(struct HttpContext *pContext, HttpSqlCmd *cmd);
+  bool (*buildQueryJsonFp)(struct HttpContext *pContext, HttpSqlCmd *cmd, void *result, int numOfRows);
+  void (*buildAffectRowJsonFp)(struct HttpContext *pContext, HttpSqlCmd *cmd, int affectRows);
+  void (*initJsonFp)(struct HttpContext *pContext);
+  void (*cleanJsonFp)(struct HttpContext *pContext);
+  bool (*checkFinishedFp)(struct HttpContext *pContext, HttpSqlCmd *cmd, int code);
+  void (*setNextCmdFp)(struct HttpContext *pContext, HttpSqlCmd *cmd, int code);
+} HttpEncodeMethod;
+
+typedef struct {
+  char   *pos;
+  int32_t len;
+} HttpBuf;
+
+typedef struct {
+  char              buffer[HTTP_BUFFER_SIZE];
+  int               bufsize;
+  char             *pLast;
+  char             *pCur;
+  HttpBuf           method;
+  HttpBuf           path[HTTP_MAX_URL];  // url: dbname/meter/query
+  HttpBuf           data;                // body content
+  HttpBuf           token;               // auth token
+  HttpDecodeMethod *pMethod;
+} HttpParser;
+
+typedef struct HttpContext {
+  int32_t      refCount;
+  int          fd;
+  uint32_t     accessTimes;
+  uint32_t     lastAccessTime;
+  int32_t      state;
+  uint8_t      httpVersion;
+  uint8_t      httpChunked;
+  uint8_t      httpKeepAlive;  // http1.0 and not keep-alive, close connection immediately
+  uint8_t      acceptEncoding;
+  uint8_t      contentEncoding;
+  uint8_t      reqType;
+  uint8_t      parsed;
+  char         ipstr[22];
+  char         user[TSDB_USER_LEN];  // parsed from auth token or login message
+  char         pass[TSDB_PASSWORD_LEN];
+  void *       taos;
+  void *       ppContext;
+  HttpSession *session;
+  z_stream     gzipStream;
+  HttpParser   parser;
+  HttpSqlCmd   singleCmd;
+  HttpSqlCmds *multiCmds;
+  JsonBuf *    jsonBuf;
+  void *       timer;
+  HttpEncodeMethod * encodeMethod;
+  struct HttpThread *pThread;
+} HttpContext;
+
+typedef struct HttpThread {
+  pthread_t       thread;
+  HttpContext *   pHead;
+  pthread_mutex_t threadMutex;
+  bool            stop;
+  int             pollFd;
+  int             numOfFds;
+  int             threadId;
+  char            label[HTTP_LABEL_SIZE];
+  bool (*processData)(HttpContext *pContext);
+} HttpThread;
+
+typedef struct HttpServer {
+  char              label[HTTP_LABEL_SIZE];
+  uint32_t          serverIp;
+  uint16_t          serverPort;
+  int               fd;
+  int               numOfThreads;
+  int               methodScannerLen;
+  int32_t           requestNum;
+  int32_t           status;
+  pthread_t         thread;
+  HttpThread *      pThreads;
+  void *            contextCache;
+  void *            sessionCache;
+  pthread_mutex_t   serverMutex;
+  HttpDecodeMethod *methodScanner[HTTP_METHOD_SCANNER_SIZE];
+  bool (*processData)(HttpContext *pContext);
+} HttpServer;
+
+extern const char *httpKeepAliveStr[];
+extern const char *httpVersionStr[];
+extern HttpServer  tsHttpServer;
+
+#endif

src/plugins/http/inc/httpJson.h

@@ -97,4 +97,8 @@ void httpJsonPrint(JsonBuf* buf, const char* json, int len);
 // quick
 void httpJsonPairStatus(JsonBuf* buf, int code);
 
+// http json printer
+JsonBuf* httpMallocJsonBuf(struct HttpContext* pContext);
+void     httpFreeJsonBuf(struct HttpContext* pContext);
+
 #endif

src/plugins/http/inc/httpResp.h

@@ -16,7 +16,7 @@
 #ifndef TDENGINE_HTTP_RESP_H
 #define TDENGINE_HTTP_RESP_H
 
-#include "httpHandle.h"
+#include "httpInt.h"
 
 enum _httpRespTempl {
   HTTP_RESPONSE_JSON_OK,

src/plugins/http/inc/httpServer.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_HTTP_SERVER_H
+#define TDENGINE_HTTP_SERVER_H
+
+#include "httpInt.h"
+
+bool httpInitConnect();
+void httpCleanUpConnect();
+
+void *httpInitServer(char *ip, uint16_t port, char *label, int numOfThreads, void *fp, void *shandle);
+void httpCleanUpServer(HttpServer *pServer);
+bool httpReadDataImp(HttpContext *pContext);
+
+#endif

src/plugins/http/inc/httpSession.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_HTTP_SESSION_H
+#define TDENGINE_HTTP_SESSION_H
+
+bool httpInitSessions();
+void httpCleanUpSessions();
+
+// http session method
+void httpCreateSession(HttpContext *pContext, void *taos);
+void httpGetSession(HttpContext *pContext);
+void httpReleaseSession(HttpContext *pContext);
+
+#endif

src/plugins/http/inc/httpSql.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_HTTP_SQL_H
+#define TDENGINE_HTTP_SQL_H
+
+
+int32_t httpAddToSqlCmdBuffer(HttpContext *pContext, const char *const format, ...);
+int32_t httpAddToSqlCmdBufferNoTerminal(HttpContext *pContext, const char *const format, ...);
+int32_t httpAddToSqlCmdBufferWithSize(HttpContext *pContext, int mallocSize);
+int32_t httpAddToSqlCmdBufferTerminal(HttpContext *pContext);
+
+bool httpMallocMultiCmds(HttpContext *pContext, int cmdSize, int bufferSize);
+bool httpReMallocMultiCmdsSize(HttpContext *pContext, int cmdSize);
+bool httpReMallocMultiCmdsBuffer(HttpContext *pContext, int bufferSize);
+void httpFreeMultiCmds(HttpContext *pContext);
+
+HttpSqlCmd *httpNewSqlCmd(HttpContext *pContext);
+HttpSqlCmd *httpCurrSqlCmd(HttpContext *pContext);
+int httpCurSqlCmdPos(HttpContext *pContext);
+
+void httpTrimTableName(char *name);
+int httpShrinkTableName(HttpContext *pContext, int pos, char *name);
+char *httpGetCmdsString(HttpContext *pContext, int pos);
+
+#endif

src/plugins/http/inc/httpUtil.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_HTTP_UTIL_H
+#define TDENGINE_HTTP_UTIL_H
+
+bool httpCheckUsedbSql(char *sql);
+void httpTimeToString(time_t t, char *buf, int buflen);
+
+bool httpUrlMatch(HttpContext *pContext, int pos, char *cmp);
+bool httpParseRequest(HttpContext *pContext);
+int  httpCheckReadCompleted(HttpContext *pContext);
+void httpReadDirtyData(HttpContext *pContext);
+
+int httpGzipDeCompress(char *srcData, int32_t nSrcData, char *destData, int32_t *nDestData);
+int httpGzipCompressInit(HttpContext *pContext);
+int httpGzipCompress(HttpContext *pContext, char *inSrcData, int32_t inSrcDataLen,
+                     char *outDestData, int32_t *outDestDataLen, bool isTheLast);
+
+// http request parser
+void httpAddMethod(HttpServer *pServer, HttpDecodeMethod *pMethod);
+
+
+
+#endif

src/plugins/http/inc/restHandle.h

@@ -16,15 +16,11 @@
 #ifndef TDENGINE_REST_HANDLE_H
 #define TDENGINE_REST_HANDLE_H
 
-#include <stdbool.h>
-#include <stdint.h>
-#include <string.h>
-#include <unistd.h>
-
 #include "http.h"
-#include "httpCode.h"
-#include "httpHandle.h"
+#include "httpInt.h"
+#include "httpUtil.h"
 #include "httpResp.h"
+#include "httpSql.h"
 
 #define REST_ROOT_URL_POS   0
 #define REST_ACTION_URL_POS 1

src/plugins/http/inc/tgHandle.h

@@ -16,16 +16,11 @@
 #ifndef TDENGINE_TG_HANDLE_H
 #define TDENGINE_TG_HANDLE_H
 
-#include <stdbool.h>
-#include <stdint.h>
-#include <string.h>
-#include <unistd.h>
-
-#include "cJSON.h"
 #include "http.h"
-#include "httpCode.h"
-#include "httpHandle.h"
+#include "httpInt.h"
+#include "httpUtil.h"
 #include "httpResp.h"
+#include "httpSql.h"
 
 #define TG_ROOT_URL_POS   0
 #define TG_DB_URL_POS     1

src/plugins/http/src/httpAuth.c

@@ -18,8 +18,8 @@
 #include "tkey.h"
 #include "tutil.h"
 #include "http.h"
-#include "httpLog.h"
-#include "httpHandle.h"
+#include "httpInt.h"
+#include "httpAuth.h"
 
 #define KEY_DES_4 4971256377704625728L
 
@@ -73,6 +73,7 @@ bool httpParseTaosdAuthToken(HttpContext *pContext, char *token, int len) {
   unsigned char *base64 = base64_decode(token, len, &outlen);
   if (base64 == NULL || outlen == 0) {
     httpError("context:%p, fd:%d, ip:%s, taosd token:%s parsed error", pContext, pContext->fd, pContext->ipstr, token);
+    if (base64) free(base64);
     return false;
   }
   if (outlen != (TSDB_USER_LEN + TSDB_PASSWORD_LEN)) {

src/plugins/http/src/httpContext.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "taosmsg.h"
+#include "tsocket.h"
+#include "tutil.h"
+#include "ttime.h"
+#include "ttimer.h"
+#include "tglobal.h"
+#include "tcache.h"
+#include "hash.h"
+#include "httpInt.h"
+#include "httpResp.h"
+#include "httpSql.h"
+#include "httpSession.h"
+
+static void httpRemoveContextFromEpoll(HttpContext *pContext) {
+  HttpThread *pThread = pContext->pThread;
+  if (pContext->fd >= 0) {
+    epoll_ctl(pThread->pollFd, EPOLL_CTL_DEL, pContext->fd, NULL);
+    taosCloseSocket(pContext->fd);
+    pContext->fd = -1;
+  }
+}
+
+static void httpDestroyContext(void *data) {
+  HttpContext *pContext = *(HttpContext **)data;
+  if (pContext->fd > 0) tclose(pContext->fd);
+
+  HttpThread *pThread = pContext->pThread;
+  httpRemoveContextFromEpoll(pContext);
+  httpReleaseSession(pContext);
+  atomic_sub_fetch_32(&pThread->numOfFds, 1);
+  
+  pContext->pThread = 0;
+  pContext->state = HTTP_CONTEXT_STATE_CLOSED;
+
+  // avoid double free
+  httpFreeJsonBuf(pContext);
+  httpFreeMultiCmds(pContext);
+  
+  httpTrace("context:%p, is destroyed, refCount:%d", pContext, pContext->refCount);
+  tfree(pContext);
+}
+
+bool httpInitContexts() {
+  tsHttpServer.contextCache = taosCacheInitWithCb(2, httpDestroyContext);
+  if (tsHttpServer.contextCache == NULL) {
+    httpError("failed to init context cache");
+    return false;
+  }
+
+  return true;
+}
+
+void httpCleanupContexts() {
+  if (tsHttpServer.contextCache != NULL) {
+    SCacheObj *cache = tsHttpServer.contextCache;
+    httpPrint("context cache is cleanuping, size:%d", taosHashGetSize(cache->pHashTable));
+    taosCacheCleanup(tsHttpServer.contextCache);
+    tsHttpServer.contextCache = NULL;
+  }
+}
+
+const char *httpContextStateStr(HttpContextState state) {
+  switch (state) {
+    case HTTP_CONTEXT_STATE_READY:
+      return "ready";
+    case HTTP_CONTEXT_STATE_HANDLING:
+      return "handling";
+    case HTTP_CONTEXT_STATE_DROPPING:
+      return "dropping";
+    case HTTP_CONTEXT_STATE_CLOSED:
+      return "closed";
+    default:
+      return "unknown";
+  }
+}
+
+void httpNotifyContextClose(HttpContext *pContext) { 
+  shutdown(pContext->fd, SHUT_WR); 
+}
+
+bool httpAlterContextState(HttpContext *pContext, HttpContextState srcState, HttpContextState destState) {
+  return (atomic_val_compare_exchange_32(&pContext->state, srcState, destState) == srcState);
+}
+
+HttpContext *httpCreateContext(int32_t fd) {
+  HttpContext *pContext = calloc(1, sizeof(HttpContext));
+  if (pContext == NULL) return NULL;
+
+  char contextStr[16] = {0};
+  snprintf(contextStr, sizeof(contextStr), "%p", pContext);
+  
+  pContext->fd = fd;
+  pContext->httpVersion = HTTP_VERSION_10;
+  pContext->lastAccessTime = taosGetTimestampSec();
+  pContext->state = HTTP_CONTEXT_STATE_READY;
+  
+  HttpContext **ppContext = taosCachePut(tsHttpServer.contextCache, contextStr, &pContext, sizeof(HttpContext *), 3);
+  pContext->ppContext = ppContext;
+  httpTrace("context:%p, fd:%d, is created, item:%p", pContext, fd, ppContext);
+
+  // set the ref to 0 
+  taosCacheRelease(tsHttpServer.contextCache, (void**)&ppContext, false);
+
+  return pContext;
+}
+
+HttpContext *httpGetContext(void *ptr) {
+  char contextStr[16] = {0};
+  snprintf(contextStr, sizeof(contextStr), "%p", ptr);
+  
+  HttpContext **ppContext = taosCacheAcquireByName(tsHttpServer.contextCache, contextStr);
+  
+  if (ppContext) {
+    HttpContext *pContext = *ppContext;
+    if (pContext) {
+      int32_t refCount = atomic_add_fetch_32(&pContext->refCount, 1);
+      httpTrace("context:%p, fd:%d, is accquired, refCount:%d", pContext, pContext->fd, refCount);
+      return pContext;
+    }
+  }
+  return NULL;
+}
+
+void httpReleaseContext(HttpContext *pContext) {
+  int32_t refCount = atomic_sub_fetch_32(&pContext->refCount, 1);
+  assert(refCount >= 0);
+  httpTrace("context:%p, fd:%d, is releasd, refCount:%d", pContext, pContext->fd, refCount);
+
+  HttpContext **ppContext = pContext->ppContext;
+  taosCacheRelease(tsHttpServer.contextCache, (void **)(&ppContext), false);
+}
+
+bool httpInitContext(HttpContext *pContext) {
+  pContext->accessTimes++;
+  pContext->lastAccessTime = taosGetTimestampSec();
+  pContext->httpVersion = HTTP_VERSION_10;
+  pContext->httpKeepAlive = HTTP_KEEPALIVE_NO_INPUT;
+  pContext->httpChunked = HTTP_UNCUNKED;
+  pContext->acceptEncoding = HTTP_COMPRESS_IDENTITY;
+  pContext->contentEncoding = HTTP_COMPRESS_IDENTITY;
+  pContext->reqType = HTTP_REQTYPE_OTHERS;
+  pContext->encodeMethod = NULL;
+  pContext->timer = NULL;
+  memset(&pContext->singleCmd, 0, sizeof(HttpSqlCmd));
+
+  HttpParser *pParser = &pContext->parser;
+  memset(pParser, 0, sizeof(HttpParser));
+  pParser->pCur = pParser->pLast = pParser->buffer;
+
+  httpTrace("context:%p, fd:%d, ip:%s, thread:%s, accessTimes:%d, parsed:%d",
+          pContext, pContext->fd, pContext->ipstr, pContext->pThread->label, pContext->accessTimes, pContext->parsed);
+  return true;
+}
+
+void httpCloseContextByApp(HttpContext *pContext) {
+  pContext->parsed = false;
+
+  bool keepAlive = true;
+  if (pContext->httpVersion == HTTP_VERSION_10 && pContext->httpKeepAlive != HTTP_KEEPALIVE_ENABLE) {
+    keepAlive = false;
+  } else if (pContext->httpVersion != HTTP_VERSION_10 && pContext->httpKeepAlive == HTTP_KEEPALIVE_DISABLE) {
+    keepAlive = false;
+  } else {}
+
+  if (keepAlive) {
+    if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_HANDLING, HTTP_CONTEXT_STATE_READY)) {
+      httpTrace("context:%p, fd:%d, ip:%s, last state:handling, keepAlive:true, reuse connect",
+              pContext, pContext->fd, pContext->ipstr);
+    } else if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_DROPPING, HTTP_CONTEXT_STATE_CLOSED)) {
+      httpRemoveContextFromEpoll(pContext);
+      httpTrace("context:%p, fd:%d, ip:%s, last state:dropping, keepAlive:true, close connect",
+              pContext, pContext->fd, pContext->ipstr);
+    } else if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_READY, HTTP_CONTEXT_STATE_READY)) {
+      httpTrace("context:%p, fd:%d, ip:%s, last state:ready, keepAlive:true, reuse connect",
+              pContext, pContext->fd, pContext->ipstr);
+    } else if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_CLOSED, HTTP_CONTEXT_STATE_CLOSED)) {
+      httpRemoveContextFromEpoll(pContext);
+      httpTrace("context:%p, fd:%d, ip:%s, last state:ready, keepAlive:true, close connect",
+                pContext, pContext->fd, pContext->ipstr);
+    } else {
+      httpRemoveContextFromEpoll(pContext);
+      httpError("context:%p, fd:%d, ip:%s, last state:%s:%d, keepAlive:true, close connect",
+              pContext, pContext->fd, pContext->ipstr, httpContextStateStr(pContext->state), pContext->state);
+    }
+  } else {
+    httpRemoveContextFromEpoll(pContext);
+    httpTrace("context:%p, fd:%d, ip:%s, last state:%s:%d, keepAlive:false, close connect",
+              pContext, pContext->fd, pContext->ipstr, httpContextStateStr(pContext->state), pContext->state);
+  }
+
+  httpReleaseContext(pContext);
+}
+
+void httpCloseContextByServer(HttpContext *pContext) {
+  if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_HANDLING, HTTP_CONTEXT_STATE_DROPPING)) {
+    httpTrace("context:%p, fd:%d, ip:%s, epoll finished, still used by app", pContext, pContext->fd, pContext->ipstr);
+  } else if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_DROPPING, HTTP_CONTEXT_STATE_DROPPING)) {
+    httpTrace("context:%p, fd:%d, ip:%s, epoll already finished, wait app finished", pContext, pContext->fd, pContext->ipstr);
+  } else if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_READY, HTTP_CONTEXT_STATE_CLOSED)) {
+    httpTrace("context:%p, fd:%d, ip:%s, epoll finished, close context", pContext, pContext->fd, pContext->ipstr);
+  } else if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_CLOSED, HTTP_CONTEXT_STATE_CLOSED)) {
+    httpTrace("context:%p, fd:%d, ip:%s, epoll finished, will be closed soon", pContext, pContext->fd, pContext->ipstr);
+  } else {
+    httpError("context:%p, fd:%d, ip:%s, unknown state:%d", pContext, pContext->fd, pContext->ipstr, pContext->state);
+  }
+
+  pContext->parsed = false;
+  httpRemoveContextFromEpoll(pContext);
+  httpReleaseContext(pContext);
+}

src/plugins/http/src/httpHandle.c

@@ -19,11 +19,12 @@
 #include "tglobal.h"
 #include "tsocket.h"
 #include "ttimer.h"
-#include "http.h"
-#include "httpLog.h"
-#include "httpCode.h"
-#include "httpHandle.h"
+#include "httpInt.h"
 #include "httpResp.h"
+#include "httpAuth.h"
+#include "httpServer.h"
+#include "httpContext.h"
+#include "httpHandle.h"
 
 void httpToLowerUrl(char* url) {
   /*ignore case */
@@ -58,6 +59,10 @@ bool httpParseURL(HttpContext* pContext) {
   HttpParser* pParser = &pContext->parser;
   char* pSeek;
   char* pEnd = strchr(pParser->pLast, ' ');
+  if (pEnd == NULL) {
+    return false;
+  }
+
   if (*pParser->pLast != '/') {
     httpSendErrorResp(pContext, HTTP_UNSUPPORT_URL);
     return false;
@@ -159,7 +164,7 @@ bool httpGetHttpMethod(HttpContext* pContext) {
 bool httpGetDecodeMethod(HttpContext* pContext) {
   HttpParser* pParser = &pContext->parser;
 
-  HttpServer* pServer = pContext->pThread->pServer;
+  HttpServer* pServer = &tsHttpServer;
   int         methodLen = pServer->methodScannerLen;
   for (int i = 0; i < methodLen; i++) {
     HttpDecodeMethod* method = pServer->methodScanner[i];

src/plugins/http/src/httpJson.c

@@ -22,6 +22,7 @@
 #include "httpCode.h"
 #include "httpJson.h"
 #include "httpResp.h"
+#include "httpUtil.h"
 
 #define MAX_NUM_STR_SZ 25
 

src/plugins/http/src/httpResp.c

@@ -21,6 +21,7 @@
 #include "httpResp.h"
 #include "httpCode.h"
 #include "httpJson.h"
+#include "httpContext.h"
 
 const char *httpKeepAliveStr[] = {"", "Connection: Keep-Alive\r\n", "Connection: Close\r\n"};
 

src/plugins/http/src/httpServer.c

@@ -21,244 +21,15 @@
 #include "ttime.h"
 #include "ttimer.h"
 #include "tglobal.h"
-#include "http.h"
-#include "httpLog.h"
-#include "httpCode.h"
-#include "httpHandle.h"
+#include "httpInt.h"
+#include "httpContext.h"
 #include "httpResp.h"
+#include "httpUtil.h"
 
 #ifndef EPOLLWAKEUP
  #define EPOLLWAKEUP (1u << 29)
 #endif
 
-const char* httpContextStateStr(HttpContextState state) {
-  switch (state) {
-    case HTTP_CONTEXT_STATE_READY:
-      return "ready";
-    case HTTP_CONTEXT_STATE_HANDLING:
-      return "handling";
-    case HTTP_CONTEXT_STATE_DROPPING:
-      return "dropping";
-    case HTTP_CONTEXT_STATE_CLOSED:
-      return "closed";
-    default:
-      return "unknown";
-  }
-}
-
-void httpRemoveContextFromEpoll(HttpThread *pThread, HttpContext *pContext) {
-  if (pContext->fd >= 0) {
-    epoll_ctl(pThread->pollFd, EPOLL_CTL_DEL, pContext->fd, NULL);
-    taosCloseSocket(pContext->fd);
-    pContext->fd = -1;
-  }
-}
-
-bool httpAlterContextState(HttpContext *pContext, HttpContextState srcState, HttpContextState destState) {
-  return (atomic_val_compare_exchange_32(&pContext->state, srcState, destState) == srcState);
-}
-
-void httpFreeContext(HttpServer *pServer, HttpContext *pContext);
-
-/**
- * context will be reused while connection exist
- * multiCmds and jsonBuf will be malloc after taos_query_a called
- * and won't be freed until connection closed
- */
-HttpContext *httpCreateContext(HttpServer *pServer) {
-  HttpContext *pContext = (HttpContext *)taosMemPoolMalloc(pServer->pContextPool);
-  if (pContext != NULL) {
-    pContext->fromMemPool = 1;
-    httpTrace("context:%p, is malloced from mempool", pContext);
-  } else {
-    pContext = (HttpContext *)malloc(sizeof(HttpContext));
-    if (pContext == NULL) {
-      return NULL;
-    } else {
-      memset(pContext, 0, sizeof(HttpContext));
-    }
-    httpTrace("context:%p, is malloced from raw memory", pContext);
-  }
-
-  pContext->signature = pContext;
-  pContext->httpVersion = HTTP_VERSION_10;
-  pContext->lastAccessTime = taosGetTimestampSec();
-  pContext->state = HTTP_CONTEXT_STATE_READY;
-  return pContext;
-}
-
-void httpFreeContext(HttpServer *pServer, HttpContext *pContext) {
-  if (pContext->fromMemPool) {
-    httpTrace("context:%p, is freed from mempool", pContext);
-    taosMemPoolFree(pServer->pContextPool, (char *)pContext);
-  } else {
-    httpTrace("context:%p, is freed from raw memory", pContext);
-    tfree(pContext);
-  }
-}
-
-void httpCleanUpContextTimer(HttpContext *pContext) {
-  if (pContext->timer != NULL) {
-    taosTmrStopA(&pContext->timer);
-    //httpTrace("context:%p, ip:%s, close timer:%p", pContext, pContext->ipstr, pContext->timer);
-    pContext->timer = NULL;
-  }
-}
-
-void httpCleanUpContext(HttpContext *pContext, void *unused) {
-  httpTrace("context:%p, start the clean up operation, sig:%p", pContext, pContext->signature);
-  void *sig = atomic_val_compare_exchange_ptr(&pContext->signature, pContext, 0);
-  if (sig == NULL) {
-    httpTrace("context:%p is freed by another thread.", pContext);
-    return;
-  }
-
-  HttpThread *pThread = pContext->pThread;
-
-  httpCleanUpContextTimer(pContext);
-
-  httpRemoveContextFromEpoll(pThread, pContext);
-
-  httpRestoreSession(pContext);
-
-  pthread_mutex_lock(&pThread->threadMutex);
-
-  pThread->numOfFds--;
-  if (pThread->numOfFds < 0) {
-    httpError("context:%p, ip:%s, thread:%s, number of FDs:%d shall never be negative",
-              pContext, pContext->ipstr, pThread->label, pThread->numOfFds);
-    pThread->numOfFds = 0;
-  }
-
-  // remove from the link list
-  if (pContext->prev) {
-    (pContext->prev)->next = pContext->next;
-  } else {
-    pThread->pHead = pContext->next;
-  }
-
-  if (pContext->next) {
-    (pContext->next)->prev = pContext->prev;
-  }
-
-  pthread_mutex_unlock(&pThread->threadMutex);
-
-  httpTrace("context:%p, ip:%s, thread:%s, numOfFds:%d, context is cleaned up", pContext, pContext->ipstr,
-            pThread->label, pThread->numOfFds);
-
-  pContext->signature = 0;
-  pContext->fd = -1;
-  pContext->pThread = 0;
-  pContext->prev = 0;
-  pContext->next = 0;
-  pContext->state = HTTP_CONTEXT_STATE_READY;
-
-  // avoid double free
-  httpFreeJsonBuf(pContext);
-  httpFreeMultiCmds(pContext);
-  httpFreeContext(pThread->pServer, pContext);
-}
-
-bool httpInitContext(HttpContext *pContext) {
-  pContext->accessTimes++;
-  pContext->lastAccessTime = taosGetTimestampSec();
-  pContext->httpVersion = HTTP_VERSION_10;
-  pContext->httpKeepAlive = HTTP_KEEPALIVE_NO_INPUT;
-  pContext->httpChunked = HTTP_UNCUNKED;
-  pContext->acceptEncoding = HTTP_COMPRESS_IDENTITY;
-  pContext->contentEncoding = HTTP_COMPRESS_IDENTITY;
-  pContext->reqType = HTTP_REQTYPE_OTHERS;
-  pContext->encodeMethod = NULL;
-  pContext->timer = NULL;
-  memset(&pContext->singleCmd, 0, sizeof(HttpSqlCmd));
-
-  HttpParser *pParser = &pContext->parser;
-  memset(pParser, 0, sizeof(HttpParser));
-  pParser->pCur = pParser->pLast = pParser->buffer;
-
-  httpTrace("context:%p, fd:%d, ip:%s, thread:%s, accessTimes:%d, parsed:%d",
-          pContext, pContext->fd, pContext->ipstr, pContext->pThread->label, pContext->accessTimes, pContext->parsed);
-  return true;
-}
-
-
-void httpCloseContext(HttpThread *pThread, HttpContext *pContext) {
-  taosTmrReset((TAOS_TMR_CALLBACK)httpCleanUpContext, HTTP_DELAY_CLOSE_TIME_MS, pContext, pThread->pServer->timerHandle, &pContext->timer);
-  httpTrace("context:%p, fd:%d, ip:%s, state:%s will be closed after:%d ms, timer:%p",
-          pContext, pContext->fd, pContext->ipstr, httpContextStateStr(pContext->state), HTTP_DELAY_CLOSE_TIME_MS, pContext->timer);
-}
-
-void httpCloseContextByApp(HttpContext *pContext) {
-  HttpThread *pThread = pContext->pThread;
-  pContext->parsed = false;
-
-  bool keepAlive = true;
-  if (pContext->httpVersion == HTTP_VERSION_10 && pContext->httpKeepAlive != HTTP_KEEPALIVE_ENABLE) {
-    keepAlive = false;
-  } else if (pContext->httpVersion != HTTP_VERSION_10 && pContext->httpKeepAlive == HTTP_KEEPALIVE_DISABLE) {
-    keepAlive = false;
-  } else {}
-
-  if (keepAlive) {
-    if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_HANDLING, HTTP_CONTEXT_STATE_READY)) {
-      httpTrace("context:%p, fd:%d, ip:%s, last state:handling, keepAlive:true, reuse connect",
-              pContext, pContext->fd, pContext->ipstr);
-    } else if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_DROPPING, HTTP_CONTEXT_STATE_CLOSED)) {
-      httpRemoveContextFromEpoll(pThread, pContext);
-      httpTrace("context:%p, fd:%d, ip:%s, last state:dropping, keepAlive:true, close connect",
-              pContext, pContext->fd, pContext->ipstr);
-      httpCloseContext(pThread, pContext);
-    } else if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_READY, HTTP_CONTEXT_STATE_READY)) {
-      httpTrace("context:%p, fd:%d, ip:%s, last state:ready, keepAlive:true, reuse connect",
-              pContext, pContext->fd, pContext->ipstr);
-    } else if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_CLOSED, HTTP_CONTEXT_STATE_CLOSED)) {
-      httpRemoveContextFromEpoll(pThread, pContext);
-      httpTrace("context:%p, fd:%d, ip:%s, last state:ready, keepAlive:true, close connect",
-                pContext, pContext->fd, pContext->ipstr);
-      httpCloseContext(pThread, pContext);
-    } else {
-      httpRemoveContextFromEpoll(pThread, pContext);
-      httpError("context:%p, fd:%d, ip:%s, last state:%s:%d, keepAlive:true, close connect",
-              pContext, pContext->fd, pContext->ipstr, httpContextStateStr(pContext->state), pContext->state);
-      httpCloseContext(pThread, pContext);
-    }
-  } else {
-    httpRemoveContextFromEpoll(pThread, pContext);
-    httpTrace("context:%p, fd:%d, ip:%s, last state:%s:%d, keepAlive:false, close connect",
-              pContext, pContext->fd, pContext->ipstr, httpContextStateStr(pContext->state), pContext->state);
-    httpCloseContext(pThread, pContext);
-  }
-}
-
-void httpCloseContextByServer(HttpThread *pThread, HttpContext *pContext) {
-  httpRemoveContextFromEpoll(pThread, pContext);
-  pContext->parsed = false;
-  
-  if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_HANDLING, HTTP_CONTEXT_STATE_DROPPING)) {
-    httpTrace("context:%p, fd:%d, ip:%s, epoll finished, still used by app", pContext, pContext->fd, pContext->ipstr);
-  } else if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_DROPPING, HTTP_CONTEXT_STATE_DROPPING)) {
-    httpTrace("context:%p, fd:%d, ip:%s, epoll already finished, wait app finished", pContext, pContext->fd, pContext->ipstr);
-  } else if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_READY, HTTP_CONTEXT_STATE_CLOSED)) {
-    httpTrace("context:%p, fd:%d, ip:%s, epoll finished, close context", pContext, pContext->fd, pContext->ipstr);
-    httpCloseContext(pThread, pContext);
-  } else if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_CLOSED, HTTP_CONTEXT_STATE_CLOSED)) {
-    httpTrace("context:%p, fd:%d, ip:%s, epoll finished, will be closed soon", pContext, pContext->fd, pContext->ipstr);
-    httpCloseContext(pThread, pContext);
-  } else {
-    httpError("context:%p, fd:%d, ip:%s, unknown state:%d", pContext, pContext->fd, pContext->ipstr, pContext->state);
-    httpCloseContext(pThread, pContext);
-  }
-}
-
-void httpCloseContextByServerForExpired(void *param, void *tmrId) {
-  HttpContext *pContext = (HttpContext *)param;
-  httpRemoveContextFromEpoll(pContext->pThread, pContext);
-  httpError("context:%p, fd:%d, ip:%s, read http body error, time expired, timer:%p", pContext, pContext->fd, pContext->ipstr, tmrId);
-  httpSendErrorResp(pContext, HTTP_PARSE_BODY_ERROR);
-  httpCloseContextByServer(pContext->pThread, pContext);
-}
-
-
 static void httpStopThread(HttpThread* pThread) {
   pThread->stop = true;
 
@@ -281,19 +52,13 @@ static void httpStopThread(HttpThread* pThread) {
 
   close(pThread->pollFd);
   pthread_mutex_destroy(&(pThread->threadMutex));
-
-  //while (pThread->pHead) {
-  //  httpCleanUpContext(pThread->pHead, 0);
-  //}
 }
 
+void httpCleanUpConnect() {
+  HttpServer *pServer = &tsHttpServer;
+  if (pServer->pThreads == NULL) return;
 
-void httpCleanUpConnect(HttpServer *pServer) {
-  if (pServer == NULL) return;
-
-  shutdown(pServer->fd, SHUT_RD);
   pthread_join(pServer->thread, NULL);
-
   for (int i = 0; i < pServer->numOfThreads; ++i) {
     HttpThread* pThread = pServer->pThreads + i;
     if (pThread != NULL) {
@@ -302,19 +67,10 @@ void httpCleanUpConnect(HttpServer *pServer) {
   }
 
   tfree(pServer->pThreads);
+  pServer->pThreads = NULL;
   httpTrace("http server:%s is cleaned up", pServer->label);
 }
 
-// read all the data, then just discard it
-void httpReadDirtyData(HttpContext *pContext) {
-  int fd = pContext->fd;
-  char data[1024] = {0};
-  int  len = (int)taosReadSocket(fd, data, 1024);
-  while (len >= sizeof(data)) {
-    len = (int)taosReadSocket(fd, data, 1024);
-  }
-}
-
 bool httpReadDataImp(HttpContext *pContext) {
   HttpParser *pParser = &pContext->parser;
 
@@ -338,11 +94,10 @@ bool httpReadDataImp(HttpContext *pContext) {
     }
 
     if (pParser->bufsize >= (HTTP_BUFFER_SIZE - HTTP_STEP_SIZE)) {
-      httpReadDirtyData(pContext);
       httpError("context:%p, fd:%d, ip:%s, thread:%s, request big than:%d",
                 pContext, pContext->fd, pContext->ipstr, pContext->pThread->label, HTTP_BUFFER_SIZE);
-      httpRemoveContextFromEpoll(pContext->pThread, pContext);
       httpSendErrorResp(pContext, HTTP_REQUSET_TOO_BIG);
+      httpNotifyContextClose(pContext);
       return false;
     }
   }
@@ -352,7 +107,7 @@ bool httpReadDataImp(HttpContext *pContext) {
   return true;
 }
 
-bool httpDecompressData(HttpContext *pContext) {
+static bool httpDecompressData(HttpContext *pContext) {
   if (pContext->contentEncoding != HTTP_COMPRESS_GZIP) {
     httpDump("context:%p, fd:%d, ip:%s, content:%s", pContext, pContext->fd, pContext->ipstr, pContext->parser.data.pos);
     return true;
@@ -382,45 +137,43 @@ bool httpDecompressData(HttpContext *pContext) {
   return ret == 0;
 }
 
-bool httpReadData(HttpThread *pThread, HttpContext *pContext) {
+static bool httpReadData(HttpContext *pContext) {
   if (!pContext->parsed) {
     httpInitContext(pContext);
   }
 
   if (!httpReadDataImp(pContext)) {
-    httpCloseContextByServer(pThread, pContext);
+    httpNotifyContextClose(pContext);
     return false;
   }
 
   if (!httpParseRequest(pContext)) {
-    httpCloseContextByServer(pThread, pContext);
+    httpNotifyContextClose(pContext);
     return false;
   }
 
   int ret = httpCheckReadCompleted(pContext);
   if (ret == HTTP_CHECK_BODY_CONTINUE) {
-    taosTmrReset(httpCloseContextByServerForExpired, HTTP_EXPIRED_TIME, pContext, pThread->pServer->timerHandle, &pContext->timer);
-    //httpTrace("context:%p, fd:%d, ip:%s, not finished yet, try another times, timer:%p", pContext, pContext->fd, pContext->ipstr, pContext->timer);
+    //httpTrace("context:%p, fd:%d, ip:%s, not finished yet, wait another event", pContext, pContext->fd, pContext->ipstr);
     return false;
   } else if (ret == HTTP_CHECK_BODY_SUCCESS){
-    httpCleanUpContextTimer(pContext);
     httpTrace("context:%p, fd:%d, ip:%s, thread:%s, read size:%d, dataLen:%d",
               pContext, pContext->fd, pContext->ipstr, pContext->pThread->label, pContext->parser.bufsize, pContext->parser.data.len);
     if (httpDecompressData(pContext)) {
       return true;
     } else {
-      httpCloseContextByServer(pThread, pContext);
+      httpNotifyContextClose(pContext);
       return false;
     }
   } else {
-    httpCleanUpContextTimer(pContext);
     httpError("context:%p, fd:%d, ip:%s, failed to read http body, close connect", pContext, pContext->fd, pContext->ipstr);
-    httpCloseContextByServer(pThread, pContext);
+    httpNotifyContextClose(pContext);
     return false;
   }
 }
 
-void httpProcessHttpData(void *param) {
+static void httpProcessHttpData(void *param) {
+  HttpServer  *pServer = &tsHttpServer;
   HttpThread  *pThread = (HttpThread *)param;
   HttpContext *pContext;
   int          fdNum;
@@ -441,77 +194,72 @@ void httpProcessHttpData(void *param) {
     if (fdNum <= 0) continue;
 
     for (int i = 0; i < fdNum; ++i) {
-      pContext = events[i].data.ptr;
-      if (pContext->signature != pContext || pContext->pThread != pThread || pContext->fd <= 0) {
+      pContext = httpGetContext(events[i].data.ptr);
+      if (pContext == NULL) {
+        httpError("context:%p, is already released, close connect", events[i].data.ptr);
+        //epoll_ctl(pThread->pollFd, EPOLL_CTL_DEL, events[i].data.fd, NULL);
+        //tclose(events[i].data.fd);
         continue;
       }
 
       if (events[i].events & EPOLLPRI) {
         httpTrace("context:%p, fd:%d, ip:%s, state:%s, EPOLLPRI events occured, accessed:%d, close connect",
                   pContext, pContext->fd, pContext->ipstr, httpContextStateStr(pContext->state), pContext->accessTimes);
-        httpRemoveContextFromEpoll(pThread, pContext);
-        httpCloseContextByServer(pThread, pContext);
+        httpCloseContextByServer(pContext);
         continue;
       }
 
       if (events[i].events & EPOLLRDHUP) {
         httpTrace("context:%p, fd:%d, ip:%s, state:%s, EPOLLRDHUP events occured, accessed:%d, close connect",
                   pContext, pContext->fd, pContext->ipstr, httpContextStateStr(pContext->state), pContext->accessTimes);
-        httpRemoveContextFromEpoll(pThread, pContext);
-        httpCloseContextByServer(pThread, pContext);
+        httpCloseContextByServer(pContext);
         continue;
       }
 
       if (events[i].events & EPOLLERR) {
         httpTrace("context:%p, fd:%d, ip:%s, state:%s, EPOLLERR events occured, accessed:%d, close connect",
                   pContext, pContext->fd, pContext->ipstr, httpContextStateStr(pContext->state), pContext->accessTimes);
-        httpRemoveContextFromEpoll(pThread, pContext);
-        httpCloseContextByServer(pThread, pContext);
+        httpCloseContextByServer(pContext);
         continue;
       }
 
       if (events[i].events & EPOLLHUP) {
         httpTrace("context:%p, fd:%d, ip:%s, state:%s, EPOLLHUP events occured, accessed:%d, close connect",
                   pContext, pContext->fd, pContext->ipstr, httpContextStateStr(pContext->state), pContext->accessTimes);
-        httpRemoveContextFromEpoll(pThread, pContext);
-        httpCloseContextByServer(pThread, pContext);
+        httpCloseContextByServer(pContext);
         continue;
       }
 
       if (!httpAlterContextState(pContext, HTTP_CONTEXT_STATE_READY, HTTP_CONTEXT_STATE_READY)) {
         httpTrace("context:%p, fd:%d, ip:%s, state:%s, not in ready state, ignore read events",
                 pContext, pContext->fd, pContext->ipstr, httpContextStateStr(pContext->state));
+        httpReleaseContext(pContext);
         continue;
       }
 
-      if (!pContext->pThread->pServer->online) {
-        httpTrace("context:%p, fd:%d, ip:%s, state:%s, server is not online, accessed:%d, close connect",
-                  pContext, pContext->fd, pContext->ipstr, httpContextStateStr(pContext->state), pContext->accessTimes);
-        httpRemoveContextFromEpoll(pThread, pContext);
-        httpReadDirtyData(pContext);
+      if (pServer->status != HTTP_SERVER_RUNNING) {
+        httpTrace("context:%p, fd:%d, ip:%s, state:%s, server is not running, accessed:%d, close connect", pContext,
+                  pContext->fd, pContext->ipstr, httpContextStateStr(pContext->state), pContext->accessTimes);
         httpSendErrorResp(pContext, HTTP_SERVER_OFFLINE);
-        httpCloseContextByServer(pThread, pContext);
-        continue;
+        httpNotifyContextClose(pContext);
       } else {
-        if (httpReadData(pThread, pContext)) {
+        if (httpReadData(pContext)) {
           (*(pThread->processData))(pContext);
-          atomic_fetch_add_32(&pThread->pServer->requestNum, 1);
+          atomic_fetch_add_32(&pServer->requestNum, 1);
         }
       }
     }
   }
 }
 
-void* httpAcceptHttpConnection(void *arg) {
+static void *httpAcceptHttpConnection(void *arg) {
   int                connFd = -1;
   struct sockaddr_in clientAddr;
   int                threadId = 0;
-  HttpThread *       pThread;
-  HttpServer *       pServer;
-  HttpContext *      pContext;
-  int                totalFds;
-
-  pServer = (HttpServer *)arg;
+  HttpServer *       pServer = &tsHttpServer;
+  HttpThread *       pThread = NULL;
+  HttpContext *      pContext = NULL;
+  int                totalFds = 0;
 
   sigset_t set;
   sigemptyset(&set);
@@ -521,12 +269,12 @@ void* httpAcceptHttpConnection(void *arg) {
   pServer->fd = taosOpenTcpServerSocket(pServer->serverIp, pServer->serverPort);
 
   if (pServer->fd < 0) {
-    httpError("http server:%s, failed to open http socket, ip:%s:%u error:%s", pServer->label, taosIpStr(pServer->serverIp),
-              pServer->serverPort, strerror(errno));
+    httpError("http server:%s, failed to open http socket, ip:%s:%u error:%s", pServer->label,
+              taosIpStr(pServer->serverIp), pServer->serverPort, strerror(errno));
     return NULL;
   } else {
-    httpPrint("http service init success at %u", pServer->serverPort);
-    pServer->online = true;
+    httpPrint("http server init success at %u", pServer->serverPort);
+    pServer->status = HTTP_SERVER_RUNNING;
   }
 
   while (1) {
@@ -534,10 +282,10 @@ void* httpAcceptHttpConnection(void *arg) {
     connFd = (int)accept(pServer->fd, (struct sockaddr *)&clientAddr, &addrlen);
     if (connFd == -1) {
       if (errno == EINVAL) {
-        httpTrace("%s HTTP server socket was shutdown, exiting...", pServer->label);
+        httpTrace("http server:%s socket was shutdown, exiting...", pServer->label);
         break;
       }
-      httpError("http server:%s, accept connect failure, errno:%d, reason:%s", pServer->label, errno, strerror(errno));
+      httpError("http server:%s, accept connect failure, errno:%d reason:%s", pServer->label, errno, strerror(errno));
       continue;
     }
 
@@ -547,8 +295,8 @@ void* httpAcceptHttpConnection(void *arg) {
     }
 
     if (totalFds > tsHttpCacheSessions * 100) {
-      httpError("fd:%d, ip:%s:%u, totalFds:%d larger than httpCacheSessions:%d*100, refuse connection",
-              connFd, inet_ntoa(clientAddr.sin_addr), htons(clientAddr.sin_port), totalFds, tsHttpCacheSessions);
+      httpError("fd:%d, ip:%s:%u, totalFds:%d larger than httpCacheSessions:%d*100, refuse connection", connFd,
+                inet_ntoa(clientAddr.sin_addr), htons(clientAddr.sin_port), totalFds, tsHttpCacheSessions);
       taosCloseSocket(connFd);
       continue;
     }
@@ -559,7 +307,7 @@ void* httpAcceptHttpConnection(void *arg) {
     // pick up the thread to handle this connection
     pThread = pServer->pThreads + threadId;
 
-    pContext = httpCreateContext(pServer);
+    pContext = httpCreateContext(connFd);
     if (pContext == NULL) {
       httpError("fd:%d, ip:%s:%u, no enough resource to allocate http context", connFd, inet_ntoa(clientAddr.sin_addr),
                 htons(clientAddr.sin_port));
@@ -567,39 +315,24 @@ void* httpAcceptHttpConnection(void *arg) {
       continue;
     }
 
-    httpTrace("context:%p, fd:%d, ip:%s:%u, thread:%s, numOfFds:%d, totalFds:%d, accept a new connection",
-            pContext, connFd, inet_ntoa(clientAddr.sin_addr), htons(clientAddr.sin_port), pThread->label,
-            pThread->numOfFds, totalFds);
-
-    pContext->fd = connFd;
-    sprintf(pContext->ipstr, "%s:%d", inet_ntoa(clientAddr.sin_addr), htons(clientAddr.sin_port));
     pContext->pThread = pThread;
-
+    sprintf(pContext->ipstr, "%s:%u", inet_ntoa(clientAddr.sin_addr), htons(clientAddr.sin_port));
+    
     struct epoll_event event;
     event.events = EPOLLIN | EPOLLPRI | EPOLLWAKEUP | EPOLLERR | EPOLLHUP | EPOLLRDHUP;
-
     event.data.ptr = pContext;
     if (epoll_ctl(pThread->pollFd, EPOLL_CTL_ADD, connFd, &event) < 0) {
-      httpError("context:%p, fd:%d, ip:%s:%u, thread:%s, failed to add http fd for epoll, error:%s",
-                pContext, connFd, inet_ntoa(clientAddr.sin_addr), htons(clientAddr.sin_port), pThread->label,
-                strerror(errno));
-      httpFreeContext(pThread->pServer, pContext);
-      tclose(connFd);
+      httpError("context:%p, fd:%d, ip:%s, thread:%s, failed to add http fd for epoll, error:%s", pContext, connFd,
+                pContext->ipstr, pThread->label, strerror(errno));
+      tclose(pContext->fd);
+      httpReleaseContext(pContext);
       continue;
     }
 
     // notify the data process, add into the FdObj list
-    pthread_mutex_lock(&(pThread->threadMutex));
-
-    pContext->next = pThread->pHead;
-
-    if (pThread->pHead) (pThread->pHead)->prev = pContext;
-
-    pThread->pHead = pContext;
-
-    pThread->numOfFds++;
-
-    pthread_mutex_unlock(&(pThread->threadMutex));
+    atomic_add_fetch_32(&pThread->numOfFds, 1);
+    httpTrace("context:%p, fd:%d, ip:%s, thread:%s numOfFds:%d totalFds:%d, accept a new connection", pContext, connFd,
+              pContext->ipstr, pThread->label, pThread->numOfFds, totalFds);
 
     // pick up next thread for next connection
     threadId++;
@@ -610,21 +343,17 @@ void* httpAcceptHttpConnection(void *arg) {
   return NULL;
 }
 
-bool httpInitConnect(HttpServer *pServer) {
-  int            i;
-  HttpThread *   pThread;
-
-  pServer->pThreads = (HttpThread *)malloc(sizeof(HttpThread) * (size_t)pServer->numOfThreads);
+bool httpInitConnect() {
+  HttpServer *pServer = &tsHttpServer;
+  pServer->pThreads = calloc(pServer->numOfThreads, sizeof(HttpThread));
   if (pServer->pThreads == NULL) {
     httpError("init error no enough memory");
     return false;
   }
-  memset(pServer->pThreads, 0, sizeof(HttpThread) * (size_t)pServer->numOfThreads);
 
-  pThread = pServer->pThreads;
-  for (i = 0; i < pServer->numOfThreads; ++i) {
+  HttpThread *pThread = pServer->pThreads;
+  for (int i = 0; i < pServer->numOfThreads; ++i) {
     sprintf(pThread->label, "%s%d", pServer->label, i);
-    pThread->pServer = pServer;
     pThread->processData = pServer->processData;
     pThread->threadId = i;
 
@@ -643,8 +372,8 @@ bool httpInitConnect(HttpServer *pServer) {
     pthread_attr_init(&thattr);
     pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
     if (pthread_create(&(pThread->thread), &thattr, (void *)httpProcessHttpData, (void *)(pThread)) != 0) {
-      httpError("http thread:%s, failed to create HTTP process data thread, reason:%s",
-                pThread->label, strerror(errno));
+      httpError("http thread:%s, failed to create HTTP process data thread, reason:%s", pThread->label,
+                strerror(errno));
       return false;
     }
     pthread_attr_destroy(&thattr);

src/plugins/http/src/httpSession.c

@@ -15,44 +15,28 @@
 
 #define _DEFAULT_SOURCE
 #include "os.h"
-#include "hash.h"
 #include "taos.h"
 #include "ttime.h"
-#include "ttimer.h"
-#include "http.h"
-#include "httpLog.h"
-#include "httpCode.h"
-#include "httpHandle.h"
-#include "httpResp.h"
-
-void httpAccessSession(HttpContext *pContext) {
-  HttpServer *server = pContext->pThread->pServer;
-  pthread_mutex_lock(&server->serverMutex);
-  if (pContext->session == pContext->session->signature) {
-    pContext->session->expire = (int) taosGetTimestampSec() + pContext->pThread->pServer->sessionExpire;
-  }
-  pthread_mutex_unlock(&server->serverMutex);
-}
+#include "tglobal.h"
+#include "tcache.h"
+#include "httpInt.h"
+#include "httpContext.h"
+#include "httpSession.h"
 
 void httpCreateSession(HttpContext *pContext, void *taos) {
-  HttpServer *server = pContext->pThread->pServer;
-  pthread_mutex_lock(&server->serverMutex);
+  HttpServer *server = &tsHttpServer;
+  httpReleaseSession(pContext);
 
-  if (pContext->session != NULL && pContext->session == pContext->session->signature) {
-    httpTrace("context:%p, fd:%d, ip:%s, user:%s, set exist session:%p:%p expired", pContext, pContext->fd,
-              pContext->ipstr, pContext->user, pContext->session, pContext->session->taos);
-    pContext->session->expire = 0;
-    pContext->session->access--;
-  }
+  pthread_mutex_lock(&server->serverMutex);
 
-  HttpSession session;
+  HttpSession session = {0};
   session.taos = taos;
-  session.expire = (int)taosGetTimestampSec() + server->sessionExpire;
-  session.access = 1;
-  int sessionIdLen = snprintf(session.id, HTTP_SESSION_ID_LEN, "%s.%s", pContext->user, pContext->pass);
+  session.refCount = 1;
+  snprintf(session.id, HTTP_SESSION_ID_LEN, "%s.%s", pContext->user, pContext->pass);
 
-  taosHashPut(server->pSessionHash, session.id, sessionIdLen, (char *)(&session), sizeof(HttpSession));
-  pContext->session = taosHashGet(server->pSessionHash, session.id, sessionIdLen);
+  pContext->session = taosCachePut(server->sessionCache, session.id, &session, sizeof(HttpSession), tsHttpSessionExpire);
+  // void *temp = pContext->session;
+  // taosCacheRelease(server->sessionCache, (void **)&temp, false);
 
   if (pContext->session == NULL) {
     httpError("context:%p, fd:%d, ip:%s, user:%s, error:%s", pContext, pContext->fd, pContext->ipstr, pContext->user,
@@ -62,26 +46,23 @@ void httpCreateSession(HttpContext *pContext, void *taos) {
     return;
   }
 
-  pContext->session->signature = pContext->session;
-  httpTrace("context:%p, fd:%d, ip:%s, user:%s, create a new session:%p:%p", pContext, pContext->fd, pContext->ipstr,
-            pContext->user, pContext->session, pContext->session->taos);
+  httpTrace("context:%p, fd:%d, ip:%s, user:%s, create a new session:%p:%p sessionRef:%d", pContext, pContext->fd,
+            pContext->ipstr, pContext->user, pContext->session, pContext->session->taos, pContext->session->refCount);
   pthread_mutex_unlock(&server->serverMutex);
 }
 
-void httpFetchSessionImp(HttpContext *pContext) {
-  HttpServer *server = pContext->pThread->pServer;
+static void httpFetchSessionImp(HttpContext *pContext) {
+  HttpServer *server = &tsHttpServer;
   pthread_mutex_lock(&server->serverMutex);
 
   char sessionId[HTTP_SESSION_ID_LEN];
-  int sessonIdLen = snprintf(sessionId, HTTP_SESSION_ID_LEN, "%s.%s", pContext->user, pContext->pass);
-
-  pContext->session = taosHashGet(server->pSessionHash, sessionId, sessonIdLen);
-  if (pContext->session != NULL && pContext->session == pContext->session->signature) {
-    pContext->session->access++;
-    httpTrace("context:%p, fd:%d, ip:%s, user:%s, find an exist session:%p:%p, access:%d, expire:%d",
-              pContext, pContext->fd, pContext->ipstr, pContext->user, pContext->session,
-              pContext->session->taos, pContext->session->access, pContext->session->expire);
-    pContext->session->expire = (int)taosGetTimestampSec() + server->sessionExpire;
+  snprintf(sessionId, HTTP_SESSION_ID_LEN, "%s.%s", pContext->user, pContext->pass);
+
+  pContext->session = taosCacheAcquireByName(server->sessionCache, sessionId);
+  if (pContext->session != NULL) {
+    atomic_add_fetch_32(&pContext->session->refCount, 1);
+    httpTrace("context:%p, fd:%d, ip:%s, user:%s, find an exist session:%p:%p, sessionRef:%d", pContext, pContext->fd,
+              pContext->ipstr, pContext->user, pContext->session, pContext->session->taos, pContext->session->refCount);
   } else {
     httpTrace("context:%p, fd:%d, ip:%s, user:%s, session not found", pContext, pContext->fd, pContext->ipstr,
               pContext->user);
@@ -90,113 +71,54 @@ void httpFetchSessionImp(HttpContext *pContext) {
   pthread_mutex_unlock(&server->serverMutex);
 }
 
-void httpFetchSession(HttpContext *pContext) {
+void httpGetSession(HttpContext *pContext) {
   if (pContext->session == NULL) {
     httpFetchSessionImp(pContext);
   } else {
     char sessionId[HTTP_SESSION_ID_LEN];
     snprintf(sessionId, HTTP_SESSION_ID_LEN, "%s.%s", pContext->user, pContext->pass);
-    if (strcmp(pContext->session->id, sessionId) != 0) {
-      httpError("context:%p, fd:%d, ip:%s, user:%s, password may be changed", pContext, pContext->fd, pContext->ipstr, pContext->user);
-      httpRestoreSession(pContext);
-      httpFetchSessionImp(pContext);
-    }
+    httpReleaseSession(pContext);
+    httpFetchSessionImp(pContext);
   }
 }
 
-void httpRestoreSession(HttpContext *pContext) {
-  HttpServer * server = pContext->pThread->pServer;
+void httpReleaseSession(HttpContext *pContext) {
+  if (pContext == NULL || pContext->session == NULL) return;
 
-  // all access to the session is via serverMutex
-  pthread_mutex_lock(&server->serverMutex);
-  HttpSession *session = pContext->session;
-  if (session == NULL || session != session->signature) {
-    pthread_mutex_unlock(&server->serverMutex);
-    return;
-  }
-  session->access--;
-  httpTrace("context:%p, ip:%s, user:%s, restore session:%p:%p, access:%d, expire:%d",
-            pContext, pContext->ipstr, pContext->user, session, session->taos,
-            session->access, pContext->session->expire);
-  pContext->session = NULL;
-  pthread_mutex_unlock(&server->serverMutex);
-}
+  int32_t refCount = atomic_sub_fetch_32(&pContext->session->refCount, 1);
+  assert(refCount >= 0);
+  httpTrace("context:%p, release session:%p:%p, sessionRef:%d", pContext, pContext->session, pContext->session->taos,
+            pContext->session->refCount);
 
-void httpResetSession(HttpSession *pSession) {
-  httpTrace("close session:%p:%p", pSession, pSession->taos);
-  if (pSession->taos != NULL) {
-    taos_close(pSession->taos);
-    pSession->taos = NULL;
-  }
-  pSession->signature = NULL;
+  taosCacheRelease(tsHttpServer.sessionCache, (void **)&pContext->session, false);
+  pContext->session = NULL;
 }
 
-void httpRemoveAllSessions(HttpServer *pServer) {
-  SHashMutableIterator *pIter = taosHashCreateIter(pServer->pSessionHash);
+static void httpDestroySession(void *data) {
+  HttpSession *session = data;
+  httpTrace("session:%p:%p, is destroyed, sessionRef:%d", session, session->taos, session->refCount);
 
-  while (taosHashIterNext(pIter)) {
-    HttpSession *pSession = taosHashIterGet(pIter);
-    if (pSession == NULL) continue;
-    httpResetSession(pSession);
+  if (session->taos != NULL) {
+    taos_close(session->taos);
+    session->taos = NULL;
   }
-
-  taosHashDestroyIter(pIter);
 }
 
-bool httpInitAllSessions(HttpServer *pServer) {
-  if (pServer->pSessionHash == NULL) {
-    pServer->pSessionHash = taosHashInit(10, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true);
-  }
-  if (pServer->pSessionHash == NULL) {
-    httpError("http init session pool failed");
-    return false;
-  }
-  if (pServer->expireTimer == NULL) {
-    taosTmrReset(httpProcessSessionExpire, 50000, pServer, pServer->timerHandle, &pServer->expireTimer);
+void httpCleanUpSessions() {
+  if (tsHttpServer.sessionCache != NULL) {
+    SCacheObj *cache = tsHttpServer.sessionCache;
+    httpPrint("session cache is cleanuping, size:%d", taosHashGetSize(cache->pHashTable));
+    taosCacheCleanup(tsHttpServer.sessionCache);
+    tsHttpServer.sessionCache = NULL;
   }
-
-  return true;
 }
 
-bool httpSessionExpired(HttpSession *pSession) {
-  time_t cur = taosGetTimestampSec();
-
-  if (pSession->taos != NULL) {
-    if (pSession->expire > cur) {
-      return false;  // un-expired, so return false
-    }
-    if (pSession->access > 0) {
-      httpTrace("session:%p:%p is expired, but still access:%d", pSession, pSession->taos,
-                pSession->access);
-      return false;  // still used, so return false
-    }
-    httpTrace("need close session:%p:%p for it expired, cur:%d, expire:%d, invertal:%d",
-              pSession, pSession->taos, cur, pSession->expire, cur - pSession->expire);
+bool httpInitSessions() {
+  tsHttpServer.sessionCache = taosCacheInitWithCb(5, httpDestroySession);
+  if (tsHttpServer.sessionCache == NULL) {
+    httpError("failed to init session cache");
+    return false;
   }
 
   return true;
 }
-
-void httpRemoveExpireSessions(HttpServer *pServer) {  
-  SHashMutableIterator *pIter = taosHashCreateIter(pServer->pSessionHash);
-
-  while (taosHashIterNext(pIter)) {
-    HttpSession *pSession = taosHashIterGet(pIter);
-    if (pSession == NULL) continue;
-
-    pthread_mutex_lock(&pServer->serverMutex);
-    if (httpSessionExpired(pSession)) {
-      httpResetSession(pSession);
-      taosHashRemove(pServer->pSessionHash, pSession->id, strlen(pSession->id));
-    }
-    pthread_mutex_unlock(&pServer->serverMutex);
-  }
-
-  taosHashDestroyIter(pIter);
-}
-
-void httpProcessSessionExpire(void *handle, void *tmrId) {
-  HttpServer *pServer = (HttpServer *)handle;
-  httpRemoveExpireSessions(pServer);
-  taosTmrReset(httpProcessSessionExpire, 60000, pServer, pServer->timerHandle, &pServer->expireTimer);
-}
\ No newline at end of file

src/plugins/http/src/httpSql.c

@@ -18,11 +18,12 @@
 #include "tnote.h"
 #include "taos.h"
 #include "tsclient.h"
-#include "http.h"
-#include "httpLog.h"
-#include "httpCode.h"
-#include "httpHandle.h"
+#include "httpInt.h"
+#include "httpContext.h"
+#include "httpSql.h"
 #include "httpResp.h"
+#include "httpAuth.h"
+#include "httpSession.h"
 
 void *taos_connect_a(char *ip, char *user, char *pass, char *db, uint16_t port, void (*fp)(void *, TAOS_RES *, int),
                      void *param, void **taos);
@@ -30,7 +31,7 @@ void httpProcessMultiSql(HttpContext *pContext);
 
 void httpProcessMultiSqlRetrieveCallBack(void *param, TAOS_RES *result, int numOfRows) {
   HttpContext *pContext = (HttpContext *)param;
-  if (pContext == NULL || pContext->signature != pContext) return;
+  if (pContext == NULL) return;
 
   HttpSqlCmds *     multiCmds = pContext->multiCmds;
   HttpEncodeMethod *encode = pContext->encodeMethod;
@@ -72,7 +73,7 @@ void httpProcessMultiSqlRetrieveCallBack(void *param, TAOS_RES *result, int numO
 
 void httpProcessMultiSqlCallBack(void *param, TAOS_RES *result, int code) {
   HttpContext *pContext = (HttpContext *)param;
-  if (pContext == NULL || pContext->signature != pContext) return;
+  if (pContext == NULL) return;
 
   HttpSqlCmds *     multiCmds = pContext->multiCmds;
   HttpEncodeMethod *encode = pContext->encodeMethod;
@@ -172,7 +173,7 @@ void httpProcessMultiSql(HttpContext *pContext) {
 }
 
 void httpProcessMultiSqlCmd(HttpContext *pContext) {
-  if (pContext == NULL || pContext->signature != pContext) return;
+  if (pContext == NULL) return;
 
   HttpSqlCmds *multiCmds = pContext->multiCmds;
   if (multiCmds == NULL || multiCmds->size <= 0 || multiCmds->pos >= multiCmds->size || multiCmds->pos < 0) {
@@ -192,7 +193,7 @@ void httpProcessMultiSqlCmd(HttpContext *pContext) {
 
 void httpProcessSingleSqlRetrieveCallBack(void *param, TAOS_RES *result, int numOfRows) {
   HttpContext *pContext = (HttpContext *)param;
-  if (pContext == NULL || pContext->signature != pContext) return;
+  if (pContext == NULL) return;
 
   HttpEncodeMethod *encode = pContext->encodeMethod;
 
@@ -230,7 +231,7 @@ void httpProcessSingleSqlRetrieveCallBack(void *param, TAOS_RES *result, int num
 
 void httpProcessSingleSqlCallBack(void *param, TAOS_RES *result, int code) {
   HttpContext *pContext = (HttpContext *)param;
-  if (pContext == NULL || pContext->signature != pContext) return;
+  if (pContext == NULL) return;
 
   HttpEncodeMethod *encode = pContext->encodeMethod;
 
@@ -354,7 +355,7 @@ void httpExecCmd(HttpContext *pContext) {
 
 void httpProcessRequestCb(void *param, TAOS_RES *result, int code) {
   HttpContext *pContext = param;
-  if (pContext == NULL || pContext->signature != pContext) return;
+  if (pContext == NULL) return;
 
   if (code < 0) {
     httpError("context:%p, fd:%d, ip:%s, user:%s, login error, code:%s", pContext, pContext->fd, pContext->ipstr,
@@ -383,16 +384,14 @@ void httpProcessRequestCb(void *param, TAOS_RES *result, int code) {
 }
 
 void httpProcessRequest(HttpContext *pContext) {
-  httpFetchSession(pContext);
+  httpGetSession(pContext);
 
-  if (pContext->session == NULL || pContext->session != pContext->session->signature ||
-      pContext->reqType == HTTP_REQTYPE_LOGIN) {
+  if (pContext->session == NULL || pContext->reqType == HTTP_REQTYPE_LOGIN) {
     taos_connect_a(NULL, pContext->user, pContext->pass, "", 0, httpProcessRequestCb, (void *)pContext,
                    &(pContext->taos));
     httpTrace("context:%p, fd:%d, ip:%s, user:%s, try connect tdengine, taos:%p", pContext, pContext->fd,
               pContext->ipstr, pContext->user, pContext->taos);
   } else {
-    httpAccessSession(pContext);
     httpExecCmd(pContext);
   }
 }

src/plugins/http/src/httpSystem.c

@@ -20,84 +20,64 @@
 #include "tsocket.h"
 #include "ttimer.h"
 #include "tadmin.h"
-#include "http.h"
-#include "httpCode.h"
-#include "httpHandle.h"
+#include "httpInt.h"
+#include "httpContext.h"
+#include "httpSession.h"
+#include "httpServer.h"
 #include "httpResp.h"
-#include "httpLog.h"
-#include "gcHandle.h"
 #include "httpHandle.h"
+#include "gcHandle.h"
 #include "restHandle.h"
 #include "tgHandle.h"
 
 #ifndef _ADMIN
-
 void adminInitHandle(HttpServer* pServer) {}
 void opInitHandle(HttpServer* pServer) {}
-
 #endif
 
-static HttpServer *httpServer = NULL;
+HttpServer tsHttpServer;
 void taosInitNote(int numOfNoteLines, int maxNotes, char* lable);
 
 int httpInitSystem() {
-  // taos_init();
-
-  httpServer = (HttpServer *)malloc(sizeof(HttpServer));
-  memset(httpServer, 0, sizeof(HttpServer));
-
-  strcpy(httpServer->label, "rest");
-  httpServer->serverIp = 0;
-  httpServer->serverPort = tsHttpPort;
-  httpServer->cacheContext = tsHttpCacheSessions;
-  httpServer->sessionExpire = tsHttpSessionExpire;
-  httpServer->numOfThreads = tsHttpMaxThreads;
-  httpServer->processData = httpProcessData;
+  strcpy(tsHttpServer.label, "rest");
+  tsHttpServer.serverIp = 0;
+  tsHttpServer.serverPort = tsHttpPort;
+  tsHttpServer.numOfThreads = tsHttpMaxThreads;
+  tsHttpServer.processData = httpProcessData;
 
-  pthread_mutex_init(&httpServer->serverMutex, NULL);
+  pthread_mutex_init(&tsHttpServer.serverMutex, NULL);
 
   if (tsHttpEnableRecordSql != 0) {
     taosInitNote(tsNumOfLogLines / 10, 1, (char*)"http_note");
   }
-  restInitHandle(httpServer);
-  adminInitHandle(httpServer);
-  gcInitHandle(httpServer);
-  tgInitHandle(httpServer);
-  opInitHandle(httpServer);
+  restInitHandle(&tsHttpServer);
+  adminInitHandle(&tsHttpServer);
+  gcInitHandle(&tsHttpServer);
+  tgInitHandle(&tsHttpServer);
+  opInitHandle(&tsHttpServer);
 
   return 0;
 }
 
 int httpStartSystem() {
-  httpPrint("starting to initialize http service ...");
+  httpPrint("start http server ...");
 
-  if (httpServer == NULL) {
-    httpError("http server is null");
-    httpInitSystem();
-  }
-
-  if (httpServer->pContextPool == NULL) {
-    httpServer->pContextPool = taosMemPoolInit(httpServer->cacheContext, sizeof(HttpContext));
-  }
-  if (httpServer->pContextPool == NULL) {
-    httpError("http init context pool failed");
+  if (tsHttpServer.status != HTTP_SERVER_INIT) {
+    httpError("http server is already started");
     return -1;
   }
 
-  if (httpServer->timerHandle == NULL) {
-    httpServer->timerHandle = taosTmrInit(tsHttpCacheSessions * 100 + 100, 200, 60000, "http");
-  }
-  if (httpServer->timerHandle == NULL) {
-    httpError("http init timer failed");
+  if (!httpInitContexts()) {
+    httpError("http init contexts failed");
     return -1;
   }
 
-  if (!httpInitAllSessions(httpServer)) {
+  if (!httpInitSessions()) {
     httpError("http init session failed");
     return -1;
   }
 
-  if (!httpInitConnect(httpServer)) {
+  if (!httpInitConnect()) {
     httpError("http init server failed");
     return -1;
   }
@@ -106,53 +86,23 @@ int httpStartSystem() {
 }
 
 void httpStopSystem() {
-  if (httpServer != NULL) {
-    httpServer->online = false;
-  }
+  tsHttpServer.status = HTTP_SERVER_CLOSING;
+  shutdown(tsHttpServer.fd, SHUT_RD);
   tgCleanupHandle();
 }
 
 void httpCleanUpSystem() {
-  httpPrint("http service cleanup");
+  httpPrint("http server cleanup");
   httpStopSystem();
 
-//#if 0
-  if (httpServer == NULL) {
-    return;
-  }
-
-  if (httpServer->expireTimer != NULL) {
-    taosTmrStopA(&(httpServer->expireTimer));
-  }
-
-  if (httpServer->timerHandle != NULL) {
-    taosTmrCleanUp(httpServer->timerHandle);
-    httpServer->timerHandle = NULL;
-  }
-
-  if (httpServer->pThreads != NULL) {
-    httpCleanUpConnect(httpServer);
-    httpServer->pThreads = NULL;
-  }
-  
-
-#if 0
-  httpRemoveAllSessions(httpServer);
+  httpCleanupContexts();
+  httpCleanUpSessions();
+  httpCleanUpConnect();
+  pthread_mutex_destroy(&tsHttpServer.serverMutex);
 
-  if (httpServer->pContextPool != NULL) {
-    taosMemPoolCleanUp(httpServer->pContextPool);
-    httpServer->pContextPool = NULL;
-  }
-
-  pthread_mutex_destroy(&httpServer->serverMutex);
-
-  tfree(httpServer);
-#endif
+  tsHttpServer.status = HTTP_SERVER_CLOSED;
 }
 
 int32_t httpGetReqCount() {
-  if (httpServer != NULL) {
-    return atomic_exchange_32(&httpServer->requestNum, 0);
-  }
-  return 0;
+  return atomic_exchange_32(&tsHttpServer.requestNum, 0);
 }

src/plugins/http/src/httpUtil.c

@@ -17,11 +17,10 @@
 #include "os.h"
 #include "tmd5.h"
 #include "taos.h"
-#include "http.h"
-#include "httpLog.h"
-#include "httpCode.h"
-#include "httpHandle.h"
+#include "httpInt.h"
 #include "httpResp.h"
+#include "httpSql.h"
+#include "httpUtil.h"
 
 bool httpCheckUsedbSql(char *sql) {
   if (strstr(sql, "use ") != NULL) {

src/plugins/http/src/tgHandle.c

@@ -18,9 +18,10 @@
 #include "tglobal.h"
 #include "taosdef.h"
 #include "taosmsg.h"
+#include "httpInt.h"
 #include "tgHandle.h"
 #include "tgJson.h"
-#include "httpLog.h"
+#include "cJSON.h"
 
 /*
  * taos.telegraf.cfg formats like

src/query/src/qExecutor.c

@@ -354,7 +354,7 @@ static SWindowResult *doSetTimeWindowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SWin
                                              int16_t bytes) {
   SQuery *pQuery = pRuntimeEnv->pQuery;
 
-  int32_t *p1 = (int32_t *)taosHashGet(pWindowResInfo->hashList, pData, bytes);
+  int32_t *p1 = (int32_t *) taosHashGet(pWindowResInfo->hashList, pData, bytes);
   if (p1 != NULL) {
     pWindowResInfo->curIndex = *p1;
   } else {  // more than the capacity, reallocate the resources
@@ -919,12 +919,25 @@ static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pDat
 
   SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf;
 
+  int64_t v = -1;
+  // not assign result buffer yet, add new result buffer
+  switch(type) {
+    case TSDB_DATA_TYPE_BOOL:
+    case TSDB_DATA_TYPE_TINYINT:  v = GET_INT8_VAL(pData);  break;
+    case TSDB_DATA_TYPE_SMALLINT: v = GET_INT16_VAL(pData); break;
+    case TSDB_DATA_TYPE_INT:      v = GET_INT32_VAL(pData); break;
+    case TSDB_DATA_TYPE_BIGINT:   v = GET_INT64_VAL(pData); break;
+  }
+
+//  assert(pRuntimeEnv->windowResInfo.hashList->size <= 2);
   SWindowResult *pWindowRes = doSetTimeWindowFromKey(pRuntimeEnv, &pRuntimeEnv->windowResInfo, pData, bytes);
   if (pWindowRes == NULL) {
     return -1;
   }
 
-  // not assign result buffer yet, add new result buffer
+  pWindowRes->window.skey = v;
+  pWindowRes->window.ekey = v;
+
   if (pWindowRes->pos.pageId == -1) {
     int32_t ret = addNewWindowResultBuf(pWindowRes, pResultBuf, GROUPRESULTID, pRuntimeEnv->numOfRowsPerPage);
     if (ret != 0) {
@@ -1022,12 +1035,16 @@ static bool functionNeedToExecute(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx
     return false;
   }
 
-  if (functionId == TSDB_FUNC_LAST_DST || functionId == TSDB_FUNC_LAST) {
-    return !QUERY_IS_ASC_QUERY(pQuery);
-  } else if (functionId == TSDB_FUNC_FIRST_DST || functionId == TSDB_FUNC_FIRST) {
+  if (functionId == TSDB_FUNC_FIRST_DST || functionId == TSDB_FUNC_FIRST) {
     return QUERY_IS_ASC_QUERY(pQuery);
   }
-  
+
+  // todo add comments
+  if ((functionId == TSDB_FUNC_LAST_DST || functionId == TSDB_FUNC_LAST)) {
+    return pCtx->param[0].i64Key == pQuery->order.order;
+//    return !QUERY_IS_ASC_QUERY(pQuery);
+  }
+
   // in the supplementary scan, only the following functions need to be executed
   if (IS_REVERSE_SCAN(pRuntimeEnv)) {
     return false;
@@ -1079,7 +1096,7 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS
 
   int32_t j = 0;
   int32_t offset = -1;
-  
+
   for (j = 0; j < pDataBlockInfo->rows; ++j) {
     offset = GET_COL_DATA_POS(pQuery, j, step);
 
@@ -1479,19 +1496,6 @@ static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) {
 
 static bool isQueryKilled(SQInfo *pQInfo) {
   return (pQInfo->code == TSDB_CODE_TSC_QUERY_CANCELLED);
-#if 0
-  /*
-   * check if the queried meter is going to be deleted.
-   * if it will be deleted soon, stop current query ASAP.
-   */
-  SMeterObj *pMeterObj = pQInfo->pObj;
-  if (vnodeIsMeterState(pMeterObj, TSDB_METER_STATE_DROPPING)) {
-    pQInfo->killed = 1;
-    return true;
-  }
-
-  return (pQInfo->killed == 1);
-#endif
 }
 
 static void setQueryKilled(SQInfo *pQInfo) { pQInfo->code = TSDB_CODE_TSC_QUERY_CANCELLED; }
@@ -1574,10 +1578,14 @@ static bool needReverseScan(SQuery *pQuery) {
       continue;
     }
 
-    if (((functionId == TSDB_FUNC_LAST || functionId == TSDB_FUNC_LAST_DST) && QUERY_IS_ASC_QUERY(pQuery)) ||
-        ((functionId == TSDB_FUNC_FIRST || functionId == TSDB_FUNC_FIRST_DST) && !QUERY_IS_ASC_QUERY(pQuery))) {
+    if ((functionId == TSDB_FUNC_FIRST || functionId == TSDB_FUNC_FIRST_DST) && !QUERY_IS_ASC_QUERY(pQuery)) {
       return true;
     }
+
+    if (functionId == TSDB_FUNC_LAST || functionId == TSDB_FUNC_LAST_DST) {
+      int32_t order = pQuery->pSelectExpr[i].base.arg->argValue.i64;
+      return order != pQuery->order.order;
+    }
   }
 
   return false;
@@ -2030,6 +2038,34 @@ int32_t binarySearchForKey(char *pValue, int num, TSKEY key, int order) {
   return midPos;
 }
 
+static void ensureOutputBufferSimple(SQueryRuntimeEnv* pRuntimeEnv, int32_t capacity) {
+  SQuery* pQuery = pRuntimeEnv->pQuery;
+
+  if (capacity < pQuery->rec.capacity) {
+    return;
+  }
+
+  for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
+    int32_t bytes = pQuery->pSelectExpr[i].bytes;
+    assert(bytes > 0 && capacity > 0);
+
+    char *tmp = realloc(pQuery->sdata[i], bytes * capacity + sizeof(tFilePage));
+    if (tmp == NULL) {  // todo handle the oom
+      assert(0);
+    } else {
+      pQuery->sdata[i] = (tFilePage *)tmp;
+    }
+
+    // set the pCtx output buffer position
+    pRuntimeEnv->pCtx[i].aOutputBuf = pQuery->sdata[i]->data;
+  }
+
+  qTrace("QInfo:%p realloc output buffer to inc output buffer from: %d rows to:%d rows", GET_QINFO_ADDR(pRuntimeEnv),
+         pQuery->rec.capacity, capacity);
+
+  pQuery->rec.capacity = capacity;
+}
+
 static void ensureOutputBuffer(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo* pBlockInfo) {
   // in case of prj/diff query, ensure the output buffer is sufficient to accommodate the results of current block
   SQuery* pQuery = pRuntimeEnv->pQuery;
@@ -2916,8 +2952,7 @@ void skipResults(SQueryRuntimeEnv *pRuntimeEnv) {
         pRuntimeEnv->pCtx[i].ptsOutputBuf = pRuntimeEnv->pCtx[0].aOutputBuf;
       }
     }
-  
-    
+
     updateNumOfResult(pRuntimeEnv, pQuery->rec.rows);
   }
 }
@@ -3054,7 +3089,7 @@ static void clearEnvAfterReverseScan(SQueryRuntimeEnv *pRuntimeEnv, SQueryStatus
   pQuery->window = pTableQueryInfo->win;
 }
 
-void scanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) {
+void scanOneTableDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) {
   SQInfo *pQInfo = (SQInfo *) GET_QINFO_ADDR(pRuntimeEnv);
   SQuery *pQuery = pRuntimeEnv->pQuery;
   STableQueryInfo *pTableQueryInfo = pQuery->current;
@@ -3496,18 +3531,32 @@ void copyFromWindowResToSData(SQInfo *pQInfo, SWindowResult *result) {
   assert(pQuery->rec.rows <= pQuery->rec.capacity);
 }
 
-static void updateWindowResNumOfRes(SQueryRuntimeEnv *pRuntimeEnv, STableQueryInfo *pTableQueryInfo) {
+static UNUSED_FUNC void updateWindowResNumOfRes(SQueryRuntimeEnv *pRuntimeEnv, STableQueryInfo *pTableQueryInfo) {
   SQuery *pQuery = pRuntimeEnv->pQuery;
 
   // update the number of result for each, only update the number of rows for the corresponding window result.
   if (pQuery->intervalTime == 0) {
-    int32_t g = pTableQueryInfo->groupIndex;
-    assert(pRuntimeEnv->windowResInfo.size > 0);
 
-    SWindowResult *pWindowRes = doSetTimeWindowFromKey(pRuntimeEnv, &pRuntimeEnv->windowResInfo, (char *)&g, sizeof(g));
-    if (pWindowRes->numOfRows == 0) {
-      pWindowRes->numOfRows = getNumOfResult(pRuntimeEnv);
+    for (int32_t i = 0; i < pRuntimeEnv->windowResInfo.size; ++i) {
+      SWindowResult *pResult = &pRuntimeEnv->windowResInfo.pResult[i];
+
+      for (int32_t j = 0; j < pQuery->numOfOutput; ++j) {
+        int32_t functionId = pRuntimeEnv->pCtx[j].functionId;
+        if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_TAGPRJ) {
+          continue;
+        }
+
+        pResult->numOfRows = MAX(pResult->numOfRows, pResult->resultInfo[j].numOfRes);
+      }
     }
+
+//    int32_t g = pTableQueryInfo->groupIndex;
+//    assert(pRuntimeEnv->windowResInfo.size > 0);
+//
+//    SWindowResult *pWindowRes = doSetTimeWindowFromKey(pRuntimeEnv, &pRuntimeEnv->windowResInfo, (char *)&g, sizeof(g));
+//    if (pWindowRes->numOfRows == 0) {
+//      pWindowRes->numOfRows = getNumOfResult(pRuntimeEnv);
+//    }
   }
 }
 
@@ -3519,7 +3568,7 @@ void stableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *
   SWindowResInfo * pWindowResInfo = &pTableQueryInfo->windowResInfo;
   pQuery->pos = QUERY_IS_ASC_QUERY(pQuery)? 0 : pDataBlockInfo->rows - 1;
 
-  if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTSBuf != NULL) {
+  if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTSBuf != NULL || isGroupbyNormalCol(pQuery->pGroupbyExpr)) {
     rowwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, pDataBlock);
   } else {
     blockwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, searchFn, pDataBlock);
@@ -4081,21 +4130,22 @@ static int64_t scanMultiTableDataBlocks(SQInfo *pQInfo) {
     SDataStatis *pStatis = NULL;
     SArray *pDataBlock = loadDataBlockOnDemand(pRuntimeEnv, pQueryHandle, &blockInfo, &pStatis);
 
-    if (!isIntervalQuery(pQuery)) {
-      int32_t step = QUERY_IS_ASC_QUERY(pQuery)? 1:-1;
-      setExecutionContext(pQInfo, &pTableQueryInfo->id, pTableQueryInfo->groupIndex, blockInfo.window.ekey + step);
-    } else {  // interval query
-      TSKEY nextKey = blockInfo.window.skey;
-      setIntervalQueryRange(pQInfo, nextKey);
-      /*int32_t ret = */setAdditionalInfo(pQInfo, &pTableQueryInfo->id, pTableQueryInfo);
+    if (!isGroupbyNormalCol(pQuery->pGroupbyExpr)) {
+      if (!isIntervalQuery(pQuery)) {
+        int32_t step = QUERY_IS_ASC_QUERY(pQuery)? 1:-1;
+        setExecutionContext(pQInfo, &pTableQueryInfo->id, pTableQueryInfo->groupIndex, blockInfo.window.ekey + step);
+      } else {  // interval query
+        TSKEY nextKey = blockInfo.window.skey;
+        setIntervalQueryRange(pQInfo, nextKey);
+        /*int32_t ret = */setAdditionalInfo(pQInfo, &pTableQueryInfo->id, pTableQueryInfo);
+      }
     }
 
     summary->totalRows += blockInfo.rows;
     stableApplyFunctionsOnBlock(pRuntimeEnv, &blockInfo, pStatis, pDataBlock, binarySearchForKey);
   
     qTrace("QInfo:%p check data block, uid:%"PRId64", tid:%d, brange:%" PRId64 "-%" PRId64 ", numOfRows:%d, lastKey:%" PRId64,
-           GET_QINFO_ADDR(pRuntimeEnv), blockInfo.uid, blockInfo.tid, blockInfo.window.skey, blockInfo.window.ekey,
-           blockInfo.rows, pQuery->current->lastKey);
+           pQInfo, blockInfo.uid, blockInfo.tid, blockInfo.window.skey, blockInfo.window.ekey, blockInfo.rows, pQuery->current->lastKey);
   }
 
   int64_t et = taosGetTimestampMs();
@@ -4220,7 +4270,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
       
       // here we simply set the first table as current table
       pQuery->current = ((SGroupItem*) taosArrayGet(group, 0))->info;
-      scanAllDataBlocks(pRuntimeEnv, pQuery->current->lastKey);
+      scanOneTableDataBlocks(pRuntimeEnv, pQuery->current->lastKey);
       
       int64_t numOfRes = getNumOfResult(pRuntimeEnv);
       if (numOfRes > 0) {
@@ -4233,10 +4283,84 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
 
       // enable execution for next table, when handling the projection query
       enableExecutionForNextTable(pRuntimeEnv);
+
+      if (pQuery->rec.rows >= pQuery->rec.capacity) {
+        setQueryStatus(pQuery, QUERY_RESBUF_FULL);
+        break;
+      }
+    }
+  } else if (isGroupbyNormalCol(pQuery->pGroupbyExpr)) { // group-by on normal columns query
+    while (pQInfo->groupIndex < numOfGroups) {
+      SArray* group = taosArrayGetP(pQInfo->groupInfo.pGroupList, pQInfo->groupIndex);
+
+      qTrace("QInfo:%p group by normal columns group:%d, total group:%d", pQInfo, pQInfo->groupIndex, numOfGroups);
+
+      STsdbQueryCond cond = {
+          .twindow = pQuery->window,
+          .colList = pQuery->colList,
+          .order   = pQuery->order.order,
+          .numOfCols = pQuery->numOfCols,
+      };
+
+      SArray *g1 = taosArrayInit(1, POINTER_BYTES);
+      SArray *tx = taosArrayClone(group);
+      taosArrayPush(g1, &tx);
+
+      STableGroupInfo gp = {.numOfTables = taosArrayGetSize(tx), .pGroupList = g1};
+
+      // include only current table
+      if (pRuntimeEnv->pQueryHandle != NULL) {
+        tsdbCleanupQueryHandle(pRuntimeEnv->pQueryHandle);
+        pRuntimeEnv->pQueryHandle = NULL;
+      }
+
+      pRuntimeEnv->pQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &gp, pQInfo);
+
+      SArray* s = tsdbGetQueriedTableIdList(pRuntimeEnv->pQueryHandle);
+      assert(taosArrayGetSize(s) >= 1);
+
+      setTagVal(pRuntimeEnv, (STableId*) taosArrayGet(s, 0), pQInfo->tsdb);
+
+      // here we simply set the first table as current table
+      scanMultiTableDataBlocks(pQInfo);
+      pQInfo->groupIndex += 1;
+
+      SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo;
+
+        // no results generated for current group, continue to try the next group
+      if (pWindowResInfo->size <= 0) {
+        continue;
+      }
+
+      for (int32_t i = 0; i < pWindowResInfo->size; ++i) {
+        SWindowStatus *pStatus = &pWindowResInfo->pResult[i].status;
+        pStatus->closed = true;  // enable return all results for group by normal columns
+
+        SWindowResult *pResult = &pWindowResInfo->pResult[i];
+        for (int32_t j = 0; j < pQuery->numOfOutput; ++j) {
+          pResult->numOfRows = MAX(pResult->numOfRows, pResult->resultInfo[j].numOfRes);
+        }
+      }
+
+      qTrace("QInfo:%p generated groupby columns results %d rows for group %d completed", pQInfo, pWindowResInfo->size,
+          pQInfo->groupIndex);
+      int32_t currentGroupIndex = pQInfo->groupIndex;
+
+      pQuery->rec.rows = 0;
+      pQInfo->groupIndex = 0;
+
+      ensureOutputBufferSimple(pRuntimeEnv, pWindowResInfo->size);
+      copyFromWindowResToSData(pQInfo, pWindowResInfo->pResult);
+
+      pQInfo->groupIndex = currentGroupIndex;  //restore the group index
+      assert(pQuery->rec.rows == pWindowResInfo->size);
+
+      clearClosedTimeWindow(pRuntimeEnv);
+      break;
     }
   } else {
     /*
-     * 1. super table projection query, 2. group-by on normal columns query, 3. ts-comp query
+     * 1. super table projection query, 2. ts-comp query
      * if the subgroup index is larger than 0, results generated by group by tbname,k is existed.
      * we need to return it to client in the first place.
      */
@@ -4283,7 +4407,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
         }
       }
 
-      scanAllDataBlocks(pRuntimeEnv, pQuery->current->lastKey);
+      scanOneTableDataBlocks(pRuntimeEnv, pQuery->current->lastKey);
       skipResults(pRuntimeEnv);
 
       // the limitation of output result is reached, set the query completed
@@ -4349,25 +4473,6 @@ static void sequentialTableProcess(SQInfo *pQInfo) {
     pRuntimeEnv->cur = pRuntimeEnv->pTSBuf->cur;
   }
 
-  // todo refactor
-  if (isGroupbyNormalCol(pQuery->pGroupbyExpr)) {
-    SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo;
-
-    for (int32_t i = 0; i < pWindowResInfo->size; ++i) {
-      SWindowStatus *pStatus = &pWindowResInfo->pResult[i].status;
-      pStatus->closed = true;  // enable return all results for group by normal columns
-
-      SWindowResult *pResult = &pWindowResInfo->pResult[i];
-      for (int32_t j = 0; j < pQuery->numOfOutput; ++j) {
-        pResult->numOfRows = MAX(pResult->numOfRows, pResult->resultInfo[j].numOfRes);
-      }
-    }
-
-    pQInfo->groupIndex = 0;
-    pQuery->rec.rows = 0;
-    copyFromWindowResToSData(pQInfo, pWindowResInfo->pResult);
-  }
-
   qTrace(
       "QInfo %p numOfTables:%d, index:%d, numOfGroups:%d, %d points returned, total:%"PRId64", offset:%" PRId64,
       pQInfo, pQInfo->groupInfo.numOfTables, pQInfo->tableIndex, numOfGroups, pQuery->rec.rows, pQuery->rec.total,
@@ -4449,7 +4554,6 @@ static void multiTableQueryProcess(SQInfo *pQInfo) {
      */
     if (isIntervalQuery(pQuery)) {
       copyResToQueryResultBuf(pQInfo, pQuery);
-
 #ifdef _DEBUG_VIEW
       displayInterResult(pQuery->sdata, pRuntimeEnv, pQuery->sdata[0]->num);
 #endif
@@ -4527,7 +4631,7 @@ static void tableFixedOutputProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo)
   
   pQuery->current = pTableInfo;  // set current query table info
   
-  scanAllDataBlocks(pRuntimeEnv, pTableInfo->lastKey);
+  scanOneTableDataBlocks(pRuntimeEnv, pTableInfo->lastKey);
   finalizeQueryResult(pRuntimeEnv);
 
   if (isQueryKilled(pQInfo)) {
@@ -4560,7 +4664,7 @@ static void tableMultiOutputProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo)
   }
 
   while (1) {
-    scanAllDataBlocks(pRuntimeEnv, pQuery->current->lastKey);
+    scanOneTableDataBlocks(pRuntimeEnv, pQuery->current->lastKey);
     finalizeQueryResult(pRuntimeEnv);
 
     if (isQueryKilled(pQInfo)) {
@@ -4607,7 +4711,7 @@ static void tableIntervalProcessImpl(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start)
   SQuery *pQuery = pRuntimeEnv->pQuery;
 
   while (1) {
-    scanAllDataBlocks(pRuntimeEnv, start);
+    scanOneTableDataBlocks(pRuntimeEnv, start);
 
     if (isQueryKilled(GET_QINFO_ADDR(pRuntimeEnv))) {
       return;

src/query/src/qUtil.c

@@ -113,7 +113,9 @@ void clearFirstNTimeWindow(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) {
   for (int32_t i = 0; i < num; ++i) {
     SWindowResult *pResult = &pWindowResInfo->pResult[i];
     if (pResult->status.closed) {  // remove the window slot from hash table
-      taosHashRemove(pWindowResInfo->hashList, (const char *)&pResult->window.skey, TSDB_KEYSIZE);
+      taosHashRemove(pWindowResInfo->hashList, (const char *)&pResult->window.skey, pWindowResInfo->type);
+      printf("remove ============>%ld, remain size:%ld\n", pResult->window.skey, pWindowResInfo->hashList->size);
+
     } else {
       break;
     }
@@ -133,14 +135,16 @@ void clearFirstNTimeWindow(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) {
   }
   
   pWindowResInfo->size = remain;
-  
+  printf("---------------size:%ld\n", taosHashGetSize(pWindowResInfo->hashList));
   for (int32_t k = 0; k < pWindowResInfo->size; ++k) {
     SWindowResult *pResult = &pWindowResInfo->pResult[k];
-    int32_t *p = (int32_t *)taosHashGet(pWindowResInfo->hashList, (const char *)&pResult->window.skey, TSDB_KEYSIZE);
+    int32_t *p = (int32_t *)taosHashGet(pWindowResInfo->hashList, (const char *)&pResult->window.skey,
+        tDataTypeDesc[pWindowResInfo->type].nSize);
     
     int32_t  v = (*p - num);
     assert(v >= 0 && v <= pWindowResInfo->size);
-    taosHashPut(pWindowResInfo->hashList, (char *)&pResult->window.skey, TSDB_KEYSIZE, (char *)&v, sizeof(int32_t));
+    taosHashPut(pWindowResInfo->hashList, (char *)&pResult->window.skey, tDataTypeDesc[pWindowResInfo->type].nSize,
+        (char *)&v, sizeof(int32_t));
   }
   
   pWindowResInfo->curIndex = -1;

src/query/src/tvariant.c

@@ -800,12 +800,13 @@ int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool inclu
       break;
     }
     case TSDB_DATA_TYPE_NCHAR: {
+      int32_t newlen = 0;
       if (!includeLengthPrefix) {
         if (pVariant->nType == TSDB_DATA_TYPE_NULL) {
           *(uint32_t *)payload = TSDB_DATA_NCHAR_NULL;
         } else {
           if (pVariant->nType != TSDB_DATA_TYPE_NCHAR) {
-            toNchar(pVariant, &payload, &pVariant->nLen);
+            toNchar(pVariant, &payload, &newlen);
           } else {
             wcsncpy((wchar_t *)payload, pVariant->wpz, pVariant->nLen);
           }
@@ -817,12 +818,13 @@ int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool inclu
           char *p = varDataVal(payload);
 
           if (pVariant->nType != TSDB_DATA_TYPE_NCHAR) {
-            toNchar(pVariant, &p, &pVariant->nLen);
+            toNchar(pVariant, &p, &newlen);
           } else {
             wcsncpy((wchar_t *)p, pVariant->wpz, pVariant->nLen);
+            newlen = pVariant->nLen;
           }
 
-          varDataSetLen(payload, pVariant->nLen);  // the length may be changed after toNchar function called
+          varDataSetLen(payload, newlen);  // the length may be changed after toNchar function called
           assert(p == varDataVal(payload));
         }
       }

src/rpc/src/rpcMain.c

@@ -73,6 +73,7 @@ typedef struct {
   SRpcInfo *pRpc;       // associated SRpcInfo
   SRpcIpSet ipSet;      // ip list provided by app
   void     *ahandle;    // handle provided by app
+  struct SRpcConn *pConn; // pConn allocated
   char      msgType;    // message type
   uint8_t  *pCont;      // content provided by app
   int32_t   contLen;    // content length
@@ -339,7 +340,7 @@ void *rpcReallocCont(void *ptr, int contLen) {
   return start + sizeof(SRpcReqContext) + sizeof(SRpcHead);
 }
 
-void rpcSendRequest(void *shandle, const SRpcIpSet *pIpSet, const SRpcMsg *pMsg) {
+void *rpcSendRequest(void *shandle, const SRpcIpSet *pIpSet, const SRpcMsg *pMsg) {
   SRpcInfo       *pRpc = (SRpcInfo *)shandle;
   SRpcReqContext *pContext;
 
@@ -367,7 +368,7 @@ void rpcSendRequest(void *shandle, const SRpcIpSet *pIpSet, const SRpcMsg *pMsg)
   
   rpcSendReqToServer(pRpc, pContext);
 
-  return;
+  return pContext;
 }
 
 void rpcSendResponse(const SRpcMsg *pRsp) {
@@ -501,6 +502,19 @@ int rpcReportProgress(void *handle, char *pCont, int contLen) {
   return -1;
 }
 
+/* todo: cancel process may have race condition, pContext may have been released 
+   just before app calls the rpcCancelRequest */
+void rpcCancelRequest(void *handle) {
+  SRpcReqContext *pContext = handle;
+
+  if (pContext->pConn) {
+    tTrace("%s, app trys to cancel request", pContext->pConn->info);
+    rpcCloseConn(pContext->pConn);
+    pContext->pConn = NULL;
+    rpcFreeCont(pContext->pCont);
+  }
+}
+
 static void rpcFreeMsg(void *msg) {
   if ( msg ) {
     char *temp = (char *)msg - sizeof(SRpcReqContext);
@@ -874,6 +888,7 @@ static void rpcReportBrokenLinkToServer(SRpcConn *pConn) {
   SRpcMsg rpcMsg;
   rpcMsg.pCont = pConn->pReqMsg;     // pReqMsg is re-used to store the APP context from server
   rpcMsg.contLen = pConn->reqMsgLen; // reqMsgLen is re-used to store the APP context length
+  rpcMsg.ahandle = pConn->ahandle;
   rpcMsg.handle = pConn;
   rpcMsg.msgType = pConn->inType;
   rpcMsg.code = TSDB_CODE_RPC_NETWORK_UNAVAIL; 
@@ -909,7 +924,7 @@ static void *rpcProcessMsgFromPeer(SRecvInfo *pRecv) {
   // underlying UDP layer does not know it is server or client
   pRecv->connType = pRecv->connType | pRpc->connType;  
 
-  if (pRecv->ip == 0) {
+  if (pRecv->msg == NULL) {
     rpcProcessBrokenLink(pConn);
     return NULL;
   }
@@ -942,6 +957,7 @@ static void *rpcProcessMsgFromPeer(SRecvInfo *pRecv) {
 static void rpcNotifyClient(SRpcReqContext *pContext, SRpcMsg *pMsg) {
   SRpcInfo       *pRpc = pContext->pRpc;
 
+  pContext->pConn = NULL;
   if (pContext->pRsp) { 
     // for synchronous API
     memcpy(pContext->pSet, &pContext->ipSet, sizeof(SRpcIpSet));
@@ -1110,6 +1126,7 @@ static void rpcSendReqToServer(SRpcInfo *pRpc, SRpcReqContext *pContext) {
     return;
   }
 
+  pContext->pConn = pConn;
   pConn->ahandle = pContext->ahandle;
   rpcLockConn(pConn);
 

src/util/inc/ttime.h

@@ -56,7 +56,7 @@ static FORCE_INLINE int64_t taosGetTimestamp(int32_t precision) {
 
 int32_t getTimestampInUsFromStr(char* token, int32_t tokenlen, int64_t* ts);
 
-int32_t taosParseTime(char* timestr, int64_t* time, int32_t len, int32_t timePrec);
+int32_t taosParseTime(char* timestr, int64_t* time, int32_t len, int32_t timePrec, int8_t dayligth);
 void deltaToUtcInitOnce();
 
 #ifdef __cplusplus

src/util/src/tcache.c

@@ -63,10 +63,12 @@ static FORCE_INLINE void __cache_lock_destroy(SCacheObj *pCacheObj) {
 #endif
 }
 
+#if 0
 static FORCE_INLINE void taosFreeNode(void *data) {
   SCacheDataNode *pNode = *(SCacheDataNode **)data;
   free(pNode);
 }
+#endif
 
 /**
  * @param key      key of object for hash, usually a null-terminated string
@@ -241,7 +243,7 @@ SCacheObj *taosCacheInitWithCb(int64_t refreshTime, void (*freeCb)(void *data))
   }
   
   // set free cache node callback function for hash table
-  taosHashSetFreecb(pCacheObj->pHashTable, taosFreeNode);
+  // taosHashSetFreecb(pCacheObj->pHashTable, taosFreeNode);
   
   pCacheObj->freeFp = freeCb;
   pCacheObj->refreshTime = refreshTime * 1000;
@@ -565,7 +567,17 @@ void taosTrashCanEmpty(SCacheObj *pCacheObj, bool force) {
 
 void doCleanupDataCache(SCacheObj *pCacheObj) {
   __cache_wr_lock(pCacheObj);
-  taosHashCleanup(pCacheObj->pHashTable);
+
+  SHashMutableIterator *pIter = taosHashCreateIter(pCacheObj->pHashTable);
+  while (taosHashIterNext(pIter)) {
+    SCacheDataNode *pNode = *(SCacheDataNode **)taosHashIterGet(pIter);
+    // if (pNode->expiredTime <= expiredTime && T_REF_VAL_GET(pNode) <= 0) {
+    taosCacheReleaseNode(pCacheObj, pNode);
+    //}
+  }
+  taosHashDestroyIter(pIter);
+
+  taosHashCleanup(pCacheObj->pHashTable); 
   __cache_unlock(pCacheObj);
 
   taosTrashCanEmpty(pCacheObj, true);

src/util/src/ttime.c

@@ -24,7 +24,6 @@
 #include "taosdef.h"
 #include "ttime.h"
 #include "tutil.h"
-
 /*
  * mktime64 - Converts date to seconds.
  * Converts Gregorian date to seconds since 1970-01-01 00:00:00.
@@ -119,15 +118,21 @@ static int month[12] = {
 static int64_t parseFraction(char* str, char** end, int32_t timePrec);
 static int32_t parseTimeWithTz(char* timestr, int64_t* time, int32_t timePrec);
 static int32_t parseLocaltime(char* timestr, int64_t* time, int32_t timePrec);
+static int32_t parseLocaltimeWithDst(char* timestr, int64_t* time, int32_t timePrec);
+
+static int32_t (*parseLocaltimeFp[]) (char* timestr, int64_t* time, int32_t timePrec) = {
+  parseLocaltime,
+  parseLocaltimeWithDst
+}; 
 
 int32_t taosGetTimestampSec() { return (int32_t)time(NULL); }
 
-int32_t taosParseTime(char* timestr, int64_t* time, int32_t len, int32_t timePrec) {
+int32_t taosParseTime(char* timestr, int64_t* time, int32_t len, int32_t timePrec, int8_t daylight) {
   /* parse datatime string in with tz */
   if (strnchr(timestr, 'T', len, false) != NULL) {
     return parseTimeWithTz(timestr, time, timePrec);
   } else {
-    return parseLocaltime(timestr, time, timePrec);
+    return (*parseLocaltimeFp[daylight])(timestr, time, timePrec);
   }
 }
 
@@ -304,9 +309,6 @@ int32_t parseLocaltime(char* timestr, int64_t* time, int32_t timePrec) {
     return -1;
   }
 
-  /* mktime will be affected by TZ, set by using taos_options */
-  //int64_t seconds = mktime(&tm);
-  //int64_t seconds = (int64_t)user_mktime(&tm);
   int64_t seconds = user_mktime64(tm.tm_year+1900, tm.tm_mon+1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec);
   
   int64_t fraction = 0;
@@ -324,6 +326,32 @@ int32_t parseLocaltime(char* timestr, int64_t* time, int32_t timePrec) {
   return 0;
 }
 
+int32_t parseLocaltimeWithDst(char* timestr, int64_t* time, int32_t timePrec) {
+  *time = 0;
+  struct tm tm = {0};
+  tm.tm_isdst = -1;
+
+  char* str = strptime(timestr, "%Y-%m-%d %H:%M:%S", &tm);
+  if (str == NULL) {
+    return -1;
+  }
+
+  /* mktime will be affected by TZ, set by using taos_options */
+  int64_t seconds = mktime(&tm);
+  
+  int64_t fraction = 0;
+
+  if (*str == '.') {
+    /* parse the second fraction part */
+    if ((fraction = parseFraction(str + 1, &str, timePrec)) < 0) {
+      return -1;
+    }
+  }
+
+  int64_t factor = (timePrec == TSDB_TIME_PRECISION_MILLI) ? 1000 : 1000000;
+  *time = factor * seconds + fraction;
+  return 0;
+}
 static int32_t getTimestampInUsFromStrImpl(int64_t val, char unit, int64_t* result) {
   *result = val;
 

src/vnode/src/vnodeMain.c

@@ -321,6 +321,22 @@ void vnodeRelease(void *pVnodeRaw) {
     return;
   }
 
+  if (pVnode->tsdb)
+    tsdbCloseRepo(pVnode->tsdb, 1);
+  pVnode->tsdb = NULL;
+
+  if (pVnode->wal) 
+    walClose(pVnode->wal);
+  pVnode->wal = NULL;
+
+  if (pVnode->wqueue) 
+    dnodeFreeVnodeWqueue(pVnode->wqueue);
+  pVnode->wqueue = NULL;
+
+  if (pVnode->rqueue) 
+    dnodeFreeVnodeRqueue(pVnode->rqueue);
+  pVnode->rqueue = NULL;
+ 
   tfree(pVnode->rootDir);
 
   if (pVnode->status == TAOS_VN_STATUS_DELETING) {
@@ -411,33 +427,21 @@ void vnodeBuildStatusMsg(void *param) {
 }
 
 static void vnodeCleanUp(SVnodeObj *pVnode) {
+  // remove from hash, so new messages wont be consumed
   taosHashRemove(tsDnodeVnodesHash, (const char *)&pVnode->vgId, sizeof(int32_t));
 
+  // stop replication module
   if (pVnode->sync) {
     syncStop(pVnode->sync);
     pVnode->sync = NULL;
   }
 
-  if (pVnode->tsdb)
-    tsdbCloseRepo(pVnode->tsdb, 1);
-  pVnode->tsdb = NULL;
-
-  if (pVnode->wal) 
-    walClose(pVnode->wal);
-  pVnode->wal = NULL;
-
+  // stop continuous query
   if (pVnode->cq) 
     cqClose(pVnode->cq);
   pVnode->cq = NULL;
 
-  if (pVnode->wqueue) 
-    dnodeFreeVnodeWqueue(pVnode->wqueue);
-  pVnode->wqueue = NULL;
-
-  if (pVnode->rqueue) 
-    dnodeFreeVnodeRqueue(pVnode->rqueue);
-  pVnode->rqueue = NULL;
- 
+  // release local resources only after cutting off outside connections
   vnodeRelease(pVnode);
 }
 

src/vnode/src/vnodeRead.c

@@ -61,7 +61,7 @@ int32_t vnodeProcessRead(void *param, SReadMsg *pReadMsg) {
 
 // notify connection(handle) that current qhandle is created, if current connection from
 // client is broken, the query needs to be killed immediately.
-static void vnodeNotifyCurrentQhandle(void* handle, void* qhandle, int32_t vgId) {
+static int32_t vnodeNotifyCurrentQhandle(void* handle, void* qhandle, int32_t vgId) {
   SRetrieveTableMsg* killQueryMsg = rpcMallocCont(sizeof(SRetrieveTableMsg));
   killQueryMsg->qhandle = htobe64((uint64_t) qhandle);
   killQueryMsg->free = htons(1);
@@ -69,7 +69,7 @@ static void vnodeNotifyCurrentQhandle(void* handle, void* qhandle, int32_t vgId)
   killQueryMsg->header.contLen = htonl(sizeof(SRetrieveTableMsg));
 
   vTrace("QInfo:%p register qhandle to connect:%p", qhandle, handle);
-  rpcReportProgress(handle, (char*) killQueryMsg, sizeof(SRetrieveTableMsg));
+  return rpcReportProgress(handle, (char*) killQueryMsg, sizeof(SRetrieveTableMsg));
 }
 
 static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
@@ -106,7 +106,17 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
     pRet->len = sizeof(SQueryTableRsp);
     pRet->rsp = pRsp;
 
-    vnodeNotifyCurrentQhandle(pReadMsg->rpcMsg.handle, pQInfo, pVnode->vgId);
+    // current connect is broken
+    if (vnodeNotifyCurrentQhandle(pReadMsg->rpcMsg.handle, pQInfo, pVnode->vgId) != TSDB_CODE_SUCCESS) {
+      vError("vgId:%d, QInfo:%p, dnode query discarded since link is broken, %p", pVnode->vgId, pQInfo, pReadMsg->rpcMsg.handle);
+      pRsp->code = TSDB_CODE_RPC_NETWORK_UNAVAIL;
+
+      //NOTE: there two refcount, needs to kill twice, todo refactor
+      qKillQuery(pQInfo);
+      qKillQuery(pQInfo);
+
+      return pRsp->code;
+    }
 
     vTrace("vgId:%d, QInfo:%p, dnode query msg disposed", pVnode->vgId, pQInfo);
   } else {

tests/How-To-Run-Test-And-How-To-Add-New-Test-Case.md

@@ -11,6 +11,8 @@
 4.  pip install ../src/connector/python/linux/python2 ; pip3 install
     ../src/connector/python/linux/python3
 
+5.  pip install numpy; pip3 install numpy    
+
 >   Note: Both Python2 and Python3 are currently supported by the Python test
 >   framework. Since Python2 is no longer officially supported by Python Software
 >   Foundation since January 1, 2020, it is recommended that subsequent test case

tests/pytest/fulltest.sh

@@ -134,6 +134,10 @@ python3 ./test.py -f query/filter.py
 python3 ./test.py -f query/filterCombo.py
 python3 ./test.py -f query/queryNormal.py
 python3 ./test.py -f query/queryError.py
+python3 ./test.py -f query/filterAllIntTypes.py
+python3 ./test.py -f query/filterFloatAndDouble.py
+python3 ./test.py -f query/filterOtherTypes.py
+python3 ./test.py -f query/querySort.py
 
 #stream
 python3 ./test.py -f stream/stream1.py

tests/pytest/query/filterAllIntTypes.py

+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+from util.log import *
+from util.cases import *
+from util.sql import *
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor())
+    
+        self.powers = [7, 15, 31, 63]
+        self.types = ["tinyint", "smallint", "int", "bigint"]
+        self.rowNum = 10
+        self.ts = 1537146000000       
+        
+    def run(self):
+        tdSql.prepare()
+
+        for i in range(len(self.powers)):
+            curType = self.types[i]
+            print("======= Verify filter for %s type =========" % (curType))
+            tdLog.debug(
+                "create table st%s(ts timestamp, num %s) tags(id %s)" % (curType, curType, curType))
+            tdSql.execute(
+                "create table st%s(ts timestamp, num %s) tags(id %s)" % (curType, curType, curType))
+            
+            #create 10 tables, insert 10 rows for each table
+            for j in range(self.rowNum):            
+                tdSql.execute("create table st%s%d using st%s tags(%d)" % (curType, j + 1, curType, j + 1))
+                for k in range(self.rowNum):                
+                    tdSql.execute("insert into st%s%d values(%d, %d)" % (curType, j + 1, self.ts + k + 1, j * 10 + k + 1))
+            
+            tdSql.error("insert into st%s10 values(%d, %d)" % (curType, self.ts + 11, pow(2, self.powers[i])))
+            tdSql.execute("insert into st%s10 values(%d, %d)" % (curType, self.ts + 12, pow(2, self.powers[i]) - 1))
+            tdSql.error("insert into st%s10 values(%d, %d)" % (curType, self.ts + 13, pow(-2, self.powers[i])))
+            tdSql.execute("insert into st%s10 values(%d, %d)" % (curType, self.ts + 14, pow(-2, self.powers[i]) + 1)) 
+
+            # > for int type on column
+            tdSql.query("select * from st%s where num > 50" % curType)
+            tdSql.checkRows(51)
+
+            # >= for int type on column
+            tdSql.query("select * from st%s where num >= 50" % curType)
+            tdSql.checkRows(52)
+
+            # = for int type on column
+            tdSql.query("select * from st%s where num = 50" % curType)
+            tdSql.checkRows(1)
+
+            # < for int type on column
+            tdSql.query("select * from st%s where num < 50" % curType)
+            tdSql.checkRows(50)
+
+            # <= for int type on column
+            tdSql.query("select * from st%s where num <= 50" % curType)
+            tdSql.checkRows(51)
+
+            # <> for int type on column
+            tdSql.query("select * from st%s where num <> 50" % curType)
+            tdSql.checkRows(101)
+
+            # != for int type on column
+            tdSql.query("select * from st%s where num != 50" % curType)
+            tdSql.checkRows(101)
+
+            # > for int type on tag
+            tdSql.query("select * from st%s where id > 5" % curType)
+            tdSql.checkRows(52)
+
+            # >= for int type on tag
+            tdSql.query("select * from st%s where id >= 5" % curType)
+            tdSql.checkRows(62)
+
+            # = for int type on tag
+            tdSql.query("select * from st%s where id = 5" % curType)
+            tdSql.checkRows(10)
+
+            # < for int type on tag
+            tdSql.query("select * from st%s where id < 5" % curType)
+            tdSql.checkRows(40)
+
+            # <= for int type on tag
+            tdSql.query("select * from st%s where id <= 5" % curType)
+            tdSql.checkRows(50)
+
+            # <> for int type on tag
+            tdSql.query("select * from st%s where id <> 5" % curType)
+            tdSql.checkRows(92)
+
+            # != for int type on tag
+            tdSql.query("select * from st%s where id != 5" % curType)
+            tdSql.checkRows(92)            
+
+            print("======= Verify filter for %s type finished =========" % curType)
+                                                   
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())

tests/pytest/query/filterFloatAndDouble.py

+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+from util.log import *
+from util.cases import *
+from util.sql import *
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor())    
+
+        self.rowNum = 10
+        self.ts = 1537146000000
+        
+    def run(self):
+        tdSql.prepare()
+
+        print("======= Verify filter for float and double type =========")
+        tdLog.debug(
+                "create table st(ts timestamp, num float, speed double) tags(tagcol1 float, tagcol2 double)")
+        tdSql.execute(
+                "create table st(ts timestamp, num float, speed double) tags(tagcol1 float, tagcol2 double)")
+                
+        for j in range(self.rowNum):             
+            tdSql.execute(
+                "insert into st1 using st tags(1.1, 2.3) values(%d, %f, %f)" % (self.ts + j + 1, 1.1 * (j + 1), 2.3 * (j + 1)))    
+
+        # > for float type on column
+        tdSql.query("select * from st where num > 5.5")
+        tdSql.checkRows(5)
+        
+        # >= for float type on column
+        tdSql.query("select * from st where num >= 5.5")
+        tdSql.checkRows(6)
+
+        # = for float type on column
+        tdSql.query("select * from st where num = 5.5")
+        tdSql.checkRows(1)
+
+        # <> for float type on column
+        tdSql.query("select * from st where num <> 5.5")
+        tdSql.checkRows(9)
+
+        # != for float type on column
+        tdSql.query("select * from st where num != 5.5")
+        tdSql.checkRows(9)
+
+        # <= for float type on column
+        tdSql.query("select * from st where num <= 5.5")
+        tdSql.checkRows(5)
+
+        # < for float type on column
+        tdSql.query("select * from st where num < 5.5")
+        tdSql.checkRows(4)
+
+        # > for float type on tag
+        tdSql.query("select * from st where tagcol1 > 1.1")
+        tdSql.checkRows(0)
+        
+        # >= for float type on tag
+        tdSql.query("select * from st where tagcol1 >= 1.1")
+        tdSql.checkRows(10)
+        
+        # = for float type on tag
+        tdSql.query("select * from st where tagcol1 = 1.1")
+        tdSql.checkRows(10)
+
+        # <> for float type on tag
+        tdSql.query("select * from st where tagcol1 <> 1.1")
+        tdSql.checkRows(0)
+
+        # != for float type on tag
+        tdSql.query("select * from st where tagcol1 != 1.1")
+        tdSql.checkRows(0)
+        
+        # <= for float type on tag
+        tdSql.query("select * from st where tagcol1 <= 1.1")
+        tdSql.checkRows(10)
+
+        # < for float type on tag
+        tdSql.query("select * from st where tagcol1 < 1.1")
+        tdSql.checkRows(0)
+        
+        # > for double type on column
+        tdSql.query("select * from st where speed > 11.5")
+        tdSql.checkRows(5)
+        
+        # >= for double type on column
+        tdSql.query("select * from st where speed >= 11.5")
+        tdSql.checkRows(6)
+
+        # = for double type on column
+        tdSql.query("select * from st where speed = 11.5")
+        tdSql.checkRows(1)
+
+        # <> for double type on column
+        tdSql.query("select * from st where speed <> 11.5")
+        tdSql.checkRows(9)
+
+        # != for double type on column
+        tdSql.query("select * from st where speed != 11.5")
+        tdSql.checkRows(9)
+
+        # <= for double type on column
+        tdSql.query("select * from st where speed <= 11.5")
+        tdSql.checkRows(5)
+
+        # < for double type on column
+        tdSql.query("select * from st where speed < 11.5")
+        tdSql.checkRows(4)
+
+        # > for double type on tag
+        tdSql.query("select * from st where tagcol2 > 2.3")
+        tdSql.checkRows(0)
+        
+        # >= for double type on tag
+        tdSql.query("select * from st where tagcol2 >= 2.3")
+        tdSql.checkRows(10)
+        
+        # = for double type on tag
+        tdSql.query("select * from st where tagcol2 = 2.3")
+        tdSql.checkRows(10)
+
+        # <> for double type on tag
+        tdSql.query("select * from st where tagcol2 <> 2.3")
+        tdSql.checkRows(0)
+
+        # != for double type on tag
+        tdSql.query("select * from st where tagcol2 != 2.3")
+        tdSql.checkRows(0)
+        
+        # <= for double type on tag
+        tdSql.query("select * from st where tagcol2 <= 2.3")
+        tdSql.checkRows(10)
+
+        # < for double type on tag
+        tdSql.query("select * from st where tagcol2 < 2.3")
+        tdSql.checkRows(0)
+                                           
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())

tests/pytest/query/filterOtherTypes.py

+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+from util.log import *
+from util.cases import *
+from util.sql import *
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor())   
+        
+        self.ts = 1537146000000
+        
+    def run(self):
+        tdSql.prepare()
+                
+        print("======= Verify filter for bool, nchar and binary type =========")
+        tdLog.debug(
+            "create table st(ts timestamp, tbcol1 bool, tbcol2 nchar(10), tbcol3 binary(20)) tags(tagcol1 bool, tagcol2 nchar(10), tagcol3 binary(10))")
+        tdSql.execute(
+            "create table st(ts timestamp, tbcol1 bool, tbcol2 nchar(10), tbcol3 binary(20)) tags(tagcol1 bool, tagcol2 nchar(10), tagcol3 binary(10))")                
+
+        tdSql.execute("create table st1 using st tags(true, 'table1', '水表')")
+        for i in range(1, 6):                
+            tdSql.execute("insert into st1 values(%d, %d, 'taosdata%d', '涛思数据%d')" % (self.ts + i, i % 2, i, i)) 
+
+        tdSql.execute("create table st2 using st tags(false, 'table2', '电表')")
+        for i in range(6, 11):                
+            tdSql.execute("insert into st2 values(%d, %d, 'taosdata%d', '涛思数据%d')" % (self.ts + i, i % 2, i, i))  
+
+        # =============Verify stable columns====================
+        # > for bool type on column
+        tdSql.error("select * from st where tbcol1 > false")
+
+        # >= for bool type on column
+        tdSql.error("select * from st where tbcol1 >= false") 
+
+        # = for bool type on column
+        tdSql.query("select * from st where tbcol1 = false")
+        tdSql.checkRows(5)
+
+        # <> for bool type on column
+        tdSql.query("select * from st where tbcol1 <> true")
+        tdSql.checkRows(5)
+
+        # != for bool type on column
+        tdSql.query("select * from st where tbcol1 != true")
+        tdSql.checkRows(5)
+
+        # > for bool type on column
+        tdSql.error("select * from st where tbcol1 < true")
+
+        # >= for bool type on column
+        tdSql.error("select * from st where tbcol1 <= true")
+
+        # % for bool type on column
+        tdSql.error("select * from st where tbcol1 like '%'")
+
+        # _ for bool type on column
+        tdSql.error("select * from st where tbcol1 like '____'")
+
+        # > for nchar type on column
+        tdSql.error("select * from st where tbcol2 > 'taosdata'")
+
+        # >= for nchar type on column
+        tdSql.error("select * from st where tbcol2 >= 'taosdata'") 
+
+        # = for nchar type on column
+        tdSql.query("select * from st where tbcol2 = 'taosdata1'")
+        tdSql.checkRows(1)
+
+        # <> for nchar type on column
+        tdSql.query("select * from st where tbcol2 <> 'taosdata1'")        
+        tdSql.checkRows(9)
+
+        # != for nchar type on column
+        tdSql.query("select * from st where tbcol2 != 'taosdata1'")        
+        tdSql.checkRows(9)
+
+        # > for nchar type on column
+        tdSql.error("select * from st where tbcol2 < 'taodata'")
+
+        # >= for nchar type on column
+        tdSql.error("select * from st where tbcol2 <= 'taodata'")
+
+        # % for nchar type on column case 1
+        tdSql.query("select * from st where tbcol2 like '%'")        
+        tdSql.checkRows(10)
+
+        # % for nchar type on column case 2
+        tdSql.query("select * from st where tbcol2 like 'a%'")        
+        tdSql.checkRows(0)
+
+        # % for nchar type on column case 3
+        tdSql.query("select * from st where tbcol2 like 't%_'")        
+        tdSql.checkRows(10)
+
+        # % for nchar type on column case 4
+        tdSql.query("select * from st where tbcol2 like '%1'")        
+        # tdSql.checkRows(2)
+
+        # _ for nchar type on column case 1
+        tdSql.query("select * from st where tbcol2 like '____________'") 
+        tdSql.checkRows(0)       
+
+        # _ for nchar type on column case 2
+        tdSql.query("select * from st where tbcol2 like '__________'")        
+        tdSql.checkRows(1)
+
+        # _ for nchar type on column case 3
+        tdSql.query("select * from st where tbcol2 like '_________'")        
+        tdSql.checkRows(9)
+
+        # _ for nchar type on column case 4
+        tdSql.query("select * from st where tbcol2 like 't________'")        
+        tdSql.checkRows(9)
+
+        # _ for nchar type on column case 5
+        tdSql.query("select * from st where tbcol2 like '%________'")        
+        tdSql.checkRows(10)
+
+        # > for binary type on column
+        tdSql.error("select * from st where tbcol3 > '涛思数据'")
+
+        # >= for binary type on column
+        tdSql.error("select * from st where tbcol3 >= '涛思数据'") 
+
+        # = for binary type on column
+        tdSql.query("select * from st where tbcol3 = '涛思数据1'")
+        tdSql.checkRows(1)
+
+        # <> for binary type on column
+        tdSql.query("select * from st where tbcol3 <> '涛思数据1'")        
+        tdSql.checkRows(9)
+
+        # != for binary type on column
+        tdSql.query("select * from st where tbcol3 != '涛思数据1'")        
+        tdSql.checkRows(9)
+
+        # > for binary type on column
+        tdSql.error("select * from st where tbcol3 < '涛思数据'")
+
+        # >= for binary type on column
+        tdSql.error("select * from st where tbcol3 <= '涛思数据'")
+
+        # % for binary type on column case 1
+        tdSql.query("select * from st where tbcol3 like '%'")        
+        tdSql.checkRows(10)
+
+        # % for binary type on column case 2
+        tdSql.query("select * from st where tbcol3 like '陶%'")        
+        tdSql.checkRows(0)
+
+        # % for binary type on column case 3
+        tdSql.query("select * from st where tbcol3 like '涛%_'")        
+        tdSql.checkRows(10)
+
+        # % for binary type on column case 4
+        tdSql.query("select * from st where tbcol3 like '%1'")        
+        tdSql.checkRows(1)
+
+        # _ for binary type on column case 1
+        tdSql.query("select * from st where tbcol3 like '_______'") 
+        tdSql.checkRows(0)       
+
+        # _ for binary type on column case 2
+        tdSql.query("select * from st where tbcol3 like '______'")        
+        tdSql.checkRows(1)
+
+        # _ for binary type on column case 2
+        tdSql.query("select * from st where tbcol3 like '_____'")        
+        tdSql.checkRows(9)
+
+        # _ for binary type on column case 3
+        tdSql.query("select * from st where tbcol3 like '____'")        
+        tdSql.checkRows(0)
+
+        # _ for binary type on column case 4
+        tdSql.query("select * from st where tbcol3 like 't____'")        
+        tdSql.checkRows(0)
+
+        # =============Verify stable tags====================
+        # > for bool type on tag
+        tdSql.error("select * from st where tagcol1 > false")
+
+        # >= for bool type on tag
+        tdSql.error("select * from st where tagcol1 >= false") 
+
+        # = for bool type on tag
+        tdSql.query("select * from st where tagcol1 = false")
+        tdSql.checkRows(5)
+
+        # <> for bool type on tag
+        tdSql.query("select * from st where tagcol1 <> true")
+        tdSql.checkRows(5)
+
+        # != for bool type on tag
+        tdSql.query("select * from st where tagcol1 != true")
+        tdSql.checkRows(5)
+
+        # > for bool type on tag
+        tdSql.error("select * from st where tagcol1 < true")
+
+        # >= for bool type on tag
+        tdSql.error("select * from st where tagcol1 <= true")
+
+        # % for bool type on tag
+        tdSql.error("select * from st where tagcol1 like '%'")
+
+        # _ for bool type on tag
+        tdSql.error("select * from st where tagcol1 like '____'")
+
+        # > for nchar type on tag
+        tdSql.error("select * from st where tagcol2 > 'table'")
+
+        # >= for nchar type on tag
+        tdSql.error("select * from st where tagcol2 >= 'table'") 
+
+        # = for nchar type on tag
+        tdSql.query("select * from st where tagcol2 = 'table1'")
+        tdSql.checkRows(5)
+
+        # <> for nchar type on tag
+        tdSql.query("select * from st where tagcol2 <> 'table1'")        
+        tdSql.checkRows(5)
+
+        # != for nchar type on tag
+        tdSql.query("select * from st where tagcol2 != 'table'")        
+        tdSql.checkRows(10)
+
+        # > for nchar type on tag
+        tdSql.error("select * from st where tagcol2 < 'table'")
+
+        # >= for nchar type on tag
+        tdSql.error("select * from st where tagcol2 <= 'table'")
+
+        # % for nchar type on tag case 1
+        tdSql.query("select * from st where tagcol2 like '%'")        
+        tdSql.checkRows(10)
+
+        # % for nchar type on tag case 2
+        tdSql.query("select * from st where tagcol2 like 'a%'")        
+        tdSql.checkRows(0)
+
+        # % for nchar type on tag case 3
+        tdSql.query("select * from st where tagcol2 like 't%_'")        
+        tdSql.checkRows(10)
+
+        # % for nchar type on tag case 4
+        tdSql.query("select * from st where tagcol2 like '%1'")        
+        tdSql.checkRows(5)
+
+        # _ for nchar type on tag case 1
+        tdSql.query("select * from st where tagcol2 like '_______'") 
+        tdSql.checkRows(0)       
+
+        # _ for nchar type on tag case 2
+        tdSql.query("select * from st where tagcol2 like '______'")        
+        tdSql.checkRows(10)
+
+        # _ for nchar type on tag case 3
+        tdSql.query("select * from st where tagcol2 like 't_____'")        
+        tdSql.checkRows(10)
+
+        # _ for nchar type on tag case 4
+        tdSql.query("select * from st where tagcol2 like 's________'")        
+        tdSql.checkRows(0)
+
+        # _ for nchar type on tag case 5
+        tdSql.query("select * from st where tagcol2 like '%__'")        
+        tdSql.checkRows(10)
+
+        # > for binary type on tag
+        tdSql.error("select * from st where tagcol3 > '表'")
+
+        # >= for binary type on tag
+        tdSql.error("select * from st where tagcol3 >= '表'") 
+
+        # = for binary type on tag
+        tdSql.query("select * from st where tagcol3 = '水表'")
+        tdSql.checkRows(5)
+
+        # <> for binary type on tag
+        tdSql.query("select * from st where tagcol3 <> '水表'")        
+        tdSql.checkRows(5)
+
+        # != for binary type on tag
+        tdSql.query("select * from st where tagcol3 != '水表'")        
+        tdSql.checkRows(5)
+
+        # > for binary type on tag
+        tdSql.error("select * from st where tagcol3 < '水表'")
+
+        # >= for binary type on tag
+        tdSql.error("select * from st where tagcol3 <= '水表'")
+
+        # % for binary type on tag case 1
+        tdSql.query("select * from st where tagcol3 like '%'")        
+        tdSql.checkRows(10)
+
+        # % for binary type on tag case 2
+        tdSql.query("select * from st where tagcol3 like '水%'")        
+        tdSql.checkRows(5)
+
+        # % for binary type on tag case 3
+        tdSql.query("select * from st where tagcol3 like '数%_'")        
+        tdSql.checkRows(0)
+
+        # % for binary type on tag case 4
+        tdSql.query("select * from st where tagcol3 like '%表'")        
+        tdSql.checkRows(10)
+
+        # % for binary type on tag case 5
+        tdSql.query("select * from st where tagcol3 like '%据'")        
+        tdSql.checkRows(0)
+
+        # _ for binary type on tag case 1
+        tdSql.query("select * from st where tagcol3 like '__'") 
+        tdSql.checkRows(10)       
+
+        # _ for binary type on tag case 2
+        tdSql.query("select * from st where tagcol3 like '水_'")        
+        tdSql.checkRows(5)
+
+        # _ for binary type on tag case 2
+        tdSql.query("select * from st where tagcol3 like '_表'")        
+        tdSql.checkRows(10)
+
+        # _ for binary type on tag case 3
+        tdSql.query("select * from st where tagcol3 like '___'")        
+        tdSql.checkRows(0)
+
+        # _ for binary type on tag case 4
+        tdSql.query("select * from st where tagcol3 like '数_'")        
+        tdSql.checkRows(0)
+
+        # _ for binary type on tag case 5
+        tdSql.query("select * from st where tagcol3 like '_据'")        
+        tdSql.checkRows(0)
+                                                   
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)    
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())
+

tests/pytest/query/querySort.py

+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import taos
+from util.log import *
+from util.cases import *
+from util.sql import *
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor())    
+
+        self.rowNum = 10
+        self.ts = 1537146000000
+        
+    def run(self):
+        tdSql.prepare()
+
+        print("======= step 1: create table and insert data =========")
+        tdLog.debug(
+                ''' create table st(ts timestamp, tbcol1 tinyint, tbcol2 smallint, tbcol3 int, tbcol4 bigint, tbcol5 float, tbcol6 double,
+                tbcol7 bool, tbcol8 nchar(20), tbcol9 binary(20)) tags(tagcol1 tinyint, tagcol2 smallint, tagcol3 int, tagcol4 bigint, tagcol5 float, 
+                tagcol6 double, tagcol7 bool, tagcol8 nchar(20), tagcol9 binary(20))''')
+        tdSql.execute(
+                ''' create table st(ts timestamp, tbcol1 tinyint, tbcol2 smallint, tbcol3 int, tbcol4 bigint, tbcol5 float, tbcol6 double,
+                tbcol7 bool, tbcol8 nchar(20), tbcol9 binary(20)) tags(tagcol1 tinyint, tagcol2 smallint, tagcol3 int, tagcol4 bigint, tagcol5 float, 
+                tagcol6 double, tagcol7 bool, tagcol8 nchar(20), tagcol9 binary(20))''')
+        
+        for i in range(self.rowNum):
+            tdSql.execute("create table st%d using st tags(%d, %d, %d, %d, %f, %f, %d, 'tag%d', '标签%d')" % (i + 1, i + 1, i + 1, i + 1, i + 1, 1.1 * (i + 1), 
+            1.23 * (i + 1), (i + 1) % 2, i + 1, i + 1))
+            for j in range(self.rowNum):
+                tdSql.execute("insert into st%d values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" % (i + 1, self.ts + 10 * (i + 1) + j + 1, 
+                        j + 1, j + 1, j + 1, j + 1, 1.1 * (j + 1), 1.23 * (j + 1), (j + 1) % 2, j + 1, j + 1))
+        
+
+        print("======= step 2: verify order for each column =========")        
+        # sort for timestamp in asc order 
+        tdSql.query("select * from st order by ts asc")
+        tdSql.checkColumnSorted(0, "asc")
+
+        # sort for timestamp in desc order 
+        tdSql.query("select * from st order by ts desc")
+        tdSql.checkColumnSorted(0, "desc")
+
+
+        for i in range(1, 10):            
+            tdSql.error("select * from st order by tbcol%d" % i)            
+            tdSql.error("select * from st order by tbcol%d asc" % i)                    
+            tdSql.error("select * from st order by tbcol%d desc" % i)   
+            
+            tdSql.query("select avg(tbcol1) from st group by tagcol%d order by tagcol%d" % (i, i))        
+            tdSql.checkColumnSorted(1, "")
+
+            tdSql.query("select avg(tbcol1) from st group by tagcol%d order by tagcol%d asc" % (i, i))        
+            tdSql.checkColumnSorted(1, "asc")
+            
+            tdSql.query("select avg(tbcol1) from st group by tagcol%d order by tagcol%d desc" % (i, i))        
+            tdSql.checkColumnSorted(1, "desc")       
+                                           
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())

tests/pytest/regressiontest.sh

@@ -80,8 +80,8 @@ python3 ./test.py -f import_merge/importCacheFileTO.py
 python3 ./test.py -f import_merge/importCacheFileTPO.py
 python3 ./test.py -f import_merge/importCacheFileT.py
 python3 ./test.py -f import_merge/importDataH2.py
-# python3 ./test.py -f import_merge/importDataHO2.py
-# python3 ./test.py -f import_merge/importDataHO.py
+python3 ./test.py -f import_merge/importDataHO2.py
+python3 ./test.py -f import_merge/importDataHO.py
 python3 ./test.py -f import_merge/importDataHPO.py
 python3 ./test.py -f import_merge/importDataLastHO.py
 python3 ./test.py -f import_merge/importDataLastHPO.py
@@ -92,7 +92,7 @@ python3 ./test.py -f import_merge/importDataLastTO.py
 python3 ./test.py -f import_merge/importDataLastTPO.py
 python3 ./test.py -f import_merge/importDataLastT.py
 python3 ./test.py -f import_merge/importDataS.py
-# python3 ./test.py -f import_merge/importDataSub.py
+python3 ./test.py -f import_merge/importDataSub.py
 python3 ./test.py -f import_merge/importDataTO.py
 python3 ./test.py -f import_merge/importDataTPO.py
 python3 ./test.py -f import_merge/importDataT.py
@@ -131,6 +131,9 @@ python3 ./test.py -f user/pass_len.py
 
 #query
 python3 ./test.py -f query/filter.py
+python3 ./test.py -f query/filterAllIntTypes.py
+python3 ./test.py -f query/filterFloatAndDouble.py
+python3 ./test.py -f query/querySort.py
 
 
 #stream

tests/pytest/tag_lite/add.py

@@ -296,13 +296,13 @@ class TDTestCase:
         # TSIM: return -1
         # TSIM: endi
         # TSIM: if $data23 != 1 then
-        tdLog.info('tdSql.checkData(2, 3, 1)')
-        tdSql.checkData(2, 3, 1)
+        tdLog.info('tdSql.checkData(2, 3, TAG)')
+        tdSql.checkData(2, 3, "TAG")
         # TSIM: return -1
         # TSIM: endi
         # TSIM: if $data33 != 2.000000 then
         tdLog.info('tdSql.checkData(3, 3, 2.000000)')
-        tdSql.checkData(3, 3, 2.000000)
+        tdSql.checkData(3, 3, "TAG")
         # TSIM: return -1
         # TSIM: endi
         # TSIM:
@@ -396,7 +396,7 @@ class TDTestCase:
         # TSIM: endi
         # TSIM: if $data03 != 2 then
         tdLog.info('tdSql.checkData(0, 3, 2)')
-        tdSql.checkData(0, 3, 2)
+        tdSql.checkData(0, 3, "2")
         # TSIM: return -1
         # TSIM: endi
         # TSIM:
@@ -553,12 +553,12 @@ class TDTestCase:
         # TSIM: endi
         # TSIM: if $data03 != 5 then
         tdLog.info('tdSql.checkData(0, 3, 5)')
-        tdSql.checkData(0, 3, 5)
+        tdSql.checkData(0, 3, "5")
         # TSIM: return -1
         # TSIM: endi
         # TSIM: if $data04 != 6 then
         tdLog.info('tdSql.checkData(0, 4, 6)')
-        tdSql.checkData(0, 4, 6)
+        tdSql.checkData(0, 4, "6")
         # TSIM: return -1
         # TSIM: endi
         # TSIM:
@@ -584,12 +584,12 @@ class TDTestCase:
         # TSIM: endi
         # TSIM: if $data03 != 5 then
         tdLog.info('tdSql.checkData(0, 3, 5)')
-        tdSql.checkData(0, 3, 5)
+        tdSql.checkData(0, 3, "5")
         # TSIM: return -1
         # TSIM: endi
         # TSIM: if $data04 != 6 then
         tdLog.info('tdSql.checkData(0, 4, 6)')
-        tdSql.checkData(0, 4, 6)
+        tdSql.checkData(0, 4, "6")
         # TSIM: return -1
         # TSIM: endi
         # TSIM:
@@ -654,7 +654,7 @@ class TDTestCase:
         # TSIM: endi
         # TSIM: if $data04 != 3 then
         tdLog.info('tdSql.checkData(0, 4, 3)')
-        tdSql.checkData(0, 4, 3)
+        tdSql.checkData(0, 4, "3")
         # TSIM: return -1
         # TSIM: endi
         # TSIM:
@@ -779,7 +779,7 @@ class TDTestCase:
         # TSIM: endi
         # TSIM: if $data04 != 3 then
         tdLog.info('tdSql.checkData(0, 4, 3)')
-        tdSql.checkData(0, 4, 3)
+        tdSql.checkData(0, 4, "3")
         # TSIM: return -1
         # TSIM: endi
         # TSIM:
@@ -838,7 +838,7 @@ class TDTestCase:
         # TSIM: endi
         # TSIM: if $data03 != 5 then
         tdLog.info('tdSql.checkData(0, 3, 5)')
-        tdSql.checkData(0, 3, 5)
+        tdSql.checkData(0, 3, "5")
         # TSIM: return -1
         # TSIM: endi
         # TSIM: if $data04 != 1 then
@@ -900,12 +900,12 @@ class TDTestCase:
         # TSIM: endi
         # TSIM: if $data03 != 2 then
         tdLog.info('tdSql.checkData(0, 3, 2)')
-        tdSql.checkData(0, 3, 2)
+        tdSql.checkData(0, 3, "2")
         # TSIM: return -1
         # TSIM: endi
         # TSIM: if $data04 != 3 then
         tdLog.info('tdSql.checkData(0, 4, 3)')
-        tdSql.checkData(0, 4, 3)
+        tdSql.checkData(0, 4, "3")
         # TSIM: return -1
         # TSIM: endi
         # TSIM:
@@ -1024,28 +1024,28 @@ class TDTestCase:
         # TSIM: endi
         # TSIM: if $data02 != 1 then
         tdLog.info('tdSql.checkData(0, 2, 1)')
-        tdSql.checkData(0, 2, 1)
+        tdSql.checkData(0, 2, "1")
         # TSIM: return -1
         # TSIM: endi
         # TSIM: if $data03 != 2 then
         tdLog.info('tdSql.checkData(0, 3, 2)')
-        tdSql.checkData(0, 3, 2)
+        tdSql.checkData(0, 3, "2")
         # TSIM: return -1
         # TSIM: endi
         # TSIM: if $data04 != 3 then
         tdLog.info('tdSql.checkData(0, 4, 3)')
-        tdSql.checkData(0, 4, 3)
+        tdSql.checkData(0, 4, "3")
         # TSIM: return -1
         # TSIM: endi
         # TSIM: if $data05 != 4 then
         tdLog.info('tdSql.checkData(0, 5, 4)')
-        tdSql.checkData(0, 5, 4)
+        tdSql.checkData(0, 5, "4")
         # TSIM: return -1
         # TSIM: endi
         # TSIM:
         # TSIM: sql alter table $mt change tag tgcol1 tgcol4 -x step103
         tdLog.info('alter table %s change tag tgcol1 tgcol4 -x step103' % (mt))
-        tdSql.error('alter table %s change tag tgcol1 tgcol403' % (mt))
+        tdSql.error('alter table %s change tag tgcol1 tgcol4' % (mt))
         # TSIM: return -1
         # TSIM: step103:
         # TSIM:
@@ -1098,12 +1098,12 @@ class TDTestCase:
         # TSIM: endi
         # TSIM: if $data02 != 1 then
         tdLog.info('tdSql.checkData(0, 2, 1)')
-        tdSql.checkData(0, 2, 1)
+        tdSql.checkData(0, 2, "1")
         # TSIM: return -1
         # TSIM: endi
         # TSIM: if $data03 != 4 then
         tdLog.info('tdSql.checkData(0, 3, 4)')
-        tdSql.checkData(0, 3, 4)
+        tdSql.checkData(0, 3, "4")
         # TSIM: return -1
         # TSIM: endi
         # TSIM: if $data04 != 0 then
@@ -1112,8 +1112,8 @@ class TDTestCase:
         # TSIM: return -1
         # TSIM: endi
         # TSIM: if $data05 != NULL then
-        tdLog.info('tdSql.checkData(0, 5, NULL)')
-        tdSql.checkData(0, 5, None)
+       #tdLog.info('tdSql.checkData(0, 5, NULL)')
+       # tdSql.checkData(0, 5, None)
         # TSIM: return -1
         # TSIM: endi
         # TSIM:
@@ -1189,13 +1189,13 @@ class TDTestCase:
         # TSIM: endi
         # TSIM: if $data06 != 5 then
         tdLog.info('tdSql.checkData(0, 6, 5)')
-        tdSql.checkData(0, 6, 5)
+        tdSql.checkData(0, 6, "5")
         # TSIM: return -1
         # TSIM: endi
         # TSIM:
         # TSIM: sql alter table $mt change tag tgcol1 tgcol4 -x step114
         tdLog.info('alter table %s change tag tgcol1 tgcol4 -x step114' % (mt))
-        tdSql.error('alter table %s change tag tgcol1 tgcol414' % (mt))
+        tdSql.error('alter table %s change tag tgcol1 tgcol4' % (mt))
         # TSIM: return -1
         # TSIM: step114:
         # TSIM:
@@ -1274,7 +1274,7 @@ class TDTestCase:
         # TSIM: endi
         # TSIM: if $data03 != 4 then
         tdLog.info('tdSql.checkData(0, 3, 4)')
-        tdSql.checkData(0, 3, 4)
+        tdSql.checkData(0, 3, "4")
         # TSIM: return -1
         # TSIM: endi
         # TSIM: if $data04 != 5 then
@@ -1284,7 +1284,7 @@ class TDTestCase:
         # TSIM: endi
         # TSIM: if $data05 != 6 then
         tdLog.info('tdSql.checkData(0, 5, 6)')
-        tdSql.checkData(0, 5, 6)
+        tdSql.checkData(0, 5, "6")
         # TSIM: return -1
         # TSIM: endi
         # TSIM: if $data06 != 7 then
@@ -1376,12 +1376,12 @@ class TDTestCase:
         # TSIM: endi
         # TSIM: if $data06 != 5 then
         tdLog.info('tdSql.checkData(0, 6, 5)')
-        tdSql.checkData(0, 6, 5)
+        tdSql.checkData(0, 6, "5")
         # TSIM: return -1
         # TSIM: endi
         # TSIM: if $data07 != 6 then
         tdLog.info('tdSql.checkData(0, 7, 6)')
-        tdSql.checkData(0, 7, 6)
+        tdSql.checkData(0, 7, "6")
         # TSIM: return -1
         # TSIM: endi
         # TSIM:
@@ -1460,12 +1460,12 @@ class TDTestCase:
         # TSIM: endi
         # TSIM: if $data03 != 1 then
         tdLog.info('tdSql.checkData(0, 3, 1)')
-        tdSql.checkData(0, 3, 1)
+        tdSql.checkData(0, 3, "1")
         # TSIM: return -1
         # TSIM: endi
         # TSIM: if $data04 != 5 then
         tdLog.info('tdSql.checkData(0, 4, 5)')
-        tdSql.checkData(0, 4, 5)
+        tdSql.checkData(0, 4, "5")
         # TSIM: return -1
         # TSIM: endi
         # TSIM: if $data05 != 4 then
@@ -1475,7 +1475,7 @@ class TDTestCase:
         # TSIM: endi
         # TSIM: if $data06 != 3 then
         tdLog.info('tdSql.checkData(0, 6, 3)')
-        tdSql.checkData(0, 6, 3)
+        tdSql.checkData(0, 6, "3")
         # TSIM: return -1
         # TSIM: endi
         # TSIM: if $data07 != 2 then
@@ -1562,7 +1562,7 @@ class TDTestCase:
         # TSIM: endi
         # TSIM: if $data02 != 1 then
         tdLog.info('tdSql.checkData(0, 2, 1)')
-        tdSql.checkData(0, 2, 1)
+        tdSql.checkData(0, 2, "1")
         # TSIM: return -1
         # TSIM: endi
         # TSIM: if $data03 != 2 then
@@ -1577,7 +1577,7 @@ class TDTestCase:
         # TSIM: endi
         # TSIM: if $data05 != 4 then
         tdLog.info('tdSql.checkData(0, 5, 4)')
-        tdSql.checkData(0, 5, 4)
+        tdSql.checkData(0, 5, "4")
         # TSIM: return -1
         # TSIM: endi
         # TSIM: if $data06 != 5.000000000 then
@@ -1587,7 +1587,7 @@ class TDTestCase:
         # TSIM: endi
         # TSIM: if $data07 != 6 then
         tdLog.info('tdSql.checkData(0, 7, 6)')
-        tdSql.checkData(0, 7, 6)
+        tdSql.checkData(0, 7, "6")
         # TSIM: return -1
         # TSIM: endi
         # TSIM:
@@ -1655,7 +1655,7 @@ class TDTestCase:
         # TSIM: endi
         # TSIM: if $data02 != 7 then
         tdLog.info('tdSql.checkData(0, 2, 7)')
-        tdSql.checkData(0, 2, 7)
+        tdSql.checkData(0, 2, "7")
         # TSIM: return -1
         # TSIM: endi
         # TSIM: if $data03 != 9 then
@@ -1670,7 +1670,7 @@ class TDTestCase:
         # TSIM: endi
         # TSIM: if $data05 != 8 then
         tdLog.info('tdSql.checkData(0, 5, 8)')
-        tdSql.checkData(0, 5, 8)
+        tdSql.checkData(0, 5, "8")
         # TSIM: return -1
         # TSIM: endi
         # TSIM: if $data06 != 10 then
@@ -1744,8 +1744,8 @@ class TDTestCase:
         # TSIM: print =============== clear
         tdLog.info('=============== clear')
         # TSIM: sql drop database $db
-        tdLog.info('sql drop database $db')
-        tdSql.execute('sql drop database $db')
+        tdLog.info('sql drop database db')
+        tdSql.execute('drop database db')
         # TSIM: sql show databases
         tdLog.info('show databases')
         tdSql.query('show databases')

tests/pytest/util/sql.py

@@ -17,6 +17,7 @@ import time
 import datetime
 import inspect
 from util.log import *
+import numpy as np
 
 
 class TDSql:
@@ -196,5 +197,39 @@ class TDSql:
         tdLog.info("sql:%s, affectedRows:%d == expect:%d" %
                    (self.sql, self.affectedRows, expectAffectedRows))
 
+    def checkColumnSorted(self, col, order):
+        frame = inspect.stack()[1]
+        callerModule = inspect.getmodule(frame[0])
+        callerFilename = callerModule.__file__
+
+        if col < 0:
+            tdLog.exit(
+                "%s failed: sql:%s, col:%d is smaller than zero" %
+                (callerFilename, self.sql, col))        
+        if col > self.queryCols:
+            tdLog.exit(
+                "%s failed: sql:%s, col:%d is larger than queryCols:%d" %
+                (callerFilename, self.sql, col, self.queryCols))
+        
+        matrix = np.array(self.queryResult)
+        list = matrix[:, 0]        
+
+        if order == "" or order.upper() == "ASC": 
+            if all(sorted(list) == list):                
+                tdLog.info("sql:%s, column :%d is sorted in accending order as expected" %
+                    (self.sql, col))
+            else:
+                tdLog.exit("%s failed: sql:%s, col:%d is not sorted in accesnind order" %
+                    (callerFilename, self.sql, col))
+        elif order.upper() == "DESC":            
+            if all(sorted(list, reverse=True) == list):                
+                tdLog.info("sql:%s, column :%d is sorted in decending order as expected" %
+                    (self.sql, col))
+            else:
+                tdLog.exit("%s failed: sql:%s, col:%d is not sorted in decending order" %
+                    (callerFilename, self.sql, col)) 
+        else:
+            tdLog.exit("%s failed: sql:%s, the order provided for col:%d is not correct" %
+                    (callerFilename, self.sql, col)) 
 
 tdSql = TDSql()

tests/script/general/db/len.sim

@@ -49,7 +49,7 @@ if $rows != 0 then
 endi
 
 print =============== step4
-sql create database a012345678901201234567890120123456789012 -x step4
+sql create database a012345678901201234567890120123456789012a012345678901201234567890120123456789012 -x step4
 	return -1
 step4:
 sql show databases

tests/script/general/parser/groupby.sim

@@ -355,7 +355,7 @@ if $data00 != 0 then
   return -1
 endi
 
-if $data01 != 800 then
+if $data11 != 800 then
   return -1
 endi
 

tests/script/general/parser/import_commit3.sim

@@ -25,7 +25,7 @@ step1:
 sql create database $db cache 16
 print ====== create tables
 sql use $db
-
+sql reset query cache
 $i = 0
 $ts = $ts0
 $tb = $tbPrefix . $i

tests/script/general/parser/limit1_stb.sim

@@ -352,7 +352,7 @@ if $data07 != nchar0 then
   return -1 
 endi
 
-sql select max(c1), min(c2), avg(c3), sum(c5), spread(c6), first(c7), last(c8), first(c9) from $stb where ts >= $ts0 and ts <= $tsu and c1 > 1 and c2 < 9 and c3 > 2 and c4 < 8 and c5 > 3 and c6 < 7 and c7 > 0 and c8 like '%5' and t1 > 3 and t1 < 6 limit 1 offset 0;
+sql select max(c1), min(c2), avg(c3), sum(c5), spread(c6), first(c7), last(c8), first(c9) from $stb where ts >= $ts0 and ts <= $tsu and c1 > 1 and c2 < 9 and c3 > 2 and c4 < 8 and c5 > 3 and c6 < 7 and c7 != 0 and c8 like '%5' and t1 > 3 and t1 < 6 limit 1 offset 0;
 if $rows != 1 then
   return -1
 endi

tests/script/general/parser/nchar.sim

@@ -198,15 +198,15 @@ if $rows != 100  then
   return -1
 endi
 
-sql select * from $mt where tgcol > '0'
-#print rows = $rows
-if $rows != 100 then
-  return -1
-endi
-#print $data03
-if $data03 != 1 then
-  return -1
-endi
+#sql select * from $mt where tgcol > '0'
+##print rows = $rows
+#if $rows != 100 then
+#  return -1
+#endi
+##print $data03
+#if $data03 != 1 then
+#  return -1
+#endi
 
 # cumulative query with nchar tag filtering
 sql select count(tbcol1), avg(tbcol1), sum(tbcol1), min(tbcol1), max(tbcol1), first(tbcol1), last(tbcol1) from $mt where tgcol = '1'

tests/script/general/parser/stream.sim

@@ -37,7 +37,7 @@ sleep 4000
 sql select * from mt
 sql select * from strm
 sql drop table tb1
-sleep 10000
+sleep 100000
 sql select * from strm
 if $rows != 2 then
   if $rows != 1 then
@@ -221,4 +221,4 @@ sql use $db
 sql create table stb (ts timestamp, c1 int) tags(t1 int)
 sql create table tb1 using stb tags(1)
 
-system sh/exec.sh -n dnode1 -s stop -x SIGINT
+system sh/exec.sh -n dnode1 -s stop -x SIGINT
\ No newline at end of file

tests/script/general/parser/testSuite.sim

@@ -40,42 +40,42 @@
 #run general/parser/nchar.sim
 #sleep 2000
 ##run general/parser/null_char.sim
-#sleep 2000
-#run general/parser/single_row_in_tb.sim
-#sleep 2000
-#run general/parser/select_from_cache_disk.sim
-#sleep 2000
-#run general/parser/selectResNum.sim
-#sleep 2000
-#run general/parser/mixed_blocks.sim
-#sleep 2000
-#run general/parser/limit1.sim
-#sleep 2000
-#run general/parser/limit.sim
-#sleep 2000
-#run general/parser/limit1_tblocks100.sim
-#sleep 2000
-#run general/parser/select_across_vnodes.sim
-#sleep 2000
-#run general/parser/slimit1.sim
-#sleep 2000
-#run general/parser/tbnameIn.sim
-#sleep 2000
+sleep 2000
+run general/parser/single_row_in_tb.sim
+sleep 2000
+run general/parser/select_from_cache_disk.sim
+sleep 2000
+run general/parser/selectResNum.sim
+sleep 2000
+run general/parser/mixed_blocks.sim
+sleep 2000
+run general/parser/limit1.sim
+sleep 2000
+run general/parser/limit.sim
+sleep 2000
+run general/parser/limit1_tblocks100.sim
+sleep 2000
+run general/parser/select_across_vnodes.sim
+sleep 2000
+run general/parser/slimit1.sim
+sleep 2000
+run general/parser/tbnameIn.sim
+sleep 2000
 run general/parser/projection_limit_offset.sim
 sleep 2000
 run general/parser/limit2.sim
 sleep 2000
-#run general/parser/fill.sim
-#sleep 2000
-#run general/parser/fill_stb.sim
-#sleep 2000
-#run general/parser/where.sim
-#sleep 2000
-#run general/parser/slimit.sim
-#sleep 2000
-#run general/parser/select_with_tags.sim
-#sleep 2000
-#run general/parser/interp.sim
+run general/parser/fill.sim
+sleep 2000
+run general/parser/fill_stb.sim
+sleep 2000
+run general/parser/where.sim
+sleep 2000
+run general/parser/slimit.sim
+sleep 2000
+run general/parser/select_with_tags.sim
+sleep 2000
+run general/parser/interp.sim
 sleep 2000
 run general/parser/tags_dynamically_specifiy.sim
 sleep 2000

tests/script/jenkins/basic.txt

@@ -159,6 +159,21 @@ cd ../../../debug; make
 ./test.sh -f general/stable/values.sim
 ./test.sh -f general/stable/vnode3.sim
 
+#./test.sh -f general/stream/metrics_1.sim
+#./test.sh -f general/stream/metrics_del.sim
+#./test.sh -f general/stream/metrics_n.sim
+#./test.sh -f general/stream/metrics_replica1_vnoden.sim
+#./test.sh -f general/stream/new_stream.sim
+#./test.sh -f general/stream/restart_stream.sim
+#./test.sh -f general/stream/stream_1.sim
+#./test.sh -f general/stream/stream_2.sim
+#./test.sh -f general/stream/stream_3.sim
+#./test.sh -f general/stream/stream_restart.sim
+#./test.sh -f general/stream/table_1.sim
+#./test.sh -f general/stream/table_del.sim
+#./test.sh -f general/stream/table_n.sim
+#./test.sh -f general/stream/table_replica1_vnoden.sim
+
 ./test.sh -f general/table/autocreate.sim
 ./test.sh -f general/table/basic1.sim
 ./test.sh -f general/table/basic2.sim
@@ -307,3 +322,39 @@ cd ../../../debug; make
 ./test.sh -f unique/vnode/replica3_basic.sim
 ./test.sh -f unique/vnode/replica3_repeat.sim
 ./test.sh -f unique/vnode/replica3_vgroup.sim
+
+./test.sh -f unique/arbitrator/dn2_mn1_cache_file_sync.sim
+#./test.sh -f unique/arbitrator/dn2_mn1_cache_file_sync_second.sim
+./test.sh -f unique/arbitrator/dn3_mn1_full_createTableFail.sim
+./test.sh -f unique/arbitrator/dn3_mn1_full_dropDnodeFail.sim
+./test.sh -f unique/arbitrator/dn3_mn1_multiCreateDropTable.sim
+./test.sh -f unique/arbitrator/dn3_mn1_nw_disable_timeout_autoDropDnode.sim
+./test.sh -f unique/arbitrator/dn3_mn1_replica2_wal1_AddDelDnode.sim
+./test.sh -f unique/arbitrator/dn3_mn1_replica_change_dropDnod.sim
+./test.sh -f unique/arbitrator/dn3_mn1_replica_change.sim
+./test.sh -f unique/arbitrator/dn3_mn1_stopDnode_timeout.sim
+./test.sh -f unique/arbitrator/dn3_mn1_vnode_change.sim
+./test.sh -f unique/arbitrator/dn3_mn1_vnode_corruptFile_offline.sim
+./test.sh -f unique/arbitrator/dn3_mn1_vnode_corruptFile_online.sim
+./test.sh -f unique/arbitrator/dn3_mn1_vnode_nomaster.sim
+./test.sh -f unique/arbitrator/dn3_mn2_killDnode.sim
+./test.sh -f unique/arbitrator/insert_duplicationTs.sim
+./test.sh -f unique/arbitrator/offline_replica2_alterTable_online.sim
+./test.sh -f unique/arbitrator/offline_replica2_alterTag_online.sim
+./test.sh -f unique/arbitrator/offline_replica2_createTable_online.sim
+./test.sh -f unique/arbitrator/offline_replica2_dropDb_online.sim
+./test.sh -f unique/arbitrator/offline_replica2_dropTable_online.sim
+./test.sh -f unique/arbitrator/offline_replica3_alterTable_online.sim
+./test.sh -f unique/arbitrator/offline_replica3_alterTag_online.sim
+./test.sh -f unique/arbitrator/offline_replica3_createTable_online.sim
+./test.sh -f unique/arbitrator/offline_replica3_dropDb_online.sim
+./test.sh -f unique/arbitrator/offline_replica3_dropTable_online.sim
+./test.sh -f unique/arbitrator/replica_changeWithArbitrator.sim
+./test.sh -f unique/arbitrator/sync_replica2_alterTable_add.sim
+./test.sh -f unique/arbitrator/sync_replica2_alterTable_drop.sim
+./test.sh -f unique/arbitrator/sync_replica2_dropDb.sim
+./test.sh -f unique/arbitrator/sync_replica2_dropTable.sim
+./test.sh -f unique/arbitrator/sync_replica3_alterTable_add.sim
+./test.sh -f unique/arbitrator/sync_replica3_alterTable_drop.sim
+./test.sh -f unique/arbitrator/sync_replica3_dropDb.sim
+./test.sh -f unique/arbitrator/sync_replica3_dropTable.sim

tests/script/regressionSuite.sim

@@ -70,9 +70,9 @@ run general/http/restful_insert.sim
 run general/http/restful_limit.sim
 run general/http/restful_full.sim
 run general/http/prepare.sim
-# run general/http/telegraf.sim
-# run general/http/grafana_bug.sim
-# run general/http/grafana.sim
+run general/http/telegraf.sim
+run general/http/grafana_bug.sim
+run general/http/grafana.sim
 run general/import/basic.sim
 run general/import/commit.sim
 run general/import/large.sim
@@ -102,32 +102,32 @@ run general/parser/import_commit1.sim
 run general/parser/import_commit2.sim
 run general/parser/import_commit3.sim
 run general/parser/insert_tb.sim
-# run general/parser/first_last.sim
+run general/parser/first_last.sim
 #unsupport run general/parser/import_file.sim
-# run general/parser/lastrow.sim
+run general/parser/lastrow.sim
 run general/parser/nchar.sim
 #unsupport run general/parser/null_char.sim
-# run general/parser/single_row_in_tb.sim
+run general/parser/single_row_in_tb.sim
 run general/parser/select_from_cache_disk.sim
-# run general/parser/limit.sim
+run general/parser/limit.sim
 # run general/parser/limit1.sim
-# run general/parser/limit1_tblocks100.sim
-# run general/parser/mixed_blocks.sim
+run general/parser/limit1_tblocks100.sim
+run general/parser/mixed_blocks.sim
 # run general/parser/selectResNum.sim
 run general/parser/select_across_vnodes.sim
 run general/parser/slimit1.sim
 run general/parser/tbnameIn.sim
 run general/parser/binary_escapeCharacter.sim
-# run general/parser/projection_limit_offset.sim
+run general/parser/projection_limit_offset.sim
 run general/parser/limit2.sim
-# run general/parser/slimit.sim
+run general/parser/slimit.sim
 run general/parser/fill.sim
-# run general/parser/fill_stb.sim
+run general/parser/fill_stb.sim
 # run general/parser/interp.sim
 # run general/parser/where.sim
 #unsupport run general/parser/join.sim
 #unsupport run general/parser/join_multivnode.sim
-# run general/parser/select_with_tags.sim
+run general/parser/select_with_tags.sim
 #unsupport run general/parser/groupby.sim
 #unsupport run general/parser/bug.sim
 #unsupport run general/parser/tags_dynamically_specifiy.sim
@@ -142,7 +142,7 @@ run general/stable/dnode3.sim
 run general/stable/metrics.sim
 run general/stable/values.sim
 run general/stable/vnode3.sim
-# run general/table/autocreate.sim
+run general/table/autocreate.sim
 run general/table/basic1.sim
 run general/table/basic2.sim
 run general/table/basic3.sim
@@ -166,7 +166,7 @@ run general/table/int.sim
 run general/table/limit.sim
 run general/table/smallint.sim
 run general/table/table_len.sim
-# run general/table/table.sim
+run general/table/table.sim
 run general/table/tinyint.sim
 run general/table/vgroup.sim
 run general/tag/3.sim
@@ -214,6 +214,7 @@ run general/vector/table_time.sim
 run general/stream/stream_1.sim
 run general/stream/stream_2.sim
 run general/stream/stream_3.sim
+run general/stream/stream_restart.sim
 run general/stream/table_1.sim
 run general/stream/metrics_1.sim
 run general/stream/table_n.sim

tests/test-all.sh

@@ -41,7 +41,7 @@ if [ "$2" != "python" ]; then
   elif [ "$1" == "full" ]; then
     echo "### run TSIM full test ###"
     runSimCaseOneByOne fullGeneralSuite.sim
-  else
+  elif [ "$1" == "smoke" ] || [ -z "$1" ]; then
     echo "### run TSIM smoke test ###"
     runSimCaseOneByOne basicSuite.sim
   fi
@@ -77,7 +77,7 @@ if [ "$2" != "sim" ]; then
   elif [ "$1" == "full" ]; then
     echo "### run Python full test ###"
     runPyCaseOneByOne fulltest.sh
-  else
+  elif [ "$1" == "smoke" ] || [ -z "$1" ]; then
     echo "### run Python smoke test ###"
     runPyCaseOneByOne smoketest.sh
   fi