Merge pull request #2609 from taosdata/feature/query

Feature/query

src/client/inc/tscUtil.h

@@ -87,6 +87,16 @@ typedef struct SVgroupTableInfo {
   SArray*       itemList;   //SArray<STableIdInfo>
 } SVgroupTableInfo;
 
+static FORCE_INLINE SQueryInfo* tscGetQueryInfoDetail(SSqlCmd* pCmd, int32_t subClauseIndex) {
+  assert(pCmd != NULL && subClauseIndex >= 0 && subClauseIndex < TSDB_MAX_UNION_CLAUSE);
+
+  if (pCmd->pQueryInfo == NULL || subClauseIndex >= pCmd->numOfClause) {
+    return NULL;
+  }
+
+  return pCmd->pQueryInfo[subClauseIndex];
+}
+
 int32_t tscCreateDataBlock(size_t initialSize, int32_t rowSize, int32_t startOffset, const char* name,
                            STableMeta* pTableMeta, STableDataBlocks** dataBlocks);
 void tscDestroyDataBlock(STableDataBlocks* pDataBlock);

src/client/src/tscAsync.c

@@ -477,7 +477,13 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) {
           tscDebug("%p redo parse sql string to build submit block", pSql);
 
           pCmd->parseFinished = false;
-          if ((code = tsParseSql(pSql, true)) == TSDB_CODE_SUCCESS) {
+          code = tsParseSql(pSql, true);
+
+          if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
+            return;
+          }
+
+          if (code == TSDB_CODE_SUCCESS) {
             /*
              * Discard previous built submit blocks, and then parse the sql string again and build up all submit blocks,
              * and send the required submit block according to index value in supporter to server.

src/client/src/tscFunctionImpl.c

@@ -340,13 +340,12 @@ bool stableQueryFunctChanged(int32_t funcId) {
  */
 void resetResultInfo(SResultInfo *pResInfo) { pResInfo->initialized = false; }
 
-void setResultInfoBuf(SResultInfo *pResInfo, int32_t size, bool superTable) {
+void setResultInfoBuf(SResultInfo *pResInfo, int32_t size, bool superTable, char* buf) {
   assert(pResInfo->interResultBuf == NULL);
   
   pResInfo->bufLen = size;
   pResInfo->superTableQ = superTable;
-  
-  pResInfo->interResultBuf = calloc(1, (size_t)size);
+  pResInfo->interResultBuf = buf;
 }
 
 // set the query flag to denote that query is completed

src/client/src/tscUtil.c

@@ -1464,16 +1464,6 @@ STableMetaInfo* tscGetMetaInfo(SQueryInfo* pQueryInfo, int32_t tableIndex) {
   return pQueryInfo->pTableMetaInfo[tableIndex];
 }
 
-SQueryInfo* tscGetQueryInfoDetail(SSqlCmd* pCmd, int32_t subClauseIndex) {
-  assert(pCmd != NULL && subClauseIndex >= 0 && subClauseIndex < TSDB_MAX_UNION_CLAUSE);
-
-  if (pCmd->pQueryInfo == NULL || subClauseIndex >= pCmd->numOfClause) {
-    return NULL;
-  }
-
-  return pCmd->pQueryInfo[subClauseIndex];
-}
-
 int32_t tscGetQueryInfoDetailSafely(SSqlCmd* pCmd, int32_t subClauseIndex, SQueryInfo** pQueryInfo) {
   int32_t ret = TSDB_CODE_SUCCESS;
 
@@ -2097,7 +2087,7 @@ void tscTryQueryNextClause(SSqlObj* pSql, void (*queryFp)()) {
 }
 
 void tscGetResultColumnChr(SSqlRes* pRes, SFieldInfo* pFieldInfo, int32_t columnIndex) {
-  SFieldSupInfo* pInfo = taosArrayGet(pFieldInfo->pSupportInfo, columnIndex);//tscFieldInfoGetSupp(pFieldInfo, columnIndex);
+  SFieldSupInfo* pInfo = taosArrayGet(pFieldInfo->pSupportInfo, columnIndex);
   assert(pInfo->pSqlExpr != NULL);
 
   int32_t type = pInfo->pSqlExpr->resType;
@@ -2112,7 +2102,7 @@ void tscGetResultColumnChr(SSqlRes* pRes, SFieldInfo* pFieldInfo, int32_t column
     if (isNull(pData, type)) {
       pRes->tsrow[columnIndex] = NULL;
     } else {
-      pRes->tsrow[columnIndex] = pData + VARSTR_HEADER_SIZE;
+      pRes->tsrow[columnIndex] = ((tstr*)pData)->data;
     }
   
     if (realLen < pInfo->pSqlExpr->resBytes - VARSTR_HEADER_SIZE) { // todo refactor

src/query/inc/qExecutor.h

@@ -172,6 +172,7 @@ typedef struct SQueryRuntimeEnv {
   bool                 topBotQuery;      // false
   bool                 groupbyNormalCol; // denote if this is a groupby normal column query
   bool                 hasTagResults;    // if there are tag values in final result or not
+  int32_t              interBufSize;     // intermediate buffer sizse
   int32_t              prevGroupId;      // previous executed group id
   SDiskbasedResultBuf* pResultBuf;       // query result buffer based on blocked-wised disk file
 } SQueryRuntimeEnv;

src/query/inc/qUtil.h

@@ -15,6 +15,8 @@
 #ifndef TDENGINE_QUERYUTIL_H
 #define TDENGINE_QUERYUTIL_H
 
+int32_t getOutputInterResultBufSize(SQuery* pQuery);
+
 void clearTimeWindowResBuf(SQueryRuntimeEnv* pRuntimeEnv, SWindowResult* pOneOutputRes);
 void copyTimeWindowResBuf(SQueryRuntimeEnv* pRuntimeEnv, SWindowResult* dst, const SWindowResult* src);
 
@@ -35,7 +37,7 @@ SWindowResult *getWindowResult(SWindowResInfo *pWindowResInfo, int32_t slot);
 #define curTimeWindow(_winres)  ((_winres)->curIndex)
 bool isWindowResClosed(SWindowResInfo *pWindowResInfo, int32_t slot);
 
-void createQueryResultInfo(SQuery *pQuery, SWindowResult *pResultRow, bool isSTableQuery, SPosInfo *posInfo);
+void createQueryResultInfo(SQuery *pQuery, SWindowResult *pResultRow, bool isSTableQuery, SPosInfo *posInfo, size_t interBufSize);
 
 char *getPosInResultPage(SQueryRuntimeEnv *pRuntimeEnv, int32_t columnIndex, SWindowResult *pResult);
 

src/query/inc/tsqlfunction.h

@@ -272,7 +272,7 @@ bool top_bot_datablock_filter(SQLFunctionCtx *pCtx, int32_t functionId, char *mi
 bool stableQueryFunctChanged(int32_t funcId);
 
 void resetResultInfo(SResultInfo *pResInfo);
-void setResultInfoBuf(SResultInfo *pResInfo, int32_t size, bool superTable);
+void setResultInfoBuf(SResultInfo *pResInfo, int32_t size, bool superTable, char* buf);
 
 static FORCE_INLINE void initResultInfo(SResultInfo *pResInfo) {
   pResInfo->initialized = true;  // the this struct has been initialized flag

src/query/src/qUtil.c

@@ -17,15 +17,24 @@
 
 #include "hash.h"
 #include "taosmsg.h"
-#include "qextbuffer.h"
-#include "ttime.h"
-
-#include "qfill.h"
 #include "ttime.h"
 
 #include "qExecutor.h"
 #include "qUtil.h"
 
+int32_t getOutputInterResultBufSize(SQuery* pQuery) {
+  int32_t size = 0;
+
+  for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
+    assert(pQuery->pSelectExpr[i].interBytes <= DEFAULT_INTERN_BUF_PAGE_SIZE);
+    size += pQuery->pSelectExpr[i].interBytes;
+  }
+
+  assert(size > 0);
+
+  return size;
+}
+
 int32_t initWindowResInfo(SWindowResInfo *pWindowResInfo, SQueryRuntimeEnv *pRuntimeEnv, int32_t size,
                           int32_t threshold, int16_t type) {
   pWindowResInfo->capacity = size;
@@ -43,7 +52,7 @@ int32_t initWindowResInfo(SWindowResInfo *pWindowResInfo, SQueryRuntimeEnv *pRun
   pWindowResInfo->pResult = calloc(threshold, sizeof(SWindowResult));
   for (int32_t i = 0; i < pWindowResInfo->capacity; ++i) {
     SPosInfo posInfo = {-1, -1};
-    createQueryResultInfo(pRuntimeEnv->pQuery, &pWindowResInfo->pResult[i], pRuntimeEnv->stableQuery, &posInfo);
+    createQueryResultInfo(pRuntimeEnv->pQuery, &pWindowResInfo->pResult[i], pRuntimeEnv->stableQuery, &posInfo, pRuntimeEnv->interBufSize);
   }
   
   return TSDB_CODE_SUCCESS;
@@ -54,11 +63,7 @@ void destroyTimeWindowRes(SWindowResult *pWindowRes, int32_t nOutputCols) {
     return;
   }
 
-  // TODO opt malloc strategy
-  for (int32_t i = 0; i < nOutputCols; ++i) {
-    free(pWindowRes->resultInfo[i].interResultBuf);
-  }
-  
+  free(pWindowRes->resultInfo[0].interResultBuf);
   free(pWindowRes->resultInfo);
 }
 
@@ -241,10 +246,9 @@ void clearTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pWindow
   }
   
   pWindowRes->numOfRows = 0;
-  //  pWindowRes->nAlloc = 0;
   pWindowRes->pos = (SPosInfo){-1, -1};
   pWindowRes->status.closed = false;
-  pWindowRes->window = (STimeWindow){0, 0};
+  pWindowRes->window = TSWINDOW_INITIALIZER;
 }
 
 /**
@@ -254,7 +258,6 @@ void clearTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pWindow
  */
 void copyTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *dst, const SWindowResult *src) {
   dst->numOfRows = src->numOfRows;
-  //  dst->nAlloc = src->nAlloc;
   dst->window = src->window;
   dst->status = src->status;
   

tests/script/general/parser/fill_us.sim

 system sh/stop_dnodes.sh
-system sh/ip.sh -i 1 -s up
-system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1
-system sh/cfg.sh -n dnode1 -c commitLog -v 0
+
+system sh/deploy.sh -n dnode1 -i 1
+system sh/cfg.sh -n dnode1 -c walLevel -v 0
 system sh/exec.sh -n dnode1 -s start
 sleep 3000
 sql connect

tests/script/general/parser/testSuite.sim

+sleep 2000
 run general/parser/alter.sim
 sleep 2000
 run general/parser/alter1.sim
@@ -7,7 +8,6 @@ sleep 2000
 run general/parser/auto_create_tb.sim
 sleep 2000
 run general/parser/auto_create_tb_drop_tb.sim
-
 sleep 2000
 run general/parser/col_arithmetic_operation.sim
 sleep 2000
@@ -23,65 +23,61 @@ run general/parser/create_tb.sim
 sleep 2000
 run general/parser/dbtbnameValidate.sim
 sleep 2000
+run general/parser/fill.sim
+sleep 2000
+run general/parser/fill_stb.sim
+sleep 2000                                    
+#run general/parser/fill_us.sim               #
+sleep 2000
+run general/parser/first_last.sim
+sleep 2000
 run general/parser/import_commit1.sim
 sleep 2000
 run general/parser/import_commit2.sim
 sleep 2000
 run general/parser/import_commit3.sim
 sleep 2000
+#run general/parser/import_file.sim
+sleep 2000
 run general/parser/insert_tb.sim
 sleep 2000
-run general/parser/first_last.sim
+run general/parser/tags_dynamically_specifiy.sim
 sleep 2000
-#run general/parser/import_file.sim
+run general/parser/interp.sim
 sleep 2000
 run general/parser/lastrow.sim
 sleep 2000
-run general/parser/nchar.sim
-sleep 2000
-#run general/parser/null_char.sim
+run general/parser/limit.sim
 sleep 2000
-run general/parser/single_row_in_tb.sim
+run general/parser/limit1.sim
 sleep 2000
-run general/parser/select_from_cache_disk.sim
+run general/parser/limit1_tblocks100.sim
 sleep 2000
-run general/parser/selectResNum.sim
+run general/parser/limit2.sim
 sleep 2000
 run general/parser/mixed_blocks.sim
 sleep 2000
-run general/parser/limit1.sim
+run general/parser/nchar.sim
 sleep 2000
-run general/parser/limit.sim
+run general/parser/null_char.sim
 sleep 2000
-run general/parser/limit1_tblocks100.sim
+run general/parser/selectResNum.sim
 sleep 2000
 run general/parser/select_across_vnodes.sim
 sleep 2000
-run general/parser/slimit1.sim
-sleep 2000
-run general/parser/tbnameIn.sim
-sleep 2000
-run general/parser/projection_limit_offset.sim
-sleep 2000
-run general/parser/limit2.sim
-sleep 2000
-run general/parser/fill.sim
+run general/parser/select_from_cache_disk.sim
 sleep 2000
-run general/parser/fill_stb.sim
+run general/parser/set_tag_vals.sim
 sleep 2000
-run general/parser/where.sim
+run general/parser/single_row_in_tb.sim
 sleep 2000
 run general/parser/slimit.sim
 sleep 2000
-run general/parser/select_with_tags.sim
-sleep 2000
-run general/parser/interp.sim
+run general/parser/slimit1.sim
 sleep 2000
-run general/parser/tags_dynamically_specifiy.sim
+run general/parser/slimit_alter_tags.sim
 sleep 2000
-run general/parser/groupby.sim
-sleep 2000
-run general/parser/set_tag_vals.sim
+run general/parser/tbnameIn.sim
 sleep 2000
 run general/parser/slimit_alter_tags.sim  # persistent failed
 sleep 2000
@@ -89,11 +85,19 @@ run general/parser/join.sim
 sleep 2000
 run general/parser/join_multivnode.sim
 sleep 2000
-run general/parser/repeatAlter.sim
+run general/parser/projection_limit_offset.sim
+sleep 2000
+run general/parser/select_with_tags.sim
+sleep 2000
+run general/parser/groupby.sim
+sleep 2000
+run general/parser/union.sim
+sleep 2000
+run general/parser/sliding.sim
 sleep 2000
-run general/parser/binary_escapeCharacter.sim
+run general/parser/fill_us.sim
 sleep 2000
-run general/parser/bug.sim
+run general/parser/tags_filter.sim
 
 #sleep 2000
 #run general/parser/repeatStream.sim