Merge branch 'feature/syncFC' of https://github.com/taosdata/TDengine into feature/syncFC

CMakeLists.txt

@@ -16,8 +16,8 @@ MESSAGE(STATUS "Community directory: " ${TD_COMMUNITY_DIR})
 
 INCLUDE(cmake/input.inc)
 INCLUDE(cmake/platform.inc)
-INCLUDE(cmake/env.inc)
 INCLUDE(cmake/define.inc)
+INCLUDE(cmake/env.inc)
 INCLUDE(cmake/install.inc)
 
 ADD_SUBDIRECTORY(deps)

cmake/define.inc

@@ -28,3 +28,107 @@ ENDIF ()
 IF (TD_RANDOM_NETWORK_FAIL)
   ADD_DEFINITIONS(-DTAOS_RANDOM_NETWORK_FAIL)
 ENDIF ()
+
+IF (TD_ARM)
+  ADD_DEFINITIONS(-D_TD_ARM_)
+  IF (TD_ARM_64)
+    ADD_DEFINITIONS(-D_TD_ARM_64_)
+  ENDIF ()
+  IF (TD_ARM_32)
+    ADD_DEFINITIONS(-D_TD_ARM_32_)
+  ENDIF ()
+ENDIF ()
+
+IF (TD_MIPS)
+  ADD_DEFINITIONS(-D_TD_MIPS_)
+  IF (TD_MIPS_64)
+    ADD_DEFINITIONS(-D_TD_MIPS_64_)
+  ENDIF ()
+  IF (TD_MIPS_32)
+    ADD_DEFINITIONS(-D_TD_MIPS_32_)
+  ENDIF ()
+ENDIF ()
+
+IF (TD_LINUX_64)
+  ADD_DEFINITIONS(-D_M_X64)
+  ADD_DEFINITIONS(-D_TD_LINUX_64)
+  IF (NOT TD_ARM)
+    IF (${CMAKE_CXX_COMPILER_ID} MATCHES "Clang")
+      SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -Wno-missing-braces -fPIC -g3 -gdwarf-2 -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
+    ELSE ()
+      SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -malign-double -g3 -gdwarf-2 -malign-stringops -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
+    ENDIF ()
+  ELSE ()
+    SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -g -fsigned-char -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
+  ENDIF ()
+  FIND_PATH(ICONV_INCLUDE_EXIST iconv.h /usr/include/ /usr/local/include/)
+  IF (ICONV_INCLUDE_EXIST)
+    ADD_DEFINITIONS(-DUSE_LIBICONV)
+  ENDIF ()
+ENDIF ()
+
+IF (TD_LINUX_32)
+  ADD_DEFINITIONS(-D_TD_LINUX_32)
+  ADD_DEFINITIONS(-DUSE_LIBICONV)
+  SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -g -fsigned-char -munaligned-access -fpack-struct=8 -latomic -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
+  IF (NOT TD_ARM)
+    EXIT ()
+  ENDIF ()
+ENDIF ()
+
+IF (TD_LINUX)
+  ADD_DEFINITIONS(-DLINUX)
+  ADD_DEFINITIONS(-D_TD_LINUX)
+  ADD_DEFINITIONS(-D_REENTRANT -D__USE_POSIX -D_LIBC_REENTRANT)
+  
+  SET(DEBUG_FLAGS "-O0 -DDEBUG")
+  SET(RELEASE_FLAGS "-O0")
+  
+  IF (${COVER} MATCHES "true")
+    MESSAGE(STATUS "Test coverage mode, add extra flags")
+    SET(GCC_COVERAGE_COMPILE_FLAGS "-fprofile-arcs -ftest-coverage")
+    SET(GCC_COVERAGE_LINK_FLAGS    "-lgcov --coverage")
+    SET(COMMON_FLAGS "${COMMON_FLAGS} ${GCC_COVERAGE_COMPILE_FLAGS} ${GCC_COVERAGE_LINK_FLAGS}")
+  ENDIF ()
+ENDIF ()
+
+IF (TD_APLHINE)
+  SET(COMMON_FLAGS "${COMMON_FLAGS} -largp")
+  link_libraries(/usr/lib/libargp.a)
+  ADD_DEFINITIONS(-D_ALPINE)
+ENDIF ()
+
+IF (TD_DARWIN_64)
+  ADD_DEFINITIONS(-D_TD_DARWIN_64)
+  ADD_DEFINITIONS(-DDARWIN)
+  ADD_DEFINITIONS(-D_REENTRANT -D__USE_POSIX -D_LIBC_REENTRANT)
+  ADD_DEFINITIONS(-DUSE_LIBICONV)
+  SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -Wno-missing-braces -fPIC -g -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
+  SET(DEBUG_FLAGS "-O0 -DDEBUG")
+  SET(RELEASE_FLAGS "-O0")
+ENDIF ()
+
+IF (TD_WINDOWS)
+  ADD_DEFINITIONS(-DWINDOWS)
+  ADD_DEFINITIONS(-D__CLEANUP_C)
+  ADD_DEFINITIONS(-DPTW32_STATIC_LIB)
+  ADD_DEFINITIONS(-DPTW32_BUILD)
+  ADD_DEFINITIONS(-D_MBCS -D_CRT_SECURE_NO_DEPRECATE -D_CRT_NONSTDC_NO_DEPRECATE)
+  SET(CMAKE_GENERATOR "NMake Makefiles" CACHE INTERNAL "" FORCE)
+  IF (NOT TD_GODLL)
+    SET(COMMON_FLAGS "/nologo /WX /Oi /Oy- /Gm- /EHsc /MT /GS /Gy /fp:precise /Zc:wchar_t /Zc:forScope /Gd /errorReport:prompt /analyze-")
+    SET(DEBUG_FLAGS "/Zi /W3 /GL")
+    SET(RELEASE_FLAGS "/W0 /GL")
+  ENDIF ()
+ENDIF () 
+
+IF (TD_WINDOWS_64)
+  ADD_DEFINITIONS(-D_M_X64)
+  ADD_DEFINITIONS(-D_TD_WINDOWS_64)
+  ADD_DEFINITIONS(-DUSE_LIBICONV)
+ENDIF ()
+
+IF (TD_WINDOWS_32)
+  ADD_DEFINITIONS(-D_TD_WINDOWS_32)
+ENDIF ()
+

cmake/platform.inc

@@ -14,6 +14,7 @@ PROJECT(TDengine)
 #
 
 # Set macro definitions according to os platform
+SET(TD_LINUX FALSE)
 SET(TD_LINUX_64 FALSE)
 SET(TD_LINUX_32 FALSE)
 SET(TD_ARM FALSE)
@@ -22,8 +23,11 @@ SET(TD_ARM_32 FALSE)
 SET(TD_MIPS FALSE)
 SET(TD_MIPS_64 FALSE)
 SET(TD_MIPS_32 FALSE)
-SET(TD_DARWIN_64 FALSE)
+SET(TD_WINDOWS FALSE)
 SET(TD_WINDOWS_64 FALSE)
+SET(TD_WINDOWS_32 FALSE)
+SET(TD_DARWIN_64 FALSE)
+SET(TD_APLHINE FALSE)
 
 # if generate ARM version:
 #   cmake -DCPUTYPE=aarch32 .. or cmake -DCPUTYPE=aarch64
@@ -31,27 +35,21 @@ IF (${CPUTYPE} MATCHES "aarch32")
   SET(TD_ARM TRUE)
   SET(TD_ARM_32 TRUE)
   SET(TD_PAGMODE_LITE TRUE)
-  ADD_DEFINITIONS(-D_TD_ARM_)
-  ADD_DEFINITIONS(-D_TD_ARM_32_)
   MESSAGE(STATUS "input cpuType: aarch32")
 ELSEIF (${CPUTYPE} MATCHES "aarch64")
   SET(TD_ARM TRUE)
   SET(TD_ARM_64 TRUE)
-  ADD_DEFINITIONS(-D_TD_ARM_)
-  ADD_DEFINITIONS(-D_TD_ARM_64_)
-   MESSAGE(STATUS "input cpuType: aarch64")
+  MESSAGE(STATUS "input cpuType: aarch64")
 ELSEIF (${CPUTYPE} MATCHES "mips64")
   SET(TD_MIPS TRUE)
   SET(TD_MIPS_64 TRUE)
-  ADD_DEFINITIONS(-D_TD_MIPS_)
-  ADD_DEFINITIONS(-D_TD_MIPS_64_)
   MESSAGE(STATUS "input cpuType: mips64")
 ELSEIF (${CPUTYPE} MATCHES "x64")
   MESSAGE(STATUS "input cpuType: x64")
 ELSEIF (${CPUTYPE} MATCHES "x86")
   MESSAGE(STATUS "input cpuType: x86")
 ELSE ()
-  MESSAGE(STATUS "input cpuType: " ${CPUTYPE})
+  MESSAGE(STATUS "input cpuType unknown " ${CPUTYPE})
 ENDIF ()
 
 #
@@ -62,16 +60,13 @@ execute_process(COMMAND ${TD_COMMUNITY_DIR}/packaging/tools/get_os.sh "" OUTPUT_
 MESSAGE(STATUS "The current os is " ${TD_OS_INFO})
 
 IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux")
+  SET(TD_LINUX TRUE)
   IF (${CMAKE_SIZEOF_VOID_P} MATCHES 8)
     SET(TD_LINUX_64 TRUE)
-    ADD_DEFINITIONS(-D_M_X64)
-    ADD_DEFINITIONS(-D_TD_LINUX_64)
     MESSAGE(STATUS "The current platform is Linux 64-bit")
   ELSEIF (${CMAKE_SIZEOF_VOID_P} MATCHES 4)
     SET(TD_LINUX_32 TRUE)
-    ADD_DEFINITIONS(-D_TD_LINUX_32)
     IF (TD_ARM)
-      #ADD_DEFINITIONS(-D_M_IX86)
       MESSAGE(STATUS "The current platform is Linux 32-bit")
     ELSE ()
       MESSAGE(FATAL_ERROR "The current platform is Linux 32-bit, but no ARM not supported yet")
@@ -81,24 +76,26 @@ IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux")
     MESSAGE(FATAL_ERROR "The current platform is Linux neither 32-bit nor 64-bit, not supported yet")
     EXIT ()
   ENDIF ()
+
+  IF (${TD_OS_INFO} MATCHES "Alpine")
+    SET(TD_APLHINE TRUE)
+    MESSAGE(STATUS "The current OS is Alpine, append extra flags")
+  ENDIF()
 ELSEIF (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
   IF (${CMAKE_SIZEOF_VOID_P} MATCHES 8)
     SET(TD_DARWIN_64 TRUE)
-    ADD_DEFINITIONS(-D_TD_DARWIN_64)
     MESSAGE(STATUS "The current platform is Darwin 64-bit")
   ELSE ()
     MESSAGE(FATAL_ERROR "The current platform is Darwin 32-bit, not supported yet")
     EXIT ()
   ENDIF ()
 ELSEIF (${CMAKE_SYSTEM_NAME} MATCHES "Windows")
+  SET(TD_WINDOWS TRUE)
   IF (${CMAKE_SIZEOF_VOID_P} MATCHES 8)
     SET(TD_WINDOWS_64 TRUE)
-    ADD_DEFINITIONS(-D_M_X64)
-    ADD_DEFINITIONS(-D_TD_WINDOWS_64)
     MESSAGE(STATUS "The current platform is Windows 64-bit")
   ELSE ()
     SET(TD_WINDOWS_32 TRUE)
-    ADD_DEFINITIONS(-D_TD_WINDOWS_32)
     MESSAGE(FATAL_ERROR "The current platform is Windows 32-bit, not supported yet")
     EXIT ()
   ENDIF ()
@@ -106,74 +103,3 @@ ELSE()
   MESSAGE(FATAL_ERROR "The current platform is not Linux/Darwin/Windows, stop compile")
   EXIT ()
 ENDIF ()
-
-IF (TD_LINUX_64)
-    SET(DEBUG_FLAGS "-O0 -DDEBUG")
-    SET(RELEASE_FLAGS "-O0")
-    IF (NOT TD_ARM)
-      IF (${CMAKE_CXX_COMPILER_ID} MATCHES "Clang")
-        SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -Wno-missing-braces -fPIC -g3 -gdwarf-2 -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
-      ELSE ()
-        SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -malign-double -g3 -gdwarf-2 -malign-stringops -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
-      ENDIF ()
-    ELSE ()
-      SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -g -fsigned-char -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
-    ENDIF ()
-    ADD_DEFINITIONS(-DLINUX)
-    ADD_DEFINITIONS(-D_REENTRANT -D__USE_POSIX -D_LIBC_REENTRANT)
-    IF (${TD_OS_INFO} MATCHES "Alpine")
-      MESSAGE(STATUS "The current OS is Alpine, append extra flags")
-      SET(COMMON_FLAGS "${COMMON_FLAGS} -largp")
-      link_libraries(/usr/lib/libargp.a)
-      ADD_DEFINITIONS(-D_ALPINE)
-    ENDIF ()
-    IF (${COVER} MATCHES "true")
-      MESSAGE(STATUS "Test coverage mode, add extra flags")
-      SET(GCC_COVERAGE_COMPILE_FLAGS "-fprofile-arcs -ftest-coverage")
-      SET(GCC_COVERAGE_LINK_FLAGS    "-lgcov --coverage")
-      SET(COMMON_FLAGS "${COMMON_FLAGS} ${GCC_COVERAGE_COMPILE_FLAGS} ${GCC_COVERAGE_LINK_FLAGS}")
-    ENDIF ()
-  ELSEIF (TD_LINUX_32)
-    IF (NOT TD_ARM)
-      EXIT ()
-    ENDIF ()
-    SET(DEBUG_FLAGS "-O0 -DDEBUG")
-    SET(RELEASE_FLAGS "-O0")
-    SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -g -fsigned-char -munaligned-access -fpack-struct=8 -latomic -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
-    ADD_DEFINITIONS(-DLINUX)
-    ADD_DEFINITIONS(-D_REENTRANT -D__USE_POSIX -D_LIBC_REENTRANT)
-    ADD_DEFINITIONS(-DUSE_LIBICONV)
-    IF (${TD_OS_INFO} MATCHES "Alpine")
-      MESSAGE(STATUS "The current OS is Alpine, add extra flags")
-      SET(COMMON_FLAGS "${COMMON_FLAGS} -largp")
-      link_library(/usr/lib/libargp.a)
-      ADD_DEFINITIONS(-D_ALPINE)
-    ENDIF ()
-    IF (${COVER} MATCHES "true")
-      MESSAGE(STATUS "Test coverage mode, add extra flags")
-      SET(GCC_COVERAGE_COMPILE_FLAGS "-fprofile-arcs -ftest-coverage")
-      SET(GCC_COVERAGE_LINK_FLAGS    "-lgcov --coverage")
-      SET(COMMON_FLAGS "${COMMON_FLAGS} ${GCC_COVERAGE_COMPILE_FLAGS} ${GCC_COVERAGE_LINK_FLAGS}")
-    ENDIF ()
-  ELSEIF (TD_WINDOWS_64)
-    SET(CMAKE_GENERATOR "NMake Makefiles" CACHE INTERNAL "" FORCE)
-    IF (NOT TD_GODLL)
-      SET(COMMON_FLAGS "/nologo /WX /Oi /Oy- /Gm- /EHsc /MT /GS /Gy /fp:precise /Zc:wchar_t /Zc:forScope /Gd /errorReport:prompt /analyze-")
-      SET(DEBUG_FLAGS "/Zi /W3 /GL")
-      SET(RELEASE_FLAGS "/W0 /GL")
-    ENDIF ()
-    ADD_DEFINITIONS(-DWINDOWS)
-    ADD_DEFINITIONS(-D__CLEANUP_C)
-    ADD_DEFINITIONS(-DPTW32_STATIC_LIB)
-    ADD_DEFINITIONS(-DPTW32_BUILD)
-    ADD_DEFINITIONS(-D_MBCS -D_CRT_SECURE_NO_DEPRECATE -D_CRT_NONSTDC_NO_DEPRECATE)
-  ELSEIF (TD_DARWIN_64)
-    SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -Wno-missing-braces -fPIC -g -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
-    SET(DEBUG_FLAGS "-O0 -DDEBUG")
-    SET(RELEASE_FLAGS "-O0")
-    ADD_DEFINITIONS(-DDARWIN)
-    ADD_DEFINITIONS(-D_REENTRANT -D__USE_POSIX -D_LIBC_REENTRANT)
-  ELSE ()
-    MESSAGE(FATAL_ERROR "The current platform is not support yet, stop compile")
-    EXIT ()
-  ENDIF ()

src/client/inc/tsclient.h

@@ -31,8 +31,8 @@ extern "C" {
 #include "tutil.h"
 
 #include "qExecutor.h"
+#include "qSqlparser.h"
 #include "qTsbuf.h"
-#include "qsqlparser.h"
 #include "tcmdtype.h"
 
 // forward declaration

src/client/src/tscAsync.c

@@ -430,7 +430,7 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) {
   pRes->code = code;
 
   if (code != TSDB_CODE_SUCCESS) {
-    tscError("%p ge tableMeta failed, code:%s", pSql, tstrerror(code));
+    tscError("%p get tableMeta failed, code:%s", pSql, tstrerror(code));
     goto _error;
   } else {
     tscDebug("%p get tableMeta successfully", pSql);

src/client/src/tscFunctionImpl.c

@@ -2131,6 +2131,11 @@ static STopBotInfo *getTopBotOutputInfo(SQLFunctionCtx *pCtx) {
 }
 
 bool topbot_datablock_filter(SQLFunctionCtx *pCtx, int32_t functionId, const char *minval, const char *maxval) {
+  SResultInfo *pResInfo = GET_RES_INFO(pCtx);
+  if (pResInfo == NULL) {
+    return true;
+  }
+
   STopBotInfo *pTopBotInfo = getTopBotOutputInfo(pCtx);
   
   // required number of results are not reached, continue load data block

src/client/src/tscLocalMerge.c

@@ -691,9 +691,15 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr
   
   pModel = createColumnModel(pSchema, size, capacity);
 
+  int32_t pg = DEFAULT_PAGE_SIZE;
+  int32_t overhead = sizeof(tFilePage);
+  while((pg - overhead) < pModel->rowSize * 2) {
+    pg *= 2;
+  }
+
   size_t numOfSubs = pTableMetaInfo->vgroupList->numOfVgroups;
   for (int32_t i = 0; i < numOfSubs; ++i) {
-    (*pMemBuffer)[i] = createExtMemBuffer(nBufferSizes, rlen, pModel);
+    (*pMemBuffer)[i] = createExtMemBuffer(nBufferSizes, rlen, pg, pModel);
     (*pMemBuffer)[i]->flushModel = MULTIPLE_APPEND_MODEL;
   }
 

src/client/src/tscSub.c

@@ -334,7 +334,7 @@ void tscSaveSubscriptionProgress(void* sub) {
 
   char path[256];
   sprintf(path, "%s/subscribe", tsDataDir);
-  if (tmkdir(path, 0777) != 0) {
+  if (taosMkDir(path, 0777) != 0) {
     tscError("failed to create subscribe dir: %s", path);
   }
 

src/client/src/tscSubquery.c

@@ -1505,12 +1505,11 @@ static int32_t tscReissueSubquery(SRetrieveSupport *trsupport, SSqlObj *pSql, in
 
   SSqlObj *pNew = tscCreateSqlObjForSubquery(trsupport->pParentSql, trsupport, pSql);
 
-  // todo add to async res or not??
   if (pNew == NULL) {
-    tscError("%p sub:%p failed to create new subquery due to out of memory, abort retry, vgId:%d, orderOfSub:%d",
-             trsupport->pParentSql, pSql, pVgroup->vgId, trsupport->subqueryIndex);
+    tscError("%p sub:%p failed to create new subquery due to error:%s, abort retry, vgId:%d, orderOfSub:%d",
+             trsupport->pParentSql, pSql, tstrerror(terrno), pVgroup->vgId, trsupport->subqueryIndex);
 
-    pParentSql->res.code = TSDB_CODE_TSC_OUT_OF_MEMORY;
+    pParentSql->res.code = terrno;
     trsupport->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY;
 
     return pParentSql->res.code;

src/client/src/tscSystem.c

@@ -148,7 +148,7 @@ void taos_init_imp() {
   refreshTime = refreshTime < 10 ? 10 : refreshTime;
 
   if (tscCacheHandle == NULL) {
-    tscCacheHandle = taosCacheInit(TSDB_DATA_TYPE_BINARY, refreshTime, false, NULL, "client");
+    tscCacheHandle = taosCacheInit(TSDB_DATA_TYPE_BINARY, refreshTime, false, NULL, "tableMeta");
   }
 
   tscDebug("client is initialized successfully");

src/client/src/tscUtil.c

@@ -356,9 +356,9 @@ void tscPartiallyFreeSqlObj(SSqlObj* pSql) {
   
   // pSql->sqlstr will be used by tscBuildQueryStreamDesc
   if (pObj->signature == pObj) {
-    pthread_mutex_lock(&pObj->mutex);
+    //pthread_mutex_lock(&pObj->mutex);
     tfree(pSql->sqlstr);
-    pthread_mutex_unlock(&pObj->mutex);
+    //pthread_mutex_unlock(&pObj->mutex);
   }
   
   tscFreeSqlResult(pSql);
@@ -1675,6 +1675,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void
   SSqlObj* pNew = (SSqlObj*)calloc(1, sizeof(SSqlObj));
   if (pNew == NULL) {
     tscError("%p new subquery failed, tableIndex:%d", pSql, tableIndex);
+    terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
     return NULL;
   }
   
@@ -1688,6 +1689,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void
     tscError("%p new subquery failed, tableIndex:%d, vgroupIndex:%d", pSql, tableIndex, pTableMetaInfo->vgroupIndex);
 
     free(pNew);
+    terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
     return NULL;
   }
 
@@ -1706,6 +1708,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void
 
   if (tscAddSubqueryInfo(pnCmd) != TSDB_CODE_SUCCESS) {
     tscFreeSqlObj(pNew);
+    terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
     return NULL;
   }
 
@@ -1743,6 +1746,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void
   if (tscAllocPayload(pnCmd, TSDB_DEFAULT_PAYLOAD_SIZE) != TSDB_CODE_SUCCESS) {
     tscError("%p new subquery failed, tableIndex:%d, vgroupIndex:%d", pSql, tableIndex, pTableMetaInfo->vgroupIndex);
     tscFreeSqlObj(pNew);
+    terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
     return NULL;
   }
   
@@ -1827,8 +1831,16 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void
   }
 
   if (pFinalInfo->pTableMeta == NULL) {
-    tscError("%p new subquery failed for get tableMeta is NULL from cache", pSql);
+    tscError("%p new subquery failed since no tableMeta in cache, name:%s", pSql, name);
     tscFreeSqlObj(pNew);
+
+    if (pPrevSql != NULL) {
+      assert(pPrevSql->res.code != TSDB_CODE_SUCCESS);
+      terrno = pPrevSql->res.code;
+    } else {
+      terrno = TSDB_CODE_TSC_APP_ERROR;
+    }
+
     return NULL;
   }
   

src/dnode/src/dnodeVRead.c

@@ -49,7 +49,7 @@ static taos_qset       readQset;
 int32_t dnodeInitVnodeRead() {
   readQset = taosOpenQset();
 
-  readPool.min = 2;
+  readPool.min = tsNumOfCores;
   readPool.max = tsNumOfCores * tsNumOfThreadsPerCore;
   if (readPool.max <= readPool.min * 2) readPool.max = 2 * readPool.min;
   readPool.readWorker = (SReadWorker *)calloc(sizeof(SReadWorker), readPool.max);
@@ -206,10 +206,14 @@ static void *dnodeProcessReadQueue(void *param) {
            taosMsg[pReadMsg->rpcMsg.msgType], type);
     int32_t code = vnodeProcessRead(pVnode, pReadMsg);
 
-    if (type == TAOS_QTYPE_RPC) {
+    if (type == TAOS_QTYPE_RPC && code != TSDB_CODE_QRY_NOT_READY) {
       dnodeSendRpcReadRsp(pVnode, pReadMsg, code);
     } else {
-      dnodeDispatchNonRspMsg(pVnode, pReadMsg, code);
+      if (code == TSDB_CODE_QRY_HAS_RSP) {
+        dnodeSendRpcReadRsp(pVnode, pReadMsg, TSDB_CODE_SUCCESS);
+      } else {
+        dnodeDispatchNonRspMsg(pVnode, pReadMsg, code);
+      }
     }
 
     taosFreeQitem(pReadMsg);

src/inc/query.h

@@ -28,7 +28,7 @@ typedef void* qinfo_t;
  * @param qinfo
  * @return
  */
-int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryTableMsg, void* param, qinfo_t* qinfo);
+int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryTableMsg, qinfo_t* qinfo);
 
 
 /**
@@ -38,7 +38,7 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryTableMs
  * @param qinfo
  * @return
  */
-void qTableQuery(qinfo_t qinfo);
+bool qTableQuery(qinfo_t qinfo);
 
 /**
  * Retrieve the produced results information, if current query is not paused or completed,
@@ -48,7 +48,7 @@ void qTableQuery(qinfo_t qinfo);
  * @param qinfo
  * @return
  */
-int32_t qRetrieveQueryResultInfo(qinfo_t qinfo);
+int32_t qRetrieveQueryResultInfo(qinfo_t qinfo, bool* buildRes, void* pRspContext);
 
 /**
  *
@@ -60,16 +60,14 @@ int32_t qRetrieveQueryResultInfo(qinfo_t qinfo);
  * @param contLen payload length
  * @return
  */
-int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp** pRsp, int32_t* contLen);
+int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp** pRsp, int32_t* contLen, bool* continueExec);
 
 /**
- * Decide if more results will be produced or not, NOTE: this function will increase the ref count of QInfo,
- * so it can be only called once for each retrieve
  *
  * @param qinfo
  * @return
  */
-bool qHasMoreResultsToRetrieve(qinfo_t qinfo);
+void* qGetResultRetrieveMsg(qinfo_t qinfo);
 
 /**
  * kill current ongoing query and free query handle automatically
@@ -89,7 +87,7 @@ void  qQueryMgmtNotifyClosed(void* pExecutor);
 void  qCleanupQueryMgmt(void* pExecutor);
 void** qRegisterQInfo(void* pMgmt, uint64_t qInfo);
 void** qAcquireQInfo(void* pMgmt, uint64_t key);
-void** qReleaseQInfo(void* pMgmt, void* pQInfo, bool needFree);
+void** qReleaseQInfo(void* pMgmt, void* pQInfo, bool freeHandle);
 
 #ifdef __cplusplus
 }

src/inc/taoserror.h

@@ -216,6 +216,8 @@ TAOS_DEFINE_ERROR(TSDB_CODE_QRY_OUT_OF_MEMORY,            0, 0x0703, "query out
 TAOS_DEFINE_ERROR(TSDB_CODE_QRY_APP_ERROR,                0, 0x0704, "query app error")
 TAOS_DEFINE_ERROR(TSDB_CODE_QRY_DUP_JOIN_KEY,             0, 0x0705, "query duplicated join key")
 TAOS_DEFINE_ERROR(TSDB_CODE_QRY_EXCEED_TAGS_LIMIT,        0, 0x0706, "query tag conditon too many")
+TAOS_DEFINE_ERROR(TSDB_CODE_QRY_NOT_READY,                0, 0x0707, "query not ready")
+TAOS_DEFINE_ERROR(TSDB_CODE_QRY_HAS_RSP,                  0, 0x0708, "query should response")
 
 // grant
 TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_EXPIRED,                0, 0x0800, "grant expired")

src/mnode/src/mnodeProfile.c

@@ -68,7 +68,7 @@ int32_t mnodeInitProfile() {
   mnodeAddWriteMsgHandle(TSDB_MSG_TYPE_CM_KILL_STREAM, mnodeProcessKillStreamMsg);
   mnodeAddWriteMsgHandle(TSDB_MSG_TYPE_CM_KILL_CONN, mnodeProcessKillConnectionMsg);
 
-  tsMnodeConnCache = taosCacheInit(TSDB_DATA_TYPE_INT, CONN_CHECK_TIME, false, mnodeFreeConn, "conn");
+  tsMnodeConnCache = taosCacheInit(TSDB_DATA_TYPE_INT, CONN_CHECK_TIME, true, mnodeFreeConn, "conn");
   return 0;
 }
 
@@ -119,7 +119,7 @@ SConnObj *mnodeAccquireConn(int32_t connId, char *user, uint32_t ip, uint16_t po
     return NULL;
   }
 
-  if (pConn->ip != ip || pConn->port != port /* || strcmp(pConn->user, user) != 0 */) {
+  if (/* pConn->ip != ip || */ pConn->port != port /* || strcmp(pConn->user, user) != 0 */) {
     mError("connId:%d, incoming conn user:%s ip:%s:%u, not match exist conn user:%s ip:%s:%u", connId, user,
            taosIpStr(ip), port, pConn->user, taosIpStr(pConn->ip), pConn->port);
     taosCacheRelease(tsMnodeConnCache, (void **)&pConn, false);

src/os/inc/osDarwin64.h

@@ -205,7 +205,6 @@ ssize_t tread(int fd, void *buf, size_t count);
 
 ssize_t twrite(int fd, void *buf, size_t n);
 
-char *taosCharsetReplace(char *charsetstr);
 
 bool taosCheckPthreadValid(pthread_t thread);
 
@@ -219,8 +218,6 @@ int taosSetSockOpt(int socketfd, int level, int optname, void *optval, int optle
 
 void taosPrintOsInfo();
 
-char *taosCharsetReplace(char *charsetstr);
-
 void taosPrintOsInfo();
 
 void taosGetSystemInfo();

src/os/inc/osSpec.h

@@ -20,6 +20,89 @@
 extern "C" {
 #endif
 
+#define tclose(x) taosCloseSocket(x)
+#define tfree(x)         \
+  do {                   \
+    if (x) {             \
+      free((void *)(x)); \
+      x = 0;             \
+    }                    \
+  } while (0);
+
+#define tstrncpy(dst, src, size)   \
+  do {                             \
+    strncpy((dst), (src), (size)); \
+    (dst)[(size)-1] = 0;           \
+  } while (0);
+
+#ifndef STDERR_FILENO
+#define STDERR_FILENO (2)
+#endif
+
+#define FD_VALID(x) ((x) > STDERR_FILENO)
+#define FD_INITIALIZER  ((int32_t)-1)
+
+#define WCHAR wchar_t
+
+#define POINTER_SHIFT(p, b) ((void *)((char *)(p) + (b)))
+#define POINTER_DISTANCE(p1, p2) ((char *)(p1) - (char *)(p2)) 
+
+#ifndef NDEBUG
+#define ASSERT(x) assert(x)
+#else
+#define ASSERT(x)
+#endif
+
+#ifdef UNUSED
+#undefine UNUSED
+#endif
+#define UNUSED(x) ((void)(x))
+
+#ifdef UNUSED_FUNC
+#undefine UNUSED_FUNC
+#endif
+
+#ifdef UNUSED_PARAM
+#undef UNUSED_PARAM
+#endif
+
+#if defined(__GNUC__)
+#define UNUSED_PARAM(x) _UNUSED##x __attribute__((unused))
+#define UNUSED_FUNC __attribute__((unused))
+#else
+#define UNUSED_PARAM(x) x
+#define UNUSED_FUNC
+#endif
+
+#ifdef tListLen
+#undefine tListLen
+#endif
+#define tListLen(x) (sizeof(x) / sizeof((x)[0]))
+
+#if defined(__GNUC__)
+#define FORCE_INLINE inline __attribute__((always_inline))
+#else
+#define FORCE_INLINE
+#endif
+
+#define DEFAULT_UNICODE_ENCODEC "UCS-4LE"
+  
+#define DEFAULT_COMP(x, y)       \
+  do {                           \
+    if ((x) == (y)) {            \
+      return 0;                  \
+    } else {                     \
+      return (x) < (y) ? -1 : 1; \
+    }                            \
+  } while (0)
+
+#define ALIGN_NUM(n, align) (((n) + ((align)-1)) & (~((align)-1)))
+
+// align to 8bytes
+#define ALIGN8(n) ALIGN_NUM(n, 8)
+
+#define POW2(x) ((x) * (x))
+
 #ifndef TAOS_OS_FUNC_MATH
   #define SWAP(a, b, c)        \
     do {                       \
@@ -46,6 +129,12 @@ extern "C" {
 #ifndef TAOS_OS_DEF_TIME
   #define MILLISECOND_PER_SECOND ((int64_t)1000L)
 #endif
+#define MILLISECOND_PER_MINUTE (MILLISECOND_PER_SECOND * 60)
+#define MILLISECOND_PER_HOUR   (MILLISECOND_PER_MINUTE * 60)
+#define MILLISECOND_PER_DAY    (MILLISECOND_PER_HOUR * 24)
+#define MILLISECOND_PER_WEEK   (MILLISECOND_PER_DAY * 7)
+#define MILLISECOND_PER_MONTH  (MILLISECOND_PER_DAY * 30)
+#define MILLISECOND_PER_YEAR   (MILLISECOND_PER_DAY * 365)
 
 #ifndef TAOS_OS_FUNC_SEMPHONE
   #define tsem_t sem_t
@@ -144,24 +233,13 @@ extern "C" {
 ssize_t taosTReadImp(int fd, void *buf, size_t count);
 ssize_t taosTWriteImp(int fd, void *buf, size_t count);
 ssize_t taosTSendFileImp(int dfd, int sfd, off_t *offset, size_t size);
-#ifndef TAOS_OS_FUNC_FILE
+#ifndef TAOS_OS_FUNC_FILE_OP
   #define taosTRead(fd, buf, count) taosTReadImp(fd, buf, count)
   #define taosTWrite(fd, buf, count) taosTWriteImp(fd, buf, count)
   #define taosLSeek(fd, offset, whence) lseek(fd, offset, whence)
   #define taosTSendFile(dfd, sfd, offset, size) taosTSendFileImp(dfd, sfd, offset, size)
 #endif
 
-#ifdef TAOS_RANDOM_FILE_FAIL
-  void taosSetRandomFileFailFactor(int factor);
-  void taosSetRandomFileFailOutput(const char *path);
-  ssize_t taosReadFileRandomFail(int fd, void *buf, size_t count, const char *file, uint32_t line);
-  ssize_t taosWriteFileRandomFail(int fd, void *buf, size_t count, const char *file, uint32_t line);
-  off_t taosLSeekRandomFail(int fd, off_t offset, int whence, const char *file, uint32_t line);
-  #define taosTRead(fd, buf, count) taosReadFileRandomFail(fd, buf, count, __FILE__, __LINE__)
-  #define taosTWrite(fd, buf, count) taosWriteFileRandomFail(fd, buf, count, __FILE__, __LINE__)
-  #define taosLSeek(fd, offset, whence) taosLSeekRandomFail(fd, offset, whence, __FILE__, __LINE__)
-#endif
-
 #ifndef TAOS_OS_FUNC_NETWORK
   #define taosSend(sockfd, buf, len, flags) send(sockfd, buf, len, flags)
   #define taosSendto(sockfd, buf, len, flags, dest_addr, addrlen) sendto(sockfd, buf, len, flags, dest_addr, addrlen)
@@ -176,17 +254,6 @@ ssize_t taosTSendFileImp(int dfd, int sfd, off_t *offset, size_t size);
     }
 #endif
 
-#ifdef TAOS_RANDOM_NETWORK_FAIL
-  ssize_t taosSendRandomFail(int sockfd, const void *buf, size_t len, int flags);
-  ssize_t taosSendToRandomFail(int sockfd, const void *buf, size_t len, int flags, const struct sockaddr *dest_addr, socklen_t addrlen);
-  ssize_t taosReadSocketRandomFail(int fd, void *buf, size_t count);
-  ssize_t taosWriteSocketRandomFail(int fd, const void *buf, size_t count);
-  #define taosSend(sockfd, buf, len, flags) taosSendRandomFail(sockfd, buf, len, flags)
-  #define taosSendto(sockfd, buf, len, flags, dest_addr, addrlen) taosSendToRandomFail(sockfd, buf, len, flags, dest_addr, addrlen)
-  #define taosReadSocket(fd, buf, len) taosReadSocketRandomFail(fd, buf, len)
-  #define taosWriteSocket(fd, buf, len) taosWriteSocketRandomFail(fd, buf, len)
-#endif
-
 #ifndef TAOS_OS_FUNC_LZ4
   #define BUILDIN_CLZL(val) __builtin_clzl(val)
   #define BUILDIN_CTZL(val) __builtin_ctzl(val)
@@ -194,6 +261,10 @@ ssize_t taosTSendFileImp(int dfd, int sfd, off_t *offset, size_t size);
   #define BUILDIN_CTZ(val) __builtin_ctz(val)
 #endif
 
+#ifndef TAOS_OS_FUNC_WCHAR
+  #define twcslen wcslen
+#endif  
+
 #undef threadlocal
 #ifdef _ISOC11_SOURCE
   #define threadlocal _Thread_local
@@ -233,6 +304,81 @@ void taosMsleep(int mseconds);
 int taosInitTimer(void (*callback)(int), int ms);
 void taosUninitTimer();
 
+// TAOS_OS_FUNC_RAND
+uint32_t taosRand(void);
+void taosRandStr(char* str, int32_t size);
+uint32_t trand(void);
+
+// TAOS_OS_FUNC_FILE
+void getTmpfilePath(const char *fileNamePrefix, char *dstPath);
+int32_t taosFileRename(char *fullPath, char *suffix, char delimiter, char **dstPath);
+
+// USE_LIBICONV
+int32_t taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs);
+bool    taosMbsToUcs4(char *mbs, size_t mbs_len, char *ucs4, int32_t ucs4_max_len, size_t *len);
+int     tasoUcs4Compare(void *f1_ucs4, void *f2_ucs4, int bytes);
+bool    taosValidateEncodec(const char *encodec);
+char *  taosCharsetReplace(char *charsetstr);
+
+// TAOS_OS_FUNC_MALLOC
+#define TAOS_ALLOC_MODE_DEFAULT 0
+#define TAOS_ALLOC_MODE_RANDOM_FAIL 1
+#define TAOS_ALLOC_MODE_DETECT_LEAK 2
+void   taosSetAllocMode(int mode, const char *path, bool autoDump);
+void   taosDumpMemoryLeak();
+void * tmalloc(size_t size);
+void * tcalloc(size_t nmemb, size_t size);
+size_t tsizeof(void *ptr);
+void   tmemset(void *ptr, int c);
+void * trealloc(void *ptr, size_t size);
+void   tzfree(void *ptr);
+
+// TAOS_OS_FUNC_DIR
+void taosRemoveDir(char *rootDir);
+int  taosMkDir(const char *pathname, mode_t mode); 
+void taosMvDir(char* destDir, char *srcDir);
+
+
+
+#ifdef TAOS_RANDOM_FILE_FAIL
+  void taosSetRandomFileFailFactor(int factor);
+  void taosSetRandomFileFailOutput(const char *path);
+  ssize_t taosReadFileRandomFail(int fd, void *buf, size_t count, const char *file, uint32_t line);
+  ssize_t taosWriteFileRandomFail(int fd, void *buf, size_t count, const char *file, uint32_t line);
+  off_t taosLSeekRandomFail(int fd, off_t offset, int whence, const char *file, uint32_t line);
+  #define taosTRead(fd, buf, count) taosReadFileRandomFail(fd, buf, count, __FILE__, __LINE__)
+  #define taosTWrite(fd, buf, count) taosWriteFileRandomFail(fd, buf, count, __FILE__, __LINE__)
+  #define taosLSeek(fd, offset, whence) taosLSeekRandomFail(fd, offset, whence, __FILE__, __LINE__)
+#endif
+
+#ifdef TAOS_RANDOM_NETWORK_FAIL
+  ssize_t taosSendRandomFail(int sockfd, const void *buf, size_t len, int flags);
+  ssize_t taosSendToRandomFail(int sockfd, const void *buf, size_t len, int flags, const struct sockaddr *dest_addr, socklen_t addrlen);
+  ssize_t taosReadSocketRandomFail(int fd, void *buf, size_t count);
+  ssize_t taosWriteSocketRandomFail(int fd, const void *buf, size_t count);
+  #define taosSend(sockfd, buf, len, flags) taosSendRandomFail(sockfd, buf, len, flags)
+  #define taosSendto(sockfd, buf, len, flags, dest_addr, addrlen) taosSendToRandomFail(sockfd, buf, len, flags, dest_addr, addrlen)
+  #define taosReadSocket(fd, buf, len) taosReadSocketRandomFail(fd, buf, len)
+  #define taosWriteSocket(fd, buf, len) taosWriteSocketRandomFail(fd, buf, len)
+#endif
+
+#ifdef TAOS_MEM_CHECK
+  void *  taos_malloc(size_t size, const char *file, uint32_t line);
+  void *  taos_calloc(size_t num, size_t size, const char *file, uint32_t line);
+  void *  taos_realloc(void *ptr, size_t size, const char *file, uint32_t line);
+  void    taos_free(void *ptr, const char *file, uint32_t line);
+  char *  taos_strdup(const char *str, const char *file, uint32_t line);
+  char *  taos_strndup(const char *str, size_t size, const char *file, uint32_t line);
+  ssize_t taos_getline(char **lineptr, size_t *n, FILE *stream, const char *file, uint32_t line);
+  #define malloc(size) taos_malloc(size, __FILE__, __LINE__)
+  #define calloc(num, size) taos_calloc(num, size, __FILE__, __LINE__)
+  #define realloc(ptr, size) taos_realloc(ptr, size, __FILE__, __LINE__)
+  #define free(ptr) taos_free(ptr, __FILE__, __LINE__)
+  #define strdup(str) taos_strdup(str, __FILE__, __LINE__)
+  #define strndup(str, size) taos_strndup(str, size, __FILE__, __LINE__)
+  #define getline(lineptr, n, stream) taos_getline(lineptr, n, stream, __FILE__, __LINE__)
+#endif  // TAOS_MEM_CHECK
+
 #ifdef __cplusplus
 }
 #endif

src/os/inc/osWindows64.h

@@ -48,6 +48,8 @@
 extern "C" {
 #endif
 
+#define TAOS_OS_FUNC_WCHAR
+
 // for function open in stat.h 
 #define S_IRWXU                  _S_IREAD
 #define S_IRWXG                  _S_IWRITE
@@ -359,7 +361,6 @@ int taosSetNonblocking(int sock, int on);
 
 int taosSetSockOpt(int socketfd, int level, int optname, void *optval, int optlen);
 
-char *taosCharsetReplace(char *charsetstr);
 
 void taosPrintOsInfo();
 

src/os/src/detail/osCoredump.c

@@ -18,7 +18,6 @@
 #include "tconfig.h"
 #include "tglobal.h"
 #include "tulog.h"
-#include "tutil.h"
 #include "tsystem.h"
 
 #ifndef TAOS_OS_FUNC_CORE

src/os/src/detail/osDir.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "tglobal.h"
+#include "tulog.h"
+
+#ifndef TAOS_OS_FUNC_DIR
+
+void taosRemoveDir(char *rootDir) {
+  DIR *dir = opendir(rootDir);
+  if (dir == NULL) return;
+
+  struct dirent *de = NULL;
+  while ((de = readdir(dir)) != NULL) {
+    if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) continue;
+     
+    char filename[1024];
+    snprintf(filename, 1023, "%s/%s", rootDir, de->d_name);
+    if (de->d_type & DT_DIR) {
+      taosRemoveDir(filename);
+    } else {
+      (void)remove(filename);
+      uInfo("file:%s is removed", filename);
+    }
+  }
+
+  closedir(dir);
+  rmdir(rootDir);
+
+  uInfo("dir:%s is removed", rootDir);
+}
+
+int taosMkDir(const char *path, mode_t mode) {
+  int code = mkdir(path, 0755);
+  if (code < 0 && errno == EEXIST) code = 0;
+  return code;
+}
+
+void taosMvDir(char* destDir, char *srcDir) {
+  if (0 == tsEnableVnodeBak) {
+    uInfo("vnode backup not enabled");
+    return;
+  }
+
+  char shellCmd[1024+1] = {0}; 
+  
+  //(void)snprintf(shellCmd, 1024, "cp -rf %s %s", srcDir, destDir);
+  (void)snprintf(shellCmd, 1024, "mv %s %s", srcDir, destDir);
+  tSystem(shellCmd);
+  uInfo("shell cmd:%s is executed", shellCmd);
+}
+
+
+#endif
\ No newline at end of file

src/os/src/detail/osFile.c

@@ -15,74 +15,54 @@
 
 #define _DEFAULT_SOURCE
 #include "os.h"
+#include "ttime.h"
 
-ssize_t taosTReadImp(int fd, void *buf, size_t count) {
-  size_t  leftbytes = count;
-  ssize_t readbytes;
-  char *  tbuf = (char *)buf;
+#ifndef TAOS_OS_FUNC_FILE
 
-  while (leftbytes > 0) {
-    readbytes = read(fd, (void *)tbuf, leftbytes);
-    if (readbytes < 0) {
-      if (errno == EINTR) {
-        continue;
-      } else {
-        return -1;
-      }
-    } else if (readbytes == 0) {
-      return (ssize_t)(count - leftbytes);
-    }
-
-    leftbytes -= readbytes;
-    tbuf += readbytes;
+void getTmpfilePath(const char *fileNamePrefix, char *dstPath) {
+  const char* tdengineTmpFileNamePrefix = "tdengine-";
+  
+  char tmpPath[PATH_MAX];
+  char *tmpDir = "/tmp/";
+  
+  strcpy(tmpPath, tmpDir);
+  strcat(tmpPath, tdengineTmpFileNamePrefix);
+  if (strlen(tmpPath) + strlen(fileNamePrefix) + strlen("-%d-%s") < PATH_MAX) {
+    strcat(tmpPath, fileNamePrefix);
+    strcat(tmpPath, "-%d-%s");
   }
-
-  return (ssize_t)count;
+  
+  char rand[8] = {0};
+  taosRandStr(rand, tListLen(rand) - 1);
+  snprintf(dstPath, PATH_MAX, tmpPath, getpid(), rand);
 }
 
-ssize_t taosTSendFileImp(int dfd, int sfd, off_t *offset, size_t size) {
-  size_t  leftbytes = size;
-  ssize_t sentbytes;
+// rename file name
+int32_t taosFileRename(char *fullPath, char *suffix, char delimiter, char **dstPath) {
+  int32_t ts = taosGetTimestampSec();
 
-  while (leftbytes > 0) {
-    /*
-     * TODO : Think to check if file is larger than 1GB
-     */
-    //if (leftbytes > 1000000000) leftbytes = 1000000000;
-    sentbytes = sendfile(dfd, sfd, offset, leftbytes);
-    if (sentbytes == -1) {
-      if (errno == EINTR) {
-        continue;
-      }
-      else {
-        return -1;
-      }
-    } else if (sentbytes == 0) {
-      return (ssize_t)(size - leftbytes);
-    }
+  char fname[PATH_MAX] = {0};  // max file name length must be less than 255
 
-    leftbytes -= sentbytes;
-  }
+  char *delimiterPos = strrchr(fullPath, delimiter);
+  if (delimiterPos == NULL) return -1;
 
-  return size;
-}
+  int32_t fileNameLen = 0;
+  if (suffix)
+    fileNameLen = snprintf(fname, PATH_MAX, "%s.%d.%s", delimiterPos + 1, ts, suffix);
+  else
+    fileNameLen = snprintf(fname, PATH_MAX, "%s.%d", delimiterPos + 1, ts);
 
-ssize_t taosTWriteImp(int fd, void *buf, size_t n) {
-  size_t nleft = n; 
-  ssize_t nwritten = 0;
-  char *tbuf = (char *)buf;
-
-  while (nleft > 0) {
-    nwritten = write(fd, (void *)tbuf, nleft);
-    if (nwritten < 0) {
-      if (errno == EINTR) {
-        continue;
-      }
-      return -1;
-    }
-    nleft -= nwritten;
-    tbuf += nwritten;
+  size_t len = (size_t)((delimiterPos - fullPath) + fileNameLen + 1);
+  if (*dstPath == NULL) {
+    *dstPath = calloc(1, len + 1);
+    if (*dstPath == NULL) return -1;
   }
 
-  return n;
+  strncpy(*dstPath, fullPath, (size_t)(delimiterPos - fullPath + 1));
+  strncat(*dstPath, fname, (size_t)fileNameLen);
+  (*dstPath)[len] = 0;
+
+  return rename(fullPath, *dstPath);
 }
+
+#endif
\ No newline at end of file

src/os/src/detail/osFileOp.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+
+ssize_t taosTReadImp(int fd, void *buf, size_t count) {
+  size_t  leftbytes = count;
+  ssize_t readbytes;
+  char *  tbuf = (char *)buf;
+
+  while (leftbytes > 0) {
+    readbytes = read(fd, (void *)tbuf, leftbytes);
+    if (readbytes < 0) {
+      if (errno == EINTR) {
+        continue;
+      } else {
+        return -1;
+      }
+    } else if (readbytes == 0) {
+      return (ssize_t)(count - leftbytes);
+    }
+
+    leftbytes -= readbytes;
+    tbuf += readbytes;
+  }
+
+  return (ssize_t)count;
+}
+
+ssize_t taosTSendFileImp(int dfd, int sfd, off_t *offset, size_t size) {
+  size_t  leftbytes = size;
+  ssize_t sentbytes;
+
+  while (leftbytes > 0) {
+    /*
+     * TODO : Think to check if file is larger than 1GB
+     */
+    //if (leftbytes > 1000000000) leftbytes = 1000000000;
+    sentbytes = sendfile(dfd, sfd, offset, leftbytes);
+    if (sentbytes == -1) {
+      if (errno == EINTR) {
+        continue;
+      }
+      else {
+        return -1;
+      }
+    } else if (sentbytes == 0) {
+      return (ssize_t)(size - leftbytes);
+    }
+
+    leftbytes -= sentbytes;
+  }
+
+  return size;
+}
+
+ssize_t taosTWriteImp(int fd, void *buf, size_t n) {
+  size_t nleft = n; 
+  ssize_t nwritten = 0;
+  char *tbuf = (char *)buf;
+
+  while (nleft > 0) {
+    nwritten = write(fd, (void *)tbuf, nleft);
+    if (nwritten < 0) {
+      if (errno == EINTR) {
+        continue;
+      }
+      return -1;
+    }
+    nleft -= nwritten;
+    tbuf += nwritten;
+  }
+
+  return n;
+}

src/os/src/detail/osMalloc.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+
+#ifndef TAOS_OS_FUNC_MALLOC
+
+void *tmalloc(size_t size) {
+  if (size <= 0) return NULL;
+
+  void *ret = malloc(size + sizeof(size_t));
+  if (ret == NULL) return NULL;
+
+  *(size_t *)ret = size;
+
+  return (void *)((char *)ret + sizeof(size_t));
+}
+
+void *tcalloc(size_t nmemb, size_t size) {
+  size_t tsize = nmemb * size;
+  void * ret = tmalloc(tsize);
+  if (ret == NULL) return NULL;
+
+  tmemset(ret, 0);
+  return ret;
+}
+
+size_t tsizeof(void *ptr) { return (ptr) ? (*(size_t *)((char *)ptr - sizeof(size_t))) : 0; }
+
+void tmemset(void *ptr, int c) { memset(ptr, c, tsizeof(ptr)); }
+
+void * trealloc(void *ptr, size_t size) {
+  if (ptr == NULL) return tmalloc(size);
+
+  if (size <= tsizeof(ptr)) return ptr;
+
+  void * tptr = (void *)((char *)ptr - sizeof(size_t));
+  size_t tsize = size + sizeof(size_t);
+  tptr = realloc(tptr, tsize);
+  if (tptr == NULL) return NULL;
+
+  *(size_t *)tptr = size;
+
+  return (void *)((char *)tptr + sizeof(size_t));
+}
+
+void tzfree(void *ptr) {
+  if (ptr) {
+    free((void *)((char *)ptr - sizeof(size_t)));
+  }
+}
+
+#endif
\ No newline at end of file

src/util/src/tmem.c → src/os/src/detail/osMem.c

@@ -13,13 +13,10 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
+#define _DEFAULT_SOURCE
 #include "os.h"
 #include "tulog.h"
 
-#define TAOS_MEM_CHECK_IMPL
-#include "tutil.h"
-
-
 #ifdef TAOS_MEM_CHECK
 
 static int allocMode = TAOS_ALLOC_MODE_DEFAULT;

src/os/src/detail/osRand.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+
+#ifndef TAOS_OS_FUNC_RAND
+
+uint32_t taosRand(void) { return rand(); }
+
+uint32_t trand(void) {
+  int fd;
+  int seed;
+
+  fd = open("/dev/urandom", 0);
+  if (fd < 0) {
+    seed = time(0);
+  } else {
+    int len = read(fd, &seed, sizeof(seed));
+    if (len < 0) {
+      seed = time(0);
+    }
+    close(fd);
+  }
+
+  return (uint32_t)seed;
+}
+
+#endif
+
+void taosRandStr(char* str, int32_t size) {
+  const char* set = "abcdefghijklmnopqrstuvwxyz0123456789-_.";
+  int32_t     len = 39;
+
+  for (int32_t i = 0; i < size; ++i) {
+    str[i] = set[taosRand() % len];
+  }
+}
\ No newline at end of file

src/os/src/detail/osSysinfo.c

@@ -18,8 +18,8 @@
 #include "tconfig.h"
 #include "tglobal.h"
 #include "tulog.h"
-#include "tutil.h"
 #include "tsystem.h"
+#include "taosdef.h"
 
 #ifndef TAOS_OS_FUNC_SYSINFO
 

src/os/src/detail/osTimer.c

@@ -13,13 +13,12 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
+#define _DEFAULT_SOURCE
 #include "os.h"
 #include "taosdef.h"
 #include "tglobal.h"
 #include "ttimer.h"
 #include "tulog.h"
-#include "tutil.h"
-#include <signal.h>
 
 #ifndef TAOS_OS_FUNC_TIMER
 

src/os/src/detail/osWchar.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "tglobal.h"
+#include "taosdef.h"
+
+#ifndef TAOS_OS_FUNC_WCHAR
+
+int tasoUcs4Compare(void* f1_ucs4, void *f2_ucs4, int bytes) {
+  return wcsncmp((wchar_t *)f1_ucs4, (wchar_t *)f2_ucs4, bytes / TSDB_NCHAR_SIZE);
+}
+
+#endif
+
+#ifdef USE_LIBICONV
+#include "iconv.h"
+
+int32_t taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs) {
+  iconv_t cd = iconv_open(tsCharset, DEFAULT_UNICODE_ENCODEC);
+  size_t  ucs4_input_len = ucs4_max_len;
+  size_t  outLen = ucs4_max_len;
+  if (iconv(cd, (char **)&ucs4, &ucs4_input_len, &mbs, &outLen) == -1) {
+    iconv_close(cd);
+    return -1;
+  }
+
+  iconv_close(cd);
+  return (int32_t)(ucs4_max_len - outLen);
+}
+
+bool taosMbsToUcs4(char *mbs, size_t mbsLength, char *ucs4, int32_t ucs4_max_len, size_t *len) {
+  memset(ucs4, 0, ucs4_max_len);
+  iconv_t cd = iconv_open(DEFAULT_UNICODE_ENCODEC, tsCharset);
+  size_t  ucs4_input_len = mbsLength;
+  size_t  outLeft = ucs4_max_len;
+  if (iconv(cd, &mbs, &ucs4_input_len, &ucs4, &outLeft) == -1) {
+    iconv_close(cd);
+    return false;
+  }
+
+  iconv_close(cd);
+  if (len != NULL) {
+    *len = ucs4_max_len - outLeft;
+  }
+
+  return true;
+}
+
+bool taosValidateEncodec(const char *encodec) {
+  iconv_t cd = iconv_open(encodec, DEFAULT_UNICODE_ENCODEC);
+  if (cd == (iconv_t)(-1)) {
+    return false;
+  }
+
+  iconv_close(cd);
+  return true;
+}
+
+#else
+
+int32_t taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs) {
+  mbstate_t state = {0};
+  int32_t   len = (int32_t)wcsnrtombs(NULL, (const wchar_t **)&ucs4, ucs4_max_len / 4, 0, &state);
+  if (len < 0) {
+    return -1;
+  }
+
+  memset(&state, 0, sizeof(state));
+  len = wcsnrtombs(mbs, (const wchar_t **)&ucs4, ucs4_max_len / 4, (size_t)len, &state);
+  if (len < 0) {
+    return -1;
+  }
+
+  return len;
+}
+
+bool taosMbsToUcs4(char *mbs, size_t mbsLength, char *ucs4, int32_t ucs4_max_len, size_t *len) {
+  memset(ucs4, 0, ucs4_max_len);
+  mbstate_t state = {0};
+  int32_t retlen = mbsnrtowcs((wchar_t *)ucs4, (const char **)&mbs, mbsLength, ucs4_max_len / 4, &state);
+  *len = retlen;
+
+  return retlen >= 0;
+}
+
+bool taosValidateEncodec(const char *encodec) {
+  return true;
+}
+
+#endif
+
+typedef struct CharsetPair {
+  char *oldCharset;
+  char *newCharset;
+} CharsetPair;
+
+char *taosCharsetReplace(char *charsetstr) {
+  CharsetPair charsetRep[] = {
+      { "utf8", "UTF-8" }, { "936", "CP936" },
+  };
+
+  for (int32_t i = 0; i < tListLen(charsetRep); ++i) {
+    if (strcasecmp(charsetRep[i].oldCharset, charsetstr) == 0) {
+      return strdup(charsetRep[i].newCharset);
+    }
+  }
+
+  return strdup(charsetstr);
+}
\ No newline at end of file

src/os/src/windows64/w64File.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+
+void getTmpfilePath(const char *fileNamePrefix, char *dstPath) {
+  const char* tdengineTmpFileNamePrefix = "tdengine-";
+    char tmpPath[PATH_MAX];
+
+  char *tmpDir = getenv("tmp");
+  if (tmpDir == NULL) {
+    tmpDir = "";
+  }
+  
+  strcpy(tmpPath, tmpDir);
+  strcat(tmpPath, tdengineTmpFileNamePrefix);
+  if (strlen(tmpPath) + strlen(fileNamePrefix) + strlen("-%d-%s") < PATH_MAX) {
+    strcat(tmpPath, fileNamePrefix);
+    strcat(tmpPath, "-%d-%s");
+  }
+  
+  char rand[8] = {0};
+  taosRandStr(rand, tListLen(rand) - 1);
+  snprintf(dstPath, PATH_MAX, tmpPath, getpid(), rand);
+}

src/os/src/windows64/w64Wchar.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "tglobal.h"
+#include "tulog.h"
+
+size_t twcslen(const wchar_t *wcs) {
+  int *wstr = (int *)wcs;
+  if (NULL == wstr) {
+    return 0;
+  }
+
+  size_t n = 0;
+  while (1) {
+    if (0 == *wstr++) {
+      break;
+    }
+    n++;
+  }
+
+  return n;
+}
+
+int tasoUcs4Compare(void *f1_ucs4, void *f2_ucs4, int bytes) {
+  for (int i = 0; i < bytes; ++i) {
+    int32_t f1 = *(int32_t *)((char *)f1_ucs4 + i * 4);
+    int32_t f2 = *(int32_t *)((char *)f2_ucs4 + i * 4);
+
+    if ((f1 == 0 && f2 != 0) || (f1 != 0 && f2 == 0)) {
+      return f1 - f2;
+    } else if (f1 == 0 && f2 == 0) {
+      return 0;
+    }
+
+    if (f1 != f2) {
+      return f1 - f2;
+    }
+  }
+
+  return 0;
+
+#if 0
+  int32_t ucs4_max_len = bytes + 4;
+  char *f1_mbs = calloc(bytes, 1);
+  char *f2_mbs = calloc(bytes, 1);
+  if (taosUcs4ToMbs(f1_ucs4, ucs4_max_len, f1_mbs) < 0) {
+    return -1;
+  }
+  if (taosUcs4ToMbs(f2_ucs4, ucs4_max_len, f2_mbs) < 0) {
+    return -1;
+  }
+  int32_t ret = strcmp(f1_mbs, f2_mbs);
+  free(f1_mbs);
+  free(f2_mbs);
+  return ret;
+#endif
+}

src/plugins/http/src/httpContext.c

@@ -58,7 +58,7 @@ static void httpDestroyContext(void *data) {
 }
 
 bool httpInitContexts() {
-  tsHttpServer.contextCache = taosCacheInit(TSDB_DATA_TYPE_BIGINT, 2, false, httpDestroyContext, "restc");
+  tsHttpServer.contextCache = taosCacheInit(TSDB_DATA_TYPE_BIGINT, 2, true, httpDestroyContext, "restc");
   if (tsHttpServer.contextCache == NULL) {
     httpError("failed to init context cache");
     return false;

src/query/inc/qAst.h

@@ -42,7 +42,6 @@ typedef void (*__do_filter_suppl_fn_t)(void *, void *);
 
 /**
  * this structure is used to filter data in tags, so the offset of filtered tag column in tagdata string is required
- *
  */
 typedef struct tQueryInfo {
   uint8_t       optr;     // expression operator

src/query/inc/qExecutor.h

@@ -20,8 +20,8 @@
 #include "hash.h"
 #include "qFill.h"
 #include "qResultbuf.h"
+#include "qSqlparser.h"
 #include "qTsbuf.h"
-#include "qsqlparser.h"
 #include "query.h"
 #include "taosdef.h"
 #include "tarray.h"
@@ -43,7 +43,7 @@ typedef struct SSqlGroupbyExpr {
 
 typedef struct SPosInfo {
   int32_t pageId;
-  int16_t rowId;
+  int32_t rowId;
 } SPosInfo;
 
 typedef struct SWindowStatus {
@@ -177,13 +177,18 @@ typedef struct SQueryRuntimeEnv {
   SDiskbasedResultBuf* pResultBuf;       // query result buffer based on blocked-wised disk file
 } SQueryRuntimeEnv;
 
+enum {
+  QUERY_RESULT_NOT_READY = 1,
+  QUERY_RESULT_READY     = 2,
+};
+
 typedef struct SQInfo {
   void*            signature;
   int32_t          pointsInterpo;
   int32_t          code;  // error code to returned to client
-  sem_t            dataReady;
+//  sem_t            dataReady;
+
   void*            tsdb;
-  void*            param;
   int32_t          vgId;
   STableGroupInfo  tableGroupInfo;       // table id list < only includes the STable list>
   STableGroupInfo  tableqinfoGroupInfo;  // this is a group array list, including SArray<STableQueryInfo*> structure
@@ -200,8 +205,11 @@ typedef struct SQInfo {
    */
   int32_t          tableIndex;
   int32_t          numOfGroupResultPages;
-  void*            pBuf; // allocated buffer for STableQueryInfo, sizeof(STableQueryInfo)*numOfTables;
+  void*            pBuf;        // allocated buffer for STableQueryInfo, sizeof(STableQueryInfo)*numOfTables;
 
+  pthread_mutex_t  lock;        // used to synchronize the rsp/query threads
+  int32_t          dataReady;   // denote if query result is ready or not
+  void*            rspContext;  // response context
 } SQInfo;
 
 #endif  // TDENGINE_QUERYEXECUTOR_H

src/query/inc/qExtbuffer.h

@@ -19,7 +19,6 @@
 extern "C" {
 #endif
 
-
 #include "os.h"
 #include "taosmsg.h"
 
@@ -28,9 +27,9 @@ extern "C" {
 #include "tdataformat.h"
 #include "talgo.h"
 
-#define DEFAULT_PAGE_SIZE (1024L*4)  // 16k larger than the SHistoInfo
-#define MAX_TMPFILE_PATH_LENGTH PATH_MAX
+#define MAX_TMPFILE_PATH_LENGTH        PATH_MAX
 #define INITIAL_ALLOCATION_BUFFER_SIZE 64
+#define DEFAULT_PAGE_SIZE              (4096L)  // 16k larger than the SHistoInfo
 
 typedef enum EXT_BUFFER_FLUSH_MODEL {
   /*
@@ -126,7 +125,7 @@ typedef struct tExtMemBuffer {
  * @param pModel
  * @return
  */
-tExtMemBuffer *createExtMemBuffer(int32_t inMemSize, int32_t elemSize, SColumnModel *pModel);
+tExtMemBuffer *createExtMemBuffer(int32_t inMemSize, int32_t elemSize, int32_t pagesize, SColumnModel *pModel);
 
 /**
  *

src/query/inc/qResultbuf.h

@@ -13,50 +13,85 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef TDENGINE_VNODEQUERYUTIL_H
-#define TDENGINE_VNODEQUERYUTIL_H
+#ifndef TDENGINE_QRESULTBUF_H
+#define TDENGINE_QRESULTBUF_H
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
+#include <tlist.h>
 #include "hash.h"
 #include "os.h"
 #include "qExtbuffer.h"
+#include "tlockfree.h"
 
 typedef struct SArray* SIDList;
 
+typedef struct SPageDiskInfo {
+  int32_t offset;
+  int32_t length;
+} SPageDiskInfo;
+
+typedef struct SPageInfo {
+  SListNode*    pn;       // point to list node
+  int32_t       pageId;
+  SPageDiskInfo info;
+  void*         pData;
+  bool          used;     // set current page is in used
+} SPageInfo;
+
+typedef struct SFreeListItem {
+  int32_t offset;
+  int32_t len;
+} SFreeListItem;
+
+typedef struct SResultBufStatis {
+  int32_t flushBytes;
+  int32_t loadBytes;
+  int32_t getPages;
+  int32_t releasePages;
+  int32_t flushPages;
+} SResultBufStatis;
+
 typedef struct SDiskbasedResultBuf {
   int32_t   numOfRowsPerPage;
   int32_t   numOfPages;
   int64_t   totalBufSize;
-  int32_t   fd;                  // data file fd
+  int64_t   fileSize;            // disk file size
+  FILE*     file;
   int32_t   allocateId;          // allocated page id
-  int32_t   incStep;             // minimum allocated pages
-  void*     pBuf;                // mmap buffer pointer
   char*     path;                // file path
   int32_t   pageSize;            // current used page size
   int32_t   inMemPages;          // numOfPages that are allocated in memory
-  SHashObj* idsTable;            // id hash table
-  SIDList   list;                // for each id, there is a page id list
-
-  void*     iBuf;                // inmemory buf
-  void*     handle;              // for debug purpose
+  SHashObj* groupSet;            // id hash table
+  SHashObj* all;
+  SList*    lruList;
   void*     emptyDummyIdList;    // dummy id list
+  void*     assistBuf;           // assistant buffer for compress/decompress data
+  SArray*   pFree;               // free area in file
+  bool      comp;                // compressed before flushed to disk
+  int32_t   nextPos;             // next page flush position
+
+  const void*      handle;        // for debug purpose
+  SResultBufStatis statis;
 } SDiskbasedResultBuf;
 
-#define DEFAULT_INTERN_BUF_PAGE_SIZE (1024L)
+#define DEFAULT_INTERN_BUF_PAGE_SIZE  (4096L)
 #define DEFAULT_INMEM_BUF_PAGES       10
+#define PAGE_INFO_INITIALIZER         (SPageDiskInfo){-1, -1}
 
 /**
  * create disk-based result buffer
  * @param pResultBuf
- * @param size
  * @param rowSize
+ * @param pagesize
+ * @param inMemPages
+ * @param handle
  * @return
  */
-int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t numOfPages, int32_t rowSize, int32_t pagesize,
-    int32_t inMemPages, void* handle);
+int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t rowSize, int32_t pagesize,
+                                    int32_t inMemBufSize, const void* handle);
 
 /**
  *
@@ -72,7 +107,7 @@ tFilePage* getNewDataBuf(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int32
  * @param pResultBuf
  * @return
  */
-int32_t getNumOfRowsPerPage(SDiskbasedResultBuf* pResultBuf);
+size_t getNumOfRowsPerPage(const SDiskbasedResultBuf* pResultBuf);
 
 /**
  *
@@ -88,42 +123,52 @@ SIDList getDataBufPagesIdList(SDiskbasedResultBuf* pResultBuf, int32_t groupId);
  * @param id
  * @return
  */
-static FORCE_INLINE tFilePage* getResBufPage(SDiskbasedResultBuf* pResultBuf, int32_t id) {
-  if (id < pResultBuf->inMemPages) {
-    return (tFilePage*) ((char*) pResultBuf->iBuf + id * pResultBuf->pageSize);
-  } else {
-    return (tFilePage*) ((char*) pResultBuf->pBuf + (id - pResultBuf->inMemPages) * pResultBuf->pageSize);
-  }
-}
+tFilePage* getResBufPage(SDiskbasedResultBuf* pResultBuf, int32_t id);
+
+/**
+ * release the referenced buf pages
+ * @param pResultBuf
+ * @param page
+ */
+void releaseResBufPage(SDiskbasedResultBuf* pResultBuf, void* page);
+
+/**
+ *
+ * @param pResultBuf
+ * @param pi
+ */
+void releaseResBufPageInfo(SDiskbasedResultBuf* pResultBuf, SPageInfo* pi);
+
+
 /**
  * get the total buffer size in the format of disk file
  * @param pResultBuf
  * @return
  */
-int32_t getResBufSize(SDiskbasedResultBuf* pResultBuf);
+size_t getResBufSize(const SDiskbasedResultBuf* pResultBuf);
 
 /**
  * get the number of groups in the result buffer
  * @param pResultBuf
  * @return
  */
-int32_t getNumOfResultBufGroupId(SDiskbasedResultBuf* pResultBuf);
+size_t getNumOfResultBufGroupId(const SDiskbasedResultBuf* pResultBuf);
 
 /**
  * destroy result buffer
  * @param pResultBuf
  */
-void destroyResultBuf(SDiskbasedResultBuf* pResultBuf, void* handle);
+void destroyResultBuf(SDiskbasedResultBuf* pResultBuf);
 
 /**
  *
  * @param pList
  * @return
  */
-int32_t getLastPageId(SIDList pList);
+SPageInfo* getLastPageInfo(SIDList pList);
 
 #ifdef __cplusplus
 }
 #endif
 
-#endif  // TDENGINE_VNODEQUERYUTIL_H
+#endif  // TDENGINE_QRESULTBUF_H

src/query/inc/qUtil.h

@@ -45,13 +45,14 @@ bool isWindowResClosed(SWindowResInfo *pWindowResInfo, int32_t slot);
 
 int32_t createQueryResultInfo(SQuery *pQuery, SWindowResult *pResultRow, bool isSTableQuery, size_t interBufSize);
 
-static FORCE_INLINE char *getPosInResultPage(SQueryRuntimeEnv *pRuntimeEnv, int32_t columnIndex, SWindowResult *pResult) {
+static FORCE_INLINE char *getPosInResultPage(SQueryRuntimeEnv *pRuntimeEnv, int32_t columnIndex, SWindowResult *pResult,
+    tFilePage* page) {
   assert(pResult != NULL && pRuntimeEnv != NULL);
 
   SQuery    *pQuery = pRuntimeEnv->pQuery;
-  tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pResult->pos.pageId);
-  int32_t realRowId = pResult->pos.rowId * GET_ROW_PARAM_FOR_MULTIOUTPUT(pQuery, pRuntimeEnv->topBotQuery, pRuntimeEnv->stableQuery);
+//  tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pResult->pos.pageId);
 
+  int32_t realRowId = pResult->pos.rowId * GET_ROW_PARAM_FOR_MULTIOUTPUT(pQuery, pRuntimeEnv->topBotQuery, pRuntimeEnv->stableQuery);
   return ((char *)page->data) + pRuntimeEnv->offset[columnIndex] * pRuntimeEnv->numOfRowsPerPage +
       pQuery->pSelectExpr[columnIndex].bytes * realRowId;
 }

src/query/inc/tsqlfunction.h

@@ -22,7 +22,7 @@ extern "C" {
 
 #include "os.h"
 
-#include "../../common/inc/tname.h"
+#include "tname.h"
 #include "taosdef.h"
 #include "trpc.h"
 #include "tvariant.h"

src/query/src/qAst.c

@@ -18,8 +18,8 @@
 
 #include "exception.h"
 #include "qAst.h"
+#include "qSqlparser.h"
 #include "qSyntaxtreefunction.h"
-#include "qsqlparser.h"
 #include "taosdef.h"
 #include "taosmsg.h"
 #include "tarray.h"
@@ -878,28 +878,6 @@ void tExprTreeCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput,
   free(pRightOutput);
 }
 
-//void tSQLBinaryExprTrv(tExprNode *pExprs, SArray* res) {
-//  if (pExprs == NULL) {
-//    return;
-//  }
-//
-//  tExprNode *pLeft = pExprs->_node.pLeft;
-//  tExprNode *pRight = pExprs->_node.pRight;
-//
-//  // recursive traverse left child branch
-//  if (pLeft->nodeType == TSQL_NODE_EXPR) {
-//    tSQLBinaryExprTrv(pLeft, res);
-//  } else if (pLeft->nodeType == TSQL_NODE_COL) {
-//    taosArrayPush(res, &pLeft->pSchema->colId);
-//  }
-//
-//  if (pRight->nodeType == TSQL_NODE_EXPR) {
-//    tSQLBinaryExprTrv(pRight, res);
-//  } else if (pRight->nodeType == TSQL_NODE_COL) {
-//    taosArrayPush(res, &pRight->pSchema->colId);
-//  }
-//}
-
 static void exprTreeToBinaryImpl(SBufferWriter* bw, tExprNode* expr) {
   tbufWriteUint8(bw, expr->nodeType);
   

src/query/src/qExecutor.c

@@ -221,7 +221,7 @@ void updateNumOfResult(SQueryRuntimeEnv *pRuntimeEnv, int32_t numOfRes) {
 }
 
 static int32_t getGroupResultId(int32_t groupIndex) {
-  int32_t base = 200000;
+  int32_t base = 20000000;
   return base + (groupIndex * 10000);
 }
 
@@ -478,10 +478,14 @@ static int32_t addNewWindowResultBuf(SWindowResult *pWindowRes, SDiskbasedResult
   if (taosArrayGetSize(list) == 0) {
     pData = getNewDataBuf(pResultBuf, sid, &pageId);
   } else {
-    pageId = getLastPageId(list);
-    pData = getResBufPage(pResultBuf, pageId);
+    SPageInfo* pi = getLastPageInfo(list);
+    pData = getResBufPage(pResultBuf, pi->pageId);
+    pageId = pi->pageId;
 
     if (pData->num >= numOfRowsPerPage) {
+      // release current page first, and prepare the next one
+      releaseResBufPageInfo(pResultBuf, pi);
+
       pData = getNewDataBuf(pResultBuf, sid, &pageId);
       if (pData != NULL) {
         assert(pData->num == 0);  // number of elements must be 0 for new allocated buffer
@@ -497,6 +501,8 @@ static int32_t addNewWindowResultBuf(SWindowResult *pWindowRes, SDiskbasedResult
   if (pWindowRes->pos.pageId == -1) {  // not allocated yet, allocate new buffer
     pWindowRes->pos.pageId = pageId;
     pWindowRes->pos.rowId = pData->num++;
+
+    assert(pWindowRes->pos.pageId >= 0);
   }
 
   return 0;
@@ -1490,8 +1496,6 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order
     goto _clean;
   }
 
-  qDebug("QInfo:%p setup runtime env1", GET_QINFO_ADDR(pRuntimeEnv));
-
   pRuntimeEnv->offset[0] = 0;
   for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
     SSqlFuncMsg *pSqlFuncMsg = &pQuery->pSelectExpr[i].base;
@@ -1536,8 +1540,6 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order
       }
     }
 
-    qDebug("QInfo:%p setup runtime env2", GET_QINFO_ADDR(pRuntimeEnv));
-
     // set the order information for top/bottom query
     int32_t functionId = pCtx->functionId;
 
@@ -1558,25 +1560,19 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order
     }
   }
 
-  qDebug("QInfo:%p setup runtime env3", GET_QINFO_ADDR(pRuntimeEnv));
-
   char* buf = (char*) pRuntimeEnv->resultInfo + sizeof(SResultInfo) * pQuery->numOfOutput;
 
   // set the intermediate result output buffer
   setWindowResultInfo(pRuntimeEnv->resultInfo, pQuery, pRuntimeEnv->stableQuery, buf);
 
-  qDebug("QInfo:%p setup runtime env4", GET_QINFO_ADDR(pRuntimeEnv));
-
   // if it is group by normal column, do not set output buffer, the output buffer is pResult
   if (!pRuntimeEnv->groupbyNormalCol && !pRuntimeEnv->stableQuery) {
     resetCtxOutputBuf(pRuntimeEnv);
   }
 
-  qDebug("QInfo:%p setup runtime env5", GET_QINFO_ADDR(pRuntimeEnv));
-
   setCtxTagColumnInfo(pRuntimeEnv, pRuntimeEnv->pCtx);
 
-  qDebug("QInfo:%p init completed", GET_QINFO_ADDR(pRuntimeEnv));
+  qDebug("QInfo:%p init runtime completed", GET_QINFO_ADDR(pRuntimeEnv));
   return TSDB_CODE_SUCCESS;
 
 _clean:
@@ -1615,7 +1611,7 @@ static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) {
 
   pRuntimeEnv->pFillInfo = taosDestoryFillInfo(pRuntimeEnv->pFillInfo);
 
-  destroyResultBuf(pRuntimeEnv->pResultBuf, pQInfo);
+  destroyResultBuf(pRuntimeEnv->pResultBuf);
   tsdbCleanupQueryHandle(pRuntimeEnv->pQueryHandle);
   tsdbCleanupQueryHandle(pRuntimeEnv->pSecQueryHandle);
 
@@ -2111,9 +2107,6 @@ int32_t loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, void* pQueryHandle,
     }
     
     if (!needToLoadDataBlock(pRuntimeEnv, *pStatis, pRuntimeEnv->pCtx, pBlockInfo->rows)) {
-#if defined(_DEBUG_VIEW)
-      qDebug("QInfo:%p block discarded by per-filter", GET_QINFO_ADDR(pRuntimeEnv));
-#endif
       // current block has been discard due to filter applied
       pRuntimeEnv->summary.discardBlocks += 1;
       qDebug("QInfo:%p data block discard, brange:%"PRId64 "-%"PRId64", rows:%d", GET_QINFO_ADDR(pRuntimeEnv),
@@ -2446,6 +2439,8 @@ static void doMerge(SQueryRuntimeEnv *pRuntimeEnv, int64_t timestamp, SWindowRes
   SQuery *        pQuery = pRuntimeEnv->pQuery;
   SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx;
 
+  tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes->pos.pageId);
+
   for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
     int32_t functionId = pQuery->pSelectExpr[i].base.functionId;
     if (!mergeFlag) {
@@ -2458,7 +2453,7 @@ static void doMerge(SQueryRuntimeEnv *pRuntimeEnv, int64_t timestamp, SWindowRes
 
     pCtx[i].hasNull = true;
     pCtx[i].nStartQueryTimestamp = timestamp;
-    pCtx[i].aInputElemBuf = getPosInResultPage(pRuntimeEnv, i, pWindowRes);
+    pCtx[i].aInputElemBuf = getPosInResultPage(pRuntimeEnv, i, pWindowRes, page);
 
     // in case of tag column, the tag information should be extracted from input buffer
     if (functionId == TSDB_FUNC_TAG_DUMMY || functionId == TSDB_FUNC_TAG) {
@@ -2615,14 +2610,16 @@ int32_t tableResultComparFn(const void *pLeft, const void *pRight, void *param)
 
   SWindowResInfo *pWindowResInfo1 = &supporter->pTableQueryInfo[left]->windowResInfo;
   SWindowResult * pWindowRes1 = getWindowResult(pWindowResInfo1, leftPos);
+  tFilePage *page1 = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes1->pos.pageId);
 
-  char *b1 = getPosInResultPage(pRuntimeEnv, PRIMARYKEY_TIMESTAMP_COL_INDEX, pWindowRes1);
+  char *b1 = getPosInResultPage(pRuntimeEnv, PRIMARYKEY_TIMESTAMP_COL_INDEX, pWindowRes1, page1);
   TSKEY leftTimestamp = GET_INT64_VAL(b1);
 
   SWindowResInfo *pWindowResInfo2 = &supporter->pTableQueryInfo[right]->windowResInfo;
   SWindowResult * pWindowRes2 = getWindowResult(pWindowResInfo2, rightPos);
+  tFilePage *page2 = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes2->pos.pageId);
 
-  char *b2 = getPosInResultPage(pRuntimeEnv, PRIMARYKEY_TIMESTAMP_COL_INDEX, pWindowRes2);
+  char *b2 = getPosInResultPage(pRuntimeEnv, PRIMARYKEY_TIMESTAMP_COL_INDEX, pWindowRes2, page2);
   TSKEY rightTimestamp = GET_INT64_VAL(b2);
 
   if (leftTimestamp == rightTimestamp) {
@@ -2685,35 +2682,26 @@ void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) {
   int32_t id = getGroupResultId(pQInfo->groupIndex - 1);
   SIDList list = getDataBufPagesIdList(pResultBuf, pQInfo->offset + id);
 
-  int32_t total = 0;
   int32_t size = taosArrayGetSize(list);
-  for (int32_t i = 0; i < size; ++i) {
-    int32_t* pgId = taosArrayGet(list, i);
-    tFilePage *pData = getResBufPage(pResultBuf, *pgId);
-    total += pData->num;
-  }
-
-  int32_t rows = total;
 
   int32_t offset = 0;
   for (int32_t j = 0; j < size; ++j) {
-    int32_t* pgId = taosArrayGet(list, j);
-    tFilePage *pData = getResBufPage(pResultBuf, *pgId);
+    SPageInfo* pi = *(SPageInfo**) taosArrayGet(list, j);
+    tFilePage *pData = getResBufPage(pResultBuf, pi->pageId);
 
     for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
       int32_t bytes = pRuntimeEnv->pCtx[i].outputBytes;
       char *  pDest = pQuery->sdata[i]->data;
-
-      memcpy(pDest + offset * bytes, pData->data + pRuntimeEnv->offset[i] * pData->num,
-             bytes * pData->num);
+      memcpy(pDest + offset * bytes, pData->data + pRuntimeEnv->offset[i] * pData->num, bytes * pData->num);
     }
 
+//    rows += pData->num;
     offset += pData->num;
   }
 
   assert(pQuery->rec.rows == 0);
 
-  pQuery->rec.rows += rows;
+  pQuery->rec.rows += offset;
   pQInfo->offset += 1;
 }
 
@@ -2777,7 +2765,6 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) {
     assert(pQInfo->numOfGroupResultPages == 0);
     return 0;
   } else if (numOfTables == 1) { // no need to merge results since only one table in each group
-
   }
 
   SCompSupporter cs = {pTableList, posList, pQInfo};
@@ -2802,8 +2789,9 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) {
 
     SWindowResInfo *pWindowResInfo = &pTableList[pos]->windowResInfo;
     SWindowResult * pWindowRes = getWindowResult(pWindowResInfo, cs.position[pos]);
+    tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes->pos.pageId);
 
-    char *b = getPosInResultPage(pRuntimeEnv, PRIMARYKEY_TIMESTAMP_COL_INDEX, pWindowRes);
+    char *b = getPosInResultPage(pRuntimeEnv, PRIMARYKEY_TIMESTAMP_COL_INDEX, pWindowRes, page);
     TSKEY ts = GET_INT64_VAL(b);
 
     assert(ts == pWindowRes->window.skey);
@@ -3517,9 +3505,11 @@ void setWindowResOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pResult
   SQuery *pQuery = pRuntimeEnv->pQuery;
 
   // Note: pResult->pos[i]->num == 0, there is only fixed number of results for each group
+  tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pResult->pos.pageId);
+
   for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
     SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i];
-    pCtx->aOutputBuf = getPosInResultPage(pRuntimeEnv, i, pResult);
+    pCtx->aOutputBuf = getPosInResultPage(pRuntimeEnv, i, pResult, page);
 
     int32_t functionId = pQuery->pSelectExpr[i].base.functionId;
     if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF) {
@@ -3542,6 +3532,8 @@ void setWindowResOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *
   SQuery *pQuery = pRuntimeEnv->pQuery;
 
   // Note: pResult->pos[i]->num == 0, there is only fixed number of results for each group
+  tFilePage* bufPage = getResBufPage(pRuntimeEnv->pResultBuf, pResult->pos.pageId);
+
   for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
     SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i];
 
@@ -3550,7 +3542,7 @@ void setWindowResOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *
       continue;
     }
 
-    pCtx->aOutputBuf = getPosInResultPage(pRuntimeEnv, i, pResult);
+    pCtx->aOutputBuf = getPosInResultPage(pRuntimeEnv, i, pResult, bufPage);
     pCtx->currentStage = 0;
 
     int32_t functionId = pCtx->functionId;
@@ -3713,11 +3705,13 @@ static int32_t doCopyToSData(SQInfo *pQInfo, SWindowResInfo *pResultInfo, int32_
       pQInfo->groupIndex += 1;
     }
 
+    tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, result[i].pos.pageId);
+
     for (int32_t j = 0; j < pQuery->numOfOutput; ++j) {
       int32_t size = pRuntimeEnv->pCtx[j].outputBytes;
 
       char *out = pQuery->sdata[j]->data + numOfResult * size;
-      char *in = getPosInResultPage(pRuntimeEnv, j, &result[i]);
+      char *in = getPosInResultPage(pRuntimeEnv, j, &result[i], page);
       memcpy(out, in + oldOffset * size, size * numOfRowsToCopy);
     }
 
@@ -4238,10 +4232,10 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo
   int32_t ps = DEFAULT_PAGE_SIZE;
   int32_t rowsize = 0;
   getIntermediateBufInfo(pRuntimeEnv, &ps, &rowsize);
+  int32_t TWOMB = 1024*1024*2;
 
   if (isSTableQuery && !onlyQueryTags(pRuntimeEnv->pQuery)) {
-    int32_t numOfPages = getInitialPageNum(pQInfo);
-    code = createDiskbasedResultBuffer(&pRuntimeEnv->pResultBuf, numOfPages, rowsize, ps, numOfPages, pQInfo);
+    code = createDiskbasedResultBuffer(&pRuntimeEnv->pResultBuf, rowsize, ps, TWOMB, pQInfo);
     if (code != TSDB_CODE_SUCCESS) {
       return code;
     }
@@ -4269,8 +4263,7 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo
   } else if (pRuntimeEnv->groupbyNormalCol || QUERY_IS_INTERVAL_QUERY(pQuery)) {
     int32_t numOfResultRows = getInitialPageNum(pQInfo);
     getIntermediateBufInfo(pRuntimeEnv, &ps, &rowsize);
-
-    code = createDiskbasedResultBuffer(&pRuntimeEnv->pResultBuf, numOfResultRows, rowsize, ps, numOfResultRows, pQInfo);
+    code = createDiskbasedResultBuffer(&pRuntimeEnv->pResultBuf, rowsize, ps, TWOMB, pQInfo);
     if (code != TSDB_CODE_SUCCESS) {
       return code;
     }
@@ -5894,16 +5887,11 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SArray* pTableIdList,
   }
 
   pQInfo->arrTableIdInfo = taosArrayInit(tableIndex, sizeof(STableIdInfo));
+  pQInfo->dataReady = QUERY_RESULT_NOT_READY;
+  pthread_mutex_init(&pQInfo->lock, NULL);
 
   pQuery->pos = -1;
   pQuery->window = pQueryMsg->window;
-
-  if (sem_init(&pQInfo->dataReady, 0, 0) != 0) {
-    int32_t code = TAOS_SYSTEM_ERROR(errno);
-    qError("QInfo:%p init dataReady sem failed, reason:%s", pQInfo, tstrerror(code));
-    goto _cleanup;
-  }
-
   colIdCheck(pQuery);
 
   qDebug("qmsg:%p QInfo:%p created", pQueryMsg, pQInfo);
@@ -5945,7 +5933,7 @@ static bool isValidQInfo(void *param) {
   return (sig == (uint64_t)pQInfo);
 }
 
-static int32_t initQInfo(SQueryTableMsg *pQueryMsg, void *tsdb, int32_t vgId, SQInfo *pQInfo, bool isSTable, void* param) {
+static int32_t initQInfo(SQueryTableMsg *pQueryMsg, void *tsdb, int32_t vgId, SQInfo *pQInfo, bool isSTable) {
   int32_t code = TSDB_CODE_SUCCESS;
   SQuery *pQuery = pQInfo->runtimeEnv.pQuery;
 
@@ -5965,18 +5953,12 @@ static int32_t initQInfo(SQueryTableMsg *pQueryMsg, void *tsdb, int32_t vgId, SQ
            pQuery->window.ekey, pQuery->order.order);
     setQueryStatus(pQuery, QUERY_COMPLETED);
     pQInfo->tableqinfoGroupInfo.numOfTables = 0;
-
-    sem_post(&pQInfo->dataReady);
     return TSDB_CODE_SUCCESS;
   }
 
-  pQInfo->param = param;
-
   if (pQInfo->tableqinfoGroupInfo.numOfTables == 0) {
     qDebug("QInfo:%p no table qualified for tag filter, abort query", pQInfo);
     setQueryStatus(pQuery, QUERY_COMPLETED);
-  
-    sem_post(&pQInfo->dataReady);
     return TSDB_CODE_SUCCESS;
   }
 
@@ -6018,7 +6000,6 @@ static void freeQInfo(SQInfo *pQInfo) {
     tfree(pQuery->sdata[col]);
   }
 
-  sem_destroy(&(pQInfo->dataReady));
   teardownQueryRuntimeEnv(&pQInfo->runtimeEnv);
 
   for (int32_t i = 0; i < pQuery->numOfFilterCols; ++i) {
@@ -6170,7 +6151,7 @@ typedef struct SQueryMgmt {
   pthread_mutex_t lock;
 } SQueryMgmt;
 
-int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, void* param, qinfo_t* pQInfo) {
+int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, qinfo_t* pQInfo) {
   assert(pQueryMsg != NULL && tsdb != NULL);
 
   int32_t code = TSDB_CODE_SUCCESS;
@@ -6266,7 +6247,7 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, vo
     goto _over;
   }
 
-  code = initQInfo(pQueryMsg, tsdb, vgId, *pQInfo, isSTableQuery, param);
+  code = initQInfo(pQueryMsg, tsdb, vgId, *pQInfo, isSTableQuery);
 
 _over:
   free(tagCond);
@@ -6306,24 +6287,32 @@ void qDestroyQueryInfo(qinfo_t qHandle) {
   freeQInfo(pQInfo);
 }
 
-void qTableQuery(qinfo_t qinfo) {
+static void setQueryResultReady(SQInfo* pQInfo) {
+  pthread_mutex_lock(&pQInfo->lock);
+  pQInfo->dataReady = QUERY_RESULT_READY;
+  pthread_mutex_unlock(&pQInfo->lock);
+}
+
+bool qTableQuery(qinfo_t qinfo) {
   SQInfo *pQInfo = (SQInfo *)qinfo;
 
   if (pQInfo == NULL || pQInfo->signature != pQInfo) {
     qDebug("QInfo:%p has been freed, no need to execute", pQInfo);
-    return;
+    return false;
   }
 
   if (IS_QUERY_KILLED(pQInfo)) {
     qDebug("QInfo:%p it is already killed, abort", pQInfo);
-    sem_post(&pQInfo->dataReady);
-    return;
+    setQueryResultReady(pQInfo);
+    return false;
   }
 
   if (pQInfo->tableqinfoGroupInfo.numOfTables == 0) {
+    setQueryStatus(pQInfo->runtimeEnv.pQuery, QUERY_COMPLETED);
+    setQueryResultReady(pQInfo);
+
     qDebug("QInfo:%p no table exists for query, abort", pQInfo);
-    sem_post(&pQInfo->dataReady);
-    return;
+    return false;
   }
 
   // error occurs, record the error code and return to client
@@ -6331,8 +6320,9 @@ void qTableQuery(qinfo_t qinfo) {
   if (ret != TSDB_CODE_SUCCESS) {
     pQInfo->code = ret;
     qDebug("QInfo:%p query abort due to error/cancel occurs, code:%s", pQInfo, tstrerror(pQInfo->code));
-    sem_post(&pQInfo->dataReady);
-    return;
+
+    setQueryResultReady(pQInfo);
+    return false;
   }
 
   qDebug("QInfo:%p query task is launched", pQInfo);
@@ -6357,10 +6347,20 @@ void qTableQuery(qinfo_t qinfo) {
            pQInfo, pQuery->rec.rows, pQuery->rec.total + pQuery->rec.rows);
   }
 
-  sem_post(&pQInfo->dataReady);
+  bool buildRes = false;
+  pthread_mutex_lock(&pQInfo->lock);
+  pQInfo->dataReady = QUERY_RESULT_READY;
+
+  if (pQInfo->rspContext != NULL) {
+    buildRes = true;
+  }
+
+
+  pthread_mutex_unlock(&pQInfo->lock);
+  return buildRes;
 }
 
-int32_t qRetrieveQueryResultInfo(qinfo_t qinfo) {
+int32_t qRetrieveQueryResultInfo(qinfo_t qinfo, bool* buildRes, void* pRspContext) {
   SQInfo *pQInfo = (SQInfo *)qinfo;
 
   if (pQInfo == NULL || !isValidQInfo(pQInfo)) {
@@ -6373,11 +6373,21 @@ int32_t qRetrieveQueryResultInfo(qinfo_t qinfo) {
     return pQInfo->code;
   }
 
-  sem_wait(&pQInfo->dataReady);
-  qDebug("QInfo:%p retrieve result info, rowsize:%d, rows:%"PRId64", code:%d", pQInfo, pQuery->rowSize, pQuery->rec.rows,
-         pQInfo->code);
+  int32_t code = TSDB_CODE_SUCCESS;
+  pthread_mutex_lock(&pQInfo->lock);
+  if (pQInfo->dataReady == QUERY_RESULT_READY) {
+    *buildRes = true;
+    qDebug("QInfo:%p retrieve result info, rowsize:%d, rows:%"PRId64", code:%d", pQInfo, pQuery->rowSize, pQuery->rec.rows,
+           pQInfo->code);
+  } else {
+    *buildRes = false;
+    qDebug("QInfo:%p retrieve req set query return result after paused", pQInfo);
+    pQInfo->rspContext = pRspContext;
+  }
 
-  return pQInfo->code;
+  code = pQInfo->code;
+  pthread_mutex_unlock(&pQInfo->lock);
+  return code;
 }
 
 bool qHasMoreResultsToRetrieve(qinfo_t qinfo) {
@@ -6389,6 +6399,7 @@ bool qHasMoreResultsToRetrieve(qinfo_t qinfo) {
   }
 
   SQuery *pQuery = pQInfo->runtimeEnv.pQuery;
+
   bool ret = false;
   if (Q_STATUS_EQUAL(pQuery->status, QUERY_OVER)) {
     ret = false;
@@ -6407,7 +6418,7 @@ bool qHasMoreResultsToRetrieve(qinfo_t qinfo) {
   return ret;
 }
 
-int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *contLen) {
+int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *contLen, bool* continueExec) {
   SQInfo *pQInfo = (SQInfo *)qinfo;
 
   if (pQInfo == NULL || !isValidQInfo(pQInfo)) {
@@ -6417,8 +6428,10 @@ int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *co
   SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv;
   SQuery *pQuery = pQInfo->runtimeEnv.pQuery;
   size_t  size = getResultSize(pQInfo, &pQuery->rec.rows);
+
   size += sizeof(int32_t);
   size += sizeof(STableIdInfo) * taosArrayGetSize(pQInfo->arrTableIdInfo);
+
   *contLen = size + sizeof(SRetrieveTableRsp);
 
   // todo proper handle failed to allocate memory,
@@ -6427,6 +6440,7 @@ int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *co
   if (*pRsp == NULL) {
     return TSDB_CODE_QRY_OUT_OF_MEMORY;
   }
+
   (*pRsp)->numOfRows = htonl(pQuery->rec.rows);
 
   int32_t code = pQInfo->code;
@@ -6434,8 +6448,8 @@ int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *co
     (*pRsp)->offset = htobe64(pQuery->limit.offset);
     (*pRsp)->useconds = htobe64(pRuntimeEnv->summary.elapsedTime);
   } else {
-    (*pRsp)->offset = 0;
     (*pRsp)->useconds = 0;
+    (*pRsp)->offset = 0;
   }
   
   (*pRsp)->precision = htons(pQuery->precision);
@@ -6446,10 +6460,20 @@ int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *co
     code = pQInfo->code;
   }
 
+  pQInfo->rspContext = NULL;
+  pQInfo->dataReady = QUERY_RESULT_NOT_READY;
+
   if (IS_QUERY_KILLED(pQInfo) || Q_STATUS_EQUAL(pQuery->status, QUERY_OVER)) {
     (*pRsp)->completed = 1;  // notify no more result to client
   }
 
+  if (qHasMoreResultsToRetrieve(pQInfo)) {
+    *continueExec = true;
+  } else { // failed to dump result, free qhandle immediately
+    *continueExec = false;
+    qKillQuery(pQInfo);
+  }
+
   return code;
 }
 
@@ -6460,7 +6484,7 @@ int32_t qKillQuery(qinfo_t qinfo) {
     return TSDB_CODE_QRY_INVALID_QHANDLE;
   }
 
-  sem_post(&pQInfo->dataReady);
+//  sem_post(&pQInfo->dataReady);
   setQueryKilled(pQInfo);
   return TSDB_CODE_SUCCESS;
 }
@@ -6607,6 +6631,13 @@ static void buildTagQueryResult(SQInfo* pQInfo) {
   setQueryStatus(pQuery, QUERY_COMPLETED);
 }
 
+void* qGetResultRetrieveMsg(qinfo_t qinfo) {
+  SQInfo* pQInfo = (SQInfo*) qinfo;
+  assert(pQInfo != NULL);
+
+  return pQInfo->rspContext;
+}
+
 void freeqinfoFn(void *qhandle) {
   void** handle = qhandle;
   if (handle == NULL || *handle == NULL) {
@@ -6618,19 +6649,21 @@ void freeqinfoFn(void *qhandle) {
 }
 
 void* qOpenQueryMgmt(int32_t vgId) {
-  const int32_t REFRESH_HANDLE_INTERVAL = 2; // every 2 seconds, refresh handle pool
+  const int32_t REFRESH_HANDLE_INTERVAL = 30; // every 30 seconds, refresh handle pool
 
   char cacheName[128] = {0};
   sprintf(cacheName, "qhandle_%d", vgId);
 
-  SQueryMgmt* pQueryHandle = calloc(1, sizeof(SQueryMgmt));
+  SQueryMgmt* pQueryMgmt = calloc(1, sizeof(SQueryMgmt));
+
+  pQueryMgmt->qinfoPool = taosCacheInit(TSDB_DATA_TYPE_BIGINT, REFRESH_HANDLE_INTERVAL, true, freeqinfoFn, cacheName);
+  pQueryMgmt->closed    = false;
+  pQueryMgmt->vgId      = vgId;
 
-  pQueryHandle->qinfoPool = taosCacheInit(TSDB_DATA_TYPE_BIGINT, REFRESH_HANDLE_INTERVAL, true, freeqinfoFn, cacheName);
-  pQueryHandle->closed    = false;
-  pthread_mutex_init(&pQueryHandle->lock, NULL);
+  pthread_mutex_init(&pQueryMgmt->lock, NULL);
 
   qDebug("vgId:%d, open querymgmt success", vgId);
-  return pQueryHandle;
+  return pQueryMgmt;
 }
 
 static void queryMgmtKillQueryFn(void* handle) {
@@ -6670,7 +6703,7 @@ void qCleanupQueryMgmt(void* pQMgmt) {
   pthread_mutex_destroy(&pQueryMgmt->lock);
   tfree(pQueryMgmt);
 
-  qDebug("vgId:%d querymgmt cleanup completed", vgId);
+  qDebug("vgId:%d queryMgmt cleanup completed", vgId);
 }
 
 void** qRegisterQInfo(void* pMgmt, uint64_t qInfo) {
@@ -6716,14 +6749,15 @@ void** qAcquireQInfo(void* pMgmt, uint64_t key) {
   }
 }
 
-void** qReleaseQInfo(void* pMgmt, void* pQInfo, bool needFree) {
+void** qReleaseQInfo(void* pMgmt, void* pQInfo, bool freeHandle) {
   SQueryMgmt *pQueryMgmt = pMgmt;
 
   if (pQueryMgmt->qinfoPool == NULL) {
     return NULL;
   }
 
-  taosCacheRelease(pQueryMgmt->qinfoPool, pQInfo, needFree);
+  taosCacheRelease(pQueryMgmt->qinfoPool, pQInfo, freeHandle);
   return 0;
 }
 
+

src/query/src/qExtbuffer.c

@@ -28,10 +28,10 @@
 /*
  * SColumnModel is deeply copy
  */
-tExtMemBuffer* createExtMemBuffer(int32_t inMemSize, int32_t elemSize, SColumnModel *pModel) {
+tExtMemBuffer* createExtMemBuffer(int32_t inMemSize, int32_t elemSize, int32_t pagesize, SColumnModel *pModel) {
   tExtMemBuffer* pMemBuffer = (tExtMemBuffer *)calloc(1, sizeof(tExtMemBuffer));
 
-  pMemBuffer->pageSize = DEFAULT_PAGE_SIZE;
+  pMemBuffer->pageSize = pagesize;
   pMemBuffer->inMemCapacity = ALIGN8(inMemSize) / pMemBuffer->pageSize;
   pMemBuffer->nElemSize = elemSize;
 

src/query/src/qParserImpl.c

@@ -14,7 +14,7 @@
  */
 
 #include "os.h"
-#include "qsqlparser.h"
+#include "qSqlparser.h"
 #include "queryLog.h"
 #include "taosdef.h"
 #include "taosmsg.h"

src/query/src/qPercentile.c

@@ -535,7 +535,7 @@ void tMemBucketPut(tMemBucket *pBucket, void *data, int32_t numOfRows) {
 
     if (pSeg->pBuffer[slotIdx] == NULL) {
       pSeg->pBuffer[slotIdx] = createExtMemBuffer(pBucket->numOfTotalPages * pBucket->pageSize, pBucket->nElemSize,
-                                                  pBucket->pOrderDesc->pColumnModel);
+                                                  pBucket->pageSize, pBucket->pOrderDesc->pColumnModel);
       pSeg->pBuffer[slotIdx]->flushModel = SINGLE_APPEND_MODEL;
       pBucket->pOrderDesc->pColumnModel->capacity = pSeg->pBuffer[slotIdx]->numOfElemsPerPage;
     }

src/query/src/qResultbuf.c

 #include "qResultbuf.h"
+#include "stddef.h"
+#include "tscompression.h"
 #include "hash.h"
 #include "qExtbuffer.h"
 #include "queryLog.h"
 #include "taoserror.h"
 
-int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t numOfPages, int32_t rowSize,
-    int32_t pagesize, int32_t inMemPages, void* handle) {
+#define GET_DATA_PAYLOAD(_p) ((_p)->pData + POINTER_BYTES)
 
+int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t rowSize, int32_t pagesize,
+                                    int32_t inMemBufSize, const void* handle) {
   *pResultBuf = calloc(1, sizeof(SDiskbasedResultBuf));
+
   SDiskbasedResultBuf* pResBuf = *pResultBuf;
   if (pResBuf == NULL) {
     return TSDB_CODE_COM_OUT_OF_MEMORY;  
   }
 
-  pResBuf->pageSize   = pagesize;
-  pResBuf->numOfPages = inMemPages;  // all pages are in buffer in the first place
-  pResBuf->inMemPages = inMemPages;
-  assert(inMemPages <= numOfPages);
-
-  pResBuf->numOfRowsPerPage = (pagesize - sizeof(tFilePage)) / rowSize;
+  pResBuf->pageSize     = pagesize;
+  pResBuf->numOfPages   = 0;                        // all pages are in buffer in the first place
+  pResBuf->totalBufSize = 0;
+  pResBuf->inMemPages   = inMemBufSize/pagesize;    // maximum allowed pages, it is a soft limit.
+  pResBuf->allocateId   = -1;
+  pResBuf->comp         = true;
+  pResBuf->file         = NULL;
+  pResBuf->handle       = handle;
+  pResBuf->fileSize = 0;
 
-  pResBuf->totalBufSize = pResBuf->numOfPages * pagesize;
-  pResBuf->incStep = 4;
-  pResBuf->allocateId = -1;
+  // at least more than 2 pages must be in memory
+  assert(inMemBufSize >= pagesize * 2);
 
-  pResBuf->iBuf = calloc(pResBuf->inMemPages, pResBuf->pageSize);
+  pResBuf->numOfRowsPerPage = (pagesize - sizeof(tFilePage)) / rowSize;
+  pResBuf->lruList = tdListNew(POINTER_BYTES);
 
   // init id hash table
-  pResBuf->idsTable = taosHashInit(numOfPages, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false);
-  pResBuf->list = taosArrayInit(numOfPages, POINTER_BYTES);
+  pResBuf->groupSet  = taosHashInit(10, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false);
+  pResBuf->assistBuf = malloc(pResBuf->pageSize + 2); // EXTRA BYTES
+  pResBuf->all = taosHashInit(10, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false);
 
   char path[PATH_MAX] = {0};
-  getTmpfilePath("tsdb_qbuf", path);
+  getTmpfilePath("qbuf", path);
   pResBuf->path = strdup(path);
 
-  pResBuf->fd = FD_INITIALIZER;
-  pResBuf->pBuf = NULL;
   pResBuf->emptyDummyIdList = taosArrayInit(1, sizeof(int32_t));
 
-  qDebug("QInfo:%p create resBuf for output, page size:%d, initial pages:%d, %" PRId64 "bytes", handle,
-      pResBuf->pageSize, pResBuf->numOfPages, pResBuf->totalBufSize);
-  
+  qDebug("QInfo:%p create resBuf for output, page size:%d, inmem buf pages:%d, file:%s", handle, pResBuf->pageSize,
+         pResBuf->inMemPages, pResBuf->path);
+
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t getNumOfResultBufGroupId(SDiskbasedResultBuf* pResultBuf) { return taosHashGetSize(pResultBuf->idsTable); }
-
-int32_t getResBufSize(SDiskbasedResultBuf* pResultBuf) { return pResultBuf->totalBufSize; }
-
-#define NUM_OF_PAGES_ON_DISK(_r) ((_r)->numOfPages - (_r)->inMemPages)
-#define FILE_SIZE_ON_DISK(_r) (NUM_OF_PAGES_ON_DISK(_r) * (_r)->pageSize)
-
-static int32_t createDiskResidesBuf(SDiskbasedResultBuf* pResultBuf) {
-  pResultBuf->fd = open(pResultBuf->path, O_CREAT | O_RDWR | O_TRUNC, 0666);
-  if (!FD_VALID(pResultBuf->fd)) {
+static int32_t createDiskFile(SDiskbasedResultBuf* pResultBuf) {
+  pResultBuf->file = fopen(pResultBuf->path, "wb+");
+  if (pResultBuf->file == NULL) {
     qError("failed to create tmp file: %s on disk. %s", pResultBuf->path, strerror(errno));
     return TAOS_SYSTEM_ERROR(errno);
   }
 
-  assert(pResultBuf->numOfPages == pResultBuf->inMemPages);
-  pResultBuf->numOfPages += pResultBuf->incStep;
+  return TSDB_CODE_SUCCESS;
+}
 
-  int32_t ret = ftruncate(pResultBuf->fd, NUM_OF_PAGES_ON_DISK(pResultBuf) * pResultBuf->pageSize);
-  if (ret != TSDB_CODE_SUCCESS) {
-    qError("failed to create tmp file: %s on disk. %s", pResultBuf->path, strerror(errno));
-    return TAOS_SYSTEM_ERROR(errno);
+static char* doCompressData(void* data, int32_t srcSize, int32_t *dst, SDiskbasedResultBuf* pResultBuf) { // do nothing
+  if (!pResultBuf->comp) {
+    *dst = srcSize;
+    return data;
   }
 
-  pResultBuf->pBuf = mmap(NULL, FILE_SIZE_ON_DISK(pResultBuf), PROT_READ | PROT_WRITE, MAP_SHARED, pResultBuf->fd, 0);
-  if (pResultBuf->pBuf == MAP_FAILED) {
-    qError("QInfo:%p failed to map temp file: %s. %s", pResultBuf->handle, pResultBuf->path, strerror(errno));
-    return TAOS_SYSTEM_ERROR(errno);
+  *dst = tsCompressString(data, srcSize, 1, pResultBuf->assistBuf, srcSize, ONE_STAGE_COMP, NULL, 0);
+
+  memcpy(data, pResultBuf->assistBuf, *dst);
+  return data;
+}
+
+static char* doDecompressData(void* data, int32_t srcSize, int32_t *dst, SDiskbasedResultBuf* pResultBuf) { // do nothing
+  if (!pResultBuf->comp) {
+    *dst = srcSize;
+    return data;
   }
 
-  pResultBuf->totalBufSize = pResultBuf->numOfPages * pResultBuf->pageSize;
-  return TSDB_CODE_SUCCESS;
+  *dst = tsDecompressString(data, srcSize, 1, pResultBuf->assistBuf, pResultBuf->pageSize, ONE_STAGE_COMP, NULL, 0);
+
+  memcpy(data, pResultBuf->assistBuf, *dst);
+  return data;
 }
 
-static int32_t extendDiskFileSize(SDiskbasedResultBuf* pResultBuf, int32_t incNumOfPages) {
-  assert(pResultBuf->numOfPages * pResultBuf->pageSize == pResultBuf->totalBufSize);
-  int32_t ret = TSDB_CODE_SUCCESS;
+static int32_t allocatePositionInFile(SDiskbasedResultBuf* pResultBuf, size_t size) {
+  if (pResultBuf->pFree == NULL) {
+    return pResultBuf->nextPos;
+  } else {
+    int32_t offset = -1;
+
+    size_t num = taosArrayGetSize(pResultBuf->pFree);
+    for(int32_t i = 0; i < num; ++i) {
+      SFreeListItem* pi = taosArrayGet(pResultBuf->pFree, i);
+      if (pi->len >= size) {
+        offset = pi->offset;
+        pi->offset += size;
+        pi->len -= size;
+
+        return offset;
+      }
+    }
+
+    // no available recycle space, allocate new area in file
+    return pResultBuf->nextPos;
+  }
+}
+
+static char* doFlushPageToDisk(SDiskbasedResultBuf* pResultBuf, SPageInfo* pg) {
+  assert(!pg->used && pg->pData != NULL);
+
+  int32_t size = -1;
+  char* t = doCompressData(GET_DATA_PAYLOAD(pg), pResultBuf->pageSize, &size, pResultBuf);
+
+  // this page is flushed to disk for the first time
+  if (pg->info.offset == -1) {
+    pg->info.offset = allocatePositionInFile(pResultBuf, size);
+    pResultBuf->nextPos += size;
 
-  if (pResultBuf->pBuf == NULL) {
-    assert(pResultBuf->fd == FD_INITIALIZER);
+    fseek(pResultBuf->file, pg->info.offset, SEEK_SET);
+    /*int32_t ret =*/ fwrite(t, 1, size, pResultBuf->file);
 
-    if ((ret = createDiskResidesBuf(pResultBuf)) != TSDB_CODE_SUCCESS) {
-      return ret;
+    if (pResultBuf->fileSize < pg->info.offset + pg->info.length) {
+      pResultBuf->fileSize = pg->info.offset + pg->info.length;
     }
   } else {
-    ret = munmap(pResultBuf->pBuf, FILE_SIZE_ON_DISK(pResultBuf));
-    pResultBuf->numOfPages += incNumOfPages;
-
-    /*
-     * disk-based output buffer is exhausted, try to extend the disk-based buffer, the available disk space may
-     * be insufficient
-     */
-    ret = ftruncate(pResultBuf->fd, NUM_OF_PAGES_ON_DISK(pResultBuf) * pResultBuf->pageSize);
-    if (ret != TSDB_CODE_SUCCESS) {
-      //    dError("QInfo:%p failed to create intermediate result output file:%s. %s", pQInfo, pSupporter->extBufFile,
-      //           strerror(errno));
-      return TSDB_CODE_QRY_NO_DISKSPACE;
+    // length becomes greater, current space is not enough, allocate new place, otherwise, do nothing
+    if (pg->info.length < size) {
+      // 1. add current space to free list
+      taosArrayPush(pResultBuf->pFree, &pg->info);
+
+      // 2. allocate new position, and update the info
+      pg->info.offset = allocatePositionInFile(pResultBuf, size);
+      pResultBuf->nextPos += size;
     }
 
-    pResultBuf->totalBufSize = pResultBuf->numOfPages * pResultBuf->pageSize;
-    pResultBuf->pBuf = mmap(NULL, FILE_SIZE_ON_DISK(pResultBuf), PROT_READ | PROT_WRITE, MAP_SHARED, pResultBuf->fd, 0);
+    //3. write to disk.
+    fseek(pResultBuf->file, pg->info.offset, SEEK_SET);
+    fwrite(t, size, 1, pResultBuf->file);
 
-    if (pResultBuf->pBuf == MAP_FAILED) {
-      //    dError("QInfo:%p failed to map temp file: %s. %s", pQInfo, pSupporter->extBufFile, strerror(errno));
-      return TSDB_CODE_QRY_OUT_OF_MEMORY;
+    if (pResultBuf->fileSize < pg->info.offset + pg->info.length) {
+      pResultBuf->fileSize = pg->info.offset + pg->info.length;
     }
   }
 
-  return TSDB_CODE_SUCCESS;
+  char* ret = pg->pData;
+  memset(ret, 0, pResultBuf->pageSize);
+
+  pg->pData = NULL;
+  pg->info.length = size;
+
+  pResultBuf->statis.flushBytes += pg->info.length;
+
+  return ret;
 }
 
-#define NO_AVAILABLE_PAGES(_b) ((_b)->allocateId == (_b)->numOfPages - 1)
+static char* flushPageToDisk(SDiskbasedResultBuf* pResultBuf, SPageInfo* pg) {
+  int32_t ret = TSDB_CODE_SUCCESS;
+  assert(pResultBuf->numOfPages * pResultBuf->pageSize == pResultBuf->totalBufSize && pResultBuf->numOfPages >= pResultBuf->inMemPages);
 
-static FORCE_INLINE int32_t getGroupIndex(SDiskbasedResultBuf* pResultBuf, int32_t groupId) {
-  assert(pResultBuf != NULL);
+  if (pResultBuf->file == NULL) {
+    if ((ret = createDiskFile(pResultBuf)) != TSDB_CODE_SUCCESS) {
+      terrno = ret;
+      return NULL;
+    }
+  }
+
+  return doFlushPageToDisk(pResultBuf, pg);
+}
+
+// load file block data in disk
+static char* loadPageFromDisk(SDiskbasedResultBuf* pResultBuf, SPageInfo* pg) {
+  int32_t ret = fseek(pResultBuf->file, pg->info.offset, SEEK_SET);
+  ret = fread(GET_DATA_PAYLOAD(pg), 1, pg->info.length, pResultBuf->file);
+  if (ret != pg->info.length) {
+    terrno = errno;
+    return NULL;
+  }
+
+  pResultBuf->statis.loadBytes += pg->info.length;
 
-  char* p = taosHashGet(pResultBuf->idsTable, (const char*)&groupId, sizeof(int32_t));
+  int32_t fullSize = 0;
+  doDecompressData(GET_DATA_PAYLOAD(pg), pg->info.length, &fullSize, pResultBuf);
+
+  return GET_DATA_PAYLOAD(pg);
+}
+
+#define NO_AVAILABLE_PAGES(_b) ((_b)->numOfPages >= (_b)->inMemPages)
+
+static SIDList addNewGroup(SDiskbasedResultBuf* pResultBuf, int32_t groupId) {
+  assert(taosHashGet(pResultBuf->groupSet, (const char*) &groupId, sizeof(int32_t)) == NULL);
+
+  SArray* pa = taosArrayInit(1, POINTER_BYTES);
+  int32_t ret = taosHashPut(pResultBuf->groupSet, (const char*)&groupId, sizeof(int32_t), &pa, POINTER_BYTES);
+  assert(ret == 0);
+
+  return pa;
+}
+
+static SPageInfo* registerPage(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int32_t pageId) {
+  SIDList list = NULL;
+
+  char** p = taosHashGet(pResultBuf->groupSet, (const char*)&groupId, sizeof(int32_t));
   if (p == NULL) {  // it is a new group id
-    return -1;
+    list = addNewGroup(pResultBuf, groupId);
+  } else {
+    list = (SIDList) (*p);
   }
 
-  int32_t slot = GET_INT32_VAL(p);
-  assert(slot >= 0 && slot < taosHashGetSize(pResultBuf->idsTable));
+  pResultBuf->numOfPages += 1;
 
-  return slot;
+  SPageInfo* ppi = malloc(sizeof(SPageInfo));//{ .info = PAGE_INFO_INITIALIZER, .pageId = pageId, .pn = NULL};
+  ppi->info     = PAGE_INFO_INITIALIZER;
+  ppi->pageId   = pageId;
+  ppi->pData    = NULL;
+  ppi->pn       = NULL;
+  ppi->used     = true;
+
+  return *(SPageInfo**) taosArrayPush(list, &ppi);
 }
 
-static int32_t addNewGroupId(SDiskbasedResultBuf* pResultBuf, int32_t groupId) {
-  int32_t num = getNumOfResultBufGroupId(pResultBuf); // the num is the newest allocated group id slot
-  taosHashPut(pResultBuf->idsTable, (const char*)&groupId, sizeof(int32_t), &num, sizeof(int32_t));
+static SListNode* getEldestUnrefedPage(SDiskbasedResultBuf* pResultBuf) {
+  SListIter iter = {0};
+  tdListInitIter(pResultBuf->lruList, &iter, TD_LIST_BACKWARD);
+
+  SListNode* pn = NULL;
+  while((pn = tdListNext(&iter)) != NULL) {
+    assert(pn != NULL);
+
+    SPageInfo* pageInfo = *(SPageInfo**) pn->data;
+    assert(pageInfo->pageId >= 0 && pageInfo->pn == pn);
 
-  SArray* pa = taosArrayInit(1, sizeof(int32_t));
-  taosArrayPush(pResultBuf->list, &pa);
+    if (!pageInfo->used) {
+      break;
+    }
+  }
 
-  assert(taosArrayGetSize(pResultBuf->list) == taosHashGetSize(pResultBuf->idsTable));
-  return num;
+  return pn;
 }
 
-static void registerPageId(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int32_t pageId) {
-  int32_t slot = getGroupIndex(pResultBuf, groupId);
-  if (slot < 0) {
-    slot = addNewGroupId(pResultBuf, groupId);
+static char* evicOneDataPage(SDiskbasedResultBuf* pResultBuf) {
+  char* bufPage = NULL;
+  SListNode* pn = getEldestUnrefedPage(pResultBuf);
+
+  // all pages are referenced by user, try to allocate new space
+  if (pn == NULL) {
+    int32_t prev = pResultBuf->inMemPages;
+    pResultBuf->inMemPages = pResultBuf->inMemPages * 1.5;
+
+    qWarn("%p in memory buf page not sufficient, expand from %d to %d, page size:%d", pResultBuf, prev,
+          pResultBuf->inMemPages, pResultBuf->pageSize);
+  } else {
+    pResultBuf->statis.flushPages += 1;
+    tdListPopNode(pResultBuf->lruList, pn);
+
+    SPageInfo* d = *(SPageInfo**) pn->data;
+    assert(d->pn == pn);
+
+    d->pn = NULL;
+    tfree(pn);
+
+    bufPage = flushPageToDisk(pResultBuf, d);
   }
 
-  SIDList pList = taosArrayGetP(pResultBuf->list, slot);
-  taosArrayPush(pList, &pageId);
+  return bufPage;
+}
+
+static void lruListPushFront(SList *pList, SPageInfo* pi) {
+  tdListPrepend(pList, &pi);
+  SListNode* front = tdListGetHead(pList);
+  pi->pn = front;
+}
+
+static void lruListMoveToFront(SList *pList, SPageInfo* pi) {
+  tdListPopNode(pList, pi->pn);
+  tdListPrependNode(pList, pi->pn);
 }
 
 tFilePage* getNewDataBuf(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int32_t* pageId) {
+  pResultBuf->statis.getPages += 1;
+
+  char* availablePage = NULL;
   if (NO_AVAILABLE_PAGES(pResultBuf)) {
-    if (extendDiskFileSize(pResultBuf, pResultBuf->incStep) != TSDB_CODE_SUCCESS) {
-      return NULL;
-    }
+    availablePage = evicOneDataPage(pResultBuf);
   }
 
   // register new id in this group
   *pageId = (++pResultBuf->allocateId);
-  registerPageId(pResultBuf, groupId, *pageId);
 
-  // clear memory for the new page
-  tFilePage* page = getResBufPage(pResultBuf, *pageId);
-  memset(page, 0, pResultBuf->pageSize);
-  
-  return page;
+  // register page id info
+  SPageInfo* pi = registerPage(pResultBuf, groupId, *pageId);
+
+  // add to LRU list
+  assert(listNEles(pResultBuf->lruList) < pResultBuf->inMemPages && pResultBuf->inMemPages > 0);
+
+  lruListPushFront(pResultBuf->lruList, pi);
+
+  // add to hash map
+  taosHashPut(pResultBuf->all, pageId, sizeof(int32_t), &pi, POINTER_BYTES);
+
+  // allocate buf
+  if (availablePage == NULL) {
+    pi->pData = calloc(1, pResultBuf->pageSize + POINTER_BYTES);
+  } else {
+    pi->pData = availablePage;
+  }
+
+  pResultBuf->totalBufSize += pResultBuf->pageSize;
+
+  ((void**)pi->pData)[0] = pi;
+  pi->used = true;
+
+  return GET_DATA_PAYLOAD(pi);
+}
+
+tFilePage* getResBufPage(SDiskbasedResultBuf* pResultBuf, int32_t id) {
+  assert(pResultBuf != NULL && id >= 0);
+  pResultBuf->statis.getPages += 1;
+
+  SPageInfo** pi = taosHashGet(pResultBuf->all, &id, sizeof(int32_t));
+  assert(pi != NULL && *pi != NULL);
+
+  if ((*pi)->pData != NULL) { // it is in memory
+    // no need to update the LRU list if only one page exists
+    if (pResultBuf->numOfPages == 1) {
+      (*pi)->used = true;
+      return GET_DATA_PAYLOAD(*pi);
+    }
+
+    SPageInfo** pInfo = (SPageInfo**) ((*pi)->pn->data);
+    assert(*pInfo == *pi);
+
+    lruListMoveToFront(pResultBuf->lruList, (*pi));
+    (*pi)->used = true;
+
+    return GET_DATA_PAYLOAD(*pi);
+
+  } else { // not in memory
+    assert((*pi)->pData == NULL && (*pi)->pn == NULL && (*pi)->info.length >= 0 && (*pi)->info.offset >= 0);
+
+    char* availablePage = NULL;
+    if (NO_AVAILABLE_PAGES(pResultBuf)) {
+      availablePage = evicOneDataPage(pResultBuf);
+    }
+
+    if (availablePage == NULL) {
+      (*pi)->pData = calloc(1, pResultBuf->pageSize + POINTER_BYTES);
+    } else {
+      (*pi)->pData = availablePage;
+    }
+
+    ((void**)((*pi)->pData))[0] = (*pi);
+
+    lruListPushFront(pResultBuf->lruList, *pi);
+    loadPageFromDisk(pResultBuf, *pi);
+    return GET_DATA_PAYLOAD(*pi);
+  }
 }
 
-int32_t getNumOfRowsPerPage(SDiskbasedResultBuf* pResultBuf) { return pResultBuf->numOfRowsPerPage; }
+void releaseResBufPage(SDiskbasedResultBuf* pResultBuf, void* page) {
+  assert(pResultBuf != NULL && page != NULL);
+  char* p = (char*) page - POINTER_BYTES;
+
+  SPageInfo* ppi = ((SPageInfo**) p)[0];
+  releaseResBufPageInfo(pResultBuf, ppi);
+}
+
+void releaseResBufPageInfo(SDiskbasedResultBuf* pResultBuf, SPageInfo* pi) {
+  assert(pi->pData != NULL && pi->used);
+
+  pi->used = false;
+  pResultBuf->statis.releasePages += 1;
+}
+
+size_t getNumOfRowsPerPage(const SDiskbasedResultBuf* pResultBuf) { return pResultBuf->numOfRowsPerPage; }
+
+size_t getNumOfResultBufGroupId(const SDiskbasedResultBuf* pResultBuf) { return taosHashGetSize(pResultBuf->groupSet); }
+
+size_t getResBufSize(const SDiskbasedResultBuf* pResultBuf) { return pResultBuf->totalBufSize; }
 
 SIDList getDataBufPagesIdList(SDiskbasedResultBuf* pResultBuf, int32_t groupId) {
-  int32_t slot = getGroupIndex(pResultBuf, groupId);
-  if (slot < 0) {
+  assert(pResultBuf != NULL);
+
+  char** p = taosHashGet(pResultBuf->groupSet, (const char*)&groupId, sizeof(int32_t));
+  if (p == NULL) {  // it is a new group id
     return pResultBuf->emptyDummyIdList;
   } else {
-    return taosArrayGetP(pResultBuf->list, slot);
+    return (SArray*) (*p);
   }
 }
 
-void destroyResultBuf(SDiskbasedResultBuf* pResultBuf, void* handle) {
+void destroyResultBuf(SDiskbasedResultBuf* pResultBuf) {
   if (pResultBuf == NULL) {
     return;
   }
 
-  if (FD_VALID(pResultBuf->fd)) {
-    qDebug("QInfo:%p disk-based output buffer closed, total:%" PRId64 " bytes, file created:%s, file size:%d", handle,
-        pResultBuf->totalBufSize, pResultBuf->path, FILE_SIZE_ON_DISK(pResultBuf));
+  if (pResultBuf->file != NULL) {
+    qDebug("QInfo:%p disk-based output buffer closed, total:%" PRId64 " bytes, file size:%"PRId64" bytes",
+        pResultBuf->handle, pResultBuf->totalBufSize, pResultBuf->fileSize);
 
-    close(pResultBuf->fd);
-    munmap(pResultBuf->pBuf, FILE_SIZE_ON_DISK(pResultBuf));
-    pResultBuf->pBuf = NULL;
+    fclose(pResultBuf->file);
   } else {
-    qDebug("QInfo:%p disk-based output buffer closed, total:%" PRId64 " bytes, no file created", handle,
+    qDebug("QInfo:%p disk-based output buffer closed, total:%" PRId64 " bytes, no file created", pResultBuf->handle,
            pResultBuf->totalBufSize);
   }
 
   unlink(pResultBuf->path);
   tfree(pResultBuf->path);
 
-  size_t size = taosArrayGetSize(pResultBuf->list);
-  for (int32_t i = 0; i < size; ++i) {
-    SArray* pa = taosArrayGetP(pResultBuf->list, i);
-    taosArrayDestroy(pa);
+  SHashMutableIterator* iter = taosHashCreateIter(pResultBuf->groupSet);
+  while(taosHashIterNext(iter)) {
+    SArray** p = (SArray**) taosHashIterGet(iter);
+    size_t n = taosArrayGetSize(*p);
+    for(int32_t i = 0; i < n; ++i) {
+      SPageInfo* pi = taosArrayGetP(*p, i);
+      tfree(pi->pData);
+      tfree(pi);
+    }
+
+    taosArrayDestroy(*p);
   }
 
-  taosArrayDestroy(pResultBuf->list);
+  taosHashDestroyIter(iter);
+
+  tdListFree(pResultBuf->lruList);
   taosArrayDestroy(pResultBuf->emptyDummyIdList);
-  taosHashCleanup(pResultBuf->idsTable);
+  taosHashCleanup(pResultBuf->groupSet);
+  taosHashCleanup(pResultBuf->all);
 
-  tfree(pResultBuf->iBuf);
+  tfree(pResultBuf->assistBuf);
   tfree(pResultBuf);
 }
 
-int32_t getLastPageId(SIDList pList) {
+SPageInfo* getLastPageInfo(SIDList pList) {
   size_t size = taosArrayGetSize(pList);
-  return *(int32_t*) taosArrayGet(pList, size - 1);
+  return (SPageInfo*) taosArrayGetP(pList, size - 1);
 }
 

src/query/src/qUtil.c

@@ -236,11 +236,13 @@ void clearTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pWindow
   if (pWindowRes == NULL) {
     return;
   }
-  
+
+  tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes->pos.pageId);
+
   for (int32_t i = 0; i < pRuntimeEnv->pQuery->numOfOutput; ++i) {
     SResultInfo *pResultInfo = &pWindowRes->resultInfo[i];
     
-    char * s = getPosInResultPage(pRuntimeEnv, i, pWindowRes);
+    char * s = getPosInResultPage(pRuntimeEnv, i, pWindowRes, page);
     size_t size = pRuntimeEnv->pQuery->pSelectExpr[i].bytes;
     memset(s, 0, size);
     
@@ -277,8 +279,11 @@ void copyTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *dst, con
     memcpy(pDst->interResultBuf, pSrc->interResultBuf, pDst->bufLen);
     
     // copy the output buffer data from src to dst, the position info keep unchanged
-    char * dstBuf = getPosInResultPage(pRuntimeEnv, i, dst);
-    char * srcBuf = getPosInResultPage(pRuntimeEnv, i, (SWindowResult *)src);
+    tFilePage *dstpage = getResBufPage(pRuntimeEnv->pResultBuf, dst->pos.pageId);
+    char * dstBuf = getPosInResultPage(pRuntimeEnv, i, dst, dstpage);
+
+    tFilePage *srcpage = getResBufPage(pRuntimeEnv->pResultBuf, src->pos.pageId);
+    char * srcBuf = getPosInResultPage(pRuntimeEnv, i, (SWindowResult *)src, srcpage);
     size_t s = pRuntimeEnv->pQuery->pSelectExpr[i].bytes;
     
     memcpy(dstBuf, srcBuf, s);

src/query/src/sql.c

@@ -30,7 +30,7 @@
 #include <string.h>
 #include <assert.h>
 #include <stdbool.h>
-#include "qsqlparser.h"
+#include "qSqlparser.h"
 #include "tcmdtype.h"
 #include "tstoken.h"
 #include "ttokendef.h"

src/query/tests/resultBufferTest.cpp

@@ -10,7 +10,7 @@ namespace {
 // simple test
 void simpleTest() {
   SDiskbasedResultBuf* pResultBuf = NULL;
-  int32_t ret = createDiskbasedResultBuffer(&pResultBuf, 1000, 64, 1024, 4, NULL);
+  int32_t ret = createDiskbasedResultBuffer(&pResultBuf, 64, 1024, 4096, NULL);
   
   int32_t pageId = 0;
   int32_t groupId = 0;
@@ -18,17 +18,144 @@ void simpleTest() {
   tFilePage* pBufPage = getNewDataBuf(pResultBuf, groupId, &pageId);
   ASSERT_TRUE(pBufPage != NULL);
   
-  ASSERT_EQ(getNumOfRowsPerPage(pResultBuf), (16384L - sizeof(int64_t))/64);
-  ASSERT_EQ(getResBufSize(pResultBuf), 1000*16384L);
+  ASSERT_EQ(getResBufSize(pResultBuf), 1024);
   
   SIDList list = getDataBufPagesIdList(pResultBuf, groupId);
   ASSERT_EQ(taosArrayGetSize(list), 1);
   ASSERT_EQ(getNumOfResultBufGroupId(pResultBuf), 1);
-  
-  destroyResultBuf(pResultBuf, NULL);
+
+  releaseResBufPage(pResultBuf, pBufPage);
+
+  tFilePage* pBufPage1 = getNewDataBuf(pResultBuf, groupId, &pageId);
+
+  tFilePage* t = getResBufPage(pResultBuf, pageId);
+  ASSERT_TRUE(t == pBufPage1);
+
+  tFilePage* pBufPage2 = getNewDataBuf(pResultBuf, groupId, &pageId);
+  tFilePage* t1 = getResBufPage(pResultBuf, pageId);
+  ASSERT_TRUE(t1 == pBufPage2);
+
+  tFilePage* pBufPage3 = getNewDataBuf(pResultBuf, groupId, &pageId);
+  tFilePage* t2 = getResBufPage(pResultBuf, pageId);
+  ASSERT_TRUE(t2 == pBufPage3);
+
+  tFilePage* pBufPage4 = getNewDataBuf(pResultBuf, groupId, &pageId);
+  tFilePage* t3 = getResBufPage(pResultBuf, pageId);
+  ASSERT_TRUE(t3 == pBufPage4);
+
+  tFilePage* pBufPage5 = getNewDataBuf(pResultBuf, groupId, &pageId);
+  tFilePage* t4 = getResBufPage(pResultBuf, pageId);
+  ASSERT_TRUE(t4 == pBufPage5);
+
+  destroyResultBuf(pResultBuf);
+}
+
+void writeDownTest() {
+  SDiskbasedResultBuf* pResultBuf = NULL;
+  int32_t ret = createDiskbasedResultBuffer(&pResultBuf, 64, 1024, 4*1024, NULL);
+
+  int32_t pageId = 0;
+  int32_t writePageId = 0;
+  int32_t groupId = 0;
+  int32_t nx = 12345;
+
+  tFilePage* pBufPage = getNewDataBuf(pResultBuf, groupId, &pageId);
+  ASSERT_TRUE(pBufPage != NULL);
+
+  *(int32_t*)(pBufPage->data) = nx;
+  writePageId = pageId;
+  releaseResBufPage(pResultBuf, pBufPage);
+
+  tFilePage* pBufPage1 = getNewDataBuf(pResultBuf, groupId, &pageId);
+  tFilePage* t1 = getResBufPage(pResultBuf, pageId);
+  ASSERT_TRUE(t1 == pBufPage1);
+  ASSERT_TRUE(pageId == 1);
+
+  tFilePage* pBufPage2 = getNewDataBuf(pResultBuf, groupId, &pageId);
+  tFilePage* t2 = getResBufPage(pResultBuf, pageId);
+  ASSERT_TRUE(t2 == pBufPage2);
+  ASSERT_TRUE(pageId == 2);
+
+  tFilePage* pBufPage3 = getNewDataBuf(pResultBuf, groupId, &pageId);
+  tFilePage* t3 = getResBufPage(pResultBuf, pageId);
+  ASSERT_TRUE(t3 == pBufPage3);
+  ASSERT_TRUE(pageId == 3);
+
+  tFilePage* pBufPage4 = getNewDataBuf(pResultBuf, groupId, &pageId);
+  tFilePage* t4 = getResBufPage(pResultBuf, pageId);
+  ASSERT_TRUE(t4 == pBufPage4);
+  ASSERT_TRUE(pageId == 4);
+  releaseResBufPage(pResultBuf, t4);
+
+  // flush the written page to disk, and read it out again
+  tFilePage* pBufPagex = getResBufPage(pResultBuf, writePageId);
+  ASSERT_EQ(*(int32_t*)pBufPagex->data, nx);
+
+  SArray* pa = getDataBufPagesIdList(pResultBuf, groupId);
+  ASSERT_EQ(taosArrayGetSize(pa), 5);
+
+  destroyResultBuf(pResultBuf);
+}
+
+void recyclePageTest() {
+  SDiskbasedResultBuf* pResultBuf = NULL;
+  int32_t ret = createDiskbasedResultBuffer(&pResultBuf, 64, 1024, 4*1024, NULL);
+
+  int32_t pageId = 0;
+  int32_t writePageId = 0;
+  int32_t groupId = 0;
+  int32_t nx = 12345;
+
+  tFilePage* pBufPage = getNewDataBuf(pResultBuf, groupId, &pageId);
+  ASSERT_TRUE(pBufPage != NULL);
+  releaseResBufPage(pResultBuf, pBufPage);
+
+  tFilePage* pBufPage1 = getNewDataBuf(pResultBuf, groupId, &pageId);
+  tFilePage* t1 = getResBufPage(pResultBuf, pageId);
+  ASSERT_TRUE(t1 == pBufPage1);
+  ASSERT_TRUE(pageId == 1);
+
+  tFilePage* pBufPage2 = getNewDataBuf(pResultBuf, groupId, &pageId);
+  tFilePage* t2 = getResBufPage(pResultBuf, pageId);
+  ASSERT_TRUE(t2 == pBufPage2);
+  ASSERT_TRUE(pageId == 2);
+
+  tFilePage* pBufPage3 = getNewDataBuf(pResultBuf, groupId, &pageId);
+  tFilePage* t3 = getResBufPage(pResultBuf, pageId);
+  ASSERT_TRUE(t3 == pBufPage3);
+  ASSERT_TRUE(pageId == 3);
+
+  tFilePage* pBufPage4 = getNewDataBuf(pResultBuf, groupId, &pageId);
+  tFilePage* t4 = getResBufPage(pResultBuf, pageId);
+  ASSERT_TRUE(t4 == pBufPage4);
+  ASSERT_TRUE(pageId == 4);
+  releaseResBufPage(pResultBuf, t4);
+  releaseResBufPage(pResultBuf, t4);
+
+  tFilePage* pBufPage5 = getNewDataBuf(pResultBuf, groupId, &pageId);
+  tFilePage* t5 = getResBufPage(pResultBuf, pageId);
+  ASSERT_TRUE(t5 == pBufPage5);
+  ASSERT_TRUE(pageId == 5);
+
+  // flush the written page to disk, and read it out again
+  tFilePage* pBufPagex = getResBufPage(pResultBuf, writePageId);
+  *(int32_t*)(pBufPagex->data) = nx;
+  writePageId = pageId;   // update the data
+  releaseResBufPage(pResultBuf, pBufPagex);
+
+  tFilePage* pBufPagex1 = getResBufPage(pResultBuf, 1);
+
+  SArray* pa = getDataBufPagesIdList(pResultBuf, groupId);
+  ASSERT_EQ(taosArrayGetSize(pa), 6);
+
+  destroyResultBuf(pResultBuf);
 }
 } // namespace
 
+
 TEST(testCase, resultBufferTest) {
+  srand(time(NULL));
   simpleTest();
+  writeDownTest();
+  recyclePageTest();
 }

src/tsdb/src/tsdbRead.c

@@ -30,11 +30,6 @@
 #define ASCENDING_TRAVERSE(o)   (o == TSDB_ORDER_ASC)
 #define QH_GET_NUM_OF_COLS(handle) ((size_t)(taosArrayGetSize((handle)->pColumns)))
 
-enum {
-  QUERY_RANGE_LESS_EQUAL = 0,
-  QUERY_RANGE_GREATER_EQUAL = 1,
-};
-
 enum {
   TSDB_QUERY_TYPE_ALL      = 1,
   TSDB_QUERY_TYPE_LAST     = 2,
@@ -210,7 +205,7 @@ TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STab
   if (pQueryHandle->pColumns == NULL) {
     goto out_of_memory;
   }
-  
+
   for (int32_t i = 0; i < numOfCols; ++i) {
     SColumnInfoData  colInfo = {{0}, 0};
   
@@ -222,29 +217,29 @@ TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STab
     taosArrayPush(pQueryHandle->pColumns, &colInfo);
     pQueryHandle->statis[i].colId = colInfo.info.colId;
   }
-  
+
   pQueryHandle->pTableCheckInfo = taosArrayInit(groupList->numOfTables, sizeof(STableCheckInfo));
   if (pQueryHandle->pTableCheckInfo == NULL) {
     goto out_of_memory;
   }
   STsdbMeta* pMeta = tsdbGetMeta(tsdb);
   assert(pMeta != NULL);
-  
+
   for (int32_t i = 0; i < sizeOfGroup; ++i) {
     SArray* group = *(SArray**) taosArrayGet(groupList->pGroupList, i);
-    
+
     size_t gsize = taosArrayGetSize(group);
     assert(gsize > 0);
-    
+
     for (int32_t j = 0; j < gsize; ++j) {
       STable* pTable = (STable*) taosArrayGetP(group, j);
-      
+
       STableCheckInfo info = {
           .lastKey = pQueryHandle->window.skey,
           .tableId = pTable->tableId,
           .pTableObj = pTable,
       };
-      
+
       assert(info.pTableObj != NULL && (info.pTableObj->type == TSDB_NORMAL_TABLE ||
       info.pTableObj->type == TSDB_CHILD_TABLE || info.pTableObj->type == TSDB_STREAM_TABLE));
 
@@ -280,17 +275,17 @@ TsdbQueryHandleT tsdbQueryLastRow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STab
 
 SArray* tsdbGetQueriedTableList(TsdbQueryHandleT *pHandle) {
   assert(pHandle != NULL);
-  
+
   STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) pHandle;
-  
+
   size_t size = taosArrayGetSize(pQueryHandle->pTableCheckInfo);
   SArray* res = taosArrayInit(size, POINTER_BYTES);
-  
+
   for(int32_t i = 0; i < size; ++i) {
     STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i);
     taosArrayPush(res, &pCheckInfo->pTableObj);
   }
-  
+
   return res;
 }
 
@@ -306,11 +301,11 @@ TsdbQueryHandleT tsdbQueryRowsInExternalWindow(TSDB_REPO_T *tsdb, STsdbQueryCond
 static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCheckInfo) {
   STable* pTable = pCheckInfo->pTableObj;
   assert(pTable != NULL);
-  
+
   if (pCheckInfo->initBuf) {
     return true;
   }
-  
+
   pCheckInfo->initBuf = true;
   int32_t order = pHandle->order;
 
@@ -318,7 +313,7 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh
   if (pHandle->mem == NULL && pHandle->imem == NULL) {
     return false;
   }
-  
+
   assert(pCheckInfo->iter == NULL && pCheckInfo->iiter == NULL);
 
   // TODO: add uid check
@@ -338,17 +333,17 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh
   if (pCheckInfo->iter == NULL && pCheckInfo->iiter == NULL) {
     return false;
   }
-  
+
   bool memEmpty  = (pCheckInfo->iter == NULL) || (pCheckInfo->iter != NULL && !tSkipListIterNext(pCheckInfo->iter));
   bool imemEmpty = (pCheckInfo->iiter == NULL) || (pCheckInfo->iiter != NULL && !tSkipListIterNext(pCheckInfo->iiter));
   if (memEmpty && imemEmpty) { // buffer is empty
     return false;
   }
-  
+
   if (!memEmpty) {
     SSkipListNode* node = tSkipListIterGet(pCheckInfo->iter);
     assert(node != NULL);
-  
+
     SDataRow row = SL_GET_NODE_DATA(node);
     TSKEY key = dataRowKey(row);  // first timestamp in buffer
     tsdbDebug("%p uid:%" PRId64", tid:%d check data in mem from skey:%" PRId64 ", order:%d, %p", pHandle,
@@ -357,11 +352,11 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh
     tsdbDebug("%p uid:%"PRId64", tid:%d no data in mem, %p", pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid,
         pHandle->qinfo);
   }
-  
+
   if (!imemEmpty) {
     SSkipListNode* node = tSkipListIterGet(pCheckInfo->iiter);
     assert(node != NULL);
-  
+
     SDataRow row = SL_GET_NODE_DATA(node);
     TSKEY key = dataRowKey(row);  // first timestamp in buffer
     tsdbDebug("%p uid:%" PRId64", tid:%d check data in imem from skey:%" PRId64 ", order:%d, %p", pHandle,
@@ -370,7 +365,7 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh
     tsdbDebug("%p uid:%"PRId64", tid:%d no data in imem, %p", pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid,
         pHandle->qinfo);
   }
-  
+
   return true;
 }
 
@@ -473,7 +468,7 @@ static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) {
   size_t size = taosArrayGetSize(pHandle->pTableCheckInfo);
   assert(pHandle->activeIndex < size && pHandle->activeIndex >= 0 && size >= 1);
   pHandle->cur.fid = -1;
-  
+
   STableCheckInfo* pCheckInfo = taosArrayGet(pHandle->pTableCheckInfo, pHandle->activeIndex);
 
   STable* pTable = pCheckInfo->pTableObj;
@@ -491,17 +486,17 @@ static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) {
   pCheckInfo->lastKey = dataRowKey(row);  // first timestamp in buffer
   tsdbDebug("%p uid:%" PRId64", tid:%d check data in buffer from skey:%" PRId64 ", order:%d, %p", pHandle,
       pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, pCheckInfo->lastKey, pHandle->order, pHandle->qinfo);
-  
+
   // all data in mem are checked already.
   if ((pCheckInfo->lastKey > pHandle->window.ekey && ASCENDING_TRAVERSE(pHandle->order)) ||
       (pCheckInfo->lastKey < pHandle->window.ekey && !ASCENDING_TRAVERSE(pHandle->order))) {
     return false;
   }
-  
+
   int32_t step = ASCENDING_TRAVERSE(pHandle->order)? 1:-1;
   STimeWindow* win = &pHandle->cur.win;
   pHandle->cur.rows = tsdbReadRowsFromCache(pCheckInfo, pHandle->window.ekey, pHandle->outputCapacity, win, pHandle);
-  
+
   // update the last key value
   pCheckInfo->lastKey = win->ekey + step;
   pHandle->cur.lastKey = win->ekey + step;
@@ -510,7 +505,7 @@ static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) {
   if (!ASCENDING_TRAVERSE(pHandle->order)) {
     SWAP(win->skey, win->ekey, TSKEY);
   }
-  
+
   return true;
 }
 
@@ -519,31 +514,31 @@ static int32_t getFileIdFromKey(TSKEY key, int32_t daysPerFile, int32_t precisio
   if (key == TSKEY_INITIAL_VAL) {
     return INT32_MIN;
   }
-  
+
   int64_t fid = (int64_t)(key / (daysPerFile * tsMsPerDay[precision]));  // set the starting fileId
   if (fid < 0L && llabs(fid) > INT32_MAX) { // data value overflow for INT32
     fid = INT32_MIN;
   }
-  
+
   if (fid > 0L && fid > INT32_MAX) {
     fid = INT32_MAX;
   }
-  
+
   return fid;
 }
 
 static int32_t binarySearchForBlock(SCompBlock* pBlock, int32_t numOfBlocks, TSKEY skey, int32_t order) {
   int32_t firstSlot = 0;
   int32_t lastSlot = numOfBlocks - 1;
-  
+
   int32_t midSlot = firstSlot;
-  
+
   while (1) {
     numOfBlocks = lastSlot - firstSlot + 1;
     midSlot = (firstSlot + (numOfBlocks >> 1));
-    
+
     if (numOfBlocks == 1) break;
-    
+
     if (skey > pBlock[midSlot].keyLast) {
       if (numOfBlocks == 2) break;
       if ((order == TSDB_ORDER_DESC) && (skey < pBlock[midSlot + 1].keyFirst)) break;
@@ -555,7 +550,7 @@ static int32_t binarySearchForBlock(SCompBlock* pBlock, int32_t numOfBlocks, TSK
       break;  // got the slot
     }
   }
-  
+
   return midSlot;
 }
 
@@ -644,7 +639,7 @@ static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlo
                     .uid = (_checkInfo)->tableId.uid})
 
 
-static bool doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock, STableCheckInfo* pCheckInfo) {
+static bool doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock, STableCheckInfo* pCheckInfo, int32_t slotIndex) {
   STsdbRepo *pRepo = pQueryHandle->pTsdb;
   bool       blockLoaded = false;
   int64_t    st = taosGetTimestampUs();
@@ -678,8 +673,9 @@ static bool doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlo
 
   int64_t elapsedTime = (taosGetTimestampUs() - st);
   pQueryHandle->cost.blockLoadTime += elapsedTime;
-  tsdbDebug("%p load file block into buffer, elapsed time:%"PRId64 " us", pQueryHandle, elapsedTime);
 
+  tsdbDebug("%p load file block into buffer, index:%d, brange:%"PRId64"-%"PRId64" , rows:%d, elapsed time:%"PRId64 " us, %p",
+      pQueryHandle, slotIndex, pBlock->keyFirst, pBlock->keyLast, pBlock->numOfRows, elapsedTime, pQueryHandle->qinfo);
   return blockLoaded;
 }
 
@@ -692,18 +688,17 @@ static void handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SCompBlock*
 
   TSKEY key = (row != NULL)? dataRowKey(row):TSKEY_INITIAL_VAL;
   cur->pos = ASCENDING_TRAVERSE(pQueryHandle->order)? 0:(binfo.rows-1);
-  
+
   if ((ASCENDING_TRAVERSE(pQueryHandle->order) && (key != TSKEY_INITIAL_VAL && key <= binfo.window.ekey)) ||
       (!ASCENDING_TRAVERSE(pQueryHandle->order) && (key != TSKEY_INITIAL_VAL && key >= binfo.window.skey))) {
-    
+
     if ((ASCENDING_TRAVERSE(pQueryHandle->order) && (key != TSKEY_INITIAL_VAL && key < binfo.window.skey)) ||
         (!ASCENDING_TRAVERSE(pQueryHandle->order) && (key != TSKEY_INITIAL_VAL && key > binfo.window.ekey))) {
 
       // do not load file block into buffer
       int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order) ? 1 : -1;
 
-      cur->rows = tsdbReadRowsFromCache(pCheckInfo, binfo.window.skey - step,
-                                        pQueryHandle->outputCapacity, &cur->win, pQueryHandle);
+      cur->rows = tsdbReadRowsFromCache(pCheckInfo, binfo.window.skey - step, pQueryHandle->outputCapacity, &cur->win, pQueryHandle);
       pQueryHandle->realNumOfRows = cur->rows;
 
       // update the last key value
@@ -711,13 +706,13 @@ static void handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SCompBlock*
       if (!ASCENDING_TRAVERSE(pQueryHandle->order)) {
         SWAP(cur->win.skey, cur->win.ekey, TSKEY);
       }
-      
+
       cur->mixBlock = true;
       cur->blockCompleted = false;
       return;
     }
-  
-    doLoadFileDataBlock(pQueryHandle, pBlock, pCheckInfo);
+
+    doLoadFileDataBlock(pQueryHandle, pBlock, pCheckInfo, cur->slot);
     doMergeTwoLevelData(pQueryHandle, pCheckInfo, pBlock);
   } else {
     /*
@@ -742,39 +737,41 @@ static bool loadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock
   SQueryFilePos* cur = &pQueryHandle->cur;
 
   if (ASCENDING_TRAVERSE(pQueryHandle->order)) {
-    // query ended in current block
+    // query ended in/started from current block
     if (pQueryHandle->window.ekey < pBlock->keyLast || pCheckInfo->lastKey > pBlock->keyFirst) {
-      if (!doLoadFileDataBlock(pQueryHandle, pBlock, pCheckInfo)) {
+      if (!doLoadFileDataBlock(pQueryHandle, pBlock, pCheckInfo, cur->slot)) {
         return false;
       }
 
       SDataCols* pTSCol = pQueryHandle->rhelper.pDataCols[0];
       assert(pTSCol->cols->type == TSDB_DATA_TYPE_TIMESTAMP && pTSCol->numOfRows == pBlock->numOfRows);
-      
+
       if (pCheckInfo->lastKey > pBlock->keyFirst) {
         cur->pos =
             binarySearchForKey(pTSCol->cols[0].pData, pBlock->numOfRows, pCheckInfo->lastKey, pQueryHandle->order);
       } else {
         cur->pos = 0;
       }
-      
+
+      assert(pCheckInfo->lastKey <= pBlock->keyLast);
       doMergeTwoLevelData(pQueryHandle, pCheckInfo, pBlock);
     } else {  // the whole block is loaded in to buffer
       handleDataMergeIfNeeded(pQueryHandle, pBlock, pCheckInfo);
     }
   } else {  //desc order, query ended in current block
     if (pQueryHandle->window.ekey > pBlock->keyFirst || pCheckInfo->lastKey < pBlock->keyLast) {
-      if (!doLoadFileDataBlock(pQueryHandle, pBlock, pCheckInfo)) {
+      if (!doLoadFileDataBlock(pQueryHandle, pBlock, pCheckInfo, cur->slot)) {
         return false;
       }
-  
+
       SDataCols* pTSCol = pQueryHandle->rhelper.pDataCols[0];
       if (pCheckInfo->lastKey < pBlock->keyLast) {
         cur->pos = binarySearchForKey(pTSCol->cols[0].pData, pBlock->numOfRows, pCheckInfo->lastKey, pQueryHandle->order);
       } else {
         cur->pos = pBlock->numOfRows - 1;
       }
-      
+
+      assert(pCheckInfo->lastKey >= pBlock->keyFirst);
       doMergeTwoLevelData(pQueryHandle, pCheckInfo, pBlock);
     } else {
       handleDataMergeIfNeeded(pQueryHandle, pBlock, pCheckInfo);
@@ -790,7 +787,7 @@ static int doBinarySearchKey(char* pValue, int num, TSKEY key, int order) {
   TSKEY* keyList;
 
   assert(order == TSDB_ORDER_ASC || order == TSDB_ORDER_DESC);
-  
+
   if (num <= 0) return -1;
 
   keyList = (TSKEY*)pValue;
@@ -849,13 +846,19 @@ static int doBinarySearchKey(char* pValue, int num, TSKEY key, int order) {
 static int32_t copyDataFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t capacity, int32_t numOfRows, int32_t start, int32_t end) {
   char* pData = NULL;
   int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1 : -1;
-  
+
   SDataCols* pCols = pQueryHandle->rhelper.pDataCols[0];
   TSKEY* tsArray = pCols->cols[0].pData;
-  
+
   int32_t num = end - start + 1;
+  assert(num >= 0);
+
+  if (num == 0) {
+    return numOfRows;
+  }
+
   int32_t requiredNumOfCols = taosArrayGetSize(pQueryHandle->pColumns);
-  
+
   //data in buffer has greater timestamp, copy data in file block
   int32_t i = 0, j = 0;
   while(i < requiredNumOfCols && j < pCols->numOfCols) {
@@ -928,7 +931,7 @@ static int32_t copyDataFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t cap
 
     i++;
   }
-  
+
   pQueryHandle->cur.win.ekey = tsArray[end];
   pQueryHandle->cur.lastKey = tsArray[end] + step;
 
@@ -995,24 +998,99 @@ static void copyOneRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity,
   }
 }
 
+static void moveDataToFront(STsdbQueryHandle* pQueryHandle, int32_t numOfRows, int32_t numOfCols) {
+  if (numOfRows == 0 || ASCENDING_TRAVERSE(pQueryHandle->order)) {
+    return;
+  }
+
+  // if the buffer is not full in case of descending order query, move the data in the front of the buffer
+  if (numOfRows < pQueryHandle->outputCapacity) {
+    int32_t emptySize = pQueryHandle->outputCapacity - numOfRows;
+    for(int32_t i = 0; i < numOfCols; ++i) {
+      SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i);
+      memmove(pColInfo->pData, pColInfo->pData + emptySize * pColInfo->info.bytes, numOfRows * pColInfo->info.bytes);
+    }
+  }
+}
+
+static void getQualifiedRowsPos(STsdbQueryHandle* pQueryHandle, int32_t startPos, int32_t endPos,
+    int32_t numOfExisted, int32_t *start, int32_t *end) {
+  *start = -1;
+
+  if (ASCENDING_TRAVERSE(pQueryHandle->order)) {
+    int32_t remain = endPos - startPos + 1;
+    if (remain + numOfExisted > pQueryHandle->outputCapacity) {
+      *end = (pQueryHandle->outputCapacity - numOfExisted) + startPos - 1;
+    } else {
+      *end = endPos;
+    }
+
+    *start = startPos;
+  } else {
+    int32_t remain = (startPos - endPos) + 1;
+    if (remain + numOfExisted > pQueryHandle->outputCapacity) {
+      *end = startPos + 1 - (pQueryHandle->outputCapacity - numOfExisted);
+    } else {
+      *end = endPos;
+    }
+
+    *start = *end;
+    *end = startPos;
+  }
+}
+
+static void updateInfoAfterMerge(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, int32_t numOfRows, int32_t endPos) {
+  SQueryFilePos* cur = &pQueryHandle->cur;
+
+  pCheckInfo->lastKey = cur->lastKey;
+  pQueryHandle->realNumOfRows = numOfRows;
+  cur->rows = numOfRows;
+  cur->pos = endPos;
+}
+
+static void doCheckGeneratedBlockRange(STsdbQueryHandle* pQueryHandle) {
+  SQueryFilePos* cur = &pQueryHandle->cur;
+
+  if (cur->rows > 0) {
+    if (ASCENDING_TRAVERSE(pQueryHandle->order)) {
+      assert(cur->win.skey >= pQueryHandle->window.skey && cur->win.ekey <= pQueryHandle->window.ekey);
+    } else {
+      assert(cur->win.skey >= pQueryHandle->window.ekey && cur->win.ekey <= pQueryHandle->window.skey);
+    }
+
+    SColumnInfoData* pColInfoData = taosArrayGet(pQueryHandle->pColumns, 0);
+    assert(cur->win.skey == ((TSKEY*)pColInfoData->pData)[0] && cur->win.ekey == ((TSKEY*)pColInfoData->pData)[cur->rows-1]);
+  } else {
+    cur->win = pQueryHandle->window;
+
+    int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1:-1;
+    cur->lastKey = pQueryHandle->window.ekey + step;
+  }
+}
+
 // only return the qualified data to client in terms of query time window, data rows in the same block but do not
 // be included in the query time window will be discarded
 static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SCompBlock* pBlock) {
   SQueryFilePos* cur = &pQueryHandle->cur;
   SDataBlockInfo blockInfo = GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlock);
-  
+
   initTableMemIterator(pQueryHandle, pCheckInfo);
+
   SDataCols* pCols = pQueryHandle->rhelper.pDataCols[0];
+  assert(pCols->cols[0].type == TSDB_DATA_TYPE_TIMESTAMP && pCols->cols[0].colId == PRIMARYKEY_TIMESTAMP_COL_INDEX &&
+      cur->pos >= 0 && cur->pos < pBlock->numOfRows);
+
+  TSKEY* tsArray = pCols->cols[0].pData;
 
   // for search the endPos, so the order needs to reverse
   int32_t order = (pQueryHandle->order == TSDB_ORDER_ASC)? TSDB_ORDER_DESC:TSDB_ORDER_ASC;
 
   int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1:-1;
-  int32_t numOfCols = taosArrayGetSize(pQueryHandle->pColumns);
+  int32_t numOfCols = QH_GET_NUM_OF_COLS(pQueryHandle);
 
   STable* pTable = pCheckInfo->pTableObj;
-
   int32_t endPos = cur->pos;
+
   if (ASCENDING_TRAVERSE(pQueryHandle->order) && pQueryHandle->window.ekey > blockInfo.window.ekey) {
     endPos = blockInfo.rows - 1;
     cur->mixBlock = (cur->pos != 0);
@@ -1024,48 +1102,36 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
     endPos = doBinarySearchKey(pCols->cols[0].pData, pCols->numOfRows, pQueryHandle->window.ekey, order);
     cur->mixBlock = true;
   }
-  
+
   // compared with the data from in-memory buffer, to generate the correct timestamp array list
-  int32_t pos = cur->pos;
-  
-  assert(pCols->cols[0].type == TSDB_DATA_TYPE_TIMESTAMP && pCols->cols[0].colId == 0);
-  TSKEY* tsArray = pCols->cols[0].pData;
-  
   int32_t numOfRows = 0;
-  pQueryHandle->cur.win = TSWINDOW_INITIALIZER;
+  int32_t pos = cur->pos;
+  cur->win = TSWINDOW_INITIALIZER;
 
   // no data in buffer, load data from file directly
   if (pCheckInfo->iiter == NULL && pCheckInfo->iter == NULL) {
     int32_t start = cur->pos;
     int32_t end = endPos;
+
     if (!ASCENDING_TRAVERSE(pQueryHandle->order)) {
-      end = cur->pos;
-      start = endPos;
-    }
-    
-    cur->win.skey = tsArray[start];
-    cur->win.ekey = tsArray[end];
-    
-    // todo opt in case of no data in buffer
+      SWAP(start, end, int32_t);
+    }
+
     numOfRows = copyDataFromFileBlock(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, start, end);
-    
-    // if the buffer is not full in case of descending order query, move the data in the front of the buffer
-    if (!ASCENDING_TRAVERSE(pQueryHandle->order) && numOfRows < pQueryHandle->outputCapacity) {
-      int32_t emptySize = pQueryHandle->outputCapacity - numOfRows;
 
-      for(int32_t i = 0; i < numOfCols; ++i) {
-        SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i);
-        memmove(pColInfo->pData, pColInfo->pData + emptySize * pColInfo->info.bytes, numOfRows * pColInfo->info.bytes);
-      }
-    }
-  
+    // the time window should always be right order: skey <= ekey
+    cur->win = (STimeWindow) {.skey = tsArray[start], .ekey = tsArray[end]};
+    cur->lastKey = tsArray[endPos];
     pos += (end - start + 1) * step;
-    cur->blockCompleted = (((pos >= endPos || cur->lastKey > pQueryHandle->window.ekey) && ASCENDING_TRAVERSE(pQueryHandle->order)) ||
-        ((pos <= endPos || cur->lastKey < pQueryHandle->window.ekey) && !ASCENDING_TRAVERSE(pQueryHandle->order)));
-    
-    pCheckInfo->lastKey = cur->lastKey;
-    pQueryHandle->realNumOfRows = numOfRows;
-    cur->rows = numOfRows;
+
+    cur->blockCompleted =
+        (((pos >= endPos || cur->lastKey > pQueryHandle->window.ekey) && ASCENDING_TRAVERSE(pQueryHandle->order)) ||
+         ((pos <= endPos || cur->lastKey < pQueryHandle->window.ekey) && !ASCENDING_TRAVERSE(pQueryHandle->order)));
+
+    // if the buffer is not full in case of descending order query, move the data in the front of the buffer
+    moveDataToFront(pQueryHandle, numOfRows, numOfCols);
+    updateInfoAfterMerge(pQueryHandle, pCheckInfo, numOfRows, pos);
+    doCheckGeneratedBlockRange(pQueryHandle);
     return;
   } else if (pCheckInfo->iter != NULL || pCheckInfo->iiter != NULL) {
     SSkipListNode* node = NULL;
@@ -1111,30 +1177,18 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
         if (tsArray[end] == key) { // the value of key in cache equals to the end timestamp value, ignore it
           moveToNextRowInMem(pCheckInfo);
         }
-        
-        int32_t start = -1;
-        if (ASCENDING_TRAVERSE(pQueryHandle->order)) {
-          int32_t remain = end - pos + 1;
-          if (remain + numOfRows > pQueryHandle->outputCapacity) {
-            end = (pQueryHandle->outputCapacity - numOfRows) + pos - 1;
-          }
 
-          start = pos;
-        } else {
-          int32_t remain = (pos - end) + 1;
-          if (remain + numOfRows > pQueryHandle->outputCapacity) {
-            end = pos + 1 - (pQueryHandle->outputCapacity - numOfRows);
-          }
+        int32_t qstart = 0, qend = 0;
+        getQualifiedRowsPos(pQueryHandle, pos, end, numOfRows, &qstart, &qend);
 
-          start = end;
-          end = pos;
-        }
+        numOfRows = copyDataFromFileBlock(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, qstart, qend);
+        pos += (qend - qstart + 1) * step;
 
-        numOfRows = copyDataFromFileBlock(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, start, end);
-        pos += (end - start + 1) * step;
+        cur->win.ekey = ASCENDING_TRAVERSE(pQueryHandle->order)? tsArray[qend]:tsArray[qstart];
+        cur->lastKey  = cur->win.ekey + step;
       }
     } while (numOfRows < pQueryHandle->outputCapacity);
-    
+
     if (numOfRows < pQueryHandle->outputCapacity) {
       /**
        * if cache is empty, load remain file block data. In contrast, if there are remain data in cache, do NOT
@@ -1148,54 +1202,29 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
           cur->win.skey = tsArray[pos];
         }
 
-        int32_t start = -1;
-        int32_t end = -1;
-
-        // all remain data are qualified, but check the remain capacity in the first place.
-        if (ASCENDING_TRAVERSE(pQueryHandle->order)) {
-          int32_t remain = endPos - pos + 1;
-          if (remain + numOfRows > pQueryHandle->outputCapacity) {
-            endPos = (pQueryHandle->outputCapacity - numOfRows) + pos - 1;
-          }
-
-          start = pos;
-          end = endPos;
-        } else {
-          int32_t remain = pos + 1;
-          if (remain + numOfRows > pQueryHandle->outputCapacity) {
-            endPos = pos + 1 - (pQueryHandle->outputCapacity - numOfRows);
-          }
-
-          start = endPos;
-          end = pos;
-        }
+        int32_t start = -1, end = -1;
+        getQualifiedRowsPos(pQueryHandle, pos, endPos, numOfRows, &start, &end);
 
         numOfRows = copyDataFromFileBlock(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, start, end);
         pos += (end - start + 1) * step;
+
+        cur->win.ekey = ASCENDING_TRAVERSE(pQueryHandle->order)? tsArray[end]:tsArray[start];
+        cur->lastKey  = cur->win.ekey + step;
       }
     }
   }
-  
-  cur->blockCompleted = (((pos >= endPos || cur->lastKey > pQueryHandle->window.ekey) && ASCENDING_TRAVERSE(pQueryHandle->order)) ||
-      ((pos <= endPos || cur->lastKey < pQueryHandle->window.ekey) && !ASCENDING_TRAVERSE(pQueryHandle->order)));
+
+  cur->blockCompleted =
+      (((pos >= endPos || cur->lastKey > pQueryHandle->window.ekey) && ASCENDING_TRAVERSE(pQueryHandle->order)) ||
+       ((pos <= endPos || cur->lastKey < pQueryHandle->window.ekey) && !ASCENDING_TRAVERSE(pQueryHandle->order)));
 
   if (!ASCENDING_TRAVERSE(pQueryHandle->order)) {
     SWAP(cur->win.skey, cur->win.ekey, TSKEY);
-  
-    // if the buffer is not full in case of descending order query, move the data in the front of the buffer
-    if (numOfRows < pQueryHandle->outputCapacity) {
-      int32_t emptySize = pQueryHandle->outputCapacity - numOfRows;
-      for(int32_t i = 0; i < numOfCols; ++i) {
-        SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i);
-        memmove(pColInfo->pData, pColInfo->pData + emptySize * pColInfo->info.bytes, numOfRows * pColInfo->info.bytes);
-      }
-    }
   }
-  
-  pCheckInfo->lastKey = cur->lastKey;
-  pQueryHandle->realNumOfRows = numOfRows;
-  cur->rows = numOfRows;
-  cur->pos = pos;
+
+  moveDataToFront(pQueryHandle, numOfRows, numOfCols);
+  updateInfoAfterMerge(pQueryHandle, pCheckInfo, numOfRows, pos);
+  doCheckGeneratedBlockRange(pQueryHandle);
 
   tsdbDebug("%p uid:%" PRIu64",tid:%d data block created, brange:%"PRIu64"-%"PRIu64" rows:%d, %p", pQueryHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, cur->win.skey,
       cur->win.ekey, cur->rows, pQueryHandle->qinfo);
@@ -1332,16 +1361,16 @@ static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numO
     cleanBlockOrderSupporter(&sup, 0);
     return TSDB_CODE_TDB_OUT_OF_MEMORY;
   }
-  
+
   int32_t cnt = 0;
   int32_t numOfQualTables = 0;
-  
+
   for (int32_t j = 0; j < numOfTables; ++j) {
     STableCheckInfo* pTableCheck = (STableCheckInfo*)taosArrayGet(pQueryHandle->pTableCheckInfo, j);
     if (pTableCheck->numOfBlocks <= 0) {
       continue;
     }
-    
+
     SCompBlock* pBlock = pTableCheck->pCompInfo->blocks;
     sup.numOfBlocksPerTable[numOfQualTables] = pTableCheck->numOfBlocks;
 
@@ -1437,37 +1466,39 @@ static int32_t getDataBlocksInFilesImpl(STsdbQueryHandle* pQueryHandle, bool* ex
 
     // current file are not overlapped with query time window, ignore remain files
     if ((ASCENDING_TRAVERSE(pQueryHandle->order) && win.skey > pQueryHandle->window.ekey) ||
-    (!ASCENDING_TRAVERSE(pQueryHandle->order) && win.ekey < pQueryHandle->window.ekey)) {
-      tsdbDebug("%p remain files are not qualified for qrange:%"PRId64"-%"PRId64", ignore, %p", pQueryHandle, pQueryHandle->window.skey, pQueryHandle->window.ekey, pQueryHandle->qinfo)
+        (!ASCENDING_TRAVERSE(pQueryHandle->order) && win.ekey < pQueryHandle->window.ekey)) {
+      tsdbDebug("%p remain files are not qualified for qrange:%" PRId64 "-%" PRId64 ", ignore, %p", pQueryHandle,
+                pQueryHandle->window.skey, pQueryHandle->window.ekey, pQueryHandle->qinfo);
       pQueryHandle->pFileGroup = NULL;
+      assert(pQueryHandle->numOfBlocks == 0);
       break;
     }
 
     if ((code = getFileCompInfo(pQueryHandle, &numOfBlocks)) != TSDB_CODE_SUCCESS) {
       break;
     }
-    
-    tsdbDebug("%p %d blocks found in file for %d table(s), fid:%d, %p", pQueryHandle, numOfBlocks,
-           numOfTables, pQueryHandle->pFileGroup->fileId, pQueryHandle->qinfo);
-    
+
+    tsdbDebug("%p %d blocks found in file for %d table(s), fid:%d, %p", pQueryHandle, numOfBlocks, numOfTables,
+              pQueryHandle->pFileGroup->fileId, pQueryHandle->qinfo);
+
     assert(numOfBlocks >= 0);
     if (numOfBlocks == 0) {
       continue;
     }
-    
+
     // todo return error code to query engine
-    if (createDataBlocksInfo(pQueryHandle, numOfBlocks, &pQueryHandle->numOfBlocks) != TSDB_CODE_SUCCESS) {
+    if ((code = createDataBlocksInfo(pQueryHandle, numOfBlocks, &pQueryHandle->numOfBlocks)) != TSDB_CODE_SUCCESS) {
       break;
     }
-    
+
     assert(numOfBlocks >= pQueryHandle->numOfBlocks);
     if (pQueryHandle->numOfBlocks > 0) {
       break;
     }
   }
-  
+
   // no data in file anymore
-  if (pQueryHandle->numOfBlocks <= 0) {
+  if (pQueryHandle->numOfBlocks <= 0 || code != TSDB_CODE_SUCCESS) {
     if (code == TSDB_CODE_SUCCESS) {
       assert(pQueryHandle->pFileGroup == NULL);
     }
@@ -1476,10 +1507,11 @@ static int32_t getDataBlocksInFilesImpl(STsdbQueryHandle* pQueryHandle, bool* ex
     *exists = false;
     return code;
   }
-  
+
+  assert(pQueryHandle->pFileGroup != NULL && pQueryHandle->numOfBlocks > 0);
   cur->slot = ASCENDING_TRAVERSE(pQueryHandle->order)? 0:pQueryHandle->numOfBlocks-1;
   cur->fid = pQueryHandle->pFileGroup->fileId;
-  
+
   STableBlockInfo* pBlockInfo = &pQueryHandle->pDataBlockInfo[cur->slot];
   *exists = loadFileDataBlock(pQueryHandle, pBlockInfo->compBlock, pBlockInfo->pTableCheckInfo);
 
@@ -1495,7 +1527,7 @@ static int32_t getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle, bool* exists
     pQueryHandle->locateStart = true;
     STsdbCfg* pCfg = &pQueryHandle->pTsdb->config;
     int32_t fid = getFileIdFromKey(pQueryHandle->window.skey, pCfg->daysPerFile, pCfg->precision);
-    
+
     tsdbInitFileGroupIter(pFileHandle, &pQueryHandle->fileIter, pQueryHandle->order);
     tsdbSeekFileGroupIter(&pQueryHandle->fileIter, fid);
 
@@ -1504,7 +1536,7 @@ static int32_t getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle, bool* exists
     // check if current file block is all consumed
     STableBlockInfo* pBlockInfo = &pQueryHandle->pDataBlockInfo[cur->slot];
     STableCheckInfo* pCheckInfo = pBlockInfo->pTableCheckInfo;
-    
+
     // current block is done, try next
     if (!cur->mixBlock || cur->blockCompleted) {
       if ((cur->slot == pQueryHandle->numOfBlocks - 1 && ASCENDING_TRAVERSE(pQueryHandle->order)) ||
@@ -1515,10 +1547,10 @@ static int32_t getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle, bool* exists
         // next block of the same file
         int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order) ? 1 : -1;
         cur->slot += step;
-        
+
         cur->mixBlock = false;
         cur->blockCompleted = false;
-        
+
         STableBlockInfo* pNext = &pQueryHandle->pDataBlockInfo[cur->slot];
         *exists = loadFileDataBlock(pQueryHandle, pNext->compBlock, pNext->pTableCheckInfo);
 
@@ -1540,10 +1572,10 @@ static bool doHasDataInBuffer(STsdbQueryHandle* pQueryHandle) {
     if (hasMoreDataInCache(pQueryHandle)) {
       return true;
     }
-    
+
     pQueryHandle->activeIndex += 1;
   }
-  
+
   return false;
 }
 
@@ -1561,14 +1593,14 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) {
   if (pQueryHandle->type == TSDB_QUERY_TYPE_EXTERNAL) {
     pQueryHandle->type = TSDB_QUERY_TYPE_ALL;
     pQueryHandle->order = TSDB_ORDER_DESC;
-    
+
     if (!tsdbNextDataBlock(pHandle)) {
       return false;
     }
-    
+
     /*SDataBlockInfo* pBlockInfo =*/ tsdbRetrieveDataBlockInfo(pHandle, &blockInfo);
     /*SArray *pDataBlock = */tsdbRetrieveDataBlock(pHandle, pQueryHandle->defaultLoadColumn);
-  
+
     if (pQueryHandle->cur.win.ekey == pQueryHandle->window.skey) {
       // data already retrieve, discard other data rows and return
       int32_t numOfCols = QH_GET_NUM_OF_COLS(pQueryHandle);
@@ -1576,7 +1608,7 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) {
         SColumnInfoData* pCol = taosArrayGet(pQueryHandle->pColumns, i);
         memcpy(pCol->pData, pCol->pData + pCol->info.bytes * (pQueryHandle->cur.rows-1), pCol->info.bytes);
       }
-  
+
       pQueryHandle->cur.win  = (STimeWindow){pQueryHandle->window.skey, pQueryHandle->window.skey};
       pQueryHandle->window   = pQueryHandle->cur.win;
       pQueryHandle->cur.rows = 1;
@@ -1593,7 +1625,7 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) {
       pSecQueryHandle->checkFiles  = true;
       pSecQueryHandle->activeIndex = 0;
       pSecQueryHandle->outputCapacity = ((STsdbRepo*)pSecQueryHandle->pTsdb)->config.maxRowsPerFileBlock;
-  
+
       if (tsdbInitReadHelper(&pSecQueryHandle->rhelper, (STsdbRepo*) pSecQueryHandle->pTsdb) != 0) {
         free(pSecQueryHandle);
         return false;
@@ -1603,24 +1635,24 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) {
 
       // allocate buffer in order to load data blocks from file
       int32_t numOfCols = QH_GET_NUM_OF_COLS(pQueryHandle);
-  
+
       pSecQueryHandle->statis = calloc(numOfCols, sizeof(SDataStatis));
       pSecQueryHandle->pColumns = taosArrayInit(numOfCols, sizeof(SColumnInfoData));
 
       for (int32_t i = 0; i < numOfCols; ++i) {
         SColumnInfoData colInfo = {{0}, 0};
         SColumnInfoData* pCol = taosArrayGet(pQueryHandle->pColumns, i);
-        
+
         colInfo.info = pCol->info;
         colInfo.pData = calloc(1, EXTRA_BYTES + pQueryHandle->outputCapacity * pCol->info.bytes);
         taosArrayPush(pSecQueryHandle->pColumns, &colInfo);
       }
-  
+
       size_t si = taosArrayGetSize(pQueryHandle->pTableCheckInfo);
       pSecQueryHandle->pTableCheckInfo = taosArrayInit(si, sizeof(STableCheckInfo));
       STsdbMeta* pMeta = tsdbGetMeta(pQueryHandle->pTsdb);
       assert(pMeta != NULL);
-  
+
       for (int32_t j = 0; j < si; ++j) {
         STableCheckInfo* pCheckInfo = (STableCheckInfo*) taosArrayGet(pQueryHandle->pTableCheckInfo, j);
         STableCheckInfo info = {
@@ -1628,10 +1660,10 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) {
             .tableId = pCheckInfo->tableId,
             .pTableObj = pCheckInfo->pTableObj,
         };
-        
+
         taosArrayPush(pSecQueryHandle->pTableCheckInfo, &info);
       }
-  
+
       tsdbInitDataBlockLoadInfo(&pSecQueryHandle->dataBlockLoadInfo);
       tsdbInitCompBlockLoadInfo(&pSecQueryHandle->compBlockLoadInfo);
       pSecQueryHandle->defaultLoadColumn = taosArrayClone(pQueryHandle->defaultLoadColumn);
@@ -1641,17 +1673,17 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) {
 
       tsdbRetrieveDataBlockInfo((void*) pSecQueryHandle, &blockInfo);
       tsdbRetrieveDataBlock((void*) pSecQueryHandle, pSecQueryHandle->defaultLoadColumn);
-  
+
       for (int32_t i = 0; i < numOfCols; ++i) {
         SColumnInfoData* pCol = taosArrayGet(pQueryHandle->pColumns, i);
         memcpy(pCol->pData, pCol->pData + pCol->info.bytes * (pQueryHandle->cur.rows-1), pCol->info.bytes);
-  
+
         SColumnInfoData* pCol1 = taosArrayGet(pSecQueryHandle->pColumns, i);
         assert(pCol->info.colId == pCol1->info.colId);
-        
+
         memcpy(pCol->pData + pCol->info.bytes, pCol1->pData, pCol1->info.bytes);
       }
-  
+
       SColumnInfoData* pTSCol = taosArrayGet(pQueryHandle->pColumns, 0);
 
       // it is ascending order
@@ -1675,7 +1707,7 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) {
     pQueryHandle->checkFiles = false;
     return true;
   }
-  
+
   if (pQueryHandle->checkFiles) {
     bool exists = true;
     int32_t code = getDataBlocksInFiles(pQueryHandle, &exists);
@@ -1688,11 +1720,11 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) {
       pQueryHandle->cost.checkForNextTime += elapsedTime;
       return exists;
     }
-  
+
     pQueryHandle->activeIndex = 0;
     pQueryHandle->checkFiles  = false;
   }
-  
+
   // TODO: opt by consider the scan order
   bool ret = doHasDataInBuffer(pQueryHandle);
   terrno = TSDB_CODE_SUCCESS;
@@ -1705,15 +1737,15 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) {
 void changeQueryHandleForLastrowQuery(TsdbQueryHandleT pqHandle) {
   STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pqHandle;
   assert(!ASCENDING_TRAVERSE(pQueryHandle->order));
-  
+
   // starts from the buffer in case of descending timestamp order check data blocks
-  
+
   // todo consider the query time window, current last_row does not apply the query time window
   size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo);
-  
+
   TSKEY key = TSKEY_INITIAL_VAL;
   int32_t index = -1;
-  
+
   for(int32_t i = 0; i < numOfTables; ++i) {
     STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i);
     if (pCheckInfo->pTableObj->lastKey > key) {
@@ -1721,36 +1753,36 @@ void changeQueryHandleForLastrowQuery(TsdbQueryHandleT pqHandle) {
       index = i;
     }
   }
-  
+
   if (index == -1) {
     // todo add failure test cases
     return;
   }
-  
+
   // erase all other elements in array list
   size_t size = taosArrayGetSize(pQueryHandle->pTableCheckInfo);
   for (int32_t i = 0; i < size; ++i) {
     if (i == index) {
       continue;
     }
-    
+
     STableCheckInfo* pTableCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i);
     tSkipListDestroyIter(pTableCheckInfo->iter);
-    
+
     if (pTableCheckInfo->pDataCols != NULL) {
       tfree(pTableCheckInfo->pDataCols->buf);
     }
-    
+
     tfree(pTableCheckInfo->pDataCols);
     tfree(pTableCheckInfo->pCompInfo);
   }
-  
+
   STableCheckInfo info = *(STableCheckInfo*) taosArrayGet(pQueryHandle->pTableCheckInfo, index);
   taosArrayClear(pQueryHandle->pTableCheckInfo);
-  
+
   info.lastKey = key;
   taosArrayPush(pQueryHandle->pTableCheckInfo, &info);
-  
+
   // update the query time window according to the chosen last timestamp
   pQueryHandle->window = (STimeWindow) {key, key};
 }
@@ -1759,13 +1791,13 @@ static void changeQueryHandleForInterpQuery(TsdbQueryHandleT pHandle) {
   // filter the queried time stamp in the first place
   STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pHandle;
   pQueryHandle->order = TSDB_ORDER_DESC;
-  
+
   assert(pQueryHandle->window.skey == pQueryHandle->window.ekey);
-  
+
   // starts from the buffer in case of descending timestamp order check data blocks
   // todo consider the query time window, current last_row does not apply the query time window
   size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo);
-  
+
   int32_t i = 0;
   while(i < numOfTables) {
     STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i);
@@ -1773,21 +1805,21 @@ static void changeQueryHandleForInterpQuery(TsdbQueryHandleT pHandle) {
         pCheckInfo->pTableObj->lastKey != TSKEY_INITIAL_VAL) {
       break;
     }
-    
+
     i++;
   }
-  
+
   // there are no data in all the tables
   if (i == numOfTables) {
     return;
   }
-  
+
   STableCheckInfo info = *(STableCheckInfo*) taosArrayGet(pQueryHandle->pTableCheckInfo, i);
   taosArrayClear(pQueryHandle->pTableCheckInfo);
-  
+
   info.lastKey = pQueryHandle->window.skey;
   taosArrayPush(pQueryHandle->pTableCheckInfo, &info);
-  
+
   // update the query time window according to the chosen last timestamp
   pQueryHandle->window = (STimeWindow) {info.lastKey, TSKEY_INITIAL_VAL};
 }
@@ -1811,7 +1843,7 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int
     if ((key > maxKey && ASCENDING_TRAVERSE(pQueryHandle->order)) || (key < maxKey && !ASCENDING_TRAVERSE(pQueryHandle->order))) {
       tsdbDebug("%p key:%"PRIu64" beyond qrange:%"PRId64" - %"PRId64", no more data in buffer", pQueryHandle, key, pQueryHandle->window.skey,
           pQueryHandle->window.ekey);
-      
+
       break;
     }
 
@@ -1826,24 +1858,24 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int
       moveToNextRowInMem(pCheckInfo);
       break;
     }
-    
+
   } while(moveToNextRowInMem(pCheckInfo));
 
   assert(numOfRows <= maxRowsToRead);
-  
+
   // if the buffer is not full in case of descending order query, move the data in the front of the buffer
   if (!ASCENDING_TRAVERSE(pQueryHandle->order) && numOfRows < maxRowsToRead) {
     int32_t emptySize = maxRowsToRead - numOfRows;
-    
+
     for(int32_t i = 0; i < numOfCols; ++i) {
       SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i);
       memmove(pColInfo->pData, pColInfo->pData + emptySize * pColInfo->info.bytes, numOfRows * pColInfo->info.bytes);
     }
   }
-  
+
   int64_t elapsedTime = taosGetTimestampUs() - st;
-  tsdbDebug("%p build data block from cache completed, elapsed time:%"PRId64" us, numOfRows:%d, numOfCols:%d", pQueryHandle,
-            elapsedTime, numOfRows, numOfCols);
+  tsdbDebug("%p build data block from cache completed, elapsed time:%"PRId64" us, numOfRows:%d, numOfCols:%d, %p", pQueryHandle,
+            elapsedTime, numOfRows, numOfCols, pQueryHandle->qinfo);
 
   return numOfRows;
 }
@@ -1852,7 +1884,7 @@ void tsdbRetrieveDataBlockInfo(TsdbQueryHandleT* pQueryHandle, SDataBlockInfo* p
   STsdbQueryHandle* pHandle = (STsdbQueryHandle*)pQueryHandle;
   SQueryFilePos* cur = &pHandle->cur;
   STable* pTable = NULL;
-  
+
   // there are data in file
   if (pHandle->cur.fid >= 0) {
     STableBlockInfo* pBlockInfo = &pHandle->pDataBlockInfo[cur->slot];
@@ -1874,13 +1906,13 @@ void tsdbRetrieveDataBlockInfo(TsdbQueryHandleT* pQueryHandle, SDataBlockInfo* p
  */
 int32_t tsdbRetrieveDataBlockStatisInfo(TsdbQueryHandleT* pQueryHandle, SDataStatis** pBlockStatis) {
   STsdbQueryHandle* pHandle = (STsdbQueryHandle*) pQueryHandle;
-  
+
   SQueryFilePos* c = &pHandle->cur;
   if (c->mixBlock) {
     *pBlockStatis = NULL;
     return TSDB_CODE_SUCCESS;
   }
-  
+
   STableBlockInfo* pBlockInfo = &pHandle->pDataBlockInfo[c->slot];
   assert((c->slot >= 0 && c->slot < pHandle->numOfBlocks) || ((c->slot == pHandle->numOfBlocks) && (c->slot == 0)));
 
@@ -1900,7 +1932,7 @@ int32_t tsdbRetrieveDataBlockStatisInfo(TsdbQueryHandleT* pQueryHandle, SDataSta
   for(int32_t i = 0; i < numOfCols; ++i) {
     pHandle->statis[i].colId = colIds[i];
   }
-  
+
   tsdbGetDataStatis(&pHandle->rhelper, pHandle->statis, numOfCols);
 
   // always load the first primary timestamp column data
@@ -1949,31 +1981,31 @@ SArray* tsdbRetrieveDataBlock(TsdbQueryHandleT* pQueryHandle, SArray* pIdList) {
     } else {
       SDataBlockInfo binfo = GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlockInfo->compBlock);
       assert(pHandle->realNumOfRows <= binfo.rows);
-  
+
       // data block has been loaded, todo extract method
       SDataBlockLoadInfo* pBlockLoadInfo = &pHandle->dataBlockLoadInfo;
-      
+
       if (pBlockLoadInfo->slot == pHandle->cur.slot && pBlockLoadInfo->fileGroup->fileId == pHandle->cur.fid &&
           pBlockLoadInfo->tid == pCheckInfo->pTableObj->tableId.tid) {
         return pHandle->pColumns;
       } else {  // only load the file block
         SCompBlock* pBlock = pBlockInfo->compBlock;
-        doLoadFileDataBlock(pHandle, pBlock, pCheckInfo);
-  
+        doLoadFileDataBlock(pHandle, pBlock, pCheckInfo, pHandle->cur.slot);
+
         // todo refactor
         int32_t numOfRows = copyDataFromFileBlock(pHandle, pHandle->outputCapacity, 0, 0, pBlock->numOfRows - 1);
-  
+
         // if the buffer is not full in case of descending order query, move the data in the front of the buffer
         if (!ASCENDING_TRAVERSE(pHandle->order) && numOfRows < pHandle->outputCapacity) {
           int32_t emptySize = pHandle->outputCapacity - numOfRows;
           int32_t reqNumOfCols = taosArrayGetSize(pHandle->pColumns);
-    
+
           for(int32_t i = 0; i < reqNumOfCols; ++i) {
             SColumnInfoData* pColInfo = taosArrayGet(pHandle->pColumns, i);
             memmove(pColInfo->pData, pColInfo->pData + emptySize * pColInfo->info.bytes, numOfRows * pColInfo->info.bytes);
           }
         }
-        
+
         return pHandle->pColumns;
       }
     }
@@ -1984,11 +2016,11 @@ static int32_t getAllTableList(STable* pSuperTable, SArray* list) {
   SSkipListIterator* iter = tSkipListCreateIter(pSuperTable->pIndex);
   while (tSkipListIterNext(iter)) {
     SSkipListNode* pNode = tSkipListIterGet(iter);
-    
+
     STable** pTable = (STable**) SL_GET_NODE_DATA((SSkipListNode*) pNode);
     taosArrayPush(list, pTable);
   }
-  
+
   tSkipListDestroyIter(iter);
   return TSDB_CODE_SUCCESS;
 }
@@ -1998,12 +2030,12 @@ static void destroyHelper(void* param) {
     return;
   }
 
-  
+
   tQueryInfo* pInfo = (tQueryInfo*)param;
   if (pInfo->optr != TSDB_RELATION_IN) {
     tfree(pInfo->q);
   }
-  
+
 //  tVariantDestroy(&(pInfo->q));
   free(param);
 }
@@ -2015,7 +2047,7 @@ void filterPrepare(void* expr, void* param) {
   }
 
   pExpr->_node.info = calloc(1, sizeof(tQueryInfo));
-  
+
   STSchema*   pTSSchema = (STSchema*) param;
   tQueryInfo* pInfo = pExpr->_node.info;
   tVariant*   pCond = pExpr->_node.pRight->pVal;
@@ -2025,7 +2057,7 @@ void filterPrepare(void* expr, void* param) {
   pInfo->optr     = pExpr->_node.optr;
   pInfo->compare  = getComparFunc(pSchema->type, pInfo->optr);
   pInfo->param    = pTSSchema;
-  
+
   if (pInfo->optr == TSDB_RELATION_IN) {
     pInfo->q = (char*) pCond->arr;
   } else {
@@ -2038,25 +2070,24 @@ typedef struct STableGroupSupporter {
   int32_t    numOfCols;
   SColIndex* pCols;
   STSchema*  pTagSchema;
-//  void*      tsdbMeta;
 } STableGroupSupporter;
 
 int32_t tableGroupComparFn(const void *p1, const void *p2, const void *param) {
   STableGroupSupporter* pTableGroupSupp = (STableGroupSupporter*) param;
   STable* pTable1 = *(STable**) p1;
   STable* pTable2 = *(STable**) p2;
-  
+
   for (int32_t i = 0; i < pTableGroupSupp->numOfCols; ++i) {
     SColIndex* pColIndex = &pTableGroupSupp->pCols[i];
     int32_t colIndex = pColIndex->colIndex;
-    
+
     assert(colIndex >= TSDB_TBNAME_COLUMN_INDEX);
-    
+
     char *  f1 = NULL;
     char *  f2 = NULL;
     int32_t type = 0;
     int32_t bytes = 0;
-    
+
     if (colIndex == TSDB_TBNAME_COLUMN_INDEX) {
       f1 = (char*) TABLE_NAME(pTable1);
       f2 = (char*) TABLE_NAME(pTable2);
@@ -2090,14 +2121,14 @@ int32_t tableGroupComparFn(const void *p1, const void *p2, const void *param) {
       return ret;
     }
   }
-  
+
   return 0;
 }
 
 void createTableGroupImpl(SArray* pGroups, SArray* pTableList, size_t numOfTables, STableGroupSupporter* pSupp,
     __ext_compar_fn_t compareFn) {
   STable* pTable = taosArrayGetP(pTableList, 0);
-  
+
   SArray* g = taosArrayInit(16, POINTER_BYTES);
   taosArrayPush(g, &pTable);
   tsdbRefTable(pTable);
@@ -2105,10 +2136,10 @@ void createTableGroupImpl(SArray* pGroups, SArray* pTableList, size_t numOfTable
   for (int32_t i = 1; i < numOfTables; ++i) {
     STable** prev = taosArrayGet(pTableList, i - 1);
     STable** p = taosArrayGet(pTableList, i);
-    
+
     int32_t ret = compareFn(prev, p, pSupp);
     assert(ret == 0 || ret == -1);
-    
+
     tsdbRefTable(*p);
     assert((*p)->type == TSDB_CHILD_TABLE);
 
@@ -2120,20 +2151,20 @@ void createTableGroupImpl(SArray* pGroups, SArray* pTableList, size_t numOfTable
       taosArrayPush(g, p);
     }
   }
-  
+
   taosArrayPush(pGroups, &g);
 }
 
 SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pCols, int32_t numOfOrderCols) {
   assert(pTableList != NULL);
   SArray* pTableGroup = taosArrayInit(1, POINTER_BYTES);
-  
+
   size_t size = taosArrayGetSize(pTableList);
   if (size == 0) {
     tsdbDebug("no qualified tables");
     return pTableGroup;
   }
-  
+
   if (numOfOrderCols == 0 || size == 1) { // no group by tags clause or only one table
     SArray* sa = taosArrayInit(size, POINTER_BYTES);
     for(int32_t i = 0; i < size; ++i) {
@@ -2143,7 +2174,7 @@ SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pC
       tsdbRefTable(*pTable);
       taosArrayPush(sa, pTable);
     }
-    
+
     taosArrayPush(pTableGroup, &sa);
     tsdbDebug("all %zu tables belong to one group", size);
   } else {
@@ -2151,18 +2182,18 @@ SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pC
     pSupp->numOfCols = numOfOrderCols;
     pSupp->pTagSchema = pTagSchema;
     pSupp->pCols = pCols;
-    
+
     taosqsort(pTableList->pData, size, POINTER_BYTES, pSupp, tableGroupComparFn);
     createTableGroupImpl(pTableGroup, pTableList, size, pSupp, tableGroupComparFn);
     tfree(pSupp);
   }
-  
+
   return pTableGroup;
 }
 
 bool indexedNodeFilterFp(const void* pNode, void* param) {
   tQueryInfo* pInfo = (tQueryInfo*) param;
-  
+
   STable* pTable = *(STable**)(SL_GET_NODE_DATA((SSkipListNode*)pNode));
 
   char*  val = NULL;
@@ -2172,7 +2203,7 @@ bool indexedNodeFilterFp(const void* pNode, void* param) {
   } else {
     val = tdGetKVRowValOfCol(pTable->tagVal, pInfo->sch.colId);
   }
-  
+
   int32_t ret = 0;
   if (val == NULL) { //the val is possible to be null, so check it out carefully
     ret = -1; // val is missing in table tags value pairs
@@ -2209,7 +2240,7 @@ bool indexedNodeFilterFp(const void* pNode, void* param) {
     default:
       assert(false);
   }
-  
+
   return true;
 }
 
@@ -2239,7 +2270,7 @@ int32_t tsdbQuerySTableByTagCond(TSDB_REPO_T* tsdb, uint64_t uid, const char* pT
 
     goto _error;
   }
-  
+
   if (pTable->type != TSDB_SUPER_TABLE) {
     tsdbError("%p query normal tag not allowed, uid:%" PRIu64 ", tid:%d, name:%s", tsdb, uid, pTable->tableId.tid,
         pTable->name->data);
@@ -2252,7 +2283,7 @@ int32_t tsdbQuerySTableByTagCond(TSDB_REPO_T* tsdb, uint64_t uid, const char* pT
   //NOTE: not add ref count for super table
   SArray* res = taosArrayInit(8, POINTER_BYTES);
   STSchema* pTagSchema = tsdbGetTableTagSchema(pTable);
-  
+
   // no tags and tbname condition, all child tables of this stable are involved
   if (tbnameCond == NULL && (pTagCond == NULL || len == 0)) {
     int32_t ret = getAllTableList(pTable, res);
@@ -2263,7 +2294,7 @@ int32_t tsdbQuerySTableByTagCond(TSDB_REPO_T* tsdb, uint64_t uid, const char* pT
 
     pGroupInfo->numOfTables = taosArrayGetSize(res);
     pGroupInfo->pGroupList  = createTableGroup(res, pTagSchema, pColIndex, numOfCols);
-      
+
     tsdbDebug("%p no table name/tag condition, all tables belong to one group, numOfTables:%zu", tsdb, pGroupInfo->numOfTables);
     taosArrayDestroy(res);
 
@@ -2299,7 +2330,7 @@ int32_t tsdbQuerySTableByTagCond(TSDB_REPO_T* tsdb, uint64_t uid, const char* pT
   } CATCH( code ) {
     CLEANUP_EXECUTE();
     terrno = code;
-    goto _error; 
+    goto _error;
     // TODO: more error handling
   } END_TRY
 
@@ -2335,12 +2366,12 @@ int32_t tsdbGetOneTableGroup(TSDB_REPO_T* tsdb, uint64_t uid, STableGroupInfo* p
 
   pGroupInfo->numOfTables = 1;
   pGroupInfo->pGroupList = taosArrayInit(1, POINTER_BYTES);
-  
+
   SArray* group = taosArrayInit(1, POINTER_BYTES);
-  
+
   taosArrayPush(group, &pTable);
   taosArrayPush(pGroupInfo->pGroupList, &group);
-  
+
   return TSDB_CODE_SUCCESS;
 
   _error:
@@ -2428,8 +2459,8 @@ void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle) {
   tsdbDestroyHelper(&pQueryHandle->rhelper);
 
   SIOCostSummary* pCost = &pQueryHandle->cost;
-  tsdbDebug(":io-cost summary: statis-info:%"PRId64"us, datablock:%" PRId64"us, check data:%"PRId64"us, %p",
-      pCost->statisInfoLoadTime, pCost->blockLoadTime, pCost->checkForNextTime, pQueryHandle->qinfo);
+  tsdbDebug("%p :io-cost summary: statis-info:%"PRId64"us, datablock:%" PRId64"us, check data:%"PRId64"us, %p",
+      pQueryHandle, pCost->statisInfoLoadTime, pCost->blockLoadTime, pCost->checkForNextTime, pQueryHandle->qinfo);
 
   tfree(pQueryHandle);
 }

src/util/CMakeLists.txt

@@ -12,9 +12,10 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
   AUX_SOURCE_DIRECTORY(src SRC)
   ADD_LIBRARY(tutil ${SRC})
   TARGET_LINK_LIBRARIES(tutil pthread osdetail m rt lz4)
+  ADD_SUBDIRECTORY(tests)
+
   FIND_PATH(ICONV_INCLUDE_EXIST iconv.h /usr/include/ /usr/local/include/)
   IF (ICONV_INCLUDE_EXIST)
-    ADD_DEFINITIONS(-DUSE_LIBICONV)
     FIND_PATH(ICONV_LIBRARY_A_EXIST libiconv.a /usr/lib/ /usr/local/lib/ /lib64)
     FIND_PATH(ICONV_LIBRARY_SO_EXIST libiconv.so /usr/lib/ /usr/local/lib/ /lib64)
     IF (ICONV_LIBRARY_A_EXIST OR ICONV_LIBRARY_SO_EXIST)
@@ -28,9 +29,7 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
     MESSAGE(STATUS "Failed to find iconv, use default encoding method")
   ENDIF ()
 
-  ADD_SUBDIRECTORY(tests)
 ELSEIF (TD_WINDOWS_64)
-  ADD_DEFINITIONS(-DUSE_LIBICONV)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/pthread)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/iconv)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/regex)
@@ -70,7 +69,6 @@ ELSEIF (TD_WINDOWS_64)
   ADD_LIBRARY(tutil ${SRC}) 
   TARGET_LINK_LIBRARIES(tutil iconv regex pthread osdetail winmm IPHLPAPI ws2_32 lz4)
 ELSEIF(TD_DARWIN_64)
-  ADD_DEFINITIONS(-DUSE_LIBICONV)
   AUX_SOURCE_DIRECTORY(src SRC)
   ADD_LIBRARY(tutil ${SRC})
   TARGET_LINK_LIBRARIES(tutil iconv pthread osdetail lz4)

src/util/inc/tcache.h

@@ -68,8 +68,6 @@ typedef struct {
   int64_t         refreshTime;
   STrashElem *    pTrash;
   char*           name;
-//  void *          tmrCtrl;
-//  void *          pTimer;
   SCacheStatis    statistics;
   SHashObj *      pHashTable;
   __cache_free_fn_t freeFp;

src/util/inc/tlist.h

@@ -55,6 +55,8 @@ int        tdListPrepend(SList *list, void *data);
 int        tdListAppend(SList *list, void *data);
 SListNode *tdListPopHead(SList *list);
 SListNode *tdListPopTail(SList *list);
+SListNode *tdListGetHead(SList *list);
+SListNode *tsListGetTail(SList *list);
 SListNode *tdListPopNode(SList *list, SListNode *node);
 void       tdListMove(SList *src, SList *dst);
 void       tdListDiscard(SList *list);

src/util/inc/ttime.h

@@ -20,9 +20,8 @@
 extern "C" {
 #endif
 
-#include <stdint.h>
-#include <time.h>
-#include "tutil.h"
+#include "os.h"
+#include "taosdef.h"
 
 //@return timestamp in second
 int32_t taosGetTimestampSec();

src/util/inc/tutil.h

@@ -25,148 +25,24 @@ extern "C" {
 #include "tcrc32c.h"
 #include "taosdef.h"
 
-#ifndef STDERR_FILENO
-#define STDERR_FILENO (2)
-#endif
-
-#define FD_VALID(x) ((x) > STDERR_FILENO)
-#define FD_INITIALIZER  ((int32_t)-1)
-
-#define WCHAR wchar_t
-
-#define tfree(x)         \
-  do {                   \
-    if (x) {             \
-      free((void *)(x)); \
-      x = 0;             \
-    }                    \
-  } while(0);
-
-#define tstrncpy(dst, src, size)   \
-  do {                             \
-    strncpy((dst), (src), (size)); \
-    (dst)[(size)-1] = 0;           \
-  } while (0);
-
-#define tclose(x) taosCloseSocket(x)
-
-// Pointer p drift right by b bytes
-#define POINTER_SHIFT(p, b) ((void *)((char *)(p) + (b)))
-#define POINTER_DISTANCE(p1, p2) ((char *)(p1) - (char *)(p2)) 
-
-#ifndef NDEBUG
-#define ASSERT(x) assert(x)
-#else
-#define ASSERT(x)
-#endif
-
-#ifdef UNUSED
-#undefine UNUSED
-#endif
-#define UNUSED(x) ((void)(x))
-
-#ifdef UNUSED_FUNC
-#undefine UNUSED_FUNC
-#endif
-
-#ifdef UNUSED_PARAM
-#undef UNUSED_PARAM
-#endif
-
-#if defined(__GNUC__)
-#define UNUSED_PARAM(x) _UNUSED##x __attribute__((unused))
-#define UNUSED_FUNC __attribute__((unused))
-#else
-#define UNUSED_PARAM(x) x
-#define UNUSED_FUNC
-#endif
-
-#ifdef tListLen
-#undefine tListLen
-#endif
-#define tListLen(x) (sizeof(x) / sizeof((x)[0]))
-
-#if defined(__GNUC__)
-#define FORCE_INLINE inline __attribute__((always_inline))
-#else
-#define FORCE_INLINE
-#endif
-
-#define DEFAULT_UNICODE_ENCODEC "UCS-4LE"
-  
-#define DEFAULT_COMP(x, y)       \
-  do {                           \
-    if ((x) == (y)) {            \
-      return 0;                  \
-    } else {                     \
-      return (x) < (y) ? -1 : 1; \
-    }                            \
-  } while (0)
-
-#define ALIGN_NUM(n, align) (((n) + ((align)-1)) & (~((align)-1)))
-
-// align to 8bytes
-#define ALIGN8(n) ALIGN_NUM(n, 8)
-
-#define MILLISECOND_PER_MINUTE (MILLISECOND_PER_SECOND * 60)
-#define MILLISECOND_PER_HOUR   (MILLISECOND_PER_MINUTE * 60)
-#define MILLISECOND_PER_DAY    (MILLISECOND_PER_HOUR * 24)
-#define MILLISECOND_PER_WEEK   (MILLISECOND_PER_DAY * 7)
-#define MILLISECOND_PER_MONTH  (MILLISECOND_PER_DAY * 30)
-#define MILLISECOND_PER_YEAR   (MILLISECOND_PER_DAY * 365)
-
-#define POW2(x) ((x) * (x))
-
-uint32_t taosRand(void);
-
-uint32_t trand(void);
-
-size_t twcslen(const wchar_t *wcs);
-
 int32_t strdequote(char *src);
-
-size_t strtrim(char *src);
-
-char *strnchr(char *haystack, char needle, int32_t len, bool skipquote);
-
-char **strsplit(char *src, const char *delim, int32_t *num);
-
-char* strtolower(char *dst, const char *src);
-
-char* strntolower(char *dst, const char *src, int32_t n);
-
+size_t  strtrim(char *src);
+char *  strnchr(char *haystack, char needle, int32_t len, bool skipquote);
+char ** strsplit(char *src, const char *delim, int32_t *num);
+char *  strtolower(char *dst, const char *src);
+char *  strntolower(char *dst, const char *src, int32_t n);
 int64_t strnatoi(char *num, int32_t len);
-
-//char* strreplace(const char* str, const char* pattern, const char* rep);
-
-char *strbetween(char *string, char *begin, char *end);
-
-char *paGetToken(char *src, char **token, int32_t *tokenLen);
+char *  strbetween(char *string, char *begin, char *end);
+char *  paGetToken(char *src, char **token, int32_t *tokenLen);
 
 int32_t taosByteArrayToHexStr(char bytes[], int32_t len, char hexstr[]);
-
 int32_t taosHexStrToByteArray(char hexstr[], char bytes[]);
 
-int32_t taosFileRename(char *fullPath, char *suffix, char delimiter, char **dstPath);
-
-/**
- *
- * @param fileNamePattern
- * @param dstPath
- */
-void getTmpfilePath(const char *fileNamePattern, char *dstPath);
-
-bool taosMbsToUcs4(char *mbs, size_t mbs_len, char *ucs4, int32_t ucs4_max_len, size_t* len);
-
-int tasoUcs4Compare(void* f1_ucs4, void *f2_ucs4, int bytes);
-
-void taosRandStr(char* str, int32_t size);
-
-int32_t taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs);
-
-bool taosValidateEncodec(const char *encodec);
-
 bool taosGetVersionNumber(char *versionStr, int *versionNubmer);
+int  taosCheckVersion(char *input_client_version, char *input_server_version, int compared_segments);
+
+char *   taosIpStr(uint32_t ipInt);
+uint32_t ip2uint(const char *const ip_addr);
 
 static FORCE_INLINE void taosEncryptPass(uint8_t *inBuf, unsigned int inLen, char *target) {
   MD5_CTX context;
@@ -176,56 +52,6 @@ static FORCE_INLINE void taosEncryptPass(uint8_t *inBuf, unsigned int inLen, cha
   memcpy(target, context.digest, TSDB_KEY_LEN);
 }
 
-int taosCheckVersion(char *input_client_version, char *input_server_version, int compared_segments);
-
-char *taosIpStr(uint32_t ipInt);
-
-uint32_t ip2uint(const char *const ip_addr);
-
-void taosRemoveDir(char *rootDir);
-int  tmkdir(const char *pathname, mode_t mode); 
-void taosMvDir(char* destDir, char *srcDir);
-
-#define TAOS_ALLOC_MODE_DEFAULT 0
-#define TAOS_ALLOC_MODE_RANDOM_FAIL 1
-#define TAOS_ALLOC_MODE_DETECT_LEAK 2
-void taosSetAllocMode(int mode, const char* path, bool autoDump);
-void taosDumpMemoryLeak();
-
-void * tmalloc(size_t size);
-void * tcalloc(size_t nmemb, size_t size);
-size_t tsizeof(void *ptr);
-void   tmemset(void *ptr, int c);
-void * trealloc(void *ptr, size_t size);
-void   tzfree(void *ptr);
-
-#ifdef TAOS_MEM_CHECK
-
-void *  taos_malloc(size_t size, const char *file, uint32_t line);
-void *  taos_calloc(size_t num, size_t size, const char *file, uint32_t line);
-void *  taos_realloc(void *ptr, size_t size, const char *file, uint32_t line);
-void    taos_free(void *ptr, const char *file, uint32_t line);
-char *  taos_strdup(const char *str, const char *file, uint32_t line);
-char *  taos_strndup(const char *str, size_t size, const char *file, uint32_t line);
-ssize_t taos_getline(char **lineptr, size_t *n, FILE *stream, const char *file, uint32_t line);
-
-#ifndef TAOS_MEM_CHECK_IMPL
-
-#define malloc(size) taos_malloc(size, __FILE__, __LINE__)
-#define calloc(num, size) taos_calloc(num, size, __FILE__, __LINE__)
-#define realloc(ptr, size) taos_realloc(ptr, size, __FILE__, __LINE__)
-#define free(ptr) taos_free(ptr, __FILE__, __LINE__)
-#define strdup(str) taos_strdup(str, __FILE__, __LINE__)
-#define strndup(str, size) taos_strndup(str, size, __FILE__, __LINE__)
-#define getline(lineptr, n, stream) taos_getline(lineptr, n, stream, __FILE__, __LINE__)
-
-#endif  // TAOS_MEM_CHECK_IMPL
-
-#endif // TAOS_MEM_CHECK
-
-
-char *taosCharsetReplace(char *charsetstr);
-
 #ifdef __cplusplus
 }
 #endif

src/util/src/hash.c

@@ -14,10 +14,9 @@
  */
 
 #include "os.h"
-
 #include "hash.h"
 #include "tulog.h"
-#include "tutil.h"
+#include "taosdef.h"
 
 static FORCE_INLINE void __wr_lock(void *lock) {
   if (lock == NULL) {

src/util/src/tcache.c

@@ -343,7 +343,7 @@ void* taosCacheUpdateExpireTimeByName(SCacheObj *pCacheObj, void *key, size_t ke
   SCacheDataNode **ptNode = (SCacheDataNode **)taosHashGet(pCacheObj->pHashTable, key, keyLen);
   if (ptNode != NULL) {
      T_REF_INC(*ptNode);
-    (*ptNode)->expireTime = taosGetTimestampMs() + (*ptNode)->lifespan;
+    (*ptNode)->expireTime = expireTime; // taosGetTimestampMs() + (*ptNode)->lifespan;
   }
 
   __cache_unlock(pCacheObj);
@@ -381,7 +381,7 @@ void *taosCacheAcquireByData(SCacheObj *pCacheObj, void *data) {
 }
 
 void *taosCacheTransfer(SCacheObj *pCacheObj, void **data) {
-  if (pCacheObj == NULL || data == NULL) return NULL;
+  if (pCacheObj == NULL || data == NULL || (*data) == NULL) return NULL;
   
   size_t          offset = offsetof(SCacheDataNode, data);
   SCacheDataNode *ptNode = (SCacheDataNode *)((char *)(*data) - offset);
@@ -419,7 +419,7 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) {
   // note: extend lifespan before dec ref count
   bool inTrashCan = pNode->inTrashCan;
 
-  if (pCacheObj->extendLifespan && (!inTrashCan)) {
+  if (pCacheObj->extendLifespan && (!inTrashCan) && (!_remove)) {
     atomic_store_64(&pNode->expireTime, pNode->lifespan + taosGetTimestampMs());
     uDebug("cache:%s data:%p extend life time to %"PRId64 "  before release", pCacheObj->name, pNode->data, pNode->expireTime);
   }
@@ -457,8 +457,9 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) {
   } else {
     // NOTE: once refcount is decrease, pNode may be freed by other thread immediately.
     int32_t ref = T_REF_DEC(pNode);
-    uDebug("cache:%s, key:%p, %p is released, refcnt:%d, in trashcan:%d", pCacheObj->name, pNode->key, pNode->data, ref,
-        inTrashCan);
+
+    uDebug("cache:%s, key:%p, %p released, refcnt:%d, data in trancan:%d", pCacheObj->name, pNode->key, pNode->data,
+           ref, inTrashCan);
   }
 }
 
@@ -572,6 +573,7 @@ void taosRemoveFromTrashCan(SCacheObj *pCacheObj, STrashElem *pElem) {
   free(pElem);
 }
 
+// TODO add another lock when scanning trashcan
 void taosTrashCanEmpty(SCacheObj *pCacheObj, bool force) {
   __cache_wr_lock(pCacheObj);
 
@@ -643,6 +645,7 @@ static void doCacheRefresh(SCacheObj* pCacheObj, int64_t time, __cache_free_fn_t
   __cache_wr_lock(pCacheObj);
   while (taosHashIterNext(pIter)) {
     SCacheDataNode *pNode = *(SCacheDataNode **)taosHashIterGet(pIter);
+
     if (pNode->expireTime < time && T_REF_VAL_GET(pNode) <= 0) {
       taosCacheReleaseNode(pCacheObj, pNode);
       continue;
@@ -674,6 +677,7 @@ void* taosCacheTimedRefresh(void *handle) {
 
     // check if current cache object will be deleted every 500ms.
     if (pCacheObj->deleting) {
+      uDebug("%s refresh threads quit", pCacheObj->name);
       break;
     }
 

src/util/src/tcompare.c

 #include "taosdef.h"
 #include "tcompare.h"
-#include <tarray.h>
+#include "tarray.h"
 #include "tutil.h"
 
 int32_t compareInt32Val(const void *pLeft, const void *pRight) {

src/util/src/tconfig.c

@@ -118,7 +118,7 @@ static void taosReadDirectoryConfig(SGlobalCfg *cfg, char *input_value) {
       
       wordfree(&full_path);
 
-      int code = tmkdir(option, 0755);
+      int code = taosMkDir(option, 0755);
       if (code != 0) {
         terrno = TAOS_SYSTEM_ERROR(errno);
         uError("config option:%s, input value:%s, directory not exist, create fail:%s",

src/util/src/tlist.c

@@ -76,6 +76,7 @@ int tdListPrepend(SList *list, void *data) {
   SListNode *node = (SListNode *)malloc(sizeof(SListNode) + list->eleSize);
   if (node == NULL) return -1;
 
+  node->next = node->prev = NULL;
   memcpy((void *)(node->data), data, list->eleSize);
   tdListPrependNode(list, node);
 
@@ -121,6 +122,22 @@ SListNode *tdListPopTail(SList *list) {
   return node;
 }
 
+SListNode *tdListGetHead(SList *list) {
+  if (list == NULL || list->numOfEles == 0) {
+    return NULL;
+  }
+
+  return list->head;
+}
+
+SListNode *tsListGetTail(SList *list) {
+  if (list == NULL || list->numOfEles == 0) {
+    return NULL;
+  }
+
+  return list->tail;
+}
+
 SListNode *tdListPopNode(SList *list, SListNode *node) {
   if (list->head == node) {
     list->head = node->next;

src/util/src/tutil.c

@@ -14,11 +14,6 @@
  */
 
 #include "os.h"
-
-#ifdef USE_LIBICONV
-#include "iconv.h"
-#endif
-
 #include "tcrc32c.h"
 #include "tglobal.h"
 #include "ttime.h"
@@ -27,76 +22,6 @@
 #include "tulog.h"
 #include "taoserror.h"
 
-
-#ifdef WINDOWS
-uint32_t taosRand(void)
-{
-  return rand();
-}
-#else
-uint32_t taosRand(void)
-{
-/*
-  int fd;
-  int seed;
-  
-  fd = open("/dev/urandom", 0);
-  if (fd < 0) {
-    seed = time(0);
-  } else {
-    int len = read(fd, &seed, sizeof(seed));
-    if (len < 0) {
-      seed = time(0);
-    }  
-    close(fd);
-  }
-
-  return (uint32_t)seed;
-*/
-  return rand();
-}
-
-uint32_t trand(void)
-{
-  int fd;
-  int seed;
-  
-  fd = open("/dev/urandom", 0);
-  if (fd < 0) {
-    seed = time(0);
-  } else {
-    int len = read(fd, &seed, sizeof(seed));
-    if (len < 0) {
-      seed = time(0);
-    }  
-    close(fd);
-  }
-
-  return (uint32_t)seed;
-}
-#endif
-
-size_t twcslen(const wchar_t *wcs) {
-#ifdef WINDOWS
-  int *wstr = (int *)wcs;
-  if (NULL == wstr) {
-    return 0;
-  }
-
-  size_t n = 0;
-  while (1) {
-    if (0 == *wstr++) {
-      break;
-    }
-    n++;
-  }
-
-  return n;
-#else
-  return wcslen(wcs);
-#endif
-}
-
 int32_t strdequote(char *z) {
   if (z == NULL) {
     return 0;
@@ -234,6 +159,7 @@ char* strtolower(char *dst, const char *src) {
   *p = 0;
   return dst;
 }
+
 char* strntolower(char *dst, const char *src, int32_t n) {
   int esc = 0;
   char quote = 0, *p = dst, c;
@@ -343,96 +269,6 @@ int64_t strnatoi(char *num, int32_t len) {
   return ret;
 }
 
-#if 0
-FORCE_INLINE size_t getLen(size_t old, size_t size) {
-  if (old == 1) {
-    old = 2;
-  }
-
-  while (old < size) {
-    old = (old * 1.5);
-  }
-
-  return old;
-}
-
-static char *ensureSpace(char *dest, size_t *curSize, size_t size) {
-  if (*curSize < size) {
-    *curSize = getLen(*curSize, size);
-
-    char *tmp = realloc(dest, *curSize);
-    if (tmp == NULL) {
-      free(dest);
-      return NULL;
-    }
-
-    return tmp;
-  }
-
-  return dest;
-}
-
-char *strreplace(const char *str, const char *pattern, const char *rep) {
-  if (str == NULL || pattern == NULL || rep == NULL) {
-    return NULL;
-  }
-
-  const char *s = str;
-
-  size_t oldLen = strlen(str);
-  size_t newLen = oldLen;
-
-  size_t repLen = strlen(rep);
-  size_t patternLen = strlen(pattern);
-
-  char *dest = calloc(1, oldLen + 1);
-  if (dest == NULL) {
-    return NULL;
-  }
-
-  if (patternLen == 0) {
-    return strcpy(dest, str);
-  }
-
-  int32_t start = 0;
-
-  while (1) {
-    char *p = strstr(str, pattern);
-    if (p == NULL) {  // remain does not contain pattern
-      size_t remain = (oldLen - (str - s));
-      size_t size = remain + start + 1;
-
-      dest = ensureSpace(dest, &newLen, size);
-      if (dest == NULL) {
-        return NULL;
-      }
-
-      strcpy(dest + start, str);
-      dest[start + remain] = 0;
-      break;
-    }
-
-    size_t len = p - str;
-    size_t size = start + len + repLen + 1;
-
-    dest = ensureSpace(dest, &newLen, size);
-    if (dest == NULL) {
-      return NULL;
-    }
-
-    memcpy(dest + start, str, len);
-
-    str += (len + patternLen);
-    start += len;
-
-    memcpy(dest + start, rep, repLen);
-    start += repLen;
-  }
-
-  return dest;
-}
-#endif
-
 char *strbetween(char *string, char *begin, char *end) {
   char *result = NULL;
   char *_begin = strstr(string, begin);
@@ -491,170 +327,6 @@ int32_t taosHexStrToByteArray(char hexstr[], char bytes[]) {
   return 0;
 }
 
-// rename file name
-int32_t taosFileRename(char *fullPath, char *suffix, char delimiter, char **dstPath) {
-  int32_t ts = taosGetTimestampSec();
-
-  char fname[PATH_MAX] = {0};  // max file name length must be less than 255
-
-  char *delimiterPos = strrchr(fullPath, delimiter);
-  if (delimiterPos == NULL) return -1;
-
-  int32_t fileNameLen = 0;
-  if (suffix)
-    fileNameLen = snprintf(fname, PATH_MAX, "%s.%d.%s", delimiterPos + 1, ts, suffix);
-  else
-    fileNameLen = snprintf(fname, PATH_MAX, "%s.%d", delimiterPos + 1, ts);
-
-  size_t len = (size_t)((delimiterPos - fullPath) + fileNameLen + 1);
-  if (*dstPath == NULL) {
-    *dstPath = calloc(1, len + 1);
-    if (*dstPath == NULL) return -1;
-  }
-
-  strncpy(*dstPath, fullPath, (size_t)(delimiterPos - fullPath + 1));
-  strncat(*dstPath, fname, (size_t)fileNameLen);
-  (*dstPath)[len] = 0;
-
-  return rename(fullPath, *dstPath);
-}
-
-void getTmpfilePath(const char *fileNamePrefix, char *dstPath) {
-  const char* tdengineTmpFileNamePrefix = "tdengine-";
-  
-  char tmpPath[PATH_MAX];
-
-#ifdef WINDOWS
-  char *tmpDir = getenv("tmp");
-  if (tmpDir == NULL) {
-    tmpDir = "";
-  }
-#else
-  char *tmpDir = "/tmp/";
-#endif
-  
-  strcpy(tmpPath, tmpDir);
-  strcat(tmpPath, tdengineTmpFileNamePrefix);
-  if (strlen(tmpPath) + strlen(fileNamePrefix) + strlen("-%d-%s") < PATH_MAX) {
-    strcat(tmpPath, fileNamePrefix);
-    strcat(tmpPath, "-%d-%s");
-  }
-  
-  char rand[8] = {0};
-  taosRandStr(rand, tListLen(rand) - 1);
-  snprintf(dstPath, PATH_MAX, tmpPath, getpid(), rand);
-}
-
-void taosRandStr(char* str, int32_t size) {
-  const char* set = "abcdefghijklmnopqrstuvwxyz0123456789-_.";
-  int32_t len = 39;
-
-  for (int32_t i = 0; i < size; ++i) {
-    str[i] = set[taosRand() % len];
-  }
-}
-
-int tasoUcs4Compare(void* f1_ucs4, void *f2_ucs4, int bytes) {
-#if defined WINDOWS
-  for (int i = 0; i < bytes; ++i) {
-    int32_t f1 = *(int32_t*)((char*)f1_ucs4 + i * 4);
-    int32_t f2 = *(int32_t*)((char*)f2_ucs4 + i * 4);
-
-    if ((f1 == 0 && f2 != 0) || (f1 != 0 && f2 == 0)) {
-      return f1 - f2;
-    }
-    else if (f1 == 0 && f2 == 0) {
-      return 0;
-    }
-
-    if (f1 != f2) {
-      return f1 - f2;
-    }
-  }
-  return 0;
-
-#if 0
-  int32_t ucs4_max_len = bytes + 4;
-  char *f1_mbs = calloc(bytes, 1);
-  char *f2_mbs = calloc(bytes, 1);
-  if (taosUcs4ToMbs(f1_ucs4, ucs4_max_len, f1_mbs) < 0) {
-    return -1;
-  }
-  if (taosUcs4ToMbs(f2_ucs4, ucs4_max_len, f2_mbs) < 0) {
-    return -1;
-  }
-  int32_t ret = strcmp(f1_mbs, f2_mbs);
-  free(f1_mbs);
-  free(f2_mbs);
-  return ret;
-#endif
-
-#else
-  return wcsncmp((wchar_t *)f1_ucs4, (wchar_t *)f2_ucs4, bytes / TSDB_NCHAR_SIZE);
-#endif
-}
-
-int32_t taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs) {
-#ifdef USE_LIBICONV
-  iconv_t cd = iconv_open(tsCharset, DEFAULT_UNICODE_ENCODEC);
-  size_t ucs4_input_len = ucs4_max_len;
-  size_t outLen = ucs4_max_len;
-  if (iconv(cd, (char **)&ucs4, &ucs4_input_len, &mbs, &outLen) == -1) {
-    iconv_close(cd);
-    return -1;
-  }
-  iconv_close(cd);
-  return (int32_t)(ucs4_max_len - outLen);
-#else
-  mbstate_t state = {0};
-  int32_t len = (int32_t) wcsnrtombs(NULL, (const wchar_t **) &ucs4, ucs4_max_len / 4, 0, &state);
-  if (len < 0) {
-    return -1;
-  }
-  memset(&state, 0, sizeof(state));
-  len = wcsnrtombs(mbs, (const wchar_t **) &ucs4, ucs4_max_len / 4, (size_t) len, &state);
-  if (len < 0) {
-    return -1;
-  }
-  return len;
-#endif
-}
-
-bool taosMbsToUcs4(char *mbs, size_t mbsLength, char *ucs4, int32_t ucs4_max_len, size_t* len) {
-  memset(ucs4, 0, ucs4_max_len);
-#ifdef USE_LIBICONV
-  iconv_t cd = iconv_open(DEFAULT_UNICODE_ENCODEC, tsCharset);
-  size_t ucs4_input_len = mbsLength;
-  size_t outLeft = ucs4_max_len;
-  if (iconv(cd, &mbs, &ucs4_input_len, &ucs4, &outLeft) == -1) {
-    iconv_close(cd);
-    return false;
-  }
-  
-  iconv_close(cd);
-  if (len != NULL) {
-    *len = ucs4_max_len - outLeft;
-  }
-  
-  return true;
-#else
-  mbstate_t state = {0};
-  int32_t len = mbsnrtowcs((wchar_t *) ucs4, (const char **) &mbs, mbsLength, ucs4_max_len / 4, &state);
-  return len >= 0;
-#endif
-}
-
-bool taosValidateEncodec(const char *encodec) {
-#ifdef USE_LIBICONV
-  iconv_t cd = iconv_open(encodec, DEFAULT_UNICODE_ENCODEC);
-  if (cd == (iconv_t)(-1)) {
-    return false;
-  }
-  iconv_close(cd);
-#endif
-  return true;
-}
-
 bool taosGetVersionNumber(char *versionStr, int *versionNubmer) {
   if (versionStr == NULL || versionNubmer == NULL) {
     return false;
@@ -734,112 +406,3 @@ FORCE_INLINE double taos_align_get_double(const char* pBuf) {
   *(int64_t*)(&dv) = *(int64_t*)pBuf;
   return dv; 
 }
-
-typedef struct CharsetPair {
-  char *oldCharset;
-  char *newCharset;
-} CharsetPair;
-
-char *taosCharsetReplace(char *charsetstr) {
-  CharsetPair charsetRep[] = {
-      { "utf8", "UTF-8" }, { "936", "CP936" },
-  };
-
-  for (int32_t i = 0; i < tListLen(charsetRep); ++i) {
-    if (strcasecmp(charsetRep[i].oldCharset, charsetstr) == 0) {
-      return strdup(charsetRep[i].newCharset);
-    }
-  }
-
-  return strdup(charsetstr);
-}
-
-void *tmalloc(size_t size) {
-  if (size <= 0) return NULL;
-
-  void *ret = malloc(size + sizeof(size_t));
-  if (ret == NULL) return NULL;
-
-  *(size_t *)ret = size;
-
-  return (void *)((char *)ret + sizeof(size_t));
-}
-
-void *tcalloc(size_t nmemb, size_t size) {
-  size_t tsize = nmemb * size;
-  void * ret = tmalloc(tsize);
-  if (ret == NULL) return NULL;
-
-  tmemset(ret, 0);
-  return ret;
-}
-
-size_t tsizeof(void *ptr) { return (ptr) ? (*(size_t *)((char *)ptr - sizeof(size_t))) : 0; }
-
-void tmemset(void *ptr, int c) { memset(ptr, c, tsizeof(ptr)); }
-
-void * trealloc(void *ptr, size_t size) {
-  if (ptr == NULL) return tmalloc(size);
-
-  if (size <= tsizeof(ptr)) return ptr;
-
-  void * tptr = (void *)((char *)ptr - sizeof(size_t));
-  size_t tsize = size + sizeof(size_t);
-  tptr = realloc(tptr, tsize);
-  if (tptr == NULL) return NULL;
-
-  *(size_t *)tptr = size;
-
-  return (void *)((char *)tptr + sizeof(size_t));
-}
-
-void tzfree(void *ptr) {
-  if (ptr) {
-    free((void *)((char *)ptr - sizeof(size_t)));
-  }
-}
-
-void taosRemoveDir(char *rootDir) {
-  DIR *dir = opendir(rootDir);
-  if (dir == NULL) return;
-
-  struct dirent *de = NULL;
-  while ((de = readdir(dir)) != NULL) {
-    if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) continue;
-     
-    char filename[1024];
-    snprintf(filename, 1023, "%s/%s", rootDir, de->d_name);
-    if (de->d_type & DT_DIR) {
-      taosRemoveDir(filename);
-    } else {
-      (void)remove(filename);
-      uInfo("file:%s is removed", filename);
-    }
-  }
-
-  closedir(dir);
-  rmdir(rootDir);
-
-  uInfo("dir:%s is removed", rootDir);
-}
-
-int tmkdir(const char *path, mode_t mode) {
-  int code = mkdir(path, 0755);
-  if (code < 0 && errno == EEXIST) code = 0;
-  return code;
-}
-
-void taosMvDir(char* destDir, char *srcDir) {
-  if (0 == tsEnableVnodeBak) {
-    uInfo("vnode backup not enabled");
-    return;
-  }
-
-  char shellCmd[1024+1] = {0}; 
-  
-  //(void)snprintf(shellCmd, 1024, "cp -rf %s %s", srcDir, destDir);
-  (void)snprintf(shellCmd, 1024, "mv %s %s", srcDir, destDir);
-  tSystem(shellCmd);
-  uInfo("shell cmd:%s is executed", shellCmd);
-}
-

src/vnode/src/vnodeRead.c

@@ -71,11 +71,45 @@ static void vnodePutItemIntoReadQueue(SVnodeObj *pVnode, void *qhandle) {
   pRead->rpcMsg.msgType = TSDB_MSG_TYPE_QUERY;
   pRead->pCont = qhandle;
   pRead->contLen = 0;
+  pRead->rpcMsg.handle = NULL;
 
   atomic_add_fetch_32(&pVnode->refCount, 1);
   taosWriteQitem(pVnode->rqueue, TAOS_QTYPE_QUERY, pRead);
 }
 
+static int32_t vnodeDumpQueryResult(SRspRet *pRet, void* pVnode, void* handle, bool* freeHandle) {
+  bool continueExec = false;
+
+  int32_t code = TSDB_CODE_SUCCESS;
+  if ((code = qDumpRetrieveResult(handle, (SRetrieveTableRsp **)&pRet->rsp, &pRet->len, &continueExec)) == TSDB_CODE_SUCCESS) {
+    if (continueExec) {
+      vDebug("QInfo:%p add to query task queue for exec", handle);
+      vnodePutItemIntoReadQueue(pVnode, handle);
+      pRet->qhandle = handle;
+      *freeHandle = false;
+    } else {
+      vDebug("QInfo:%p exec completed", handle);
+      *freeHandle = true;
+    }
+  } else {
+    pRet->rsp = (SRetrieveTableRsp *)rpcMallocCont(sizeof(SRetrieveTableRsp));
+    memset(pRet->rsp, 0, sizeof(SRetrieveTableRsp));
+    *freeHandle = true;
+  }
+
+  return code;
+}
+
+static void vnodeBuildNoResultQueryRsp(SRspRet* pRet) {
+  pRet->rsp = (SRetrieveTableRsp *)rpcMallocCont(sizeof(SRetrieveTableRsp));
+  pRet->len = sizeof(SRetrieveTableRsp);
+
+  memset(pRet->rsp, 0, sizeof(SRetrieveTableRsp));
+  SRetrieveTableRsp* pRsp = pRet->rsp;
+
+  pRsp->completed = true;
+}
+
 static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
   void    *pCont = pReadMsg->pCont;
   int32_t  contLen = pReadMsg->contLen;
@@ -98,6 +132,7 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
       vWarn("QInfo:%p invalid qhandle, no matched query handle, conn:%p", (void*) killQueryMsg->qhandle, pReadMsg->rpcMsg.handle);
     } else {
       assert(*qhandle == (void*) killQueryMsg->qhandle);
+
       qKillQuery(*qhandle);
       qReleaseQInfo(pVnode->qMgmt, (void**) &qhandle, true);
     }
@@ -110,7 +145,7 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
 
   if (contLen != 0) {
     qinfo_t pQInfo = NULL;
-    code = qCreateQueryInfo(pVnode->tsdb, pVnode->vgId, pQueryTableMsg, pVnode, &pQInfo);
+    code = qCreateQueryInfo(pVnode->tsdb, pVnode->vgId, pQueryTableMsg, &pQInfo);
 
     SQueryTableRsp *pRsp = (SQueryTableRsp *) rpcMallocCont(sizeof(SQueryTableRsp));
     pRsp->code    = code;
@@ -133,7 +168,6 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
         pRsp->qhandle = htobe64((uint64_t) pQInfo);
       }
 
-      pQInfo = NULL;
       if (handle != NULL && vnodeNotifyCurrentQhandle(pReadMsg->rpcMsg.handle, *handle, pVnode->vgId) != TSDB_CODE_SUCCESS) {
         vError("vgId:%d, QInfo:%p, query discarded since link is broken, %p", pVnode->vgId, *handle, pReadMsg->rpcMsg.handle);
         pRsp->code = TSDB_CODE_RPC_NETWORK_UNAVAIL;
@@ -153,16 +187,34 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
 
   } else {
     assert(pCont != NULL);
+
     handle = qAcquireQInfo(pVnode->qMgmt, (uint64_t) pCont);
     if (handle == NULL) {
       vWarn("QInfo:%p invalid qhandle in continuing exec query, conn:%p", (void*) pCont, pReadMsg->rpcMsg.handle);
       code = TSDB_CODE_QRY_INVALID_QHANDLE;
     } else {
       vDebug("vgId:%d, QInfo:%p, dnode continue exec query", pVnode->vgId, (void*) pCont);
-      code = TSDB_CODE_VND_ACTION_IN_PROGRESS;
-      qTableQuery(*handle); // do execute query
+
+      bool freehandle = false;
+      bool buildRes = qTableQuery(*handle); // do execute query
+
+      // build query rsp
+      if (buildRes) {
+        // update the connection info according to the retrieve connection
+        pReadMsg->rpcMsg.handle = qGetResultRetrieveMsg(*handle);
+        assert(pReadMsg->rpcMsg.handle != NULL);
+
+        vDebug("vgId:%d, QInfo:%p, start to build result rsp after query paused, %p", pVnode->vgId, *handle, pReadMsg->rpcMsg.handle);
+        code = vnodeDumpQueryResult(&pReadMsg->rspRet, pVnode, *handle, &freehandle);
+
+        // todo test the error code case
+        if (code == TSDB_CODE_SUCCESS) {
+          code = TSDB_CODE_QRY_HAS_RSP;
+        }
+      }
+
+      qReleaseQInfo(pVnode->qMgmt, (void**) &handle, freehandle);
     }
-    qReleaseQInfo(pVnode->qMgmt, (void**) &handle, false);
   }
 
   return code;
@@ -176,7 +228,7 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
   pRetrieve->qhandle = htobe64(pRetrieve->qhandle);
   pRetrieve->free = htons(pRetrieve->free);
 
-  vDebug("vgId:%d, QInfo:%p, retrieve msg is disposed", pVnode->vgId, (void*) pRetrieve->qhandle);
+  vDebug("vgId:%d, QInfo:%p, retrieve msg is disposed, free:%d, conn:%p", pVnode->vgId, (void*) pRetrieve->qhandle, pRetrieve->free, pReadMsg->rpcMsg.handle);
 
   memset(pRet, 0, sizeof(SRspRet));
 
@@ -185,16 +237,8 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
   if (handle == NULL || (*handle) != (void*) pRetrieve->qhandle) {
     code = TSDB_CODE_QRY_INVALID_QHANDLE;
     vDebug("vgId:%d, invalid qhandle in fetch result, QInfo:%p", pVnode->vgId, (void*) pRetrieve->qhandle);
-
-    pRet->rsp = (SRetrieveTableRsp *)rpcMallocCont(sizeof(SRetrieveTableRsp));
-    pRet->len = sizeof(SRetrieveTableRsp);
-
-    memset(pRet->rsp, 0, sizeof(SRetrieveTableRsp));
-    SRetrieveTableRsp* pRsp = pRet->rsp;
-    pRsp->numOfRows = 0;
-    pRsp->useconds  = 0;
-    pRsp->completed = true;
-
+    
+    vnodeBuildNoResultQueryRsp(pRet);
     return code;
   }
 
@@ -203,35 +247,25 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
     qKillQuery(*handle);
     qReleaseQInfo(pVnode->qMgmt, (void**) &handle, true);
 
-    pRet->rsp = (SRetrieveTableRsp *)rpcMallocCont(sizeof(SRetrieveTableRsp));
-    pRet->len = sizeof(SRetrieveTableRsp);
-
-    memset(pRet->rsp, 0, sizeof(SRetrieveTableRsp));
-    SRetrieveTableRsp* pRsp = pRet->rsp;
-    pRsp->numOfRows = 0;
-    pRsp->completed = true;
-    pRsp->useconds  = 0;
-
+    vnodeBuildNoResultQueryRsp(pRet);
     return code;
   }
 
   bool freeHandle = true;
-  code = qRetrieveQueryResultInfo(*handle);
+  bool buildRes   = false;
+
+  code = qRetrieveQueryResultInfo(*handle, &buildRes, pReadMsg->rpcMsg.handle);
   if (code != TSDB_CODE_SUCCESS) {
     //TODO handle malloc failure
     pRet->rsp = (SRetrieveTableRsp *)rpcMallocCont(sizeof(SRetrieveTableRsp));
     memset(pRet->rsp, 0, sizeof(SRetrieveTableRsp));
-  } else { // if failed to dump result, free qhandle immediately
-    if ((code = qDumpRetrieveResult(*handle, (SRetrieveTableRsp **)&pRet->rsp, &pRet->len)) == TSDB_CODE_SUCCESS) {
-      if (qHasMoreResultsToRetrieve(*handle)) {
-        vnodePutItemIntoReadQueue(pVnode, *handle);
-        pRet->qhandle = *handle;
-        freeHandle = false;
-      } else {
-        qKillQuery(*handle);
-        freeHandle = true;
-      }
+  } else { // result is not ready, return immediately
+    if (!buildRes) {
+      qReleaseQInfo(pVnode->qMgmt, (void**) &handle, false);
+      return TSDB_CODE_QRY_NOT_READY;
     }
+
+    code = vnodeDumpQueryResult(pRet, pVnode, *handle, &freeHandle);
   }
 
   qReleaseQInfo(pVnode->qMgmt, (void**) &handle, freeHandle);

src/wal/src/walMain.c

@@ -101,7 +101,7 @@ void *walOpen(const char *path, const SWalCfg *pCfg) {
     }
   }
 
-  if (tmkdir(path, 0755) != 0) {
+  if (taosMkDir(path, 0755) != 0) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     wError("wal:%s, failed to create directory(%s)", path, strerror(errno));
     walRelease(pWal);
@@ -420,7 +420,7 @@ int walHandleExistingFiles(const char *path) {
       if ( strncmp(ent->d_name, walPrefix, plen) == 0) {
         snprintf(oname, sizeof(oname), "%s/%s", path, ent->d_name);
         snprintf(nname, sizeof(nname), "%s/old/%s", path, ent->d_name);
-        if (tmkdir(opath, 0755) != 0) {
+        if (taosMkDir(opath, 0755) != 0) {
           wError("wal:%s, failed to create directory:%s(%s)", oname, opath, strerror(errno));
           terrno = TAOS_SYSTEM_ERROR(errno);
           break; 

tests/script/general/parser/testSuite.sim

@@ -93,6 +93,8 @@ run general/parser/groupby.sim
 sleep 2000
 run general/parser/tags_filter.sim
 sleep 2000
+run general/parser/topbot.sim
+sleep 2000
 run general/parser/union.sim
 sleep 2000
 run general/parser/sliding.sim

tests/script/general/parser/topbot.sim

+system sh/stop_dnodes.sh
+
+system sh/deploy.sh -n dnode1 -i 1
+system sh/cfg.sh -n dnode1 -c walLevel -v 0
+system sh/exec.sh -n dnode1 -s start
+sleep 3000
+sql connect
+
+$dbPrefix = tb_db
+$tbPrefix = tb_tb
+$stbPrefix = tb_stb
+$tbNum = 10
+$rowNum = 1000
+$totalNum = $tbNum * $rowNum
+$loops = 200000
+$log = 10000
+$ts0 = 1537146000000
+$delta = 600000
+print ========== topbot.sim
+$i = 0
+$db = $dbPrefix . $i
+$stb = $stbPrefix . $i
+
+sql drop database $db -x step1
+step1:
+sql create database $db cache 16 maxtables 200
+print ====== create tables
+sql use $db
+sql create table $stb (ts timestamp, c1 int, c2 bigint, c3 float, c4 double, c5 smallint, c6 tinyint, c7 bool, c8 binary(10), c9 nchar(10)) tags(t1 int)
+
+$i = 0
+$ts = $ts0
+$halfNum = $tbNum / 2
+while $i < $halfNum
+  $tbId = $i + $halfNum
+  $tb = $tbPrefix . $i
+  $tb1 = $tbPrefix . $tbId
+  sql create table $tb using $stb tags( $i )
+  sql create table $tb1 using $stb tags( $tbId )
+
+  $x = 0
+  while $x < $rowNum
+    $xs = $x * $delta
+    $ts = $ts0 + $xs
+    $c = $x / 10
+    $c = $c * 10
+    $c = $x - $c
+    $binary = 'binary . $c
+    $binary = $binary . '
+    $nchar = 'nchar . $c
+    $nchar = $nchar . '
+    sql insert into $tb values ( $ts , $c , $c , $c , $c , $c , $c , true, $binary , $nchar )
+    sql insert into $tb1 values ( $ts , $c , NULL , $c , NULL , $c , $c , true, $binary , $nchar )
+    $x = $x + 1
+  endw
+
+  $i = $i + 1
+endw
+print ====== tables created
+
+sql use $db
+##### select from table
+print ====== select top/bot from table and check num of rows returned
+sql select top(c1, 100) from tb_stb0
+if $row != 100 then
+  return -1
+endi
+
+sql select last(c2) from tb_tb9
+if $row != 1 then
+  return -1
+endi
+
+system sh/exec.sh -n dnode1 -s stop -x SIGINT
\ No newline at end of file