[td-10564] Add codes for query parsing.

include/common/common.h

@@ -36,4 +36,32 @@
 //  int16_t bytes;
 //} SSchema;
 
+typedef struct SColumnDataAgg {
+  int16_t colId;
+  int64_t sum;
+  int64_t max;
+  int64_t min;
+  int16_t maxIndex;
+  int16_t minIndex;
+  int16_t numOfNull;
+} SColumnDataAgg;
+
+typedef struct SDataBlockInfo {
+  STimeWindow window;
+  int32_t     rows;
+  int32_t     numOfCols;
+  int64_t     uid;
+} SDataBlockInfo;
+
+typedef struct SSDataBlock {
+  SColumnDataAgg *pBlockAgg;
+  SArray         *pDataBlock;   // SArray<SColumnInfoData>
+  SDataBlockInfo info;
+} SSDataBlock;
+
+typedef struct SColumnInfoData {
+  SColumnInfo info;     // TODO filter info needs to be removed
+  char       *pData;    // the corresponding block data in memory
+} SColumnInfoData;
+
 #endif  // TDENGINE_COMMON_H

include/libs/executor/executor.h

@@ -184,16 +184,6 @@ void** qReleaseQInfo(void* pMgmt, void* pQInfo);
  */
 void** qDeregisterQInfo(void* pMgmt, void* pQInfo);
 
-//======================================================================================================================
-// built-in sql functions
-/**
- * If the given name is a valid built-in sql function, the value of true will be returned.
- * @param name
- * @param len
- * @return
- */
-bool isBuiltinFunction(const char* name, int32_t len);
-
 #ifdef __cplusplus
 }
 #endif

include/libs/function/function.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_TFUNCTION_H
+#define TDENGINE_TFUNCTION_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "common.h"
+
+#define FUNCTION_SCALAR       1
+#define FUNCTION_AGG          2
+
+#define FUNCTIONS_NAME_MAX_LENGTH 16
+
+#define FUNCTION_INVALID_ID  -1
+#define FUNCTION_COUNT        0
+#define FUNCTION_SUM          1
+#define FUNCTION_AVG          2
+#define FUNCTION_MIN          3
+#define FUNCTION_MAX          4
+#define FUNCTION_STDDEV       5
+#define FUNCTION_PERCT        6
+#define FUNCTION_APERCT       7
+#define FUNCTION_FIRST        8
+#define FUNCTION_LAST         9
+#define FUNCTION_LAST_ROW     10
+#define FUNCTION_TOP          11
+#define FUNCTION_BOTTOM       12
+#define FUNCTION_SPREAD       13
+#define FUNCTION_TWA          14
+#define FUNCTION_LEASTSQR     15
+
+#define FUNCTION_TS           16
+#define FUNCTION_TS_DUMMY     17
+#define FUNCTION_TAG_DUMMY    18
+#define FUNCTION_TS_COMP      19
+
+#define FUNCTION_TAG          20
+#define FUNCTION_PRJ          21
+
+#define FUNCTION_TAGPRJ       22
+#define FUNCTION_ARITHM       23
+#define FUNCTION_DIFF         24
+
+#define FUNCTION_FIRST_DST    25
+#define FUNCTION_LAST_DST     26
+#define FUNCTION_STDDEV_DST   27
+#define FUNCTION_INTERP       28
+
+#define FUNCTION_RATE         29
+#define FUNCTION_IRATE        30
+#define FUNCTION_TID_TAG      31
+#define FUNCTION_DERIVATIVE   32
+#define FUNCTION_BLKINFO      33
+
+#define FUNCTION_HISTOGRAM    34
+#define FUNCTION_HLL          35
+#define FUNCTION_MODE         36
+#define FUNCTION_SAMPLE       37
+
+typedef struct SPoint1 {
+  int64_t   key;
+  union{double  val; char* ptr;};
+} SPoint1;
+
+struct SQLFunctionCtx;
+struct SResultRowCellInfo;
+
+//for selectivity query, the corresponding tag value is assigned if the data is qualified
+typedef struct SExtTagsInfo {
+  int16_t                 tagsLen;      // keep the tags data for top/bottom query result
+  int16_t                 numOfTagCols;
+  struct SQLFunctionCtx **pTagCtxList;
+} SExtTagsInfo;
+
+// sql function runtime context
+typedef struct SQLFunctionCtx {
+  int32_t      size;      // number of rows
+  void *       pInput;    // input data buffer
+  uint32_t     order;     // asc|desc
+  int16_t      inputType;
+  int16_t      inputBytes;
+
+  int16_t      outputType;
+  int16_t      outputBytes;   // size of results, determined by function and input column data type
+  int32_t      interBufBytes; // internal buffer size
+  bool         hasNull;       // null value exist in current block
+  bool         requireNull;   // require null in some function
+  bool         stableQuery;
+  int16_t      functionId;    // function id
+  char *       pOutput;       // final result output buffer, point to sdata->data
+  uint8_t      currentStage;  // record current running step, default: 0
+  int64_t      startTs;       // timestamp range of current query when function is executed on a specific data block
+  int32_t      numOfParams;
+  SVariant     param[4];      // input parameter, e.g., top(k, 20), the number of results for top query is kept in param
+  int64_t     *ptsList;       // corresponding timestamp array list
+  void        *ptsOutputBuf;  // corresponding output buffer for timestamp of each result, e.g., top/bottom*/
+  SVariant     tag;
+
+  bool         isSmaSet;
+  SColumnDataAgg sma;
+  struct  SResultRowCellInfo *resultInfo;
+  SExtTagsInfo tagInfo;
+  SPoint1      start;
+  SPoint1      end;
+} SQLFunctionCtx;
+
+typedef struct SAggFunctionInfo {
+  char     name[FUNCTIONS_NAME_MAX_LENGTH];
+  int8_t   type;         // Scalar function or aggregation function
+  uint8_t  functionId;   // Function Id
+  int8_t   sFunctionId;  // Transfer function for super table query
+  uint16_t status;
+
+  bool (*init)(SQLFunctionCtx *pCtx, struct SResultRowCellInfo* pResultCellInfo);  // setup the execute environment
+  void (*exec)(SQLFunctionCtx *pCtx);
+
+  // finalizer must be called after all exec has been executed to generated final result.
+  void (*xFinalize)(SQLFunctionCtx *pCtx);
+  void (*mergeFunc)(SQLFunctionCtx *pCtx);
+
+  int32_t (*dataReqFunc)(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId);
+} SAggFunctionInfo;
+
+typedef struct SScalarFunctionInfo {
+  char     name[FUNCTIONS_NAME_MAX_LENGTH];
+  int8_t   type;              // scalar function or aggregation function
+  uint8_t  functionId;        // index of scalar function
+
+  bool    (*init)(SQLFunctionCtx *pCtx, struct SResultRowCellInfo* pResultCellInfo);  // setup the execute environment
+  void    (*exec)(SQLFunctionCtx *pCtx);
+} SScalarFunctionInfo;
+
+int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionId, int32_t param, int16_t *type,
+                          int16_t *len, int32_t *interBytes, int16_t extLength, bool isSuperTable/*, SUdfInfo* pUdfInfo*/);
+
+/**
+ * If the given name is a valid built-in sql function, the value of true will be returned.
+ * @param name
+ * @param len
+ * @return
+ */
+int32_t qIsBuiltinFunction(const char* name, int32_t len);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TDENGINE_TFUNCTION_H

include/libs/parser/parser.h

@@ -95,10 +95,7 @@ typedef struct STagCond {
 
 typedef struct STableMetaInfo {
   STableMeta    *pTableMeta;      // table meta, cached in client side and acquired by name
-  uint32_t       tableMetaSize;
-//  size_t         tableMetaCapacity;
   SVgroupsInfo  *vgroupList;
-  SArray        *pVgroupTables;   // SArray<SVgroupTableInfo>
 
   /*
    * 1. keep the vgroup index during the multi-vnode super table projection query
@@ -110,6 +107,20 @@ typedef struct STableMetaInfo {
   SArray       *tagColList;                        // SArray<SColumn*>, involved tag columns
 } STableMetaInfo;
 
+typedef struct SQueryType {
+  bool             stableQuery;
+  bool             groupbyColumn;
+  bool             simpleAgg;
+  bool             arithmeticOnAgg;
+  bool             projectionQuery;
+  bool             hasFilter;
+  bool             onlyTagQuery;
+  bool             orderProjectQuery;
+  bool             stateWindow;
+  bool             globalMerge;
+  bool             multigroupResult;
+} SQueryType;
+
 typedef struct SQueryStmtInfo {
   int16_t          command;       // the command may be different for each subclause, so keep it seperately.
   uint32_t         type;          // query/insert type
@@ -152,17 +163,6 @@ typedef struct SQueryStmtInfo {
   SArray            *pUpstream;   // SArray<struct SQueryStmtInfo>
   struct SQueryStmtInfo *pDownstream;
   int32_t            havingFieldNum;
-  bool               stableQuery;
-  bool               groupbyColumn;
-  bool               simpleAgg;
-  bool               arithmeticOnAgg;
-  bool               projectionQuery;
-  bool               hasFilter;
-  bool               onlyTagQuery;
-  bool               orderProjectQuery;
-  bool               stateWindow;
-  bool               globalMerge;
-  bool               multigroupResult;
 } SQueryStmtInfo;
 
 struct SInsertStmtInfo;

include/os/os.h

@@ -44,6 +44,7 @@ extern "C" {
 #include <errno.h>
 #include <float.h>
 #include <math.h>
+#include <sys/stat.h>
 
 #include "osAtomic.h"
 #include "osDef.h"

include/util/tpagedfile.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_TPAGEDFILE_H
+#define TDENGINE_TPAGEDFILE_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "tlist.h"
+#include "thash.h"
+#include "os.h"
+#include "tlockfree.h"
+
+typedef struct SArray* SIDList;
+
+typedef struct SPageDiskInfo {
+  int32_t offset;
+  int32_t length;
+} SPageDiskInfo;
+
+typedef struct SPageInfo {
+  SListNode*    pn;       // point to list node
+  int32_t       pageId;
+  SPageDiskInfo info;
+  void*         pData;
+  bool          used;     // set current page is in used
+} SPageInfo;
+
+typedef struct SFreeListItem {
+  int32_t offset;
+  int32_t len;
+} SFreeListItem;
+
+typedef struct SResultBufStatis {
+  int32_t flushBytes;
+  int32_t loadBytes;
+  int32_t getPages;
+  int32_t releasePages;
+  int32_t flushPages;
+} SResultBufStatis;
+
+typedef struct SDiskbasedResultBuf {
+  int32_t   numOfPages;
+  int64_t   totalBufSize;
+  int64_t   fileSize;            // disk file size
+  FILE*     file;
+  int32_t   allocateId;          // allocated page id
+  char*     path;                // file path
+  int32_t   pageSize;            // current used page size
+  int32_t   inMemPages;          // numOfPages that are allocated in memory
+  SHashObj* groupSet;            // id hash table
+  SHashObj* all;
+  SList*    lruList;
+  void*     emptyDummyIdList;    // dummy id list
+  void*     assistBuf;           // assistant buffer for compress/decompress data
+  SArray*   pFree;               // free area in file
+  bool      comp;                // compressed before flushed to disk
+  int32_t   nextPos;             // next page flush position
+
+  uint64_t  qId;                 // for debug purpose
+  SResultBufStatis statis;
+} SDiskbasedResultBuf;
+
+#define DEFAULT_INTERN_BUF_PAGE_SIZE  (1024L)                          // in bytes
+#define PAGE_INFO_INITIALIZER         (SPageDiskInfo){-1, -1}
+#define DEFAULT_PAGE_SIZE             (16384L)
+
+typedef struct SFilePage {
+  int64_t num;
+  char    data[];
+} SFilePage;
+
+/**
+ * create disk-based result buffer
+ * @param pResultBuf
+ * @param rowSize
+ * @param pagesize
+ * @param inMemPages
+ * @param handle
+ * @return
+ */
+int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t pagesize, int32_t inMemBufSize, uint64_t qId);
+
+/**
+ *
+ * @param pResultBuf
+ * @param groupId
+ * @param pageId
+ * @return
+ */
+SFilePage* getNewDataBuf(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int32_t* pageId);
+
+/**
+ *
+ * @param pResultBuf
+ * @param groupId
+ * @return
+ */
+SIDList getDataBufPagesIdList(SDiskbasedResultBuf* pResultBuf, int32_t groupId);
+
+/**
+ * get the specified buffer page by id
+ * @param pResultBuf
+ * @param id
+ * @return
+ */
+tFilePage* getResBufPage(SDiskbasedResultBuf* pResultBuf, int32_t id);
+
+/**
+ * release the referenced buf pages
+ * @param pResultBuf
+ * @param page
+ */
+void releaseResBufPage(SDiskbasedResultBuf* pResultBuf, void* page);
+
+/**
+ *
+ * @param pResultBuf
+ * @param pi
+ */
+void releaseResBufPageInfo(SDiskbasedResultBuf* pResultBuf, SPageInfo* pi);
+
+
+/**
+ * get the total buffer size in the format of disk file
+ * @param pResultBuf
+ * @return
+ */
+size_t getResBufSize(const SDiskbasedResultBuf* pResultBuf);
+
+/**
+ * get the number of groups in the result buffer
+ * @param pResultBuf
+ * @return
+ */
+size_t getNumOfResultBufGroupId(const SDiskbasedResultBuf* pResultBuf);
+
+/**
+ * destroy result buffer
+ * @param pResultBuf
+ */
+void destroyResultBuf(SDiskbasedResultBuf* pResultBuf);
+
+/**
+ *
+ * @param pList
+ * @return
+ */
+SPageInfo* getLastPageInfo(SIDList pList);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TDENGINE_TPAGEDFILE_H

source/libs/CMakeLists.txt

@@ -9,3 +9,4 @@ add_subdirectory(lru)
 add_subdirectory(catalog)
 add_subdirectory(executor)
 add_subdirectory(planner)
+add_subdirectory(function)
\ No newline at end of file

source/libs/function/CMakeLists.txt

+aux_source_directory(src FUNCTION_SRC)
+add_library(function ${FUNCTION_SRC})
+target_include_directories(
+    function
+    PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/function"
+    PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
+)
+
+target_link_libraries(
+        function
+        PRIVATE os util common
+)
\ No newline at end of file

source/libs/function/inc/taggfunction.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_TAGGFUNCTION_H
+#define TDENGINE_TAGGFUNCTION_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "os.h"
+
+#include "tname.h"
+#include "taosdef.h"
+#include "tvariant.h"
+#include "function.h"
+#include "tudf.h"
+
+extern SAggFunctionInfo aggFunc[34];
+
+typedef struct SResultRowCellInfo {
+  int8_t   hasResult;       // result generated, not NULL value
+  bool     initialized;     // output buffer has been initialized
+  bool     complete;        // query has completed
+  uint32_t numOfRes;        // num of output result in current buffer
+} SResultRowCellInfo;
+
+#define FUNCSTATE_SO           0x0u
+#define FUNCSTATE_MO           0x1u    // dynamic number of output, not multinumber of output e.g., TOP/BOTTOM
+#define FUNCSTATE_STREAM       0x2u    // function avail for stream
+#define FUNCSTATE_STABLE       0x4u    // function avail for super table
+#define FUNCSTATE_NEED_TS      0x8u    // timestamp is required during query processing
+#define FUNCSTATE_SELECTIVITY  0x10u   // selectivity functions, can exists along with tag columns
+
+#define BASIC_FUNC_SO FUNCSTATE_SO | FUNCSTATE_STREAM | FUNCSTATE_STABLE
+#define BASIC_FUNC_MO FUNCSTATE_MO | FUNCSTATE_STREAM | FUNCSTATE_STABLE
+
+#define AVG_FUNCTION_INTER_BUFFER_SIZE 50
+
+#define DATA_SET_FLAG ','  // to denote the output area has data, not null value
+#define DATA_SET_FLAG_SIZE sizeof(DATA_SET_FLAG)
+
+#define QUERY_ASC_FORWARD_STEP   1
+#define QUERY_DESC_FORWARD_STEP -1
+
+#define GET_FORWARD_DIRECTION_FACTOR(ord) (((ord) == TSDB_ORDER_ASC) ? QUERY_ASC_FORWARD_STEP : QUERY_DESC_FORWARD_STEP)
+
+#define MAX_INTERVAL_TIME_WINDOW 1000000  // maximum allowed time windows in final results
+#define TOP_BOTTOM_QUERY_LIMIT   100
+
+enum {
+  MASTER_SCAN   = 0x0u,
+  REVERSE_SCAN  = 0x1u,
+  REPEAT_SCAN   = 0x2u,  //repeat scan belongs to the master scan
+  MERGE_STAGE   = 0x20u,
+};
+
+#define QUERY_IS_STABLE_QUERY(type)      (((type)&TSDB_QUERY_TYPE_STABLE_QUERY) != 0)
+#define QUERY_IS_JOIN_QUERY(type)        (TSDB_QUERY_HAS_TYPE(type, TSDB_QUERY_TYPE_JOIN_QUERY))
+#define QUERY_IS_PROJECTION_QUERY(type)  (((type)&TSDB_QUERY_TYPE_PROJECTION_QUERY) != 0)
+#define QUERY_IS_FREE_RESOURCE(type)     (((type)&TSDB_QUERY_TYPE_FREE_RESOURCE) != 0)
+
+typedef struct SArithmeticSupport {
+  SExprInfo   *pExprInfo;
+  int32_t      numOfCols;
+  SColumnInfo *colList;
+  void        *exprList;   // client side used
+  int32_t      offset;
+  char**       data;
+} SArithmeticSupport;
+
+typedef struct SInterpInfoDetail {
+  TSKEY  ts;  // interp specified timestamp
+  int8_t type;
+  int8_t primaryCol;
+} SInterpInfoDetail;
+
+#define GET_ROWCELL_INTERBUF(_c) ((void*) ((char*)(_c) + sizeof(SResultRowCellInfo)))
+
+#define GET_RES_INFO(ctx) ((ctx)->resultInfo)
+
+#define IS_STREAM_QUERY_VALID(x)  (((x)&TSDB_FUNCSTATE_STREAM) != 0)
+#define IS_MULTIOUTPUT(x)         (((x)&TSDB_FUNCSTATE_MO) != 0)
+
+// determine the real data need to calculated the result
+enum {
+  BLK_DATA_NO_NEEDED     = 0x0,
+  BLK_DATA_STATIS_NEEDED = 0x1,
+  BLK_DATA_ALL_NEEDED    = 0x3,
+  BLK_DATA_DISCARD       = 0x4,   // discard current data block since it is not qualified for filter
+};
+
+typedef struct STwaInfo {
+  int8_t      hasResult;  // flag to denote has value
+  double      dOutput;
+  SPoint1     p;
+  STimeWindow win;
+} STwaInfo;
+
+extern int32_t functionCompatList[]; // compatible check array list
+
+bool topbot_datablock_filter(SQLFunctionCtx *pCtx, const char *minval, const char *maxval);
+
+/**
+ * the numOfRes should be kept, since it may be used later
+ * and allow the ResultInfo to be re initialized
+ */
+#define RESET_RESULT_INFO(_r)  \
+  do {                         \
+    (_r)->initialized = false; \
+  } while (0)
+
+static FORCE_INLINE void initResultInfo(SResultRowCellInfo *pResInfo, int32_t bufLen) {
+  pResInfo->initialized = true;  // the this struct has been initialized flag
+  
+  pResInfo->complete  = false;
+  pResInfo->hasResult = false;
+  pResInfo->numOfRes  = 0;
+  
+  memset(GET_ROWCELL_INTERBUF(pResInfo), 0, bufLen);
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TDENGINE_TAGGFUNCTION_H

source/libs/function/inc/tfill.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_TFILL_H
+#define TDENGINE_TFILL_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "os.h"
+#include "taosdef.h"
+
+struct SSDataBlock;
+
+typedef struct {
+  STColumn col;             // column info
+  int16_t  functionId;      // sql function id
+  int16_t  flag;            // column flag: TAG COLUMN|NORMAL COLUMN
+  int16_t  tagIndex;        // index of current tag in SFillTagColInfo array list
+  union {int64_t i; double d;} fillVal;
+} SFillColInfo;
+
+typedef struct {
+  SSchema col;
+  char*   tagVal;
+} SFillTagColInfo;
+  
+typedef struct SFillInfo {
+  TSKEY     start;                // start timestamp
+  TSKEY     end;                  // endKey for fill
+  TSKEY     currentKey;           // current active timestamp, the value may be changed during the fill procedure.
+  int32_t   order;                // order [TSDB_ORDER_ASC|TSDB_ORDER_DESC]
+  int32_t   type;                 // fill type
+  int32_t   numOfRows;            // number of rows in the input data block
+  int32_t   index;                // active row index
+  int32_t   numOfTotal;           // number of filled rows in one round
+  int32_t   numOfCurrent;         // number of filled rows in current results
+
+  int32_t   numOfTags;            // number of tags
+  int32_t   numOfCols;            // number of columns, including the tags columns
+  int32_t   rowSize;              // size of each row
+  SInterval interval;
+  char *    prevValues;           // previous row of data, to generate the interpolation results
+  char *    nextValues;           // next row of data
+  char**    pData;                // original result data block involved in filling data
+  int32_t   alloc;                // data buffer size in rows
+  int8_t    precision;            // time resoluation
+
+  SFillColInfo* pFillCol;         // column info for fill operations
+  SFillTagColInfo* pTags;         // tags value for filling gap
+  void*     handle;               // for debug purpose
+} SFillInfo;
+
+typedef struct SPoint {
+  int64_t key;
+  void *  val;
+} SPoint;
+
+SFillInfo* taosCreateFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_t capacity, int32_t numOfCols,
+                            int64_t slidingTime, int8_t slidingUnit, int8_t precision, int32_t fillType,
+                            SFillColInfo* pFillCol, void* handle);
+
+void taosResetFillInfo(SFillInfo* pFillInfo, TSKEY startTimestamp);
+
+void* taosDestroyFillInfo(SFillInfo *pFillInfo);
+
+void taosFillSetStartInfo(SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey);
+
+void taosFillSetInputDataBlock(SFillInfo* pFillInfo, const struct SSDataBlock* pInput);
+
+bool taosFillHasMoreResults(SFillInfo* pFillInfo);
+
+int64_t getNumOfResultsAfterFillGap(SFillInfo* pFillInfo, int64_t ekey, int32_t maxNumOfRows);
+
+int32_t taosGetLinearInterpolationVal(SPoint* point, int32_t outputType, SPoint* point1, SPoint* point2, int32_t inputType);
+
+int64_t taosFillResultDataBlock(SFillInfo* pFillInfo, void** output, int32_t capacity);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TDENGINE_TFILL_H

source/libs/function/inc/thistogram.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_HISTOGRAM_H
+#define TDENGINE_HISTOGRAM_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define USE_ARRAYLIST
+
+#define MAX_HISTOGRAM_BIN 500
+
+typedef struct SHistBin {
+  double  val;
+  int64_t num;
+
+#if !defined(USE_ARRAYLIST)
+  double  delta;
+  int32_t index;  // index in min-heap list
+#endif
+} SHistBin;
+
+typedef struct SHeapEntry {
+  void*  pData;
+  double val;
+} SHeapEntry;
+
+typedef struct SHistogramInfo {
+  int64_t numOfElems;
+  int32_t numOfEntries;
+  int32_t maxEntries;
+  double min;
+  double max;
+#if defined(USE_ARRAYLIST)
+  SHistBin* elems;
+#else
+  tSkipList*      pList;
+  SLoserTreeInfo* pLoserTree;
+  int32_t         maxIndex;
+  bool            ordered;
+#endif
+} SHistogramInfo;
+
+SHistogramInfo* tHistogramCreate(int32_t numOfBins);
+SHistogramInfo* tHistogramCreateFrom(void* pBuf, int32_t numOfBins);
+
+int32_t tHistogramAdd(SHistogramInfo** pHisto, double val);
+int64_t tHistogramSum(SHistogramInfo* pHisto, double v);
+
+double* tHistogramUniform(SHistogramInfo* pHisto, double* ratio, int32_t num);
+SHistogramInfo* tHistogramMerge(SHistogramInfo* pHisto1, SHistogramInfo* pHisto2, int32_t numOfEntries);
+void tHistogramDestroy(SHistogramInfo** pHisto);
+
+void tHistogramPrint(SHistogramInfo* pHisto);
+
+int32_t histoBinarySearch(SHistBin* pEntry, int32_t len, double val);
+
+SHeapEntry* tHeapCreate(int32_t numOfEntries);
+void tHeapSort(SHeapEntry* pEntry, int32_t len);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TDENGINE_HISTOGRAM_H

source/libs/function/inc/tpercentile.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_TPERCENTILE_H
+#define TDENGINE_TPERCENTILE_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "qExtbuffer.h"
+#include "qResultbuf.h"
+#include "ttszip.h"
+
+typedef struct MinMaxEntry {
+  union {
+    double   dMinVal;
+    int64_t  i64MinVal;
+    uint64_t u64MinVal;
+  };
+  union {
+    double  dMaxVal;
+    int64_t i64MaxVal;
+    int64_t u64MaxVal;
+  };
+} MinMaxEntry;
+
+typedef struct {
+  int32_t    size;
+  int32_t    pageId;
+  tFilePage *data;
+} SSlotInfo;
+
+typedef struct tMemBucketSlot {
+  SSlotInfo   info;
+  MinMaxEntry range;
+} tMemBucketSlot;
+
+struct tMemBucket;
+typedef int32_t (*__perc_hash_func_t)(struct tMemBucket *pBucket, const void *value);
+
+typedef struct tMemBucket {
+  int16_t       numOfSlots;
+  int16_t       type;
+  int16_t       bytes;
+  int32_t       total;
+  int32_t       elemPerPage;  // number of elements for each object
+  int32_t       maxCapacity;  // maximum allowed number of elements that can be sort directly to get the result
+  int32_t       bufPageSize;  // disk page size
+  MinMaxEntry   range;        // value range
+  int32_t       times;        // count that has been checked for deciding the correct data value buckets.
+  __compar_fn_t comparFn;
+
+  tMemBucketSlot *     pSlots;
+  SDiskbasedResultBuf *pBuffer;
+  __perc_hash_func_t   hashFunc;
+} tMemBucket;
+
+tMemBucket *tMemBucketCreate(int16_t nElemSize, int16_t dataType, double minval, double maxval);
+
+void tMemBucketDestroy(tMemBucket *pBucket);
+
+int32_t tMemBucketPut(tMemBucket *pBucket, const void *data, size_t size);
+
+double getPercentile(tMemBucket *pMemBucket, double percent);
+
+#endif  // TDENGINE_TPERCENTILE_H
+
+#ifdef __cplusplus
+}
+#endif
\ No newline at end of file

source/libs/function/inc/tscalarfunction.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+#ifndef TDENGINE_TSCALARFUNCTION_H
+#define TDENGINE_TSCALARFUNCTION_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "function.h"
+
+extern struct SScalarFunctionInfo scalarFunc[1];
+
+#define FUNCTION_CEIL         38
+#define FUNCTION_FLOOR        39
+#define FUNCTION_ROUND        40
+#define FUNCTION_MAVG         41
+#define FUNCTION_CSUM         42
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TDENGINE_TSCALARFUNCTION_H

source/libs/function/inc/ttszip.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_TTSZIP_H
+#define TDENGINE_TTSZIP_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "os.h"
+#include "taosdef.h"
+#include "tvariant.h"
+
+#define MEM_BUF_SIZE (1 << 20)
+#define TS_COMP_FILE_MAGIC 0x87F5EC4C
+#define TS_COMP_FILE_GROUP_MAX 512
+
+typedef struct STSList {
+  char*   rawBuf;
+  int32_t allocSize;
+  int32_t threshold;
+  int32_t len;
+} STSList;
+
+typedef struct STSElem {
+  TSKEY     ts;
+  tVariant* tag;
+  int32_t   id;
+} STSElem;
+
+typedef struct STSCursor {
+  int32_t  vgroupIndex;
+  int32_t  blockIndex;
+  int32_t  tsIndex;
+  uint32_t order;
+} STSCursor;
+
+typedef struct STSBlock {
+  tVariant tag;        // tag value
+  int32_t  numOfElem;  // number of elements
+  int32_t  compLen;    // size after compressed
+  int32_t  padding;    // 0xFFFFFFFF by default, after the payload
+  char*    payload;    // actual data that is compressed
+} STSBlock;
+
+/*
+ * The size of buffer file should not be greater than 2G,
+ * and the offset of int32_t type is enough
+ */
+typedef struct STSGroupBlockInfo {
+  int32_t id;          // group id
+  int32_t offset;      // offset set value in file
+  int32_t numOfBlocks; // number of total blocks
+  int32_t compLen;     // compressed size
+} STSGroupBlockInfo;
+
+typedef struct STSGroupBlockInfoEx {
+  STSGroupBlockInfo info;
+  int32_t           len;  // length before compress
+} STSGroupBlockInfoEx;
+
+typedef struct STSBuf {
+  FILE*    f;
+  char     path[PATH_MAX];
+  uint32_t fileSize;
+
+  // todo use array
+  STSGroupBlockInfoEx* pData;
+  uint32_t             numOfAlloc;
+  uint32_t             numOfGroups;
+
+  char*     assistBuf;
+  int32_t   bufSize;
+  STSBlock  block;
+  STSList   tsData;  // uncompressed raw ts data
+  uint64_t  numOfTotal;
+  bool      autoDelete;
+  bool      remainOpen;
+  int32_t   tsOrder;  // order of timestamp in ts comp buffer
+  STSCursor cur;
+} STSBuf;
+
+typedef struct STSBufFileHeader {
+  uint32_t magic;       // file magic number
+  uint32_t numOfGroup;  // number of group stored in current file
+  int32_t  tsOrder;     // timestamp order in current file
+} STSBufFileHeader;
+
+STSBuf* tsBufCreate(bool autoDelete, int32_t order);
+STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete);
+STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_t len, int32_t tsOrder, int32_t id);
+
+void* tsBufDestroy(STSBuf* pTSBuf);
+
+void    tsBufAppend(STSBuf* pTSBuf, int32_t id, tVariant* tag, const char* pData, int32_t len);
+int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf);
+
+STSBuf* tsBufClone(STSBuf* pTSBuf);
+
+STSGroupBlockInfo* tsBufGetGroupBlockInfo(STSBuf* pTSBuf, int32_t id);
+
+void    tsBufFlush(STSBuf* pTSBuf);
+void    tsBufResetPos(STSBuf* pTSBuf);
+bool    tsBufNextPos(STSBuf* pTSBuf);
+
+STSElem tsBufGetElem(STSBuf* pTSBuf);
+STSElem tsBufGetElemStartPos(STSBuf* pTSBuf, int32_t id, tVariant* tag);
+
+STSCursor tsBufGetCursor(STSBuf* pTSBuf);
+void      tsBufSetTraverseOrder(STSBuf* pTSBuf, int32_t order);
+
+void tsBufSetCursor(STSBuf* pTSBuf, STSCursor* pCur);
+
+/**
+ * display all data in comp block file, for debug purpose only
+ * @param pTSBuf
+ */
+void tsBufDisplay(STSBuf* pTSBuf);
+
+int32_t tsBufGetNumOfGroup(STSBuf* pTSBuf);
+
+void tsBufGetGroupIdList(STSBuf* pTSBuf, int32_t* num, int32_t** id);
+
+int32_t dumpFileBlockByGroupId(STSBuf* pTSBuf, int32_t id, void* buf, int32_t* len, int32_t* numOfBlocks);
+
+STSElem tsBufFindElemStartPosByTag(STSBuf* pTSBuf, tVariant* pTag);
+
+bool tsBufIsValidElem(STSElem* pElem);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TDENGINE_TTSZIP_H

source/libs/function/inc/tudf.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_TUDF_H
+#define TDENGINE_TUDF_H
+
+enum {
+  TSDB_UDF_FUNC_NORMAL = 0,
+  TSDB_UDF_FUNC_INIT,
+  TSDB_UDF_FUNC_FINALIZE,
+  TSDB_UDF_FUNC_MERGE,
+  TSDB_UDF_FUNC_DESTROY,
+  TSDB_UDF_FUNC_MAX_NUM
+};
+
+typedef struct SUdfInit {
+  int32_t  maybe_null; /* 1 if function can return NULL */
+  uint32_t decimals;   /* for real functions */
+  uint64_t length;     /* For string functions */
+  char*    ptr;        /* free pointer for function data */
+  int32_t  const_item; /* 0 if result is independent of arguments */
+
+  // script like lua/javascript
+  void* script_ctx;
+  void (*destroyCtxFunc)(void* script_ctx);
+} SUdfInit;
+
+typedef struct SUdfInfo {
+  int32_t functionId;                    // system assigned function id
+  int32_t funcType;                      // scalar function or aggregate function
+  int8_t  resType;                       // result type
+  int16_t resBytes;                      // result byte
+  int32_t contLen;                       // content length
+  int32_t bufSize;                       // interbuf size
+  char*   name;                          // function name
+  void*   handle;                        // handle loaded in mem
+  void*   funcs[TSDB_UDF_FUNC_MAX_NUM];  // function ptr
+
+  // for script like lua/javascript only
+  int   isScript;
+  void* pScriptCtx;
+
+  SUdfInit init;
+  char*    content;
+  char*    path;
+} SUdfInfo;
+
+// script
+
+typedef int32_t (*scriptInitFunc)(void* pCtx);
+typedef void (*scriptNormalFunc)(void* pCtx, char* data, int16_t iType, int16_t iBytes, int32_t numOfRows,
+                                 int64_t* ptList, int64_t key, char* dataOutput, char* tsOutput, int32_t* numOfOutput,
+                                 int16_t oType, int16_t oBytes);
+typedef void (*scriptFinalizeFunc)(void* pCtx, int64_t key, char* dataOutput, int32_t* numOfOutput);
+typedef void (*scriptMergeFunc)(void* pCtx, char* data, int32_t numOfRows, char* dataOutput, int32_t* numOfOutput);
+typedef void (*scriptDestroyFunc)(void* pCtx);
+
+// dynamic lib
+typedef void (*udfNormalFunc)(char* data, int16_t itype, int16_t iBytes, int32_t numOfRows, int64_t* ts,
+                              char* dataOutput, char* interBuf, char* tsOutput, int32_t* numOfOutput, int16_t oType,
+                              int16_t oBytes, SUdfInit* buf);
+typedef int32_t (*udfInitFunc)(SUdfInit* data);
+typedef void (*udfFinalizeFunc)(char* dataOutput, char* interBuf, int32_t* numOfOutput, SUdfInit* buf);
+typedef void (*udfMergeFunc)(char* data, int32_t numOfRows, char* dataOutput, int32_t* numOfOutput, SUdfInit* buf);
+typedef void (*udfDestroyFunc)(SUdfInit* buf);
+
+#endif  // TDENGINE_TUDF_H

source/libs/function/src/taggfunction.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "os.h"
+#include "taosdef.h"
+#include "taosmsg.h"
+//#include "texpr.h"
+#include "ttypes.h"
+#include "tglobal.h"
+#include "thash.h"
+
+#include "taggfunction.h"
+#include "tfill.h"
+#include "thistogram.h"
+#include "ttszip.h"
+#include "tpercentile.h"
+#include "tbuffer.h"
+#include "tcompression.h"
+//#include "queryLog.h"
+#include "tudf.h"
+
+#define GET_INPUT_DATA_LIST(x) ((char *)((x)->pInput))
+#define GET_INPUT_DATA(x, y) (GET_INPUT_DATA_LIST(x) + (y) * (x)->inputBytes)
+
+#define GET_TS_LIST(x)    ((TSKEY*)((x)->ptsList))
+#define GET_TS_DATA(x, y) (GET_TS_LIST(x)[(y)])
+
+#define GET_TRUE_DATA_TYPE()                          \
+  int32_t type = 0;                                   \
+  if (pCtx->currentStage == MERGE_STAGE) {            \
+    type = pCtx->outputType;                          \
+    assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); \
+  } else {                                            \
+    type = pCtx->inputType;                           \
+  }
+
+#define SET_VAL(ctx, numOfElem, res)     \
+  do {                                   \
+    if ((numOfElem) <= 0) {              \
+      break;                             \
+    }                                    \
+    GET_RES_INFO(ctx)->numOfRes = (res); \
+  } while (0)
+
+#define INC_INIT_VAL(ctx, res) (GET_RES_INFO(ctx)->numOfRes += (res));
+
+#define DO_UPDATE_TAG_COLUMNS(ctx, ts)                             \
+  do {                                                             \
+    for (int32_t _i = 0; _i < (ctx)->tagInfo.numOfTagCols; ++_i) { \
+      SQLFunctionCtx *__ctx = (ctx)->tagInfo.pTagCtxList[_i];      \
+      if (__ctx->functionId == FUNCTION_TS_DUMMY) {                \
+        __ctx->tag.i64 = (ts);                                     \
+        __ctx->tag.nType = TSDB_DATA_TYPE_BIGINT;                  \
+      }                                                            \
+      aggFunc[FUNCTION_TAG].exec(__ctx);                             \
+    }                                                              \
+  } while (0)
+
+#define DO_UPDATE_TAG_COLUMNS_WITHOUT_TS(ctx)                      \
+  do {                                                             \
+    for (int32_t _i = 0; _i < (ctx)->tagInfo.numOfTagCols; ++_i) { \
+      SQLFunctionCtx *__ctx = (ctx)->tagInfo.pTagCtxList[_i];      \
+      aggFunc[FUNCTION_TAG].exec(__ctx);                             \
+    }                                                              \
+  } while (0);
+
+void noop1(SQLFunctionCtx *UNUSED_PARAM(pCtx)) {}
+
+void doFinalizer(SQLFunctionCtx *pCtx) { RESET_RESULT_INFO(GET_RES_INFO(pCtx)); }
+
+typedef struct tValuePair {
+  SVariant v;
+  int64_t  timestamp;
+  char *   pTags;  // the corresponding tags of each record in the final result
+} tValuePair;
+
+typedef struct SSpreadInfo {
+  double min;
+  double max;
+  int8_t hasResult;
+} SSpreadInfo;
+
+typedef struct SSumInfo {
+  union {
+    int64_t  isum;
+    uint64_t usum;
+    double   dsum;
+  };
+  int8_t hasResult;
+} SSumInfo;
+
+// the attribute of hasResult is not needed since the num attribute would server as this purpose
+typedef struct SAvgInfo {
+  double  sum;
+  int64_t num;
+} SAvgInfo;
+
+typedef struct SStddevInfo {
+  double  avg;
+  int64_t num;
+  double  res;
+  int8_t  stage;
+} SStddevInfo;
+
+typedef struct SStddevdstInfo {
+  int64_t num;
+  double  res;
+} SStddevdstInfo;
+
+typedef struct SFirstLastInfo {
+  int8_t hasResult;
+  TSKEY  ts;
+} SFirstLastInfo;
+
+typedef struct SFirstLastInfo SLastrowInfo;
+typedef struct SPercentileInfo {
+  tMemBucket *pMemBucket;
+  int32_t     stage;
+  double      minval;
+  double      maxval;
+  int64_t     numOfElems;
+} SPercentileInfo;
+
+typedef struct STopBotInfo {
+  int32_t      num;
+  tValuePair **res;
+} STopBotInfo;
+
+// leastsquares do not apply to super table
+typedef struct SLeastsquaresInfo {
+  double  mat[2][3];
+  double  startVal;
+  int64_t num;
+} SLeastsquaresInfo;
+
+typedef struct SAPercentileInfo {
+  SHistogramInfo *pHisto;
+} SAPercentileInfo;
+
+typedef struct STSCompInfo {
+  STSBuf *pTSBuf;
+} STSCompInfo;
+
+typedef struct SRateInfo {
+  double  correctionValue;
+  double  firstValue;
+  TSKEY   firstKey;
+  double  lastValue;
+  TSKEY   lastKey;
+  int8_t  hasResult;  // flag to denote has value
+  bool    isIRate;    // true for IRate functions, false for Rate functions
+} SRateInfo;
+
+typedef struct SDerivInfo {
+  double   prevValue;     // previous value
+  TSKEY    prevTs;        // previous timestamp
+  bool     ignoreNegative;// ignore the negative value
+  int64_t  tsWindow;      // time window for derivative
+  bool     valueSet;      // the value has been set already
+} SDerivInfo;
+
+typedef struct SResPair {
+  TSKEY  key;
+  double avg;
+} SResPair;
+
+#define TSDB_BLOCK_DIST_STEP_ROWS 16
+
+typedef struct STableBlockDist {
+  uint16_t  rowSize;
+  uint16_t  numOfFiles;
+  uint32_t  numOfTables;
+  uint64_t  totalSize;
+  uint64_t  totalRows;
+  int32_t   maxRows;
+  int32_t   minRows;
+  int32_t   firstSeekTimeUs;
+  uint32_t  numOfRowsInMemTable;
+  uint32_t  numOfSmallBlocks;
+  SArray   *dataBlockInfos;
+} STableBlockDist;
+
+typedef struct SFileBlockInfo {
+  int32_t numBlocksOfStep;
+} SFileBlockInfo;
+
+int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionId, int32_t param, int16_t *type,
+                          int16_t *bytes, int32_t *interBytes, int16_t extLength, bool isSuperTable/*, SUdfInfo* pUdfInfo*/) {
+  if (!isValidDataType(dataType)) {
+//    qError("Illegal data type %d or data type length %d", dataType, dataBytes);
+    return TSDB_CODE_TSC_INVALID_OPERATION;
+  }
+
+
+  if (functionId == FUNCTION_TS || functionId == FUNCTION_TS_DUMMY || functionId == FUNCTION_TAG_DUMMY ||
+      functionId == FUNCTION_DIFF || functionId == FUNCTION_PRJ || functionId == FUNCTION_TAGPRJ ||
+      functionId == FUNCTION_TAG || functionId == FUNCTION_INTERP) {
+    *type = (int16_t)dataType;
+    *bytes = (int16_t)dataBytes;
+
+    if (functionId == FUNCTION_INTERP) {
+      *interBytes = sizeof(SInterpInfoDetail);
+    } else {
+      *interBytes = 0;
+    }
+
+    return TSDB_CODE_SUCCESS;
+  }
+  
+  // (uid, tid) + VGID + TAGSIZE + VARSTR_HEADER_SIZE
+  if (functionId == FUNCTION_TID_TAG) { // todo use struct
+    *type = TSDB_DATA_TYPE_BINARY;
+    *bytes = (int16_t)(dataBytes + sizeof(int16_t) + sizeof(int64_t) + sizeof(int32_t) + sizeof(int32_t) + VARSTR_HEADER_SIZE);
+    *interBytes = 0;
+    return TSDB_CODE_SUCCESS;
+  }
+
+  if (functionId == FUNCTION_BLKINFO) {
+    *type = TSDB_DATA_TYPE_BINARY;
+    *bytes = 16384;
+    *interBytes = 0;
+    return TSDB_CODE_SUCCESS;
+  }
+  
+  if (functionId == FUNCTION_COUNT) {
+    *type = TSDB_DATA_TYPE_BIGINT;
+    *bytes = sizeof(int64_t);
+    *interBytes = 0;
+    return TSDB_CODE_SUCCESS;
+  }
+  
+  if (functionId == FUNCTION_ARITHM) {
+    *type = TSDB_DATA_TYPE_DOUBLE;
+    *bytes = sizeof(double);
+    *interBytes = 0;
+    return TSDB_CODE_SUCCESS;
+  }
+  
+  if (functionId == FUNCTION_TS_COMP) {
+    *type = TSDB_DATA_TYPE_BINARY;
+    *bytes = 1;  // this results is compressed ts data, only one byte
+    *interBytes = POINTER_BYTES;
+    return TSDB_CODE_SUCCESS;
+  }
+
+  if (functionId == FUNCTION_DERIVATIVE) {
+    *type = TSDB_DATA_TYPE_DOUBLE;
+    *bytes = sizeof(double);  // this results is compressed ts data, only one byte
+    *interBytes = sizeof(SDerivInfo);
+    return TSDB_CODE_SUCCESS;
+  }
+
+  if (isSuperTable) {
+//    if (functionId < 0) {
+//      if (pUdfInfo->bufSize > 0) {
+//        *type = TSDB_DATA_TYPE_BINARY;
+//        *bytes = pUdfInfo->bufSize;
+//        *interBytes = *bytes;
+//      } else {
+//        *type = pUdfInfo->resType;
+//        *bytes = pUdfInfo->resBytes;
+//        *interBytes = *bytes;
+//      }
+//
+//      return TSDB_CODE_SUCCESS;
+//    }
+
+    if (functionId == FUNCTION_MIN || functionId == FUNCTION_MAX) {
+      *type = TSDB_DATA_TYPE_BINARY;
+      *bytes = (int16_t)(dataBytes + DATA_SET_FLAG_SIZE);
+      *interBytes = *bytes;
+      
+      return TSDB_CODE_SUCCESS;
+    } else if (functionId == FUNCTION_SUM) {
+      *type = TSDB_DATA_TYPE_BINARY;
+      *bytes = sizeof(SSumInfo);
+      *interBytes = *bytes;
+      
+      return TSDB_CODE_SUCCESS;
+    } else if (functionId == FUNCTION_AVG) {
+      *type = TSDB_DATA_TYPE_BINARY;
+      *bytes = sizeof(SAvgInfo);
+      *interBytes = *bytes;
+      return TSDB_CODE_SUCCESS;
+      
+    } else if (functionId >= FUNCTION_RATE && functionId <= FUNCTION_IRATE) {
+      *type = TSDB_DATA_TYPE_DOUBLE;
+      *bytes = sizeof(SRateInfo);
+      *interBytes = sizeof(SRateInfo);
+      return TSDB_CODE_SUCCESS;
+    } else if (functionId == FUNCTION_TOP || functionId == FUNCTION_BOTTOM) {
+      *type = TSDB_DATA_TYPE_BINARY;
+      *bytes = (int16_t)(sizeof(STopBotInfo) + (sizeof(tValuePair) + POINTER_BYTES + extLength) * param);
+      *interBytes = *bytes;
+      
+      return TSDB_CODE_SUCCESS;
+    } else if (functionId == FUNCTION_SPREAD) {
+      *type = TSDB_DATA_TYPE_BINARY;
+      *bytes = sizeof(SSpreadInfo);
+      *interBytes = *bytes;
+      
+      return TSDB_CODE_SUCCESS;
+    } else if (functionId == FUNCTION_APERCT) {
+      *type = TSDB_DATA_TYPE_BINARY;
+      *bytes = sizeof(SHistBin) * (MAX_HISTOGRAM_BIN + 1) + sizeof(SHistogramInfo) + sizeof(SAPercentileInfo);
+      *interBytes = *bytes;
+      
+      return TSDB_CODE_SUCCESS;
+    } else if (functionId == FUNCTION_LAST_ROW) {
+      *type = TSDB_DATA_TYPE_BINARY;
+      *bytes = (int16_t)(sizeof(SLastrowInfo) + dataBytes);
+      *interBytes = *bytes;
+      
+      return TSDB_CODE_SUCCESS;
+    } else if (functionId == FUNCTION_TWA) {
+      *type = TSDB_DATA_TYPE_DOUBLE;
+      *bytes = sizeof(STwaInfo);
+      *interBytes = *bytes;
+      return TSDB_CODE_SUCCESS;
+    }
+  }
+
+  if (functionId == FUNCTION_SUM) {
+    if (IS_SIGNED_NUMERIC_TYPE(dataType)) {
+      *type = TSDB_DATA_TYPE_BIGINT;
+    } else if (IS_UNSIGNED_NUMERIC_TYPE(dataType)) {
+      *type = TSDB_DATA_TYPE_UBIGINT;
+    } else {
+      *type = TSDB_DATA_TYPE_DOUBLE;
+    }
+    
+    *bytes = sizeof(int64_t);
+    *interBytes = sizeof(SSumInfo);
+    return TSDB_CODE_SUCCESS;
+  } else if (functionId == FUNCTION_APERCT) {
+    *type = TSDB_DATA_TYPE_DOUBLE;
+    *bytes = sizeof(double);
+    *interBytes =
+        sizeof(SAPercentileInfo) + sizeof(SHistogramInfo) + sizeof(SHistBin) * (MAX_HISTOGRAM_BIN + 1);
+    return TSDB_CODE_SUCCESS;
+  } else if (functionId == FUNCTION_TWA) {
+    *type = TSDB_DATA_TYPE_DOUBLE;
+    *bytes = sizeof(double);
+    *interBytes = sizeof(STwaInfo);
+    return TSDB_CODE_SUCCESS;
+  }
+
+//  if (functionId < 0) {
+//    *type = pUdfInfo->resType;
+//    *bytes = pUdfInfo->resBytes;
+//
+//    if (pUdfInfo->bufSize > 0) {
+//      *interBytes = pUdfInfo->bufSize;
+//    } else {
+//      *interBytes = *bytes;
+//    }
+//
+//    return TSDB_CODE_SUCCESS;
+//  }
+
+  if (functionId == FUNCTION_AVG) {
+    *type = TSDB_DATA_TYPE_DOUBLE;
+    *bytes = sizeof(double);
+    *interBytes = sizeof(SAvgInfo);
+  } else if (functionId >= FUNCTION_RATE && functionId <= FUNCTION_IRATE) {
+    *type = TSDB_DATA_TYPE_DOUBLE;
+    *bytes = sizeof(double);
+    *interBytes = sizeof(SRateInfo);
+  } else if (functionId == FUNCTION_STDDEV) {
+    *type = TSDB_DATA_TYPE_DOUBLE;
+    *bytes = sizeof(double);
+    *interBytes = sizeof(SStddevInfo);
+  } else if (functionId == FUNCTION_MIN || functionId == FUNCTION_MAX) {
+    *type = (int16_t)dataType;
+    *bytes = (int16_t)dataBytes;
+    *interBytes = dataBytes + DATA_SET_FLAG_SIZE;
+  } else if (functionId == FUNCTION_FIRST || functionId == FUNCTION_LAST) {
+    *type = (int16_t)dataType;
+    *bytes = (int16_t)dataBytes;
+    *interBytes = (int16_t)(dataBytes + sizeof(SFirstLastInfo));
+  } else if (functionId == FUNCTION_SPREAD) {
+    *type = (int16_t)TSDB_DATA_TYPE_DOUBLE;
+    *bytes = sizeof(double);
+    *interBytes = sizeof(SSpreadInfo);
+  } else if (functionId == FUNCTION_PERCT) {
+    *type = (int16_t)TSDB_DATA_TYPE_DOUBLE;
+    *bytes = (int16_t)sizeof(double);
+    *interBytes = (int16_t)sizeof(SPercentileInfo);
+  } else if (functionId == FUNCTION_LEASTSQR) {
+    *type = TSDB_DATA_TYPE_BINARY;
+    *bytes = MAX(AVG_FUNCTION_INTER_BUFFER_SIZE, sizeof(SLeastsquaresInfo));  // string
+    *interBytes = *bytes;
+  } else if (functionId == FUNCTION_FIRST_DST || functionId == FUNCTION_LAST_DST) {
+    *type = TSDB_DATA_TYPE_BINARY;
+    *bytes = (int16_t)(dataBytes + sizeof(SFirstLastInfo));
+    *interBytes = *bytes;
+  } else if (functionId == FUNCTION_TOP || functionId == FUNCTION_BOTTOM) {
+    *type = (int16_t)dataType;
+    *bytes = (int16_t)dataBytes;
+    
+    size_t size = sizeof(STopBotInfo) + (sizeof(tValuePair) + POINTER_BYTES + extLength) * param;
+    
+    // the output column may be larger than sizeof(STopBotInfo)
+    *interBytes = (int32_t)size;
+  } else if (functionId == FUNCTION_LAST_ROW) {
+    *type = (int16_t)dataType;
+    *bytes = (int16_t)dataBytes;
+    *interBytes = dataBytes;
+  } else if (functionId == FUNCTION_STDDEV_DST) {
+    *type = TSDB_DATA_TYPE_BINARY;
+    *bytes = sizeof(SStddevdstInfo);
+    *interBytes = (*bytes);
+
+  } else {
+    return TSDB_CODE_TSC_INVALID_OPERATION;
+  }
+  
+  return TSDB_CODE_SUCCESS;
+}
+
+static bool function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResultInfo) {
+  if (pResultInfo->initialized) {
+    return false;
+  }
+  
+  memset(pCtx->pOutput, 0, (size_t)pCtx->outputBytes);
+  initResultInfo(pResultInfo, pCtx->interBufBytes);
+  return true;
+}
+
+/**
+ * in handling the stable query, function_finalizer is called after the secondary
+ * merge being completed, during the first merge procedure, which is executed at the
+ * vnode side, the finalize will never be called.
+ *
+ * @param pCtx
+ */
+static void function_finalizer(SQLFunctionCtx *pCtx) {
+  SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+  if (pResInfo->hasResult != DATA_SET_FLAG) {
+    setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes);
+  }
+  
+  doFinalizer(pCtx);
+}
+
+/*
+ * count function does need the finalize, if data is missing, the default value, which is 0, is used
+ * count function does not use the pCtx->interResBuf to keep the intermediate buffer
+ */
+static void count_function(SQLFunctionCtx *pCtx) {
+  int32_t numOfElem = 0;
+  
+  /*
+   * 1. column data missing (schema modified) causes pCtx->hasNull == true. pCtx->isSmaSet == true;
+   * 2. for general non-primary key columns, pCtx->hasNull may be true or false, pCtx->isSmaSet == true;
+   * 3. for primary key column, pCtx->hasNull always be false, pCtx->isSmaSet == false;
+   */
+  if (pCtx->isSmaSet) {
+    numOfElem = pCtx->size - pCtx->sma.numOfNull;
+  } else {
+    if (pCtx->hasNull) {
+      for (int32_t i = 0; i < pCtx->size; ++i) {
+        char *val = GET_INPUT_DATA(pCtx, i);
+        if (isNull(val, pCtx->inputType)) {
+          continue;
+        }
+        
+        numOfElem += 1;
+      }
+    } else {
+      //when counting on the primary time stamp column and no statistics data is presented, use the size value directly.
+      numOfElem = pCtx->size;
+    }
+  }
+  
+  if (numOfElem > 0) {
+    GET_RES_INFO(pCtx)->hasResult = DATA_SET_FLAG;
+  }
+  
+  *((int64_t *)pCtx->pOutput) += numOfElem;
+  SET_VAL(pCtx, numOfElem, 1);
+}
+
+static void count_func_merge(SQLFunctionCtx *pCtx) {
+  int64_t *pData = (int64_t *)GET_INPUT_DATA_LIST(pCtx);
+  for (int32_t i = 0; i < pCtx->size; ++i) {
+    *((int64_t *)pCtx->pOutput) += pData[i];
+  }
+  
+  SET_VAL(pCtx, pCtx->size, 1);
+}
+
+/**
+ * 1. If the column value for filter exists, we need to load the SFields, which serves
+ *    as the pre-filter to decide if the actual data block is required or not.
+ * 2. If it queries on the non-primary timestamp column, SFields is also required to get the not-null value.
+ *
+ * @param colId
+ * @param filterCols
+ * @return
+ */
+int32_t countRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId) {
+  if (colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) {
+    return BLK_DATA_NO_NEEDED;
+  } else {
+    return BLK_DATA_STATIS_NEEDED;
+  }
+}
+
+int32_t noDataRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId) {
+  return BLK_DATA_NO_NEEDED;
+}
+#define LIST_ADD_N_DOUBLE_FLOAT(x, ctx, p, t, numOfElem, tsdbType)              \
+  do {                                                                \
+    t *d = (t *)(p);                                               \
+    for (int32_t i = 0; i < (ctx)->size; ++i) {                    \
+      if (((ctx)->hasNull) && isNull((char *)&(d)[i], tsdbType)) { \
+        continue;                                                  \
+      };                                                           \
+      SET_DOUBLE_VAL(&(x) , GET_DOUBLE_VAL(&(x)) + GET_FLOAT_VAL(&(d)[i]));                                               \
+      (numOfElem)++;                                               \
+    }                                                              \
+  } while(0)
+#define LIST_ADD_N_DOUBLE(x, ctx, p, t, numOfElem, tsdbType)              \
+  do {                                                                \
+    t *d = (t *)(p);                                               \
+    for (int32_t i = 0; i < (ctx)->size; ++i) {                    \
+      if (((ctx)->hasNull) && isNull((char *)&(d)[i], tsdbType)) { \
+        continue;                                                  \
+      };                                                           \
+      SET_DOUBLE_VAL(&(x) , (x) + (d)[i]);                                               \
+      (numOfElem)++;                                               \
+    }                                                              \
+  } while(0)
+
+#define LIST_ADD_N(x, ctx, p, t, numOfElem, tsdbType)              \
+  do {                                                                \
+    t *d = (t *)(p);                                               \
+    for (int32_t i = 0; i < (ctx)->size; ++i) {                    \
+      if (((ctx)->hasNull) && isNull((char *)&(d)[i], tsdbType)) { \
+        continue;                                                  \
+      };                                                           \
+      (x) += (d)[i];                                               \
+      (numOfElem)++;                                               \
+    }                                                              \
+  } while(0)
+
+#define UPDATE_DATA(ctx, left, right, num, sign, k) \
+  do {                                              \
+    if (((left) < (right)) ^ (sign)) {              \
+      (left) = (right);                             \
+      DO_UPDATE_TAG_COLUMNS(ctx, k);                \
+      (num) += 1;                                   \
+    }                                               \
+  } while (0)
+
+#define DUPATE_DATA_WITHOUT_TS(ctx, left, right, num, sign) \
+  do {                                                      \
+    if (((left) < (right)) ^ (sign)) {                      \
+      (left) = (right);                                     \
+      DO_UPDATE_TAG_COLUMNS_WITHOUT_TS(ctx);                \
+      (num) += 1;                                           \
+    }                                                       \
+  } while (0)
+
+#define LOOPCHECK_N(val, list, ctx, tsdbType, sign, num)          \
+  for (int32_t i = 0; i < ((ctx)->size); ++i) {                   \
+    if ((ctx)->hasNull && isNull((char *)&(list)[i], tsdbType)) { \
+      continue;                                                   \
+    }                                                             \
+    TSKEY key = (ctx)->ptsList != NULL? GET_TS_DATA(ctx, i):0;    \
+    UPDATE_DATA(ctx, val, (list)[i], num, sign, key);             \
+  }
+
+#define TYPED_LOOPCHECK_N(type, data, list, ctx, tsdbType, sign, notNullElems) \
+  do {                                                                         \
+    type *_data = (type *)data;                                                \
+    type *_list = (type *)list;                                                \
+    LOOPCHECK_N(*_data, _list, ctx, tsdbType, sign, notNullElems);             \
+  } while (0)
+
+static void do_sum(SQLFunctionCtx *pCtx) {
+  int32_t notNullElems = 0;
+  
+  // Only the pre-computing information loaded and actual data does not loaded
+  if (pCtx->isSmaSet) {
+    notNullElems = pCtx->size - pCtx->sma.numOfNull;
+    assert(pCtx->size >= pCtx->sma.numOfNull);
+    
+    if (IS_SIGNED_NUMERIC_TYPE(pCtx->inputType)) {
+      int64_t *retVal = (int64_t *)pCtx->pOutput;
+      *retVal += pCtx->sma.sum;
+    } else if (IS_UNSIGNED_NUMERIC_TYPE(pCtx->inputType)) {
+      uint64_t *retVal = (uint64_t *)pCtx->pOutput;
+      *retVal += (uint64_t)pCtx->sma.sum;
+    } else if (IS_FLOAT_TYPE(pCtx->inputType)) {
+      double *retVal = (double*) pCtx->pOutput;
+      SET_DOUBLE_VAL(retVal, *retVal + GET_DOUBLE_VAL((const char*)&(pCtx->sma.sum)));
+    }
+  } else {  // computing based on the true data block
+    void *pData = GET_INPUT_DATA_LIST(pCtx);
+    notNullElems = 0;
+
+    if (IS_SIGNED_NUMERIC_TYPE(pCtx->inputType)) {
+      int64_t *retVal = (int64_t *)pCtx->pOutput;
+
+      if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) {
+        LIST_ADD_N(*retVal, pCtx, pData, int8_t, notNullElems, pCtx->inputType);
+      } else if (pCtx->inputType == TSDB_DATA_TYPE_SMALLINT) {
+        LIST_ADD_N(*retVal, pCtx, pData, int16_t, notNullElems, pCtx->inputType);
+      } else if (pCtx->inputType == TSDB_DATA_TYPE_INT) {
+        LIST_ADD_N(*retVal, pCtx, pData, int32_t, notNullElems, pCtx->inputType);
+      } else if (pCtx->inputType == TSDB_DATA_TYPE_BIGINT) {
+        LIST_ADD_N(*retVal, pCtx, pData, int64_t, notNullElems, pCtx->inputType);
+      }
+    } else if (IS_UNSIGNED_NUMERIC_TYPE(pCtx->inputType)) {
+      uint64_t *retVal = (uint64_t *)pCtx->pOutput;
+
+      if (pCtx->inputType == TSDB_DATA_TYPE_UTINYINT) {
+        LIST_ADD_N(*retVal, pCtx, pData, uint8_t, notNullElems, pCtx->inputType);
+      } else if (pCtx->inputType == TSDB_DATA_TYPE_USMALLINT) {
+        LIST_ADD_N(*retVal, pCtx, pData, uint16_t, notNullElems, pCtx->inputType);
+      } else if (pCtx->inputType == TSDB_DATA_TYPE_UINT) {
+        LIST_ADD_N(*retVal, pCtx, pData, uint32_t, notNullElems, pCtx->inputType);
+      } else if (pCtx->inputType == TSDB_DATA_TYPE_UBIGINT) {
+        LIST_ADD_N(*retVal, pCtx, pData, uint64_t, notNullElems, pCtx->inputType);
+      }
+    } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) {
+      double *retVal = (double *)pCtx->pOutput;
+      LIST_ADD_N_DOUBLE(*retVal, pCtx, pData, double, notNullElems, pCtx->inputType);
+    } else if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT) {
+      double *retVal = (double *)pCtx->pOutput;
+      LIST_ADD_N_DOUBLE_FLOAT(*retVal, pCtx, pData, float, notNullElems, pCtx->inputType);
+    }
+  }
+  
+  // data in the check operation are all null, not output
+  SET_VAL(pCtx, notNullElems, 1);
+  
+  if (notNullElems > 0) {
+    GET_RES_INFO(pCtx)->hasResult = DATA_SET_FLAG;
+  }
+}
+
+static void sum_function(SQLFunctionCtx *pCtx) {
+  do_sum(pCtx);
+  
+  // keep the result data in output buffer, not in the intermediate buffer
+  SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+  if (pResInfo->hasResult == DATA_SET_FLAG && pCtx->stableQuery) {
+    // set the flag for super table query
+    SSumInfo *pSum = (SSumInfo *)pCtx->pOutput;
+    pSum->hasResult = DATA_SET_FLAG;
+  }
+}
+
+static void sum_func_merge(SQLFunctionCtx *pCtx) {
+  int32_t notNullElems = 0;
+
+  GET_TRUE_DATA_TYPE();
+  assert(pCtx->stableQuery);
+
+  for (int32_t i = 0; i < pCtx->size; ++i) {
+    char *    input = GET_INPUT_DATA(pCtx, i);
+    SSumInfo *pInput = (SSumInfo *)input;
+    if (pInput->hasResult != DATA_SET_FLAG) {
+      continue;
+    }
+
+    notNullElems++;
+
+    if (IS_SIGNED_NUMERIC_TYPE(type)) {
+      *(int64_t *)pCtx->pOutput += pInput->isum;
+    } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) {
+      *(uint64_t *) pCtx->pOutput += pInput->usum;
+    } else {
+      SET_DOUBLE_VAL((double *)pCtx->pOutput, *(double *)pCtx->pOutput + pInput->dsum);
+    }
+  }
+
+  SET_VAL(pCtx, notNullElems, 1);
+  SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+  
+  if (notNullElems > 0) {
+    pResInfo->hasResult = DATA_SET_FLAG;
+  }
+}
+
+static int32_t statisRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId) {
+  return BLK_DATA_STATIS_NEEDED;
+}
+
+static int32_t dataBlockRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId) {
+  return BLK_DATA_ALL_NEEDED;
+}
+
+// todo: if column in current data block are null, opt for this case
+static int32_t firstFuncRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId) {
+  if (pCtx->order == TSDB_ORDER_DESC) {
+    return BLK_DATA_NO_NEEDED;
+  }
+  
+  // no result for first query, data block is required
+  if (GET_RES_INFO(pCtx) == NULL || GET_RES_INFO(pCtx)->numOfRes <= 0) {
+    return BLK_DATA_ALL_NEEDED;
+  } else {
+    return BLK_DATA_NO_NEEDED;
+  }
+}
+
+static int32_t lastFuncRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId) {
+  if (pCtx->order != pCtx->param[0].i64) {
+    return BLK_DATA_NO_NEEDED;
+  }
+  
+  if (GET_RES_INFO(pCtx) == NULL || GET_RES_INFO(pCtx)->numOfRes <= 0) {
+    return BLK_DATA_ALL_NEEDED;
+  } else {
+    return BLK_DATA_NO_NEEDED;
+  }
+}
+
+static int32_t firstDistFuncRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId) {
+  if (pCtx->order == TSDB_ORDER_DESC) {
+    return BLK_DATA_NO_NEEDED;
+  }
+
+  // not initialized yet, it is the first block, load it.
+  if (pCtx->pOutput == NULL) {
+    return BLK_DATA_ALL_NEEDED;
+  }
+
+  // the pCtx should be set to current Ctx and output buffer before call this function. Otherwise, pCtx->pOutput is
+  // the previous windowRes output buffer, not current unloaded block. In this case, the following filter is invalid
+  SFirstLastInfo *pInfo = (SFirstLastInfo*) (pCtx->pOutput + pCtx->inputBytes);
+  if (pInfo->hasResult != DATA_SET_FLAG) {
+    return BLK_DATA_ALL_NEEDED;
+  } else {  // data in current block is not earlier than current result
+    return (pInfo->ts <= w->skey) ? BLK_DATA_NO_NEEDED : BLK_DATA_ALL_NEEDED;
+  }
+}
+
+static int32_t lastDistFuncRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId) {
+  if (pCtx->order != pCtx->param[0].i64) {
+    return BLK_DATA_NO_NEEDED;
+  }
+
+  // not initialized yet, it is the first block, load it.
+  if (pCtx->pOutput == NULL) {
+    return BLK_DATA_ALL_NEEDED;
+  }
+
+  // the pCtx should be set to current Ctx and output buffer before call this function. Otherwise, pCtx->pOutput is
+  // the previous windowRes output buffer, not current unloaded block. In this case, the following filter is invalid
+  SFirstLastInfo *pInfo = (SFirstLastInfo*) (pCtx->pOutput + pCtx->inputBytes);
+  if (pInfo->hasResult != DATA_SET_FLAG) {
+    return BLK_DATA_ALL_NEEDED;
+  } else {
+    return (pInfo->ts > w->ekey) ? BLK_DATA_NO_NEEDED : BLK_DATA_ALL_NEEDED;
+  }
+}
+
+//////////////////////////////////////////////////////////////////////////////////////////////
+/*
+ * The intermediate result of average is kept in the interResultBuf.
+ * For super table query, once the avg_function/avg_function_f is finished, copy the intermediate
+ * result into output buffer.
+ */
+static void avg_function(SQLFunctionCtx *pCtx) {
+  int32_t notNullElems = 0;
+  
+  // NOTE: keep the intermediate result into the interResultBuf
+  SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+  
+  SAvgInfo *pAvgInfo = (SAvgInfo *)GET_ROWCELL_INTERBUF(pResInfo);
+  double   *pVal = &pAvgInfo->sum;
+  
+  if (pCtx->isSmaSet) { // Pre-aggregation
+    notNullElems = pCtx->size - pCtx->sma.numOfNull;
+    assert(notNullElems >= 0);
+    
+    if (IS_SIGNED_NUMERIC_TYPE(pCtx->inputType)) {
+      *pVal += pCtx->sma.sum;
+    }  else if (IS_UNSIGNED_NUMERIC_TYPE(pCtx->inputType)) {
+      *pVal += (uint64_t) pCtx->sma.sum;
+    } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE || pCtx->inputType == TSDB_DATA_TYPE_FLOAT) {
+      *pVal += GET_DOUBLE_VAL((const char *)&(pCtx->sma.sum));
+    }
+  } else {
+    void *pData = GET_INPUT_DATA_LIST(pCtx);
+    
+    if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) {
+      LIST_ADD_N(*pVal, pCtx, pData, int8_t, notNullElems, pCtx->inputType);
+    } else if (pCtx->inputType == TSDB_DATA_TYPE_SMALLINT) {
+      LIST_ADD_N(*pVal, pCtx, pData, int16_t, notNullElems, pCtx->inputType);
+    } else if (pCtx->inputType == TSDB_DATA_TYPE_INT) {
+      LIST_ADD_N(*pVal, pCtx, pData, int32_t, notNullElems, pCtx->inputType);
+    } else if (pCtx->inputType == TSDB_DATA_TYPE_BIGINT) {
+      LIST_ADD_N(*pVal, pCtx, pData, int64_t, notNullElems, pCtx->inputType);
+    } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) {
+      LIST_ADD_N_DOUBLE(*pVal, pCtx, pData, double, notNullElems, pCtx->inputType);
+    } else if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT) {
+      LIST_ADD_N_DOUBLE_FLOAT(*pVal, pCtx, pData, float, notNullElems, pCtx->inputType);
+    } else if (pCtx->inputType == TSDB_DATA_TYPE_UTINYINT) {
+      LIST_ADD_N(*pVal, pCtx, pData, uint8_t, notNullElems, pCtx->inputType);
+    } else if (pCtx->inputType == TSDB_DATA_TYPE_USMALLINT) {
+      LIST_ADD_N(*pVal, pCtx, pData, uint16_t, notNullElems, pCtx->inputType);
+    } else if (pCtx->inputType == TSDB_DATA_TYPE_UINT) {
+      LIST_ADD_N(*pVal, pCtx, pData, uint32_t, notNullElems, pCtx->inputType);
+    } else if (pCtx->inputType == TSDB_DATA_TYPE_UBIGINT) {
+      LIST_ADD_N(*pVal, pCtx, pData, uint64_t, notNullElems, pCtx->inputType);
+    }
+  }
+  
+  if (!pCtx->hasNull) {
+    assert(notNullElems == pCtx->size);
+  }
+  
+  SET_VAL(pCtx, notNullElems, 1);
+  pAvgInfo->num += notNullElems;
+  
+  if (notNullElems > 0) {
+    pResInfo->hasResult = DATA_SET_FLAG;
+  }
+  
+  // keep the data into the final output buffer for super table query since this execution may be the last one
+  if (pCtx->stableQuery) {
+    memcpy(pCtx->pOutput, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SAvgInfo));
+  }
+}
+
+static void avg_func_merge(SQLFunctionCtx *pCtx) {
+  SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+  
+  double *sum = (double*) pCtx->pOutput;
+  char   *input = GET_INPUT_DATA_LIST(pCtx);
+  
+  for (int32_t i = 0; i < pCtx->size; ++i, input += pCtx->inputBytes) {
+    SAvgInfo *pInput = (SAvgInfo *)input;
+    if (pInput->num == 0) {  // current input is null
+      continue;
+    }
+    
+    SET_DOUBLE_VAL(sum, *sum + pInput->sum);
+    
+    // keep the number of data into the temp buffer
+    *(int64_t *)GET_ROWCELL_INTERBUF(pResInfo) += pInput->num;
+  }
+}
+
+/*
+ * the average value is calculated in finalize routine, since current routine does not know the exact number of points
+ */
+static void avg_finalizer(SQLFunctionCtx *pCtx) {
+  SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+  
+  if (pCtx->currentStage == MERGE_STAGE) {
+    assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY);
+    
+    if (GET_INT64_VAL(GET_ROWCELL_INTERBUF(pResInfo)) <= 0) {
+      setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes);
+      return;
+    }
+
+    SET_DOUBLE_VAL((double *)pCtx->pOutput,(*(double *)pCtx->pOutput) / *(int64_t *)GET_ROWCELL_INTERBUF(pResInfo));
+  } else {  // this is the secondary merge, only in the secondary merge, the input type is TSDB_DATA_TYPE_BINARY
+    assert(IS_NUMERIC_TYPE(pCtx->inputType));
+    SAvgInfo *pAvgInfo = (SAvgInfo *)GET_ROWCELL_INTERBUF(pResInfo);
+    
+    if (pAvgInfo->num == 0) {  // all data are NULL or empty table
+      setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes);
+      return;
+    }
+    
+    SET_DOUBLE_VAL((double *)pCtx->pOutput, pAvgInfo->sum / pAvgInfo->num);
+  }
+  
+  // cannot set the numOfIteratedElems again since it is set during previous iteration
+  GET_RES_INFO(pCtx)->numOfRes = 1;
+  doFinalizer(pCtx);
+}
+
+/////////////////////////////////////////////////////////////////////////////////////////////
+
+static void minMax_function(SQLFunctionCtx *pCtx, char *pOutput, int32_t isMin, int32_t *notNullElems) {
+  // data in current data block are qualified to the query
+  if (pCtx->isSmaSet) {
+    *notNullElems = pCtx->size - pCtx->sma.numOfNull;
+    assert(*notNullElems >= 0);
+
+    if (*notNullElems == 0) {
+      return;
+    }
+
+    void*   tval = NULL;
+    int16_t index = 0;
+    
+    if (isMin) {
+      tval = &pCtx->sma.min;
+      index = pCtx->sma.minIndex;
+    } else {
+      tval = &pCtx->sma.max;
+      index = pCtx->sma.maxIndex;
+    }
+    
+    TSKEY key = TSKEY_INITIAL_VAL;
+    if (pCtx->ptsList != NULL) {
+      /**
+       * NOTE: work around the bug caused by invalid pre-calculated function.
+       * Here the selectivity + ts will not return correct value.
+       *
+       * The following codes of 3 lines will be removed later.
+       */
+//      if (index < 0 || index >= pCtx->size + pCtx->startOffset) {
+//        index = 0;
+//      }
+
+      // the index is the original position, not the relative position
+      key = pCtx->ptsList[index];
+    }
+    
+    if (IS_SIGNED_NUMERIC_TYPE(pCtx->inputType)) {
+      int64_t val = GET_INT64_VAL(tval);
+      if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) {
+        int8_t *data = (int8_t *)pOutput;
+        
+        UPDATE_DATA(pCtx, *data, (int8_t)val, notNullElems, isMin, key);
+      } else if (pCtx->inputType == TSDB_DATA_TYPE_SMALLINT) {
+        int16_t *data = (int16_t *)pOutput;
+        
+        UPDATE_DATA(pCtx, *data, (int16_t)val, notNullElems, isMin, key);
+      } else if (pCtx->inputType == TSDB_DATA_TYPE_INT) {
+        int32_t *data = (int32_t *)pOutput;
+#if defined(_DEBUG_VIEW)
+        qDebug("max value updated according to pre-cal:%d", *data);
+#endif
+        
+        if ((*data < val) ^ isMin) {
+          *data = (int32_t)val;
+          for (int32_t i = 0; i < (pCtx)->tagInfo.numOfTagCols; ++i) {
+            SQLFunctionCtx *__ctx = pCtx->tagInfo.pTagCtxList[i];
+            if (__ctx->functionId == FUNCTION_TS_DUMMY) {
+              __ctx->tag.i64 = key;
+              __ctx->tag.nType = TSDB_DATA_TYPE_BIGINT;
+            }
+            
+            aggFunc[FUNCTION_TAG].exec(__ctx);
+          }
+        }
+      } else if (pCtx->inputType == TSDB_DATA_TYPE_BIGINT) {
+        int64_t *data = (int64_t *)pOutput;
+        UPDATE_DATA(pCtx, *data, val, notNullElems, isMin, key);
+      }
+    } else if (IS_UNSIGNED_NUMERIC_TYPE(pCtx->inputType)) {
+      uint64_t val = GET_UINT64_VAL(tval);
+      if (pCtx->inputType == TSDB_DATA_TYPE_UTINYINT) {
+        uint8_t *data = (uint8_t *)pOutput;
+
+        UPDATE_DATA(pCtx, *data, (uint8_t)val, notNullElems, isMin, key);
+      } else if (pCtx->inputType == TSDB_DATA_TYPE_USMALLINT) {
+        uint16_t *data = (uint16_t *)pOutput;
+        UPDATE_DATA(pCtx, *data, (uint16_t)val, notNullElems, isMin, key);
+      } else if (pCtx->inputType == TSDB_DATA_TYPE_UINT) {
+        uint32_t *data = (uint32_t *)pOutput;
+        UPDATE_DATA(pCtx, *data, (uint32_t)val, notNullElems, isMin, key);
+      } else if (pCtx->inputType == TSDB_DATA_TYPE_UBIGINT) {
+        uint64_t *data = (uint64_t *)pOutput;
+        UPDATE_DATA(pCtx, *data, val, notNullElems, isMin, key);
+      }
+    } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) {
+        double *data = (double *)pOutput;
+        double  val = GET_DOUBLE_VAL(tval);
+
+        UPDATE_DATA(pCtx, *data, val, notNullElems, isMin, key);
+    } else if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT) {
+      float *data = (float *)pOutput;
+      double val = GET_DOUBLE_VAL(tval);
+      
+      UPDATE_DATA(pCtx, *data, (float)val, notNullElems, isMin, key);
+    }
+    
+    return;
+  }
+  
+  void  *p = GET_INPUT_DATA_LIST(pCtx);
+  TSKEY *tsList = GET_TS_LIST(pCtx);
+
+  *notNullElems = 0;
+  
+  if (IS_SIGNED_NUMERIC_TYPE(pCtx->inputType)) {
+    if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) {
+      TYPED_LOOPCHECK_N(int8_t, pOutput, p, pCtx, pCtx->inputType, isMin, *notNullElems);
+    } else if (pCtx->inputType == TSDB_DATA_TYPE_SMALLINT) {
+      TYPED_LOOPCHECK_N(int16_t, pOutput, p, pCtx, pCtx->inputType, isMin, *notNullElems);
+    } else if (pCtx->inputType == TSDB_DATA_TYPE_INT) {
+      int32_t *pData = p;
+      int32_t *retVal = (int32_t*) pOutput;
+      
+      for (int32_t i = 0; i < pCtx->size; ++i) {
+        if (pCtx->hasNull && isNull((const char*)&pData[i], pCtx->inputType)) {
+          continue;
+        }
+        
+        if ((*retVal < pData[i]) ^ isMin) {
+          *retVal = pData[i];
+          TSKEY k = tsList[i];
+          
+          DO_UPDATE_TAG_COLUMNS(pCtx, k);
+        }
+        
+        *notNullElems += 1;
+      }
+#if defined(_DEBUG_VIEW)
+      qDebug("max value updated:%d", *retVal);
+#endif
+    } else if (pCtx->inputType == TSDB_DATA_TYPE_BIGINT) {
+      TYPED_LOOPCHECK_N(int64_t, pOutput, p, pCtx, pCtx->inputType, isMin, *notNullElems);
+    }
+  } else if (IS_UNSIGNED_NUMERIC_TYPE(pCtx->inputType)) {
+    if (pCtx->inputType == TSDB_DATA_TYPE_UTINYINT) {
+      TYPED_LOOPCHECK_N(uint8_t, pOutput, p, pCtx, pCtx->inputType, isMin, *notNullElems);
+    } else if (pCtx->inputType == TSDB_DATA_TYPE_USMALLINT) {
+      TYPED_LOOPCHECK_N(uint16_t, pOutput, p, pCtx, pCtx->inputType, isMin, *notNullElems);
+    } else if (pCtx->inputType == TSDB_DATA_TYPE_UINT) {
+      TYPED_LOOPCHECK_N(uint32_t, pOutput, p, pCtx, pCtx->inputType, isMin, *notNullElems);
+    } else if (pCtx->inputType == TSDB_DATA_TYPE_UBIGINT) {
+      TYPED_LOOPCHECK_N(uint64_t, pOutput, p, pCtx, pCtx->inputType, isMin, *notNullElems);
+    }
+  } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) {
+    TYPED_LOOPCHECK_N(double, pOutput, p, pCtx, pCtx->inputType, isMin, *notNullElems);
+  } else if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT) {
+    TYPED_LOOPCHECK_N(float, pOutput, p, pCtx, pCtx->inputType, isMin, *notNullElems);
+  }
+}
+
+static bool min_func_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResultInfo) {
+  if (!function_setup(pCtx, pResultInfo)) {
+    return false;  // not initialized since it has been initialized
+  }
+  
+  GET_TRUE_DATA_TYPE();
+  
+  switch (type) {
+    case TSDB_DATA_TYPE_TINYINT:
+      *((int8_t *)pCtx->pOutput) = INT8_MAX;
+      break;
+    case TSDB_DATA_TYPE_UTINYINT:
+      *(uint8_t *) pCtx->pOutput = UINT8_MAX;
+      break;
+    case TSDB_DATA_TYPE_SMALLINT:
+      *((int16_t *)pCtx->pOutput) = INT16_MAX;
+      break;
+    case TSDB_DATA_TYPE_USMALLINT:
+      *((uint16_t *)pCtx->pOutput) = UINT16_MAX;
+      break;
+    case TSDB_DATA_TYPE_INT:
+      *((int32_t *)pCtx->pOutput) = INT32_MAX;
+      break;
+    case TSDB_DATA_TYPE_UINT:
+      *((uint32_t *)pCtx->pOutput) = UINT32_MAX;
+      break;
+    case TSDB_DATA_TYPE_BIGINT:
+      *((int64_t *)pCtx->pOutput) = INT64_MAX;
+      break;
+    case TSDB_DATA_TYPE_UBIGINT:
+      *((uint64_t *)pCtx->pOutput) = UINT64_MAX;
+      break;
+    case TSDB_DATA_TYPE_FLOAT:
+      *((float *)pCtx->pOutput) = FLT_MAX;
+      break;
+    case TSDB_DATA_TYPE_DOUBLE:
+      SET_DOUBLE_VAL(((double *)pCtx->pOutput), DBL_MAX);
+      break;
+    default:
+      assert(0);
+//      qError("illegal data type:%d in min/max query", pCtx->inputType);
+  }
+
+  return true;
+}
+
+static bool max_func_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResultInfo) {
+  if (!function_setup(pCtx, pResultInfo)) {
+    return false;  // not initialized since it has been initialized
+  }
+  
+  GET_TRUE_DATA_TYPE();
+  
+  switch (type) {
+    case TSDB_DATA_TYPE_INT:
+      *((int32_t *)pCtx->pOutput) = INT32_MIN;
+      break;
+    case TSDB_DATA_TYPE_UINT:
+      *((uint32_t *)pCtx->pOutput) = 0;
+      break;
+    case TSDB_DATA_TYPE_FLOAT:
+      *((float *)pCtx->pOutput) = -FLT_MAX;
+      break;
+    case TSDB_DATA_TYPE_DOUBLE:
+      SET_DOUBLE_VAL(((double *)pCtx->pOutput), -DBL_MAX);
+      break;
+    case TSDB_DATA_TYPE_BIGINT:
+      *((int64_t *)pCtx->pOutput) = INT64_MIN;
+      break;
+    case TSDB_DATA_TYPE_UBIGINT:
+      *((uint64_t *)pCtx->pOutput) = 0;
+      break;
+    case TSDB_DATA_TYPE_SMALLINT:
+      *((int16_t *)pCtx->pOutput) = INT16_MIN;
+      break;
+    case TSDB_DATA_TYPE_USMALLINT:
+      *((uint16_t *)pCtx->pOutput) = 0;
+      break;
+    case TSDB_DATA_TYPE_TINYINT:
+      *((int8_t *)pCtx->pOutput) = INT8_MIN;
+      break;
+    case TSDB_DATA_TYPE_UTINYINT:
+      *((uint8_t *)pCtx->pOutput) = 0;
+      break;
+    default:
+      assert(0);
+//      qError("illegal data type:%d in min/max query", pCtx->inputType);
+  }
+  
+  return true;
+}
+
+/*
+ * the output result of min/max function is the final output buffer, not the intermediate result buffer
+ */
+static void min_function(SQLFunctionCtx *pCtx) {
+  int32_t notNullElems = 0;
+  minMax_function(pCtx, pCtx->pOutput, 1, &notNullElems);
+  
+  SET_VAL(pCtx, notNullElems, 1);
+  
+  if (notNullElems > 0) {
+    SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+    pResInfo->hasResult = DATA_SET_FLAG;
+    
+    // set the flag for super table query
+    if (pCtx->stableQuery) {
+      *(pCtx->pOutput + pCtx->inputBytes) = DATA_SET_FLAG;
+    }
+  }
+}
+
+static void max_function(SQLFunctionCtx *pCtx) {
+  int32_t notNullElems = 0;
+  minMax_function(pCtx, pCtx->pOutput, 0, &notNullElems);
+  
+  SET_VAL(pCtx, notNullElems, 1);
+  
+  if (notNullElems > 0) {
+    SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+    pResInfo->hasResult = DATA_SET_FLAG;
+    
+    // set the flag for super table query
+    if (pCtx->stableQuery) {
+      *(pCtx->pOutput + pCtx->inputBytes) = DATA_SET_FLAG;
+    }
+  }
+}
+
+static int32_t minmax_merge_impl(SQLFunctionCtx *pCtx, int32_t bytes, char *output, bool isMin) {
+  int32_t notNullElems = 0;
+  
+  GET_TRUE_DATA_TYPE();
+  assert(pCtx->stableQuery);
+  
+  for (int32_t i = 0; i < pCtx->size; ++i) {
+    char *input = GET_INPUT_DATA(pCtx, i);
+    if (input[bytes] != DATA_SET_FLAG) {
+      continue;
+    }
+    
+    switch (type) {
+      case TSDB_DATA_TYPE_TINYINT: {
+        int8_t v = GET_INT8_VAL(input);
+        DUPATE_DATA_WITHOUT_TS(pCtx, *(int8_t *)output, v, notNullElems, isMin);
+        break;
+      }
+      case TSDB_DATA_TYPE_SMALLINT: {
+        int16_t v = GET_INT16_VAL(input);
+        DUPATE_DATA_WITHOUT_TS(pCtx, *(int16_t *)output, v, notNullElems, isMin);
+        break;
+      }
+      case TSDB_DATA_TYPE_INT: {
+        int32_t v = GET_INT32_VAL(input);
+        if ((*(int32_t *)output < v) ^ isMin) {
+          *(int32_t *)output = v;
+          
+          for (int32_t j = 0; j < pCtx->tagInfo.numOfTagCols; ++j) {
+            SQLFunctionCtx *__ctx = pCtx->tagInfo.pTagCtxList[j];
+            aggFunc[FUNCTION_TAG].exec(__ctx);
+          }
+          
+          notNullElems++;
+        }
+        break;
+      }
+      case TSDB_DATA_TYPE_FLOAT: {
+        float v = GET_FLOAT_VAL(input);
+        DUPATE_DATA_WITHOUT_TS(pCtx, *(float *)output, v, notNullElems, isMin);
+        break;
+      }
+      case TSDB_DATA_TYPE_DOUBLE: {
+        double v = GET_DOUBLE_VAL(input);
+        DUPATE_DATA_WITHOUT_TS(pCtx, *(double *)output, v, notNullElems, isMin);
+        break;
+      }
+      case TSDB_DATA_TYPE_BIGINT: {
+        int64_t v = GET_INT64_VAL(input);
+        DUPATE_DATA_WITHOUT_TS(pCtx, *(int64_t *)output, v, notNullElems, isMin);
+        break;
+      }
+
+      case TSDB_DATA_TYPE_UTINYINT: {
+        uint8_t v = GET_UINT8_VAL(input);
+        DUPATE_DATA_WITHOUT_TS(pCtx, *(uint8_t *)output, v, notNullElems, isMin);
+        break;
+      }
+
+      case TSDB_DATA_TYPE_USMALLINT: {
+        uint16_t v = GET_UINT16_VAL(input);
+        DUPATE_DATA_WITHOUT_TS(pCtx, *(uint16_t *)output, v, notNullElems, isMin);
+        break;
+      }
+
+      case TSDB_DATA_TYPE_UINT: {
+        uint32_t v = GET_UINT32_VAL(input);
+        DUPATE_DATA_WITHOUT_TS(pCtx, *(uint32_t *)output, v, notNullElems, isMin);
+        break;
+      }
+
+      case TSDB_DATA_TYPE_UBIGINT: {
+        uint64_t v = GET_UINT64_VAL(input);
+        DUPATE_DATA_WITHOUT_TS(pCtx, *(uint64_t *)output, v, notNullElems, isMin);
+        break;
+      }
+
+      default:
+        break;
+    }
+  }
+  
+  return notNullElems;
+}
+
+static void min_func_merge(SQLFunctionCtx *pCtx) {
+  int32_t notNullElems = minmax_merge_impl(pCtx, pCtx->outputBytes, pCtx->pOutput, 1);
+  
+  SET_VAL(pCtx, notNullElems, 1);
+  
+  SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+  if (notNullElems > 0) {
+    pResInfo->hasResult = DATA_SET_FLAG;
+  }
+}
+
+static void max_func_merge(SQLFunctionCtx *pCtx) {
+  int32_t numOfElem = minmax_merge_impl(pCtx, pCtx->outputBytes, pCtx->pOutput, 0);
+  
+  SET_VAL(pCtx, numOfElem, 1);
+  
+  SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+  if (numOfElem > 0) {
+    pResInfo->hasResult = DATA_SET_FLAG;
+  }
+}
+
+#define LOOP_STDDEV_IMPL(type, r, d, ctx, delta, _type, num)          \
+  for (int32_t i = 0; i < (ctx)->size; ++i) {                         \
+    if ((ctx)->hasNull && isNull((char *)&((type *)d)[i], (_type))) { \
+      continue;                                                       \
+    }                                                                 \
+    (num) += 1;                                                       \
+    (r) += POW2(((type *)d)[i] - (delta));                            \
+  }
+
+static void stddev_function(SQLFunctionCtx *pCtx) {
+  SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+  SStddevInfo *pStd = GET_ROWCELL_INTERBUF(pResInfo);
+
+  if (pCtx->currentStage == REPEAT_SCAN && pStd->stage == 0) {
+    pStd->stage++;
+    avg_finalizer(pCtx);
+
+    pResInfo->initialized = true; // set it initialized to avoid re-initialization
+
+    // save average value into tmpBuf, for second stage scan
+    SAvgInfo *pAvg = GET_ROWCELL_INTERBUF(pResInfo);
+
+    pStd->avg = GET_DOUBLE_VAL(pCtx->pOutput);
+    assert((isnan(pAvg->sum) && pAvg->num == 0) || (pStd->num == pAvg->num && pStd->avg == pAvg->sum));
+  }
+  
+  if (pStd->stage == 0) {
+    // the first stage is to calculate average value
+    avg_function(pCtx);
+  } else if (pStd->num > 0) {
+    // the second stage to calculate standard deviation
+    // if pStd->num == 0, there are no numbers in the first round check. No need to do the second round
+    double *retVal = &pStd->res;
+    double  avg = pStd->avg;
+    
+    void *pData = GET_INPUT_DATA_LIST(pCtx);
+    int32_t num = 0;
+
+    switch (pCtx->inputType) {
+      case TSDB_DATA_TYPE_INT: {
+        for (int32_t i = 0; i < pCtx->size; ++i) {
+          if (pCtx->hasNull && isNull((const char*) (&((int32_t *)pData)[i]), pCtx->inputType)) {
+            continue;
+          }
+          num += 1;
+          *retVal += POW2(((int32_t *)pData)[i] - avg);
+        }
+        break;
+      }
+      case TSDB_DATA_TYPE_FLOAT: {
+        LOOP_STDDEV_IMPL(float, *retVal, pData, pCtx, avg, pCtx->inputType, num);
+        break;
+      }
+      case TSDB_DATA_TYPE_DOUBLE: {
+        LOOP_STDDEV_IMPL(double, *retVal, pData, pCtx, avg, pCtx->inputType, num);
+        break;
+      }
+      case TSDB_DATA_TYPE_BIGINT: {
+        LOOP_STDDEV_IMPL(int64_t, *retVal, pData, pCtx, avg, pCtx->inputType, num);
+        break;
+      }
+      case TSDB_DATA_TYPE_SMALLINT: {
+        LOOP_STDDEV_IMPL(int16_t, *retVal, pData, pCtx, avg, pCtx->inputType, num);
+        break;
+      }
+      case TSDB_DATA_TYPE_TINYINT: {
+        LOOP_STDDEV_IMPL(int8_t, *retVal, pData, pCtx, avg, pCtx->inputType, num);
+        break;
+      }
+      case TSDB_DATA_TYPE_UBIGINT: {
+        LOOP_STDDEV_IMPL(uint64_t, *retVal, pData, pCtx, avg, pCtx->inputType, num);
+        break;
+      }
+      case TSDB_DATA_TYPE_USMALLINT: {
+        LOOP_STDDEV_IMPL(uint16_t, *retVal, pData, pCtx, avg, pCtx->inputType, num);
+        break;
+      }
+      case TSDB_DATA_TYPE_UTINYINT: {
+        LOOP_STDDEV_IMPL(uint8_t, *retVal, pData, pCtx, avg, pCtx->inputType, num);
+        break;
+      }
+      case TSDB_DATA_TYPE_UINT: {
+        LOOP_STDDEV_IMPL(uint32_t, *retVal, pData, pCtx, avg, pCtx->inputType, num);
+        break;
+      }
+      default:
+        assert(0);
+//        qError("stddev function not support data type:%d", pCtx->inputType);
+    }
+    
+    SET_VAL(pCtx, 1, 1);
+  }
+}
+
+static void stddev_finalizer(SQLFunctionCtx *pCtx) {
+  SStddevInfo *pStd = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
+  
+  if (pStd->num <= 0) {
+    setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes);
+  } else {
+    double *retValue = (double *)pCtx->pOutput;
+    SET_DOUBLE_VAL(retValue, sqrt(pStd->res / pStd->num));
+    SET_VAL(pCtx, 1, 1);
+  }
+  
+  doFinalizer(pCtx);
+}
+
+//////////////////////////////////////////////////////////////////////////////////////
+int32_t tsCompare(const void* p1, const void* p2) {
+  TSKEY k = *(TSKEY*)p1;
+  SResPair* pair = (SResPair*)p2;
+
+  if (k == pair->key) {
+    return 0;
+  } else {
+    return k < pair->key? -1:1;
+  }
+}
+
+static void stddev_dst_function(SQLFunctionCtx *pCtx) {
+  SStddevdstInfo *pStd = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
+
+  // the second stage to calculate standard deviation
+  double *retVal = &pStd->res;
+
+  // all data are null, no need to proceed
+  SArray* resList = (SArray*) pCtx->param[0].pz;
+  if (resList == NULL) {
+    return;
+  }
+
+  // find the correct group average results according to the tag value
+  int32_t len = (int32_t) taosArrayGetSize(resList);
+  assert(len > 0);
+
+  double avg = 0;
+  if (len == 1) {
+    SResPair* p = taosArrayGet(resList, 0);
+    avg = p->avg;
+  } else {  // todo opt performance by using iterator since the timestamp lsit is matched with the output result
+    SResPair* p = bsearch(&pCtx->startTs, resList->pData, len, sizeof(SResPair), tsCompare);
+    if (p == NULL) {
+      return;
+    }
+
+    avg = p->avg;
+  }
+
+  void *pData = GET_INPUT_DATA_LIST(pCtx);
+  int32_t num = 0;
+
+  switch (pCtx->inputType) {
+    case TSDB_DATA_TYPE_INT: {
+      for (int32_t i = 0; i < pCtx->size; ++i) {
+        if (pCtx->hasNull && isNull((const char*) (&((int32_t *)pData)[i]), pCtx->inputType)) {
+          continue;
+        }
+        num += 1;
+        *retVal += POW2(((int32_t *)pData)[i] - avg);
+      }
+      break;
+    }
+    case TSDB_DATA_TYPE_FLOAT: {
+      LOOP_STDDEV_IMPL(float, *retVal, pData, pCtx, avg, pCtx->inputType, num);
+      break;
+    }
+    case TSDB_DATA_TYPE_DOUBLE: {
+      LOOP_STDDEV_IMPL(double, *retVal, pData, pCtx, avg, pCtx->inputType, num);
+      break;
+    }
+    case TSDB_DATA_TYPE_TINYINT: {
+      LOOP_STDDEV_IMPL(int8_t, *retVal, pData, pCtx, avg, pCtx->inputType, num);
+      break;
+    }
+    case TSDB_DATA_TYPE_UTINYINT: {
+      LOOP_STDDEV_IMPL(int8_t, *retVal, pData, pCtx, avg, pCtx->inputType, num);
+      break;
+    }
+    case TSDB_DATA_TYPE_SMALLINT: {
+      LOOP_STDDEV_IMPL(int16_t, *retVal, pData, pCtx, avg, pCtx->inputType, num);
+      break;
+    }
+    case TSDB_DATA_TYPE_USMALLINT: {
+      LOOP_STDDEV_IMPL(uint16_t, *retVal, pData, pCtx, avg, pCtx->inputType, num);
+      break;
+    }
+    case TSDB_DATA_TYPE_UINT: {
+      LOOP_STDDEV_IMPL(uint32_t, *retVal, pData, pCtx, avg, pCtx->inputType, num);
+      break;
+    }
+    case TSDB_DATA_TYPE_BIGINT: {
+      LOOP_STDDEV_IMPL(int64_t, *retVal, pData, pCtx, avg, pCtx->inputType, num);
+      break;
+    }
+    case TSDB_DATA_TYPE_UBIGINT: {
+      LOOP_STDDEV_IMPL(uint64_t, *retVal, pData, pCtx, avg, pCtx->inputType, num);
+      break;
+    }
+    default:
+      assert(0);
+//      qError("stddev function not support data type:%d", pCtx->inputType);
+  }
+
+  pStd->num += num;
+  SET_VAL(pCtx, num, 1);
+
+  // copy to the final output buffer for super table
+  memcpy(pCtx->pOutput, GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)), sizeof(SAvgInfo));
+}
+
+static void stddev_dst_merge(SQLFunctionCtx *pCtx) {
+  SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+  SStddevdstInfo* pRes = GET_ROWCELL_INTERBUF(pResInfo);
+
+  char   *input = GET_INPUT_DATA_LIST(pCtx);
+
+  for (int32_t i = 0; i < pCtx->size; ++i, input += pCtx->inputBytes) {
+    SStddevdstInfo *pInput = (SStddevdstInfo *)input;
+    if (pInput->num == 0) {  // current input is null
+      continue;
+    }
+
+    pRes->num += pInput->num;
+    pRes->res += pInput->res;
+  }
+}
+
+static void stddev_dst_finalizer(SQLFunctionCtx *pCtx) {
+  SStddevdstInfo *pStd = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
+
+  if (pStd->num <= 0) {
+    setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes);
+  } else {
+    double *retValue = (double *)pCtx->pOutput;
+    SET_DOUBLE_VAL(retValue, sqrt(pStd->res / pStd->num));
+    SET_VAL(pCtx, 1, 1);
+  }
+
+  doFinalizer(pCtx);
+}
+
+//////////////////////////////////////////////////////////////////////////////////////
+static bool first_last_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResInfo) {
+  if (!function_setup(pCtx, pResInfo)) {
+    return false;
+  }
+  
+  // used to keep the timestamp for comparison
+  pCtx->param[1].nType = 0;
+  pCtx->param[1].i64 = 0;
+  
+  return true;
+}
+
+// todo opt for null block
+static void first_function(SQLFunctionCtx *pCtx) {
+  if (pCtx->order == TSDB_ORDER_DESC) {
+    return;
+  }
+  
+  int32_t notNullElems = 0;
+  
+  // handle the null value
+  for (int32_t i = 0; i < pCtx->size; ++i) {
+    char *data = GET_INPUT_DATA(pCtx, i);
+    if (pCtx->hasNull && isNull(data, pCtx->inputType)) {
+      continue;
+    }
+    
+    memcpy(pCtx->pOutput, data, pCtx->inputBytes);
+    if (pCtx->ptsList != NULL) {
+      TSKEY k = GET_TS_DATA(pCtx, i);
+      DO_UPDATE_TAG_COLUMNS(pCtx, k);
+    }
+
+    SResultRowCellInfo *pInfo = GET_RES_INFO(pCtx);
+    pInfo->hasResult = DATA_SET_FLAG;
+    pInfo->complete = true;
+    
+    notNullElems++;
+    break;
+  }
+  
+  SET_VAL(pCtx, notNullElems, 1);
+}
+
+static void first_data_assign_impl(SQLFunctionCtx *pCtx, char *pData, int32_t index) {
+  int64_t *timestamp = GET_TS_LIST(pCtx);
+  
+  SFirstLastInfo *pInfo = (SFirstLastInfo *)(pCtx->pOutput + pCtx->inputBytes);
+  
+  if (pInfo->hasResult != DATA_SET_FLAG || timestamp[index] < pInfo->ts) {
+    memcpy(pCtx->pOutput, pData, pCtx->inputBytes);
+    pInfo->hasResult = DATA_SET_FLAG;
+    pInfo->ts = timestamp[index];
+    
+    DO_UPDATE_TAG_COLUMNS(pCtx, pInfo->ts);
+  }
+}
+
+/*
+ * format of intermediate result: "timestamp,value" need to compare the timestamp in the first part (before the comma)
+ * to decide if the value is earlier than current intermediate result
+ */
+static void first_dist_function(SQLFunctionCtx *pCtx) {
+  /*
+   * do not to check data in the following cases:
+   * 1. data block that are not loaded
+   * 2. scan data files in desc order
+   */
+  if (pCtx->order == TSDB_ORDER_DESC) {
+    return;
+  }
+  
+  int32_t notNullElems = 0;
+
+  // find the first not null value
+  for (int32_t i = 0; i < pCtx->size; ++i) {
+    char *data = GET_INPUT_DATA(pCtx, i);
+    if (pCtx->hasNull && isNull(data, pCtx->inputType)) {
+      continue;
+    }
+    
+    first_data_assign_impl(pCtx, data, i);
+    
+    SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+    pResInfo->hasResult = DATA_SET_FLAG;
+    
+    notNullElems++;
+    break;
+  }
+  
+  SET_VAL(pCtx, notNullElems, 1);
+}
+
+static void first_dist_func_merge(SQLFunctionCtx *pCtx) {
+  assert(pCtx->stableQuery);
+
+  char *          pData = GET_INPUT_DATA_LIST(pCtx);
+  SFirstLastInfo *pInput = (SFirstLastInfo*) (pData + pCtx->outputBytes);
+  if (pInput->hasResult != DATA_SET_FLAG) {
+    return;
+  }
+  
+  // The param[1] is used to keep the initial value of max ts value
+  if (pCtx->param[1].nType != pCtx->outputType || pCtx->param[1].i64 > pInput->ts) {
+    memcpy(pCtx->pOutput, pData, pCtx->outputBytes);
+    pCtx->param[1].i64 = pInput->ts;
+    pCtx->param[1].nType = pCtx->outputType;
+    
+    DO_UPDATE_TAG_COLUMNS(pCtx, pInput->ts);
+  }
+  
+  SET_VAL(pCtx, 1, 1);
+  GET_RES_INFO(pCtx)->hasResult = DATA_SET_FLAG;
+}
+
+//////////////////////////////////////////////////////////////////////////////////////////
+/*
+ * last function:
+ * 1. since the last block may be all null value, so, we simply access the last block is not valid
+ *    each block need to be checked.
+ * 2. If numOfNull == pBlock->numOfBlocks, the whole block is empty. Otherwise, there is at
+ *    least one data in this block that is not null.(TODO opt for this case)
+ */
+static void last_function(SQLFunctionCtx *pCtx) {
+  if (pCtx->order != pCtx->param[0].i64) {
+    return;
+  }
+
+  SResultRowCellInfo* pResInfo = GET_RES_INFO(pCtx);
+
+  int32_t notNullElems = 0;
+  if (pCtx->order == TSDB_ORDER_DESC) {
+
+    for (int32_t i = pCtx->size - 1; i >= 0; --i) {
+      char *data = GET_INPUT_DATA(pCtx, i);
+      if (pCtx->hasNull && isNull(data, pCtx->inputType) && (!pCtx->requireNull)) {
+        continue;
+      }
+
+      memcpy(pCtx->pOutput, data, pCtx->inputBytes);
+
+      TSKEY ts = pCtx->ptsList ? GET_TS_DATA(pCtx, i) : 0;
+      DO_UPDATE_TAG_COLUMNS(pCtx, ts);
+
+      pResInfo->hasResult = DATA_SET_FLAG;
+      pResInfo->complete = true;  // set query completed on this column
+      notNullElems++;
+      break;
+    }
+  } else {  // ascending order
+    for (int32_t i = pCtx->size - 1; i >= 0; --i) {
+      char *data = GET_INPUT_DATA(pCtx, i);
+      if (pCtx->hasNull && isNull(data, pCtx->inputType) && (!pCtx->requireNull)) {
+        continue;
+      }
+
+      TSKEY ts = pCtx->ptsList ? GET_TS_DATA(pCtx, i) : 0;
+
+      char* buf = GET_ROWCELL_INTERBUF(pResInfo);
+      if (pResInfo->hasResult != DATA_SET_FLAG || (*(TSKEY*)buf) < ts) {
+        pResInfo->hasResult = DATA_SET_FLAG;
+        memcpy(pCtx->pOutput, data, pCtx->inputBytes);
+
+        *(TSKEY*)buf = ts;
+        DO_UPDATE_TAG_COLUMNS(pCtx, ts);
+      }
+
+      notNullElems++;
+      break;
+    }
+  }
+
+  SET_VAL(pCtx, notNullElems, 1);
+}
+
+static void last_data_assign_impl(SQLFunctionCtx *pCtx, char *pData, int32_t index) {
+  int64_t *timestamp = GET_TS_LIST(pCtx);
+  
+  SFirstLastInfo *pInfo = (SFirstLastInfo *)(pCtx->pOutput + pCtx->inputBytes);
+  
+  if (pInfo->hasResult != DATA_SET_FLAG || pInfo->ts < timestamp[index]) {
+#if defined(_DEBUG_VIEW)
+    qDebug("assign index:%d, ts:%" PRId64 ", val:%d, ", index, timestamp[index], *(int32_t *)pData);
+#endif
+    
+    memcpy(pCtx->pOutput, pData, pCtx->inputBytes);
+    pInfo->hasResult = DATA_SET_FLAG;
+    pInfo->ts = timestamp[index];
+    
+    DO_UPDATE_TAG_COLUMNS(pCtx, pInfo->ts);
+  }
+}
+
+static void last_dist_function(SQLFunctionCtx *pCtx) {
+  /*
+   * 1. for scan data is not the required order
+   * 2. for data blocks that are not loaded, no need to check data
+   */
+  if (pCtx->order != pCtx->param[0].i64) {
+    return;
+  }
+
+  int32_t notNullElems = 0;
+  for (int32_t i = pCtx->size - 1; i >= 0; --i) {
+    char *data = GET_INPUT_DATA(pCtx, i);
+    if (pCtx->hasNull && isNull(data, pCtx->inputType)) {
+      if (!pCtx->requireNull) {
+        continue; 
+      }
+    }
+    
+    last_data_assign_impl(pCtx, data, i);
+    
+    SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+    pResInfo->hasResult = DATA_SET_FLAG;
+    
+    notNullElems++;
+    break;
+  }
+  
+  SET_VAL(pCtx, notNullElems, 1);
+}
+
+/*
+ * in the secondary merge(local reduce), the output is limited by the
+ * final output size, so the main difference between last_dist_func_merge and second_merge
+ * is: the output data format in computing
+ */
+static void last_dist_func_merge(SQLFunctionCtx *pCtx) {
+  char *pData = GET_INPUT_DATA_LIST(pCtx);
+  
+  SFirstLastInfo *pInput = (SFirstLastInfo*) (pData + pCtx->outputBytes);
+  if (pInput->hasResult != DATA_SET_FLAG) {
+    return;
+  }
+  
+  /*
+   * param[1] used to keep the corresponding timestamp to decide if current result is
+   * the true last result
+   */
+  if (pCtx->param[1].nType != pCtx->outputType || pCtx->param[1].i64 < pInput->ts) {
+    memcpy(pCtx->pOutput, pData, pCtx->outputBytes);
+    pCtx->param[1].i64 = pInput->ts;
+    pCtx->param[1].nType = pCtx->outputType;
+    
+    DO_UPDATE_TAG_COLUMNS(pCtx, pInput->ts);
+  }
+  
+  SET_VAL(pCtx, 1, 1);
+  GET_RES_INFO(pCtx)->hasResult = DATA_SET_FLAG;
+}
+
+//////////////////////////////////////////////////////////////////////////////////
+/*
+ * NOTE: last_row does not use the interResultBuf to keep the result
+ */
+static void last_row_function(SQLFunctionCtx *pCtx) {
+  assert(pCtx->size >= 1);
+  char *pData = GET_INPUT_DATA_LIST(pCtx);
+
+  // assign the last element in current data block
+  assignVal(pCtx->pOutput, pData + (pCtx->size - 1) * pCtx->inputBytes, pCtx->inputBytes, pCtx->inputType);
+  
+  SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+  pResInfo->hasResult = DATA_SET_FLAG;
+  
+  // set the result to final result buffer in case of super table query
+  if (pCtx->stableQuery) {
+    SLastrowInfo *pInfo1 = (SLastrowInfo *)(pCtx->pOutput + pCtx->inputBytes);
+    pInfo1->ts = GET_TS_DATA(pCtx, pCtx->size - 1);
+    pInfo1->hasResult = DATA_SET_FLAG;
+    
+    DO_UPDATE_TAG_COLUMNS(pCtx, pInfo1->ts);
+  } else {
+    TSKEY ts = GET_TS_DATA(pCtx, pCtx->size - 1);
+    DO_UPDATE_TAG_COLUMNS(pCtx, ts);
+  }
+
+  SET_VAL(pCtx, pCtx->size, 1);
+}
+
+static void last_row_finalizer(SQLFunctionCtx *pCtx) {
+  // do nothing at the first stage
+  SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+  if (pResInfo->hasResult != DATA_SET_FLAG) {
+    setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes);
+    return;
+  }
+  
+  GET_RES_INFO(pCtx)->numOfRes = 1;
+  doFinalizer(pCtx);
+}
+
+//////////////////////////////////////////////////////////////////////////////////
+static void valuePairAssign(tValuePair *dst, int16_t type, const char *val, int64_t tsKey, char *pTags,
+                            SExtTagsInfo *pTagInfo, int16_t stage) {
+  dst->v.nType = type;
+  dst->v.i64 = *(int64_t *)val;
+  dst->timestamp = tsKey;
+  
+  int32_t size = 0;
+  if (stage == MERGE_STAGE) {
+    memcpy(dst->pTags, pTags, (size_t)pTagInfo->tagsLen);
+  } else {  // the tags are dumped from the ctx tag fields
+    for (int32_t i = 0; i < pTagInfo->numOfTagCols; ++i) {
+      SQLFunctionCtx* ctx = pTagInfo->pTagCtxList[i];
+      if (ctx->functionId == FUNCTION_TS_DUMMY) {
+        ctx->tag.nType = TSDB_DATA_TYPE_BIGINT;
+        ctx->tag.i64 = tsKey;
+      }
+      
+      taosVariantDump(&ctx->tag, dst->pTags + size, ctx->tag.nType, true);
+      size += pTagInfo->pTagCtxList[i]->outputBytes;
+    }
+  }
+}
+
+#define VALUEPAIRASSIGN(dst, src, __l)                 \
+  do {                                                 \
+    (dst)->timestamp = (src)->timestamp;               \
+    (dst)->v = (src)->v;                               \
+    memcpy((dst)->pTags, (src)->pTags, (size_t)(__l)); \
+  } while (0)
+
+static int32_t topBotComparFn(const void *p1, const void *p2, const void *param)
+{
+  uint16_t     type = *(uint16_t *) param;
+  tValuePair  *val1 = *(tValuePair **) p1;
+  tValuePair  *val2 = *(tValuePair **) p2;
+
+  if (IS_SIGNED_NUMERIC_TYPE(type)) {
+    if (val1->v.i64 == val2->v.i64) {
+      return 0;
+    }
+
+    return (val1->v.i64 > val2->v.i64) ? 1 : -1;
+   } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) {
+     if (val1->v.u64 == val2->v.u64) {
+       return 0;
+     }
+
+     return (val1->v.u64 > val2->v.u64) ? 1 : -1;
+  }
+
+  if (val1->v.d == val2->v.d) {
+    return 0;
+  }
+
+  return (val1->v.d > val2->v.d) ? 1 : -1;
+}
+
+static void topBotSwapFn(void *dst, void *src, const void *param)
+{
+  char         tag[32768];
+  tValuePair   temp;
+  uint16_t     tagLen = *(uint16_t *) param;
+  tValuePair  *vdst = *(tValuePair **) dst;
+  tValuePair  *vsrc = *(tValuePair **) src;
+
+  memset(tag, 0, sizeof(tag));
+  temp.pTags = tag;
+
+  VALUEPAIRASSIGN(&temp, vdst, tagLen);
+  VALUEPAIRASSIGN(vdst, vsrc, tagLen);
+  VALUEPAIRASSIGN(vsrc, &temp, tagLen);
+}
+
+static void do_top_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pData, int64_t ts, uint16_t type,
+                                SExtTagsInfo *pTagInfo, char *pTags, int16_t stage) {
+  SVariant val = {0};
+  taosVariantCreateFromBinary(&val, pData, tDataTypes[type].bytes, type);
+  
+  tValuePair **pList = pInfo->res;
+  assert(pList != NULL);
+
+  if (pInfo->num < maxLen) {
+    valuePairAssign(pList[pInfo->num], type, (const char *)&val.i64, ts, pTags, pTagInfo, stage);
+
+    taosheapsort((void *) pList, sizeof(tValuePair **), pInfo->num + 1, (const void *) &type, topBotComparFn, (const void *) &pTagInfo->tagsLen, topBotSwapFn, 0);
+ 
+    pInfo->num++;
+  } else {
+    if ((IS_SIGNED_NUMERIC_TYPE(type) && val.i64 > pList[0]->v.i64) ||
+        (IS_UNSIGNED_NUMERIC_TYPE(type) && val.u64 > pList[0]->v.u64) ||
+        (IS_FLOAT_TYPE(type) && val.d > pList[0]->v.d)) {
+      valuePairAssign(pList[0], type, (const char *)&val.i64, ts, pTags, pTagInfo, stage);
+      taosheapadjust((void *) pList, sizeof(tValuePair **), 0, maxLen - 1, (const void *) &type, topBotComparFn, (const void *) &pTagInfo->tagsLen, topBotSwapFn, 0);
+    }
+  }
+}
+
+static void do_bottom_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pData, int64_t ts, uint16_t type,
+                                   SExtTagsInfo *pTagInfo, char *pTags, int16_t stage) {
+  SVariant val = {0};
+  taosVariantCreateFromBinary(&val, pData, tDataTypes[type].bytes, type);
+
+  tValuePair **pList = pInfo->res;
+  assert(pList != NULL);
+
+  if (pInfo->num < maxLen) {
+    valuePairAssign(pList[pInfo->num], type, (const char *)&val.i64, ts, pTags, pTagInfo, stage);
+
+    taosheapsort((void *) pList, sizeof(tValuePair **), pInfo->num + 1, (const void *) &type, topBotComparFn, (const void *) &pTagInfo->tagsLen, topBotSwapFn, 1);
+
+    pInfo->num++;
+  } else {
+    if ((IS_SIGNED_NUMERIC_TYPE(type) && val.i64 < pList[0]->v.i64) ||
+        (IS_UNSIGNED_NUMERIC_TYPE(type) && val.u64 < pList[0]->v.u64) ||
+        (IS_FLOAT_TYPE(type) && val.d < pList[0]->v.d)) {
+      valuePairAssign(pList[0], type, (const char *)&val.i64, ts, pTags, pTagInfo, stage);
+      taosheapadjust((void *) pList, sizeof(tValuePair **), 0, maxLen - 1, (const void *) &type, topBotComparFn, (const void *) &pTagInfo->tagsLen, topBotSwapFn, 1);
+    }
+  }
+}
+
+static int32_t resAscComparFn(const void *pLeft, const void *pRight) {
+  tValuePair *pLeftElem = *(tValuePair **)pLeft;
+  tValuePair *pRightElem = *(tValuePair **)pRight;
+  
+  if (pLeftElem->timestamp == pRightElem->timestamp) {
+    return 0;
+  } else {
+    return pLeftElem->timestamp > pRightElem->timestamp ? 1 : -1;
+  }
+}
+
+static int32_t resDescComparFn(const void *pLeft, const void *pRight) { return -resAscComparFn(pLeft, pRight); }
+
+static int32_t resDataAscComparFn(const void *pLeft, const void *pRight) {
+  tValuePair *pLeftElem = *(tValuePair **)pLeft;
+  tValuePair *pRightElem = *(tValuePair **)pRight;
+  
+  if (IS_FLOAT_TYPE(pLeftElem->v.nType)) {
+    if (pLeftElem->v.d == pRightElem->v.d) {
+      return 0;
+    } else {
+      return pLeftElem->v.d > pRightElem->v.d ? 1 : -1;
+    }
+  } else if (IS_SIGNED_NUMERIC_TYPE(pLeftElem->v.nType)){
+    if (pLeftElem->v.i64 == pRightElem->v.i64) {
+      return 0;
+    } else {
+      return pLeftElem->v.i64 > pRightElem->v.i64 ? 1 : -1;
+    }
+  } else {
+    if (pLeftElem->v.u64 == pRightElem->v.u64) {
+      return 0;
+    } else {
+      return pLeftElem->v.u64 > pRightElem->v.u64 ? 1 : -1;
+    }
+  }
+}
+
+static int32_t resDataDescComparFn(const void *pLeft, const void *pRight) { return -resDataAscComparFn(pLeft, pRight); }
+
+static void copyTopBotRes(SQLFunctionCtx *pCtx, int32_t type) {
+  SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+  STopBotInfo *pRes = GET_ROWCELL_INTERBUF(pResInfo);
+  
+  tValuePair **tvp = pRes->res;
+  
+  int32_t step = QUERY_ASC_FORWARD_STEP;
+  int32_t len = (int32_t)(GET_RES_INFO(pCtx)->numOfRes);
+  
+  switch (type) {
+    case TSDB_DATA_TYPE_UINT:
+    case TSDB_DATA_TYPE_INT: {
+      int32_t *output = (int32_t *)pCtx->pOutput;
+      for (int32_t i = 0; i < len; ++i, output += step) {
+        *output = (int32_t)tvp[i]->v.i64;
+      }
+      break;
+    }
+    case TSDB_DATA_TYPE_UBIGINT:
+    case TSDB_DATA_TYPE_BIGINT: {
+      int64_t *output = (int64_t *)pCtx->pOutput;
+      for (int32_t i = 0; i < len; ++i, output += step) {
+        *output = tvp[i]->v.i64;
+      }
+      break;
+    }
+    case TSDB_DATA_TYPE_DOUBLE: {
+      double *output = (double *)pCtx->pOutput;
+      for (int32_t i = 0; i < len; ++i, output += step) {
+        SET_DOUBLE_VAL(output, tvp[i]->v.d);
+      }
+      break;
+    }
+    case TSDB_DATA_TYPE_FLOAT: {
+      float *output = (float *)pCtx->pOutput;
+      for (int32_t i = 0; i < len; ++i, output += step) {
+        *output = (float)tvp[i]->v.d;
+      }
+      break;
+    }
+    case TSDB_DATA_TYPE_USMALLINT:
+    case TSDB_DATA_TYPE_SMALLINT: {
+      int16_t *output = (int16_t *)pCtx->pOutput;
+      for (int32_t i = 0; i < len; ++i, output += step) {
+        *output = (int16_t)tvp[i]->v.i64;
+      }
+      break;
+    }
+    case TSDB_DATA_TYPE_UTINYINT:
+    case TSDB_DATA_TYPE_TINYINT: {
+      int8_t *output = (int8_t *)pCtx->pOutput;
+      for (int32_t i = 0; i < len; ++i, output += step) {
+        *output = (int8_t)tvp[i]->v.i64;
+      }
+      break;
+    }
+    default: {
+//      qError("top/bottom function not support data type:%d", pCtx->inputType);
+      return;
+    }
+  }
+  
+  // set the output timestamp of each record.
+  TSKEY *output = pCtx->ptsOutputBuf;
+  for (int32_t i = 0; i < len; ++i, output += step) {
+    *output = tvp[i]->timestamp;
+  }
+  
+  // set the corresponding tag data for each record
+  // todo check malloc failure
+  char **pData = calloc(pCtx->tagInfo.numOfTagCols, POINTER_BYTES);
+  for (int32_t i = 0; i < pCtx->tagInfo.numOfTagCols; ++i) {
+    pData[i] = pCtx->tagInfo.pTagCtxList[i]->pOutput;
+  }
+  
+  for (int32_t i = 0; i < len; ++i, output += step) {
+    int16_t offset = 0;
+    for (int32_t j = 0; j < pCtx->tagInfo.numOfTagCols; ++j) {
+      memcpy(pData[j], tvp[i]->pTags + offset, (size_t)pCtx->tagInfo.pTagCtxList[j]->outputBytes);
+      offset += pCtx->tagInfo.pTagCtxList[j]->outputBytes;
+      pData[j] += pCtx->tagInfo.pTagCtxList[j]->outputBytes;
+    }
+  }
+  
+  tfree(pData);
+}
+
+/*
+ * Parameters values:
+ * 1. param[0]: maximum allowable results
+ * 2. param[1]: order by type (time or value)
+ * 3. param[2]: asc/desc order
+ *
+ * top/bottom use the intermediate result buffer to keep the intermediate result
+ */
+static STopBotInfo *getTopBotOutputInfo(SQLFunctionCtx *pCtx) {
+  SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+
+  // only the first_stage_merge is directly written data into final output buffer
+  if (pCtx->stableQuery && pCtx->currentStage != MERGE_STAGE) {
+    return (STopBotInfo*) pCtx->pOutput;
+  } else { // during normal table query and super table at the secondary_stage, result is written to intermediate buffer
+    return GET_ROWCELL_INTERBUF(pResInfo);
+  }
+}
+
+
+/*
+ * keep the intermediate results during scan data blocks in the format of:
+ * +-----------------------------------+-------------one value pair-----------+------------next value pair-----------+
+ * |-------------pointer area----------|----ts---+-----+-----n tags-----------|----ts---+-----+-----n tags-----------|
+ * +..[Value Pointer1][Value Pointer2].|timestamp|value|tags1|tags2|....|tagsn|timestamp|value|tags1|tags2|....|tagsn+
+ */
+static void buildTopBotStruct(STopBotInfo *pTopBotInfo, SQLFunctionCtx *pCtx) {
+  char *tmp = (char *)pTopBotInfo + sizeof(STopBotInfo);
+  pTopBotInfo->res = (tValuePair**) tmp;
+  tmp += POINTER_BYTES * pCtx->param[0].i64;
+
+  size_t size = sizeof(tValuePair) + pCtx->tagInfo.tagsLen;
+
+  for (int32_t i = 0; i < pCtx->param[0].i64; ++i) {
+    pTopBotInfo->res[i] = (tValuePair*) tmp;
+    pTopBotInfo->res[i]->pTags = tmp + sizeof(tValuePair);
+    tmp += size;
+  }
+}
+
+bool topbot_datablock_filter(SQLFunctionCtx *pCtx, const char *minval, const char *maxval) {
+  SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+  if (pResInfo == NULL) {
+    return true;
+  }
+
+  STopBotInfo *pTopBotInfo = getTopBotOutputInfo(pCtx);
+  
+  // required number of results are not reached, continue load data block
+  if (pTopBotInfo->num < pCtx->param[0].i64) {
+    return true;
+  }
+  
+  if ((void *)pTopBotInfo->res[0] != (void *)((char *)pTopBotInfo + sizeof(STopBotInfo) + POINTER_BYTES * pCtx->param[0].i64)) {
+    buildTopBotStruct(pTopBotInfo, pCtx);
+  }
+
+  tValuePair **pRes = (tValuePair**) pTopBotInfo->res;
+  
+  if (pCtx->functionId == FUNCTION_TOP) {
+    switch (pCtx->inputType) {
+      case TSDB_DATA_TYPE_TINYINT:
+        return GET_INT8_VAL(maxval) > pRes[0]->v.i64;
+      case TSDB_DATA_TYPE_SMALLINT:
+        return GET_INT16_VAL(maxval) > pRes[0]->v.i64;
+      case TSDB_DATA_TYPE_INT:
+        return GET_INT32_VAL(maxval) > pRes[0]->v.i64;
+      case TSDB_DATA_TYPE_BIGINT:
+        return GET_INT64_VAL(maxval) > pRes[0]->v.i64;
+      case TSDB_DATA_TYPE_FLOAT:
+        return GET_FLOAT_VAL(maxval) > pRes[0]->v.d;
+      case TSDB_DATA_TYPE_DOUBLE:
+        return GET_DOUBLE_VAL(maxval) > pRes[0]->v.d;
+      default:
+        return true;
+    }
+  } else {
+    switch (pCtx->inputType) {
+      case TSDB_DATA_TYPE_TINYINT:
+        return GET_INT8_VAL(minval) < pRes[0]->v.i64;
+      case TSDB_DATA_TYPE_SMALLINT:
+        return GET_INT16_VAL(minval) < pRes[0]->v.i64;
+      case TSDB_DATA_TYPE_INT:
+        return GET_INT32_VAL(minval) < pRes[0]->v.i64;
+      case TSDB_DATA_TYPE_BIGINT:
+        return GET_INT64_VAL(minval) < pRes[0]->v.i64;
+      case TSDB_DATA_TYPE_FLOAT:
+        return GET_FLOAT_VAL(minval) < pRes[0]->v.d;
+      case TSDB_DATA_TYPE_DOUBLE:
+        return GET_DOUBLE_VAL(minval) < pRes[0]->v.d;
+      default:
+        return true;
+    }
+  }
+}
+
+static bool top_bottom_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResInfo) {
+  if (!function_setup(pCtx, pResInfo)) {
+    return false;
+  }
+  
+  STopBotInfo *pInfo = getTopBotOutputInfo(pCtx);
+  buildTopBotStruct(pInfo, pCtx);
+  return true;
+}
+
+static void top_function(SQLFunctionCtx *pCtx) {
+  int32_t notNullElems = 0;
+
+  STopBotInfo *pRes = getTopBotOutputInfo(pCtx);
+  assert(pRes->num >= 0);
+
+  if ((void *)pRes->res[0] != (void *)((char *)pRes + sizeof(STopBotInfo) + POINTER_BYTES * pCtx->param[0].i64)) {
+    buildTopBotStruct(pRes, pCtx);
+  }
+  
+  for (int32_t i = 0; i < pCtx->size; ++i) {
+    char *data = GET_INPUT_DATA(pCtx, i);
+    if (pCtx->hasNull && isNull(data, pCtx->inputType)) {
+      continue;
+    }
+    
+    notNullElems++;
+
+    // NOTE: Set the default timestamp if it is missing [todo refactor]
+    TSKEY ts = (pCtx->ptsList != NULL)? GET_TS_DATA(pCtx, i):0;
+    do_top_function_add(pRes, (int32_t)pCtx->param[0].i64, data, ts, pCtx->inputType, &pCtx->tagInfo, NULL, 0);
+  }
+  
+  if (!pCtx->hasNull) {
+    assert(pCtx->size == notNullElems);
+  }
+  
+  // treat the result as only one result
+  SET_VAL(pCtx, notNullElems, 1);
+  
+  if (notNullElems > 0) {
+    SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+    pResInfo->hasResult = DATA_SET_FLAG;
+  }
+}
+
+static void top_func_merge(SQLFunctionCtx *pCtx) {
+  STopBotInfo *pInput = (STopBotInfo *)GET_INPUT_DATA_LIST(pCtx);
+  
+  // construct the input data struct from binary data
+  buildTopBotStruct(pInput, pCtx);
+  
+  STopBotInfo *pOutput = getTopBotOutputInfo(pCtx);
+  
+  // the intermediate result is binary, we only use the output data type
+  for (int32_t i = 0; i < pInput->num; ++i) {
+    int16_t type = (pCtx->outputType == TSDB_DATA_TYPE_FLOAT)? TSDB_DATA_TYPE_DOUBLE:pCtx->outputType;
+    do_top_function_add(pOutput, (int32_t)pCtx->param[0].i64, &pInput->res[i]->v.i64, pInput->res[i]->timestamp,
+                        type, &pCtx->tagInfo, pInput->res[i]->pTags, pCtx->currentStage);
+  }
+  
+  SET_VAL(pCtx, pInput->num, pOutput->num);
+  
+  if (pOutput->num > 0) {
+    SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+    pResInfo->hasResult = DATA_SET_FLAG;
+  }
+}
+
+static void bottom_function(SQLFunctionCtx *pCtx) {
+  int32_t notNullElems = 0;
+  
+  STopBotInfo *pRes = getTopBotOutputInfo(pCtx);
+  
+  if ((void *)pRes->res[0] != (void *)((char *)pRes + sizeof(STopBotInfo) + POINTER_BYTES * pCtx->param[0].i64)) {
+    buildTopBotStruct(pRes, pCtx);
+  }
+
+  for (int32_t i = 0; i < pCtx->size; ++i) {
+    char *data = GET_INPUT_DATA(pCtx, i);
+    if (pCtx->hasNull && isNull(data, pCtx->inputType)) {
+      continue;
+    }
+
+    notNullElems++;
+    // NOTE: Set the default timestamp if it is missing [todo refactor]
+    TSKEY ts = (pCtx->ptsList != NULL)? GET_TS_DATA(pCtx, i):0;
+    do_bottom_function_add(pRes, (int32_t)pCtx->param[0].i64, data, ts, pCtx->inputType, &pCtx->tagInfo, NULL, 0);
+  }
+  
+  if (!pCtx->hasNull) {
+    assert(pCtx->size == notNullElems);
+  }
+  
+  // treat the result as only one result
+  SET_VAL(pCtx, notNullElems, 1);
+  
+  if (notNullElems > 0) {
+    SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+    pResInfo->hasResult = DATA_SET_FLAG;
+  }
+}
+
+static void bottom_func_merge(SQLFunctionCtx *pCtx) {
+  STopBotInfo *pInput = (STopBotInfo *)GET_INPUT_DATA_LIST(pCtx);
+  
+  // construct the input data struct from binary data
+  buildTopBotStruct(pInput, pCtx);
+  
+  STopBotInfo *pOutput = getTopBotOutputInfo(pCtx);
+  
+  // the intermediate result is binary, we only use the output data type
+  for (int32_t i = 0; i < pInput->num; ++i) {
+    int16_t type = (pCtx->outputType == TSDB_DATA_TYPE_FLOAT) ? TSDB_DATA_TYPE_DOUBLE : pCtx->outputType;
+    do_bottom_function_add(pOutput, (int32_t)pCtx->param[0].i64, &pInput->res[i]->v.i64, pInput->res[i]->timestamp, type,
+                           &pCtx->tagInfo, pInput->res[i]->pTags, pCtx->currentStage);
+  }
+
+  SET_VAL(pCtx, pInput->num, pOutput->num);
+  
+  if (pOutput->num > 0) {
+    SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+    pResInfo->hasResult = DATA_SET_FLAG;
+  }
+}
+
+static void top_bottom_func_finalizer(SQLFunctionCtx *pCtx) {
+  SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+  
+  // data in temporary list is less than the required number of results, not enough qualified number of results
+  STopBotInfo *pRes = GET_ROWCELL_INTERBUF(pResInfo);
+  if (pRes->num == 0) {  // no result
+    assert(pResInfo->hasResult != DATA_SET_FLAG);
+    // TODO:
+  }
+  
+  GET_RES_INFO(pCtx)->numOfRes = pRes->num;
+  tValuePair **tvp = pRes->res;
+  
+  // user specify the order of output by sort the result according to timestamp
+  if (pCtx->param[1].i64 == PRIMARYKEY_TIMESTAMP_COL_INDEX) {
+    __compar_fn_t comparator = (pCtx->param[2].i64 == TSDB_ORDER_ASC) ? resAscComparFn : resDescComparFn;
+    qsort(tvp, (size_t)pResInfo->numOfRes, POINTER_BYTES, comparator);
+  } else /*if (pCtx->param[1].i64 > PRIMARYKEY_TIMESTAMP_COL_INDEX)*/ {
+    __compar_fn_t comparator = (pCtx->param[2].i64 == TSDB_ORDER_ASC) ? resDataAscComparFn : resDataDescComparFn;
+    qsort(tvp, (size_t)pResInfo->numOfRes, POINTER_BYTES, comparator);
+  }
+  
+  GET_TRUE_DATA_TYPE();
+  copyTopBotRes(pCtx, type);
+  
+  doFinalizer(pCtx);
+}
+
+///////////////////////////////////////////////////////////////////////////////////////////////
+static bool percentile_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResultInfo) {
+  if (!function_setup(pCtx, pResultInfo)) {
+    return false;
+  }
+
+  // in the first round, get the min-max value of all involved data
+  SPercentileInfo *pInfo = GET_ROWCELL_INTERBUF(pResultInfo);
+  SET_DOUBLE_VAL(&pInfo->minval, DBL_MAX);
+  SET_DOUBLE_VAL(&pInfo->maxval, -DBL_MAX);
+  pInfo->numOfElems = 0;
+
+  return true;
+}
+
+static void percentile_function(SQLFunctionCtx *pCtx) {
+  int32_t notNullElems = 0;
+  
+  SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+  SPercentileInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo);
+
+  if (pCtx->currentStage == REPEAT_SCAN && pInfo->stage == 0) {
+    pInfo->stage += 1;
+
+    // all data are null, set it completed
+    if (pInfo->numOfElems == 0) {
+      pResInfo->complete = true;
+      
+      return;
+    } else {
+      pInfo->pMemBucket = tMemBucketCreate(pCtx->inputBytes, pCtx->inputType, pInfo->minval, pInfo->maxval);
+    }
+  }
+
+  // the first stage, only acquire the min/max value
+  if (pInfo->stage == 0) {
+    if (pCtx->isSmaSet) {
+      double tmin = 0.0, tmax = 0.0;
+      if (IS_SIGNED_NUMERIC_TYPE(pCtx->inputType)) {
+        tmin = (double)GET_INT64_VAL(&pCtx->sma.min);
+        tmax = (double)GET_INT64_VAL(&pCtx->sma.max);
+      } else if (IS_FLOAT_TYPE(pCtx->inputType)) {
+        tmin = GET_DOUBLE_VAL(&pCtx->sma.min);
+        tmax = GET_DOUBLE_VAL(&pCtx->sma.max);
+      } else if (IS_UNSIGNED_NUMERIC_TYPE(pCtx->inputType)) {
+        tmin = (double)GET_UINT64_VAL(&pCtx->sma.min);
+        tmax = (double)GET_UINT64_VAL(&pCtx->sma.max);
+      } else {
+        assert(true);
+      }
+
+      if (GET_DOUBLE_VAL(&pInfo->minval) > tmin) {
+        SET_DOUBLE_VAL(&pInfo->minval, tmin);
+      }
+
+      if (GET_DOUBLE_VAL(&pInfo->maxval) < tmax) {
+        SET_DOUBLE_VAL(&pInfo->maxval, tmax);
+      }
+
+      pInfo->numOfElems += (pCtx->size - pCtx->sma.numOfNull);
+    } else {
+      for (int32_t i = 0; i < pCtx->size; ++i) {
+        char *data = GET_INPUT_DATA(pCtx, i);
+        if (pCtx->hasNull && isNull(data, pCtx->inputType)) {
+          continue;
+        }
+
+        double v = 0;
+        GET_TYPED_DATA(v, double, pCtx->inputType, data);
+
+        if (v < GET_DOUBLE_VAL(&pInfo->minval)) {
+          SET_DOUBLE_VAL(&pInfo->minval, v);
+        }
+
+        if (v > GET_DOUBLE_VAL(&pInfo->maxval)) {
+          SET_DOUBLE_VAL(&pInfo->maxval, v);
+        }
+
+        pInfo->numOfElems += 1;
+      }
+    }
+
+    return;
+  }
+
+  // the second stage, calculate the true percentile value
+  for (int32_t i = 0; i < pCtx->size; ++i) {
+    char *data = GET_INPUT_DATA(pCtx, i);
+    if (pCtx->hasNull && isNull(data, pCtx->inputType)) {
+      continue;
+    }
+    
+    notNullElems += 1;
+    tMemBucketPut(pInfo->pMemBucket, data, 1);
+  }
+  
+  SET_VAL(pCtx, notNullElems, 1);
+  pResInfo->hasResult = DATA_SET_FLAG;
+}
+
+static void percentile_finalizer(SQLFunctionCtx *pCtx) {
+  double v = pCtx->param[0].nType == TSDB_DATA_TYPE_INT ? pCtx->param[0].i64 : pCtx->param[0].d;
+  
+  SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+  SPercentileInfo* ppInfo = (SPercentileInfo *) GET_ROWCELL_INTERBUF(pResInfo);
+
+  tMemBucket * pMemBucket = ppInfo->pMemBucket;
+  if (pMemBucket == NULL || pMemBucket->total == 0) {  // check for null
+    assert(ppInfo->numOfElems == 0);
+    setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes);
+  } else {
+    SET_DOUBLE_VAL((double *)pCtx->pOutput, getPercentile(pMemBucket, v));
+  }
+  
+  tMemBucketDestroy(pMemBucket);
+  doFinalizer(pCtx);
+}
+
+//////////////////////////////////////////////////////////////////////////////////
+static void buildHistogramInfo(SAPercentileInfo* pInfo) {
+  pInfo->pHisto = (SHistogramInfo*) ((char*) pInfo + sizeof(SAPercentileInfo));
+  pInfo->pHisto->elems = (SHistBin*) ((char*)pInfo->pHisto + sizeof(SHistogramInfo));
+}
+
+static SAPercentileInfo *getAPerctInfo(SQLFunctionCtx *pCtx) {
+  SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+  SAPercentileInfo* pInfo = NULL;
+
+  if (pCtx->stableQuery && pCtx->currentStage != MERGE_STAGE) {
+    pInfo = (SAPercentileInfo*) pCtx->pOutput;
+  } else {
+    pInfo = GET_ROWCELL_INTERBUF(pResInfo);
+  }
+
+  buildHistogramInfo(pInfo);
+  return pInfo;
+}
+
+static bool apercentile_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResultInfo) {
+  if (!function_setup(pCtx, pResultInfo)) {
+    return false;
+  }
+  
+  SAPercentileInfo *pInfo = getAPerctInfo(pCtx);
+  
+  char *tmp = (char *)pInfo + sizeof(SAPercentileInfo);
+  pInfo->pHisto = tHistogramCreateFrom(tmp, MAX_HISTOGRAM_BIN);
+  return true;
+}
+
+static void apercentile_function(SQLFunctionCtx *pCtx) {
+  int32_t notNullElems = 0;
+  
+  SResultRowCellInfo *     pResInfo = GET_RES_INFO(pCtx);
+  SAPercentileInfo *pInfo = getAPerctInfo(pCtx);
+
+  assert(pInfo->pHisto->elems != NULL);
+  
+  for (int32_t i = 0; i < pCtx->size; ++i) {
+    char *data = GET_INPUT_DATA(pCtx, i);
+    if (pCtx->hasNull && isNull(data, pCtx->inputType)) {
+      continue;
+    }
+    
+    notNullElems += 1;
+
+    double v = 0;
+    GET_TYPED_DATA(v, double, pCtx->inputType, data);
+    tHistogramAdd(&pInfo->pHisto, v);
+  }
+  
+  if (!pCtx->hasNull) {
+    assert(pCtx->size == notNullElems);
+  }
+  
+  SET_VAL(pCtx, notNullElems, 1);
+  
+  if (notNullElems > 0) {
+    pResInfo->hasResult = DATA_SET_FLAG;
+  }
+}
+
+static void apercentile_func_merge(SQLFunctionCtx *pCtx) {
+  SAPercentileInfo *pInput = (SAPercentileInfo *)GET_INPUT_DATA_LIST(pCtx);
+  
+  pInput->pHisto = (SHistogramInfo*) ((char *)pInput + sizeof(SAPercentileInfo));
+  pInput->pHisto->elems = (SHistBin*) ((char *)pInput->pHisto + sizeof(SHistogramInfo));
+
+  if (pInput->pHisto->numOfElems <= 0) {
+    return;
+  }
+  
+  SAPercentileInfo *pOutput = getAPerctInfo(pCtx);
+  SHistogramInfo  *pHisto = pOutput->pHisto;
+  
+  if (pHisto->numOfElems <= 0) {
+    memcpy(pHisto, pInput->pHisto, sizeof(SHistogramInfo) + sizeof(SHistBin) * (MAX_HISTOGRAM_BIN + 1));
+    pHisto->elems = (SHistBin*) ((char *)pHisto + sizeof(SHistogramInfo));
+  } else {
+    //TODO(dengyihao): avoid memcpy   
+    pHisto->elems = (SHistBin*) ((char *)pHisto + sizeof(SHistogramInfo));
+    SHistogramInfo *pRes = tHistogramMerge(pHisto, pInput->pHisto, MAX_HISTOGRAM_BIN);
+    memcpy(pHisto, pRes, sizeof(SHistogramInfo) + sizeof(SHistBin) * MAX_HISTOGRAM_BIN);
+    pHisto->elems = (SHistBin*) ((char *)pHisto + sizeof(SHistogramInfo));
+    tHistogramDestroy(&pRes);
+  }
+
+  SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+  pResInfo->hasResult = DATA_SET_FLAG;
+  SET_VAL(pCtx, 1, 1);
+}
+
+static void apercentile_finalizer(SQLFunctionCtx *pCtx) {
+  double v = (pCtx->param[0].nType == TSDB_DATA_TYPE_INT) ? pCtx->param[0].i64 : pCtx->param[0].d;
+  
+  SResultRowCellInfo *     pResInfo = GET_RES_INFO(pCtx);
+  SAPercentileInfo *pOutput = GET_ROWCELL_INTERBUF(pResInfo);
+
+  if (pCtx->currentStage == MERGE_STAGE) {
+    if (pResInfo->hasResult == DATA_SET_FLAG) {  // check for null
+      assert(pOutput->pHisto->numOfElems > 0);
+      
+      double  ratio[] = {v};
+      double *res = tHistogramUniform(pOutput->pHisto, ratio, 1);
+      
+      memcpy(pCtx->pOutput, res, sizeof(double));
+      free(res);
+    } else {
+      setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes);
+      return;
+    }
+  } else {
+    if (pOutput->pHisto->numOfElems > 0) {
+      double ratio[] = {v};
+      
+      double *res = tHistogramUniform(pOutput->pHisto, ratio, 1);
+      memcpy(pCtx->pOutput, res, sizeof(double));
+      free(res);
+    } else {  // no need to free
+      setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes);
+      return;
+    }
+  }
+  
+  doFinalizer(pCtx);
+}
+
+/////////////////////////////////////////////////////////////////////////////////
+static bool leastsquares_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResInfo) {
+  if (!function_setup(pCtx, pResInfo)) {
+    return false;
+  }
+
+  SLeastsquaresInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo);
+  
+  // 2*3 matrix
+  pInfo->startVal = pCtx->param[0].d;
+  return true;
+}
+
+#define LEASTSQR_CAL(p, x, y, index, step) \
+  do {                                     \
+    (p)[0][0] += (double)(x) * (x);        \
+    (p)[0][1] += (double)(x);              \
+    (p)[0][2] += (double)(x) * (y)[index]; \
+    (p)[1][2] += (y)[index];               \
+    (x) += step;                           \
+  } while (0)
+
+#define LEASTSQR_CAL_LOOP(ctx, param, x, y, tsdbType, n, step) \
+  for (int32_t i = 0; i < (ctx)->size; ++i) {                  \
+    if ((ctx)->hasNull && isNull((char *)&(y)[i], tsdbType)) { \
+      continue;                                                \
+    }                                                          \
+    (n)++;                                                     \
+    LEASTSQR_CAL(param, x, y, i, step);                        \
+  }
+
+static void leastsquares_function(SQLFunctionCtx *pCtx) {
+  SResultRowCellInfo *     pResInfo = GET_RES_INFO(pCtx);
+  SLeastsquaresInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo);
+  
+  double(*param)[3] = pInfo->mat;
+  double x = pInfo->startVal;
+  
+  void *pData = GET_INPUT_DATA_LIST(pCtx);
+  
+  int32_t numOfElem = 0;
+  switch (pCtx->inputType) {
+    case TSDB_DATA_TYPE_INT: {
+      int32_t *p = pData;
+      //            LEASTSQR_CAL_LOOP(pCtx, param, pParamData, p);
+      for (int32_t i = 0; i < pCtx->size; ++i) {
+        if (pCtx->hasNull && isNull((const char*) p, pCtx->inputType)) {
+          continue;
+        }
+        
+        param[0][0] += x * x;
+        param[0][1] += x;
+        param[0][2] += x * p[i];
+        param[1][2] += p[i];
+        
+        x += pCtx->param[1].d;
+        numOfElem++;
+      }
+      break;
+    }
+    case TSDB_DATA_TYPE_BIGINT: {
+      int64_t *p = pData;
+      LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].d);
+      break;
+    }
+    case TSDB_DATA_TYPE_DOUBLE: {
+      double *p = pData;
+      LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].d);
+      break;
+    }
+    case TSDB_DATA_TYPE_FLOAT: {
+      float *p = pData;
+      LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].d);
+      break;
+    };
+    case TSDB_DATA_TYPE_SMALLINT: {
+      int16_t *p = pData;
+      LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].d);
+      break;
+    }
+    case TSDB_DATA_TYPE_TINYINT: {
+      int8_t *p = pData;
+      LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].d);
+      break;
+    }
+    case TSDB_DATA_TYPE_UTINYINT: {
+      uint8_t *p = pData;
+      LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].d);
+      break;
+    }
+    case TSDB_DATA_TYPE_USMALLINT: {
+      uint16_t *p = pData;
+      LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].d);
+      break;
+    }
+    case TSDB_DATA_TYPE_UINT: {
+      uint32_t *p = pData;
+      LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].d);
+      break;
+    }
+    case TSDB_DATA_TYPE_UBIGINT: {
+      uint64_t *p = pData;
+      LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].d);
+      break;
+    }
+  }
+  
+  pInfo->startVal = x;
+  pInfo->num += numOfElem;
+  
+  if (pInfo->num > 0) {
+    pResInfo->hasResult = DATA_SET_FLAG;
+  }
+  
+  SET_VAL(pCtx, numOfElem, 1);
+}
+
+static void leastsquares_finalizer(SQLFunctionCtx *pCtx) {
+  // no data in query
+  SResultRowCellInfo *     pResInfo = GET_RES_INFO(pCtx);
+  SLeastsquaresInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo);
+  
+  if (pInfo->num == 0) {
+    setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes);
+    return;
+  }
+  
+  double(*param)[3] = pInfo->mat;
+  
+  param[1][1] = (double)pInfo->num;
+  param[1][0] = param[0][1];
+  
+  param[0][0] -= param[1][0] * (param[0][1] / param[1][1]);
+  param[0][2] -= param[1][2] * (param[0][1] / param[1][1]);
+  param[0][1] = 0;
+  param[1][2] -= param[0][2] * (param[1][0] / param[0][0]);
+  param[1][0] = 0;
+  param[0][2] /= param[0][0];
+  
+  param[1][2] /= param[1][1];
+  
+  int32_t maxOutputSize = AVG_FUNCTION_INTER_BUFFER_SIZE - VARSTR_HEADER_SIZE;
+  size_t n = snprintf(varDataVal(pCtx->pOutput), maxOutputSize, "{slop:%.6lf, intercept:%.6lf}",
+      param[0][2], param[1][2]);
+  
+  varDataSetLen(pCtx->pOutput, n);
+  doFinalizer(pCtx);
+}
+
+static void date_col_output_function(SQLFunctionCtx *pCtx) {
+  SET_VAL(pCtx, pCtx->size, 1);
+  *(int64_t *)(pCtx->pOutput) = pCtx->startTs;
+}
+
+static void col_project_function(SQLFunctionCtx *pCtx) {
+  // the number of output rows should not affect the final number of rows, so set it to be 0
+  if (pCtx->numOfParams == 2) {
+    return;
+  }
+
+  // only one row is required.
+  if (pCtx->param[0].i64 == 1) {
+    SET_VAL(pCtx, pCtx->size, 1);
+  } else {
+    INC_INIT_VAL(pCtx, pCtx->size);
+  }
+
+  char *pData = GET_INPUT_DATA_LIST(pCtx);
+  if (pCtx->order == TSDB_ORDER_ASC) {
+    int32_t numOfRows = (pCtx->param[0].i64 == 1)? 1:pCtx->size;
+    memcpy(pCtx->pOutput, pData, (size_t) numOfRows * pCtx->inputBytes);
+  } else {
+    for(int32_t i = 0; i < pCtx->size; ++i) {
+      memcpy(pCtx->pOutput + (pCtx->size - 1 - i) * pCtx->inputBytes, pData + i * pCtx->inputBytes,
+             pCtx->inputBytes);
+    }
+  }
+}
+
+/**
+ * only used for tag projection query in select clause
+ * @param pCtx
+ * @return
+ */
+static void tag_project_function(SQLFunctionCtx *pCtx) {
+  INC_INIT_VAL(pCtx, pCtx->size);
+  
+  assert(pCtx->inputBytes == pCtx->outputBytes);
+
+  taosVariantDump(&pCtx->tag, pCtx->pOutput, pCtx->outputType, true);
+  char* data = pCtx->pOutput;
+  pCtx->pOutput += pCtx->outputBytes;
+
+  // directly copy from the first one
+  for (int32_t i = 1; i < pCtx->size; ++i) {
+    memmove(pCtx->pOutput, data, pCtx->outputBytes);
+    pCtx->pOutput += pCtx->outputBytes;
+  }
+}
+
+/**
+ * used in group by clause. when applying group by tags, the tags value is
+ * assign by using tag function.
+ * NOTE: there is only ONE output for ONE query range
+ * @param pCtx
+ * @return
+ */
+static void copy_function(SQLFunctionCtx *pCtx);
+
+static void tag_function(SQLFunctionCtx *pCtx) {
+  SET_VAL(pCtx, 1, 1);
+  if (pCtx->currentStage == MERGE_STAGE) {
+    copy_function(pCtx);
+  } else {
+    taosVariantDump(&pCtx->tag, pCtx->pOutput, pCtx->outputType, true);
+  }
+}
+
+static void copy_function(SQLFunctionCtx *pCtx) {
+  SET_VAL(pCtx, pCtx->size, 1);
+  
+  char *pData = GET_INPUT_DATA_LIST(pCtx);
+  assignVal(pCtx->pOutput, pData, pCtx->inputBytes, pCtx->inputType);
+}
+
+enum {
+  INITIAL_VALUE_NOT_ASSIGNED = 0,
+};
+
+static bool diff_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResInfo) {
+  if (!function_setup(pCtx, pResInfo)) {
+    return false;
+  }
+  
+  // diff function require the value is set to -1
+  pCtx->param[1].nType = INITIAL_VALUE_NOT_ASSIGNED;
+  return false;
+}
+
+static bool deriv_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResultInfo) {
+  if (!function_setup(pCtx, pResultInfo)) {
+    return false;
+  }
+
+  // diff function require the value is set to -1
+  SDerivInfo* pDerivInfo = GET_ROWCELL_INTERBUF(pResultInfo);
+
+  pDerivInfo->ignoreNegative = pCtx->param[1].i64;
+  pDerivInfo->prevTs   = -1;
+  pDerivInfo->tsWindow = pCtx->param[0].i64;
+  pDerivInfo->valueSet = false;
+  return false;
+}
+
+static void deriv_function(SQLFunctionCtx *pCtx) {
+  SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+  SDerivInfo* pDerivInfo = GET_ROWCELL_INTERBUF(pResInfo);
+
+  void *data = GET_INPUT_DATA_LIST(pCtx);
+
+  int32_t notNullElems = 0;
+  int32_t step = GET_FORWARD_DIRECTION_FACTOR(pCtx->order);
+  int32_t i = (pCtx->order == TSDB_ORDER_ASC) ? 0 : pCtx->size - 1;
+
+  TSKEY *pTimestamp = pCtx->ptsOutputBuf;
+  TSKEY *tsList = GET_TS_LIST(pCtx);
+
+  double *pOutput = (double *)pCtx->pOutput;
+
+  switch (pCtx->inputType) {
+    case TSDB_DATA_TYPE_INT: {
+      int32_t *pData = (int32_t *)data;
+      for (; i < pCtx->size && i >= 0; i += step) {
+        if (pCtx->hasNull && isNull((const char *)&pData[i], pCtx->inputType)) {
+          continue;
+        }
+
+        if (!pDerivInfo->valueSet) {  // initial value is not set yet
+          pDerivInfo->valueSet  = true;
+        } else {
+          SET_DOUBLE_VAL(pOutput, ((pData[i] - pDerivInfo->prevValue) * pDerivInfo->tsWindow) / (tsList[i] - pDerivInfo->prevTs));
+          if (pDerivInfo->ignoreNegative && *pOutput < 0) {
+          } else {
+            *pTimestamp = tsList[i];
+            pOutput    += 1;
+            pTimestamp += 1;
+            notNullElems++;
+          }
+        }
+
+        pDerivInfo->prevValue = pData[i];
+        pDerivInfo->prevTs    = tsList[i];
+      }
+
+      break;
+    };
+
+    case TSDB_DATA_TYPE_BIGINT: {
+      int64_t *pData = (int64_t *)data;
+      for (; i < pCtx->size && i >= 0; i += step) {
+        if (pCtx->hasNull && isNull((const char *)&pData[i], pCtx->inputType)) {
+          continue;
+        }
+
+        if (!pDerivInfo->valueSet) {  // initial value is not set yet
+          pDerivInfo->valueSet  = true;
+        } else {
+          *pOutput = ((pData[i] - pDerivInfo->prevValue) * pDerivInfo->tsWindow) / (tsList[i] - pDerivInfo->prevTs);
+          if (pDerivInfo->ignoreNegative && *pOutput < 0) {
+          } else {
+            *pTimestamp = tsList[i];
+            pOutput    += 1;
+            pTimestamp += 1;
+            notNullElems++;
+          }
+        }
+
+        pDerivInfo->prevValue = (double) pData[i];
+        pDerivInfo->prevTs    = tsList[i];
+      }
+      break;
+    }
+    case TSDB_DATA_TYPE_DOUBLE: {
+      double *pData = (double *)data;
+
+      for (; i < pCtx->size && i >= 0; i += step) {
+        if (pCtx->hasNull && isNull((const char *)&pData[i], pCtx->inputType)) {
+          continue;
+        }
+
+        if (!pDerivInfo->valueSet) {  // initial value is not set yet
+          pDerivInfo->valueSet  = true;
+        } else {
+          *pOutput = ((pData[i] - pDerivInfo->prevValue) * pDerivInfo->tsWindow) / (tsList[i] - pDerivInfo->prevTs);
+          if (pDerivInfo->ignoreNegative && *pOutput < 0) {
+          } else {
+            *pTimestamp = tsList[i];
+            pOutput    += 1;
+            pTimestamp += 1;
+            notNullElems++;
+          }
+        }
+
+        pDerivInfo->prevValue = pData[i];
+        pDerivInfo->prevTs    = tsList[i];
+      }
+      break;
+    }
+
+    case TSDB_DATA_TYPE_FLOAT: {
+      float *pData = (float *)data;
+
+      for (; i < pCtx->size && i >= 0; i += step) {
+        if (pCtx->hasNull && isNull((const char *)&pData[i], pCtx->inputType)) {
+          continue;
+        }
+
+        if (!pDerivInfo->valueSet) {  // initial value is not set yet
+          pDerivInfo->valueSet  = true;
+        } else {
+          *pOutput = ((pData[i] - pDerivInfo->prevValue) * pDerivInfo->tsWindow) / (tsList[i] - pDerivInfo->prevTs);
+          if (pDerivInfo->ignoreNegative && *pOutput < 0) {
+          } else {
+            *pTimestamp = tsList[i];
+            pOutput    += 1;
+            pTimestamp += 1;
+            notNullElems++;
+          }
+        }
+
+        pDerivInfo->prevValue = pData[i];
+        pDerivInfo->prevTs    = tsList[i];
+      }
+      break;
+    }
+
+    case TSDB_DATA_TYPE_SMALLINT: {
+      int16_t *pData = (int16_t *)data;
+      for (; i < pCtx->size && i >= 0; i += step) {
+        if (pCtx->hasNull && isNull((const char *)&pData[i], pCtx->inputType)) {
+          continue;
+        }
+
+        if (!pDerivInfo->valueSet) {  // initial value is not set yet
+          pDerivInfo->valueSet  = true;
+        } else {
+          *pOutput = ((pData[i] - pDerivInfo->prevValue) * pDerivInfo->tsWindow) / (tsList[i] - pDerivInfo->prevTs);
+          if (pDerivInfo->ignoreNegative && *pOutput < 0) {
+          } else {
+            *pTimestamp = tsList[i];
+            pOutput    += 1;
+            pTimestamp += 1;
+            notNullElems++;
+          }
+        }
+
+        pDerivInfo->prevValue = pData[i];
+        pDerivInfo->prevTs    = tsList[i];
+      }
+      break;
+    }
+
+    case TSDB_DATA_TYPE_TINYINT: {
+      int8_t *pData = (int8_t *)data;
+      for (; i < pCtx->size && i >= 0; i += step) {
+        if (pCtx->hasNull && isNull((char *)&pData[i], pCtx->inputType)) {
+          continue;
+        }
+
+        if (!pDerivInfo->valueSet) {  // initial value is not set yet
+          pDerivInfo->valueSet  = true;
+        } else {
+          *pOutput = ((pData[i] - pDerivInfo->prevValue) * pDerivInfo->tsWindow) / (tsList[i] - pDerivInfo->prevTs);
+          if (pDerivInfo->ignoreNegative && *pOutput < 0) {
+          } else {
+            *pTimestamp = tsList[i];
+
+            pOutput    += 1;
+            pTimestamp += 1;
+            notNullElems++;
+          }
+        }
+
+        pDerivInfo->prevValue = pData[i];
+        pDerivInfo->prevTs    = tsList[i];
+      }
+      break;
+    }
+    default:
+      assert(0);
+//      qError("error input type");
+  }
+
+  GET_RES_INFO(pCtx)->numOfRes += notNullElems;
+}
+
+#define DIFF_IMPL(ctx, d, type)                                                              \
+  do {                                                                                       \
+    if ((ctx)->param[1].nType == INITIAL_VALUE_NOT_ASSIGNED) {                               \
+      (ctx)->param[1].nType = (ctx)->inputType;                                              \
+      *(type *)&(ctx)->param[1].i64 = *(type *)(d);                                       \
+    } else {                                                                                 \
+      *(type *)(ctx)->pOutput = *(type *)(d) - (*(type *)(&(ctx)->param[1].i64));      \
+      *(type *)(&(ctx)->param[1].i64) = *(type *)(d);                                     \
+      *(int64_t *)(ctx)->ptsOutputBuf = GET_TS_DATA(ctx, index);                             \
+    }                                                                                        \
+  } while (0);
+
+// TODO difference in date column
+static void diff_function(SQLFunctionCtx *pCtx) {
+  void *data = GET_INPUT_DATA_LIST(pCtx);
+  bool  isFirstBlock = (pCtx->param[1].nType == INITIAL_VALUE_NOT_ASSIGNED);
+
+  int32_t notNullElems = 0;
+
+  int32_t step = GET_FORWARD_DIRECTION_FACTOR(pCtx->order);
+  int32_t i = (pCtx->order == TSDB_ORDER_ASC) ? 0 : pCtx->size - 1;
+
+  TSKEY* pTimestamp = pCtx->ptsOutputBuf;
+  TSKEY* tsList = GET_TS_LIST(pCtx);
+
+  switch (pCtx->inputType) {
+    case TSDB_DATA_TYPE_INT: {
+      int32_t *pData = (int32_t *)data;
+      int32_t *pOutput = (int32_t *)pCtx->pOutput;
+
+      for (; i < pCtx->size && i >= 0; i += step) {
+        if (pCtx->hasNull && isNull((const char*) &pData[i], pCtx->inputType)) {
+          continue;
+        }
+
+        if (pCtx->param[1].nType != INITIAL_VALUE_NOT_ASSIGNED) {  // initial value is not set yet
+          *pOutput = (int32_t)(pData[i] - pCtx->param[1].i64);  // direct previous may be null
+          *pTimestamp = (tsList != NULL)? tsList[i]:0;
+          pOutput    += 1;
+          pTimestamp += 1;
+        }
+
+        pCtx->param[1].i64 = pData[i];
+        pCtx->param[1].nType = pCtx->inputType;
+        notNullElems++;
+      }
+      break;
+    };
+    case TSDB_DATA_TYPE_BIGINT: {
+      int64_t *pData = (int64_t *)data;
+      int64_t *pOutput = (int64_t *)pCtx->pOutput;
+
+      for (; i < pCtx->size && i >= 0; i += step) {
+        if (pCtx->hasNull && isNull((const char*) &pData[i], pCtx->inputType)) {
+          continue;
+        }
+
+        if (pCtx->param[1].nType != INITIAL_VALUE_NOT_ASSIGNED) {  // initial value is not set yet
+          *pOutput = pData[i] - pCtx->param[1].i64;  // direct previous may be null
+          *pTimestamp = (tsList != NULL)? tsList[i]:0;
+          pOutput    += 1;
+          pTimestamp += 1;
+        }
+
+        pCtx->param[1].i64 = pData[i];
+        pCtx->param[1].nType = pCtx->inputType;
+        notNullElems++;
+      }
+      break;
+    }
+    case TSDB_DATA_TYPE_DOUBLE: {
+      double *pData = (double *)data;
+      double *pOutput = (double *)pCtx->pOutput;
+
+      for (; i < pCtx->size && i >= 0; i += step) {
+        if (pCtx->hasNull && isNull((const char*) &pData[i], pCtx->inputType)) {
+          continue;
+        }
+
+        if (pCtx->param[1].nType != INITIAL_VALUE_NOT_ASSIGNED) {  // initial value is not set yet
+          SET_DOUBLE_VAL(pOutput, pData[i] - pCtx->param[1].d);  // direct previous may be null
+          *pTimestamp = (tsList != NULL)? tsList[i]:0;
+          pOutput    += 1;
+          pTimestamp += 1;
+        }
+
+        pCtx->param[1].d = pData[i];
+        pCtx->param[1].nType = pCtx->inputType;
+        notNullElems++;
+      }
+      break;
+    }
+    case TSDB_DATA_TYPE_FLOAT: {
+      float *pData = (float *)data;
+      float *pOutput = (float *)pCtx->pOutput;
+
+      for (; i < pCtx->size && i >= 0; i += step) {
+        if (pCtx->hasNull && isNull((const char*) &pData[i], pCtx->inputType)) {
+          continue;
+        }
+
+        if (pCtx->param[1].nType != INITIAL_VALUE_NOT_ASSIGNED) {  // initial value is not set yet
+          *pOutput = (float)(pData[i] - pCtx->param[1].d);  // direct previous may be null
+          *pTimestamp = (tsList != NULL)? tsList[i]:0;
+          pOutput    += 1;
+          pTimestamp += 1;
+        }
+
+        pCtx->param[1].d = pData[i];
+        pCtx->param[1].nType = pCtx->inputType;
+        notNullElems++;
+      }
+      break;
+    }
+    case TSDB_DATA_TYPE_SMALLINT: {
+      int16_t *pData = (int16_t *)data;
+      int16_t *pOutput = (int16_t *)pCtx->pOutput;
+
+      for (; i < pCtx->size && i >= 0; i += step) {
+        if (pCtx->hasNull && isNull((const char*) &pData[i], pCtx->inputType)) {
+          continue;
+        }
+
+        if (pCtx->param[1].nType != INITIAL_VALUE_NOT_ASSIGNED) {  // initial value is not set yet
+          *pOutput = (int16_t)(pData[i] - pCtx->param[1].i64);  // direct previous may be null
+          *pTimestamp = (tsList != NULL)? tsList[i]:0;
+          pOutput    += 1;
+          pTimestamp += 1;
+        }
+
+        pCtx->param[1].i64 = pData[i];
+        pCtx->param[1].nType = pCtx->inputType;
+        notNullElems++;
+      }
+      break;
+    }
+
+    case TSDB_DATA_TYPE_TINYINT: {
+      int8_t *pData = (int8_t *)data;
+      int8_t *pOutput = (int8_t *)pCtx->pOutput;
+
+      for (; i < pCtx->size && i >= 0; i += step) {
+        if (pCtx->hasNull && isNull((char *)&pData[i], pCtx->inputType)) {
+          continue;
+        }
+
+        if (pCtx->param[1].nType != INITIAL_VALUE_NOT_ASSIGNED) {  // initial value is not set yet
+          *pOutput = (int8_t)(pData[i] - pCtx->param[1].i64);  // direct previous may be null
+          *pTimestamp = (tsList != NULL)? tsList[i]:0;
+          pOutput    += 1;
+          pTimestamp += 1;
+        }
+
+        pCtx->param[1].i64 = pData[i];
+        pCtx->param[1].nType = pCtx->inputType;
+        notNullElems++;
+      }
+      break;
+    }
+    default:
+      assert(0);
+//      qError("error input type");
+  }
+
+  // initial value is not set yet
+  if (pCtx->param[1].nType == INITIAL_VALUE_NOT_ASSIGNED || notNullElems <= 0) {
+    /*
+     * 1. current block and blocks before are full of null
+     * 2. current block may be null value
+     */
+    assert(pCtx->hasNull);
+  } else {
+    int32_t forwardStep = (isFirstBlock) ? notNullElems - 1 : notNullElems;
+
+    GET_RES_INFO(pCtx)->numOfRes += forwardStep;
+  }
+}
+
+char *getArithColumnData(void *param, const char* name, int32_t colId) {
+  SArithmeticSupport *pSupport = (SArithmeticSupport *)param;
+  
+  int32_t index = -1;
+  for (int32_t i = 0; i < pSupport->numOfCols; ++i) {
+    if (colId == pSupport->colList[i].colId) {
+      index = i;
+      break;
+    }
+  }
+  
+  assert(index >= 0);
+  return pSupport->data[index] + pSupport->offset * pSupport->colList[index].bytes;
+}
+
+static void arithmetic_function(SQLFunctionCtx *pCtx) {
+  GET_RES_INFO(pCtx)->numOfRes += pCtx->size;
+  SArithmeticSupport *sas = (SArithmeticSupport *)pCtx->param[1].pz;
+  
+//  arithmeticTreeTraverse(sas->pExprInfo->pExpr, pCtx->size, pCtx->pOutput, sas, pCtx->order, getArithColumnData);
+}
+
+#define LIST_MINMAX_N(ctx, minOutput, maxOutput, elemCnt, data, type, tsdbType, numOfNotNullElem) \
+  {                                                                                               \
+    type *inputData = (type *)data;                                                               \
+    for (int32_t i = 0; i < elemCnt; ++i) {                                                       \
+      if ((ctx)->hasNull && isNull((char *)&inputData[i], tsdbType)) {                            \
+        continue;                                                                                 \
+      }                                                                                           \
+      if (inputData[i] < minOutput) {                                                             \
+        minOutput = (double)inputData[i];                                                                 \
+      }                                                                                           \
+      if (inputData[i] > maxOutput) {                                                             \
+        maxOutput = (double)inputData[i];                                                                 \
+      }                                                                                           \
+      numOfNotNullElem++;                                                                         \
+    }                                                                                             \
+  }
+
+/////////////////////////////////////////////////////////////////////////////////
+static bool spread_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResInfo) {
+  if (!function_setup(pCtx, pResInfo)) {
+    return false;
+  }
+  
+  SSpreadInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo);
+  
+  // this is the server-side setup function in client-side, the secondary merge do not need this procedure
+  if (pCtx->currentStage == MERGE_STAGE) {
+    pCtx->param[0].d = DBL_MAX;
+    pCtx->param[3].d = -DBL_MAX;
+  } else {
+    pInfo->min = DBL_MAX;
+    pInfo->max = -DBL_MAX;
+  }
+  
+  return true;
+}
+
+static void spread_function(SQLFunctionCtx *pCtx) {
+  SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+  SSpreadInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo);
+  
+  int32_t numOfElems = 0;
+  
+  // todo : opt with pre-calculated result
+  // column missing cause the hasNull to be true
+  if (pCtx->isSmaSet) {
+    numOfElems = pCtx->size - pCtx->sma.numOfNull;
+    
+    // all data are null in current data block, ignore current data block
+    if (numOfElems == 0) {
+      goto _spread_over;
+    }
+
+    if (IS_SIGNED_NUMERIC_TYPE(pCtx->inputType) || IS_UNSIGNED_NUMERIC_TYPE(pCtx->inputType) ||
+        (pCtx->inputType == TSDB_DATA_TYPE_TIMESTAMP)) {
+      if (pInfo->min > pCtx->sma.min) {
+        pInfo->min = (double)pCtx->sma.min;
+      }
+
+      if (pInfo->max < pCtx->sma.max) {
+        pInfo->max = (double)pCtx->sma.max;
+      }
+    } else if (IS_FLOAT_TYPE(pCtx->inputType)) {
+      if (pInfo->min > GET_DOUBLE_VAL((const char *)&(pCtx->sma.min))) {
+        pInfo->min = GET_DOUBLE_VAL((const char *)&(pCtx->sma.min));
+      }
+      
+      if (pInfo->max < GET_DOUBLE_VAL((const char *)&(pCtx->sma.max))) {
+        pInfo->max = GET_DOUBLE_VAL((const char *)&(pCtx->sma.max));
+      }
+    }
+    
+    goto _spread_over;
+  }
+  
+  void *pData = GET_INPUT_DATA_LIST(pCtx);
+  numOfElems = 0;
+  
+  if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) {
+    LIST_MINMAX_N(pCtx, pInfo->min, pInfo->max, pCtx->size, pData, int8_t, pCtx->inputType, numOfElems);
+  } else if (pCtx->inputType == TSDB_DATA_TYPE_SMALLINT) {
+    LIST_MINMAX_N(pCtx, pInfo->min, pInfo->max, pCtx->size, pData, int16_t, pCtx->inputType, numOfElems);
+  } else if (pCtx->inputType == TSDB_DATA_TYPE_INT) {
+    LIST_MINMAX_N(pCtx, pInfo->min, pInfo->max, pCtx->size, pData, int32_t, pCtx->inputType, numOfElems);
+  } else if (pCtx->inputType == TSDB_DATA_TYPE_BIGINT || pCtx->inputType == TSDB_DATA_TYPE_TIMESTAMP) {
+    LIST_MINMAX_N(pCtx, pInfo->min, pInfo->max, pCtx->size, pData, int64_t, pCtx->inputType, numOfElems);
+  } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) {
+    LIST_MINMAX_N(pCtx, pInfo->min, pInfo->max, pCtx->size, pData, double, pCtx->inputType, numOfElems);
+  } else if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT) {
+    LIST_MINMAX_N(pCtx, pInfo->min, pInfo->max, pCtx->size, pData, float, pCtx->inputType, numOfElems);
+  } else if (pCtx->inputType == TSDB_DATA_TYPE_UTINYINT) {
+    LIST_MINMAX_N(pCtx, pInfo->min, pInfo->max, pCtx->size, pData, uint8_t, pCtx->inputType, numOfElems);
+  } else if (pCtx->inputType == TSDB_DATA_TYPE_USMALLINT) {
+    LIST_MINMAX_N(pCtx, pInfo->min, pInfo->max, pCtx->size, pData, uint16_t, pCtx->inputType, numOfElems);
+  } else if (pCtx->inputType == TSDB_DATA_TYPE_UINT) {
+    LIST_MINMAX_N(pCtx, pInfo->min, pInfo->max, pCtx->size, pData, uint32_t, pCtx->inputType, numOfElems);
+  } else if (pCtx->inputType == TSDB_DATA_TYPE_UBIGINT) {
+    LIST_MINMAX_N(pCtx, pInfo->min, pInfo->max, pCtx->size, pData, uint64_t, pCtx->inputType, numOfElems);
+  }
+  
+  if (!pCtx->hasNull) {
+    assert(pCtx->size == numOfElems);
+  }
+  
+  _spread_over:
+  SET_VAL(pCtx, numOfElems, 1);
+  
+  if (numOfElems > 0) {
+    pResInfo->hasResult = DATA_SET_FLAG;
+    pInfo->hasResult = DATA_SET_FLAG;
+  }
+  
+  // keep the data into the final output buffer for super table query since this execution may be the last one
+  if (pCtx->stableQuery) {
+    memcpy(pCtx->pOutput, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SSpreadInfo));
+  }
+}
+
+/*
+ * here we set the result value back to the intermediate buffer, to apply the finalize the function
+ * the final result is generated in spread_function_finalizer
+ */
+void spread_func_merge(SQLFunctionCtx *pCtx) {
+  SSpreadInfo *pData = (SSpreadInfo *)GET_INPUT_DATA_LIST(pCtx);
+  if (pData->hasResult != DATA_SET_FLAG) {
+    return;
+  }
+  
+  if (pCtx->param[0].d > pData->min) {
+    pCtx->param[0].d = pData->min;
+  }
+  
+  if (pCtx->param[3].d < pData->max) {
+    pCtx->param[3].d = pData->max;
+  }
+  
+  GET_RES_INFO(pCtx)->hasResult = DATA_SET_FLAG;
+}
+
+void spread_function_finalizer(SQLFunctionCtx *pCtx) {
+  /*
+   * here we do not check the input data types, because in case of metric query,
+   * the type of intermediate data is binary
+   */
+  SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+  
+  if (pCtx->currentStage == MERGE_STAGE) {
+    assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY);
+    
+    if (pResInfo->hasResult != DATA_SET_FLAG) {
+      setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes);
+      return;
+    }
+    
+    SET_DOUBLE_VAL((double *)pCtx->pOutput, pCtx->param[3].d - pCtx->param[0].d);
+  } else {
+    assert(IS_NUMERIC_TYPE(pCtx->inputType) || (pCtx->inputType == TSDB_DATA_TYPE_TIMESTAMP));
+    
+    SSpreadInfo *pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
+    if (pInfo->hasResult != DATA_SET_FLAG) {
+      setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes);
+      return;
+    }
+    
+    SET_DOUBLE_VAL((double *)pCtx->pOutput, pInfo->max - pInfo->min);
+  }
+  
+  GET_RES_INFO(pCtx)->numOfRes = 1;  // todo add test case
+  doFinalizer(pCtx);
+}
+
+
+/**
+ * param[1]: start time
+ * param[2]: end time
+ * @param pCtx
+ */
+static bool twa_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResInfo) {
+  if (!function_setup(pCtx, pResInfo)) {
+    return false;
+  }
+
+  STwaInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo);
+  pInfo->p.key    = INT64_MIN;
+  pInfo->win      = TSWINDOW_INITIALIZER;
+  return true;
+}
+
+static double twa_get_area(SPoint1 s, SPoint1 e) {
+  if ((s.val >= 0 && e.val >= 0)|| (s.val <=0 && e.val <= 0)) {
+    return (s.val + e.val) * (e.key - s.key) / 2;
+  }
+
+  double x = (s.key * e.val - e.key * s.val)/(e.val - s.val);
+  double val = (s.val * (x - s.key) + e.val * (e.key - x)) / 2;
+  return val;
+}
+
+static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t index, int32_t size) {
+  int32_t notNullElems = 0;
+  SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+
+  STwaInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo);
+  TSKEY    *tsList = GET_TS_LIST(pCtx);
+
+  int32_t i = index;
+  int32_t step = GET_FORWARD_DIRECTION_FACTOR(pCtx->order);
+  SPoint1* last = &pInfo->p;
+
+  if (pCtx->start.key != INT64_MIN) {
+    assert((pCtx->start.key < tsList[i] && pCtx->order == TSDB_ORDER_ASC) ||
+               (pCtx->start.key > tsList[i] && pCtx->order == TSDB_ORDER_DESC));
+
+    assert(last->key == INT64_MIN);
+
+    last->key = tsList[i];
+    GET_TYPED_DATA(last->val, double, pCtx->inputType, GET_INPUT_DATA(pCtx, index));
+
+    pInfo->dOutput += twa_get_area(pCtx->start, *last);
+
+    pInfo->hasResult = DATA_SET_FLAG;
+    pInfo->win.skey = pCtx->start.key;
+    notNullElems++;
+    i += step;
+  } else if (pInfo->p.key == INT64_MIN) {
+    last->key = tsList[i];
+    GET_TYPED_DATA(last->val, double, pCtx->inputType, GET_INPUT_DATA(pCtx, index));
+
+    pInfo->hasResult = DATA_SET_FLAG;
+    pInfo->win.skey = last->key;
+    notNullElems++;
+    i += step;
+  }
+
+  // calculate the value of
+  switch(pCtx->inputType) {
+    case TSDB_DATA_TYPE_TINYINT: {
+      int8_t *val = (int8_t*) GET_INPUT_DATA(pCtx, 0);
+      for (; i < size && i >= 0; i += step) {
+        if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) {
+          continue;
+        }
+
+#ifndef _TD_NINGSI_60
+        SPoint1 st = {.key = tsList[i], .val = val[i]};
+#else
+        SPoint1 st;
+        st.key = tsList[i];
+        st.val = val[i];
+#endif        
+        pInfo->dOutput += twa_get_area(pInfo->p, st);
+        pInfo->p = st;
+      }
+      break;
+    }
+    case TSDB_DATA_TYPE_SMALLINT: {
+      int16_t *val = (int16_t*) GET_INPUT_DATA(pCtx, 0);
+      for (; i < size && i >= 0; i += step) {
+        if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) {
+          continue;
+        }
+        
+#ifndef _TD_NINGSI_60
+        SPoint1 st = {.key = tsList[i], .val = val[i]};
+#else
+        SPoint1 st;
+        st.key = tsList[i];
+        st.val = val[i];
+#endif        
+        pInfo->dOutput += twa_get_area(pInfo->p, st);
+        pInfo->p = st;
+      }
+      break;
+    }
+    case TSDB_DATA_TYPE_INT: {
+      int32_t *val = (int32_t*) GET_INPUT_DATA(pCtx, 0);
+      for (; i < size && i >= 0; i += step) {
+        if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) {
+          continue;
+        }
+        
+#ifndef _TD_NINGSI_60
+        SPoint1 st = {.key = tsList[i], .val = val[i]};
+#else
+        SPoint1 st;
+        st.key = tsList[i];
+        st.val = val[i];
+#endif        
+        pInfo->dOutput += twa_get_area(pInfo->p, st);
+        pInfo->p = st;
+      }
+      break;
+    }
+    case TSDB_DATA_TYPE_BIGINT: {
+      int64_t *val = (int64_t*) GET_INPUT_DATA(pCtx, 0);
+      for (; i < size && i >= 0; i += step) {
+        if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) {
+          continue;
+        }
+        
+#ifndef _TD_NINGSI_60
+        SPoint1 st = {.key = tsList[i], .val = (double) val[i]};
+#else
+        SPoint1 st;
+        st.key = tsList[i];
+        st.val = (double)val[i];
+#endif        
+        pInfo->dOutput += twa_get_area(pInfo->p, st);
+        pInfo->p = st;
+      }
+      break;
+    }
+    case TSDB_DATA_TYPE_FLOAT: {
+      float *val = (float*) GET_INPUT_DATA(pCtx, 0);
+      for (; i < size && i >= 0; i += step) {
+        if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) {
+          continue;
+        }
+        
+#ifndef _TD_NINGSI_60
+        SPoint1 st = {.key = tsList[i], .val = val[i]};
+#else
+        SPoint1 st;
+        st.key = tsList[i];
+        st.val = (double)val[i];
+#endif        
+        pInfo->dOutput += twa_get_area(pInfo->p, st);
+        pInfo->p = st;
+      }
+      break;
+    }
+    case TSDB_DATA_TYPE_DOUBLE: {
+      double *val = (double*) GET_INPUT_DATA(pCtx, 0);
+      for (; i < size && i >= 0; i += step) {
+        if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) {
+          continue;
+        }
+        
+#ifndef _TD_NINGSI_60
+        SPoint1 st = {.key = tsList[i], .val = val[i]};
+#else
+        SPoint1 st;
+        st.key = tsList[i];
+        st.val = val[i];
+#endif        
+        pInfo->dOutput += twa_get_area(pInfo->p, st);
+        pInfo->p = st;
+      }
+      break;
+    }
+    case TSDB_DATA_TYPE_UTINYINT: {
+      uint8_t *val = (uint8_t*) GET_INPUT_DATA(pCtx, 0);
+      for (; i < size && i >= 0; i += step) {
+        if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) {
+          continue;
+        }
+
+#ifndef _TD_NINGSI_60
+        SPoint1 st = {.key = tsList[i], .val = val[i]};
+#else
+        SPoint1 st;
+        st.key = tsList[i];
+        st.val = val[i];
+#endif        
+        pInfo->dOutput += twa_get_area(pInfo->p, st);
+        pInfo->p = st;
+      }
+      break;
+    }
+    case TSDB_DATA_TYPE_USMALLINT: {
+      uint16_t *val = (uint16_t*) GET_INPUT_DATA(pCtx, 0);
+      for (; i < size && i >= 0; i += step) {
+        if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) {
+          continue;
+        }
+
+#ifndef _TD_NINGSI_60
+        SPoint1 st = {.key = tsList[i], .val = val[i]};
+#else
+        SPoint1 st;
+        st.key = tsList[i];
+        st.val = val[i];
+#endif        
+        pInfo->dOutput += twa_get_area(pInfo->p, st);
+        pInfo->p = st;
+      }
+      break;
+    }
+    case TSDB_DATA_TYPE_UINT: {
+      uint32_t *val = (uint32_t*) GET_INPUT_DATA(pCtx, 0);
+      for (; i < size && i >= 0; i += step) {
+        if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) {
+          continue;
+        }
+
+#ifndef _TD_NINGSI_60
+        SPoint1 st = {.key = tsList[i], .val = val[i]};
+#else
+        SPoint1 st;
+        st.key = tsList[i];
+        st.val = val[i];
+#endif        
+        pInfo->dOutput += twa_get_area(pInfo->p, st);
+        pInfo->p = st;
+      }
+      break;
+    }
+    case TSDB_DATA_TYPE_UBIGINT: {
+      uint64_t *val = (uint64_t*) GET_INPUT_DATA(pCtx, 0);
+      for (; i < size && i >= 0; i += step) {
+        if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) {
+          continue;
+        }
+        
+#ifndef _TD_NINGSI_60
+        SPoint1 st = {.key = tsList[i], .val = (double) val[i]};
+#else
+        SPoint1 st;
+        st.key = tsList[i];
+        st.val = (double) val[i];
+#endif        
+        pInfo->dOutput += twa_get_area(pInfo->p, st);
+        pInfo->p = st;
+      }
+      break;
+    }
+    default: assert(0);
+  }
+
+  // the last interpolated time window value
+  if (pCtx->end.key != INT64_MIN) {
+    pInfo->dOutput  += twa_get_area(pInfo->p, pCtx->end);
+    pInfo->p = pCtx->end;
+  }
+
+  pInfo->win.ekey  = pInfo->p.key;
+  return notNullElems;
+}
+
+static void twa_function(SQLFunctionCtx *pCtx) {
+  void *data = GET_INPUT_DATA_LIST(pCtx);
+
+  SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+  STwaInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo);
+  
+  // skip null value
+  int32_t step = GET_FORWARD_DIRECTION_FACTOR(pCtx->order);
+  int32_t i = (pCtx->order == TSDB_ORDER_ASC)? 0:(pCtx->size - 1);
+  while (pCtx->hasNull && i < pCtx->size && i >= 0 && isNull((char *)data + pCtx->inputBytes * i, pCtx->inputType)) {
+    i += step;
+  }
+
+  int32_t notNullElems = 0;
+  if (i >= 0 && i < pCtx->size) {
+    notNullElems = twa_function_impl(pCtx, i, pCtx->size);
+  }
+
+  SET_VAL(pCtx, notNullElems, 1);
+  
+  if (notNullElems > 0) {
+    pResInfo->hasResult = DATA_SET_FLAG;
+  }
+  
+  if (pCtx->stableQuery) {
+    memcpy(pCtx->pOutput, pInfo, sizeof(STwaInfo));
+  }
+}
+
+/*
+ * To copy the input to interResBuf to avoid the input buffer space be over writen
+ * by next input data. The TWA function only applies to each table, so no merge procedure
+ * is required, we simply copy to the resut ot interResBuffer.
+ */
+void twa_function_copy(SQLFunctionCtx *pCtx) {
+  assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY);
+  SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+  
+  memcpy(GET_ROWCELL_INTERBUF(pResInfo), pCtx->pInput, (size_t)pCtx->inputBytes);
+  pResInfo->hasResult = ((STwaInfo *)pCtx->pInput)->hasResult;
+}
+
+void twa_function_finalizer(SQLFunctionCtx *pCtx) {
+  SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+  
+  STwaInfo *pInfo = (STwaInfo *)GET_ROWCELL_INTERBUF(pResInfo);
+  if (pInfo->hasResult != DATA_SET_FLAG) {
+    setNull(pCtx->pOutput, TSDB_DATA_TYPE_DOUBLE, sizeof(double));
+    return;
+  }
+
+  assert(pInfo->win.ekey == pInfo->p.key && pInfo->hasResult == pResInfo->hasResult);
+  if (pInfo->win.ekey == pInfo->win.skey) {
+    SET_DOUBLE_VAL((double *)pCtx->pOutput, pInfo->p.val);
+  } else {
+    SET_DOUBLE_VAL((double *)pCtx->pOutput , pInfo->dOutput / (pInfo->win.ekey - pInfo->win.skey));
+  }
+  
+  GET_RES_INFO(pCtx)->numOfRes = 1;
+  doFinalizer(pCtx);
+}
+
+/**
+ *
+ * @param pCtx
+ */
+
+static void interp_function_impl(SQLFunctionCtx *pCtx) {
+  int32_t type = (int32_t) pCtx->param[2].i64;
+  if (type == TSDB_FILL_NONE) {
+    return;
+  }
+
+  bool ascQuery = (pCtx->order == TSDB_ORDER_ASC);
+
+  if (pCtx->inputType == TSDB_DATA_TYPE_TIMESTAMP) {
+    *(TSKEY *)pCtx->pOutput = pCtx->startTs;
+  } else if (type == TSDB_FILL_NULL) {
+    setNull(pCtx->pOutput, pCtx->outputType, pCtx->outputBytes);
+  } else if (type == TSDB_FILL_SET_VALUE) {
+    taosVariantDump(&pCtx->param[1], pCtx->pOutput, pCtx->inputType, true);
+  } else {
+    if (pCtx->start.key != INT64_MIN && ((ascQuery && pCtx->start.key <= pCtx->startTs && pCtx->end.key >= pCtx->startTs) || ((!ascQuery) && pCtx->start.key >= pCtx->startTs && pCtx->end.key <= pCtx->startTs))) {
+      if (type == TSDB_FILL_PREV) {
+        if (IS_NUMERIC_TYPE(pCtx->inputType) || pCtx->inputType == TSDB_DATA_TYPE_BOOL) {
+          SET_TYPED_DATA(pCtx->pOutput, pCtx->inputType, pCtx->start.val);
+        } else {
+          assignVal(pCtx->pOutput, pCtx->start.ptr, pCtx->outputBytes, pCtx->inputType);
+        }
+      } else if (type == TSDB_FILL_NEXT) {
+        if (IS_NUMERIC_TYPE(pCtx->inputType) || pCtx->inputType == TSDB_DATA_TYPE_BOOL) {
+          SET_TYPED_DATA(pCtx->pOutput, pCtx->inputType, pCtx->end.val);
+        } else {
+          assignVal(pCtx->pOutput, pCtx->end.ptr, pCtx->outputBytes, pCtx->inputType);
+        }
+      } else if (type == TSDB_FILL_LINEAR) {
+        SPoint point1 = {.key = pCtx->start.key, .val = &pCtx->start.val};
+        SPoint point2 = {.key = pCtx->end.key, .val = &pCtx->end.val};
+        SPoint point  = {.key = pCtx->startTs, .val = pCtx->pOutput};
+
+        int32_t srcType = pCtx->inputType;
+        if (IS_NUMERIC_TYPE(srcType)) {  // TODO should find the not null data?
+          if (isNull((char *)&pCtx->start.val, srcType) || isNull((char *)&pCtx->end.val, srcType)) {
+            setNull(pCtx->pOutput, srcType, pCtx->inputBytes);
+          } else {
+            taosGetLinearInterpolationVal(&point, pCtx->outputType, &point1, &point2, TSDB_DATA_TYPE_DOUBLE);
+          }
+        } else {
+          setNull(pCtx->pOutput, srcType, pCtx->inputBytes);
+        }
+      }
+    } else {
+      // no data generated yet
+      if (pCtx->size < 1) {
+        return;
+      }
+
+      // check the timestamp in input buffer
+      TSKEY skey = GET_TS_DATA(pCtx, 0);
+
+      if (type == TSDB_FILL_PREV) {
+        if ((ascQuery && skey > pCtx->startTs) || ((!ascQuery) && skey < pCtx->startTs)) {
+          return;
+        }
+
+        if (pCtx->size > 1) {
+          TSKEY ekey = GET_TS_DATA(pCtx, 1);
+          if ((ascQuery &&  ekey > skey && ekey <= pCtx->startTs) ||
+             ((!ascQuery) && ekey < skey && ekey >= pCtx->startTs)){
+            skey = ekey;
+          }
+        }
+        assignVal(pCtx->pOutput, pCtx->pInput, pCtx->outputBytes, pCtx->inputType);
+      } else if (type == TSDB_FILL_NEXT) {
+        TSKEY ekey = skey;
+        char* val = NULL;
+        
+        if ((ascQuery && ekey < pCtx->startTs) || ((!ascQuery) && ekey > pCtx->startTs)) {
+          if (pCtx->size > 1) {
+            ekey = GET_TS_DATA(pCtx, 1);
+            if ((ascQuery && ekey < pCtx->startTs) || ((!ascQuery) && ekey > pCtx->startTs)) {
+              return;
+            }
+
+            val = ((char*)pCtx->pInput) + pCtx->inputBytes;            
+          } else {
+            return;
+          }
+        } else {
+          val = (char*)pCtx->pInput;
+        }
+        
+        assignVal(pCtx->pOutput, val, pCtx->outputBytes, pCtx->inputType);
+      } else if (type == TSDB_FILL_LINEAR) {
+        if (pCtx->size <= 1) {
+          return;
+        }
+      
+        TSKEY ekey = GET_TS_DATA(pCtx, 1);
+      
+        // no data generated yet
+        if ((ascQuery && !(skey <= pCtx->startTs && ekey >= pCtx->startTs))
+           || ((!ascQuery) && !(skey >= pCtx->startTs && ekey <= pCtx->startTs))) {
+          return;
+        }
+        
+        char *start = GET_INPUT_DATA(pCtx, 0);
+        char *end = GET_INPUT_DATA(pCtx, 1);
+
+        SPoint point1 = {.key = skey, .val = start};
+        SPoint point2 = {.key = ekey, .val = end};
+        SPoint point = {.key = pCtx->startTs, .val = pCtx->pOutput};
+
+        int32_t srcType = pCtx->inputType;
+        if (IS_NUMERIC_TYPE(srcType)) {  // TODO should find the not null data?
+          if (isNull(start, srcType) || isNull(end, srcType)) {
+            setNull(pCtx->pOutput, srcType, pCtx->inputBytes);
+          } else {
+            taosGetLinearInterpolationVal(&point, pCtx->outputType, &point1, &point2, srcType);
+          }
+        } else {
+          setNull(pCtx->pOutput, srcType, pCtx->inputBytes);
+        }
+      }
+    }
+  }
+
+  SET_VAL(pCtx, 1, 1);
+}
+
+static void interp_function(SQLFunctionCtx *pCtx) {
+  // at this point, the value is existed, return directly
+  if (pCtx->size > 0) {
+    bool ascQuery = (pCtx->order == TSDB_ORDER_ASC);
+    TSKEY key;
+    char *pData;
+    int32_t typedData = 0;
+    
+    if (ascQuery) {
+      key = GET_TS_DATA(pCtx, 0);
+      pData = GET_INPUT_DATA(pCtx, 0);
+    } else {
+      key = pCtx->start.key;
+      if (key == INT64_MIN) {
+        key = GET_TS_DATA(pCtx, 0);
+        pData = GET_INPUT_DATA(pCtx, 0);
+      } else {        
+        if (!(IS_NUMERIC_TYPE(pCtx->inputType) || pCtx->inputType == TSDB_DATA_TYPE_BOOL)) {
+          pData = pCtx->start.ptr;
+        } else {
+          typedData = 1;
+          pData = (char *)&pCtx->start.val;
+        }
+      }
+    }
+    
+    //if (key == pCtx->startTs && (ascQuery || !(IS_NUMERIC_TYPE(pCtx->inputType) || pCtx->inputType == TSDB_DATA_TYPE_BOOL))) {
+    if (key == pCtx->startTs) {
+      if (typedData) {
+        SET_TYPED_DATA(pCtx->pOutput, pCtx->inputType, *(double *)pData);
+      } else {
+        assignVal(pCtx->pOutput, pData, pCtx->inputBytes, pCtx->inputType);
+      }
+      
+      SET_VAL(pCtx, 1, 1);
+    } else {
+      interp_function_impl(pCtx);
+    }
+  } else {  //no qualified data rows and interpolation is required
+    interp_function_impl(pCtx);
+  }
+}
+
+static bool ts_comp_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResInfo) {
+  if (!function_setup(pCtx, pResInfo)) {
+    return false;  // not initialized since it has been initialized
+  }
+
+  STSCompInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo);
+  pInfo->pTSBuf = tsBufCreate(false, pCtx->order);
+  pInfo->pTSBuf->tsOrder = pCtx->order;
+  return true;
+}
+
+static void ts_comp_function(SQLFunctionCtx *pCtx) {
+  SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+  STSBuf *     pTSbuf = ((STSCompInfo *)(GET_ROWCELL_INTERBUF(pResInfo)))->pTSBuf;
+  
+  const char *input = GET_INPUT_DATA_LIST(pCtx);
+  
+  // primary ts must be existed, so no need to check its existance
+  if (pCtx->order == TSDB_ORDER_ASC) {
+    tsBufAppend(pTSbuf, (int32_t)pCtx->param[0].i64, &pCtx->tag, input, pCtx->size * TSDB_KEYSIZE);
+  } else {
+    for (int32_t i = pCtx->size - 1; i >= 0; --i) {
+      char *d = GET_INPUT_DATA(pCtx, i);
+      tsBufAppend(pTSbuf, (int32_t)pCtx->param[0].i64, &pCtx->tag, d, (int32_t)TSDB_KEYSIZE);
+    }
+  }
+  
+  SET_VAL(pCtx, pCtx->size, 1);
+  pResInfo->hasResult = DATA_SET_FLAG;
+}
+
+static void ts_comp_finalize(SQLFunctionCtx *pCtx) {
+  SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+  
+  STSCompInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo);
+  STSBuf *     pTSbuf = pInfo->pTSBuf;
+  
+  tsBufFlush(pTSbuf);
+//  qDebug("total timestamp :%"PRId64, pTSbuf->numOfTotal);
+
+  // TODO refactor transfer ownership of current file
+  *(FILE **)pCtx->pOutput = pTSbuf->f;
+
+  pResInfo->complete = true;
+
+  // get the file size
+  struct stat fStat;
+  if ((fstat(fileno(pTSbuf->f), &fStat) == 0)) {
+    pResInfo->numOfRes = fStat.st_size;
+  }
+
+  pTSbuf->remainOpen = true;
+  tsBufDestroy(pTSbuf);
+
+  doFinalizer(pCtx);
+}
+
+//////////////////////////////////////////////////////////////////////////////////////////////
+// rate functions
+static double do_calc_rate(const SRateInfo* pRateInfo, double tickPerSec) {
+  if ((INT64_MIN == pRateInfo->lastKey) || (INT64_MIN == pRateInfo->firstKey) ||
+      (pRateInfo->firstKey >= pRateInfo->lastKey)) {
+    return 0.0;
+  }
+
+  double diff = 0;
+  if (pRateInfo->isIRate) {
+    // If the previous value of the last is greater than the last value, only keep the last point instead of the delta
+    // value between two values.
+    diff = pRateInfo->lastValue;
+    if (diff >= pRateInfo->firstValue) {
+      diff -= pRateInfo->firstValue;
+    }
+  } else {
+    diff = pRateInfo->correctionValue + pRateInfo->lastValue -  pRateInfo->firstValue;
+    if (diff <= 0) {
+      return 0;
+    }
+  }
+  
+  int64_t duration = pRateInfo->lastKey - pRateInfo->firstKey;
+  if (duration == 0) {
+    return 0;
+  }
+
+  return (duration > 0)? ((double)diff) / (duration/tickPerSec):0.0;
+}
+
+static bool rate_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResInfo) {
+  if (!function_setup(pCtx, pResInfo)) {
+    return false;
+  }
+  
+  SRateInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo);
+  pInfo->correctionValue = 0;
+  pInfo->firstKey    = INT64_MIN;
+  pInfo->lastKey     = INT64_MIN;
+  pInfo->firstValue  = (double) INT64_MIN;
+  pInfo->lastValue   = (double) INT64_MIN;
+
+  pInfo->hasResult = 0;
+  pInfo->isIRate = (pCtx->functionId == FUNCTION_IRATE);
+  return true;
+}
+
+static void rate_function(SQLFunctionCtx *pCtx) {
+  SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+  
+  int32_t    notNullElems = 0;
+  SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo);
+  TSKEY     *primaryKey = GET_TS_LIST(pCtx);
+  
+//  qDebug("%p rate_function() size:%d, hasNull:%d", pCtx, pCtx->size, pCtx->hasNull);
+  
+  for (int32_t i = 0; i < pCtx->size; ++i) {
+    char *pData = GET_INPUT_DATA(pCtx, i);
+    if (pCtx->hasNull && isNull(pData, pCtx->inputType)) {
+//      qDebug("%p rate_function() index of null data:%d", pCtx, i);
+      continue;
+    }
+    
+    notNullElems++;
+    
+    double v = 0;
+    GET_TYPED_DATA(v, double, pCtx->inputType, pData);
+    
+    if ((INT64_MIN == pRateInfo->firstValue) || (INT64_MIN == pRateInfo->firstKey)) {
+      pRateInfo->firstValue = v;
+      pRateInfo->firstKey = primaryKey[i];
+    }
+    
+    if (INT64_MIN == pRateInfo->lastValue) {
+      pRateInfo->lastValue = v;
+    } else if (v < pRateInfo->lastValue) {
+      pRateInfo->correctionValue += pRateInfo->lastValue;
+    }
+    
+    pRateInfo->lastValue = v;
+    pRateInfo->lastKey   = primaryKey[i];
+  }
+  
+  if (!pCtx->hasNull) {
+    assert(pCtx->size == notNullElems);
+  }
+  
+  SET_VAL(pCtx, notNullElems, 1);
+  
+  if (notNullElems > 0) {
+    pRateInfo->hasResult = DATA_SET_FLAG;
+    pResInfo->hasResult  = DATA_SET_FLAG;
+  }
+  
+  // keep the data into the final output buffer for super table query since this execution may be the last one
+  if (pCtx->stableQuery) {
+    memcpy(pCtx->pOutput, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SRateInfo));
+  }
+}
+
+static void rate_func_copy(SQLFunctionCtx *pCtx) {
+  assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY);
+  
+  SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+  memcpy(GET_ROWCELL_INTERBUF(pResInfo), pCtx->pInput, (size_t)pCtx->inputBytes);
+  pResInfo->hasResult = ((SRateInfo*)pCtx->pInput)->hasResult;
+}
+
+static void rate_finalizer(SQLFunctionCtx *pCtx) {
+  SResultRowCellInfo *pResInfo  = GET_RES_INFO(pCtx);
+  SRateInfo   *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo);
+
+  if (pRateInfo->hasResult != DATA_SET_FLAG) {
+    setNull(pCtx->pOutput, TSDB_DATA_TYPE_DOUBLE, sizeof(double));
+    return;
+  }
+  
+  SET_DOUBLE_VAL((double*) pCtx->pOutput, do_calc_rate(pRateInfo, (double) TSDB_TICK_PER_SECOND(pCtx->param[0].i64)));
+
+  // cannot set the numOfIteratedElems again since it is set during previous iteration
+  pResInfo->numOfRes  = 1;
+  pResInfo->hasResult = DATA_SET_FLAG;
+  
+  doFinalizer(pCtx);
+}
+
+static void irate_function(SQLFunctionCtx *pCtx) {
+  SResultRowCellInfo  *pResInfo = GET_RES_INFO(pCtx);
+
+  int32_t    notNullElems = 0;
+  SRateInfo *pRateInfo    = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo);
+  TSKEY     *primaryKey   = GET_TS_LIST(pCtx);
+
+  for (int32_t i = pCtx->size - 1; i >= 0; --i) {
+    char *pData = GET_INPUT_DATA(pCtx, i);
+    if (pCtx->hasNull && isNull(pData, pCtx->inputType)) {
+      continue;
+    }
+    
+    notNullElems++;
+    
+    double v = 0;
+    GET_TYPED_DATA(v, double, pCtx->inputType, pData);
+
+    if ((INT64_MIN == pRateInfo->lastKey) || primaryKey[i] > pRateInfo->lastKey) {
+      pRateInfo->lastValue = v;
+      pRateInfo->lastKey   = primaryKey[i];
+      continue;
+    }
+    
+    if ((INT64_MIN == pRateInfo->firstKey) || primaryKey[i] > pRateInfo->firstKey) {
+      pRateInfo->firstValue = v;
+      pRateInfo->firstKey = primaryKey[i];
+      break;
+    }
+  }
+  
+  SET_VAL(pCtx, notNullElems, 1);
+  
+  if (notNullElems > 0) {
+    pRateInfo->hasResult = DATA_SET_FLAG;
+    pResInfo->hasResult  = DATA_SET_FLAG;
+  }
+  
+  // keep the data into the final output buffer for super table query since this execution may be the last one
+  if (pCtx->stableQuery) {
+    memcpy(pCtx->pOutput, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SRateInfo));
+  }
+}
+
+static void blockDistInfoFromBinary(const char* data, int32_t len, STableBlockDist* pDist) {
+  SBufferReader br = tbufInitReader(data, len, false);
+
+  pDist->numOfTables = tbufReadUint32(&br);
+  pDist->numOfFiles  = tbufReadUint16(&br);
+  pDist->totalSize   = tbufReadUint64(&br);
+  pDist->totalRows   = tbufReadUint64(&br);
+  pDist->maxRows     = tbufReadInt32(&br);
+  pDist->minRows     = tbufReadInt32(&br);
+  pDist->numOfRowsInMemTable = tbufReadUint32(&br);
+  pDist->numOfSmallBlocks = tbufReadUint32(&br);
+  int64_t numSteps = tbufReadUint64(&br);
+
+  bool comp = tbufReadUint8(&br);
+  uint32_t compLen = tbufReadUint32(&br);
+
+  size_t originalLen = (size_t) (numSteps *sizeof(SFileBlockInfo));
+
+  char* outputBuf = NULL;
+  if (comp) {
+    outputBuf = malloc(originalLen);
+
+    size_t actualLen = compLen;
+    const char* compStr = tbufReadBinary(&br, &actualLen);
+
+    int32_t orignalLen = tsDecompressString(compStr, compLen, 1, outputBuf,
+                                            (int32_t)originalLen , ONE_STAGE_COMP, NULL, 0);
+    assert(orignalLen == numSteps *sizeof(SFileBlockInfo));
+  } else {
+    outputBuf = (char*) tbufReadBinary(&br, &originalLen);
+  }
+
+  pDist->dataBlockInfos = taosArrayFromList(outputBuf, (uint32_t)numSteps, sizeof(SFileBlockInfo));
+  if (comp) {
+    tfree(outputBuf);
+  }
+}
+
+static void blockInfo_func(SQLFunctionCtx* pCtx) {
+  SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+  STableBlockDist* pDist = (STableBlockDist*) GET_ROWCELL_INTERBUF(pResInfo);
+
+  int32_t len = *(int32_t*) pCtx->pInput;
+  blockDistInfoFromBinary((char*)pCtx->pInput + sizeof(int32_t), len, pDist);
+  pDist->rowSize = (uint16_t)pCtx->param[0].i64;
+
+  memcpy(pCtx->pOutput, pCtx->pInput, sizeof(int32_t) + len);
+
+  pResInfo->numOfRes  = 1;
+  pResInfo->hasResult = DATA_SET_FLAG;
+}
+
+static void mergeTableBlockDist(SResultRowCellInfo* pResInfo, const STableBlockDist* pSrc) {
+  STableBlockDist* pDist = (STableBlockDist*) GET_ROWCELL_INTERBUF(pResInfo);
+  assert(pDist != NULL && pSrc != NULL);
+
+  pDist->numOfTables += pSrc->numOfTables;
+  pDist->numOfRowsInMemTable += pSrc->numOfRowsInMemTable;
+  pDist->numOfSmallBlocks += pSrc->numOfSmallBlocks;
+  pDist->numOfFiles += pSrc->numOfFiles;
+  pDist->totalSize += pSrc->totalSize;
+  pDist->totalRows += pSrc->totalRows;
+
+  if (pResInfo->hasResult == DATA_SET_FLAG) {
+    pDist->maxRows = MAX(pDist->maxRows, pSrc->maxRows);
+    pDist->minRows = MIN(pDist->minRows, pSrc->minRows);
+  } else {
+    pDist->maxRows = pSrc->maxRows;
+    pDist->minRows = pSrc->minRows;
+
+    int32_t maxSteps = TSDB_MAX_MAX_ROW_FBLOCK/TSDB_BLOCK_DIST_STEP_ROWS;
+    if (TSDB_MAX_MAX_ROW_FBLOCK % TSDB_BLOCK_DIST_STEP_ROWS != 0) {
+      ++maxSteps;
+    }
+    pDist->dataBlockInfos = taosArrayInit(maxSteps, sizeof(SFileBlockInfo));
+    taosArraySetSize(pDist->dataBlockInfos, maxSteps);
+  }
+
+  size_t steps = taosArrayGetSize(pSrc->dataBlockInfos);
+  for (int32_t i = 0; i < steps; ++i) {
+    int32_t srcNumBlocks = ((SFileBlockInfo*)taosArrayGet(pSrc->dataBlockInfos, i))->numBlocksOfStep;
+    SFileBlockInfo* blockInfo = (SFileBlockInfo*)taosArrayGet(pDist->dataBlockInfos, i);
+    blockInfo->numBlocksOfStep += srcNumBlocks;
+  }
+}
+
+void block_func_merge(SQLFunctionCtx* pCtx) {
+  STableBlockDist info = {0};
+  int32_t len = *(int32_t*) pCtx->pInput;
+  blockDistInfoFromBinary(((char*)pCtx->pInput) + sizeof(int32_t), len, &info);
+  SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+  mergeTableBlockDist(pResInfo, &info);
+  taosArrayDestroy(info.dataBlockInfos); 
+
+  pResInfo->numOfRes = 1;
+  pResInfo->hasResult = DATA_SET_FLAG;
+}
+
+void getPercentiles(STableBlockDist *pTableBlockDist, int64_t totalBlocks, int32_t numOfPercents,
+                    double* percents, int32_t* percentiles) {
+  if (totalBlocks == 0) {
+    for (int32_t i = 0; i < numOfPercents; ++i) {
+      percentiles[i] = 0;
+    }
+    return;
+  }
+
+  SArray *blocksInfos = pTableBlockDist->dataBlockInfos;
+  size_t  numSteps = taosArrayGetSize(blocksInfos);
+  size_t  cumulativeBlocks = 0;
+
+  int percentIndex = 0;
+  for (int32_t indexStep = 0; indexStep < numSteps; ++indexStep) {
+    int32_t numStepBlocks = ((SFileBlockInfo *)taosArrayGet(blocksInfos, indexStep))->numBlocksOfStep;
+    if (numStepBlocks == 0) continue;
+    cumulativeBlocks += numStepBlocks;
+
+    while (percentIndex < numOfPercents) {
+      double blockRank = totalBlocks * percents[percentIndex];
+      if (blockRank <= cumulativeBlocks) {
+        percentiles[percentIndex] = indexStep;
+        ++percentIndex;
+      } else {
+        break;
+      }
+    }
+  }
+
+  for (int32_t i = 0; i < numOfPercents; ++i) {
+    percentiles[i] = (percentiles[i]+1) * TSDB_BLOCK_DIST_STEP_ROWS - TSDB_BLOCK_DIST_STEP_ROWS/2;
+  }
+}
+
+void generateBlockDistResult(STableBlockDist *pTableBlockDist, char* result) {
+  if (pTableBlockDist == NULL) {
+    return;
+  }
+
+  SArray* blockInfos = pTableBlockDist->dataBlockInfos;
+  uint64_t totalRows = pTableBlockDist->totalRows;
+  size_t   numSteps = taosArrayGetSize(blockInfos);
+  int64_t totalBlocks = 0;
+  int64_t min = -1, max = -1, avg = 0;
+
+  for (int32_t i = 0; i < numSteps; i++) {
+    SFileBlockInfo *blockInfo = taosArrayGet(blockInfos, i);
+    int64_t blocks = blockInfo->numBlocksOfStep;
+    totalBlocks += blocks;
+  }
+
+  avg = totalBlocks > 0 ? (int64_t)(totalRows/totalBlocks) : 0;
+  min = totalBlocks > 0 ? pTableBlockDist->minRows : 0;
+  max = totalBlocks > 0 ? pTableBlockDist->maxRows : 0;
+
+  double stdDev = 0;
+  if (totalBlocks > 0) {
+    double variance = 0;
+    for (int32_t i = 0; i < numSteps; i++) {
+      SFileBlockInfo *blockInfo = taosArrayGet(blockInfos, i);
+      int64_t         blocks = blockInfo->numBlocksOfStep;
+      int32_t         rows = (i * TSDB_BLOCK_DIST_STEP_ROWS + TSDB_BLOCK_DIST_STEP_ROWS / 2);
+      variance += blocks * (rows - avg) * (rows - avg);
+    }
+    variance = variance / totalBlocks;
+    stdDev = sqrt(variance);
+  }
+
+  double percents[] = {0.05, 0.10, 0.20, 0.30, 0.40, 0.50, 0.60, 0.70, 0.80, 0.90, 0.95, 0.99};
+  int32_t percentiles[] = {-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1};
+  assert(sizeof(percents)/sizeof(double) == sizeof(percentiles)/sizeof(int32_t));
+  getPercentiles(pTableBlockDist, totalBlocks, sizeof(percents)/sizeof(double), percents, percentiles);
+
+  uint64_t totalLen = pTableBlockDist->totalSize;
+  int32_t rowSize = pTableBlockDist->rowSize;
+  int32_t smallBlocks = pTableBlockDist->numOfSmallBlocks;
+  double compRatio = (totalRows>0) ? ((double)(totalLen)/(rowSize*totalRows)) : 1;
+  int sz = sprintf(result + VARSTR_HEADER_SIZE,
+                   "summary: \n\t "
+                   "5th=[%d], 10th=[%d], 20th=[%d], 30th=[%d], 40th=[%d], 50th=[%d]\n\t "
+                   "60th=[%d], 70th=[%d], 80th=[%d], 90th=[%d], 95th=[%d], 99th=[%d]\n\t "
+                   "Min=[%"PRId64"(Rows)] Max=[%"PRId64"(Rows)] Avg=[%"PRId64"(Rows)] Stddev=[%.2f] \n\t "
+                   "Rows=[%"PRIu64"], Blocks=[%"PRId64"], SmallBlocks=[%d], Size=[%.3f(Kb)] Comp=[%.2f]\n\t "
+                   "RowsInMem=[%d] \n\t",
+                   percentiles[0], percentiles[1], percentiles[2], percentiles[3], percentiles[4], percentiles[5],
+                   percentiles[6], percentiles[7], percentiles[8], percentiles[9], percentiles[10], percentiles[11],
+                   min, max, avg, stdDev,
+                   totalRows, totalBlocks, smallBlocks, totalLen/1024.0, compRatio,
+                   pTableBlockDist->numOfRowsInMemTable);
+  varDataSetLen(result, sz);
+  UNUSED(sz);
+}
+
+void blockinfo_func_finalizer(SQLFunctionCtx* pCtx) {
+  SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+  STableBlockDist* pDist = (STableBlockDist*) GET_ROWCELL_INTERBUF(pResInfo);
+
+  pDist->rowSize = (uint16_t)pCtx->param[0].i64;
+  generateBlockDistResult(pDist, pCtx->pOutput);
+
+  if (pDist->dataBlockInfos != NULL) {
+    taosArrayDestroy(pDist->dataBlockInfos);
+    pDist->dataBlockInfos = NULL;
+  }
+
+  // cannot set the numOfIteratedElems again since it is set during previous iteration
+  pResInfo->numOfRes  = 1;
+  pResInfo->hasResult = DATA_SET_FLAG;
+
+  doFinalizer(pCtx);
+}
+
+/////////////////////////////////////////////////////////////////////////////////////////////
+/*
+ * function compatible list.
+ * tag and ts are not involved in the compatibility check
+ *
+ * 1. functions that are not simultaneously present with any other functions. e.g., diff/ts_z/top/bottom
+ * 2. functions that are only allowed to be present only with same functions. e.g., last_row, interp
+ * 3. functions that are allowed to be present with other functions.
+ *    e.g., count/sum/avg/min/max/stddev/percentile/apercentile/first/last...
+ *
+ */
+int32_t functionCompatList[] = {
+    // count,   sum,      avg,       min,      max,    stddev,    percentile,   apercentile, first,   last
+    1,          1,        1,         1,        1,      1,          1,           1,           1,      1,
+    // last_row,top,      bottom,    spread,   twa,    leastsqr,   ts,          ts_dummy, tag_dummy, ts_comp
+    4,         -1,       -1,         1,        1,      1,          1,           1,        1,     -1,
+    //  tag,    colprj,   tagprj,    arithmetic, diff, first_dist, last_dist,   stddev_dst, interp    rate    irate
+    1,          1,        1,         1,       -1,      1,          1,           1,          5,        1,      1,
+    // tid_tag, derivative, blk_info
+    6,          8,        7,
+};
+
+SAggFunctionInfo aggFunc[34] = {{
+                              // 0, count function does not invoke the finalize function
+                              "count",
+                              FUNCTION_AGG,
+                              FUNCTION_COUNT,
+                              FUNCTION_COUNT,
+                              BASIC_FUNC_SO,
+                              function_setup,
+                              count_function,
+                              doFinalizer,
+                              count_func_merge,
+                              countRequired,
+                          },
+                          {
+                              // 1
+                              "sum",
+                              FUNCTION_SUM,
+                              FUNCTION_SUM,
+                              BASIC_FUNC_SO,
+                              FUNCTION_AGG,
+                              function_setup,
+                              sum_function,
+                              function_finalizer,
+                              sum_func_merge,
+                              statisRequired,
+                          },
+                          {
+                              // 2
+                              "avg",
+                              FUNCTION_AVG,
+                              FUNCTION_AVG,
+                              BASIC_FUNC_SO,
+                                  FUNCTION_AGG,
+                              function_setup,
+                              avg_function,
+                              avg_finalizer,
+                              avg_func_merge,
+                              statisRequired,
+                          },
+                          {
+                              // 3
+                              "min",
+                              FUNCTION_MIN,
+                              FUNCTION_MIN,
+                              BASIC_FUNC_SO | FUNCSTATE_SELECTIVITY,
+                                  FUNCTION_AGG,
+                              min_func_setup,
+                              min_function,
+                              function_finalizer,
+                              min_func_merge,
+                              statisRequired,
+                          },
+                          {
+                              // 4
+                              "max",
+                              FUNCTION_MAX,
+                              FUNCTION_MAX,
+                              BASIC_FUNC_SO | FUNCSTATE_SELECTIVITY,
+                                  FUNCTION_AGG,
+                              max_func_setup,
+                              max_function,
+                              function_finalizer,
+                              max_func_merge,
+                              statisRequired,
+                          },
+                          {
+                              // 5
+                              "stddev",
+                              FUNCTION_STDDEV,
+                              FUNCTION_STDDEV_DST,
+                              FUNCSTATE_SO | FUNCSTATE_STREAM,
+                                  FUNCTION_AGG,
+                              function_setup,
+                              stddev_function,
+                              stddev_finalizer,
+                              noop1,
+                              dataBlockRequired,
+                          },
+                          {
+                              // 6
+                              "percentile",
+                              FUNCTION_PERCT,
+                              FUNCTION_INVALID_ID,
+                              FUNCSTATE_SO | FUNCSTATE_STREAM,
+                                  FUNCTION_AGG,
+                              percentile_function_setup,
+                              percentile_function,
+                              percentile_finalizer,
+                              noop1,
+                              dataBlockRequired,
+                          },
+                          {
+                              // 7
+                              "apercentile",
+                              FUNCTION_APERCT,
+                              FUNCTION_APERCT,
+                              FUNCSTATE_SO | FUNCSTATE_STREAM | FUNCSTATE_STABLE,
+                                  FUNCTION_AGG,
+                              apercentile_function_setup,
+                              apercentile_function,
+                              apercentile_finalizer,
+                              apercentile_func_merge,
+                              dataBlockRequired,
+                          },
+                          {
+                              // 8
+                              "first",
+                              FUNCTION_FIRST,
+                              FUNCTION_FIRST_DST,
+                              BASIC_FUNC_SO | FUNCSTATE_SELECTIVITY,
+                                  FUNCTION_AGG,
+                              function_setup,
+                              first_function,
+                              function_finalizer,
+                              noop1,
+                              firstFuncRequired,
+                          },
+                          {
+                              // 9
+                              "last",
+                              FUNCTION_LAST,
+                              FUNCTION_LAST_DST,
+                              BASIC_FUNC_SO | FUNCSTATE_SELECTIVITY,
+                                  FUNCTION_AGG,
+                              function_setup,
+                              last_function,
+                              function_finalizer,
+                              noop1,
+                              lastFuncRequired,
+                          },
+                          {
+                              // 10
+                              "last_row",
+                              FUNCTION_LAST_ROW,
+                              FUNCTION_LAST_ROW,
+                              FUNCSTATE_SO | FUNCSTATE_STABLE | FUNCSTATE_NEED_TS | FUNCSTATE_SELECTIVITY,
+                                  FUNCTION_AGG,
+                              first_last_function_setup,
+                              last_row_function,
+                              last_row_finalizer,
+                              last_dist_func_merge,
+                              dataBlockRequired,
+                          },
+                          {
+                              // 11
+                              "top",
+                              FUNCTION_TOP,
+                              FUNCTION_TOP,
+                              FUNCSTATE_MO | FUNCSTATE_STABLE | FUNCSTATE_NEED_TS | FUNCSTATE_SELECTIVITY,
+                                  FUNCTION_AGG,
+                              top_bottom_function_setup,
+                              top_function,
+                              top_bottom_func_finalizer,
+                              top_func_merge,
+                              dataBlockRequired,
+                          },
+                          {
+                              // 12
+                              "bottom",
+                              FUNCTION_BOTTOM,
+                              FUNCTION_BOTTOM,
+                              FUNCSTATE_MO | FUNCSTATE_STABLE | FUNCSTATE_NEED_TS | FUNCSTATE_SELECTIVITY,
+                                  FUNCTION_AGG,
+                              top_bottom_function_setup,
+                              bottom_function,
+                              top_bottom_func_finalizer,
+                              bottom_func_merge,
+                              dataBlockRequired,
+                          },
+                          {
+                              // 13
+                              "spread",
+                              FUNCTION_SPREAD,
+                              FUNCTION_SPREAD,
+                              BASIC_FUNC_SO,
+                                  FUNCTION_AGG,
+                              spread_function_setup,
+                              spread_function,
+                              spread_function_finalizer,
+                              spread_func_merge,
+                              countRequired,
+                          },
+                          {
+                              // 14
+                              "twa",
+                              FUNCTION_TWA,
+                              FUNCTION_TWA,
+                              BASIC_FUNC_SO | FUNCSTATE_NEED_TS,
+                                  FUNCTION_AGG,
+                              twa_function_setup,
+                              twa_function,
+                              twa_function_finalizer,
+                              twa_function_copy,
+                              dataBlockRequired,
+                          },
+                          {
+                              // 15
+                              "leastsquares",
+                              FUNCTION_LEASTSQR,
+                              FUNCTION_INVALID_ID,
+                              FUNCSTATE_SO | FUNCSTATE_STREAM,
+                                  FUNCTION_AGG,
+                              leastsquares_function_setup,
+                              leastsquares_function,
+                              leastsquares_finalizer,
+                              noop1,
+                              dataBlockRequired,
+                          },
+                          {
+                              // 16
+                              "ts",
+                              FUNCTION_TS,
+                              FUNCTION_TS,
+                              BASIC_FUNC_SO | FUNCSTATE_NEED_TS,
+                                  FUNCTION_AGG,
+                              function_setup,
+                              date_col_output_function,
+                              doFinalizer,
+                              copy_function,
+                              noDataRequired,
+                          },
+                          {
+                              // 17
+                              "ts",
+                              FUNCTION_TS_DUMMY,
+                              FUNCTION_TS_DUMMY,
+                              BASIC_FUNC_SO | FUNCSTATE_NEED_TS,
+                                  FUNCTION_AGG,
+                              function_setup,
+                              noop1,
+                              doFinalizer,
+                              copy_function,
+                              dataBlockRequired,
+                          },
+                          {
+                              // 18
+                              "tag_dummy",
+                              FUNCTION_TAG_DUMMY,
+                              FUNCTION_TAG_DUMMY,
+                              BASIC_FUNC_SO,
+                                  FUNCTION_AGG,
+                              function_setup,
+                              tag_function,
+                              doFinalizer,
+                              copy_function,
+                              noDataRequired,
+                          },
+                          {
+                              // 19
+                              "ts",
+                              FUNCTION_TS_COMP,
+                              FUNCTION_TS_COMP,
+                              FUNCSTATE_MO | FUNCSTATE_NEED_TS,
+                                  FUNCTION_AGG,
+                              ts_comp_function_setup,
+                              ts_comp_function,
+                              ts_comp_finalize,
+                              copy_function,
+                              dataBlockRequired,
+                          },
+                          {
+                              // 20
+                              "tag",
+                              FUNCTION_TAG,
+                              FUNCTION_TAG,
+                              BASIC_FUNC_SO,
+                                  FUNCTION_AGG,
+                              function_setup,
+                              tag_function,
+                              doFinalizer,
+                              copy_function,
+                              noDataRequired,
+                          },
+                          {//TODO this is a scala function
+                              // 21, column project sql function
+                              "colprj",
+                              FUNCTION_PRJ,
+                              FUNCTION_PRJ,
+                              BASIC_FUNC_MO | FUNCSTATE_NEED_TS,
+                              FUNCTION_AGG,
+                              function_setup,
+                              col_project_function,
+                              doFinalizer,
+                              copy_function,
+                              dataBlockRequired,
+                          },
+                          {
+                              // 22, multi-output, tag function has only one result
+                              "tagprj",
+                              FUNCTION_TAGPRJ,
+                              FUNCTION_TAGPRJ,
+                              BASIC_FUNC_MO,
+                              FUNCTION_AGG,
+                              function_setup,
+                              tag_project_function,
+                              doFinalizer,
+                              copy_function,
+                              noDataRequired,
+                          },
+                          {
+                              // 23
+                              "arithmetic",
+                              FUNCTION_ARITHM,
+                              FUNCTION_ARITHM,
+                              FUNCSTATE_MO | FUNCSTATE_STABLE | FUNCSTATE_NEED_TS,
+                              FUNCTION_AGG,
+                              function_setup,
+                              arithmetic_function,
+                              doFinalizer,
+                              copy_function,
+                              dataBlockRequired,
+                          },
+                          {
+                              // 24
+                              "diff",
+                              FUNCTION_DIFF,
+                              FUNCTION_INVALID_ID,
+                              FUNCSTATE_MO | FUNCSTATE_STABLE | FUNCSTATE_NEED_TS | FUNCSTATE_SELECTIVITY,
+                              FUNCTION_AGG,
+                              diff_function_setup,
+                              diff_function,
+                              doFinalizer,
+                              noop1,
+                              dataBlockRequired,
+                          },
+    // distributed version used in two-stage aggregation processes
+                          {
+                              // 25
+                              "first_dist",
+                              FUNCTION_FIRST_DST,
+                              FUNCTION_FIRST_DST,
+                              BASIC_FUNC_SO | FUNCSTATE_NEED_TS | FUNCSTATE_SELECTIVITY,
+                              FUNCTION_AGG,
+                              first_last_function_setup,
+                              first_dist_function,
+                              function_finalizer,
+                              first_dist_func_merge,
+                              firstDistFuncRequired,
+                          },
+                          {
+                              // 26
+                              "last_dist",
+                              FUNCTION_LAST_DST,
+                              FUNCTION_LAST_DST,
+                              BASIC_FUNC_SO | FUNCSTATE_NEED_TS | FUNCSTATE_SELECTIVITY,
+                              FUNCTION_AGG,
+                              first_last_function_setup,
+                              last_dist_function,
+                              function_finalizer,
+                              last_dist_func_merge,
+                              lastDistFuncRequired,
+                          },
+                          {
+                              // 27
+                              "stddev",   // return table id and the corresponding tags for join match and subscribe
+                              FUNCTION_STDDEV_DST,
+                              FUNCTION_AVG,
+                              FUNCSTATE_SO | FUNCSTATE_STABLE,
+                                  FUNCTION_AGG,
+                              function_setup,
+                              stddev_dst_function,
+                              stddev_dst_finalizer,
+                              stddev_dst_merge,
+                              dataBlockRequired,
+                          },
+                          {
+                              // 28
+                              "interp",
+                              FUNCTION_INTERP,
+                              FUNCTION_INTERP,
+                              FUNCSTATE_SO | FUNCSTATE_STABLE | FUNCSTATE_NEED_TS ,
+                                  FUNCTION_AGG,
+                              function_setup,
+                              interp_function,
+                              doFinalizer,
+                              copy_function,
+                              dataBlockRequired,
+                          },
+                          {
+                              // 29
+                              "rate",
+                              FUNCTION_RATE,
+                              FUNCTION_RATE,
+                              BASIC_FUNC_SO | FUNCSTATE_NEED_TS,
+                                  FUNCTION_AGG,
+                              rate_function_setup,
+                              rate_function,
+                              rate_finalizer,
+                              rate_func_copy,
+                              dataBlockRequired,
+                          },
+                          {
+                              // 30
+                              "irate",
+                              FUNCTION_IRATE,
+                              FUNCTION_IRATE,
+                              BASIC_FUNC_SO | FUNCSTATE_NEED_TS,
+                                  FUNCTION_AGG,
+                              rate_function_setup,
+                              irate_function,
+                              rate_finalizer,
+                              rate_func_copy,
+                              dataBlockRequired,
+                          },
+                          {
+                              // 31
+                              "tbid",   // return table id and the corresponding tags for join match and subscribe
+                              FUNCTION_TID_TAG,
+                              FUNCTION_TID_TAG,
+                              FUNCSTATE_MO | FUNCSTATE_STABLE,
+                              FUNCTION_AGG,
+                              function_setup,
+                              noop1,
+                              noop1,
+                              noop1,
+                              dataBlockRequired,
+                          },
+                          {   //32
+                              "derivative",   // return table id and the corresponding tags for join match and subscribe
+                              FUNCTION_DERIVATIVE,
+                              FUNCTION_INVALID_ID,
+                              FUNCSTATE_MO | FUNCSTATE_STABLE | FUNCSTATE_NEED_TS | FUNCSTATE_SELECTIVITY,
+                              FUNCTION_AGG,
+                              deriv_function_setup,
+                              deriv_function,
+                              doFinalizer,
+                              noop1,
+                              dataBlockRequired,
+                          },
+                          {
+                                // 33
+                              "_block_dist",   // return table id and the corresponding tags for join match and subscribe
+                              FUNCTION_BLKINFO,
+                              FUNCTION_BLKINFO,
+                              FUNCSTATE_SO | FUNCSTATE_STABLE,
+                              FUNCTION_AGG,
+                              function_setup,
+                              blockInfo_func,
+                              blockinfo_func_finalizer,
+                              block_func_merge,
+                              dataBlockRequired,
+                          }};

source/libs/function/src/tfill.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "os.h"
+
+#include "taosdef.h"
+#include "taosmsg.h"
+#include "ttypes.h"
+
+#include "tfill.h"
+#include "thash.h"
+#include "function.h"
+#include "common.h"
+#include "ttime.h"
+
+#define FILL_IS_ASC_FILL(_f) ((_f)->order == TSDB_ORDER_ASC)
+#define DO_INTERPOLATION(_v1, _v2, _k1, _k2, _k) ((_v1) + ((_v2) - (_v1)) * (((double)(_k)) - ((double)(_k1))) / (((double)(_k2)) - ((double)(_k1))))
+#define GET_FORWARD_DIRECTION_FACTOR(_ord) (((_ord) == TSDB_ORDER_ASC)? 1:-1)
+
+static void setTagsValue(SFillInfo* pFillInfo, void** data, int32_t genRows) {
+  for(int32_t j = 0; j < pFillInfo->numOfCols; ++j) {
+    SFillColInfo* pCol = &pFillInfo->pFillCol[j];
+    if (TSDB_COL_IS_NORMAL_COL(pCol->flag) || TSDB_COL_IS_UD_COL(pCol->flag)) {
+      continue;
+    }
+
+    char* val1 = elePtrAt(data[j], pCol->col.bytes, genRows);
+
+    assert(pCol->tagIndex >= 0 && pCol->tagIndex < pFillInfo->numOfTags);
+    SFillTagColInfo* pTag = &pFillInfo->pTags[pCol->tagIndex];
+
+    assert (pTag->col.colId == pCol->col.colId);
+    assignVal(val1, pTag->tagVal, pCol->col.bytes, pCol->col.type);
+  }
+}
+
+static void setNullValueForRow(SFillInfo* pFillInfo, void** data, int32_t numOfCol, int32_t rowIndex) {
+  // the first are always the timestamp column, so start from the second column.
+  for (int32_t i = 1; i < numOfCol; ++i) {
+    SFillColInfo* pCol = &pFillInfo->pFillCol[i];
+
+    char* output = elePtrAt(data[i], pCol->col.bytes, rowIndex);
+    setNull(output, pCol->col.type, pCol->col.bytes);
+  }
+}
+
+static void doFillOneRowResult(SFillInfo* pFillInfo, void** data, char** srcData, int64_t ts, bool outOfBound) {
+  char* prev = pFillInfo->prevValues;
+  char* next = pFillInfo->nextValues;
+
+  SPoint point1, point2, point;
+  int32_t step = GET_FORWARD_DIRECTION_FACTOR(pFillInfo->order);
+
+  // set the primary timestamp column value
+  int32_t index = pFillInfo->numOfCurrent;
+  char* val = elePtrAt(data[0], TSDB_KEYSIZE, index);
+  *(TSKEY*) val = pFillInfo->currentKey;
+
+  // set the other values
+  if (pFillInfo->type == TSDB_FILL_PREV) {
+    char* p = FILL_IS_ASC_FILL(pFillInfo) ? prev : next;
+
+    if (p != NULL) {
+      for (int32_t i = 1; i < pFillInfo->numOfCols; ++i) {
+        SFillColInfo* pCol = &pFillInfo->pFillCol[i];
+        if (TSDB_COL_IS_TAG(pCol->flag)) {
+          continue;
+        }
+
+        char* output = elePtrAt(data[i], pCol->col.bytes, index);
+        assignVal(output, p + pCol->col.offset, pCol->col.bytes, pCol->col.type);
+      }
+    } else {  // no prev value yet, set the value for NULL
+      setNullValueForRow(pFillInfo, data, pFillInfo->numOfCols, index);
+    }
+  } else if (pFillInfo->type == TSDB_FILL_NEXT) {
+    char* p = FILL_IS_ASC_FILL(pFillInfo)? next : prev;
+
+    if (p != NULL) {
+      for (int32_t i = 1; i < pFillInfo->numOfCols; ++i) {
+        SFillColInfo* pCol = &pFillInfo->pFillCol[i];
+        if (TSDB_COL_IS_TAG(pCol->flag)) {
+          continue;
+        }
+
+        char* output = elePtrAt(data[i], pCol->col.bytes, index);
+        assignVal(output, p + pCol->col.offset, pCol->col.bytes, pCol->col.type);
+      }
+    } else { // no prev value yet, set the value for NULL
+      setNullValueForRow(pFillInfo, data, pFillInfo->numOfCols, index);
+    }
+  } else if (pFillInfo->type == TSDB_FILL_LINEAR) {
+    // TODO : linear interpolation supports NULL value
+    if (prev != NULL && !outOfBound) {
+      for (int32_t i = 1; i < pFillInfo->numOfCols; ++i) {
+        SFillColInfo* pCol = &pFillInfo->pFillCol[i];
+        if (TSDB_COL_IS_TAG(pCol->flag)) {
+          continue;
+        }
+
+        int16_t type  = pCol->col.type;
+        int16_t bytes = pCol->col.bytes;
+
+        char *val1 = elePtrAt(data[i], pCol->col.bytes, index);
+        if (type == TSDB_DATA_TYPE_BINARY|| type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_BOOL) {
+          setNull(val1, pCol->col.type, bytes);
+          continue;
+        }
+
+        point1 = (SPoint){.key = *(TSKEY*)(prev), .val = prev + pCol->col.offset};
+        point2 = (SPoint){.key = ts, .val = srcData[i] + pFillInfo->index * bytes};
+        point  = (SPoint){.key = pFillInfo->currentKey, .val = val1};
+        taosGetLinearInterpolationVal(&point, type, &point1, &point2, type);
+      }
+    } else {
+      setNullValueForRow(pFillInfo, data, pFillInfo->numOfCols, index);
+    }
+  } else { // fill the default value */
+    for (int32_t i = 1; i < pFillInfo->numOfCols; ++i) {
+      SFillColInfo* pCol = &pFillInfo->pFillCol[i];
+      if (TSDB_COL_IS_TAG(pCol->flag)/* || IS_VAR_DATA_TYPE(pCol->col.type)*/) {
+        continue;
+      }
+
+      char* val1 = elePtrAt(data[i], pCol->col.bytes, index);
+      assignVal(val1, (char*)&pCol->fillVal.i, pCol->col.bytes, pCol->col.type);
+    }
+  }
+
+  setTagsValue(pFillInfo, data, index);
+  pFillInfo->currentKey = taosTimeAdd(pFillInfo->currentKey, pFillInfo->interval.sliding * step, pFillInfo->interval.slidingUnit, pFillInfo->precision);
+  pFillInfo->numOfCurrent++;
+}
+
+static void initBeforeAfterDataBuf(SFillInfo* pFillInfo, char** next) {
+  if (*next != NULL) {
+    return;
+  }
+
+  *next = calloc(1, pFillInfo->rowSize);
+  for (int i = 1; i < pFillInfo->numOfCols; i++) {
+    SFillColInfo* pCol = &pFillInfo->pFillCol[i];
+    setNull(*next + pCol->col.offset, pCol->col.type, pCol->col.bytes);
+  }
+}
+
+static void copyCurrentRowIntoBuf(SFillInfo* pFillInfo, char** srcData, char* buf) {
+  int32_t rowIndex = pFillInfo->index;
+  for (int32_t i = 0; i < pFillInfo->numOfCols; ++i) {
+    SFillColInfo* pCol = &pFillInfo->pFillCol[i];
+    memcpy(buf + pCol->col.offset, srcData[i] + rowIndex * pCol->col.bytes, pCol->col.bytes);
+  }
+}
+
+static int32_t fillResultImpl(SFillInfo* pFillInfo, void** data, int32_t outputRows) {
+  pFillInfo->numOfCurrent = 0;
+
+  char** srcData = pFillInfo->pData;
+  char** prev = &pFillInfo->prevValues;
+  char** next = &pFillInfo->nextValues;
+
+  int32_t step = GET_FORWARD_DIRECTION_FACTOR(pFillInfo->order);
+
+  if (FILL_IS_ASC_FILL(pFillInfo)) {
+    assert(pFillInfo->currentKey >= pFillInfo->start);
+  } else {
+    assert(pFillInfo->currentKey <= pFillInfo->start);
+  }
+
+  while (pFillInfo->numOfCurrent < outputRows) {
+    int64_t ts = ((int64_t*)pFillInfo->pData[0])[pFillInfo->index];
+
+    // set the next value for interpolation
+    if ((pFillInfo->currentKey < ts && FILL_IS_ASC_FILL(pFillInfo)) ||
+        (pFillInfo->currentKey > ts && !FILL_IS_ASC_FILL(pFillInfo))) {
+      initBeforeAfterDataBuf(pFillInfo, next);
+      copyCurrentRowIntoBuf(pFillInfo, srcData, *next);
+    }
+
+    if (((pFillInfo->currentKey < ts && FILL_IS_ASC_FILL(pFillInfo)) || (pFillInfo->currentKey > ts && !FILL_IS_ASC_FILL(pFillInfo))) &&
+        pFillInfo->numOfCurrent < outputRows) {
+
+      // fill the gap between two actual input rows
+      while (((pFillInfo->currentKey < ts && FILL_IS_ASC_FILL(pFillInfo)) ||
+              (pFillInfo->currentKey > ts && !FILL_IS_ASC_FILL(pFillInfo))) &&
+             pFillInfo->numOfCurrent < outputRows) {
+        doFillOneRowResult(pFillInfo, data, srcData, ts, false);
+      }
+
+      // output buffer is full, abort
+      if (pFillInfo->numOfCurrent == outputRows) {
+        pFillInfo->numOfTotal += pFillInfo->numOfCurrent;
+        return outputRows;
+      }
+    } else {
+      assert(pFillInfo->currentKey == ts);
+      initBeforeAfterDataBuf(pFillInfo, prev);
+      if (pFillInfo->type == TSDB_FILL_NEXT && (pFillInfo->index + 1) < pFillInfo->numOfRows) {
+        initBeforeAfterDataBuf(pFillInfo, next);
+        ++pFillInfo->index;
+        copyCurrentRowIntoBuf(pFillInfo, srcData, *next);
+        --pFillInfo->index;
+      }
+
+      // assign rows to dst buffer
+      for (int32_t i = 0; i < pFillInfo->numOfCols; ++i) {
+        SFillColInfo* pCol = &pFillInfo->pFillCol[i];
+        if (TSDB_COL_IS_TAG(pCol->flag)/* || IS_VAR_DATA_TYPE(pCol->col.type)*/) {
+          continue;
+        }
+
+        char* output = elePtrAt(data[i], pCol->col.bytes, pFillInfo->numOfCurrent);
+        char* src = elePtrAt(srcData[i], pCol->col.bytes, pFillInfo->index);
+
+        if (i == 0 || (pCol->functionId != FUNCTION_COUNT && !isNull(src, pCol->col.type)) ||
+            (pCol->functionId == FUNCTION_COUNT && GET_INT64_VAL(src) != 0)) {
+          assignVal(output, src, pCol->col.bytes, pCol->col.type);
+          memcpy(*prev + pCol->col.offset, src, pCol->col.bytes);
+        } else {  // i > 0 and data is null , do interpolation
+          if (pFillInfo->type == TSDB_FILL_PREV) {
+            assignVal(output, *prev + pCol->col.offset, pCol->col.bytes, pCol->col.type);
+          } else if (pFillInfo->type == TSDB_FILL_LINEAR) {
+            assignVal(output, src, pCol->col.bytes, pCol->col.type);
+            memcpy(*prev + pCol->col.offset, src, pCol->col.bytes);
+          } else if (pFillInfo->type == TSDB_FILL_NEXT) {
+            if (*next) {
+              assignVal(output, *next + pCol->col.offset, pCol->col.bytes, pCol->col.type);
+            } else {
+              setNull(output, pCol->col.type, pCol->col.bytes);
+            }
+          } else {
+            assignVal(output, (char*)&pCol->fillVal.i, pCol->col.bytes, pCol->col.type);
+          }
+        }
+      }
+
+      // set the tag value for final result
+      setTagsValue(pFillInfo, data, pFillInfo->numOfCurrent);
+
+      pFillInfo->currentKey = taosTimeAdd(pFillInfo->currentKey, pFillInfo->interval.sliding * step,
+                                          pFillInfo->interval.slidingUnit, pFillInfo->precision);
+      pFillInfo->index += 1;
+      pFillInfo->numOfCurrent += 1;
+    }
+
+    if (pFillInfo->index >= pFillInfo->numOfRows || pFillInfo->numOfCurrent >= outputRows) {
+      /* the raw data block is exhausted, next value does not exists */
+      if (pFillInfo->index >= pFillInfo->numOfRows) {
+        tfree(*next);
+      }
+
+      pFillInfo->numOfTotal += pFillInfo->numOfCurrent;
+      return pFillInfo->numOfCurrent;
+    }
+  }
+
+  return pFillInfo->numOfCurrent;
+}
+
+static int64_t appendFilledResult(SFillInfo* pFillInfo, void** output, int64_t resultCapacity) {
+  /*
+   * These data are generated according to fill strategy, since the current timestamp is out of the time window of
+   * real result set. Note that we need to keep the direct previous result rows, to generated the filled data.
+   */
+  pFillInfo->numOfCurrent = 0;
+  while (pFillInfo->numOfCurrent < resultCapacity) {
+    doFillOneRowResult(pFillInfo, output, pFillInfo->pData, pFillInfo->start, true);
+  }
+
+  pFillInfo->numOfTotal += pFillInfo->numOfCurrent;
+
+  assert(pFillInfo->numOfCurrent == resultCapacity);
+  return resultCapacity;
+}
+
+// there are no duplicated tags in the SFillTagColInfo list
+static int32_t setTagColumnInfo(SFillInfo* pFillInfo, int32_t numOfCols, int32_t capacity) {
+  int32_t rowsize = 0;
+  int32_t numOfTags = 0;
+
+  int32_t k = 0;
+  for (int32_t i = 0; i < numOfCols; ++i) {
+    SFillColInfo* pColInfo = &pFillInfo->pFillCol[i];
+    pFillInfo->pData[i] = NULL;
+
+    if (TSDB_COL_IS_TAG(pColInfo->flag) || pColInfo->col.type == TSDB_DATA_TYPE_BINARY) {
+      numOfTags += 1;
+
+      bool exists = false;
+      int32_t index = -1;
+      for (int32_t j = 0; j < k; ++j) {
+        if (pFillInfo->pTags[j].col.colId == pColInfo->col.colId) {
+          exists = true;
+          index = j;
+          break;
+        }
+      }
+
+      if (!exists) {
+        SSchema* pSchema = &pFillInfo->pTags[k].col;
+        pSchema->colId = pColInfo->col.colId;
+        pSchema->type  = pColInfo->col.type;
+        pSchema->bytes = pColInfo->col.bytes;
+
+        pFillInfo->pTags[k].tagVal = calloc(1, pColInfo->col.bytes);
+        pColInfo->tagIndex = k;
+
+        k += 1;
+      } else {
+        pColInfo->tagIndex = index;
+      }
+    }
+
+    rowsize += pColInfo->col.bytes;
+  }
+
+  pFillInfo->numOfTags = numOfTags;
+
+  assert(k <= pFillInfo->numOfTags);
+  return rowsize;
+}
+
+static int32_t taosNumOfRemainRows(SFillInfo* pFillInfo) {
+  if (pFillInfo->numOfRows == 0 || (pFillInfo->numOfRows > 0 && pFillInfo->index >= pFillInfo->numOfRows)) {
+    return 0;
+  }
+
+  return pFillInfo->numOfRows - pFillInfo->index;
+}
+
+SFillInfo* taosCreateFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_t capacity, int32_t numOfCols,
+                            int64_t slidingTime, int8_t slidingUnit, int8_t precision, int32_t fillType,
+                            SFillColInfo* pCol, void* handle) {
+  if (fillType == TSDB_FILL_NONE) {
+    return NULL;
+  }
+
+  SFillInfo* pFillInfo = calloc(1, sizeof(SFillInfo));
+  taosResetFillInfo(pFillInfo, skey);
+
+  pFillInfo->order     = order;
+  pFillInfo->type      = fillType;
+  pFillInfo->pFillCol  = pCol;
+  pFillInfo->numOfTags = numOfTags;
+  pFillInfo->numOfCols = numOfCols;
+  pFillInfo->precision = precision;
+  pFillInfo->alloc     = capacity;
+  pFillInfo->handle    = handle;
+
+  pFillInfo->interval.interval     = slidingTime;
+  pFillInfo->interval.intervalUnit = slidingUnit;
+  pFillInfo->interval.sliding      = slidingTime;
+  pFillInfo->interval.slidingUnit  = slidingUnit;
+
+  pFillInfo->pData = malloc(POINTER_BYTES * numOfCols);
+
+//  if (numOfTags > 0) {
+    pFillInfo->pTags = calloc(numOfCols, sizeof(SFillTagColInfo));
+    for (int32_t i = 0; i < numOfCols; ++i) {
+      pFillInfo->pTags[i].col.colId = -2;  // TODO
+    }
+//  }
+
+  pFillInfo->rowSize = setTagColumnInfo(pFillInfo, pFillInfo->numOfCols, pFillInfo->alloc);
+  assert(pFillInfo->rowSize > 0);
+
+  return pFillInfo;
+}
+
+void taosResetFillInfo(SFillInfo* pFillInfo, TSKEY startTimestamp) {
+  pFillInfo->start        = startTimestamp;
+  pFillInfo->currentKey   = startTimestamp;
+  pFillInfo->end          = startTimestamp;
+  pFillInfo->index        = -1;
+  pFillInfo->numOfRows    = 0;
+  pFillInfo->numOfCurrent = 0;
+  pFillInfo->numOfTotal   = 0;
+}
+
+void* taosDestroyFillInfo(SFillInfo* pFillInfo) {
+  if (pFillInfo == NULL) {
+    return NULL;
+  }
+
+  tfree(pFillInfo->prevValues);
+  tfree(pFillInfo->nextValues);
+
+  for(int32_t i = 0; i < pFillInfo->numOfTags; ++i) {
+    tfree(pFillInfo->pTags[i].tagVal);
+  }
+
+  tfree(pFillInfo->pTags);
+  
+  tfree(pFillInfo->pData);
+  tfree(pFillInfo->pFillCol);
+  
+  tfree(pFillInfo);
+  return NULL;
+}
+
+void taosFillSetStartInfo(SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey) {
+  if (pFillInfo->type == TSDB_FILL_NONE) {
+    return;
+  }
+
+  pFillInfo->end = endKey;
+  if (!FILL_IS_ASC_FILL(pFillInfo)) {
+    pFillInfo->end = taosTimeTruncate(endKey, &pFillInfo->interval, pFillInfo->precision);
+  }
+
+  pFillInfo->index     = 0;
+  pFillInfo->numOfRows = numOfRows;
+}
+
+void taosFillSetInputDataBlock(SFillInfo* pFillInfo, const SSDataBlock* pInput) {
+  for (int32_t i = 0; i < pFillInfo->numOfCols; ++i) {
+    SFillColInfo* pCol = &pFillInfo->pFillCol[i];
+
+    SColumnInfoData* pColData = taosArrayGet(pInput->pDataBlock, i);
+    pFillInfo->pData[i] = pColData->pData;
+
+    if (TSDB_COL_IS_TAG(pCol->flag)) {  // copy the tag value to tag value buffer
+      SFillTagColInfo* pTag = &pFillInfo->pTags[pCol->tagIndex];
+      assert (pTag->col.colId == pCol->col.colId);
+      memcpy(pTag->tagVal, pColData->pData, pCol->col.bytes);  // TODO not memcpy??
+    }
+  }
+}
+
+bool taosFillHasMoreResults(SFillInfo* pFillInfo) {
+  int32_t remain = taosNumOfRemainRows(pFillInfo);
+  if (remain > 0) {
+    return true;
+  }
+
+  if (pFillInfo->numOfTotal > 0 && (((pFillInfo->end > pFillInfo->start) && FILL_IS_ASC_FILL(pFillInfo)) ||
+                                    (pFillInfo->end < pFillInfo->start && !FILL_IS_ASC_FILL(pFillInfo)))) {
+    return getNumOfResultsAfterFillGap(pFillInfo, pFillInfo->end, 4096) > 0;
+  }
+
+  return false;
+}
+
+int64_t getNumOfResultsAfterFillGap(SFillInfo* pFillInfo, TSKEY ekey, int32_t maxNumOfRows) {
+  int64_t* tsList = (int64_t*) pFillInfo->pData[0];
+
+  int32_t numOfRows = taosNumOfRemainRows(pFillInfo);
+
+  TSKEY ekey1 = ekey;
+  if (!FILL_IS_ASC_FILL(pFillInfo)) {
+    pFillInfo->end = taosTimeTruncate(ekey, &pFillInfo->interval, pFillInfo->precision);
+  }
+
+  int64_t numOfRes = -1;
+  if (numOfRows > 0) {  // still fill gap within current data block, not generating data after the result set.
+    TSKEY lastKey = tsList[pFillInfo->numOfRows - 1];
+    numOfRes = taosTimeCountInterval(
+      lastKey,
+      pFillInfo->currentKey,
+      pFillInfo->interval.sliding,
+      pFillInfo->interval.slidingUnit,
+      pFillInfo->precision);
+    numOfRes += 1;
+    assert(numOfRes >= numOfRows);
+  } else { // reach the end of data
+    if ((ekey1 < pFillInfo->currentKey && FILL_IS_ASC_FILL(pFillInfo)) ||
+        (ekey1 > pFillInfo->currentKey && !FILL_IS_ASC_FILL(pFillInfo))) {
+      return 0;
+    }
+    numOfRes = taosTimeCountInterval(
+      ekey1,
+      pFillInfo->currentKey,
+      pFillInfo->interval.sliding,
+      pFillInfo->interval.slidingUnit,
+      pFillInfo->precision);
+    numOfRes += 1;
+  }
+
+  return (numOfRes > maxNumOfRows) ? maxNumOfRows : numOfRes;
+}
+
+int32_t taosGetLinearInterpolationVal(SPoint* point, int32_t outputType, SPoint* point1, SPoint* point2, int32_t inputType) {
+  double v1 = -1, v2 = -1;
+  GET_TYPED_DATA(v1, double, inputType, point1->val);
+  GET_TYPED_DATA(v2, double, inputType, point2->val);
+
+  double r = DO_INTERPOLATION(v1, v2, point1->key, point2->key, point->key);
+  SET_TYPED_DATA(point->val, outputType, r);
+
+  return TSDB_CODE_SUCCESS;
+}
+
+int64_t taosFillResultDataBlock(SFillInfo* pFillInfo, void** output, int32_t capacity) {
+  int32_t remain = taosNumOfRemainRows(pFillInfo);
+
+  int64_t numOfRes = getNumOfResultsAfterFillGap(pFillInfo, pFillInfo->end, capacity);
+  assert(numOfRes <= capacity);
+
+  // no data existed for fill operation now, append result according to the fill strategy
+  if (remain == 0) {
+    appendFilledResult(pFillInfo, output, numOfRes);
+  } else {
+    fillResultImpl(pFillInfo, output, (int32_t) numOfRes);
+    assert(numOfRes == pFillInfo->numOfCurrent);
+  }
+
+//  qDebug("fill:%p, generated fill result, src block:%d, index:%d, brange:%"PRId64"-%"PRId64", currentKey:%"PRId64", current:%d, total:%d, %p",
+//      pFillInfo, pFillInfo->numOfRows, pFillInfo->index, pFillInfo->start, pFillInfo->end, pFillInfo->currentKey, pFillInfo->numOfCurrent,
+//         pFillInfo->numOfTotal, pFillInfo->handle);
+
+  return numOfRes;
+}

source/libs/function/src/tfunction.c

+#include "os.h"
+#include "tarray.h"
+#include "function.h"
+#include "thash.h"
+#include "taggfunction.h"
+#include "tscalarfunction.h"
+
+static SHashObj* functionHashTable = NULL;
+
+static void doInitFunctionHashTable() {
+  int numOfEntries = tListLen(aggFunc);
+  functionHashTable = taosHashInit(numOfEntries, MurmurHash3_32, false, false);
+  for (int32_t i = 0; i < numOfEntries; i++) {
+    int32_t len = (uint32_t)strlen(aggFunc[i].name);
+    taosHashPut(functionHashTable, aggFunc[i].name, len, (void*)&aggFunc[i], POINTER_BYTES);
+  }
+
+  numOfEntries = tListLen(scalarFunc);
+  for(int32_t i = 0; i < numOfEntries; ++i) {
+    int32_t len = (int32_t) strlen(scalarFunc[i].name);
+    taosHashPut(functionHashTable, scalarFunc[i].name, len, (void*)&scalarFunc[i], POINTER_BYTES);
+  }
+}
+
+static pthread_once_t functionHashTableInit = PTHREAD_ONCE_INIT;
+
+int32_t qIsBuiltinFunction(const char* name, int32_t len) {
+  pthread_once(&functionHashTableInit, doInitFunctionHashTable);
+
+  SAggFunctionInfo** pInfo = taosHashGet(functionHashTable, name, len);
+  if (pInfo != NULL) {
+    return (*pInfo)->functionId;
+  } else {
+    return -1;
+  }
+}
+
+bool isTagsQuery(SArray* pFunctionIdList) {
+  int32_t num = (int32_t) taosArrayGetSize(pFunctionIdList);
+  for (int32_t i = 0; i < num; ++i) {
+    int16_t f = *(int16_t*) taosArrayGet(pFunctionIdList, i);
+
+    // "select count(tbname)" query
+//    if (functId == FUNCTION_COUNT && pExpr->base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) {
+//      continue;
+//    }
+
+    if (f != FUNCTION_TAGPRJ && f != FUNCTION_TID_TAG) {
+      return false;
+    }
+  }
+
+  return true;
+}
+
+//bool tscMultiRoundQuery(SArray* pFunctionIdList, int32_t index) {
+//  if (!UTIL_TABLE_IS_SUPER_TABLE(pQueryInfo->pTableMetaInfo[index])) {
+//    return false;
+//  }
+//
+//  size_t numOfExprs = (int32_t) getNumOfExprs(pQueryInfo);
+//  for(int32_t i = 0; i < numOfExprs; ++i) {
+//    SExprInfo* pExpr = getExprInfo(pQueryInfo, i);
+//    if (pExpr->base.functionId == FUNCTION_STDDEV_DST) {
+//      return true;
+//    }
+//  }
+//
+//  return false;
+//}
+
+bool isBlockInfoQuery(SArray* pFunctionIdList) {
+  int32_t num = (int32_t) taosArrayGetSize(pFunctionIdList);
+  for (int32_t i = 0; i < num; ++i) {
+    int32_t f = *(int16_t*) taosArrayGet(pFunctionIdList, i);
+
+    if (f == FUNCTION_BLKINFO) {
+      return true;
+    }
+  }
+
+  return false;
+}
+
+bool isProjectionQuery(SArray* pFunctionIdList) {
+  int32_t num = (int32_t) taosArrayGetSize(pFunctionIdList);
+  for (int32_t i = 0; i < num; ++i) {
+    int32_t f = *(int16_t*) taosArrayGet(pFunctionIdList, i);
+    if (f == FUNCTION_TS_DUMMY) {
+      continue;
+    }
+
+    if (f != FUNCTION_PRJ && f != FUNCTION_TAGPRJ && f != FUNCTION_TAG &&
+        f != FUNCTION_TS && f != FUNCTION_ARITHM && f != FUNCTION_DIFF &&
+        f != FUNCTION_DERIVATIVE) {
+      return false;
+    }
+  }
+
+  return true;
+}
+
+bool isDiffDerivQuery(SArray* pFunctionIdList) {
+  int32_t num = (int32_t) taosArrayGetSize(pFunctionIdList);
+  for (int32_t i = 0; i < num; ++i) {
+    int32_t f = *(int16_t*) taosArrayGet(pFunctionIdList, i);
+    if (f == FUNCTION_TS_DUMMY) {
+      continue;
+    }
+
+    if (f == FUNCTION_DIFF || f == FUNCTION_DERIVATIVE) {
+      return true;
+    }
+  }
+
+  return false;
+}
+
+bool isPointInterpQuery(SArray* pFunctionIdList) {
+  int32_t num = (int32_t) taosArrayGetSize(pFunctionIdList);
+  for (int32_t i = 0; i < num; ++i) {
+    int32_t f = *(int16_t*) taosArrayGet(pFunctionIdList, i);
+    if (f == FUNCTION_TAG || f == FUNCTION_TS) {
+      continue;
+    }
+
+    if (f != FUNCTION_INTERP) {
+      return false;
+    }
+  }
+
+  return true;
+}
+
+bool isArithmeticQueryOnAggResult(SArray* pFunctionIdList) {
+  if (isProjectionQuery(pFunctionIdList)) {
+    return false;
+  }
+
+  assert(0);
+
+//  size_t numOfOutput = getNumOfFields(pQueryInfo);
+//  for(int32_t i = 0; i < numOfOutput; ++i) {
+//    SExprInfo* pExprInfo = tscFieldInfoGetInternalField(&pQueryInfo->fieldsInfo, i)->pExpr;
+//    if (pExprInfo->pExpr != NULL) {
+//      return true;
+//    }
+//  }
+
+  return false;
+}
+
+bool isGroupbyColumn(SArray* pFunctionIdList) {
+//  STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
+//  int32_t         numOfCols = getNumOfColumns(pTableMetaInfo->pTableMeta);
+//
+//  SGroupbyExpr* pGroupbyExpr = &pQueryInfo->groupbyExpr;
+//  for (int32_t k = 0; k < pGroupbyExpr->numOfGroupCols; ++k) {
+//    SColIndex* pIndex = taosArrayGet(pGroupbyExpr->columnInfo, k);
+//    if (!TSDB_COL_IS_TAG(pIndex->flag) && pIndex->colIndex < numOfCols) {  // group by normal columns
+//      return true;
+//    }
+//  }
+
+  return false;
+}
+
+bool isTopBotQuery(SArray* pFunctionIdList) {
+  int32_t num = (int32_t) taosArrayGetSize(pFunctionIdList);
+  for (int32_t i = 0; i < num; ++i) {
+    int32_t f = *(int16_t*) taosArrayGet(pFunctionIdList, i);
+    if (f == FUNCTION_TS) {
+      continue;
+    }
+
+    if (f == FUNCTION_TOP || f == FUNCTION_BOTTOM) {
+      return true;
+    }
+  }
+
+  return false;
+}
+
+bool isTsCompQuery(SArray* pFunctionIdList) {
+  int32_t num = (int32_t) taosArrayGetSize(pFunctionIdList);
+  if (num != 1) {
+    return false;
+  }
+
+  int32_t f = *(int16_t*) taosArrayGet(pFunctionIdList, 0);
+  return f == FUNCTION_TS_COMP;
+}
+
+bool isTWAQuery(SArray* pFunctionIdList) {
+  int32_t num = (int32_t) taosArrayGetSize(pFunctionIdList);
+  for (int32_t i = 0; i < num; ++i) {
+    int32_t f = *(int16_t*) taosArrayGet(pFunctionIdList, i);
+    if (f == FUNCTION_TWA) {
+      return true;
+    }
+  }
+
+  return false;
+}
+
+bool isIrateQuery(SArray* pFunctionIdList) {
+  int32_t num = (int32_t) taosArrayGetSize(pFunctionIdList);
+  for (int32_t i = 0; i < num; ++i) {
+    int32_t f = *(int16_t*) taosArrayGet(pFunctionIdList, i);
+    if (f == FUNCTION_IRATE) {
+      return true;
+    }
+  }
+
+  return false;
+}
+
+bool isStabledev(SArray* pFunctionIdList) {
+  int32_t num = (int32_t) taosArrayGetSize(pFunctionIdList);
+  for (int32_t i = 0; i < num; ++i) {
+    int32_t f = *(int16_t*) taosArrayGet(pFunctionIdList, i);
+    if (f == FUNCTION_STDDEV_DST) {
+      return true;
+    }
+  }
+
+  return false;
+}
+
+bool needReverseScan(SArray* pFunctionIdList) {
+  assert(0);
+  int32_t num = (int32_t) taosArrayGetSize(pFunctionIdList);
+  for (int32_t i = 0; i < num; ++i) {
+    int32_t f = *(int16_t*) taosArrayGet(pFunctionIdList, i);
+    if (f == FUNCTION_TS || f == FUNCTION_TS_DUMMY || f == FUNCTION_TAG) {
+      continue;
+    }
+
+//    if ((f == FUNCTION_FIRST || f == FUNCTION_FIRST_DST) && pQueryInfo->order.order == TSDB_ORDER_DESC) {
+//      return true;
+//    }
+
+    if (f == FUNCTION_LAST || f == FUNCTION_LAST_DST) {
+      // the scan order to acquire the last result of the specified column
+//      int32_t order = (int32_t)pExpr->base.param[0].i64;
+//      if (order != pQueryInfo->order.order) {
+//        return true;
+//      }
+    }
+  }
+
+  return false;
+}
+
+bool isSimpleAggregateRv(SArray* pFunctionIdList) {
+  assert(0);
+
+//  if (pQueryInfo->interval.interval > 0 || pQueryInfo->sessionWindow.gap > 0) {
+//    return false;
+//  }
+//
+//  if (tscIsDiffDerivQuery(pQueryInfo)) {
+//    return false;
+//  }
+//
+//  size_t numOfExprs = getNumOfExprs(pQueryInfo);
+//  for (int32_t i = 0; i < numOfExprs; ++i) {
+//    SExprInfo* pExpr = getExprInfo(pQueryInfo, i);
+//    if (pExpr == NULL) {
+//      continue;
+//    }
+//
+//    int32_t functionId = pExpr->base.functionId;
+//    if (functionId < 0) {
+//      SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, -1 * functionId - 1);
+//      if (pUdfInfo->funcType == TSDB_UDF_TYPE_AGGREGATE) {
+//        return true;
+//      }
+//
+//      continue;
+//    }
+//
+//    if (functionId == FUNCTION_TS || functionId == FUNCTION_TS_DUMMY) {
+//      continue;
+//    }
+//
+//    if ((!IS_MULTIOUTPUT(aAggs[functionId].status)) ||
+//        (functionId == FUNCTION_TOP || functionId == FUNCTION_BOTTOM || functionId == FUNCTION_TS_COMP)) {
+//      return true;
+//    }
+//  }
+
+  return false;
+}
+
+bool isBlockDistQuery(SArray* pFunctionIdList) {
+  int32_t num = (int32_t) taosArrayGetSize(pFunctionIdList);
+  int32_t f = *(int16_t*) taosArrayGet(pFunctionIdList, 0);
+  return (num == 1 && f == FUNCTION_BLKINFO);
+}
+
+bool isTwoStageSTableQuery(SArray* pFunctionIdList, int32_t tableIndex) {
+//  if (pQueryInfo == NULL) {
+//    return false;
+//  }
+//
+//  STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tableIndex);
+//  if (pTableMetaInfo == NULL) {
+//    return false;
+//  }
+//
+//  if ((pQueryInfo->type & TSDB_QUERY_TYPE_FREE_RESOURCE) == TSDB_QUERY_TYPE_FREE_RESOURCE) {
+//    return false;
+//  }
+//
+//  // for ordered projection query, iterate all qualified vnodes sequentially
+//  if (tscNonOrderedProjectionQueryOnSTable(pQueryInfo, tableIndex)) {
+//    return false;
+//  }
+//
+//  if (!TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_STABLE_SUBQUERY) && pQueryInfo->command == TSDB_SQL_SELECT) {
+//    return UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo);
+//  }
+
+  return false;
+}
+
+bool isProjectionQueryOnSTable(SArray* pFunctionIdList, int32_t tableIndex) {
+//  STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tableIndex);
+//
+//  /*
+//   * In following cases, return false for non ordered project query on super table
+//   * 1. failed to get tableMeta from server; 2. not a super table; 3. limitation is 0;
+//   * 4. show queries, instead of a select query
+//   */
+//  size_t numOfExprs = getNumOfExprs(pQueryInfo);
+//  if (pTableMetaInfo == NULL || !UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo) ||
+//      pQueryInfo->command == TSDB_SQL_RETRIEVE_EMPTY_RESULT || numOfExprs == 0) {
+//    return false;
+//  }
+//
+//  for (int32_t i = 0; i < numOfExprs; ++i) {
+//    int32_t functionId = getExprInfo(pQueryInfo, i)->base.functionId;
+//
+//    if (functionId < 0) {
+//      SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, -1 * functionId - 1);
+//      if (pUdfInfo->funcType == TSDB_UDF_TYPE_AGGREGATE) {
+//        return false;
+//      }
+//
+//      continue;
+//    }
+//
+//    if (functionId != FUNCTION_PRJ &&
+//        functionId != FUNCTION_TAGPRJ &&
+//        functionId != FUNCTION_TAG &&
+//        functionId != FUNCTION_TS &&
+//        functionId != FUNCTION_ARITHM &&
+//        functionId != FUNCTION_TS_COMP &&
+//        functionId != FUNCTION_DIFF &&
+//        functionId != FUNCTION_DERIVATIVE &&
+//        functionId != FUNCTION_TS_DUMMY &&
+//        functionId != FUNCTION_TID_TAG) {
+//      return false;
+//    }
+//  }
+
+  return true;
+}
+
+bool hasTagValOutput(SArray* pFunctionIdList) {
+//  size_t    numOfExprs = getNumOfExprs(pQueryInfo);
+//  SExprInfo* pExpr1 = getExprInfo(pQueryInfo, 0);
+//
+//  if (numOfExprs == 1 && pExpr1->base.functionId == FUNCTION_TS_COMP) {
+//    return true;
+//  }
+//
+//  for (int32_t i = 0; i < numOfExprs; ++i) {
+//    SExprInfo* pExpr = getExprInfo(pQueryInfo, i);
+//    if (pExpr == NULL) {
+//      continue;
+//    }
+//
+//    // ts_comp column required the tag value for join filter
+//    if (TSDB_COL_IS_TAG(pExpr->base.colInfo.flag)) {
+//      return true;
+//    }
+//  }
+
+  return false;
+}
+
+bool timeWindowInterpoRequired(SArray* pFunctionIdList) {
+  int32_t num = (int32_t) taosArrayGetSize(pFunctionIdList);
+  for (int32_t i = 0; i < num; ++i) {
+    int32_t f = *(int16_t*) taosArrayGet(pFunctionIdList, i);
+    if (f == FUNCTION_TWA || f == FUNCTION_INTERP) {
+      return true;
+    }
+  }
+
+  return false;
+}
+
+//SQueryType setQueryType(SArray* pFunctionIdList) {
+//  assert(pFunctionIdList != NULL);
+//
+//
+//}
\ No newline at end of file

source/libs/function/src/thistogram.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+#include "os.h"
+
+#include "thistogram.h"
+#include "taosdef.h"
+#include "taosmsg.h"
+#include "tlosertree.h"
+
+/**
+ *
+ * implement the histogram and percentile_approx based on the paper:
+ * Yael Ben-Haim, Elad Tom-Tov. A Streaming Parallel Decision Tree Algorithm,
+ * The Journal of Machine Learning Research.Volume 11, 3/1/2010 pp.849-872
+ * https://dl.acm.org/citation.cfm?id=1756034
+ *
+ * @data 2018-12-14
+ * @version 0.1
+ *
+ */
+static int32_t histogramCreateBin(SHistogramInfo* pHisto, int32_t index, double val);
+
+SHistogramInfo* tHistogramCreate(int32_t numOfEntries) {
+  /* need one redundant slot */
+  SHistogramInfo* pHisto = malloc(sizeof(SHistogramInfo) + sizeof(SHistBin) * (numOfEntries + 1));
+
+#if !defined(USE_ARRAYLIST)
+  pHisto->pList = SSkipListCreate(MAX_SKIP_LIST_LEVEL, TSDB_DATA_TYPE_DOUBLE, sizeof(double));
+  SInsertSupporter* pss = malloc(sizeof(SInsertSupporter));
+  pss->numOfEntries = pHisto->maxEntries;
+  pss->pSkipList = pHisto->pList;
+
+  int32_t ret = tLoserTreeCreate1(&pHisto->pLoserTree, numOfEntries, pss, compare);
+  pss->pTree = pHisto->pLoserTree;
+#endif
+
+  return tHistogramCreateFrom(pHisto, numOfEntries);
+}
+
+SHistogramInfo* tHistogramCreateFrom(void* pBuf, int32_t numOfBins) {
+  memset(pBuf, 0, sizeof(SHistogramInfo) + sizeof(SHistBin) * (numOfBins + 1));
+
+  SHistogramInfo* pHisto = (SHistogramInfo*)pBuf;
+  pHisto->elems = (SHistBin*)((char*)pBuf + sizeof(SHistogramInfo));
+  for(int32_t i = 0; i < numOfBins; ++i) {
+    pHisto->elems[i].val = -DBL_MAX;
+  }
+
+  pHisto->maxEntries = numOfBins;
+
+  pHisto->min = DBL_MAX;
+  pHisto->max = -DBL_MAX;
+
+  return pBuf;
+}
+
+int32_t tHistogramAdd(SHistogramInfo** pHisto, double val) {
+  if (*pHisto == NULL) {
+    *pHisto = tHistogramCreate(MAX_HISTOGRAM_BIN);
+  }
+
+#if defined(USE_ARRAYLIST)
+  int32_t idx = histoBinarySearch((*pHisto)->elems, (*pHisto)->numOfEntries, val);
+  assert(idx >= 0 && idx <= (*pHisto)->maxEntries && (*pHisto)->elems != NULL);
+
+  if ((*pHisto)->elems[idx].val == val && idx >= 0) {
+    (*pHisto)->elems[idx].num += 1;
+
+    if ((*pHisto)->numOfEntries == 0) {
+      (*pHisto)->numOfEntries += 1;
+    }
+  } else { /* insert a new slot */
+    if ((*pHisto)->numOfElems >= 1 && idx < (*pHisto)->numOfEntries) {
+      if (idx > 0) {
+        assert((*pHisto)->elems[idx - 1].val <= val);
+      } else {
+        assert((*pHisto)->elems[idx].val > val);
+      }
+    } else if ((*pHisto)->numOfElems > 0) {
+      assert((*pHisto)->elems[(*pHisto)->numOfEntries].val <= val);
+    }
+
+    histogramCreateBin(*pHisto, idx, val);
+  }
+#else
+  tSkipListKey key = tSkipListCreateKey(TSDB_DATA_TYPE_DOUBLE, &val, tDataTypes[TSDB_DATA_TYPE_DOUBLE].nSize);
+  SHistBin*    entry = calloc(1, sizeof(SHistBin));
+  entry->val = val;
+
+  tSkipListNode* pResNode = SSkipListPut((*pHisto)->pList, entry, &key, 0);
+
+  SHistBin* pEntry1 = (SHistBin*)pResNode->pData;
+  pEntry1->index = -1;
+
+  tSkipListNode* pLast = NULL;
+
+  if (pEntry1->num == 0) { /* it is a new node */
+    (*pHisto)->numOfEntries += 1;
+    pEntry1->num += 1;
+
+    /* number of entries reaches the upper limitation */
+    if (pResNode->pForward[0] != NULL) {
+      /* we need to update the last updated slot in loser tree*/
+      pEntry1->delta = ((SHistBin*)pResNode->pForward[0]->pData)->val - val;
+
+      if ((*pHisto)->ordered) {
+        int32_t         lastIndex = (*pHisto)->maxIndex;
+        SLoserTreeInfo* pTree = (*pHisto)->pLoserTree;
+
+        (*pHisto)->pLoserTree->pNode[lastIndex + pTree->numOfEntries].pData = pResNode;
+        pEntry1->index = (*pHisto)->pLoserTree->pNode[lastIndex + pTree->numOfEntries].index;
+
+        // update the loser tree
+        if ((*pHisto)->ordered) {
+          tLoserTreeAdjust(pTree, pEntry1->index + pTree->numOfEntries);
+        }
+
+        tSkipListKey kx =
+            tSkipListCreateKey(TSDB_DATA_TYPE_DOUBLE, &(*pHisto)->max, tDataTypes[TSDB_DATA_TYPE_DOUBLE].nSize);
+        pLast = tSkipListGetOne((*pHisto)->pList, &kx);
+      }
+    } else {
+      /* this node located at the last position of the skiplist, we do not
+       * update the loser-tree */
+      pEntry1->delta = DBL_MAX;
+      pLast = pResNode;
+    }
+
+    if (pResNode->pBackward[0] != &(*pHisto)->pList->pHead) {
+      SHistBin* pPrevEntry = (SHistBin*)pResNode->pBackward[0]->pData;
+      pPrevEntry->delta = val - pPrevEntry->val;
+
+      SLoserTreeInfo* pTree = (*pHisto)->pLoserTree;
+      if ((*pHisto)->ordered) {
+        tLoserTreeAdjust(pTree, pPrevEntry->index + pTree->numOfEntries);
+        tLoserTreeDisplay(pTree);
+      }
+    }
+
+    if ((*pHisto)->numOfEntries >= (*pHisto)->maxEntries + 1) {
+      // set the right value for loser-tree
+      assert((*pHisto)->pLoserTree != NULL);
+      if (!(*pHisto)->ordered) {
+        SSkipListPrint((*pHisto)->pList, 1);
+
+        SLoserTreeInfo* pTree = (*pHisto)->pLoserTree;
+        tSkipListNode*  pHead = (*pHisto)->pList->pHead.pForward[0];
+
+        tSkipListNode* p1 = pHead;
+
+        printf("\n");
+        while (p1 != NULL) {
+          printf("%f\t", ((SHistBin*)(p1->pData))->delta);
+          p1 = p1->pForward[0];
+        }
+        printf("\n");
+
+        /* last one in skiplist is ignored */
+        for (int32_t i = pTree->numOfEntries; i < pTree->totalEntries; ++i) {
+          pTree->pNode[i].pData = pHead;
+          pTree->pNode[i].index = i - pTree->numOfEntries;
+          SHistBin* pBin = (SHistBin*)pHead->pData;
+          pBin->index = pTree->pNode[i].index;
+
+          pHead = pHead->pForward[0];
+        }
+
+        pLast = pHead;
+
+        for (int32_t i = 0; i < pTree->numOfEntries; ++i) {
+          pTree->pNode[i].index = -1;
+        }
+
+        tLoserTreeDisplay(pTree);
+
+        for (int32_t i = pTree->totalEntries - 1; i >= pTree->numOfEntries; i--) {
+          tLoserTreeAdjust(pTree, i);
+        }
+
+        tLoserTreeDisplay(pTree);
+        (*pHisto)->ordered = true;
+      }
+
+      printf("delta is:%lf\n", pEntry1->delta);
+
+      SSkipListPrint((*pHisto)->pList, 1);
+
+      /* the chosen node */
+      tSkipListNode* pNode = (*pHisto)->pLoserTree->pNode[0].pData;
+      SHistBin*      pEntry = (SHistBin*)pNode->pData;
+
+      tSkipListNode* pNext = pNode->pForward[0];
+      SHistBin*      pNextEntry = (SHistBin*)pNext->pData;
+      assert(pNextEntry->val - pEntry->val == pEntry->delta);
+
+      double newVal = (pEntry->val * pEntry->num + pNextEntry->val * pNextEntry->num) / (pEntry->num + pNextEntry->num);
+      pEntry->val = newVal;
+      pNode->key.dKey = newVal;
+      pEntry->num = pEntry->num + pNextEntry->num;
+
+      // update delta value in current node
+      pEntry->delta = (pNextEntry->delta + pNextEntry->val) - pEntry->val;
+
+      // reset delta value in the previous node
+      SHistBin* pPrevEntry = (SHistBin*)pNode->pBackward[0]->pData;
+      if (pPrevEntry) {
+        pPrevEntry->delta = pEntry->val - pPrevEntry->val;
+      }
+
+      SLoserTreeInfo* pTree = (*pHisto)->pLoserTree;
+      if (pNextEntry->index != -1) {
+        (*pHisto)->maxIndex = pNextEntry->index;
+
+        // set the last element in skiplist, of which delta is FLT_MAX;
+        pTree->pNode[pNextEntry->index + pTree->numOfEntries].pData = pLast;
+        ((SHistBin*)pLast->pData)->index = pNextEntry->index;
+        int32_t f = pTree->pNode[pNextEntry->index + pTree->numOfEntries].index;
+        printf("disappear index is:%d\n", f);
+      }
+
+      tLoserTreeAdjust(pTree, pEntry->index + pTree->numOfEntries);
+      // remove the next node in skiplist
+      tSkipListRemoveNode((*pHisto)->pList, pNext);
+      SSkipListPrint((*pHisto)->pList, 1);
+
+      tLoserTreeDisplay((*pHisto)->pLoserTree);
+    } else {  // add to heap
+      if (pResNode->pForward[0] != NULL) {
+        pEntry1->delta = ((SHistBin*)pResNode->pForward[0]->pData)->val - val;
+      } else {
+        pEntry1->delta = DBL_MAX;
+      }
+
+      if (pResNode->pBackward[0] != &(*pHisto)->pList->pHead) {
+        SHistBin* pPrevEntry = (SHistBin*)pResNode->pBackward[0]->pData;
+        pEntry1->delta = val - pPrevEntry->val;
+      }
+
+      printf("delta is:%9lf\n", pEntry1->delta);
+    }
+
+  } else {
+    SHistBin* pEntry = (SHistBin*)pResNode->pData;
+    assert(pEntry->val == val);
+    pEntry->num += 1;
+  }
+
+#endif
+  if (val > (*pHisto)->max) {
+    (*pHisto)->max = val;
+  }
+
+  if (val < (*pHisto)->min) {
+    (*pHisto)->min = val;
+  }
+
+  (*pHisto)->numOfElems += 1;
+  return 0;
+}
+
+int32_t histoBinarySearch(SHistBin* pEntry, int32_t len, double val) {
+  int32_t end = len - 1;
+  int32_t start = 0;
+
+  while (start <= end) {
+    int32_t mid = (end - start) / 2 + start;
+    if (pEntry[mid].val == val) {
+      return mid;
+    }
+
+    if (pEntry[mid].val < val) {
+      start = mid + 1;
+    } else {
+      end = mid - 1;
+    }
+  }
+
+  int32_t ret = start > end ? start : end;
+  if (ret < 0) {
+    return 0;
+  } else {
+    return ret;
+  }
+}
+
+static void histogramMergeImpl(SHistBin* pHistBin, int32_t* size) {
+#if defined(USE_ARRAYLIST)
+  int32_t oldSize = *size;
+
+  double  delta = DBL_MAX;
+  int32_t index = -1;
+  for (int32_t i = 1; i < oldSize; ++i) {
+    double d = pHistBin[i].val - pHistBin[i - 1].val;
+    if (d < delta) {
+      delta = d;
+      index = i - 1;
+    }
+  }
+
+  SHistBin* s1 = &pHistBin[index];
+  SHistBin* s2 = &pHistBin[index + 1];
+
+  double newVal = (s1->val * s1->num + s2->val * s2->num) / (s1->num + s2->num);
+  s1->val = newVal;
+  s1->num = s1->num + s2->num;
+
+  memmove(&pHistBin[index + 1], &pHistBin[index + 2], (oldSize - index - 2) * sizeof(SHistBin));
+  (*size) -= 1;
+#endif
+}
+
+/* optimize this procedure */
+int32_t histogramCreateBin(SHistogramInfo* pHisto, int32_t index, double val) {
+#if defined(USE_ARRAYLIST)
+  int32_t remain = pHisto->numOfEntries - index;
+  if (remain > 0) {
+    memmove(&pHisto->elems[index + 1], &pHisto->elems[index], sizeof(SHistBin) * remain);
+  }
+
+  assert(index >= 0 && index <= pHisto->maxEntries);
+
+  pHisto->elems[index].num = 1;
+  pHisto->elems[index].val = val;
+  pHisto->numOfEntries += 1;
+
+  /* we need to merge the slot */
+  if (pHisto->numOfEntries == pHisto->maxEntries + 1) {
+    histogramMergeImpl(pHisto->elems, &pHisto->numOfEntries);
+
+    pHisto->elems[pHisto->maxEntries].val = 0;
+    pHisto->elems[pHisto->maxEntries].num = 0;
+  }
+#endif
+  assert(pHisto->numOfEntries <= pHisto->maxEntries);
+  return 0;
+}
+
+void tHistogramDestroy(SHistogramInfo** pHisto) {
+  if (*pHisto == NULL) {
+    return;
+  }
+
+  free(*pHisto);
+  *pHisto = NULL;
+}
+
+void tHistogramPrint(SHistogramInfo* pHisto) {
+  printf("total entries: %d, elements: %"PRId64 "\n", pHisto->numOfEntries, pHisto->numOfElems);
+#if defined(USE_ARRAYLIST)
+  for (int32_t i = 0; i < pHisto->numOfEntries; ++i) {
+    printf("%d: (%f, %" PRId64 ")\n", i + 1, pHisto->elems[i].val, pHisto->elems[i].num);
+  }
+#else
+  tSkipListNode* pNode = pHisto->pList->pHead.pForward[0];
+
+  for (int32_t i = 0; i < pHisto->numOfEntries; ++i) {
+    SHistBin* pEntry = (SHistBin*)pNode->pData;
+    printf("%d: (%f, %" PRId64 ")\n", i + 1, pEntry->val, pEntry->num);
+    pNode = pNode->pForward[0];
+  }
+#endif
+}
+
+/**
+ * Estimated number of points in the interval (−inf,b].
+ * @param pHisto
+ * @param v
+ */
+int64_t tHistogramSum(SHistogramInfo* pHisto, double v) {
+#if defined(USE_ARRAYLIST)
+  int32_t slotIdx = histoBinarySearch(pHisto->elems, pHisto->numOfEntries, v);
+  if (pHisto->elems[slotIdx].val != v) {
+    slotIdx -= 1;
+
+    if (slotIdx < 0) {
+      slotIdx = 0;
+      assert(v <= pHisto->elems[slotIdx].val);
+    } else {
+      assert(v >= pHisto->elems[slotIdx].val);
+
+      if (slotIdx + 1 < pHisto->numOfEntries) {
+        assert(v < pHisto->elems[slotIdx + 1].val);
+      }
+    }
+  }
+
+  double m1 = (double)pHisto->elems[slotIdx].num;
+  double v1 = pHisto->elems[slotIdx].val;
+
+  double m2 = (double)pHisto->elems[slotIdx + 1].num;
+  double v2 = pHisto->elems[slotIdx + 1].val;
+
+  double estNum = m1 + (m2 - m1) * (v - v1) / (v2 - v1);
+  double s1 = (m1 + estNum) * (v - v1) / (2 * (v2 - v1));
+
+  for (int32_t i = 0; i < slotIdx; ++i) {
+    s1 += pHisto->elems[i].num;
+  }
+
+  s1 = s1 + m1 / 2;
+
+  return (int64_t)s1;
+#endif
+}
+
+double* tHistogramUniform(SHistogramInfo* pHisto, double* ratio, int32_t num) {
+#if defined(USE_ARRAYLIST)
+  double* pVal = malloc(num * sizeof(double));
+
+  for (int32_t i = 0; i < num; ++i) {
+    double numOfElem = (ratio[i] / 100) * pHisto->numOfElems;
+
+    if (numOfElem == 0) {
+      pVal[i] = pHisto->min;
+      continue;
+    } else if (numOfElem <= pHisto->elems[0].num) {
+      pVal[i] = pHisto->elems[0].val;
+      continue;
+    } else if (numOfElem == pHisto->numOfElems) {
+      pVal[i] = pHisto->max;
+      continue;
+    }
+
+    int32_t j = 0;
+    int64_t total = 0;
+
+    while (j < pHisto->numOfEntries) {
+      total += pHisto->elems[j].num;
+      if (total <= numOfElem && total + pHisto->elems[j + 1].num > numOfElem) {
+        break;
+      }
+
+      j += 1;
+    }
+
+    assert(total <= numOfElem && total + pHisto->elems[j + 1].num > numOfElem);
+
+    double delta = numOfElem - total;
+    if (fabs(delta) < FLT_EPSILON) {
+      pVal[i] = pHisto->elems[j].val;
+    }
+
+    double start = (double)pHisto->elems[j].num;
+    double range = pHisto->elems[j + 1].num - start;
+
+    if (range == 0) {
+      pVal[i] = (pHisto->elems[j + 1].val - pHisto->elems[j].val) * delta / start + pHisto->elems[j].val;
+    } else {
+      double factor = (-2 * start + sqrt(4 * start * start - 4 * range * (-2 * delta))) / (2 * range);
+      pVal[i] = pHisto->elems[j].val + (pHisto->elems[j + 1].val - pHisto->elems[j].val) * factor;
+    }
+  }
+#else
+  double* pVal = malloc(num * sizeof(double));
+
+  for (int32_t i = 0; i < num; ++i) {
+    double numOfElem = ratio[i] * pHisto->numOfElems;
+
+    tSkipListNode* pFirst = pHisto->pList->pHead.pForward[0];
+    SHistBin*      pEntry = (SHistBin*)pFirst->pData;
+    if (numOfElem == 0) {
+      pVal[i] = pHisto->min;
+      printf("i/numofSlot: %f, v:%f, %f\n", ratio[i], numOfElem, pVal[i]);
+      continue;
+    } else if (numOfElem <= pEntry->num) {
+      pVal[i] = pEntry->val;
+      printf("i/numofSlot: %f, v:%f, %f\n", ratio[i], numOfElem, pVal[i]);
+      continue;
+    } else if (numOfElem == pHisto->numOfElems) {
+      pVal[i] = pHisto->max;
+      printf("i/numofSlot: %f, v:%f, %f\n", ratio[i], numOfElem, pVal[i]);
+      continue;
+    }
+
+    int32_t   j = 0;
+    int64_t   total = 0;
+    SHistBin* pPrev = pEntry;
+
+    while (j < pHisto->numOfEntries) {
+      if (total <= numOfElem && total + pEntry->num > numOfElem) {
+        break;
+      }
+
+      total += pEntry->num;
+      pPrev = pEntry;
+
+      pFirst = pFirst->pForward[0];
+      pEntry = (SHistBin*)pFirst->pData;
+
+      j += 1;
+    }
+
+    assert(total <= numOfElem && total + pEntry->num > numOfElem);
+
+    double delta = numOfElem - total;
+    if (fabs(delta) < FLT_EPSILON) {
+      //                printf("i/numofSlot: %f, v:%f, %f\n",
+      //                (double)i/numOfSlots, numOfElem, pHisto->elems[j].val);
+      pVal[i] = pPrev->val;
+    }
+
+    double start = pPrev->num;
+    double range = pEntry->num - start;
+
+    if (range == 0) {
+      pVal[i] = (pEntry->val - pPrev->val) * delta / start + pPrev->val;
+    } else {
+      double factor = (-2 * start + sqrt(4 * start * start - 4 * range * (-2 * delta))) / (2 * range);
+      pVal[i] = pPrev->val + (pEntry->val - pPrev->val) * factor;
+    }
+    //            printf("i/numofSlot: %f, v:%f, %f\n", (double)i/numOfSlots,
+    //            numOfElem, val);
+  }
+#endif
+  return pVal;
+}
+
+SHistogramInfo* tHistogramMerge(SHistogramInfo* pHisto1, SHistogramInfo* pHisto2, int32_t numOfEntries) {
+  SHistogramInfo* pResHistogram = tHistogramCreate(numOfEntries);
+
+  // error in histogram info
+  if (pHisto1->numOfEntries > MAX_HISTOGRAM_BIN || pHisto2->numOfEntries > MAX_HISTOGRAM_BIN) {
+    return pResHistogram;
+  }
+
+  SHistBin* pHistoBins = calloc(1, sizeof(SHistBin) * (pHisto1->numOfEntries + pHisto2->numOfEntries));
+  int32_t i = 0, j = 0, k = 0;
+
+  while (i < pHisto1->numOfEntries && j < pHisto2->numOfEntries) {
+    if (pHisto1->elems[i].val < pHisto2->elems[j].val) {
+      pHistoBins[k++] = pHisto1->elems[i++];
+    } else if (pHisto1->elems[i].val > pHisto2->elems[j].val) {
+      pHistoBins[k++] = pHisto2->elems[j++];
+    } else {
+      pHistoBins[k] = pHisto1->elems[i++];
+      pHistoBins[k++].num += pHisto2->elems[j++].num;
+    }
+  }
+
+  if (i < pHisto1->numOfEntries) {
+    int32_t remain = pHisto1->numOfEntries - i;
+    memcpy(&pHistoBins[k], &pHisto1->elems[i], sizeof(SHistBin) * remain);
+    k += remain;
+  }
+
+  if (j < pHisto2->numOfEntries) {
+    int32_t remain = pHisto2->numOfEntries - j;
+    memcpy(&pHistoBins[k], &pHisto2->elems[j], sizeof(SHistBin) * remain);
+    k += remain;
+  }
+
+  /* update other information */
+  pResHistogram->numOfElems = pHisto1->numOfElems + pHisto2->numOfElems;
+  pResHistogram->min = (pHisto1->min < pHisto2->min) ? pHisto1->min : pHisto2->min;
+  pResHistogram->max = (pHisto1->max > pHisto2->max) ? pHisto1->max : pHisto2->max;
+
+  while (k > numOfEntries) {
+    histogramMergeImpl(pHistoBins, &k);
+  }
+
+  pResHistogram->numOfEntries = k;
+  memcpy(pResHistogram->elems, pHistoBins, sizeof(SHistBin) * k);
+
+  free(pHistoBins);
+  return pResHistogram;
+}

source/libs/function/src/tpercentile.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+#include "os.h"
+
+#include "tpercentile.h"
+#include "tpagedfile.h"
+#include "taosdef.h"
+#include "tcompare.h"
+#include "ttypes.h"
+
+#define DEFAULT_NUM_OF_SLOT 1024
+
+int32_t getGroupId(int32_t numOfSlots, int32_t slotIndex, int32_t times) {
+  return (times * numOfSlots) + slotIndex;
+}
+
+static SFilePage *loadDataFromFilePage(tMemBucket *pMemBucket, int32_t slotIdx) {
+  SFilePage *buffer = (SFilePage *)calloc(1, pMemBucket->bytes * pMemBucket->pSlots[slotIdx].info.size + sizeof(SFilePage));
+
+  int32_t groupId = getGroupId(pMemBucket->numOfSlots, slotIdx, pMemBucket->times);
+  SIDList list = getDataBufPagesIdList(pMemBucket->pBuffer, groupId);
+
+  int32_t offset = 0;
+  for(int32_t i = 0; i < list->size; ++i) {
+    SPageInfo* pgInfo = *(SPageInfo**) taosArrayGet(list, i);
+
+    SFilePage* pg = getResBufPage(pMemBucket->pBuffer, pgInfo->pageId);
+    memcpy(buffer->data + offset, pg->data, (size_t)(pg->num * pMemBucket->bytes));
+
+    offset += (int32_t)(pg->num * pMemBucket->bytes);
+  }
+
+  qsort(buffer->data, pMemBucket->pSlots[slotIdx].info.size, pMemBucket->bytes, pMemBucket->comparFn);
+  return buffer;
+}
+
+static void resetBoundingBox(MinMaxEntry* range, int32_t type) {
+  if (IS_SIGNED_NUMERIC_TYPE(type)) {
+    range->i64MaxVal = INT64_MIN;
+    range->i64MinVal = INT64_MAX;
+  } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) {
+    range->u64MaxVal = 0;
+    range->u64MinVal = UINT64_MAX;
+  } else {
+    range->dMaxVal = -DBL_MAX;
+    range->dMinVal = DBL_MAX;
+  }
+}
+
+static int32_t setBoundingBox(MinMaxEntry* range, int16_t type, double minval, double maxval) {
+  if (minval > maxval) {
+    return -1;
+  }
+
+  if (IS_SIGNED_NUMERIC_TYPE(type)) {
+    range->i64MinVal = (int64_t) minval;
+    range->i64MaxVal = (int64_t) maxval;
+  } else if (IS_UNSIGNED_NUMERIC_TYPE(type)){
+    range->u64MinVal = (uint64_t) minval;
+    range->u64MaxVal = (uint64_t) maxval;
+  } else {
+    range->dMinVal = minval;
+    range->dMaxVal = maxval;
+  }
+
+  return 0;
+}
+
+static void resetPosInfo(SSlotInfo* pInfo) {
+  pInfo->size   = 0;
+  pInfo->pageId = -1;
+  pInfo->data   = NULL;
+}
+
+double findOnlyResult(tMemBucket *pMemBucket) {
+  assert(pMemBucket->total == 1);
+
+  for (int32_t i = 0; i < pMemBucket->numOfSlots; ++i) {
+    tMemBucketSlot *pSlot = &pMemBucket->pSlots[i];
+    if (pSlot->info.size  == 0) {
+      continue;
+    }
+
+    int32_t groupId = getGroupId(pMemBucket->numOfSlots, i, pMemBucket->times);
+    SIDList list = getDataBufPagesIdList(pMemBucket->pBuffer, groupId);
+    assert(list->size == 1);
+
+    SPageInfo* pgInfo = (SPageInfo*) taosArrayGetP(list, 0);
+    SFilePage* pPage = getResBufPage(pMemBucket->pBuffer, pgInfo->pageId);
+    assert(pPage->num == 1);
+
+    double v = 0;
+    GET_TYPED_DATA(v, double, pMemBucket->type, pPage->data);
+    return v;
+  }
+
+  return 0;
+}
+
+int32_t tBucketIntHash(tMemBucket *pBucket, const void *value) {
+  int64_t v = 0;
+  GET_TYPED_DATA(v, int64_t, pBucket->type, value);
+
+  int32_t index = -1;
+
+  if (v > pBucket->range.i64MaxVal || v < pBucket->range.i64MinVal) {
+    return index;
+  }
+  
+  // divide the value range into 1024 buckets
+  uint64_t span = pBucket->range.i64MaxVal - pBucket->range.i64MinVal;
+  if (span < pBucket->numOfSlots) {
+    int64_t delta = v - pBucket->range.i64MinVal;
+    index = (delta % pBucket->numOfSlots);
+  } else {
+    double slotSpan = (double)span / pBucket->numOfSlots;
+    index = (int32_t)((v - pBucket->range.i64MinVal) / slotSpan);
+    if (v == pBucket->range.i64MaxVal) {
+      index -= 1;
+    }
+  }
+
+  assert(index >= 0 && index < pBucket->numOfSlots);
+  return index;
+}
+
+int32_t tBucketUintHash(tMemBucket *pBucket, const void *value) {
+  int64_t v = 0;
+  GET_TYPED_DATA(v, uint64_t, pBucket->type, value);
+
+  int32_t index = -1;
+
+  if (v > pBucket->range.u64MaxVal || v < pBucket->range.u64MinVal) {
+    return index;
+  }
+  
+  // divide the value range into 1024 buckets
+  uint64_t span = pBucket->range.u64MaxVal - pBucket->range.u64MinVal;
+  if (span < pBucket->numOfSlots) {
+    int64_t delta = v - pBucket->range.u64MinVal;
+    index = (int32_t) (delta % pBucket->numOfSlots);
+  } else {
+    double slotSpan = (double)span / pBucket->numOfSlots;
+    index = (int32_t)((v - pBucket->range.u64MinVal) / slotSpan);
+    if (v == pBucket->range.u64MaxVal) {
+      index -= 1;
+    }
+  }
+
+  assert(index >= 0 && index < pBucket->numOfSlots);
+  return index;
+}
+
+int32_t tBucketDoubleHash(tMemBucket *pBucket, const void *value) {
+  double v = 0;
+  if (pBucket->type == TSDB_DATA_TYPE_FLOAT) {
+    v = GET_FLOAT_VAL(value);
+  } else {
+    v = GET_DOUBLE_VAL(value);
+  }
+
+  int32_t index = -1;
+
+  if (v > pBucket->range.dMaxVal || v < pBucket->range.dMinVal) {
+    return index;
+  }
+
+  // divide a range of [dMinVal, dMaxVal] into 1024 buckets
+  double span = pBucket->range.dMaxVal - pBucket->range.dMinVal;
+  if (span < pBucket->numOfSlots) {
+    int32_t delta = (int32_t)(v - pBucket->range.dMinVal);
+    index = (delta % pBucket->numOfSlots);
+  } else {
+    double slotSpan = span / pBucket->numOfSlots;
+    index = (int32_t)((v - pBucket->range.dMinVal) / slotSpan);
+    if (v == pBucket->range.dMaxVal) {
+      index -= 1;
+    }
+  }
+
+  assert(index >= 0 && index < pBucket->numOfSlots);
+  return index;
+}
+
+static __perc_hash_func_t getHashFunc(int32_t type) {
+  if (IS_SIGNED_NUMERIC_TYPE(type)) {
+    return tBucketIntHash;
+  } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) {
+    return tBucketUintHash;
+  } else {
+    return tBucketDoubleHash;
+  }
+}
+
+static void resetSlotInfo(tMemBucket* pBucket) {
+  for (int32_t i = 0; i < pBucket->numOfSlots; ++i) {
+    tMemBucketSlot* pSlot = &pBucket->pSlots[i];
+
+    resetBoundingBox(&pSlot->range, pBucket->type);
+    resetPosInfo(&pSlot->info);
+  }
+}
+
+tMemBucket *tMemBucketCreate(int16_t nElemSize, int16_t dataType, double minval, double maxval) {
+  tMemBucket *pBucket = (tMemBucket *)calloc(1, sizeof(tMemBucket));
+  if (pBucket == NULL) {
+    return NULL;
+  }
+
+  pBucket->numOfSlots = DEFAULT_NUM_OF_SLOT;
+  pBucket->bufPageSize = DEFAULT_PAGE_SIZE * 4;   // 4k per page
+
+  pBucket->type  = dataType;
+  pBucket->bytes = nElemSize;
+  pBucket->total = 0;
+  pBucket->times = 1;
+
+  pBucket->maxCapacity = 200000;
+
+  if (setBoundingBox(&pBucket->range, pBucket->type, minval, maxval) != 0) {
+//    qError("MemBucket:%p, invalid value range: %f-%f", pBucket, minval, maxval);
+    free(pBucket);
+    return NULL;
+  }
+
+  pBucket->elemPerPage = (pBucket->bufPageSize - sizeof(SFilePage))/pBucket->bytes;
+  pBucket->comparFn = getKeyComparFunc(pBucket->type, TSDB_ORDER_ASC);
+
+  pBucket->hashFunc = getHashFunc(pBucket->type);
+  if (pBucket->hashFunc == NULL) {
+//    qError("MemBucket:%p, not support data type %d, failed", pBucket, pBucket->type);
+    free(pBucket);
+    return NULL;
+  }
+
+  pBucket->pSlots = (tMemBucketSlot *)calloc(pBucket->numOfSlots, sizeof(tMemBucketSlot));
+  if (pBucket->pSlots == NULL) {
+    free(pBucket);
+    return NULL;
+  }
+
+  resetSlotInfo(pBucket);
+
+  int32_t ret = createDiskbasedResultBuffer(&pBucket->pBuffer, pBucket->bufPageSize, pBucket->bufPageSize * 512, 1);
+  if (ret != 0) {
+    tMemBucketDestroy(pBucket);
+    return NULL;
+  }
+  
+//  qDebug("MemBucket:%p, elem size:%d", pBucket, pBucket->bytes);
+  return pBucket;
+}
+
+void tMemBucketDestroy(tMemBucket *pBucket) {
+  if (pBucket == NULL) {
+    return;
+  }
+
+  destroyResultBuf(pBucket->pBuffer);
+  tfree(pBucket->pSlots);
+  tfree(pBucket);
+}
+
+void tMemBucketUpdateBoundingBox(MinMaxEntry *r, const char *data, int32_t dataType) {
+  if (IS_SIGNED_NUMERIC_TYPE(dataType)) {
+    int64_t v = 0;
+    GET_TYPED_DATA(v, int64_t, dataType, data);
+
+    if (r->i64MinVal > v) {
+      r->i64MinVal = v;
+    }
+
+    if (r->i64MaxVal < v) {
+      r->i64MaxVal = v;
+    }
+  } else if (IS_UNSIGNED_NUMERIC_TYPE(dataType)) {
+    uint64_t v = 0;
+    GET_TYPED_DATA(v, uint64_t, dataType, data);
+
+    if (r->i64MinVal > v) {
+      r->i64MinVal = v;
+    }
+
+    if (r->i64MaxVal < v) {
+      r->i64MaxVal = v;
+    }
+  } else if (IS_FLOAT_TYPE(dataType)) {
+    double v = 0;
+    GET_TYPED_DATA(v, double, dataType, data);
+
+    if (r->dMinVal > v) {
+      r->dMinVal = v;
+    }
+
+    if (r->dMaxVal < v) {
+      r->dMaxVal = v;
+    }
+  } else {
+    assert(0);
+  }
+}
+
+/*
+ * in memory bucket, we only accept data array list
+ */
+int32_t tMemBucketPut(tMemBucket *pBucket, const void *data, size_t size) {
+  assert(pBucket != NULL && data != NULL && size > 0);
+
+  int32_t count = 0;
+  int32_t bytes = pBucket->bytes;
+  for (int32_t i = 0; i < size; ++i) {
+    char *d = (char *) data + i * bytes;
+
+    int32_t index = (pBucket->hashFunc)(pBucket, d);
+    if (index < 0) {
+      continue;
+    }
+
+    count += 1;
+
+    tMemBucketSlot *pSlot = &pBucket->pSlots[index];
+    tMemBucketUpdateBoundingBox(&pSlot->range, d, pBucket->type);
+
+    // ensure available memory pages to allocate
+    int32_t groupId = getGroupId(pBucket->numOfSlots, index, pBucket->times);
+    int32_t pageId = -1;
+
+    if (pSlot->info.data == NULL || pSlot->info.data->num >= pBucket->elemPerPage) {
+      if (pSlot->info.data != NULL) {
+        assert(pSlot->info.data->num >= pBucket->elemPerPage && pSlot->info.size > 0);
+
+        // keep the pointer in memory
+        releaseResBufPage(pBucket->pBuffer, pSlot->info.data);
+        pSlot->info.data = NULL;
+      }
+
+      pSlot->info.data = getNewDataBuf(pBucket->pBuffer, groupId, &pageId);
+      pSlot->info.pageId = pageId;
+    }
+
+    memcpy(pSlot->info.data->data + pSlot->info.data->num * pBucket->bytes, d, pBucket->bytes);
+
+    pSlot->info.data->num += 1;
+    pSlot->info.size += 1;
+  }
+
+  pBucket->total += count;
+  return 0;
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////
+/*
+ *
+ * now, we need to find the minimum value of the next slot for
+ * interpolating the percentile value
+ * j is the last slot of current segment, we need to get the first
+ * slot of the next segment.
+ */
+static MinMaxEntry getMinMaxEntryOfNextSlotWithData(tMemBucket *pMemBucket, int32_t slotIdx) {
+    int32_t j = slotIdx + 1;
+    while (j < pMemBucket->numOfSlots && (pMemBucket->pSlots[j].info.size == 0)) {
+      ++j;
+    }
+
+    assert(j < pMemBucket->numOfSlots);
+    return pMemBucket->pSlots[j].range;
+}
+
+static bool isIdenticalData(tMemBucket *pMemBucket, int32_t index);
+
+static double getIdenticalDataVal(tMemBucket* pMemBucket, int32_t slotIndex) {
+  assert(isIdenticalData(pMemBucket, slotIndex));
+
+  tMemBucketSlot *pSlot = &pMemBucket->pSlots[slotIndex];
+
+  double finalResult = 0.0;
+  if (IS_SIGNED_NUMERIC_TYPE(pMemBucket->type)) {
+    finalResult = (double) pSlot->range.i64MinVal;
+  } else if (IS_UNSIGNED_NUMERIC_TYPE(pMemBucket->type)) {
+    finalResult = (double) pSlot->range.u64MinVal;
+  } else {
+    finalResult = (double) pSlot->range.dMinVal;
+  }
+
+  return finalResult;
+}
+
+double getPercentileImpl(tMemBucket *pMemBucket, int32_t count, double fraction) {
+  int32_t num = 0;
+
+  for (int32_t i = 0; i < pMemBucket->numOfSlots; ++i) {
+    tMemBucketSlot *pSlot = &pMemBucket->pSlots[i];
+    if (pSlot->info.size == 0) {
+      continue;
+    }
+
+    // required value in current slot
+    if (num < (count + 1) && num + pSlot->info.size >= (count + 1)) {
+      if (pSlot->info.size + num == (count + 1)) {
+        /*
+         * now, we need to find the minimum value of the next slot for interpolating the percentile value
+         * j is the last slot of current segment, we need to get the first slot of the next segment.
+         */
+        MinMaxEntry next = getMinMaxEntryOfNextSlotWithData(pMemBucket, i);
+
+        double maxOfThisSlot = 0;
+        double minOfNextSlot = 0;
+        if (IS_SIGNED_NUMERIC_TYPE(pMemBucket->type)) {
+          maxOfThisSlot = (double) pSlot->range.i64MaxVal;
+          minOfNextSlot = (double) next.i64MinVal;
+        } else if (IS_UNSIGNED_NUMERIC_TYPE(pMemBucket->type)) {
+          maxOfThisSlot = (double) pSlot->range.u64MaxVal;
+          minOfNextSlot = (double) next.u64MinVal;
+        } else {
+          maxOfThisSlot = (double) pSlot->range.dMaxVal;
+          minOfNextSlot = (double) next.dMinVal;
+        }
+
+        assert(minOfNextSlot > maxOfThisSlot);
+
+        double val = (1 - fraction) * maxOfThisSlot + fraction * minOfNextSlot;
+        return val;
+      }
+
+      if (pSlot->info.size <= pMemBucket->maxCapacity) {
+        // data in buffer and file are merged together to be processed.
+        SFilePage *buffer = loadDataFromFilePage(pMemBucket, i);
+        int32_t    currentIdx = count - num;
+
+        char *thisVal = buffer->data + pMemBucket->bytes * currentIdx;
+        char *nextVal = thisVal + pMemBucket->bytes;
+
+        double td = 1.0, nd = 1.0;
+        GET_TYPED_DATA(td, double, pMemBucket->type, thisVal);
+        GET_TYPED_DATA(nd, double, pMemBucket->type, nextVal);
+
+        double val = (1 - fraction) * td + fraction * nd;
+        tfree(buffer);
+
+        return val;
+      } else {  // incur a second round bucket split
+       if (isIdenticalData(pMemBucket, i)) {
+         return getIdenticalDataVal(pMemBucket, i);
+       }
+
+       // try next round
+       pMemBucket->times += 1;
+//       qDebug("MemBucket:%p, start next round data bucketing, time:%d", pMemBucket, pMemBucket->times);
+
+       pMemBucket->range = pSlot->range;
+       pMemBucket->total = 0;
+
+       resetSlotInfo(pMemBucket);
+
+       int32_t groupId = getGroupId(pMemBucket->numOfSlots, i, pMemBucket->times - 1);
+       SIDList list = getDataBufPagesIdList(pMemBucket->pBuffer, groupId);
+       assert(list->size > 0);
+
+       for (int32_t f = 0; f < list->size; ++f) {
+         SPageInfo *pgInfo = *(SPageInfo **)taosArrayGet(list, f);
+         SFilePage *pg = getResBufPage(pMemBucket->pBuffer, pgInfo->pageId);
+
+         tMemBucketPut(pMemBucket, pg->data, (int32_t)pg->num);
+         releaseResBufPageInfo(pMemBucket->pBuffer, pgInfo);
+       }
+
+       return getPercentileImpl(pMemBucket, count - num, fraction);
+      }
+    } else {
+      num += pSlot->info.size;
+    }
+  }
+
+  return 0;
+}
+
+double getPercentile(tMemBucket *pMemBucket, double percent) {
+  if (pMemBucket->total == 0) {
+    return 0.0;
+  }
+
+  // if only one elements exists, return it
+  if (pMemBucket->total == 1) {
+    return findOnlyResult(pMemBucket);
+  }
+
+  percent = fabs(percent);
+
+  // find the min/max value, no need to scan all data in bucket
+  if (fabs(percent - 100.0) < DBL_EPSILON || (percent < DBL_EPSILON)) {
+    MinMaxEntry* pRange = &pMemBucket->range;
+
+    if (IS_SIGNED_NUMERIC_TYPE(pMemBucket->type)) {
+      double v = (double)(fabs(percent - 100) < DBL_EPSILON ? pRange->i64MaxVal : pRange->i64MinVal);
+      return v;
+    } else if (IS_UNSIGNED_NUMERIC_TYPE(pMemBucket->type)) {
+      double v = (double)(fabs(percent - 100) < DBL_EPSILON ? pRange->u64MaxVal : pRange->u64MinVal);
+      return v;
+    } else {
+      return fabs(percent - 100) < DBL_EPSILON? pRange->dMaxVal:pRange->dMinVal;
+    }
+  }
+
+  double  percentVal = (percent * (pMemBucket->total - 1)) / ((double)100.0);
+
+  // do put data by using buckets
+  int32_t orderIdx = (int32_t)percentVal;
+  return getPercentileImpl(pMemBucket, orderIdx, percentVal - orderIdx);
+}
+
+/*
+ * check if data in one slot are all identical only need to compare with the bounding box
+ */
+bool isIdenticalData(tMemBucket *pMemBucket, int32_t index) {
+  tMemBucketSlot *pSeg = &pMemBucket->pSlots[index];
+
+  if (IS_FLOAT_TYPE(pMemBucket->type)) {
+    return fabs(pSeg->range.dMaxVal - pSeg->range.dMinVal) < DBL_EPSILON;
+  } else {
+    return pSeg->range.i64MinVal == pSeg->range.i64MaxVal;
+  }
+}

source/libs/function/src/tscalarfunction.c

+#include "tscalarfunction.h"
+
+SScalarFunctionInfo scalarFunc[1] = {
+    {
+
+    },
+
+};
+
+

source/libs/function/src/ttszip.c

+#include "ttszip.h"
+#include <tglobal.h>
+#include "taoserror.h"
+#include "tcompression.h"
+#include "tutil.h"
+
+static int32_t getDataStartOffset();
+static void TSBufUpdateGroupInfo(STSBuf* pTSBuf, int32_t index, STSGroupBlockInfo* pBlockInfo);
+static STSBuf* allocResForTSBuf(STSBuf* pTSBuf);
+static int32_t STSBufUpdateHeader(STSBuf* pTSBuf, STSBufFileHeader* pHeader);
+
+/**
+ * todo error handling
+ * support auto closeable tmp file
+ * @param path
+ * @return
+ */
+STSBuf* tsBufCreate(bool autoDelete, int32_t order) {
+  STSBuf* pTSBuf = calloc(1, sizeof(STSBuf));
+  if (pTSBuf == NULL) {
+    return NULL;
+  }
+
+  pTSBuf->autoDelete = autoDelete;
+  
+  taosGetTmpfilePath(tsTempDir, "join", pTSBuf->path);
+  pTSBuf->f = fopen(pTSBuf->path, "wb+");
+  if (pTSBuf->f == NULL) {
+    free(pTSBuf);
+    return NULL;
+  }
+
+  if (!autoDelete) {
+    remove(pTSBuf->path);
+  }
+  
+  if (NULL == allocResForTSBuf(pTSBuf)) {
+    return NULL;
+  }
+  
+  // update the header info
+  STSBufFileHeader header = {.magic = TS_COMP_FILE_MAGIC, .numOfGroup = pTSBuf->numOfGroups, .tsOrder = TSDB_ORDER_ASC};
+  STSBufUpdateHeader(pTSBuf, &header);
+  
+  tsBufResetPos(pTSBuf);
+  pTSBuf->cur.order = TSDB_ORDER_ASC;
+
+  pTSBuf->tsOrder = order;
+  
+  return pTSBuf;
+}
+
+STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) {
+  STSBuf* pTSBuf = calloc(1, sizeof(STSBuf));
+  if (pTSBuf == NULL) {
+    return NULL;
+  }
+
+  pTSBuf->autoDelete = autoDelete;
+  
+  tstrncpy(pTSBuf->path, path, sizeof(pTSBuf->path));
+  
+  pTSBuf->f = fopen(pTSBuf->path, "rb+");
+  if (pTSBuf->f == NULL) {
+    free(pTSBuf);
+    return NULL;
+  }
+  
+  if (allocResForTSBuf(pTSBuf) == NULL) {
+    return NULL;
+  }
+  
+  // validate the file magic number
+  STSBufFileHeader header = {0};
+  int32_t ret = fseek(pTSBuf->f, 0, SEEK_SET);
+  UNUSED(ret);
+  size_t sz = fread(&header, 1, sizeof(STSBufFileHeader), pTSBuf->f);
+  UNUSED(sz);
+
+  // invalid file
+  if (header.magic != TS_COMP_FILE_MAGIC) {
+    tsBufDestroy(pTSBuf);
+    return NULL;
+  }
+  
+  if (header.numOfGroup > pTSBuf->numOfAlloc) {
+    pTSBuf->numOfAlloc = header.numOfGroup;
+    STSGroupBlockInfoEx* tmp = realloc(pTSBuf->pData, sizeof(STSGroupBlockInfoEx) * pTSBuf->numOfAlloc);
+    if (tmp == NULL) {
+      tsBufDestroy(pTSBuf);
+      return NULL;
+    }
+    
+    pTSBuf->pData = tmp;
+  }
+  
+  pTSBuf->numOfGroups = header.numOfGroup;
+  
+  // check the ts order
+  pTSBuf->tsOrder = header.tsOrder;
+  if (pTSBuf->tsOrder != TSDB_ORDER_ASC && pTSBuf->tsOrder != TSDB_ORDER_DESC) {
+//    tscError("invalid order info in buf:%d", pTSBuf->tsOrder);
+    tsBufDestroy(pTSBuf);
+    return NULL;
+  }
+  
+  size_t infoSize = sizeof(STSGroupBlockInfo) * pTSBuf->numOfGroups;
+  
+  STSGroupBlockInfo* buf = (STSGroupBlockInfo*)calloc(1, infoSize);
+  if (buf == NULL) {
+    tsBufDestroy(pTSBuf);
+    return NULL; 
+  } 
+  
+  //int64_t pos = ftell(pTSBuf->f); //pos not used
+  sz = fread(buf, infoSize, 1, pTSBuf->f);
+  UNUSED(sz);
+  
+  // the length value for each vnode is not kept in file, so does not set the length value
+  for (int32_t i = 0; i < pTSBuf->numOfGroups; ++i) {
+    STSGroupBlockInfoEx* pBlockList = &pTSBuf->pData[i];
+    memcpy(&pBlockList->info, &buf[i], sizeof(STSGroupBlockInfo));
+  }
+  free(buf);
+  
+  ret = fseek(pTSBuf->f, 0, SEEK_END);
+  UNUSED(ret);
+  
+  struct stat fileStat;
+  if (fstat(fileno(pTSBuf->f), &fileStat) != 0) {
+    tsBufDestroy(pTSBuf);
+    return NULL;
+  }
+  
+  pTSBuf->fileSize = (uint32_t)fileStat.st_size;
+  tsBufResetPos(pTSBuf);
+  
+  // ascending by default
+  pTSBuf->cur.order = TSDB_ORDER_ASC;
+  
+//  tscDebug("create tsBuf from file:%s, fd:%d, size:%d, numOfGroups:%d, autoDelete:%d", pTSBuf->path, fileno(pTSBuf->f),
+//           pTSBuf->fileSize, pTSBuf->numOfGroups, pTSBuf->autoDelete);
+  
+  return pTSBuf;
+}
+
+void* tsBufDestroy(STSBuf* pTSBuf) {
+  if (pTSBuf == NULL) {
+    return NULL;
+  }
+  
+  tfree(pTSBuf->assistBuf);
+  tfree(pTSBuf->tsData.rawBuf);
+  
+  tfree(pTSBuf->pData);
+  tfree(pTSBuf->block.payload);
+
+  if (!pTSBuf->remainOpen) {
+    fclose(pTSBuf->f);
+  }
+  
+  if (pTSBuf->autoDelete) {
+//    ("tsBuf %p destroyed, delete tmp file:%s", pTSBuf, pTSBuf->path);
+    remove(pTSBuf->path);
+  } else {
+//    tscDebug("tsBuf %p destroyed, tmp file:%s, remains", pTSBuf, pTSBuf->path);
+  }
+
+  taosVariantDestroy(&pTSBuf->block.tag);
+  free(pTSBuf);
+  return NULL;
+}
+
+static STSGroupBlockInfoEx* tsBufGetLastGroupInfo(STSBuf* pTSBuf) {
+  int32_t last = pTSBuf->numOfGroups - 1;
+  
+  assert(last >= 0);
+  return &pTSBuf->pData[last];
+}
+
+static STSGroupBlockInfoEx* addOneGroupInfo(STSBuf* pTSBuf, int32_t id) {
+  if (pTSBuf->numOfAlloc <= pTSBuf->numOfGroups) {
+    uint32_t newSize = (uint32_t)(pTSBuf->numOfAlloc * 1.5);
+    assert((int32_t)newSize > pTSBuf->numOfAlloc);
+    
+    STSGroupBlockInfoEx* tmp = (STSGroupBlockInfoEx*)realloc(pTSBuf->pData, sizeof(STSGroupBlockInfoEx) * newSize);
+    if (tmp == NULL) {
+      return NULL;
+    }
+    
+    pTSBuf->pData = tmp;
+    pTSBuf->numOfAlloc = newSize;
+    memset(&pTSBuf->pData[pTSBuf->numOfGroups], 0, sizeof(STSGroupBlockInfoEx) * (newSize - pTSBuf->numOfGroups));
+  }
+  
+  if (pTSBuf->numOfGroups > 0) {
+    STSGroupBlockInfoEx* pPrevBlockInfoEx = tsBufGetLastGroupInfo(pTSBuf);
+    
+    // update prev vnode length info in file
+    TSBufUpdateGroupInfo(pTSBuf, pTSBuf->numOfGroups - 1, &pPrevBlockInfoEx->info);
+  }
+  
+  // set initial value for vnode block
+  STSGroupBlockInfo* pBlockInfo = &pTSBuf->pData[pTSBuf->numOfGroups].info;
+  pBlockInfo->id = id;
+  pBlockInfo->offset = pTSBuf->fileSize;
+  assert(pBlockInfo->offset >= getDataStartOffset());
+  
+  // update vnode info in file
+  TSBufUpdateGroupInfo(pTSBuf, pTSBuf->numOfGroups, pBlockInfo);
+  
+  // add one vnode info
+  pTSBuf->numOfGroups += 1;
+  
+  // update the header info
+  STSBufFileHeader header = {
+      .magic = TS_COMP_FILE_MAGIC, .numOfGroup = pTSBuf->numOfGroups, .tsOrder = pTSBuf->tsOrder};
+  
+  STSBufUpdateHeader(pTSBuf, &header);
+  return tsBufGetLastGroupInfo(pTSBuf);
+}
+
+static void shrinkBuffer(STSList* ptsData) {
+  // shrink tmp buffer size if it consumes too many memory compared to the pre-defined size
+  if (ptsData->allocSize >= ptsData->threshold * 2) {
+    char* rawBuf = realloc(ptsData->rawBuf, MEM_BUF_SIZE);
+    if(rawBuf) {
+      ptsData->rawBuf = rawBuf;
+      ptsData->allocSize = MEM_BUF_SIZE;
+    }
+  }
+}
+
+static int32_t getTagAreaLength(SVariant* pa) {
+  int32_t t = sizeof(pa->nLen) * 2 + sizeof(pa->nType);
+  if (pa->nType != TSDB_DATA_TYPE_NULL) {
+    t += pa->nLen;
+  }
+
+  return t;
+}
+
+static void writeDataToDisk(STSBuf* pTSBuf) {
+  if (pTSBuf->tsData.len == 0) {
+    return;
+  }
+  
+  STSBlock* pBlock = &pTSBuf->block;
+  STSList*  pTsData = &pTSBuf->tsData;
+
+  pBlock->numOfElem = pTsData->len / TSDB_KEYSIZE;
+  pBlock->compLen =
+      tsCompressTimestamp(pTsData->rawBuf, pTsData->len, pTsData->len/TSDB_KEYSIZE, pBlock->payload, pTsData->allocSize,
+          TWO_STAGE_COMP, pTSBuf->assistBuf, pTSBuf->bufSize);
+  
+  int64_t r = fseek(pTSBuf->f, pTSBuf->fileSize, SEEK_SET);
+  assert(r == 0);
+  
+  /*
+   * format for output data:
+   * 1. tags, number of ts, size after compressed, payload, size after compressed
+   * 2. tags, number of ts, size after compressed, payload, size after compressed
+   *
+   * both side has the compressed length is used to support load data forwards/backwords.
+   */
+  int32_t metaLen = 0;
+  metaLen += (int32_t)fwrite(&pBlock->tag.nType, 1, sizeof(pBlock->tag.nType), pTSBuf->f);
+
+  int32_t trueLen = pBlock->tag.nLen;
+  if (pBlock->tag.nType == TSDB_DATA_TYPE_BINARY || pBlock->tag.nType == TSDB_DATA_TYPE_NCHAR) {
+    metaLen += (int32_t)fwrite(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f);
+    metaLen += (int32_t)fwrite(pBlock->tag.pz, 1, (size_t)pBlock->tag.nLen, pTSBuf->f);
+  } else if (pBlock->tag.nType == TSDB_DATA_TYPE_FLOAT) {
+    metaLen += (int32_t)fwrite(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f);
+    float tfloat = (float)pBlock->tag.d;
+    metaLen += (int32_t)fwrite(&tfloat, 1, (size_t) pBlock->tag.nLen, pTSBuf->f);  
+  } else if (pBlock->tag.nType != TSDB_DATA_TYPE_NULL) {
+    metaLen += (int32_t)fwrite(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f);
+    metaLen += (int32_t)fwrite(&pBlock->tag.i64, 1, (size_t) pBlock->tag.nLen, pTSBuf->f);
+  } else {
+    trueLen = 0;
+    metaLen += (int32_t)fwrite(&trueLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f);
+  }
+
+  fwrite(&pBlock->numOfElem, sizeof(pBlock->numOfElem), 1, pTSBuf->f);
+  fwrite(&pBlock->compLen, sizeof(pBlock->compLen), 1, pTSBuf->f);
+  fwrite(pBlock->payload, (size_t)pBlock->compLen, 1, pTSBuf->f);
+  fwrite(&pBlock->compLen, sizeof(pBlock->compLen), 1, pTSBuf->f);
+
+  metaLen += (int32_t) fwrite(&trueLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f);
+  assert(metaLen == getTagAreaLength(&pBlock->tag));
+
+  int32_t blockSize = metaLen + sizeof(pBlock->numOfElem) + sizeof(pBlock->compLen) * 2 + pBlock->compLen;
+  pTSBuf->fileSize += blockSize;
+  
+  pTSBuf->tsData.len = 0;
+  
+  STSGroupBlockInfoEx* pGroupBlockInfoEx = tsBufGetLastGroupInfo(pTSBuf);
+  
+  pGroupBlockInfoEx->info.compLen += blockSize;
+  pGroupBlockInfoEx->info.numOfBlocks += 1;
+  
+  shrinkBuffer(&pTSBuf->tsData);
+}
+
+static void expandBuffer(STSList* ptsData, int32_t inputSize) {
+  if (ptsData->allocSize - ptsData->len < inputSize) {
+    int32_t newSize = inputSize + ptsData->len;
+    char*   tmp = realloc(ptsData->rawBuf, (size_t)newSize);
+    if (tmp == NULL) {
+      // todo
+    }
+    
+    ptsData->rawBuf = tmp;
+    ptsData->allocSize = newSize;
+  }
+}
+
+STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) {
+  STSBlock* pBlock = &pTSBuf->block;
+
+  // clear the memory buffer
+  pBlock->compLen   = 0;
+  pBlock->padding   = 0;
+  pBlock->numOfElem = 0;
+
+  int32_t offset = -1;
+
+  if (order == TSDB_ORDER_DESC) {
+    /*
+     * set the right position for the reversed traverse, the reversed traverse is started from
+     * the end of each comp data block
+     */
+    int32_t prev = -(int32_t) (sizeof(pBlock->padding) + sizeof(pBlock->tag.nLen));
+    int32_t ret = fseek(pTSBuf->f, prev, SEEK_CUR);
+    size_t sz = fread(&pBlock->padding, 1, sizeof(pBlock->padding), pTSBuf->f);
+    sz = fread(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f);
+    UNUSED(sz); 
+
+    pBlock->compLen = pBlock->padding;
+
+    offset = pBlock->compLen + sizeof(pBlock->compLen) * 2 + sizeof(pBlock->numOfElem) + getTagAreaLength(&pBlock->tag);
+    ret = fseek(pTSBuf->f, -offset, SEEK_CUR);
+    UNUSED(ret);
+  }
+
+  fread(&pBlock->tag.nType, sizeof(pBlock->tag.nType), 1, pTSBuf->f);
+  fread(&pBlock->tag.nLen, sizeof(pBlock->tag.nLen), 1, pTSBuf->f);
+
+  // NOTE: mix types tags are not supported
+  size_t sz = 0;
+  if (pBlock->tag.nType == TSDB_DATA_TYPE_BINARY || pBlock->tag.nType == TSDB_DATA_TYPE_NCHAR) {
+    char* tp = realloc(pBlock->tag.pz, pBlock->tag.nLen + 1);
+    assert(tp != NULL);
+
+    memset(tp, 0, pBlock->tag.nLen + 1);
+    pBlock->tag.pz = tp;
+
+    sz = fread(pBlock->tag.pz, (size_t)pBlock->tag.nLen, 1, pTSBuf->f);
+    UNUSED(sz);
+  } else if (pBlock->tag.nType == TSDB_DATA_TYPE_FLOAT) {
+    float tfloat = 0;
+    sz = fread(&tfloat, (size_t) pBlock->tag.nLen, 1, pTSBuf->f);
+    pBlock->tag.d = (double)tfloat;
+    UNUSED(sz);
+  } else if (pBlock->tag.nType != TSDB_DATA_TYPE_NULL) { //TODO check the return value
+    sz = fread(&pBlock->tag.i64, (size_t) pBlock->tag.nLen, 1, pTSBuf->f);
+    UNUSED(sz);
+  }
+
+  sz = fread(&pBlock->numOfElem, sizeof(pBlock->numOfElem), 1, pTSBuf->f);
+  UNUSED(sz);
+  sz = fread(&pBlock->compLen, sizeof(pBlock->compLen), 1, pTSBuf->f);
+  UNUSED(sz);
+  sz = fread(pBlock->payload, (size_t)pBlock->compLen, 1, pTSBuf->f);
+
+  if (decomp) {
+    pTSBuf->tsData.len =
+        tsDecompressTimestamp(pBlock->payload, pBlock->compLen, pBlock->numOfElem, pTSBuf->tsData.rawBuf,
+                              pTSBuf->tsData.allocSize, TWO_STAGE_COMP, pTSBuf->assistBuf, pTSBuf->bufSize);
+  }
+  
+  // read the comp length at the length of comp block
+  sz = fread(&pBlock->padding, sizeof(pBlock->padding), 1, pTSBuf->f);
+  assert(pBlock->padding == pBlock->compLen);
+
+  int32_t n = 0;
+  sz = fread(&n, sizeof(pBlock->tag.nLen), 1, pTSBuf->f);
+  if (pBlock->tag.nType == TSDB_DATA_TYPE_NULL) {
+    assert(n == 0);
+  } else {
+    assert(n == pBlock->tag.nLen);
+  }
+
+  UNUSED(sz);
+  
+  // for backwards traverse, set the start position at the end of previous block
+  if (order == TSDB_ORDER_DESC) {
+    int32_t r = fseek(pTSBuf->f, -offset, SEEK_CUR);
+    UNUSED(r);
+  }
+  
+  return pBlock;
+}
+
+// set the order of ts buffer if the ts order has not been set yet
+static int32_t setCheckTSOrder(STSBuf* pTSBuf, const char* pData, int32_t len) {
+  STSList* ptsData = &pTSBuf->tsData;
+  
+  if (pTSBuf->tsOrder == -1) {
+    if (ptsData->len > 0) {
+      TSKEY lastKey = *(TSKEY*)(ptsData->rawBuf + ptsData->len - TSDB_KEYSIZE);
+      
+      if (lastKey > *(TSKEY*)pData) {
+        pTSBuf->tsOrder = TSDB_ORDER_DESC;
+      } else {
+        pTSBuf->tsOrder = TSDB_ORDER_ASC;
+      }
+    } else if (len > TSDB_KEYSIZE) {
+      // no data in current vnode, more than one ts is added, check the orders
+      TSKEY k1 = *(TSKEY*)(pData);
+      TSKEY k2 = *(TSKEY*)(pData + TSDB_KEYSIZE);
+      
+      if (k1 < k2) {
+        pTSBuf->tsOrder = TSDB_ORDER_ASC;
+      } else if (k1 > k2) {
+        pTSBuf->tsOrder = TSDB_ORDER_DESC;
+      } else {
+        // todo handle error
+      }
+    }
+  } else {
+    // todo the timestamp order is set, check the asc/desc order of appended data
+  }
+  
+  return TSDB_CODE_SUCCESS;
+}
+
+void tsBufAppend(STSBuf* pTSBuf, int32_t id, SVariant* tag, const char* pData, int32_t len) {
+  STSGroupBlockInfoEx* pBlockInfo = NULL;
+  STSList*             ptsData = &pTSBuf->tsData;
+  
+  if (pTSBuf->numOfGroups == 0 || tsBufGetLastGroupInfo(pTSBuf)->info.id != id) {
+    writeDataToDisk(pTSBuf);
+    shrinkBuffer(ptsData);
+    
+    pBlockInfo = addOneGroupInfo(pTSBuf, id);
+  } else {
+    pBlockInfo = tsBufGetLastGroupInfo(pTSBuf);
+  }
+  
+  assert(pBlockInfo->info.id == id);
+
+  if ((taosVariantCompare(&pTSBuf->block.tag, tag) != 0) && ptsData->len > 0) {
+    // new arrived data with different tags value, save current value into disk first
+    writeDataToDisk(pTSBuf);
+  } else {
+    expandBuffer(ptsData, len);
+  }
+
+  taosVariantAssign(&pTSBuf->block.tag, tag);
+  memcpy(ptsData->rawBuf + ptsData->len, pData, (size_t)len);
+  
+  // todo check return value
+  setCheckTSOrder(pTSBuf, pData, len);
+  
+  ptsData->len += len;
+  pBlockInfo->len += len;
+  
+  pTSBuf->numOfTotal += len / TSDB_KEYSIZE;
+  
+  // the size of raw data exceeds the size of the default prepared buffer, so
+  // during getBufBlock, the output buffer needs to be large enough.
+  if (ptsData->len >= ptsData->threshold) {
+    writeDataToDisk(pTSBuf);
+    shrinkBuffer(ptsData);
+  }
+  
+  tsBufResetPos(pTSBuf);
+}
+
+void tsBufFlush(STSBuf* pTSBuf) {
+  if (pTSBuf->tsData.len <= 0) {
+    return;
+  }
+  
+  writeDataToDisk(pTSBuf);
+  shrinkBuffer(&pTSBuf->tsData);
+  
+  STSGroupBlockInfoEx* pBlockInfoEx = tsBufGetLastGroupInfo(pTSBuf);
+  
+  // update prev vnode length info in file
+  TSBufUpdateGroupInfo(pTSBuf, pTSBuf->numOfGroups - 1, &pBlockInfoEx->info);
+  
+  // save the ts order into header
+  STSBufFileHeader header = {
+      .magic = TS_COMP_FILE_MAGIC, .numOfGroup = pTSBuf->numOfGroups, .tsOrder = pTSBuf->tsOrder};
+  STSBufUpdateHeader(pTSBuf, &header);
+}
+
+static int32_t tsBufFindGroupById(STSGroupBlockInfoEx* pGroupInfoEx, int32_t numOfGroups, int32_t id) {
+  int32_t j = -1;
+  for (int32_t i = 0; i < numOfGroups; ++i) {
+    if (pGroupInfoEx[i].info.id == id) {
+      j = i;
+      break;
+    }
+  }
+  
+  return j;
+}
+
+// todo opt performance by cache blocks info
+static int32_t tsBufFindBlock(STSBuf* pTSBuf, STSGroupBlockInfo* pBlockInfo, int32_t blockIndex) {
+  if (fseek(pTSBuf->f, pBlockInfo->offset, SEEK_SET) != 0) {
+    return -1;
+  }
+  
+  // sequentially read the compressed data blocks, start from the beginning of the comp data block of this vnode
+  int32_t i = 0;
+  bool    decomp = false;
+  
+  while ((i++) <= blockIndex) {
+    if (readDataFromDisk(pTSBuf, TSDB_ORDER_ASC, decomp) == NULL) {
+      return -1;
+    }
+  }
+  
+  // set the file position to be the end of previous comp block
+  if (pTSBuf->cur.order == TSDB_ORDER_DESC) {
+    STSBlock* pBlock = &pTSBuf->block;
+    int32_t   compBlockSize =
+        pBlock->compLen + sizeof(pBlock->compLen) * 2 + sizeof(pBlock->numOfElem) + getTagAreaLength(&pBlock->tag);
+    int32_t ret = fseek(pTSBuf->f, -compBlockSize, SEEK_CUR);
+    UNUSED(ret);
+  }
+  
+  return 0;
+}
+
+static int32_t tsBufFindBlockByTag(STSBuf* pTSBuf, STSGroupBlockInfo* pBlockInfo, SVariant* tag) {
+  bool decomp = false;
+  
+  int64_t offset = 0;
+  if (pTSBuf->cur.order == TSDB_ORDER_ASC) {
+    offset = pBlockInfo->offset;
+  } else {  // reversed traverse starts from the end of block
+    offset = pBlockInfo->offset + pBlockInfo->compLen;
+  }
+  
+  if (fseek(pTSBuf->f, (int32_t)offset, SEEK_SET) != 0) {
+    return -1;
+  }
+  
+  for (int32_t i = 0; i < pBlockInfo->numOfBlocks; ++i) {
+    if (readDataFromDisk(pTSBuf, pTSBuf->cur.order, decomp) == NULL) {
+      return -1;
+    }
+    
+    if (taosVariantCompare(&pTSBuf->block.tag, tag) == 0) {
+      return (pTSBuf->cur.order == TSDB_ORDER_ASC)? i: (pBlockInfo->numOfBlocks - (i + 1));
+    }
+  }
+  
+  return -1;
+}
+
+static void tsBufGetBlock(STSBuf* pTSBuf, int32_t groupIndex, int32_t blockIndex) {
+  STSGroupBlockInfo* pBlockInfo = &pTSBuf->pData[groupIndex].info;
+  if (pBlockInfo->numOfBlocks <= blockIndex) {
+    assert(false);
+  }
+  
+  STSCursor* pCur = &pTSBuf->cur;
+  if (pCur->vgroupIndex == groupIndex && ((pCur->blockIndex <= blockIndex && pCur->order == TSDB_ORDER_ASC) ||
+      (pCur->blockIndex >= blockIndex && pCur->order == TSDB_ORDER_DESC))) {
+    int32_t i = 0;
+    bool    decomp = false;
+    int32_t step = abs(blockIndex - pCur->blockIndex);
+    
+    while ((++i) <= step) {
+      if (readDataFromDisk(pTSBuf, pCur->order, decomp) == NULL) {
+        return;
+      }
+    }
+  } else {
+    if (tsBufFindBlock(pTSBuf, pBlockInfo, blockIndex) == -1) {
+      assert(false);
+    }
+  }
+  
+  STSBlock* pBlock = &pTSBuf->block;
+  
+  size_t s = pBlock->numOfElem * TSDB_KEYSIZE;
+  
+  /*
+   * In order to accommodate all the qualified data, the actual buffer size for one block with identical tags value
+   * may exceed the maximum allowed size during *tsBufAppend* function by invoking expandBuffer function
+   */
+  if (s > pTSBuf->tsData.allocSize) {
+    expandBuffer(&pTSBuf->tsData, (int32_t)s);
+  }
+  
+  pTSBuf->tsData.len =
+      tsDecompressTimestamp(pBlock->payload, pBlock->compLen, pBlock->numOfElem, pTSBuf->tsData.rawBuf,
+                            pTSBuf->tsData.allocSize, TWO_STAGE_COMP, pTSBuf->assistBuf, pTSBuf->bufSize);
+  
+  assert((pTSBuf->tsData.len / TSDB_KEYSIZE == pBlock->numOfElem) && (pTSBuf->tsData.allocSize >= pTSBuf->tsData.len));
+  
+  pCur->vgroupIndex = groupIndex;
+  pCur->blockIndex = blockIndex;
+  
+  pCur->tsIndex = (pCur->order == TSDB_ORDER_ASC) ? 0 : pBlock->numOfElem - 1;
+}
+
+static int32_t doUpdateGroupInfo(STSBuf* pTSBuf, int64_t offset, STSGroupBlockInfo* pVInfo) {
+  if (offset < 0 || offset >= getDataStartOffset()) {
+    return -1;
+  }
+
+  if (fseek(pTSBuf->f, (int32_t)offset, SEEK_SET) != 0) {
+    return -1;
+  }
+
+  fwrite(pVInfo, sizeof(STSGroupBlockInfo), 1, pTSBuf->f);
+  return 0;
+}
+
+STSGroupBlockInfo* tsBufGetGroupBlockInfo(STSBuf* pTSBuf, int32_t id) {
+  int32_t j = tsBufFindGroupById(pTSBuf->pData, pTSBuf->numOfGroups, id);
+  if (j == -1) {
+    return NULL;
+  }
+  
+  return &pTSBuf->pData[j].info;
+}
+
+int32_t STSBufUpdateHeader(STSBuf* pTSBuf, STSBufFileHeader* pHeader) {
+  if ((pTSBuf->f == NULL) || pHeader == NULL || pHeader->numOfGroup == 0 || pHeader->magic != TS_COMP_FILE_MAGIC) {
+    return -1;
+  }
+
+  assert(pHeader->tsOrder == TSDB_ORDER_ASC || pHeader->tsOrder == TSDB_ORDER_DESC);
+
+  int32_t r = fseek(pTSBuf->f, 0, SEEK_SET);
+  if (r != 0) {
+//    qError("fseek failed, errno:%d", errno);
+    return -1;
+  }
+
+  size_t ws = fwrite(pHeader, sizeof(STSBufFileHeader), 1, pTSBuf->f);
+  if (ws != 1) {    
+//    qError("ts update header fwrite failed, size:%d, expected size:%d", (int32_t)ws, (int32_t)sizeof(STSBufFileHeader));
+    return -1;
+  }
+  return 0;
+}
+
+bool tsBufNextPos(STSBuf* pTSBuf) {
+  if (pTSBuf == NULL || pTSBuf->numOfGroups == 0) {
+    return false;
+  }
+  
+  STSCursor* pCur = &pTSBuf->cur;
+  
+  // get the first/last position according to traverse order
+  if (pCur->vgroupIndex == -1) {
+    if (pCur->order == TSDB_ORDER_ASC) {
+      tsBufGetBlock(pTSBuf, 0, 0);
+      
+      if (pTSBuf->block.numOfElem == 0) {  // the whole list is empty, return
+        tsBufResetPos(pTSBuf);
+        return false;
+      } else {
+        return true;
+      }
+      
+    } else {  // get the last timestamp record in the last block of the last vnode
+      assert(pTSBuf->numOfGroups > 0);
+      
+      int32_t groupIndex = pTSBuf->numOfGroups - 1;
+      pCur->vgroupIndex = groupIndex;
+      
+      int32_t            id = pTSBuf->pData[pCur->vgroupIndex].info.id;
+      STSGroupBlockInfo* pBlockInfo = tsBufGetGroupBlockInfo(pTSBuf, id);
+      int32_t            blockIndex = pBlockInfo->numOfBlocks - 1;
+      
+      tsBufGetBlock(pTSBuf, groupIndex, blockIndex);
+      
+      pCur->tsIndex = pTSBuf->block.numOfElem - 1;
+      if (pTSBuf->block.numOfElem == 0) {
+        tsBufResetPos(pTSBuf);
+        return false;
+      } else {
+        return true;
+      }
+    }
+  }
+  
+  int32_t step = pCur->order == TSDB_ORDER_ASC ? 1 : -1;
+  
+  while (1) {
+    assert(pTSBuf->tsData.len == pTSBuf->block.numOfElem * TSDB_KEYSIZE);
+    
+    if ((pCur->order == TSDB_ORDER_ASC && pCur->tsIndex >= pTSBuf->block.numOfElem - 1) ||
+        (pCur->order == TSDB_ORDER_DESC && pCur->tsIndex <= 0)) {
+      int32_t id = pTSBuf->pData[pCur->vgroupIndex].info.id;
+      
+      STSGroupBlockInfo* pBlockInfo = tsBufGetGroupBlockInfo(pTSBuf, id);
+      if (pBlockInfo == NULL || (pCur->blockIndex >= pBlockInfo->numOfBlocks - 1 && pCur->order == TSDB_ORDER_ASC) ||
+          (pCur->blockIndex <= 0 && pCur->order == TSDB_ORDER_DESC)) {
+        if ((pCur->vgroupIndex >= pTSBuf->numOfGroups - 1 && pCur->order == TSDB_ORDER_ASC) ||
+            (pCur->vgroupIndex <= 0 && pCur->order == TSDB_ORDER_DESC)) {
+          pCur->vgroupIndex = -1;
+          return false;
+        }
+        
+        if (pBlockInfo == NULL) {
+          return false;
+        }
+        
+        int32_t blockIndex = (pCur->order == TSDB_ORDER_ASC) ? 0 : (pBlockInfo->numOfBlocks - 1);
+        tsBufGetBlock(pTSBuf, pCur->vgroupIndex + step, blockIndex);
+        break;
+        
+      } else {
+        tsBufGetBlock(pTSBuf, pCur->vgroupIndex, pCur->blockIndex + step);
+        break;
+      }
+    } else {
+      pCur->tsIndex += step;
+      break;
+    }
+  }
+  
+  return true;
+}
+
+void tsBufResetPos(STSBuf* pTSBuf) {
+  if (pTSBuf == NULL) {
+    return;
+  }
+  
+  pTSBuf->cur = (STSCursor){.tsIndex = -1, .blockIndex = -1, .vgroupIndex = -1, .order = pTSBuf->cur.order};
+}
+
+STSElem tsBufGetElem(STSBuf* pTSBuf) {
+  STSElem elem1 = {.id = -1};
+  if (pTSBuf == NULL) {
+    return elem1;
+  }
+  
+  STSCursor* pCur = &pTSBuf->cur;
+  if (pCur != NULL && pCur->vgroupIndex < 0) {
+    return elem1;
+  }
+
+  STSBlock* pBlock = &pTSBuf->block;
+  
+  elem1.id = pTSBuf->pData[pCur->vgroupIndex].info.id;
+  elem1.ts = *(TSKEY*)(pTSBuf->tsData.rawBuf + pCur->tsIndex * TSDB_KEYSIZE);
+  elem1.tag = &pBlock->tag;
+
+  return elem1;
+}
+
+/**
+ * current only support ts comp data from two vnode merge
+ * @param pDestBuf
+ * @param pSrcBuf
+ * @param id
+ * @return
+ */
+int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf) {
+  if (pDestBuf == NULL || pSrcBuf == NULL || pSrcBuf->numOfGroups <= 0) {
+    return 0;
+  }
+  
+  if (pDestBuf->numOfGroups + pSrcBuf->numOfGroups > TS_COMP_FILE_GROUP_MAX) {
+    return -1;
+  }
+  
+  // src can only have one vnode index
+  assert(pSrcBuf->numOfGroups == 1);
+
+  // there are data in buffer, flush to disk first
+  tsBufFlush(pDestBuf);
+  
+  // compared with the last vnode id
+  int32_t id = tsBufGetLastGroupInfo((STSBuf*) pSrcBuf)->info.id;
+  if (id != tsBufGetLastGroupInfo(pDestBuf)->info.id) {
+    int32_t oldSize = pDestBuf->numOfGroups;
+    int32_t newSize = oldSize + pSrcBuf->numOfGroups;
+    
+    if (pDestBuf->numOfAlloc < newSize) {
+      pDestBuf->numOfAlloc = newSize;
+      
+      STSGroupBlockInfoEx* tmp = realloc(pDestBuf->pData, sizeof(STSGroupBlockInfoEx) * newSize);
+      if (tmp == NULL) {
+        return -1;
+      }
+      
+      pDestBuf->pData = tmp;
+    }
+    
+    // directly copy the vnode index information
+    memcpy(&pDestBuf->pData[oldSize], pSrcBuf->pData, (size_t)pSrcBuf->numOfGroups * sizeof(STSGroupBlockInfoEx));
+    
+    // set the new offset value
+    for (int32_t i = 0; i < pSrcBuf->numOfGroups; ++i) {
+      STSGroupBlockInfoEx* pBlockInfoEx = &pDestBuf->pData[i + oldSize];
+      pBlockInfoEx->info.offset = (pSrcBuf->pData[i].info.offset - getDataStartOffset()) + pDestBuf->fileSize;
+      pBlockInfoEx->info.id = id;
+    }
+    
+    pDestBuf->numOfGroups = newSize;
+  } else {
+    STSGroupBlockInfoEx* pBlockInfoEx = tsBufGetLastGroupInfo(pDestBuf);
+    
+    pBlockInfoEx->len += pSrcBuf->pData[0].len;
+    pBlockInfoEx->info.numOfBlocks += pSrcBuf->pData[0].info.numOfBlocks;
+    pBlockInfoEx->info.compLen += pSrcBuf->pData[0].info.compLen;
+    pBlockInfoEx->info.id = id;
+  }
+  
+  int32_t r = fseek(pDestBuf->f, 0, SEEK_END);
+  assert(r == 0);
+  
+  int64_t offset = getDataStartOffset();
+  int32_t size = (int32_t)pSrcBuf->fileSize - (int32_t)offset;
+  int64_t written = taosFSendFile(pDestBuf->f, pSrcBuf->f, &offset, size);
+  
+  if (written == -1 || written != size) {
+    return -1;
+  }
+  
+  pDestBuf->numOfTotal += pSrcBuf->numOfTotal;
+
+  int32_t oldSize = pDestBuf->fileSize;
+
+  // file meta data may be cached, close and reopen the file for accurate file size.
+  fclose(pDestBuf->f);
+  pDestBuf->f = fopen(pDestBuf->path, "rb+");
+  if (pDestBuf->f == NULL) {
+    return -1;
+  }
+
+  struct stat fileStat;
+  if (fstat(fileno(pDestBuf->f), &fileStat) != 0) {
+    return -1;  
+  }
+  pDestBuf->fileSize = (uint32_t)fileStat.st_size;
+
+  assert(pDestBuf->fileSize == oldSize + size);
+
+  return 0;
+}
+
+STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_t len, int32_t order, int32_t id) {
+  STSBuf* pTSBuf = tsBufCreate(true, order);
+  
+  STSGroupBlockInfo* pBlockInfo = &(addOneGroupInfo(pTSBuf, 0)->info);
+  pBlockInfo->numOfBlocks = numOfBlocks;
+  pBlockInfo->compLen = len;
+  pBlockInfo->offset = getDataStartOffset();
+  pBlockInfo->id = id;
+  
+  // update prev vnode length info in file
+  TSBufUpdateGroupInfo(pTSBuf, pTSBuf->numOfGroups - 1, pBlockInfo);
+  
+  int32_t ret = fseek(pTSBuf->f, pBlockInfo->offset, SEEK_SET);
+  if (ret == -1) {
+//    qError("fseek failed, errno:%d", errno);
+    tsBufDestroy(pTSBuf);
+    return NULL;
+  }
+  size_t sz = fwrite((void*)pData, 1, len, pTSBuf->f);
+  if (sz != len) {
+//    qError("ts data fwrite failed, write size:%d, expected size:%d", (int32_t)sz, len);
+    tsBufDestroy(pTSBuf);
+    return NULL;
+  }
+  pTSBuf->fileSize += len;
+  
+  pTSBuf->tsOrder = order;
+  assert(order == TSDB_ORDER_ASC || order == TSDB_ORDER_DESC);
+  
+  STSBufFileHeader header = {
+      .magic = TS_COMP_FILE_MAGIC, .numOfGroup = pTSBuf->numOfGroups, .tsOrder = pTSBuf->tsOrder};
+  if (STSBufUpdateHeader(pTSBuf, &header) < 0) {
+    tsBufDestroy(pTSBuf);
+    return NULL;
+  }
+
+  // TODO taosFsync??
+//  if (taosFsync(fileno(pTSBuf->f)) == -1) {
+////    qError("fsync failed, errno:%d", errno);
+//    tsBufDestroy(pTSBuf);
+//    return NULL;
+//  }
+  
+  return pTSBuf;
+}
+
+STSElem tsBufGetElemStartPos(STSBuf* pTSBuf, int32_t id, SVariant* tag) {
+  STSElem elem = {.id = -1};
+  
+  if (pTSBuf == NULL) {
+    return elem;
+  }
+  
+  int32_t j = tsBufFindGroupById(pTSBuf->pData, pTSBuf->numOfGroups, id);
+  if (j == -1) {
+    return elem;
+  }
+  
+  // for debug purpose
+  //  tsBufDisplay(pTSBuf);
+  
+  STSCursor*         pCur = &pTSBuf->cur;
+  STSGroupBlockInfo* pBlockInfo = &pTSBuf->pData[j].info;
+  
+  int32_t blockIndex = tsBufFindBlockByTag(pTSBuf, pBlockInfo, tag);
+  if (blockIndex < 0) {
+    return elem;
+  }
+  
+  pCur->vgroupIndex = j;
+  pCur->blockIndex = blockIndex;
+  tsBufGetBlock(pTSBuf, j, blockIndex);
+  
+  return tsBufGetElem(pTSBuf);
+}
+
+STSCursor tsBufGetCursor(STSBuf* pTSBuf) {
+  STSCursor c = {.vgroupIndex = -1};
+  if (pTSBuf == NULL) {
+    return c;
+  }
+  
+  return pTSBuf->cur;
+}
+
+void tsBufSetCursor(STSBuf* pTSBuf, STSCursor* pCur) {
+  if (pTSBuf == NULL || pCur == NULL) {
+    return;
+  }
+  
+  //  assert(pCur->vgroupIndex != -1 && pCur->tsIndex >= 0 && pCur->blockIndex >= 0);
+  if (pCur->vgroupIndex != -1) {
+    tsBufGetBlock(pTSBuf, pCur->vgroupIndex, pCur->blockIndex);
+  }
+  
+  pTSBuf->cur = *pCur;
+}
+
+void tsBufSetTraverseOrder(STSBuf* pTSBuf, int32_t order) {
+  if (pTSBuf == NULL) {
+    return;
+  }
+  
+  pTSBuf->cur.order = order;
+}
+
+STSBuf* tsBufClone(STSBuf* pTSBuf) {
+  if (pTSBuf == NULL) {
+    return NULL;
+  }
+
+  tsBufFlush(pTSBuf);
+
+  return tsBufCreateFromFile(pTSBuf->path, false);
+}
+
+void tsBufDisplay(STSBuf* pTSBuf) {
+  printf("-------start of ts comp file-------\n");
+  printf("number of vnode:%d\n", pTSBuf->numOfGroups);
+  
+  int32_t old = pTSBuf->cur.order;
+  pTSBuf->cur.order = TSDB_ORDER_ASC;
+  
+  tsBufResetPos(pTSBuf);
+  
+  while (tsBufNextPos(pTSBuf)) {
+    STSElem elem = tsBufGetElem(pTSBuf);
+    if (elem.tag->nType == TSDB_DATA_TYPE_BIGINT) {
+      printf("%d-%" PRId64 "-%" PRId64 "\n", elem.id, elem.tag->i64, elem.ts);
+    }
+  }
+  
+  pTSBuf->cur.order = old;
+  printf("-------end of ts comp file-------\n");
+}
+
+static int32_t getDataStartOffset() {
+  return sizeof(STSBufFileHeader) + TS_COMP_FILE_GROUP_MAX * sizeof(STSGroupBlockInfo);
+}
+
+// update prev vnode length info in file
+static void TSBufUpdateGroupInfo(STSBuf* pTSBuf, int32_t index, STSGroupBlockInfo* pBlockInfo) {
+  int32_t offset = sizeof(STSBufFileHeader) + index * sizeof(STSGroupBlockInfo);
+  doUpdateGroupInfo(pTSBuf, offset, pBlockInfo);
+}
+
+static STSBuf* allocResForTSBuf(STSBuf* pTSBuf) {
+  const int32_t INITIAL_GROUPINFO_SIZE = 4;
+  
+  pTSBuf->numOfAlloc = INITIAL_GROUPINFO_SIZE;
+  pTSBuf->pData = calloc(pTSBuf->numOfAlloc, sizeof(STSGroupBlockInfoEx));
+  if (pTSBuf->pData == NULL) {
+    tsBufDestroy(pTSBuf);
+    return NULL;
+  }
+  
+  pTSBuf->tsData.rawBuf = malloc(MEM_BUF_SIZE);
+  if (pTSBuf->tsData.rawBuf == NULL) {
+    tsBufDestroy(pTSBuf);
+    return NULL;
+  }
+  
+  pTSBuf->bufSize = MEM_BUF_SIZE;
+  pTSBuf->tsData.threshold = MEM_BUF_SIZE;
+  pTSBuf->tsData.allocSize = MEM_BUF_SIZE;
+  
+  pTSBuf->assistBuf = malloc(MEM_BUF_SIZE);
+  if (pTSBuf->assistBuf == NULL) {
+    tsBufDestroy(pTSBuf);
+    return NULL;
+  }
+  
+  pTSBuf->block.payload = malloc(MEM_BUF_SIZE);
+  if (pTSBuf->block.payload == NULL) {
+    tsBufDestroy(pTSBuf);
+    return NULL;
+  }
+  
+  pTSBuf->fileSize += getDataStartOffset();
+  return pTSBuf;
+}
+
+int32_t tsBufGetNumOfGroup(STSBuf* pTSBuf) {
+  if (pTSBuf == NULL) {
+    return 0;
+  }
+
+  return pTSBuf->numOfGroups;
+}
+
+void tsBufGetGroupIdList(STSBuf* pTSBuf, int32_t* num, int32_t** id) {
+  int32_t size = tsBufGetNumOfGroup(pTSBuf);
+  if (num != NULL) {
+    *num = size;
+  }
+
+  *id = NULL;
+  if (size == 0) {
+    return;
+  }
+
+  (*id) = malloc(tsBufGetNumOfGroup(pTSBuf) * sizeof(int32_t));
+
+  for(int32_t i = 0; i < size; ++i) {
+    (*id)[i] = pTSBuf->pData[i].info.id;
+  }
+}
+
+int32_t dumpFileBlockByGroupId(STSBuf* pTSBuf, int32_t groupIndex, void* buf, int32_t* len, int32_t* numOfBlocks) {
+  assert(groupIndex >= 0 && groupIndex < pTSBuf->numOfGroups);
+  STSGroupBlockInfo *pBlockInfo = &pTSBuf->pData[groupIndex].info;
+
+  *len = 0;
+  *numOfBlocks = 0;
+
+  if (fseek(pTSBuf->f, pBlockInfo->offset, SEEK_SET) != 0) {
+    int code = TAOS_SYSTEM_ERROR(ferror(pTSBuf->f));
+//    qError("%p: fseek failed: %s", pSql, tstrerror(code));
+    return code;
+  }
+
+  size_t s = fread(buf, 1, pBlockInfo->compLen, pTSBuf->f);
+  if (s != pBlockInfo->compLen) {
+    int code = TAOS_SYSTEM_ERROR(ferror(pTSBuf->f));
+//    tscError("%p: fread didn't return expected data: %s", pSql, tstrerror(code));
+    return code;
+  }
+
+  *len = pBlockInfo->compLen;
+  *numOfBlocks = pBlockInfo->numOfBlocks;
+
+  return TSDB_CODE_SUCCESS;
+}
+
+STSElem tsBufFindElemStartPosByTag(STSBuf* pTSBuf, SVariant* pTag) {
+  STSElem el = {.id = -1};
+
+  for (int32_t i = 0; i < pTSBuf->numOfGroups; ++i) {
+    el = tsBufGetElemStartPos(pTSBuf, pTSBuf->pData[i].info.id, pTag);
+    if (el.id == pTSBuf->pData[i].info.id) {
+      return el;
+    }
+  }
+
+  return el;
+}
+
+bool tsBufIsValidElem(STSElem* pElem) {
+  return pElem->id >= 0;
+}

source/libs/parser/CMakeLists.txt

@@ -8,7 +8,7 @@ target_include_directories(
 
 target_link_libraries(
         parser
-        PRIVATE os util common catalog executor transport
+        PRIVATE os util common catalog function transport
 )
 
 ADD_SUBDIRECTORY(test)
\ No newline at end of file

source/libs/parser/inc/astGenerator.h

@@ -246,6 +246,7 @@ typedef struct tSqlExpr {
 // used in select clause. select <SArray> from xxx
 typedef struct tSqlExprItem {
   tSqlExpr          *pNode;      // The list of expressions
+  int32_t            functionId;
   char              *aliasName;  // alias name, null-terminated string
   bool               distinct;
 } tSqlExprItem;

source/libs/parser/inc/parserInt.h

@@ -26,6 +26,11 @@ extern "C" {
 
 struct SSqlNode;
 
+typedef struct SColumnIndex {
+  int16_t tableIndex;
+  int16_t columnIndex;
+} SColumnIndex;
+
 typedef struct SInsertStmtInfo {
   SHashObj *pTableBlockHashList;     // data block for each table
   SArray   *pDataBlocks;             // SArray<STableDataBlocks*>. Merged submit block for each vgroup
@@ -35,6 +40,51 @@ typedef struct SInsertStmtInfo {
   char     *sql;                     // current sql statement position
 } SInsertStmtInfo;
 
+// the structure for sql function in select clause
+typedef struct SSqlExpr {
+  char      aliasName[TSDB_COL_NAME_LEN];  // as aliasName
+  char      token[TSDB_COL_NAME_LEN];      // original token
+  SColIndex colInfo;
+  uint64_t  uid;            // table uid, todo refactor use the pointer
+
+  int16_t   functionId;     // function id in aAgg array
+
+  int16_t   resType;        // return value type
+  int16_t   resBytes;       // length of return value
+  int32_t   interBytes;     // inter result buffer size
+
+  int16_t   colType;        // table column type
+  int16_t   colBytes;       // table column bytes
+
+  int16_t   numOfParams;    // argument value of each function
+  SVariant  param[3];       // parameters are not more than 3
+  int32_t   offset;         // sub result column value of arithmetic expression.
+  int16_t   resColId;       // result column id
+
+  SColumnFilterList flist;
+} SSqlExpr;
+
+typedef struct SExprInfo {
+  SSqlExpr           base;
+  struct tExprNode  *pExpr;
+} SExprInfo;
+
+typedef struct SColumn {
+  uint64_t     tableUid;
+  int32_t      columnIndex;
+  SColumnInfo  info;
+} SColumn;
+
+typedef struct SInternalField {
+  TAOS_FIELD      field;
+  bool            visible;
+  SExprInfo      *pExpr;
+} SInternalField;
+
+void clearTableMetaInfo(STableMetaInfo* pTableMetaInfo);
+
+void clearAllTableMetaInfo(SQueryStmtInfo* pQueryInfo, bool removeMeta, uint64_t id);
+
 /**
  * Validate the sql info, according to the corresponding metadata info from catalog.
  * @param pCatalog

source/libs/parser/inc/parserUtil.h

@@ -23,10 +23,31 @@ extern "C" {
 #include "os.h"
 #include "ttoken.h"
 
+#define UTIL_TABLE_IS_SUPER_TABLE(metaInfo) \
+  (((metaInfo)->pTableMeta != NULL) && ((metaInfo)->pTableMeta->tableType == TSDB_SUPER_TABLE))
+
+#define UTIL_TABLE_IS_CHILD_TABLE(metaInfo) \
+  (((metaInfo)->pTableMeta != NULL) && ((metaInfo)->pTableMeta->tableType == TSDB_CHILD_TABLE))
+
+#define UTIL_TABLE_IS_NORMAL_TABLE(metaInfo) \
+  (!(UTIL_TABLE_IS_SUPER_TABLE(metaInfo) || UTIL_TABLE_IS_CHILD_TABLE(metaInfo)))
+
+#define UTIL_TABLE_IS_TMP_TABLE(metaInfo) \
+  (((metaInfo)->pTableMeta != NULL) && ((metaInfo)->pTableMeta->tableType == TSDB_TEMP_TABLE))
+
+
 int32_t parserValidateIdToken(SToken* pToken);
-int32_t parserSetInvalidOperatorMsg(char* dst, int32_t dstBufLen, const char* msg);
+int32_t buildInvalidOperationMsg(char* dst, int32_t dstBufLen, const char* msg);
 int32_t parserSetSyntaxErrMsg(char* dst, int32_t dstBufLen, const char* additionalInfo,  const char* sourceStr);
 
+
+void columnListCopy(SArray* dst, const SArray* src, uint64_t tableUid);
+void columnListCopyAll(SArray* dst, const SArray* src);
+void columnListDestroy(SArray* pColumnList);
+
+void cleanupTagCond(STagCond* pTagCond);
+void cleanupColumnCond(SArray** pCond);
+
 #ifdef __cplusplus
 }
 #endif

source/libs/parser/inc/queryInfoUtil.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+
+#ifndef TDENGINE_QUERYINFOUTIL_H
+#define TDENGINE_QUERYINFOUTIL_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+#include "parserInt.h"
+
+SSchema* getTbnameColumnSchema();
+
+int32_t  getNumOfColumns(const STableMeta* pTableMeta);
+int32_t  getNumOfTags(const STableMeta* pTableMeta);
+SSchema *getTableColumnSchema(const STableMeta *pTableMeta);
+SSchema *getTableTagSchema(const STableMeta* pTableMeta);
+SSchema *getOneColumnSchema(const STableMeta* pTableMeta, int32_t colIndex);
+
+size_t     getNumOfExprs(SQueryStmtInfo* pQueryInfo);
+SExprInfo* createExprInfo(STableMetaInfo* pTableMetaInfo, int16_t functionId, SColumnIndex* pColIndex, int16_t type,
+                          int16_t size, int16_t resColId, int16_t interSize, int32_t colType);
+void       addExprInfo(SQueryStmtInfo* pQueryInfo, int32_t index, SExprInfo* pExprInfo);
+void       updateExprInfo(SExprInfo* pExprInfo, int16_t functionId, int32_t colId, int16_t srcColumnIndex, int16_t resType, int16_t resSize);
+void       assignExprInfo(SExprInfo* dst, const SExprInfo* src);
+
+SExprInfo* getExprInfo(SQueryStmtInfo* pQueryInfo, int32_t index);
+int32_t    copyAllExprInfo(SArray* dst, const SArray* src, bool deepcopy);
+
+void       cleanupFieldInfo(SFieldInfo* pFieldInfo);
+
+STableComInfo getTableInfo(const STableMeta* pTableMeta);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TDENGINE_QUERYINFOUTIL_H

source/libs/parser/src/astValidate.c

@@ -16,6 +16,7 @@
 #include "ttime.h"
 #include "parserInt.h"
 #include "parserUtil.h"
+#include "queryInfoUtil.h"
 
 #define TSQL_TBNAME_L "tbname"
 #define DEFAULT_PRIMARY_TIMESTAMP_COL_NAME "_c0"
@@ -25,18 +26,13 @@
 #define COLUMN_INDEX_INITIAL_VAL (-2)
 #define COLUMN_INDEX_INITIALIZER { COLUMN_INDEX_INITIAL_VAL, COLUMN_INDEX_INITIAL_VAL }
 
-typedef struct SColumn {
-  uint64_t     tableUid;
-  int32_t      columnIndex;
-  SColumnInfo  info;
-} SColumn;
+static int32_t validateSelectNodeList(SQueryStmtInfo* pQueryInfo, SArray* pSelNodeList, bool joinQuery, bool timeWindowQuery, bool outerQuery, char* msg, int32_t msgBufLen);
 
-typedef struct SColumnIndex {
-  int16_t tableIndex;
-  int16_t columnIndex;
-} SColumnIndex;
+size_t tscNumOfExprs(SQueryStmtInfo* pQueryInfo) {
+  return taosArrayGetSize(pQueryInfo->exprList);
+}
 
-static int32_t evaluateImpl(tSqlExpr* pExpr, int32_t tsPrecision) {
+  static int32_t evaluateImpl(tSqlExpr* pExpr, int32_t tsPrecision) {
   int32_t code = 0;
   if (pExpr->type == SQL_NODE_EXPR) {
     code = evaluateImpl(pExpr->pLeft, tsPrecision);
@@ -147,7 +143,7 @@ void columnDestroy(SColumn* pCol) {
   free(pCol);
 }
 
-void tscColumnListDestroy(SArray* pColumnList) {
+void destroyColumnList(SArray* pColumnList) {
   if (pColumnList == NULL) {
     return;
   }
@@ -169,7 +165,7 @@ void clearTableMetaInfo(STableMetaInfo* pTableMetaInfo) {
   tfree(pTableMetaInfo->pTableMeta);
   tfree(pTableMetaInfo->vgroupList);
 
-  tscColumnListDestroy(pTableMetaInfo->tagColList);
+  destroyColumnList(pTableMetaInfo->tagColList);
   pTableMetaInfo->tagColList = NULL;
 
   free(pTableMetaInfo);
@@ -183,7 +179,6 @@ void clearAllTableMetaInfo(SQueryStmtInfo* pQueryInfo, bool removeMeta, uint64_t
 //      removeCachedTableMeta(pTableMetaInfo, id);
     }
 
-    freeVgroupTableInfo(pTableMetaInfo->pVgroupTables);
     clearTableMetaInfo(pTableMetaInfo);
   }
 
@@ -224,7 +219,7 @@ static STableMeta* extractTempTableMetaFromSubquery(SQueryStmtInfo* pUpstream) {
   return meta;
 }
 
-void tscInitQueryInfo(SQueryStmtInfo* pQueryInfo) {
+void initQueryInfo(SQueryStmtInfo* pQueryInfo) {
   assert(pQueryInfo->fieldsInfo.internalField == NULL);
   //assert(0);
 //  pQueryInfo->fieldsInfo.internalField = taosArrayInit(4, sizeof(SInternalField));
@@ -241,7 +236,7 @@ void tscInitQueryInfo(SQueryStmtInfo* pQueryInfo) {
   pQueryInfo->slimit.offset  = 0;
   pQueryInfo->pUpstream      = taosArrayInit(4, POINTER_BYTES);
   pQueryInfo->window         = TSWINDOW_INITIALIZER;
-  pQueryInfo->multigroupResult = true;
+//  pQueryInfo->multigroupResult = true;
 }
 
 int32_t validateSqlNode(SSqlNode* pSqlNode, SQueryStmtInfo* pQueryInfo, char* msg, int32_t msgBufLen);
@@ -307,11 +302,11 @@ static int32_t doValidateSubquery(SSqlNode* pSqlNode, int32_t index, SQueryStmtI
   // union all is not support currently
   SSqlNode* p = taosArrayGetP(subInfo->pSubquery, 0);
   if (taosArrayGetSize(subInfo->pSubquery) >= 2) {
-    return parserSetInvalidOperatorMsg(msgBuf, msgBufLen, "not support union in subquery");
+    return buildInvalidOperationMsg(msgBuf, msgBufLen, "not support union in subquery");
   }
 
   SQueryStmtInfo* pSub = calloc(1, sizeof(SQueryStmtInfo));
-  tscInitQueryInfo(pSub);
+  initQueryInfo(pSub);
 
   SArray *pUdfInfo = NULL;
   if (pQueryInfo->pUdfInfo) {
@@ -336,7 +331,7 @@ static int32_t doValidateSubquery(SSqlNode* pSqlNode, int32_t index, SQueryStmtI
   if (subInfo->aliasName.n > 0) {
     if (subInfo->aliasName.n >= TSDB_TABLE_FNAME_LEN) {
       tfree(pTableMetaInfo1);
-      return parserSetInvalidOperatorMsg(msgBuf, msgBufLen, "subquery alias name too long");
+      return buildInvalidOperationMsg(msgBuf, msgBufLen, "subquery alias name too long");
     }
 
     tstrncpy(pTableMetaInfo1->aliasName, subInfo->aliasName.z, subInfo->aliasName.n + 1);
@@ -528,8 +523,6 @@ STableMetaInfo* tscAddTableMetaInfo(SQueryStmtInfo* pQueryInfo, char* name, STab
     tscColumnListCopy(pTableMetaInfo->tagColList, pTagCols, pTableMetaInfo->pTableMeta->uid);
   }
 
-  pTableMetaInfo->pVgroupTables = tscVgroupTableInfoDup(pVgroupTables);
-
   pQueryInfo->numOfTables += 1;
   return pTableMetaInfo;
 }
@@ -600,36 +593,15 @@ int32_t getTableIndexByName(SToken* pToken, SQueryStmtInfo* pQueryInfo, SColumnI
   return TSDB_CODE_SUCCESS;
 }
 
-STableComInfo getTableInfo(const STableMeta* pTableMeta) {
-  assert(pTableMeta != NULL);
-  return pTableMeta->tableInfo;
-}
-
-int32_t getNumOfColumns(const STableMeta* pTableMeta) {
-  assert(pTableMeta != NULL);
-  // table created according to super table, use data from super table
-  return getTableInfo(pTableMeta).numOfColumns;
-}
-
-int32_t getNumOfTags(const STableMeta* pTableMeta) {
-  assert(pTableMeta != NULL);
-  return getTableInfo(pTableMeta).numOfTags;
-}
-
-SSchema *getTableSchema(const STableMeta *pTableMeta) {
-  assert(pTableMeta != NULL);
-  return (SSchema*) pTableMeta->schema;
-}
-
 static int16_t doGetColumnIndex(SQueryStmtInfo* pQueryInfo, int32_t index, SToken* pToken) {
   STableMeta* pTableMeta = getMetaInfo(pQueryInfo, index)->pTableMeta;
 
   int32_t  numOfCols = getNumOfColumns(pTableMeta) + getNumOfTags(pTableMeta);
-  SSchema* pSchema = getTableSchema(pTableMeta);
+  SSchema* pSchema = getTableColumnSchema(pTableMeta);
 
   int16_t columnIndex = COLUMN_INDEX_INITIAL_VAL;
 
-  for (int16_t i = 0; i < numOfCols; ++i) {
+  for (int32_t i = 0; i < numOfCols; ++i) {
     if (pToken->n != strlen(pSchema[i].name)) {
       continue;
     }
@@ -670,7 +642,7 @@ int32_t doGetColumnIndexByName(SToken* pToken, SQueryStmtInfo* pQueryInfo, SColu
 
         if (colIndex != COLUMN_INDEX_INITIAL_VAL) {
           if (pIndex->columnIndex != COLUMN_INDEX_INITIAL_VAL) {
-            return parserSetInvalidOperatorMsg(msg, msgBufLen, msg0);
+            return buildInvalidOperationMsg(msg, msgBufLen, msg0);
           } else {
             pIndex->tableIndex = i;
             pIndex->columnIndex = colIndex;
@@ -685,7 +657,7 @@ int32_t doGetColumnIndexByName(SToken* pToken, SQueryStmtInfo* pQueryInfo, SColu
     }
 
     if (pIndex->columnIndex == COLUMN_INDEX_INITIAL_VAL) {
-      return parserSetInvalidOperatorMsg(msg, msgBufLen, msg1);
+      return buildInvalidOperationMsg(msg, msgBufLen, msg1);
     }
   }
 
@@ -696,7 +668,7 @@ int32_t doGetColumnIndexByName(SToken* pToken, SQueryStmtInfo* pQueryInfo, SColu
   }
 }
 
-int32_t getColumnIndexByName(const SToken* pToken, SQueryStmtInfo* pQueryInfo, SColumnIndex* pIndex, char* msg) {
+int32_t getColumnIndexByName(const SToken* pToken, SQueryStmtInfo* pQueryInfo, SColumnIndex* pIndex, char* msg, int32_t msgBufLen) {
   if (pQueryInfo->pTableMetaInfo == NULL || pQueryInfo->numOfTables == 0) {
     return TSDB_CODE_TSC_INVALID_OPERATION;
   }
@@ -707,7 +679,7 @@ int32_t getColumnIndexByName(const SToken* pToken, SQueryStmtInfo* pQueryInfo, S
     return TSDB_CODE_TSC_INVALID_OPERATION;
   }
 
-  return doGetColumnIndexByName(&tmpToken, pQueryInfo, pIndex, msg);
+  return doGetColumnIndexByName(&tmpToken, pQueryInfo, pIndex, msg, msgBufLen);
 }
 
 int32_t validateGroupbyNode(SQueryStmtInfo* pQueryInfo, SArray* pList, char* msg, int32_t msgBufLen) {
@@ -727,7 +699,7 @@ int32_t validateGroupbyNode(SQueryStmtInfo* pQueryInfo, SArray* pList, char* msg
   }
 
   if (pQueryInfo->numOfTables > 1) {
-    return parserSetInvalidOperatorMsg(msg, msgBufLen, msg4);
+    return buildInvalidOperationMsg(msg, msgBufLen, msg4);
   }
 
   SGroupbyExpr* pGroupExpr = &pQueryInfo->groupbyExpr;
@@ -745,7 +717,7 @@ int32_t validateGroupbyNode(SQueryStmtInfo* pQueryInfo, SArray* pList, char* msg
 
   int32_t numOfGroupCols = (int16_t) taosArrayGetSize(pList);
   if (numOfGroupCols > TSDB_MAX_TAGS) {
-    return parserSetInvalidOperatorMsg(msg, msgBufLen, msg1);
+    return buildInvalidOperationMsg(msg, msgBufLen, msg1);
   }
 
   SSchema *pSchema       = NULL;
@@ -753,20 +725,20 @@ int32_t validateGroupbyNode(SQueryStmtInfo* pQueryInfo, SArray* pList, char* msg
 
   size_t num = taosArrayGetSize(pList);
   for (int32_t i = 0; i < num; ++i) {
-    tVariantListItem * pItem = taosArrayGet(pList, i);
-    tVariant* pVar = &pItem->pVar;
+    SListItem * pItem = taosArrayGet(pList, i);
+    SVariant* pVar = &pItem->pVar;
 
     SToken token = {pVar->nLen, pVar->nType, pVar->pz};
 
     SColumnIndex index = COLUMN_INDEX_INITIALIZER;
-    if (getColumnIndexByName(&token, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) {
-      return parserSetInvalidOperatorMsg(msg, msgBufLen, msg2);
+    if (getColumnIndexByName(&token, pQueryInfo, &index, msg, msgBufLen) != TSDB_CODE_SUCCESS) {
+      return buildInvalidOperationMsg(msg, msgBufLen, msg2);
     }
 
     if (tableIndex == COLUMN_INDEX_INITIAL_VAL) {
       tableIndex = index.tableIndex;
     } else if (tableIndex != index.tableIndex) {
-      return parserSetInvalidOperatorMsg(msg, msgBufLen, msg3);
+      return buildInvalidOperationMsg(msg, msgBufLen, msg3);
     }
 
     pTableMetaInfo = getMetaInfo(pQueryInfo, index.tableIndex);
@@ -775,7 +747,7 @@ int32_t validateGroupbyNode(SQueryStmtInfo* pQueryInfo, SArray* pList, char* msg
     if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) {
       pSchema = getTbnameColumnSchema();
     } else {
-      pSchema = getTableColumnSchema(pTableMeta, index.columnIndex);
+      pSchema = getOneColumnSchema(pTableMeta, index.columnIndex);
     }
 
     int32_t numOfCols = getNumOfColumns(pTableMeta);
@@ -783,7 +755,7 @@ int32_t validateGroupbyNode(SQueryStmtInfo* pQueryInfo, SArray* pList, char* msg
 
     if (groupTag) {
       if (!UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
-        return parserSetInvalidOperatorMsg(msg, msgBufLen, msg6);
+        return buildInvalidOperationMsg(msg, msgBufLen, msg6);
       }
 
       int32_t relIndex = index.columnIndex;
@@ -796,14 +768,14 @@ int32_t validateGroupbyNode(SQueryStmtInfo* pQueryInfo, SArray* pList, char* msg
       taosArrayPush(pGroupExpr->columnInfo, &colIndex);
 
       index.columnIndex = relIndex;
-      tscColumnListInsert(pTableMetaInfo->tagColList, index.columnIndex, pTableMeta->id.uid, pSchema);
+      tscColumnListInsert(pTableMetaInfo->tagColList, index.columnIndex, pTableMeta->uid, pSchema);
     } else {
       // check if the column type is valid, here only support the bool/tinyint/smallint/bigint group by
       if (pSchema->type == TSDB_DATA_TYPE_TIMESTAMP || pSchema->type == TSDB_DATA_TYPE_FLOAT || pSchema->type == TSDB_DATA_TYPE_DOUBLE) {
-        return parserSetInvalidOperatorMsg(msg, msgBufLen, msg5);
+        return buildInvalidOperationMsg(msg, msgBufLen, msg5);
       }
 
-      tscColumnListInsert(pQueryInfo->colList, index.columnIndex, pTableMeta->id.uid, pSchema);
+      tscColumnListInsert(pQueryInfo->colList, index.columnIndex, pTableMeta->uid, pSchema);
 
       SColIndex colIndex = { .colIndex = index.columnIndex, .flag = TSDB_COL_NORMAL, .colId = pSchema->colId };
       strncpy(colIndex.name, pSchema->name, tListLen(colIndex.name));
@@ -817,13 +789,13 @@ int32_t validateGroupbyNode(SQueryStmtInfo* pQueryInfo, SArray* pList, char* msg
   // 1. only one normal column allowed in the group by clause
   // 2. the normal column in the group by clause can only located in the end position
   if (numOfGroupCols > 1) {
-    return parserSetInvalidOperatorMsg(msg, msgBufLen, msg7);
+    return buildInvalidOperationMsg(msg, msgBufLen, msg7);
   }
 
   for(int32_t i = 0; i < num; ++i) {
     SColIndex* pIndex = taosArrayGet(pGroupExpr->columnInfo, i);
     if (TSDB_COL_IS_NORMAL_COL(pIndex->flag) && i != num - 1) {
-      return parserSetInvalidOperatorMsg(msg, msgBufLen, msg8);
+      return buildInvalidOperationMsg(msg, msgBufLen, msg8);
     }
   }
 
@@ -863,7 +835,6 @@ int32_t validateSqlNode(SSqlNode* pSqlNode, SQueryStmtInfo* pQueryInfo, char* ms
   }
 
   if (pSqlNode->from->type == SQL_NODE_FROM_SUBQUERY) {
-//    clearAllTableMetaInfo(pQueryInfo, false, pSql->self);
     pQueryInfo->numOfTables = 0;
 
     // parse the subquery in the first place
@@ -873,7 +844,7 @@ int32_t validateSqlNode(SSqlNode* pSqlNode, SQueryStmtInfo* pQueryInfo, char* ms
       SRelElementPair* subInfo = taosArrayGet(pSqlNode->from->list, i);
       SSqlNode* p = taosArrayGetP(subInfo->pSubquery, 0);
       if (p->from->type == SQL_NODE_FROM_SUBQUERY) {
-        return parserSetInvalidOperatorMsg(msg, msgBufLen, msg9);
+        return buildInvalidOperationMsg(msg, msgBufLen, msg9);
       }
 
       code = doValidateSubquery(pSqlNode, i, pQueryInfo, msg, msgBufLen);
@@ -884,25 +855,25 @@ int32_t validateSqlNode(SSqlNode* pSqlNode, SQueryStmtInfo* pQueryInfo, char* ms
 
     int32_t timeWindowQuery =
         (TPARSER_HAS_TOKEN(pSqlNode->interval.interval) || TPARSER_HAS_TOKEN(pSqlNode->sessionVal.gap));
-    TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_TABLE_QUERY);
+//    TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_TABLE_QUERY);
 
     // parse the group by clause in the first place
-    if (validateGroupbyNode(pQueryInfo, pSqlNode->pGroupby, pCmd) != TSDB_CODE_SUCCESS) {
+    if (validateGroupbyNode(pQueryInfo, pSqlNode->pGroupby, msg, msgBufLen) != TSDB_CODE_SUCCESS) {
       return TSDB_CODE_TSC_INVALID_OPERATION;
     }
 
 
-    if (validateSelectNodeList(pCmd, pQueryInfo, pSqlNode->pSelNodeList, false, timeWindowQuery, true) !=
+    if (validateSelectNodeList(pQueryInfo, pSqlNode->pSelNodeList, false, timeWindowQuery, true, msg, msgBufLen) !=
         TSDB_CODE_SUCCESS) {
       return TSDB_CODE_TSC_INVALID_OPERATION;
     }
 
     // todo NOT support yet
     for (int32_t i = 0; i < tscNumOfExprs(pQueryInfo); ++i) {
-      SExprInfo* pExpr = tscExprGet(pQueryInfo, i);
+      SExprInfo* pExpr = getExprInfo(pQueryInfo, i);
       int32_t    f = pExpr->base.functionId;
       if (f == TSDB_FUNC_STDDEV || f == TSDB_FUNC_PERCT || f == TSDB_FUNC_INTERP) {
-        return parserSetInvalidOperatorMsg(msg, msgBufLen, msg6);
+        return buildInvalidOperationMsg(msg, msgBufLen, msg6);
       }
 
       if ((timeWindowQuery || pQueryInfo->stateWindow) && f == TSDB_FUNC_LAST) {
@@ -913,17 +884,17 @@ int32_t validateSqlNode(SSqlNode* pSqlNode, SQueryStmtInfo* pQueryInfo, char* ms
     }
 
     STableMeta* pTableMeta = getMetaInfo(pQueryInfo, 0)->pTableMeta;
-    SSchema*    pSchema = tscGetTableColumnSchema(pTableMeta, 0);
+    SSchema*    pSchema = getOneColumnSchema(pTableMeta, 0);
 
     if (pSchema->type != TSDB_DATA_TYPE_TIMESTAMP) {
       int32_t numOfExprs = (int32_t)tscNumOfExprs(pQueryInfo);
 
       for (int32_t i = 0; i < numOfExprs; ++i) {
-        SExprInfo* pExpr = tscExprGet(pQueryInfo, i);
+        SExprInfo* pExpr = getExprInfo(pQueryInfo, i);
 
         int32_t f = pExpr->base.functionId;
         if (f == TSDB_FUNC_DERIVATIVE || f == TSDB_FUNC_TWA || f == TSDB_FUNC_IRATE) {
-          return parserSetInvalidOperatorMsg(msg, msgBufLen, msg7);
+          return buildInvalidOperationMsg(msg, msgBufLen, msg7);
         }
       }
     }
@@ -935,7 +906,7 @@ int32_t validateSqlNode(SSqlNode* pSqlNode, SQueryStmtInfo* pQueryInfo, char* ms
       }
     } else {
       if (pQueryInfo->numOfTables > 1) {
-        return parserSetInvalidOperatorMsg(msg, msgBufLen, msg8);
+        return buildInvalidOperationMsg(msg, msgBufLen, msg8);
       }
     }
 
@@ -956,7 +927,7 @@ int32_t validateSqlNode(SSqlNode* pSqlNode, SQueryStmtInfo* pQueryInfo, char* ms
         // check if the first column of the nest query result is timestamp column
         SColumn* pCol = taosArrayGetP(pQueryInfo->colList, 0);
         if (pCol->info.type != TSDB_DATA_TYPE_TIMESTAMP) {
-          return parserSetInvalidOperatorMsg(msg, msgBufLen, msg4);
+          return buildInvalidOperationMsg(msg, msgBufLen, msg4);
         }
 
         if (validateFunctionsInIntervalOrGroupbyQuery(pCmd, pQueryInfo) != TSDB_CODE_SUCCESS) {
@@ -1005,7 +976,7 @@ int32_t validateSqlNode(SSqlNode* pSqlNode, SQueryStmtInfo* pQueryInfo, char* ms
 
     size_t numOfTables = taosArrayGetSize(pSqlNode->from->list);
     if (numOfTables > TSDB_MAX_JOIN_TABLE_NUM) {
-      return parserSetInvalidOperatorMsg(msg, msgBufLen, msg2);
+      return buildInvalidOperationMsg(msg, msgBufLen, msg2);
     }
 
     // set all query tables, which are maybe more than one.
@@ -1033,7 +1004,7 @@ int32_t validateSqlNode(SSqlNode* pSqlNode, SQueryStmtInfo* pQueryInfo, char* ms
       pSqlNode->pWhere = NULL;
     } else {
       if (taosArrayGetSize(pSqlNode->from->list) > 1) { // Cross join not allowed yet
-        return parserSetInvalidOperatorMsg(msg, msgBufLen, "cross join not supported yet");
+        return buildInvalidOperationMsg(msg, msgBufLen, "cross join not supported yet");
       }
     }
 
@@ -1041,7 +1012,7 @@ int32_t validateSqlNode(SSqlNode* pSqlNode, SQueryStmtInfo* pQueryInfo, char* ms
     int32_t timeWindowQuery =
         (TPARSER_HAS_TOKEN(pSqlNode->interval.interval) || TPARSER_HAS_TOKEN(pSqlNode->sessionVal.gap));
 
-    if (validateSelectNodeList(pCmd, pQueryInfo, pSqlNode->pSelNodeList, joinQuery, timeWindowQuery, false) !=
+    if (validateSelectNodeList(pQueryInfo, pSqlNode->pSelNodeList, joinQuery, timeWindowQuery, false, msg, msgBufLen) !=
         TSDB_CODE_SUCCESS) {
       return TSDB_CODE_TSC_INVALID_OPERATION;
     }
@@ -1067,11 +1038,11 @@ int32_t validateSqlNode(SSqlNode* pSqlNode, SQueryStmtInfo* pQueryInfo, char* ms
     }
 
     if (tscQueryTags(pQueryInfo)) {
-      SExprInfo* pExpr1 = tscExprGet(pQueryInfo, 0);
+      SExprInfo* pExpr1 = getExprInfo(pQueryInfo, 0);
 
       if (pExpr1->base.functionId != TSDB_FUNC_TID_TAG) {
         if ((pQueryInfo->colCond && taosArrayGetSize(pQueryInfo->colCond) > 0) || IS_TSWINDOW_SPECIFIED(pQueryInfo->window))  {
-          return parserSetInvalidOperatorMsg(msg, msgBufLen, msg5);
+          return buildInvalidOperationMsg(msg, msgBufLen, msg5);
         }
       }
     }
@@ -1108,14 +1079,14 @@ int32_t validateSqlNode(SSqlNode* pSqlNode, SQueryStmtInfo* pQueryInfo, char* ms
     }
 
     if (!hasTimestampForPointInterpQuery(pQueryInfo)) {
-      return parserSetInvalidOperatorMsg(msg, msgBufLen, msg1);
+      return buildInvalidOperationMsg(msg, msgBufLen, msg1);
     }
 
     // in case of join query, time range is required.
     if (QUERY_IS_JOIN_QUERY(pQueryInfo->type)) {
       uint64_t timeRange = (uint64_t)pQueryInfo->window.ekey - pQueryInfo->window.skey;
       if (timeRange == 0 && pQueryInfo->window.skey == 0) {
-        return parserSetInvalidOperatorMsg(msg, msgBufLen, msg3);
+        return buildInvalidOperationMsg(msg, msgBufLen, msg3);
       }
     }
 
@@ -1160,6 +1131,119 @@ int32_t validateSqlNode(SSqlNode* pSqlNode, SQueryStmtInfo* pQueryInfo, char* ms
   return TSDB_CODE_SUCCESS;  // Does not build query message here
 }
 
+int32_t validateSelectNodeList(SQueryStmtInfo* pQueryInfo, SArray* pSelNodeList, bool joinQuery, bool timeWindowQuery, bool outerQuery, char* msg, int32_t msgBufLen) {
+  assert(pSelNodeList != NULL);
+
+  const char* msg1 = "too many items in selection clause";
+  const char* msg2 = "functions or others can not be mixed up";
+  const char* msg3 = "not support query expression";
+  const char* msg4 = "not support distinct mixed with proj/agg func";
+  const char* msg5 = "invalid function name";
+  const char* msg6 = "not support distinct mixed with join";
+  const char* msg7 = "not support distinct mixed with groupby";
+  const char* msg8 = "not support distinct in nest query";
+  const char* msg9 = "_block_dist not support subquery, only support stable/table";
+
+  // too many result columns not support order by in query
+  if (taosArrayGetSize(pSelNodeList) > TSDB_MAX_COLUMNS) {
+    return buildInvalidOperationMsg(msg, msgBufLen, msg1);
+  }
+
+  if (pQueryInfo->colList == NULL) {
+    pQueryInfo->colList = taosArrayInit(4, POINTER_BYTES);
+  }
+
+
+  bool hasDistinct = false;
+  bool hasAgg      = false;
+  size_t numOfExpr = taosArrayGetSize(pSelNodeList);
+  int32_t distIdx = -1;
+  for (int32_t i = 0; i < numOfExpr; ++i) {
+    int32_t outputIndex = (int32_t)tscNumOfExprs(pQueryInfo);
+    tSqlExprItem* pItem = taosArrayGet(pSelNodeList, i);
+    if (hasDistinct == false) {
+      hasDistinct = (pItem->distinct == true);
+      distIdx     =  hasDistinct ? i : -1;
+    }
+
+    int32_t type = pItem->pNode->type;
+    if (type == SQL_NODE_SQLFUNCTION) {
+      hasAgg = true;
+      if (hasDistinct)  break;
+
+      pItem->functionId = qIsBuiltinFunction(pItem->pNode->Expr.operand.z, pItem->pNode->Expr.operand.n);
+
+      if (pItem->pNode->functionId == TSDB_FUNC_BLKINFO && taosArrayGetSize(pQueryInfo->pUpstream) > 0) {
+        return buildInvalidOperationMsg(msg, msgBufLen, msg9);
+      }
+
+      SUdfInfo* pUdfInfo = NULL;
+      if (pItem->pNode->functionId < 0) {
+        pUdfInfo = isValidUdf(pQueryInfo->pUdfInfo, pItem->pNode->Expr.operand.z, pItem->pNode->Expr.operand.n);
+        if (pUdfInfo == NULL) {
+          return buildInvalidOperationMsg(msg, msgBufLen, msg5);
+        }
+
+        pItem->pNode->functionId = pUdfInfo->functionId;
+      }
+
+      // sql function in selection clause, append sql function info in pSqlCmd structure sequentially
+      if (addExprAndResultField(pCmd, pQueryInfo, outputIndex, pItem, true, pUdfInfo) != TSDB_CODE_SUCCESS) {
+        return TSDB_CODE_TSC_INVALID_OPERATION;
+      }
+    } else if (type == SQL_NODE_TABLE_COLUMN || type == SQL_NODE_VALUE) {
+      // use the dynamic array list to decide if the function is valid or not
+      // select table_name1.field_name1, table_name2.field_name2  from table_name1, table_name2
+      if (addProjectionExprAndResultField(pCmd, pQueryInfo, pItem, outerQuery) != TSDB_CODE_SUCCESS) {
+        return TSDB_CODE_TSC_INVALID_OPERATION;
+      }
+    } else if (type == SQL_NODE_EXPR) {
+      int32_t code = handleArithmeticExpr(pCmd, pQueryInfo, i, pItem);
+      if (code != TSDB_CODE_SUCCESS) {
+        return code;
+      }
+    } else {
+      return buildInvalidOperationMsg(msg, msgBufLen, msg3);
+    }
+
+    if (pQueryInfo->fieldsInfo.numOfOutput > TSDB_MAX_COLUMNS) {
+      return buildInvalidOperationMsg(msg, msgBufLen, msg1);
+    }
+  }
+
+  //TODO(dengyihao), refactor as function
+  //handle distinct func mixed with other func
+  if (hasDistinct == true) {
+    if (distIdx != 0 || hasAgg) {
+      return buildInvalidOperationMsg(msg, msgBufLen, msg4);
+    }
+    if (joinQuery) {
+      return buildInvalidOperationMsg(msg, msgBufLen, msg6);
+    }
+    if (pQueryInfo->groupbyExpr.numOfGroupCols  != 0) {
+      return buildInvalidOperationMsg(msg, msgBufLen, msg7);
+    }
+    if (pQueryInfo->pDownstream != NULL) {
+      return buildInvalidOperationMsg(msg, msgBufLen, msg8);
+    }
+
+    pQueryInfo->distinct = true;
+  }
+
+
+  // there is only one user-defined column in the final result field, add the timestamp column.
+  size_t numOfSrcCols = taosArrayGetSize(pQueryInfo->colList);
+  if ((numOfSrcCols <= 0 || !hasNoneUserDefineExpr(pQueryInfo)) && !tscQueryTags(pQueryInfo) && !tscQueryBlockInfo(pQueryInfo)) {
+    addPrimaryTsColIntoResult(pQueryInfo, pCmd);
+  }
+
+  if (!functionCompatibleCheck(pQueryInfo, joinQuery, timeWindowQuery)) {
+    return buildInvalidOperationMsg(msg, msgBufLen, msg2);
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
 int32_t evaluateSqlNode(SSqlNode* pNode, int32_t tsPrecision, char* msg, int32_t msgBufLen) {
   assert(pNode != NULL && msg != NULL && msgBufLen > 0);
   if (pNode->pWhere == NULL) {

source/libs/parser/src/parser.c

@@ -17,10 +17,17 @@
 #include "parserInt.h"
 #include "parserUtil.h"
 #include "ttoken.h"
-#include "executor.h"
+#include "function.h"
 
 bool qIsInsertSql(const char* pStr, size_t length) {
-  return false;
+  int32_t index = 0;
+
+  do {
+    SToken t0 = tStrGetToken(pStr, &index, false);
+    if (t0.type != TK_LP) {
+      return t0.type == TK_INSERT || t0.type == TK_IMPORT;
+    }
+  } while (1);
 }
 
 int32_t qParseQuerySql(const char* pStr, size_t length, struct SQueryStmtInfo** pQueryInfo, int64_t id, char* msg, int32_t msgLen) {
@@ -110,12 +117,12 @@ int32_t getTableNameFromSqlNode(SSqlNode* pSqlNode, SArray* tableNameList, char*
 
     SToken* t = &item->tableName;
     if (t->type == TK_INTEGER || t->type == TK_FLOAT || t->type == TK_STRING) {
-      return parserSetInvalidOperatorMsg(msg, msgBufLen, msg1);
+      return buildInvalidOperationMsg(msg, msgBufLen, msg1);
     }
 
 //    tscDequoteAndTrimToken(t);
     if (parserValidateIdToken(t) != TSDB_CODE_SUCCESS) {
-      return parserSetInvalidOperatorMsg(msg, msgBufLen, msg1);
+      return buildInvalidOperationMsg(msg, msgBufLen, msg1);
     }
 
     SName name = {0};
@@ -144,7 +151,7 @@ int32_t qParserExtractRequestedMetaInfo(const SSqlInfo* pSqlInfo, SMetaReq* pMet
   for (int32_t i = 0; i < size; ++i) {
     SSqlNode* pSqlNode = taosArrayGetP(pSqlInfo->list, i);
     if (pSqlNode->from == NULL) {
-      return parserSetInvalidOperatorMsg(msg, msgBufLen, "invalid from clause");
+      return buildInvalidOperationMsg(msg, msgBufLen, "invalid from clause");
     }
 
     // load the table meta in the FROM clause
@@ -179,7 +186,7 @@ int32_t qParserExtractRequestedMetaInfo(const SSqlInfo* pSqlInfo, SMetaReq* pMet
       }
 
       // Let's assume that it is an UDF/UDAF, if it is not a built-in function.
-      if (!isBuiltinFunction(t->z, t->n)) {
+      if (!qIsBuiltinFunction(t->z, t->n)) {
         char* fname = strndup(t->z, t->n);
         taosArrayPush(pMetaInfo->pUdf, &fname);
       }

source/libs/parser/src/parserUtil.c

+#include "taosmsg.h"
+#include "parser.h"
 #include "parserUtil.h"
 #include "taoserror.h"
 #include "tutil.h"
+#include "ttypes.h"
+#include "thash.h"
+#include "tbuffer.h"
+#include "parserInt.h"
+#include "queryInfoUtil.h"
+#include "function.h"
+
+typedef struct STableFilterCond {
+  uint64_t uid;
+  int16_t  idx;  //table index
+  int32_t  len;  // length of tag query condition data
+  char *   cond;
+} STableFilterCond;
+
+static STableMetaInfo* addTableMetaInfo(SQueryStmtInfo* pQueryInfo, SName* name, STableMeta* pTableMeta,
+                                        SVgroupsInfo* vgroupList, SArray* pTagCols, SArray* pVgroupTables);
+STableMeta* tableMetaDup(STableMeta* pTableMeta);
 
 int32_t parserValidateIdToken(SToken* pToken) {
   if (pToken == NULL || pToken->z == NULL || pToken->type != TK_ID) {
@@ -59,7 +78,7 @@ int32_t parserValidateIdToken(SToken* pToken) {
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t parserSetInvalidOperatorMsg(char* dst, int32_t dstBufLen, const char* msg) {
+int32_t buildInvalidOperationMsg(char* dst, int32_t dstBufLen, const char* msg) {
   strncpy(dst, msg, dstBufLen);
   return TSDB_CODE_TSC_INVALID_OPERATION;
 }
@@ -88,3 +107,1762 @@ int32_t parserSetSyntaxErrMsg(char* dst, int32_t dstBufLen, const char* addition
 
   return TSDB_CODE_TSC_SQL_SYNTAX_ERROR;
 }
+
+SCond* getSTableQueryCond(STagCond* pTagCond, uint64_t uid) {
+  if (pTagCond->pCond == NULL) {
+    return NULL;
+  }
+
+  size_t size = taosArrayGetSize(pTagCond->pCond);
+  for (int32_t i = 0; i < size; ++i) {
+    SCond* pCond = taosArrayGet(pTagCond->pCond, i);
+
+    if (uid == pCond->uid) {
+      return pCond;
+    }
+  }
+
+  return NULL;
+}
+
+STableFilterCond* tsGetTableFilter(SArray* filters, uint64_t uid, int16_t idx) {
+  if (filters == NULL) {
+    return NULL;
+  }
+
+  size_t size = taosArrayGetSize(filters);
+  for (int32_t i = 0; i < size; ++i) {
+    STableFilterCond* cond = taosArrayGet(filters, i);
+
+    if (uid == cond->uid && (idx >= 0 && cond->idx == idx)) {
+      return cond;
+    }
+  }
+
+  return NULL;
+}
+
+void setSTableQueryCond(STagCond* pTagCond, uint64_t uid, SBufferWriter* bw) {
+  if (tbufTell(bw) == 0) {
+    return;
+  }
+
+  SCond cond = {
+      .uid = uid,
+      .len = (int32_t)(tbufTell(bw)),
+      .cond = NULL,
+  };
+
+  cond.cond = tbufGetData(bw, true);
+
+  if (pTagCond->pCond == NULL) {
+    pTagCond->pCond = taosArrayInit(3, sizeof(SCond));
+  }
+
+  taosArrayPush(pTagCond->pCond, &cond);
+}
+
+//typedef struct SJoinStatus {
+//  SSDataBlock* pBlock;   // point to the upstream block
+//  int32_t      index;
+//  bool         completed;// current upstream is completed or not
+//} SJoinStatus;
+
+/*
+static void createInputDataFilterInfo(SQueryStmtInfo* px, int32_t numOfCol1, int32_t* numOfFilterCols, SSingleColumnFilterInfo** pFilterInfo) {
+  SColumnInfo* tableCols = calloc(numOfCol1, sizeof(SColumnInfo));
+  for(int32_t i = 0; i < numOfCol1; ++i) {
+    SColumn* pCol = taosArrayGetP(px->colList, i);
+    if (pCol->info.flist.numOfFilters > 0) {
+      (*numOfFilterCols) += 1;
+    }
+
+    tableCols[i] = pCol->info;
+  }
+
+  if ((*numOfFilterCols) > 0) {
+    doCreateFilterInfo(tableCols, numOfCol1, (*numOfFilterCols), pFilterInfo, 0);
+  }
+
+  tfree(tableCols);
+}
+*/
+
+//void destroyTableNameList(SInsertStatementParam* pInsertParam) {
+//  if (pInsertParam->numOfTables == 0) {
+//    assert(pInsertParam->pTableNameList == NULL);
+//    return;
+//  }
+//
+//  for(int32_t i = 0; i < pInsertParam->numOfTables; ++i) {
+//    tfree(pInsertParam->pTableNameList[i]);
+//  }
+//
+//  pInsertParam->numOfTables = 0;
+//  tfree(pInsertParam->pTableNameList);
+//}
+
+//void tscDestroyBoundColumnInfo(SParsedDataColInfo* pColInfo) {
+//  tfree(pColInfo->boundedColumns);
+//  tfree(pColInfo->cols);
+//  tfree(pColInfo->colIdxInfo);
+//}
+//
+//void tscDestroyDataBlock(STableDataBlocks* pDataBlock, bool removeMeta) {
+//  if (pDataBlock == NULL) {
+//    return;
+//  }
+//
+//  tfree(pDataBlock->pData);
+//
+//  if (removeMeta) {
+//    char name[TSDB_TABLE_FNAME_LEN] = {0};
+//    tNameExtractFullName(&pDataBlock->tableName, name);
+//
+//    taosHashRemove(tscTableMetaMap, name, strnlen(name, TSDB_TABLE_FNAME_LEN));
+//  }
+//
+//  if (!pDataBlock->cloned) {
+//    tfree(pDataBlock->params);
+//
+//    // free the refcount for metermeta
+//    if (pDataBlock->pTableMeta != NULL) {
+//      tfree(pDataBlock->pTableMeta);
+//    }
+//
+//    tscDestroyBoundColumnInfo(&pDataBlock->boundColumnInfo);
+//  }
+//
+//  tfree(pDataBlock);
+//}
+
+//SParamInfo* tscAddParamToDataBlock(STableDataBlocks* pDataBlock, char type, uint8_t timePrec, int16_t bytes,
+//                                   uint32_t offset) {
+//  uint32_t needed = pDataBlock->numOfParams + 1;
+//  if (needed > pDataBlock->numOfAllocedParams) {
+//    needed *= 2;
+//    void* tmp = realloc(pDataBlock->params, needed * sizeof(SParamInfo));
+//    if (tmp == NULL) {
+//      return NULL;
+//    }
+//    pDataBlock->params = (SParamInfo*)tmp;
+//    pDataBlock->numOfAllocedParams = needed;
+//  }
+//
+//  SParamInfo* param = pDataBlock->params + pDataBlock->numOfParams;
+//  param->idx = -1;
+//  param->type = type;
+//  param->timePrec = timePrec;
+//  param->bytes = bytes;
+//  param->offset = offset;
+//
+//  ++pDataBlock->numOfParams;
+//  return param;
+//}
+
+//void*  tscDestroyBlockArrayList(SArray* pDataBlockList) {
+//  if (pDataBlockList == NULL) {
+//    return NULL;
+//  }
+//
+//  size_t size = taosArrayGetSize(pDataBlockList);
+//  for (int32_t i = 0; i < size; i++) {
+//    void* d = taosArrayGetP(pDataBlockList, i);
+//    tscDestroyDataBlock(d, false);
+//  }
+//
+//  taosArrayDestroy(pDataBlockList);
+//  return NULL;
+//}
+
+
+//void freeUdfInfo(SUdfInfo* pUdfInfo) {
+//  if (pUdfInfo == NULL) {
+//    return;
+//  }
+//
+//  if (pUdfInfo->funcs[TSDB_UDF_FUNC_DESTROY]) {
+//    (*(udfDestroyFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_DESTROY])(&pUdfInfo->init);
+//  }
+//
+//  tfree(pUdfInfo->name);
+//
+//  if (pUdfInfo->path) {
+//    unlink(pUdfInfo->path);
+//  }
+//
+//  tfree(pUdfInfo->path);
+//
+//  tfree(pUdfInfo->content);
+//
+//  taosCloseDll(pUdfInfo->handle);
+//}
+
+//void*  tscDestroyUdfArrayList(SArray* pUdfList) {
+//  if (pUdfList == NULL) {
+//    return NULL;
+//  }
+//
+//  size_t size = taosArrayGetSize(pUdfList);
+//  for (int32_t i = 0; i < size; i++) {
+//    SUdfInfo* udf = taosArrayGet(pUdfList, i);
+//    freeUdfInfo(udf);
+//  }
+//
+//  taosArrayDestroy(pUdfList);
+//  return NULL;
+//}
+
+//void* tscDestroyBlockHashTable(SHashObj* pBlockHashTable, bool removeMeta) {
+//  if (pBlockHashTable == NULL) {
+//    return NULL;
+//  }
+//
+//  STableDataBlocks** p = taosHashIterate(pBlockHashTable, NULL);
+//  while(p) {
+//    tscDestroyDataBlock(*p, removeMeta);
+//    p = taosHashIterate(pBlockHashTable, p);
+//  }
+//
+//  taosHashCleanup(pBlockHashTable);
+//  return NULL;
+//}
+
+/**
+ * create the in-memory buffer for each table to keep the submitted data block
+ * @param initialSize
+ * @param rowSize
+ * @param startOffset
+ * @param name
+ * @param dataBlocks
+ * @return
+ */
+//int32_t tscCreateDataBlock(size_t defaultSize, int32_t rowSize, int32_t startOffset, SName* name,
+//                           STableMeta* pTableMeta, STableDataBlocks** dataBlocks) {
+//  STableDataBlocks* dataBuf = (STableDataBlocks*)calloc(1, sizeof(STableDataBlocks));
+//  if (dataBuf == NULL) {
+//    tscError("failed to allocated memory, reason:%s", strerror(errno));
+//    return TSDB_CODE_TSC_OUT_OF_MEMORY;
+//  }
+//
+//  dataBuf->nAllocSize = (uint32_t)defaultSize;
+//  dataBuf->headerSize = startOffset;
+//
+//  // the header size will always be the startOffset value, reserved for the subumit block header
+//  if (dataBuf->nAllocSize <= dataBuf->headerSize) {
+//    dataBuf->nAllocSize = dataBuf->headerSize * 2;
+//  }
+//
+//  //dataBuf->pData = calloc(1, dataBuf->nAllocSize);
+//  dataBuf->pData = malloc(dataBuf->nAllocSize);
+//  if (dataBuf->pData == NULL) {
+//    tscError("failed to allocated memory, reason:%s", strerror(errno));
+//    tfree(dataBuf);
+//    return TSDB_CODE_TSC_OUT_OF_MEMORY;
+//  }
+//  memset(dataBuf->pData, 0, sizeof(SSubmitBlk));
+//
+//  //Here we keep the tableMeta to avoid it to be remove by other threads.
+//  dataBuf->pTableMeta = tscTableMetaDup(pTableMeta);
+//
+//  SParsedDataColInfo* pColInfo = &dataBuf->boundColumnInfo;
+//  SSchema* pSchema = getTableColumnSchema(dataBuf->pTableMeta);
+//  tscSetBoundColumnInfo(pColInfo, pSchema, dataBuf->pTableMeta->tableInfo.numOfColumns);
+//
+//  dataBuf->ordered  = true;
+//  dataBuf->prevTS   = INT64_MIN;
+//  dataBuf->rowSize  = rowSize;
+//  dataBuf->size     = startOffset;
+//  dataBuf->tsSource = -1;
+//  dataBuf->vgId     = dataBuf->pTableMeta->vgId;
+//
+//  tNameAssign(&dataBuf->tableName, name);
+//
+//  assert(defaultSize > 0 && pTableMeta != NULL && dataBuf->pTableMeta != NULL);
+//
+//  *dataBlocks = dataBuf;
+//  return TSDB_CODE_SUCCESS;
+//}
+//
+//int32_t tscGetDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, int32_t startOffset, int32_t rowSize,
+//                                SName* name, STableMeta* pTableMeta, STableDataBlocks** dataBlocks,
+//                                SArray* pBlockList) {
+//  *dataBlocks = NULL;
+//  STableDataBlocks** t1 = (STableDataBlocks**)taosHashGet(pHashList, (const char*)&id, sizeof(id));
+//  if (t1 != NULL) {
+//    *dataBlocks = *t1;
+//  }
+//
+//  if (*dataBlocks == NULL) {
+//    int32_t ret = tscCreateDataBlock((size_t)size, rowSize, startOffset, name, pTableMeta, dataBlocks);
+//    if (ret != TSDB_CODE_SUCCESS) {
+//      return ret;
+//    }
+//
+//    taosHashPut(pHashList, (const char*)&id, sizeof(int64_t), (char*)dataBlocks, POINTER_BYTES);
+//    if (pBlockList) {
+//      taosArrayPush(pBlockList, dataBlocks);
+//    }
+//  }
+//
+//  return TSDB_CODE_SUCCESS;
+//}
+//
+//// Erase the empty space reserved for binary data
+//static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, SInsertStatementParam* insertParam,
+//                         SBlockKeyTuple* blkKeyTuple) {
+//  // TODO: optimize this function, handle the case while binary is not presented
+//  STableMeta*     pTableMeta = pTableDataBlock->pTableMeta;
+//  STableComInfo   tinfo = tscGetTableInfo(pTableMeta);
+//  SSchema*        pSchema = getTableColumnSchema(pTableMeta);
+//
+//  SSubmitBlk* pBlock = pDataBlock;
+//  memcpy(pDataBlock, pTableDataBlock->pData, sizeof(SSubmitBlk));
+//  pDataBlock = (char*)pDataBlock + sizeof(SSubmitBlk);
+//
+//  int32_t flen = 0;  // original total length of row
+//
+//  // schema needs to be included into the submit data block
+//  if (insertParam->schemaAttached) {
+//    int32_t numOfCols = tscGetNumOfColumns(pTableDataBlock->pTableMeta);
+//    for(int32_t j = 0; j < numOfCols; ++j) {
+//      STColumn* pCol = (STColumn*) pDataBlock;
+//      pCol->colId = htons(pSchema[j].colId);
+//      pCol->type  = pSchema[j].type;
+//      pCol->bytes = htons(pSchema[j].bytes);
+//      pCol->offset = 0;
+//
+//      pDataBlock = (char*)pDataBlock + sizeof(STColumn);
+//      flen += TYPE_BYTES[pSchema[j].type];
+//    }
+//
+//    int32_t schemaSize = sizeof(STColumn) * numOfCols;
+//    pBlock->schemaLen = schemaSize;
+//  } else {
+//    if (IS_RAW_PAYLOAD(insertParam->payloadType)) {
+//      for (int32_t j = 0; j < tinfo.numOfColumns; ++j) {
+//        flen += TYPE_BYTES[pSchema[j].type];
+//      }
+//    }
+//    pBlock->schemaLen = 0;
+//  }
+//
+//  char* p = pTableDataBlock->pData + sizeof(SSubmitBlk);
+//  pBlock->dataLen = 0;
+//  int32_t numOfRows = htons(pBlock->numOfRows);
+//
+//  if (IS_RAW_PAYLOAD(insertParam->payloadType)) {
+//    for (int32_t i = 0; i < numOfRows; ++i) {
+//      SMemRow memRow = (SMemRow)pDataBlock;
+//      memRowSetType(memRow, SMEM_ROW_DATA);
+//      SDataRow trow = memRowDataBody(memRow);
+//      dataRowSetLen(trow, (uint16_t)(TD_DATA_ROW_HEAD_SIZE + flen));
+//      dataRowSetVersion(trow, pTableMeta->sversion);
+//
+//      int toffset = 0;
+//      for (int32_t j = 0; j < tinfo.numOfColumns; j++) {
+//        tdAppendColVal(trow, p, pSchema[j].type, toffset);
+//        toffset += TYPE_BYTES[pSchema[j].type];
+//        p += pSchema[j].bytes;
+//      }
+//
+//      pDataBlock = (char*)pDataBlock + memRowTLen(memRow);
+//      pBlock->dataLen += memRowTLen(memRow);
+//    }
+//  } else {
+//    for (int32_t i = 0; i < numOfRows; ++i) {
+//      char* payload = (blkKeyTuple + i)->payloadAddr;
+//      if (isNeedConvertRow(payload)) {
+//        convertSMemRow(pDataBlock, payload, pTableDataBlock);
+//        TDRowTLenT rowTLen = memRowTLen(pDataBlock);
+//        pDataBlock = POINTER_SHIFT(pDataBlock, rowTLen);
+//        pBlock->dataLen += rowTLen;
+//      } else {
+//        TDRowTLenT rowTLen = memRowTLen(payload);
+//        memcpy(pDataBlock, payload, rowTLen);
+//        pDataBlock = POINTER_SHIFT(pDataBlock, rowTLen);
+//        pBlock->dataLen += rowTLen;
+//      }
+//    }
+//  }
+//
+//  int32_t len = pBlock->dataLen + pBlock->schemaLen;
+//  pBlock->dataLen = htonl(pBlock->dataLen);
+//  pBlock->schemaLen = htonl(pBlock->schemaLen);
+//
+//  return len;
+//}
+
+TAOS_FIELD createField(int8_t type, const char* name, int16_t bytes) {
+  TAOS_FIELD f = { .type = type, .bytes = bytes, };
+  tstrncpy(f.name, name, sizeof(f.name));
+  return f;
+}
+
+int32_t getFirstInvisibleFieldPos(SQueryStmtInfo* pQueryInfo) {
+  if (pQueryInfo->fieldsInfo.numOfOutput <= 0 || pQueryInfo->fieldsInfo.internalField == NULL) {
+    return 0;
+  }
+
+  for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) {
+    SInternalField* pField = taosArrayGet(pQueryInfo->fieldsInfo.internalField, i);
+    if (!pField->visible) {
+      return i;
+    }
+  }
+
+  return pQueryInfo->fieldsInfo.numOfOutput;
+}
+
+SInternalField* appendFieldInfo(SFieldInfo* pFieldInfo, TAOS_FIELD* pField) {
+  assert(pFieldInfo != NULL);
+  pFieldInfo->numOfOutput++;
+
+  struct SInternalField info = { .pExpr = NULL, .visible = true };
+
+  info.field = *pField;
+  return taosArrayPush(pFieldInfo->internalField, &info);
+}
+
+SInternalField* insertFieldInfo(SFieldInfo* pFieldInfo, int32_t index, TAOS_FIELD* field) {
+  pFieldInfo->numOfOutput++;
+  struct SInternalField info = { .pExpr = NULL, .visible = true };
+
+  info.field = *field;
+  return taosArrayInsert(pFieldInfo->internalField, index, &info);
+}
+
+void fieldInfoUpdateOffset(SQueryStmtInfo* pQueryInfo) {
+  int32_t offset = 0;
+  size_t numOfExprs = getNumOfExprs(pQueryInfo);
+
+  for (int32_t i = 0; i < numOfExprs; ++i) {
+    SExprInfo* p = taosArrayGetP(pQueryInfo->exprList, i);
+
+    p->base.offset = offset;
+    offset += p->base.resBytes;
+  }
+}
+
+SInternalField* fieldInfoGetInternalField(SFieldInfo* pFieldInfo, int32_t index) {
+  assert(index < pFieldInfo->numOfOutput);
+  return TARRAY_GET_ELEM(pFieldInfo->internalField, index);
+}
+
+TAOS_FIELD* getFieldInfo(SFieldInfo* pFieldInfo, int32_t index) {
+  assert(index < pFieldInfo->numOfOutput);
+  return &((SInternalField*)TARRAY_GET_ELEM(pFieldInfo->internalField, index))->field;
+}
+
+int16_t getFieldInfoOffset(SQueryStmtInfo* pQueryInfo, int32_t index) {
+  SInternalField* pInfo = fieldInfoGetInternalField(&pQueryInfo->fieldsInfo, index);
+  assert(pInfo != NULL && pInfo->pExpr->pExpr == NULL);
+
+  return pInfo->pExpr->base.offset;
+}
+
+int32_t fieldInfoCompare(const SFieldInfo* pFieldInfo1, const SFieldInfo* pFieldInfo2, int32_t *diffSize) {
+  assert(pFieldInfo1 != NULL && pFieldInfo2 != NULL);
+
+  if (pFieldInfo1->numOfOutput != pFieldInfo2->numOfOutput) {
+    return pFieldInfo1->numOfOutput - pFieldInfo2->numOfOutput;
+  }
+
+  for (int32_t i = 0; i < pFieldInfo1->numOfOutput; ++i) {
+    TAOS_FIELD* pField1 = getFieldInfo((SFieldInfo*) pFieldInfo1, i);
+    TAOS_FIELD* pField2 = getFieldInfo((SFieldInfo*) pFieldInfo2, i);
+
+    if (pField1->type != pField2->type ||
+        strcasecmp(pField1->name, pField2->name) != 0) {
+      return 1;
+    }
+
+    if (pField1->bytes != pField2->bytes) {
+      *diffSize = 1;
+
+      if (pField2->bytes > pField1->bytes) {
+        assert(IS_VAR_DATA_TYPE(pField1->type));
+        pField1->bytes = pField2->bytes;
+      }
+    }
+  }
+
+  return 0;
+}
+
+int32_t getFieldInfoSize(const SFieldInfo* pFieldInfo1, const SFieldInfo* pFieldInfo2) {
+  assert(pFieldInfo1 != NULL && pFieldInfo2 != NULL);
+
+  for (int32_t i = 0; i < pFieldInfo1->numOfOutput; ++i) {
+    TAOS_FIELD* pField1 = getFieldInfo((SFieldInfo*) pFieldInfo1, i);
+    TAOS_FIELD* pField2 = getFieldInfo((SFieldInfo*) pFieldInfo2, i);
+
+    pField2->bytes = pField1->bytes;
+  }
+
+  return 0;
+}
+
+static void destroyFilterInfo(SColumnFilterList* pFilterList) {
+  if (pFilterList->filterInfo == NULL) {
+    pFilterList->numOfFilters = 0;
+    return;
+  }
+
+  for(int32_t i = 0; i < pFilterList->numOfFilters; ++i) {
+    if (pFilterList->filterInfo[i].filterstr) {
+      tfree(pFilterList->filterInfo[i].pz);
+    }
+  }
+
+  tfree(pFilterList->filterInfo);
+  pFilterList->numOfFilters = 0;
+}
+
+void cleanupFieldInfo(SFieldInfo* pFieldInfo) {
+  if (pFieldInfo == NULL) {
+    return;
+  }
+
+  if (pFieldInfo->internalField != NULL) {
+    size_t num = taosArrayGetSize(pFieldInfo->internalField);
+    for (int32_t i = 0; i < num; ++i) {
+//      SInternalField* pfield = taosArrayGet(pFieldInfo->internalField, i);
+//      if (pfield->pExpr != NULL && pfield->pExpr->pExpr != NULL) {
+//        sqlExprDestroy(pfield->pExpr);
+//      }
+    }
+  }
+
+  taosArrayDestroy(pFieldInfo->internalField);
+//  tfree(pFieldInfo->final);
+
+  memset(pFieldInfo, 0, sizeof(SFieldInfo));
+}
+
+void copyFieldInfo(SFieldInfo* pFieldInfo, const SFieldInfo* pSrc, const SArray* pExprList) {
+  assert(pFieldInfo != NULL && pSrc != NULL && pExprList != NULL);
+  pFieldInfo->numOfOutput = pSrc->numOfOutput;
+
+  if (pSrc->final != NULL) {
+    pFieldInfo->final = calloc(pSrc->numOfOutput, sizeof(TAOS_FIELD));
+    memcpy(pFieldInfo->final, pSrc->final, sizeof(TAOS_FIELD) * pSrc->numOfOutput);
+  }
+
+  if (pSrc->internalField != NULL) {
+    size_t num = taosArrayGetSize(pSrc->internalField);
+    size_t numOfExpr = taosArrayGetSize(pExprList);
+
+    for (int32_t i = 0; i < num; ++i) {
+      SInternalField* pfield = taosArrayGet(pSrc->internalField, i);
+
+      SInternalField p = {.visible = pfield->visible, .field = pfield->field};
+
+      bool found = false;
+      int32_t resColId = pfield->pExpr->base.resColId;
+      for(int32_t j = 0; j < numOfExpr; ++j) {
+        SExprInfo* pExpr = taosArrayGetP(pExprList, j);
+        if (pExpr->base.resColId == resColId) {
+          p.pExpr = pExpr;
+          found = true;
+          break;
+        }
+      }
+
+      if (!found) {
+        assert(pfield->pExpr->pExpr != NULL);
+        p.pExpr = calloc(1, sizeof(SExprInfo));
+        assignExprInfo(p.pExpr, pfield->pExpr);
+      }
+
+      taosArrayPush(pFieldInfo->internalField, &p);
+    }
+  }
+}
+
+// ignore the tbname columnIndex to be inserted into source list
+int32_t columnExists(SArray* pColumnList, int32_t columnId, uint64_t uid) {
+  size_t numOfCols = taosArrayGetSize(pColumnList);
+
+  int32_t i = 0;
+  while (i < numOfCols) {
+    SColumn* pCol = taosArrayGetP(pColumnList, i);
+    if ((pCol->info.colId != columnId) || (pCol->tableUid != uid)) {
+      ++i;
+      continue;
+    } else {
+      break;
+    }
+  }
+
+  if (i >= numOfCols || numOfCols == 0) {
+    return -1;
+  }
+
+  return i;
+}
+
+SColumn* columnListInsert(SArray* pColumnList, int32_t columnIndex, uint64_t uid, SSchema* pSchema) {
+  // ignore the tbname columnIndex to be inserted into source list
+  if (columnIndex < 0) {
+    return NULL;
+  }
+
+  size_t numOfCols = taosArrayGetSize(pColumnList);
+
+  int32_t i = 0;
+  while (i < numOfCols) {
+    SColumn* pCol = taosArrayGetP(pColumnList, i);
+    if (pCol->columnIndex < columnIndex) {
+      i++;
+    } else if (pCol->tableUid < uid) {
+      i++;
+    } else {
+      break;
+    }
+  }
+
+  if (i >= numOfCols || numOfCols == 0) {
+    SColumn* b = calloc(1, sizeof(SColumn));
+    if (b == NULL) {
+      return NULL;
+    }
+
+    b->columnIndex = columnIndex;
+    b->tableUid    = uid;
+    b->info.colId  = pSchema->colId;
+    b->info.bytes  = pSchema->bytes;
+    b->info.type   = pSchema->type;
+
+    taosArrayInsert(pColumnList, i, &b);
+  } else {
+    SColumn* pCol = taosArrayGetP(pColumnList, i);
+
+    if (i < numOfCols && (pCol->columnIndex > columnIndex || pCol->tableUid != uid)) {
+      SColumn* b = calloc(1, sizeof(SColumn));
+      if (b == NULL) {
+        return NULL;
+      }
+
+      b->columnIndex = columnIndex;
+      b->tableUid    = uid;
+      b->info.colId = pSchema->colId;
+      b->info.bytes = pSchema->bytes;
+      b->info.type  = pSchema->type;
+
+      taosArrayInsert(pColumnList, i, &b);
+    }
+  }
+
+  return taosArrayGetP(pColumnList, i);
+}
+
+void columnCopy(SColumn* pDest, const SColumn* pSrc);
+
+SColumn* columnClone(const SColumn* src) {
+  assert(src != NULL);
+
+  SColumn* dst = calloc(1, sizeof(SColumn));
+  if (dst == NULL) {
+    return NULL;
+  }
+
+  columnCopy(dst, src);
+  return dst;
+}
+
+void columnCopy(SColumn* pDest, const SColumn* pSrc) {
+  destroyFilterInfo(&pDest->info.flist);
+
+  pDest->columnIndex       = pSrc->columnIndex;
+  pDest->tableUid          = pSrc->tableUid;
+  pDest->info.flist.numOfFilters = pSrc->info.flist.numOfFilters;
+  pDest->info.flist.filterInfo   = tFilterInfoDup(pSrc->info.flist.filterInfo, pSrc->info.flist.numOfFilters);
+  pDest->info.type        = pSrc->info.type;
+  pDest->info.colId        = pSrc->info.colId;
+  pDest->info.bytes      = pSrc->info.bytes;
+}
+
+void columnListCopy(SArray* dst, const SArray* src, uint64_t tableUid) {
+  assert(src != NULL && dst != NULL);
+
+  size_t num = taosArrayGetSize(src);
+  for (int32_t i = 0; i < num; ++i) {
+    SColumn* pCol = taosArrayGetP(src, i);
+
+    if (pCol->tableUid == tableUid) {
+      SColumn* p = columnClone(pCol);
+      taosArrayPush(dst, &p);
+    }
+  }
+}
+
+void columnListCopyAll(SArray* dst, const SArray* src) {
+  assert(src != NULL && dst != NULL);
+
+  size_t num = taosArrayGetSize(src);
+  for (int32_t i = 0; i < num; ++i) {
+    SColumn* pCol = taosArrayGetP(src, i);
+    SColumn* p = columnClone(pCol);
+    taosArrayPush(dst, &p);
+  }
+}
+
+static void columnDestroy(SColumn* pCol) {
+  destroyFilterInfo(&pCol->info.flist);
+  free(pCol);
+}
+
+void columnListDestroy(SArray* pColumnList) {
+  if (pColumnList == NULL) {
+    return;
+  }
+
+  size_t num = taosArrayGetSize(pColumnList);
+  for (int32_t i = 0; i < num; ++i) {
+    SColumn* pCol = taosArrayGetP(pColumnList, i);
+    columnDestroy(pCol);
+  }
+
+  taosArrayDestroy(pColumnList);
+}
+
+bool validateColumnId(STableMetaInfo* pTableMetaInfo, int32_t colId, int32_t numOfParams) {
+  if (pTableMetaInfo->pTableMeta == NULL) {
+    return false;
+  }
+
+  if (colId == TSDB_TBNAME_COLUMN_INDEX || (colId <= TSDB_UD_COLUMN_INDEX && numOfParams == 2)) {
+    return true;
+  }
+
+  SSchema* pSchema = getTableColumnSchema(pTableMetaInfo->pTableMeta);
+  STableComInfo tinfo = getTableInfo(pTableMetaInfo->pTableMeta);
+
+  int32_t  numOfTotal = tinfo.numOfTags + tinfo.numOfColumns;
+
+  for (int32_t i = 0; i < numOfTotal; ++i) {
+    if (pSchema[i].colId == colId) {
+      return true;
+    }
+  }
+
+  return false;
+}
+
+int32_t tscTagCondCopy(STagCond* dest, const STagCond* src) {
+  memset(dest, 0, sizeof(STagCond));
+
+  if (src->tbnameCond.cond != NULL) {
+    dest->tbnameCond.cond = strdup(src->tbnameCond.cond);
+    if (dest->tbnameCond.cond == NULL) {
+      return -1;
+    }
+  }
+
+  dest->tbnameCond.uid = src->tbnameCond.uid;
+  dest->tbnameCond.len = src->tbnameCond.len;
+
+  dest->joinInfo.hasJoin = src->joinInfo.hasJoin;
+
+  for (int32_t i = 0; i < TSDB_MAX_JOIN_TABLE_NUM; ++i) {
+    if (src->joinInfo.joinTables[i]) {
+      dest->joinInfo.joinTables[i] = calloc(1, sizeof(SJoinNode));
+
+      memcpy(dest->joinInfo.joinTables[i], src->joinInfo.joinTables[i], sizeof(SJoinNode));
+
+      if (src->joinInfo.joinTables[i]->tsJoin) {
+        dest->joinInfo.joinTables[i]->tsJoin = taosArrayDup(src->joinInfo.joinTables[i]->tsJoin);
+      }
+
+      if (src->joinInfo.joinTables[i]->tagJoin) {
+        dest->joinInfo.joinTables[i]->tagJoin = taosArrayDup(src->joinInfo.joinTables[i]->tagJoin);
+      }
+    }
+  }
+
+
+  dest->relType = src->relType;
+
+  if (src->pCond == NULL) {
+    return 0;
+  }
+
+  size_t s = taosArrayGetSize(src->pCond);
+  dest->pCond = taosArrayInit(s, sizeof(SCond));
+
+  for (int32_t i = 0; i < s; ++i) {
+    SCond* pCond = taosArrayGet(src->pCond, i);
+
+    SCond c = {0};
+    c.len = pCond->len;
+    c.uid = pCond->uid;
+
+    if (pCond->len > 0) {
+      assert(pCond->cond != NULL);
+      c.cond = malloc(c.len);
+      if (c.cond == NULL) {
+        return -1;
+      }
+
+      memcpy(c.cond, pCond->cond, c.len);
+    }
+
+    taosArrayPush(dest->pCond, &c);
+  }
+
+  return 0;
+}
+
+int32_t tscColCondCopy(SArray** dest, const SArray* src, uint64_t uid, int16_t tidx) {
+  if (src == NULL) {
+    return 0;
+  }
+
+  size_t s = taosArrayGetSize(src);
+  *dest = taosArrayInit(s, sizeof(SCond));
+
+  for (int32_t i = 0; i < s; ++i) {
+    STableFilterCond* pCond = taosArrayGet(src, i);
+    STableFilterCond c = {0};
+
+    if (tidx > 0) {
+      if (!(pCond->uid == uid && pCond->idx == tidx)) {
+        continue;
+      }
+
+      c.idx = 0;
+    } else {
+      c.idx = pCond->idx;
+    }
+
+    c.len = pCond->len;
+    c.uid = pCond->uid;
+
+    if (pCond->len > 0) {
+      assert(pCond->cond != NULL);
+      c.cond = malloc(c.len);
+      if (c.cond == NULL) {
+        return -1;
+      }
+
+      memcpy(c.cond, pCond->cond, c.len);
+    }
+
+    taosArrayPush(*dest, &c);
+  }
+
+  return 0;
+}
+
+void cleanupColumnCond(SArray** pCond) {
+  if (*pCond == NULL) {
+    return;
+  }
+
+  size_t s = taosArrayGetSize(*pCond);
+  for (int32_t i = 0; i < s; ++i) {
+    STableFilterCond* p = taosArrayGet(*pCond, i);
+    tfree(p->cond);
+  }
+
+  taosArrayDestroy(*pCond);
+
+  *pCond = NULL;
+}
+
+void cleanupTagCond(STagCond* pTagCond) {
+  free(pTagCond->tbnameCond.cond);
+
+  if (pTagCond->pCond != NULL) {
+    size_t s = taosArrayGetSize(pTagCond->pCond);
+    for (int32_t i = 0; i < s; ++i) {
+      SCond* p = taosArrayGet(pTagCond->pCond, i);
+      tfree(p->cond);
+    }
+
+    taosArrayDestroy(pTagCond->pCond);
+  }
+
+  for (int32_t i = 0; i < TSDB_MAX_JOIN_TABLE_NUM; ++i) {
+    SJoinNode *node = pTagCond->joinInfo.joinTables[i];
+    if (node == NULL) {
+      continue;
+    }
+
+    if (node->tsJoin != NULL) {
+      taosArrayDestroy(node->tsJoin);
+    }
+
+    if (node->tagJoin != NULL) {
+      taosArrayDestroy(node->tagJoin);
+    }
+
+    tfree(node);
+  }
+
+  memset(pTagCond, 0, sizeof(STagCond));
+}
+
+//void tscGetSrcColumnInfo(SSrcColumnInfo* pColInfo, SQueryStmtInfo* pQueryInfo) {
+//  STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
+//  SSchema*        pSchema = getTableColumnSchema(pTableMetaInfo->pTableMeta);
+//
+//  size_t numOfExprs = getNumOfExprs(pQueryInfo);
+//  for (int32_t i = 0; i < numOfExprs; ++i) {
+//    SExprInfo* pExpr = getExprInfo(pQueryInfo, i);
+//    pColInfo[i].functionId = pExpr->base.functionId;
+//
+//    if (TSDB_COL_IS_TAG(pExpr->base.colInfo.flag)) {
+//      SSchema* pTagSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta);
+//
+//      int16_t index = pExpr->base.colInfo.colIndex;
+//      pColInfo[i].type = (index != -1) ? pTagSchema[index].type : TSDB_DATA_TYPE_BINARY;
+//    } else {
+//      pColInfo[i].type = pSchema[pExpr->base.colInfo.colIndex].type;
+//    }
+//  }
+//}
+
+/**
+ *
+ * @param clauseIndex denote the index of the union sub clause, usually are 0, if no union query exists.
+ * @param tableIndex  denote the table index for join query, where more than one table exists
+ * @return
+ */
+STableMetaInfo* getMetaInfo(SQueryStmtInfo* pQueryInfo, int32_t tableIndex) {
+  assert(pQueryInfo != NULL);
+  if (pQueryInfo->pTableMetaInfo == NULL) {
+    assert(pQueryInfo->numOfTables == 0);
+    return NULL;
+  }
+
+  assert(tableIndex >= 0 && tableIndex <= pQueryInfo->numOfTables && pQueryInfo->pTableMetaInfo != NULL);
+  return pQueryInfo->pTableMetaInfo[tableIndex];
+}
+
+STableMetaInfo* getTableMetaInfoByUid(SQueryStmtInfo* pQueryInfo, uint64_t uid, int32_t* index) {
+  int32_t k = -1;
+
+  for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) {
+    if (pQueryInfo->pTableMetaInfo[i]->pTableMeta->uid == uid) {
+      k = i;
+      break;
+    }
+  }
+
+  if (index != NULL) {
+    *index = k;
+  }
+
+  assert(k != -1);
+  return getMetaInfo(pQueryInfo, k);
+}
+
+int32_t queryInfoCopy(SQueryStmtInfo* pQueryInfo, const SQueryStmtInfo* pSrc) {
+  assert(pQueryInfo != NULL && pSrc != NULL);
+  int32_t code = TSDB_CODE_SUCCESS;
+
+  memcpy(&pQueryInfo->interval, &pSrc->interval, sizeof(pQueryInfo->interval));
+
+  pQueryInfo->command        = pSrc->command;
+  pQueryInfo->type           = pSrc->type;
+  pQueryInfo->window         = pSrc->window;
+  pQueryInfo->limit          = pSrc->limit;
+  pQueryInfo->slimit         = pSrc->slimit;
+  pQueryInfo->order          = pSrc->order;
+  pQueryInfo->vgroupLimit    = pSrc->vgroupLimit;
+  pQueryInfo->tsBuf          = NULL;
+  pQueryInfo->fillType       = pSrc->fillType;
+  pQueryInfo->fillVal        = NULL;
+  pQueryInfo->numOfFillVal   = 0;;
+  pQueryInfo->clauseLimit    = pSrc->clauseLimit;
+  pQueryInfo->prjOffset      = pSrc->prjOffset;
+  pQueryInfo->numOfTables    = 0;
+  pQueryInfo->window         = pSrc->window;
+  pQueryInfo->sessionWindow  = pSrc->sessionWindow;
+  pQueryInfo->pTableMetaInfo = NULL;
+
+  pQueryInfo->bufLen         = pSrc->bufLen;
+//  pQueryInfo->orderProjectQuery = pSrc->orderProjectQuery;
+//  pQueryInfo->arithmeticOnAgg   = pSrc->arithmeticOnAgg;
+  pQueryInfo->buf            = malloc(pSrc->bufLen);
+  if (pQueryInfo->buf == NULL) {
+    code = TSDB_CODE_TSC_OUT_OF_MEMORY;
+    goto _error;
+  }
+
+  if (pSrc->bufLen > 0) {
+    memcpy(pQueryInfo->buf, pSrc->buf, pSrc->bufLen);
+  }
+
+  pQueryInfo->groupbyExpr = pSrc->groupbyExpr;
+  if (pSrc->groupbyExpr.columnInfo != NULL) {
+    pQueryInfo->groupbyExpr.columnInfo = taosArrayDup(pSrc->groupbyExpr.columnInfo);
+    if (pQueryInfo->groupbyExpr.columnInfo == NULL) {
+      code = TSDB_CODE_TSC_OUT_OF_MEMORY;
+      goto _error;
+    }
+  }
+
+  if (tscTagCondCopy(&pQueryInfo->tagCond, &pSrc->tagCond) != 0) {
+    code = TSDB_CODE_TSC_OUT_OF_MEMORY;
+    goto _error;
+  }
+
+  if (tscColCondCopy(&pQueryInfo->colCond, pSrc->colCond, 0, -1) != 0) {
+    code = TSDB_CODE_TSC_OUT_OF_MEMORY;
+    goto _error;
+  }
+
+  if (pSrc->fillType != TSDB_FILL_NONE) {
+    pQueryInfo->fillVal = calloc(1, pSrc->fieldsInfo.numOfOutput * sizeof(int64_t));
+    if (pQueryInfo->fillVal == NULL) {
+      code = TSDB_CODE_TSC_OUT_OF_MEMORY;
+      goto _error;
+    }
+    pQueryInfo->numOfFillVal = pSrc->fieldsInfo.numOfOutput;
+
+    memcpy(pQueryInfo->fillVal, pSrc->fillVal, pSrc->fieldsInfo.numOfOutput * sizeof(int64_t));
+  }
+
+  if (copyAllExprInfo(pQueryInfo->exprList, pSrc->exprList, true) != 0) {
+    code = TSDB_CODE_TSC_OUT_OF_MEMORY;
+    goto _error;
+  }
+
+//  if (pQueryInfo->arithmeticOnAgg) {
+//    pQueryInfo->exprList1 = taosArrayInit(4, POINTER_BYTES);
+//    if (copyAllExprInfo(pQueryInfo->exprList1, pSrc->exprList1, true) != 0) {
+//      code = TSDB_CODE_TSC_OUT_OF_MEMORY;
+//      goto _error;
+//    }
+//  }
+
+  columnListCopyAll(pQueryInfo->colList, pSrc->colList);
+  copyFieldInfo(&pQueryInfo->fieldsInfo, &pSrc->fieldsInfo, pQueryInfo->exprList);
+
+  for(int32_t i = 0; i < pSrc->numOfTables; ++i) {
+    STableMetaInfo* p1 = getMetaInfo((SQueryStmtInfo*) pSrc, i);
+
+    STableMeta* pMeta = tableMetaDup(p1->pTableMeta);
+    if (pMeta == NULL) {
+      // todo handle the error
+    }
+
+    addTableMetaInfo(pQueryInfo, &p1->name, pMeta, p1->vgroupList, p1->tagColList, NULL);
+  }
+
+  SArray *pUdfInfo = NULL;
+  if (pSrc->pUdfInfo) {
+    pUdfInfo = taosArrayDup(pSrc->pUdfInfo);
+  }
+
+  pQueryInfo->pUdfInfo = pUdfInfo;
+
+  _error:
+  return code;
+}
+
+void clearAllTableMetaInfo(SQueryStmtInfo* pQueryInfo, bool removeMeta, uint64_t id) {
+  for(int32_t i = 0; i < pQueryInfo->numOfTables; ++i) {
+    STableMetaInfo* pTableMetaInfo = getMetaInfo(pQueryInfo, i);
+    clearTableMetaInfo(pTableMetaInfo);
+  }
+
+  tfree(pQueryInfo->pTableMetaInfo);
+}
+
+STableMetaInfo* addTableMetaInfo(SQueryStmtInfo* pQueryInfo, SName* name, STableMeta* pTableMeta,
+                                 SVgroupsInfo* vgroupList, SArray* pTagCols, SArray* pVgroupTables) {
+  void* tmp = realloc(pQueryInfo->pTableMetaInfo, (pQueryInfo->numOfTables + 1) * POINTER_BYTES);
+  if (tmp == NULL) {
+    terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
+    return NULL;
+  }
+
+  pQueryInfo->pTableMetaInfo = tmp;
+  STableMetaInfo* pTableMetaInfo = calloc(1, sizeof(STableMetaInfo));
+
+  if (pTableMetaInfo == NULL) {
+    terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
+    return NULL;
+  }
+
+  pQueryInfo->pTableMetaInfo[pQueryInfo->numOfTables] = pTableMetaInfo;
+
+  if (name != NULL) {
+    tNameAssign(&pTableMetaInfo->name, name);
+  }
+
+  pTableMetaInfo->pTableMeta = pTableMeta;
+
+  if (vgroupList != NULL) {
+//    pTableMetaInfo->vgroupList = vgroupInfoClone(vgroupList);
+  }
+
+  // TODO handle malloc failure
+  pTableMetaInfo->tagColList = taosArrayInit(4, POINTER_BYTES);
+  if (pTableMetaInfo->tagColList == NULL) {
+    return NULL;
+  }
+
+  if (pTagCols != NULL && pTableMetaInfo->pTableMeta != NULL) {
+    columnListCopy(pTableMetaInfo->tagColList, pTagCols, pTableMetaInfo->pTableMeta->uid);
+  }
+
+  pQueryInfo->numOfTables += 1;
+  return pTableMetaInfo;
+}
+
+STableMetaInfo* addEmptyMetaInfo(SQueryStmtInfo* pQueryInfo) {
+  return addTableMetaInfo(pQueryInfo, NULL, NULL, NULL, NULL, NULL);
+}
+
+SInternalField* getInternalFieldInfo(SFieldInfo* pFieldInfo, int32_t index) {
+  assert(index < pFieldInfo->numOfOutput);
+  return TARRAY_GET_ELEM(pFieldInfo->internalField, index);
+}
+
+int32_t getNumOfInternalField(SFieldInfo* pFieldInfo) {
+  return (int32_t) taosArrayGetSize(pFieldInfo->internalField);
+}
+
+static void doSetSqlExprAndResultFieldInfo(SQueryStmtInfo* pNewQueryInfo, int64_t uid) {
+  int32_t numOfOutput = (int32_t)getNumOfExprs(pNewQueryInfo);
+  if (numOfOutput == 0) {
+    return;
+  }
+
+  // set the field info in pNewQueryInfo object according to sqlExpr information
+  for (int32_t i = 0; i < numOfOutput; ++i) {
+    SExprInfo* pExpr = getExprInfo(pNewQueryInfo, i);
+
+    TAOS_FIELD f = createField((int8_t) pExpr->base.resType, pExpr->base.aliasName, pExpr->base.resBytes);
+    SInternalField* pInfo1 = appendFieldInfo(&pNewQueryInfo->fieldsInfo, &f);
+    pInfo1->pExpr = pExpr;
+  }
+
+  // update the pSqlExpr pointer in SInternalField according the field name
+  // make sure the pSqlExpr point to the correct SqlExpr in pNewQueryInfo, not SqlExpr in pQueryInfo
+  for (int32_t f = 0; f < pNewQueryInfo->fieldsInfo.numOfOutput; ++f) {
+    TAOS_FIELD* field = getFieldInfo(&pNewQueryInfo->fieldsInfo, f);
+
+    bool matched = false;
+    for (int32_t k1 = 0; k1 < numOfOutput; ++k1) {
+      SExprInfo* pExpr1 = getExprInfo(pNewQueryInfo, k1);
+
+      if (strcmp(field->name, pExpr1->base.aliasName) == 0) {  // establish link according to the result field name
+        SInternalField* pInfo = getInternalFieldInfo(&pNewQueryInfo->fieldsInfo, f);
+        pInfo->pExpr = pExpr1;
+
+        matched = true;
+        break;
+      }
+    }
+
+    assert(matched);
+    (void)matched;
+  }
+
+//  updateFieldInfoOffset(pNewQueryInfo);
+}
+
+int16_t getJoinTagColIdByUid(STagCond* pTagCond, uint64_t uid) {
+  int32_t i = 0;
+  while (i < TSDB_MAX_JOIN_TABLE_NUM) {
+    SJoinNode* node = pTagCond->joinInfo.joinTables[i];
+    if (node && node->uid == uid) {
+      return node->tagColId;
+    }
+
+    i++;
+  }
+
+  assert(0);
+  return -1;
+}
+
+int16_t getTagColIndexById(STableMeta* pTableMeta, int16_t colId) {
+  int32_t numOfTags = getNumOfTags(pTableMeta);
+
+  SSchema* pSchema = getTableTagSchema(pTableMeta);
+  for(int32_t i = 0; i < numOfTags; ++i) {
+    if (pSchema[i].colId == colId) {
+      return i;
+    }
+  }
+
+  // can not reach here
+  assert(0);
+  return INT16_MIN;
+}
+
+bool isQueryWithLimit(SQueryStmtInfo* pQueryInfo) {
+  while(pQueryInfo != NULL) {
+    if (pQueryInfo->limit.limit > 0) {
+      return true;
+    }
+
+    pQueryInfo = pQueryInfo->sibling;
+  }
+
+  return false;
+}
+
+SVgroupsInfo* vgroupInfoClone(SVgroupsInfo *vgroupList) {
+  if (vgroupList == NULL) {
+    return NULL;
+  }
+
+  size_t size = sizeof(SVgroupsInfo) + sizeof(SVgroupMsg) * vgroupList->numOfVgroups;
+  SVgroupsInfo* pNew = malloc(size);
+  if (pNew == NULL) {
+    return NULL;
+  }
+
+  pNew->numOfVgroups = vgroupList->numOfVgroups;
+
+  for(int32_t i = 0; i < vgroupList->numOfVgroups; ++i) {
+    SVgroupMsg* pNewVInfo = &pNew->vgroups[i];
+
+    SVgroupMsg* pvInfo = &vgroupList->vgroups[i];
+    pNewVInfo->vgId = pvInfo->vgId;
+    pNewVInfo->numOfEps = pvInfo->numOfEps;
+
+    for(int32_t j = 0; j < pvInfo->numOfEps; ++j) {
+      pNewVInfo->epAddr[j].port = pvInfo->epAddr[j].port;
+      tstrncpy(pNewVInfo->epAddr[j].fqdn, pvInfo->epAddr[j].fqdn, TSDB_FQDN_LEN);
+    }
+  }
+
+  return pNew;
+}
+
+void* vgroupInfoClear(SVgroupsInfo *vgroupList) {
+  if (vgroupList == NULL) {
+    return NULL;
+  }
+
+  tfree(vgroupList);
+  return NULL;
+}
+
+char* serializeTagData(STagData* pTagData, char* pMsg) {
+  int32_t n = (int32_t) strlen(pTagData->name);
+  *(int32_t*) pMsg = htonl(n);
+  pMsg += sizeof(n);
+
+  memcpy(pMsg, pTagData->name, n);
+  pMsg += n;
+
+  *(int32_t*)pMsg = htonl(pTagData->dataLen);
+  pMsg += sizeof(int32_t);
+
+  memcpy(pMsg, pTagData->data, pTagData->dataLen);
+  pMsg += pTagData->dataLen;
+
+  return pMsg;
+}
+
+int32_t copyTagData(STagData* dst, const STagData* src) {
+  dst->dataLen = src->dataLen;
+  tstrncpy(dst->name, src->name, tListLen(dst->name));
+
+  if (dst->dataLen > 0) {
+    dst->data = malloc(dst->dataLen);
+    if (dst->data == NULL) {
+      return -1;
+    }
+
+    memcpy(dst->data, src->data, dst->dataLen);
+  }
+
+  return 0;
+}
+
+STableMeta* createSuperTableMeta(STableMetaMsg* pChild) {
+  assert(pChild != NULL);
+  int32_t total = pChild->numOfColumns + pChild->numOfTags;
+
+  STableMeta* pTableMeta = calloc(1, sizeof(STableMeta) + sizeof(SSchema) * total);
+  pTableMeta->tableType = TSDB_SUPER_TABLE;
+  pTableMeta->tableInfo.numOfTags = pChild->numOfTags;
+  pTableMeta->tableInfo.numOfColumns = pChild->numOfColumns;
+  pTableMeta->tableInfo.precision = pChild->precision;
+
+  pTableMeta->uid = pChild->suid;
+  pTableMeta->tversion = pChild->tversion;
+  pTableMeta->sversion = pChild->sversion;
+
+  memcpy(pTableMeta->schema, pChild->schema, sizeof(SSchema) * total);
+
+  int32_t num = pTableMeta->tableInfo.numOfColumns;
+  for(int32_t i = 0; i < num; ++i) {
+    pTableMeta->tableInfo.rowSize += pTableMeta->schema[i].bytes;
+  }
+
+  return pTableMeta;
+}
+
+uint32_t getTableMetaSize(STableMeta* pTableMeta) {
+  assert(pTableMeta != NULL);
+
+  int32_t totalCols = 0;
+  if (pTableMeta->tableInfo.numOfColumns >= 0) {
+    totalCols = pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags;
+  }
+
+  return sizeof(STableMeta) + totalCols * sizeof(SSchema);
+}
+
+uint32_t getTableMetaMaxSize() {
+  return sizeof(STableMeta) + TSDB_MAX_COLUMNS * sizeof(SSchema);
+}
+
+STableMeta* tableMetaDup(STableMeta* pTableMeta) {
+  assert(pTableMeta != NULL);
+  size_t size = getTableMetaSize(pTableMeta);
+
+  STableMeta* p = malloc(size);
+  memcpy(p, pTableMeta, size);
+  return p;
+}
+
+SVgroupsInfo* vgroupsInfoDup(SVgroupsInfo* pVgroupsInfo) {
+  assert(pVgroupsInfo != NULL);
+
+  size_t size = sizeof(SVgroupMsg) * pVgroupsInfo->numOfVgroups + sizeof(SVgroupsInfo);
+  SVgroupsInfo* pInfo = calloc(1, size);
+  pInfo->numOfVgroups = pVgroupsInfo->numOfVgroups;
+  for (int32_t m = 0; m < pVgroupsInfo->numOfVgroups; ++m) {
+    memcpy(&pInfo->vgroups[m], &pVgroupsInfo->vgroups[m], sizeof(SVgroupMsg));
+  }
+
+  return pInfo;
+}
+
+int32_t getNumOfOutput(SFieldInfo* pFieldInfo) {
+  return pFieldInfo->numOfOutput;
+}
+
+int32_t createProjectionExpr(SQueryStmtInfo* pQueryInfo, STableMetaInfo* pTableMetaInfo, SExprInfo*** pExpr, int32_t* num) {
+//  if (!pQueryInfo->arithmeticOnAgg) {
+//    return TSDB_CODE_SUCCESS;
+//  }
+
+  *num = getNumOfOutput(pQueryInfo);
+  *pExpr = calloc(*(num), POINTER_BYTES);
+  if ((*pExpr) == NULL) {
+    return TSDB_CODE_TSC_OUT_OF_MEMORY;
+  }
+
+  for (int32_t i = 0; i < (*num); ++i) {
+    SInternalField* pField = getInternalFieldInfo(&pQueryInfo->fieldsInfo, i);
+    SExprInfo* pSource = pField->pExpr;
+
+    SExprInfo* px = calloc(1, sizeof(SExprInfo));
+    (*pExpr)[i] = px;
+
+    SSqlExpr *pse = &px->base;
+    pse->uid      = pTableMetaInfo->pTableMeta->uid;
+    pse->resColId = pSource->base.resColId;
+    strncpy(pse->aliasName, pSource->base.aliasName, tListLen(pse->aliasName));
+    strncpy(pse->token, pSource->base.token, tListLen(pse->token));
+
+    if (pSource->base.functionId != FUNCTION_ARITHM) {  // this should be switched to projection query
+      pse->numOfParams = 0;      // no params for projection query
+      pse->functionId  = FUNCTION_PRJ;
+      pse->colInfo.colId = pSource->base.resColId;
+
+      int32_t numOfOutput = (int32_t) taosArrayGetSize(pQueryInfo->exprList);
+      for (int32_t j = 0; j < numOfOutput; ++j) {
+        SExprInfo* p = taosArrayGetP(pQueryInfo->exprList, j);
+        if (p->base.resColId == pse->colInfo.colId) {
+          pse->colInfo.colIndex = j;
+          break;
+        }
+      }
+
+      pse->colInfo.flag = TSDB_COL_NORMAL;
+      pse->resType  = pSource->base.resType;
+      pse->resBytes = pSource->base.resBytes;
+      strncpy(pse->colInfo.name, pSource->base.aliasName, tListLen(pse->colInfo.name));
+
+      // TODO restore refactor
+      int32_t functionId = pSource->base.functionId;
+      if (pSource->base.functionId == FUNCTION_FIRST_DST) {
+        functionId = FUNCTION_FIRST;
+      } else if (pSource->base.functionId == FUNCTION_LAST_DST) {
+        functionId = FUNCTION_LAST;
+      } else if (pSource->base.functionId == FUNCTION_STDDEV_DST) {
+        functionId = FUNCTION_STDDEV;
+      }
+
+      int32_t inter = 0;
+      getResultDataInfo(pSource->base.colType, pSource->base.colBytes, functionId, 0, &pse->resType,
+                        &pse->resBytes, &inter, 0, false/*, NULL*/);
+      pse->colType  = pse->resType;
+      pse->colBytes = pse->resBytes;
+
+    } else {  // arithmetic expression
+      pse->colInfo.colId = pSource->base.colInfo.colId;
+      pse->colType  = pSource->base.colType;
+      pse->colBytes = pSource->base.colBytes;
+      pse->resBytes = sizeof(double);
+      pse->resType  = TSDB_DATA_TYPE_DOUBLE;
+
+      pse->functionId = pSource->base.functionId;
+      pse->numOfParams = pSource->base.numOfParams;
+
+      for (int32_t j = 0; j < pSource->base.numOfParams; ++j) {
+        taosVariantAssign(&pse->param[j], &pSource->base.param[j]);
+//        buildArithmeticExprFromMsg(px, NULL);
+      }
+    }
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t getColFilterSerializeLen(SQueryStmtInfo* pQueryInfo) {
+  int16_t numOfCols = (int16_t)taosArrayGetSize(pQueryInfo->colList);
+  int32_t len = 0;
+
+  for(int32_t i = 0; i < numOfCols; ++i) {
+    SColumn* pCol = taosArrayGetP(pQueryInfo->colList, i);
+    for (int32_t j = 0; j < pCol->info.flist.numOfFilters; ++j) {
+      len += sizeof(SColumnFilterInfo);
+      if (pCol->info.flist.filterInfo[j].filterstr) {
+        len += (int32_t)pCol->info.flist.filterInfo[j].len + 1 * TSDB_NCHAR_SIZE;
+      }
+    }
+  }
+  return len;
+}
+
+int32_t getTagFilterSerializeLen(SQueryStmtInfo* pQueryInfo) {
+  // serialize tag column query condition
+  if (pQueryInfo->tagCond.pCond != NULL && taosArrayGetSize(pQueryInfo->tagCond.pCond) > 0) {
+    STagCond* pTagCond = &pQueryInfo->tagCond;
+
+    STableMetaInfo *pTableMetaInfo = getMetaInfo(pQueryInfo, 0);
+    STableMeta * pTableMeta = pTableMetaInfo->pTableMeta;
+    SCond *pCond = getSTableQueryCond(pTagCond, pTableMeta->uid);
+    if (pCond != NULL && pCond->cond != NULL) {
+      return pCond->len;
+    }
+  }
+  return 0;
+}
+
+#if 0
+int32_t tscCreateQueryFromQueryInfo(SQueryStmtInfo* pQueryInfo, SQueryAttr* pQueryAttr, void* addr) {
+  memset(pQueryAttr, 0, sizeof(SQueryAttr));
+
+  int16_t numOfCols        = (int16_t) taosArrayGetSize(pQueryInfo->colList);
+  int16_t numOfOutput      = (int16_t) getNumOfExprs(pQueryInfo);
+
+  pQueryAttr->topBotQuery       = tscIsTopBotQuery(pQueryInfo);
+  pQueryAttr->hasTagResults     = hasTagValOutput(pQueryInfo);
+  pQueryAttr->stabledev         = isStabledev(pQueryInfo);
+  pQueryAttr->tsCompQuery       = isTsCompQuery(pQueryInfo);
+  pQueryAttr->diffQuery         = tscIsDiffDerivQuery(pQueryInfo);
+  pQueryAttr->simpleAgg         = isSimpleAggregateRv(pQueryInfo);
+  pQueryAttr->needReverseScan   = tscNeedReverseScan(pQueryInfo);
+  pQueryAttr->stableQuery       = QUERY_IS_STABLE_QUERY(pQueryInfo->type);
+  pQueryAttr->groupbyColumn     = (!pQueryInfo->stateWindow) && tscGroupbyColumn(pQueryInfo);
+  pQueryAttr->queryBlockDist    = isBlockDistQuery(pQueryInfo);
+  pQueryAttr->pointInterpQuery  = tscIsPointInterpQuery(pQueryInfo);
+  pQueryAttr->timeWindowInterpo = timeWindowInterpoRequired(pQueryInfo);
+  pQueryAttr->distinct          = pQueryInfo->distinct;
+  pQueryAttr->sw                = pQueryInfo->sessionWindow;
+  pQueryAttr->stateWindow       = pQueryInfo->stateWindow;
+  pQueryAttr->multigroupResult  = pQueryInfo->multigroupResult;
+
+  pQueryAttr->numOfCols         = numOfCols;
+  pQueryAttr->numOfOutput       = numOfOutput;
+  pQueryAttr->limit             = pQueryInfo->limit;
+  pQueryAttr->slimit            = pQueryInfo->slimit;
+  pQueryAttr->order             = pQueryInfo->order;
+  pQueryAttr->fillType          = pQueryInfo->fillType;
+  pQueryAttr->havingNum         = pQueryInfo->havingFieldNum;
+  pQueryAttr->pUdfInfo          = pQueryInfo->pUdfInfo;
+
+  if (pQueryInfo->order.order == TSDB_ORDER_ASC) {   // TODO refactor
+    pQueryAttr->window = pQueryInfo->window;
+  } else {
+    pQueryAttr->window.skey = pQueryInfo->window.ekey;
+    pQueryAttr->window.ekey = pQueryInfo->window.skey;
+  }
+
+  memcpy(&pQueryAttr->interval, &pQueryInfo->interval, sizeof(pQueryAttr->interval));
+
+  STableMetaInfo* pTableMetaInfo = pQueryInfo->pTableMetaInfo[0];
+
+  if (pQueryInfo->groupbyExpr.numOfGroupCols > 0) {
+    pQueryAttr->pGroupbyExpr    = calloc(1, sizeof(SGroupbyExpr));
+    *(pQueryAttr->pGroupbyExpr) = pQueryInfo->groupbyExpr;
+    pQueryAttr->pGroupbyExpr->columnInfo = taosArrayDup(pQueryInfo->groupbyExpr.columnInfo);
+  } else {
+    assert(pQueryInfo->groupbyExpr.columnInfo == NULL);
+  }
+
+  pQueryAttr->pExpr1 = calloc(pQueryAttr->numOfOutput, sizeof(SExprInfo));
+  for(int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) {
+    SExprInfo* pExpr = getExprInfo(pQueryInfo, i);
+    tscExprAssign(&pQueryAttr->pExpr1[i], pExpr);
+
+    if (pQueryAttr->pExpr1[i].base.functionId == FUNCTION_ARITHM) {
+      for (int32_t j = 0; j < pQueryAttr->pExpr1[i].base.numOfParams; ++j) {
+        buildArithmeticExprFromMsg(&pQueryAttr->pExpr1[i], NULL);
+      }
+    }
+  }
+
+  pQueryAttr->tableCols = calloc(numOfCols, sizeof(SColumnInfo));
+  for(int32_t i = 0; i < numOfCols; ++i) {
+    SColumn* pCol = taosArrayGetP(pQueryInfo->colList, i);
+    if (!isValidDataType(pCol->info.type) || pCol->info.type == TSDB_DATA_TYPE_NULL) {
+      assert(0);
+    }
+
+    pQueryAttr->tableCols[i] = pCol->info;
+    pQueryAttr->tableCols[i].flist.filterInfo = tFilterInfoDup(pCol->info.flist.filterInfo, pQueryAttr->tableCols[i].flist.numOfFilters);
+  }
+
+  // global aggregate query
+  if (pQueryAttr->stableQuery && (pQueryAttr->simpleAgg || pQueryAttr->interval.interval > 0) && tscIsTwoStageSTableQuery(pQueryInfo, 0)) {
+    createGlobalAggregateExpr(pQueryAttr, pQueryInfo);
+  }
+
+  // for simple table, not for super table
+  if (pQueryInfo->arithmeticOnAgg) {
+    pQueryAttr->numOfExpr2 = (int32_t) taosArrayGetSize(pQueryInfo->exprList1);
+    pQueryAttr->pExpr2 = calloc(pQueryAttr->numOfExpr2, sizeof(SExprInfo));
+    for(int32_t i = 0; i < pQueryAttr->numOfExpr2; ++i) {
+      SExprInfo* p = taosArrayGetP(pQueryInfo->exprList1, i);
+      tscExprAssign(&pQueryAttr->pExpr2[i], p);
+    }
+  }
+
+  // tag column info
+  int32_t code = createTagColumnInfo(pQueryAttr, pQueryInfo, pTableMetaInfo);
+  if (code != TSDB_CODE_SUCCESS) {
+    return code;
+  }
+
+  if (pQueryAttr->fillType != TSDB_FILL_NONE) {
+    pQueryAttr->fillVal = calloc(pQueryAttr->numOfOutput, sizeof(int64_t));
+    memcpy(pQueryAttr->fillVal, pQueryInfo->fillVal, pQueryInfo->numOfFillVal * sizeof(int64_t));
+  }
+
+  pQueryAttr->srcRowSize = 0;
+  pQueryAttr->maxTableColumnWidth = 0;
+  for (int16_t i = 0; i < numOfCols; ++i) {
+    pQueryAttr->srcRowSize += pQueryAttr->tableCols[i].bytes;
+    if (pQueryAttr->maxTableColumnWidth < pQueryAttr->tableCols[i].bytes) {
+      pQueryAttr->maxTableColumnWidth = pQueryAttr->tableCols[i].bytes;
+    }
+  }
+
+  pQueryAttr->interBufSize = getOutputInterResultBufSize(pQueryAttr);
+
+  if (pQueryAttr->numOfCols <= 0 && !tscQueryTags(pQueryInfo) && !pQueryAttr->queryBlockDist) {
+    tscError("%p illegal value of numOfCols in query msg: %" PRIu64 ", table cols:%d", addr,
+             (uint64_t)pQueryAttr->numOfCols, numOfCols);
+
+    return TSDB_CODE_TSC_INVALID_OPERATION;
+  }
+
+  if (pQueryAttr->interval.interval < 0) {
+    tscError("%p illegal value of aggregation time interval in query msg: %" PRId64, addr,
+             (int64_t)pQueryInfo->interval.interval);
+    return TSDB_CODE_TSC_INVALID_OPERATION;
+  }
+
+  if (pQueryAttr->pGroupbyExpr != NULL && pQueryAttr->pGroupbyExpr->numOfGroupCols < 0) {
+    tscError("%p illegal value of numOfGroupCols in query msg: %d", addr, pQueryInfo->groupbyExpr.numOfGroupCols);
+    return TSDB_CODE_TSC_INVALID_OPERATION;
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t doAddTableName(char* nextStr, char** str, SArray* pNameArray, SSqlObj* pSql) {
+  int32_t code = TSDB_CODE_SUCCESS;
+  SSqlCmd* pCmd = &pSql->cmd;
+
+  char  tablename[TSDB_TABLE_FNAME_LEN] = {0};
+  int32_t len = 0;
+
+  if (nextStr == NULL) {
+    tstrncpy(tablename, *str, TSDB_TABLE_FNAME_LEN);
+    len = (int32_t) strlen(tablename);
+  } else {
+    len = (int32_t)(nextStr - (*str));
+    if (len >= TSDB_TABLE_NAME_LEN) {
+      sprintf(pCmd->payload, "table name too long");
+      return TSDB_CODE_TSC_INVALID_OPERATION;
+    }
+
+    memcpy(tablename, *str, nextStr - (*str));
+    tablename[len] = '\0';
+  }
+
+  (*str) = nextStr + 1;
+  len = (int32_t)strtrim(tablename);
+
+  SToken sToken = {.n = len, .type = TK_ID, .z = tablename};
+  tGetToken(tablename, &sToken.type);
+
+  // Check if the table name available or not
+  if (tscValidateName(&sToken) != TSDB_CODE_SUCCESS) {
+    sprintf(pCmd->payload, "table name is invalid");
+    return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH;
+  }
+
+  SName name = {0};
+  if ((code = tscSetTableFullName(&name, &sToken, pSql)) != TSDB_CODE_SUCCESS) {
+    return code;
+  }
+
+  memset(tablename, 0, tListLen(tablename));
+  tNameExtractFullName(&name, tablename);
+
+  char* p = strdup(tablename);
+  taosArrayPush(pNameArray, &p);
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t nameComparFn(const void* n1, const void* n2) {
+  int32_t ret = strcmp(*(char**)n1, *(char**)n2);
+  if (ret == 0) {
+    return 0;
+  } else {
+    return ret > 0? 1:-1;
+  }
+}
+
+static void freeContent(void* p) {
+  char* ptr = *(char**)p;
+  tfree(ptr);
+}
+
+
+int tscTransferTableNameList(SSqlObj *pSql, const char *pNameList, int32_t length, SArray* pNameArray) {
+  SSqlCmd *pCmd = &pSql->cmd;
+
+  pCmd->command = TSDB_SQL_MULTI_META;
+  pCmd->msgType = TSDB_MSG_TYPE_CM_TABLES_META;
+
+  int   code = TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH;
+  char *str = (char *)pNameList;
+
+  SQueryStmtInfo *pQueryInfo = tscGetQueryInfoS(pCmd);
+  if (pQueryInfo == NULL) {
+    pSql->res.code = terrno;
+    return terrno;
+  }
+
+  char *nextStr;
+  while (1) {
+    nextStr = strchr(str, ',');
+    if (nextStr == NULL) {
+      code = doAddTableName(nextStr, &str, pNameArray, pSql);
+      break;
+    }
+
+    code = doAddTableName(nextStr, &str, pNameArray, pSql);
+    if (code != TSDB_CODE_SUCCESS) {
+      return code;
+    }
+
+    if (taosArrayGetSize(pNameArray) > TSDB_MULTI_TABLEMETA_MAX_NUM) {
+      code = TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH;
+      sprintf(pCmd->payload, "tables over the max number");
+      return code;
+    }
+  }
+
+  size_t len = taosArrayGetSize(pNameArray);
+  if (len == 1) {
+    return TSDB_CODE_SUCCESS;
+  }
+
+  if (len > TSDB_MULTI_TABLEMETA_MAX_NUM) {
+    code = TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH;
+    sprintf(pCmd->payload, "tables over the max number");
+    return code;
+  }
+
+  taosArraySort(pNameArray, nameComparFn);
+  taosArrayRemoveDuplicate(pNameArray, nameComparFn, freeContent);
+  return TSDB_CODE_SUCCESS;
+}
+
+bool vgroupInfoIdentical(SNewVgroupInfo *pExisted, SVgroupMsg* src) {
+  assert(pExisted != NULL && src != NULL);
+  if (pExisted->numOfEps != src->numOfEps) {
+    return false;
+  }
+
+  for(int32_t i = 0; i < pExisted->numOfEps; ++i) {
+    if (pExisted->ep[i].port != src->epAddr[i].port) {
+      return false;
+    }
+
+    if (strncmp(pExisted->ep[i].fqdn, src->epAddr[i].fqdn, tListLen(pExisted->ep[i].fqdn)) != 0) {
+      return false;
+    }
+  }
+
+  return true;
+}
+
+SNewVgroupInfo createNewVgroupInfo(SVgroupMsg *pVgroupMsg) {
+  assert(pVgroupMsg != NULL);
+
+  SNewVgroupInfo info = {0};
+  info.numOfEps = pVgroupMsg->numOfEps;
+  info.vgId     = pVgroupMsg->vgId;
+  info.inUse    = 0;   // 0 is the default value of inUse in case of multiple replica
+
+  assert(info.numOfEps >= 1 && info.vgId >= 1);
+  for(int32_t i = 0; i < pVgroupMsg->numOfEps; ++i) {
+    tstrncpy(info.ep[i].fqdn, pVgroupMsg->epAddr[i].fqdn, TSDB_FQDN_LEN);
+    info.ep[i].port = pVgroupMsg->epAddr[i].port;
+  }
+
+  return info;
+}
+
+char* cloneCurrentDBName(SSqlObj* pSql) {
+  char        *p = NULL;
+  HttpContext *pCtx = NULL;
+
+  pthread_mutex_lock(&pSql->pTscObj->mutex);
+  STscObj *pTscObj = pSql->pTscObj;
+  switch (pTscObj->from) {
+    case TAOS_REQ_FROM_HTTP:
+      pCtx = pSql->param;
+      if (pCtx && pCtx->db[0] != '\0') {
+        char db[TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN] = {0};
+        int32_t len = sprintf(db, "%s%s%s", pTscObj->acctId, TS_PATH_DELIMITER, pCtx->db);
+        assert(len <= sizeof(db));
+
+        p = strdup(db);
+      }
+      break;
+    default:
+      break;
+  }
+  if (p == NULL) {
+    p = strdup(pSql->pTscObj->db);
+  }
+  pthread_mutex_unlock(&pSql->pTscObj->mutex);
+
+  return p;
+}
+
+#endif
\ No newline at end of file

source/libs/parser/src/queryInfoUtil.c

+#include "queryInfoUtil.h"
+#include "tmsgtype.h"
+#include "astGenerator.h"
+#include "os.h"
+#include "parser.h"
+#include "parserInt.h"
+#include "parserUtil.h"
+
+static struct SSchema _s = {
+    .colId = TSDB_TBNAME_COLUMN_INDEX,
+    .type  = TSDB_DATA_TYPE_BINARY,
+    .bytes = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE,
+    .name = "tbname",
+};
+
+SSchema* getTbnameColumnSchema() {
+  return &_s;
+}
+
+size_t getNumOfExprs(SQueryStmtInfo* pQueryInfo) {
+  return taosArrayGetSize(pQueryInfo->exprList);
+}
+
+SSchema* getOneColumnSchema(const STableMeta* pTableMeta, int32_t colIndex) {
+  assert(pTableMeta != NULL && pTableMeta->schema != NULL && colIndex >= 0 && colIndex < getNumOfColumns(pTableMeta));
+
+  SSchema* pSchema = (SSchema*) pTableMeta->schema;
+  return &pSchema[colIndex];
+}
+
+STableComInfo getTableInfo(const STableMeta* pTableMeta) {
+  assert(pTableMeta != NULL);
+  return pTableMeta->tableInfo;
+}
+
+int32_t getNumOfColumns(const STableMeta* pTableMeta) {
+  assert(pTableMeta != NULL);
+  // table created according to super table, use data from super table
+  return getTableInfo(pTableMeta).numOfColumns;
+}
+
+int32_t getNumOfTags(const STableMeta* pTableMeta) {
+  assert(pTableMeta != NULL);
+  return getTableInfo(pTableMeta).numOfTags;
+}
+
+SSchema *getTableColumnSchema(const STableMeta *pTableMeta) {
+  assert(pTableMeta != NULL);
+  return (SSchema*) pTableMeta->schema;
+}
+
+SSchema* getTableTagSchema(const STableMeta* pTableMeta) {
+  assert(pTableMeta != NULL && (pTableMeta->tableType == TSDB_SUPER_TABLE || pTableMeta->tableType == TSDB_CHILD_TABLE));
+  return getOneColumnSchema(pTableMeta, getTableInfo(pTableMeta).numOfColumns);
+}
+
+SExprInfo* createExprInfo(STableMetaInfo* pTableMetaInfo, int16_t functionId, SColumnIndex* pColIndex, int16_t type,
+                         int16_t size, int16_t resColId, int16_t interSize, int32_t colType) {
+  SExprInfo* pExpr = calloc(1, sizeof(SExprInfo));
+  if (pExpr == NULL) {
+    return NULL;
+  }
+
+  SSqlExpr* p = &pExpr->base;
+  p->functionId = functionId;
+
+  // set the correct columnIndex index
+  if (pColIndex->columnIndex == TSDB_TBNAME_COLUMN_INDEX) {
+    SSchema* s = getTbnameColumnSchema();
+    p->colInfo.colId = TSDB_TBNAME_COLUMN_INDEX;
+    p->colBytes = s->bytes;
+    p->colType  = s->type;
+  } else if (pColIndex->columnIndex <= TSDB_UD_COLUMN_INDEX) {
+    p->colInfo.colId = pColIndex->columnIndex;
+    p->colBytes = size;
+    p->colType  = type;
+  } else if (functionId == 0/*TSDB_FUNC_BLKINFO*/) {
+    assert(0);
+    p->colInfo.colId = pColIndex->columnIndex;
+    p->colBytes = TSDB_MAX_BINARY_LEN;
+    p->colType  = TSDB_DATA_TYPE_BINARY;
+  } else {
+    int32_t len = tListLen(p->colInfo.name);
+    if (TSDB_COL_IS_TAG(colType)) {
+      SSchema* pSchema = getTableTagSchema(pTableMetaInfo->pTableMeta);
+      p->colInfo.colId = pSchema[pColIndex->columnIndex].colId;
+      p->colBytes = pSchema[pColIndex->columnIndex].bytes;
+      p->colType = pSchema[pColIndex->columnIndex].type;
+      snprintf(p->colInfo.name, len, "%s.%s", pTableMetaInfo->aliasName, pSchema[pColIndex->columnIndex].name);
+    } else if (pTableMetaInfo->pTableMeta != NULL) {
+      // in handling select database/version/server_status(), the pTableMeta is NULL
+      SSchema* pSchema = getOneColumnSchema(pTableMetaInfo->pTableMeta, pColIndex->columnIndex);
+      p->colInfo.colId = pSchema->colId;
+      p->colBytes = pSchema->bytes;
+      p->colType  = pSchema->type;
+      snprintf(p->colInfo.name, len, "%s.%s", pTableMetaInfo->aliasName, pSchema->name);
+    }
+  }
+
+  p->colInfo.flag     = colType;
+  p->colInfo.colIndex = pColIndex->columnIndex;
+
+  p->resType       = type;
+  p->resBytes      = size;
+  p->resColId      = resColId;
+  p->interBytes    = interSize;
+
+  if (pTableMetaInfo->pTableMeta) {
+    p->uid = pTableMetaInfo->pTableMeta->uid;
+  }
+
+  return pExpr;
+}
+
+void addExprInfo(SQueryStmtInfo* pQueryInfo, int32_t index, SExprInfo* pExprInfo) {
+  assert(pQueryInfo != NULL && pQueryInfo->exprList != NULL);
+
+  int32_t num = (int32_t) taosArrayGetSize(pQueryInfo->exprList);
+  if (index == num) {
+    taosArrayPush(pQueryInfo->exprList, &pExprInfo);
+  } else {
+    taosArrayInsert(pQueryInfo->exprList, index, &pExprInfo);
+  }
+}
+
+void updateExprInfo(SExprInfo* pExprInfo, int16_t functionId, int32_t colId, int16_t srcColumnIndex, int16_t resType, int16_t resSize) {
+  assert(pExprInfo != NULL);
+
+  SSqlExpr* pse = &pExprInfo->base;
+  pse->functionId = functionId;
+
+  pse->colInfo.colIndex = srcColumnIndex;
+  pse->colInfo.colId = colId;
+  pse->resType  = resType;
+  pse->resBytes = resSize;
+}
+
+SExprInfo* getExprInfo(SQueryStmtInfo* pQueryInfo, int32_t index) {
+  assert(pQueryInfo != NULL && pQueryInfo->exprList && index >= 0);
+  return taosArrayGetP(pQueryInfo->exprList, index);
+}
+
+void destroyExprInfo(SArray* pExprInfo) {
+  size_t size = taosArrayGetSize(pExprInfo);
+
+  for(int32_t i = 0; i < size; ++i) {
+    SExprInfo* pExpr = taosArrayGetP(pExprInfo, i);
+    tSqlExprDestroy(&pExpr->base);
+  }
+
+  taosArrayDestroy(pExprInfo);
+}
+
+void addExprParam(SSqlExpr* pExpr, char* argument, int32_t type, int32_t bytes) {
+  assert (pExpr != NULL || argument != NULL || bytes != 0);
+
+  // set parameter value
+  // transfer to tVariant from byte data/no ascii data
+  taosVariantCreateFromBinary(&pExpr->param[pExpr->numOfParams], argument, bytes, type);
+  pExpr->numOfParams += 1;
+
+  assert(pExpr->numOfParams <= 3);
+}
+
+void assignExprInfo(SExprInfo* dst, const SExprInfo* src) {
+  assert(dst != NULL && src != NULL);
+
+  *dst = *src;
+
+  if (src->base.flist.numOfFilters > 0) {
+    dst->base.flist.filterInfo = calloc(src->base.flist.numOfFilters, sizeof(SColumnFilterInfo));
+    memcpy(dst->base.flist.filterInfo, src->base.flist.filterInfo, sizeof(SColumnFilterInfo) * src->base.flist.numOfFilters);
+  }
+
+  assert(0);
+//  dst->pExpr = exprdup(src->pExpr);
+  memset(dst->base.param, 0, sizeof(SVariant) * tListLen(dst->base.param));
+  for (int32_t j = 0; j < src->base.numOfParams; ++j) {
+    taosVariantAssign(&dst->base.param[j], &src->base.param[j]);
+  }
+}
+
+int32_t copyOneExprInfo(SArray* dst, const SArray* src, uint64_t uid, bool deepcopy) {
+  assert(src != NULL && dst != NULL);
+
+  size_t size = taosArrayGetSize(src);
+  for (int32_t i = 0; i < size; ++i) {
+    SExprInfo* pExpr = taosArrayGetP(src, i);
+
+    if (pExpr->base.uid == uid) {
+      if (deepcopy) {
+        SExprInfo* p1 = calloc(1, sizeof(SExprInfo));
+        assignExprInfo(p1, pExpr);
+
+        taosArrayPush(dst, &p1);
+      } else {
+        taosArrayPush(dst, &pExpr);
+      }
+    }
+  }
+
+  return 0;
+}
+
+int32_t copyAllExprInfo(SArray* dst, const SArray* src, bool deepcopy) {
+  assert(src != NULL && dst != NULL);
+
+  size_t size = taosArrayGetSize(src);
+  for (int32_t i = 0; i < size; ++i) {
+    SExprInfo* pExpr = taosArrayGetP(src, i);
+
+    if (deepcopy) {
+      SExprInfo* p1 = calloc(1, sizeof(SExprInfo));
+      assignExprInfo(p1, pExpr);
+      taosArrayPush(dst, &p1);
+    } else {
+      taosArrayPush(dst, &pExpr);
+    }
+  }
+
+  return 0;
+}
+
+//void* tSqlExprDestroy(SExprInfo* pExpr) {
+//  if (pExpr == NULL) {
+//    return NULL;
+//  }
+//
+//  SSqlExpr* p = &pExpr->base;
+//  for(int32_t i = 0; i < tListLen(p->param); ++i) {
+//    taosVariantDestroy(&p->param[i]);
+//  }
+//
+//  if (p->flist.numOfFilters > 0) {
+//    tfree(p->flist.filterInfo);
+//  }
+//
+//  if (pExpr->pExpr != NULL) {
+//    tExprTreeDestroy(pExpr->pExpr, NULL);
+//  }
+//
+//  tfree(pExpr);
+//  return NULL;
+//}
+
+int32_t getResRowLength(SArray* pExprList) {
+  size_t num = taosArrayGetSize(pExprList);
+  if (num == 0) {
+    return 0;
+  }
+
+  int32_t size = 0;
+  for(int32_t i = 0; i < num; ++i) {
+    SExprInfo* pExpr = taosArrayGetP(pExprList, i);
+    size += pExpr->base.resBytes;
+  }
+
+  return size;
+}
+
+static void freeQueryInfoImpl(SQueryStmtInfo* pQueryInfo) {
+  cleanupTagCond(&pQueryInfo->tagCond);
+  cleanupColumnCond(&pQueryInfo->colCond);
+  cleanupFieldInfo(&pQueryInfo->fieldsInfo);
+
+  destroyExprInfo(pQueryInfo->exprList);
+  pQueryInfo->exprList = NULL;
+
+  if (pQueryInfo->exprList1 != NULL) {
+    destroyExprInfo(pQueryInfo->exprList1);
+    pQueryInfo->exprList1 = NULL;
+  }
+
+  columnListDestroy(pQueryInfo->colList);
+  pQueryInfo->colList = NULL;
+
+  if (pQueryInfo->groupbyExpr.columnInfo != NULL) {
+    taosArrayDestroy(pQueryInfo->groupbyExpr.columnInfo);
+    pQueryInfo->groupbyExpr.columnInfo = NULL;
+  }
+
+  pQueryInfo->fillType = 0;
+
+  tfree(pQueryInfo->fillVal);
+  tfree(pQueryInfo->buf);
+
+  taosArrayDestroy(pQueryInfo->pUpstream);
+  pQueryInfo->pUpstream = NULL;
+  pQueryInfo->bufLen = 0;
+}
+
+void freeQueryInfo(SQueryStmtInfo* pQueryInfo, bool removeCachedMeta, uint64_t id) {
+  while(pQueryInfo != NULL) {
+    SQueryStmtInfo* p = pQueryInfo->sibling;
+
+    size_t numOfUpstream = taosArrayGetSize(pQueryInfo->pUpstream);
+    for(int32_t i = 0; i < numOfUpstream; ++i) {
+      SQueryStmtInfo* pUpQueryInfo = taosArrayGetP(pQueryInfo->pUpstream, i);
+      freeQueryInfoImpl(pUpQueryInfo);
+      clearAllTableMetaInfo(pUpQueryInfo, removeCachedMeta, id);
+      tfree(pUpQueryInfo);
+    }
+
+    freeQueryInfoImpl(pQueryInfo);
+    clearAllTableMetaInfo(pQueryInfo, removeCachedMeta, id);
+
+    tfree(pQueryInfo);
+    pQueryInfo = p;
+  }
+}
+
+SArray* extractFunctionIdList(SArray* pExprInfoList) {
+  assert(pExprInfoList != NULL);
+
+  size_t len = taosArrayGetSize(pExprInfoList);
+  SArray* p = taosArrayInit(len, sizeof(int16_t));
+  for(int32_t i = 0; i < len; ++i) {
+    SExprInfo* pExprInfo = taosArrayGetP(pExprInfoList, i);
+    taosArrayPush(p, &pExprInfo->base.functionId);
+  }
+
+  return p;
+}
+
+bool tscIsProjectionQueryOnSTable(SQueryStmtInfo* pQueryInfo, int32_t tableIndex);
+
+bool tscNonOrderedProjectionQueryOnSTable(SQueryStmtInfo* pQueryInfo, int32_t tableIndex) {
+  if (!tscIsProjectionQueryOnSTable(pQueryInfo, tableIndex)) {
+    return false;
+  }
+
+  // order by columnIndex exists, not a non-ordered projection query
+  return pQueryInfo->order.orderColId < 0;
+}
+
+// not order by timestamp projection query on super table
+bool tscOrderedProjectionQueryOnSTable(SQueryStmtInfo* pQueryInfo, int32_t tableIndex) {
+  if (!tscIsProjectionQueryOnSTable(pQueryInfo, tableIndex)) {
+    return false;
+  }
+
+  // order by columnIndex exists, a non-ordered projection query
+  return pQueryInfo->order.orderColId >= 0;
+}
+
+bool tscHasColumnFilter(SQueryStmtInfo* pQueryInfo) {
+  // filter on primary timestamp column
+  if (pQueryInfo->window.skey != INT64_MIN || pQueryInfo->window.ekey != INT64_MAX) {
+    return true;
+  }
+
+  size_t size = taosArrayGetSize(pQueryInfo->colList);
+  for (int32_t i = 0; i < size; ++i) {
+    SColumn* pCol = taosArrayGetP(pQueryInfo->colList, i);
+    if (pCol->info.flist.numOfFilters > 0) {
+      return true;
+    }
+  }
+
+  return false;
+}
+
+//void tscClearInterpInfo(SQueryStmtInfo* pQueryInfo) {
+//  if (!tscIsPointInterpQuery(pQueryInfo)) {
+//    return;
+//  }
+//
+//  pQueryInfo->fillType = TSDB_FILL_NONE;
+//  tfree(pQueryInfo->fillVal);
+//}
\ No newline at end of file

source/util/src/tpagedfile.c

+#include "tpagedfile.h"
+#include "thash.h"
+#include "stddef.h"
+#include "taoserror.h"
+#include "tcompression.h"
+
+#define GET_DATA_PAYLOAD(_p) ((char *)(_p)->pData + POINTER_BYTES)
+#define NO_IN_MEM_AVAILABLE_PAGES(_b) (listNEles((_b)->lruList) >= (_b)->inMemPages)
+
+int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t pagesize, int32_t inMemBufSize, uint64_t qId) {
+  *pResultBuf = calloc(1, sizeof(SDiskbasedResultBuf));
+
+  SDiskbasedResultBuf* pResBuf = *pResultBuf;
+  if (pResBuf == NULL) {
+    return TSDB_CODE_COM_OUT_OF_MEMORY;  
+  }
+
+  pResBuf->pageSize     = pagesize;
+  pResBuf->numOfPages   = 0;                        // all pages are in buffer in the first place
+  pResBuf->totalBufSize = 0;
+  pResBuf->inMemPages   = inMemBufSize/pagesize;    // maximum allowed pages, it is a soft limit.
+  pResBuf->allocateId   = -1;
+  pResBuf->comp         = true;
+  pResBuf->file         = NULL;
+  pResBuf->qId          = qId;
+  pResBuf->fileSize     = 0;
+
+  // at least more than 2 pages must be in memory
+  assert(inMemBufSize >= pagesize * 2);
+
+  pResBuf->lruList = tdListNew(POINTER_BYTES);
+
+  // init id hash table
+  pResBuf->groupSet  = taosHashInit(10, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false);
+  pResBuf->assistBuf = malloc(pResBuf->pageSize + 2); // EXTRA BYTES
+  pResBuf->all = taosHashInit(10, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false);
+
+  char path[PATH_MAX] = {0};
+  taosGetTmpfilePath("qbuf", path);
+  pResBuf->path = strdup(path);
+
+  pResBuf->emptyDummyIdList = taosArrayInit(1, sizeof(int32_t));
+
+//  qDebug("QInfo:0x%"PRIx64" create resBuf for output, page size:%d, inmem buf pages:%d, file:%s", qId, pResBuf->pageSize,
+//         pResBuf->inMemPages, pResBuf->path);
+
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t createDiskFile(SDiskbasedResultBuf* pResultBuf) {
+  pResultBuf->file = fopen(pResultBuf->path, "wb+");
+  if (pResultBuf->file == NULL) {
+//    qError("failed to create tmp file: %s on disk. %s", pResultBuf->path, strerror(errno));
+    return TAOS_SYSTEM_ERROR(errno);
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+static char* doCompressData(void* data, int32_t srcSize, int32_t *dst, SDiskbasedResultBuf* pResultBuf) { // do nothing
+  if (!pResultBuf->comp) {
+    *dst = srcSize;
+    return data;
+  }
+
+  *dst = tsCompressString(data, srcSize, 1, pResultBuf->assistBuf, srcSize, ONE_STAGE_COMP, NULL, 0);
+
+  memcpy(data, pResultBuf->assistBuf, *dst);
+  return data;
+}
+
+static char* doDecompressData(void* data, int32_t srcSize, int32_t *dst, SDiskbasedResultBuf* pResultBuf) { // do nothing
+  if (!pResultBuf->comp) {
+    *dst = srcSize;
+    return data;
+  }
+
+  *dst = tsDecompressString(data, srcSize, 1, pResultBuf->assistBuf, pResultBuf->pageSize, ONE_STAGE_COMP, NULL, 0);
+  if (*dst > 0) {
+    memcpy(data, pResultBuf->assistBuf, *dst);
+  }
+  return data;
+}
+
+static int32_t allocatePositionInFile(SDiskbasedResultBuf* pResultBuf, size_t size) {
+  if (pResultBuf->pFree == NULL) {
+    return pResultBuf->nextPos;
+  } else {
+    int32_t offset = -1;
+
+    size_t num = taosArrayGetSize(pResultBuf->pFree);
+    for(int32_t i = 0; i < num; ++i) {
+      SFreeListItem* pi = taosArrayGet(pResultBuf->pFree, i);
+      if (pi->len >= size) {
+        offset = pi->offset;
+        pi->offset += (int32_t)size;
+        pi->len -= (int32_t)size;
+
+        return offset;
+      }
+    }
+
+    // no available recycle space, allocate new area in file
+    return pResultBuf->nextPos;
+  }
+}
+
+static char* doFlushPageToDisk(SDiskbasedResultBuf* pResultBuf, SPageInfo* pg) {
+  assert(!pg->used && pg->pData != NULL);
+
+  int32_t size = -1;
+  char* t = doCompressData(GET_DATA_PAYLOAD(pg), pResultBuf->pageSize, &size, pResultBuf);
+
+  // this page is flushed to disk for the first time
+  if (pg->info.offset == -1) {
+    pg->info.offset = allocatePositionInFile(pResultBuf, size);
+    pResultBuf->nextPos += size;
+
+    int32_t ret = fseek(pResultBuf->file, pg->info.offset, SEEK_SET);
+    assert(ret == 0);
+
+    ret = (int32_t) fwrite(t, 1, size, pResultBuf->file);
+    assert(ret == size);
+
+    if (pResultBuf->fileSize < pg->info.offset + pg->info.length) {
+      pResultBuf->fileSize = pg->info.offset + pg->info.length;
+    }
+  } else {
+    // length becomes greater, current space is not enough, allocate new place, otherwise, do nothing
+    if (pg->info.length < size) {
+      // 1. add current space to free list
+      taosArrayPush(pResultBuf->pFree, &pg->info);
+
+      // 2. allocate new position, and update the info
+      pg->info.offset = allocatePositionInFile(pResultBuf, size);
+      pResultBuf->nextPos += size;
+    }
+
+    //3. write to disk.
+    int32_t ret = fseek(pResultBuf->file, pg->info.offset, SEEK_SET);
+    if (ret != 0) {  // todo handle the error case
+
+    }
+
+    ret = (int32_t)fwrite(t, size, 1, pResultBuf->file);
+    if (ret != size) {  // todo handle the error case
+
+    }
+
+    if (pResultBuf->fileSize < pg->info.offset + pg->info.length) {
+      pResultBuf->fileSize = pg->info.offset + pg->info.length;
+    }
+  }
+
+  char* ret = pg->pData;
+  memset(ret, 0, pResultBuf->pageSize);
+
+  pg->pData = NULL;
+  pg->info.length = size;
+
+  pResultBuf->statis.flushBytes += pg->info.length;
+
+  return ret;
+}
+
+static char* flushPageToDisk(SDiskbasedResultBuf* pResultBuf, SPageInfo* pg) {
+  int32_t ret = TSDB_CODE_SUCCESS;
+  assert(((int64_t) pResultBuf->numOfPages * pResultBuf->pageSize) == pResultBuf->totalBufSize && pResultBuf->numOfPages >= pResultBuf->inMemPages);
+
+  if (pResultBuf->file == NULL) {
+    if ((ret = createDiskFile(pResultBuf)) != TSDB_CODE_SUCCESS) {
+      terrno = ret;
+      return NULL;
+    }
+  }
+
+  return doFlushPageToDisk(pResultBuf, pg);
+}
+
+// load file block data in disk
+static char* loadPageFromDisk(SDiskbasedResultBuf* pResultBuf, SPageInfo* pg) {
+  int32_t ret = fseek(pResultBuf->file, pg->info.offset, SEEK_SET);
+  ret = (int32_t)fread(GET_DATA_PAYLOAD(pg), 1, pg->info.length, pResultBuf->file);
+  if (ret != pg->info.length) {
+    terrno = errno;
+    return NULL;
+  }
+
+  pResultBuf->statis.loadBytes += pg->info.length;
+
+  int32_t fullSize = 0;
+  doDecompressData(GET_DATA_PAYLOAD(pg), pg->info.length, &fullSize, pResultBuf);
+
+  return (char*)GET_DATA_PAYLOAD(pg);
+}
+
+static SIDList addNewGroup(SDiskbasedResultBuf* pResultBuf, int32_t groupId) {
+  assert(taosHashGet(pResultBuf->groupSet, (const char*) &groupId, sizeof(int32_t)) == NULL);
+
+  SArray* pa = taosArrayInit(1, POINTER_BYTES);
+  int32_t ret = taosHashPut(pResultBuf->groupSet, (const char*)&groupId, sizeof(int32_t), &pa, POINTER_BYTES);
+  assert(ret == 0);
+
+  return pa;
+}
+
+static SPageInfo* registerPage(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int32_t pageId) {
+  SIDList list = NULL;
+
+  char** p = taosHashGet(pResultBuf->groupSet, (const char*)&groupId, sizeof(int32_t));
+  if (p == NULL) {  // it is a new group id
+    list = addNewGroup(pResultBuf, groupId);
+  } else {
+    list = (SIDList) (*p);
+  }
+
+  pResultBuf->numOfPages += 1;
+
+  SPageInfo* ppi = malloc(sizeof(SPageInfo));//{ .info = PAGE_INFO_INITIALIZER, .pageId = pageId, .pn = NULL};
+
+  ppi->pageId = pageId;
+  ppi->pData  = NULL;
+  ppi->info   = PAGE_INFO_INITIALIZER;
+  ppi->used   = true;
+  ppi->pn     = NULL;
+
+  return *(SPageInfo**) taosArrayPush(list, &ppi);
+}
+
+static SListNode* getEldestUnrefedPage(SDiskbasedResultBuf* pResultBuf) {
+  SListIter iter = {0};
+  tdListInitIter(pResultBuf->lruList, &iter, TD_LIST_BACKWARD);
+
+  SListNode* pn = NULL;
+  while((pn = tdListNext(&iter)) != NULL) {
+    assert(pn != NULL);
+
+    SPageInfo* pageInfo = *(SPageInfo**) pn->data;
+    assert(pageInfo->pageId >= 0 && pageInfo->pn == pn);
+
+    if (!pageInfo->used) {
+      break;
+    }
+  }
+
+  return pn;
+}
+
+static char* evicOneDataPage(SDiskbasedResultBuf* pResultBuf) {
+  char* bufPage = NULL;
+  SListNode* pn = getEldestUnrefedPage(pResultBuf);
+
+  // all pages are referenced by user, try to allocate new space
+  if (pn == NULL) {
+    int32_t prev = pResultBuf->inMemPages;
+
+    // increase by 50% of previous mem pages
+    pResultBuf->inMemPages = (int32_t)(pResultBuf->inMemPages * 1.5f);
+
+//    qWarn("%p in memory buf page not sufficient, expand from %d to %d, page size:%d", pResultBuf, prev,
+//          pResultBuf->inMemPages, pResultBuf->pageSize);
+  } else {
+    pResultBuf->statis.flushPages += 1;
+    tdListPopNode(pResultBuf->lruList, pn);
+
+    SPageInfo* d = *(SPageInfo**) pn->data;
+    assert(d->pn == pn);
+
+    d->pn = NULL;
+    tfree(pn);
+
+    bufPage = flushPageToDisk(pResultBuf, d);
+  }
+
+  return bufPage;
+}
+
+static void lruListPushFront(SList *pList, SPageInfo* pi) {
+  tdListPrepend(pList, &pi);
+  SListNode* front = tdListGetHead(pList);
+  pi->pn = front;
+}
+
+static void lruListMoveToFront(SList *pList, SPageInfo* pi) {
+  tdListPopNode(pList, pi->pn);
+  tdListPrependNode(pList, pi->pn);
+}
+
+static FORCE_INLINE size_t getAllocPageSize(int32_t pageSize) {
+  return pageSize + POINTER_BYTES + 2 + sizeof(SFilePage);
+}
+
+SFilePage* getNewDataBuf(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int32_t* pageId) {
+  pResultBuf->statis.getPages += 1;
+
+  char* availablePage = NULL;
+  if (NO_IN_MEM_AVAILABLE_PAGES(pResultBuf)) {
+    availablePage = evicOneDataPage(pResultBuf);
+  }
+
+  // register new id in this group
+  *pageId = (++pResultBuf->allocateId);
+
+  // register page id info
+  SPageInfo* pi = registerPage(pResultBuf, groupId, *pageId);
+
+  // add to LRU list
+  assert(listNEles(pResultBuf->lruList) < pResultBuf->inMemPages && pResultBuf->inMemPages > 0);
+
+  lruListPushFront(pResultBuf->lruList, pi);
+
+  // add to hash map
+  taosHashPut(pResultBuf->all, pageId, sizeof(int32_t), &pi, POINTER_BYTES);
+
+  // allocate buf
+  if (availablePage == NULL) {
+    pi->pData = calloc(1, getAllocPageSize(pResultBuf->pageSize));  // add extract bytes in case of zipped buffer increased.
+  } else {
+    pi->pData = availablePage;
+  }
+
+  pResultBuf->totalBufSize += pResultBuf->pageSize;
+
+  ((void**)pi->pData)[0] = pi;
+  pi->used = true;
+
+  return (void *)(GET_DATA_PAYLOAD(pi));
+}
+
+SFilePage* getResBufPage(SDiskbasedResultBuf* pResultBuf, int32_t id) {
+  assert(pResultBuf != NULL && id >= 0);
+  pResultBuf->statis.getPages += 1;
+
+  SPageInfo** pi = taosHashGet(pResultBuf->all, &id, sizeof(int32_t));
+  assert(pi != NULL && *pi != NULL);
+
+  if ((*pi)->pData != NULL) { // it is in memory
+    // no need to update the LRU list if only one page exists
+    if (pResultBuf->numOfPages == 1) {
+      (*pi)->used = true;
+      return (void *)(GET_DATA_PAYLOAD(*pi));
+    }
+
+    SPageInfo** pInfo = (SPageInfo**) ((*pi)->pn->data);
+    assert(*pInfo == *pi);
+
+    lruListMoveToFront(pResultBuf->lruList, (*pi));
+    (*pi)->used = true;
+
+    return (void *)(GET_DATA_PAYLOAD(*pi));
+
+  } else { // not in memory
+    assert((*pi)->pData == NULL && (*pi)->pn == NULL && (*pi)->info.length >= 0 && (*pi)->info.offset >= 0);
+
+    char* availablePage = NULL;
+    if (NO_IN_MEM_AVAILABLE_PAGES(pResultBuf)) {
+      availablePage = evicOneDataPage(pResultBuf);
+    }
+
+    if (availablePage == NULL) {
+      (*pi)->pData = calloc(1, getAllocPageSize(pResultBuf->pageSize));
+    } else {
+      (*pi)->pData = availablePage;
+    }
+
+    ((void**)((*pi)->pData))[0] = (*pi);
+
+    lruListPushFront(pResultBuf->lruList, *pi);
+    (*pi)->used = true;
+
+    loadPageFromDisk(pResultBuf, *pi);
+    return (void *)(GET_DATA_PAYLOAD(*pi));
+  }
+}
+
+void releaseResBufPage(SDiskbasedResultBuf* pResultBuf, void* page) {
+  assert(pResultBuf != NULL && page != NULL);
+  char* p = (char*) page - POINTER_BYTES;
+
+  SPageInfo* ppi = ((SPageInfo**) p)[0];
+  releaseResBufPageInfo(pResultBuf, ppi);
+}
+
+void releaseResBufPageInfo(SDiskbasedResultBuf* pResultBuf, SPageInfo* pi) {
+  assert(pi->pData != NULL && pi->used);
+
+  pi->used = false;
+  pResultBuf->statis.releasePages += 1;
+}
+
+size_t getNumOfResultBufGroupId(const SDiskbasedResultBuf* pResultBuf) { return taosHashGetSize(pResultBuf->groupSet); }
+
+size_t getResBufSize(const SDiskbasedResultBuf* pResultBuf) { return (size_t)pResultBuf->totalBufSize; }
+
+SIDList getDataBufPagesIdList(SDiskbasedResultBuf* pResultBuf, int32_t groupId) {
+  assert(pResultBuf != NULL);
+
+  char** p = taosHashGet(pResultBuf->groupSet, (const char*)&groupId, sizeof(int32_t));
+  if (p == NULL) {  // it is a new group id
+    return pResultBuf->emptyDummyIdList;
+  } else {
+    return (SArray*) (*p);
+  }
+}
+
+void destroyResultBuf(SDiskbasedResultBuf* pResultBuf) {
+  if (pResultBuf == NULL) {
+    return;
+  }
+
+  if (pResultBuf->file != NULL) {
+//    qDebug("QInfo:0x%"PRIx64" res output buffer closed, total:%.2f Kb, inmem size:%.2f Kb, file size:%.2f Kb",
+//        pResultBuf->qId, pResultBuf->totalBufSize/1024.0, listNEles(pResultBuf->lruList) * pResultBuf->pageSize / 1024.0,
+//        pResultBuf->fileSize/1024.0);
+
+    fclose(pResultBuf->file);
+  } else {
+//    qDebug("QInfo:0x%"PRIx64" res output buffer closed, total:%.2f Kb, no file created", pResultBuf->qId,
+//           pResultBuf->totalBufSize/1024.0);
+  }
+
+  remove(pResultBuf->path);
+  tfree(pResultBuf->path);
+
+  SArray** p = taosHashIterate(pResultBuf->groupSet, NULL);
+  while(p) {
+    size_t n = taosArrayGetSize(*p);
+    for(int32_t i = 0; i < n; ++i) {
+      SPageInfo* pi = taosArrayGetP(*p, i);
+      tfree(pi->pData);
+      tfree(pi);
+    }
+
+    taosArrayDestroy(*p);
+    p = taosHashIterate(pResultBuf->groupSet, p);
+  }
+
+  tdListFree(pResultBuf->lruList);
+  taosArrayDestroy(pResultBuf->emptyDummyIdList);
+  taosHashCleanup(pResultBuf->groupSet);
+  taosHashCleanup(pResultBuf->all);
+
+  tfree(pResultBuf->assistBuf);
+  tfree(pResultBuf);
+}
+
+SPageInfo* getLastPageInfo(SIDList pList) {
+  size_t size = taosArrayGetSize(pList);
+  return (SPageInfo*) taosArrayGetP(pList, size - 1);
+}
+