Merge remote-tracking branch 'origin/3.0' into feature/check

2.0/src/query/inc/qExecutor.h

@@ -179,7 +179,7 @@ typedef struct SQueryCostInfo {
   uint32_t totalBlocks;
   uint32_t loadBlocks;
   uint32_t loadBlockStatis;
-  uint32_t discardBlocks;
+  uint32_t skipBlocks;
   uint64_t elapsedTime;
   uint64_t firstStageMergeTime;
   uint64_t winInfoSize;

2.0/src/query/src/qExecutor.c

@@ -3144,7 +3144,7 @@ int32_t loadDataBlockOnDemand(SQueryRuntimeEnv* pRuntimeEnv, STableScanInfo* pTa
   if ((*status) == BLK_DATA_NO_NEEDED || (*status) == BLK_DATA_DISCARD) {
     qDebug("QInfo:0x%"PRIx64" data block discard, brange:%" PRId64 "-%" PRId64 ", rows:%d", pQInfo->qId, pBlockInfo->window.skey,
            pBlockInfo->window.ekey, pBlockInfo->rows);
-    pCost->discardBlocks += 1;
+    pCost->skipBlocks += 1;
   } else if ((*status) == BLK_DATA_STATIS_NEEDED) {
     // this function never returns error?
     pCost->loadBlockStatis += 1;
@@ -3184,7 +3184,7 @@ int32_t loadDataBlockOnDemand(SQueryRuntimeEnv* pRuntimeEnv, STableScanInfo* pTa
           load = topbot_datablock_filter(&pTableScanInfo->pCtx[i], (char*)&(pBlock->pBlockStatis[i].min),
                                          (char*)&(pBlock->pBlockStatis[i].max));
           if (!load) { // current block has been discard due to filter applied
-            pCost->discardBlocks += 1;
+            pCost->skipBlocks += 1;
             qDebug("QInfo:0x%"PRIx64" data block discard, brange:%" PRId64 "-%" PRId64 ", rows:%d", pQInfo->qId,
                    pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows);
             (*status) = BLK_DATA_DISCARD;
@@ -3196,7 +3196,7 @@ int32_t loadDataBlockOnDemand(SQueryRuntimeEnv* pRuntimeEnv, STableScanInfo* pTa
 
     // current block has been discard due to filter applied
     if (!doFilterByBlockStatistics(pRuntimeEnv, pBlock->pBlockStatis, pTableScanInfo->pCtx, pBlockInfo->rows)) {
-      pCost->discardBlocks += 1;
+      pCost->skipBlocks += 1;
       qDebug("QInfo:0x%"PRIx64" data block discard, brange:%" PRId64 "-%" PRId64 ", rows:%d", pQInfo->qId, pBlockInfo->window.skey,
              pBlockInfo->window.ekey, pBlockInfo->rows);
       (*status) = BLK_DATA_DISCARD;

CMakeLists.txt

@@ -17,6 +17,7 @@ include(${TD_SUPPORT_DIR}/cmake.platform)
 include(${TD_SUPPORT_DIR}/cmake.define)
 include(${TD_SUPPORT_DIR}/cmake.options)
 include(${TD_SUPPORT_DIR}/cmake.version)
+include(${TD_SUPPORT_DIR}/cmake.install)
 
 # contrib
 add_subdirectory(contrib)

cmake/cmake.define

 cmake_minimum_required(VERSION 3.16)
 
+#set output directory
+SET(LIBRARY_OUTPUT_PATH ${PROJECT_BINARY_DIR}/build/lib)
+SET(EXECUTABLE_OUTPUT_PATH ${PROJECT_BINARY_DIR}/build/bin)
+SET(TD_TESTS_OUTPUT_DIR ${PROJECT_BINARY_DIR}/test)
+
+MESSAGE(STATUS "Project source directory: " ${PROJECT_SOURCE_DIR})
+MESSAGE(STATUS "Project binary files output path: " ${PROJECT_BINARY_DIR})
+MESSAGE(STATUS "Project executable files output path: " ${EXECUTABLE_OUTPUT_PATH})
+MESSAGE(STATUS "Project library files output path: " ${LIBRARY_OUTPUT_PATH})
+
 if (NOT DEFINED TD_GRANT)
   SET(TD_GRANT FALSE)
 endif()
-if (NOT DEFINED TD_USB_DONGLE)
-  SET(TD_USB_DONGLE FALSE)
-endif()
-IF (TD_GRANT)
-  ADD_DEFINITIONS(-D_GRANT)
-ENDIF ()
 
 IF ("${BUILD_TOOLS}" STREQUAL "")
   IF (TD_LINUX)

cmake/cmake.install

+IF (TD_LINUX)
+  SET(TD_MAKE_INSTALL_SH "${TD_SOURCE_DIR}/packaging/make_install.sh")
+  INSTALL(CODE "MESSAGE(\"make install script: ${TD_MAKE_INSTALL_SH}\")")
+  INSTALL(CODE "execute_process(COMMAND bash ${TD_MAKE_INSTALL_SH} ${TD_SOURCE_DIR} ${PROJECT_BINARY_DIR} Linux ${TD_VER_NUMBER})")
+ELSEIF (TD_WINDOWS)
+  SET(CMAKE_INSTALL_PREFIX C:/TDengine)
+
+  INSTALL(DIRECTORY ${TD_SOURCE_DIR}/src/connector/go     DESTINATION connector)
+  INSTALL(DIRECTORY ${TD_SOURCE_DIR}/src/connector/nodejs DESTINATION connector)
+  INSTALL(DIRECTORY ${TD_SOURCE_DIR}/src/connector/python DESTINATION connector)
+  INSTALL(DIRECTORY ${TD_SOURCE_DIR}/src/connector/C\# DESTINATION connector)
+  INSTALL(DIRECTORY ${TD_SOURCE_DIR}/examples DESTINATION .)
+  INSTALL(FILES ${TD_SOURCE_DIR}/packaging/cfg/taos.cfg DESTINATION cfg)
+  INSTALL(FILES ${TD_SOURCE_DIR}/src/inc/taos.h DESTINATION include)
+  INSTALL(FILES ${TD_SOURCE_DIR}/src/inc/taoserror.h DESTINATION include)
+  INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos.lib DESTINATION driver)
+  INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos_static.lib DESTINATION driver)
+  INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos.exp DESTINATION driver)
+  INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos.dll DESTINATION driver)
+
+  IF (TD_MVN_INSTALLED)
+    INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-2.0.38-dist.jar DESTINATION connector/jdbc)
+  ENDIF ()
+ELSEIF (TD_DARWIN)
+  SET(TD_MAKE_INSTALL_SH "${TD_SOURCE_DIR}/packaging/tools/make_install.sh")
+  INSTALL(CODE "MESSAGE(\"make install script: ${TD_MAKE_INSTALL_SH}\")")
+  INSTALL(CODE "execute_process(COMMAND bash ${TD_MAKE_INSTALL_SH} ${TD_SOURCE_DIR} ${PROJECT_BINARY_DIR} Darwin ${TD_VER_NUMBER})")
+ENDIF ()

example/CMakeLists.txt

@@ -18,15 +18,15 @@ target_sources(demoapi
 )
 
 target_link_libraries(tmq
-    taos
+    taos_static
 )
 
 target_link_libraries(tstream
-    taos
+    taos_static
 )
 
 target_link_libraries(demoapi
-    taos
+    taos_static
 )
 
 target_include_directories(tmq

example/src/demoapi.c

@@ -30,7 +30,7 @@
         fprintf(stderr, "\033[0m"); } while(0)
 
 int64_t g_num_of_tb = 2;
-int64_t g_num_of_rec = 2;
+int64_t g_num_of_rec = 3;
 
 static struct argp_option options[] = {
     {"tables", 't', "NUMBER", 0, "Number of child tables, default is 10000."},
@@ -42,10 +42,18 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
     switch (key) {
         case 't':
             g_num_of_tb = atoll(arg);
+            if (g_num_of_tb < 1) {
+                warnPrint("minimal g_num_of_tb is %d\n", 1);
+                g_num_of_tb = 1;
+            }
             break;
 
         case 'n':
             g_num_of_rec = atoll(arg);
+            if (g_num_of_rec < 2) {
+                warnPrint("minimal g_num_of_rec is %d\n", 2);
+                g_num_of_rec = 2;
+            }
             break;
     }
 
@@ -65,15 +73,32 @@ static void prepare_data(TAOS* taos) {
     usleep(100000);
     taos_select_db(taos, "test");
 
-    res = taos_query(taos, "create table meters(ts timestamp, f float, n int, b binary(20), c nchar(20)) tags(area int, city binary(20), dist nchar(20));");
+    char command[1024] = {0};
+    sprintf(command, "%s", "create table meters(ts timestamp, f float, n int, bin1 binary(20), c nchar(20), bin2 binary(20)) tags(area int, city binary(20), dist nchar(20), street binary(20));");
+    res = taos_query(taos, command);
+    if ((res) && (0 == taos_errno(res))) {
+        okPrint("%s created\n", "meters");
+    } else {
+        errorPrint("%s() LN%d: %s\n",
+                __func__, __LINE__, taos_errstr(res));
+        taos_free_result(res);
+        exit(1);
+    }
     taos_free_result(res);
 
-    char command[1024] = {0};
     for (int64_t i = 0; i < g_num_of_tb; i ++) {
-//        sprintf(command, "create table t%"PRId64" using meters tags(%"PRId64", '%s', '%s');",
-//                i, i, (i%2)?"beijing":"shanghai", (i%2)?"朝阳区":"黄浦区");
-        sprintf(command, "create table t%"PRId64" using meters tags(%"PRId64", '%s', '%s');",
-                i, i, (i%2)?"beijing":"shanghai", (i%2)?"chaoyang":"huangpu");
+        sprintf(command, "create table t%"PRId64" using meters "
+                "tags(%"PRId64", '%s', '%s', '%s');",
+                i, i, (i%2)?"beijing":"shanghai",
+                (i%2)?"朝阳区":"黄浦区",
+                (i%2)?"长安街":"中山路");
+/*        sprintf(command, "create table t%"PRId64" using meters "
+                "tags(%"PRId64", '%s', '%s', '%s');",
+                i, i,
+                (i%2)?"beijing":"shanghai",
+                (i%2)?"chaoyang":"huangpu",
+                (i%2?"changan street":"jianguo rd"));
+                */
         res = taos_query(taos,  command);
         if ((res) && (0 == taos_errno(res))) {
             okPrint("t%" PRId64 " created\n", i);
@@ -86,11 +111,15 @@ static void prepare_data(TAOS* taos) {
         int64_t j = 0;
         int64_t total = 0;
         int64_t affected;
-        for (; j < g_num_of_rec -1; j ++) {
+        for (; j < g_num_of_rec -2; j ++) {
             sprintf(command, "insert into t%"PRId64" "
-                    "values(%" PRId64 ", %f, %"PRId64", '%c%d', '%c%d')",
-                    i, 1650000000000+j, (float)j, j, 'a'+(int)j%25, rand(),
-                    'z' - (int)j%25, rand());
+                    "values(%" PRId64 ", %f, %"PRId64", "
+                    "'%c%d', '%s%c%d', '%c%d')",
+                    i, 1650000000000+j, (float)j, j,
+                    'a'+(int)j%25, rand(),
+                    "涛思", 'z' - (int)j%25, rand(),
+                    'b' - (int)j%25, rand()
+                    );
             res = taos_query(taos,  command);
             if ((res) && (0 == taos_errno(res))) {
                 affected = taos_affected_rows(res);
@@ -101,8 +130,25 @@ static void prepare_data(TAOS* taos) {
             }
             taos_free_result(res);
         }
-        sprintf(command, "insert into t%"PRId64" values(%" PRId64 ", NULL, NULL, NULL, NULL)",
-                i, 1650000000000+j+1);
+        sprintf(command, "insert into t%"PRId64" values(%" PRId64 ", "
+                "NULL, NULL, NULL, NULL, NULL)",
+                i, 1650000000000+j);
+        res = taos_query(taos,  command);
+        if ((res) && (0 == taos_errno(res))) {
+            affected = taos_affected_rows(res);
+            total += affected;
+        } else {
+            errorPrint("%s() LN%d: %s\n",
+                    __func__, __LINE__, taos_errstr(res));
+        }
+        sprintf(command, "insert into t%"PRId64" "
+                "values(%" PRId64 ", %f, %"PRId64", "
+                "'%c%d', '%s%c%d', '%c%d')",
+                i, 1650000000000+j+1, (float)j, j,
+                'a'+(int)j%25, rand(),
+                "数据", 'z' - (int)j%25, rand(),
+                'b' - (int)j%25, rand()
+               );
         res = taos_query(taos,  command);
         if ((res) && (0 == taos_errno(res))) {
             affected = taos_affected_rows(res);
@@ -113,7 +159,8 @@ static void prepare_data(TAOS* taos) {
         }
         taos_free_result(res);
 
-        printf("insert %"PRId64" records into t%"PRId64", total affected rows: %"PRId64"\n", j, i, total);
+        okPrint("insert %"PRId64" records into t%"PRId64", "
+                "total affected rows: %"PRId64"\n", j, i, total);
     }
 }
 
@@ -127,29 +174,107 @@ static int print_result(char *tbname, TAOS_RES* res, int block) {
         printf("fields[%d].name=%s, fields[%d].type=%d, fields[%d].bytes=%d\n",
                 f, fields[f].name, f, fields[f].type, f, fields[f].bytes);
     }
+
     if (block) {
-        warnPrint("%s() LN%d, call taos_fetch_block()\n", __func__, __LINE__);
+        warnPrint("%s", "call taos_fetch_block()\n");
         int rows = 0;
         while ((rows = taos_fetch_block(res, &row))) {
+            int *lengths = taos_fetch_lengths(res);
+            for (int f = 0; f < num_fields; f++) {
+                if ((fields[f].type != TSDB_DATA_TYPE_VARCHAR)
+                        && (fields[f].type != TSDB_DATA_TYPE_NCHAR)
+                        && (fields[f].type != TSDB_DATA_TYPE_JSON)) {
+                    printf("col%d type is %d, no need get offset\n",
+                            f, fields[f].type);
+                    for (int64_t c = 0; c < rows; c++) {
+                        switch(fields[f].type) {
+                            case TSDB_DATA_TYPE_TIMESTAMP:
+                                if (taos_is_null(res, c, f)) {
+                                    printf("col%d, row: %"PRId64" "
+                                            "value: NULL\n", f, c);
+                                } else {
+                                    printf("col%d, row: %"PRId64", "
+                                            "value: %"PRId64"\n",
+                                            f, c,
+                                            *(int64_t*)(row[f]+c*sizeof(int64_t)));
+                                }
+                                break;
+
+                            case TSDB_DATA_TYPE_INT:
+                                if (taos_is_null(res, c, f)) {
+                                    printf("col%d, row: %"PRId64" "
+                                            "value: NULL\n", f, c);
+                                } else {
+                                    printf("col%d, row: %"PRId64", "
+                                            "value: %d\n",
+                                            f, c,
+                                            *(int32_t*)(row[f]+c*sizeof(int32_t)));
+                                }
+                                break;
+
+                            case TSDB_DATA_TYPE_FLOAT:
+                                if (taos_is_null(res, c, f)) {
+                                    printf("col%d, row: %"PRId64" "
+                                            "value: NULL\n", f, c);
+                                } else {
+                                    printf("col%d, row: %"PRId64", "
+                                            "value: %f\n",
+                                            f, c,
+                                            *(float*)(row[f]+c*sizeof(float)));
+                                }
+                                break;
+
+                            default:
+                                printf("type: %d is not processed\n",
+                                        fields[f].type);
+                                break;
+                        }
+                    }
+                } else {
+                    int *offsets = taos_get_column_data_offset(res, f);
+                    if (offsets) {
+                        for (int c = 0; c < rows; c++) {
+                            if (offsets[c] != -1) {
+                                int length = *(int16_t*)(row[f] + offsets[c]);
+                                char *buf = calloc(1, length + 1);
+                                strncpy(buf, (char *)(row[f] + offsets[c] + 2), length);
+                                printf("row: %d, col: %d, offset: %d, length: %d, content: %s\n",
+                                        c, f, offsets[c], length, buf);
+                                free(buf);
+                            } else {
+                                printf("row: %d, col: %d, offset: -1, means content is NULL\n",
+                                        c, f);
+                            }
+                        }
+                    } else {
+                        errorPrint("%s() LN%d: col%d's offsets is NULL\n",
+                                __func__, __LINE__, f);
+                    }
+                }
+            }
             num_rows += rows;
         }
     } else {
-        warnPrint("%s() LN%d, call taos_fetch_rows()\n", __func__, __LINE__);
+        warnPrint("%s", "call taos_fetch_rows()\n");
         while ((row = taos_fetch_row(res))) {
             char temp[256] = {0};
             taos_print_row(temp, row, fields, num_fields);
             puts(temp);
-            num_rows ++;
 
             int* lengths = taos_fetch_lengths(res);
             if (lengths) {
                 for (int c = 0; c < num_fields; c++) {
-                    printf("length of column %d is %d\n", c, lengths[c]);
+                    printf("row: %"PRId64", col: %d, is_null: %s, length of column %d is %d\n",
+                            num_rows, c,
+                            taos_is_null(res, num_rows, c)?"True":"False",
+                            c, lengths[c]);
                 }
             } else {
                 errorPrint("%s() LN%d: %s's lengths is NULL\n",
                         __func__, __LINE__, tbname);
             }
+
+            num_rows ++;
         }
     }
 
@@ -172,8 +297,8 @@ static void verify_query(TAOS* taos) {
                 int field_count = taos_field_count(res);
                 printf("field_count: %d\n", field_count);
                 int64_t rows = print_result(tbname, res, i % 2);
-                printf("rows is: %"PRId64"\n", rows);
-
+                okPrint("total query %s result rows is: %"PRId64"\n",
+                        tbname, rows);
             } else {
                 errorPrint("%s() LN%d: %s\n",
                         __func__, __LINE__, taos_errstr(res));
@@ -207,7 +332,7 @@ int main(int argc, char *argv[]) {
     verify_query(taos);
 
     taos_close(taos);
-    printf("done\n");
+    okPrint("%s", "done\n");
 
     return 0;
 }

include/common/tmsg.h

@@ -1209,12 +1209,17 @@ typedef struct {
   int32_t code;
 } STaskDropRsp;
 
+#define STREAM_TRIGGER_AT_ONCE      1
+#define STREAM_TRIGGER_WINDOW_CLOSE 2
+
 typedef struct {
   char   name[TSDB_TOPIC_FNAME_LEN];
   char   outputSTbName[TSDB_TABLE_FNAME_LEN];
   int8_t igExists;
   char*  sql;
   char*  ast;
+  int8_t triggerType;
+  int64_t watermark;
 } SCMCreateStreamReq;
 
 typedef struct {

include/common/trow.h

@@ -162,38 +162,39 @@ typedef struct {
   int32_t  extendedRowSize;
 } SRowBuilder;
 
-#define TD_ROW_HEAD_LEN (sizeof(STSRow))
+#define TD_ROW_HEAD_LEN  (sizeof(STSRow))
 #define TD_ROW_NCOLS_LEN (sizeof(col_id_t))
 
-#define TD_ROW_INFO(r) ((r)->info)
-#define TD_ROW_TYPE(r) ((r)->type)
-#define TD_ROW_DELETE(r) ((r)->del)
-#define TD_ROW_ENDIAN(r) ((r)->endian)
-#define TD_ROW_SVER(r) ((r)->sver)
-#define TD_ROW_NCOLS(r) ((r)->data)  // only valid for SKvRow
-#define TD_ROW_DATA(r) ((r)->data)
-#define TD_ROW_LEN(r) ((r)->len)
-#define TD_ROW_KEY(r) ((r)->ts)
+#define TD_ROW_INFO(r)     ((r)->info)
+#define TD_ROW_TYPE(r)     ((r)->type)
+#define TD_ROW_DELETE(r)   ((r)->del)
+#define TD_ROW_ENDIAN(r)   ((r)->endian)
+#define TD_ROW_SVER(r)     ((r)->sver)
+#define TD_ROW_NCOLS(r)    ((r)->data)  // only valid for SKvRow
+#define TD_ROW_DATA(r)     ((r)->data)
+#define TD_ROW_LEN(r)      ((r)->len)
+#define TD_ROW_KEY(r)      ((r)->ts)
+#define TD_ROW_VER(r)      ((r)->ver)
 #define TD_ROW_KEY_ADDR(r) (r)
 
 // N.B. If without STSchema, getExtendedRowSize() is used to get the rowMaxBytes and
 // (int32_t)ceil((double)nCols/TD_VTYPE_PARTS) should be added if TD_SUPPORT_BITMAP defined.
 #define TD_ROW_MAX_BYTES_FROM_SCHEMA(s) (schemaTLen(s) + TD_ROW_HEAD_LEN)
 
-#define TD_ROW_SET_INFO(r, i) (TD_ROW_INFO(r) = (i))
-#define TD_ROW_SET_TYPE(r, t) (TD_ROW_TYPE(r) = (t))
-#define TD_ROW_SET_DELETE(r) (TD_ROW_DELETE(r) = 1)
-#define TD_ROW_SET_SVER(r, v) (TD_ROW_SVER(r) = (v))
-#define TD_ROW_SET_LEN(r, l) (TD_ROW_LEN(r) = (l))
+#define TD_ROW_SET_INFO(r, i)  (TD_ROW_INFO(r) = (i))
+#define TD_ROW_SET_TYPE(r, t)  (TD_ROW_TYPE(r) = (t))
+#define TD_ROW_SET_DELETE(r)   (TD_ROW_DELETE(r) = 1)
+#define TD_ROW_SET_SVER(r, v)  (TD_ROW_SVER(r) = (v))
+#define TD_ROW_SET_LEN(r, l)   (TD_ROW_LEN(r) = (l))
 #define TD_ROW_SET_NCOLS(r, n) (*(col_id_t *)TD_ROW_NCOLS(r) = (n))
 
 #define TD_ROW_IS_DELETED(r) (TD_ROW_DELETE(r) == 1)
-#define TD_IS_TP_ROW(r) (TD_ROW_TYPE(r) == TD_ROW_TP)
-#define TD_IS_KV_ROW(r) (TD_ROW_TYPE(r) == TD_ROW_KV)
-#define TD_IS_TP_ROW_T(t) ((t) == TD_ROW_TP)
-#define TD_IS_KV_ROW_T(t) ((t) == TD_ROW_KV)
+#define TD_IS_TP_ROW(r)      (TD_ROW_TYPE(r) == TD_ROW_TP)
+#define TD_IS_KV_ROW(r)      (TD_ROW_TYPE(r) == TD_ROW_KV)
+#define TD_IS_TP_ROW_T(t)    ((t) == TD_ROW_TP)
+#define TD_IS_KV_ROW_T(t)    ((t) == TD_ROW_KV)
 
-#define TD_BOOL_STR(b) ((b) ? "true" : "false")
+#define TD_BOOL_STR(b)       ((b) ? "true" : "false")
 #define isUtilizeKVRow(k, d) ((k) < ((d)*KVRatioConvert))
 
 #define TD_ROW_COL_IDX(r) POINTER_SHIFT(TD_ROW_DATA(r), sizeof(col_id_t))
@@ -221,8 +222,10 @@ static FORCE_INLINE int32_t tdSetBitmapValTypeI(void *pBitmap, int16_t colIdx, T
 static FORCE_INLINE int32_t tdSetBitmapValTypeII(void *pBitmap, int16_t colIdx, TDRowValT valType);
 static FORCE_INLINE int32_t tdSetBitmapValType(void *pBitmap, int16_t colIdx, TDRowValT valType, int8_t bitmapMode);
 int32_t                     tdSetBitmapValTypeN(void *pBitmap, int16_t nEle, TDRowValT valType, int8_t bitmapMode);
-static FORCE_INLINE int32_t tdGetBitmapValTypeI(void *pBitmap, int16_t colIdx, TDRowValT *pValType);
-static FORCE_INLINE int32_t tdGetBitmapValTypeII(void *pBitmap, int16_t colIdx, TDRowValT *pValType);
+static FORCE_INLINE int32_t tdGetBitmapValTypeI(const void *pBitmap, int16_t colIdx, TDRowValT *pValType);
+static FORCE_INLINE int32_t tdGetBitmapValTypeII(const void *pBitmap, int16_t colIdx, TDRowValT *pValType);
+static FORCE_INLINE int32_t tdGetBitmapValType(const void *pBitmap, int16_t colIdx, TDRowValT *pValType, int8_t bitmapMode);
+static FORCE_INLINE bool    tdIsBitmapValTypeNormII(const void *pBitmap, int16_t idx);
 int32_t tdAppendValToDataCol(SDataCol *pCol, TDRowValT valType, const void *val, int32_t numOfRows, int32_t maxPoints,
                              int8_t bitmapMode);
 static FORCE_INLINE int32_t tdAppendColValToTpRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val,
@@ -324,7 +327,16 @@ static FORCE_INLINE int32_t tdSetBitmapValTypeII(void *pBitmap, int16_t colIdx,
   return TSDB_CODE_SUCCESS;
 }
 
-static FORCE_INLINE int32_t tdGetBitmapValType(void *pBitmap, int16_t colIdx, TDRowValT *pValType, int8_t bitmapMode) {
+static FORCE_INLINE bool tdIsBitmapValTypeNormII(const void *pBitmap, int16_t idx) {
+  TDRowValT valType = 0;
+  tdGetBitmapValTypeII(pBitmap, idx, &valType);
+  if (tdValTypeIsNorm(valType)) {
+    return true;
+  }
+  return false;
+}
+
+static FORCE_INLINE int32_t tdGetBitmapValType(const void *pBitmap, int16_t colIdx, TDRowValT *pValType, int8_t bitmapMode) {
   switch (bitmapMode) {
     case 0:
       tdGetBitmapValTypeII(pBitmap, colIdx, pValType);
@@ -349,7 +361,7 @@ static FORCE_INLINE int32_t tdGetBitmapValType(void *pBitmap, int16_t colIdx, TD
  * @param pValType
  * @return FORCE_INLINE
  */
-static FORCE_INLINE int32_t tdGetBitmapValTypeII(void *pBitmap, int16_t colIdx, TDRowValT *pValType) {
+static FORCE_INLINE int32_t tdGetBitmapValTypeII(const void *pBitmap, int16_t colIdx, TDRowValT *pValType) {
   if (!pBitmap || colIdx < 0) {
     TASSERT(0);
     terrno = TSDB_CODE_INVALID_PARA;
@@ -448,7 +460,7 @@ static FORCE_INLINE int32_t tdSetBitmapValTypeI(void *pBitmap, int16_t colIdx, T
  * @param pValType
  * @return FORCE_INLINE
  */
-static FORCE_INLINE int32_t tdGetBitmapValTypeI(void *pBitmap, int16_t colIdx, TDRowValT *pValType) {
+static FORCE_INLINE int32_t tdGetBitmapValTypeI(const void *pBitmap, int16_t colIdx, TDRowValT *pValType) {
   if (!pBitmap || colIdx < 0) {
     TASSERT(0);
     terrno = TSDB_CODE_INVALID_PARA;

include/common/ttokendef.h

@@ -171,60 +171,70 @@
 #define TK_BUFSIZE                        153
 #define TK_STREAM                         154
 #define TK_INTO                           155
-#define TK_KILL                           156
-#define TK_CONNECTION                     157
-#define TK_MERGE                          158
-#define TK_VGROUP                         159
-#define TK_REDISTRIBUTE                   160
-#define TK_SPLIT                          161
-#define TK_SYNCDB                         162
-#define TK_NULL                           163
-#define TK_FIRST                          164
-#define TK_LAST                           165
-#define TK_NOW                            166
-#define TK_TODAY                          167
-#define TK_CAST                           168
-#define TK_ROWTS                          169
-#define TK_TBNAME                         170
-#define TK_QSTARTTS                       171
-#define TK_QENDTS                         172
-#define TK_WSTARTTS                       173
-#define TK_WENDTS                         174
-#define TK_WDURATION                      175
-#define TK_BETWEEN                        176
-#define TK_IS                             177
-#define TK_NK_LT                          178
-#define TK_NK_GT                          179
-#define TK_NK_LE                          180
-#define TK_NK_GE                          181
-#define TK_NK_NE                          182
-#define TK_MATCH                          183
-#define TK_NMATCH                         184
-#define TK_JOIN                           185
-#define TK_INNER                          186
-#define TK_SELECT                         187
-#define TK_DISTINCT                       188
-#define TK_WHERE                          189
-#define TK_PARTITION                      190
-#define TK_BY                             191
-#define TK_SESSION                        192
-#define TK_STATE_WINDOW                   193
-#define TK_SLIDING                        194
-#define TK_FILL                           195
-#define TK_VALUE                          196
-#define TK_NONE                           197
-#define TK_PREV                           198
-#define TK_LINEAR                         199
-#define TK_NEXT                           200
-#define TK_GROUP                          201
-#define TK_HAVING                         202
-#define TK_ORDER                          203
-#define TK_SLIMIT                         204
-#define TK_SOFFSET                        205
-#define TK_LIMIT                          206
-#define TK_OFFSET                         207
-#define TK_ASC                            208
-#define TK_NULLS                          209
+#define TK_TRIGGER                        156
+#define TK_AT_ONCE                        157
+#define TK_WINDOW_CLOSE                   158
+#define TK_WATERMARK                      159
+#define TK_KILL                           160
+#define TK_CONNECTION                     161
+#define TK_MERGE                          162
+#define TK_VGROUP                         163
+#define TK_REDISTRIBUTE                   164
+#define TK_SPLIT                          165
+#define TK_SYNCDB                         166
+#define TK_NULL                           167
+#define TK_NK_QUESTION                    168
+#define TK_NK_ARROW                       169
+#define TK_ROWTS                          170
+#define TK_TBNAME                         171
+#define TK_QSTARTTS                       172
+#define TK_QENDTS                         173
+#define TK_WSTARTTS                       174
+#define TK_WENDTS                         175
+#define TK_WDURATION                      176
+#define TK_CAST                           177
+#define TK_NOW                            178
+#define TK_TODAY                          179
+#define TK_TIMEZONE                       180
+#define TK_COUNT                          181
+#define TK_FIRST                          182
+#define TK_LAST                           183
+#define TK_LAST_ROW                       184
+#define TK_BETWEEN                        185
+#define TK_IS                             186
+#define TK_NK_LT                          187
+#define TK_NK_GT                          188
+#define TK_NK_LE                          189
+#define TK_NK_GE                          190
+#define TK_NK_NE                          191
+#define TK_MATCH                          192
+#define TK_NMATCH                         193
+#define TK_CONTAINS                       194
+#define TK_JOIN                           195
+#define TK_INNER                          196
+#define TK_SELECT                         197
+#define TK_DISTINCT                       198
+#define TK_WHERE                          199
+#define TK_PARTITION                      200
+#define TK_BY                             201
+#define TK_SESSION                        202
+#define TK_STATE_WINDOW                   203
+#define TK_SLIDING                        204
+#define TK_FILL                           205
+#define TK_VALUE                          206
+#define TK_NONE                           207
+#define TK_PREV                           208
+#define TK_LINEAR                         209
+#define TK_NEXT                           210
+#define TK_GROUP                          211
+#define TK_HAVING                         212
+#define TK_ORDER                          213
+#define TK_SLIMIT                         214
+#define TK_SOFFSET                        215
+#define TK_LIMIT                          216
+#define TK_OFFSET                         217
+#define TK_ASC                            218
+#define TK_NULLS                          219
 
 #define TK_NK_SPACE                       300
 #define TK_NK_COMMENT                     301
@@ -233,7 +243,6 @@
 #define TK_NK_OCT                         304   // oct number
 #define TK_NK_BIN                         305   // bin format data 0b111
 #define TK_NK_FILE                        306
-#define TK_NK_QUESTION                    307   // denoting the placeholder of "?",when invoking statement bind query
 
 #define TK_NK_BITNOT                      501
 #define TK_INSERT                         502

include/common/ttypes.h

@@ -248,7 +248,7 @@ typedef struct tDataTypeDescriptor {
                       int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize);
   int32_t (*decompFunc)(const char *const input, int32_t compressedSize, const int32_t nelements, char *const output,
                         int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize);
-  void (*statisFunc)(const void *pData, int32_t numofrow, int64_t *min, int64_t *max, int64_t *sum, int16_t *minindex,
+  void (*statisFunc)(const void* pBitmap, const void *pData, int32_t numofrow, int64_t *min, int64_t *max, int64_t *sum, int16_t *minindex,
                      int16_t *maxindex, int16_t *numofnull);
 } tDataTypeDescriptor;
 

include/libs/function/functionMgt.h

@@ -137,12 +137,13 @@ bool fmIsWindowPseudoColumnFunc(int32_t funcId);
 bool fmIsWindowClauseFunc(int32_t funcId);
 bool fmIsSpecialDataRequiredFunc(int32_t funcId);
 bool fmIsDynamicScanOptimizedFunc(int32_t funcId);
+bool fmIsMultiResFunc(int32_t funcId);
 
 typedef enum EFuncDataRequired {
-  FUNC_DATA_REQUIRED_ALL_NEEDED = 1,
-  FUNC_DATA_REQUIRED_STATIS_NEEDED,
-  FUNC_DATA_REQUIRED_NO_NEEDED,
-  FUNC_DATA_REQUIRED_DISCARD
+  FUNC_DATA_REQUIRED_DATA_LOAD = 1,
+  FUNC_DATA_REQUIRED_STATIS_LOAD,
+  FUNC_DATA_REQUIRED_NOT_LOAD,
+  FUNC_DATA_REQUIRED_FILTEROUT,
 } EFuncDataRequired;
 
 EFuncDataRequired fmFuncDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWindow);

include/libs/nodes/cmdnodes.h

@@ -272,6 +272,28 @@ typedef struct SKillStmt {
   int32_t targetId;
 } SKillStmt;
 
+typedef struct SStreamOptions {
+  ENodeType type;
+  int8_t triggerType;
+  SNode* pWatermark;
+} SStreamOptions;
+
+typedef struct SCreateStreamStmt {
+  ENodeType type;
+  char streamName[TSDB_TABLE_NAME_LEN];
+  char targetDbName[TSDB_DB_NAME_LEN];
+  char targetTabName[TSDB_TABLE_NAME_LEN];
+  bool ignoreExists;
+  SStreamOptions* pOptions;
+  SNode* pQuery;
+} SCreateStreamStmt;
+
+typedef struct SDropStreamStmt {
+  ENodeType type;
+  char streamName[TSDB_TABLE_NAME_LEN];
+  bool ignoreNotExists;
+} SDropStreamStmt;
+
 #ifdef __cplusplus
 }
 #endif

include/libs/nodes/nodes.h

@@ -80,6 +80,7 @@ typedef enum ENodeType {
   QUERY_NODE_TABLE_OPTIONS,
   QUERY_NODE_INDEX_OPTIONS,
   QUERY_NODE_EXPLAIN_OPTIONS,
+  QUERY_NODE_STREAM_OPTIONS,
 
   // Statement nodes are used in parser and planner module.
   QUERY_NODE_SET_OPERATOR,
@@ -151,6 +152,12 @@ typedef enum ENodeType {
   QUERY_NODE_SHOW_CONNECTIONS_STMT,
   QUERY_NODE_SHOW_QUERIES_STMT,
   QUERY_NODE_SHOW_VNODES_STMT,
+  QUERY_NODE_SHOW_APPS_STMT,
+  QUERY_NODE_SHOW_SCORES_STMT,
+  QUERY_NODE_SHOW_VARIABLE_STMT,
+  QUERY_NODE_SHOW_CREATE_DATABASE_STMT,
+  QUERY_NODE_SHOW_CREATE_TABLE_STMT,
+  QUERY_NODE_SHOW_CREATE_STABLE_STMT,
   QUERY_NODE_KILL_CONNECTION_STMT,
   QUERY_NODE_KILL_QUERY_STMT,
 

include/libs/nodes/plannodes.h

@@ -53,6 +53,11 @@ typedef struct SScanLogicNode {
   double ratio;
   SNodeList* pDynamicScanFuncs;
   int32_t dataRequired;
+  int64_t interval;
+  int64_t offset;
+  int64_t sliding;
+  int8_t intervalUnit;
+  int8_t slidingUnit;
 } SScanLogicNode;
 
 typedef struct SJoinLogicNode {
@@ -109,6 +114,8 @@ typedef struct SWindowLogicNode {
   int64_t sessionGap;
   SNode* pTspk;
   SNode* pStateExpr;
+  int8_t triggerType;
+  int64_t watermark;
 } SWindowLogicNode;
 
 typedef struct SSortLogicNode {
@@ -206,6 +213,11 @@ typedef struct STableScanPhysiNode {
   double ratio;
   int32_t dataRequired;
   SNodeList* pDynamicScanFuncs;
+  int64_t interval;
+  int64_t offset;
+  int64_t sliding;
+  int8_t intervalUnit;
+  int8_t slidingUnit;
 } STableScanPhysiNode;
 
 typedef STableScanPhysiNode STableSeqScanPhysiNode;
@@ -251,6 +263,8 @@ typedef struct SWinodwPhysiNode {
   SNodeList* pExprs;   // these are expression list of parameter expression of function
   SNodeList* pFuncs;
   SNode* pTspk; // timestamp primary key
+  int8_t triggerType;
+  int64_t watermark;
 } SWinodwPhysiNode;
 
 typedef struct SIntervalPhysiNode {
@@ -260,7 +274,6 @@ typedef struct SIntervalPhysiNode {
   int64_t    sliding;
   int8_t     intervalUnit;
   int8_t     slidingUnit;
-  uint8_t    precision;
   SFillNode* pFill;
 } SIntervalPhysiNode;
 

include/libs/nodes/querynodes.h

@@ -57,6 +57,7 @@ typedef enum EColumnType {
 typedef struct SColumnNode {
   SExprNode node; // QUERY_NODE_COLUMN
   uint64_t tableId;
+  int8_t tableType;
   col_id_t colId;
   EColumnType colType; // column or tag
   char dbName[TSDB_DB_NAME_LEN];
@@ -196,8 +197,8 @@ typedef struct SStateWindowNode {
 
 typedef struct SSessionWindowNode {
   ENodeType type; // QUERY_NODE_SESSION_WINDOW
-  SNode* pCol; // timestamp primary key
-  SNode* pGap; // gap between two session window(in microseconds)
+  SColumnNode* pCol; // timestamp primary key
+  SValueNode* pGap; // gap between two session window(in microseconds)
 } SSessionWindowNode;
 
 typedef struct SIntervalWindowNode {

include/libs/planner/planner.h

@@ -30,6 +30,8 @@ typedef struct SPlanContext {
   bool topicQuery;
   bool streamQuery;
   bool showRewrite;
+  int8_t triggerType;
+  int64_t watermark;
 } SPlanContext;
 
 // Create the physical plan for the query, according to the AST.

include/libs/scalar/scalar.h

@@ -80,6 +80,7 @@ int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarPara
 int32_t timeDiffFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
 int32_t nowFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
 int32_t todayFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
+int32_t timezoneFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
 
 bool getTimePseudoFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
 

include/libs/wal/wal.h

@@ -61,16 +61,16 @@ extern "C" {
     }                                                             \
   }
 
-#define WAL_HEAD_VER 0
+#define WAL_HEAD_VER     0
 #define WAL_NOSUFFIX_LEN 20
-#define WAL_SUFFIX_AT (WAL_NOSUFFIX_LEN + 1)
-#define WAL_LOG_SUFFIX "log"
+#define WAL_SUFFIX_AT    (WAL_NOSUFFIX_LEN + 1)
+#define WAL_LOG_SUFFIX   "log"
 #define WAL_INDEX_SUFFIX "idx"
-#define WAL_REFRESH_MS 1000
-#define WAL_MAX_SIZE (TSDB_MAX_WAL_SIZE + sizeof(SWalHead))
-#define WAL_PATH_LEN (TSDB_FILENAME_LEN + 12)
-#define WAL_FILE_LEN (WAL_PATH_LEN + 32)
-#define WAL_MAGIC 0xFAFBFCFDULL
+#define WAL_REFRESH_MS   1000
+#define WAL_MAX_SIZE     (TSDB_MAX_WAL_SIZE + sizeof(SWalHead))
+#define WAL_PATH_LEN     (TSDB_FILENAME_LEN + 12)
+#define WAL_FILE_LEN     (WAL_PATH_LEN + 32)
+#define WAL_MAGIC        0xFAFBFCFDULL
 
 #define WAL_CUR_FAILED 1
 

include/util/taoserror.h

@@ -579,7 +579,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_PAR_INVALID_PORT              TAOS_DEF_ERROR_CODE(0, 0x2612)
 #define TSDB_CODE_PAR_INVALID_ENDPOINT          TAOS_DEF_ERROR_CODE(0, 0x2613)
 #define TSDB_CODE_PAR_EXPRIE_STATEMENT          TAOS_DEF_ERROR_CODE(0, 0x2614)
-#define TSDB_CODE_PAR_INTERVAL_VALUE_TOO_SMALL  TAOS_DEF_ERROR_CODE(0, 0x2615)
+#define TSDB_CODE_PAR_INTER_VALUE_TOO_SMALL     TAOS_DEF_ERROR_CODE(0, 0x2615)
 #define TSDB_CODE_PAR_DB_NOT_SPECIFIED          TAOS_DEF_ERROR_CODE(0, 0x2616)
 #define TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME   TAOS_DEF_ERROR_CODE(0, 0x2617)
 #define TSDB_CODE_PAR_CORRESPONDING_STABLE_ERR  TAOS_DEF_ERROR_CODE(0, 0x2618)
@@ -597,6 +597,18 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_PAR_GROUPBY_WINDOW_COEXIST    TAOS_DEF_ERROR_CODE(0, 0x2624)
 #define TSDB_CODE_PAR_INVALID_OPTION_UNIT       TAOS_DEF_ERROR_CODE(0, 0x2625)
 #define TSDB_CODE_PAR_INVALID_KEEP_UNIT         TAOS_DEF_ERROR_CODE(0, 0x2626)
+#define TSDB_CODE_PAR_AGG_FUNC_NESTING          TAOS_DEF_ERROR_CODE(0, 0x2627)
+#define TSDB_CODE_PAR_INVALID_STATE_WIN_TYPE    TAOS_DEF_ERROR_CODE(0, 0x2628)
+#define TSDB_CODE_PAR_INVALID_STATE_WIN_COL     TAOS_DEF_ERROR_CODE(0, 0x2629)
+#define TSDB_CODE_PAR_INVALID_STATE_WIN_TABLE   TAOS_DEF_ERROR_CODE(0, 0x262A)
+#define TSDB_CODE_PAR_INTER_SESSION_GAP         TAOS_DEF_ERROR_CODE(0, 0x262B)
+#define TSDB_CODE_PAR_INTER_SESSION_COL         TAOS_DEF_ERROR_CODE(0, 0x262C)
+#define TSDB_CODE_PAR_INTER_OFFSET_NEGATIVE     TAOS_DEF_ERROR_CODE(0, 0x262D)
+#define TSDB_CODE_PAR_INTER_OFFSET_UNIT         TAOS_DEF_ERROR_CODE(0, 0x262E)
+#define TSDB_CODE_PAR_INTER_OFFSET_TOO_BIG      TAOS_DEF_ERROR_CODE(0, 0x262F)
+#define TSDB_CODE_PAR_INTER_SLIDING_UNIT        TAOS_DEF_ERROR_CODE(0, 0x2630)
+#define TSDB_CODE_PAR_INTER_SLIDING_TOO_BIG     TAOS_DEF_ERROR_CODE(0, 0x2631)
+#define TSDB_CODE_PAR_INTER_SLIDING_TOO_SMALL   TAOS_DEF_ERROR_CODE(0, 0x2632)
 
 //planner
 #define TSDB_CODE_PLAN_INTERNAL_ERROR           TAOS_DEF_ERROR_CODE(0, 0x2700)

packaging/release.sh

@@ -64,11 +64,11 @@ cp ${install_files} ${install_dir}
 header_files="${top_dir}/include/client/taos.h ${top_dir}/include/util/taoserror.h"
 cp ${header_files} ${install_dir}/inc
  
-bin_files="${compile_dir}/source/dnode/mgmt/taosd ${compile_dir}/tools/shell/taos  ${compile_dir}/tests/test/c/create_table ${compile_dir}/tests/test/c/tmq_sim ${script_dir}/remove.sh"
+bin_files="${compile_dir}/build/bin/taosd ${compile_dir}/build/bin/taos  ${compile_dir}/build/bin/create_table ${compile_dir}/build/bin/tmq_sim ${script_dir}/remove.sh"
 cp ${bin_files} ${install_dir}/bin && chmod a+x ${install_dir}/bin/* || :
 
-cp ${compile_dir}/source/client/libtaos.so  ${install_dir}/lib/
-cp ${compile_dir}/source/libs/tdb/libtdb.so ${install_dir}/lib/
+cp ${compile_dir}/build/lib/libtaos.so  ${install_dir}/lib/
+cp ${compile_dir}/build/lib/libtdb.so ${install_dir}/lib/
 taostoolfile="${top_dir}/tools/taosTools-1.4.1-Linux-x64.tar.gz"
 cp ${taostoolfile} ${install_dir}/taos-tools 
 

source/client/CMakeLists.txt

 aux_source_directory(src CLIENT_SRC)
-add_library(taos ${CLIENT_SRC})
+add_library(taos SHARED ${CLIENT_SRC})
 target_include_directories(
     taos
     PUBLIC "${TD_SOURCE_DIR}/include/client"
@@ -11,6 +11,25 @@ target_link_libraries(
     PRIVATE os util common transport nodes parser command planner catalog scheduler function qcom
 )
 
+set_target_properties(
+    taos
+    PROPERTIES
+    VERSION ${TD_VER_NUMBER}
+    SOVERSION ${TD_VER_NUMBER}
+)
+
+add_library(taos_static STATIC ${CLIENT_SRC})
+target_include_directories(
+    taos_static
+    PUBLIC "${TD_SOURCE_DIR}/include/client"
+    PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
+)
+target_link_libraries(
+    taos_static
+    INTERFACE api
+    PRIVATE os util common transport nodes parser command planner catalog scheduler function qcom
+)
+
 if(${BUILD_TEST})
     ADD_SUBDIRECTORY(test)
 endif(${BUILD_TEST})
\ No newline at end of file

source/client/test/CMakeLists.txt

@@ -8,13 +8,13 @@ AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
 ADD_EXECUTABLE(clientTest clientTests.cpp)
 TARGET_LINK_LIBRARIES(
         clientTest
-        PUBLIC os util common transport parser catalog scheduler function gtest taos qcom
+        PUBLIC os util common transport parser catalog scheduler function gtest taos_static qcom
 )
 
 ADD_EXECUTABLE(tmqTest tmqTest.cpp)
 TARGET_LINK_LIBRARIES(
         tmqTest
-        PUBLIC os util common transport parser catalog scheduler function gtest taos qcom
+        PUBLIC os util common transport parser catalog scheduler function gtest taos_static qcom
 )
 
 TARGET_INCLUDE_DIRECTORIES(

source/client/test/clientTests.cpp

@@ -662,7 +662,7 @@ TEST(testCase, agg_query_tables) {
   TAOS_RES* pRes = taos_query(pConn, "use abc1");
   taos_free_result(pRes);
 
-  pRes = taos_query(pConn, "select * from test_block_raw.all_type");
+  pRes = taos_query(pConn, "select count(*) from tu");
   if (taos_errno(pRes) != 0) {
     printf("failed to select from table, reason:%s\n", taos_errstr(pRes));
     taos_free_result(pRes);
@@ -674,9 +674,6 @@ TEST(testCase, agg_query_tables) {
   int32_t     numOfFields = taos_num_fields(pRes);
 
   int32_t n = 0;
-  void* data = NULL;
-  int32_t code = taos_fetch_raw_block(pRes, &n, &data);
-
   char str[512] = {0};
   while ((pRow = taos_fetch_row(pRes)) != NULL) {
     int32_t* length = taos_fetch_lengths(pRes);

source/common/src/tmsg.c

@@ -3381,6 +3381,8 @@ int32_t tSerializeSCMCreateStreamReq(void *buf, int32_t bufLen, const SCMCreateS
   if (tEncodeI8(&encoder, pReq->igExists) < 0) return -1;
   if (tEncodeI32(&encoder, sqlLen) < 0) return -1;
   if (tEncodeI32(&encoder, astLen) < 0) return -1;
+  if (tEncodeI8(&encoder, pReq->triggerType) < 0) return -1;
+  if (tEncodeI64(&encoder, pReq->watermark) < 0) return -1;
   if (sqlLen > 0 && tEncodeCStr(&encoder, pReq->sql) < 0) return -1;
   if (astLen > 0 && tEncodeCStr(&encoder, pReq->ast) < 0) return -1;
 
@@ -3404,6 +3406,8 @@ int32_t tDeserializeSCMCreateStreamReq(void *buf, int32_t bufLen, SCMCreateStrea
   if (tDecodeI8(&decoder, &pReq->igExists) < 0) return -1;
   if (tDecodeI32(&decoder, &sqlLen) < 0) return -1;
   if (tDecodeI32(&decoder, &astLen) < 0) return -1;
+  if (tDecodeI8(&decoder, &pReq->triggerType) < 0) return -1;
+  if (tDecodeI64(&decoder, &pReq->watermark) < 0) return -1;
 
   if (sqlLen > 0) {
     pReq->sql = taosMemoryCalloc(1, sqlLen + 1);

source/common/src/ttypes.c

@@ -16,6 +16,7 @@
 #define _DEFAULT_SOURCE
 #include "ttypes.h"
 #include "tcompression.h"
+#include "trow.h"
 
 const int32_t TYPE_BYTES[15] = {
     -1,                      // TSDB_DATA_TYPE_NULL
@@ -49,7 +50,7 @@ const int32_t TYPE_BYTES[15] = {
     }                                                                          \
   } while (0)
 
-static void getStatics_bool(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum,
+static void getStatics_bool(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum,
                             int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
   int8_t *data = (int8_t *)pData;
   *min = INT64_MAX;
@@ -60,7 +61,8 @@ static void getStatics_bool(const void *pData, int32_t numOfRow, int64_t *min, i
   assert(numOfRow <= INT16_MAX);
 
   for (int32_t i = 0; i < numOfRow; ++i) {
-    if (data[i] == TSDB_DATA_BOOL_NULL) {
+    // if (data[i] == TSDB_DATA_BOOL_NULL) {
+    if (!tdIsBitmapValTypeNormII(pBitmap, i)) {
       (*numOfNull) += 1;
       continue;
     }
@@ -69,7 +71,7 @@ static void getStatics_bool(const void *pData, int32_t numOfRow, int64_t *min, i
   }
 }
 
-static void getStatics_i8(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum,
+static void getStatics_i8(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum,
                           int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
   int8_t *data = (int8_t *)pData;
   *min = INT64_MAX;
@@ -80,7 +82,8 @@ static void getStatics_i8(const void *pData, int32_t numOfRow, int64_t *min, int
   assert(numOfRow <= INT16_MAX);
 
   for (int32_t i = 0; i < numOfRow; ++i) {
-    if (((uint8_t)data[i]) == TSDB_DATA_TINYINT_NULL) {
+    // if (((uint8_t)data[i]) == TSDB_DATA_TINYINT_NULL) {
+    if (!tdIsBitmapValTypeNormII(pBitmap, i)) {
       (*numOfNull) += 1;
       continue;
     }
@@ -89,7 +92,7 @@ static void getStatics_i8(const void *pData, int32_t numOfRow, int64_t *min, int
   }
 }
 
-static void getStatics_u8(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum,
+static void getStatics_u8(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum,
                           int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
   uint8_t *data = (uint8_t *)pData;
   uint64_t _min = UINT64_MAX;
@@ -102,7 +105,8 @@ static void getStatics_u8(const void *pData, int32_t numOfRow, int64_t *min, int
   assert(numOfRow <= INT16_MAX);
 
   for (int32_t i = 0; i < numOfRow; ++i) {
-    if (((uint8_t)data[i]) == TSDB_DATA_UTINYINT_NULL) {
+    // if (((uint8_t)data[i]) == TSDB_DATA_UTINYINT_NULL) {
+    if (!tdIsBitmapValTypeNormII(pBitmap, i)) {
       (*numOfNull) += 1;
       continue;
     }
@@ -115,7 +119,7 @@ static void getStatics_u8(const void *pData, int32_t numOfRow, int64_t *min, int
   *sum = _sum;
 }
 
-static void getStatics_i16(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum,
+static void getStatics_i16(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum,
                            int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
   int16_t *data = (int16_t *)pData;
   *min = INT64_MAX;
@@ -126,7 +130,8 @@ static void getStatics_i16(const void *pData, int32_t numOfRow, int64_t *min, in
   assert(numOfRow <= INT16_MAX);
 
   for (int32_t i = 0; i < numOfRow; ++i) {
-    if (((uint16_t)data[i]) == TSDB_DATA_SMALLINT_NULL) {
+    // if (((uint16_t)data[i]) == TSDB_DATA_SMALLINT_NULL) {
+    if (!tdIsBitmapValTypeNormII(pBitmap, i)) {
       (*numOfNull) += 1;
       continue;
     }
@@ -135,7 +140,7 @@ static void getStatics_i16(const void *pData, int32_t numOfRow, int64_t *min, in
   }
 }
 
-static void getStatics_u16(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum,
+static void getStatics_u16(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum,
                            int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
   uint16_t *data = (uint16_t *)pData;
   uint64_t  _min = UINT64_MAX;
@@ -148,7 +153,8 @@ static void getStatics_u16(const void *pData, int32_t numOfRow, int64_t *min, in
   assert(numOfRow <= INT16_MAX);
 
   for (int32_t i = 0; i < numOfRow; ++i) {
-    if (((uint16_t)data[i]) == TSDB_DATA_USMALLINT_NULL) {
+    // if (((uint16_t)data[i]) == TSDB_DATA_USMALLINT_NULL) {
+    if (!tdIsBitmapValTypeNormII(pBitmap, i)) {
       (*numOfNull) += 1;
       continue;
     }
@@ -161,7 +167,7 @@ static void getStatics_u16(const void *pData, int32_t numOfRow, int64_t *min, in
   *sum = _sum;
 }
 
-static void getStatics_i32(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum,
+static void getStatics_i32(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum,
                            int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
   int32_t *data = (int32_t *)pData;
   *min = INT64_MAX;
@@ -172,7 +178,8 @@ static void getStatics_i32(const void *pData, int32_t numOfRow, int64_t *min, in
   assert(numOfRow <= INT16_MAX);
 
   for (int32_t i = 0; i < numOfRow; ++i) {
-    if (((uint32_t)data[i]) == TSDB_DATA_INT_NULL) {
+    // if (((uint32_t)data[i]) == TSDB_DATA_INT_NULL) {
+    if (!tdIsBitmapValTypeNormII(pBitmap, i)) {
       (*numOfNull) += 1;
       continue;
     }
@@ -181,7 +188,7 @@ static void getStatics_i32(const void *pData, int32_t numOfRow, int64_t *min, in
   }
 }
 
-static void getStatics_u32(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum,
+static void getStatics_u32(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum,
                            int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
   uint32_t *data = (uint32_t *)pData;
   uint64_t  _min = UINT64_MAX;
@@ -194,7 +201,8 @@ static void getStatics_u32(const void *pData, int32_t numOfRow, int64_t *min, in
   assert(numOfRow <= INT16_MAX);
 
   for (int32_t i = 0; i < numOfRow; ++i) {
-    if (((uint32_t)data[i]) == TSDB_DATA_UINT_NULL) {
+    // if (((uint32_t)data[i]) == TSDB_DATA_UINT_NULL) {
+    if (!tdIsBitmapValTypeNormII(pBitmap, i)) {
       (*numOfNull) += 1;
       continue;
     }
@@ -207,7 +215,7 @@ static void getStatics_u32(const void *pData, int32_t numOfRow, int64_t *min, in
   *sum = _sum;
 }
 
-static void getStatics_i64(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum,
+static void getStatics_i64(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum,
                            int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
   int64_t *data = (int64_t *)pData;
   *min = INT64_MAX;
@@ -218,7 +226,8 @@ static void getStatics_i64(const void *pData, int32_t numOfRow, int64_t *min, in
   assert(numOfRow <= INT16_MAX);
 
   for (int32_t i = 0; i < numOfRow; ++i) {
-    if (((uint64_t)data[i]) == TSDB_DATA_BIGINT_NULL) {
+    // if (((uint64_t)data[i]) == TSDB_DATA_BIGINT_NULL) {
+    if (!tdIsBitmapValTypeNormII(pBitmap, i)) {
       (*numOfNull) += 1;
       continue;
     }
@@ -227,7 +236,7 @@ static void getStatics_i64(const void *pData, int32_t numOfRow, int64_t *min, in
   }
 }
 
-static void getStatics_u64(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum,
+static void getStatics_u64(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum,
                            int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
   uint64_t *data = (uint64_t *)pData;
   uint64_t  _min = UINT64_MAX;
@@ -240,7 +249,8 @@ static void getStatics_u64(const void *pData, int32_t numOfRow, int64_t *min, in
   assert(numOfRow <= INT16_MAX);
 
   for (int32_t i = 0; i < numOfRow; ++i) {
-    if (((uint64_t)data[i]) == TSDB_DATA_UBIGINT_NULL) {
+    // if (((uint64_t)data[i]) == TSDB_DATA_UBIGINT_NULL) {
+    if (!tdIsBitmapValTypeNormII(pBitmap, i)) {
       (*numOfNull) += 1;
       continue;
     }
@@ -253,7 +263,7 @@ static void getStatics_u64(const void *pData, int32_t numOfRow, int64_t *min, in
   *sum = _sum;
 }
 
-static void getStatics_f(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum,
+static void getStatics_f(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum,
                          int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
   float *data = (float *)pData;
   float  fmin = FLT_MAX;
@@ -265,7 +275,8 @@ static void getStatics_f(const void *pData, int32_t numOfRow, int64_t *min, int6
   assert(numOfRow <= INT16_MAX);
 
   for (int32_t i = 0; i < numOfRow; ++i) {
-    if ((*(uint32_t *)&(data[i])) == TSDB_DATA_FLOAT_NULL) {
+    // if ((*(uint32_t *)&(data[i])) == TSDB_DATA_FLOAT_NULL) {
+    if (!tdIsBitmapValTypeNormII(pBitmap, i)) {
       (*numOfNull) += 1;
       continue;
     }
@@ -289,7 +300,7 @@ static void getStatics_f(const void *pData, int32_t numOfRow, int64_t *min, int6
   SET_DOUBLE_VAL(min, fmin);
 }
 
-static void getStatics_d(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum,
+static void getStatics_d(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum,
                          int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
   double *data = (double *)pData;
   double  dmin = DBL_MAX;
@@ -301,7 +312,8 @@ static void getStatics_d(const void *pData, int32_t numOfRow, int64_t *min, int6
   assert(numOfRow <= INT16_MAX);
 
   for (int32_t i = 0; i < numOfRow; ++i) {
-    if ((*(uint64_t *)&(data[i])) == TSDB_DATA_DOUBLE_NULL) {
+    // if ((*(uint64_t *)&(data[i])) == TSDB_DATA_DOUBLE_NULL) {
+    if (!tdIsBitmapValTypeNormII(pBitmap, i)) {
       (*numOfNull) += 1;
       continue;
     }
@@ -325,13 +337,14 @@ static void getStatics_d(const void *pData, int32_t numOfRow, int64_t *min, int6
   SET_DOUBLE_PTR(min, &dmin);
 }
 
-static void getStatics_bin(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum,
+static void getStatics_bin(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum,
                            int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
   const char *data = pData;
   assert(numOfRow <= INT16_MAX);
 
   for (int32_t i = 0; i < numOfRow; ++i) {
-    if (isNull(data, TSDB_DATA_TYPE_BINARY)) {
+    // if (isNull(data, TSDB_DATA_TYPE_BINARY)) {
+    if (!tdIsBitmapValTypeNormII(pBitmap, i)) {
       (*numOfNull) += 1;
     }
 
@@ -345,13 +358,14 @@ static void getStatics_bin(const void *pData, int32_t numOfRow, int64_t *min, in
   *maxIndex = 0;
 }
 
-static void getStatics_nchr(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum,
+static void getStatics_nchr(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum,
                             int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
   const char *data = pData;
   assert(numOfRow <= INT16_MAX);
 
   for (int32_t i = 0; i < numOfRow; ++i) {
-    if (isNull(data, TSDB_DATA_TYPE_NCHAR)) {
+    // if (isNull(data, TSDB_DATA_TYPE_NCHAR)) {
+    if (!tdIsBitmapValTypeNormII(pBitmap, i)) {
       (*numOfNull) += 1;
     }
 
@@ -377,7 +391,7 @@ tDataTypeDescriptor tDataTypes[15] = {
      getStatics_i64},
     {TSDB_DATA_TYPE_FLOAT, 5, FLOAT_BYTES, "FLOAT", 0, 0, tsCompressFloat, tsDecompressFloat, getStatics_f},
     {TSDB_DATA_TYPE_DOUBLE, 6, DOUBLE_BYTES, "DOUBLE", 0, 0, tsCompressDouble, tsDecompressDouble, getStatics_d},
-    {TSDB_DATA_TYPE_BINARY, 6, 0, "BINARY", 0, 0, tsCompressString, tsDecompressString, getStatics_bin},
+    {TSDB_DATA_TYPE_VARCHAR, 6, 0, "VARCHAR", 0, 0, tsCompressString, tsDecompressString, getStatics_bin},
     {TSDB_DATA_TYPE_TIMESTAMP, 9, LONG_BYTES, "TIMESTAMP", INT64_MIN, INT64_MAX, tsCompressTimestamp,
      tsDecompressTimestamp, getStatics_i64},
     {TSDB_DATA_TYPE_NCHAR, 5, 8, "NCHAR", 0, 0, tsCompressString, tsDecompressString, getStatics_nchr},
@@ -402,7 +416,7 @@ char tTokenTypeSwitcher[13] = {
     TSDB_DATA_TYPE_DOUBLE,  // TK_DOUBLE
     TSDB_DATA_TYPE_BINARY,  // TK_STRING
     TSDB_DATA_TYPE_BIGINT,  // TK_TIMESTAMP
-    TSDB_DATA_TYPE_BINARY,  // TK_BINARY
+    TSDB_DATA_TYPE_VARCHAR,  // TK_BINARY
     TSDB_DATA_TYPE_NCHAR,   // TK_NCHAR
 };
 

source/dnode/mgmt/implement/CMakeLists.txt

@@ -6,12 +6,4 @@ target_link_libraries(
 target_include_directories(
     dnode
     PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
-)
-
+)
\ No newline at end of file
-IF (TD_GRANT)
-  TARGET_LINK_LIBRARIES(dnode grant)
-ENDIF ()
-IF (TD_USB_DONGLE)
-  TARGET_LINK_LIBRARIES(dnode usb_dongle)
-else()
-ENDIF ()
\ No newline at end of file

source/dnode/mgmt/interface/CMakeLists.txt

@@ -6,5 +6,5 @@ target_include_directories(
     PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/inc"
 )
 target_link_libraries(
-    dnode_interface cjson mnode vnode qnode snode bnode wal sync taos tfs monitor
+    dnode_interface cjson mnode vnode qnode snode bnode wal sync taos_static tfs monitor
 )
\ No newline at end of file

source/dnode/mgmt/mgmt_vnode/src/vmHandle.c

@@ -83,26 +83,25 @@ int32_t vmProcessGetVnodeLoadsReq(SMgmtWrapper *pWrapper, SNodeMsg *pReq) {
 
 static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) {
   pCfg->vgId = pCreate->vgId;
-  pCfg->wsize = pCreate->cacheBlockSize;
-  pCfg->ssize = pCreate->cacheBlockSize;
-  pCfg->lsize = pCreate->cacheBlockSize;
+  pCfg->wsize = pCreate->cacheBlockSize * 1024 * 1024;
+  pCfg->ssize = 1024;
+  pCfg->lsize = 1024 * 1024;
   pCfg->isHeapAllocator = true;
   pCfg->ttl = 4;
   pCfg->keep = pCreate->daysToKeep0;
   pCfg->streamMode = pCreate->streamMode;
   pCfg->isWeak = true;
-  pCfg->tsdbCfg.keep = pCreate->daysToKeep0;
-  pCfg->tsdbCfg.keep1 = pCreate->daysToKeep2;
   pCfg->tsdbCfg.keep2 = pCreate->daysToKeep0;
+  pCfg->tsdbCfg.keep0 = pCreate->daysToKeep2;
+  pCfg->tsdbCfg.keep1 = pCreate->daysToKeep0;
   pCfg->tsdbCfg.lruCacheSize = pCreate->cacheBlockSize;
   pCfg->tsdbCfg.retentions = pCreate->pRetensions;
-  pCfg->metaCfg.lruSize = pCreate->cacheBlockSize;
-  pCfg->walCfg.fsyncPeriod = pCreate->fsyncPeriod;
-  pCfg->walCfg.level = pCreate->walLevel;
-  pCfg->walCfg.retentionPeriod = 10;
-  pCfg->walCfg.retentionSize = 128;
-  pCfg->walCfg.rollPeriod = 128;
-  pCfg->walCfg.segSize = 128;
+  pCfg->walCfg.level = TAOS_WAL_WRITE;
+  pCfg->walCfg.fsyncPeriod = 0;
+  pCfg->walCfg.retentionPeriod = 0;
+  pCfg->walCfg.retentionSize = 0;
+  pCfg->walCfg.rollPeriod = 0;
+  pCfg->walCfg.segSize = 0;
   pCfg->walCfg.vgId = pCreate->vgId;
   pCfg->hashBegin = pCreate->hashBegin;
   pCfg->hashEnd = pCreate->hashEnd;
@@ -121,6 +120,8 @@ static void vmGenerateWrapperCfg(SVnodesMgmt *pMgmt, SCreateVnodeReq *pCreate, S
 int32_t vmProcessCreateVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) {
   SRpcMsg        *pReq = &pMsg->rpcMsg;
   SCreateVnodeReq createReq = {0};
+  char            path[TSDB_FILENAME_LEN];
+
   if (tDeserializeSCreateVnodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) {
     terrno = TSDB_CODE_INVALID_MSG;
     return -1;
@@ -143,6 +144,14 @@ int32_t vmProcessCreateVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) {
     return -1;
   }
 
+  // create vnode
+  snprintf(path, TSDB_FILENAME_LEN, "vnode%svnode%d", TD_DIRSEP, vnodeCfg.vgId);
+  if (vnodeCreate(path, &vnodeCfg, pMgmt->pTfs) < 0) {
+    tFreeSCreateVnodeReq(&createReq);
+    dError("vgId:%d, failed to create vnode since %s", createReq.vgId, terrstr());
+    return -1;
+  }
+
   SMsgCb msgCb = pMgmt->pDnode->data.msgCb;
   msgCb.pWrapper = pMgmt->pWrapper;
   msgCb.queueFps[QUERY_QUEUE] = vmPutMsgToQueryQueue;
@@ -150,13 +159,10 @@ int32_t vmProcessCreateVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) {
   msgCb.queueFps[APPLY_QUEUE] = vmPutMsgToApplyQueue;
   msgCb.qsizeFp = vmGetQueueSize;
 
-  vnodeCfg.msgCb = msgCb;
-  vnodeCfg.pTfs = pMgmt->pTfs;
-  vnodeCfg.dbId = wrapperCfg.dbUid;
-  SVnode *pImpl = vnodeOpen(wrapperCfg.path, &vnodeCfg);
+  SVnode *pImpl = vnodeOpen(path, pMgmt->pTfs, msgCb);
   if (pImpl == NULL) {
-    tFreeSCreateVnodeReq(&createReq);
     dError("vgId:%d, failed to create vnode since %s", createReq.vgId, terrstr());
+    tFreeSCreateVnodeReq(&createReq);
     return -1;
   }
 
@@ -165,7 +171,7 @@ int32_t vmProcessCreateVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) {
     tFreeSCreateVnodeReq(&createReq);
     dError("vgId:%d, failed to open vnode since %s", createReq.vgId, terrstr());
     vnodeClose(pImpl);
-    vnodeDestroy(wrapperCfg.path);
+    vnodeDestroy(path, pMgmt->pTfs);
     terrno = code;
     return code;
   }
@@ -174,7 +180,7 @@ int32_t vmProcessCreateVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) {
   if (code != 0) {
     tFreeSCreateVnodeReq(&createReq);
     vnodeClose(pImpl);
-    vnodeDestroy(wrapperCfg.path);
+    vnodeDestroy(path, pMgmt->pTfs);
     terrno = code;
     return code;
   }

source/dnode/mgmt/mgmt_vnode/src/vmInt.c

@@ -84,6 +84,8 @@ int32_t vmOpenVnode(SVnodesMgmt *pMgmt, SWrapperCfg *pCfg, SVnode *pImpl) {
 }
 
 void vmCloseVnode(SVnodesMgmt *pMgmt, SVnodeObj *pVnode) {
+  char path[TSDB_FILENAME_LEN];
+
   taosWLockLatch(&pMgmt->latch);
   taosHashRemove(pMgmt->hash, &pVnode->vgId, sizeof(int32_t));
   taosWUnLockLatch(&pMgmt->latch);
@@ -104,7 +106,8 @@ void vmCloseVnode(SVnodesMgmt *pMgmt, SVnodeObj *pVnode) {
 
   if (pVnode->dropped) {
     dDebug("vgId:%d, vnode is destroyed for dropped:%d", pVnode->vgId, pVnode->dropped);
-    vnodeDestroy(pVnode->path);
+    snprintf(path, TSDB_FILENAME_LEN, "vnode%svnode%d", TD_DIRSEP, pVnode->vgId);
+    vnodeDestroy(path, pMgmt->pTfs);
   }
 
   taosMemoryFree(pVnode->path);
@@ -116,6 +119,7 @@ static void *vmOpenVnodeFunc(void *param) {
   SVnodeThread *pThread = param;
   SVnodesMgmt  *pMgmt = pThread->pMgmt;
   SDnode       *pDnode = pMgmt->pDnode;
+  char          path[TSDB_FILENAME_LEN];
 
   dDebug("thread:%d, start to open %d vnodes", pThread->threadIndex, pThread->vnodeNum);
   setThreadName("open-vnodes");
@@ -134,8 +138,8 @@ static void *vmOpenVnodeFunc(void *param) {
     msgCb.queueFps[FETCH_QUEUE] = vmPutMsgToFetchQueue;
     msgCb.queueFps[APPLY_QUEUE] = vmPutMsgToApplyQueue;
     msgCb.qsizeFp = vmGetQueueSize;
-    SVnodeCfg cfg = {.msgCb = msgCb, .pTfs = pMgmt->pTfs, .vgId = pCfg->vgId, .dbId = pCfg->dbUid};
-    SVnode   *pImpl = vnodeOpen(pCfg->path, &cfg);
+    snprintf(path, TSDB_FILENAME_LEN, "vnode%svnode%d", TD_DIRSEP, pCfg->vgId);
+    SVnode *pImpl = vnodeOpen(path, pMgmt->pTfs, msgCb);
     if (pImpl == NULL) {
       dError("vgId:%d, failed to open vnode by thread:%d", pCfg->vgId, pThread->threadIndex);
       pThread->failed++;

source/dnode/mnode/impl/CMakeLists.txt

@@ -12,8 +12,8 @@ target_link_libraries(
 IF (TD_GRANT)
   TARGET_LINK_LIBRARIES(mnode grant)
 ENDIF ()
-IF (TD_USB_DONGLE)
-  TARGET_LINK_LIBRARIES(mnode usb_dongle)
+IF (TD_GRANT)
+  ADD_DEFINITIONS(-D_GRANT)
 ENDIF ()
 
 if(${BUILD_TEST})

source/dnode/mnode/impl/inc/mndStream.h

@@ -31,7 +31,7 @@ void        mndReleaseStream(SMnode *pMnode, SStreamObj *pStream);
 SSdbRaw *mndStreamActionEncode(SStreamObj *pStream);
 SSdbRow *mndStreamActionDecode(SSdbRaw *pRaw);
 
-int32_t mndAddStreamToTrans(SMnode *pMnode, SStreamObj *pStream, const char *ast, STrans *pTrans);
+int32_t mndAddStreamToTrans(SMnode *pMnode, SStreamObj *pStream, const char *ast, int8_t triggerType, int64_t watermark, STrans *pTrans);
 
 #ifdef __cplusplus
 }

source/dnode/mnode/impl/src/mndDb.c

@@ -106,6 +106,7 @@ static SSdbRaw *mndDbActionEncode(SDbObj *pDb) {
   SDB_SET_INT8(pRaw, dataPos, pDb->cfg.cacheLastRow, DB_ENCODE_OVER)
   SDB_SET_INT32(pRaw, dataPos, pDb->cfg.numOfRetensions, DB_ENCODE_OVER)
   for (int32_t i = 0; i < pDb->cfg.numOfRetensions; ++i) {
+    TASSERT(taosArrayGetSize(pDb->cfg.pRetensions) == pDb->cfg.numOfRetensions);
     SRetention *pRetension = taosArrayGet(pDb->cfg.pRetensions, i);
     SDB_SET_INT32(pRaw, dataPos, pRetension->freq, DB_ENCODE_OVER)
     SDB_SET_INT32(pRaw, dataPos, pRetension->keep, DB_ENCODE_OVER)

source/dnode/mnode/impl/src/mndInfoSchema.c

@@ -82,7 +82,7 @@ static const SInfosTableSchema userDBSchema[] = {
     {.name = "blocks", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
     {.name = "minrows", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
     {.name = "maxrows", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
-    {.name = "wallevel", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT},
+    {.name = "wal", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT},
     {.name = "fsync", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
     {.name = "comp", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT},
     {.name = "cachelast", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT},

source/dnode/mnode/impl/src/mndSma.c

@@ -429,7 +429,7 @@ static int32_t mndCreateSma(SMnode *pMnode, SNodeMsg *pReq, SMCreateSmaReq *pCre
   if (mndSetCreateSmaRedoLogs(pMnode, pTrans, &smaObj) != 0) goto _OVER;
   if (mndSetCreateSmaCommitLogs(pMnode, pTrans, &smaObj) != 0) goto _OVER;
   if (mndSetCreateSmaRedoActions(pMnode, pTrans, pDb, &smaObj) != 0) goto _OVER;
-  if (mndAddStreamToTrans(pMnode, &streamObj, pCreate->ast, pTrans) != 0) goto _OVER;
+  if (mndAddStreamToTrans(pMnode, &streamObj, pCreate->ast, STREAM_TRIGGER_AT_ONCE, 0, pTrans) != 0) goto _OVER;
   if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
 
   code = 0;

source/dnode/mnode/impl/src/mndStream.c

@@ -218,7 +218,7 @@ static int32_t mndCheckCreateStreamReq(SCMCreateStreamReq *pCreate) {
   return 0;
 }
 
-static int32_t mndStreamGetPlanString(const char *ast, char **pStr) {
+static int32_t mndStreamGetPlanString(const char *ast, int8_t triggerType, int64_t watermark, char **pStr) {
   if (NULL == ast) {
     return TSDB_CODE_SUCCESS;
   }
@@ -232,6 +232,8 @@ static int32_t mndStreamGetPlanString(const char *ast, char **pStr) {
         .pAstRoot = pAst,
         .topicQuery = false,
         .streamQuery = true,
+        .triggerType = triggerType,
+        .watermark = watermark,
     };
     code = qCreateQueryPlan(&cxt, &pPlan, NULL);
   }
@@ -245,7 +247,7 @@ static int32_t mndStreamGetPlanString(const char *ast, char **pStr) {
   return code;
 }
 
-int32_t mndAddStreamToTrans(SMnode *pMnode, SStreamObj *pStream, const char *ast, STrans *pTrans) {
+int32_t mndAddStreamToTrans(SMnode *pMnode, SStreamObj *pStream, const char *ast, int8_t triggerType, int64_t watermark, STrans *pTrans) {
   SNode *pAst = NULL;
 
   if (nodesStringToNode(ast, &pAst) < 0) {
@@ -265,7 +267,7 @@ int32_t mndAddStreamToTrans(SMnode *pMnode, SStreamObj *pStream, const char *ast
 
 #endif
 
-  if (TSDB_CODE_SUCCESS != mndStreamGetPlanString(ast, &pStream->physicalPlan)) {
+  if (TSDB_CODE_SUCCESS != mndStreamGetPlanString(ast, triggerType, watermark, &pStream->physicalPlan)) {
     mError("topic:%s, failed to get plan since %s", pStream->name, terrstr());
     return -1;
   }
@@ -313,7 +315,7 @@ static int32_t mndCreateStream(SMnode *pMnode, SNodeMsg *pReq, SCMCreateStreamRe
   }
   mDebug("trans:%d, used to create stream:%s", pTrans->id, pCreate->name);
 
-  if (mndAddStreamToTrans(pMnode, &streamObj, pCreate->ast, pTrans) != 0) {
+  if (mndAddStreamToTrans(pMnode, &streamObj, pCreate->ast, pCreate->triggerType, pCreate->watermark, pTrans) != 0) {
     mError("trans:%d, failed to add stream since %s", pTrans->id, terrstr());
     mndTransDrop(pTrans);
     return -1;

source/dnode/vnode/CMakeLists.txt

@@ -18,12 +18,11 @@ target_sources(
     "src/vnd/vnodeSvr.c"
 
     # meta
-    # "src/meta/metaBDBImpl.c"
+    "src/meta/metaOpen.c"
     "src/meta/metaIdx.c"
-    "src/meta/metaMain.c"
     "src/meta/metaTable.c"
-    "src/meta/metaTbUid.c"
     "src/meta/metaTDBImpl.c"
+    # "src/meta/metaBDBImpl.c"
 
     # tsdb
     "src/tsdb/tsdbTDBImpl.c"

source/dnode/vnode/inc/vnode.h

@@ -37,16 +37,14 @@ extern "C" {
 
 // vnode
 typedef struct SVnode    SVnode;
-typedef struct SMetaCfg  SMetaCfg;  // todo: remove
 typedef struct STsdbCfg  STsdbCfg;  // todo: remove
-typedef struct STqCfg    STqCfg;    // todo: remove
 typedef struct SVnodeCfg SVnodeCfg;
 
 int     vnodeInit(int nthreads);
 void    vnodeCleanup();
 int     vnodeCreate(const char *path, SVnodeCfg *pCfg, STfs *pTfs);
-void    vnodeDestroy(const char *path);
-SVnode *vnodeOpen(const char *path, const SVnodeCfg *pVnodeCfg);
+void    vnodeDestroy(const char *path, STfs *pTfs);
+SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb);
 void    vnodeClose(SVnode *pVnode);
 void    vnodePreprocessWriteReqs(SVnode *pVnode, SArray *pMsgs);
 int     vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp);
@@ -73,7 +71,6 @@ char       *metaTbCursorNext(SMTbCursor *pTbCur);
 
 // tsdb
 typedef struct STsdb          STsdb;
-typedef struct SDataStatis    SDataStatis;
 typedef struct STsdbQueryCond STsdbQueryCond;
 typedef void                 *tsdbReaderT;
 
@@ -93,7 +90,7 @@ int32_t      tsdbQuerySTableByTagCond(void *pMeta, uint64_t uid, TSKEY skey, con
 int64_t      tsdbGetNumOfRowsInMemTable(tsdbReaderT *pHandle);
 bool         tsdbNextDataBlock(tsdbReaderT pTsdbReadHandle);
 void         tsdbRetrieveDataBlockInfo(tsdbReaderT *pTsdbReadHandle, SDataBlockInfo *pBlockInfo);
-int32_t      tsdbRetrieveDataBlockStatisInfo(tsdbReaderT *pTsdbReadHandle, SDataStatis **pBlockStatis);
+int32_t      tsdbRetrieveDataBlockStatisInfo(tsdbReaderT *pTsdbReadHandle, SColumnDataAgg **pBlockStatis);
 SArray      *tsdbRetrieveDataBlock(tsdbReaderT *pTsdbReadHandle, SArray *pColumnIdList);
 void         tsdbDestroyTableGroup(STableGroupInfo *pGroupList);
 int32_t      tsdbGetOneTableGroup(void *pMeta, uint64_t uid, TSKEY startKey, STableGroupInfo *pGroupInfo);
@@ -124,24 +121,19 @@ struct STsdbCfg {
   int8_t   precision;
   int8_t   update;
   int8_t   compression;
-  int32_t  daysPerFile;
-  int32_t  minRowsPerFileBlock;
-  int32_t  maxRowsPerFileBlock;
-  int32_t  keep;
-  int32_t  keep1;
+  int32_t  days;
+  int32_t  minRows;
+  int32_t  maxRows;
   int32_t  keep2;
+  int32_t  keep0;
+  int32_t  keep1;
   uint64_t lruCacheSize;
   SArray  *retentions;
 };
 
-struct STqCfg {
-  int32_t reserved;
-};
-
 struct SVnodeCfg {
   int32_t  vgId;
   uint64_t dbId;
-  STfs    *pTfs;
   uint64_t wsize;
   uint64_t ssize;
   uint64_t lsize;
@@ -151,25 +143,12 @@ struct SVnodeCfg {
   int8_t   streamMode;
   bool     isWeak;
   STsdbCfg tsdbCfg;
-  SMetaCfg metaCfg;
-  STqCfg   tqCfg;
   SWalCfg  walCfg;
-  SMsgCb   msgCb;
   uint32_t hashBegin;
   uint32_t hashEnd;
   int8_t   hashMethod;
 };
 
-struct SDataStatis {
-  int16_t colId;
-  int16_t maxIndex;
-  int16_t minIndex;
-  int16_t numOfNull;
-  int64_t sum;
-  int64_t max;
-  int64_t min;
-};
-
 struct STsdbQueryCond {
   STimeWindow  twindow;
   int32_t      order;  // desc|asc order to iterate the data block

source/dnode/vnode/src/inc/meta.h

@@ -20,7 +20,6 @@
 extern "C" {
 #endif
 
-typedef struct SMetaCache  SMetaCache;
 typedef struct SMetaIdx    SMetaIdx;
 typedef struct SMetaDB     SMetaDB;
 typedef struct SMCtbCursor SMCtbCursor;
@@ -36,13 +35,22 @@ typedef struct SMSmaCursor SMSmaCursor;
 #define metaTrace(...) do { if (metaDebugFlag & DEBUG_TRACE) { taosPrintLog("META ", DEBUG_TRACE, metaDebugFlag, __VA_ARGS__); }} while(0)
 // clang-format on
 
+// metaOpen ==================
+int metaOpen(SVnode* pVnode, SMeta** ppMeta);
+int metaClose(SMeta* pMeta);
+
+// metaIdx ==================
+int  metaOpenIdx(SMeta* pMeta);
+void metaCloseIdx(SMeta* pMeta);
+int  metaSaveTableToIdx(SMeta* pMeta, const STbCfg* pTbOptions);
+int  metaRemoveTableFromIdx(SMeta* pMeta, tb_uid_t uid);
+
+static FORCE_INLINE tb_uid_t metaGenerateUid(SMeta* pMeta) { return tGenIdPI64(); }
+
 #define META_SUPER_TABLE  TD_SUPER_TABLE
 #define META_CHILD_TABLE  TD_CHILD_TABLE
 #define META_NORMAL_TABLE TD_NORMAL_TABLE
 
-SMeta*          metaOpen(const char* path, const SMetaCfg* pMetaCfg, SMemAllocatorFactory* pMAF);
-void            metaClose(SMeta* pMeta);
-void            metaRemove(const char* path);
 int             metaCreateTable(SMeta* pMeta, STbCfg* pTbCfg);
 int             metaDropTable(SMeta* pMeta, tb_uid_t uid);
 int             metaCommit(SMeta* pMeta);
@@ -71,38 +79,15 @@ int  metaRemoveTableFromDb(SMeta* pMeta, tb_uid_t uid);
 int  metaSaveSmaToDB(SMeta* pMeta, STSma* pTbCfg);
 int  metaRemoveSmaFromDb(SMeta* pMeta, int64_t indexUid);
 
-// SMetaCache
-int  metaOpenCache(SMeta* pMeta);
-void metaCloseCache(SMeta* pMeta);
-
 // SMetaIdx
-int  metaOpenIdx(SMeta* pMeta);
-void metaCloseIdx(SMeta* pMeta);
-int  metaSaveTableToIdx(SMeta* pMeta, const STbCfg* pTbOptions);
-int  metaRemoveTableFromIdx(SMeta* pMeta, tb_uid_t uid);
-
-// STbUidGnrt
-typedef struct STbUidGenerator {
-  tb_uid_t nextUid;
-} STbUidGenerator;
-
-// STableUidGenerator
-int  metaOpenUidGnrt(SMeta* pMeta);
-void metaCloseUidGnrt(SMeta* pMeta);
 
-// tb_uid_t
-#define IVLD_TB_UID 0
 tb_uid_t metaGenerateUid(SMeta* pMeta);
 
 struct SMeta {
-  char*                 path;
-  SVnode*               pVnode;
-  SMetaCfg              options;
-  SMetaDB*              pDB;
-  SMetaIdx*             pIdx;
-  SMetaCache*           pCache;
-  STbUidGenerator       uidGnrt;
-  SMemAllocatorFactory* pmaf;
+  char*     path;
+  SVnode*   pVnode;
+  SMetaDB*  pDB;
+  SMetaIdx* pIdx;
 };
 
 #ifdef __cplusplus

source/dnode/vnode/src/inc/tq.h

@@ -160,7 +160,6 @@ struct STQ {
   // the handle of meta kvstore
   bool          writeTrigger;
   char*         path;
-  STqCfg*       tqConfig;
   STqMemRef     tqMemRef;
   STqMetaStore* tqMeta;
   // STqPushMgr*   tqPushMgr;
@@ -251,8 +250,7 @@ int  tqInit();
 void tqCleanUp();
 
 // open in each vnode
-STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal, SMeta* pMeta, STqCfg* tqConfig,
-            SMemAllocatorFactory* allocFac);
+STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal, SMeta* pMeta, SMemAllocatorFactory* allocFac);
 void tqClose(STQ*);
 // required by vnode
 int tqPushMsg(STQ*, void* msg, int32_t msgLen, tmsg_t msgType, int64_t version);

source/dnode/vnode/src/inc/tsdb.h

@@ -438,7 +438,7 @@ int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo,
 int tsdbLoadBlockStatis(SReadH *pReadh, SBlock *pBlock);
 int tsdbEncodeSBlockIdx(void **buf, SBlockIdx *pIdx);
 void *tsdbDecodeSBlockIdx(void *buf, SBlockIdx *pIdx);
-void  tsdbGetBlockStatis(SReadH *pReadh, SDataStatis *pStatis, int numOfCols, SBlock *pBlock);
+void  tsdbGetBlockStatis(SReadH *pReadh, SColumnDataAgg *pStatis, int numOfCols, SBlock *pBlock);
 
 static FORCE_INLINE int tsdbMakeRoom(void **ppBuf, size_t size) {
   void  *pBuf = *ppBuf;

source/dnode/vnode/src/inc/vnd.h

@@ -43,6 +43,12 @@ int  vnodeGetTableMeta(SVnode* pVnode, SRpcMsg* pMsg);
 // vnodeCommit ====================
 int vnodeSaveInfo(const char* dir, const SVnodeInfo* pCfg);
 int vnodeCommitInfo(const char* dir, const SVnodeInfo* pInfo);
+int vnodeLoadInfo(const char* dir, SVnodeInfo* pInfo);
+int vnodeBegin(SVnode* pVnode, int option);
+int vnodeSyncCommit(SVnode* pVnode);
+int vnodeAsyncCommit(SVnode* pVnode);
+
+#define vnodeShouldCommit vnodeBufPoolIsFull
 
 #if 1
 // SVBufPool
@@ -83,13 +89,8 @@ bool            vmaIsFull(SVMemAllocator* pVMA);
 // vnodeCfg.h
 extern const SVnodeCfg vnodeCfgDefault;
 
-int  vnodeCheckCfg(const SVnodeCfg*);
-void vnodeOptionsCopy(SVnodeCfg* pDest, const SVnodeCfg* pSrc);
+int vnodeCheckCfg(const SVnodeCfg*);
 
-// For commit
-#define vnodeShouldCommit vnodeBufPoolIsFull
-int vnodeSyncCommit(SVnode* pVnode);
-int vnodeAsyncCommit(SVnode* pVnode);
 #endif
 
 #ifdef __cplusplus

source/dnode/vnode/src/inc/vnodeInt.h

@@ -27,6 +27,7 @@
 #include "tdbInt.h"
 #include "tfs.h"
 #include "tglobal.h"
+#include "tjson.h"
 #include "tlist.h"
 #include "tlockfree.h"
 #include "tlosertree.h"
@@ -51,6 +52,11 @@ typedef struct SVState      SVState;
 typedef struct SVBufPool    SVBufPool;
 typedef struct SQWorkerMgmt SQHandle;
 
+#define VNODE_META_DIR "meta"
+#define VNODE_TSDB_DIR "tsdb"
+#define VNODE_TQ_DIR   "tq"
+#define VNODE_WAL_DIR  "wal"
+
 typedef struct {
   int8_t  streamType;  // sma or other
   int8_t  dstType;
@@ -79,10 +85,11 @@ struct SVnodeInfo {
 };
 
 struct SVnode {
-  int32_t    vgId;
   char*      path;
   SVnodeCfg  config;
   SVState    state;
+  STfs*      pTfs;
+  SMsgCb     msgCb;
   SVBufPool* pBufPool;
   SMeta*     pMeta;
   STsdb*     pTsdb;
@@ -91,10 +98,10 @@ struct SVnode {
   SSink*     pSink;
   tsem_t     canCommit;
   SQHandle*  pQuery;
-  SMsgCb     msgCb;
-  STfs*      pTfs;
 };
 
+#define TD_VID(PVNODE) (PVNODE)->config.vgId
+
 // sma
 void smaHandleRes(void* pVnode, int64_t smaId, const SArray* data);
 

source/dnode/vnode/src/meta/metaMain.c → source/dnode/vnode/src/meta/metaOpen.c

@@ -13,99 +13,54 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "tcoding.h"
-
 #include "vnodeInt.h"
 
-static SMeta *metaNew(const char *path, const SMetaCfg *pMetaCfg, SMemAllocatorFactory *pMAF);
-static void   metaFree(SMeta *pMeta);
-static int    metaOpenImpl(SMeta *pMeta);
-static void   metaCloseImpl(SMeta *pMeta);
-
-SMeta *metaOpen(const char *path, const SMetaCfg *pMetaCfg, SMemAllocatorFactory *pMAF) {
+int metaOpen(SVnode *pVnode, SMeta **ppMeta) {
   SMeta *pMeta = NULL;
+  int    slen;
 
-  // Allocate handle
-  pMeta = metaNew(path, pMetaCfg, pMAF);
-  if (pMeta == NULL) {
-    // TODO: handle error
-    return NULL;
-  }
-
-  // Create META path (TODO)
-  taosMkDir(path);
-
-  // Open meta
-  if (metaOpenImpl(pMeta) < 0) {
-    metaFree(pMeta);
-    return NULL;
-  }
-
-  return pMeta;
-}
+  *ppMeta = NULL;
 
-void metaClose(SMeta *pMeta) {
-  if (pMeta) {
-    metaCloseImpl(pMeta);
-    metaFree(pMeta);
+  // create handle
+  slen = strlen(tfsGetPrimaryPath(pVnode->pTfs)) + strlen(pVnode->path) + strlen(VNODE_META_DIR) + 3;
+  if ((pMeta = taosMemoryCalloc(1, sizeof(*pMeta) + slen)) == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    return -1;
   }
-}
 
-void metaRemove(const char *path) { taosRemoveDir(path); }
+  pMeta->path = (char *)&pMeta[1];
+  sprintf(pMeta->path, "%s%s%s%s%s", tfsGetPrimaryPath(pVnode->pTfs), TD_DIRSEP, pVnode->path, TD_DIRSEP,
+          VNODE_META_DIR);
+  pMeta->pVnode = pVnode;
 
-/* ------------------------ STATIC METHODS ------------------------ */
-static SMeta *metaNew(const char *path, const SMetaCfg *pMetaCfg, SMemAllocatorFactory *pMAF) {
-  SMeta *pMeta;
-  size_t psize = strlen(path);
+  // create path if not created yet
+  taosMkDir(pMeta->path);
 
-  pMeta = (SMeta *)taosMemoryCalloc(1, sizeof(*pMeta));
-  if (pMeta == NULL) {
-    return NULL;
+  // open meta
+  if (metaOpenDB(pMeta) < 0) {
+    goto _err;
   }
 
-  pMeta->path = strdup(path);
-  if (pMeta->path == NULL) {
-    metaFree(pMeta);
-    return NULL;
+  if (metaOpenIdx(pMeta) < 0) {
+    goto _err;
   }
 
-  return pMeta;
-};
+  *ppMeta = pMeta;
+  return 0;
 
-static void metaFree(SMeta *pMeta) {
-  if (pMeta) {
-    taosMemoryFreeClear(pMeta->path);
-    taosMemoryFree(pMeta);
-  }
+_err:
+  if (pMeta->pIdx) metaCloseIdx(pMeta);
+  if (pMeta->pDB) metaCloseDB(pMeta);
+  taosMemoryFree(pMeta);
+  return -1;
 }
 
-static int metaOpenImpl(SMeta *pMeta) {
-  // Open meta db
-  if (metaOpenDB(pMeta) < 0) {
-    // TODO: handle error
-    metaCloseImpl(pMeta);
-    return -1;
-  }
-
-  // Open meta index
-  if (metaOpenIdx(pMeta) < 0) {
-    // TODO: handle error
-    metaCloseImpl(pMeta);
-    return -1;
-  }
-
-  // Open meta table uid generator
-  if (metaOpenUidGnrt(pMeta) < 0) {
-    // TODO: handle error
-    metaCloseImpl(pMeta);
-    return -1;
+int metaClose(SMeta *pMeta) {
+  if (pMeta) {
+    metaCloseIdx(pMeta);
+    metaCloseDB(pMeta);
+    taosMemoryFree(pMeta);
   }
 
   return 0;
-}
-
-static void metaCloseImpl(SMeta *pMeta) {
-  metaCloseUidGnrt(pMeta);
-  metaCloseIdx(pMeta);
-  metaCloseDB(pMeta);
 }
\ No newline at end of file

source/dnode/vnode/src/tq/tq.c

@@ -19,15 +19,13 @@ int32_t tqInit() { return tqPushMgrInit(); }
 
 void tqCleanUp() { tqPushMgrCleanUp(); }
 
-STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal, SMeta* pVnodeMeta, STqCfg* tqConfig,
-            SMemAllocatorFactory* allocFac) {
+STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal, SMeta* pVnodeMeta, SMemAllocatorFactory* allocFac) {
   STQ* pTq = taosMemoryMalloc(sizeof(STQ));
   if (pTq == NULL) {
     terrno = TSDB_CODE_TQ_OUT_OF_MEMORY;
     return NULL;
   }
   pTq->path = strdup(path);
-  pTq->tqConfig = tqConfig;
   pTq->pVnode = pVnode;
   pTq->pWal = pWal;
   pTq->pVnodeMeta = pVnodeMeta;
@@ -267,7 +265,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
   }
 
   vDebug("tmq poll: consumer %ld (epoch %d) recv poll req in vg %d, req %ld %ld", consumerId, pReq->epoch,
-         pTq->pVnode->vgId, pReq->currentOffset, fetchOffset);
+         TD_VID(pTq->pVnode), pReq->currentOffset, fetchOffset);
 
   SMqPollRsp rsp = {
       /*.consumerId = consumerId,*/
@@ -277,7 +275,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
 
   STqConsumer* pConsumer = tqHandleGet(pTq->tqMeta, consumerId);
   if (pConsumer == NULL) {
-    vWarn("tmq poll: consumer %ld (epoch %d) not found in vg %d", consumerId, pReq->epoch, pTq->pVnode->vgId);
+    vWarn("tmq poll: consumer %ld (epoch %d) not found in vg %d", consumerId, pReq->epoch, TD_VID(pTq->pVnode));
     pMsg->pCont = NULL;
     pMsg->contLen = 0;
     pMsg->code = -1;
@@ -303,7 +301,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
   }
   if (pTopic == NULL) {
     vWarn("tmq poll: consumer %ld (epoch %d) topic %s not found in vg %d", consumerId, pReq->epoch, pReq->topic,
-          pTq->pVnode->vgId);
+          TD_VID(pTq->pVnode));
     pMsg->pCont = NULL;
     pMsg->contLen = 0;
     pMsg->code = -1;
@@ -312,7 +310,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
   }
 
   vDebug("poll topic %s from consumer %ld (epoch %d) vg %d", pTopic->topicName, consumerId, pReq->epoch,
-         pTq->pVnode->vgId);
+         TD_VID(pTq->pVnode));
 
   rsp.reqOffset = pReq->currentOffset;
   rsp.skipLogNum = 0;
@@ -323,7 +321,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
     consumerEpoch = atomic_load_32(&pConsumer->epoch);
     if (consumerEpoch > reqEpoch) {
       vDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, found new consumer epoch %d discard req epoch %d",
-             consumerId, pReq->epoch, pTq->pVnode->vgId, fetchOffset, consumerEpoch, reqEpoch);
+             consumerId, pReq->epoch, TD_VID(pTq->pVnode), fetchOffset, consumerEpoch, reqEpoch);
       break;
     }
     SWalReadHead* pHead;
@@ -332,11 +330,11 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
       // if data inserted during waiting, launch query and
       // response to user
       vDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, no more log to return", consumerId, pReq->epoch,
-             pTq->pVnode->vgId, fetchOffset);
+             TD_VID(pTq->pVnode), fetchOffset);
       break;
     }
     vDebug("tmq poll: consumer %ld (epoch %d) iter log, vg %d offset %ld msgType %d", consumerId, pReq->epoch,
-           pTq->pVnode->vgId, fetchOffset, pHead->msgType);
+           TD_VID(pTq->pVnode), fetchOffset, pHead->msgType);
     /*int8_t pos = fetchOffset % TQ_BUFFER_SIZE;*/
     /*pHead = pTopic->pReadhandle->pHead;*/
     if (pHead->msgType == TDMT_VND_SUBMIT) {
@@ -361,7 +359,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
 
       if (taosArrayGetSize(pRes) == 0) {
         vDebug("tmq poll: consumer %ld (epoch %d) iter log, vg %d skip log %ld since not wanted", consumerId,
-               pReq->epoch, pTq->pVnode->vgId, fetchOffset);
+               pReq->epoch, TD_VID(pTq->pVnode), fetchOffset);
         fetchOffset++;
         rsp.skipLogNum++;
         taosArrayDestroy(pRes);
@@ -390,7 +388,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
       pMsg->pCont = buf;
       pMsg->contLen = tlen;
       pMsg->code = 0;
-      vDebug("vg %d offset %ld msgType %d from consumer %ld (epoch %d) actual rsp", pTq->pVnode->vgId, fetchOffset,
+      vDebug("vg %d offset %ld msgType %d from consumer %ld (epoch %d) actual rsp", TD_VID(pTq->pVnode), fetchOffset,
              pHead->msgType, consumerId, pReq->epoch);
       tmsgSendRsp(pMsg);
       taosMemoryFree(pHead);
@@ -422,7 +420,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
   pMsg->contLen = tlen;
   pMsg->code = 0;
   tmsgSendRsp(pMsg);
-  vDebug("vg %d offset %ld from consumer %ld (epoch %d) not rsp", pTq->pVnode->vgId, fetchOffset, consumerId,
+  vDebug("vg %d offset %ld from consumer %ld (epoch %d) not rsp", TD_VID(pTq->pVnode), fetchOffset, consumerId,
          pReq->epoch);
   /*}*/
 
@@ -446,14 +444,14 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
   }
 
   vDebug("tmq poll: consumer %ld (epoch %d) recv poll req in vg %d, req %ld %ld", consumerId, pReq->epoch,
-         pTq->pVnode->vgId, pReq->currentOffset, fetchOffset);
+         TD_VID(pTq->pVnode), pReq->currentOffset, fetchOffset);
 
   SMqPollRspV2 rspV2 = {0};
   rspV2.dataLen = 0;
 
   STqConsumer* pConsumer = tqHandleGet(pTq->tqMeta, consumerId);
   if (pConsumer == NULL) {
-    vWarn("tmq poll: consumer %ld (epoch %d) not found in vg %d", consumerId, pReq->epoch, pTq->pVnode->vgId);
+    vWarn("tmq poll: consumer %ld (epoch %d) not found in vg %d", consumerId, pReq->epoch, TD_VID(pTq->pVnode));
     pMsg->pCont = NULL;
     pMsg->contLen = 0;
     pMsg->code = -1;
@@ -479,7 +477,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
   }
   if (pTopic == NULL) {
     vWarn("tmq poll: consumer %ld (epoch %d) topic %s not found in vg %d", consumerId, pReq->epoch, pReq->topic,
-          pTq->pVnode->vgId);
+          TD_VID(pTq->pVnode));
     pMsg->pCont = NULL;
     pMsg->contLen = 0;
     pMsg->code = -1;
@@ -488,7 +486,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
   }
 
   vDebug("poll topic %s from consumer %ld (epoch %d) vg %d", pTopic->topicName, consumerId, pReq->epoch,
-         pTq->pVnode->vgId);
+         TD_VID(pTq->pVnode));
 
   rspV2.reqOffset = pReq->currentOffset;
   rspV2.skipLogNum = 0;
@@ -499,7 +497,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
     consumerEpoch = atomic_load_32(&pConsumer->epoch);
     if (consumerEpoch > reqEpoch) {
       vDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, found new consumer epoch %d discard req epoch %d",
-             consumerId, pReq->epoch, pTq->pVnode->vgId, fetchOffset, consumerEpoch, reqEpoch);
+             consumerId, pReq->epoch, TD_VID(pTq->pVnode), fetchOffset, consumerEpoch, reqEpoch);
       break;
     }
     SWalReadHead* pHead;
@@ -508,11 +506,11 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
       // if data inserted during waiting, launch query and
       // response to user
       vDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, no more log to return", consumerId, pReq->epoch,
-             pTq->pVnode->vgId, fetchOffset);
+             TD_VID(pTq->pVnode), fetchOffset);
       break;
     }
     vDebug("tmq poll: consumer %ld (epoch %d) iter log, vg %d offset %ld msgType %d", consumerId, pReq->epoch,
-           pTq->pVnode->vgId, fetchOffset, pHead->msgType);
+           TD_VID(pTq->pVnode), fetchOffset, pHead->msgType);
     /*int8_t pos = fetchOffset % TQ_BUFFER_SIZE;*/
     /*pHead = pTopic->pReadhandle->pHead;*/
     if (pHead->msgType == TDMT_VND_SUBMIT) {
@@ -537,7 +535,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
 
       if (taosArrayGetSize(pRes) == 0) {
         vDebug("tmq poll: consumer %ld (epoch %d) iter log, vg %d skip log %ld since not wanted", consumerId,
-               pReq->epoch, pTq->pVnode->vgId, fetchOffset);
+               pReq->epoch, TD_VID(pTq->pVnode), fetchOffset);
         fetchOffset++;
         rspV2.skipLogNum++;
         taosArrayDestroy(pRes);
@@ -597,7 +595,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
       pMsg->pCont = buf;
       pMsg->contLen = msgLen;
       pMsg->code = 0;
-      vDebug("vg %d offset %ld msgType %d from consumer %ld (epoch %d) actual rsp", pTq->pVnode->vgId, fetchOffset,
+      vDebug("vg %d offset %ld msgType %d from consumer %ld (epoch %d) actual rsp", TD_VID(pTq->pVnode), fetchOffset,
              pHead->msgType, consumerId, pReq->epoch);
       tmsgSendRsp(pMsg);
       taosMemoryFree(pHead);
@@ -631,7 +629,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
   pMsg->contLen = tlen;
   pMsg->code = 0;
   tmsgSendRsp(pMsg);
-  vDebug("vg %d offset %ld from consumer %ld (epoch %d) not rsp", pTq->pVnode->vgId, fetchOffset, consumerId,
+  vDebug("vg %d offset %ld from consumer %ld (epoch %d) not rsp", TD_VID(pTq->pVnode), fetchOffset, consumerId,
          pReq->epoch);
   /*}*/
 
@@ -742,7 +740,7 @@ int32_t tqProcessSetConnReq(STQ* pTq, char* msg) {
     pTopic->buffer.output[i].task = qCreateStreamExecTaskInfo(req.qmsg, &handle);
     ASSERT(pTopic->buffer.output[i].task);
   }
-  vDebug("set topic %s to consumer %ld on vg %d", pTopic->topicName, req.consumerId, pTq->pVnode->vgId);
+  vDebug("set topic %s to consumer %ld on vg %d", pTopic->topicName, req.consumerId, TD_VID(pTq->pVnode));
   taosArrayPush(pConsumer->topics, pTopic);
   if (create) {
     tqHandleMovePut(pTq->tqMeta, req.consumerId, pConsumer);

source/dnode/vnode/src/tsdb/tsdbCommit.c

@@ -55,7 +55,7 @@ typedef struct {
 #define TSDB_COMMIT_BUF(ch)          TSDB_READ_BUF(&((ch)->readh))
 #define TSDB_COMMIT_COMP_BUF(ch)     TSDB_READ_COMP_BUF(&((ch)->readh))
 #define TSDB_COMMIT_EXBUF(ch)        TSDB_READ_EXBUF(&((ch)->readh))
-#define TSDB_COMMIT_DEFAULT_ROWS(ch) TSDB_DEFAULT_BLOCK_ROWS(TSDB_COMMIT_REPO(ch)->config.maxRowsPerFileBlock)
+#define TSDB_COMMIT_DEFAULT_ROWS(ch) TSDB_DEFAULT_BLOCK_ROWS(TSDB_COMMIT_REPO(ch)->config.maxRows)
 #define TSDB_COMMIT_TXN_VERSION(ch)  FS_TXN_VERSION(REPO_FS(TSDB_COMMIT_REPO(ch)))
 
 static void tsdbStartCommit(STsdb *pRepo);
@@ -217,14 +217,14 @@ void tsdbGetRtnSnap(STsdb *pRepo, SRtn *pRtn) {
   TSKEY     minKey, midKey, maxKey, now;
 
   now = taosGetTimestamp(pCfg->precision);
-  minKey = now - pCfg->keep * tsTickPerDay[pCfg->precision];
-  midKey = now - pCfg->keep2 * tsTickPerDay[pCfg->precision];
-  maxKey = now - pCfg->keep1 * tsTickPerDay[pCfg->precision];
+  minKey = now - pCfg->keep2 * tsTickPerDay[pCfg->precision];
+  midKey = now - pCfg->keep1 * tsTickPerDay[pCfg->precision];
+  maxKey = now - pCfg->keep0 * tsTickPerDay[pCfg->precision];
 
   pRtn->minKey = minKey;
-  pRtn->minFid = (int)(TSDB_KEY_FID(minKey, pCfg->daysPerFile, pCfg->precision));
-  pRtn->midFid = (int)(TSDB_KEY_FID(midKey, pCfg->daysPerFile, pCfg->precision));
-  pRtn->maxFid = (int)(TSDB_KEY_FID(maxKey, pCfg->daysPerFile, pCfg->precision));
+  pRtn->minFid = (int)(TSDB_KEY_FID(minKey, pCfg->days, pCfg->precision));
+  pRtn->midFid = (int)(TSDB_KEY_FID(midKey, pCfg->days, pCfg->precision));
+  pRtn->maxFid = (int)(TSDB_KEY_FID(maxKey, pCfg->days, pCfg->precision));
   tsdbDebug("vgId:%d now:%" PRId64 " minKey:%" PRId64 " minFid:%d, midFid:%d, maxFid:%d", REPO_ID(pRepo), now, minKey,
             pRtn->minFid, pRtn->midFid, pRtn->maxFid);
 }
@@ -286,7 +286,7 @@ static int tsdbInitCommitH(SCommitH *pCommith, STsdb *pRepo) {
     return -1;
   }
 
-  pCommith->pDataCols = tdNewDataCols(0, pCfg->maxRowsPerFileBlock);
+  pCommith->pDataCols = tdNewDataCols(0, pCfg->maxRows);
   if (pCommith->pDataCols == NULL) {
     terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
     tsdbDestroyCommitH(pCommith);
@@ -319,7 +319,7 @@ static int tsdbNextCommitFid(SCommitH *pCommith) {
     if (nextKey == TSDB_DATA_TIMESTAMP_NULL) {
       continue;
     } else {
-      int tfid = (int)(TSDB_KEY_FID(nextKey, pCfg->daysPerFile, pCfg->precision));
+      int tfid = (int)(TSDB_KEY_FID(nextKey, pCfg->days, pCfg->precision));
       if (fid == TSDB_IVLD_FID || fid > tfid) {
         fid = tfid;
       }
@@ -346,7 +346,7 @@ static int tsdbCommitToFile(SCommitH *pCommith, SDFileSet *pSet, int fid) {
   ASSERT(pSet == NULL || pSet->fid == fid);
 
   tsdbResetCommitFile(pCommith);
-  tsdbGetFidKeyRange(pCfg->daysPerFile, pCfg->precision, fid, &(pCommith->minKey), &(pCommith->maxKey));
+  tsdbGetFidKeyRange(pCfg->days, pCfg->precision, fid, &(pCommith->minKey), &(pCommith->maxKey));
 
   // Set and open files
   if (tsdbSetAndOpenCommitFile(pCommith, pSet, fid) < 0) {
@@ -1210,8 +1210,8 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF
   int64_t       offset = 0, offsetAggr = 0;
   int           rowsToWrite = pDataCols->numOfRows;
 
-  ASSERT(rowsToWrite > 0 && rowsToWrite <= pCfg->maxRowsPerFileBlock);
-  ASSERT((!isLast) || rowsToWrite < pCfg->minRowsPerFileBlock);
+  ASSERT(rowsToWrite > 0 && rowsToWrite <= pCfg->maxRows);
+  ASSERT((!isLast) || rowsToWrite < pCfg->minRows);
 
   // Make buffer space
   if (tsdbMakeRoom(ppBuf, tsdbBlockStatisSize(pDataCols->numOfCols, SBlockVerLatest)) < 0) {
@@ -1250,7 +1250,7 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF
                                                &(pBlockCol->sum), &(pBlockCol->minIndex), &(pBlockCol->maxIndex),
                                                &(pBlockCol->numOfNull));
 #endif
-      (*tDataTypes[pDataCol->type].statisFunc)(pDataCol->pData, rowsToWrite, &(pAggrBlkCol->min), &(pAggrBlkCol->max),
+      (*tDataTypes[pDataCol->type].statisFunc)(pDataCol->pBitmap, pDataCol->pData, rowsToWrite, &(pAggrBlkCol->min), &(pAggrBlkCol->max),
                                                &(pAggrBlkCol->sum), &(pAggrBlkCol->minIndex), &(pAggrBlkCol->maxIndex),
                                                &(pAggrBlkCol->numOfNull));
 
@@ -1460,7 +1460,7 @@ static int tsdbCommitMemData(SCommitH *pCommith, SCommitIter *pIter, TSKEY keyLi
 
     if (pCommith->pDataCols->numOfRows <= 0) break;
 
-    if (toData || pCommith->pDataCols->numOfRows >= pCfg->minRowsPerFileBlock) {
+    if (toData || pCommith->pDataCols->numOfRows >= pCfg->minRows) {
       pDFile = TSDB_COMMIT_DATA_FILE(pCommith);
       isLast = false;
     } else {
@@ -1619,7 +1619,7 @@ static int tsdbMergeBlockData(SCommitH *pCommith, SCommitIter *pIter, SDataCols
     if (pCommith->pDataCols->numOfRows == 0) break;
 
     if (isLastOneBlock) {
-      if (pCommith->pDataCols->numOfRows < pCfg->minRowsPerFileBlock) {
+      if (pCommith->pDataCols->numOfRows < pCfg->minRows) {
         pDFile = TSDB_COMMIT_LAST_FILE(pCommith);
         isLast = true;
       } else {
@@ -1667,7 +1667,8 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt
         if (tdGetColDataOfRow(&sVal, pDataCols->cols + i, *iter, pDataCols->bitmapMode) < 0) {
           TASSERT(0);
         }
-        tdAppendValToDataCol(pTarget->cols + i, sVal.valType, sVal.val, pTarget->numOfRows, pTarget->maxPoints, pTarget->bitmapMode);
+        tdAppendValToDataCol(pTarget->cols + i, sVal.valType, sVal.val, pTarget->numOfRows, pTarget->maxPoints,
+                             pTarget->bitmapMode);
       }
 
       ++pTarget->numOfRows;
@@ -1774,11 +1775,11 @@ static bool tsdbCanAddSubBlock(SCommitH *pCommith, SBlock *pBlock, SMergeInfo *p
 
   ASSERT(mergeRows > 0);
 
-  if (pBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS && pInfo->nOperations <= pCfg->maxRowsPerFileBlock) {
+  if (pBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS && pInfo->nOperations <= pCfg->maxRows) {
     if (pBlock->last) {
-      if (pCommith->isLFileSame && mergeRows < pCfg->minRowsPerFileBlock) return true;
+      if (pCommith->isLFileSame && mergeRows < pCfg->minRows) return true;
     } else {
-      if (pCommith->isDFileSame && mergeRows <= pCfg->maxRowsPerFileBlock) return true;
+      if (pCommith->isDFileSame && mergeRows <= pCfg->maxRows) return true;
     }
   }
 

source/dnode/vnode/src/tsdb/tsdbFS.c

@@ -190,8 +190,8 @@ static int tsdbAddDFileSetToStatus(SFSStatus *pStatus, const SDFileSet *pSet) {
 
 // ================== STsdbFS
 STsdbFS *tsdbNewFS(const STsdbCfg *pCfg) {
-  int      keep = pCfg->keep;
-  int      days = pCfg->daysPerFile;
+  int      keep = pCfg->keep2;
+  int      days = pCfg->days;
   int      maxFSet = TSDB_MAX_FSETS(keep, days);
   STsdbFS *pfs;
 

source/dnode/vnode/src/tsdb/tsdbMemTable.c

@@ -19,9 +19,9 @@ static int      tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg);
 static int      tsdbMemTableInsertTbData(STsdb *pRepo, SSubmitBlk *pBlock, int32_t *pAffectedRows);
 static STbData *tsdbNewTbData(tb_uid_t uid);
 static void     tsdbFreeTbData(STbData *pTbData);
-static char *   tsdbGetTsTupleKey(const void *data);
+static char    *tsdbGetTsTupleKey(const void *data);
 static int      tsdbTbDataComp(const void *arg1, const void *arg2);
-static char *   tsdbTbDataGetUid(const void *arg);
+static char    *tsdbTbDataGetUid(const void *arg);
 static int      tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema **ppSchema, STSRow *row);
 
 STsdbMemTable *tsdbNewMemTable(STsdb *pTsdb) {
@@ -74,7 +74,7 @@ void tsdbFreeMemTable(STsdb *pTsdb, STsdbMemTable *pMemTable) {
 }
 
 int tsdbMemTableInsert(STsdb *pTsdb, STsdbMemTable *pMemTable, SSubmitReq *pMsg, SSubmitRsp *pRsp) {
-  SSubmitBlk *   pBlock = NULL;
+  SSubmitBlk    *pBlock = NULL;
   SSubmitMsgIter msgIter = {0};
   int32_t        affectedrows = 0, numOfRows = 0;
 
@@ -97,8 +97,8 @@ int tsdbMemTableInsert(STsdb *pTsdb, STsdbMemTable *pMemTable, SSubmitReq *pMsg,
   }
 
   if (pRsp != NULL) {
-    pRsp->affectedRows = htonl(affectedrows);
-    pRsp->numOfRows = htonl(numOfRows);
+    pRsp->affectedRows = affectedrows;
+    pRsp->numOfRows = numOfRows;
   }
 
   return 0;
@@ -119,12 +119,12 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey
                           TKEY *filterKeys, int nFilterKeys, bool keepDup, SMergeInfo *pMergeInfo) {
   ASSERT(maxRowsToRead > 0 && nFilterKeys >= 0);
   if (pIter == NULL) return 0;
-  STSchema * pSchema = NULL;
+  STSchema  *pSchema = NULL;
   TSKEY      rowKey = 0;
   TSKEY      fKey = 0;
   bool       isRowDel = false;
   int        filterIter = 0;
-  STSRow *   row = NULL;
+  STSRow    *row = NULL;
   SMergeInfo mInfo;
 
   if (pMergeInfo == NULL) pMergeInfo = &mInfo;
@@ -259,12 +259,12 @@ static int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg) {
   ASSERT(pMsg != NULL);
   // STsdbMeta *    pMeta = pTsdb->tsdbMeta;
   SSubmitMsgIter msgIter = {0};
-  SSubmitBlk *   pBlock = NULL;
+  SSubmitBlk    *pBlock = NULL;
   SSubmitBlkIter blkIter = {0};
-  STSRow *       row = NULL;
+  STSRow        *row = NULL;
   TSKEY          now = taosGetTimestamp(pTsdb->config.precision);
-  TSKEY          minKey = now - tsTickPerDay[pTsdb->config.precision] * pTsdb->config.keep;
-  TSKEY          maxKey = now + tsTickPerDay[pTsdb->config.precision] * pTsdb->config.daysPerFile;
+  TSKEY          minKey = now - tsTickPerDay[pTsdb->config.precision] * pTsdb->config.keep2;
+  TSKEY          maxKey = now + tsTickPerDay[pTsdb->config.precision] * pTsdb->config.days;
 
   terrno = TSDB_CODE_SUCCESS;
   pMsg->length = htonl(pMsg->length);
@@ -332,9 +332,9 @@ static int tsdbMemTableInsertTbData(STsdb *pTsdb, SSubmitBlk *pBlock, int32_t *p
   // STable          *pTable = NULL;
   SSubmitBlkIter blkIter = {0};
   STsdbMemTable *pMemTable = pTsdb->mem;
-  void *         tptr;
-  STbData *      pTbData;
-  STSRow *       row;
+  void          *tptr;
+  STbData       *pTbData;
+  STSRow        *row;
   TSKEY          keyMin;
   TSKEY          keyMax;
 
@@ -504,7 +504,7 @@ int tsdbInsertDataToMemTable(STsdbMemTable *pMemTable, SSubmitReq *pMsg) {
 #include "tskiplist.h"
 
 #define TSDB_DATA_SKIPLIST_LEVEL 5
-#define TSDB_MAX_INSERT_BATCH 512
+#define TSDB_MAX_INSERT_BATCH    512
 
 typedef struct {
   int32_t  totalLen;

source/dnode/vnode/src/tsdb/tsdbOptions.c

@@ -17,12 +17,12 @@
 
 const STsdbCfg defautlTsdbOptions = {.precision = 0,
                                      .lruCacheSize = 0,
-                                     .daysPerFile = 10,
-                                     .minRowsPerFileBlock = 100,
-                                     .maxRowsPerFileBlock = 4096,
-                                     .keep = 3650,
-                                     .keep1 = 3650,
+                                     .days = 10,
+                                     .minRows = 100,
+                                     .maxRows = 4096,
                                      .keep2 = 3650,
+                                     .keep0 = 3650,
+                                     .keep1 = 3650,
                                      .update = 0,
                                      .compression = TWO_STAGE_COMP};
 

source/dnode/vnode/src/tsdb/tsdbRead.c

@@ -99,10 +99,10 @@ typedef struct SIOCostSummary {
 
 typedef struct STsdbReadHandle {
   STsdb*        pTsdb;
-  SQueryFilePos cur;  // current position
+  SQueryFilePos cur;       // current position
   int16_t       order;
-  STimeWindow   window;  // the primary query time window that applies to all queries
-  SDataStatis*  statis;  // query level statistics, only one table block statistics info exists at any time
+  STimeWindow   window;    // the primary query time window that applies to all queries
+  SColumnDataAgg* statis;  // query level statistics, only one table block statistics info exists at any time
   int32_t       numOfBlocks;
   SArray*       pColumns;  // column list, SColumnInfoData array list
   bool          locateStart;
@@ -314,7 +314,7 @@ static int64_t getEarliestValidTimestamp(STsdb* pTsdb) {
   STsdbCfg* pCfg = &pTsdb->config;
 
   int64_t now = taosGetTimestamp(pCfg->precision);
-  return now - (tsTickPerDay[pCfg->precision] * pCfg->keep) + 1;  // needs to add one tick
+  return now - (tsTickPerDay[pCfg->precision] * pCfg->keep2) + 1;  // needs to add one tick
 }
 
 static void setQueryTimewindow(STsdbReadHandle* pTsdbReadHandle, STsdbQueryCond* pCond) {
@@ -377,7 +377,7 @@ static STsdbReadHandle* tsdbQueryTablesImpl(STsdb* tsdb, STsdbQueryCond* pCond,
 
   if (pCond->numOfCols > 0) {
     // allocate buffer in order to load data blocks from file
-    pReadHandle->statis = taosMemoryCalloc(pCond->numOfCols, sizeof(SDataStatis));
+    pReadHandle->statis = taosMemoryCalloc(pCond->numOfCols, sizeof(SColumnDataAgg));
     if (pReadHandle->statis == NULL) {
       goto _end;
     }
@@ -404,7 +404,7 @@ static STsdbReadHandle* tsdbQueryTablesImpl(STsdb* tsdb, STsdbQueryCond* pCond,
     pReadHandle->defaultLoadColumn = getDefaultLoadColumns(pReadHandle, true);
   }
 
-  pReadHandle->pDataCols = tdNewDataCols(1000, pReadHandle->pTsdb->config.maxRowsPerFileBlock);
+  pReadHandle->pDataCols = tdNewDataCols(1000, pReadHandle->pTsdb->config.maxRows);
   if (pReadHandle->pDataCols == NULL) {
     tsdbError("%p failed to malloc buf for pDataCols, %s", pReadHandle, pReadHandle->idStr);
     terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
@@ -477,7 +477,7 @@ void tsdbResetQueryHandle(tsdbReaderT queryHandle, STsdbQueryCond* pCond) {
   }
 
   // allocate buffer in order to load data blocks from file
-  memset(pTsdbReadHandle->statis, 0, sizeof(SDataStatis));
+  memset(pTsdbReadHandle->statis, 0, sizeof(SColumnDataAgg));
 
   tsdbInitDataBlockLoadInfo(&pTsdbReadHandle->dataBlockLoadInfo);
   tsdbInitCompBlockLoadInfo(&pTsdbReadHandle->compBlockLoadInfo);
@@ -505,7 +505,7 @@ void tsdbResetQueryHandleForNewTable(tsdbReaderT queryHandle, STsdbQueryCond* pC
   }
 
   // allocate buffer in order to load data blocks from file
-  memset(pTsdbReadHandle->statis, 0, sizeof(SDataStatis));
+  memset(pTsdbReadHandle->statis, 0, sizeof(SColumnDataAgg));
 
   tsdbInitDataBlockLoadInfo(&pTsdbReadHandle->dataBlockLoadInfo);
   tsdbInitCompBlockLoadInfo(&pTsdbReadHandle->compBlockLoadInfo);
@@ -2199,7 +2199,7 @@ static int32_t getFirstFileDataBlock(STsdbReadHandle* pTsdbReadHandle, bool* exi
       break;
     }
 
-    tsdbGetFidKeyRange(pCfg->daysPerFile, pCfg->precision, pTsdbReadHandle->pFileGroup->fid, &win.skey, &win.ekey);
+    tsdbGetFidKeyRange(pCfg->days, pCfg->precision, pTsdbReadHandle->pFileGroup->fid, &win.skey, &win.ekey);
 
     // current file are not overlapped with query time window, ignore remain files
     if ((ASCENDING_TRAVERSE(pTsdbReadHandle->order) && win.skey > pTsdbReadHandle->window.ekey) ||
@@ -2295,7 +2295,7 @@ int32_t tsdbGetFileBlocksDistInfo(tsdbReaderT* queryHandle, STableBlockDistInfo*
   // find the start data block in file
   pTsdbReadHandle->locateStart = true;
   STsdbCfg* pCfg = &pTsdbReadHandle->pTsdb->config;
-  int32_t   fid = getFileIdFromKey(pTsdbReadHandle->window.skey, pCfg->daysPerFile, pCfg->precision);
+  int32_t   fid = getFileIdFromKey(pTsdbReadHandle->window.skey, pCfg->days, pCfg->precision);
 
   tsdbRLockFS(pFileHandle);
   tsdbFSIterInit(&pTsdbReadHandle->fileIter, pFileHandle, pTsdbReadHandle->order);
@@ -2321,7 +2321,7 @@ int32_t tsdbGetFileBlocksDistInfo(tsdbReaderT* queryHandle, STableBlockDistInfo*
       break;
     }
 
-    tsdbGetFidKeyRange(pCfg->daysPerFile, pCfg->precision, pTsdbReadHandle->pFileGroup->fid, &win.skey, &win.ekey);
+    tsdbGetFidKeyRange(pCfg->days, pCfg->precision, pTsdbReadHandle->pFileGroup->fid, &win.skey, &win.ekey);
 
     // current file are not overlapped with query time window, ignore remain files
     if ((ascTraverse && win.skey > pTsdbReadHandle->window.ekey) ||
@@ -2396,7 +2396,7 @@ static int32_t getDataBlocksInFiles(STsdbReadHandle* pTsdbReadHandle, bool* exis
   if (!pTsdbReadHandle->locateStart) {
     pTsdbReadHandle->locateStart = true;
     STsdbCfg* pCfg = &pTsdbReadHandle->pTsdb->config;
-    int32_t   fid = getFileIdFromKey(pTsdbReadHandle->window.skey, pCfg->daysPerFile, pCfg->precision);
+    int32_t   fid = getFileIdFromKey(pTsdbReadHandle->window.skey, pCfg->days, pCfg->precision);
 
     tsdbRLockFS(pFileHandle);
     tsdbFSIterInit(&pTsdbReadHandle->fileIter, pFileHandle, pTsdbReadHandle->order);
@@ -3222,7 +3222,7 @@ void tsdbRetrieveDataBlockInfo(tsdbReaderT* pTsdbReadHandle, SDataBlockInfo* pDa
 /*
  * return null for mixed data block, if not a complete file data block, the statistics value will always return NULL
  */
-int32_t tsdbRetrieveDataBlockStatisInfo(tsdbReaderT* pTsdbReadHandle, SDataStatis** pBlockStatis) {
+int32_t tsdbRetrieveDataBlockStatisInfo(tsdbReaderT* pTsdbReadHandle, SColumnDataAgg** pBlockStatis) {
   STsdbReadHandle* pHandle = (STsdbReadHandle*)pTsdbReadHandle;
 
   SQueryFilePos* c = &pHandle->cur;
@@ -3252,7 +3252,7 @@ int32_t tsdbRetrieveDataBlockStatisInfo(tsdbReaderT* pTsdbReadHandle, SDataStati
   int16_t* colIds = pHandle->defaultLoadColumn->pData;
 
   size_t numOfCols = QH_GET_NUM_OF_COLS(pHandle);
-  memset(pHandle->statis, 0, numOfCols * sizeof(SDataStatis));
+  memset(pHandle->statis, 0, numOfCols * sizeof(SColumnDataAgg));
   for (int32_t i = 0; i < numOfCols; ++i) {
     pHandle->statis[i].colId = colIds[i];
   }
@@ -3260,7 +3260,7 @@ int32_t tsdbRetrieveDataBlockStatisInfo(tsdbReaderT* pTsdbReadHandle, SDataStati
   tsdbGetBlockStatis(&pHandle->rhelper, pHandle->statis, (int)numOfCols, pBlockInfo->compBlock);
 
   // always load the first primary timestamp column data
-  SDataStatis* pPrimaryColStatis = &pHandle->statis[0];
+  SColumnDataAgg* pPrimaryColStatis = &pHandle->statis[0];
   assert(pPrimaryColStatis->colId == PRIMARYKEY_TIMESTAMP_COL_ID);
 
   pPrimaryColStatis->numOfNull = 0;

source/dnode/vnode/src/tsdb/tsdbReadImpl.c

@@ -43,14 +43,14 @@ int tsdbInitReadH(SReadH *pReadh, STsdb *pRepo) {
     return -1;
   }
 
-  pReadh->pDCols[0] = tdNewDataCols(0, pCfg->maxRowsPerFileBlock);
+  pReadh->pDCols[0] = tdNewDataCols(0, pCfg->maxRows);
   if (pReadh->pDCols[0] == NULL) {
     terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
     tsdbDestroyReadH(pReadh);
     return -1;
   }
 
-  pReadh->pDCols[1] = tdNewDataCols(0, pCfg->maxRowsPerFileBlock);
+  pReadh->pDCols[1] = tdNewDataCols(0, pCfg->maxRows);
   if (pReadh->pDCols[1] == NULL) {
     terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
     tsdbDestroyReadH(pReadh);
@@ -276,8 +276,8 @@ int tsdbLoadBlockData(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo) {
   return 0;
 }
 
-int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo, const int16_t *colIds,
-                          int numOfColsIds, bool mergeBitmap) {
+int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo, const int16_t *colIds, int numOfColsIds,
+                          bool mergeBitmap) {
   ASSERT(pBlock->numOfSubBlocks > 0);
   int8_t update = pReadh->pRepo->config.update;
 
@@ -433,7 +433,7 @@ void *tsdbDecodeSBlockIdx(void *buf, SBlockIdx *pIdx) {
   return buf;
 }
 
-void tsdbGetBlockStatis(SReadH *pReadh, SDataStatis *pStatis, int numOfCols, SBlock *pBlock) {
+void tsdbGetBlockStatis(SReadH *pReadh, SColumnDataAgg *pStatis, int numOfCols, SBlock *pBlock) {
 #ifdef TD_REFACTOR_3
   SBlockData *pBlockData = pReadh->pBlkData;
 
@@ -513,7 +513,7 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat
 
   tdResetDataCols(pDataCols);
 
-  if(tsdbIsSupBlock(pBlock)) {
+  if (tsdbIsSupBlock(pBlock)) {
     tdDataColsSetBitmapI(pDataCols);
   }
 
@@ -606,9 +606,10 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat
         if (tsdbMakeRoom((void **)(&TSDB_READ_COMP_BUF(pReadh)), zsize) < 0) return -1;
       }
 
-      if (tsdbCheckAndDecodeColumnData(pDataCol, POINTER_SHIFT(pBlockData, tsize + toffset), tlen, pBlockCol->blen,
-                                       pBlock->algorithm, pBlock->numOfRows, tLenBitmap, pDataCols->maxPoints,
-                                       TSDB_READ_COMP_BUF(pReadh), (int)taosTSizeof(TSDB_READ_COMP_BUF(pReadh))) < 0) {
+      if (tsdbCheckAndDecodeColumnData(pDataCol, POINTER_SHIFT(pBlockData, tsize + toffset), tlen,
+                                       pBlockCol ? pBlockCol->blen : 0, pBlock->algorithm, pBlock->numOfRows,
+                                       tLenBitmap, pDataCols->maxPoints, TSDB_READ_COMP_BUF(pReadh),
+                                       (int)taosTSizeof(TSDB_READ_COMP_BUF(pReadh))) < 0) {
         tsdbError("vgId:%d file %s is broken at column %d block offset %" PRId64 " column offset %u",
                   TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), tcolId, (int64_t)pBlock->offset, toffset);
         return -1;
@@ -710,7 +711,7 @@ static int tsdbLoadBlockDataColsImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *
 
   tdResetDataCols(pDataCols);
 
-  if(tsdbIsSupBlock(pBlock)) {
+  if (tsdbIsSupBlock(pBlock)) {
     tdDataColsSetBitmapI(pDataCols);
   }
 
@@ -747,6 +748,7 @@ static int tsdbLoadBlockDataColsImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *
     if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) {  // load the key row
       blockCol.colId = colId;
       TD_SET_COL_ROWS_NORM(&blockCol);  // default is NORM for the primary key column
+      blockCol.blen = 0;
       blockCol.len = pBlock->keyLen;
       blockCol.type = pDataCol->type;
       blockCol.offset = TSDB_KEY_COL_OFFSET;
@@ -836,7 +838,7 @@ static int tsdbLoadColData(SReadH *pReadh, SDFile *pDFile, SBlock *pBlock, SBloc
   }
 
   if (tsdbCheckAndDecodeColumnData(pDataCol, pReadh->pBuf, pBlockCol->len, pBlockCol->blen, pBlock->algorithm,
-                                   pBlock->numOfRows, tLenBitmap, pCfg->maxRowsPerFileBlock, pReadh->pCBuf,
+                                   pBlock->numOfRows, tLenBitmap, pCfg->maxRows, pReadh->pCBuf,
                                    (int32_t)taosTSizeof(pReadh->pCBuf)) < 0) {
     tsdbError("vgId:%d file %s is broken at column %d offset %" PRId64, REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile),
               pBlockCol->colId, offset);

source/dnode/vnode/src/tsdb/tsdbSma.c

@@ -106,7 +106,8 @@ struct SSmaStat {
 
 // expired window
 static int32_t  tsdbUpdateExpiredWindowImpl(STsdb *pTsdb, SSubmitReq *pMsg, int64_t version);
-static int32_t  tsdbSetExpiredWindow(STsdb *pTsdb, SHashObj *pItemsHash, int64_t indexUid, int64_t winSKey, int64_t version);
+static int32_t  tsdbSetExpiredWindow(STsdb *pTsdb, SHashObj *pItemsHash, int64_t indexUid, int64_t winSKey,
+                                     int64_t version);
 static int32_t  tsdbInitSmaStat(SSmaStat **pSmaStat);
 static void    *tsdbFreeSmaStatItem(SSmaStatItem *pSmaStatItem);
 static int32_t  tsdbDestroySmaState(SSmaStat *pSmaStat);
@@ -197,7 +198,7 @@ static SPoolMem *openPool() {
 
 static void clearPool(SPoolMem *pPool) {
   if (!pPool) return;
-  
+
   SPoolMem *pMem;
 
   do {
@@ -313,8 +314,7 @@ static FORCE_INLINE void tsdbSmaStatSetDropped(SSmaStatItem *pStatItem) {
 }
 
 static void tsdbGetSmaDir(int32_t vgId, ETsdbSmaType smaType, char dirName[]) {
-  snprintf(dirName, TSDB_FILENAME_LEN, "vnode%svnode%d%stsdb%s%s", TD_DIRSEP, vgId, TD_DIRSEP, TD_DIRSEP,
-           TSDB_SMA_DNAME[smaType]);
+  snprintf(dirName, TSDB_FILENAME_LEN, "vnode%svnode%d%s%s", TD_DIRSEP, vgId, TD_DIRSEP, TSDB_SMA_DNAME[smaType]);
 }
 
 static SSmaEnv *tsdbNewSmaEnv(const STsdb *pTsdb, const char *path, SDiskID did) {
@@ -544,7 +544,8 @@ static int32_t tsdbCheckAndInitSmaEnv(STsdb *pTsdb, int8_t smaType) {
   return TSDB_CODE_SUCCESS;
 };
 
-static int32_t tsdbSetExpiredWindow(STsdb *pTsdb, SHashObj *pItemsHash, int64_t indexUid, int64_t winSKey, int64_t version) {
+static int32_t tsdbSetExpiredWindow(STsdb *pTsdb, SHashObj *pItemsHash, int64_t indexUid, int64_t winSKey,
+                                    int64_t version) {
   SSmaStatItem *pItem = taosHashGet(pItemsHash, &indexUid, sizeof(indexUid));
   if (pItem == NULL) {
     // TODO: use TSDB_SMA_STAT_EXPIRED and update by stream computing later
@@ -946,7 +947,7 @@ static int32_t tsdbSetTSmaDataFile(STSmaWriteH *pSmaH, int64_t indexUid, int32_t
  */
 static int32_t tsdbGetTSmaDays(STsdb *pTsdb, int64_t interval, int32_t storageLevel) {
   STsdbCfg *pCfg = REPO_CFG(pTsdb);
-  int32_t   daysPerFile = pCfg->daysPerFile;
+  int32_t   daysPerFile = pCfg->days;
 
   if (storageLevel == SMA_STORAGE_LEVEL_TSDB) {
     int32_t days = SMA_STORAGE_TSDB_TIMES * (interval / tsTickPerDay[pCfg->precision]);

source/dnode/vnode/src/vnd/vnodeCfg.c

@@ -23,11 +23,6 @@ int vnodeCheckCfg(const SVnodeCfg *pCfg) {
   return 0;
 }
 
-#if 1  //======================================================================
-void vnodeOptionsCopy(SVnodeCfg *pDest, const SVnodeCfg *pSrc) {
-  memcpy((void *)pDest, (void *)pSrc, sizeof(SVnodeCfg));
-}
-
 int vnodeValidateTableHash(SVnodeCfg *pVnodeOptions, char *tableFName) {
   uint32_t hashValue = 0;
 
@@ -47,5 +42,3 @@ int vnodeValidateTableHash(SVnodeCfg *pVnodeOptions, char *tableFName) {
 
   return TSDB_CODE_SUCCESS;
 }
-
-#endif
\ No newline at end of file

source/dnode/vnode/src/vnd/vnodeCommit.c

@@ -18,8 +18,8 @@
 #define VND_INFO_FNAME     "vnode.json"
 #define VND_INFO_FNAME_TMP "vnode_tmp.json"
 
-static int  vnodeEncodeInfo(const SVnodeInfo *pInfo, uint8_t **ppData, int *len);
-static int  vnodeDecodeInfo(uint8_t *pData, int len, SVnodeInfo *pInfo);
+static int  vnodeEncodeInfo(const SVnodeInfo *pInfo, char **ppData);
+static int  vnodeDecodeInfo(uint8_t *pData, SVnodeInfo *pInfo);
 static int  vnodeStartCommit(SVnode *pVnode);
 static int  vnodeEndCommit(SVnode *pVnode);
 static int  vnodeCommit(void *arg);
@@ -28,16 +28,14 @@ static void vnodeWaitCommit(SVnode *pVnode);
 int vnodeSaveInfo(const char *dir, const SVnodeInfo *pInfo) {
   char      fname[TSDB_FILENAME_LEN];
   TdFilePtr pFile;
-  uint8_t  *data;
-  int       len;
+  char     *data;
 
   snprintf(fname, TSDB_FILENAME_LEN, "%s%s%s", dir, TD_DIRSEP, VND_INFO_FNAME_TMP);
 
   // encode info
   data = NULL;
-  len = 0;
 
-  if (vnodeEncodeInfo(pInfo, &data, &len) < 0) {
+  if (vnodeEncodeInfo(pInfo, &data) < 0) {
     return -1;
   }
 
@@ -48,7 +46,7 @@ int vnodeSaveInfo(const char *dir, const SVnodeInfo *pInfo) {
     return -1;
   }
 
-  if (taosWriteFile(pFile, data, len) < 0) {
+  if (taosWriteFile(pFile, data, strlen(data)) < 0) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     goto _err;
   }
@@ -90,9 +88,55 @@ int vnodeCommitInfo(const char *dir, const SVnodeInfo *pInfo) {
   return 0;
 }
 
-int vnodeLoadInfo(const char *dir) {
-  // TODO
+int vnodeLoadInfo(const char *dir, SVnodeInfo *pInfo) {
+  char      fname[TSDB_FILENAME_LEN];
+  TdFilePtr pFile = NULL;
+  char     *pData = NULL;
+  int64_t   size;
+
+  snprintf(fname, TSDB_FILENAME_LEN, "%s%s%s", dir, TD_DIRSEP, VND_INFO_FNAME);
+
+  // read info
+  pFile = taosOpenFile(fname, TD_FILE_READ);
+  if (pFile == NULL) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  if (taosFStatFile(pFile, &size, NULL) < 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    goto _err;
+  }
+
+  pData = taosMemoryMalloc(size + 1);
+  if (pData == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    goto _err;
+  }
+
+  if (taosReadFile(pFile, pData, size) < 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    goto _err;
+  }
+
+  pData[size] = '\0';
+
+  taosCloseFile(&pFile);
+
+  // decode info
+  if (vnodeDecodeInfo(pData, pInfo) < 0) {
+    taosMemoryFree(pData);
+    return -1;
+  }
+
+  taosMemoryFree(pData);
+
   return 0;
+
+_err:
+  taosCloseFile(&pFile);
+  taosMemoryFree(pData);
+  return -1;
 }
 
 int vnodeAsyncCommit(SVnode *pVnode) {
@@ -137,12 +181,151 @@ static int vnodeEndCommit(SVnode *pVnode) {
 
 static FORCE_INLINE void vnodeWaitCommit(SVnode *pVnode) { tsem_wait(&pVnode->canCommit); }
 
-static int vnodeEncodeInfo(const SVnodeInfo *pInfo, uint8_t **ppData, int *len) {
-  // TODO
+static int vnodeEncodeConfig(const void *pObj, SJson *pJson) {
+  const SVnodeCfg *pCfg = (SVnodeCfg *)pObj;
+
+  if (tjsonAddIntegerToObject(pJson, "vgId", pCfg->vgId) < 0) return -1;
+  if (tjsonAddIntegerToObject(pJson, "dbId", pCfg->dbId) < 0) return -1;
+  if (tjsonAddIntegerToObject(pJson, "wsize", pCfg->wsize) < 0) return -1;
+  if (tjsonAddIntegerToObject(pJson, "ssize", pCfg->ssize) < 0) return -1;
+  if (tjsonAddIntegerToObject(pJson, "lsize", pCfg->lsize) < 0) return -1;
+  if (tjsonAddIntegerToObject(pJson, "isHeap", pCfg->isHeapAllocator) < 0) return -1;
+  if (tjsonAddIntegerToObject(pJson, "ttl", pCfg->ttl) < 0) return -1;
+  if (tjsonAddIntegerToObject(pJson, "keep", pCfg->keep) < 0) return -1;
+  if (tjsonAddIntegerToObject(pJson, "streamMode", pCfg->streamMode) < 0) return -1;
+  if (tjsonAddIntegerToObject(pJson, "isWeak", pCfg->isWeak) < 0) return -1;
+  if (tjsonAddIntegerToObject(pJson, "precision", pCfg->tsdbCfg.precision) < 0) return -1;
+  if (tjsonAddIntegerToObject(pJson, "update", pCfg->tsdbCfg.update) < 0) return -1;
+  if (tjsonAddIntegerToObject(pJson, "compression", pCfg->tsdbCfg.compression) < 0) return -1;
+  if (tjsonAddIntegerToObject(pJson, "daysPerFile", pCfg->tsdbCfg.days) < 0) return -1;
+  if (tjsonAddIntegerToObject(pJson, "minRows", pCfg->tsdbCfg.minRows) < 0) return -1;
+  if (tjsonAddIntegerToObject(pJson, "maxRows", pCfg->tsdbCfg.maxRows) < 0) return -1;
+  if (tjsonAddIntegerToObject(pJson, "keep0", pCfg->tsdbCfg.keep0) < 0) return -1;
+  if (tjsonAddIntegerToObject(pJson, "keep1", pCfg->tsdbCfg.keep1) < 0) return -1;
+  if (tjsonAddIntegerToObject(pJson, "keep2", pCfg->tsdbCfg.keep2) < 0) return -1;
+  if (tjsonAddIntegerToObject(pJson, "lruCacheSize", pCfg->tsdbCfg.lruCacheSize) < 0) return -1;
+  if (tjsonAddIntegerToObject(pJson, "wal.vgId", pCfg->walCfg.vgId) < 0) return -1;
+  if (tjsonAddIntegerToObject(pJson, "wal.fsyncPeriod", pCfg->walCfg.fsyncPeriod) < 0) return -1;
+  if (tjsonAddIntegerToObject(pJson, "wal.retentionPeriod", pCfg->walCfg.retentionPeriod) < 0) return -1;
+  if (tjsonAddIntegerToObject(pJson, "wal.rollPeriod", pCfg->walCfg.rollPeriod) < 0) return -1;
+  if (tjsonAddIntegerToObject(pJson, "wal.retentionSize", pCfg->walCfg.retentionSize) < 0) return -1;
+  if (tjsonAddIntegerToObject(pJson, "wal.segSize", pCfg->walCfg.segSize) < 0) return -1;
+  if (tjsonAddIntegerToObject(pJson, "wal.level", pCfg->walCfg.level) < 0) return -1;
+  if (tjsonAddIntegerToObject(pJson, "hashBegin", pCfg->hashBegin) < 0) return -1;
+  if (tjsonAddIntegerToObject(pJson, "hashEnd", pCfg->hashEnd) < 0) return -1;
+  if (tjsonAddIntegerToObject(pJson, "hashMethod", pCfg->hashMethod) < 0) return -1;
+
   return 0;
 }
 
-static int vnodeDecodeInfo(uint8_t *pData, int len, SVnodeInfo *pInfo) {
-  // TODO
+static int vnodeDecodeConfig(const SJson *pJson, void *pObj) {
+  SVnodeCfg *pCfg = (SVnodeCfg *)pObj;
+
+  if (tjsonGetNumberValue(pJson, "vgId", pCfg->vgId) < 0) return -1;
+  if (tjsonGetNumberValue(pJson, "dbId", pCfg->dbId) < 0) return -1;
+  if (tjsonGetNumberValue(pJson, "wsize", pCfg->wsize) < 0) return -1;
+  if (tjsonGetNumberValue(pJson, "ssize", pCfg->ssize) < 0) return -1;
+  if (tjsonGetNumberValue(pJson, "lsize", pCfg->lsize) < 0) return -1;
+  if (tjsonGetNumberValue(pJson, "isHeap", pCfg->isHeapAllocator) < 0) return -1;
+  if (tjsonGetNumberValue(pJson, "ttl", pCfg->ttl) < 0) return -1;
+  if (tjsonGetNumberValue(pJson, "keep", pCfg->keep) < 0) return -1;
+  if (tjsonGetNumberValue(pJson, "streamMode", pCfg->streamMode) < 0) return -1;
+  if (tjsonGetNumberValue(pJson, "isWeak", pCfg->isWeak) < 0) return -1;
+  if (tjsonGetNumberValue(pJson, "precision", pCfg->tsdbCfg.precision) < 0) return -1;
+  if (tjsonGetNumberValue(pJson, "update", pCfg->tsdbCfg.update) < 0) return -1;
+  if (tjsonGetNumberValue(pJson, "compression", pCfg->tsdbCfg.compression) < 0) return -1;
+  if (tjsonGetNumberValue(pJson, "daysPerFile", pCfg->tsdbCfg.days) < 0) return -1;
+  if (tjsonGetNumberValue(pJson, "minRows", pCfg->tsdbCfg.minRows) < 0) return -1;
+  if (tjsonGetNumberValue(pJson, "maxRows", pCfg->tsdbCfg.maxRows) < 0) return -1;
+  if (tjsonGetNumberValue(pJson, "keep0", pCfg->tsdbCfg.keep0) < 0) return -1;
+  if (tjsonGetNumberValue(pJson, "keep1", pCfg->tsdbCfg.keep1) < 0) return -1;
+  if (tjsonGetNumberValue(pJson, "keep2", pCfg->tsdbCfg.keep2) < 0) return -1;
+  if (tjsonGetNumberValue(pJson, "lruCacheSize", pCfg->tsdbCfg.lruCacheSize) < 0) return -1;
+  if (tjsonGetNumberValue(pJson, "wal.vgId", pCfg->walCfg.vgId) < 0) return -1;
+  if (tjsonGetNumberValue(pJson, "wal.fsyncPeriod", pCfg->walCfg.fsyncPeriod) < 0) return -1;
+  if (tjsonGetNumberValue(pJson, "wal.retentionPeriod", pCfg->walCfg.retentionPeriod) < 0) return -1;
+  if (tjsonGetNumberValue(pJson, "wal.rollPeriod", pCfg->walCfg.rollPeriod) < 0) return -1;
+  if (tjsonGetNumberValue(pJson, "wal.retentionSize", pCfg->walCfg.retentionSize) < 0) return -1;
+  if (tjsonGetNumberValue(pJson, "wal.segSize", pCfg->walCfg.segSize) < 0) return -1;
+  if (tjsonGetNumberValue(pJson, "wal.level", pCfg->walCfg.level) < 0) return -1;
+  if (tjsonGetNumberValue(pJson, "hashBegin", pCfg->hashBegin) < 0) return -1;
+  if (tjsonGetNumberValue(pJson, "hashEnd", pCfg->hashEnd) < 0) return -1;
+  if (tjsonGetNumberValue(pJson, "hashMethod", pCfg->hashMethod) < 0) return -1;
+
+  return 0;
+}
+
+static int vnodeEncodeState(const void *pObj, SJson *pJson) {
+  const SVState *pState = (SVState *)pObj;
+
+  if (tjsonAddIntegerToObject(pJson, "commit version", pState->committed) < 0) return -1;
+
+  return 0;
+}
+
+static int vnodeDecodeState(const SJson *pJson, void *pObj) {
+  SVState *pState = (SVState *)pObj;
+
+  if (tjsonGetNumberValue(pJson, "commit version", pState->committed) < 0) return -1;
+
+  return 0;
+}
+
+static int vnodeEncodeInfo(const SVnodeInfo *pInfo, char **ppData) {
+  SJson *pJson;
+  char  *pData;
+
+  *ppData = NULL;
+
+  pJson = tjsonCreateObject();
+  if (pJson == NULL) {
+    return -1;
+  }
+
+  if (tjsonAddObject(pJson, "config", vnodeEncodeConfig, (void *)&pInfo->config) < 0) {
+    goto _err;
+  }
+
+  if (tjsonAddObject(pJson, "state", vnodeEncodeState, (void *)&pInfo->state) < 0) {
+    goto _err;
+  }
+
+  pData = tjsonToString(pJson);
+  if (pData == NULL) {
+    goto _err;
+  }
+
+  tjsonDelete(pJson);
+
+  *ppData = pData;
   return 0;
+
+_err:
+  tjsonDelete(pJson);
+  return -1;
+}
+
+static int vnodeDecodeInfo(uint8_t *pData, SVnodeInfo *pInfo) {
+  SJson *pJson = NULL;
+
+  pJson = tjsonParse(pData);
+  if (pJson == NULL) {
+    return -1;
+  }
+
+  if (tjsonToObject(pJson, "config", vnodeDecodeConfig, (void *)&pInfo->config) < 0) {
+    goto _err;
+  }
+
+  if (tjsonToObject(pJson, "state", vnodeDecodeState, (void *)&pInfo->state) < 0) {
+    goto _err;
+  }
+
+  tjsonDelete(pJson);
+
+  return 0;
+
+_err:
+  tjsonDelete(pJson);
+  return -1;
 }

source/dnode/vnode/src/vnd/vnodeOpen.c

@@ -15,11 +15,6 @@
 
 #include "vnodeInt.h"
 
-static SVnode *vnodeNew(const char *path, const SVnodeCfg *pVnodeCfg);
-static void    vnodeFree(SVnode *pVnode);
-static int     vnodeOpenImpl(SVnode *pVnode);
-static void    vnodeCloseImpl(SVnode *pVnode);
-
 int vnodeCreate(const char *path, SVnodeCfg *pCfg, STfs *pTfs) {
   SVnodeInfo info = {0};
   char       dir[TSDB_FILENAME_LEN];
@@ -46,144 +41,122 @@ int vnodeCreate(const char *path, SVnodeCfg *pCfg, STfs *pTfs) {
     return -1;
   }
 
+  vInfo("vgId: %d vnode is created", pCfg->vgId);
+
   return 0;
 }
 
-SVnode *vnodeOpen(const char *path, const SVnodeCfg *pVnodeCfg) {
-  SVnode *pVnode = NULL;
-
-  // Set default options
-  SVnodeCfg cfg = vnodeCfgDefault;
-  if (pVnodeCfg != NULL) {
-    cfg.vgId = pVnodeCfg->vgId;
-    cfg.msgCb = pVnodeCfg->msgCb;
-    cfg.pTfs = pVnodeCfg->pTfs;
-    cfg.dbId = pVnodeCfg->dbId;
-    cfg.hashBegin = pVnodeCfg->hashBegin;
-    cfg.hashEnd = pVnodeCfg->hashEnd;
-    cfg.hashMethod = pVnodeCfg->hashMethod;
-  }
+void vnodeDestroy(const char *path, STfs *pTfs) { tfsRmdir(pTfs, path); }
 
-  // Validate options
-  if (vnodeCheckCfg(&cfg) < 0) {
-    // TODO
-    return NULL;
-  }
-
-  // Create the handle
-  pVnode = vnodeNew(path, &cfg);
-  if (pVnode == NULL) {
-    // TODO: handle error
-    return NULL;
-  }
+SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) {
+  SVnode    *pVnode = NULL;
+  SVnodeInfo info = {0};
+  char       dir[TSDB_FILENAME_LEN];
+  char       tdir[TSDB_FILENAME_LEN * 2];
+  int        ret;
 
-  taosMkDir(path);
+  snprintf(dir, TSDB_FILENAME_LEN, "%s%s%s", tfsGetPrimaryPath(pTfs), TD_DIRSEP, path);
 
-  // Open the vnode
-  if (vnodeOpenImpl(pVnode) < 0) {
-    // TODO: handle error
+  // load vnode info
+  ret = vnodeLoadInfo(dir, &info);
+  if (ret < 0) {
+    vError("failed to open vnode from %s since %s", path, tstrerror(terrno));
     return NULL;
   }
 
-  return pVnode;
-}
-
-void vnodeClose(SVnode *pVnode) {
-  if (pVnode) {
-    vnodeCloseImpl(pVnode);
-    vnodeFree(pVnode);
-  }
-}
-
-void vnodeDestroy(const char *path) { taosRemoveDir(path); }
-
-/* ------------------------ STATIC METHODS ------------------------ */
-static SVnode *vnodeNew(const char *path, const SVnodeCfg *pVnodeCfg) {
-  SVnode *pVnode = NULL;
-
-  pVnode = (SVnode *)taosMemoryCalloc(1, sizeof(*pVnode));
+  // create handle
+  pVnode = (SVnode *)taosMemoryCalloc(1, sizeof(*pVnode) + strlen(path) + 1);
   if (pVnode == NULL) {
-    // TODO
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
+    vError("vgId: %d failed to open vnode since %s", info.config.vgId, tstrerror(terrno));
     return NULL;
   }
 
-  pVnode->vgId = pVnodeCfg->vgId;
-  pVnode->msgCb = pVnodeCfg->msgCb;
-  pVnode->pTfs = pVnodeCfg->pTfs;
-  pVnode->path = strdup(path);
-  vnodeOptionsCopy(&(pVnode->config), pVnodeCfg);
+  pVnode->path = (char *)&pVnode[1];
+  strcpy(pVnode->path, path);
+  pVnode->config = info.config;
+  pVnode->state.committed = info.state.committed;
+  pVnode->state.processed = pVnode->state.applied = pVnode->state.committed;
+  pVnode->pTfs = pTfs;
+  pVnode->msgCb = msgCb;
 
   tsem_init(&(pVnode->canCommit), 0, 1);
 
-  return pVnode;
-}
-
-static void vnodeFree(SVnode *pVnode) {
-  if (pVnode) {
-    tsem_destroy(&(pVnode->canCommit));
-    taosMemoryFreeClear(pVnode->path);
-    taosMemoryFree(pVnode);
-  }
-}
-
-static int vnodeOpenImpl(SVnode *pVnode) {
-  char dir[TSDB_FILENAME_LEN];
-
+  // open buffer pool
   if (vnodeOpenBufPool(pVnode) < 0) {
-    // TODO: handle error
-    return -1;
+    vError("vgId: %d failed to open vnode buffer pool since %s", TD_VID(pVnode), tstrerror(terrno));
+    goto _err;
   }
 
-  // Open meta
-  sprintf(dir, "%s/meta", pVnode->path);
-  pVnode->pMeta = metaOpen(dir, &(pVnode->config.metaCfg), vBufPoolGetMAF(pVnode));
-  if (pVnode->pMeta == NULL) {
-    // TODO: handle error
-    return -1;
+  // open meta
+  if (metaOpen(pVnode, &pVnode->pMeta) < 0) {
+    vError("vgId: %d failed to open vnode meta since %s", TD_VID(pVnode), tstrerror(terrno));
+    goto _err;
   }
 
-  // Open tsdb
-  sprintf(dir, "%s/tsdb", pVnode->path);
+  // open tsdb
+  sprintf(tdir, "%s%s%s", dir, TD_DIRSEP, VNODE_TSDB_DIR);
   pVnode->pTsdb =
-      tsdbOpen(dir, pVnode->vgId, &(pVnode->config.tsdbCfg), vBufPoolGetMAF(pVnode), pVnode->pMeta, pVnode->pTfs);
+      tsdbOpen(tdir, TD_VID(pVnode), &(pVnode->config.tsdbCfg), vBufPoolGetMAF(pVnode), pVnode->pMeta, pVnode->pTfs);
   if (pVnode->pTsdb == NULL) {
-    // TODO: handle error
-    return -1;
+    vError("vgId: %d failed to open vnode tsdb since %s", TD_VID(pVnode), tstrerror(terrno));
+    goto _err;
   }
 
-  // Open WAL
-  sprintf(dir, "%s/wal", pVnode->path);
-  pVnode->pWal = walOpen(dir, &(pVnode->config.walCfg));
+  // open wal
+  sprintf(tdir, "%s%s%s", dir, TD_DIRSEP, VNODE_WAL_DIR);
+  pVnode->pWal = walOpen(tdir, &(pVnode->config.walCfg));
   if (pVnode->pWal == NULL) {
-    // TODO: handle error
-    return -1;
+    vError("vgId: %d failed to open vnode wal since %s", TD_VID(pVnode), tstrerror(terrno));
+    goto _err;
   }
 
-  // Open TQ
-  sprintf(dir, "%s/tq", pVnode->path);
-  pVnode->pTq = tqOpen(dir, pVnode, pVnode->pWal, pVnode->pMeta, &(pVnode->config.tqCfg), vBufPoolGetMAF(pVnode));
+  // open tq
+  sprintf(tdir, "%s%s%s", dir, TD_DIRSEP, VNODE_TQ_DIR);
+  pVnode->pTq = tqOpen(tdir, pVnode, pVnode->pWal, pVnode->pMeta, vBufPoolGetMAF(pVnode));
   if (pVnode->pTq == NULL) {
-    // TODO: handle error
-    return -1;
+    vError("vgId: %d failed to open vnode tq since %s", TD_VID(pVnode), tstrerror(terrno));
+    goto _err;
   }
 
-  // Open Query
+  // open query
   if (vnodeQueryOpen(pVnode)) {
-    return -1;
+    vError("vgId: %d failed to open vnode query since %s", TD_VID(pVnode), tstrerror(terrno));
+    goto _err;
   }
 
-  // TODO
-  return 0;
+#if 0
+  if (vnodeBegin() < 0) {
+    goto _err;
+  }
+#endif
+
+  return pVnode;
+
+_err:
+  if (pVnode->pQuery) vnodeQueryClose(pVnode);
+  if (pVnode->pTq) tqClose(pVnode->pTq);
+  if (pVnode->pWal) walClose(pVnode->pWal);
+  if (pVnode->pTsdb) tsdbClose(pVnode->pTsdb);
+  if (pVnode->pMeta) metaClose(pVnode->pMeta);
+  tsem_destroy(&(pVnode->canCommit));
+  taosMemoryFree(pVnode);
+  return NULL;
 }
 
-static void vnodeCloseImpl(SVnode *pVnode) {
-  vnodeSyncCommit(pVnode);
+void vnodeClose(SVnode *pVnode) {
   if (pVnode) {
-    vnodeCloseBufPool(pVnode);
-    metaClose(pVnode->pMeta);
-    tsdbClose(pVnode->pTsdb);
-    tqClose(pVnode->pTq);
-    walClose(pVnode->pWal);
+    // commit (TODO: use option to control)
+    vnodeSyncCommit(pVnode);
+    // close vnode
     vnodeQueryClose(pVnode);
+    walClose(pVnode->pWal);
+    tqClose(pVnode->pTq);
+    tsdbClose(pVnode->pTsdb);
+    metaClose(pVnode->pMeta);
+    vnodeCloseBufPool(pVnode);
+    // destroy handle
+    tsem_destroy(&(pVnode->canCommit));
+    taosMemoryFree(pVnode);
   }
 }

source/dnode/vnode/src/vnd/vnodeQuery.c

@@ -16,7 +16,7 @@
 #include "vnodeInt.h"
 
 int vnodeQueryOpen(SVnode *pVnode) {
-  return qWorkerInit(NODE_TYPE_VNODE, pVnode->vgId, NULL, (void **)&pVnode->pQuery, &pVnode->msgCb);
+  return qWorkerInit(NODE_TYPE_VNODE, TD_VID(pVnode), NULL, (void **)&pVnode->pQuery, &pVnode->msgCb);
 }
 
 void vnodeQueryClose(SVnode *pVnode) { qWorkerDestroy((void **)&pVnode->pQuery); }
@@ -101,7 +101,7 @@ int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg) {
   metaRsp.numOfColumns = nCols;
   metaRsp.tableType = pTbCfg->type;
   metaRsp.tuid = uid;
-  metaRsp.vgId = pVnode->vgId;
+  metaRsp.vgId = TD_VID(pVnode);
 
   memcpy(metaRsp.pSchemas, pSW->pSchema, sizeof(SSchema) * pSW->nCols);
   if (nTagCols) {
@@ -151,7 +151,7 @@ _exit:
 }
 
 int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad) {
-  pLoad->vgId = pVnode->vgId;
+  pLoad->vgId = TD_VID(pVnode);
   pLoad->role = TAOS_SYNC_STATE_LEADER;
   pLoad->numOfTables = metaGetTbNum(pVnode->pMeta);
   pLoad->numOfTimeSeries = 400;

source/dnode/vnode/src/vnd/vnodeSvr.c

@@ -18,6 +18,7 @@
 static int vnodeProcessCreateStbReq(SVnode *pVnode, void *pReq);
 static int vnodeProcessCreateTbReq(SVnode *pVnode, SRpcMsg *pMsg, void *pReq, SRpcMsg **pRsp);
 static int vnodeProcessAlterStbReq(SVnode *pVnode, void *pReq);
+static int vnodeProcessSubmitReq(SVnode *pVnode, SSubmitReq *pSubmitReq, SRpcMsg *pRsp);
 
 void vnodePreprocessWriteReqs(SVnode *pVnode, SArray *pMsgs) {
   SNodeMsg *pMsg;
@@ -35,7 +36,7 @@ void vnodePreprocessWriteReqs(SVnode *pVnode, SArray *pMsgs) {
     if (walWrite(pVnode->pWal, ver, pRpc->msgType, pRpc->pCont, pRpc->contLen) < 0) {
       // TODO: handle error
       /*ASSERT(false);*/
-      vError("vnode:%d  write wal error since %s", pVnode->vgId, terrstr());
+      vError("vnode:%d  write wal error since %s", TD_VID(pVnode), terrstr());
     }
   }
 
@@ -72,16 +73,17 @@ int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
     case TDMT_VND_ALTER_STB:
       return vnodeProcessAlterStbReq(pVnode, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)));
     case TDMT_VND_DROP_STB:
-      vTrace("vgId:%d, process drop stb req", pVnode->vgId);
+      vTrace("vgId:%d, process drop stb req", TD_VID(pVnode));
       break;
     case TDMT_VND_DROP_TABLE:
       break;
     case TDMT_VND_SUBMIT:
-      /*printf("vnode %d write data %ld\n", pVnode->vgId, ver);*/
+      /*printf("vnode %d write data %ld\n", TD_VID(pVnode), ver);*/
       if (pVnode->config.streamMode == 0) {
-        if (tsdbInsertData(pVnode->pTsdb, (SSubmitReq *)ptr, NULL) < 0) {
-          // TODO: handle error
-        }
+        *pRsp = taosMemoryCalloc(1, sizeof(SRpcMsg));
+        (*pRsp)->handle = pMsg->handle;
+        (*pRsp)->ahandle = pMsg->ahandle;
+        return vnodeProcessSubmitReq(pVnode, ptr, *pRsp);
       }
       break;
     case TDMT_VND_MQ_SET_CONN: {
@@ -243,7 +245,7 @@ static int vnodeProcessCreateTbReq(SVnode *pVnode, SRpcMsg *pMsg, void *pReq, SR
 
     if (metaCreateTable(pVnode->pMeta, pCreateTbReq) < 0) {
       // TODO: handle error
-      vError("vgId:%d, failed to create table: %s", pVnode->vgId, pCreateTbReq->name);
+      vError("vgId:%d, failed to create table: %s", TD_VID(pVnode), pCreateTbReq->name);
     }
     // TODO: to encapsule a free API
     taosMemoryFree(pCreateTbReq->name);
@@ -266,7 +268,7 @@ static int vnodeProcessCreateTbReq(SVnode *pVnode, SRpcMsg *pMsg, void *pReq, SR
     }
   }
 
-  vTrace("vgId:%d process create %" PRIzu " tables", pVnode->vgId, taosArrayGetSize(vCreateTbBatchReq.pArray));
+  vTrace("vgId:%d process create %" PRIzu " tables", TD_VID(pVnode), taosArrayGetSize(vCreateTbBatchReq.pArray));
   taosArrayDestroy(vCreateTbBatchReq.pArray);
   if (vCreateTbBatchRsp.rspList) {
     int32_t contLen = tSerializeSVCreateTbBatchRsp(NULL, 0, &vCreateTbBatchRsp);
@@ -287,7 +289,7 @@ static int vnodeProcessCreateTbReq(SVnode *pVnode, SRpcMsg *pMsg, void *pReq, SR
 
 static int vnodeProcessAlterStbReq(SVnode *pVnode, void *pReq) {
   SVCreateTbReq vAlterTbReq = {0};
-  vTrace("vgId:%d, process alter stb req", pVnode->vgId);
+  vTrace("vgId:%d, process alter stb req", TD_VID(pVnode));
   tDeserializeSVCreateTbReq(pReq, &vAlterTbReq);
   // TODO: to encapsule a free API
   taosMemoryFree(vAlterTbReq.stbCfg.pSchema);
@@ -298,5 +300,25 @@ static int vnodeProcessAlterStbReq(SVnode *pVnode, void *pReq) {
   }
   taosMemoryFree(vAlterTbReq.dbFName);
   taosMemoryFree(vAlterTbReq.name);
+  return 0;
+}
+
+static int vnodeProcessSubmitReq(SVnode *pVnode, SSubmitReq *pSubmitReq, SRpcMsg *pRsp) {
+  SSubmitRsp rsp = {0};
+
+  pRsp->code = 0;
+
+  // handle the request
+  if (tsdbInsertData(pVnode->pTsdb, pSubmitReq, &rsp) < 0) {
+    pRsp->code = terrno;
+    return -1;
+  }
+
+  // encode the response (TODO)
+  pRsp->msgType = TDMT_VND_SUBMIT_RSP;
+  pRsp->pCont = rpcMallocCont(sizeof(SSubmitRsp));
+  memcpy(pRsp->pCont, &rsp, sizeof(rsp));
+  pRsp->contLen = sizeof(SSubmitRsp);
+
   return 0;
 }
\ No newline at end of file

source/libs/catalog/test/CMakeLists.txt

@@ -8,7 +8,7 @@ AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
 ADD_EXECUTABLE(catalogTest ${SOURCE_LIST})
 TARGET_LINK_LIBRARIES(
         catalogTest
-        PUBLIC os util common catalog transport gtest qcom taos
+        PUBLIC os util common catalog transport gtest qcom taos_static
 )
 
 TARGET_INCLUDE_DIRECTORIES(

source/libs/command/src/explain.c

@@ -316,6 +316,9 @@ int32_t qExplainResAppendRow(SExplainCtx *ctx, char *tbuf, int32_t len, int32_t
   return TSDB_CODE_SUCCESS;
 }
 
+static uint8_t getIntervalPrecision(SIntervalPhysiNode* pIntNode) {
+  return ((SColumnNode*)pIntNode->window.pTspk)->node.resType.precision;
+}
 
 int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, int32_t level) {
   int32_t tlen = 0;
@@ -658,10 +661,10 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
         EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pIntNode->window.node.pOutputDataBlockDesc->outputRowSize);
         EXPLAIN_ROW_END();
         QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
-
-        EXPLAIN_ROW_NEW(level + 1, EXPLAIN_TIME_WINDOWS_FORMAT, INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->interval, pIntNode->intervalUnit, pIntNode->precision),  
-              pIntNode->intervalUnit, pIntNode->offset, getPrecisionUnit(pIntNode->precision), 
-              INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->sliding, pIntNode->slidingUnit, pIntNode->precision), pIntNode->slidingUnit);
+        uint8_t precision = getIntervalPrecision(pIntNode);
+        EXPLAIN_ROW_NEW(level + 1, EXPLAIN_TIME_WINDOWS_FORMAT, INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->interval, pIntNode->intervalUnit, precision),  
+              pIntNode->intervalUnit, pIntNode->offset, getPrecisionUnit(precision), 
+              INVERAL_TIME_FROM_PRECISION_TO_UNIT(pIntNode->sliding, pIntNode->slidingUnit, precision), pIntNode->slidingUnit);
         EXPLAIN_ROW_END();
         QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
         

source/libs/executor/inc/executil.h

@@ -42,10 +42,6 @@
 
 #define curTimeWindowIndex(_winres)        ((_winres)->curIndex)
 
-struct SColumnFilterElem;
-
-typedef bool (*__filter_func_t)(struct SColumnFilterElem* pFilter, const char* val1, const char* val2, int16_t type);
-
 typedef struct SGroupResInfo {
   int32_t totalGroup;
   int32_t currentGroup;
@@ -76,7 +72,8 @@ typedef struct SResultRowInfo {
   SResultRowPosition *pPosition;
   int32_t      size;       // number of result set
   int32_t      capacity;   // max capacity
-  int32_t      curPos;     // current active result row index of pResult list
+//  int32_t      curPos;     // current active result row index of pResult list
+  SResultRowPosition cur;
 } SResultRowInfo;
 
 struct STaskAttr;

source/dnode/vnode/src/meta/metaTbUid.c → source/libs/executor/inc/indexoperator.h

@@ -13,18 +13,11 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "vnodeInt.h"
+#include "filter.h"
+#include "tglobal.h"
 
-int metaOpenUidGnrt(SMeta *pMeta) {
-  // Init a generator
-  pMeta->uidGnrt.nextUid = IVLD_TB_UID;
-  return 0;
-}
+typedef enum { SFLT_NOT_INDEX, SFLT_COARSE_INDEX, SFLT_ACCURATE_INDEX } SIdxFltStatus;
 
-void metaCloseUidGnrt(SMeta *pMeta) { /* TODO */
-}
-
-tb_uid_t metaGenerateUid(SMeta *pMeta) {
-  // Generate a new table UID
-  return tGenIdPI64();
-}
+SIdxFltStatus idxGetFltStatus(SNode *pFilterNode);
+// construct tag filter operator later
+int32_t doFilterTag(const SNode *pFilterNode, SArray *resutl);

source/libs/executor/src/executil.c

@@ -52,11 +52,11 @@ int32_t getOutputInterResultBufSize(STaskAttr* pQueryAttr) {
 }
 
 int32_t initResultRowInfo(SResultRowInfo *pResultRowInfo, int32_t size) {
-  pResultRowInfo->size     = 0;
-  pResultRowInfo->curPos   = -1;
-  pResultRowInfo->capacity = size;
+  pResultRowInfo->size       = 0;
+  pResultRowInfo->capacity   = size;
+  pResultRowInfo->cur.pageId = -1;
+  
   pResultRowInfo->pPosition = taosMemoryCalloc(pResultRowInfo->capacity, sizeof(SResultRowPosition));
-
   if (pResultRowInfo->pPosition == NULL) {
     return TSDB_CODE_QRY_OUT_OF_MEMORY;
   }
@@ -114,7 +114,6 @@ void closeAllResultRows(SResultRowInfo *pResultRowInfo) {
   assert(pResultRowInfo->size >= 0 && pResultRowInfo->capacity >= pResultRowInfo->size);
   
   for (int32_t i = 0; i < pResultRowInfo->size; ++i) {
-//    ASSERT(0);
 //    SResultRow* pRow = pResultRowInfo->pResult[i];
 //    if (pRow->closed) {
 //      continue;
@@ -378,7 +377,7 @@ static int32_t mergeIntoGroupResultImplRv(STaskRuntimeEnv *pRuntimeEnv, SGroupRe
 static UNUSED_FUNC int32_t mergeIntoGroupResultImpl(STaskRuntimeEnv *pRuntimeEnv, SGroupResInfo* pGroupResInfo, SArray *pTableList,
     int32_t* rowCellInfoOffset) {
   bool ascQuery = true;
-
+#if 0
   int32_t code = TSDB_CODE_SUCCESS;
 
   int32_t *posList = NULL;
@@ -402,16 +401,16 @@ static UNUSED_FUNC int32_t mergeIntoGroupResultImpl(STaskRuntimeEnv *pRuntimeEnv
   int32_t numOfTables = 0;
   for (int32_t i = 0; i < size; ++i) {
     STableQueryInfo *item = taosArrayGetP(pTableList, i);
-    if (item->resInfo.size > 0) {
-      pTableQueryInfoList[numOfTables++] = item;
-    }
+//    if (item->resInfo.size > 0) {
+//      pTableQueryInfoList[numOfTables++] = item;
+//    }
   }
 
   // there is no data in current group
   // no need to merge results since only one table in each group
-  if (numOfTables == 0) {
-    goto _end;
-  }
+//  if (numOfTables == 0) {
+//    goto _end;
+//  }
 
   int32_t order = TSDB_ORDER_ASC;
   SCompSupporter cs = {pTableQueryInfoList, posList, order};
@@ -498,6 +497,7 @@ int32_t mergeIntoGroupResult(SGroupResInfo* pGroupResInfo, STaskRuntimeEnv* pRun
 //  int64_t elapsedTime = taosGetTimestampUs() - st;
 //  qDebug("QInfo:%"PRIu64" merge res data into group, index:%d, total group:%d, elapsed time:%" PRId64 "us", GET_TASKID(pRuntimeEnv),
 //         pGroupResInfo->currentGroup, pGroupResInfo->totalGroup, elapsedTime);
+#endif
 
   return TSDB_CODE_SUCCESS;
 }

source/libs/executor/src/groupoperator.c

@@ -286,7 +286,7 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator, bool* newgrou
     // the pDataBlock are always the same one, no need to call this again
     setInputDataBlock(pOperator, pInfo->binfo.pCtx, pBlock, order);
 
-    // there is an scalar expression that needs to be calculated before apply the group aggregation.
+    // there is an scalar expression that needs to be calculated right before apply the group aggregation.
     if (pInfo->pScalarExprInfo != NULL) {
       projectApplyFunctions(pInfo->pScalarExprInfo, pBlock, pBlock, pInfo->pScalarFuncCtx, pInfo->numOfScalarExpr, NULL);
     }
@@ -343,7 +343,6 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pEx
   pInfo->numOfScalarExpr = numOfScalarExpr;
   pInfo->pScalarFuncCtx = createSqlFunctionCtx(pExprInfo, numOfCols, &pInfo->binfo.rowCellInfoOffset);
 
-
   int32_t code = initGroupOptrInfo(&pInfo->pGroupColVals, &pInfo->groupKeyLen, &pInfo->keyBuf, pGroupColList);
   if (code != TSDB_CODE_SUCCESS) {
     goto _error;

source/libs/executor/src/scanoperator.c

@@ -13,11 +13,13 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "tglobal.h"
+#include <common/ttime.h>
 #include "filter.h"
 #include "function.h"
+#include "functionMgt.h"
 #include "os.h"
 #include "querynodes.h"
+#include "tglobal.h"
 #include "tname.h"
 #include "vnode.h"
 
@@ -64,35 +66,193 @@ static void setupQueryRangeForReverseScan(STableScanInfo* pTableScanInfo) {
 #endif
 }
 
-int32_t loadDataBlock(SExecTaskInfo* pTaskInfo, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, uint32_t* status) {
+// relocated the column data according to the slotId
+static void relocateColumnData(SSDataBlock* pBlock, const SArray* pColMatchInfo, SArray* pCols) {
+  int32_t numOfCols = pBlock->info.numOfCols;
+  for (int32_t i = 0; i < numOfCols; ++i) {
+    SColumnInfoData* p = taosArrayGet(pCols, i);
+    SColMatchInfo*   pmInfo = taosArrayGet(pColMatchInfo, i);
+    if (!pmInfo->output) {
+      continue;
+    }
+
+    ASSERT(pmInfo->colId == p->info.colId);
+    taosArraySet(pBlock->pDataBlock, pmInfo->targetSlotId, p);
+  }
+}
+
+static void getNextTimeWindow(SInterval* pInterval, STimeWindow* tw, int32_t order) {
+  int32_t factor = GET_FORWARD_DIRECTION_FACTOR(order);
+  if (pInterval->intervalUnit != 'n' && pInterval->intervalUnit != 'y') {
+    tw->skey += pInterval->sliding * factor;
+    tw->ekey = tw->skey + pInterval->interval - 1;
+    return;
+  }
+
+  int64_t key = tw->skey, interval = pInterval->interval;
+  //convert key to second
+  key = convertTimePrecision(key, pInterval->precision, TSDB_TIME_PRECISION_MILLI) / 1000;
+
+  if (pInterval->intervalUnit == 'y') {
+    interval *= 12;
+  }
+
+  struct tm tm;
+  time_t t = (time_t)key;
+  taosLocalTime(&t, &tm);
+
+  int mon = (int)(tm.tm_year * 12 + tm.tm_mon + interval * factor);
+  tm.tm_year = mon / 12;
+  tm.tm_mon = mon % 12;
+  tw->skey = convertTimePrecision((int64_t)taosMktime(&tm) * 1000L, TSDB_TIME_PRECISION_MILLI, pInterval->precision);
+
+  mon = (int)(mon + interval);
+  tm.tm_year = mon / 12;
+  tm.tm_mon = mon % 12;
+  tw->ekey = convertTimePrecision((int64_t)taosMktime(&tm) * 1000L, TSDB_TIME_PRECISION_MILLI, pInterval->precision);
+
+  tw->ekey -= 1;
+}
+
+static bool overlapWithTimeWindow(SInterval* pInterval, SDataBlockInfo* pBlockInfo) {
+  STimeWindow w = {0};
+
+  // 0 by default, which means it is not a interval operator of the upstream operator.
+  if (pInterval->interval == 0) {
+    return false;
+  }
+
+  // todo handle the time range case
+  TSKEY sk = INT64_MIN;
+  TSKEY ek = INT64_MAX;
+//  TSKEY sk = MIN(pQueryAttr->window.skey, pQueryAttr->window.ekey);
+//  TSKEY ek = MAX(pQueryAttr->window.skey, pQueryAttr->window.ekey);
+
+  if (true) {
+    getAlignQueryTimeWindow(pInterval, pInterval->precision, pBlockInfo->window.skey, sk, ek, &w);
+    assert(w.ekey >= pBlockInfo->window.skey);
+
+    if (w.ekey < pBlockInfo->window.ekey) {
+      return true;
+    }
+
+    while(1) { // todo handle the desc order scan case
+      getNextTimeWindow(pInterval, &w, TSDB_ORDER_ASC);
+      if (w.skey > pBlockInfo->window.ekey) {
+        break;
+      }
+
+      assert(w.ekey > pBlockInfo->window.ekey);
+      if (w.skey <= pBlockInfo->window.ekey && w.skey > pBlockInfo->window.skey) {
+        return true;
+      }
+    }
+  } else {
+//    getAlignQueryTimeWindow(pQueryAttr, pBlockInfo->window.ekey, sk, ek, &w);
+//    assert(w.skey <= pBlockInfo->window.ekey);
+//
+//    if (w.skey > pBlockInfo->window.skey) {
+//      return true;
+//    }
+//
+//    while(1) {
+//      getNextTimeWindow(pQueryAttr, &w);
+//      if (w.ekey < pBlockInfo->window.skey) {
+//        break;
+//      }
+//
+//      assert(w.skey < pBlockInfo->window.skey);
+//      if (w.ekey < pBlockInfo->window.ekey && w.ekey >= pBlockInfo->window.skey) {
+//        return true;
+//      }
+//    }
+  }
+
+  return false;
+}
+
+int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, uint32_t* status) {
+  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
+  STableScanInfo* pInfo = pOperator->info;
+
   STaskCostInfo* pCost = &pTaskInfo->cost;
 
   pCost->totalBlocks += 1;
   pCost->totalRows += pBlock->info.rows;
 
+  *status = pInfo->dataBlockLoadFlag;
+  if (pTableScanInfo->pFilterNode != NULL || overlapWithTimeWindow(&pTableScanInfo->interval, &pBlock->info)) {
+    (*status) = FUNC_DATA_REQUIRED_DATA_LOAD;
+  }
+
+  SDataBlockInfo* pBlockInfo = &pBlock->info;
+  taosMemoryFreeClear(pBlock->pBlockAgg);
+
+  if (*status == FUNC_DATA_REQUIRED_FILTEROUT) {
+    qDebug("%s data block filter out, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), pBlockInfo->window.skey,
+           pBlockInfo->window.ekey, pBlockInfo->rows);
+    pCost->filterOutBlocks += 1;
+    return TSDB_CODE_SUCCESS;
+  } else if (*status == FUNC_DATA_REQUIRED_NOT_LOAD) {
+    qDebug("%s data block skipped, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), pBlockInfo->window.skey,
+           pBlockInfo->window.ekey, pBlockInfo->rows);
+    pCost->skipBlocks += 1;
+    return TSDB_CODE_SUCCESS;
+  } else if (*status == FUNC_DATA_REQUIRED_STATIS_LOAD) {
+    pCost->loadBlockStatis += 1;
+
+    SColumnDataAgg* pColAgg = NULL;
+    tsdbRetrieveDataBlockStatisInfo(pTableScanInfo->dataReader, &pColAgg);
+
+    if (pColAgg != NULL) {
+      int32_t numOfCols = pBlock->info.numOfCols;
+
+      // todo create this buffer during creating operator
+      pBlock->pBlockAgg = taosMemoryCalloc(numOfCols, sizeof(SColumnDataAgg));
+      for (int32_t i = 0; i < numOfCols; ++i) {
+        SColMatchInfo* pColMatchInfo = taosArrayGet(pTableScanInfo->pColMatchInfo, i);
+        if (!pColMatchInfo->output) {
+          continue;
+        }
+        pBlock->pBlockAgg[pColMatchInfo->targetSlotId] = pColAgg[i];
+      }
+
+      return TSDB_CODE_SUCCESS;
+    } else { // failed to load the block sma data, data block statistics does not exist, load data block instead
+      *status = FUNC_DATA_REQUIRED_DATA_LOAD;
+    }
+  }
+
+  ASSERT (*status == FUNC_DATA_REQUIRED_DATA_LOAD);
+
+  // todo filter data block according to the block sma data firstly
+#if 0
+  if (!doFilterByBlockStatistics(pBlock->pBlockStatis, pTableScanInfo->pCtx, pBlockInfo->rows)) {
+    pCost->filterOutBlocks += 1;
+    qDebug("%s data block filter out, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), pBlockInfo->window.skey,
+           pBlockInfo->window.ekey, pBlockInfo->rows);
+    (*status) = FUNC_DATA_REQUIRED_FILTEROUT;
+    return TSDB_CODE_SUCCESS;
+  }
+#endif
+
   pCost->totalCheckedRows += pBlock->info.rows;
   pCost->loadBlocks += 1;
 
-  *status = BLK_DATA_DATA_LOAD;
-
   SArray* pCols = tsdbRetrieveDataBlock(pTableScanInfo->dataReader, NULL);
   if (pCols == NULL) {
     return terrno;
   }
 
-  int32_t numOfCols = pBlock->info.numOfCols;
-  for (int32_t i = 0; i < numOfCols; ++i) {
-    SColumnInfoData* p = taosArrayGet(pCols, i);
-    SColMatchInfo*   pColMatchInfo = taosArrayGet(pTableScanInfo->pColMatchInfo, i);
-    if (!pColMatchInfo->output) {
-      continue;
-    }
+  relocateColumnData(pBlock, pTableScanInfo->pColMatchInfo, pCols);
 
-    ASSERT(pColMatchInfo->colId == p->info.colId);
-    taosArraySet(pBlock->pDataBlock, pColMatchInfo->targetSlotId, p);
+  doFilter(pTableScanInfo->pFilterNode, pBlock);
+  if (pBlock->info.rows == 0) {
+    pCost->filterOutBlocks += 1;
+    qDebug("%s data block filter out, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), pBlockInfo->window.skey,
+           pBlockInfo->window.ekey, pBlockInfo->rows);
   }
 
-  doFilter(pTableScanInfo->pFilterNode, pBlock);
   return TSDB_CODE_SUCCESS;
 }
 
@@ -111,7 +271,6 @@ static void setupEnvForReverseScan(STableScanInfo* pTableScanInfo, SqlFunctionCt
 
 static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator, bool* newgroup) {
   STableScanInfo* pTableScanInfo = pOperator->info;
-  SExecTaskInfo*  pTaskInfo = pOperator->pTaskInfo;
 
   SSDataBlock*     pBlock = pTableScanInfo->pResBlock;
   STableGroupInfo* pTableGroupInfo = &pOperator->pTaskInfo->tableqinfoGroupInfo;
@@ -138,15 +297,15 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator, bool* newgroup) {
     //    }
 
     // this function never returns error?
-    uint32_t status = BLK_DATA_DATA_LOAD;
-    int32_t  code = loadDataBlock(pTaskInfo, pTableScanInfo, pBlock, &status);
+    uint32_t status = 0;
+    int32_t  code = loadDataBlock(pOperator, pTableScanInfo, pBlock, &status);
     //    int32_t  code = loadDataBlockOnDemand(pOperator->pRuntimeEnv, pTableScanInfo, pBlock, &status);
     if (code != TSDB_CODE_SUCCESS) {
       longjmp(pOperator->pTaskInfo->env, code);
     }
 
-    // current block is ignored according to filter result by block statistics data, continue load the next block
-    if (status == BLK_DATA_FILTEROUT || pBlock->info.rows == 0) {
+    // current block is filter out according to filter condition, continue load the next block
+    if (status == FUNC_DATA_REQUIRED_FILTEROUT || pBlock->info.rows == 0) {
       continue;
     }
 
@@ -189,9 +348,9 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator, bool* newgroup) {
     setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED);
     pTableScanInfo->scanFlag = REPEAT_SCAN;
 
-    if (pResultRowInfo->size > 0) {
-      pResultRowInfo->curPos = 0;
-    }
+//    if (pResultRowInfo->size > 0) {
+//      pResultRowInfo->curPos = 0;
+//    }
 
     qDebug("%s start to repeat scan data blocks due to query func required, qrange:%" PRId64 "-%" PRId64,
            GET_TASKID(pTaskInfo), pTaskInfo->window.skey, pTaskInfo->window.ekey);
@@ -208,7 +367,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator, bool* newgroup) {
            GET_TASKID(pTaskInfo), pTaskInfo->window.skey, pTaskInfo->window.ekey);
 
     if (pResultRowInfo->size > 0) {
-      pResultRowInfo->curPos = pResultRowInfo->size - 1;
+//      pResultRowInfo->curPos = pResultRowInfo->size - 1;
     }
 
     p = doTableScanImpl(pOperator, newgroup);
@@ -217,9 +376,9 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator, bool* newgroup) {
   return p;
 }
 
-SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order, int32_t numOfOutput,
+SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order, int32_t numOfOutput, int32_t dataLoadFlag,
                                            int32_t repeatTime, int32_t reverseTime, SArray* pColMatchInfo, SSDataBlock* pResBlock,
-                                           SNode* pCondition, SExecTaskInfo* pTaskInfo) {
+                                           SNode* pCondition, SInterval* pInterval, double sampleRatio, SExecTaskInfo* pTaskInfo) {
   assert(repeatTime > 0);
 
   STableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableScanInfo));
@@ -232,6 +391,9 @@ SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order,
     return NULL;
   }
 
+  pInfo->interval         = *pInterval;
+  pInfo->sampleRatio      = sampleRatio;
+  pInfo->dataBlockLoadFlag= dataLoadFlag;
   pInfo->pResBlock        = pResBlock;
   pInfo->pFilterNode      = pCondition;
   pInfo->dataReader       = pTsdbReadHandle;

source/libs/executor/test/CMakeLists.txt

@@ -8,7 +8,7 @@ AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
 ADD_EXECUTABLE(executorTest ${SOURCE_LIST})
 TARGET_LINK_LIBRARIES(
         executorTest
-        PRIVATE os util common transport gtest taos qcom executor function planner scalar nodes
+        PRIVATE os util common transport gtest taos_static qcom executor function planner scalar nodes
 )
 
 TARGET_INCLUDE_DIRECTORIES(

source/libs/function/inc/builtins.h

@@ -37,6 +37,7 @@ extern "C" {
 #define FUNC_MGT_WINDOW_PC_FUNC         FUNC_MGT_FUNC_CLASSIFICATION_MASK(8)
 #define FUNC_MGT_SPECIAL_DATA_REQUIRED  FUNC_MGT_FUNC_CLASSIFICATION_MASK(9)
 #define FUNC_MGT_DYNAMIC_SCAN_OPTIMIZED FUNC_MGT_FUNC_CLASSIFICATION_MASK(10)
+#define FUNC_MGT_MULTI_RES_FUNC         FUNC_MGT_FUNC_CLASSIFICATION_MASK(11)
 
 #define FUNC_MGT_TEST_MASK(val, mask) (((val) & (mask)) != 0)
 

source/libs/function/inc/builtinsimpl.h

@@ -30,6 +30,7 @@ EFuncDataRequired countDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWind
 bool getCountFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
 int32_t countFunction(SqlFunctionCtx *pCtx);
 
+EFuncDataRequired statisDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWindow);
 bool getSumFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
 int32_t sumFunction(SqlFunctionCtx *pCtx);
 

source/libs/function/src/builtinsimpl.c

@@ -58,9 +58,9 @@ void functionFinalize(SqlFunctionCtx *pCtx) {
 EFuncDataRequired countDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWindow) {
   SNode* pParam = nodesListGetNode(pFunc->pParameterList, 0);
   if (QUERY_NODE_COLUMN == nodeType(pParam) && PRIMARYKEY_TIMESTAMP_COL_ID == ((SColumnNode*)pParam)->colId) {
-    return FUNC_DATA_REQUIRED_NO_NEEDED;
+    return FUNC_DATA_REQUIRED_NOT_LOAD;
   }
-  return FUNC_DATA_REQUIRED_STATIS_NEEDED;
+  return FUNC_DATA_REQUIRED_STATIS_LOAD;
 }
 
 bool getCountFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) {
@@ -183,6 +183,10 @@ bool getSumFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) {
   return true;
 }
 
+EFuncDataRequired statisDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWindow){
+  return FUNC_DATA_REQUIRED_STATIS_LOAD;
+}
+
 bool maxFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) {
   if (!functionSetup(pCtx, pResultInfo)) {
     return false;
@@ -357,20 +361,15 @@ int32_t doMinMaxHelper(SqlFunctionCtx *pCtx, int32_t isMinFunc) {
       index = pInput->pColumnDataAgg[0]->maxIndex;
     }
 
-    TSKEY key = TSKEY_INITIAL_VAL;
-    if (pCtx->ptsList != NULL) {
-      // the index is the original position, not the relative position
-      key = pCtx->ptsList[index];
-    }
+    // the index is the original position, not the relative position
+    TSKEY key = (pCtx->ptsList != NULL)? pCtx->ptsList[index]:TSKEY_INITIAL_VAL;
 
     if (IS_SIGNED_NUMERIC_TYPE(type)) {
-      int64_t val = GET_INT64_VAL(tval);
-
-#if defined(_DEBUG_VIEW)
-      qDebug("max value updated according to pre-cal:%d", *data);
-#endif
+      int64_t prev = 0;
+      GET_TYPED_DATA(prev, int64_t, type, buf);
 
-      if ((*(int64_t*)buf < val) ^ isMinFunc) {
+      int64_t val = GET_INT64_VAL(tval);
+      if ((prev < val) ^ isMinFunc) {
         *(int64_t*) buf = val;
         for (int32_t i = 0; i < (pCtx)->subsidiaryRes.numOfCols; ++i) {
           SqlFunctionCtx* __ctx = pCtx->subsidiaryRes.pCtx[i];
@@ -383,14 +382,28 @@ int32_t doMinMaxHelper(SqlFunctionCtx *pCtx, int32_t isMinFunc) {
         }
       }
     } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) {
+      uint64_t prev = 0;
+      GET_TYPED_DATA(prev, uint64_t, type, buf);
+
       uint64_t val = GET_UINT64_VAL(tval);
-      UPDATE_DATA(pCtx, *(uint64_t*)buf, val, numOfElems, isMinFunc, key);
+      if ((prev < val) ^ isMinFunc) {
+        *(uint64_t*) buf = val;
+        for (int32_t i = 0; i < (pCtx)->subsidiaryRes.numOfCols; ++i) {
+          SqlFunctionCtx* __ctx = pCtx->subsidiaryRes.pCtx[i];
+          if (__ctx->functionId == FUNCTION_TS_DUMMY) {  // TODO refactor
+            __ctx->tag.i = key;
+            __ctx->tag.nType = TSDB_DATA_TYPE_BIGINT;
+          }
+
+          __ctx->fpSet.process(__ctx);
+        }
+      }
     } else if (type == TSDB_DATA_TYPE_DOUBLE) {
       double  val = GET_DOUBLE_VAL(tval);
-      UPDATE_DATA(pCtx, *(double*)buf, val, numOfElems, isMinFunc, key);
+      UPDATE_DATA(pCtx, *(double*) buf, val, numOfElems, isMinFunc, key);
     } else if (type == TSDB_DATA_TYPE_FLOAT) {
       double val = GET_DOUBLE_VAL(tval);
-      UPDATE_DATA(pCtx, *(float*)buf, (float)val, numOfElems, isMinFunc, key);
+      UPDATE_DATA(pCtx, *(float*) buf, val, numOfElems, isMinFunc, key);
     }
 
     return numOfElems;

source/libs/parser/test/mockCatalogService.cpp

@@ -264,7 +264,7 @@ private:
   }
 
   std::string ftToString(int16_t colid, int16_t numOfColumns) const {
-    return (0 == colid ? "column" : (colid <= numOfColumns ? "tag" : "column"));
+    return (0 == colid ? "column" : (colid < numOfColumns ? "column" : "tag"));
   }
 
   STableMeta* getTableSchemaMeta(const std::string& db, const std::string& tbname) const {