Merge remote-tracking branch 'origin/3.0' into feature/3.0_liaohj

CMakeLists.txt

@@ -6,13 +6,17 @@ project(
     DESCRIPTION "An open-source big data platform designed and optimized for the Internet of Things(IOT)"
 )
 
-set(CMAKE_SUPPORT_DIR "${CMAKE_SOURCE_DIR}/cmake")
-set(CMAKE_CONTRIB_DIR "${CMAKE_SOURCE_DIR}/contrib")
+if (NOT DEFINED TD_SOURCE_DIR)
+  set( TD_SOURCE_DIR ${CMAKE_SOURCE_DIR} )
+endif()
 
-include(${CMAKE_SUPPORT_DIR}/cmake.platform)
-include(${CMAKE_SUPPORT_DIR}/cmake.define)
-include(${CMAKE_SUPPORT_DIR}/cmake.options)
-include(${CMAKE_SUPPORT_DIR}/cmake.version)
+set(TD_SUPPORT_DIR "${TD_SOURCE_DIR}/cmake")
+set(TD_CONTRIB_DIR "${TD_SOURCE_DIR}/contrib")
+
+include(${TD_SUPPORT_DIR}/cmake.platform)
+include(${TD_SUPPORT_DIR}/cmake.define)
+include(${TD_SUPPORT_DIR}/cmake.options)
+include(${TD_SUPPORT_DIR}/cmake.version)
 
 # contrib
 add_subdirectory(contrib)

cmake/bdb_CMakeLists.txt.in.bak

@@ -3,8 +3,8 @@
 ExternalProject_Add(bdb
         GIT_REPOSITORY https://github.com/berkeleydb/libdb.git
         GIT_TAG v5.3.28
-        SOURCE_DIR "${CMAKE_CONTRIB_DIR}/bdb"
-        BINARY_DIR "${CMAKE_CONTRIB_DIR}/bdb"
+        SOURCE_DIR "${TD_CONTRIB_DIR}/bdb"
+        BINARY_DIR "${TD_CONTRIB_DIR}/bdb"
         #BUILD_IN_SOURCE TRUE
         CONFIGURE_COMMAND COMMAND ./dist/configure --enable-debug
         BUILD_COMMAND "$(MAKE)"

cmake/cjson_CMakeLists.txt.in

@@ -3,7 +3,7 @@
 ExternalProject_Add(cjson
         GIT_REPOSITORY https://github.com/taosdata-contrib/cJSON.git
         GIT_TAG v1.7.15
-        SOURCE_DIR "${CMAKE_CONTRIB_DIR}/cJson"
+        SOURCE_DIR "${TD_CONTRIB_DIR}/cJson"
         BINARY_DIR ""
         CONFIGURE_COMMAND ""
         BUILD_COMMAND ""

cmake/craft_CMakeLists.txt.in

@@ -3,8 +3,8 @@
 ExternalProject_Add(craft
         GIT_REPOSITORY https://github.com/canonical/raft.git
         GIT_TAG v0.11.2
-        SOURCE_DIR "${CMAKE_CONTRIB_DIR}/craft"
-        BINARY_DIR "${CMAKE_CONTRIB_DIR}/craft"
+        SOURCE_DIR "${TD_CONTRIB_DIR}/craft"
+        BINARY_DIR "${TD_CONTRIB_DIR}/craft"
         #BUILD_IN_SOURCE TRUE
         # https://answers.ros.org/question/333125/how-to-include-external-automakeautoconf-projects-into-ament_cmake/
         CONFIGURE_COMMAND COMMAND autoreconf -i COMMAND ./configure --enable-example

cmake/gnuregex_CMakeLists.txt.in

@@ -3,7 +3,7 @@
 ExternalProject_Add(gnuregex
         URL https://launchpad.net/gnuregex/trunk/2.9/+download/libgnurx-src-2.9.zip
         DOWNLOAD_NAME libgnurx-src.zip
-        SOURCE_DIR "${CMAKE_CONTRIB_DIR}/gnuregex"
+        SOURCE_DIR "${TD_CONTRIB_DIR}/gnuregex"
         BINARY_DIR ""
         #BUILD_IN_SOURCE TRUE
         CONFIGURE_COMMAND ""

cmake/gtest_CMakeLists.txt.in

@@ -3,7 +3,7 @@
 ExternalProject_Add(googletest
         GIT_REPOSITORY https://github.com/taosdata-contrib/googletest.git
         GIT_TAG release-1.11.0
-        SOURCE_DIR "${CMAKE_CONTRIB_DIR}/googletest"
+        SOURCE_DIR "${TD_CONTRIB_DIR}/googletest"
         BINARY_DIR ""
         CONFIGURE_COMMAND ""
         BUILD_COMMAND ""

cmake/iconv_CMakeLists.txt.in

@@ -3,7 +3,7 @@
 ExternalProject_Add(iconv
         GIT_REPOSITORY https://github.com/win-iconv/win-iconv.git
         GIT_TAG v0.0.8
-        SOURCE_DIR "${CMAKE_CONTRIB_DIR}/iconv"
+        SOURCE_DIR "${TD_CONTRIB_DIR}/iconv"
         BINARY_DIR ""
         CONFIGURE_COMMAND ""
         BUILD_COMMAND ""

cmake/leveldb_CMakeLists.txt.in

@@ -3,7 +3,7 @@
 ExternalProject_Add(leveldb
         GIT_REPOSITORY https://github.com/taosdata-contrib/leveldb.git
         GIT_TAG master
-        SOURCE_DIR "${CMAKE_CONTRIB_DIR}/leveldb"
+        SOURCE_DIR "${TD_CONTRIB_DIR}/leveldb"
         BINARY_DIR ""
         #BUILD_IN_SOURCE TRUE
         CONFIGURE_COMMAND ""

cmake/libuv_CMakeLists.txt.in

@@ -3,8 +3,8 @@
 ExternalProject_Add(libuv
         GIT_REPOSITORY https://github.com/libuv/libuv.git
         GIT_TAG v1.42.0
-        SOURCE_DIR "${CMAKE_CONTRIB_DIR}/libuv"
-        BINARY_DIR "${CMAKE_CONTRIB_DIR}/libuv"
+        SOURCE_DIR "${TD_CONTRIB_DIR}/libuv"
+        BINARY_DIR "${TD_CONTRIB_DIR}/libuv"
         CONFIGURE_COMMAND "" 
         BUILD_COMMAND ""  
         INSTALL_COMMAND ""

cmake/lucene_CMakeLists.txt.in

@@ -2,7 +2,7 @@
 # lucene
 ExternalProject_Add(lucene
         GIT_REPOSITORY https://github.com/yihaoDeng/LucenePlusPlus.git
-        SOURCE_DIR "${CMAKE_CONTRIB_DIR}/lucene"
+        SOURCE_DIR "${TD_CONTRIB_DIR}/lucene"
         BINARY_DIR ""
         #BUILD_IN_SOURCE TRUE
         CONFIGURE_COMMAND ""

cmake/lz4_CMakeLists.txt.in

@@ -3,7 +3,7 @@
 ExternalProject_Add(lz4
         GIT_REPOSITORY https://github.com/taosdata-contrib/lz4.git
         GIT_TAG v1.9.3
-        SOURCE_DIR "${CMAKE_CONTRIB_DIR}/lz4"
+        SOURCE_DIR "${TD_CONTRIB_DIR}/lz4"
         BINARY_DIR ""
         CONFIGURE_COMMAND ""
         BUILD_COMMAND ""

cmake/nuraft_CMakeLists.txt.in

@@ -3,8 +3,8 @@
 ExternalProject_Add(NuRaft
         GIT_REPOSITORY https://github.com/eBay/NuRaft.git
         GIT_TAG v1.3.0
-        SOURCE_DIR "${CMAKE_CONTRIB_DIR}/nuraft"
-        BINARY_DIR "${CMAKE_CONTRIB_DIR}/nuraft"
+        SOURCE_DIR "${TD_CONTRIB_DIR}/nuraft"
+        BINARY_DIR "${TD_CONTRIB_DIR}/nuraft"
         CONFIGURE_COMMAND "./prepare.sh"
         BUILD_COMMAND ""
         INSTALL_COMMAND ""

cmake/pthread_CMakeLists.txt.in

@@ -3,7 +3,7 @@
 ExternalProject_Add(pthread
         GIT_REPOSITORY https://github.com/GerHobbelt/pthread-win32
         GIT_TAG v3.0.3.1
-        SOURCE_DIR "${CMAKE_CONTRIB_DIR}/pthread-win32"
+        SOURCE_DIR "${TD_CONTRIB_DIR}/pthread-win32"
         BINARY_DIR ""
         #BUILD_IN_SOURCE TRUE
         CONFIGURE_COMMAND ""

cmake/rocksdb_CMakeLists.txt.in

@@ -3,7 +3,7 @@
 ExternalProject_Add(rocksdb
         GIT_REPOSITORY https://github.com/taosdata-contrib/rocksdb.git
         GIT_TAG v6.23.3
-        SOURCE_DIR "${CMAKE_CONTRIB_DIR}/rocksdb"
+        SOURCE_DIR "${TD_CONTRIB_DIR}/rocksdb"
         CONFIGURE_COMMAND ""
         BUILD_COMMAND ""
         INSTALL_COMMAND ""

cmake/sqlite_CMakeLists.txt.in

@@ -3,8 +3,8 @@
 ExternalProject_Add(sqlite
         GIT_REPOSITORY https://github.com/sqlite/sqlite.git
         GIT_TAG version-3.36.0
-        SOURCE_DIR "${CMAKE_CONTRIB_DIR}/sqlite"
-        BINARY_DIR "${CMAKE_CONTRIB_DIR}/sqlite"
+        SOURCE_DIR "${TD_CONTRIB_DIR}/sqlite"
+        BINARY_DIR "${TD_CONTRIB_DIR}/sqlite"
         #BUILD_IN_SOURCE TRUE
         CONFIGURE_COMMAND "./configure"
         BUILD_COMMAND "$(MAKE)"

cmake/stub_CMakeLists.txt.in

@@ -3,8 +3,8 @@
 ExternalProject_Add(stub
         GIT_REPOSITORY https://github.com/coolxv/cpp-stub.git
         GIT_SUBMODULES "src"
-        SOURCE_DIR "${CMAKE_CONTRIB_DIR}/cpp-stub"
-        BINARY_DIR "${CMAKE_CONTRIB_DIR}/cpp-stub/src"
+        SOURCE_DIR "${TD_CONTRIB_DIR}/cpp-stub"
+        BINARY_DIR "${TD_CONTRIB_DIR}/cpp-stub/src"
         CONFIGURE_COMMAND ""
         BUILD_COMMAND ""
         INSTALL_COMMAND ""

cmake/traft_CMakeLists.txt.in

@@ -3,8 +3,8 @@
 ExternalProject_Add(traft
         GIT_REPOSITORY https://github.com/taosdata/traft.git
         GIT_TAG for_3.0
-        SOURCE_DIR "${CMAKE_CONTRIB_DIR}/traft"
-        BINARY_DIR "${CMAKE_CONTRIB_DIR}/traft"
+        SOURCE_DIR "${TD_CONTRIB_DIR}/traft"
+        BINARY_DIR "${TD_CONTRIB_DIR}/traft"
         #BUILD_IN_SOURCE TRUE
         # https://answers.ros.org/question/333125/how-to-include-external-automakeautoconf-projects-into-ament_cmake/
         CONFIGURE_COMMAND COMMAND autoreconf -i COMMAND ./configure

cmake/zlib_CMakeLists.txt.in

@@ -3,7 +3,7 @@
 ExternalProject_Add(zlib
         GIT_REPOSITORY https://github.com/taosdata-contrib/zlib.git
         GIT_TAG v1.2.11
-        SOURCE_DIR "${CMAKE_CONTRIB_DIR}/zlib"
+        SOURCE_DIR "${TD_CONTRIB_DIR}/zlib"
         BINARY_DIR ""
         #BUILD_IN_SOURCE TRUE
         CONFIGURE_COMMAND ""

contrib/CMakeLists.txt

@@ -7,88 +7,88 @@ function(cat IN_FILE OUT_FILE)
 endfunction(cat IN_FILE OUT_FILE)
 
 set(CONTRIB_TMP_FILE "${CMAKE_BINARY_DIR}/deps_tmp_CMakeLists.txt.in")
-configure_file("${CMAKE_SUPPORT_DIR}/deps_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
+configure_file("${TD_SUPPORT_DIR}/deps_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
 
 # pthread
 if(${BUILD_PTHREAD})
-    cat("${CMAKE_SUPPORT_DIR}/pthread_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
+    cat("${TD_SUPPORT_DIR}/pthread_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
 endif()
 
 # gnu regex
 if(${BUILD_GNUREGEX})
-    cat("${CMAKE_SUPPORT_DIR}/gnuregex_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
+    cat("${TD_SUPPORT_DIR}/gnuregex_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
 endif()
 
 # googletest
 if(${BUILD_TEST})
-    cat("${CMAKE_SUPPORT_DIR}/gtest_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
-    cat("${CMAKE_SUPPORT_DIR}/stub_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
+    cat("${TD_SUPPORT_DIR}/gtest_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
+    cat("${TD_SUPPORT_DIR}/stub_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
 endif(${BUILD_TEST})
 
 # lz4
-cat("${CMAKE_SUPPORT_DIR}/lz4_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
+cat("${TD_SUPPORT_DIR}/lz4_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
 
 # zlib
-cat("${CMAKE_SUPPORT_DIR}/zlib_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
+cat("${TD_SUPPORT_DIR}/zlib_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
 
 # cJson
-cat("${CMAKE_SUPPORT_DIR}/cjson_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
+cat("${TD_SUPPORT_DIR}/cjson_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
 
 # leveldb
 if(${BUILD_WITH_LEVELDB})
-    cat("${CMAKE_SUPPORT_DIR}/leveldb_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
+    cat("${TD_SUPPORT_DIR}/leveldb_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
 endif(${BUILD_WITH_LEVELDB})
 
 # rocksdb
 if(${BUILD_WITH_ROCKSDB})
-    cat("${CMAKE_SUPPORT_DIR}/rocksdb_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
+    cat("${TD_SUPPORT_DIR}/rocksdb_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
     add_definitions(-DUSE_ROCKSDB)
 endif(${BUILD_WITH_ROCKSDB})
 
 # canonical-raft
 if(${BUILD_WITH_CRAFT})
-    cat("${CMAKE_SUPPORT_DIR}/craft_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
+    cat("${TD_SUPPORT_DIR}/craft_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
     SET(BUILD_WITH_UV ON CACHE BOOL "craft need libuv" FORCE)
 endif(${BUILD_WITH_CRAFT})
 
 # traft
 if(${BUILD_WITH_TRAFT})
-    cat("${CMAKE_SUPPORT_DIR}/traft_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
+    cat("${TD_SUPPORT_DIR}/traft_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
     SET(BUILD_WITH_UV ON CACHE BOOL "traft need libuv" FORCE)
 endif(${BUILD_WITH_TRAFT})
 
 #libuv
 if(${BUILD_WITH_UV})
-    cat("${CMAKE_SUPPORT_DIR}/libuv_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
+    cat("${TD_SUPPORT_DIR}/libuv_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
 endif(${BUILD_WITH_UV})
 
 # bdb
 #if(${BUILD_WITH_BDB})
-    #cat("${CMAKE_SUPPORT_DIR}/bdb_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
+    #cat("${TD_SUPPORT_DIR}/bdb_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
 #endif(${BUILD_WITH_BDB})
 
 # sqlite
 if(${BUILD_WITH_SQLITE})
-    cat("${CMAKE_SUPPORT_DIR}/sqlite_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
+    cat("${TD_SUPPORT_DIR}/sqlite_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
 endif(${BUILD_WITH_SQLITE})
 
 # lucene
 if(${BUILD_WITH_LUCENE})
-    cat("${CMAKE_SUPPORT_DIR}/lucene_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
+    cat("${TD_SUPPORT_DIR}/lucene_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
     add_definitions(-DUSE_LUCENE)
 endif(${BUILD_WITH_LUCENE})
 
 # NuRaft
 if(${BUILD_WITH_NURAFT})
-    cat("${CMAKE_SUPPORT_DIR}/nuraft_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
+    cat("${TD_SUPPORT_DIR}/nuraft_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
 endif(${BUILD_WITH_NURAFT})
 
 # download dependencies
-configure_file(${CONTRIB_TMP_FILE} "${CMAKE_CONTRIB_DIR}/deps-download/CMakeLists.txt")
+configure_file(${CONTRIB_TMP_FILE} "${TD_CONTRIB_DIR}/deps-download/CMakeLists.txt")
 execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" .
-        WORKING_DIRECTORY "${CMAKE_CONTRIB_DIR}/deps-download")
+        WORKING_DIRECTORY "${TD_CONTRIB_DIR}/deps-download")
 execute_process(COMMAND "${CMAKE_COMMAND}" --build .
-        WORKING_DIRECTORY "${CMAKE_CONTRIB_DIR}/deps-download")
+        WORKING_DIRECTORY "${TD_CONTRIB_DIR}/deps-download")
 
 # ================================================================================================
 # Build
@@ -105,7 +105,7 @@ endif(${BUILD_TEST})
 
 # cJson
 # see https://stackoverflow.com/questions/37582508/silence-cmp0048-warnings-in-vendored-projects
-set(CMAKE_PROJECT_INCLUDE_BEFORE "${CMAKE_SUPPORT_DIR}/EnableCMP0048.txt.in")
+set(CMAKE_PROJECT_INCLUDE_BEFORE "${TD_SUPPORT_DIR}/EnableCMP0048.txt.in")
 option(ENABLE_CJSON_TEST "Enable building cJSON test" OFF)
 option(CJSON_OVERRIDE_BUILD_SHARED_LIBS "Override BUILD_SHARED_LIBS with CJSON_BUILD_SHARED_LIBS" ON)
 option(CJSON_BUILD_SHARED_LIBS "Overrides BUILD_SHARED_LIBS if CJSON_OVERRIDE_BUILD_SHARED_LIBS is enabled" OFF)
@@ -125,7 +125,7 @@ target_include_directories(
 )
 
 # zlib
-set(CMAKE_PROJECT_INCLUDE_BEFORE "${CMAKE_SUPPORT_DIR}/EnableCMP0048.txt.in")
+set(CMAKE_PROJECT_INCLUDE_BEFORE "${TD_SUPPORT_DIR}/EnableCMP0048.txt.in")
 add_subdirectory(zlib)
 target_include_directories(
     zlibstatic

docs/CMakeLists.txt

@@ -4,7 +4,7 @@ if(${BUILD_DOCS})
     find_package(Doxygen)
     if (DOXYGEN_FOUND)
         # Build the doc
-        set(DOXYGEN_IN ${CMAKE_SOURCE_DIR}/docs/Doxyfile.in)
+        set(DOXYGEN_IN ${TD_SOURCE_DIR}/docs/Doxyfile.in)
         set(DOXYGEN_OUT ${CMAKE_BINARY_DIR}/Doxyfile)
 
         configure_file(${DOXYGEN_IN} ${DOXYGEN_OUT} @ONLY)

docs/Doxyfile.in

@@ -829,7 +829,7 @@ WARN_LOGFILE           =
 # spaces. See also FILE_PATTERNS and EXTENSION_MAPPING
 # Note: If this tag is empty the current directory is searched.
 
-INPUT                  = @CMAKE_SOURCE_DIR@/include @CMAKE_SOURCE_DIR@/source
+INPUT                  = @TD_SOURCE_DIR@/include @TD_SOURCE_DIR@/source
 
 # This tag can be used to specify the character encoding of the source files
 # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses

include/common/taosdef.h

@@ -57,6 +57,8 @@ typedef enum {
   TD_ROW_PARTIAL_UPDATE = 2,
 } TDUpdateConfig;
 
+#define TD_SUPPORT_UPDATE(u) ((u) > 0)
+
 typedef enum {
   TSDB_STATIS_OK = 0,    // statis part exist and load successfully
   TSDB_STATIS_NONE = 1,  // statis part not exist

include/common/tdataformat.h

@@ -131,12 +131,17 @@ typedef struct {
   STColumn    *columns;
 } STSchemaBuilder;
 
-#define TD_VTYPE_BITS  2  // val type
-#define TD_VTYPE_PARTS 4  // 8 bits / TD_VTYPE_BITS = 4
-#define TD_VTYPE_OPTR  3  // TD_VTYPE_PARTS - 1, utilize to get remainder
-
-#define TD_BITMAP_BYTES(cnt) (ceil((double)(cnt) / TD_VTYPE_PARTS))
-#define TD_BIT_TO_BYTES(cnt) (ceil((double)(cnt) / 8))
+// use 2 bits for bitmap(default: STSRow/sub block)
+#define TD_VTYPE_BITS        2
+#define TD_VTYPE_PARTS       4  // PARTITIONS: 1 byte / 2 bits
+#define TD_VTYPE_OPTR        3  // OPERATOR: 4 - 1, utilize to get remainder
+#define TD_BITMAP_BYTES(cnt) (((cnt) + TD_VTYPE_OPTR) >> 2)
+
+// use 1 bit for bitmap(super block)
+#define TD_VTYPE_BITS_I        1
+#define TD_VTYPE_PARTS_I       8  // PARTITIONS: 1 byte / 1 bit
+#define TD_VTYPE_OPTR_I        7  // OPERATOR: 8 - 1, utilize to get remainder
+#define TD_BITMAP_BYTES_I(cnt) (((cnt) + TD_VTYPE_OPTR_I) >> 3)
 
 int32_t   tdInitTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version);
 void      tdDestroyTSchemaBuilder(STSchemaBuilder *pBuilder);
@@ -367,9 +372,10 @@ static FORCE_INLINE void tdCopyColOfRowBySchema(SDataRow dst, STSchema *pDstSche
 // ----------------- Data column structure
 // SDataCol arrangement: data => bitmap => dataOffset
 typedef struct SDataCol {
-  int8_t          type;        // column type
-  uint8_t         bitmap : 1;  // 0: has bitmap if has NULL/NORM rows, 1: no bitmap if all rows are NORM
-  uint8_t         reserve : 7;
+  int8_t          type;            // column type
+  uint8_t         bitmap : 1;      // 0: no bitmap if all rows are NORM, 1: has bitmap if has NULL/NORM rows
+  uint8_t         bitmapMode : 1;  // default is 0(2 bits), otherwise 1(1 bit)
+  uint8_t         reserve : 6;
   int16_t         colId;      // column ID
   int32_t         bytes;      // column data bytes defined
   int32_t         offset;     // data offset in a SDataRow (including the header size)
@@ -381,6 +387,8 @@ typedef struct SDataCol {
   TSKEY           ts;         // only used in last NULL column
 } SDataCol;
 
+
+
 #define isAllRowsNull(pCol) ((pCol)->len == 0)
 #define isAllRowsNone(pCol) ((pCol)->len == 0)
 static FORCE_INLINE void dataColReset(SDataCol *pDataCol) { pDataCol->len = 0; }
@@ -421,10 +429,14 @@ typedef struct {
   col_id_t  numOfCols;  // Total number of cols
   int32_t   maxPoints;  // max number of points
   int32_t   numOfRows;
-  int32_t   sversion;  // TODO: set sversion
+  int32_t   bitmapMode : 1;  // default is 0(2 bits), otherwise 1(1 bit)
+  int32_t   sversion : 31;   // TODO: set sversion(not used yet)
   SDataCol *cols;
 } SDataCols;
 
+static FORCE_INLINE bool tdDataColsIsBitmapI(SDataCols *pCols) { return pCols->bitmapMode != 0; }
+static FORCE_INLINE void tdDataColsSetBitmapI(SDataCols *pCols) { pCols->bitmapMode = 1; }
+
 #define keyCol(pCols)              (&((pCols)->cols[0]))                    // Key column
 #define dataColsTKeyAt(pCols, idx) ((TKEY *)(keyCol(pCols)->pData))[(idx)]  // the idx row of column-wised data
 #define dataColsKeyAt(pCols, idx)  tdGetKey(dataColsTKeyAt(pCols, idx))
@@ -470,7 +482,7 @@ void       tdResetDataCols(SDataCols *pCols);
 int32_t    tdInitDataCols(SDataCols *pCols, STSchema *pSchema);
 SDataCols *tdDupDataCols(SDataCols *pCols, bool keepData);
 SDataCols *tdFreeDataCols(SDataCols *pCols);
-int32_t tdMergeDataCols(SDataCols *target, SDataCols *source, int32_t rowsToMerge, int32_t *pOffset, bool forceSetNull);
+int32_t tdMergeDataCols(SDataCols *target, SDataCols *source, int32_t rowsToMerge, int32_t *pOffset, bool forceSetNull, TDRowVerT maxVer);
 
 // ----------------- K-V data row structure
 /* |<-------------------------------------- len -------------------------------------------->|

include/common/tglobal.h

@@ -126,6 +126,7 @@ int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDi
 int32_t taosInitCfg(const char *cfgDir, const char *envFile, const char *apolloUrl, SArray *pArgs, bool tsc);
 void    taosCleanupCfg();
 void    taosCfgDynamicOptions(const char *option, const char *value);
+void    taosAddDataDir(int32_t index, char *v1, int32_t level, int32_t primary);
 
 struct SConfig *taosGetCfg();
 

include/common/tgrant.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_GTANT_H
+#define TDENGINE_GTANT_H
+
+#ifdef __cplusplus
+"C" {
+#endif
+
+typedef enum {
+  TSDB_GRANT_ALL,
+  TSDB_GRANT_TIME,
+  TSDB_GRANT_USER,
+  TSDB_GRANT_DB,
+  TSDB_GRANT_TIMESERIES,
+  TSDB_GRANT_DNODE,
+  TSDB_GRANT_ACCT,
+  TSDB_GRANT_STORAGE,
+  TSDB_GRANT_SPEED,
+  TSDB_GRANT_QUERY_TIME,
+  TSDB_GRANT_CONNS,
+  TSDB_GRANT_STREAMS,
+  TSDB_GRANT_CPU_CORES,
+} EGrantType;
+
+int32_t grantInit();
+void    grantCleanUp();
+void    grantParseParameter();
+int32_t grantCheck(EGrantType grant);
+void    grantReset(EGrantType grant, uint64_t value);
+void    grantAdd(EGrantType grant, uint64_t value);
+void    grantRestore(EGrantType grant, uint64_t value);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

include/common/tmsg.h

@@ -484,7 +484,7 @@ typedef struct {
   char    intervalUnit;
   char    slidingUnit;
   char
-      offsetUnit;  // TODO Remove it, the offset is the number of precision tickle, and it must be a immutable duration.
+          offsetUnit;  // TODO Remove it, the offset is the number of precision tickle, and it must be a immutable duration.
   int8_t  precision;
   int64_t interval;
   int64_t sliding;
@@ -589,7 +589,7 @@ int32_t tDeserializeSUseDbRspImp(SCoder* pDecoder, SUseDbRsp* pRsp);
 void    tFreeSUsedbRsp(SUseDbRsp* pRsp);
 
 typedef struct {
-  char    db[TSDB_DB_FNAME_LEN];
+  char db[TSDB_DB_FNAME_LEN];
 } SDbCfgReq;
 
 int32_t tSerializeSDbCfgReq(void* buf, int32_t bufLen, SDbCfgReq* pReq);
@@ -622,7 +622,6 @@ typedef struct {
 int32_t tSerializeSDbCfgRsp(void* buf, int32_t bufLen, const SDbCfgRsp* pRsp);
 int32_t tDeserializeSDbCfgRsp(void* buf, int32_t bufLen, SDbCfgRsp* pRsp);
 
-
 typedef struct {
   int32_t rowNum;
 } SQnodeListReq;
@@ -1023,8 +1022,9 @@ int32_t tDeserializeSMCfgDnodeReq(void* buf, int32_t bufLen, SMCfgDnodeReq* pReq
 
 typedef struct {
   int32_t dnodeId;
-} SMCreateMnodeReq, SMDropMnodeReq, SDDropMnodeReq, SMCreateQnodeReq, SMDropQnodeReq, SDCreateQnodeReq, SDDropQnodeReq, SMCreateSnodeReq, SMDropSnodeReq,
-    SDCreateSnodeReq, SDDropSnodeReq, SMCreateBnodeReq, SMDropBnodeReq, SDCreateBnodeReq, SDDropBnodeReq;
+} SMCreateMnodeReq, SMDropMnodeReq, SDDropMnodeReq, SMCreateQnodeReq, SMDropQnodeReq, SDCreateQnodeReq, SDDropQnodeReq,
+    SMCreateSnodeReq, SMDropSnodeReq, SDCreateSnodeReq, SDDropSnodeReq, SMCreateBnodeReq, SMDropBnodeReq,
+    SDCreateBnodeReq, SDDropBnodeReq;
 
 int32_t tSerializeSCreateDropMQSBNodeReq(void* buf, int32_t bufLen, SMCreateQnodeReq* pReq);
 int32_t tDeserializeSCreateDropMQSBNodeReq(void* buf, int32_t bufLen, SMCreateQnodeReq* pReq);
@@ -2298,6 +2298,22 @@ static FORCE_INLINE void* tDecodeTSmaWrapper(void* buf, STSmaWrapper* pSW) {
   return buf;
 }
 
+typedef struct {
+  int idx;
+} SMCreateFullTextReq;
+
+int32_t tSerializeSMCreateFullTextReq(void* buf, int32_t bufLen, SMCreateFullTextReq* pReq);
+int32_t tDeserializeSMCreateFullTextReq(void* buf, int32_t bufLen, SMCreateFullTextReq* pReq);
+void    tFreeSMCreateFullTextReq(SMCreateFullTextReq* pReq);
+
+typedef struct {
+  char   name[TSDB_TABLE_FNAME_LEN];
+  int8_t igNotExists;
+} SMDropFullTextReq;
+
+int32_t tSerializeSMDropFullTextReq(void* buf, int32_t bufLen, SMDropFullTextReq* pReq);
+int32_t tDeserializeSMDropFullTextReq(void* buf, int32_t bufLen, SMDropFullTextReq* pReq);
+
 typedef struct {
   char indexFName[TSDB_INDEX_FNAME_LEN];
 } SUserIndexReq;
@@ -2332,9 +2348,9 @@ typedef struct {
   int8_t  withSchema;
   char    cgroup[TSDB_CGROUP_LEN];
 
-  int64_t currentOffset;
+  int64_t  currentOffset;
   uint64_t reqId;
-  char    topic[TSDB_TOPIC_FNAME_LEN];
+  char     topic[TSDB_TOPIC_FNAME_LEN];
 } SMqPollReq;
 
 typedef struct {

include/common/tmsgdef.h

@@ -155,6 +155,8 @@ enum {
   TD_DEF_MSG_TYPE(TDMT_MND_CREATE_STREAM, "mnode-create-stream", SCMCreateStreamReq, SCMCreateStreamRsp)
   TD_DEF_MSG_TYPE(TDMT_MND_ALTER_STREAM, "mnode-alter-stream", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_DROP_STREAM, "mnode-drop-stream", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_MND_CREATE_INDEX, "mnode-create-index", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_MND_DROP_INDEX, "mnode-drop-index", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_GET_DB_CFG, "mnode-get-db-cfg", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_MND_GET_INDEX, "mnode-get-index", NULL, NULL)
 

include/common/trow.h

@@ -40,21 +40,20 @@ extern "C" {
 #define TD_ROW_KV 0x01U
 
 /**
- * @brief val type
- *  - for data from client input and STSRow in memory, 3 types of val none/null/norm available
- *  - for data in
+ * @brief value type
+ *  - for data from client input and STSRow in memory, 3 types of value none/null/norm available
  */
-#define TD_VTYPE_NONE 0x0U   // none or unknown/undefined
+#define TD_VTYPE_NORM 0x00U  // normal val: not none, not null(no need assign value)
 #define TD_VTYPE_NULL 0x01U  // null val
-#define TD_VTYPE_NORM 0x02U  // normal val: not none, not null
-#define TD_VTYPE_MAX 0x03U   //
+#define TD_VTYPE_NONE 0x02U  // none or unknown/undefined
+#define TD_VTYPE_MAX  0x03U  //
 
-#define TD_VTYPE_NONE_BYTE 0x0U
+#define TD_VTYPE_NORM_BYTE 0x0U
 #define TD_VTYPE_NULL_BYTE 0x55U
-#define TD_VTYPE_NORM_BYTE 0xAAU
+#define TD_VTYPE_NONE_BYTE 0xAAU
 
-#define TD_ROWS_ALL_NORM 0x01U
-#define TD_ROWS_NULL_NORM 0x0U
+#define TD_ROWS_ALL_NORM  0x00U
+#define TD_ROWS_NULL_NORM 0x01U
 
 #define TD_COL_ROWS_NORM(c) ((c)->bitmap == TD_ROWS_ALL_NORM)  // all rows of SDataCol/SBlockCol is NORM
 #define TD_SET_COL_ROWS_BTIMAP(c, v) ((c)->bitmap = (v))
@@ -126,15 +125,15 @@ typedef struct {
     uint32_t info;
     struct {
       /// row type
-      uint32_t type : 2;
+      uint16_t type : 2;
       /// is delete row(0 not delete, 1 delete)
-      uint32_t del : 1;
+      uint16_t del : 1;
       /// endian(0 little endian, 1 big endian)
-      uint32_t endian : 1;
+      uint16_t endian : 1;
       /// reserved for back compatibility
-      uint32_t reserve : 12;
+      uint16_t reserve : 12;
       /// row schema version
-      uint32_t sver : 16;
+      uint16_t sver;
     };
   };
   /// row total length
@@ -216,17 +215,22 @@ static FORCE_INLINE void *tdKVRowColVal(STSRow *pRow, SKvRowIdx *pIdx) { return
 
 #define TD_ROW_OFFSET(p) ((p)->toffset);  // During ParseInsert when without STSchema, how to get the offset for STpRow?
 
+void                        tdMergeBitmap(uint8_t *srcBitmap, int32_t srcLen, uint8_t *dstBitmap);
 static FORCE_INLINE void    tdRowCopy(void *dst, STSRow *row) { memcpy(dst, row, TD_ROW_LEN(row)); }
-static FORCE_INLINE int32_t tdSetBitmapValType(void *pBitmap, int16_t colIdx, TDRowValT valType);
-int32_t                     tdSetBitmapValTypeN(void *pBitmap, int16_t nEle, TDRowValT valType);
-static FORCE_INLINE int32_t tdGetBitmapValType(void *pBitmap, int16_t colIdx, TDRowValT *pValType);
-int32_t tdAppendValToDataCol(SDataCol *pCol, TDRowValT valType, const void *val, int32_t numOfRows, int32_t maxPoints);
+static FORCE_INLINE int32_t tdSetBitmapValTypeI(void *pBitmap, int16_t colIdx, TDRowValT valType);
+static FORCE_INLINE int32_t tdSetBitmapValTypeII(void *pBitmap, int16_t colIdx, TDRowValT valType);
+static FORCE_INLINE int32_t tdSetBitmapValType(void *pBitmap, int16_t colIdx, TDRowValT valType, int8_t bitmapMode);
+int32_t                     tdSetBitmapValTypeN(void *pBitmap, int16_t nEle, TDRowValT valType, int8_t bitmapMode);
+static FORCE_INLINE int32_t tdGetBitmapValTypeI(void *pBitmap, int16_t colIdx, TDRowValT *pValType);
+static FORCE_INLINE int32_t tdGetBitmapValTypeII(void *pBitmap, int16_t colIdx, TDRowValT *pValType);
+int32_t tdAppendValToDataCol(SDataCol *pCol, TDRowValT valType, const void *val, int32_t numOfRows, int32_t maxPoints,
+                             int8_t bitmapMode);
 static FORCE_INLINE int32_t tdAppendColValToTpRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val,
                                                   bool isCopyVarData, int8_t colType, int16_t colIdx, int32_t offset);
 static FORCE_INLINE int32_t tdAppendColValToKvRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val,
                                                   bool isCopyVarData, int8_t colType, int16_t colIdx, int32_t offset,
                                                   col_id_t colId);
-int32_t tdAppendSTSRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols *pCols, bool forceSetNull);
+int32_t tdAppendSTSRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols *pCols);
 
 /**
  * @brief
@@ -259,15 +263,32 @@ static FORCE_INLINE void *tdGetBitmapAddr(STSRow *pRow, uint8_t rowType, uint32_
   return NULL;
 }
 
+static FORCE_INLINE int32_t tdSetBitmapValType(void *pBitmap, int16_t colIdx, TDRowValT valType, int8_t bitmapMode) {
+  switch (bitmapMode) {
+    case 0:
+      tdSetBitmapValTypeII(pBitmap, colIdx, valType);
+      break;
+    case -1:
+    case 1:
+      tdSetBitmapValTypeI(pBitmap, colIdx, valType);
+      break;
+    default:
+      TASSERT(0);
+      terrno = TSDB_CODE_INVALID_PARA;
+      return TSDB_CODE_FAILED;
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
 /**
- * @brief
+ * @brief Use 2 bits at default
  *
  * @param pBitmap
  * @param colIdx The relative index of colId, may have minus value as parameter.
  * @param valType
  * @return FORCE_INLINE
  */
-static FORCE_INLINE int32_t tdSetBitmapValType(void *pBitmap, int16_t colIdx, TDRowValT valType) {
+static FORCE_INLINE int32_t tdSetBitmapValTypeII(void *pBitmap, int16_t colIdx, TDRowValT valType) {
   if (!pBitmap || colIdx < 0) {
     TASSERT(0);
     terrno = TSDB_CODE_INVALID_PARA;
@@ -276,18 +297,24 @@ static FORCE_INLINE int32_t tdSetBitmapValType(void *pBitmap, int16_t colIdx, TD
   int16_t nBytes = colIdx / TD_VTYPE_PARTS;
   int16_t nOffset = colIdx & TD_VTYPE_OPTR;
   char   *pDestByte = (char *)POINTER_SHIFT(pBitmap, nBytes);
+  // use literal value directly and not use formula to simplify the codes
   switch (nOffset) {
     case 0:
-      *pDestByte = ((*pDestByte) & 0x3F) | (valType << 6);
+      // *pDestByte = ((*pDestByte) & 0x3F) | (valType << 6);
+      // set the value and clear other partitions for offset 0
+      *pDestByte = (valType << 6);
       break;
     case 1:
-      *pDestByte = ((*pDestByte) & 0xCF) | (valType << 4);
+      // *pDestByte = ((*pDestByte) & 0xCF) | (valType << 4);
+      *pDestByte |= (valType << 4);
       break;
     case 2:
-      *pDestByte = ((*pDestByte) & 0xF3) | (valType << 2);
+      // *pDestByte = ((*pDestByte) & 0xF3) | (valType << 2);
+      *pDestByte |= (valType << 2);
       break;
     case 3:
-      *pDestByte = ((*pDestByte) & 0xFC) | valType;
+      // *pDestByte = ((*pDestByte) & 0xFC) | valType;
+      *pDestByte |= (valType);
       break;
     default:
       TASSERT(0);
@@ -297,15 +324,32 @@ static FORCE_INLINE int32_t tdSetBitmapValType(void *pBitmap, int16_t colIdx, TD
   return TSDB_CODE_SUCCESS;
 }
 
+static FORCE_INLINE int32_t tdGetBitmapValType(void *pBitmap, int16_t colIdx, TDRowValT *pValType, int8_t bitmapMode) {
+  switch (bitmapMode) {
+    case 0:
+      tdGetBitmapValTypeII(pBitmap, colIdx, pValType);
+      break;
+    case -1:
+    case 1:
+      tdGetBitmapValTypeI(pBitmap, colIdx, pValType);
+      break;
+    default:
+      TASSERT(0);
+      terrno = TSDB_CODE_INVALID_PARA;
+      return TSDB_CODE_FAILED;
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
 /**
- * @brief
+ * @brief Use 2 bits at default
  *
  * @param pBitmap
  * @param colIdx The relative index of colId, may have minus value as parameter.
  * @param pValType
  * @return FORCE_INLINE
  */
-static FORCE_INLINE int32_t tdGetBitmapValType(void *pBitmap, int16_t colIdx, TDRowValT *pValType) {
+static FORCE_INLINE int32_t tdGetBitmapValTypeII(void *pBitmap, int16_t colIdx, TDRowValT *pValType) {
   if (!pBitmap || colIdx < 0) {
     TASSERT(0);
     terrno = TSDB_CODE_INVALID_PARA;
@@ -314,6 +358,7 @@ static FORCE_INLINE int32_t tdGetBitmapValType(void *pBitmap, int16_t colIdx, TD
   int16_t nBytes = colIdx / TD_VTYPE_PARTS;
   int16_t nOffset = colIdx & TD_VTYPE_OPTR;
   char   *pDestByte = (char *)POINTER_SHIFT(pBitmap, nBytes);
+  // use literal value directly and not use formula to simplify the codes
   switch (nOffset) {
     case 0:
       *pValType = (((*pDestByte) & 0xC0) >> 6);
@@ -335,6 +380,117 @@ static FORCE_INLINE int32_t tdGetBitmapValType(void *pBitmap, int16_t colIdx, TD
   return TSDB_CODE_SUCCESS;
 }
 
+/**
+ * @brief
+ *
+ * @param pBitmap
+ * @param colIdx The relative index of colId, may have minus value as parameter.
+ * @param valType
+ * @return FORCE_INLINE
+ */
+static FORCE_INLINE int32_t tdSetBitmapValTypeI(void *pBitmap, int16_t colIdx, TDRowValT valType) {
+  if (!pBitmap || colIdx < 0) {
+    TASSERT(0);
+    terrno = TSDB_CODE_INVALID_PARA;
+    return terrno;
+  }
+  int16_t nBytes = colIdx / TD_VTYPE_PARTS_I;
+  int16_t nOffset = colIdx & TD_VTYPE_OPTR_I;
+  char   *pDestByte = (char *)POINTER_SHIFT(pBitmap, nBytes);
+  // use literal value directly and not use formula to simplify the codes
+  switch (nOffset) {
+    case 0:
+      // *pDestByte = ((*pDestByte) & 0x7F) | (valType << 7);
+      // set the value and clear other partitions for offset 0
+      *pDestByte = (valType << 7);
+      break;
+    case 1:
+      // *pDestByte = ((*pDestByte) & 0xBF) | (valType << 6);
+      *pDestByte |= (valType << 6);
+      break;
+    case 2:
+      // *pDestByte = ((*pDestByte) & 0xDF) | (valType << 5);
+      *pDestByte |= (valType << 5);
+      break;
+    case 3:
+      // *pDestByte = ((*pDestByte) & 0xEF) | (valType << 4);
+      *pDestByte |= (valType << 4);
+      break;
+    case 4:
+      // *pDestByte = ((*pDestByte) & 0xF7) | (valType << 3);
+      *pDestByte |= (valType << 3);
+      break;
+    case 5:
+      // *pDestByte = ((*pDestByte) & 0xFB) | (valType << 2);
+      *pDestByte |= (valType << 2);
+      break;
+    case 6:
+      // *pDestByte = ((*pDestByte) & 0xFD) | (valType << 1);
+      *pDestByte |= (valType << 1);
+      break;
+    case 7:
+      // *pDestByte = ((*pDestByte) & 0xFE) | valType;
+      *pDestByte |= (valType);
+      break;
+    default:
+      TASSERT(0);
+      terrno = TSDB_CODE_INVALID_PARA;
+      return terrno;
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
+/**
+ * @brief
+ *
+ * @param pBitmap
+ * @param colIdx The relative index of colId, may have minus value as parameter.
+ * @param pValType
+ * @return FORCE_INLINE
+ */
+static FORCE_INLINE int32_t tdGetBitmapValTypeI(void *pBitmap, int16_t colIdx, TDRowValT *pValType) {
+  if (!pBitmap || colIdx < 0) {
+    TASSERT(0);
+    terrno = TSDB_CODE_INVALID_PARA;
+    return terrno;
+  }
+  int16_t nBytes = colIdx / TD_VTYPE_PARTS_I;
+  int16_t nOffset = colIdx & TD_VTYPE_OPTR_I;
+  char   *pDestByte = (char *)POINTER_SHIFT(pBitmap, nBytes);
+  // use literal value directly and not use formula to simplify the codes
+  switch (nOffset) {
+    case 0:
+      *pValType = (((*pDestByte) & 0x80) >> 7);
+      break;
+    case 1:
+      *pValType = (((*pDestByte) & 0x40) >> 6);
+      break;
+    case 2:
+      *pValType = (((*pDestByte) & 0x20) >> 5);
+      break;
+    case 3:
+      *pValType = (((*pDestByte) & 0x10) >> 4);
+      break;
+    case 4:
+      *pValType = (((*pDestByte) & 0x08) >> 3);
+      break;
+    case 5:
+      *pValType = (((*pDestByte) & 0x04) >> 2);
+      break;
+    case 6:
+      *pValType = (((*pDestByte) & 0x02) >> 1);
+      break;
+    case 7:
+      *pValType = ((*pDestByte) & 0x01);
+      break;
+    default:
+      TASSERT(0);
+      terrno = TSDB_CODE_INVALID_PARA;
+      return terrno;
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
 // ----------------- Tuple row structure(STpRow)
 /*
  * |<----------------------------- tlen ---------------------------------->|
@@ -552,7 +708,7 @@ static FORCE_INLINE int32_t tdAppendColValToTpRow(SRowBuilder *pBuilder, TDRowVa
   --colIdx;
 
 #ifdef TD_SUPPORT_BITMAP
-  if (tdSetBitmapValType(pBuilder->pBitmap, colIdx, valType) != TSDB_CODE_SUCCESS) {
+  if (tdSetBitmapValType(pBuilder->pBitmap, colIdx, valType, 0) != TSDB_CODE_SUCCESS) {
     return terrno;
   }
 #endif
@@ -610,7 +766,7 @@ static FORCE_INLINE int32_t tdAppendColValToKvRow(SRowBuilder *pBuilder, TDRowVa
   --colIdx;
 
 #ifdef TD_SUPPORT_BITMAP
-  if (tdSetBitmapValType(pBuilder->pBitmap, colIdx, valType) != TSDB_CODE_SUCCESS) {
+  if (tdSetBitmapValType(pBuilder->pBitmap, colIdx, valType, 0) != TSDB_CODE_SUCCESS) {
     return terrno;
   }
 #endif
@@ -691,12 +847,6 @@ static FORCE_INLINE int32_t tdAppendColValToRow(SRowBuilder *pBuilder, col_id_t
   if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
     TD_ROW_KEY(pRow) = *(TSKEY *)val;
     // The primary TS key is Norm all the time, thus its valType is not stored in bitmap.
-    // #ifdef TD_SUPPORT_BITMAP
-    //     pBitmap = tdGetBitmapAddr(pRow, pRow->type, pBuilder->flen, pRow->ncols);
-    //     if (tdSetBitmapValType(pBitmap, colIdx, valType) != TSDB_CODE_SUCCESS) {
-    //       return terrno;
-    //     }
-    // #endif
     return TSDB_CODE_SUCCESS;
   }
   // TODO:  We can avoid the type judegement by FP, but would prevent the inline scheme.
@@ -712,7 +862,7 @@ static FORCE_INLINE int32_t tdAppendColValToRow(SRowBuilder *pBuilder, col_id_t
 static FORCE_INLINE int32_t tdGetTpRowValOfCol(SCellVal *output, STSRow *pRow, void *pBitmap, int8_t colType,
                                                int32_t offset, int16_t colIdx) {
 #ifdef TD_SUPPORT_BITMAP
-  if (tdGetBitmapValType(pBitmap, colIdx, &output->valType) != TSDB_CODE_SUCCESS) {
+  if (tdGetBitmapValType(pBitmap, colIdx, &output->valType, 0) != TSDB_CODE_SUCCESS) {
     output->valType = TD_VTYPE_NONE;
     return terrno;
   }
@@ -748,7 +898,7 @@ static FORCE_INLINE int32_t tdGetKvRowValOfCol(SCellVal *output, STSRow *pRow, v
                                                int16_t colIdx) {
 #ifdef TD_SUPPORT_BITMAP
   TASSERT(colIdx < tdRowGetNCols(pRow) - 1);
-  if (tdGetBitmapValType(pBitmap, colIdx, &output->valType) != TSDB_CODE_SUCCESS) {
+  if (tdGetBitmapValType(pBitmap, colIdx, &output->valType, 0) != TSDB_CODE_SUCCESS) {
     output->valType = TD_VTYPE_NONE;
     return terrno;
   }
@@ -868,7 +1018,7 @@ static FORCE_INLINE bool tdGetTpRowDataOfCol(STSRowIter *pIter, col_type_t colTy
   }
 
 #ifdef TD_SUPPORT_BITMAP
-  if (tdGetBitmapValType(pIter->pBitmap, pIter->colIdx - 1, &pVal->valType) != TSDB_CODE_SUCCESS) {
+  if (tdGetBitmapValType(pIter->pBitmap, pIter->colIdx - 1, &pVal->valType, 0) != TSDB_CODE_SUCCESS) {
     pVal->valType = TD_VTYPE_NONE;
   }
 #else
@@ -905,7 +1055,7 @@ static FORCE_INLINE bool tdGetKvRowValOfColEx(STSRowIter *pIter, col_id_t colId,
 #ifdef TD_SUPPORT_BITMAP
   int16_t colIdx = -1;
   if (pKvIdx) colIdx = POINTER_DISTANCE(TD_ROW_COL_IDX(pRow), pKvIdx) / sizeof(SKvRowIdx);
-  if (tdGetBitmapValType(pIter->pBitmap, colIdx, &pVal->valType) != TSDB_CODE_SUCCESS) {
+  if (tdGetBitmapValType(pIter->pBitmap, colIdx, &pVal->valType, 0) != TSDB_CODE_SUCCESS) {
     pVal->valType = TD_VTYPE_NONE;
   }
 #else
@@ -962,7 +1112,7 @@ static FORCE_INLINE bool tdSTSRowIterNext(STSRowIter *pIter, col_id_t colId, col
 STSRow *mergeTwoRows(void *buffer, STSRow *row1, STSRow *row2, STSchema *pSchema1, STSchema *pSchema2);
 
 // Get the data pointer from a column-wised data
-static FORCE_INLINE int32_t tdGetColDataOfRow(SCellVal *pVal, SDataCol *pCol, int32_t row) {
+static FORCE_INLINE int32_t tdGetColDataOfRow(SCellVal *pVal, SDataCol *pCol, int32_t row, int8_t bitmapMode) {
   if (isAllRowsNone(pCol)) {
     pVal->valType = TD_VTYPE_NULL;
 #ifdef TD_SUPPORT_READ2
@@ -975,7 +1125,7 @@ static FORCE_INLINE int32_t tdGetColDataOfRow(SCellVal *pVal, SDataCol *pCol, in
 
   if (TD_COL_ROWS_NORM(pCol)) {
     pVal->valType = TD_VTYPE_NORM;
-  } else if (tdGetBitmapValType(pCol->pBitmap, row, &(pVal->valType)) < 0) {
+  } else if (tdGetBitmapValType(pCol->pBitmap, row, &(pVal->valType), bitmapMode) < 0) {
     return terrno;
   }
 
@@ -1018,14 +1168,14 @@ static FORCE_INLINE bool tdSKvRowGetVal(STSRow *pRow, col_id_t colId, uint32_t o
   return true;
 }
 
-static FORCE_INLINE int32_t dataColGetNEleLen(SDataCol *pDataCol, int32_t rows) {
+static FORCE_INLINE int32_t dataColGetNEleLen(SDataCol *pDataCol, int32_t rows, int8_t bitmapMode) {
   ASSERT(rows > 0);
   int32_t result = 0;
 
   if (IS_VAR_DATA_TYPE(pDataCol->type)) {
     result += pDataCol->dataOff[rows - 1];
     SCellVal val = {0};
-    if (tdGetColDataOfRow(&val, pDataCol, rows - 1) < 0) {
+    if (tdGetColDataOfRow(&val, pDataCol, rows - 1, bitmapMode) < 0) {
       TASSERT(0);
     }
 

include/common/ttypes.h

@@ -27,6 +27,7 @@ extern "C" {
 typedef int32_t  VarDataOffsetT;
 typedef uint32_t TDRowLenT;
 typedef uint8_t  TDRowValT;
+typedef uint64_t TDRowVerT;
 typedef int16_t  col_id_t;
 typedef int8_t   col_type_t;
 typedef int32_t  col_bytes_t;

include/libs/command/command.h

@@ -22,7 +22,7 @@ typedef struct SExplainCtx SExplainCtx;
 int32_t qExecCommand(SNode* pStmt, SRetrieveTableRsp** pRsp);
 
 int32_t qExecStaticExplain(SQueryPlan *pDag, SRetrieveTableRsp **pRsp);
-int32_t qExecExplainBegin(SQueryPlan *pDag, SExplainCtx **pCtx, int32_t startTs);
+int32_t qExecExplainBegin(SQueryPlan *pDag, SExplainCtx **pCtx, int64_t startTs);
 int32_t qExecExplainEnd(SExplainCtx *pCtx, SRetrieveTableRsp **pRsp);
 int32_t qExplainUpdateExecInfo(SExplainCtx        *pCtx, SExplainRsp *pRspMsg, int32_t groupId, SRetrieveTableRsp **pRsp);
 void    qExplainFreeCtx(SExplainCtx *pCtx);

include/libs/function/functionMgt.h

@@ -135,8 +135,17 @@ bool fmIsTimeorderFunc(int32_t funcId);
 bool fmIsPseudoColumnFunc(int32_t funcId);
 bool fmIsWindowPseudoColumnFunc(int32_t funcId);
 bool fmIsWindowClauseFunc(int32_t funcId);
+bool fmIsSpecialDataRequiredFunc(int32_t funcId);
+bool fmIsDynamicScanOptimizedFunc(int32_t funcId);
 
-int32_t fmFuncScanType(int32_t funcId);
+typedef enum EFuncDataRequired {
+  FUNC_DATA_REQUIRED_ALL_NEEDED = 1,
+  FUNC_DATA_REQUIRED_STATIS_NEEDED,
+  FUNC_DATA_REQUIRED_NO_NEEDED,
+  FUNC_DATA_REQUIRED_DISCARD
+} EFuncDataRequired;
+
+EFuncDataRequired fmFuncDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWindow);
 
 int32_t fmGetFuncExecFuncs(int32_t funcId, SFuncExecFuncs* pFpSet);
 int32_t fmGetScalarFuncExecFuncs(int32_t funcId, SScalarFuncExecFuncs* pFpSet);

include/libs/nodes/nodes.h

@@ -216,6 +216,7 @@ SNodeList* nodesMakeList();
 int32_t nodesListAppend(SNodeList* pList, SNodeptr pNode);
 int32_t nodesListStrictAppend(SNodeList* pList, SNodeptr pNode);
 int32_t nodesListMakeAppend(SNodeList** pList, SNodeptr pNode);
+int32_t nodesListMakeStrictAppend(SNodeList** pList, SNodeptr pNode);
 int32_t nodesListAppendList(SNodeList* pTarget, SNodeList* pSrc);
 int32_t nodesListStrictAppendList(SNodeList* pTarget, SNodeList* pSrc);
 int32_t nodesListPushFront(SNodeList* pList, SNodeptr pNode);

include/libs/nodes/plannodes.h

@@ -30,6 +30,7 @@ typedef struct SLogicNode {
   SNode* pConditions;
   SNodeList* pChildren;
   struct SLogicNode* pParent;
+  int32_t optimizedFlag;
 } SLogicNode;
 
 typedef enum EScanType {
@@ -50,6 +51,8 @@ typedef struct SScanLogicNode {
   SName tableName;
   bool showRewrite;
   double ratio;
+  SNodeList* pDynamicScanFuncs;
+  int32_t dataRequired;
 } SScanLogicNode;
 
 typedef struct SJoinLogicNode {
@@ -84,6 +87,7 @@ typedef struct SVnodeModifLogicNode {
 typedef struct SExchangeLogicNode {
   SLogicNode node;
   int32_t srcGroupId;
+  uint8_t precision;
 } SExchangeLogicNode;
 
 typedef enum EWindowType {
@@ -163,7 +167,7 @@ typedef struct SDataBlockDescNode {
   SNodeList* pSlots;
   int32_t totalRowSize;
   int32_t outputRowSize;
-  int16_t precision;
+  uint8_t precision;
 } SDataBlockDescNode;
 
 typedef struct SPhysiNode {
@@ -195,20 +199,13 @@ typedef struct SSystemTableScanPhysiNode {
   int32_t accountId;
 } SSystemTableScanPhysiNode;
 
-typedef enum EScanRequired {
-  SCAN_REQUIRED_DATA_NO_NEEDED = 1,
-  SCAN_REQUIRED_DATA_STATIS_NEEDED,
-  SCAN_REQUIRED_DATA_ALL_NEEDED,
-  SCAN_REQUIRED_DATA_DISCARD,
-} EScanRequired;
-
 typedef struct STableScanPhysiNode {
   SScanPhysiNode scan;
   uint8_t scanFlag;         // denotes reversed scan of data or not
   STimeWindow scanRange;
   double ratio;
-  EScanRequired scanRequired;
-  SNodeList* pScanReferFuncs;
+  int32_t dataRequired;
+  SNodeList* pDynamicScanFuncs;
 } STableScanPhysiNode;
 
 typedef STableScanPhysiNode STableSeqScanPhysiNode;
@@ -253,11 +250,11 @@ typedef struct SWinodwPhysiNode {
   SPhysiNode node;
   SNodeList* pExprs;   // these are expression list of parameter expression of function
   SNodeList* pFuncs;
+  SNode* pTspk; // timestamp primary key
 } SWinodwPhysiNode;
 
 typedef struct SIntervalPhysiNode {
   SWinodwPhysiNode window;
-  SNode* pTspk; // timestamp primary key
   int64_t    interval;
   int64_t    offset;
   int64_t    sliding;
@@ -274,7 +271,7 @@ typedef struct SMultiTableIntervalPhysiNode {
 
 typedef struct SSessionWinodwPhysiNode {
   SWinodwPhysiNode window;
-  int64_t    gap;
+  int64_t gap;
 } SSessionWinodwPhysiNode;
 
 typedef struct SStateWinodwPhysiNode {

include/libs/nodes/querynodes.h

@@ -190,6 +190,7 @@ typedef struct SLimitNode {
 
 typedef struct SStateWindowNode {
   ENodeType type; // QUERY_NODE_STATE_WINDOW
+  SNode* pCol; // timestamp primary key
   SNode* pExpr;
 } SStateWindowNode;
 

include/os/osFile.h

@@ -44,7 +44,7 @@ extern "C" {
 
 typedef struct TdFile *TdFilePtr;
  
-#define TD_FILE_CTEATE    0x0001
+#define TD_FILE_CREATE    0x0001
 #define TD_FILE_WRITE     0x0002
 #define TD_FILE_READ      0x0004
 #define TD_FILE_TRUNC     0x0008

include/util/taoserror.h

@@ -195,6 +195,9 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_MND_SNODE_NOT_EXIST           TAOS_DEF_ERROR_CODE(0, 0x0355)
 #define TSDB_CODE_MND_BNODE_ALREADY_EXIST       TAOS_DEF_ERROR_CODE(0, 0x0356)
 #define TSDB_CODE_MND_BNODE_NOT_EXIST           TAOS_DEF_ERROR_CODE(0, 0x0357)
+#define TSDB_CODE_MND_TOO_FEW_MNODES            TAOS_DEF_ERROR_CODE(0, 0x0358)
+#define TSDB_CODE_MND_MNODE_DEPLOYED            TAOS_DEF_ERROR_CODE(0, 0x0359)
+#define TSDB_CODE_MND_CANT_DROP_MASTER          TAOS_DEF_ERROR_CODE(0, 0x035A)
 
 // mnode-acct
 #define TSDB_CODE_MND_ACCT_ALREADY_EXIST        TAOS_DEF_ERROR_CODE(0, 0x0360)
@@ -283,14 +286,15 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_MND_INVALID_STREAM_OPTION     TAOS_DEF_ERROR_CODE(0, 0x03F2)
 
 // mnode-sma
-#define TSDB_CODE_MND_SMA_ALREADY_EXIST         TAOS_DEF_ERROR_CODE(0, 0x0400)
-#define TSDB_CODE_MND_SMA_NOT_EXIST             TAOS_DEF_ERROR_CODE(0, 0x0401)
-#define TSDB_CODE_MND_INVALID_SMA_OPTION        TAOS_DEF_ERROR_CODE(0, 0x0402)
+#define TSDB_CODE_MND_SMA_ALREADY_EXIST         TAOS_DEF_ERROR_CODE(0, 0x0480)
+#define TSDB_CODE_MND_SMA_NOT_EXIST             TAOS_DEF_ERROR_CODE(0, 0x0481)
+#define TSDB_CODE_MND_INVALID_SMA_OPTION        TAOS_DEF_ERROR_CODE(0, 0x0482)
 
 // dnode
-#define TSDB_CODE_NODE_OFFLINE                  TAOS_DEF_ERROR_CODE(0, 0x04A1)
-#define TSDB_CODE_NODE_ALREADY_DEPLOYED         TAOS_DEF_ERROR_CODE(0, 0x04A3)
-#define TSDB_CODE_NODE_NOT_DEPLOYED             TAOS_DEF_ERROR_CODE(0, 0x04A4)
+#define TSDB_CODE_NODE_REDIRECT                 TAOS_DEF_ERROR_CODE(0, 0x0400)
+#define TSDB_CODE_NODE_OFFLINE                  TAOS_DEF_ERROR_CODE(0, 0x0401)
+#define TSDB_CODE_NODE_ALREADY_DEPLOYED         TAOS_DEF_ERROR_CODE(0, 0x0403)
+#define TSDB_CODE_NODE_NOT_DEPLOYED             TAOS_DEF_ERROR_CODE(0, 0x0404)
 
 // vnode
 #define TSDB_CODE_VND_ACTION_IN_PROGRESS        TAOS_DEF_ERROR_CODE(0, 0x0500)
@@ -424,6 +428,108 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_WAL_INVALID_VER               TAOS_DEF_ERROR_CODE(0, 0x1003)
 #define TSDB_CODE_WAL_OUT_OF_MEMORY             TAOS_DEF_ERROR_CODE(0, 0x1004)
 
+// http
+#define TSDB_CODE_HTTP_SERVER_OFFLINE           TAOS_DEF_ERROR_CODE(0, 0x1100)  //"http server is not online"
+#define TSDB_CODE_HTTP_UNSUPPORT_URL            TAOS_DEF_ERROR_CODE(0, 0x1101)  //"url is not support"
+#define TSDB_CODE_HTTP_INVALID_URL              TAOS_DEF_ERROR_CODE(0, 0x1102)  //invalid url format"
+#define TSDB_CODE_HTTP_NO_ENOUGH_MEMORY         TAOS_DEF_ERROR_CODE(0, 0x1103)  //"no enough memory"
+#define TSDB_CODE_HTTP_REQUSET_TOO_BIG          TAOS_DEF_ERROR_CODE(0, 0x1104)  //"request size is too big"
+#define TSDB_CODE_HTTP_NO_AUTH_INFO             TAOS_DEF_ERROR_CODE(0, 0x1105)  //"no auth info input"
+#define TSDB_CODE_HTTP_NO_MSG_INPUT             TAOS_DEF_ERROR_CODE(0, 0x1106)  //"request is empty"
+#define TSDB_CODE_HTTP_NO_SQL_INPUT             TAOS_DEF_ERROR_CODE(0, 0x1107)  //"no sql input"
+#define TSDB_CODE_HTTP_NO_EXEC_USEDB            TAOS_DEF_ERROR_CODE(0, 0x1108)  //"no need to execute use db cmd"
+#define TSDB_CODE_HTTP_SESSION_FULL             TAOS_DEF_ERROR_CODE(0, 0x1109)  //"session list was full"
+#define TSDB_CODE_HTTP_GEN_TAOSD_TOKEN_ERR      TAOS_DEF_ERROR_CODE(0, 0x110A)  //"generate taosd token error"
+#define TSDB_CODE_HTTP_INVALID_MULTI_REQUEST    TAOS_DEF_ERROR_CODE(0, 0x110B)  //"size of multi request is 0"
+#define TSDB_CODE_HTTP_CREATE_GZIP_FAILED       TAOS_DEF_ERROR_CODE(0, 0x110C)  //"failed to create gzip"
+#define TSDB_CODE_HTTP_FINISH_GZIP_FAILED       TAOS_DEF_ERROR_CODE(0, 0x110D)  //"failed to finish gzip"
+#define TSDB_CODE_HTTP_LOGIN_FAILED             TAOS_DEF_ERROR_CODE(0, 0x110E)  //"failed to login"
+
+#define TSDB_CODE_HTTP_INVALID_VERSION          TAOS_DEF_ERROR_CODE(0, 0x1120)  //"invalid http version"
+#define TSDB_CODE_HTTP_INVALID_CONTENT_LENGTH   TAOS_DEF_ERROR_CODE(0, 0x1121)  //"invalid content length"
+#define TSDB_CODE_HTTP_INVALID_AUTH_TYPE        TAOS_DEF_ERROR_CODE(0, 0x1122)  //"invalid type of Authorization"
+#define TSDB_CODE_HTTP_INVALID_AUTH_FORMAT      TAOS_DEF_ERROR_CODE(0, 0x1123)  //"invalid format of Authorization"
+#define TSDB_CODE_HTTP_INVALID_BASIC_AUTH       TAOS_DEF_ERROR_CODE(0, 0x1124)  //"invalid basic Authorization"
+#define TSDB_CODE_HTTP_INVALID_TAOSD_AUTH       TAOS_DEF_ERROR_CODE(0, 0x1125)  //"invalid taosd Authorization"
+#define TSDB_CODE_HTTP_PARSE_METHOD_FAILED      TAOS_DEF_ERROR_CODE(0, 0x1126)  //"failed to parse method"
+#define TSDB_CODE_HTTP_PARSE_TARGET_FAILED      TAOS_DEF_ERROR_CODE(0, 0x1127)  //"failed to parse target"
+#define TSDB_CODE_HTTP_PARSE_VERSION_FAILED     TAOS_DEF_ERROR_CODE(0, 0x1128)  //"failed to parse http version"
+#define TSDB_CODE_HTTP_PARSE_SP_FAILED          TAOS_DEF_ERROR_CODE(0, 0x1129)  //"failed to parse sp"
+#define TSDB_CODE_HTTP_PARSE_STATUS_FAILED      TAOS_DEF_ERROR_CODE(0, 0x112A)  //"failed to parse status"
+#define TSDB_CODE_HTTP_PARSE_PHRASE_FAILED      TAOS_DEF_ERROR_CODE(0, 0x112B)  //"failed to parse phrase"
+#define TSDB_CODE_HTTP_PARSE_CRLF_FAILED        TAOS_DEF_ERROR_CODE(0, 0x112C)  //"failed to parse crlf"
+#define TSDB_CODE_HTTP_PARSE_HEADER_FAILED      TAOS_DEF_ERROR_CODE(0, 0x112D)  //"failed to parse header"
+#define TSDB_CODE_HTTP_PARSE_HEADER_KEY_FAILED  TAOS_DEF_ERROR_CODE(0, 0x112E)  //"failed to parse header key"
+#define TSDB_CODE_HTTP_PARSE_HEADER_VAL_FAILED  TAOS_DEF_ERROR_CODE(0, 0x112F)  //"failed to parse header val"
+#define TSDB_CODE_HTTP_PARSE_CHUNK_SIZE_FAILED  TAOS_DEF_ERROR_CODE(0, 0x1130)  //"failed to parse chunk size"
+#define TSDB_CODE_HTTP_PARSE_CHUNK_FAILED       TAOS_DEF_ERROR_CODE(0, 0x1131)  //"failed to parse chunk"
+#define TSDB_CODE_HTTP_PARSE_END_FAILED         TAOS_DEF_ERROR_CODE(0, 0x1132)  //"failed to parse end section"
+#define TSDB_CODE_HTTP_PARSE_INVALID_STATE      TAOS_DEF_ERROR_CODE(0, 0x1134)  //"invalid parse state"
+#define TSDB_CODE_HTTP_PARSE_ERROR_STATE        TAOS_DEF_ERROR_CODE(0, 0x1135)  //"failed to parse error section"
+
+#define TSDB_CODE_HTTP_GC_QUERY_NULL            TAOS_DEF_ERROR_CODE(0, 0x1150)  //"query size is 0"
+#define TSDB_CODE_HTTP_GC_QUERY_SIZE            TAOS_DEF_ERROR_CODE(0, 0x1151)  //"query size can not more than 100"
+#define TSDB_CODE_HTTP_GC_REQ_PARSE_ERROR       TAOS_DEF_ERROR_CODE(0, 0x1152)  //"parse grafana json error"
+
+#define TSDB_CODE_HTTP_TG_DB_NOT_INPUT          TAOS_DEF_ERROR_CODE(0, 0x1160)  //"database name can not be null"
+#define TSDB_CODE_HTTP_TG_DB_TOO_LONG           TAOS_DEF_ERROR_CODE(0, 0x1161)  //"database name too long"
+#define TSDB_CODE_HTTP_TG_INVALID_JSON          TAOS_DEF_ERROR_CODE(0, 0x1162)  //"invalid telegraf json fromat"
+#define TSDB_CODE_HTTP_TG_METRICS_NULL          TAOS_DEF_ERROR_CODE(0, 0x1163)  //"metrics size is 0"
+#define TSDB_CODE_HTTP_TG_METRICS_SIZE          TAOS_DEF_ERROR_CODE(0, 0x1164)  //"metrics size can not more than 1K"
+#define TSDB_CODE_HTTP_TG_METRIC_NULL           TAOS_DEF_ERROR_CODE(0, 0x1165)  //"metric name not find"
+#define TSDB_CODE_HTTP_TG_METRIC_TYPE           TAOS_DEF_ERROR_CODE(0, 0x1166)  //"metric name type should be string"
+#define TSDB_CODE_HTTP_TG_METRIC_NAME_NULL      TAOS_DEF_ERROR_CODE(0, 0x1167)  //"metric name length is 0"
+#define TSDB_CODE_HTTP_TG_METRIC_NAME_LONG      TAOS_DEF_ERROR_CODE(0, 0x1168)  //"metric name length too long"
+#define TSDB_CODE_HTTP_TG_TIMESTAMP_NULL        TAOS_DEF_ERROR_CODE(0, 0x1169)  //"timestamp not find"
+#define TSDB_CODE_HTTP_TG_TIMESTAMP_TYPE        TAOS_DEF_ERROR_CODE(0, 0x116A)  //"timestamp type should be integer"
+#define TSDB_CODE_HTTP_TG_TIMESTAMP_VAL_NULL    TAOS_DEF_ERROR_CODE(0, 0x116B)  //"timestamp value smaller than 0"
+#define TSDB_CODE_HTTP_TG_TAGS_NULL             TAOS_DEF_ERROR_CODE(0, 0x116C)  //"tags not find"
+#define TSDB_CODE_HTTP_TG_TAGS_SIZE_0           TAOS_DEF_ERROR_CODE(0, 0x116D)  //"tags size is 0"
+#define TSDB_CODE_HTTP_TG_TAGS_SIZE_LONG        TAOS_DEF_ERROR_CODE(0, 0x116E)  //"tags size too long"
+#define TSDB_CODE_HTTP_TG_TAG_NULL              TAOS_DEF_ERROR_CODE(0, 0x116F)  //"tag is null"
+#define TSDB_CODE_HTTP_TG_TAG_NAME_NULL         TAOS_DEF_ERROR_CODE(0, 0x1170)  //"tag name is null"
+#define TSDB_CODE_HTTP_TG_TAG_NAME_SIZE         TAOS_DEF_ERROR_CODE(0, 0x1171)  //"tag name length too long"
+#define TSDB_CODE_HTTP_TG_TAG_VALUE_TYPE        TAOS_DEF_ERROR_CODE(0, 0x1172)  //"tag value type should be number or string"
+#define TSDB_CODE_HTTP_TG_TAG_VALUE_NULL        TAOS_DEF_ERROR_CODE(0, 0x1173)  //"tag value is null"
+#define TSDB_CODE_HTTP_TG_TABLE_NULL            TAOS_DEF_ERROR_CODE(0, 0x1174)  //"table is null"
+#define TSDB_CODE_HTTP_TG_TABLE_SIZE            TAOS_DEF_ERROR_CODE(0, 0x1175)  //"table name length too long"
+#define TSDB_CODE_HTTP_TG_FIELDS_NULL           TAOS_DEF_ERROR_CODE(0, 0x1176)  //"fields not find"
+#define TSDB_CODE_HTTP_TG_FIELDS_SIZE_0         TAOS_DEF_ERROR_CODE(0, 0x1177)  //"fields size is 0"
+#define TSDB_CODE_HTTP_TG_FIELDS_SIZE_LONG      TAOS_DEF_ERROR_CODE(0, 0x1178)  //"fields size too long"
+#define TSDB_CODE_HTTP_TG_FIELD_NULL            TAOS_DEF_ERROR_CODE(0, 0x1179)  //"field is null"
+#define TSDB_CODE_HTTP_TG_FIELD_NAME_NULL       TAOS_DEF_ERROR_CODE(0, 0x117A)  //"field name is null"
+#define TSDB_CODE_HTTP_TG_FIELD_NAME_SIZE       TAOS_DEF_ERROR_CODE(0, 0x117B)  //"field name length too long"
+#define TSDB_CODE_HTTP_TG_FIELD_VALUE_TYPE      TAOS_DEF_ERROR_CODE(0, 0x117C)  //"field value type should be number or string"
+#define TSDB_CODE_HTTP_TG_FIELD_VALUE_NULL      TAOS_DEF_ERROR_CODE(0, 0x117D)  //"field value is null"
+#define TSDB_CODE_HTTP_TG_HOST_NOT_STRING       TAOS_DEF_ERROR_CODE(0, 0x117E)  //"host type should be string"
+#define TSDB_CODE_HTTP_TG_STABLE_NOT_EXIST      TAOS_DEF_ERROR_CODE(0, 0x117F)  //"stable not exist"
+
+#define TSDB_CODE_HTTP_OP_DB_NOT_INPUT          TAOS_DEF_ERROR_CODE(0, 0x1190)  //"database name can not be null"
+#define TSDB_CODE_HTTP_OP_DB_TOO_LONG           TAOS_DEF_ERROR_CODE(0, 0x1191)  //"database name too long"
+#define TSDB_CODE_HTTP_OP_INVALID_JSON          TAOS_DEF_ERROR_CODE(0, 0x1192)  //"invalid opentsdb json fromat"
+#define TSDB_CODE_HTTP_OP_METRICS_NULL          TAOS_DEF_ERROR_CODE(0, 0x1193)  //"metrics size is 0"
+#define TSDB_CODE_HTTP_OP_METRICS_SIZE          TAOS_DEF_ERROR_CODE(0, 0x1194)  //"metrics size can not more than 10K"
+#define TSDB_CODE_HTTP_OP_METRIC_NULL           TAOS_DEF_ERROR_CODE(0, 0x1195)  //"metric name not find"
+#define TSDB_CODE_HTTP_OP_METRIC_TYPE           TAOS_DEF_ERROR_CODE(0, 0x1196)  //"metric name type should be string"
+#define TSDB_CODE_HTTP_OP_METRIC_NAME_NULL      TAOS_DEF_ERROR_CODE(0, 0x1197)  //"metric name length is 0"
+#define TSDB_CODE_HTTP_OP_METRIC_NAME_LONG      TAOS_DEF_ERROR_CODE(0, 0x1198)  //"metric name length can not more than 22"
+#define TSDB_CODE_HTTP_OP_TIMESTAMP_NULL        TAOS_DEF_ERROR_CODE(0, 0x1199)  //"timestamp not find"
+#define TSDB_CODE_HTTP_OP_TIMESTAMP_TYPE        TAOS_DEF_ERROR_CODE(0, 0x119A)  //"timestamp type should be integer"
+#define TSDB_CODE_HTTP_OP_TIMESTAMP_VAL_NULL    TAOS_DEF_ERROR_CODE(0, 0x119B)  //"timestamp value smaller than 0"
+#define TSDB_CODE_HTTP_OP_TAGS_NULL             TAOS_DEF_ERROR_CODE(0, 0x119C)  //"tags not find"
+#define TSDB_CODE_HTTP_OP_TAGS_SIZE_0           TAOS_DEF_ERROR_CODE(0, 0x119D)  //"tags size is 0"
+#define TSDB_CODE_HTTP_OP_TAGS_SIZE_LONG        TAOS_DEF_ERROR_CODE(0, 0x119E)  //"tags size too long"
+#define TSDB_CODE_HTTP_OP_TAG_NULL              TAOS_DEF_ERROR_CODE(0, 0x119F)  //"tag is null"
+#define TSDB_CODE_HTTP_OP_TAG_NAME_NULL         TAOS_DEF_ERROR_CODE(0, 0x11A0)  //"tag name is null"
+#define TSDB_CODE_HTTP_OP_TAG_NAME_SIZE         TAOS_DEF_ERROR_CODE(0, 0x11A1)  //"tag name length too long"
+#define TSDB_CODE_HTTP_OP_TAG_VALUE_TYPE        TAOS_DEF_ERROR_CODE(0, 0x11A2)  //"tag value type should be boolean number or string"
+#define TSDB_CODE_HTTP_OP_TAG_VALUE_NULL        TAOS_DEF_ERROR_CODE(0, 0x11A3)  //"tag value is null"
+#define TSDB_CODE_HTTP_OP_TAG_VALUE_TOO_LONG    TAOS_DEF_ERROR_CODE(0, 0x11A4)  //"tag value can not more than 64"
+#define TSDB_CODE_HTTP_OP_VALUE_NULL            TAOS_DEF_ERROR_CODE(0, 0x11A5)  //"value not find"
+#define TSDB_CODE_HTTP_OP_VALUE_TYPE            TAOS_DEF_ERROR_CODE(0, 0x11A6)  //"value type should be boolean number or string"
+
+#define TSDB_CODE_HTTP_REQUEST_JSON_ERROR       TAOS_DEF_ERROR_CODE(0, 0x1F00)  //"http request json error"
+
 // tfs
 #define TSDB_CODE_FS_APP_ERROR                  TAOS_DEF_ERROR_CODE(0, 0x2200)
 #define TSDB_CODE_FS_INVLD_CFG                  TAOS_DEF_ERROR_CODE(0, 0x2201)

source/client/CMakeLists.txt

@@ -2,7 +2,7 @@ aux_source_directory(src CLIENT_SRC)
 add_library(taos ${CLIENT_SRC})
 target_include_directories(
     taos
-    PUBLIC "${CMAKE_SOURCE_DIR}/include/client"
+    PUBLIC "${TD_SOURCE_DIR}/include/client"
     PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
 )
 target_link_libraries(

source/client/inc/clientInt.h

@@ -84,10 +84,10 @@ typedef struct {
 } SClientHbMgr;
 
 typedef struct SQueryExecMetric {
-  int64_t start;   // start timestamp
-  int64_t parsed;  // start to parse
-  int64_t send;    // start to send to server
-  int64_t rsp;     // receive response from server
+  int64_t start;   // start timestamp, us
+  int64_t parsed;  // start to parse, us
+  int64_t send;    // start to send to server, us
+  int64_t rsp;     // receive response from server, us
 } SQueryExecMetric;
 
 typedef struct SInstanceSummary {

source/client/src/clientEnv.c

@@ -65,10 +65,10 @@ static void deregisterRequest(SRequestObj *pRequest) {
   int32_t currentInst = atomic_sub_fetch_64(&pActivity->currentRequests, 1);
   int32_t num = atomic_sub_fetch_32(&pTscObj->numOfReqs, 1);
 
-  int64_t duration = taosGetTimestampMs() - pRequest->metric.start;
+  int64_t duration = taosGetTimestampUs() - pRequest->metric.start;
   tscDebug("0x%" PRIx64 " free Request from connObj: 0x%" PRIx64 ", reqId:0x%" PRIx64 " elapsed:%" PRIu64
            " ms, current:%d, app current:%d",
-           pRequest->self, pTscObj->id, pRequest->requestId, duration, num, currentInst);
+           pRequest->self, pTscObj->id, pRequest->requestId, duration/1000, num, currentInst);
   taosReleaseRef(clientConnRefPool, pTscObj->id);
 }
 
@@ -151,7 +151,7 @@ void *createRequest(STscObj *pObj, __taos_async_fn_t fp, void *param, int32_t ty
 
   pRequest->pDb = getDbOfConnection(pObj);
   pRequest->requestId = generateRequestId();
-  pRequest->metric.start = taosGetTimestampMs();
+  pRequest->metric.start = taosGetTimestampUs();
 
   pRequest->type = type;
   pRequest->pTscObj = pObj;

source/client/src/clientHb.c

@@ -435,11 +435,11 @@ static int32_t hbCreateThread() {
   taosThreadAttrInit(&thAttr);
   taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE);
 
-//  if (taosThreadCreate(&clientHbMgr.thread, &thAttr, hbThreadFunc, NULL) != 0) {
-//    terrno = TAOS_SYSTEM_ERROR(errno);
-//    return -1;
-//  }
-//  taosThreadAttrDestroy(&thAttr);
+  if (taosThreadCreate(&clientHbMgr.thread, &thAttr, hbThreadFunc, NULL) != 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+ }
+  taosThreadAttrDestroy(&thAttr);
   return 0;
 }
 
@@ -500,8 +500,6 @@ SAppHbMgr *appHbMgrInit(SAppInstInfo *pAppInstInfo, char *key) {
 }
 
 void appHbMgrCleanup(void) {
-  taosThreadMutexLock(&clientHbMgr.lock);
-
   int sz = taosArrayGetSize(clientHbMgr.appHbMgrs);
   for (int i = 0; i < sz; i++) {
     SAppHbMgr *pTarget = taosArrayGetP(clientHbMgr.appHbMgrs, i);
@@ -510,8 +508,6 @@ void appHbMgrCleanup(void) {
     taosHashCleanup(pTarget->connInfo);
     pTarget->connInfo = NULL;
   }
-
-  taosThreadMutexUnlock(&clientHbMgr.lock);
 }
 
 int hbMgrInit() {
@@ -532,7 +528,6 @@ int hbMgrInit() {
 }
 
 void hbMgrCleanUp() {
-#if 0
   hbStopThread();
 
   // destroy all appHbMgr
@@ -545,7 +540,6 @@ void hbMgrCleanUp() {
   taosThreadMutexUnlock(&clientHbMgr.lock);
 
   clientHbMgr.appHbMgrs = NULL;
-#endif
 }
 
 int hbRegisterConnImpl(SAppHbMgr *pAppHbMgr, SClientHbKey connKey, SHbConnInfo *info) {

source/client/src/clientImpl.c

@@ -520,7 +520,7 @@ void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) {
     SRequestObj* pRequest = (SRequestObj*)taosAcquireRef(clientReqRefPool, pSendInfo->requestObjRefId);
     assert(pRequest->self == pSendInfo->requestObjRefId);
 
-    pRequest->metric.rsp = taosGetTimestampMs();
+    pRequest->metric.rsp = taosGetTimestampUs();
 
     STscObj* pTscObj = pRequest->pTscObj;
     if (pEpSet) {
@@ -536,10 +536,10 @@ void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) {
     int32_t elapsed = pRequest->metric.rsp - pRequest->metric.start;
     if (pMsg->code == TSDB_CODE_SUCCESS) {
       tscDebug("0x%" PRIx64 " message:%s, code:%s rspLen:%d, elapsed:%d ms, reqId:0x%" PRIx64, pRequest->self,
-               TMSG_INFO(pMsg->msgType), tstrerror(pMsg->code), pMsg->contLen, elapsed, pRequest->requestId);
+               TMSG_INFO(pMsg->msgType), tstrerror(pMsg->code), pMsg->contLen, elapsed/1000, pRequest->requestId);
     } else {
       tscError("0x%" PRIx64 " SQL cmd:%s, code:%s rspLen:%d, elapsed time:%d ms, reqId:0x%" PRIx64, pRequest->self,
-               TMSG_INFO(pMsg->msgType), tstrerror(pMsg->code), pMsg->contLen, elapsed, pRequest->requestId);
+               TMSG_INFO(pMsg->msgType), tstrerror(pMsg->code), pMsg->contLen, elapsed/1000, pRequest->requestId);
     }
 
     taosReleaseRef(clientReqRefPool, pSendInfo->requestObjRefId);

source/client/test/CMakeLists.txt

@@ -19,12 +19,12 @@ TARGET_LINK_LIBRARIES(
 
 TARGET_INCLUDE_DIRECTORIES(
         clientTest
-        PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/client/"
-        PRIVATE "${CMAKE_SOURCE_DIR}/source/libs/client/inc"
+        PUBLIC "${TD_SOURCE_DIR}/include/libs/client/"
+        PRIVATE "${TD_SOURCE_DIR}/source/libs/client/inc"
 )
 
 TARGET_INCLUDE_DIRECTORIES(
         tmqTest
-        PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/client/"
-        PRIVATE "${CMAKE_SOURCE_DIR}/source/libs/client/inc"
+        PUBLIC "${TD_SOURCE_DIR}/include/libs/client/"
+        PRIVATE "${TD_SOURCE_DIR}/source/libs/client/inc"
 )

source/common/CMakeLists.txt

@@ -2,11 +2,11 @@ aux_source_directory(src COMMON_SRC)
 add_library(common STATIC ${COMMON_SRC})
 target_include_directories(
     common
-    PUBLIC "${CMAKE_SOURCE_DIR}/include/common"
+    PUBLIC "${TD_SOURCE_DIR}/include/common"
     PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
 IF(${TD_WINDOWS})
-    PRIVATE "${CMAKE_SOURCE_DIR}/contrib/pthread-win32"
-    PRIVATE "${CMAKE_SOURCE_DIR}/contrib/gnuregex"
+    PRIVATE "${TD_SOURCE_DIR}/contrib/pthread-win32"
+    PRIVATE "${TD_SOURCE_DIR}/contrib/gnuregex"
 ENDIF ()
 )
 target_link_libraries(

source/common/src/tdataformat.c

@@ -266,6 +266,8 @@ void dataColInit(SDataCol *pDataCol, STColumn *pCol, int maxPoints) {
 
   pDataCol->len = 0;
 }
+
+#if 0
 // value from timestamp should be TKEY here instead of TSKEY
 int dataColAppendVal(SDataCol *pCol, const void *value, int numOfRows, int maxPoints) {
   ASSERT(pCol != NULL && value != NULL);
@@ -297,7 +299,7 @@ int dataColAppendVal(SDataCol *pCol, const void *value, int numOfRows, int maxPo
   }
   return 0;
 }
-
+#endif
 static FORCE_INLINE const void *tdGetColDataOfRowUnsafe(SDataCol *pCol, int row) {
   if (IS_VAR_DATA_TYPE(pCol->type)) {
     return POINTER_SHIFT(pCol->pData, pCol->dataOff[row]);
@@ -314,6 +316,7 @@ bool isNEleNull(SDataCol *pCol, int nEle) {
   return true;
 }
 
+#if 0
 static FORCE_INLINE void dataColSetNullAt(SDataCol *pCol, int index) {
   if (IS_VAR_DATA_TYPE(pCol->type)) {
     pCol->dataOff[index] = pCol->len;
@@ -326,7 +329,7 @@ static FORCE_INLINE void dataColSetNullAt(SDataCol *pCol, int index) {
   }
 }
 
-static void dataColSetNEleNull(SDataCol *pCol, int nEle) {
+static void dataColSetNEleNull(SDataCol *pCol, int nEle, int8_t bitmapMode) {
   if (IS_VAR_DATA_TYPE(pCol->type)) {
     pCol->len = 0;
     for (int i = 0; i < nEle; ++i) {
@@ -337,7 +340,7 @@ static void dataColSetNEleNull(SDataCol *pCol, int nEle) {
     pCol->len = TYPE_BYTES[pCol->type] * nEle;
   }
 }
-
+#endif
 void *dataColSetOffset(SDataCol *pCol, int nEle) {
   ASSERT(((pCol->type == TSDB_DATA_TYPE_BINARY) || (pCol->type == TSDB_DATA_TYPE_NCHAR)));
 
@@ -364,6 +367,7 @@ SDataCols *tdNewDataCols(int maxCols, int maxRows) {
   pCols->maxCols = maxCols;
   pCols->numOfRows = 0;
   pCols->numOfCols = 0;
+  // pCols->bitmapMode = 0; // calloc already set 0
 
   if (maxCols > 0) {
     pCols->cols = (SDataCol *)taosMemoryCalloc(maxCols, sizeof(SDataCol));
@@ -407,6 +411,7 @@ int tdInitDataCols(SDataCols *pCols, STSchema *pSchema) {
 #endif
 
   pCols->numOfRows = 0;
+  pCols->bitmapMode = 0;
   pCols->numOfCols = schemaNCols(pSchema);
 
   for (i = 0; i < schemaNCols(pSchema); ++i) {
@@ -472,6 +477,7 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) {
 void tdResetDataCols(SDataCols *pCols) {
   if (pCols != NULL) {
     pCols->numOfRows = 0;
+    pCols->bitmapMode = 0;
     for (int i = 0; i < pCols->maxCols; ++i) {
       dataColReset(pCols->cols + i);
     }

source/common/src/tglobal.c

@@ -173,7 +173,7 @@ uint32_t tsMaxRange = 500;                      // max range
 uint32_t tsCurRange = 100;                      // range
 char     tsCompressor[32] = "ZSTD_COMPRESSOR";  // ZSTD_COMPRESSOR or GZIP_COMPRESSOR
 
-static void taosAddDataDir(int32_t index, char *v1, int32_t level, int32_t primary) {
+void taosAddDataDir(int32_t index, char *v1, int32_t level, int32_t primary) {
   tstrncpy(tsDiskCfg[index].dir, v1, TSDB_FILENAME_LEN);
   tsDiskCfg[index].level = level;
   tsDiskCfg[index].primary = primary;

source/common/src/tmsg.c

@@ -385,7 +385,7 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) {
       if (pReq->rollup) {
         pReq->stbCfg.pRSmaParam = (SRSmaParam *)taosMemoryMalloc(sizeof(SRSmaParam));
         SRSmaParam *param = pReq->stbCfg.pRSmaParam;
-        buf = taosDecodeBinaryTo(buf, (void*)&param->xFilesFactor, sizeof(param->xFilesFactor));
+        buf = taosDecodeBinaryTo(buf, (void *)&param->xFilesFactor, sizeof(param->xFilesFactor));
         buf = taosDecodeFixedI32(buf, &param->delay);
         buf = taosDecodeFixedI8(buf, &param->nFuncIds);
         if (param->nFuncIds > 0) {
@@ -418,7 +418,7 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) {
       if (pReq->rollup) {
         pReq->ntbCfg.pRSmaParam = (SRSmaParam *)taosMemoryMalloc(sizeof(SRSmaParam));
         SRSmaParam *param = pReq->ntbCfg.pRSmaParam;
-        buf = taosDecodeBinaryTo(buf, (void*)&param->xFilesFactor, sizeof(param->xFilesFactor));
+        buf = taosDecodeBinaryTo(buf, (void *)&param->xFilesFactor, sizeof(param->xFilesFactor));
         buf = taosDecodeFixedI32(buf, &param->delay);
         buf = taosDecodeFixedI8(buf, &param->nFuncIds);
         if (param->nFuncIds > 0) {
@@ -788,6 +788,7 @@ int32_t tSerializeSMDropSmaReq(void *buf, int32_t bufLen, SMDropSmaReq *pReq) {
 
   if (tStartEncode(&encoder) < 0) return -1;
   if (tEncodeCStr(&encoder, pReq->name) < 0) return -1;
+
   if (tEncodeI8(&encoder, pReq->igNotExists) < 0) return -1;
   tEndEncode(&encoder);
 
@@ -808,6 +809,56 @@ int32_t tDeserializeSMDropSmaReq(void *buf, int32_t bufLen, SMDropSmaReq *pReq)
   tCoderClear(&decoder);
   return 0;
 }
+int32_t tSerializeSMCreateFullTextReq(void *buf, int32_t bufLen, SMCreateFullTextReq *pReq) {
+  SCoder encoder = {0};
+  tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER);
+
+  if (tStartEncode(&encoder) < 0) return -1;
+
+  tEndEncode(&encoder);
+  int32_t tlen = encoder.pos;
+  tCoderClear(&encoder);
+  return tlen;
+}
+int32_t tDeserializeSMCreateFullTextReq(void *buf, int32_t bufLen, SMCreateFullTextReq *pReq) {
+  SCoder decoder = {0};
+  tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER);
+  if (tStartDecode(&decoder) < 0) return -1;
+
+  tEndDecode(&decoder);
+  tCoderClear(&decoder);
+  return 0;
+}
+void tFreeSMCreateFullTextReq(SMCreateFullTextReq *pReq) {
+  // impl later
+  return;
+}
+int32_t tSerializeSMDropFullTextReq(void *buf, int32_t bufLen, SMDropFullTextReq *pReq) {
+  SCoder encoder = {0};
+  tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER);
+
+  if (tStartEncode(&encoder) < 0) return -1;
+
+  if (tEncodeCStr(&encoder, pReq->name) < 0) return -1;
+
+  if (tEncodeI8(&encoder, pReq->igNotExists) < 0) return -1;
+
+  tEndEncode(&encoder);
+  int32_t tlen = encoder.pos;
+  tCoderClear(&encoder);
+  return tlen;
+}
+int32_t tDeserializeSMDropFullTextReq(void *buf, int32_t bufLen, SMDropFullTextReq *pReq) {
+  SCoder decoder = {0};
+  tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER);
+  if (tStartDecode(&decoder) < 0) return -1;
+  if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1;
+  if (tDecodeI8(&decoder, &pReq->igNotExists) < 0) return -1;
+
+  tEndDecode(&decoder);
+  tCoderClear(&decoder);
+  return 0;
+}
 
 int32_t tSerializeSStatusReq(void *buf, int32_t bufLen, SStatusReq *pReq) {
   SCoder encoder = {0};
@@ -1909,7 +1960,7 @@ int32_t tDeserializeSUseDbBatchRsp(void *buf, int32_t bufLen, SUseDbBatchRsp *pR
   int32_t numOfBatch = taosArrayGetSize(pRsp->pArray);
   if (tDecodeI32(&decoder, &numOfBatch) < 0) return -1;
 
-  pRsp->pArray = taosArrayInit(numOfBatch, sizeof(SUseDbBatchRsp));
+  pRsp->pArray = taosArrayInit(numOfBatch, sizeof(SUseDbRsp));
   if (pRsp->pArray == NULL) {
     terrno = TSDB_CODE_OUT_OF_MEMORY;
     return -1;
@@ -2912,7 +2963,7 @@ int32_t tDecodeSMqCMCommitOffsetReq(SCoder *decoder, SMqCMCommitOffsetReq *pReq)
   return 0;
 }
 
-int32_t tSerializeSExplainRsp(void* buf, int32_t bufLen, SExplainRsp* pRsp) {
+int32_t tSerializeSExplainRsp(void *buf, int32_t bufLen, SExplainRsp *pRsp) {
   SCoder encoder = {0};
   tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER);
 
@@ -2932,7 +2983,7 @@ int32_t tSerializeSExplainRsp(void* buf, int32_t bufLen, SExplainRsp* pRsp) {
   return tlen;
 }
 
-int32_t tDeserializeSExplainRsp(void* buf, int32_t bufLen, SExplainRsp* pRsp) {
+int32_t tDeserializeSExplainRsp(void *buf, int32_t bufLen, SExplainRsp *pRsp) {
   SCoder decoder = {0};
   tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER);
 

source/common/src/trow.c

@@ -22,11 +22,224 @@ const uint8_t tdVTypeByte[3] = {
     TD_VTYPE_NULL_BYTE,  // TD_VTYPE_NULL
 };
 
+// declaration
+static uint8_t tdGetBitmapByte(uint8_t byte);
+
+// implementation
+/**
+ * @brief Compress bitmap bytes comprised of 2-bits to counterpart of 1-bit.
+ * e.g.
+ * TD_VTYPE_NORM 0x00U(00000000) to 00000000 Normal
+ * TD_VTYPE_NULL 0x01U(00000001) to 00000001 Null
+ * TD_VTYPE_NONE 0x02U(00000010) to 00000001 Null
+ *
+ * 00000000 0x00 0x00
+ * 01000000 0x40 0x08
+ * 10000000 0x80 0x08
+ *  ...
+ * @param byte
+ * @return uint8_t
+ */
+static uint8_t tdGetMergedBitmapByte(uint8_t byte) {
+  switch (byte) {
+    case 0x00:
+      return 0x00;
+    case 0x40:
+      return 0x08;
+    case 0x80:
+      return 0x08;
+    case 0x10:
+      return 0x04;
+    case 0x50:
+      return 0x0c;
+    case 0x90:
+      return 0x0c;
+    case 0x20:
+      return 0x04;
+    case 0x60:
+      return 0x0c;
+    case 0xa0:
+      return 0x0c;
+    case 0x04:
+      return 0x02;
+    case 0x44:
+      return 0x0a;
+    case 0x84:
+      return 0x0a;
+    case 0x14:
+      return 0x06;
+    case 0x54:
+      return 0x0e;
+    case 0x94:
+      return 0x0e;
+    case 0x24:
+      return 0x06;
+    case 0x64:
+      return 0x0e;
+    case 0xa4:
+      return 0x0e;
+    case 0x08:
+      return 0x02;
+    case 0x48:
+      return 0x0a;
+    case 0x88:
+      return 0x0a;
+    case 0x18:
+      return 0x06;
+    case 0x58:
+      return 0x0e;
+    case 0x98:
+      return 0x0e;
+    case 0x28:
+      return 0x06;
+    case 0x68:
+      return 0x0e;
+    case 0xa8:
+      return 0x0e;
+    case 0x01:
+      return 0x01;
+    case 0x41:
+      return 0x09;
+    case 0x81:
+      return 0x09;
+    case 0x11:
+      return 0x05;
+    case 0x51:
+      return 0x0d;
+    case 0x91:
+      return 0x0d;
+    case 0x21:
+      return 0x05;
+    case 0x61:
+      return 0x0d;
+    case 0xa1:
+      return 0x0d;
+    case 0x05:
+      return 0x03;
+    case 0x45:
+      return 0x0b;
+    case 0x85:
+      return 0x0b;
+    case 0x15:
+      return 0x07;
+    case 0x55:
+      return 0x0f;
+    case 0x95:
+      return 0x0f;
+    case 0x25:
+      return 0x07;
+    case 0x65:
+      return 0x0f;
+    case 0xa5:
+      return 0x0f;
+    case 0x09:
+      return 0x03;
+    case 0x49:
+      return 0x0b;
+    case 0x89:
+      return 0x0b;
+    case 0x19:
+      return 0x07;
+    case 0x59:
+      return 0x0f;
+    case 0x99:
+      return 0x0f;
+    case 0x29:
+      return 0x07;
+    case 0x69:
+      return 0x0f;
+    case 0xa9:
+      return 0x0f;
+    case 0x02:
+      return 0x01;
+    case 0x42:
+      return 0x09;
+    case 0x82:
+      return 0x09;
+    case 0x12:
+      return 0x05;
+    case 0x52:
+      return 0x0d;
+    case 0x92:
+      return 0x0d;
+    case 0x22:
+      return 0x05;
+    case 0x62:
+      return 0x0d;
+    case 0xa2:
+      return 0x0d;
+    case 0x06:
+      return 0x03;
+    case 0x46:
+      return 0x0b;
+    case 0x86:
+      return 0x0b;
+    case 0x16:
+      return 0x07;
+    case 0x56:
+      return 0x0f;
+    case 0x96:
+      return 0x0f;
+    case 0x26:
+      return 0x07;
+    case 0x66:
+      return 0x0f;
+    case 0xa6:
+      return 0x0f;
+    case 0x0a:
+      return 0x03;
+    case 0x4a:
+      return 0x0b;
+    case 0x8a:
+      return 0x0b;
+    case 0x1a:
+      return 0x07;
+    case 0x5a:
+      return 0x0f;
+    case 0x9a:
+      return 0x0f;
+    case 0x2a:
+      return 0x07;
+    case 0x6a:
+      return 0x0f;
+    case 0xaa:
+      return 0x0f;
+    default:
+      // make sure the bitmap area is set to 0 firstly
+      ASSERT(0);
+      return 0x0f;  // return NULL bitmap for exception
+  }
+}
+
+/**
+ * @brief Merge bitmap from 2 bits to 1 bits, and the memory buffer should be guaranteed by the invoker.
+ *
+ * @param srcBitmap
+ * @param srcLen
+ * @param dstBitmap
+ */
+void tdMergeBitmap(uint8_t *srcBitmap, int32_t srcLen, uint8_t *dstBitmap) {
+  int32_t i = 0, j = 0;
+
+  if (srcLen > 0) {
+    dstBitmap[j] = (tdGetMergedBitmapByte(srcBitmap[i]) << 4);
+  }
+
+  while ((++i) < srcLen) {
+    if ((i & 1) == 0) {
+      dstBitmap[j] = (tdGetMergedBitmapByte(srcBitmap[i]) << 4);
+    } else {
+      dstBitmap[j] |= tdGetMergedBitmapByte(srcBitmap[i]);
+      ++j;
+    }
+  }
+}
+
 // static void dataColSetNEleNull(SDataCol *pCol, int nEle);
 static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2,
                                int limit2, int tRows, bool forceSetNull);
 
-static FORCE_INLINE void dataColSetNullAt(SDataCol *pCol, int index, bool setBitmap) {
+static FORCE_INLINE void dataColSetNullAt(SDataCol *pCol, int index, bool setBitmap, int8_t bitmapMode) {
   if (IS_VAR_DATA_TYPE(pCol->type)) {
     pCol->dataOff[index] = pCol->len;
     char *ptr = POINTER_SHIFT(pCol->pData, pCol->len);
@@ -37,7 +250,7 @@ static FORCE_INLINE void dataColSetNullAt(SDataCol *pCol, int index, bool setBit
     pCol->len += TYPE_BYTES[pCol->type];
   }
   if (setBitmap) {
-    tdSetBitmapValType(pCol->pBitmap, index, TD_VTYPE_NONE);
+    tdSetBitmapValType(pCol->pBitmap, index, TD_VTYPE_NONE, bitmapMode);
   }
 }
 
@@ -53,7 +266,7 @@ static FORCE_INLINE void dataColSetNullAt(SDataCol *pCol, int index, bool setBit
 //   }
 // }
 
-int32_t tdSetBitmapValTypeN(void *pBitmap, int16_t nEle, TDRowValT valType) {
+int32_t tdSetBitmapValTypeN(void *pBitmap, int16_t nEle, TDRowValT valType, int8_t bitmapMode) {
   TASSERT(valType < TD_VTYPE_MAX);
   int16_t nBytes = nEle / TD_VTYPE_PARTS;
   for (int i = 0; i < nBytes; ++i) {
@@ -63,12 +276,12 @@ int32_t tdSetBitmapValTypeN(void *pBitmap, int16_t nEle, TDRowValT valType) {
   int16_t nLeft = nEle - nBytes * TD_VTYPE_BITS;
 
   for (int j = 0; j < nLeft; ++j) {
-    tdSetBitmapValType(pBitmap, j, valType);
+    tdSetBitmapValType(pBitmap, j, valType, bitmapMode);
   }
   return TSDB_CODE_SUCCESS;
 }
 
-static FORCE_INLINE void dataColSetNoneAt(SDataCol *pCol, int index, bool setBitmap) {
+static FORCE_INLINE void dataColSetNoneAt(SDataCol *pCol, int index, bool setBitmap, int8_t bitmapMode) {
   if (IS_VAR_DATA_TYPE(pCol->type)) {
     pCol->dataOff[index] = pCol->len;
     char *ptr = POINTER_SHIFT(pCol->pData, pCol->len);
@@ -79,22 +292,22 @@ static FORCE_INLINE void dataColSetNoneAt(SDataCol *pCol, int index, bool setBit
     pCol->len += TYPE_BYTES[pCol->type];
   }
   if (setBitmap) {
-    tdSetBitmapValType(pCol->pBitmap, index, TD_VTYPE_NONE);
+    tdSetBitmapValType(pCol->pBitmap, index, TD_VTYPE_NONE, bitmapMode);
   }
 }
 
-static void dataColSetNEleNone(SDataCol *pCol, int nEle) {
+static void dataColSetNEleNone(SDataCol *pCol, int nEle, int8_t bitmapMode) {
   if (IS_VAR_DATA_TYPE(pCol->type)) {
     pCol->len = 0;
     for (int i = 0; i < nEle; ++i) {
-      dataColSetNoneAt(pCol, i, false);
+      dataColSetNoneAt(pCol, i, false, bitmapMode);
     }
   } else {
     setNullN(pCol->pData, pCol->type, pCol->bytes, nEle);
     pCol->len = TYPE_BYTES[pCol->type] * nEle;
   }
 #ifdef TD_SUPPORT_BITMAP
-  tdSetBitmapValTypeN(pCol->pBitmap, nEle, TD_VTYPE_NONE);
+  tdSetBitmapValTypeN(pCol->pBitmap, nEle, TD_VTYPE_NONE, bitmapMode);
 #endif
 }
 
@@ -126,7 +339,18 @@ STSRow *tdRowDup(STSRow *row) {
   return trow;
 }
 
-int tdAppendValToDataCol(SDataCol *pCol, TDRowValT valType, const void *val, int numOfRows, int maxPoints) {
+/**
+ * @brief 
+ * 
+ * @param pCol 
+ * @param valType 
+ * @param val 
+ * @param numOfRows 
+ * @param maxPoints 
+ * @param bitmapMode  default is 0(2 bits), otherwise 1(1 bit)
+ * @return int 
+ */
+int tdAppendValToDataCol(SDataCol *pCol, TDRowValT valType, const void *val, int numOfRows, int maxPoints, int8_t bitmapMode) {
   TASSERT(pCol != NULL);
 
   // Assume that the columns not specified during insert/upsert mean None.
@@ -139,7 +363,7 @@ int tdAppendValToDataCol(SDataCol *pCol, TDRowValT valType, const void *val, int
     if (tdAllocMemForCol(pCol, maxPoints) < 0) return -1;
     if (numOfRows > 0) {
       // Find the first not None value, fill all previous values as None
-      dataColSetNEleNone(pCol, numOfRows);
+      dataColSetNEleNone(pCol, numOfRows, bitmapMode);
     }
   }
   if (!tdValTypeIsNorm(valType)) {
@@ -161,7 +385,7 @@ int tdAppendValToDataCol(SDataCol *pCol, TDRowValT valType, const void *val, int
     pCol->len += pCol->bytes;
   }
 #ifdef TD_SUPPORT_BITMAP
-  tdSetBitmapValType(pCol->pBitmap, numOfRows, valType);
+    tdSetBitmapValType(pCol->pBitmap, numOfRows, valType, bitmapMode);
 #endif
   return 0;
 }
@@ -175,14 +399,13 @@ static int32_t tdAppendTpRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols
   void *pBitmap = tdGetBitmapAddrTp(pRow, pSchema->flen);
 
   SDataCol *pDataCol = &(pCols->cols[0]);
-  if (pDataCol->colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
-    tdAppendValToDataCol(pDataCol, TD_VTYPE_NORM, &pRow->ts, pCols->numOfRows, pCols->maxPoints);
-  }
+  ASSERT(pDataCol->colId == PRIMARYKEY_TIMESTAMP_COL_ID);
+  tdAppendValToDataCol(pDataCol, TD_VTYPE_NORM, &pRow->ts, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode);
 
   while (dcol < pCols->numOfCols) {
     pDataCol = &(pCols->cols[dcol]);
     if (rcol >= schemaNCols(pSchema)) {
-      tdAppendValToDataCol(pDataCol, TD_VTYPE_NULL, NULL, pCols->numOfRows, pCols->maxPoints);
+      tdAppendValToDataCol(pDataCol, TD_VTYPE_NULL, NULL, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode);
       ++dcol;
       continue;
     }
@@ -193,13 +416,13 @@ static int32_t tdAppendTpRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols
       if (tdGetTpRowValOfCol(&sVal, pRow, pBitmap, pRowCol->type, pRowCol->offset - sizeof(TSKEY), rcol - 1) < 0) {
         return terrno;
       }
-      tdAppendValToDataCol(pDataCol, sVal.valType, sVal.val, pCols->numOfRows, pCols->maxPoints);
+      tdAppendValToDataCol(pDataCol, sVal.valType, sVal.val, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode);
       ++dcol;
       ++rcol;
     } else if (pRowCol->colId < pDataCol->colId) {
       ++rcol;
     } else {
-      tdAppendValToDataCol(pDataCol, TD_VTYPE_NULL, NULL, pCols->numOfRows, pCols->maxPoints);
+      tdAppendValToDataCol(pDataCol, TD_VTYPE_NULL, NULL, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode);
       ++dcol;
     }
   }
@@ -218,14 +441,13 @@ static int32_t tdAppendKvRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols
   void *pBitmap = tdGetBitmapAddrKv(pRow, tdRowGetNCols(pRow));
 
   SDataCol *pDataCol = &(pCols->cols[0]);
-  if (pDataCol->colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
-    tdAppendValToDataCol(pDataCol, TD_VTYPE_NORM, &pRow->ts, pCols->numOfRows, pCols->maxPoints);
-  }
+  ASSERT(pDataCol->colId == PRIMARYKEY_TIMESTAMP_COL_ID);
+  tdAppendValToDataCol(pDataCol, TD_VTYPE_NORM, &pRow->ts, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode);
 
   while (dcol < pCols->numOfCols) {
     pDataCol = &(pCols->cols[dcol]);
     if (rcol >= tRowCols || rcol >= tSchemaCols) {
-      tdAppendValToDataCol(pDataCol, TD_VTYPE_NULL, NULL, pCols->numOfRows, pCols->maxPoints);
+      tdAppendValToDataCol(pDataCol, TD_VTYPE_NULL, NULL, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode);
       ++dcol;
       continue;
     }
@@ -240,13 +462,13 @@ static int32_t tdAppendKvRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols
       if (tdGetKvRowValOfCol(&sVal, pRow, pBitmap, pIdx->offset, colIdx) < 0) {
         return terrno;
       }
-      tdAppendValToDataCol(pDataCol, sVal.valType, sVal.val, pCols->numOfRows, pCols->maxPoints);
+      tdAppendValToDataCol(pDataCol, sVal.valType, sVal.val, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode);
       ++dcol;
       ++rcol;
     } else if (pIdx->colId < pDataCol->colId) {
       ++rcol;
     } else {
-      tdAppendValToDataCol(pDataCol, TD_VTYPE_NULL, NULL, pCols->numOfRows, pCols->maxPoints);
+      tdAppendValToDataCol(pDataCol, TD_VTYPE_NULL, NULL, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode);
       ++dcol;
     }
   }
@@ -263,7 +485,7 @@ static int32_t tdAppendKvRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols
  * @param pCols
  * @param forceSetNull
  */
-int32_t tdAppendSTSRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols *pCols, bool forceSetNull) {
+int32_t tdAppendSTSRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols *pCols) {
   if (TD_IS_TP_ROW(pRow)) {
     return tdAppendTpRowToDataCol(pRow, pSchema, pCols);
   } else if (TD_IS_KV_ROW(pRow)) {
@@ -274,7 +496,7 @@ int32_t tdAppendSTSRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols *pCol
   return TSDB_CODE_SUCCESS;
 }
 
-int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge, int *pOffset, bool forceSetNull) {
+int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge, int *pOffset, bool forceSetNull, TDRowVerT maxVer) {
   ASSERT(rowsToMerge > 0 && rowsToMerge <= source->numOfRows);
   ASSERT(target->numOfCols == source->numOfCols);
   int offset = 0;
@@ -287,14 +509,15 @@ int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge, int *
 
   if ((target->numOfRows == 0) || (dataColsKeyLast(target) < dataColsKeyAtRow(source, *pOffset))) {  // No overlap
     ASSERT(target->numOfRows + rowsToMerge <= target->maxPoints);
+    // TODO: filter the maxVer
     for (int i = 0; i < rowsToMerge; i++) {
       for (int j = 0; j < source->numOfCols; j++) {
         if (source->cols[j].len > 0 || target->cols[j].len > 0) {
           SCellVal sVal = {0};
-          if (tdGetColDataOfRow(&sVal, source->cols + j, i + (*pOffset)) < 0) {
+          if (tdGetColDataOfRow(&sVal, source->cols + j, i + (*pOffset), source->bitmapMode) < 0) {
             TASSERT(0);
           }
-          tdAppendValToDataCol(target->cols + j, sVal.valType, sVal.val, target->numOfRows, target->maxPoints);
+          tdAppendValToDataCol(target->cols + j, sVal.valType, sVal.val, target->numOfRows, target->maxPoints, target->bitmapMode);
         }
       }
       ++target->numOfRows;
@@ -332,46 +555,46 @@ static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, i
     // TKEY  tkey2 = (*iter2 >= limit2) ? TKEY_NULL : dataColsTKeyAt(src2, *iter2);
 
     ASSERT(tkey1 == TKEY_NULL || (!TKEY_IS_DELETED(tkey1)));
-
+    // TODO: filter the maxVer
     if (key1 < key2) {
-      for (int i = 0; i < src1->numOfCols; i++) {
+      for (int i = 0; i < src1->numOfCols; ++i) {
         ASSERT(target->cols[i].type == src1->cols[i].type);
         if (src1->cols[i].len > 0 || target->cols[i].len > 0) {
           SCellVal sVal = {0};
-          if (tdGetColDataOfRow(&sVal, src1->cols + i, *iter1) < 0) {
+          if (tdGetColDataOfRow(&sVal, src1->cols + i, *iter1, src1->bitmapMode) < 0) {
             TASSERT(0);
           }
-          tdAppendValToDataCol(&(target->cols[i]), sVal.valType, sVal.val, target->numOfRows, target->maxPoints);
+          tdAppendValToDataCol(&(target->cols[i]), sVal.valType, sVal.val, target->numOfRows, target->maxPoints, target->bitmapMode);
         }
       }
 
-      target->numOfRows++;
-      (*iter1)++;
+      ++target->numOfRows;
+      ++(*iter1);
     } else if (key1 >= key2) {
-      // if ((key1 > key2) || (key1 == key2 && !TKEY_IS_DELETED(tkey2))) {
-      if ((key1 > key2) || (key1 == key2)) {
-        for (int i = 0; i < src2->numOfCols; i++) {
+      // TODO: filter the maxVer
+      if ((key1 > key2) || ((key1 == key2) && !TKEY_IS_DELETED(key2))) {
+        for (int i = 0; i < src2->numOfCols; ++i) {
           SCellVal sVal = {0};
           ASSERT(target->cols[i].type == src2->cols[i].type);
-          if (tdGetColDataOfRow(&sVal, src2->cols + i, *iter2) < 0) {
+          if (tdGetColDataOfRow(&sVal, src2->cols + i, *iter2, src2->bitmapMode) < 0) {
             TASSERT(0);
           }
           if (src2->cols[i].len > 0 && !tdValTypeIsNull(sVal.valType)) {
-            tdAppendValToDataCol(&(target->cols[i]), sVal.valType, sVal.val, target->numOfRows, target->maxPoints);
+            tdAppendValToDataCol(&(target->cols[i]), sVal.valType, sVal.val, target->numOfRows, target->maxPoints, target->bitmapMode);
           } else if (!forceSetNull && key1 == key2 && src1->cols[i].len > 0) {
-            if (tdGetColDataOfRow(&sVal, src1->cols + i, *iter1) < 0) {
+            if (tdGetColDataOfRow(&sVal, src1->cols + i, *iter1, src1->bitmapMode) < 0) {
               TASSERT(0);
             }
-            tdAppendValToDataCol(&(target->cols[i]), sVal.valType, sVal.val, target->numOfRows, target->maxPoints);
+            tdAppendValToDataCol(&(target->cols[i]), sVal.valType, sVal.val, target->numOfRows, target->maxPoints, target->bitmapMode);
           } else if (target->cols[i].len > 0) {
-            dataColSetNullAt(&target->cols[i], target->numOfRows, true);
+            dataColSetNullAt(&target->cols[i], target->numOfRows, true, target->bitmapMode);
           }
         }
-        target->numOfRows++;
+        ++target->numOfRows;
       }
 
-      (*iter2)++;
-      if (key1 == key2) (*iter1)++;
+      ++(*iter2);
+      if (key1 == key2) ++(*iter1);
     }
 
     ASSERT(target->numOfRows <= target->maxPoints);
@@ -480,6 +703,7 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) {
   if (pRet == NULL) return NULL;
 
   pRet->numOfCols = pDataCols->numOfCols;
+  pRet->bitmapMode = pDataCols->bitmapMode;
   pRet->sversion = pDataCols->sversion;
   if (keepData) pRet->numOfRows = pDataCols->numOfRows;
 

source/common/src/ttszip.c

@@ -39,7 +39,7 @@ STSBuf* tsBufCreate(bool autoDelete, int32_t order) {
 
   taosGetTmpfilePath(tsTempDir, "join", pTSBuf->path);
   // pTSBuf->pFile = fopen(pTSBuf->path, "wb+");
-  pTSBuf->pFile = taosOpenFile(pTSBuf->path, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC);
+  pTSBuf->pFile = taosOpenFile(pTSBuf->path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC);
   if (pTSBuf->pFile == NULL) {
     taosMemoryFree(pTSBuf);
     return NULL;

source/common/test/CMakeLists.txt

@@ -13,8 +13,8 @@ TARGET_LINK_LIBRARIES(
 
 TARGET_INCLUDE_DIRECTORIES(
         commonTest
-        PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/common/"
-        PRIVATE "${CMAKE_SOURCE_DIR}/source/libs/common/inc"
+        PUBLIC "${TD_SOURCE_DIR}/include/libs/common/"
+        PRIVATE "${TD_SOURCE_DIR}/source/libs/common/inc"
 )
 
 # tmsg test
@@ -24,5 +24,5 @@ TARGET_INCLUDE_DIRECTORIES(
 #         "tmsgTest.cpp"
 #         "../src/tmsg.c"
 # )
-# target_include_directories(tmsgTest PUBLIC "${CMAKE_SOURCE_DIR}/include/common/")
+# target_include_directories(tmsgTest PUBLIC "${TD_SOURCE_DIR}/include/common/")
 # target_link_libraries(tmsgTest PUBLIC os util gtest gtest_main)
\ No newline at end of file

source/dnode/bnode/CMakeLists.txt

@@ -2,7 +2,7 @@ aux_source_directory(src BNODE_SRC)
 add_library(bnode STATIC ${BNODE_SRC})
 target_include_directories(
     bnode
-    PUBLIC "${CMAKE_SOURCE_DIR}/include/dnode/bnode"
+    PUBLIC "${TD_SOURCE_DIR}/include/dnode/bnode"
     private "${CMAKE_CURRENT_SOURCE_DIR}/inc"
 )
 target_link_libraries(

source/dnode/mgmt/CMakeLists.txt

@@ -11,7 +11,7 @@ target_link_libraries(
 )
 target_include_directories(
     dnode
-    PUBLIC "${CMAKE_SOURCE_DIR}/include/dnode/mgmt"
+    PUBLIC "${TD_SOURCE_DIR}/include/dnode/mgmt"
     PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
 )
 

source/dnode/mgmt/dm/dmFile.c

@@ -162,7 +162,7 @@ int32_t dmWriteFile(SDnodeMgmt *pMgmt) {
   char file[PATH_MAX];
   snprintf(file, sizeof(file), "%s%sdnode.json.bak", pMgmt->path, TD_DIRSEP);
 
-  TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
+  TdFilePtr pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
   if (pFile == NULL) {
     dError("failed to write %s since %s", file, strerror(errno));
     terrno = TAOS_SYSTEM_ERROR(errno);

source/dnode/mgmt/inc/mmInt.h

@@ -62,6 +62,7 @@ int32_t mmProcessSyncMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
 int32_t mmProcessReadMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
 int32_t mmProcessQueryMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
 int32_t mmProcessMonitorMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg);
+
 int32_t mmPutMsgToQueryQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc);
 int32_t mmPutMsgToReadQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc);
 int32_t mmPutMsgToWriteQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc);

source/dnode/mgmt/main/dndFile.c

@@ -75,7 +75,7 @@ int32_t dndWriteFile(SMgmtWrapper *pWrapper, bool deployed) {
   snprintf(file, sizeof(file), "%s%s%s.json", pWrapper->path, TD_DIRSEP, pWrapper->name);
   snprintf(realfile, sizeof(realfile), "%s%s%s.json", pWrapper->path, TD_DIRSEP, pWrapper->name);
 
-  pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
+  pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
   if (pFile == NULL) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     dError("failed to write %s since %s", file, terrstr());
@@ -121,7 +121,7 @@ TdFilePtr dndCheckRunning(const char *dataDir) {
   char filepath[PATH_MAX] = {0};
   snprintf(filepath, sizeof(filepath), "%s%s.running", dataDir, TD_DIRSEP);
 
-  TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
+  TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
   if (pFile == NULL) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     dError("failed to lock file:%s since %s", filepath, terrstr());
@@ -218,7 +218,7 @@ int32_t dndWriteShmFile(SDnode *pDnode) {
   snprintf(file, sizeof(file), "%s%s.shmfile.bak", pDnode->dataDir, TD_DIRSEP);
   snprintf(realfile, sizeof(realfile), "%s%s.shmfile", pDnode->dataDir, TD_DIRSEP);
 
-  pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
+  pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
   if (pFile == NULL) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     dError("failed to open file:%s since %s", file, terrstr());

source/dnode/mgmt/main/dndTransport.c

@@ -53,8 +53,9 @@ static void dndProcessRpcMsg(SMgmtWrapper *pWrapper, SRpcMsg *pRpc, SEpSet *pEpS
   int32_t   code = -1;
   SNodeMsg *pMsg = NULL;
   NodeMsgFp msgFp = NULL;
+  uint16_t msgType = pRpc->msgType;
 
-  if (pEpSet && pEpSet->numOfEps > 0 && pRpc->msgType == TDMT_MND_STATUS_RSP) {
+  if (pEpSet && pEpSet->numOfEps > 0 && msgType == TDMT_MND_STATUS_RSP) {
     dndUpdateMnodeEpSet(pWrapper->pDnode, pEpSet);
   }
 
@@ -84,8 +85,15 @@ _OVER:
     }
   } else {
     dError("msg:%p, failed to process since 0x%04x:%s", pMsg, code & 0XFFFF, terrstr());
-    if (pRpc->msgType & 1U) {
-      SRpcMsg rsp = {.handle = pRpc->handle, .ahandle = pRpc->ahandle, .code = terrno};
+    if (msgType & 1U) {
+      if (terrno != 0) code = terrno;
+      if (code == TSDB_CODE_NODE_NOT_DEPLOYED || code == TSDB_CODE_NODE_OFFLINE) {
+        if (msgType > TDMT_MND_MSG && msgType < TDMT_VND_MSG) {
+          code = TSDB_CODE_NODE_REDIRECT;
+        }
+      }
+
+      SRpcMsg rsp = {.handle = pRpc->handle, .ahandle = pRpc->ahandle, .code = code};
       tmsgSendRsp(&rsp);
     }
     dTrace("msg:%p, is freed", pMsg);
@@ -347,14 +355,11 @@ static int32_t dndSendRpcReq(STransMgmt *pMgmt, const SEpSet *pEpSet, SRpcMsg *p
 }
 
 static void dndSendRpcRsp(SMgmtWrapper *pWrapper, const SRpcMsg *pRsp) {
-  if (pRsp->code == TSDB_CODE_APP_NOT_READY) {
-    if (pWrapper->ntype == MNODE) {
-      dmSendRedirectRsp(pWrapper->pMgmt, pRsp);
-      return;
-    }
+  if (pRsp->code == TSDB_CODE_NODE_REDIRECT) {
+    dmSendRedirectRsp(pWrapper->pMgmt, pRsp);
+  } else {
+    rpcSendResponse(pRsp);
   }
-
-  rpcSendResponse(pRsp);
 }
 
 static int32_t dndSendReq(SMgmtWrapper *pWrapper, const SEpSet *pEpSet, SRpcMsg *pReq) {
@@ -443,7 +448,8 @@ static void dndConsumeChildQueue(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int16_t
 static void dndConsumeParentQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg, int16_t msgLen, void *pCont, int32_t contLen,
                                   ProcFuncType ftype) {
   pMsg->pCont = pCont;
-  dTrace("msg:%p, get from parent queue, ftype:%d handle:%p, app:%p", pMsg, ftype, pMsg->handle, pMsg->ahandle);
+  dTrace("msg:%p, get from parent queue, ftype:%d handle:%p code:0x%04x mtype:%d, app:%p", pMsg, ftype, pMsg->handle,
+         pMsg->code & 0xFFFF, pMsg->msgType, pMsg->ahandle);
 
   switch (ftype) {
     case PROC_REGIST:

source/dnode/mgmt/mm/mmFile.c

@@ -109,7 +109,7 @@ int32_t mmWriteFile(SMnodeMgmt *pMgmt, bool deployed) {
   char file[PATH_MAX];
   snprintf(file, sizeof(file), "%s%smnode.json.bak", pMgmt->path, TD_DIRSEP);
 
-  TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
+  TdFilePtr pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
   if (pFile == NULL) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     dError("failed to write %s since %s", file, terrstr());

source/dnode/mgmt/mm/mmHandle.c

@@ -96,7 +96,7 @@ int32_t mmProcessAlterReq(SMnodeMgmt *pMgmt, SNodeMsg *pMsg) {
 
   if (alterReq.dnodeId != pDnode->dnodeId) {
     terrno = TSDB_CODE_INVALID_OPTION;
-    dError("failed to alter mnode since %s", terrstr());
+    dError("failed to alter mnode since %s, dnodeId:%d input:%d", terrstr(), pDnode->dnodeId, alterReq.dnodeId);
     return -1;
   } else {
     return mmAlter(pMgmt, &alterReq);

source/dnode/mgmt/mm/mmInt.c

@@ -48,15 +48,13 @@ static void mmInitOption(SMnodeMgmt *pMgmt, SMnodeOpt *pOption) {
 }
 
 static void mmBuildOptionForDeploy(SMnodeMgmt *pMgmt, SMnodeOpt *pOption) {
-  SDnode *pDnode = pMgmt->pDnode;
-
   mmInitOption(pMgmt, pOption);
   pOption->replica = 1;
   pOption->selfIndex = 0;
   SReplica *pReplica = &pOption->replicas[0];
   pReplica->id = 1;
-  pReplica->port = pDnode->serverPort;
-  tstrncpy(pReplica->fqdn, pDnode->localFqdn, TSDB_FQDN_LEN);
+  pReplica->port = pMgmt->pDnode->serverPort;
+  tstrncpy(pReplica->fqdn, pMgmt->pDnode->localFqdn, TSDB_FQDN_LEN);
   pOption->deploy = true;
 
   pMgmt->selfIndex = pOption->selfIndex;
@@ -114,6 +112,9 @@ static int32_t mmOpenImp(SMnodeMgmt *pMgmt, SDCreateMnodeReq *pReq) {
 
     if (!deployed) {
       dInfo("mnode start to deploy");
+      if (pMgmt->pWrapper->procType == PROC_CHILD) {
+        pMgmt->pDnode->dnodeId = 1;
+      }
       mmBuildOptionForDeploy(pMgmt, &option);
     } else {
       dInfo("mnode start to open");
@@ -151,12 +152,9 @@ static void mmCloseImp(SMnodeMgmt *pMgmt) {
 
 int32_t mmAlter(SMnodeMgmt *pMgmt, SDAlterMnodeReq *pReq) {
   SMnodeOpt option = {0};
-  if (pReq != NULL) {
-    if (mmBuildOptionFromReq(pMgmt, &option, pReq) != 0) {
-      return -1;
-    }
+  if (mmBuildOptionFromReq(pMgmt, &option, pReq) != 0) {
+    return -1;
   }
-
   return mndAlter(pMgmt->pMnode, &option);
 }
 
@@ -240,4 +238,3 @@ void mmSetMgmtFp(SMgmtWrapper *pWrapper) {
   pWrapper->name = "mnode";
   pWrapper->fp = mgmtFp;
 }
-

source/dnode/mgmt/mm/mmWorker.c

@@ -16,6 +16,15 @@
 #define _DEFAULT_SOURCE
 #include "mmInt.h"
 
+static inline void mmSendRsp(SNodeMsg *pMsg, int32_t code) {
+  SRpcMsg rsp = {.handle = pMsg->rpcMsg.handle,
+                 .ahandle = pMsg->rpcMsg.ahandle,
+                 .code = code,
+                 .pCont = pMsg->pRsp,
+                 .contLen = pMsg->rspLen};
+  tmsgSendRsp(&rsp);
+}
+
 static void mmProcessQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) {
   SMnodeMgmt *pMgmt = pInfo->ahandle;
 
@@ -35,8 +44,7 @@ static void mmProcessQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) {
   if (pRpc->msgType & 1U) {
     if (pRpc->handle != NULL && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
       if (code != 0 && terrno != 0) code = terrno;
-      SRpcMsg rsp = {.handle = pRpc->handle, .code = code, .contLen = pMsg->rspLen, .pCont = pMsg->pRsp};
-      tmsgSendRsp(&rsp);
+      mmSendRsp(pMsg, code);
     }
   }
 
@@ -48,7 +56,7 @@ static void mmProcessQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) {
 static void mmProcessQueryQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) {
   SMnodeMgmt *pMgmt = pInfo->ahandle;
 
-  dTrace("msg:%p, get from mnode query queue", pMsg);
+  dTrace("msg:%p, get from mnode-query queue", pMsg);
   SRpcMsg *pRpc = &pMsg->rpcMsg;
   int32_t  code = -1;
 
@@ -58,8 +66,7 @@ static void mmProcessQueryQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) {
   if (pRpc->msgType & 1U) {
     if (pRpc->handle != NULL && code != 0) {
       dError("msg:%p, failed to process since %s", pMsg, terrstr());
-      SRpcMsg rsp = {.handle = pRpc->handle, .code = code, .ahandle = pRpc->ahandle};
-      tmsgSendRsp(&rsp);
+      mmSendRsp(pMsg, code);
     }
   }
 
@@ -98,11 +105,8 @@ int32_t mmProcessQueryMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
 }
 
 int32_t mmProcessMonitorMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
-  SMnodeMgmt    *pMgmt = pWrapper->pMgmt;
-  SSingleWorker *pWorker = &pMgmt->monitorWorker;
-
-  dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name);
-  taosWriteQitem(pWorker->queue, pMsg);
+  SMnodeMgmt *pMgmt = pWrapper->pMgmt;
+  mmPutMsgToWorker(&pMgmt->monitorWorker, pMsg);
   return 0;
 }
 

source/dnode/mgmt/test/bnode/CMakeLists.txt

 aux_source_directory(. BQTEST_SRC)
-add_executable(dnode_test_bnode ${BQTEST_SRC})
+add_executable(dbnodeTest ${BQTEST_SRC})
 target_link_libraries(
-    dnode_test_bnode
+    dbnodeTest
     PUBLIC sut
 )
 
 add_test(
-    NAME dnode_test_bnode
-    COMMAND dnode_test_bnode
+    NAME dbnodeTest
+    COMMAND dbnodeTest
 )

source/dnode/mgmt/test/mnode/CMakeLists.txt

 aux_source_directory(. DMTEST_SRC)
-add_executable(dnode_test_mnode ${DMTEST_SRC})
+add_executable(dmnodeTest ${DMTEST_SRC})
 target_link_libraries(
-    dnode_test_mnode
+    dmnodeTest
     PUBLIC sut
 )
 
 add_test(
-    NAME dnode_test_mnode
-    COMMAND dnode_test_mnode
+    NAME dmnodeTest
+    COMMAND dmnodeTest
 )

source/dnode/mgmt/test/qnode/CMakeLists.txt

 aux_source_directory(. DQTEST_SRC)
-add_executable(dnode_test_qnode ${DQTEST_SRC})
+add_executable(dqnodeTest ${DQTEST_SRC})
 target_link_libraries(
-    dnode_test_qnode
+    dqnodeTest
     PUBLIC sut
 )
 
 add_test(
-    NAME dnode_test_qnode
-    COMMAND dnode_test_qnode
+    NAME dqnodeTest
+    COMMAND dqnodeTest
 )

source/dnode/mgmt/test/snode/CMakeLists.txt

 aux_source_directory(. SQTEST_SRC)
-add_executable(dnode_test_snode ${SQTEST_SRC})
+add_executable(dsnodeTest ${SQTEST_SRC})
 target_link_libraries(
-    dnode_test_snode
+    dsnodeTest
     PUBLIC sut
 )
 
 add_test(
-    NAME dnode_test_snode
-    COMMAND dnode_test_snode
+    NAME dsnodeTest
+    COMMAND dsnodeTest
 )

source/dnode/mgmt/test/vnode/CMakeLists.txt

 aux_source_directory(. VNODE_SRC)
-add_executable(dnode_test_vnode ${VNODE_SRC})
+add_executable(dvnodeTest ${VNODE_SRC})
 target_link_libraries(
-    dnode_test_vnode
+    dvnodeTest
     PUBLIC sut
 )
 
 add_test(
-    NAME dnode_test_vnode
-    COMMAND dnode_test_vnode
+    NAME dvnodeTest
+    COMMAND dvnodeTest
 )

source/dnode/mgmt/vm/vmFile.c

@@ -154,7 +154,7 @@ int32_t vmWriteVnodesToFile(SVnodesMgmt *pMgmt) {
   snprintf(file, sizeof(file), "%s%svnodes.json.bak", pMgmt->path, TD_DIRSEP);
   snprintf(realfile, sizeof(file), "%s%svnodes.json", pMgmt->path, TD_DIRSEP);
 
-  TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
+  TdFilePtr pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
   if (pFile == NULL) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     dError("failed to write %s since %s", file, terrstr());

source/dnode/mnode/impl/CMakeLists.txt

@@ -2,7 +2,7 @@ aux_source_directory(src MNODE_SRC)
 add_library(mnode STATIC ${MNODE_SRC})
 target_include_directories(
     mnode
-    PUBLIC "${CMAKE_SOURCE_DIR}/include/dnode/mnode"
+    PUBLIC "${TD_SOURCE_DIR}/include/dnode/mnode"
     PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
 )
 target_link_libraries(

source/dnode/mnode/impl/inc/mndInt.h

@@ -99,6 +99,7 @@ typedef struct {
 } SGrantInfo;
 
 typedef struct SMnode {
+  int32_t           selfId;
   int64_t           clusterId;
   int8_t            replica;
   int8_t            selfIndex;
@@ -120,7 +121,7 @@ typedef struct SMnode {
   SHashObj         *infosMeta;
   SGrantInfo        grant;
   MndMsgFp          msgFp[TDMT_MAX];
-  SMsgCb            msgCb;  
+  SMsgCb            msgCb;
 } SMnode;
 
 void    mndSetMsgHandle(SMnode *pMnode, tmsg_t msgType, MndMsgFp fp);

source/dnode/mnode/impl/inc/mndMnode.h

@@ -22,12 +22,14 @@
 extern "C" {
 #endif
 
-int32_t mndInitMnode(SMnode *pMnode);
-void    mndCleanupMnode(SMnode *pMnode);
-bool    mndIsMnode(SMnode *pMnode, int32_t dnodeId);
-void    mndGetMnodeEpSet(SMnode *pMnode, SEpSet *pEpSet);
-char   *mndGetRoleStr(int32_t role);
-void    mndUpdateMnodeRole(SMnode *pMnode);
+int32_t     mndInitMnode(SMnode *pMnode);
+void        mndCleanupMnode(SMnode *pMnode);
+SMnodeObj  *mndAcquireMnode(SMnode *pMnode, int32_t mnodeId);
+void        mndReleaseMnode(SMnode *pMnode, SMnodeObj *pObj);
+bool        mndIsMnode(SMnode *pMnode, int32_t dnodeId);
+void        mndGetMnodeEpSet(SMnode *pMnode, SEpSet *pEpSet);
+void        mndUpdateMnodeRole(SMnode *pMnode);
+const char *mndGetRoleStr(int32_t role);
 
 #ifdef __cplusplus
 }

source/dnode/mnode/impl/src/mndBnode.c

@@ -395,7 +395,7 @@ static int32_t mndProcessDropBnodeReq(SNodeMsg *pReq) {
   mDebug("bnode:%d, start to drop", dropReq.dnodeId);
 
   if (dropReq.dnodeId <= 0) {
-    terrno = TSDB_CODE_SDB_APP_ERROR;
+    terrno = TSDB_CODE_INVALID_MSG;
     goto _OVER;
   }
 

source/dnode/mnode/impl/src/mndDb.c

@@ -1038,7 +1038,7 @@ static void mndBuildDBVgroupInfo(SDbObj *pDb, SMnode *pMnode, SArray *pVgList) {
   SSdb   *pSdb = pMnode->pSdb;
 
   void *pIter = NULL;
-  while (true) {
+  while (1) {
     SVgObj *pVgroup = NULL;
     pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup);
     if (pIter == NULL) break;

source/dnode/mnode/impl/src/mndDnode.c

@@ -549,6 +549,7 @@ static int32_t mndProcessDropDnodeReq(SNodeMsg *pReq) {
   int32_t        code = -1;
   SUserObj      *pUser = NULL;
   SDnodeObj     *pDnode = NULL;
+  SMnodeObj     *pMObj = NULL;
   SMDropMnodeReq dropReq = {0};
 
   if (tDeserializeSCreateDropMQSBNodeReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &dropReq) != 0) {
@@ -569,6 +570,12 @@ static int32_t mndProcessDropDnodeReq(SNodeMsg *pReq) {
     goto DROP_DNODE_OVER;
   }
 
+  pMObj = mndAcquireMnode(pMnode, dropReq.dnodeId);
+  if (pMObj != NULL) {
+    terrno = TSDB_CODE_MND_MNODE_DEPLOYED;
+    goto DROP_DNODE_OVER;
+  }
+
   pUser = mndAcquireUser(pMnode, pReq->user);
   if (pUser == NULL) {
     terrno = TSDB_CODE_MND_NO_USER_FROM_CONN;
@@ -589,6 +596,7 @@ DROP_DNODE_OVER:
 
   mndReleaseDnode(pMnode, pDnode);
   mndReleaseUser(pMnode, pUser);
+  mndReleaseMnode(pMnode, pMObj);
 
   return code;
 }

source/dnode/mnode/impl/src/mndMnode.c

@@ -21,8 +21,8 @@
 #include "mndTrans.h"
 #include "mndUser.h"
 
-#define TSDB_MNODE_VER_NUMBER   1
-#define TSDB_MNODE_RESERVE_SIZE 64
+#define MNODE_VER_NUMBER   1
+#define MNODE_RESERVE_SIZE 64
 
 static int32_t  mndCreateDefaultMnode(SMnode *pMnode);
 static SSdbRaw *mndMnodeActionEncode(SMnodeObj *pObj);
@@ -62,21 +62,20 @@ int32_t mndInitMnode(SMnode *pMnode) {
 
 void mndCleanupMnode(SMnode *pMnode) {}
 
-static SMnodeObj *mndAcquireMnode(SMnode *pMnode, int32_t mnodeId) {
-  SSdb      *pSdb = pMnode->pSdb;
-  SMnodeObj *pObj = sdbAcquire(pSdb, SDB_MNODE, &mnodeId);
+SMnodeObj *mndAcquireMnode(SMnode *pMnode, int32_t mnodeId) {
+  SMnodeObj *pObj = sdbAcquire(pMnode->pSdb, SDB_MNODE, &mnodeId);
   if (pObj == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) {
     terrno = TSDB_CODE_MND_MNODE_NOT_EXIST;
   }
   return pObj;
 }
 
-static void mndReleaseMnode(SMnode *pMnode, SMnodeObj *pObj) {
+void mndReleaseMnode(SMnode *pMnode, SMnodeObj *pObj) {
   SSdb *pSdb = pMnode->pSdb;
-  sdbRelease(pSdb, pObj);
+  sdbRelease(pMnode->pSdb, pObj);
 }
 
-char *mndGetRoleStr(int32_t showType) {
+const char *mndGetRoleStr(int32_t showType) {
   switch (showType) {
     case TAOS_SYNC_STATE_FOLLOWER:
       return "unsynced";
@@ -128,18 +127,18 @@ static int32_t mndCreateDefaultMnode(SMnode *pMnode) {
 static SSdbRaw *mndMnodeActionEncode(SMnodeObj *pObj) {
   terrno = TSDB_CODE_OUT_OF_MEMORY;
 
-  SSdbRaw *pRaw = sdbAllocRaw(SDB_MNODE, TSDB_MNODE_VER_NUMBER, sizeof(SMnodeObj) + TSDB_MNODE_RESERVE_SIZE);
-  if (pRaw == NULL) goto MNODE_ENCODE_OVER;
+  SSdbRaw *pRaw = sdbAllocRaw(SDB_MNODE, MNODE_VER_NUMBER, sizeof(SMnodeObj) + MNODE_RESERVE_SIZE);
+  if (pRaw == NULL) goto _OVER;
 
   int32_t dataPos = 0;
-  SDB_SET_INT32(pRaw, dataPos, pObj->id, MNODE_ENCODE_OVER)
-  SDB_SET_INT64(pRaw, dataPos, pObj->createdTime, MNODE_ENCODE_OVER)
-  SDB_SET_INT64(pRaw, dataPos, pObj->updateTime, MNODE_ENCODE_OVER)
-  SDB_SET_RESERVE(pRaw, dataPos, TSDB_MNODE_RESERVE_SIZE, MNODE_ENCODE_OVER)
+  SDB_SET_INT32(pRaw, dataPos, pObj->id, _OVER)
+  SDB_SET_INT64(pRaw, dataPos, pObj->createdTime, _OVER)
+  SDB_SET_INT64(pRaw, dataPos, pObj->updateTime, _OVER)
+  SDB_SET_RESERVE(pRaw, dataPos, MNODE_RESERVE_SIZE, _OVER)
 
   terrno = 0;
 
-MNODE_ENCODE_OVER:
+_OVER:
   if (terrno != 0) {
     mError("mnode:%d, failed to encode to raw:%p since %s", pObj->id, pRaw, terrstr());
     sdbFreeRaw(pRaw);
@@ -156,26 +155,26 @@ static SSdbRow *mndMnodeActionDecode(SSdbRaw *pRaw) {
   int8_t sver = 0;
   if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL;
 
-  if (sver != TSDB_MNODE_VER_NUMBER) {
+  if (sver != MNODE_VER_NUMBER) {
     terrno = TSDB_CODE_SDB_INVALID_DATA_VER;
-    goto MNODE_DECODE_OVER;
+    goto _OVER;
   }
 
   SSdbRow *pRow = sdbAllocRow(sizeof(SMnodeObj));
-  if (pRow == NULL) goto MNODE_DECODE_OVER;
+  if (pRow == NULL) goto _OVER;
 
   SMnodeObj *pObj = sdbGetRowObj(pRow);
-  if (pObj == NULL) goto MNODE_DECODE_OVER;
+  if (pObj == NULL) goto _OVER;
 
   int32_t dataPos = 0;
-  SDB_GET_INT32(pRaw, dataPos, &pObj->id, MNODE_DECODE_OVER)
-  SDB_GET_INT64(pRaw, dataPos, &pObj->createdTime, MNODE_DECODE_OVER)
-  SDB_GET_INT64(pRaw, dataPos, &pObj->updateTime, MNODE_DECODE_OVER)
-  SDB_GET_RESERVE(pRaw, dataPos, TSDB_MNODE_RESERVE_SIZE, MNODE_DECODE_OVER)
+  SDB_GET_INT32(pRaw, dataPos, &pObj->id, _OVER)
+  SDB_GET_INT64(pRaw, dataPos, &pObj->createdTime, _OVER)
+  SDB_GET_INT64(pRaw, dataPos, &pObj->updateTime, _OVER)
+  SDB_GET_RESERVE(pRaw, dataPos, MNODE_RESERVE_SIZE, _OVER)
 
   terrno = 0;
 
-MNODE_DECODE_OVER:
+_OVER:
   if (terrno != 0) {
     mError("mnode:%d, failed to decode from raw:%p since %s", pObj->id, pRaw, terrstr());
     taosMemoryFreeClear(pRow);
@@ -186,8 +185,6 @@ MNODE_DECODE_OVER:
   return pRow;
 }
 
-static void mnodeResetMnode(SMnodeObj *pObj) { pObj->role = TAOS_SYNC_STATE_FOLLOWER; }
-
 static int32_t mndMnodeActionInsert(SSdb *pSdb, SMnodeObj *pObj) {
   mTrace("mnode:%d, perform insert action, row:%p", pObj->id, pObj);
   pObj->pDnode = sdbAcquire(pSdb, SDB_DNODE, &pObj->id);
@@ -197,7 +194,7 @@ static int32_t mndMnodeActionInsert(SSdb *pSdb, SMnodeObj *pObj) {
     return -1;
   }
 
-  mnodeResetMnode(pObj);
+  pObj->role = TAOS_SYNC_STATE_FOLLOWER;
   return 0;
 }
 
@@ -231,7 +228,6 @@ bool mndIsMnode(SMnode *pMnode, int32_t dnodeId) {
 
 void mndGetMnodeEpSet(SMnode *pMnode, SEpSet *pEpSet) {
   SSdb *pSdb = pMnode->pSdb;
-
   pEpSet->numOfEps = 0;
 
   void *pIter = NULL;
@@ -239,14 +235,15 @@ void mndGetMnodeEpSet(SMnode *pMnode, SEpSet *pEpSet) {
     SMnodeObj *pObj = NULL;
     pIter = sdbFetch(pSdb, SDB_MNODE, pIter, (void **)&pObj);
     if (pIter == NULL) break;
-    if (pObj->pDnode == NULL) break;
-
-    if (pObj->role == TAOS_SYNC_STATE_LEADER) {
-      pEpSet->inUse = pEpSet->numOfEps;
+    if (pObj->pDnode == NULL) {
+      mError("mnode:%d, no corresponding dnode exists", pObj->id);
+    } else {
+      if (pObj->role == TAOS_SYNC_STATE_LEADER) {
+        pEpSet->inUse = pEpSet->numOfEps;
+      }
+      addEpIntoEpSet(pEpSet, pObj->pDnode->fqdn, pObj->pDnode->port);
+      sdbRelease(pSdb, pObj);
     }
-
-    addEpIntoEpSet(pEpSet, pObj->pDnode->fqdn, pObj->pDnode->port);
-    sdbRelease(pSdb, pObj);
   }
 }
 
@@ -362,18 +359,18 @@ static int32_t mndCreateMnode(SMnode *pMnode, SNodeMsg *pReq, SDnodeObj *pDnode,
   mnodeObj.updateTime = mnodeObj.createdTime;
 
   STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_CREATE_MNODE, &pReq->rpcMsg);
-  if (pTrans == NULL) goto CREATE_MNODE_OVER;
+  if (pTrans == NULL) goto _OVER;
 
   mDebug("trans:%d, used to create mnode:%d", pTrans->id, pCreate->dnodeId);
-  if (mndSetCreateMnodeRedoLogs(pMnode, pTrans, &mnodeObj) != 0) goto CREATE_MNODE_OVER;
-  if (mndSetCreateMnodeCommitLogs(pMnode, pTrans, &mnodeObj) != 0) goto CREATE_MNODE_OVER;
-  if (mndSetCreateMnodeRedoActions(pMnode, pTrans, pDnode, &mnodeObj) != 0) goto CREATE_MNODE_OVER;
+  if (mndSetCreateMnodeRedoLogs(pMnode, pTrans, &mnodeObj) != 0) goto _OVER;
+  if (mndSetCreateMnodeCommitLogs(pMnode, pTrans, &mnodeObj) != 0) goto _OVER;
+  if (mndSetCreateMnodeRedoActions(pMnode, pTrans, pDnode, &mnodeObj) != 0) goto _OVER;
 
-  if (mndTransPrepare(pMnode, pTrans) != 0) goto CREATE_MNODE_OVER;
+  if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
 
   code = 0;
 
-CREATE_MNODE_OVER:
+_OVER:
   mndTransDrop(pTrans);
   return code;
 }
@@ -388,7 +385,7 @@ static int32_t mndProcessCreateMnodeReq(SNodeMsg *pReq) {
 
   if (tDeserializeSCreateDropMQSBNodeReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &createReq) != 0) {
     terrno = TSDB_CODE_INVALID_MSG;
-    goto CREATE_MNODE_OVER;
+    goto _OVER;
   }
 
   mDebug("mnode:%d, start to create", createReq.dnodeId);
@@ -396,31 +393,31 @@ static int32_t mndProcessCreateMnodeReq(SNodeMsg *pReq) {
   pObj = mndAcquireMnode(pMnode, createReq.dnodeId);
   if (pObj != NULL) {
     terrno = TSDB_CODE_MND_MNODE_ALREADY_EXIST;
-    goto CREATE_MNODE_OVER;
+    goto _OVER;
   } else if (terrno != TSDB_CODE_MND_MNODE_NOT_EXIST) {
-    goto CREATE_MNODE_OVER;
+    goto _OVER;
   }
 
   pDnode = mndAcquireDnode(pMnode, createReq.dnodeId);
   if (pDnode == NULL) {
     terrno = TSDB_CODE_MND_DNODE_NOT_EXIST;
-    goto CREATE_MNODE_OVER;
+    goto _OVER;
   }
 
   pUser = mndAcquireUser(pMnode, pReq->user);
   if (pUser == NULL) {
     terrno = TSDB_CODE_MND_NO_USER_FROM_CONN;
-    goto CREATE_MNODE_OVER;
+    goto _OVER;
   }
 
   if (mndCheckNodeAuth(pUser)) {
-    goto CREATE_MNODE_OVER;
+    goto _OVER;
   }
 
   code = mndCreateMnode(pMnode, pReq, pDnode, &createReq);
   if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
 
-CREATE_MNODE_OVER:
+_OVER:
   if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
     mError("mnode:%d, failed to create since %s", createReq.dnodeId, terrstr());
   }
@@ -529,18 +526,18 @@ static int32_t mndDropMnode(SMnode *pMnode, SNodeMsg *pReq, SMnodeObj *pObj) {
   int32_t code = -1;
 
   STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_DROP_MNODE, &pReq->rpcMsg);
-  if (pTrans == NULL) goto DROP_MNODE_OVER;
+  if (pTrans == NULL) goto _OVER;
 
   mDebug("trans:%d, used to drop mnode:%d", pTrans->id, pObj->id);
 
-  if (mndSetDropMnodeRedoLogs(pMnode, pTrans, pObj) != 0) goto DROP_MNODE_OVER;
-  if (mndSetDropMnodeCommitLogs(pMnode, pTrans, pObj) != 0) goto DROP_MNODE_OVER;
-  if (mndSetDropMnodeRedoActions(pMnode, pTrans, pObj->pDnode, pObj) != 0) goto DROP_MNODE_OVER;
-  if (mndTransPrepare(pMnode, pTrans) != 0) goto DROP_MNODE_OVER;
+  if (mndSetDropMnodeRedoLogs(pMnode, pTrans, pObj) != 0) goto _OVER;
+  if (mndSetDropMnodeCommitLogs(pMnode, pTrans, pObj) != 0) goto _OVER;
+  if (mndSetDropMnodeRedoActions(pMnode, pTrans, pObj->pDnode, pObj) != 0) goto _OVER;
+  if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
 
   code = 0;
 
-DROP_MNODE_OVER:
+_OVER:
   mndTransDrop(pTrans);
   return code;
 }
@@ -554,35 +551,45 @@ static int32_t mndProcessDropMnodeReq(SNodeMsg *pReq) {
 
   if (tDeserializeSCreateDropMQSBNodeReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &dropReq) != 0) {
     terrno = TSDB_CODE_INVALID_MSG;
-    goto DROP_MNODE_OVER;
+    goto _OVER;
   }
 
   mDebug("mnode:%d, start to drop", dropReq.dnodeId);
 
   if (dropReq.dnodeId <= 0) {
-    terrno = TSDB_CODE_SDB_APP_ERROR;
-    goto DROP_MNODE_OVER;
+    terrno = TSDB_CODE_INVALID_MSG;
+    goto _OVER;
   }
 
   pObj = mndAcquireMnode(pMnode, dropReq.dnodeId);
   if (pObj == NULL) {
-    goto DROP_MNODE_OVER;
+    goto _OVER;
+  }
+
+  if (pMnode->selfId == dropReq.dnodeId) {
+    terrno = TSDB_CODE_MND_CANT_DROP_MASTER;
+    goto _OVER;
+  }
+
+  if (sdbGetSize(pMnode->pSdb, SDB_MNODE) <= 1) {
+    terrno = TSDB_CODE_MND_TOO_FEW_MNODES;
+    goto _OVER;
   }
 
   pUser = mndAcquireUser(pMnode, pReq->user);
   if (pUser == NULL) {
     terrno = TSDB_CODE_MND_NO_USER_FROM_CONN;
-    goto DROP_MNODE_OVER;
+    goto _OVER;
   }
 
   if (mndCheckNodeAuth(pUser)) {
-    goto DROP_MNODE_OVER;
+    goto _OVER;
   }
 
   code = mndDropMnode(pMnode, pReq, pObj);
   if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
 
-DROP_MNODE_OVER:
+_OVER:
   if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
     mError("mnode:%d, failed to drop since %s", dropReq.dnodeId, terrstr());
   }
@@ -632,7 +639,7 @@ static int32_t mndRetrieveMnodes(SNodeMsg *pReq, SShowObj *pShow, char *data, in
     cols++;
 
     pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
-    char *roles = mndGetRoleStr(pObj->role);
+    const char *roles = mndGetRoleStr(pObj->role);
     STR_WITH_MAXSIZE_TO_VARSTR(pWrite, roles, pShow->bytes[cols]);
     cols++;
 

source/dnode/mnode/impl/src/mndQnode.c

@@ -396,7 +396,7 @@ static int32_t mndProcessDropQnodeReq(SNodeMsg *pReq) {
   mDebug("qnode:%d, start to drop", dropReq.dnodeId);
 
   if (dropReq.dnodeId <= 0) {
-    terrno = TSDB_CODE_SDB_APP_ERROR;
+    terrno = TSDB_CODE_INVALID_MSG;
     goto _OVER;
   }
 
@@ -451,7 +451,7 @@ static int32_t mndProcessQnodeListReq(SNodeMsg *pReq) {
     goto _OVER;
   }
 
-  while (true) {
+  while (1) {
     void *pIter = sdbFetch(pSdb, SDB_QNODE, NULL, (void **)&pObj);
     if (pIter == NULL) break;
 

source/dnode/mnode/impl/src/mndSnode.c

@@ -404,7 +404,7 @@ static int32_t mndProcessDropSnodeReq(SNodeMsg *pReq) {
   mDebug("snode:%d, start to drop", dropReq.dnodeId);
 
   if (dropReq.dnodeId <= 0) {
-    terrno = TSDB_CODE_SDB_APP_ERROR;
+    terrno = TSDB_CODE_INVALID_MSG;
     goto _OVER;
   }
 

source/dnode/mnode/impl/src/mndSync.c

@@ -105,7 +105,6 @@ static int32_t mndRestoreWal(SMnode *pMnode) {
     if (walEndSnapshot(pWal) < 0) {
       goto WAL_RESTORE_OVER;
     }
-
   }
 
   code = 0;
@@ -129,7 +128,9 @@ int32_t mndInitSync(SMnode *pMnode) {
     return -1;
   }
 
-  pMgmt->state = TAOS_SYNC_STATE_LEADER;
+  if (pMnode->selfId == 1) {
+    pMgmt->state = TAOS_SYNC_STATE_LEADER;
+  }
   pMgmt->pSyncNode = NULL;
   return 0;
 }

source/dnode/mnode/impl/src/mnode.c

@@ -215,7 +215,6 @@ static int32_t mndInitSteps(SMnode *pMnode, bool deploy) {
   } else {
     if (mndAllocStep(pMnode, "mnode-sdb-read", mndReadSdb, NULL) != 0) return -1;
   }
-  // if (mndAllocStep(pMnode, "mnode-timer", mndInitTimer, NULL) != 0) return -1;
   if (mndAllocStep(pMnode, "mnode-profile", mndInitProfile, mndCleanupProfile) != 0) return -1;
   if (mndAllocStep(pMnode, "mnode-show", mndInitShow, mndCleanupShow) != 0) return -1;
   if (mndAllocStep(pMnode, "mnode-query", mndInitQuery, mndCleanupQuery) != 0) return -1;
@@ -272,6 +271,7 @@ static void mndSetOptions(SMnode *pMnode, const SMnodeOpt *pOption) {
   pMnode->selfIndex = pOption->selfIndex;
   memcpy(&pMnode->replicas, pOption->replicas, sizeof(SReplica) * TSDB_MAX_REPLICA);
   pMnode->msgCb = pOption->msgCb;
+  pMnode->selfId = pOption->replicas[pOption->selfIndex].id;
 }
 
 SMnode *mndOpen(const char *path, const SMnodeOpt *pOption) {

source/dnode/mnode/impl/test/trans/trans.cpp

@@ -38,7 +38,7 @@ class MndTestTrans : public ::testing::Test {
 
     test.ServerStop();
 
-    pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
+    pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
     int32_t writeLen = taosWriteFile(pFile, buffer, readLen);
     if (writeLen < 0 || writeLen == readLen) {
       ASSERT(1);

source/dnode/mnode/sdb/CMakeLists.txt

@@ -2,7 +2,7 @@ aux_source_directory(src MNODE_SRC)
 add_library(sdb STATIC ${MNODE_SRC})
 target_include_directories(
     sdb
-    PUBLIC "${CMAKE_SOURCE_DIR}/include/dnode/mnode/sdb"
+    PUBLIC "${TD_SOURCE_DIR}/include/dnode/mnode/sdb"
     PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
 )
 target_link_libraries(

source/dnode/mnode/sdb/src/sdbFile.c

@@ -232,7 +232,7 @@ static int32_t sdbWriteFileImp(SSdb *pSdb) {
   mDebug("start to write file:%s, current ver:%" PRId64 ", commit ver:%" PRId64, curfile, pSdb->curVer,
          pSdb->lastCommitVer);
 
-  TdFilePtr pFile = taosOpenFile(tmpfile, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
+  TdFilePtr pFile = taosOpenFile(tmpfile, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
   if (pFile == NULL) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     mError("failed to open file:%s for write since %s", tmpfile, terrstr());

source/dnode/qnode/CMakeLists.txt

@@ -2,7 +2,7 @@ aux_source_directory(src QNODE_SRC)
 add_library(qnode STATIC ${QNODE_SRC})
 target_include_directories(
     qnode
-    PUBLIC "${CMAKE_SOURCE_DIR}/include/dnode/qnode"
+    PUBLIC "${TD_SOURCE_DIR}/include/dnode/qnode"
     private "${CMAKE_CURRENT_SOURCE_DIR}/inc"
 )
 target_link_libraries(

source/dnode/snode/CMakeLists.txt

@@ -2,7 +2,7 @@ aux_source_directory(src SNODE_SRC)
 add_library(snode STATIC ${SNODE_SRC})
 target_include_directories(
     snode
-    PUBLIC "${CMAKE_SOURCE_DIR}/include/dnode/snode"
+    PUBLIC "${TD_SOURCE_DIR}/include/dnode/snode"
     private "${CMAKE_CURRENT_SOURCE_DIR}/inc"
 )
 target_link_libraries(

source/dnode/vnode/src/inc/tsdb.h

@@ -264,6 +264,9 @@ typedef struct {
 
 #define SBlock SBlockV0  // latest SBlock definition
 
+static FORCE_INLINE bool tsdbIsSupBlock(SBlock *pBlock) { return pBlock->numOfSubBlocks == 1; }
+static FORCE_INLINE bool tsdbIsSubBlock(SBlock *pBlock) { return pBlock->numOfSubBlocks == 0; }
+
 #endif
 
 typedef struct {
@@ -276,7 +279,7 @@ typedef struct {
 #ifdef TD_REFACTOR_3
 typedef struct {
   int16_t  colId;
-  uint16_t bitmap : 1;  // 0: has bitmap if has NULL/NORM rows, 1: no bitmap if all rows are NORM
+  uint16_t bitmap : 1;  // 0: no bitmap if all rows are NORM, 1: has bitmap if has NULL/NORM rows
   uint16_t reserve : 15;
   int32_t  len;
   uint32_t type : 8;
@@ -295,7 +298,7 @@ typedef struct {
   int16_t  colId;
   uint16_t type : 6;
   uint16_t blen : 10;   // bitmap length(TODO: full UT for the bitmap compress of various data input)
-  uint32_t bitmap : 1;  // 0: has bitmap if has NULL/NORM rows, 1: no bitmap if all rows are NORM
+  uint32_t bitmap : 1;  // 0: no bitmap if all rows are NORM, 1: has bitmap if has NULL/NORM rows
   uint32_t len : 31;    // data length + bitmap length
   uint32_t offset;
 } SBlockColV0;
@@ -404,7 +407,7 @@ int   tsdbSetReadTable(SReadH *pReadh, STable *pTable);
 int   tsdbLoadBlockInfo(SReadH *pReadh, void *pTarget);
 int   tsdbLoadBlockData(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlockInfo);
 int   tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo, const int16_t *colIds,
-                            int numOfColsIds);
+                            int numOfColsIds, bool mergeBitmap);
 int   tsdbLoadBlockStatis(SReadH *pReadh, SBlock *pBlock);
 int   tsdbEncodeSBlockIdx(void **buf, SBlockIdx *pIdx);
 void *tsdbDecodeSBlockIdx(void *buf, SBlockIdx *pIdx);

source/dnode/vnode/src/tq/tqMetaStore.c

@@ -95,7 +95,7 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F
     tqError("failed to create dir:%s since %s ", name, terrstr());
   }
   strcat(name, "/" TQ_IDX_NAME);
-  TdFilePtr pIdxFile = taosOpenFile(name, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_READ);
+  TdFilePtr pIdxFile = taosOpenFile(name, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ);
   if (pIdxFile == NULL) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     tqError("failed to open file:%s since %s ", name, terrstr());
@@ -113,7 +113,7 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F
 
   strcpy(name, path);
   strcat(name, "/" TQ_META_NAME);
-  TdFilePtr pFile = taosOpenFile(name, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_READ);
+  TdFilePtr pFile = taosOpenFile(name, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ);
   if (pFile == NULL) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     tqError("failed to open file:%s since %s", name, terrstr());

source/dnode/vnode/src/tsdb/tsdbCommit.c

@@ -1280,7 +1280,7 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF
   uint32_t tsizeAggr = (uint32_t)tsdbBlockAggrSize(nColsNotAllNull, SBlockVerLatest);
   int32_t  keyLen = 0;
   int32_t  nBitmaps = (int32_t)TD_BITMAP_BYTES(rowsToWrite);
-  // int32_t  tBitmaps = 0;
+  int32_t  sBitmaps = isSuper ? (int32_t)TD_BITMAP_BYTES_I(rowsToWrite) : nBitmaps;
 
   for (int ncol = 0; ncol < pDataCols->numOfCols; ++ncol) {
     // All not NULL columns finish
@@ -1292,25 +1292,13 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF
     if (ncol != 0 && (pDataCol->colId != pBlockCol->colId)) continue;
 
     int32_t flen;  // final length
-    int32_t tlen = dataColGetNEleLen(pDataCol, rowsToWrite);
+    int32_t tlen = dataColGetNEleLen(pDataCol, rowsToWrite, pDataCols->bitmapMode);
 
 #ifdef TD_SUPPORT_BITMAP
     int32_t tBitmaps = 0;
     int32_t tBitmapsLen = 0;
     if ((ncol != 0) && !TD_COL_ROWS_NORM(pBlockCol)) {
-      tBitmaps = nBitmaps;
-#if 0
-      if (IS_VAR_DATA_TYPE(pDataCol->type)) {
-        tBitmaps = nBitmaps;
-        tlen += tBitmaps;
-      } else {
-        tBitmaps = (int32_t)ceil((double)nBitmaps / TYPE_BYTES[pDataCol->type]);
-        tlen += tBitmaps * TYPE_BYTES[pDataCol->type];
-      }
-#endif
-      // move bitmap parts ahead
-      // TODO: put bitmap part to the 1st location(pBitmap points to pData) to avoid the memmove
-      // memcpy(POINTER_SHIFT(pDataCol->pData, pDataCol->len), pDataCol->pBitmap, nBitmaps);
+      tBitmaps = isSuper ? sBitmaps : nBitmaps;
     }
 #endif
 
@@ -1340,6 +1328,9 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF
                                                       tlen + COMP_OVERFLOW_BYTES);
       if (tBitmaps > 0) {
         bptr = POINTER_SHIFT(pBlockData, lsize + flen);
+        if (isSuper && !tdDataColsIsBitmapI(pDataCols)) {
+          tdMergeBitmap((uint8_t *)pDataCol->pBitmap, nBitmaps, (uint8_t *)pDataCol->pBitmap);
+        }
         tBitmapsLen =
             tsCompressTinyint((char *)pDataCol->pBitmap, tBitmaps, tBitmaps, bptr, tBitmaps + COMP_OVERFLOW_BYTES,
                               pCfg->compression, *ppCBuf, tBitmaps + COMP_OVERFLOW_BYTES);
@@ -1506,7 +1497,7 @@ static int tsdbMergeMemData(SCommitH *pCommith, SCommitIter *pIter, int bidx) {
   }
 
   SSkipListIterator titer = *(pIter->pIter);
-  if (tsdbLoadBlockDataCols(&(pCommith->readh), pBlock, NULL, &colId, 1) < 0) return -1;
+  if (tsdbLoadBlockDataCols(&(pCommith->readh), pBlock, NULL, &colId, 1, false) < 0) return -1;
 
   tsdbLoadDataFromCache(pIter->pTable, &titer, keyLimit, INT32_MAX, NULL, pCommith->readh.pDCols[0]->cols[0].pData,
                         pCommith->readh.pDCols[0]->numOfRows, pCfg->update, &mInfo);
@@ -1656,6 +1647,8 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt
   ASSERT(maxRows > 0 && dataColsKeyLast(pDataCols) <= maxKey);
   tdResetDataCols(pTarget);
 
+  pTarget->bitmapMode = pDataCols->bitmapMode;
+
   while (true) {
     key1 = (*iter >= pDataCols->numOfRows) ? INT64_MAX : dataColsKeyAt(pDataCols, *iter);
     STSRow *row = tsdbNextIterRow(pCommitIter->pIter);
@@ -1668,36 +1661,37 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt
     if (key1 == INT64_MAX && key2 == INT64_MAX) break;
 
     if (key1 < key2) {
-      for (int i = 0; i < pDataCols->numOfCols; i++) {
+      for (int i = 0; i < pDataCols->numOfCols; ++i) {
         // TODO: dataColAppendVal may fail
         SCellVal sVal = {0};
-        if (tdGetColDataOfRow(&sVal, pDataCols->cols + i, *iter) < 0) {
+        if (tdGetColDataOfRow(&sVal, pDataCols->cols + i, *iter, pDataCols->bitmapMode) < 0) {
           TASSERT(0);
         }
-        tdAppendValToDataCol(pTarget->cols + i, sVal.valType, sVal.val, pTarget->numOfRows, pTarget->maxPoints);
+        tdAppendValToDataCol(pTarget->cols + i, sVal.valType, sVal.val, pTarget->numOfRows, pTarget->maxPoints, pTarget->bitmapMode);
       }
 
-      pTarget->numOfRows++;
-      (*iter)++;
+      ++pTarget->numOfRows;
+      ++(*iter);
     } else if (key1 > key2) {
       if (pSchema == NULL || schemaVersion(pSchema) != TD_ROW_SVER(row)) {
         pSchema = tsdbGetTableSchemaImpl(pCommitIter->pTable, false, false, TD_ROW_SVER(row));
         ASSERT(pSchema != NULL);
       }
 
-      tdAppendSTSRowToDataCol(row, pSchema, pTarget, true);
+      tdAppendSTSRowToDataCol(row, pSchema, pTarget);
 
       tSkipListIterNext(pCommitIter->pIter);
     } else {
+#if 0
       if (update != TD_ROW_OVERWRITE_UPDATE) {
         // copy disk data
-        for (int i = 0; i < pDataCols->numOfCols; i++) {
+        for (int i = 0; i < pDataCols->numOfCols; ++i) {
           // TODO: dataColAppendVal may fail
           SCellVal sVal = {0};
-          if (tdGetColDataOfRow(&sVal, pDataCols->cols + i, *iter) < 0) {
+          if (tdGetColDataOfRow(&sVal, pDataCols->cols + i, *iter, pDataCols->bitmapMode) < 0) {
             TASSERT(0);
           }
-          tdAppendValToDataCol(pTarget->cols + i, sVal.valType, sVal.val, pTarget->numOfRows, pTarget->maxPoints);
+          tdAppendValToDataCol(pTarget->cols + i, sVal.valType, sVal.val, pTarget->numOfRows, pTarget->maxPoints, pTarget->bitmapMode);
         }
 
         if (update == TD_ROW_DISCARD_UPDATE) pTarget->numOfRows++;
@@ -1711,8 +1705,45 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt
 
         tdAppendSTSRowToDataCol(row, pSchema, pTarget, update == TD_ROW_OVERWRITE_UPDATE);
       }
-      (*iter)++;
+      ++(*iter);
       tSkipListIterNext(pCommitIter->pIter);
+#endif
+      // copy disk data
+      for (int i = 0; i < pDataCols->numOfCols; ++i) {
+        SCellVal sVal = {0};
+        if (tdGetColDataOfRow(&sVal, pDataCols->cols + i, *iter, pDataCols->bitmapMode) < 0) {
+          TASSERT(0);
+        }
+        // TODO: tdAppendValToDataCol may fail
+        tdAppendValToDataCol(pTarget->cols + i, sVal.valType, sVal.val, pTarget->numOfRows, pTarget->maxPoints,
+                             pTarget->bitmapMode);
+      }
+
+      if (TD_SUPPORT_UPDATE(update)) {
+        // copy mem data(Multi-Version)
+        if (pSchema == NULL || schemaVersion(pSchema) != TD_ROW_SVER(row)) {
+          pSchema = tsdbGetTableSchemaImpl(pCommitIter->pTable, false, false, TD_ROW_SVER(row));
+          ASSERT(pSchema != NULL);
+        }
+
+        // TODO: merge with Multi-Version
+        STSRow *curRow = row;
+
+        ++(*iter);
+        tSkipListIterNext(pCommitIter->pIter);
+        STSRow *nextRow = tsdbNextIterRow(pCommitIter->pIter);
+
+        if (key2 < TD_ROW_KEY(nextRow)) {
+          tdAppendSTSRowToDataCol(row, pSchema, pTarget);
+        } else {
+          tdAppendSTSRowToDataCol(row, pSchema, pTarget);
+        }
+        // TODO: merge with Multi-Version
+      } else {
+        ++pTarget->numOfRows;
+        ++(*iter);
+        tSkipListIterNext(pCommitIter->pIter);
+      }
     }
 
     if (pTarget->numOfRows >= maxRows) break;

source/dnode/vnode/src/tsdb/tsdbFS.c

@@ -416,7 +416,7 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) {
   tsdbGetTxnFname(pRepo, TSDB_TXN_TEMP_FILE, tfname);
   tsdbGetTxnFname(pRepo, TSDB_TXN_CURR_FILE, cfname);
 
-  TdFilePtr pFile = taosOpenFile(tfname, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
+  TdFilePtr pFile = taosOpenFile(tfname, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
   if (pFile == NULL) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     return -1;

source/dnode/vnode/src/tsdb/tsdbFile.c

@@ -360,7 +360,7 @@ static void *tsdbDecodeSDFileEx(void *buf, SDFile *pDFile) {
 int tsdbCreateDFile(STsdb *pRepo, SDFile *pDFile, bool updateHeader, TSDB_FILE_T fType) {
   ASSERT(pDFile->info.size == 0 && pDFile->info.magic == TSDB_FILE_INIT_MAGIC);
 
-  pDFile->pFile = taosOpenFile(TSDB_FILE_FULL_NAME(pDFile), TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
+  pDFile->pFile = taosOpenFile(TSDB_FILE_FULL_NAME(pDFile), TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
   if (pDFile->pFile == NULL) {
     if (errno == ENOENT) {
       // Try to create directory recursively
@@ -371,7 +371,7 @@ int tsdbCreateDFile(STsdb *pRepo, SDFile *pDFile, bool updateHeader, TSDB_FILE_T
       }
       taosMemoryFreeClear(s);
 
-      pDFile->pFile = taosOpenFile(TSDB_FILE_FULL_NAME(pDFile), TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC);
+      pDFile->pFile = taosOpenFile(TSDB_FILE_FULL_NAME(pDFile), TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
       if (pDFile->pFile == NULL) {
         terrno = TAOS_SYSTEM_ERROR(errno);
         return -1;
@@ -406,7 +406,7 @@ int tsdbUpdateDFileHeader(SDFile *pDFile) {
   }
 
   void *ptr = buf;
-  taosEncodeFixedU32(&ptr, 0);
+  // taosEncodeFixedU32(&ptr, 0); // fver moved to SDFInfo and saved to current
   tsdbEncodeDFInfo(&ptr, &(pDFile->info));
 
   taosCalcChecksumAppend(0, (uint8_t *)buf, TSDB_FILE_HEAD_SIZE);
@@ -437,7 +437,7 @@ int tsdbLoadDFileHeader(SDFile *pDFile, SDFInfo *pInfo) {
   }
 
   void *pBuf = buf;
-  pBuf = taosDecodeFixedU32(pBuf, &_version);
+  // pBuf = taosDecodeFixedU32(pBuf, &_version);
   pBuf = tsdbDecodeDFInfo(pBuf, pInfo);
   return 0;
 }

source/dnode/vnode/src/tsdb/tsdbMemTable.c

@@ -475,7 +475,7 @@ static int tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema *
       }
     }
 
-    tdAppendSTSRowToDataCol(row, *ppSchema, pCols, true);
+    tdAppendSTSRowToDataCol(row, *ppSchema, pCols);
   }
 
   return 0;

source/dnode/vnode/src/tsdb/tsdbRead.c

@@ -761,6 +761,7 @@ static TSKEY extractFirstTraverseKey(STableCheckInfo* pCheckInfo, int32_t order,
   TSKEY r2 = TD_ROW_KEY(rimem);
 
   if (r1 == r2) {
+#if 0
     if(update == TD_ROW_DISCARD_UPDATE){
       pCheckInfo->chosen = CHECKINFO_CHOSEN_IMEM;
       tSkipListIterNext(pCheckInfo->iter);
@@ -771,6 +772,13 @@ static TSKEY extractFirstTraverseKey(STableCheckInfo* pCheckInfo, int32_t order,
     } else {
       pCheckInfo->chosen = CHECKINFO_CHOSEN_BOTH;
     }
+#endif
+    if (TD_SUPPORT_UPDATE(update)) {
+      pCheckInfo->chosen = CHECKINFO_CHOSEN_BOTH;
+    } else {
+      pCheckInfo->chosen = CHECKINFO_CHOSEN_IMEM;
+      tSkipListIterNext(pCheckInfo->iter);
+    }
     return r1;
   } else if (r1 < r2 && ASCENDING_TRAVERSE(order)) {
     pCheckInfo->chosen = CHECKINFO_CHOSEN_MEM;
@@ -1112,7 +1120,7 @@ static int32_t doLoadFileDataBlock(STsdbReadHandle* pTsdbReadHandle, SBlock* pBl
 
   int16_t* colIds = pTsdbReadHandle->defaultLoadColumn->pData;
 
-  int32_t ret = tsdbLoadBlockDataCols(&(pTsdbReadHandle->rhelper), pBlock, pCheckInfo->pCompInfo, colIds, (int)(QH_GET_NUM_OF_COLS(pTsdbReadHandle)));
+  int32_t ret = tsdbLoadBlockDataCols(&(pTsdbReadHandle->rhelper), pBlock, pCheckInfo->pCompInfo, colIds, (int)(QH_GET_NUM_OF_COLS(pTsdbReadHandle)), true);
   if (ret != TSDB_CODE_SUCCESS) {
     int32_t c = terrno;
     assert(c != TSDB_CODE_SUCCESS);
@@ -1401,7 +1409,7 @@ static int32_t doCopyRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, int32_t
 //        memmove(pData, (char*)src->pData + bytes * start, bytes * num);
         for(int32_t k = start; k < num + start; ++k) {
           SCellVal sVal = {0};
-          if (tdGetColDataOfRow(&sVal, src, k) < 0) {
+          if (tdGetColDataOfRow(&sVal, src, k, pCols->bitmapMode) < 0) {
             TASSERT(0);
           }
 
@@ -1415,7 +1423,7 @@ static int32_t doCopyRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, int32_t
         // todo refactor, only copy one-by-one
         for (int32_t k = start; k < num + start; ++k) {
           SCellVal sVal = {0};
-          if(tdGetColDataOfRow(&sVal, src, k) < 0){
+          if(tdGetColDataOfRow(&sVal, src, k, pCols->bitmapMode) < 0){
             TASSERT(0);
           }
 

source/dnode/vnode/src/tsdb/tsdbReadImpl.c

@@ -263,8 +263,9 @@ int tsdbLoadBlockData(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo) {
   for (int i = 1; i < pBlock->numOfSubBlocks; i++) {
     iBlock++;
     if (tsdbLoadBlockDataImpl(pReadh, iBlock, pReadh->pDCols[1]) < 0) return -1;
+    // TODO: use the real maxVersion to replace the UINT64_MAX to support Multi-Version
     if (tdMergeDataCols(pReadh->pDCols[0], pReadh->pDCols[1], pReadh->pDCols[1]->numOfRows, NULL,
-                        update != TD_ROW_PARTIAL_UPDATE) < 0)
+                        update != TD_ROW_PARTIAL_UPDATE, UINT64_MAX) < 0)
       return -1;
   }
 
@@ -276,7 +277,7 @@ int tsdbLoadBlockData(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo) {
 }
 
 int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo, const int16_t *colIds,
-                          int numOfColsIds) {
+                          int numOfColsIds, bool mergeBitmap) {
   ASSERT(pBlock->numOfSubBlocks > 0);
   int8_t update = pReadh->pRepo->config.update;
 
@@ -293,11 +294,23 @@ int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo,
   for (int i = 1; i < pBlock->numOfSubBlocks; i++) {
     iBlock++;
     if (tsdbLoadBlockDataColsImpl(pReadh, iBlock, pReadh->pDCols[1], colIds, numOfColsIds) < 0) return -1;
+    // TODO: use the real maxVersion to replace the UINT64_MAX to support Multi-Version
     if (tdMergeDataCols(pReadh->pDCols[0], pReadh->pDCols[1], pReadh->pDCols[1]->numOfRows, NULL,
-                        update != TD_ROW_PARTIAL_UPDATE) < 0)
+                        update != TD_ROW_PARTIAL_UPDATE, UINT64_MAX) < 0)
       return -1;
   }
 
+  if (mergeBitmap && !tdDataColsIsBitmapI(pReadh->pDCols[0])) {
+    for (int i = 0; i < numOfColsIds; ++i) {
+      SDataCol *pDataCol = pReadh->pDCols[0]->cols + i;
+      if (pDataCol->bitmap) {
+        ASSERT(pDataCol->colId != PRIMARYKEY_TIMESTAMP_COL_ID);
+        tdMergeBitmap(pDataCol->pBitmap, TD_BITMAP_BYTES(pReadh->pDCols[0]->numOfRows), pDataCol->pBitmap);
+        tdDataColsSetBitmapI(pReadh->pDCols[0]);
+      }
+    }
+  }
+
   ASSERT(pReadh->pDCols[0]->numOfRows == pBlock->numOfRows);
   ASSERT(dataColsKeyFirst(pReadh->pDCols[0]) == pBlock->keyFirst);
   ASSERT(dataColsKeyLast(pReadh->pDCols[0]) == pBlock->keyLast);
@@ -499,6 +512,11 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat
   SDFile *pDFile = (pBlock->last) ? TSDB_READ_LAST_FILE(pReadh) : TSDB_READ_DATA_FILE(pReadh);
 
   tdResetDataCols(pDataCols);
+
+  if(tsdbIsSupBlock(pBlock)) {
+    tdDataColsSetBitmapI(pDataCols);
+  }
+
   if (tsdbMakeRoom((void **)(&TSDB_READ_BUF(pReadh)), pBlock->len) < 0) return -1;
 
   SBlockData *pBlockData = (SBlockData *)TSDB_READ_BUF(pReadh);
@@ -692,6 +710,10 @@ static int tsdbLoadBlockDataColsImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *
 
   tdResetDataCols(pDataCols);
 
+  if(tsdbIsSupBlock(pBlock)) {
+    tdDataColsSetBitmapI(pDataCols);
+  }
+
   // If only load timestamp column, no need to load SBlockData part
   if (numOfColIds > 1 && tsdbLoadBlockOffset(pReadh, pBlock) < 0) return -1;
 

source/dnode/vnode/src/tsdb/tsdbSma.c

@@ -465,16 +465,16 @@ static int32_t tsdbSetExpiredWindow(STsdb *pTsdb, SHashObj *pItemsHash, int64_t
  * @return int32_t
  */
 int32_t tsdbUpdateExpiredWindowImpl(STsdb *pTsdb, SSubmitReq *pMsg) {
+  if (atomic_load_16(&REPO_TSMA_NUM(pTsdb)) <= 0) {
+    tsdbTrace("vgId:%d not update expire window since no tSma", REPO_ID(pTsdb));
+    return TSDB_CODE_SUCCESS;
+  }
+
   if (!pTsdb->pMeta) {
     terrno = TSDB_CODE_INVALID_PTR;
     return TSDB_CODE_FAILED;
   }
 
-  if (atomic_load_16(&REPO_TSMA_NUM(pTsdb)) <= 0) {
-    tsdbWarn("vgId:%d not update expire window since no tSma", REPO_ID(pTsdb));
-    return TSDB_CODE_SUCCESS;
-  }
-
   if (tdScanAndConvertSubmitMsg(pMsg) != TSDB_CODE_SUCCESS) {
     return TSDB_CODE_FAILED;
   }

source/dnode/vnode/test/CMakeLists.txt

@@ -11,7 +11,7 @@ AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
 # )
 # target_include_directories(tqTest
 #     PUBLIC
-#     "${CMAKE_SOURCE_DIR}/include/server/vnode/tq"
+#     "${TD_SOURCE_DIR}/include/server/vnode/tq"
 #     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 # )
 
@@ -33,7 +33,7 @@ AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
 
 # TARGET_INCLUDE_DIRECTORIES(
 #         tsdbSmaTest
-#         PUBLIC "${CMAKE_SOURCE_DIR}/include/common"
+#         PUBLIC "${TD_SOURCE_DIR}/include/common"
 #         PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/../src/inc"
 #         PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 # )
\ No newline at end of file

source/libs/cache/CMakeLists.txt

@@ -2,6 +2,6 @@ aux_source_directory(src CACHE_SRC)
 add_library(cache STATIC ${CACHE_SRC})
 target_include_directories(
     cache
-    PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/cache"
+    PUBLIC "${TD_SOURCE_DIR}/include/libs/cache"
     PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
 )
\ No newline at end of file

source/libs/catalog/CMakeLists.txt

@@ -2,7 +2,7 @@ aux_source_directory(src CATALOG_SRC)
 add_library(catalog STATIC ${CATALOG_SRC})
 target_include_directories(
     catalog
-    PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/catalog"
+    PUBLIC "${TD_SOURCE_DIR}/include/libs/catalog"
     PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
 )
 

source/libs/catalog/test/CMakeLists.txt

@@ -13,8 +13,8 @@ TARGET_LINK_LIBRARIES(
 
 TARGET_INCLUDE_DIRECTORIES(
         catalogTest
-        PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/catalog/"
-        PRIVATE "${CMAKE_SOURCE_DIR}/source/libs/catalog/inc"
+        PUBLIC "${TD_SOURCE_DIR}/include/libs/catalog/"
+        PRIVATE "${TD_SOURCE_DIR}/source/libs/catalog/inc"
 )
 
 # add_test(

source/libs/command/CMakeLists.txt

@@ -2,7 +2,7 @@ aux_source_directory(src COMMAND_SRC)
 add_library(command STATIC ${COMMAND_SRC})
 target_include_directories(
         command
-        PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/command"
+        PUBLIC "${TD_SOURCE_DIR}/include/libs/command"
         PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
 )
 

source/libs/command/inc/commandInt.h

@@ -44,6 +44,9 @@ extern "C" {
 #define EXPLAIN_OUTPUT_FORMAT "Output: "
 #define EXPLAIN_TIME_WINDOWS_FORMAT "Time Window: interval=%" PRId64 "%c offset=%" PRId64 "%c sliding=%" PRId64 "%c"
 #define EXPLAIN_WINDOW_FORMAT "Window: gap=%" PRId64
+#define EXPLAIN_RATIO_TIME_FORMAT "Ratio: %f"
+#define EXPLAIN_PLANNING_TIME_FORMAT "Planning Time: %.3f ms"
+#define EXPLAIN_EXEC_TIME_FORMAT "Execution Time: %.3f ms"
 
 //append area
 #define EXPLAIN_LEFT_PARENTHESIS_FORMAT " ("
@@ -108,16 +111,19 @@ typedef struct SExplainCtx {
 #define EXPLAIN_ROW_NEW(level, ...)                                                                               \
   do {                                                                                                            \
     if (isVerboseLine) {                                                                                          \
-      tlen = snprintf(tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, "%*s", (level) * 2 + 3, "");       \
+      tlen = snprintf(tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE - VARSTR_HEADER_SIZE, "%*s", (level) * 2 + 3, "");       \
     } else {                                                                                                      \
-      tlen = snprintf(tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, "%*s%s", (level) * 2, "", "-> ");  \
+      tlen = snprintf(tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE - VARSTR_HEADER_SIZE, "%*s%s", (level) * 2, "", "-> ");  \
     }                                                                                                             \
-    tlen += snprintf(tbuf + VARSTR_HEADER_SIZE + tlen, TSDB_EXPLAIN_RESULT_ROW_SIZE - tlen, __VA_ARGS__);         \
+    tlen += snprintf(tbuf + VARSTR_HEADER_SIZE + tlen, TSDB_EXPLAIN_RESULT_ROW_SIZE - VARSTR_HEADER_SIZE - tlen, __VA_ARGS__);         \
   } while (0)
   
-#define EXPLAIN_ROW_APPEND(...) tlen += snprintf(tbuf + VARSTR_HEADER_SIZE + tlen, TSDB_EXPLAIN_RESULT_ROW_SIZE - tlen, __VA_ARGS__)
+#define EXPLAIN_ROW_APPEND(...) tlen += snprintf(tbuf + VARSTR_HEADER_SIZE + tlen, TSDB_EXPLAIN_RESULT_ROW_SIZE - VARSTR_HEADER_SIZE - tlen, __VA_ARGS__)
 #define EXPLAIN_ROW_END() do { varDataSetLen(tbuf, tlen); tlen += VARSTR_HEADER_SIZE; isVerboseLine = true; } while (0)
 
+#define EXPLAIN_SUM_ROW_NEW(...) tlen = snprintf(tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE - VARSTR_HEADER_SIZE, __VA_ARGS__)
+#define EXPLAIN_SUM_ROW_END() do { varDataSetLen(tbuf, tlen); tlen += VARSTR_HEADER_SIZE; } while (0)
+
 #ifdef __cplusplus
 }
 #endif

source/libs/command/src/explain.c

@@ -52,16 +52,14 @@ void qExplainFreeCtx(SExplainCtx *pCtx) {
     void *pIter = taosHashIterate(pCtx->groupHash, NULL);
     while (pIter) {
       SExplainGroup *group = (SExplainGroup *)pIter;
-      if (NULL == group->nodeExecInfo) {
-        continue;
+      if (group->nodeExecInfo) {
+        int32_t num = taosArrayGetSize(group->nodeExecInfo);
+        for (int32_t i = 0; i < num; ++i) {
+          SExplainRsp *rsp = taosArrayGet(group->nodeExecInfo, i);
+          taosMemoryFreeClear(rsp->subplanInfo);
+        }
       }
       
-      int32_t num = taosArrayGetSize(group->nodeExecInfo);
-      for (int32_t i = 0; i < num; ++i) {
-        SExplainRsp *rsp = taosArrayGet(group->nodeExecInfo, i);
-        taosMemoryFreeClear(rsp->subplanInfo);
-      }
-    
       pIter = taosHashIterate(pCtx->groupHash, pIter);
     }
   }
@@ -641,7 +639,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
     }
     case QUERY_NODE_PHYSICAL_PLAN_INTERVAL:{
       SIntervalPhysiNode *pIntNode = (SIntervalPhysiNode *)pNode;
-      EXPLAIN_ROW_NEW(level, EXPLAIN_INTERVAL_FORMAT, nodesGetNameFromColumnNode(pIntNode->pTspk));
+      EXPLAIN_ROW_NEW(level, EXPLAIN_INTERVAL_FORMAT, nodesGetNameFromColumnNode(pIntNode->window.pTspk));
       EXPLAIN_ROW_APPEND(EXPLAIN_LEFT_PARENTHESIS_FORMAT);
       if (pResNode->pExecInfo) {
         QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen));
@@ -896,9 +894,33 @@ _return:
   QRY_RET(code);
 }
 
+int32_t qExplainAppendPlanRows(SExplainCtx *pCtx) {
+  if (EXPLAIN_MODE_ANALYZE != pCtx->mode) {
+    return TSDB_CODE_SUCCESS;
+  }
+
+  int32_t tlen = 0;
+  char *tbuf = pCtx->tbuf;
+
+  EXPLAIN_SUM_ROW_NEW(EXPLAIN_RATIO_TIME_FORMAT, pCtx->ratio);
+  EXPLAIN_SUM_ROW_END();
+  QRY_ERR_RET(qExplainResAppendRow(pCtx, tbuf, tlen, 0));
+
+  EXPLAIN_SUM_ROW_NEW(EXPLAIN_PLANNING_TIME_FORMAT, (double)(pCtx->jobStartTs - pCtx->reqStartTs) / 1000.0);
+  EXPLAIN_SUM_ROW_END();
+  QRY_ERR_RET(qExplainResAppendRow(pCtx, tbuf, tlen, 0));
+
+  EXPLAIN_SUM_ROW_NEW(EXPLAIN_EXEC_TIME_FORMAT, (double)(pCtx->jobDoneTs - pCtx->jobStartTs) / 1000.0);
+  EXPLAIN_SUM_ROW_END();
+  QRY_ERR_RET(qExplainResAppendRow(pCtx, tbuf, tlen, 0));
+
+  return TSDB_CODE_SUCCESS;
+}
 
 int32_t qExplainGenerateRsp(SExplainCtx *pCtx, SRetrieveTableRsp **pRsp) {
   QRY_ERR_RET(qExplainAppendGroupResRows(pCtx, pCtx->rootGroupId, 0));
+
+  QRY_ERR_RET(qExplainAppendPlanRows(pCtx));
   
   QRY_ERR_RET(qExplainGetRspFromCtx(pCtx, pRsp));
 
@@ -979,18 +1001,18 @@ _return:
   QRY_RET(code);
 }
 
-int32_t qExecExplainBegin(SQueryPlan *pDag, SExplainCtx **pCtx, int32_t startTs) {
+int32_t qExecExplainBegin(SQueryPlan *pDag, SExplainCtx **pCtx, int64_t startTs) {
   QRY_ERR_RET(qExplainPrepareCtx(pDag, pCtx));
   
   (*pCtx)->reqStartTs = startTs;
-  (*pCtx)->jobStartTs = taosGetTimestampMs();
+  (*pCtx)->jobStartTs = taosGetTimestampUs();
 
   return TSDB_CODE_SUCCESS;
 }
 
 int32_t qExecExplainEnd(SExplainCtx *pCtx, SRetrieveTableRsp **pRsp) {
   int32_t code = 0;
-  pCtx->jobDoneTs = taosGetTimestampMs();
+  pCtx->jobDoneTs = taosGetTimestampUs();
   
   atomic_store_8((int8_t *)&pCtx->execDone, true);
 

source/libs/command/test/CMakeLists.txt

@@ -8,8 +8,8 @@ ADD_EXECUTABLE(commandTest ${SOURCE_LIST})
 
 TARGET_INCLUDE_DIRECTORIES(
         commandTest
-        PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/command/"
-        PRIVATE "${CMAKE_SOURCE_DIR}/source/libs/command/inc"
+        PUBLIC "${TD_SOURCE_DIR}/include/libs/command/"
+        PRIVATE "${TD_SOURCE_DIR}/source/libs/command/inc"
 )
 
 TARGET_LINK_LIBRARIES(

source/libs/executor/CMakeLists.txt

@@ -4,7 +4,7 @@ aux_source_directory(src EXECUTOR_SRC)
 add_library(executor STATIC ${EXECUTOR_SRC})
 #set_target_properties(executor PROPERTIES
 #        IMPORTED_LOCATION "${CMAKE_CURRENT_SOURCE_DIR}/libexecutor.a"
-#        INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_SOURCE_DIR}/include/libs/executor"
+#        INTERFACE_INCLUDE_DIRECTORIES "${TD_SOURCE_DIR}/include/libs/executor"
 #        )
 
 target_link_libraries(executor
@@ -13,7 +13,7 @@ target_link_libraries(executor
 
 target_include_directories(
         executor
-        PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/executor"
+        PUBLIC "${TD_SOURCE_DIR}/include/libs/executor"
         PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
 )
 

source/libs/executor/inc/executorimpl.h

@@ -232,9 +232,11 @@ typedef struct STaskAttr {
 } STaskAttr;
 
 struct SOperatorInfo;
+struct SAggSupporter;
+struct SOptrBasicInfo;
 
-typedef void (*__optr_encode_fn_t)(struct SOperatorInfo* pOperator, char **result, int32_t *length);
-typedef bool (*__optr_decode_fn_t)(struct SOperatorInfo* pOperator, char *result, int32_t length);
+typedef void (*__optr_encode_fn_t)(struct SOperatorInfo* pOperator, struct SAggSupporter *pSup, struct SOptrBasicInfo *pInfo, char **result, int32_t *length);
+typedef bool (*__optr_decode_fn_t)(struct SOperatorInfo* pOperator, struct SAggSupporter *pSup, struct SOptrBasicInfo *pInfo, char *result, int32_t length);
 
 typedef int32_t (*__optr_open_fn_t)(struct SOperatorInfo* pOptr);
 typedef SSDataBlock* (*__optr_fn_t)(struct SOperatorInfo* pOptr, bool* newgroup);
@@ -753,6 +755,9 @@ void    setTaskStatus(SExecTaskInfo* pTaskInfo, int8_t status);
 int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SReadHandle* pHandle, uint64_t taskId, EOPTR_EXEC_MODEL model);
 int32_t getOperatorExplainExecInfo(SOperatorInfo *operatorInfo, SExplainExecInfo **pRes, int32_t *capacity, int32_t *resNum);
 
+bool aggDecodeResultRow(SOperatorInfo* pOperator, SAggSupporter *pSup, SOptrBasicInfo *pInfo, char* result, int32_t length);
+void aggEncodeResultRow(SOperatorInfo* pOperator, SAggSupporter *pSup, SOptrBasicInfo *pInfo, char **result, int32_t *length);
+
 #ifdef __cplusplus
 }
 #endif