[td-186] merge develop branch

.travis.yml

@@ -32,15 +32,15 @@ matrix:
       - cd debug
 
     script:
-      - cmake ..
-      - make
+      - cmake .. > /dev/null
+      - make > /dev/null
 
     after_success:
       - |-
         case $TRAVIS_OS_NAME in
           linux)
           cd ${TRAVIS_BUILD_DIR}/debug
-          make install || travis_terminate $?
+          make install > /dev/null || travis_terminate $?
 
           pip install --user ${TRAVIS_BUILD_DIR}/src/connector/python/linux/python2/
           pip3 install --user ${TRAVIS_BUILD_DIR}/src/connector/python/linux/python3/
@@ -98,11 +98,11 @@ matrix:
 
         # Commands to prepare for build_command
         # ** likely specific to your build **
-        build_command_prepend: cmake .
+        build_command_prepend: cmake . > /dev/null
 
         # The command that will be added as an argument to "cov-build" to compile your project for analysis,
         # ** likely specific to your build **
-        build_command: make
+        build_command: make > /dev/null
 
         # Pattern to match selecting branches that will run analysis. We recommend leaving this set to 'coverity_scan'.
         # Take care in resource usage, and consider the build frequency allowances per
@@ -135,15 +135,15 @@ matrix:
       - cd debug
 
     script:
-      - cmake -DCOVER=true ..
-      - make
+      - cmake -DCOVER=true .. > /dev/null
+      - make > /dev/null
 
     after_success:
       - |-
         case $TRAVIS_OS_NAME in
           linux)
           cd ${TRAVIS_BUILD_DIR}/debug
-          make install || travis_terminate $?
+          make install > /dev/null || travis_terminate $?
 
           pip install --user ${TRAVIS_BUILD_DIR}/src/connector/python/linux/python2/
           pip3 install --user ${TRAVIS_BUILD_DIR}/src/connector/python/linux/python3/
@@ -208,8 +208,8 @@ matrix:
       - cd debug
 
     script:
-      - cmake ..
-      - make
+      - cmake .. > /dev/null
+      - make > /dev/null
 
         #  - os: osx
         #    language: c
@@ -225,5 +225,5 @@ matrix:
         #      - cd ${TRAVIS_BUILD_DIR}
         #      - mkdir debug
         #      - cd debug
-        #      - cmake ..
-        #      - make
+        #      - cmake .. > /dev/null
+        #      - make > /dev/null

src/client/src/tscSQLParser.c

@@ -303,11 +303,8 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) {
         return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg);
       }
 
-//      SSQLToken* pIpAddr = &pInfo->pDCLInfo->a[0];
-//      if (!validateIpAddress(pIpAddr->z, pIpAddr->n)) {
-//        return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg);
-//      }
-
+      SSQLToken* pIpAddr = &pInfo->pDCLInfo->a[0];
+      pIpAddr->n = strdequote(pIpAddr->z);
       break;
     }
 

src/client/src/tscServer.c

@@ -79,15 +79,18 @@ void tscSetMgmtIpListFromEdge() {
   if (tscMgmtIpSet.numOfIps != 1) {
     tscMgmtIpSet.numOfIps = 1;
     tscMgmtIpSet.inUse = 0;
-    taosGetFqdnPortFromEp(tsMaster, tscMgmtIpSet.fqdn[0], &tscMgmtIpSet.port[0]);
+    taosGetFqdnPortFromEp(tsFirst, tscMgmtIpSet.fqdn[0], &tscMgmtIpSet.port[0]);
     tscTrace("edge mgmt IP list:");
     tscPrintMgmtIp();
   }
 }
 
 void tscUpdateIpSet(void *ahandle, SRpcIpSet *pIpSet) {
-  tscTrace("mgmt IP list is changed for ufp is called");
   tscMgmtIpSet = *pIpSet;
+  tscTrace("mgmt IP list is changed for ufp is called, numOfIps:%d inUse:%d", tscMgmtIpSet.numOfIps, tscMgmtIpSet.inUse);
+  for (int32_t i = 0; i < tscMgmtIpSet.numOfIps; ++i) {
+    tscTrace("index:%d fqdn:%s port:%d", i, tscMgmtIpSet.fqdn[i], tscMgmtIpSet.port[i]);
+  }
 }
 
 void tscSetMgmtIpList(SRpcIpSet *pIpList) {

src/client/src/tscSql.c

@@ -80,8 +80,8 @@ STscObj *taosConnectImpl(const char *ip, const char *user, const char *pass, con
     strcpy(tscMgmtIpSet.fqdn[0], ip);
     tscMgmtIpSet.port[0] = port? port: tsMnodeShellPort;
   } else {
-    if (tsMaster[0] != 0) {
-      taosGetFqdnPortFromEp(tsMaster, tscMgmtIpSet.fqdn[tscMgmtIpSet.numOfIps], &tscMgmtIpSet.port[tscMgmtIpSet.numOfIps]);
+    if (tsFirst[0] != 0) {
+      taosGetFqdnPortFromEp(tsFirst, tscMgmtIpSet.fqdn[tscMgmtIpSet.numOfIps], &tscMgmtIpSet.port[tscMgmtIpSet.numOfIps]);
       tscMgmtIpSet.numOfIps++;
     }
 

src/client/src/tscSystem.c

@@ -136,9 +136,9 @@ void taos_init_imp() {
 
   tscMgmtIpSet.inUse = 0;
   tscMgmtIpSet.numOfIps = 1;
-  taosGetFqdnPortFromEp(tsMaster, tscMgmtIpSet.fqdn[0], &tscMgmtIpSet.port[0]);
+  taosGetFqdnPortFromEp(tsFirst, tscMgmtIpSet.fqdn[0], &tscMgmtIpSet.port[0]);
 
-  if (tsSecond[0] && strcmp(tsSecond, tsMaster) != 0) {
+  if (tsSecond[0] && strcmp(tsSecond, tsFirst) != 0) {
     tscMgmtIpSet.numOfIps = 2;
     taosGetFqdnPortFromEp(tsSecond, tscMgmtIpSet.fqdn[1], &tscMgmtIpSet.port[1]);
   }

src/client/src/tscUtil.c

@@ -591,47 +591,44 @@ int32_t tscGetDataBlockFromList(void* pHashList, SDataBlockList* pDataBlockList,
   return TSDB_CODE_SUCCESS;
 }
 
-static void trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock) {
-  int32_t firstPartLen = 0;
+static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock) {
+  // TODO: optimize this function
+  int len = 0;
 
   STableMeta*   pTableMeta = pTableDataBlock->pTableMeta;
   STableComInfo tinfo = tscGetTableInfo(pTableMeta);
   SSchema*      pSchema = tscGetTableSchema(pTableMeta);
 
+  SSubmitBlk* pBlock = pDataBlock;
   memcpy(pDataBlock, pTableDataBlock->pData, sizeof(SSubmitBlk));
   pDataBlock += sizeof(SSubmitBlk);
 
-  int32_t total = sizeof(int32_t)*2;
-  for(int32_t i = 0; i < tinfo.numOfColumns; ++i) {
-    switch (pSchema[i].type) {
-      case TSDB_DATA_TYPE_NCHAR:
-      case TSDB_DATA_TYPE_BINARY: {
-        assert(0);  // not support binary yet
-        firstPartLen += sizeof(int32_t);break;
-      }
-      default:
-        firstPartLen += tDataTypeDesc[pSchema[i].type].nSize;
-        total += tDataTypeDesc[pSchema[i].type].nSize;
-    }
+  int32_t flen = 0;
+  for (int32_t i = 0; i < tinfo.numOfColumns; ++i) {
+    flen += TYPE_BYTES[pSchema[i].type];
   }
 
   char* p = pTableDataBlock->pData + sizeof(SSubmitBlk);
+  pBlock->len = 0;
+  for (int32_t i = 0; i < htons(pBlock->numOfRows); ++i) {
+    SDataRow trow = (SDataRow)pDataBlock;
+    dataRowSetLen(trow, TD_DATA_ROW_HEAD_SIZE + flen);
 
-  SSubmitBlk* pBlock = (SSubmitBlk*) pTableDataBlock->pData;
-  int32_t rows = htons(pBlock->numOfRows);
-  
-  for(int32_t i = 0; i < rows; ++i) {
-    *(int32_t*) pDataBlock = total;
-    pDataBlock += sizeof(int32_t);
-    
-    *(int32_t*) pDataBlock = firstPartLen;
-    pDataBlock += sizeof(int32_t);
-    
-    memcpy(pDataBlock, p, pTableDataBlock->rowSize);
+    int toffset = 0;
+    for (int32_t j = 0; j < tinfo.numOfColumns; j++) {
+      tdAppendColVal(trow, p, pSchema[j].type, pSchema[j].bytes, toffset);
+      toffset += TYPE_BYTES[pSchema[j].type];
+      p += pSchema[j].bytes;
+    }
 
-    p += pTableDataBlock->rowSize;
-    pDataBlock += pTableDataBlock->rowSize;
+    // p += pTableDataBlock->rowSize;
+    pDataBlock += dataRowLen(trow);
+    pBlock->len += dataRowLen(trow);
   }
+
+  len = pBlock->len;
+  pBlock->len = htonl(pBlock->len);
+  return len;
 }
 
 int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SDataBlockList* pTableDataBlockList) {
@@ -694,7 +691,7 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SDataBlockList* pTableDataBlockLi
     pBlocks->len = htonl(len);
     
     // erase the empty space reserved for binary data
-    trimDataBlock(dataBuf->pData + dataBuf->size, pOneTableBlock);
+    len = trimDataBlock(dataBuf->pData + dataBuf->size, pOneTableBlock);
     dataBuf->size += (len + sizeof(SSubmitBlk));
     dataBuf->numOfTables += 1;
   }

src/common/inc/tdataformat.h

@@ -20,6 +20,7 @@
 #include <string.h>
 
 #include "taosdef.h"
+#include "tutil.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -30,7 +31,7 @@ typedef struct {
   int8_t  type;    // Column type
   int16_t colId;   // column ID
   int32_t bytes;   // column bytes
-  int32_t offset;  // point offset in a row data
+  int32_t offset;  // point offset in SDataRow after the header part
 } STColumn;
 
 #define colType(col) ((col)->type)
@@ -43,26 +44,25 @@ typedef struct {
 #define colSetBytes(col, b) (colBytes(col) = (b))
 #define colSetOffset(col, o) (colOffset(col) = (o))
 
-STColumn *tdNewCol(int8_t type, int16_t colId, int16_t bytes);
-void      tdFreeCol(STColumn *pCol);
-void      tdColCpy(STColumn *dst, STColumn *src);
-void      tdSetCol(STColumn *pCol, int8_t type, int16_t colId, int32_t bytes);
-
 // ----------------- TSDB SCHEMA DEFINITION
 typedef struct {
+  int      totalCols;  // Total columns allocated
   int      numOfCols;  // Number of columns appended
-  int      padding;  // Total columns allocated
+  int      tlen;       // maximum length of a SDataRow without the header part
+  int      flen;       // First part length in a SDataRow after the header part
   STColumn columns[];
 } STSchema;
 
 #define schemaNCols(s) ((s)->numOfCols)
+#define schemaTotalCols(s) ((s)->totalCols)
+#define schemaTLen(s) ((s)->tlen)
+#define schemaFLen(s) ((s)->flen)
 #define schemaColAt(s, i) ((s)->columns + i)
 
 STSchema *tdNewSchema(int32_t nCols);
-int       tdSchemaAppendCol(STSchema *pSchema, int8_t type, int16_t colId, int32_t bytes);
+#define   tdFreeSchema(s) tfree((s))
+int       tdSchemaAddCol(STSchema *pSchema, int8_t type, int16_t colId, int32_t bytes);
 STSchema *tdDupSchema(STSchema *pSchema);
-void      tdFreeSchema(STSchema *pSchema);
-void      tdUpdateSchema(STSchema *pSchema);
 int       tdGetSchemaEncodeSize(STSchema *pSchema);
 void *    tdEncodeSchema(void *dst, STSchema *pSchema);
 STSchema *tdDecodeSchema(void **psrc);
@@ -70,53 +70,100 @@ STSchema *tdDecodeSchema(void **psrc);
 // ----------------- Data row structure
 
 /* A data row, the format is like below:
- * +----------+---------+---------------------------------+---------------------------------+
- * | int32_t  | int32_t |                                 |                                 |
- * +----------+---------+---------------------------------+---------------------------------+
- * |   len    |  flen   |           First part            |             Second part         |
- * +----------+---------+---------------------------------+---------------------------------+
- * plen: first part length
- * len: the length including sizeof(row) + sizeof(len)
- * row: actual row data encoding
+ * |<------------------------------------- len ---------------------------------->|
+ * |<--Head ->|<---------   flen -------------->|                                 |
+ * +----------+---------------------------------+---------------------------------+
+ * | int32_t  |                                 |                                 |
+ * +----------+---------------------------------+---------------------------------+
+ * |   len    |           First part            |             Second part         |
+ * +----------+---------------------------------+---------------------------------+
  */
 typedef void *SDataRow;
 
-
-#define TD_DATA_ROW_HEAD_SIZE (2 * sizeof(int32_t))
+#define TD_DATA_ROW_HEAD_SIZE sizeof(int32_t)
 
 #define dataRowLen(r) (*(int32_t *)(r))
-#define dataRowFLen(r) (*(int32_t *)((char *)(r) + sizeof(int32_t)))
-#define dataRowTuple(r) ((char *)(r) + TD_DATA_ROW_HEAD_SIZE)
+#define dataRowTuple(r) POINTER_DRIFT(r, TD_DATA_ROW_HEAD_SIZE)
 #define dataRowKey(r) (*(TSKEY *)(dataRowTuple(r)))
 #define dataRowSetLen(r, l) (dataRowLen(r) = (l))
-#define dataRowSetFLen(r, l) (dataRowFLen(r) = (l))
-#define dataRowIdx(r, i) ((char *)(r) + i)
 #define dataRowCpy(dst, r) memcpy((dst), (r), dataRowLen(r))
-#define dataRowAt(r, idx) ((char *)(r) + (idx))
+#define dataRowMaxBytesFromSchema(s) (schemaTLen(s) + TD_DATA_ROW_HEAD_SIZE)
 
-void     tdInitDataRow(SDataRow row, STSchema *pSchema);
-int      tdMaxRowBytesFromSchema(STSchema *pSchema);
-SDataRow tdNewDataRow(int32_t bytes, STSchema *pSchema);
 SDataRow tdNewDataRowFromSchema(STSchema *pSchema);
 void     tdFreeDataRow(SDataRow row);
-int      tdAppendColVal(SDataRow row, void *value, STColumn *pCol);
-void     tdDataRowReset(SDataRow row, STSchema *pSchema);
+void     tdInitDataRow(SDataRow row, STSchema *pSchema);
+int      tdAppendColVal(SDataRow row, void *value, int8_t type, int32_t bytes, int32_t offset);
 SDataRow tdDataRowDup(SDataRow row);
 
+// NOTE: offset here including the header size
+static FORCE_INLINE void *tdGetRowDataOfCol(SDataRow row, int8_t type, int32_t offset) {
+  switch (type) {
+    case TSDB_DATA_TYPE_BINARY:
+    case TSDB_DATA_TYPE_NCHAR:
+      return POINTER_DRIFT(row, *(VarDataOffsetT *)POINTER_DRIFT(row, offset));
+      break;
+    default:
+      return POINTER_DRIFT(row, offset);
+      break;
+  }
+}
+
 // ----------------- Data column structure
 typedef struct SDataCol {
-  int8_t  type;
-  int16_t colId;
-  int     bytes;
-  int     len;
-  int     offset;
-  void *  pData; // Original data
+  int8_t          type;       // column type
+  int16_t         colId;      // column ID
+  int             bytes;      // column data bytes defined
+  int             offset;     // data offset in a SDataRow (including the header size)
+  int             spaceSize;  // Total space size for this column
+  int             len;        // column data length
+  VarDataOffsetT *dataOff;    // For binary and nchar data, the offset in the data column
+  void *          pData;      // Actual data pointer
 } SDataCol;
 
+static FORCE_INLINE void dataColReset(SDataCol *pDataCol) { pDataCol->len = 0; }
+
+void dataColInit(SDataCol *pDataCol, STColumn *pCol, void **pBuf, int maxPoints);
+void dataColAppendVal(SDataCol *pCol, void *value, int numOfPoints, int maxPoints);
+void dataColPopPoints(SDataCol *pCol, int pointsToPop, int numOfPoints);
+void dataColSetOffset(SDataCol *pCol, int nEle);
+
+bool isNEleNull(SDataCol *pCol, int nEle);
+void dataColSetNEleNull(SDataCol *pCol, int nEle, int maxPoints);
+
+// Get the data pointer from a column-wised data
+static FORCE_INLINE void *tdGetColDataOfRow(SDataCol *pCol, int row) {
+  switch (pCol->type) {
+    case TSDB_DATA_TYPE_BINARY:
+    case TSDB_DATA_TYPE_NCHAR:
+      return POINTER_DRIFT(pCol->pData, pCol->dataOff[row]);
+      break;
+
+    default:
+      return POINTER_DRIFT(pCol->pData, TYPE_BYTES[pCol->type] * row);
+      break;
+  }
+}
+
+static FORCE_INLINE int32_t dataColGetNEleLen(SDataCol *pDataCol, int rows) {
+  ASSERT(rows > 0);
+
+  switch (pDataCol->type) {
+    case TSDB_DATA_TYPE_BINARY:
+    case TSDB_DATA_TYPE_NCHAR:
+      return pDataCol->dataOff[rows - 1] + varDataTLen(tdGetColDataOfRow(pDataCol, rows - 1));
+      break;
+    default:
+      return TYPE_BYTES[pDataCol->type] * rows;
+  }
+}
+
+
 typedef struct {
   int      maxRowSize;
   int      maxCols;    // max number of columns
   int      maxPoints;  // max number of points
+  int      bufSize;
+
   int      numOfPoints;
   int      numOfCols;  // Total number of cols
   int      sversion;   // TODO: set sversion
@@ -125,7 +172,7 @@ typedef struct {
 } SDataCols;
 
 #define keyCol(pCols) (&((pCols)->cols[0]))  // Key column
-#define dataColsKeyAt(pCols, idx) ((int64_t *)(keyCol(pCols)->pData))[(idx)]
+#define dataColsKeyAt(pCols, idx) ((TSKEY *)(keyCol(pCols)->pData))[(idx)]
 #define dataColsKeyFirst(pCols) dataColsKeyAt(pCols, 0)
 #define dataColsKeyLast(pCols) dataColsKeyAt(pCols, (pCols)->numOfPoints - 1)
 

src/common/inc/tglobal.h

@@ -51,7 +51,7 @@ extern int32_t tsVersion;
 extern int32_t tscEmbedded;
 extern int64_t tsMsPerDay[2];
 
-extern char  tsMaster[];
+extern char  tsFirst[];
 extern char  tsSecond[];
 extern char  tsLocalEp[];
 extern uint16_t tsServerPort;

src/common/inc/tulog.h

@@ -43,6 +43,8 @@ extern int32_t tscEmbedded;
   }
 #define uPrint(...) \
   { taosPrintLog("UTL ", tscEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }
+#define uForcePrint(...) \
+  { taosPrintLog("ERROR UTL ", 255, __VA_ARGS__); }
 
 #define pError(...) \
   { taosPrintLog("ERROR APP ", 255, __VA_ARGS__); }  

src/common/src/tglobal.c

@@ -25,6 +25,7 @@
 #include "tutil.h"
 #include "tlocale.h"
 #include "ttimezone.h"
+#include "tsync.h"
 
 char configDir[TSDB_FILENAME_LEN] = "/etc/taos";
 char tsVnodeDir[TSDB_FILENAME_LEN] = {0};
@@ -60,8 +61,9 @@ int32_t tscEmbedded = 0;
  */
 int64_t tsMsPerDay[] = {86400000L, 86400000000L};
 
-char  tsMaster[TSDB_FQDN_LEN] = {0};
+char  tsFirst[TSDB_FQDN_LEN] = {0};
 char  tsSecond[TSDB_FQDN_LEN] = {0};
+char  tsArbitrator[TSDB_FQDN_LEN] = {0};
 char  tsLocalEp[TSDB_FQDN_LEN] = {0};  // Local End Point, hostname:port
 uint16_t tsServerPort = 6030;
 uint16_t tsMnodeShellPort = 6030;   // udp[6030-6034] tcp[6030]
@@ -141,6 +143,7 @@ int32_t qdebugFlag = 131;
 int32_t rpcDebugFlag = 131;
 int32_t uDebugFlag = 131;
 int32_t debugFlag = 131;
+int32_t sDebugFlag = 131;
 
 // the maximum number of results for projection query on super table that are returned from
 // one virtual node, to order according to timestamp
@@ -269,9 +272,9 @@ static void doInitGlobalConfig() {
   SGlobalCfg cfg = {0};
   
   // ip address
-  cfg.option = "master";
-  cfg.ptr = tsMaster;
-  cfg.valType = TAOS_CFG_VTYPE_IPSTR;
+  cfg.option = "first";
+  cfg.ptr = tsFirst;
+  cfg.valType = TAOS_CFG_VTYPE_STRING;
   cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT;
   cfg.minValue = 0;
   cfg.maxValue = 0;
@@ -281,7 +284,7 @@ static void doInitGlobalConfig() {
 
   cfg.option = "second";
   cfg.ptr = tsSecond;
-  cfg.valType = TAOS_CFG_VTYPE_IPSTR;
+  cfg.valType = TAOS_CFG_VTYPE_STRING;
   cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT;
   cfg.minValue = 0;
   cfg.maxValue = 0;
@@ -341,6 +344,16 @@ static void doInitGlobalConfig() {
   cfg.unitType = TAOS_CFG_UTYPE_NONE;
   taosInitConfigOption(cfg);
 
+  cfg.option = "arbitrator";
+  cfg.ptr = tsArbitrator;
+  cfg.valType = TAOS_CFG_VTYPE_STRING;
+  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT;
+  cfg.minValue = 0;
+  cfg.maxValue = 0;
+  cfg.ptrLength = TSDB_FQDN_LEN;
+  cfg.unitType = TAOS_CFG_UTYPE_NONE;
+  taosInitConfigOption(cfg);
+
   // dnode configs
   cfg.option = "numOfThreadsPerCore";
   cfg.ptr = &tsNumOfThreadsPerCore;
@@ -1012,6 +1025,16 @@ static void doInitGlobalConfig() {
   cfg.unitType = TAOS_CFG_UTYPE_NONE;
   taosInitConfigOption(cfg);
 
+  cfg.option = "sDebugFlag";
+  cfg.ptr = &sDebugFlag;
+  cfg.valType = TAOS_CFG_VTYPE_INT32;
+  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG;
+  cfg.minValue = 0;
+  cfg.maxValue = 255;
+  cfg.ptrLength = 0;
+  cfg.unitType = TAOS_CFG_UTYPE_NONE;
+  taosInitConfigOption(cfg);
+
   cfg.option = "sdbDebugFlag";
   cfg.ptr = &sdbDebugFlag;
   cfg.valType = TAOS_CFG_VTYPE_INT32;
@@ -1181,9 +1204,10 @@ void taosInitGlobalCfg() {
 bool taosCheckGlobalCfg() {
   taosGetFqdn(tsLocalEp);
   sprintf(tsLocalEp + strlen(tsLocalEp), ":%d", tsServerPort);
+  uPrint("localEp is %s", tsLocalEp);
 
-  if (tsMaster[0] == 0) {
-    strcpy(tsMaster, tsLocalEp);
+  if (tsFirst[0] == 0) {
+    strcpy(tsFirst, tsLocalEp);
   }
 
   if (tsSecond[0] == 0) {

src/common/src/tlocale.c

@@ -27,27 +27,20 @@
  * In case that the setLocale failed to be executed, the right charset needs to be set.
  */
 void tsSetLocale() {
-  char msgLocale[] = "Invalid locale:%s, please set the valid locale in config file\n";
-  char msgCharset[] = "Invalid charset:%s, please set the valid charset in config file\n";
-  char msgCharset1[] = "failed to get charset, please set the valid charset in config file\n";
-
   char *locale = setlocale(LC_CTYPE, tsLocale);
 
   // default locale or user specified locale is not valid, abort launch
   if (locale == NULL) {
-    printf(msgLocale, tsLocale);
-    uPrint(msgLocale, tsLocale);
+    uForcePrint("Invalid locale:%s, please set the valid locale in config file", tsLocale);
   }
 
   if (strlen(tsCharset) == 0) {
-    printf("%s\n", msgCharset1);
-    uPrint(msgCharset1);
+    uForcePrint("failed to get charset, please set the valid charset in config file");
     exit(-1);
   }
 
   if (!taosValidateEncodec(tsCharset)) {
-    printf(msgCharset, tsCharset);
-    uPrint(msgCharset, tsCharset);
+    uForcePrint("Invalid charset:%s, please set the valid charset in config file", tsCharset);
     exit(-1);
   }
 }
\ No newline at end of file

src/common/src/ttypes.c

@@ -16,6 +16,7 @@
 
 #include "taosdef.h"
 #include "ttokendef.h"
+#include "tscompression.h"
 
 const int32_t TYPE_BYTES[11] = {
     -1,                      // TSDB_DATA_TYPE_NULL
@@ -26,23 +27,23 @@ const int32_t TYPE_BYTES[11] = {
     sizeof(int64_t),         // TSDB_DATA_TYPE_BIGINT
     sizeof(float),           // TSDB_DATA_TYPE_FLOAT
     sizeof(double),          // TSDB_DATA_TYPE_DOUBLE
-    sizeof(int32_t),  // TSDB_DATA_TYPE_BINARY
+    sizeof(VarDataOffsetT),  // TSDB_DATA_TYPE_BINARY
     sizeof(TSKEY),           // TSDB_DATA_TYPE_TIMESTAMP
-    sizeof(int32_t)   // TSDB_DATA_TYPE_NCHAR
+    sizeof(VarDataOffsetT)   // TSDB_DATA_TYPE_NCHAR
 };
 
 tDataTypeDescriptor tDataTypeDesc[11] = {
-  {TSDB_DATA_TYPE_NULL,      6, 1,            "NOTYPE"},
-  {TSDB_DATA_TYPE_BOOL,      4, CHAR_BYTES,   "BOOL"},
-  {TSDB_DATA_TYPE_TINYINT,   7, CHAR_BYTES,   "TINYINT"},
-  {TSDB_DATA_TYPE_SMALLINT,  8, SHORT_BYTES,  "SMALLINT"},
-  {TSDB_DATA_TYPE_INT,       3, INT_BYTES,    "INT"},
-  {TSDB_DATA_TYPE_BIGINT,    6, LONG_BYTES,   "BIGINT"},
-  {TSDB_DATA_TYPE_FLOAT,     5, FLOAT_BYTES,  "FLOAT"},
-  {TSDB_DATA_TYPE_DOUBLE,    6, DOUBLE_BYTES, "DOUBLE"},
-  {TSDB_DATA_TYPE_BINARY,    6, 0,            "BINARY"},
-  {TSDB_DATA_TYPE_TIMESTAMP, 9, LONG_BYTES,   "TIMESTAMP"},
-  {TSDB_DATA_TYPE_NCHAR,     5, 8,            "NCHAR"},
+  {TSDB_DATA_TYPE_NULL,      6, 1,            "NOTYPE",    NULL,                NULL},
+  {TSDB_DATA_TYPE_BOOL,      4, CHAR_BYTES,   "BOOL",      tsCompressBool,      tsDecompressBool},
+  {TSDB_DATA_TYPE_TINYINT,   7, CHAR_BYTES,   "TINYINT",   tsCompressTinyint,   tsDecompressTinyint},
+  {TSDB_DATA_TYPE_SMALLINT,  8, SHORT_BYTES,  "SMALLINT",  tsCompressSmallint,  tsDecompressSmallint},
+  {TSDB_DATA_TYPE_INT,       3, INT_BYTES,    "INT",       tsCompressInt,       tsDecompressInt},
+  {TSDB_DATA_TYPE_BIGINT,    6, LONG_BYTES,   "BIGINT",    tsCompressBigint,    tsDecompressBigint},
+  {TSDB_DATA_TYPE_FLOAT,     5, FLOAT_BYTES,  "FLOAT",     tsCompressFloat,     tsDecompressFloat},
+  {TSDB_DATA_TYPE_DOUBLE,    6, DOUBLE_BYTES, "DOUBLE",    tsCompressDouble,    tsDecompressDouble},
+  {TSDB_DATA_TYPE_BINARY,    6, 0,            "BINARY",    tsCompressString,    tsDecompressString},
+  {TSDB_DATA_TYPE_TIMESTAMP, 9, LONG_BYTES,   "TIMESTAMP", tsCompressTimestamp, tsDecompressTimestamp},
+  {TSDB_DATA_TYPE_NCHAR,     5, 8,            "NCHAR",     tsCompressString,    tsDecompressString},
 };
 
 char tTokenTypeSwitcher[13] = {

src/dnode/src/dnodeMClient.c

@@ -83,17 +83,19 @@ int32_t dnodeInitMClient() {
     memset(&tsMnodeIpSet, 0, sizeof(SRpcIpSet));
     memset(&tsMnodeInfos, 0, sizeof(SDMMnodeInfos));
     tsMnodeIpSet.numOfIps = 1;
-    taosGetFqdnPortFromEp(tsMaster, tsMnodeIpSet.fqdn[0], &tsMnodeIpSet.port[0]);
+    taosGetFqdnPortFromEp(tsFirst, tsMnodeIpSet.fqdn[0], &tsMnodeIpSet.port[0]);
     tsMnodeIpSet.port[0] += TSDB_PORT_MNODEDNODE;
-    if (strcmp(tsSecond, tsMaster) != 0) {
+    if (strcmp(tsSecond, tsFirst) != 0) {
       tsMnodeIpSet.numOfIps = 2;
       taosGetFqdnPortFromEp(tsSecond, tsMnodeIpSet.fqdn[1], &tsMnodeIpSet.port[1]);
+      tsMnodeIpSet.port[1] += TSDB_PORT_MNODEDNODE;
     }
   } else {
     tsMnodeIpSet.inUse = tsMnodeInfos.inUse;
     tsMnodeIpSet.numOfIps = tsMnodeInfos.nodeNum;
     for (int32_t i = 0; i < tsMnodeInfos.nodeNum; i++) {
       taosGetFqdnPortFromEp(tsMnodeInfos.nodeInfos[i].nodeEp, tsMnodeIpSet.fqdn[i], &tsMnodeIpSet.port[i]);
+      tsMnodeIpSet.port[i] += TSDB_PORT_MNODEDNODE;
     }
   }
 

src/dnode/src/dnodeMain.c

@@ -15,6 +15,7 @@
 
 #define _DEFAULT_SOURCE
 #include "os.h"
+#include "taos.h"
 #include "tglobal.h"
 #include "trpc.h"
 #include "tutil.h"
@@ -190,6 +191,7 @@ static void dnodeCleanUpSystem() {
     dnodeCleanupWrite();
     dnodeCleanupRead();
     dnodeCleanUpModules();
+    taos_cleanup();
     dnodeCleanupStorage();
     taosCloseLog();
   }
@@ -236,5 +238,5 @@ static int32_t dnodeInitStorage() {
 static void dnodeCleanupStorage() {}
 
 bool  dnodeIsFirstDeploy() {
-  return strcmp(tsMaster, tsLocalEp) == 0;
+  return strcmp(tsFirst, tsLocalEp) == 0;
 }

src/inc/taosdef.h

@@ -32,6 +32,13 @@ extern "C" {
 #define TSKEY int64_t
 #endif
 
+// ----------------- For variable data types such as TSDB_DATA_TYPE_BINARY and TSDB_DATA_TYPE_NCHAR
+typedef int32_t VarDataOffsetT;
+typedef int16_t VarDataLenT;
+#define varDataLen(v) ((VarDataLenT *)(v))[0]
+#define varDataTLen(v) (sizeof(VarDataLenT) + varDataLen(v))
+#define varDataVal(v) ((void *)((char *)v + sizeof(VarDataLenT)))
+
 // this data type is internally used only in 'in' query to hold the values
 #define TSDB_DATA_TYPE_ARRAY      (TSDB_DATA_TYPE_NCHAR + 1)
 
@@ -121,6 +128,10 @@ typedef struct tDataTypeDescriptor {
   int16_t nameLen;
   int32_t nSize;
   char *  aName;
+  int (*compFunc)(const char *const input, int inputSize, const int nelements, char *const output, int outputSize,
+                  char algorithm, char *const buffer, int bufferSize);
+  int (*decompFunc)(const char *const input, int compressedSize, const int nelements, char *const output,
+                    int outputSize, char algorithm, char *const buffer, int bufferSize);
 } tDataTypeDescriptor;
 
 extern tDataTypeDescriptor tDataTypeDesc[11];
@@ -190,7 +201,7 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size);
 #define TSDB_LOCALE_LEN           64
 #define TSDB_TIMEZONE_LEN         64
 
-#define TSDB_FQDN_LEN             64
+#define TSDB_FQDN_LEN             72
 #define TSDB_IPv4ADDR_LEN      	  16
 #define TSDB_FILENAME_LEN         128
 #define TSDB_METER_VNODE_BITS     20

src/inc/tsdb.h

@@ -107,6 +107,7 @@ int32_t tsdbGetTableName(TsdbRepoT *repo, STableId* id, char** name);
 int   tsdbCreateTable(TsdbRepoT *repo, STableCfg *pCfg);
 int   tsdbDropTable(TsdbRepoT *pRepo, STableId tableId);
 int   tsdbAlterTable(TsdbRepoT *repo, STableCfg *pCfg);
+TSKEY tsdbGetTableLastKey(TsdbRepoT *repo, int64_t uid);
 
 // the TSDB repository info
 typedef struct STsdbRepoInfo {

src/inc/tsync.h

@@ -45,8 +45,6 @@ typedef struct {
 typedef struct {
   int8_t     quorum;    // number of confirms required, >=1 
   int8_t     replica;   // number of replications, >=1
-  uint16_t   arbitratorPort;  // arbitrator port
-  char       arbitratorFqdn[TSDB_FQDN_LEN]; // arbitrator IP address
   SNodeInfo  nodeInfo[TAOS_SYNC_MAX_REPLICA];
 } SSyncCfg;
 
@@ -108,6 +106,7 @@ extern  int   tsMaxWatchFiles;
 extern  int   tsSyncTimer;
 extern  int   tsMaxFwdInfo; 
 extern  int   sDebugFlag;
+extern  char  tsArbitrator[];
 extern  uint16_t tsSyncPort;
 
 #ifdef __cplusplus

src/mnode/inc/mgmtDb.h

@@ -32,6 +32,7 @@ int32_t mgmtInitDbs();
 void    mgmtCleanUpDbs();
 SDbObj *mgmtGetDb(char *db);
 SDbObj *mgmtGetDbByTableId(char *db);
+void *  mgmtGetNextDb(void *pNode, SDbObj **pDb);
 void    mgmtIncDbRef(SDbObj *pDb);
 void    mgmtDecDbRef(SDbObj *pDb);
 bool    mgmtCheckIsMonitorDB(char *db, char *monitordb);

src/mnode/inc/mgmtDef.h

@@ -237,7 +237,6 @@ typedef struct {
 
 typedef struct {
   uint8_t  msgType;
-  int8_t   usePublicIp;
   int8_t   received;
   int8_t   successed;
   int8_t   expected;

src/mnode/inc/mgmtMnode.h

@@ -33,16 +33,19 @@ void    mgmtCleanupMnodes();
 
 int32_t mgmtAddMnode(int32_t dnodeId);
 int32_t mgmtDropMnode(int32_t dnodeId);
+void    mgmtDropMnodeLocal(int32_t dnodeId);
 
 void *  mgmtGetMnode(int32_t mnodeId);
 int32_t mgmtGetMnodesNum();
 void *  mgmtGetNextMnode(void *pNode, struct SMnodeObj **pMnode);
-void    mgmtReleaseMnode(struct SMnodeObj *pMnode);
+void    mgmtIncMnodeRef(struct SMnodeObj *pMnode);
+void    mgmtDecMnodeRef(struct SMnodeObj *pMnode);
 
 char *  mgmtGetMnodeRoleStr();
 void    mgmtGetMnodeIpSet(SRpcIpSet *ipSet);
 void    mgmtGetMnodeInfos(void *mnodes);
 
+
 #ifdef __cplusplus
 }
 #endif

src/mnode/inc/mgmtProfile.h

@@ -24,14 +24,6 @@ extern "C" {
 int32_t mgmtInitProfile();
 void    mgmtCleanUpProfile();
 
-bool    mgmtCheckQhandle(uint64_t qhandle);
-void    mgmtSaveQhandle(void *qhandle);
-void    mgmtFreeQhandle(void *qhandle);
-
-void *  mgmtMallocQueuedMsg(SRpcMsg *rpcMsg);
-void *  mgmtCloneQueuedMsg(SQueuedMsg *pSrcMsg);
-void    mgmtFreeQueuedMsg(SQueuedMsg *pMsg);
-
 #ifdef __cplusplus
 }
 #endif

src/mnode/inc/mgmtSdb.h

@@ -72,6 +72,7 @@ void    sdbCleanUp();
 void *  sdbOpenTable(SSdbTableDesc *desc);
 void    sdbCloseTable(void *handle);
 bool    sdbIsMaster();
+bool    sdbIsServing();
 void    sdbUpdateMnodeRoles();
 
 int32_t sdbInsertRow(SSdbOper *pOper);

src/mnode/inc/mgmtShell.h

@@ -34,6 +34,14 @@ void mgmtAddToShellQueue(SQueuedMsg *queuedMsg);
 void mgmtDealyedAddToShellQueue(SQueuedMsg *queuedMsg);
 void mgmtSendSimpleResp(void *thandle, int32_t code);
 
+bool  mgmtCheckQhandle(uint64_t qhandle);
+void *mgmtSaveQhandle(void *qhandle, int32_t size);
+void  mgmtFreeQhandle(void *qhandle, bool forceRemove);
+
+void *mgmtMallocQueuedMsg(SRpcMsg *rpcMsg);
+void *mgmtCloneQueuedMsg(SQueuedMsg *pSrcMsg);
+void  mgmtFreeQueuedMsg(SQueuedMsg *pMsg);
+
 #ifdef __cplusplus
 }
 #endif

src/mnode/inc/mgmtTable.h

@@ -24,9 +24,11 @@ extern "C" {
 
 int32_t mgmtInitTables();
 void    mgmtCleanUpTables();
-STableObj*  mgmtGetTable(char* tableId);
+void *  mgmtGetTable(char *tableId);
 void    mgmtIncTableRef(void *pTable);
 void    mgmtDecTableRef(void *pTable);
+void *  mgmtGetNextChildTable(void *pNode, SChildTableObj **pTable);
+void *  mgmtGetNextSuperTable(void *pNode, SSuperTableObj **pTable);
 void    mgmtDropAllChildTables(SDbObj *pDropDb);
 void    mgmtDropAllSuperTables(SDbObj *pDropDb);
 

src/mnode/inc/mgmtUser.h

@@ -27,7 +27,7 @@ SUserObj *mgmtGetUser(char *name);
 void *    mgmtGetNextUser(void *pNode, SUserObj **pUser);
 void      mgmtIncUserRef(SUserObj *pUser);
 void      mgmtDecUserRef(SUserObj *pUser);
-SUserObj *mgmtGetUserFromConn(void *pConn, bool *usePublicIp);
+SUserObj *mgmtGetUserFromConn(void *pConn);
 int32_t   mgmtCreateUser(SAcctObj *pAcct, char *name, char *pass);
 void      mgmtDropAllUsers(SAcctObj *pAcct);
 

src/mnode/inc/mgmtVgroup.h

@@ -32,7 +32,8 @@ void    mgmtCleanUpVgroups();
 SVgObj *mgmtGetVgroup(int32_t vgId);
 void    mgmtIncVgroupRef(SVgObj *pVgroup);
 void    mgmtDecVgroupRef(SVgObj *pVgroup);
-void    mgmtDropAllVgroups(SDbObj *pDropDb);
+void    mgmtDropAllDbVgroups(SDbObj *pDropDb);
+void    mgmtDropAllDnodeVgroups(SDnodeObj *pDropDnode);
 
 void *  mgmtGetNextVgroup(void *pNode, SVgObj **pVgroup);
 void    mgmtUpdateVgroup(SVgObj *pVgroup);

src/mnode/src/mgmtAcct.c

@@ -27,7 +27,7 @@
 #include "mgmtUser.h"
 
 void *  tsAcctSdb = NULL;
-int32_t tsAcctUpdateSize;
+static int32_t tsAcctUpdateSize;
 static void    mgmtCreateRootAcct();
 
 static int32_t mgmtActionAcctDestroy(SSdbOper *pOper) {

src/mnode/src/mgmtBalance.c

@@ -35,7 +35,7 @@ int32_t balanceAllocVnodes(SVgObj *pVgroup) {
   void *     pNode = NULL;
   SDnodeObj *pDnode = NULL;
   SDnodeObj *pSelDnode = NULL;
-  float      vnodeUsage = 1.0;
+  float      vnodeUsage = 1000.0;
 
   while (1) {
     pNode = mgmtGetNextDnode(pNode, &pDnode);

src/mnode/src/mgmtDServer.c

@@ -107,6 +107,7 @@ static void mgmtProcessMsgFromDnode(SRpcMsg *rpcMsg) {
     
     SRpcIpSet ipSet = {0};
     dnodeGetMnodeDnodeIpSet(&ipSet);
+    
     mTrace("conn from dnode ip:%s user:%s redirect msg, inUse:%d", taosIpStr(connInfo.clientIp), connInfo.user, ipSet.inUse);
     for (int32_t i = 0; i < ipSet.numOfIps; ++i) {
       mTrace("index:%d %s:%d", i, ipSet.fqdn[i], ipSet.port[i]);

src/mnode/src/mgmtDb.c

@@ -36,7 +36,7 @@
 #include "mgmtUser.h"
 #include "mgmtVgroup.h"
 
-void *  tsDbSdb = NULL;
+static void *  tsDbSdb = NULL;
 static int32_t tsDbUpdateSize;
 
 static int32_t mgmtCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate);
@@ -82,7 +82,7 @@ static int32_t mgmtDbActionDelete(SSdbOper *pOper) {
   mgmtDropDbFromAcct(pAcct, pDb);
   mgmtDropAllChildTables(pDb);
   mgmtDropAllSuperTables(pDb);
-  mgmtDropAllVgroups(pDb);
+  mgmtDropAllDbVgroups(pDb);
   mgmtDecAcctRef(pAcct);
   
   return TSDB_CODE_SUCCESS;
@@ -95,6 +95,7 @@ static int32_t mgmtDbActionUpdate(SSdbOper *pOper) {
     memcpy(pSaved, pDb, pOper->rowSize);
     free(pDb);
   }
+  mgmtDecDbRef(pSaved);
   return TSDB_CODE_SUCCESS;
 }
 
@@ -154,6 +155,10 @@ int32_t mgmtInitDbs() {
   return 0;
 }
 
+void *mgmtGetNextDb(void *pNode, SDbObj **pDb) {
+  return sdbFetchRow(tsDbSdb, pNode, (void **)pDb);
+}
+
 SDbObj *mgmtGetDb(char *db) {
   return (SDbObj *)sdbGetRow(tsDbSdb, db);
 }
@@ -174,7 +179,7 @@ SDbObj *mgmtGetDbByTableId(char *tableId) {
   memset(db, 0, sizeof(db));
   strncpy(db, tableId, pos - tableId);
 
-  return (SDbObj *)sdbGetRow(tsDbSdb, db);
+  return mgmtGetDb(db);
 }
 
 static int32_t mgmtCheckDbCfg(SDbCfg *pCfg) {
@@ -346,8 +351,27 @@ bool mgmtCheckIsMonitorDB(char *db, char *monitordb) {
   return (strncasecmp(dbName, monitordb, len) == 0 && len == strlen(monitordb));
 }
 
+#if 0
+void mgmtPrintVgroups(SDbObj *pDb, char *oper) {
+  mPrint("db:%s, vgroup link from head, oper:%s", pDb->name, oper);  
+  SVgObj *pVgroup = pDb->pHead;
+  while (pVgroup != NULL) {
+    mPrint("vgId:%d", pVgroup->vgId);
+    pVgroup = pVgroup->next;
+  }
+
+  mPrint("db:%s, vgroup link from tail", pDb->name, pDb->numOfVgroups);
+  pVgroup = pDb->pTail;
+  while (pVgroup != NULL) {
+    mPrint("vgId:%d", pVgroup->vgId);
+    pVgroup = pVgroup->prev;
+  }
+}
+#endif
+
 void mgmtAddVgroupIntoDb(SVgObj *pVgroup) {
   SDbObj *pDb = pVgroup->pDb;
+
   pVgroup->next = pDb->pHead;
   pVgroup->prev = NULL;
 
@@ -397,7 +421,7 @@ static int32_t mgmtGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn)
   int32_t cols = 0;
 
   SSchema *pSchema = pMeta->schema;
-  SUserObj *pUser = mgmtGetUserFromConn(pConn, NULL);
+  SUserObj *pUser = mgmtGetUserFromConn(pConn);
   if (pUser == NULL) return 0;
 
   pShow->bytes[cols] = TSDB_DB_NAME_LEN;
@@ -545,11 +569,11 @@ static int32_t mgmtRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void *
   SDbObj *pDb = NULL;
   char *  pWrite;
   int32_t cols = 0;
-  SUserObj *pUser = mgmtGetUserFromConn(pConn, NULL);
+  SUserObj *pUser = mgmtGetUserFromConn(pConn);
   if (pUser == NULL) return 0;
 
   while (numOfRows < rows) {
-    pShow->pNode = sdbFetchRow(tsDbSdb, pShow->pNode, (void **) &pDb);
+    pShow->pNode = mgmtGetNextDb(pShow->pNode, &pDb);
     if (pDb == NULL) break;
 
     cols = 0;
@@ -674,8 +698,7 @@ static int32_t mgmtSetDbDropping(SDbObj *pDb) {
   SSdbOper oper = {
     .type = SDB_OPER_GLOBAL,
     .table = tsDbSdb,
-    .pObj = pDb,
-    .rowSize = tsDbUpdateSize
+    .pObj = pDb
   };
 
   int32_t code = sdbUpdateRow(&oper);
@@ -803,8 +826,7 @@ static int32_t mgmtAlterDb(SDbObj *pDb, SCMAlterDbMsg *pAlter) {
     SSdbOper oper = {
       .type = SDB_OPER_GLOBAL,
       .table = tsDbSdb,
-      .pObj = pDb,
-      .rowSize = tsDbUpdateSize
+      .pObj = pDb
     };
 
     int32_t code = sdbUpdateRow(&oper);
@@ -839,21 +861,21 @@ static void mgmtProcessAlterDbMsg(SQueuedMsg *pMsg) {
     return;
   }
 
-  SDbObj *pDb = pMsg->pDb = mgmtGetDb(pAlter->db);
-  if (pDb == NULL) {
+  if (pMsg->pDb == NULL) pMsg->pDb = mgmtGetDb(pAlter->db);
+  if (pMsg->pDb == NULL) {
     mError("db:%s, failed to alter, invalid db", pAlter->db);
     mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_INVALID_DB);
     return;
   }
 
-  int32_t code = mgmtAlterDb(pDb, pAlter);
+  int32_t code = mgmtAlterDb(pMsg->pDb, pAlter);
   if (code != TSDB_CODE_SUCCESS) {
     mError("db:%s, failed to alter, invalid db option", pAlter->db);
     mgmtSendSimpleResp(pMsg->thandle, code);
     return;
   }
 
-  mTrace("db:%s, all vgroups is altered", pDb->name);
+  mTrace("db:%s, all vgroups is altered", pMsg->pDb->name);
   mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_SUCCESS);
 }
 
@@ -884,8 +906,8 @@ static void mgmtProcessDropDbMsg(SQueuedMsg *pMsg) {
     return;
   }
 
-  SDbObj *pDb = pMsg->pDb = mgmtGetDb(pDrop->db);
-  if (pDb == NULL) {
+  if (pMsg->pDb == NULL) pMsg->pDb = mgmtGetDb(pDrop->db);
+  if (pMsg->pDb == NULL) {
     if (pDrop->ignoreNotExists) {
       mTrace("db:%s, db is not exist, think drop success", pDrop->db);
       mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_SUCCESS);
@@ -897,30 +919,32 @@ static void mgmtProcessDropDbMsg(SQueuedMsg *pMsg) {
     }
   }
 
-  if (mgmtCheckIsMonitorDB(pDb->name, tsMonitorDbName)) {
+  if (mgmtCheckIsMonitorDB(pMsg->pDb->name, tsMonitorDbName)) {
     mError("db:%s, can't drop monitor database", pDrop->db);
     mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_MONITOR_DB_FORBIDDEN);
     return;
   }
 
-  int32_t code = mgmtSetDbDropping(pDb);
+  int32_t code = mgmtSetDbDropping(pMsg->pDb);
   if (code != TSDB_CODE_SUCCESS) {
     mError("db:%s, failed to drop, reason:%s", pDrop->db, tstrerror(code));
     mgmtSendSimpleResp(pMsg->thandle, code);
     return;
   }
 
-  SVgObj *pVgroup = pDb->pHead;
+#if 0
+  SVgObj *pVgroup = pMsg->pDb->pHead;
   if (pVgroup != NULL) {
-    mPrint("vgroup:%d, will be dropped", pVgroup->vgId);
+    mPrint("vgId:%d, will be dropped", pVgroup->vgId);
     SQueuedMsg *newMsg = mgmtCloneQueuedMsg(pMsg);
     newMsg->ahandle = pVgroup;
     newMsg->expected = pVgroup->numOfVnodes;
     mgmtDropVgroup(pVgroup, newMsg);
     return;
   }
+#endif  
 
-  mTrace("db:%s, all vgroups is dropped", pDb->name);
+  mTrace("db:%s, all vgroups is dropped", pMsg->pDb->name);
   mgmtDropDb(pMsg);
 }
 
@@ -932,7 +956,7 @@ void  mgmtDropAllDbs(SAcctObj *pAcct)  {
   mPrint("acct:%s, all dbs will be dropped from sdb", pAcct->user);
 
   while (1) {
-    pNode = sdbFetchRow(tsDbSdb, pNode, (void **)&pDb);
+    pNode = mgmtGetNextDb(pNode, &pDb);
     if (pDb == NULL) break;
 
     if (pDb->pAcct == pAcct) {

src/mnode/src/mgmtDnode.c

@@ -36,9 +36,9 @@
 #include "mgmtUser.h"
 #include "mgmtVgroup.h"
 
-void   *tsDnodeSdb = NULL;
-int32_t tsDnodeUpdateSize = 0;
 int32_t tsAccessSquence = 0;
+static void   *tsDnodeSdb = NULL;
+static int32_t tsDnodeUpdateSize = 0;
 extern void *  tsMnodeSdb;
 extern void *  tsVgroupSdb;
 
@@ -73,39 +73,12 @@ static int32_t mgmtDnodeActionInsert(SSdbOper *pOper) {
 
 static int32_t mgmtDnodeActionDelete(SSdbOper *pOper) {
   SDnodeObj *pDnode = pOper->pObj;
-  void *     pNode = NULL;
-  void *     pLastNode = NULL;
-  SVgObj *   pVgroup = NULL;
-  int32_t    numOfVgroups = 0;
-
-  while (1) {
-    pLastNode = pNode;
-    pNode = sdbFetchRow(tsVgroupSdb, pNode, (void **)&pVgroup);
-    if (pVgroup == NULL) break;
-
-    if (pVgroup->vnodeGid[0].dnodeId == pDnode->dnodeId) {
-      SSdbOper oper = {
-        .type = SDB_OPER_LOCAL,
-        .table = tsVgroupSdb,
-        .pObj = pVgroup,
-      };
-      sdbDeleteRow(&oper);
-      pNode = pLastNode;
-      numOfVgroups++;
-      continue;
-    }
-  }
-
-  SMnodeObj *pMnode = mgmtGetMnode(pDnode->dnodeId);
-  if (pMnode != NULL) {
-    SSdbOper oper = {.type = SDB_OPER_LOCAL, .table = tsMnodeSdb, .pObj = pMnode};
-    sdbDeleteRow(&oper);
-    mgmtReleaseMnode(pMnode);
-  }
  
+  mgmtDropAllDnodeVgroups(pDnode);
+  mgmtDropMnodeLocal(pDnode->dnodeId);
   balanceNotify();
 
-  mTrace("dnode:%d, all vgroups:%d is dropped from sdb", pDnode->dnodeId, numOfVgroups);
+  mTrace("dnode:%d, all vgroups is dropped from sdb", pDnode->dnodeId);
   return TSDB_CODE_SUCCESS;
 }
 
@@ -116,6 +89,7 @@ static int32_t mgmtDnodeActionUpdate(SSdbOper *pOper) {
     memcpy(pSaved, pDnode, pOper->rowSize);
     free(pDnode);
   }
+  mgmtDecDnodeRef(pSaved);
   return TSDB_CODE_SUCCESS;
 }
 
@@ -212,7 +186,7 @@ void *mgmtGetDnodeByIp(char *ep) {
   void *     pNode = NULL;
 
   while (1) {
-    pNode = sdbFetchRow(tsDnodeSdb, pNode, (void**)&pDnode);
+    pNode = mgmtGetNextDnode(pNode, &pDnode);
     if (pDnode == NULL) break;
     if (strcmp(ep, pDnode->dnodeEp) == 0) {
       return pDnode;
@@ -235,8 +209,7 @@ void mgmtUpdateDnode(SDnodeObj *pDnode) {
   SSdbOper oper = {
     .type = SDB_OPER_GLOBAL,
     .table = tsDnodeSdb,
-    .pObj = pDnode,
-    .rowSize = tsDnodeUpdateSize
+    .pObj = pDnode
   };
 
   sdbUpdateRow(&oper);
@@ -336,7 +309,7 @@ void mgmtProcessDnodeStatusMsg(SRpcMsg *rpcMsg) {
     SVgObj *pVgroup = mgmtGetVgroup(pVload->vgId);
     if (pVgroup == NULL) {
       SRpcIpSet ipSet = mgmtGetIpSetFromIp(pDnode->dnodeEp);
-      mPrint("dnode:%d, vgroup:%d not exist in mnode, drop it", pDnode->dnodeId, pVload->vgId);
+      mPrint("dnode:%d, vgId:%d not exist in mnode, drop it", pDnode->dnodeId, pVload->vgId);
       mgmtSendDropVnodeMsg(pVload->vgId, &ipSet, NULL);
     } else {
       mgmtUpdateVgroupStatus(pVgroup, pDnode, pVload);
@@ -387,6 +360,7 @@ static int32_t mgmtCreateDnode(char *ep) {
 
   SDnodeObj *pDnode = mgmtGetDnodeByIp(ep);
   if (pDnode != NULL) {
+    mgmtDecDnodeRef(pDnode);
     mError("dnode:%d is alredy exist, %s:%d", pDnode->dnodeId, pDnode->dnodeFqdn, pDnode->dnodePort);
     return TSDB_CODE_DNODE_ALREADY_EXIST;
   }
@@ -440,6 +414,7 @@ static int32_t mgmtDropDnodeByIp(char *ep) {
     return TSDB_CODE_DNODE_NOT_EXIST;
   }
 
+  mgmtDecDnodeRef(pDnode);
   if (strcmp(pDnode->dnodeEp, dnodeGetMnodeMasterEp()) == 0) {
     mError("dnode:%d, can't drop dnode:%s which is master", pDnode->dnodeId, ep);
     return TSDB_CODE_NO_REMOVE_MASTER;
@@ -464,6 +439,7 @@ static void mgmtProcessCreateDnodeMsg(SQueuedMsg *pMsg) {
     if (rpcRsp.code == TSDB_CODE_SUCCESS) {
       SDnodeObj *pDnode = mgmtGetDnodeByIp(pCreate->ep);
       mLPrint("dnode:%d, %s is created by %s", pDnode->dnodeId, pCreate->ep, pMsg->pUser->user);
+      mgmtDecDnodeRef(pDnode);
     } else {
       mError("failed to create dnode:%s, reason:%s", pCreate->ep, tstrerror(rpcRsp.code));
     }
@@ -492,7 +468,7 @@ static void mgmtProcessDropDnodeMsg(SQueuedMsg *pMsg) {
 }
 
 static int32_t mgmtGetDnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
-  SUserObj *pUser = mgmtGetUserFromConn(pConn, NULL);
+  SUserObj *pUser = mgmtGetUserFromConn(pConn);
   if (pUser == NULL) return 0;
 
   if (strcmp(pUser->pAcct->user, "root") != 0) {
@@ -609,7 +585,7 @@ static bool mgmtCheckModuleInDnode(SDnodeObj *pDnode, int32_t moduleType) {
 static int32_t mgmtGetModuleMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
   int32_t cols = 0;
 
-  SUserObj *pUser = mgmtGetUserFromConn(pConn, NULL);
+  SUserObj *pUser = mgmtGetUserFromConn(pConn);
   if (pUser == NULL) return 0;
 
   if (strcmp(pUser->user, "root") != 0)  {
@@ -719,7 +695,7 @@ static bool mgmtCheckConfigShow(SGlobalCfg *cfg) {
 static int32_t mgmtGetConfigMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
   int32_t cols = 0;
 
-  SUserObj *pUser = mgmtGetUserFromConn(pConn, NULL);
+  SUserObj *pUser = mgmtGetUserFromConn(pConn);
   if (pUser == NULL) return 0;
 
   if (strcmp(pUser->user, "root") != 0)  {
@@ -806,7 +782,7 @@ static int32_t mgmtRetrieveConfigs(SShowObj *pShow, char *data, int32_t rows, vo
 
 static int32_t mgmtGetVnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
   int32_t cols = 0;
-  SUserObj *pUser = mgmtGetUserFromConn(pConn, NULL);
+  SUserObj *pUser = mgmtGetUserFromConn(pConn);
   if (pUser == NULL) return 0;
   
   if (strcmp(pUser->user, "root") != 0)  {

src/mnode/src/mgmtMain.c

@@ -36,7 +36,7 @@
 #include "mgmtTable.h"
 #include "mgmtShell.h"
 
-void *tsMgmtTmr = NULL;
+extern void *tsMgmtTmr;
 static bool tsMgmtIsRunning = false;
 
 int32_t mgmtStartSystem() {
@@ -51,12 +51,6 @@ int32_t mgmtStartSystem() {
     mkdir(tsMnodeDir, 0755);
   }
 
-  tsMgmtTmr = taosTmrInit((tsMaxShellConns) * 3, 200, 3600000, "MND");
-  if (tsMgmtTmr == NULL) {
-    mError("failed to init timer");
-    return -1;
-  }
-
   if (mgmtInitAccts() < 0) {
     mError("failed to init accts");
     return -1;
@@ -130,7 +124,7 @@ int32_t mgmtInitSystem() {
 
   struct stat dirstat;
   bool fileExist  = (stat(tsMnodeDir, &dirstat) == 0);
-  bool asMaster = (strcmp(tsMaster, tsLocalEp) == 0);
+  bool asMaster = (strcmp(tsFirst, tsLocalEp) == 0);
 
   if (asMaster || fileExist) {
     if (mgmtStartSystem() != 0) {

src/mnode/src/mgmtMnode.c

@@ -30,7 +30,7 @@
 #include "mgmtShell.h"
 #include "mgmtUser.h"
 
-void * tsMnodeSdb = NULL;
+static void *  tsMnodeSdb = NULL;
 static int32_t tsMnodeUpdateSize = 0;
 static int32_t mgmtGetMnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
 static int32_t mgmtRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn);
@@ -71,7 +71,7 @@ static int32_t mgmtMnodeActionUpdate(SSdbOper *pOper) {
     memcpy(pSaved, pMnode, pOper->rowSize);
     free(pMnode);
   }
-
+  mgmtDecMnodeRef(pSaved);
   return TSDB_CODE_SUCCESS;
 }
 
@@ -97,7 +97,7 @@ static int32_t mgmtMnodeActionRestored() {
     mgmtGetNextMnode(NULL, &pMnode);
     if (pMnode != NULL) {
       pMnode->role = TAOS_SYNC_ROLE_MASTER;
-      mgmtReleaseMnode(pMnode);
+      mgmtDecMnodeRef(pMnode);
     }
   }
   return TSDB_CODE_SUCCESS;
@@ -148,7 +148,11 @@ void *mgmtGetMnode(int32_t mnodeId) {
   return sdbGetRow(tsMnodeSdb, &mnodeId);
 }
 
-void mgmtReleaseMnode(SMnodeObj *pMnode) {
+void mgmtIncMnodeRef(SMnodeObj *pMnode) {
+  sdbIncRef(tsMnodeSdb, pMnode);
+}
+
+void mgmtDecMnodeRef(SMnodeObj *pMnode) {
   sdbDecRef(tsMnodeSdb, pMnode);
 }
 
@@ -187,7 +191,7 @@ void mgmtGetMnodeIpSet(SRpcIpSet *ipSet) {
 
     ipSet->numOfIps++;
     
-    mgmtReleaseMnode(pMnode);
+    mgmtDecMnodeRef(pMnode);
   }
 }
 
@@ -209,7 +213,7 @@ void mgmtGetMnodeInfos(void *param) {
     }
 
     index++;
-    mgmtReleaseMnode(pMnode);
+    mgmtDecMnodeRef(pMnode);
   }
 
   mnodes->nodeNum = index;
@@ -235,8 +239,17 @@ int32_t mgmtAddMnode(int32_t dnodeId) {
   return code;
 }
 
+void mgmtDropMnodeLocal(int32_t dnodeId) {
+  SMnodeObj *pMnode = mgmtGetMnode(dnodeId);
+  if (pMnode != NULL) {
+    SSdbOper oper = {.type = SDB_OPER_LOCAL, .table = tsMnodeSdb, .pObj = pMnode};
+    sdbDeleteRow(&oper);
+    mgmtDecMnodeRef(pMnode);
+  }
+}
+
 int32_t mgmtDropMnode(int32_t dnodeId) {
-  SMnodeObj *pMnode = sdbGetRow(tsMnodeSdb, &dnodeId);
+  SMnodeObj *pMnode = mgmtGetMnode(dnodeId);
   if (pMnode == NULL) {
     return TSDB_CODE_DNODE_NOT_EXIST;
   }
@@ -258,7 +271,7 @@ int32_t mgmtDropMnode(int32_t dnodeId) {
 
 static int32_t mgmtGetMnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
   sdbUpdateMnodeRoles();
-  SUserObj *pUser = mgmtGetUserFromConn(pConn, NULL);
+  SUserObj *pUser = mgmtGetUserFromConn(pConn);
   if (pUser == NULL) return 0;
 
   if (strcmp(pUser->pAcct->user, "root") != 0)  {
@@ -339,7 +352,7 @@ static int32_t mgmtRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, voi
     
     numOfRows++;
 
-    mgmtReleaseMnode(pMnode);
+    mgmtDecMnodeRef(pMnode);
   }
 
   pShow->numOfReads += numOfRows;

src/mnode/src/mgmtProfile.c

@@ -561,17 +561,6 @@ int32_t mgmtKillConnection(char *qidstr, void *pConn) {
   return TSDB_CODE_INVALID_CONNECTION;
 }
 
-bool mgmtCheckQhandle(uint64_t qhandle) {
-  return true;
-}
-
-void mgmtSaveQhandle(void *qhandle) {
-  mTrace("qhandle:%p is allocated", qhandle);
-}
-
-void mgmtFreeQhandle(void *qhandle) {
-  mTrace("qhandle:%p is freed", qhandle);
-}
 
 int mgmtGetConns(SShowObj *pShow, void *pConn) {
   //  SAcctObj * pAcct = pConn->pAcct;
@@ -686,7 +675,7 @@ int32_t mgmtRetrieveConns(SShowObj *pShow, char *data, int32_t rows, void *pConn
 void mgmtProcessKillQueryMsg(SQueuedMsg *pMsg) {
   SRpcMsg rpcRsp = {.handle = pMsg->thandle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0};
   
-  SUserObj *pUser = mgmtGetUserFromConn(pMsg->thandle, NULL);
+  SUserObj *pUser = mgmtGetUserFromConn(pMsg->thandle);
   if (pUser == NULL) {
     rpcRsp.code = TSDB_CODE_INVALID_USER;
     rpcSendResponse(&rpcRsp);
@@ -710,7 +699,7 @@ void mgmtProcessKillQueryMsg(SQueuedMsg *pMsg) {
 void mgmtProcessKillStreamMsg(SQueuedMsg *pMsg) {
   SRpcMsg rpcRsp = {.handle = pMsg->thandle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0};
   
-  SUserObj *pUser = mgmtGetUserFromConn(pMsg->thandle, NULL);
+  SUserObj *pUser = mgmtGetUserFromConn(pMsg->thandle);
   if (pUser == NULL) {
     rpcRsp.code = TSDB_CODE_INVALID_USER;
     rpcSendResponse(&rpcRsp);
@@ -734,7 +723,7 @@ void mgmtProcessKillStreamMsg(SQueuedMsg *pMsg) {
 void mgmtProcessKillConnectionMsg(SQueuedMsg *pMsg) {
   SRpcMsg rpcRsp = {.handle = pMsg->thandle, .pCont = NULL, .contLen = 0, .code = 0, .msgType = 0};
   
-  SUserObj *pUser = mgmtGetUserFromConn(pMsg->thandle, NULL);
+  SUserObj *pUser = mgmtGetUserFromConn(pMsg->thandle);
   if (pUser == NULL) {
     rpcRsp.code = TSDB_CODE_INVALID_USER;
     rpcSendResponse(&rpcRsp);
@@ -771,52 +760,3 @@ int32_t mgmtInitProfile() {
 
 void mgmtCleanUpProfile() {
 }
-
-void *mgmtMallocQueuedMsg(SRpcMsg *rpcMsg) {
-  bool usePublicIp = false;
-  SUserObj *pUser = mgmtGetUserFromConn(rpcMsg->handle, &usePublicIp);
-  if (pUser == NULL) {
-    return NULL;
-  }
-
-  SQueuedMsg *pMsg = calloc(1, sizeof(SQueuedMsg));
-  pMsg->thandle = rpcMsg->handle;
-  pMsg->msgType = rpcMsg->msgType;
-  pMsg->contLen = rpcMsg->contLen;
-  pMsg->pCont = rpcMsg->pCont;
-  pMsg->pUser = pUser;
-  pMsg->usePublicIp = usePublicIp;
-
-  return pMsg;
-}
-
-void mgmtFreeQueuedMsg(SQueuedMsg *pMsg) {
-  if (pMsg != NULL) {
-    rpcFreeCont(pMsg->pCont);
-    if (pMsg->pUser) mgmtDecUserRef(pMsg->pUser);
-    if (pMsg->pDb) mgmtDecDbRef(pMsg->pDb);
-    if (pMsg->pVgroup) mgmtDecVgroupRef(pMsg->pVgroup);
-    if (pMsg->pTable) mgmtDecTableRef(pMsg->pTable);
-    if (pMsg->pAcct) mgmtDecAcctRef(pMsg->pAcct);
-    if (pMsg->pDnode) mgmtDecDnodeRef(pMsg->pDnode);
-    free(pMsg);
-  }
-}
-
-void* mgmtCloneQueuedMsg(SQueuedMsg *pSrcMsg) {
-  SQueuedMsg *pDestMsg = calloc(1, sizeof(SQueuedMsg));
-  
-  pDestMsg->thandle = pSrcMsg->thandle;
-  pDestMsg->msgType = pSrcMsg->msgType;
-  pDestMsg->pCont   = pSrcMsg->pCont;
-  pDestMsg->contLen = pSrcMsg->contLen;
-  pDestMsg->retry   = pSrcMsg->retry;
-  pDestMsg->maxRetry= pSrcMsg->maxRetry;
-  pDestMsg->pUser   = pSrcMsg->pUser;
-  pDestMsg->usePublicIp = pSrcMsg->usePublicIp;
-
-  pSrcMsg->pCont = NULL;
-  pSrcMsg->pUser = NULL;
-  
-  return pDestMsg;
-}
\ No newline at end of file

src/mnode/src/mgmtSdb.c

@@ -40,7 +40,7 @@ typedef enum {
 typedef enum {
   SDB_STATUS_OFFLINE,
   SDB_STATUS_SERVING,
-  SDB_ACTION_CLOSING
+  SDB_STATUS_CLOSING
 } ESdbStatus;
 
 typedef struct _SSdbTable {
@@ -107,6 +107,10 @@ bool sdbIsMaster() {
   return tsSdbObj.role == TAOS_SYNC_ROLE_MASTER; 
 }
 
+bool sdbIsServing() {
+  return tsSdbObj.status == SDB_STATUS_SERVING; 
+}
+
 static char *sdbGetActionStr(int32_t action) {
   switch (action) {
     case SDB_ACTION_INSERT:
@@ -180,7 +184,7 @@ void sdbUpdateMnodeRoles() {
     if (pMnode != NULL) {
       pMnode->role = roles.role[i];
       sdbPrint("mnode:%d, role:%s", pMnode->mnodeId, mgmtGetMnodeRoleStr(pMnode->role));
-      mgmtReleaseMnode(pMnode);
+      mgmtDecMnodeRef(pMnode);
     }
   }
 }
@@ -248,18 +252,12 @@ void sdbUpdateSync() {
       strcpy(syncCfg.nodeInfo[index].nodeFqdn, pMnode->pDnode->dnodeEp);
       index++;
 
-      mgmtReleaseMnode(pMnode);
+      mgmtDecMnodeRef(pMnode);
     }
   }
 
   syncCfg.replica = index;
-  syncCfg.arbitratorPort = syncCfg.nodeInfo[0].nodePort;
-  strcpy(syncCfg.arbitratorFqdn, syncCfg.nodeInfo[0].nodeFqdn);
-  if (syncCfg.replica == 1) {
-    syncCfg.quorum = 1;
-  } else {
-    syncCfg.quorum = 2;
-  }
+  syncCfg.quorum = (syncCfg.replica == 1) ? 1:2;
 
   bool hasThisDnode = false;
   for (int32_t i = 0; i < syncCfg.replica; ++i) {
@@ -272,9 +270,9 @@ void sdbUpdateSync() {
   if (!hasThisDnode) return;
   if (memcmp(&syncCfg, &tsSdbObj.cfg, sizeof(SSyncCfg)) == 0) return;
 
-  sdbPrint("work as mnode, replica:%d arbitrator:%s", syncCfg.replica, syncCfg.arbitratorFqdn);
+  sdbPrint("work as mnode, replica:%d", syncCfg.replica);
   for (int32_t i = 0; i < syncCfg.replica; ++i) {
-    sdbPrint("mnode:%d, ip:%s", syncCfg.nodeInfo[i].nodeId, syncCfg.nodeInfo[i].nodeFqdn);
+    sdbPrint("mnode:%d, %s:%d", syncCfg.nodeInfo[i].nodeId, syncCfg.nodeInfo[i].nodeFqdn, syncCfg.nodeInfo[i].nodePort);
   }
 
   SSyncInfo syncInfo;
@@ -320,6 +318,7 @@ int32_t sdbInit() {
 void sdbCleanUp() {
   if (tsSdbObj.status != SDB_STATUS_SERVING) return;
 
+  tsSdbObj.status = SDB_STATUS_CLOSING;
   syncStop(tsSdbObj.sync);
   free(tsSdbObj.sync);
   walClose(tsSdbObj.wal);
@@ -333,7 +332,7 @@ void sdbIncRef(void *handle, void *pRow) {
     SSdbTable *pTable = handle;
     int32_t *  pRefCount = (int32_t *)(pRow + pTable->refCountPos);
     atomic_add_fetch_32(pRefCount, 1);
-    if (0 && strcmp(pTable->tableName, "accounts") == 0) {
+    if (0 && pTable->tableId == SDB_TABLE_CTABLE) {
       sdbTrace("table:%s, add ref to record:%s:%s:%d", pTable->tableName, pTable->tableName, sdbGetkeyStr(pTable, pRow),
                *pRefCount);
     }
@@ -345,7 +344,7 @@ void sdbDecRef(void *handle, void *pRow) {
     SSdbTable *pTable = handle;
     int32_t *  pRefCount = (int32_t *)(pRow + pTable->refCountPos);
     int32_t    refCount = atomic_sub_fetch_32(pRefCount, 1);
-    if (0 && strcmp(pTable->tableName, "accounts") == 0) {
+    if (0 && pTable->tableId == SDB_TABLE_CTABLE) {
       sdbTrace("table:%s, def ref of record:%s:%s:%d", pTable->tableName, pTable->tableName, sdbGetkeyStr(pTable, pRow),
                *pRefCount);
     }

src/mnode/src/mgmtShell.c

@@ -23,6 +23,7 @@
 #include "ttimer.h"
 #include "tgrant.h"
 #include "tglobal.h"
+#include "tcache.h"
 #include "dnode.h"
 #include "mgmtDef.h"
 #include "mgmtLog.h"
@@ -50,10 +51,11 @@ static void mgmtProcessHeartBeatMsg(SQueuedMsg *queuedMsg);
 static void mgmtProcessConnectMsg(SQueuedMsg *queuedMsg);
 static void mgmtProcessUseMsg(SQueuedMsg *queuedMsg);
 
-extern void *tsMgmtTmr;
+void *tsMgmtTmr;
 static void *tsMgmtShellRpc = NULL;
 static void *tsMgmtTranQhandle = NULL;
 static void (*tsMgmtProcessShellMsgFp[TSDB_MSG_TYPE_MAX])(SQueuedMsg *) = {0};
+static void *tsQhandleCache = NULL;
 static SShowMetaFp     tsMgmtShowMetaFp[TSDB_MGMT_TABLE_MAX]     = {0};
 static SShowRetrieveFp tsMgmtShowRetrieveFp[TSDB_MGMT_TABLE_MAX] = {0};
 
@@ -64,7 +66,9 @@ int32_t mgmtInitShell() {
   mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_CONNECT, mgmtProcessConnectMsg);
   mgmtAddShellMsgHandle(TSDB_MSG_TYPE_CM_USE_DB, mgmtProcessUseMsg);
   
+  tsMgmtTmr = taosTmrInit((tsMaxShellConns) * 3, 200, 3600000, "MND");
   tsMgmtTranQhandle = taosInitScheduler(tsMaxShellConns, 1, "mnodeT");
+  tsQhandleCache = taosCacheInit(tsMgmtTmr, 2);
 
   int32_t numOfThreads = tsNumOfCores * tsNumOfThreadsPerCore / 4.0;
   if (numOfThreads < 1) {
@@ -102,6 +106,12 @@ void mgmtCleanUpShell() {
     tsMgmtShellRpc = NULL;
     mPrint("server connection to shell is closed");
   }
+
+  if (tsQhandleCache) {
+    taosCacheEmpty(tsQhandleCache);
+    taosCacheCleanup(tsQhandleCache);
+    tsQhandleCache = NULL;
+  }
 }
 
 void mgmtAddShellMsgHandle(uint8_t showType, void (*fp)(SQueuedMsg *queuedMsg)) {
@@ -233,14 +243,15 @@ static void mgmtProcessShowMsg(SQueuedMsg *pMsg) {
     return;
   }
 
-  SShowObj *pShow = (SShowObj *) calloc(1, sizeof(SShowObj) + htons(pShowMsg->payloadLen));
+  int32_t showObjSize = sizeof(SShowObj) + htons(pShowMsg->payloadLen);
+  SShowObj *pShow = (SShowObj *) calloc(1, showObjSize);
   pShow->signature  = pShow;
   pShow->type       = pShowMsg->type;
   pShow->payloadLen = htons(pShowMsg->payloadLen);
   strcpy(pShow->db, pShowMsg->db);
   memcpy(pShow->payload, pShowMsg->payload, pShow->payloadLen);
 
-  mgmtSaveQhandle(pShow);
+  pShow = mgmtSaveQhandle(pShow, showObjSize);
   pShowRsp->qhandle = htobe64((uint64_t) pShow);
 
   mTrace("show:%p, type:%s, start to get meta", pShow, mgmtGetShowTypeStr(pShowMsg->type));
@@ -255,7 +266,7 @@ static void mgmtProcessShowMsg(SQueuedMsg *pMsg) {
     rpcSendResponse(&rpcRsp);
   } else {
     mError("show:%p, type:%s, failed to get meta, reason:%s", pShow, mgmtGetShowTypeStr(pShowMsg->type), tstrerror(code));
-    mgmtFreeQhandle(pShow);
+    mgmtFreeQhandle(pShow, false);
     SRpcMsg rpcRsp = {
       .handle = pMsg->thandle,
       .code   = code
@@ -284,11 +295,6 @@ static void mgmtProcessRetrieveMsg(SQueuedMsg *pMsg) {
   SShowObj *pShow = (SShowObj *)pRetrieve->qhandle;
   mTrace("show:%p, type:%s, retrieve data", pShow, mgmtGetShowTypeStr(pShow->type));
 
-  if (!mgmtCheckQhandle(pRetrieve->qhandle)) {
-    mError("pShow:%p, query memory is corrupted", pShow);
-    mgmtSendSimpleResp(pMsg->thandle, TSDB_CODE_MEMORY_CORRUPTED);
-    return;
-  } else {
   if ((pRetrieve->free & TSDB_QUERY_TYPE_FREE_RESOURCE) != TSDB_QUERY_TYPE_FREE_RESOURCE) {
     rowsToRead = pShow->numOfRows - pShow->numOfReads;
   }
@@ -302,7 +308,6 @@ static void mgmtProcessRetrieveMsg(SQueuedMsg *pMsg) {
    */
   if (rowsToRead < 0) rowsToRead = 0;
   size = pShow->rowSize * rowsToRead;
-  }
 
   size += 100;
   SRetrieveTableRsp *pRsp = rpcMallocCont(size);
@@ -313,6 +318,7 @@ static void mgmtProcessRetrieveMsg(SQueuedMsg *pMsg) {
 
   if (rowsRead < 0) {  // TSDB_CODE_ACTION_IN_PROGRESS;
     rpcFreeCont(pRsp);
+    mgmtFreeQhandle(pShow, false);
     return;
   }
 
@@ -329,7 +335,9 @@ static void mgmtProcessRetrieveMsg(SQueuedMsg *pMsg) {
   rpcSendResponse(&rpcRsp);
 
   if (rowsToRead == 0) {
-    mgmtFreeQhandle(pShow);
+    mgmtFreeQhandle(pShow, true);
+  } else {
+    mgmtFreeQhandle(pShow, false);
   }
 }
 
@@ -413,6 +421,7 @@ static void mgmtProcessConnectMsg(SQueuedMsg *pMsg) {
       code = TSDB_CODE_INVALID_DB;
       goto connect_over;
     }
+    mgmtDecDbRef(pDb);
   }
 
   SCMConnectRsp *pConnectRsp = rpcMallocCont(sizeof(SCMConnectRsp));
@@ -446,9 +455,8 @@ static void mgmtProcessUseMsg(SQueuedMsg *pMsg) {
   SCMUseDbMsg *pUseDbMsg = pMsg->pCont;
   
   // todo check for priority of current user
-  pMsg->pDb = mgmtGetDb(pUseDbMsg->db);
-  
   int32_t code = TSDB_CODE_SUCCESS;
+  if (pMsg->pDb == NULL) pMsg->pDb = mgmtGetDb(pUseDbMsg->db);
   if (pMsg->pDb == NULL) {
     code = TSDB_CODE_INVALID_DB;
   }
@@ -462,7 +470,7 @@ static void mgmtProcessUseMsg(SQueuedMsg *pMsg) {
  */
 static bool mgmtCheckTableMetaMsgReadOnly(SQueuedMsg *pMsg) {
   SCMTableInfoMsg *pInfo = pMsg->pCont;
-  pMsg->pTable = mgmtGetTable(pInfo->tableId);
+  if (pMsg->pTable == NULL) pMsg->pTable = mgmtGetTable(pInfo->tableId);
   if (pMsg->pTable != NULL) return true;
 
   // If table does not exists and autoCreate flag is set, we add the handler into task queue
@@ -511,3 +519,79 @@ void mgmtSendSimpleResp(void *thandle, int32_t code) {
   };
   rpcSendResponse(&rpcRsp);
 }
+
+bool mgmtCheckQhandle(uint64_t qhandle) {
+  void *pSaved = taosCacheAcquireByData(tsQhandleCache, (void *)qhandle);
+  if (pSaved == (void *)qhandle) {
+    mTrace("qhandle:%p is retrived", qhandle);
+    return true;
+  } else {
+    mTrace("qhandle:%p is already freed", qhandle);
+    return false;
+  }
+}
+
+void* mgmtSaveQhandle(void *qhandle, int32_t size) {
+  if (tsQhandleCache != NULL) {
+    char key[24];
+    sprintf(key, "show:%p", qhandle);
+    void *newQhandle = taosCachePut(tsQhandleCache, key, qhandle, size, 60);
+    free(qhandle);
+    
+    mTrace("qhandle:%p is saved", newQhandle);
+    return newQhandle;
+  }
+  
+  return NULL;
+}
+
+void mgmtFreeQhandle(void *qhandle, bool forceRemove) {
+  mTrace("qhandle:%p is freed", qhandle);
+  taosCacheRelease(tsQhandleCache, &qhandle, forceRemove);
+}
+
+void *mgmtMallocQueuedMsg(SRpcMsg *rpcMsg) {
+  SUserObj *pUser = mgmtGetUserFromConn(rpcMsg->handle);
+  if (pUser == NULL) {
+    return NULL;
+  }
+
+  SQueuedMsg *pMsg = calloc(1, sizeof(SQueuedMsg));
+  pMsg->thandle = rpcMsg->handle;
+  pMsg->msgType = rpcMsg->msgType;
+  pMsg->contLen = rpcMsg->contLen;
+  pMsg->pCont = rpcMsg->pCont;
+  pMsg->pUser = pUser;
+
+  return pMsg;
+}
+
+void mgmtFreeQueuedMsg(SQueuedMsg *pMsg) {
+  if (pMsg != NULL) {
+    rpcFreeCont(pMsg->pCont);
+    if (pMsg->pUser) mgmtDecUserRef(pMsg->pUser);
+    if (pMsg->pDb) mgmtDecDbRef(pMsg->pDb);
+    if (pMsg->pVgroup) mgmtDecVgroupRef(pMsg->pVgroup);
+    if (pMsg->pTable) mgmtDecTableRef(pMsg->pTable);
+    if (pMsg->pAcct) mgmtDecAcctRef(pMsg->pAcct);
+    if (pMsg->pDnode) mgmtDecDnodeRef(pMsg->pDnode);
+    free(pMsg);
+  }
+}
+
+void* mgmtCloneQueuedMsg(SQueuedMsg *pSrcMsg) {
+  SQueuedMsg *pDestMsg = calloc(1, sizeof(SQueuedMsg));
+  
+  pDestMsg->thandle = pSrcMsg->thandle;
+  pDestMsg->msgType = pSrcMsg->msgType;
+  pDestMsg->pCont   = pSrcMsg->pCont;
+  pDestMsg->contLen = pSrcMsg->contLen;
+  pDestMsg->retry   = pSrcMsg->retry;
+  pDestMsg->maxRetry= pSrcMsg->maxRetry;
+  pDestMsg->pUser   = pSrcMsg->pUser;
+ 
+  pSrcMsg->pCont = NULL;
+  pSrcMsg->pUser = NULL;
+  
+  return pDestMsg;
+}
\ No newline at end of file

src/mnode/src/mgmtUser.c

@@ -168,8 +168,7 @@ static int32_t mgmtUpdateUser(SUserObj *pUser) {
   SSdbOper oper = {
     .type = SDB_OPER_GLOBAL,
     .table = tsUserSdb,
-    .pObj = pUser,
-    .rowSize = tsUserUpdateSize
+    .pObj = pUser
   };
 
   int32_t code = sdbUpdateRow(&oper);
@@ -249,7 +248,7 @@ static int32_t mgmtDropUser(SUserObj *pUser) {
 }
 
 static int32_t mgmtGetUserMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
-  SUserObj *pUser = mgmtGetUserFromConn(pConn, NULL);
+  SUserObj *pUser = mgmtGetUserFromConn(pConn);
   if (pUser == NULL) {
     return TSDB_CODE_NO_USER_FROM_CONN;
   }
@@ -298,7 +297,7 @@ static int32_t mgmtRetrieveUsers(SShowObj *pShow, char *data, int32_t rows, void
   char     *pWrite;
 
   while (numOfRows < rows) {
-    pShow->pNode = sdbFetchRow(tsUserSdb, pShow->pNode, (void **) &pUser);
+    pShow->pNode = mgmtGetNextUser(pShow->pNode, &pUser);
     if (pUser == NULL) break;
     
     cols = 0;
@@ -329,12 +328,9 @@ static int32_t mgmtRetrieveUsers(SShowObj *pShow, char *data, int32_t rows, void
   return numOfRows;
 }
 
-SUserObj *mgmtGetUserFromConn(void *pConn, bool *usePublicIp) {
+SUserObj *mgmtGetUserFromConn(void *pConn) {
   SRpcConnInfo connInfo;
   if (rpcGetConnInfo(pConn, &connInfo) == 0) {
-    if (usePublicIp) {
-      *usePublicIp = (connInfo.serverIp == tsPublicIpInt);
-    }
     return mgmtGetUser(connInfo.user);
   } else {
     mError("can not get user from conn:%p", pConn);
@@ -510,7 +506,7 @@ void  mgmtDropAllUsers(SAcctObj *pAcct)  {
 
   while (1) {
     pLastNode = pNode;
-    pNode = sdbFetchRow(tsUserSdb, pNode, (void **)&pUser);
+    pNode = mgmtGetNextUser(pNode, &pUser);
     if (pUser == NULL) break;
 
     if (strncmp(pUser->acct, pAcct->user, acctNameLen) == 0) {

src/mnode/src/mgmtVgroup.c

@@ -36,8 +36,8 @@
 #include "mgmtTable.h"
 #include "mgmtVgroup.h"
 
-void   *tsVgroupSdb = NULL;
-int32_t tsVgUpdateSize = 0;
+static void   *tsVgroupSdb = NULL;
+static int32_t tsVgUpdateSize = 0;
 
 static int32_t mgmtGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
 static int32_t mgmtRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, void *pConn);
@@ -62,6 +62,8 @@ static int32_t mgmtVgroupActionDestroy(SSdbOper *pOper) {
 
 static int32_t mgmtVgroupActionInsert(SSdbOper *pOper) {
   SVgObj *pVgroup = pOper->pObj;
+
+  // refer to db
   SDbObj *pDb = mgmtGetDb(pVgroup->dbName);
   if (pDb == NULL) {
     return TSDB_CODE_INVALID_DB;
@@ -74,13 +76,13 @@ static int32_t mgmtVgroupActionInsert(SSdbOper *pOper) {
   int32_t size = sizeof(SChildTableObj *) * pDb->cfg.maxTables;
   pVgroup->tableList = calloc(pDb->cfg.maxTables, sizeof(SChildTableObj *));
   if (pVgroup->tableList == NULL) {
-    mError("vgroup:%d, failed to malloc(size:%d) for the tableList of vgroups", pVgroup->vgId, size);
+    mError("vgId:%d, failed to malloc(size:%d) for the tableList of vgroups", pVgroup->vgId, size);
     return -1;
   }
 
   pVgroup->idPool = taosInitIdPool(pDb->cfg.maxTables);
   if (pVgroup->idPool == NULL) {
-    mError("vgroup:%d, failed to taosInitIdPool for vgroups", pVgroup->vgId);
+    mError("vgId:%d, failed to taosInitIdPool for vgroups", pVgroup->vgId);
     tfree(pVgroup->tableList);
     return -1;
   }
@@ -140,6 +142,7 @@ static int32_t mgmtVgroupActionUpdate(SSdbOper *pOper) {
       if (pDnode != NULL) {
         atomic_add_fetch_32(&pDnode->openVnodes, 1);
       }
+      mgmtDecDnodeRef(pDnode);
     }
   }
 
@@ -147,14 +150,15 @@ static int32_t mgmtVgroupActionUpdate(SSdbOper *pOper) {
   SDbObj *pDb = pVgroup->pDb;
   if (pDb != NULL) {
     if (pDb->cfg.maxTables != oldTables) {
-      mPrint("vgroup:%d tables change from %d to %d", pVgroup->vgId, oldTables, pDb->cfg.maxTables);
+      mPrint("vgId:%d tables change from %d to %d", pVgroup->vgId, oldTables, pDb->cfg.maxTables);
       taosUpdateIdPool(pVgroup->idPool, pDb->cfg.maxTables);
       int32_t size = sizeof(SChildTableObj *) * pDb->cfg.maxTables;
       pVgroup->tableList = (SChildTableObj **)realloc(pVgroup->tableList, size);
     }
   }
 
-  mTrace("vgroup:%d, is updated, tables:%d numOfVnode:%d", pVgroup->vgId, pDb->cfg.maxTables, pVgroup->numOfVnodes);
+  mgmtDecVgroupRef(pVgroup);
+  mTrace("vgId:%d, is updated, tables:%d numOfVnode:%d", pVgroup->vgId, pDb->cfg.maxTables, pVgroup->numOfVnodes);
   return TSDB_CODE_SUCCESS;
 }
 
@@ -237,8 +241,7 @@ void mgmtUpdateVgroup(SVgObj *pVgroup) {
   SSdbOper oper = {
     .type = SDB_OPER_GLOBAL,
     .table = tsVgroupSdb,
-    .pObj = pVgroup,
-    .rowSize = tsVgUpdateSize
+    .pObj = pVgroup
   };
 
   sdbUpdateRow(&oper);
@@ -261,7 +264,7 @@ void mgmtUpdateVgroupStatus(SVgObj *pVgroup, SDnodeObj *pDnode, SVnodeLoad *pVlo
 
   if (!dnodeExist) {
     SRpcIpSet ipSet = mgmtGetIpSetFromIp(pDnode->dnodeEp);
-    mError("vgroup:%d, dnode:%d not exist in mnode, drop it", pVload->vgId, pDnode->dnodeId);
+    mError("vgId:%d, dnode:%d not exist in mnode, drop it", pVload->vgId, pDnode->dnodeId);
     mgmtSendDropVnodeMsg(pVload->vgId, &ipSet, NULL);
     return;
   }
@@ -273,7 +276,7 @@ void mgmtUpdateVgroupStatus(SVgObj *pVgroup, SDnodeObj *pDnode, SVnodeLoad *pVlo
   }
 
   if (pVload->cfgVersion != pVgroup->pDb->cfgVersion || pVload->replica != pVgroup->numOfVnodes) {
-    mError("dnode:%d, vgroup:%d, vnode cfgVersion:%d repica:%d not match with mgmt cfgVersion:%d replica:%d",
+    mError("dnode:%d, vgId:%d, vnode cfgVersion:%d repica:%d not match with mgmt cfgVersion:%d replica:%d",
            pDnode->dnodeId, pVload->vgId, pVload->cfgVersion, pVload->replica, pVgroup->pDb->cfgVersion,
            pVgroup->numOfVnodes);
     mgmtSendCreateVgroupMsg(pVgroup, NULL);
@@ -317,9 +320,9 @@ void mgmtCreateVgroup(SQueuedMsg *pMsg, SDbObj *pDb) {
     return;
   }
 
-  mPrint("vgroup:%d, is created in mnode, db:%s replica:%d", pVgroup->vgId, pDb->name, pVgroup->numOfVnodes);
+  mPrint("vgId:%d, is created in mnode, db:%s replica:%d", pVgroup->vgId, pDb->name, pVgroup->numOfVnodes);
   for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
-    mPrint("vgroup:%d, index:%d, dnode:%d", pVgroup->vgId, i, pVgroup->vnodeGid[i].dnodeId);
+    mPrint("vgId:%d, index:%d, dnode:%d", pVgroup->vgId, i, pVgroup->vnodeGid[i].dnodeId);
   }
 
   pMsg->ahandle = pVgroup;
@@ -331,7 +334,7 @@ void mgmtDropVgroup(SVgObj *pVgroup, void *ahandle) {
   if (ahandle != NULL) {
     mgmtSendDropVgroupMsg(pVgroup, ahandle);
   } else {
-    mTrace("vgroup:%d, replica:%d is deleting from sdb", pVgroup->vgId, pVgroup->numOfVnodes);
+    mTrace("vgId:%d, replica:%d is deleting from sdb", pVgroup->vgId, pVgroup->numOfVnodes);
     mgmtSendDropVgroupMsg(pVgroup, NULL);
     SSdbOper oper = {
       .type = SDB_OPER_GLOBAL,
@@ -379,6 +382,7 @@ int32_t mgmtGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
   if (pShow->payloadLen > 0 ) {
     pTable = mgmtGetTable(pShow->payload);
     if (NULL == pTable || pTable->type == TSDB_SUPER_TABLE) {
+      mgmtDecTableRef(pTable);
       return TSDB_CODE_INVALID_TABLE_ID;
     }
     mgmtDecTableRef(pTable);
@@ -505,25 +509,28 @@ int32_t mgmtRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, void *pCo
 }
 
 void mgmtAddTableIntoVgroup(SVgObj *pVgroup, SChildTableObj *pTable) {
-  if (pTable->sid >= 0 && pVgroup->tableList[pTable->sid] == NULL) {
-    pVgroup->tableList[pTable->sid] = pTable;
+  if (pTable->sid >= 1 && pVgroup->tableList[pTable->sid - 1] == NULL) {
+    pVgroup->tableList[pTable->sid - 1] = pTable;
     taosIdPoolMarkStatus(pVgroup->idPool, pTable->sid);
     pVgroup->numOfTables++;
   }
   
-  if (pVgroup->numOfTables >= pVgroup->pDb->cfg.maxTables)
-    mgmtAddVgroupIntoDbTail(pVgroup);
+  if (pVgroup->numOfTables >= pVgroup->pDb->cfg.maxTables) {
+    mgmtMoveVgroupToTail(pVgroup);
+  }
+
+  mgmtIncVgroupRef(pVgroup);
 }
 
 void mgmtRemoveTableFromVgroup(SVgObj *pVgroup, SChildTableObj *pTable) {
-  if (pTable->sid >= 0 && pVgroup->tableList[pTable->sid] != NULL) {
-    pVgroup->tableList[pTable->sid] = NULL;
+  if (pTable->sid >= 1 && pVgroup->tableList[pTable->sid - 1] != NULL) {
+    pVgroup->tableList[pTable->sid - 1] = NULL;
     taosFreeId(pVgroup->idPool, pTable->sid);
     pVgroup->numOfTables--;
   }
 
-  if (pVgroup->numOfTables >= pVgroup->pDb->cfg.maxTables)
-    mgmtAddVgroupIntoDbTail(pVgroup);
+  mgmtMoveVgroupToHead(pVgroup);
+  mgmtDecVgroupRef(pVgroup);
 }
 
 SMDCreateVnodeMsg *mgmtBuildCreateVnodeMsg(SVgObj *pVgroup) {
@@ -588,7 +595,7 @@ SRpcIpSet mgmtGetIpSetFromIp(char *ep) {
 }
 
 void mgmtSendCreateVnodeMsg(SVgObj *pVgroup, SRpcIpSet *ipSet, void *ahandle) {
-  mTrace("vgroup:%d, send create vnode:%d msg, ahandle:%p", pVgroup->vgId, pVgroup->vgId, ahandle);
+  mTrace("vgId:%d, send create vnode:%d msg, ahandle:%p", pVgroup->vgId, pVgroup->vgId, ahandle);
   SMDCreateVnodeMsg *pCreate = mgmtBuildCreateVnodeMsg(pVgroup);
   SRpcMsg rpcMsg = {
     .handle  = ahandle,
@@ -601,7 +608,7 @@ void mgmtSendCreateVnodeMsg(SVgObj *pVgroup, SRpcIpSet *ipSet, void *ahandle) {
 }
 
 void mgmtSendCreateVgroupMsg(SVgObj *pVgroup, void *ahandle) {
-  mTrace("vgroup:%d, send create all vnodes msg, ahandle:%p", pVgroup->vgId, ahandle);
+  mTrace("vgId:%d, send create all vnodes msg, ahandle:%p", pVgroup->vgId, ahandle);
   for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
     SRpcIpSet ipSet = mgmtGetIpSetFromIp(pVgroup->vnodeGid[i].pDnode->dnodeEp);
     mgmtSendCreateVnodeMsg(pVgroup, &ipSet, ahandle);
@@ -619,7 +626,7 @@ static void mgmtProcessCreateVnodeRsp(SRpcMsg *rpcMsg) {
   }
 
   SVgObj *pVgroup = queueMsg->ahandle;
-  mTrace("vgroup:%d, create vnode rsp received, result:%s received:%d successed:%d expected:%d, thandle:%p ahandle:%p",
+  mTrace("vgId:%d, create vnode rsp received, result:%s received:%d successed:%d expected:%d, thandle:%p ahandle:%p",
          pVgroup->vgId, tstrerror(rpcMsg->code), queueMsg->received, queueMsg->successed, queueMsg->expected,
          queueMsg->thandle, rpcMsg->handle);
 
@@ -654,7 +661,7 @@ static SMDDropVnodeMsg *mgmtBuildDropVnodeMsg(int32_t vgId) {
 }
 
 void mgmtSendDropVnodeMsg(int32_t vgId, SRpcIpSet *ipSet, void *ahandle) {
-  mTrace("vgroup:%d, send drop vnode msg, ahandle:%p", vgId, ahandle);
+  mTrace("vgId:%d, send drop vnode msg, ahandle:%p", vgId, ahandle);
   SMDDropVnodeMsg *pDrop = mgmtBuildDropVnodeMsg(vgId);
   SRpcMsg rpcMsg = {
       .handle  = ahandle,
@@ -667,7 +674,7 @@ void mgmtSendDropVnodeMsg(int32_t vgId, SRpcIpSet *ipSet, void *ahandle) {
 }
 
 static void mgmtSendDropVgroupMsg(SVgObj *pVgroup, void *ahandle) {
-  mTrace("vgroup:%d, send drop all vnodes msg, ahandle:%p", pVgroup->vgId, ahandle);
+  mTrace("vgId:%d, send drop all vnodes msg, ahandle:%p", pVgroup->vgId, ahandle);
   for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
     SRpcIpSet ipSet = mgmtGetIpSetFromIp(pVgroup->vnodeGid[i].pDnode->dnodeEp);
     mgmtSendDropVnodeMsg(pVgroup->vgId, &ipSet, ahandle);
@@ -675,7 +682,7 @@ static void mgmtSendDropVgroupMsg(SVgObj *pVgroup, void *ahandle) {
 }
 
 static void mgmtProcessDropVnodeRsp(SRpcMsg *rpcMsg) {
-  mTrace("drop vnode rsp is received");
+  mTrace("drop vnode rsp is received, handle:%p", rpcMsg->handle);
   if (rpcMsg->handle == NULL) return;
 
   SQueuedMsg *queueMsg = rpcMsg->handle;
@@ -686,7 +693,7 @@ static void mgmtProcessDropVnodeRsp(SRpcMsg *rpcMsg) {
   }
 
   SVgObj *pVgroup = queueMsg->ahandle;
-  mTrace("vgroup:%d, drop vnode rsp received, result:%s received:%d successed:%d expected:%d, thandle:%p ahandle:%p",
+  mTrace("vgId:%d, drop vnode rsp received, result:%s received:%d successed:%d expected:%d, thandle:%p ahandle:%p",
          pVgroup->vgId, tstrerror(rpcMsg->code), queueMsg->received, queueMsg->successed, queueMsg->expected,
          queueMsg->thandle, rpcMsg->handle);
 
@@ -736,7 +743,33 @@ static void mgmtProcessVnodeCfgMsg(SRpcMsg *rpcMsg) {
   mgmtSendCreateVnodeMsg(pVgroup, &ipSet, NULL);
 }
 
-void mgmtDropAllVgroups(SDbObj *pDropDb) {
+void mgmtDropAllDnodeVgroups(SDnodeObj *pDropDnode) {
+  void *  pNode = NULL;
+  void *  pLastNode = NULL;
+  SVgObj *pVgroup = NULL;
+  int32_t numOfVgroups = 0;
+
+  while (1) {
+    pLastNode = pNode;
+    pNode = mgmtGetNextVgroup(pNode, &pVgroup);
+    if (pVgroup == NULL) break;
+
+    if (pVgroup->vnodeGid[0].dnodeId == pDropDnode->dnodeId) {
+      SSdbOper oper = {
+        .type = SDB_OPER_LOCAL,
+        .table = tsVgroupSdb,
+        .pObj = pVgroup,
+      };
+      sdbDeleteRow(&oper);
+      pNode = pLastNode;
+      numOfVgroups++;
+      continue;
+    }
+    mgmtDecVgroupRef(pVgroup);
+  }
+}
+
+void mgmtDropAllDbVgroups(SDbObj *pDropDb) {
   void *pNode = NULL;
   void *pLastNode = NULL;
   int32_t numOfVgroups = 0;
@@ -744,7 +777,8 @@ void mgmtDropAllVgroups(SDbObj *pDropDb) {
 
   mPrint("db:%s, all vgroups will be dropped from sdb", pDropDb->name);
   while (1) {
-    pNode = sdbFetchRow(tsVgroupSdb, pNode, (void **)&pVgroup);
+    pLastNode = pNode;
+    pNode = mgmtGetNextVgroup(pNode, &pVgroup);
     if (pVgroup == NULL) break;
 
     if (pVgroup->pDb == pDropDb) {

src/os/darwin/src/darwinPlatform.c

@@ -151,67 +151,6 @@ int taosSetSockOpt(int socketfd, int level, int optname, void *optval, int optle
   return setsockopt(socketfd, level, optname, optval, (socklen_t)optlen);
 }
 
-int taosOpenUDClientSocket(char *ip, uint16_t port) {
-  int                sockFd = 0;
-  struct sockaddr_un serverAddr;
-  int                ret;
-  char               name[128];
-  sprintf(name, "%s.%hu", ip, port);
-
-  sockFd = socket(AF_UNIX, SOCK_STREAM, 0);
-
-  if (sockFd < 0) {
-    uError("failed to open the UD socket:%s, reason:%s", name, strerror(errno));
-    return -1;
-  }
-
-  memset((char *)&serverAddr, 0, sizeof(serverAddr));
-  serverAddr.sun_family = AF_UNIX;
-  strcpy(serverAddr.sun_path + 1, name);
-
-  ret = connect(sockFd, (struct sockaddr *)&serverAddr, sizeof(serverAddr));
-
-  if (ret != 0) {
-    uError("failed to connect UD socket, name:%d, reason: %s", name, strerror(errno));
-    sockFd = -1;
-  }
-
-  return sockFd;
-}
-
-int taosOpenUDServerSocket(char *ip, uint16_t port) {
-  struct sockaddr_un serverAdd;
-  int                sockFd;
-  char               name[128];
-
-  uTrace("open ud socket:%s", name);
-  sprintf(name, "%s.%hu", ip, port);
-
-  bzero((char *)&serverAdd, sizeof(serverAdd));
-  serverAdd.sun_family = AF_UNIX;
-  strcpy(serverAdd.sun_path + 1, name);
-  unlink(name);
-
-  if ((sockFd = socket(AF_UNIX, SOCK_STREAM, 0)) < 0) {
-    uError("failed to open UD socket:%s, reason:%s", name, strerror(errno));
-    return -1;
-  }
-
-  /* bind socket to server address */
-  if (bind(sockFd, (struct sockaddr *)&serverAdd, sizeof(serverAdd)) < 0) {
-    uError("bind socket:%s failed, reason:%s", name, strerror(errno));
-    tclose(sockFd);
-    return -1;
-  }
-
-  if (listen(sockFd, 10) < 0) {
-    uError("listen socket:%s failed, reason:%s", name, strerror(errno));
-    return -1;
-  }
-
-  return sockFd;
-}
-
 int taosInitTimer(void (*callback)(int), int ms) {
   signal(SIGALRM, callback);
 

src/os/linux/src/linuxPlatform.c

@@ -143,68 +143,6 @@ int taosSetNonblocking(int sock, int on) {
 int taosSetSockOpt(int socketfd, int level, int optname, void *optval, int optlen) {
   return setsockopt(socketfd, level, optname, optval, (socklen_t)optlen);
 }
-
-int taosOpenUDClientSocket(char *ip, uint16_t port) {
-  int                sockFd = 0;
-  struct sockaddr_un serverAddr;
-  int                ret;
-  char               name[128];
-  sprintf(name, "%s.%hu", ip, port);
-
-  sockFd = socket(AF_UNIX, SOCK_STREAM, 0);
-
-  if (sockFd < 0) {
-    uError("failed to open the UD socket:%s, reason:%s", name, strerror(errno));
-    return -1;
-  }
-
-  memset((char *)&serverAddr, 0, sizeof(serverAddr));
-  serverAddr.sun_family = AF_UNIX;
-  strcpy(serverAddr.sun_path + 1, name);
-
-  ret = connect(sockFd, (struct sockaddr *)&serverAddr, sizeof(serverAddr));
-
-  if (ret != 0) {
-    uError("failed to connect UD socket, name:%d, reason: %s", name, strerror(errno));
-    sockFd = -1;
-  }
-
-  return sockFd;
-}
-
-int taosOpenUDServerSocket(char *ip, uint16_t port) {
-  struct sockaddr_un serverAdd;
-  int                sockFd;
-  char               name[128];
-
-  uTrace("open ud socket:%s", name);
-  sprintf(name, "%s.%hu", ip, port);
-
-  bzero((char *)&serverAdd, sizeof(serverAdd));
-  serverAdd.sun_family = AF_UNIX;
-  strcpy(serverAdd.sun_path + 1, name);
-  unlink(name);
-
-  if ((sockFd = socket(AF_UNIX, SOCK_STREAM, 0)) < 0) {
-    uError("failed to open UD socket:%s, reason:%s", name, strerror(errno));
-    return -1;
-  }
-
-  /* bind socket to server address */
-  if (bind(sockFd, (struct sockaddr *)&serverAdd, sizeof(serverAdd)) < 0) {
-    uError("bind socket:%s failed, reason:%s", name, strerror(errno));
-    tclose(sockFd);
-    return -1;
-  }
-
-  if (listen(sockFd, 10) < 0) {
-    uError("listen socket:%s failed, reason:%s", name, strerror(errno));
-    return -1;
-  }
-
-  return sockFd;
-}
-
 static void taosDeleteTimer(void *tharg) {
   timer_t *pTimer = tharg;
   timer_delete(*pTimer);

src/os/linux/src/linuxSysPara.c

@@ -225,10 +225,11 @@ static void taosGetSystemLocale() {  // get and set default locale
   if (cfg_locale && cfg_locale->cfgStatus < TAOS_CFG_CSTATUS_DEFAULT) {
     locale = setlocale(LC_CTYPE, "");
     if (locale == NULL) {
-      uError("can't get locale from system");
+      uForcePrint("can't get locale from system, set it to en_US.UTF-8");
+      strcpy(tsLocale, "en_US.UTF-8");
     } else {
       strncpy(tsLocale, locale, tListLen(tsLocale));
-      uPrint("locale not configured, set to system default:%s", tsLocale);
+      uForcePrint("locale not configured, set to system default:%s", tsLocale);
     }
   }
 
@@ -243,7 +244,10 @@ static void taosGetSystemLocale() {  // get and set default locale
       strncpy(tsCharset, revisedCharset, tListLen(tsCharset));
 
       free(revisedCharset);
-      uPrint("charset not configured, set to system default:%s", tsCharset);
+      uForcePrint("charset not configured, set to system default:%s", tsCharset);
+    } else {
+      strcpy(tsCharset, "UTF-8");
+      uForcePrint("can't get locale and charset from system, set it to UTF-8");
     }
   }
 }

src/plugins/http/src/httpServer.c

@@ -508,7 +508,7 @@ void httpAcceptHttpConnection(void *arg) {
               pServer->serverPort);
     return;
   } else {
-    httpPrint("http service init success at ip:%s:%u", pServer->serverIp, pServer->serverPort);
+    httpPrint("http service init success at %u", pServer->serverPort);
     pServer->online = true;
   }
 

src/query/tests/astTest.cpp

@@ -582,7 +582,7 @@ void exprSerializeTest1() {
   tExprTreeDestroy(&p1, nullptr);
   tExprTreeDestroy(&p2, nullptr);
   
-  tbufClose(&bw);
+  // tbufClose(&bw);
 }
 
 void exprSerializeTest2() {
@@ -627,7 +627,7 @@ void exprSerializeTest2() {
   tExprTreeDestroy(&p1, nullptr);
   tExprTreeDestroy(&p2, nullptr);
 
-  tbufClose(&bw);
+  // tbufClose(&bw);
 }
 }  // namespace
 TEST(testCase, astTest) {

src/rpc/src/rpcMain.c

@@ -643,10 +643,11 @@ static SRpcConn *rpcGetConnObj(SRpcInfo *pRpc, int sid, SRecvInfo *pRecv) {
 
   if (sid) {
     pConn = pRpc->connList + sid;
-  } else {
-    pConn = rpcAllocateServerConn(pRpc, pRecv);
+    if (pConn->user[0] == 0) pConn = NULL;
   } 
 
+  if (pConn == NULL) pConn = rpcAllocateServerConn(pRpc, pRecv);
+
   if (pConn) {
     if (pConn->linkUid != pHead->linkUid) {
       tTrace("%s %p, linkUid:0x%x not matched, received:0x%x", pRpc->label, pConn, pConn->linkUid, pHead->linkUid);

src/tsdb/inc/tsdbMain.h

@@ -153,7 +153,6 @@ typedef struct {
 } SCacheMem;
 
 typedef struct {
-  int              maxBytes;
   int              cacheBlockSize;
   int              totalCacheBlocks;
   STsdbCachePool   pool;
@@ -163,7 +162,7 @@ typedef struct {
   TsdbRepoT *      pRepo;
 } STsdbCache;
 
-STsdbCache *tsdbInitCache(int maxBytes, int cacheBlockSize, TsdbRepoT *pRepo);
+STsdbCache *tsdbInitCache(int cacheBlockSize, int totalBlocks, TsdbRepoT *pRepo);
 void        tsdbFreeCache(STsdbCache *pCache);
 void *      tsdbAllocFromCache(STsdbCache *pCache, int bytes, TSKEY key);
 
@@ -297,7 +296,7 @@ typedef struct {
 // TODO: take pre-calculation into account
 typedef struct {
   int16_t colId;  // Column ID
-  int16_t len;    // Column length
+  int16_t len;    // Column length // TODO: int16_t is not enough
   int32_t type : 8;
   int32_t offset : 24;
 } SCompCol;
@@ -426,6 +425,8 @@ typedef struct {
   SCompData *pCompData;
   SDataCols *pDataCols[2];
 
+  void *blockBuffer;  // Buffer to hold the whole data block
+  void *compBuffer;   // Buffer for temperary compress/decompress purpose
 } SRWHelper;
 
 // --------- Helper state
@@ -445,13 +446,11 @@ typedef struct {
 
 int  tsdbInitReadHelper(SRWHelper *pHelper, STsdbRepo *pRepo);
 int  tsdbInitWriteHelper(SRWHelper *pHelper, STsdbRepo *pRepo);
-// int  tsdbInitHelper(SRWHelper *pHelper, SHelperCfg *pCfg);
 void tsdbDestroyHelper(SRWHelper *pHelper);
 void tsdbResetHelper(SRWHelper *pHelper);
 
 // --------- For set operations
 int tsdbSetAndOpenHelperFile(SRWHelper *pHelper, SFileGroup *pGroup);
-// void tsdbSetHelperTable(SRWHelper *pHelper, SHelperTable *pHelperTable, STSchema *pSchema);
 void tsdbSetHelperTable(SRWHelper *pHelper, STable *pTable, STsdbRepo *pRepo);
 int  tsdbCloseHelperFile(SRWHelper *pHelper, bool hasError);
 

src/tsdb/src/tsdbCache.c

@@ -21,29 +21,25 @@ static int  tsdbAllocBlockFromPool(STsdbCache *pCache);
 static void tsdbFreeBlockList(SList *list);
 static void tsdbFreeCacheMem(SCacheMem *mem);
 
-STsdbCache *tsdbInitCache(int maxBytes, int cacheBlockSize, TsdbRepoT *pRepo) {
+STsdbCache *tsdbInitCache(int cacheBlockSize, int totalBlocks, TsdbRepoT *pRepo) {
   STsdbCache *pCache = (STsdbCache *)calloc(1, sizeof(STsdbCache));
   if (pCache == NULL) return NULL;
 
   if (cacheBlockSize < 0) cacheBlockSize = TSDB_DEFAULT_CACHE_BLOCK_SIZE;
   cacheBlockSize *= (1024 * 1024);
 
-  if (maxBytes < 0) maxBytes = cacheBlockSize * TSDB_DEFAULT_TOTAL_BLOCKS;
+  if (totalBlocks <= 1) totalBlocks = TSDB_DEFAULT_TOTAL_BLOCKS;
 
-  pCache->maxBytes = maxBytes;
   pCache->cacheBlockSize = cacheBlockSize;
+  pCache->totalCacheBlocks = totalBlocks;
   pCache->pRepo = pRepo;
 
-  int nBlocks = maxBytes / cacheBlockSize + 1;
-  if (nBlocks <= 1) nBlocks = 2;
-  pCache->totalCacheBlocks = nBlocks;
-
   STsdbCachePool *pPool = &(pCache->pool);
   pPool->index = 0;
   pPool->memPool = tdListNew(sizeof(STsdbCacheBlock *));
   if (pPool->memPool == NULL) goto _err;
 
-  for (int i = 0; i < nBlocks; i++) {
+  for (int i = 0; i < totalBlocks; i++) {
     STsdbCacheBlock *pBlock = (STsdbCacheBlock *)malloc(sizeof(STsdbCacheBlock) + cacheBlockSize);
     if (pBlock == NULL) {
       goto _err;

src/tsdb/src/tsdbMain.c

@@ -5,6 +5,7 @@
 #include "tsdb.h"
 #include "tsdbMain.h"
 #include "tscompression.h"
+#include "tchecksum.h"
 
 #define TSDB_DEFAULT_PRECISION TSDB_PRECISION_MILLI  // default precision
 #define IS_VALID_PRECISION(precision) (((precision) >= TSDB_PRECISION_MILLI) && ((precision) <= TSDB_PRECISION_NANO))
@@ -157,7 +158,7 @@ static int tsdbRestoreInfo(STsdbRepo *pRepo) {
   tsdbInitFileGroupIter(pFileH, &iter, TSDB_ORDER_ASC);
   while ((pFGroup = tsdbGetFileGroupNext(&iter)) != NULL) {
     if (tsdbSetAndOpenHelperFile(&rhelper, pFGroup) < 0) goto _err;
-    for (int i = 0; i < pRepo->config.maxTables; i++) {
+    for (int i = 1; i < pRepo->config.maxTables; i++) {
       STable *  pTable = pMeta->tables[i];
       SCompIdx *pIdx = &rhelper.pCompIdx[i];
 
@@ -202,7 +203,7 @@ TsdbRepoT *tsdbOpenRepo(char *tsdbDir, STsdbAppH *pAppH) {
     return NULL;
   }
 
-  pRepo->tsdbCache = tsdbInitCache(-1, -1, (TsdbRepoT *)pRepo);
+  pRepo->tsdbCache = tsdbInitCache(pRepo->config.cacheBlockSize, pRepo->config.totalBlocks, (TsdbRepoT *)pRepo);
   if (pRepo->tsdbCache == NULL) {
     tsdbFreeMeta(pRepo->tsdbMeta);
     free(pRepo->rootDir);
@@ -259,7 +260,7 @@ int32_t tsdbCloseRepo(TsdbRepoT *repo) {
   }
   pRepo->commit = 1;
   // Loop to move pData to iData
-  for (int i = 0; i < pRepo->config.maxTables; i++) {
+  for (int i = 1; i < pRepo->config.maxTables; i++) {
     STable *pTable = pRepo->tsdbMeta->tables[i];
     if (pTable != NULL && pTable->mem != NULL) {
       pTable->imem = pTable->mem;
@@ -312,7 +313,7 @@ int32_t tsdbTriggerCommit(TsdbRepoT *repo) {
   }
   pRepo->commit = 1;
   // Loop to move pData to iData
-  for (int i = 0; i < pRepo->config.maxTables; i++) {
+  for (int i = 1; i < pRepo->config.maxTables; i++) {
     STable *pTable = pRepo->tsdbMeta->tables[i];
     if (pTable != NULL && pTable->mem != NULL) {
       pTable->imem = pTable->mem;
@@ -367,6 +368,15 @@ int tsdbAlterTable(TsdbRepoT *pRepo, STableCfg *pCfg) {
   return 0;
 }
 
+TSKEY tsdbGetTableLastKey(TsdbRepoT *repo, int64_t uid) {
+  STsdbRepo *pRepo = (STsdbRepo *)repo;
+
+  STable *pTable = tsdbGetTableByUid(pRepo->tsdbMeta, uid);
+  if (pTable == NULL) return -1;
+
+  return TSDB_GET_TABLE_LAST_KEY(pTable);
+}
+
 int tsdbDropTable(TsdbRepoT *repo, STableId tableId) {
   if (repo == NULL) return -1;
   STsdbRepo *pRepo = (STsdbRepo *)repo;
@@ -601,6 +611,10 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) {
     if (pCfg->maxTables < TSDB_MIN_TABLES || pCfg->maxTables > TSDB_MAX_TABLES) return -1;
   }
 
+  // Since tableId starts from 1, we increase maxTables by 1
+  // TODO: take a fancier way to do this
+  pCfg->maxTables++;
+
   // Check daysPerFile
   if (pCfg->daysPerFile == -1) {
     pCfg->daysPerFile = TSDB_DEFAULT_DAYS_PER_FILE;
@@ -823,7 +837,7 @@ static int tsdbReadRowsFromCache(SSkipListIterator *pIter, TSKEY maxKey, int max
 static void tsdbDestroyTableIters(SSkipListIterator **iters, int maxTables) {
   if (iters == NULL) return;
 
-  for (int tid = 0; tid < maxTables; tid++) {
+  for (int tid = 1; tid < maxTables; tid++) {
     if (iters[tid] == NULL) continue;
     tSkipListDestroyIter(iters[tid]);
   }
@@ -835,7 +849,7 @@ static SSkipListIterator **tsdbCreateTableIters(STsdbMeta *pMeta, int maxTables)
   SSkipListIterator **iters = (SSkipListIterator **)calloc(maxTables, sizeof(SSkipListIterator *));
   if (iters == NULL) return NULL;
 
-  for (int tid = 0; tid < maxTables; tid++) {
+  for (int tid = 1; tid < maxTables; tid++) {
     STable *pTable = pMeta->tables[tid];
     if (pTable == NULL || pTable->imem == NULL) continue;
 
@@ -902,7 +916,7 @@ _exit:
   free(pCache->imem);
   pCache->imem = NULL;
   pRepo->commit = 0;
-  for (int i = 0; i < pCfg->maxTables; i++) {
+  for (int i = 1; i < pCfg->maxTables; i++) {
     STable *pTable = pMeta->tables[i];
     if (pTable && pTable->imem) {
       tsdbFreeMemTable(pTable->imem);
@@ -936,7 +950,7 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters
   if (tsdbSetAndOpenHelperFile(pHelper, pGroup) < 0) goto _err;
 
   // Loop to commit data in each table
-  for (int tid = 0; tid < pCfg->maxTables; tid++) {
+  for (int tid = 1; tid < pCfg->maxTables; tid++) {
     STable *           pTable = pMeta->tables[tid];
     if (pTable == NULL) continue;
 
@@ -974,7 +988,6 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters
 
     // Write the SCompBlock part
     if (tsdbWriteCompInfo(pHelper) < 0) goto _err;
- 
   }
 
   if (tsdbWriteCompIdx(pHelper) < 0) goto _err;

src/tsdb/src/tsdbMeta.c

@@ -129,7 +129,7 @@ int tsdbRestoreTable(void *pHandle, void *cont, int contLen) {
 void tsdbOrgMeta(void *pHandle) {
   STsdbMeta *pMeta = (STsdbMeta *)pHandle;
 
-  for (int i = 0; i < pMeta->maxTables; i++) {
+  for (int i = 1; i < pMeta->maxTables; i++) {
     STable *pTable = pMeta->tables[i];
     if (pTable != NULL && pTable->type == TSDB_CHILD_TABLE) {
       tsdbAddTableIntoIndex(pMeta, pTable);
@@ -179,7 +179,7 @@ int32_t tsdbFreeMeta(STsdbMeta *pMeta) {
 
   tsdbCloseMetaFile(pMeta->mfh);
 
-  for (int i = 0; i < pMeta->maxTables; i++) {
+  for (int i = 1; i < pMeta->maxTables; i++) {
     if (pMeta->tables[i] != NULL) {
       tsdbFreeTable(pMeta->tables[i]);
     }
@@ -246,7 +246,7 @@ int32_t tsdbGetTableTagVal(TsdbRepoT* repo, STableId* id, int32_t colId, int16_t
   assert(pCol != NULL);
   
   SDataRow row = (SDataRow)pTable->tagVal;
-  char* d = dataRowAt(row, TD_DATA_ROW_HEAD_SIZE);
+  char* d = dataRowTuple(row);
   
   *val = d;
   *type  = pCol->type;
@@ -455,9 +455,8 @@ static int tsdbAddTableToMeta(STsdbMeta *pMeta, STable *pTable, bool addIdx) {
   // Update the pMeta->maxCols and pMeta->maxRowBytes
   if (pTable->type == TSDB_SUPER_TABLE || pTable->type == TSDB_NORMAL_TABLE) {
     if (schemaNCols(pTable->schema) > pMeta->maxCols) pMeta->maxCols = schemaNCols(pTable->schema);
-    int bytes = tdMaxRowBytesFromSchema(pTable->schema);
+    int bytes = dataRowMaxBytesFromSchema(pTable->schema);
     if (bytes > pMeta->maxRowBytes) pMeta->maxRowBytes = bytes;
-    tdUpdateSchema(pTable->schema);
   }
 
   return tsdbAddTableIntoMap(pMeta, pTable);
@@ -528,5 +527,5 @@ static int tsdbEstimateTableEncodeSize(STable *pTable) {
 char *getTupleKey(const void * data) {
   SDataRow row = (SDataRow)data;
 
-  return dataRowAt(row, TD_DATA_ROW_HEAD_SIZE);
+  return POINTER_DRIFT(row, TD_DATA_ROW_HEAD_SIZE);
 }
\ No newline at end of file

src/tsdb/src/tsdbRWHelper.c

@@ -131,6 +131,11 @@ static int tsdbInitHelper(SRWHelper *pHelper, STsdbRepo *pRepo, tsdb_rw_helper_t
   // Init block part
   if (tsdbInitHelperBlock(pHelper) < 0) goto _err;
 
+  pHelper->blockBuffer =
+      tmalloc(sizeof(SCompData) + (sizeof(SCompCol) + sizeof(TSCKSUM) + COMP_OVERFLOW_BYTES) * pHelper->config.maxCols +
+              pHelper->config.maxRowSize * pHelper->config.maxRowsPerFileBlock + sizeof(TSCKSUM));
+  if (pHelper->blockBuffer == NULL) goto _err;
+
   return 0;
 
 _err:
@@ -149,6 +154,8 @@ int tsdbInitWriteHelper(SRWHelper *pHelper, STsdbRepo *pRepo) {
 
 void tsdbDestroyHelper(SRWHelper *pHelper) {
   if (pHelper) {
+    tzfree(pHelper->blockBuffer);
+    tzfree(pHelper->compBuffer);
     tsdbDestroyHelperFile(pHelper);
     tsdbDestroyHelperTable(pHelper);
     tsdbDestroyHelperBlock(pHelper);
@@ -330,7 +337,7 @@ int tsdbWriteDataBlock(SRWHelper *pHelper, SDataCols *pDataCols) {
     int blkIdx = (pCompBlock == NULL) ? (pIdx->numOfBlocks - 1) : (pCompBlock - pHelper->pCompInfo->blocks);
 
     if (pCompBlock == NULL) {  // No key overlap, must has last block, just merge with the last block
-      ASSERT(pIdx->hasLast && pHelper->pCompInfo->blocks[pIdx->numOfSuperBlocks - 1].last);
+      ASSERT(pIdx->hasLast && pHelper->pCompInfo->blocks[pIdx->numOfBlocks - 1].last);
       rowsToWrite = tsdbMergeDataWithBlock(pHelper, blkIdx, pDataCols);
       if (rowsToWrite < 0) goto _err;
     } else {  // Has key overlap
@@ -552,61 +559,97 @@ int tsdbLoadBlockDataCols(SRWHelper *pHelper, SDataCols *pDataCols, int blkIdx,
   return 0;
 }
 
+static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, char *content, int32_t len, int8_t comp, int numOfPoints,
+                                        int maxPoints, char *buffer, int bufferSize) {
+  // Verify by checksum
+  if (!taosCheckChecksumWhole((uint8_t *)content, len)) return -1;
+
+  // Decode the data
+  if (comp) {
+    // // Need to decompress
+    pDataCol->len = (*(tDataTypeDesc[pDataCol->type].decompFunc))(
+        content, len - sizeof(TSCKSUM), numOfPoints, pDataCol->pData, pDataCol->spaceSize, comp, buffer, bufferSize);
+    if (pDataCol->type == TSDB_DATA_TYPE_BINARY || pDataCol->type == TSDB_DATA_TYPE_NCHAR) {
+      pDataCol->len += (sizeof(int32_t) * maxPoints);
+      dataColSetOffset(pDataCol, numOfPoints);
+    }
+  } else {
+    // No need to decompress, just memcpy it
+    switch (pDataCol->type) {
+      case TSDB_DATA_TYPE_BINARY:
+      case TSDB_DATA_TYPE_NCHAR:
+        pDataCol->len = sizeof(int32_t) * maxPoints;
+        memcpy((char *)pDataCol->pData + pDataCol->len, content, len - sizeof(TSCKSUM));
+        pDataCol->len += (len - sizeof(TSCKSUM));
+        dataColSetOffset(pDataCol, numOfPoints);
+        break;
+
+      default:
+        pDataCol->len = len - sizeof(TSCKSUM);
+        memcpy(pDataCol->pData, content, pDataCol->len);
+        break;
+    }
+  }
+  return 0;
+}
+
 /**
  * Interface to read the data of a sub-block OR the data of a super-block of which (numOfSubBlocks == 1)
  */
 static int tsdbLoadBlockDataImpl(SRWHelper *pHelper, SCompBlock *pCompBlock, SDataCols *pDataCols) {
   ASSERT(pCompBlock->numOfSubBlocks <= 1);
 
-  SCompData *pCompData = (SCompData *)malloc(pCompBlock->len);
-  if (pCompData == NULL) return -1;
+  ASSERT(tsizeof(pHelper->blockBuffer) >= pCompBlock->len);
+
+  SCompData *pCompData = (SCompData *)pHelper->blockBuffer;
 
   int fd = (pCompBlock->last) ? pHelper->files.lastF.fd : pHelper->files.dataF.fd;
   if (lseek(fd, pCompBlock->offset, SEEK_SET) < 0) goto _err;
   if (tread(fd, (void *)pCompData, pCompBlock->len) < pCompBlock->len) goto _err;
   ASSERT(pCompData->numOfCols == pCompBlock->numOfCols);
 
-  // TODO : check the checksum
-  size_t tsize = sizeof(SCompData) + sizeof(SCompCol) * pCompBlock->numOfCols + sizeof(TSCKSUM);
+  int32_t tsize = sizeof(SCompData) + sizeof(SCompCol) * pCompBlock->numOfCols + sizeof(TSCKSUM);
   if (!taosCheckChecksumWhole((uint8_t *)pCompData, tsize)) goto _err;
-  for (int i = 0; i < pCompData->numOfCols; i++) {
-    // TODO: check the data checksum
-    // if (!taosCheckChecksumWhole())
-  }
-
-  ASSERT(pCompBlock->numOfCols == pCompData->numOfCols);
 
   pDataCols->numOfPoints = pCompBlock->numOfPoints;
 
-  int ccol = 0, dcol = 0;
-  while (true) {
-    if (ccol >= pDataCols->numOfCols) {
-      // TODO: Fill rest NULL
-      break;
+  // Recover the data
+  int ccol = 0;
+  int dcol = 0;
+  while (dcol < pDataCols->numOfCols) {
+    SDataCol *pDataCol = &(pDataCols->cols[dcol]);
+    if (ccol >= pCompData->numOfCols) {
+      // Set current column as NULL and forward
+      dataColSetNEleNull(pDataCol, pCompBlock->numOfPoints, pDataCols->maxPoints);
+      dcol++;
+      continue;
     }
-    if (dcol >= pCompData->numOfCols) break;
 
     SCompCol *pCompCol = &(pCompData->cols[ccol]);
-    SDataCol *pDataCol = &(pDataCols->cols[dcol]);
 
     if (pCompCol->colId == pDataCol->colId) {
-      // TODO: uncompress
-      memcpy(pDataCol->pData, (void *)(((char *)pCompData) + tsize + pCompCol->offset), pCompCol->len);
-      ccol++;
-      dcol++;
-    } else if (pCompCol->colId > pDataCol->colId) {
-      // TODO: Fill NULL
+      if (pCompBlock->algorithm == TWO_STAGE_COMP) {
+        pHelper->compBuffer = trealloc(pHelper->compBuffer, pCompCol->len + COMP_OVERFLOW_BYTES);
+        if (pHelper->compBuffer == NULL) goto _err;
+      }
+      if (tsdbCheckAndDecodeColumnData(pDataCol, (char *)pCompData + tsize + pCompCol->offset, pCompCol->len,
+                                       pCompBlock->algorithm, pCompBlock->numOfPoints, pDataCols->maxPoints,
+                                       pHelper->compBuffer, tsizeof(pHelper->compBuffer)) < 0)
+        goto _err;
       dcol++;
-    } else {
       ccol++;
+    } else if (pCompCol->colId < pDataCol->colId) {
+      ccol++;
+    } else {
+      // Set current column as NULL and forward
+      dataColSetNEleNull(pDataCol, pCompBlock->numOfPoints, pDataCols->maxPoints);
+      dcol++;
     }
   }
 
-  tfree(pCompData);
   return 0;
 
 _err:
-  tfree(pCompData);
   return -1;
 }
 
@@ -634,36 +677,6 @@ _err:
   return -1;
 }
 
-// static int tsdbCheckHelperCfg(SHelperCfg *pCfg) {
-//   // TODO
-//   return 0;
-// }
-
-// static void tsdbClearHelperFile(SHelperFile *pHFile) {
-//   pHFile->fid = -1;
-//   if (pHFile->headF.fd > 0) {
-//     close(pHFile->headF.fd);
-//     pHFile->headF.fd = -1;
-//   }
-//   if (pHFile->dataF.fd > 0) {
-//     close(pHFile->dataF.fd);
-//     pHFile->dataF.fd = -1;
-//   }
-//   if (pHFile->lastF.fd > 0) {
-//     close(pHFile->lastF.fd);
-//     pHFile->lastF.fd = -1;
-//   }
-//   if (pHFile->nHeadF.fd > 0) {
-//     close(pHFile->nHeadF.fd);
-//     pHFile->nHeadF.fd = -1;
-//   }
-//   if (pHFile->nLastF.fd > 0) {
-//     close(pHFile->nLastF.fd);
-//     pHFile->nLastF.fd = -1;
-//   }
-
-// }
-
 static bool tsdbShouldCreateNewLast(SRWHelper *pHelper) {
   ASSERT(pHelper->files.lastF.fd > 0);
   struct stat st;
@@ -677,81 +690,94 @@ static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDa
   ASSERT(rowsToWrite > 0 && rowsToWrite <= pDataCols->numOfPoints &&
          rowsToWrite <= pHelper->config.maxRowsPerFileBlock);
 
-  SCompData *pCompData = NULL;
+  SCompData *pCompData = (SCompData *)(pHelper->blockBuffer);
   int64_t offset = 0;
 
   offset = lseek(pFile->fd, 0, SEEK_END);
   if (offset < 0) goto _err;
 
-  pCompData = (SCompData *)malloc(sizeof(SCompData) + sizeof(SCompCol) * pDataCols->numOfCols + sizeof(TSCKSUM));
-  if (pCompData == NULL) goto _err;
-
   int nColsNotAllNull = 0;
-  int32_t toffset = 0;
   for (int ncol = 0; ncol < pDataCols->numOfCols; ncol++) {
     SDataCol *pDataCol = pDataCols->cols + ncol;
     SCompCol *pCompCol = pCompData->cols + nColsNotAllNull;
 
-    if (0) {
-      // TODO: all data to commit are NULL
+    if (isNEleNull(pDataCol, rowsToWrite)) {
+      // all data to commit are NULL, just ignore it
       continue;
     }
 
-    // Compress the data here
-    {
-      // TODO
-    }
-
     pCompCol->colId = pDataCol->colId;
     pCompCol->type = pDataCol->type;
-    pCompCol->len = TYPE_BYTES[pCompCol->type] * rowsToWrite; // TODO: change it
-    pCompCol->offset = toffset;
     nColsNotAllNull++;
-
-    toffset += pCompCol->len;
   }
 
   ASSERT(nColsNotAllNull > 0 && nColsNotAllNull <= pDataCols->numOfCols);
 
-  pCompData->delimiter = TSDB_FILE_DELIMITER;
-  pCompData->uid = pHelper->tableInfo.uid;
-  pCompData->numOfCols = nColsNotAllNull;
-
-  // Write SCompData + SCompCol part
-  size_t tsize = sizeof(SCompData) + sizeof(SCompCol) * nColsNotAllNull + sizeof(TSCKSUM);
-  taosCalcChecksumAppend(0, (uint8_t *)pCompData, tsize);
-  if (twrite(pFile->fd, (void *)pCompData, tsize) < tsize) goto _err;
-  // Write true data part
-  int nCompCol = 0;
+  // Compress the data if neccessary
+  int     tcol = 0;
+  int32_t toffset = 0;
+  int32_t tsize = sizeof(SCompData) + sizeof(SCompCol) * nColsNotAllNull + sizeof(TSCKSUM);
+  int32_t lsize = tsize;
   for (int ncol = 0; ncol < pDataCols->numOfCols; ncol++) {
-    ASSERT(nCompCol < nColsNotAllNull);
+    if (tcol >= nColsNotAllNull) break;
 
     SDataCol *pDataCol = pDataCols->cols + ncol;
-    SCompCol *pCompCol = pCompData->cols + nCompCol;
+    SCompCol *pCompCol = pCompData->cols + tcol;
 
-    if (pDataCol->colId == pCompCol->colId) {
-      if (twrite(pFile->fd, (void *)(pDataCol->pData), pCompCol->len) < pCompCol->len) goto _err;
-      tsize += pCompCol->len;
-      nCompCol++;
+    if (pDataCol->colId != pCompCol->colId) continue;
+    void *tptr = (void *)((char *)pCompData + lsize);
+
+    pCompCol->offset = toffset;
+
+    int32_t tlen = dataColGetNEleLen(pDataCol, rowsToWrite);
+
+    if (pHelper->config.compress) {
+      if (pHelper->config.compress == TWO_STAGE_COMP) {
+        pHelper->compBuffer = trealloc(pHelper->compBuffer, tlen + COMP_OVERFLOW_BYTES);
+        if (pHelper->compBuffer == NULL) goto _err;
       }
+
+      pCompCol->len = (*(tDataTypeDesc[pDataCol->type].compFunc))(
+          (char *)pDataCol->pData, tlen, rowsToWrite, tptr, tsizeof(pHelper->blockBuffer) - lsize,
+          pHelper->config.compress, pHelper->compBuffer, tsizeof(pHelper->compBuffer));
+    } else {
+      pCompCol->len = tlen;
+      memcpy(tptr, pDataCol->pData, pCompCol->len);
     }
 
+    // Add checksum
+    pCompCol->len += sizeof(TSCKSUM);
+    taosCalcChecksumAppend(0, (uint8_t *)tptr, pCompCol->len);
+
+    toffset += pCompCol->len;
+    lsize += pCompCol->len;
+    tcol++;
+  }
+
+  pCompData->delimiter = TSDB_FILE_DELIMITER;
+  pCompData->uid = pHelper->tableInfo.uid;
+  pCompData->numOfCols = nColsNotAllNull;
+
+  taosCalcChecksumAppend(0, (uint8_t *)pCompData, tsize);
+
+  // Write the whole block to file
+  if (twrite(pFile->fd, (void *)pCompData, lsize) < lsize) goto _err;
+
+  // Update pCompBlock membership vairables
   pCompBlock->last = isLast;
   pCompBlock->offset = offset;
   pCompBlock->algorithm = pHelper->config.compress;
   pCompBlock->numOfPoints = rowsToWrite;
   pCompBlock->sversion = pHelper->tableInfo.sversion;
-  pCompBlock->len = (int32_t)tsize;
+  pCompBlock->len = (int32_t)lsize;
   pCompBlock->numOfSubBlocks = isSuperBlock ? 1 : 0;
   pCompBlock->numOfCols = nColsNotAllNull;
   pCompBlock->keyFirst = dataColsKeyFirst(pDataCols);
   pCompBlock->keyLast = dataColsKeyAt(pDataCols, rowsToWrite - 1);
 
-  tfree(pCompData);
   return 0;
 
   _err:
-  tfree(pCompData);
   return -1;
 }
 
@@ -782,7 +808,7 @@ static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDa
   TSKEY keyFirst = dataColsKeyFirst(pDataCols);
 
   SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid;
-  ASSERT(blkIdx < pIdx->numOfSuperBlocks);
+  ASSERT(blkIdx < pIdx->numOfBlocks);
 
   // SCompBlock *pCompBlock = pHelper->pCompInfo->blocks + blkIdx;
   ASSERT(blockAtIdx(pHelper, blkIdx)->numOfSubBlocks >= 1);
@@ -790,7 +816,7 @@ static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDa
   // ASSERT(compareKeyBlock((void *)&keyFirst, (void *)pCompBlock) == 0);
 
   if (keyFirst > blockAtIdx(pHelper, blkIdx)->keyLast) { // Merge with the last block by append
-    ASSERT(blockAtIdx(pHelper, blkIdx)->numOfPoints < pHelper->config.minRowsPerFileBlock && blkIdx == pIdx->numOfSuperBlocks-1);
+    ASSERT(blockAtIdx(pHelper, blkIdx)->numOfPoints < pHelper->config.minRowsPerFileBlock && blkIdx == pIdx->numOfBlocks-1);
     int defaultRowsToWrite = pHelper->config.maxRowsPerFileBlock * 4 / 5;  // TODO: make a interface
 
     rowsWritten = MIN((defaultRowsToWrite - blockAtIdx(pHelper, blkIdx)->numOfPoints), pDataCols->numOfPoints);
@@ -961,7 +987,7 @@ static int tsdbAdjustInfoSizeIfNeeded(SRWHelper *pHelper, size_t esize) {
 static int tsdbInsertSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx) {
   SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid;
 
-  ASSERT(blkIdx >= 0 && blkIdx <= pIdx->numOfSuperBlocks);
+  ASSERT(blkIdx >= 0 && blkIdx <= pIdx->numOfBlocks);
   ASSERT(pCompBlock->numOfSubBlocks == 1);
 
   // Adjust memory if no more room
@@ -1004,7 +1030,7 @@ static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkId
   ASSERT(pCompBlock->numOfSubBlocks == 0);
 
   SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid;
-  ASSERT(blkIdx >= 0 && blkIdx < pIdx->numOfSuperBlocks);
+  ASSERT(blkIdx >= 0 && blkIdx < pIdx->numOfBlocks);
 
   SCompBlock *pSCompBlock = pHelper->pCompInfo->blocks + blkIdx;
   ASSERT(pSCompBlock->numOfSubBlocks >= 1 && pSCompBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS);
@@ -1088,7 +1114,7 @@ static int tsdbUpdateSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int
 
   SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid;
 
-  ASSERT(blkIdx >= 0 && blkIdx < pIdx->numOfSuperBlocks);
+  ASSERT(blkIdx >= 0 && blkIdx < pIdx->numOfBlocks);
 
   SCompBlock *pSCompBlock = pHelper->pCompInfo->blocks + blkIdx;
 

src/tsdb/tests/tsdbTests.cpp

@@ -27,7 +27,7 @@ typedef struct {
 
 static int insertData(SInsertInfo *pInfo) {
   SSubmitMsg *pMsg =
-      (SSubmitMsg *)malloc(sizeof(SSubmitMsg) + sizeof(SSubmitBlk) + tdMaxRowBytesFromSchema(pInfo->pSchema) * pInfo->rowsPerSubmit);
+      (SSubmitMsg *)malloc(sizeof(SSubmitMsg) + sizeof(SSubmitBlk) + dataRowMaxBytesFromSchema(pInfo->pSchema) * pInfo->rowsPerSubmit);
   if (pMsg == NULL) return -1;
   TSKEY start_time = pInfo->startTime;
 
@@ -52,11 +52,12 @@ static int insertData(SInsertInfo *pInfo) {
       tdInitDataRow(row, pInfo->pSchema);
 
       for (int j = 0; j < schemaNCols(pInfo->pSchema); j++) {
+        STColumn *pTCol = schemaColAt(pInfo->pSchema, j);
         if (j == 0) {  // Just for timestamp
-          tdAppendColVal(row, (void *)(&start_time), schemaColAt(pInfo->pSchema, j));
+          tdAppendColVal(row, (void *)(&start_time), pTCol->type, pTCol->bytes, pTCol->offset);
         } else {  // For int
           int val = 10;
-          tdAppendColVal(row, (void *)(&val), schemaColAt(pInfo->pSchema, j));
+          tdAppendColVal(row, (void *)(&val), pTCol->type, pTCol->bytes, pTCol->offset);
         }
       }
       pBlock->len += dataRowLen(row);
@@ -105,9 +106,9 @@ TEST(TsdbTest, DISABLED_tableEncodeDecode) {
 
   for (int i = 0; i < nCols; i++) {
     if (i == 0) {
-      tdSchemaAppendCol(schema, TSDB_DATA_TYPE_TIMESTAMP, i, -1);
+      tdSchemaAddCol(schema, TSDB_DATA_TYPE_TIMESTAMP, i, -1);
     } else {
-      tdSchemaAppendCol(schema, TSDB_DATA_TYPE_INT, i, -1);
+      tdSchemaAddCol(schema, TSDB_DATA_TYPE_INT, i, -1);
     }
   }
 
@@ -149,9 +150,9 @@ TEST(TsdbTest, createRepo) {
 
   for (int i = 0; i < nCols; i++) {
     if (i == 0) {
-      tdSchemaAppendCol(schema, TSDB_DATA_TYPE_TIMESTAMP, i, -1);
+      tdSchemaAddCol(schema, TSDB_DATA_TYPE_TIMESTAMP, i, -1);
     } else {
-      tdSchemaAppendCol(schema, TSDB_DATA_TYPE_INT, i, -1);
+      tdSchemaAddCol(schema, TSDB_DATA_TYPE_INT, i, -1);
     }
   }
 
@@ -244,7 +245,7 @@ TEST(TsdbTest, DISABLED_openRepo) {
 //   tsdbLoadCompCols(&pGroup->files[TSDB_FILE_TYPE_DATA], pBlock, (void *)pCompData);
 
   // STable *pTable = tsdbGetTableByUid(pRepo->tsdbMeta, pCompData->uid);
-  // SDataCols *pDataCols = tdNewDataCols(tdMaxRowBytesFromSchema(tsdbGetTableSchema(pRepo->tsdbMeta, pTable)), 5, 10);
+  // SDataCols *pDataCols = tdNewDataCols(tdMaxRowBytesFromSchema(tsdbGetTableSchema(pRepo->tsdbMeta, pTable)), 5);
   // tdInitDataCols(pDataCols, tsdbGetTableSchema(pRepo->tsdbMeta, pTable));
 
 //   tsdbLoadDataBlock(&pGroup->files[TSDB_FILE_TYPE_DATA], pBlock, 1, pDataCols, pCompData);

src/util/CMakeLists.txt

@@ -11,7 +11,7 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/lz4/inc)
   AUX_SOURCE_DIRECTORY(src SRC)
   ADD_LIBRARY(tutil ${SRC})
-  TARGET_LINK_LIBRARIES(tutil pthread os m rt)
+  TARGET_LINK_LIBRARIES(tutil pthread os m rt lz4)
   FIND_PATH(ICONV_INCLUDE_EXIST iconv.h /usr/include/ /usr/local/include/)
   IF (ICONV_INCLUDE_EXIST)
     ADD_DEFINITIONS(-DUSE_LIBICONV)
@@ -68,7 +68,7 @@ ELSEIF (TD_WINDOWS_64)
   LIST(APPEND SRC ./src/tutil.c)
   LIST(APPEND SRC ./src/version.c)
   ADD_LIBRARY(tutil ${SRC}) 
-  TARGET_LINK_LIBRARIES(tutil iconv regex pthread os winmm IPHLPAPI ws2_32)
+  TARGET_LINK_LIBRARIES(tutil iconv regex pthread os winmm IPHLPAPI ws2_32 lz4)
 ELSEIF(TD_DARWIN_64)
   ADD_DEFINITIONS(-DUSE_LIBICONV)
   LIST(APPEND SRC ./src/hash.c)
@@ -105,7 +105,7 @@ ELSEIF(TD_DARWIN_64)
   LIST(APPEND SRC ./src/version.c)
   LIST(APPEND SRC ./src/hash.c)
   ADD_LIBRARY(tutil ${SRC})
-  TARGET_LINK_LIBRARIES(tutil iconv pthread os)
+  TARGET_LINK_LIBRARIES(tutil iconv pthread os lz4)
 ENDIF()
 
 #  TARGET_LINK_LIBRARIES(tutil mstorage)

src/util/inc/tcache.h

@@ -69,6 +69,7 @@ typedef struct {
   SHashObj *   pHashTable;
   int          numOfElemsInTrash;  // number of element in trash
   int16_t      deleting;           // set the deleting flag to stop refreshing ASAP.
+  T_REF_DECLARE()
 
 #if defined(LINUX)
   pthread_rwlock_t lock;

src/util/inc/tscompression.h

@@ -21,7 +21,9 @@ extern "C" {
 #endif
 
 #include "taosdef.h"
+#include "tutil.h"
 
+#define COMP_OVERFLOW_BYTES 2
 #define BITS_PER_BYTE 8
 // Masks
 #define INT64MASK(_x) ((1ul << _x) - 1)
@@ -32,43 +34,220 @@ extern "C" {
 #define ONE_STAGE_COMP 1
 #define TWO_STAGE_COMP 2
 
-int tsCompressTinyint(const char* const input, int inputSize, const int nelements, char* const output, int outputSize, char algorithm,
-                      char* const buffer, int bufferSize);
-int tsCompressSmallint(const char* const input, int inputSize, const int nelements, char* const output, int outputSize, char algorith,
-                       char* const buffer, int bufferSize);
-int tsCompressInt(const char* const input, int inputSize, const int nelements, char* const output, int outputSize, char algorith,
-                  char* const buffer, int bufferSize);
-int tsCompressBigint(const char* const input, int inputSize, const int nelements, char* const output, int outputSize, char algorith,
-                     char* const buffer, int bufferSize);
-int tsCompressBool(const char* const input, int inputSize, const int nelements, char* const output, int outputSize, char algorithm,
-                   char* const buffer, int bufferSize);
-int tsCompressString(const char* const input, int inputSize, const int nelements, char* const output, int outputSize, char algorith,
-                     char* const buffer, int bufferSize);
-int tsCompressFloat(const char* const input, int inputSize, const int nelements, char* const output, int outputSize, char algorith,
-                    char* const buffer, int bufferSize);
-int tsCompressDouble(const char* const input, int inputSize, const int nelements, char* const output, int outputSize, char algorith,
-                     char* const buffer, int bufferSize);
-int tsCompressTimestamp(const char* const input, int inputSize, const int nelements, char* const output, int outputSize, char algorith,
-                        char* const buffer, int bufferSize);
-
-int tsDecompressTinyint(const char* const input, int compressedSize, const int nelements, char* const output,
-                        int outputSize, char algorithm, char* const buffer, int bufferSize);
-int tsDecompressSmallint(const char* const input, int compressedSize, const int nelements, char* const output,
-                         int outputSize, char algorithm, char* const buffer, int bufferSize);
-int tsDecompressInt(const char* const input, int compressedSize, const int nelements, char* const output, int outputSize,
-                    char algorithm, char* const buffer, int bufferSize);
-int tsDecompressBigint(const char* const input, int compressedSize, const int nelements, char* const output,
-                       int outputSize, char algorithm, char* const buffer, int bufferSize);
-int tsDecompressBool(const char* const input, int compressedSize, const int nelements, char* const output,
-                     int outputSize, char algorithm, char* const buffer, int bufferSize);
-int tsDecompressString(const char* const input, int compressedSize, const int nelements, char* const output,
-                       int outputSize, char algorithm, char* const buffer, int bufferSize);
-int tsDecompressFloat(const char* const input, int compressedSize, const int nelements, char* const output,
-                      int outputSize, char algorithm, char* const buffer, int bufferSize);
-int tsDecompressDouble(const char* const input, int compressedSize, const int nelements, char* const output,
-                       int outputSize, char algorith, char* const buffer, int bufferSize);
-int tsDecompressTimestamp(const char* const input, int compressedSize, const int nelements, char* const output,
-                          int outputSize, char algorithm, char* const buffer, int bufferSize);
+extern int tsCompressINTImp(const char *const input, const int nelements, char *const output, const char type);
+extern int tsDecompressINTImp(const char *const input, const int nelements, char *const output, const char type);
+extern int tsCompressBoolImp(const char *const input, const int nelements, char *const output);
+extern int tsDecompressBoolImp(const char *const input, const int nelements, char *const output);
+extern int tsCompressStringImp(const char *const input, int inputSize, char *const output, int outputSize);
+extern int tsDecompressStringImp(const char *const input, int compressedSize, char *const output, int outputSize);
+extern int tsCompressTimestampImp(const char *const input, const int nelements, char *const output);
+extern int tsDecompressTimestampImp(const char *const input, const int nelements, char *const output);
+extern int tsCompressDoubleImp(const char *const input, const int nelements, char *const output);
+extern int tsDecompressDoubleImp(const char *const input, const int nelements, char *const output);
+extern int tsCompressFloatImp(const char *const input, const int nelements, char *const output);
+extern int tsDecompressFloatImp(const char *const input, const int nelements, char *const output);
+
+static FORCE_INLINE int tsCompressTinyint(const char *const input, int inputSize, const int nelements, char *const output, int outputSize, char algorithm,
+                      char *const buffer, int bufferSize) {
+  if (algorithm == ONE_STAGE_COMP) {
+    return tsCompressINTImp(input, nelements, output, TSDB_DATA_TYPE_TINYINT);
+  } else if (algorithm == TWO_STAGE_COMP) {
+    int len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_TINYINT);
+    return tsCompressStringImp(buffer, len, output, outputSize);
+  } else {
+    assert(0);
+  }
+}
+
+static FORCE_INLINE int tsDecompressTinyint(const char *const input, int compressedSize, const int nelements, char *const output,
+                        int outputSize, char algorithm, char *const buffer, int bufferSize) {
+  if (algorithm == ONE_STAGE_COMP) {
+    return tsDecompressINTImp(input, nelements, output, TSDB_DATA_TYPE_TINYINT);
+  } else if (algorithm == TWO_STAGE_COMP) {
+    tsDecompressStringImp(input, compressedSize, buffer, bufferSize);
+    return tsDecompressINTImp(buffer, nelements, output, TSDB_DATA_TYPE_TINYINT);
+  } else {
+    assert(0);
+  }
+}
+
+static FORCE_INLINE int tsCompressSmallint(const char *const input, int inputSize, const int nelements, char *const output, int outputSize, char algorithm,
+                       char *const buffer, int bufferSize) {
+  if (algorithm == ONE_STAGE_COMP) {
+    return tsCompressINTImp(input, nelements, output, TSDB_DATA_TYPE_SMALLINT);
+  } else if (algorithm == TWO_STAGE_COMP) {
+    int len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_SMALLINT);
+    return tsCompressStringImp(buffer, len, output, outputSize);
+  } else {
+    assert(0);
+  }
+}
+
+static FORCE_INLINE int tsDecompressSmallint(const char *const input, int compressedSize, const int nelements, char *const output,
+                         int outputSize, char algorithm, char *const buffer, int bufferSize) {
+  if (algorithm == ONE_STAGE_COMP) {
+    return tsDecompressINTImp(input, nelements, output, TSDB_DATA_TYPE_SMALLINT);
+  } else if (algorithm == TWO_STAGE_COMP) {
+    tsDecompressStringImp(input, compressedSize, buffer, bufferSize);
+    return tsDecompressINTImp(buffer, nelements, output, TSDB_DATA_TYPE_SMALLINT);
+  } else {
+    assert(0);
+  }
+}
+
+static FORCE_INLINE int tsCompressInt(const char *const input, int inputSize, const int nelements, char *const output, int outputSize, char algorithm,
+                  char *const buffer, int bufferSize) {
+  if (algorithm == ONE_STAGE_COMP) {
+    return tsCompressINTImp(input, nelements, output, TSDB_DATA_TYPE_INT);
+  } else if (algorithm == TWO_STAGE_COMP) {
+    int len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_INT);
+    return tsCompressStringImp(buffer, len, output, outputSize);
+  } else {
+    assert(0);
+  }
+}
+
+static FORCE_INLINE int tsDecompressInt(const char *const input, int compressedSize, const int nelements, char *const output,
+                    int outputSize, char algorithm, char *const buffer, int bufferSize) {
+  if (algorithm == ONE_STAGE_COMP) {
+    return tsDecompressINTImp(input, nelements, output, TSDB_DATA_TYPE_INT);
+  } else if (algorithm == TWO_STAGE_COMP) {
+    tsDecompressStringImp(input, compressedSize, buffer, bufferSize);
+    return tsDecompressINTImp(buffer, nelements, output, TSDB_DATA_TYPE_INT);
+  } else {
+    assert(0);
+  }
+}
+
+static FORCE_INLINE int tsCompressBigint(const char *const input, int inputSize, const int nelements, char *const output, int outputSize,
+                     char algorithm, char *const buffer, int bufferSize) {
+  if (algorithm == ONE_STAGE_COMP) {
+    return tsCompressINTImp(input, nelements, output, TSDB_DATA_TYPE_BIGINT);
+  } else if (algorithm == TWO_STAGE_COMP) {
+    int len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_BIGINT);
+    return tsCompressStringImp(buffer, len, output, outputSize);
+  } else {
+    assert(0);
+  }
+}
+
+static FORCE_INLINE int tsDecompressBigint(const char *const input, int compressedSize, const int nelements, char *const output,
+                       int outputSize, char algorithm, char *const buffer, int bufferSize) {
+  if (algorithm == ONE_STAGE_COMP) {
+    return tsDecompressINTImp(input, nelements, output, TSDB_DATA_TYPE_BIGINT);
+  } else if (algorithm == TWO_STAGE_COMP) {
+    tsDecompressStringImp(input, compressedSize, buffer, bufferSize);
+    return tsDecompressINTImp(buffer, nelements, output, TSDB_DATA_TYPE_BIGINT);
+  } else {
+    assert(0);
+  }
+}
+
+static FORCE_INLINE int tsCompressBool(const char *const input, int inputSize, const int nelements, char *const output, int outputSize, 
+                   char algorithm, char *const buffer, int bufferSize) {
+  if (algorithm == ONE_STAGE_COMP) {
+    return tsCompressBoolImp(input, nelements, output);
+  } else if (algorithm == TWO_STAGE_COMP) {
+    int len = tsCompressBoolImp(input, nelements, buffer);
+    return tsCompressStringImp(buffer, len, output, outputSize);
+  } else {
+    assert(0);
+  }
+}
+
+static FORCE_INLINE int tsDecompressBool(const char *const input, int compressedSize, const int nelements, char *const output,
+                     int outputSize, char algorithm, char *const buffer, int bufferSize) {
+  if (algorithm == ONE_STAGE_COMP) {
+    return tsDecompressBoolImp(input, nelements, output);
+  } else if (algorithm == TWO_STAGE_COMP) {
+    tsDecompressStringImp(input, compressedSize, buffer, bufferSize);
+    return tsDecompressBoolImp(buffer, nelements, output);
+  } else {
+    assert(0);
+  }
+}
+
+static FORCE_INLINE int tsCompressString(const char *const input, int inputSize, const int nelements, char *const output, int outputSize,
+                     char algorithm, char *const buffer, int bufferSize) {
+  return tsCompressStringImp(input, inputSize, output, outputSize);
+}
+
+static FORCE_INLINE int tsDecompressString(const char *const input, int compressedSize, const int nelements, char *const output,
+                       int outputSize, char algorithm, char *const buffer, int bufferSize) {
+  return tsDecompressStringImp(input, compressedSize, output, outputSize);
+}
+
+static FORCE_INLINE int tsCompressFloat(const char *const input, int inputSize, const int nelements, char *const output, int outputSize,
+                    char algorithm, char *const buffer, int bufferSize) {
+  if (algorithm == ONE_STAGE_COMP) {
+    return tsCompressFloatImp(input, nelements, output);
+  } else if (algorithm == TWO_STAGE_COMP) {
+    int len = tsCompressFloatImp(input, nelements, buffer);
+    return tsCompressStringImp(buffer, len, output, outputSize);
+  } else {
+    assert(0);
+  }
+}
+
+static FORCE_INLINE int tsDecompressFloat(const char *const input, int compressedSize, const int nelements, char *const output,
+                      int outputSize, char algorithm, char *const buffer, int bufferSize) {
+  if (algorithm == ONE_STAGE_COMP) {
+    return tsDecompressFloatImp(input, nelements, output);
+  } else if (algorithm == TWO_STAGE_COMP) {
+    tsDecompressStringImp(input, compressedSize, buffer, bufferSize);
+    return tsDecompressFloatImp(buffer, nelements, output);
+  } else {
+    assert(0);
+  }
+}
+
+static FORCE_INLINE int tsCompressDouble(const char *const input, int inputSize, const int nelements, char *const output, int outputSize,
+                     char algorithm, char *const buffer, int bufferSize) {
+  if (algorithm == ONE_STAGE_COMP) {
+    return tsCompressDoubleImp(input, nelements, output);
+  } else if (algorithm == TWO_STAGE_COMP) {
+    int len = tsCompressDoubleImp(input, nelements, buffer);
+    return tsCompressStringImp(buffer, len, output, outputSize);
+  } else {
+    assert(0);
+  }
+}
+
+static FORCE_INLINE int tsDecompressDouble(const char *const input, int compressedSize, const int nelements, char *const output,
+                       int outputSize, char algorithm, char *const buffer, int bufferSize) {
+  if (algorithm == ONE_STAGE_COMP) {
+    return tsDecompressDoubleImp(input, nelements, output);
+  } else if (algorithm == TWO_STAGE_COMP) {
+    tsDecompressStringImp(input, compressedSize, buffer, bufferSize);
+    return tsDecompressDoubleImp(buffer, nelements, output);
+  } else {
+    assert(0);
+  }
+}
+
+static FORCE_INLINE int tsCompressTimestamp(const char *const input, int inputSize, const int nelements, char *const output, int outputSize,
+                        char algorithm, char *const buffer, int bufferSize) {
+  if (algorithm == ONE_STAGE_COMP) {
+    return tsCompressTimestampImp(input, nelements, output);
+  } else if (algorithm == TWO_STAGE_COMP) {
+    int len = tsCompressTimestampImp(input, nelements, buffer);
+    return tsCompressStringImp(buffer, len, output, outputSize);
+  } else {
+    assert(0);
+  }
+}
+
+static FORCE_INLINE int tsDecompressTimestamp(const char *const input, int compressedSize, const int nelements, char *const output,
+                          int outputSize, char algorithm, char *const buffer, int bufferSize) {
+  if (algorithm == ONE_STAGE_COMP) {
+    return tsDecompressTimestampImp(input, nelements, output);
+  } else if (algorithm == TWO_STAGE_COMP) {
+    tsDecompressStringImp(input, compressedSize, buffer, bufferSize);
+    return tsDecompressTimestampImp(buffer, nelements, output);
+  } else {
+    assert(0);
+  }
+}
 
 #ifdef __cplusplus
 }

src/util/inc/tsocket.h

@@ -33,10 +33,6 @@ int  taosOpenTcpServerSocket(uint32_t ip, uint16_t port);
 int  taosKeepTcpAlive(int sockFd);
 void taosCloseTcpSocket(int sockFd);
 
-int taosOpenUDServerSocket(uint32_t ip, uint16_t port);
-int taosOpenUDClientSocket(uint32_t ip, uint16_t port);
-int taosOpenRawSocket(uint32_t ip);
-
 int      taosGetFqdn(char *);
 uint32_t taosGetIpFromFqdn(const char *);
 void     tinet_ntoa(char *ipstr, unsigned int ip);

src/util/inc/tutil.h

@@ -44,7 +44,10 @@ extern "C" {
 
 #define tclose(x) taosCloseSocket(x)
 
-#ifdef ASSERTION
+// Pointer p drift right by b bytes
+#define POINTER_DRIFT(p, b) ((void *)((char *)(p) + (b)))
+
+#ifndef NDEBUG
 #define ASSERT(x) assert(x)
 #else
 #define ASSERT(x)

src/util/src/hash.c

@@ -520,8 +520,8 @@ SHashMutableIterator *taosHashCreateIter(SHashObj *pHashObj) {
 
 static SHashNode *getNextHashNode(SHashMutableIterator *pIter) {
   assert(pIter != NULL);
-  pIter->entryIndex += 1;
 
+  pIter->entryIndex++;
   while (pIter->entryIndex < pIter->pHashObj->capacity) {
     SHashEntry *pEntry = pIter->pHashObj->hashList[pIter->entryIndex];
     if (pEntry->next == NULL) {

src/util/src/tcache.c

@@ -324,13 +324,26 @@ static void doCleanupDataCache(SCacheObj *pCacheObj) {
 static void taosCacheRefresh(void *handle, void *tmrId) {
   SCacheObj *pCacheObj = (SCacheObj *)handle;
   
-  if (pCacheObj == NULL || taosHashGetSize(pCacheObj->pHashTable) == 0) {
+  if (pCacheObj == NULL || T_REF_VAL_GET(pCacheObj) == 0) {
     uTrace("object is destroyed. no refresh retry");
     return;
   }
   
-  if (pCacheObj->deleting == 1) {
+  int16_t ref = T_REF_INC(pCacheObj);
+  if (ref == 1) {
+    T_REF_DEC(pCacheObj);
+    return;
+  }
+  
+  // todo add the ref before start the timer
+  int32_t num = taosHashGetSize(pCacheObj->pHashTable);
+  if (num == 0) {
+    ref = T_REF_DEC(pCacheObj);
+    if (ref == 0) {
       doCleanupDataCache(pCacheObj);
+    } else {
+      taosTmrReset(taosCacheRefresh, pCacheObj->refreshTime, pCacheObj, pCacheObj->tmrCtrl, &pCacheObj->pTimer);
+    }
     return;
   }
   
@@ -356,10 +369,13 @@ static void taosCacheRefresh(void *handle, void *tmrId) {
   
   taosHashDestroyIter(pIter);
 
-  if (pCacheObj->deleting == 1) {  // clean up resources and abort
+    taosTrashEmpty(pCacheObj, false);
+    
+    ref = T_REF_DEC(pCacheObj);
+    if (ref == 0)  {
       doCleanupDataCache(pCacheObj);
+      return;
     } else {
-    taosTrashEmpty(pCacheObj, false);
       taosTmrReset(taosCacheRefresh, pCacheObj->refreshTime, pCacheObj, pCacheObj->tmrCtrl, &pCacheObj->pTimer);
     }
 }
@@ -399,6 +415,7 @@ SCacheObj *taosCacheInit(void *tmrCtrl, int64_t refreshTime) {
     return NULL;
   }
   
+  T_REF_INC(pCacheObj);
   return pCacheObj;
 }
 
@@ -552,5 +569,8 @@ void taosCacheCleanup(SCacheObj *pCacheObj) {
     return;
   }
   
-  pCacheObj->deleting = 1;
+  int32_t ref = T_REF_DEC(pCacheObj);
+  if (ref == 0) {
+    doCleanupDataCache(pCacheObj);
+  }
 }

src/util/src/tcompression.c

@@ -56,223 +56,6 @@ const int TEST_NUMBER = 1;
 #define is_bigendian() ((*(char *)&TEST_NUMBER) == 0)
 #define SIMPLE8B_MAX_INT64 ((uint64_t)2305843009213693951L)
 
-// Function declarations
-int tsCompressINTImp(const char *const input, const int nelements, char *const output, const char type);
-int tsDecompressINTImp(const char *const input, const int nelements, char *const output, const char type);
-int tsCompressBoolImp(const char *const input, const int nelements, char *const output);
-int tsDecompressBoolImp(const char *const input, const int nelements, char *const output);
-int tsCompressStringImp(const char *const input, int inputSize, char *const output, int outputSize);
-int tsDecompressStringImp(const char *const input, int compressedSize, char *const output, int outputSize);
-int tsCompressTimestampImp(const char *const input, const int nelements, char *const output);
-int tsDecompressTimestampImp(const char *const input, const int nelements, char *const output);
-int tsCompressDoubleImp(const char *const input, const int nelements, char *const output);
-int tsDecompressDoubleImp(const char *const input, const int nelements, char *const output);
-int tsCompressFloatImp(const char *const input, const int nelements, char *const output);
-int tsDecompressFloatImp(const char *const input, const int nelements, char *const output);
-
-/* ----------------------------------------------Compression function used by
- * others ---------------------------------------------- */
-int tsCompressTinyint(const char *const input, int inputSize, const int nelements, char *const output, int outputSize, char algorithm,
-                      char *const buffer, int bufferSize) {
-  if (algorithm == ONE_STAGE_COMP) {
-    return tsCompressINTImp(input, nelements, output, TSDB_DATA_TYPE_TINYINT);
-  } else if (algorithm == TWO_STAGE_COMP) {
-    int len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_TINYINT);
-    return tsCompressStringImp(buffer, len, output, outputSize);
-  } else {
-    assert(0);
-  }
-}
-
-int tsDecompressTinyint(const char *const input, int compressedSize, const int nelements, char *const output,
-                        int outputSize, char algorithm, char *const buffer, int bufferSize) {
-  if (algorithm == ONE_STAGE_COMP) {
-    return tsDecompressINTImp(input, nelements, output, TSDB_DATA_TYPE_TINYINT);
-  } else if (algorithm == TWO_STAGE_COMP) {
-    tsDecompressStringImp(input, compressedSize, buffer, bufferSize);
-    return tsDecompressINTImp(buffer, nelements, output, TSDB_DATA_TYPE_TINYINT);
-  } else {
-    assert(0);
-  }
-}
-
-int tsCompressSmallint(const char *const input, int inputSize, const int nelements, char *const output, int outputSize, char algorithm,
-                       char *const buffer, int bufferSize) {
-  if (algorithm == ONE_STAGE_COMP) {
-    return tsCompressINTImp(input, nelements, output, TSDB_DATA_TYPE_SMALLINT);
-  } else if (algorithm == TWO_STAGE_COMP) {
-    int len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_SMALLINT);
-    return tsCompressStringImp(buffer, len, output, outputSize);
-  } else {
-    assert(0);
-  }
-}
-
-int tsDecompressSmallint(const char *const input, int compressedSize, const int nelements, char *const output,
-                         int outputSize, char algorithm, char *const buffer, int bufferSize) {
-  if (algorithm == ONE_STAGE_COMP) {
-    return tsDecompressINTImp(input, nelements, output, TSDB_DATA_TYPE_SMALLINT);
-  } else if (algorithm == TWO_STAGE_COMP) {
-    tsDecompressStringImp(input, compressedSize, buffer, bufferSize);
-    return tsDecompressINTImp(buffer, nelements, output, TSDB_DATA_TYPE_SMALLINT);
-  } else {
-    assert(0);
-  }
-}
-
-int tsCompressInt(const char *const input, int inputSize, const int nelements, char *const output, int outputSize, char algorithm,
-                  char *const buffer, int bufferSize) {
-  if (algorithm == ONE_STAGE_COMP) {
-    return tsCompressINTImp(input, nelements, output, TSDB_DATA_TYPE_INT);
-  } else if (algorithm == TWO_STAGE_COMP) {
-    int len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_INT);
-    return tsCompressStringImp(buffer, len, output, outputSize);
-  } else {
-    assert(0);
-  }
-}
-
-int tsDecompressInt(const char *const input, int compressedSize, const int nelements, char *const output,
-                    int outputSize, char algorithm, char *const buffer, int bufferSize) {
-  if (algorithm == ONE_STAGE_COMP) {
-    return tsDecompressINTImp(input, nelements, output, TSDB_DATA_TYPE_INT);
-  } else if (algorithm == TWO_STAGE_COMP) {
-    tsDecompressStringImp(input, compressedSize, buffer, bufferSize);
-    return tsDecompressINTImp(buffer, nelements, output, TSDB_DATA_TYPE_INT);
-  } else {
-    assert(0);
-  }
-}
-
-int tsCompressBigint(const char *const input, int inputSize, const int nelements, char *const output, int outputSize,
-                     char algorithm, char *const buffer, int bufferSize) {
-  if (algorithm == ONE_STAGE_COMP) {
-    return tsCompressINTImp(input, nelements, output, TSDB_DATA_TYPE_BIGINT);
-  } else if (algorithm == TWO_STAGE_COMP) {
-    int len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_BIGINT);
-    return tsCompressStringImp(buffer, len, output, outputSize);
-  } else {
-    assert(0);
-  }
-}
-
-int tsDecompressBigint(const char *const input, int compressedSize, const int nelements, char *const output,
-                       int outputSize, char algorithm, char *const buffer, int bufferSize) {
-  if (algorithm == ONE_STAGE_COMP) {
-    return tsDecompressINTImp(input, nelements, output, TSDB_DATA_TYPE_BIGINT);
-  } else if (algorithm == TWO_STAGE_COMP) {
-    tsDecompressStringImp(input, compressedSize, buffer, bufferSize);
-    return tsDecompressINTImp(buffer, nelements, output, TSDB_DATA_TYPE_BIGINT);
-  } else {
-    assert(0);
-  }
-}
-
-int tsCompressBool(const char *const input, int inputSize, const int nelements, char *const output, int outputSize, 
-                   char algorithm, char *const buffer, int bufferSize) {
-  if (algorithm == ONE_STAGE_COMP) {
-    return tsCompressBoolImp(input, nelements, output);
-  } else if (algorithm == TWO_STAGE_COMP) {
-    int len = tsCompressBoolImp(input, nelements, buffer);
-    return tsCompressStringImp(buffer, len, output, outputSize);
-  } else {
-    assert(0);
-  }
-}
-
-int tsDecompressBool(const char *const input, int compressedSize, const int nelements, char *const output,
-                     int outputSize, char algorithm, char *const buffer, int bufferSize) {
-  if (algorithm == ONE_STAGE_COMP) {
-    return tsDecompressBoolImp(input, nelements, output);
-  } else if (algorithm == TWO_STAGE_COMP) {
-    tsDecompressStringImp(input, compressedSize, buffer, bufferSize);
-    return tsDecompressBoolImp(buffer, nelements, output);
-  } else {
-    assert(0);
-  }
-}
-
-int tsCompressString(const char *const input, int inputSize, const int nelements, char *const output, int outputSize,
-                     char algorithm, char *const buffer, int bufferSize) {
-  return tsCompressStringImp(input, inputSize, output, outputSize);
-}
-
-int tsDecompressString(const char *const input, int compressedSize, const int nelements, char *const output,
-                       int outputSize, char algorithm, char *const buffer, int bufferSize) {
-  return tsDecompressStringImp(input, compressedSize, output, outputSize);
-}
-
-int tsCompressFloat(const char *const input, int inputSize, const int nelements, char *const output, int outputSize,
-                    char algorithm, char *const buffer, int bufferSize) {
-  if (algorithm == ONE_STAGE_COMP) {
-    return tsCompressFloatImp(input, nelements, output);
-  } else if (algorithm == TWO_STAGE_COMP) {
-    int len = tsCompressFloatImp(input, nelements, buffer);
-    return tsCompressStringImp(buffer, len, output, outputSize);
-  } else {
-    assert(0);
-  }
-}
-
-int tsDecompressFloat(const char *const input, int compressedSize, const int nelements, char *const output,
-                      int outputSize, char algorithm, char *const buffer, int bufferSize) {
-  if (algorithm == ONE_STAGE_COMP) {
-    return tsDecompressFloatImp(input, nelements, output);
-  } else if (algorithm == TWO_STAGE_COMP) {
-    tsDecompressStringImp(input, compressedSize, buffer, bufferSize);
-    return tsDecompressFloatImp(buffer, nelements, output);
-  } else {
-    assert(0);
-  }
-}
-int tsCompressDouble(const char *const input, int inputSize, const int nelements, char *const output, int outputSize,
-                     char algorithm, char *const buffer, int bufferSize) {
-  if (algorithm == ONE_STAGE_COMP) {
-    return tsCompressDoubleImp(input, nelements, output);
-  } else if (algorithm == TWO_STAGE_COMP) {
-    int len = tsCompressDoubleImp(input, nelements, buffer);
-    return tsCompressStringImp(buffer, len, output, outputSize);
-  } else {
-    assert(0);
-  }
-}
-
-int tsDecompressDouble(const char *const input, int compressedSize, const int nelements, char *const output,
-                       int outputSize, char algorithm, char *const buffer, int bufferSize) {
-  if (algorithm == ONE_STAGE_COMP) {
-    return tsDecompressDoubleImp(input, nelements, output);
-  } else if (algorithm == TWO_STAGE_COMP) {
-    tsDecompressStringImp(input, compressedSize, buffer, bufferSize);
-    return tsDecompressDoubleImp(buffer, nelements, output);
-  } else {
-    assert(0);
-  }
-}
-
-int tsCompressTimestamp(const char *const input, int inputSize, const int nelements, char *const output, int outputSize,
-                        char algorithm, char *const buffer, int bufferSize) {
-  if (algorithm == ONE_STAGE_COMP) {
-    return tsCompressTimestampImp(input, nelements, output);
-  } else if (algorithm == TWO_STAGE_COMP) {
-    int len = tsCompressTimestampImp(input, nelements, buffer);
-    return tsCompressStringImp(buffer, len, output, outputSize);
-  } else {
-    assert(0);
-  }
-}
-
-int tsDecompressTimestamp(const char *const input, int compressedSize, const int nelements, char *const output,
-                          int outputSize, char algorithm, char *const buffer, int bufferSize) {
-  if (algorithm == ONE_STAGE_COMP) {
-    return tsDecompressTimestampImp(input, nelements, output);
-  } else if (algorithm == TWO_STAGE_COMP) {
-    tsDecompressStringImp(input, compressedSize, buffer, bufferSize);
-    return tsDecompressTimestampImp(buffer, nelements, output);
-  } else {
-    assert(0);
-  }
-}
-
 bool safeInt64Add(int64_t a, int64_t b) {
   if ((a > 0 && b > INT64_MAX - a) || (a < 0 && b < INT64_MIN - a)) return false;
   return true;

src/util/src/tsocket.c

@@ -14,7 +14,6 @@
  */
 
 #include "os.h"
-#include "tglobal.h"
 #include "tulog.h"
 #include "tsocket.h"
 #include "tutil.h"
@@ -26,8 +25,13 @@ int taosGetFqdn(char *fqdn) {
 
   struct hostent* h;
   h = gethostbyname(hostname);
+  if (h != NULL) {
     strcpy(fqdn, h->h_name);
     return 0;
+  } else {
+    uError("failed to get host name");
+    return -1;
+  }
 }
 
 uint32_t taosGetIpFromFqdn(const char *fqdn) {
@@ -394,38 +398,6 @@ int taosOpenTcpServerSocket(uint32_t ip, uint16_t port) {
   return sockFd;
 }
 
-int taosOpenRawSocket(uint32_t ip) {
-  int                fd, hold;
-  struct sockaddr_in rawAdd;
-
-  uTrace("open udp raw socket:%s", ip);
-
-  fd = (int)socket(AF_INET, SOCK_RAW, IPPROTO_UDP);
-  if (fd < 0) {
-    uError("failed to open raw socket: %d (%s)", errno, strerror(errno));
-    return -1;
-  }
-
-  hold = 1;
-  if (taosSetSockOpt(fd, IPPROTO_IP, IP_HDRINCL, (void *)&hold, sizeof(hold)) < 0) {
-    uError("failed to set hold option: %d (%s)", errno, strerror(errno));
-    close(fd);
-    return -1;
-  }
-
-  bzero((char *)&rawAdd, sizeof(rawAdd));
-  rawAdd.sin_family = AF_INET;
-  rawAdd.sin_addr.s_addr = ip;
-
-  if (bind(fd, (struct sockaddr *)&rawAdd, sizeof(rawAdd)) < 0) {
-    uError("failed to bind RAW socket:(%s)", strerror(errno));
-    close(fd);
-    return -1;
-  }
-
-  return fd;
-}
-
 void tinet_ntoa(char *ipstr, unsigned int ip) {
   sprintf(ipstr, "%d.%d.%d.%d", ip & 0xFF, (ip >> 8) & 0xFF, (ip >> 16) & 0xFF, ip >> 24);
 }

src/util/tests/CMakeLists.txt

@@ -10,6 +10,7 @@ IF (HEADER_GTEST_INCLUDE_DIR AND LIB_GTEST_STATIC_DIR)
     INCLUDE_DIRECTORIES(${HEADER_GTEST_INCLUDE_DIR})
     AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
 
-    ADD_EXECUTABLE(utilTest ${SOURCE_LIST})
+#    ADD_EXECUTABLE(utilTest ${SOURCE_LIST})
+    ADD_EXECUTABLE(utilTest hashTest.cpp cacheTest.cpp)
     TARGET_LINK_LIBRARIES(utilTest tutil common gtest pthread)
 ENDIF()
\ No newline at end of file

src/util/tests/cacheTest.cpp

@@ -25,7 +25,7 @@ TEST(testCase, client_cache_test) {
   const char* key1 = "test1";
   char data1[] = "test11";
 
-  char* cachedObj = (char*) taosCachePut(tscCacheHandle, key1, data1, strlen(data1), 1);
+  char* cachedObj = (char*) taosCachePut(tscCacheHandle, key1, data1, strlen(data1)+1, 1);
   sleep(REFRESH_TIME_IN_SEC+1);
 
   printf("obj is still valid: %s\n", cachedObj);
@@ -34,19 +34,17 @@ TEST(testCase, client_cache_test) {
   taosCacheRelease(tscCacheHandle, (void**) &cachedObj, false);
 
   /* the object is cleared by cache clean operation */
-  cachedObj = (char*) taosCachePut(tscCacheHandle, key1, data2, strlen(data2), 20);
+  cachedObj = (char*) taosCachePut(tscCacheHandle, key1, data2, strlen(data2)+1, 20);
   printf("after updated: %s\n", cachedObj);
 
   printf("start to remove data from cache\n");
   taosCacheRelease(tscCacheHandle, (void**) &cachedObj, false);
   printf("end of removing data from cache\n");
 
-  getchar();
-
   const char* key3 = "test2";
   const char* data3 = "kkkkkkk";
 
-  char* cachedObj2 = (char*) taosCachePut(tscCacheHandle, key3, data3, strlen(data3), 1);
+  char* cachedObj2 = (char*) taosCachePut(tscCacheHandle, key3, data3, strlen(data3) + 1, 1);
   printf("%s\n", cachedObj2);
 
   taosCacheRelease(tscCacheHandle, (void**) &cachedObj2, false);
@@ -57,18 +55,18 @@ TEST(testCase, client_cache_test) {
 
   char key5[] = "test5";
   char data5[] = "data5kkkkk";
-  cachedObj2 = (char*) taosCachePut(tscCacheHandle, key5, data5, strlen(data5), 20);
+  cachedObj2 = (char*) taosCachePut(tscCacheHandle, key5, data5, strlen(data5) + 1, 20);
 
   const char* data6= "new Data after updated";
   taosCacheRelease(tscCacheHandle, (void**) &cachedObj2, false);
 
-  cachedObj2 = (char*) taosCachePut(tscCacheHandle, key5, data6, strlen(data6), 20);
+  cachedObj2 = (char*) taosCachePut(tscCacheHandle, key5, data6, strlen(data6) + 1, 20);
   printf("%s\n", cachedObj2);
 
   taosCacheRelease(tscCacheHandle, (void**) &cachedObj2, true);
 
   const char* data7 = "add call update procedure";
-  cachedObj2 = (char*) taosCachePut(tscCacheHandle, key5, data7, strlen(data7), 20);
+  cachedObj2 = (char*) taosCachePut(tscCacheHandle, key5, data7, strlen(data7) + 1, 20);
   printf("%s\n=======================================\n\n", cachedObj2);
 
   char* cc = (char*) taosCacheAcquireByName(tscCacheHandle, key5);
@@ -137,6 +135,4 @@ TEST(testCase, cache_resize_test) {
   printf("retrieve %d object cost:%" PRIu64 " us,avg:%f\n", num, endTime - startTime, (endTime - startTime)/(double)num);
 
   taosCacheCleanup(pCache);
-  taosMsleep(20000);
-  getchar();
 }
\ No newline at end of file

src/vnode/src/vnodeMain.c

@@ -96,14 +96,16 @@ int32_t vnodeCreate(SMDCreateVnodeMsg *pVnodeCfg) {
   }
 
   STsdbCfg tsdbCfg = {0};
-  tsdbCfg.precision           = pVnodeCfg->cfg.precision;
-  tsdbCfg.compression         = pVnodeCfg->cfg.compression;;
   tsdbCfg.tsdbId              = pVnodeCfg->cfg.vgId;
+  tsdbCfg.cacheBlockSize      = pVnodeCfg->cfg.cacheBlockSize;
+  tsdbCfg.totalBlocks         = pVnodeCfg->cfg.totalBlocks;
   tsdbCfg.maxTables           = pVnodeCfg->cfg.maxTables;
   tsdbCfg.daysPerFile         = pVnodeCfg->cfg.daysPerFile;
+  tsdbCfg.keep                = pVnodeCfg->cfg.daysToKeep;
   tsdbCfg.minRowsPerFileBlock = pVnodeCfg->cfg.minRowsPerFileBlock;
   tsdbCfg.maxRowsPerFileBlock = pVnodeCfg->cfg.maxRowsPerFileBlock;
-  tsdbCfg.keep                = pVnodeCfg->cfg.daysToKeep;
+  tsdbCfg.precision           = pVnodeCfg->cfg.precision;
+  tsdbCfg.compression         = pVnodeCfg->cfg.compression;;
   
   char tsdbDir[TSDB_FILENAME_LEN] = {0};
   sprintf(tsdbDir, "%s/vnode%d/tsdb", tsVnodeDir, pVnodeCfg->cfg.vgId);

src/vnode/src/vnodeWrite.c

@@ -123,7 +123,7 @@ static int32_t vnodeProcessCreateTableMsg(SVnodeObj *pVnode, void *pCont, SRspRe
 
   STSchema *pDestSchema = tdNewSchema(numOfColumns);
   for (int i = 0; i < numOfColumns; i++) {
-    tdSchemaAppendCol(pDestSchema, pSchema[i].type, htons(pSchema[i].colId), htons(pSchema[i].bytes));
+    tdSchemaAddCol(pDestSchema, pSchema[i].type, htons(pSchema[i].colId), htons(pSchema[i].bytes));
   }
   tsdbTableSetSchema(&tCfg, pDestSchema, false);
   tsdbTableSetName(&tCfg, pTable->tableId, false);
@@ -131,7 +131,7 @@ static int32_t vnodeProcessCreateTableMsg(SVnodeObj *pVnode, void *pCont, SRspRe
   if (numOfTags != 0) {
     STSchema *pDestTagSchema = tdNewSchema(numOfTags);
     for (int i = numOfColumns; i < totalCols; i++) {
-      tdSchemaAppendCol(pDestTagSchema, pSchema[i].type, htons(pSchema[i].colId), htons(pSchema[i].bytes));
+      tdSchemaAddCol(pDestTagSchema, pSchema[i].type, htons(pSchema[i].colId), htons(pSchema[i].bytes));
     }
     tsdbTableSetTagSchema(&tCfg, pDestTagSchema, false);
     tsdbTableSetSName(&tCfg, pTable->superTableId, false);
@@ -142,7 +142,8 @@ static int32_t vnodeProcessCreateTableMsg(SVnodeObj *pVnode, void *pCont, SRspRe
     SDataRow dataRow = tdNewDataRowFromSchema(pDestTagSchema);
 
     for (int i = 0; i < numOfTags; i++) {
-      tdAppendColVal(dataRow, pTagData + accumBytes, pDestTagSchema->columns + i);
+      STColumn *pTCol = schemaColAt(pDestSchema, i);
+      tdAppendColVal(dataRow, pTagData + accumBytes, pTCol->type, pTCol->bytes, pTCol->offset);
       accumBytes += htons(pSchema[i + numOfColumns].bytes);
     }
     tsdbTableSetTagValue(&tCfg, dataRow, false);
@@ -189,14 +190,14 @@ static int32_t vnodeProcessAlterTableMsg(SVnodeObj *pVnode, void *pCont, SRspRet
 
   STSchema *pDestSchema = tdNewSchema(numOfColumns);
   for (int i = 0; i < numOfColumns; i++) {
-    tdSchemaAppendCol(pDestSchema, pSchema[i].type, htons(pSchema[i].colId), htons(pSchema[i].bytes));
+    tdSchemaAddCol(pDestSchema, pSchema[i].type, htons(pSchema[i].colId), htons(pSchema[i].bytes));
   }
   tsdbTableSetSchema(&tCfg, pDestSchema, false);
 
   if (numOfTags != 0) {
     STSchema *pDestTagSchema = tdNewSchema(numOfTags);
     for (int i = numOfColumns; i < totalCols; i++) {
-      tdSchemaAppendCol(pDestTagSchema, pSchema[i].type, htons(pSchema[i].colId), htons(pSchema[i].bytes));
+      tdSchemaAddCol(pDestTagSchema, pSchema[i].type, htons(pSchema[i].colId), htons(pSchema[i].bytes));
     }
     tsdbTableSetTagSchema(&tCfg, pDestTagSchema, false);
 
@@ -205,7 +206,8 @@ static int32_t vnodeProcessAlterTableMsg(SVnodeObj *pVnode, void *pCont, SRspRet
     SDataRow dataRow = tdNewDataRowFromSchema(pDestTagSchema);
 
     for (int i = 0; i < numOfTags; i++) {
-      tdAppendColVal(dataRow, pTagData + accumBytes, pDestTagSchema->columns + i);
+      STColumn *pTCol = schemaColAt(pDestTagSchema, i);
+      tdAppendColVal(dataRow, pTagData + accumBytes, pTCol->type, pTCol->bytes, pTCol->offset);
       accumBytes += htons(pSchema[i + numOfColumns].bytes);
     }
     tsdbTableSetTagValue(&tCfg, dataRow, false);

tests/pytest/smoketest.sh

@@ -23,6 +23,17 @@ sleep 1
 python3 ./test.py $1 -f insert/tinyint.py
 python3 ./test.py -s $1
 sleep 1
+
+python3 ./test.py $1 -f table/column_name.py
+python3 ./test.py -s $1
+sleep 1
+python3 ./test.py $1 -f table/column_num.py
+python3 ./test.py -s $1
+sleep 1
+python3 ./test.py $1 -f table/db_table.py
+python3 ./test.py -s $1
+sleep 1
+
 python3 ./test.py $1 -f import_merge/importDataLastTO.py 
 python3 ./test.py -s $1
 sleep 1

tests/pytest/table/column_name.py

+# -*- coding: utf-8 -*-
+
+import sys
+from util.log import *
+from util.cases import *
+from util.sql import *
+
+
+class TDTestCase:
+    def init(self, conn):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor())
+
+    def run(self):
+        tdSql.prepare()
+
+        # TSIM: system sh/stop_dnodes.sh
+        # TSIM:
+        # TSIM: system sh/ip.sh -i 1 -s up
+        # TSIM: system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1
+        # TSIM: system sh/cfg.sh -n dnode1 -c commitLog -v 0
+        # TSIM: system sh/exec.sh -n dnode1 -s start
+        # TSIM:
+        # TSIM: sleep 3000
+        # TSIM: sql connect
+        # TSIM:
+        # TSIM: $i = 0
+        # TSIM: $dbPrefix = lm_cm_db
+        # TSIM: $tbPrefix = lm_cm_tb
+        # TSIM: $db = $dbPrefix . $i
+        # TSIM: $tb = $tbPrefix . $i
+        # TSIM:
+        # TSIM: print =============== step1
+        tdLog.info('=============== step1')
+        # TSIM: sql create database $db
+        # TSIM: sql use $db
+        # TSIM:
+        # TSIM: sql drop table dd -x step0
+        tdLog.info('drop table dd -x step0')
+        tdSql.error('drop table dd')
+        # TSIM: return -1
+        # TSIM: step0:
+        # TSIM:
+        # TSIM: sql create table $tb(ts timestamp, int) -x step1
+        tdLog.info('create table tb(ts timestamp, int) -x step1')
+        tdSql.error('create table tb(ts timestamp, int)')
+        # TSIM: return -1
+        # TSIM: step1:
+        # TSIM:
+        # TSIM: sql show tables
+        tdLog.info('show tables')
+        tdSql.query('show tables')
+        # TSIM: if $rows != 0 then
+        tdLog.info('tdSql.checkRow(0)')
+        tdSql.checkRows(0)
+        # TSIM: return -1
+        # TSIM: endi
+        # TSIM:
+        # TSIM: print =============== step2
+        tdLog.info('=============== step2')
+        # TSIM: sql create table $tb (ts timestamp, s int)
+        tdLog.info('create table tb (ts timestamp, s int)')
+        tdSql.execute('create table tb (ts timestamp, s int)')
+        # TSIM: sql show tables
+        tdLog.info('show tables')
+        tdSql.query('show tables')
+        # TSIM: if $rows != 1 then
+        tdLog.info('tdSql.checkRow(1)')
+        tdSql.checkRows(1)
+        # TSIM: return -1
+        # TSIM: endi
+        # TSIM:
+        # TSIM: sql drop table $tb
+        tdLog.info('drop table tb')
+        tdSql.execute('drop table tb')
+        # TSIM: sql show tables
+        tdLog.info('show tables')
+        tdSql.query('show tables')
+        # TSIM: if $rows != 0 then
+        tdLog.info('tdSql.checkRow(0)')
+        tdSql.checkRows(0)
+        # TSIM: return -1
+        # TSIM: endi
+        # TSIM:
+        # TSIM: print =============== step3
+        tdLog.info('=============== step3')
+        # TSIM: sql create table $tb (ts timestamp, a0123456789 int)
+        tdLog.info('create table tb (ts timestamp, a0123456789 int)')
+        tdSql.execute('create table tb (ts timestamp, a0123456789 int)')
+        # TSIM: sql show tables
+        tdLog.info('show tables')
+        tdSql.query('show tables')
+        # TSIM: if $rows != 1 then
+        tdLog.info('tdSql.checkRow(1)')
+        tdSql.checkRows(1)
+        # TSIM: return -1
+        # TSIM: endi
+        # TSIM:
+        # TSIM: sql drop table $tb
+        tdLog.info('drop table tb')
+        tdSql.execute('drop table tb')
+        # TSIM: sql show tables
+        tdLog.info('show tables')
+        tdSql.query('show tables')
+        # TSIM: if $rows != 0 then
+        tdLog.info('tdSql.checkRow(0)')
+        tdSql.checkRows(0)
+        # TSIM: return -1
+        # TSIM: endi
+        # TSIM:
+        # TSIM: print =============== step4
+        tdLog.info('=============== step4')
+        # TSIM: sql create table $tb (ts timestamp,
+        # a0123456789012345678901234567890123456789 int)
+        tdLog.info(
+            'create table tb (ts timestamp, a0123456789012345678901234567890123456789 int)')
+        tdSql.execute(
+            'create table tb (ts timestamp, a0123456789012345678901234567890123456789 int)')
+        # TSIM: sql drop table $tb
+        tdLog.info('drop table tb')
+        tdSql.execute('drop table tb')
+        # TSIM:
+        # TSIM: sql show tables
+        tdLog.info('show tables')
+        tdSql.query('show tables')
+        # TSIM: if $rows != 0 then
+        tdLog.info('tdSql.checkRow(0)')
+        tdSql.checkRows(0)
+        # TSIM: return -1
+        # TSIM: endi
+        # TSIM:
+        # TSIM: print =============== step5
+        tdLog.info('=============== step5')
+        # TSIM: sql create table $tb (ts timestamp, a0123456789 int)
+        tdLog.info('create table tb (ts timestamp, a0123456789 int)')
+        tdSql.execute('create table tb (ts timestamp, a0123456789 int)')
+        # TSIM: sql show tables
+        tdLog.info('show tables')
+        tdSql.query('show tables')
+        # TSIM: if $rows != 1 then
+        tdLog.info('tdSql.checkRow(1)')
+        tdSql.checkRows(1)
+        # TSIM: return -1
+        # TSIM: endi
+        # TSIM:
+        # TSIM: sql insert into $tb values (now , 1)
+        tdLog.info("insert into tb values (now , 1)")
+        tdSql.execute("insert into tb values (now , 1)")
+        # TSIM: sql select * from $tb
+        tdLog.info('select * from tb')
+        tdSql.query('select * from tb')
+        # TSIM: if $rows != 1 then
+        tdLog.info('tdSql.checkRow(1)')
+        tdSql.checkRows(1)
+        # TSIM: return -1
+        # TSIM: endi
+        # TSIM:
+        # TSIM: sql drop database $db
+        tdLog.info('drop database db')
+        tdSql.execute('drop database db')
+        # TSIM: sql show databases
+        tdLog.info('show databases')
+        tdSql.query('show databases')
+        # TSIM: if $rows != 0 then
+        tdLog.info('tdSql.checkRow(0)')
+        tdSql.checkRows(0)
+        # TSIM: return -1
+        # TSIM: endi
+        # TSIM:
+        # TSIM:
+        # TSIM:
+        # TSIM:
+# convert end
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())

tests/pytest/table/db_table.py

+# -*- coding: utf-8 -*-
+
+import sys
+from util.log import *
+from util.cases import *
+from util.sql import *
+
+
+class TDTestCase:
+    def init(self, conn):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor())
+
+    def run(self):
+        tdSql.prepare()
+
+        tdLog.info('=============== step1')
+        tdLog.info('=============== step2')
+        tdLog.info('create table tb (ts timestamp, speed int)')
+        tdSql.execute('create table tb (ts timestamp, speed int)')
+        tdLog.info('=============== step3')
+        tdLog.info("insert into tb values (now, 1)")
+        tdSql.execute("insert into tb values (now, 1)")
+        tdLog.info('=============== step4')
+        tdLog.info('select * from tb')
+        tdSql.query('select * from tb')
+        tdLog.info('tdSql.checkRow(1)')
+        tdSql.checkRows(1)
+        tdLog.info('tdSql.checkData(0, 1, 1)')
+        tdSql.checkData(0, 1, 1)
+        tdLog.info('=============== step5')
+        tdLog.info('describe tb')
+        tdSql.query('describe tb')
+        tdLog.info('=============== step6')
+        tdLog.info('drop database db')
+        tdSql.execute('drop database db')
+        tdLog.info('show databases')
+        tdSql.query('show databases')
+        tdLog.info('tdSql.checkRow(0)')
+        tdSql.checkRows(0)
+# convert end
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())

tests/pytest/util/log.py

@@ -23,14 +23,14 @@ class TDLog:
         self.path = ""
 
     def info(self, info):
-        printf("%s %s" % (datetime.datetime.now(), info))
+        print("%s %s" % (datetime.datetime.now(), info))
 
     def sleep(self, sec):
-        printf("%s sleep %d seconds" % (datetime.datetime.now(), sec))
+        print("%s sleep %d seconds" % (datetime.datetime.now(), sec))
         time.sleep(sec)
 
     def debug(self, err):
-        printf("\033[1;36m%s %s\033[0m" % (datetime.datetime.now(), err))
+        print("\033[1;36m%s %s\033[0m" % (datetime.datetime.now(), err))
 
     def success(self, info):
         printf("\033[1;32m%s %s\033[0m" % (datetime.datetime.now(), info))
@@ -43,7 +43,7 @@ class TDLog:
         sys.exit(1)
 
     def printNoPrefix(self, info):
-        printf("\033[1;36m%s\033[0m" % (info))
+        print("\033[1;36m%s\033[0m" % (info))
 
 
 tdLog = TDLog()

tests/script/basicSuite.sim

 ################################# 
+run general/db/basic1.sim
+run general/db/basic2.sim
+run general/db/basic3.sim
+run general/db/basic4.sim
+run general/db/basic5.sim
 
-run general/db/testSuite.sim
-run general/insert/testSuite.sim
-run general/table/testSuite.sim
-run general/user/basicSuite.sim
+run general/table/basic1.sim
+run general/table/basic2.sim
+run general/table/basic3.sim
+run general/table/column_num.sim
+run general/table/column_name.sim
+run general/table/bigint.sim
+run general/table/bool.sim
+run general/table/double.sim
+run general/table/float.sim
+run general/table/int.sim
+run general/table/smallint.sim
+run general/table/tinyint.sim
+run general/table/db.table.sim
+
+run general/user/basic1.sim
+run general/user/pass_alter.sim
+run general/user/pass_len.sim
+run general/user/user_create.sim
+run general/user/user_len.sim
 
 ################################## 

tests/script/general/agg/fill.sim

 system sh/stop_dnodes.sh
-system sh/ip.sh -i 1 -s up
-system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1
+
+system sh/deploy.sh -n dnode1 -i 1
 system sh/cfg.sh -n dnode1 -c commitLog -v 0
 system sh/exec.sh -n dnode1 -s start
 

tests/script/general/agg/stream.sim

 system sh/stop_dnodes.sh
 
-system sh/ip.sh -i 1 -s up
-system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1
+
+system sh/deploy.sh -n dnode1 -i 1
 system sh/cfg.sh -n dnode1 -c clog -v 0
 
 print ========== step1

tests/script/general/alter/cached_schema_after_alter.sim

 system sh/stop_dnodes.sh
-system sh/ip.sh -i 1 -s up
-system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1
+
+system sh/deploy.sh -n dnode1 -i 1
 system sh/cfg.sh -n dnode1 -c clog -v 0
 system sh/exec.sh -n dnode1 -s start
 sleep 3000

tests/script/general/alter/count.sim

 system sh/stop_dnodes.sh
-system sh/ip.sh -i 1 -s up
-system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1
+
+system sh/deploy.sh -n dnode1 -i 1
 
 system sh/cfg.sh -n dnode1 -c clog -v 0
 system sh/cfg.sh -n dnode1 -c numOfMPeers -v 1

tests/script/general/alter/import.sim

 system sh/stop_dnodes.sh
-system sh/ip.sh -i 1 -s up
-system sh/deploy.sh -n dnode1 -m 192.168.0.1 -i 192.168.0.1
+
+system sh/deploy.sh -n dnode1 -i 1
 
 system sh/cfg.sh -n dnode1 -c clog -v 0
 system sh/cfg.sh -n dnode1 -c numOfMPeers -v 1