Merge branch 'develop' into feature/mergeimport

CMakeLists.txt

@@ -125,7 +125,9 @@ IF (NOT DEFINED TD_CLUSTER)
   # debug flag  
   #
   # ADD_DEFINITIONS(-D_CHECK_HEADER_FILE_)
-  # ADD_DEFINITIONS(-D_TAOS_MEM_TEST_)
+  IF (${MEM_CHECK} MATCHES "true")
+    ADD_DEFINITIONS(-DTAOS_MEM_CHECK)
+  ENDIF ()
 
   IF (TD_CLUSTER)
     ADD_DEFINITIONS(-DCLUSTER)

packaging/cfg/taos.cfg

@@ -58,6 +58,12 @@
 # The server and client should have the same socket type. Otherwise, connect will fail.
 # sockettype            udp
 
+# The compressed rpc message, option:
+#  -1 (no compression)
+#   0 (all message compressed),
+# > 0 (rpc message body which larger than this value will be compressed)
+# compressMsgSize       -1
+
 # RPC re-try timer, millisecond
 # rpcTimer              300
 

src/client/inc/tscSQLParser.h

@@ -105,6 +105,7 @@ enum TSQL_TYPE {
   SHOW_MODULES = 0x6c,
   SHOW_CONNECTIONS = 0x6d,
   SHOW_GRANTS = 0x6e,
+  SHOW_VNODES = 0x6f,
 
   // create dnode
   CREATE_DNODE = 0x80,

src/client/inc/tsclient.h

@@ -34,8 +34,8 @@ extern "C" {
 #include "tglobalcfg.h"
 #include "tlog.h"
 #include "tscCache.h"
-#include "tsdb.h"
 #include "tscSQLParser.h"
+#include "tsdb.h"
 #include "tsqlfunction.h"
 #include "tutil.h"
 
@@ -247,7 +247,7 @@ typedef struct STableDataBlocks {
   // for parameter ('?') binding
   uint32_t    numOfAllocedParams;
   uint32_t    numOfParams;
-  SParamInfo* params;
+  SParamInfo *params;
 } STableDataBlocks;
 
 typedef struct SDataBlockList {
@@ -262,18 +262,17 @@ typedef struct SDataBlockList {
 typedef struct {
   SOrderVal order;
   int       command;
-
-  // TODO refactor
-  int     count;
-  int16_t isInsertFromFile;  // load data from file or not
+  int       count;// TODO refactor
 
   union {
-    bool   existsCheck;
-    int8_t showType;
+    bool   existsCheck;       // check if the table exists
+    int8_t showType;          // show command type
   };
   
+  int8_t          isInsertFromFile;  // load data from file or not
+  bool            import;     // import/insert type
   char            msgType;
-  uint16_t        type;
+  uint16_t        type;  // query type
   char            intervalTimeUnit;
   int64_t         etime, stime;
   int64_t         nAggTimeInterval;  // aggregation time interval
@@ -450,12 +449,6 @@ int taos_retrieve(TAOS_RES *res);
 int32_t tscTansformSQLFunctionForMetricQuery(SSqlCmd *pCmd);
 void    tscRestoreSQLFunctionForMetricQuery(SSqlCmd *pCmd);
 
-/**
- * release both metric/meter meta information
- * @param pCmd  SSqlCmd object that contains the metric/meter meta info
- */
-void tscClearSqlMetaInfo(SSqlCmd *pCmd);
-
 void tscClearSqlMetaInfoForce(SSqlCmd *pCmd);
 
 int32_t tscCreateResPointerInfo(SSqlCmd *pCmd, SSqlRes *pRes);
@@ -483,8 +476,8 @@ void tscProcessMultiVnodesInsert(SSqlObj *pSql);
 void    tscProcessMultiVnodesInsertForFile(SSqlObj *pSql);
 void    tscKillMetricQuery(SSqlObj *pSql);
 void    tscInitResObjForLocalQuery(SSqlObj *pObj, int32_t numOfRes, int32_t rowLen);
-int32_t tscBuildResultsForEmptyRetrieval(SSqlObj *pSql);
 bool    tscIsUpdateQuery(STscObj *pObj);
+int32_t tscInvalidSQLErrMsg(char *msg, const char *additionalInfo, const char *sql);
 
 // transfer SSqlInfo to SqlCmd struct
 int32_t tscToSQLCmd(SSqlObj *pSql, struct SSqlInfo *pInfo);

src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h

@@ -9,6 +9,22 @@ extern "C" {
 #endif
 #undef com_taosdata_jdbc_TSDBJNIConnector_INVALID_CONNECTION_POINTER_VALUE
 #define com_taosdata_jdbc_TSDBJNIConnector_INVALID_CONNECTION_POINTER_VALUE 0LL
+/*
+ * Class:     com_taosdata_jdbc_TSDBJNIConnector
+ * Method:    
+ * Signature: (Ljava/lang/String;)V
+ */
+JNIEXPORT void JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_setAllocModeImp
+  (JNIEnv *, jclass, jint, jstring, jboolean);
+
+/*
+ * Class:     com_taosdata_jdbc_TSDBJNIConnector
+ * Method:    
+ * Signature: ()Ljava/lang/String;
+ */
+JNIEXPORT void JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_dumpMemoryLeakImp
+  (JNIEnv *, jclass);
+
 /*
  * Class:     com_taosdata_jdbc_TSDBJNIConnector
  * Method:    initImp

src/client/src/TSDBJNIConnector.c

@@ -111,6 +111,20 @@ void jniGetGlobalMethod(JNIEnv *env) {
   jniTrace("native method register finished");
 }
 
+JNIEXPORT void JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_setAllocModeImp(JNIEnv *env, jobject jobj, jint jMode, jstring jPath, jboolean jAutoDump) {
+  if (jPath != NULL) {
+    const char *path = (*env)->GetStringUTFChars(env, jPath, NULL);
+    taosSetAllocMode(jMode, path, !!jAutoDump);
+    (*env)->ReleaseStringUTFChars(env, jPath, path);
+  } else {
+    taosSetAllocMode(jMode, NULL, !!jAutoDump);
+  }
+}
+
+JNIEXPORT void JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_dumpMemoryLeakImp(JNIEnv *env, jobject jobj) {
+  taosDumpMemoryLeak();
+}
+
 JNIEXPORT void JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_initImp(JNIEnv *env, jobject jobj, jstring jconfigDir) {
   if (jconfigDir != NULL) {
     const char *confDir = (*env)->GetStringUTFChars(env, jconfigDir, NULL);

src/client/src/tscAsync.c

@@ -40,6 +40,7 @@ static void tscProcessAsyncRetrieveImpl(void *param, TAOS_RES *tres, int numOfRo
  */
 static void tscProcessAsyncFetchRowsProxy(void *param, TAOS_RES *tres, int numOfRows);
 
+// TODO return the correct error code to client in tscQueueAsyncError
 void taos_query_a(TAOS *taos, const char *sqlstr, void (*fp)(void *, TAOS_RES *, int), void *param) {
   STscObj *pObj = (STscObj *)taos;
   if (pObj == NULL || pObj->signature != pObj) {
@@ -58,14 +59,13 @@ void taos_query_a(TAOS *taos, const char *sqlstr, void (*fp)(void *, TAOS_RES *,
 
   taosNotePrintTsc(sqlstr);
 
-  SSqlObj *pSql = (SSqlObj *)malloc(sizeof(SSqlObj));
+  SSqlObj *pSql = (SSqlObj *)calloc(1, sizeof(SSqlObj));
   if (pSql == NULL) {
     tscError("failed to malloc sqlObj");
     tscQueueAsyncError(fp, param);
     return;
   }
 
-  memset(pSql, 0, sizeof(SSqlObj));
   SSqlCmd *pCmd = &pSql->cmd;
   SSqlRes *pRes = &pSql->res;
 

src/client/src/tscSQLParserImpl.c

@@ -746,7 +746,7 @@ void setCreateAcctSQL(SSqlInfo *pInfo, int32_t type, SSQLToken *pName, SSQLToken
 }
 
 void setDefaultCreateDbOption(SCreateDBInfo *pDBInfo) {
-  pDBInfo->numOfBlocksPerTable = -1;
+  pDBInfo->numOfBlocksPerTable = 50;
   pDBInfo->compressionLevel = -1;
 
   pDBInfo->commitLog = -1;

src/client/src/tscSchemaUtil.c

@@ -123,6 +123,7 @@ bool tsMeterMetaIdentical(SMeterMeta* p1, SMeterMeta* p2) {
   return memcmp(p1, p2, size) == 0;
 }
 
+//todo refactor
 static FORCE_INLINE char* skipSegments(char* input, char delimiter, int32_t num) {
   for (int32_t i = 0; i < num; ++i) {
     while (*input != 0 && *input++ != delimiter) {

src/client/src/tscServer.c

@@ -1415,7 +1415,7 @@ int tscBuildSubmitMsg(SSqlObj *pSql) {
   pMsg = pStart;
 
   pShellMsg = (SShellSubmitMsg *)pMsg;
-  pShellMsg->import = pSql->cmd.order.order;
+  pShellMsg->import = pSql->cmd.import;
   pShellMsg->vnode = htons(pMeterMeta->vpeerDesc[pMeterMeta->index].vnode);
   pShellMsg->numOfSid = htonl(pSql->cmd.count);  // number of meters to be inserted
 
@@ -2175,7 +2175,7 @@ int tscBuildShowMsg(SSqlObj *pSql) {
   pShowMsg = (SShowMsg *)pMsg;
   pShowMsg->type = pCmd->showType;
 
-  if ((pShowMsg->type == TSDB_MGMT_TABLE_TABLE || pShowMsg->type == TSDB_MGMT_TABLE_METRIC) && pCmd->payloadLen != 0) {
+  if ((pShowMsg->type == TSDB_MGMT_TABLE_TABLE || pShowMsg->type == TSDB_MGMT_TABLE_METRIC || pShowMsg->type == TSDB_MGMT_TABLE_VNODES ) && pCmd->payloadLen != 0) {
     // only show tables support wildcard query
     pShowMsg->payloadLen = htons(pCmd->payloadLen);
     memcpy(pShowMsg->payload, payload, pCmd->payloadLen);
@@ -3453,31 +3453,6 @@ int tscProcessQueryRsp(SSqlObj *pSql) {
   return 0;
 }
 
-static void doDecompressPayload(SSqlCmd *pCmd, SSqlRes *pRes, int16_t compressed) {
-  if (compressed && pRes->numOfRows > 0) {
-    SRetrieveMeterRsp *pRetrieve = (SRetrieveMeterRsp *)pRes->pRsp;
-
-    int32_t numOfTotalCols = pCmd->fieldsInfo.numOfOutputCols + pCmd->fieldsInfo.numOfHiddenCols;
-    int32_t rowSize = pCmd->fieldsInfo.pOffset[numOfTotalCols - 1] + pCmd->fieldsInfo.pFields[numOfTotalCols - 1].bytes;
-
-    // TODO handle the OOM problem
-    char *  buf = malloc(rowSize * pRes->numOfRows);
-
-    int32_t payloadSize = pRes->rspLen - 1 - sizeof(SRetrieveMeterRsp);
-    assert(payloadSize > 0);
-
-    int32_t decompressedSize = tsDecompressString(pRetrieve->data, payloadSize, 1, buf, rowSize * pRes->numOfRows, 0, 0, 0);
-    assert(decompressedSize == rowSize * pRes->numOfRows);
-
-    pRes->pRsp = realloc(pRes->pRsp, pRes->rspLen - payloadSize + decompressedSize);
-    memcpy(pRes->pRsp + sizeof(SRetrieveMeterRsp), buf, decompressedSize);
-
-    free(buf);
-  }
-
-  pRes->data = ((SRetrieveMeterRsp *)pRes->pRsp)->data;
-}
-
 int tscProcessRetrieveRspFromVnode(SSqlObj *pSql) {
   SSqlRes *pRes = &pSql->res;
   SSqlCmd *pCmd = &pSql->cmd;
@@ -3490,9 +3465,7 @@ int tscProcessRetrieveRspFromVnode(SSqlObj *pSql) {
   pRes->offset = htobe64(pRetrieve->offset);
 
   pRes->useconds = htobe64(pRetrieve->useconds);
-  pRetrieve->compress = htons(pRetrieve->compress);
-
-  doDecompressPayload(pCmd, pRes, pRetrieve->compress);
+  pRes->data = pRetrieve->data;
 
   tscSetResultPointer(pCmd, pRes);
   pRes->row = 0;
@@ -3652,7 +3625,7 @@ int tscRenewMeterMeta(SSqlObj *pSql, char *meterId) {
    */
   if (pMeterMetaInfo->pMeterMeta == NULL || !tscQueryOnMetric(pCmd)) {
     if (pMeterMetaInfo->pMeterMeta) {
-      tscTrace("%p update meter meta, old: numOfTags:%d, numOfCols:%d, uid:%d, addr:%p", pSql,
+      tscTrace("%p update meter meta, old: numOfTags:%d, numOfCols:%d, uid:%lld, addr:%p", pSql,
                pMeterMetaInfo->numOfTags, pCmd->numOfCols, pMeterMetaInfo->pMeterMeta->uid, pMeterMetaInfo->pMeterMeta);
     }
     tscWaitingForCreateTable(&pSql->cmd);

src/client/src/tscSql.c

@@ -246,7 +246,12 @@ int taos_query_imp(STscObj* pObj, SSqlObj* pSql) {
     tscDoQuery(pSql);
   }
 
+  if (pRes->code == TSDB_CODE_SUCCESS) {
     tscTrace("%p SQL result:%d, %s pObj:%p", pSql, pRes->code, taos_errstr(pObj), pObj);
+  } else {
+    tscError("%p SQL result:%d, %s pObj:%p", pSql, pRes->code, taos_errstr(pObj), pObj);
+  }
+  
   if (pRes->code != TSDB_CODE_SUCCESS) {
     tscFreeSqlObjPartial(pSql);
   }
@@ -266,8 +271,9 @@ int taos_query(TAOS *taos, const char *sqlstr) {
 
   size_t sqlLen = strlen(sqlstr);
   if (sqlLen > TSDB_MAX_SQL_LEN) {
-    tscError("%p sql too long", pSql);
-    pRes->code = TSDB_CODE_INVALID_SQL;
+    pRes->code = tscInvalidSQLErrMsg(pSql->cmd.payload, "sql too long", NULL);  // set the additional error msg for invalid sql
+    tscError("%p SQL result:%d, %s pObj:%p", pSql, pRes->code, taos_errstr(taos), pObj);
+    
     return pRes->code;
   }
 
@@ -276,8 +282,9 @@ int taos_query(TAOS *taos, const char *sqlstr) {
   void *sql = realloc(pSql->sqlstr, sqlLen + 1);
   if (sql == NULL) {
     pRes->code = TSDB_CODE_CLI_OUT_OF_MEMORY;
-    tscError("%p failed to malloc sql string buffer", pSql);
-    tscTrace("%p SQL result:%d, %s pObj:%p", pSql, pRes->code, taos_errstr(taos), pObj);
+    tscError("%p failed to malloc sql string buffer, reason:%s", pSql, strerror(errno));
+  
+    tscError("%p SQL result:%d, %s pObj:%p", pSql, pRes->code, taos_errstr(taos), pObj);
     return pRes->code;
   }
 
@@ -777,9 +784,9 @@ int taos_errno(TAOS *taos) {
 }
 
 char *taos_errstr(TAOS *taos) {
-  STscObj *     pObj = (STscObj *)taos;
-  unsigned char code;
-  char          temp[256] = {0};
+  STscObj *pObj = (STscObj *)taos;
+  uint8_t  code;
+//  char          temp[256] = {0};
 
   if (pObj == NULL || pObj->signature != pObj) return tsError[globalCode];
 
@@ -788,9 +795,10 @@ char *taos_errstr(TAOS *taos) {
   else
     code = pObj->pSql->res.code;
 
+  // for invalid sql, additional information is attached to explain why the sql is invalid
   if (code == TSDB_CODE_INVALID_SQL) {
-    snprintf(temp, tListLen(temp), "invalid SQL: %s", pObj->pSql->cmd.payload);
-    strcpy(pObj->pSql->cmd.payload, temp);
+//    snprintf(temp, tListLen(temp), "invalid SQL: %s", pObj->pSql->cmd.payload);
+//    strcpy(pObj->pSql->cmd.payload, temp);
     return pObj->pSql->cmd.payload;
   } else {
     return tsError[code];

src/client/src/tscUtil.c

@@ -1294,8 +1294,7 @@ int32_t tscValidateName(SSQLToken* pToken) {
 
     // re-build the whole name string
     if (pStr[firstPartLen] == TS_PATH_DELIMITER[0]) {
-      // first part do not have quote
-      // do nothing
+      // first part do not have quote do nothing
     } else {
       pStr[firstPartLen] = TS_PATH_DELIMITER[0];
       memmove(&pStr[firstPartLen + 1], pToken->z, pToken->n);
@@ -1842,5 +1841,30 @@ bool tscIsUpdateQuery(STscObj* pObj) {
   SSqlCmd* pCmd = &pObj->pSql->cmd;
   return ((pCmd->command >= TSDB_SQL_INSERT && pCmd->command <= TSDB_SQL_DROP_DNODE) ||
       TSDB_SQL_USE_DB == pCmd->command) ? 1 : 0;
+}
+
+int32_t tscInvalidSQLErrMsg(char *msg, const char *additionalInfo, const char *sql) {
+  const char *msgFormat1 = "invalid SQL: %s";
+  const char *msgFormat2 = "invalid SQL: syntax error near \"%s\" (%s)";
+  const char *msgFormat3 = "invalid SQL: syntax error near \"%s\"";
+  
+  const int32_t BACKWARD_CHAR_STEP = 0;
+  
+  if (sql == NULL) {
+    assert(additionalInfo != NULL);
+    sprintf(msg, msgFormat1, additionalInfo);
+    return TSDB_CODE_INVALID_SQL;
+  }
   
+  char buf[64] = {0};   // only extract part of sql string
+  strncpy(buf, (sql - BACKWARD_CHAR_STEP), tListLen(buf) - 1);
+  
+  if (additionalInfo != NULL) {
+    sprintf(msg, msgFormat2, buf, additionalInfo);
+  } else {
+    sprintf(msg, msgFormat3, buf); // no additional information for invalid sql error
+  }
+  
+  return TSDB_CODE_INVALID_SQL;
 }
+

src/connector/go/src/taosSql/utils.go

@@ -12,15 +12,25 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
-
 package taosSql
 
+/*
+#cgo CFLAGS : -I/usr/include
+#include <stdlib.h>
+#cgo LDFLAGS: -L/usr/lib -ltaos
+void taosSetAllocMode(int mode, const char* path, _Bool autoDump);
+void taosDumpMemoryLeak();
+*/
+import "C"
+
+
 import (
 	"database/sql/driver"
 	"errors"
 	"fmt"
 	"sync/atomic"
 	"time"
+	"unsafe"
 )
 
 // Returns the bool value of the input.
@@ -398,3 +408,15 @@ func namedValueToValue(named []driver.NamedValue) ([]driver.Value, error) {
 }
 
 
+/******************************************************************************
+*                     Utils for C memory issues debugging                     *
+******************************************************************************/
+func SetAllocMode(mode int32, path string) {
+	cpath := C.CString(path)
+	defer C.free(unsafe.Pointer(cpath))
+	C.taosSetAllocMode(C.int(mode), cpath, false)
+}
+
+func DumpMemoryLeak() {
+	C.taosDumpMemoryLeak()
+}

src/inc/sql.y

@@ -73,6 +73,9 @@ cmd ::= SHOW CONFIGS.    { setDCLSQLElems(pInfo, SHOW_CONFIGS, 0);  }
 cmd ::= SHOW SCORES.     { setDCLSQLElems(pInfo, SHOW_SCORES, 0);   }
 cmd ::= SHOW GRANTS.     { setDCLSQLElems(pInfo, SHOW_GRANTS, 0);   }
 
+cmd ::= SHOW VNODES.                { setDCLSQLElems(pInfo, SHOW_VNODES, 0); }
+cmd ::= SHOW VNODES IPTOKEN(X).     { setDCLSQLElems(pInfo, SHOW_VNODES, 1, &X); }
+
 %type dbPrefix {SSQLToken}
 dbPrefix(A) ::=.                   {A.n = 0;}
 dbPrefix(A) ::= ids(X) DOT.        {A = X;  }

src/inc/taosmsg.h

@@ -158,6 +158,7 @@ enum _mgmt_table {
   TSDB_MGMT_TABLE_CONNS,
   TSDB_MGMT_TABLE_SCORES,
   TSDB_MGMT_TABLE_GRANTS,
+  TSDB_MGMT_TABLE_VNODES,
   TSDB_MGMT_TABLE_MAX,
 };
 
@@ -224,7 +225,7 @@ typedef struct {
   char     meterId[TSDB_UNI_LEN];
   uint16_t port;  // for UDP only
   char     empty[1];
-  char     msgType;
+  uint8_t  msgType;
   int32_t  msgLen;
   uint8_t  content[0];
 } STaosHeader;
@@ -567,7 +568,6 @@ typedef struct {
 typedef struct {
   int32_t numOfRows;
   int16_t precision;
-  int16_t compress;
   int64_t offset;  // updated offset value for multi-vnode projection query
   int64_t useconds;
   char    data[];

src/inc/tglobalcfg.h

@@ -256,6 +256,8 @@ SGlobalConfig *tsGetConfigOption(const char *option);
 #define TSDB_CFG_OPTION_LEN 24
 #define TSDB_CFG_VALUE_LEN  41
 
+#define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize)
+
 #ifdef __cplusplus
 }
 #endif

src/inc/tnote.h

@@ -20,7 +20,6 @@
 extern "C" {
 #endif
 
-#include "unistd.h"
 #include "os.h"
 #include "tutil.h"
 #include "tglobalcfg.h"

src/inc/tsqldef.h

@@ -72,145 +72,144 @@
 #define TK_CONFIGS                         54
 #define TK_SCORES                          55
 #define TK_GRANTS                          56
-#define TK_DOT                             57
-#define TK_TABLES                          58
-#define TK_STABLES                         59
-#define TK_VGROUPS                         60
-#define TK_DROP                            61
-#define TK_TABLE                           62
-#define TK_DATABASE                        63
-#define TK_DNODE                           64
-#define TK_IPTOKEN                         65
-#define TK_USER                            66
-#define TK_ACCOUNT                         67
-#define TK_USE                             68
-#define TK_DESCRIBE                        69
-#define TK_ALTER                           70
-#define TK_PASS                            71
-#define TK_PRIVILEGE                       72
-#define TK_LOCAL                           73
-#define TK_IF                              74
-#define TK_EXISTS                          75
-#define TK_CREATE                          76
-#define TK_PPS                             77
-#define TK_TSERIES                         78
-#define TK_DBS                             79
-#define TK_STORAGE                         80
-#define TK_QTIME                           81
-#define TK_CONNS                           82
-#define TK_STATE                           83
-#define TK_KEEP                            84
-#define TK_CACHE                           85
-#define TK_REPLICA                         86
-#define TK_DAYS                            87
-#define TK_ROWS                            88
-#define TK_ABLOCKS                         89
-#define TK_TBLOCKS                         90
-#define TK_CTIME                           91
-#define TK_CLOG                            92
-#define TK_COMP                            93
-#define TK_PRECISION                       94
-#define TK_LP                              95
-#define TK_RP                              96
-#define TK_TAGS                            97
-#define TK_USING                           98
-#define TK_AS                              99
-#define TK_COMMA                          100
-#define TK_NULL                           101
-#define TK_SELECT                         102
-#define TK_FROM                           103
-#define TK_VARIABLE                       104
-#define TK_INTERVAL                       105
-#define TK_FILL                           106
-#define TK_SLIDING                        107
-#define TK_ORDER                          108
-#define TK_BY                             109
-#define TK_ASC                            110
-#define TK_DESC                           111
-#define TK_GROUP                          112
-#define TK_HAVING                         113
-#define TK_LIMIT                          114
-#define TK_OFFSET                         115
-#define TK_SLIMIT                         116
-#define TK_SOFFSET                        117
-#define TK_WHERE                          118
-#define TK_NOW                            119
-#define TK_INSERT                         120
-#define TK_INTO                           121
-#define TK_VALUES                         122
-#define TK_RESET                          123
-#define TK_QUERY                          124
-#define TK_ADD                            125
-#define TK_COLUMN                         126
-#define TK_TAG                            127
-#define TK_CHANGE                         128
-#define TK_SET                            129
-#define TK_KILL                           130
-#define TK_CONNECTION                     131
-#define TK_COLON                          132
-#define TK_STREAM                         133
-#define TK_ABORT                          134
-#define TK_AFTER                          135
-#define TK_ATTACH                         136
-#define TK_BEFORE                         137
-#define TK_BEGIN                          138
-#define TK_CASCADE                        139
-#define TK_CLUSTER                        140
-#define TK_CONFLICT                       141
-#define TK_COPY                           142
-#define TK_DEFERRED                       143
-#define TK_DELIMITERS                     144
-#define TK_DETACH                         145
-#define TK_EACH                           146
-#define TK_END                            147
-#define TK_EXPLAIN                        148
-#define TK_FAIL                           149
-#define TK_FOR                            150
-#define TK_IGNORE                         151
-#define TK_IMMEDIATE                      152
-#define TK_INITIALLY                      153
-#define TK_INSTEAD                        154
-#define TK_MATCH                          155
-#define TK_KEY                            156
-#define TK_OF                             157
-#define TK_RAISE                          158
-#define TK_REPLACE                        159
-#define TK_RESTRICT                       160
-#define TK_ROW                            161
-#define TK_STATEMENT                      162
-#define TK_TRIGGER                        163
-#define TK_VIEW                           164
-#define TK_ALL                            165
-#define TK_COUNT                          166
-#define TK_SUM                            167
-#define TK_AVG                            168
-#define TK_MIN                            169
-#define TK_MAX                            170
-#define TK_FIRST                          171
-#define TK_LAST                           172
-#define TK_TOP                            173
-#define TK_BOTTOM                         174
-#define TK_STDDEV                         175
-#define TK_PERCENTILE                     176
-#define TK_APERCENTILE                    177
-#define TK_LEASTSQUARES                   178
-#define TK_HISTOGRAM                      179
-#define TK_DIFF                           180
-#define TK_SPREAD                         181
-#define TK_TWA                            182
-#define TK_INTERP                         183
-#define TK_LAST_ROW                       184
-#define TK_SEMI                           185
-#define TK_NONE                           186
-#define TK_PREV                           187
-#define TK_LINEAR                         188
-#define TK_IMPORT                         189
-#define TK_METRIC                         190
-#define TK_TBNAME                         191
-#define TK_JOIN                           192
-#define TK_METRICS                        193
-#define TK_STABLE                         194
+#define TK_VNODES                          57
+#define TK_IPTOKEN                         58
+#define TK_DOT                             59
+#define TK_TABLES                          60
+#define TK_STABLES                         61
+#define TK_VGROUPS                         62
+#define TK_DROP                            63
+#define TK_TABLE                           64
+#define TK_DATABASE                        65
+#define TK_DNODE                           66
+#define TK_USER                            67
+#define TK_ACCOUNT                         68
+#define TK_USE                             69
+#define TK_DESCRIBE                        70
+#define TK_ALTER                           71
+#define TK_PASS                            72
+#define TK_PRIVILEGE                       73
+#define TK_LOCAL                           74
+#define TK_IF                              75
+#define TK_EXISTS                          76
+#define TK_CREATE                          77
+#define TK_PPS                             78
+#define TK_TSERIES                         79
+#define TK_DBS                             80
+#define TK_STORAGE                         81
+#define TK_QTIME                           82
+#define TK_CONNS                           83
+#define TK_STATE                           84
+#define TK_KEEP                            85
+#define TK_CACHE                           86
+#define TK_REPLICA                         87
+#define TK_DAYS                            88
+#define TK_ROWS                            89
+#define TK_ABLOCKS                         90
+#define TK_TBLOCKS                         91
+#define TK_CTIME                           92
+#define TK_CLOG                            93
+#define TK_COMP                            94
+#define TK_PRECISION                       95
+#define TK_LP                              96
+#define TK_RP                              97
+#define TK_TAGS                            98
+#define TK_USING                           99
+#define TK_AS                             100
+#define TK_COMMA                          101
+#define TK_NULL                           102
+#define TK_SELECT                         103
+#define TK_FROM                           104
+#define TK_VARIABLE                       105
+#define TK_INTERVAL                       106
+#define TK_FILL                           107
+#define TK_SLIDING                        108
+#define TK_ORDER                          109
+#define TK_BY                             110
+#define TK_ASC                            111
+#define TK_DESC                           112
+#define TK_GROUP                          113
+#define TK_HAVING                         114
+#define TK_LIMIT                          115
+#define TK_OFFSET                         116
+#define TK_SLIMIT                         117
+#define TK_SOFFSET                        118
+#define TK_WHERE                          119
+#define TK_NOW                            120
+#define TK_INSERT                         121
+#define TK_INTO                           122
+#define TK_VALUES                         123
+#define TK_RESET                          124
+#define TK_QUERY                          125
+#define TK_ADD                            126
+#define TK_COLUMN                         127
+#define TK_TAG                            128
+#define TK_CHANGE                         129
+#define TK_SET                            130
+#define TK_KILL                           131
+#define TK_CONNECTION                     132
+#define TK_COLON                          133
+#define TK_STREAM                         134
+#define TK_ABORT                          135
+#define TK_AFTER                          136
+#define TK_ATTACH                         137
+#define TK_BEFORE                         138
+#define TK_BEGIN                          139
+#define TK_CASCADE                        140
+#define TK_CLUSTER                        141
+#define TK_CONFLICT                       142
+#define TK_COPY                           143
+#define TK_DEFERRED                       144
+#define TK_DELIMITERS                     145
+#define TK_DETACH                         146
+#define TK_EACH                           147
+#define TK_END                            148
+#define TK_EXPLAIN                        149
+#define TK_FAIL                           150
+#define TK_FOR                            151
+#define TK_IGNORE                         152
+#define TK_IMMEDIATE                      153
+#define TK_INITIALLY                      154
+#define TK_INSTEAD                        155
+#define TK_MATCH                          156
+#define TK_KEY                            157
+#define TK_OF                             158
+#define TK_RAISE                          159
+#define TK_REPLACE                        160
+#define TK_RESTRICT                       161
+#define TK_ROW                            162
+#define TK_STATEMENT                      163
+#define TK_TRIGGER                        164
+#define TK_VIEW                           165
+#define TK_ALL                            166
+#define TK_COUNT                          167
+#define TK_SUM                            168
+#define TK_AVG                            169
+#define TK_MIN                            170
+#define TK_MAX                            171
+#define TK_FIRST                          172
+#define TK_LAST                           173
+#define TK_TOP                            174
+#define TK_BOTTOM                         175
+#define TK_STDDEV                         176
+#define TK_PERCENTILE                     177
+#define TK_APERCENTILE                    178
+#define TK_LEASTSQUARES                   179
+#define TK_HISTOGRAM                      180
+#define TK_DIFF                           181
+#define TK_SPREAD                         182
+#define TK_TWA                            183
+#define TK_INTERP                         184
+#define TK_LAST_ROW                       185
+#define TK_SEMI                           186
+#define TK_NONE                           187
+#define TK_PREV                           188
+#define TK_LINEAR                         189
+#define TK_IMPORT                         190
+#define TK_METRIC                         191
+#define TK_TBNAME                         192
+#define TK_JOIN                           193
+#define TK_METRICS                        194
+#define TK_STABLE                         195
 
 #endif
-
-

src/inc/tutil.h

@@ -187,18 +187,35 @@ static FORCE_INLINE void taosEncryptPass(uint8_t *inBuf, unsigned int inLen, cha
 
 char *taosIpStr(uint32_t ipInt);
 
-#ifdef _TAOS_MEM_TEST_
-// Use during test to simulate the success and failure scenarios of memory allocation
-extern void* taos_malloc(unsigned int size, char* _func);
-extern void* taos_calloc(unsigned int num, unsigned int size, char* _func);
-extern void* taos_realloc(void* ptr, unsigned int size, char* _func);
-extern void  taos_free(void* ptr);
-#define malloc(size)        taos_malloc(size, __FUNCTION__)
-#define calloc(num, size)   taos_calloc(num, size, __FUNCTION__)
-#define realloc(ptr, size)  taos_realloc(ptr, size, __FUNCTION__)
-#define free(ptr)           taos_free(ptr)
-#endif
-
+#define TAOS_ALLOC_MODE_DEFAULT 0
+#define TAOS_ALLOC_MODE_RANDOM_FAIL 1
+#define TAOS_ALLOC_MODE_DETECT_LEAK 2
+void taosSetAllocMode(int mode, const char* path, bool autoDump);
+void taosDumpMemoryLeak();
+
+#ifdef TAOS_MEM_CHECK
+
+void *  taos_malloc(size_t size, const char *file, uint32_t line);
+void *  taos_calloc(size_t num, size_t size, const char *file, uint32_t line);
+void *  taos_realloc(void *ptr, size_t size, const char *file, uint32_t line);
+void    taos_free(void *ptr, const char *file, uint32_t line);
+char *  taos_strdup(const char *str, const char *file, uint32_t line);
+char *  taos_strndup(const char *str, size_t size, const char *file, uint32_t line);
+ssize_t taos_getline(char **lineptr, size_t *n, FILE *stream, const char *file, uint32_t line);
+
+#ifndef TAOS_MEM_CHECK_IMPL
+
+#define malloc(size) taos_malloc(size, __FILE__, __LINE__)
+#define calloc(num, size) taos_calloc(num, size, __FILE__, __LINE__)
+#define realloc(ptr, size) taos_realloc(ptr, size, __FILE__, __LINE__)
+#define free(ptr) taos_free(ptr, __FILE__, __LINE__)
+#define strdup(str) taos_strdup(str, __FILE__, __LINE__)
+#define strndup(str, size) taos_strndup(str, size, __FILE__, __LINE__)
+#define getline(lineptr, n, stream) taos_getline(lineptr, n, stream, __FILE__, __LINE__)
+
+#endif  // TAOS_MEM_CHECK_IMPL
+
+#endif // TAOS_MEM_CHECK
 
 #ifdef __cplusplus
 }

src/kit/shell/src/shellEngine.c

@@ -21,6 +21,7 @@
 #include "shellCommand.h"
 #include "ttime.h"
 #include "tutil.h"
+#include <regex.h>
 
 /**************** Global variables ****************/
 #ifdef WINDOWS

src/os/linux/src/tsystem.c

@@ -81,6 +81,7 @@ bool taosGetProcMemory(float *memoryUsedMB) {
   char * line = NULL;
   while (!feof(fp)) {
     tfree(line);
+    len = 0;
     getline(&line, &len, fp);
     if (line == NULL) {
       break;
@@ -137,7 +138,7 @@ bool taosGetProcCpuInfo(ProcCpuInfo *cpuInfo) {
     return false;
   }
 
-  size_t len;
+  size_t len = 0;
   char * line = NULL;
   getline(&line, &len, fp);
   if (line == NULL) {
@@ -409,6 +410,7 @@ bool taosGetCardInfo(int64_t *bytes) {
 
   while (!feof(fp)) {
     tfree(line);
+    len = 0;
     getline(&line, &len, fp);
     if (line == NULL) {
       break;
@@ -480,6 +482,7 @@ bool taosReadProcIO(int64_t *readbyte, int64_t *writebyte) {
 
   while (!feof(fp)) {
     tfree(line);
+    len = 0;
     getline(&line, &len, fp);
     if (line == NULL) {
       break;

src/os/windows/inc/os.h

@@ -16,20 +16,30 @@
 #ifndef TDENGINE_PLATFORM_WINDOWS_H
 #define TDENGINE_PLATFORM_WINDOWS_H
 
+#include <assert.h>
+#include <ctype.h>
+#include <direct.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <float.h>
+#include <locale.h>
+#include <intrin.h>
 #include <io.h>
+#include <math.h>
+#include <pthread.h>
+#include <semaphore.h>
+#include <stdarg.h>
+#include <stdbool.h>
+#include <stdint.h>
 #include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
 #include <signal.h>
-#include <stdint.h>
-#include <stdbool.h>
-#include <pthread.h>
-#include <direct.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <time.h>
 #include "winsock2.h"
 #include <WS2tcpip.h>
-#include <assert.h>
-#include <math.h>
-#include <string.h>
-#include <assert.h>
-#include <intrin.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -366,6 +376,8 @@ int fsendfile(FILE* out_file, FILE* in_file, int64_t* offset, int32_t count);
 
 char *strndup(const char *s, size_t n);
 
+void taosSetCoreDump();
+
 #ifdef __cplusplus
 }
 #endif

src/os/windows/src/twindows.c

@@ -395,3 +395,5 @@ char *strndup(const char *s, size_t n) {
   r[len] = 0;
   return r;
 }
+
+void taosSetCoreDump() {}
\ No newline at end of file

src/system/detail/inc/mgmt.h

@@ -410,6 +410,9 @@ int mgmtRetrieveScores(SShowObj *pShow, char *data, int rows, SConnObj *pConn);
 int grantGetGrantsMeta(SMeterMeta *pMeta, SShowObj *pShow, SConnObj *pConn);
 int grantRetrieveGrants(SShowObj *pShow, char *data, int rows, SConnObj *pConn);
 
+int  mgmtGetVnodeMeta(SMeterMeta *pMeta, SShowObj *pShow, SConnObj *pConn);
+int  mgmtRetrieveVnodes(SShowObj *pShow, char *data, int rows, SConnObj *pConn);
+
 // dnode balance api
 int  mgmtInitBalance();
 void mgmtCleanupBalance();

src/system/detail/inc/vnode.h

@@ -353,7 +353,7 @@ bool vnodeIsValidVnodeCfg(SVnodeCfg *pCfg);
 
 int32_t vnodeGetResultSize(void *handle, int32_t *numOfRows);
 
-int32_t vnodeCopyQueryResultToMsg(void *handle, char *data, int32_t numOfRows, int32_t *size);
+int32_t vnodeCopyQueryResultToMsg(void *handle, char *data, int32_t numOfRows);
 
 int64_t vnodeGetOffsetVal(void *thandle);
 

src/system/detail/src/dnodeService.c

@@ -61,6 +61,20 @@ int main(int argc, char *argv[]) {
       return 0;
     } else if (strcmp(argv[i], "-k") == 0) {
       dnodeParseParameterK();
+#ifdef TAOS_MEM_CHECK
+    } else if (strcmp(argv[i], "--alloc-random-fail") == 0) {
+      if ((i < argc - 1) && (argv[i+1][0] != '-')) {
+        taosSetAllocMode(TAOS_ALLOC_MODE_RANDOM_FAIL, argv[++i], true);
+      } else {
+        taosSetAllocMode(TAOS_ALLOC_MODE_RANDOM_FAIL, NULL, true);
+      }
+    } else if (strcmp(argv[i], "--detect-mem-leak") == 0) {
+      if ((i < argc - 1) && (argv[i+1][0] != '-')) {
+        taosSetAllocMode(TAOS_ALLOC_MODE_DETECT_LEAK, argv[++i], true);
+      } else {
+        taosSetAllocMode(TAOS_ALLOC_MODE_DETECT_LEAK, NULL, true);
+      }
+#endif
     }
   }
 

src/system/detail/src/mgmtMeter.c

@@ -519,10 +519,8 @@ int mgmtCreateMeter(SDbObj *pDb, SCreateTableMsg *pCreate) {
   pMeter = mgmtGetMeter(pCreate->meterId);
   if (pMeter) {
     if (pCreate->igExists) {
-      mError("table:%s, igExists is true", pCreate->meterId);
       return TSDB_CODE_SUCCESS;
     } else {
-      mError("table:%s, table is already exist", pCreate->meterId);
       return TSDB_CODE_TABLE_ALREADY_EXIST;
     }
   }
@@ -675,7 +673,7 @@ int mgmtCreateMeter(SDbObj *pDb, SCreateTableMsg *pCreate) {
 
   // send create message to the selected vnode servers
   if (pCreate->numOfTags == 0) {
-    mTrace("table:%s, send create msg to dnode, vgId:%d, sid:%d, vnode:%d",
+    mTrace("table:%s, send create table msg to dnode, vgId:%d, sid:%d, vnode:%d",
            pMeter->meterId, pMeter->gid.vgId, pMeter->gid.sid, pVgroup->vnodeGid[0].vnode);
 
     grantAddTimeSeries(pMeter->numOfColumns - 1);

src/system/detail/src/mgmtShell.c

@@ -788,12 +788,14 @@ int (*mgmtGetMetaFp[])(SMeterMeta *pMeta, SShowObj *pShow, SConnObj *pConn) = {
     mgmtGetAcctMeta,   mgmtGetUserMeta,   mgmtGetDbMeta,     mgmtGetMeterMeta,  mgmtGetDnodeMeta,
     mgmtGetMnodeMeta,  mgmtGetVgroupMeta, mgmtGetMetricMeta, mgmtGetModuleMeta, mgmtGetQueryMeta,
     mgmtGetStreamMeta, mgmtGetConfigMeta, mgmtGetConnsMeta,  mgmtGetScoresMeta, grantGetGrantsMeta,
+    mgmtGetVnodeMeta,
 };
 
 int (*mgmtRetrieveFp[])(SShowObj *pShow, char *data, int rows, SConnObj *pConn) = {
     mgmtRetrieveAccts,   mgmtRetrieveUsers,   mgmtRetrieveDbs,     mgmtRetrieveMeters,  mgmtRetrieveDnodes,
     mgmtRetrieveMnodes,  mgmtRetrieveVgroups, mgmtRetrieveMetrics, mgmtRetrieveModules, mgmtRetrieveQueries,
     mgmtRetrieveStreams, mgmtRetrieveConfigs, mgmtRetrieveConns,   mgmtRetrieveScores,  grantRetrieveGrants,
+    mgmtRetrieveVnodes,
 };
 
 int mgmtProcessShowMsg(char *pMsg, int msgLen, SConnObj *pConn) {

src/system/detail/src/vnodeFile.c

@@ -114,6 +114,7 @@ int vnodeCreateHeadDataFile(int vnode, int fileId, char *headName, char *dataNam
 
   char *path = vnodeGetDataDir(vnode, fileId);
   if (path == NULL) {
+    dError("vid:%d, fileId:%d, failed to get dataDir", vnode, fileId);
     return -1;
   }
   

src/util/src/version.c

 char version[64] = "1.6.4.0";
 char compatible_version[64] = "1.6.1.0";
-char gitinfo[128] = "d04354a8ac2f7dd9ba521d755e5d484a203783d9";
-char buildinfo[512] = "Built by root at 2019-11-11 10:23";
+char gitinfo[128] = "b6e308866e315483915f4c42a2717547ed0b9d36";
+char buildinfo[512] = "Built by ubuntu at 2019-11-26 21:56";