Merge branch 'develop' into feature/query

cmake/version.inc

@@ -4,7 +4,7 @@ PROJECT(TDengine)
 IF (DEFINED VERNUMBER)
   SET(TD_VER_NUMBER ${VERNUMBER})
 ELSE ()
-  SET(TD_VER_NUMBER "2.0.3.0")
+  SET(TD_VER_NUMBER "2.0.4.0")
 ENDIF ()
 
 IF (DEFINED VERCOMPATIBLE)

src/client/inc/tscUtil.h

@@ -191,6 +191,7 @@ SColumn* tscColumnListInsert(SArray* pColList, SColumnIndex* colIndex);
 SArray* tscColumnListClone(const SArray* src, int16_t tableIndex);
 void tscColumnListDestroy(SArray* pColList);
 
+void tscDequoteAndTrimToken(SStrToken* pToken);
 int32_t tscValidateName(SStrToken* pToken);
 
 void tscIncStreamExecutionCount(void* pStream);

src/client/inc/tsclient.h

@@ -99,6 +99,7 @@ typedef struct STableMeta {
   uint8_t        tableType;
   int16_t        sversion;
   int16_t        tversion;
+  char           sTableId[TSDB_TABLE_FNAME_LEN];
   SCMVgroupInfo  vgroupInfo;
   SCMCorVgroupInfo  corVgroupInfo;
   STableId       id;

src/client/src/tscParseInsert.c

@@ -1192,8 +1192,7 @@ int tsParseInsertSql(SSqlObj *pSql) {
         str += index;
 
         if (TK_STRING == sToken.type) {
-          strdequote(sToken.z);
-          sToken.n = (uint32_t)strtrim(sToken.z);
+          tscDequoteAndTrimToken(&sToken);
         }
 
         if (sToken.type == TK_RP) {

src/client/src/tscSQLParser.c

@@ -368,7 +368,40 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) {
 
       return tscGetTableMeta(pSql, pTableMetaInfo);
     }
+    case TSDB_SQL_SHOW_CREATE_TABLE: {
+      SStrToken*  pToken = &pInfo->pDCLInfo->a[0];
+      const char* msg1 = "invalid table name";
+      const char* msg2 = "table name is too long";
+
+      if (tscValidateName(pToken) != TSDB_CODE_SUCCESS) {
+        return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1);
+      }
+
+      if (!tscValidateTableNameLength(pToken->n)) {
+        return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2);
+      }
 
+      if (tscSetTableFullName(pTableMetaInfo, pToken, pSql) != TSDB_CODE_SUCCESS) {
+        return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2);
+      }
+      return tscGetTableMeta(pSql, pTableMetaInfo);
+    }
+    case TSDB_SQL_SHOW_CREATE_DATABASE: {
+      const char* msg1 = "invalid database name";
+      const char* msg2 = "table name is too long";
+      SStrToken*  pToken = &pInfo->pDCLInfo->a[0];
+
+      if (tscValidateName(pToken) != TSDB_CODE_SUCCESS) {
+        return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1);
+      }
+      if (pToken->n > TSDB_DB_NAME_LEN) {
+        return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1);
+      }
+      if (tscSetTableFullName(pTableMetaInfo, pToken, pSql) != TSDB_CODE_SUCCESS) {
+        return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2);
+      }
+      return TSDB_CODE_SUCCESS; 
+    }                                
     case TSDB_SQL_CFG_DNODE: {
       const char* msg2 = "invalid configure options or values, such as resetlog / debugFlag 135 / balance 'vnode:2-dnode:2' / monitor 1 ";
       const char* msg3 = "invalid dnode ep";
@@ -5722,7 +5755,7 @@ int32_t tscCheckCreateDbParams(SSqlCmd* pCmd, SCMCreateDbMsg* pCreate) {
   char msg[512] = {0};
 
   if (pCreate->walLevel != -1 && (pCreate->walLevel < TSDB_MIN_WAL_LEVEL || pCreate->walLevel > TSDB_MAX_WAL_LEVEL)) {
-    snprintf(msg, tListLen(msg), "invalid db option walLevel: %d, only 0-2 allowed", pCreate->walLevel);
+    snprintf(msg, tListLen(msg), "invalid db option walLevel: %d, only 1-2 allowed", pCreate->walLevel);
     return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg);
   }
 

src/client/src/tscSchemaUtil.c

@@ -170,6 +170,7 @@ STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size
 
   pTableMeta->sversion = pTableMetaMsg->sversion;
   pTableMeta->tversion = pTableMetaMsg->tversion;
+  tstrncpy(pTableMeta->sTableId, pTableMetaMsg->sTableId, TSDB_TABLE_FNAME_LEN);
   
   memcpy(pTableMeta->schema, pTableMetaMsg->schema, schemaSize);
   

src/client/src/tscServer.c

@@ -2093,6 +2093,9 @@ int tscProcessAlterDbMsgRsp(SSqlObj *pSql) {
   UNUSED(pSql);
   return 0;
 }
+int tscProcessShowCreateRsp(SSqlObj *pSql) {
+  return tscLocalResultCommonBuilder(pSql, 1);
+}
 
 int tscProcessQueryRsp(SSqlObj *pSql) {
   SSqlRes *pRes = &pSql->res;
@@ -2379,6 +2382,10 @@ void tscInitMsgsFp() {
   tscProcessMsgRsp[TSDB_SQL_ALTER_TABLE] = tscProcessAlterTableMsgRsp;
   tscProcessMsgRsp[TSDB_SQL_ALTER_DB] = tscProcessAlterDbMsgRsp;
 
+  tscProcessMsgRsp[TSDB_SQL_SHOW_CREATE_TABLE] = tscProcessShowCreateRsp;
+  tscProcessMsgRsp[TSDB_SQL_SHOW_CREATE_DATABASE] = tscProcessShowCreateRsp;
+  
+
   tscKeepConn[TSDB_SQL_SHOW] = 1;
   tscKeepConn[TSDB_SQL_RETRIEVE] = 1;
   tscKeepConn[TSDB_SQL_SELECT] = 1;

src/client/src/tscSql.c

@@ -477,6 +477,8 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) {
        pCmd->command == TSDB_SQL_TABLE_JOIN_RETRIEVE ||
        pCmd->command == TSDB_SQL_FETCH ||
        pCmd->command == TSDB_SQL_SHOW ||
+       pCmd->command == TSDB_SQL_SHOW_CREATE_TABLE ||
+       pCmd->command == TSDB_SQL_SHOW_CREATE_DATABASE ||
        pCmd->command == TSDB_SQL_SELECT ||
        pCmd->command == TSDB_SQL_DESCRIBE_TABLE ||
        pCmd->command == TSDB_SQL_SERV_STATUS ||

src/client/src/tscStream.c

@@ -136,7 +136,6 @@ static void tscProcessStreamTimer(void *handle, void *tmrId) {
       etime = pStream->stime + (etime - pStream->stime) / pStream->interval.interval * pStream->interval.interval;
     } else {
       etime = taosTimeTruncate(etime, &pStream->interval, pStream->precision);
-      //etime = taosGetIntervalStartTimestamp(etime, pStream->interval.sliding, pStream->interval.sliding, pStream->interval.slidingUnit, pStream->precision);
     }
     pQueryInfo->window.ekey = etime;
     if (pQueryInfo->window.skey >= pQueryInfo->window.ekey) {
@@ -454,17 +453,11 @@ static int64_t tscGetStreamStartTimestamp(SSqlObj *pSql, SSqlStream *pStream, in
     }
   } else {             // timewindow based aggregation stream
     if (stime == 0) {  // no data in meter till now
+      if (pQueryInfo->window.skey != INT64_MIN) {
         stime = pQueryInfo->window.skey;
-      if (stime == INT64_MIN) {
-        stime = (int64_t)taosGetTimestamp(pStream->precision);
-        stime = taosTimeTruncate(stime, &pStream->interval, pStream->precision);
-        stime = taosTimeTruncate(stime - 1, &pStream->interval, pStream->precision);
-        //stime = taosGetIntervalStartTimestamp(stime, pStream->interval.interval, pStream->interval.interval, pStream->interval.intervalUnit, pStream->precision);
-        //stime = taosGetIntervalStartTimestamp(stime - 1, pStream->interval.interval, pStream->interval.interval, pStream->interval.intervalUnit, pStream->precision);
-        tscWarn("%p stream:%p, last timestamp:0, reset to:%" PRId64, pSql, pStream, stime);
       }
+      stime = taosTimeTruncate(stime, &pStream->interval, pStream->precision);
     } else {
-      //int64_t newStime = taosGetIntervalStartTimestamp(stime, pStream->interval.interval, pStream->interval.interval, pStream->interval.intervalUnit, pStream->precision);
       int64_t newStime = taosTimeTruncate(stime, &pStream->interval, pStream->precision);
       if (newStime != stime) {
         tscWarn("%p stream:%p, last timestamp:%" PRId64 ", reset to:%" PRId64, pSql, pStream, stime, newStime);
@@ -477,8 +470,10 @@ static int64_t tscGetStreamStartTimestamp(SSqlObj *pSql, SSqlStream *pStream, in
 }
 
 static int64_t tscGetLaunchTimestamp(const SSqlStream *pStream) {
-  int64_t timer = pStream->stime - taosGetTimestamp(pStream->precision);
-  if (timer < 0) timer = 0;
+  int64_t timer = 0, now = taosGetTimestamp(pStream->precision);
+  if (pStream->stime > now) {
+    timer = pStream->stime - now;
+  }
 
   int64_t startDelay =
       (pStream->precision == TSDB_TIME_PRECISION_MICRO) ? tsStreamCompStartDelay * 1000L : tsStreamCompStartDelay;

src/client/src/tscSubquery.c

@@ -23,6 +23,7 @@
 #include "tscSubquery.h"
 #include "tschemautil.h"
 #include "tsclient.h"
+#include "tscSubquery.h"
 
 typedef struct SInsertSupporter {
   SSubqueryState* pState;

src/client/src/tscUtil.c

@@ -1268,6 +1268,51 @@ static int32_t validateQuoteToken(SStrToken* pToken) {
   return TSDB_CODE_SUCCESS;
 }
 
+void tscDequoteAndTrimToken(SStrToken* pToken) {
+  assert(pToken->type == TK_STRING);
+
+  uint32_t first = 0, last = pToken->n;
+
+  // trim leading spaces
+  while (first < last) {
+    char c = pToken->z[first];
+    if (c != ' ' && c != '\t') {
+      break;
+    }
+    first++;
+  }
+
+  // trim ending spaces
+  while (first < last) {
+    char c = pToken->z[last - 1];
+    if (c != ' ' && c != '\t') {
+      break;
+    }
+    last--;
+  }
+
+  // there are still at least two characters
+  if (first < last - 1) {
+    char c = pToken->z[first];
+    // dequote
+    if ((c == '\'' || c == '"') && c == pToken->z[last - 1]) {
+      first++;
+      last--;
+    }
+  }
+
+  // left shift the string and pad spaces
+  for (uint32_t i = 0; i + first < last; i++) {
+    pToken->z[i] = pToken->z[first + i];
+  }
+  for (uint32_t i = last - first; i < pToken->n; i++) {
+    pToken->z[i] = ' ';
+  }
+
+  // adjust token length
+  pToken->n = last - first;
+}
+
 int32_t tscValidateName(SStrToken* pToken) {
   if (pToken->type != TK_STRING && pToken->type != TK_ID) {
     return TSDB_CODE_TSC_INVALID_SQL;
@@ -1751,6 +1796,7 @@ SSqlObj* createSimpleSubObj(SSqlObj* pSql, void (*fp)(), void* param, int32_t cm
   SSqlCmd* pCmd = &pNew->cmd;
   pCmd->command = cmd;
   pCmd->parseFinished = 1;
+  pCmd->autoCreated = pSql->cmd.autoCreated;
 
   if (tscAddSubqueryInfo(pCmd) != TSDB_CODE_SUCCESS) {
     tscFreeSqlObj(pNew);
@@ -1869,7 +1915,6 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void
 
   pNew->pTscObj = pSql->pTscObj;
   pNew->signature = pNew;
-  T_REF_INC(pNew->pTscObj);
 
   pNew->sqlstr = strdup(pSql->sqlstr);
   if (pNew->sqlstr == NULL) {

src/common/inc/tcmdtype.h

@@ -78,6 +78,9 @@ enum {
   TSDB_DEFINE_SQL_TYPE( TSDB_SQL_RETRIEVE_LOCALMERGE, "retrieve-localmerge" )
   TSDB_DEFINE_SQL_TYPE( TSDB_SQL_TABLE_JOIN_RETRIEVE, "join-retrieve" )
 
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_SHOW_CREATE_TABLE, "show-create-table")
+  TSDB_DEFINE_SQL_TYPE( TSDB_SQL_SHOW_CREATE_DATABASE, "show-create-database")
+
   /*
    * build empty result instead of accessing dnode to fetch result
    * reset the client cache

src/common/src/tglobal.c

@@ -131,7 +131,7 @@ uint16_t tsHttpPort = 6041;  // only tcp, range tcp[6041]
 int32_t  tsHttpCacheSessions = 1000;
 int32_t  tsHttpSessionExpire = 36000;
 int32_t  tsHttpMaxThreads = 2;
-int32_t  tsHttpEnableCompress = 0;
+int32_t  tsHttpEnableCompress = 1;
 int32_t  tsHttpEnableRecordSql = 0;
 int32_t  tsTelegrafUseFieldNum = 0;
 

go @ 06ec30a0

-Subproject commit 8c58c512b6acda8bcdfa48fdc7140227b5221766
+Subproject commit 06ec30a0f1762e8169bf6b9045c82bcaa52bcdf0

src/connector/jdbc/deploy-pom.xml

@@ -5,10 +5,9 @@
 
   <groupId>com.taosdata.jdbc</groupId>
   <artifactId>taos-jdbcdriver</artifactId>
-  <version>2.0.0-SNAPSHOT</version>
+  <version>2.0.6</version>
   <packaging>jar</packaging>
 
-
   <name>JDBCDriver</name>
   <url>https://github.com/taosdata/TDengine/tree/master/src/connector/jdbc</url>
   <description>TDengine JDBC Driver</description>

src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java

@@ -291,8 +291,8 @@ public class TSDBDriver implements java.sql.Driver {
             return null;
         }
 
-        String lowerUrl = url.toLowerCase();
-        if (!lowerUrl.startsWith(URL_PREFIX) && !lowerUrl.startsWith(URL_PREFIX1)) {
+//        String lowerUrl = url.toLowerCase();
+        if (!url.startsWith(URL_PREFIX) && !url.startsWith(URL_PREFIX1)) {
             return null;
         }
 

src/inc/taosmsg.h

@@ -673,6 +673,7 @@ typedef struct {
 typedef struct STableMetaMsg {
   int32_t       contLen;
   char          tableId[TSDB_TABLE_FNAME_LEN];   // table id
+  char          sTableId[TSDB_TABLE_FNAME_LEN];
   uint8_t       numOfTags;
   uint8_t       precision;
   uint8_t       tableType;

src/inc/ttokendef.h

@@ -16,6 +16,7 @@
 #ifndef TDENGINE_TTOKENDEF_H
 #define TDENGINE_TTOKENDEF_H
 
+
 #define TK_ID                               1
 #define TK_BOOL                             2
 #define TK_TINYINT                          3
@@ -75,24 +76,24 @@
 #define TK_VNODES                          57
 #define TK_IPTOKEN                         58
 #define TK_DOT                             59
-#define TK_TABLES                          60
-#define TK_STABLES                         61
-#define TK_VGROUPS                         62
-#define TK_DROP                            63
-#define TK_TABLE                           64
-#define TK_DATABASE                        65
-#define TK_DNODE                           66
-#define TK_USER                            67
-#define TK_ACCOUNT                         68
-#define TK_USE                             69
-#define TK_DESCRIBE                        70
-#define TK_ALTER                           71
-#define TK_PASS                            72
-#define TK_PRIVILEGE                       73
-#define TK_LOCAL                           74
-#define TK_IF                              75
-#define TK_EXISTS                          76
-#define TK_CREATE                          77
+#define TK_CREATE                          60
+#define TK_TABLE                           61
+#define TK_DATABASE                        62
+#define TK_TABLES                          63
+#define TK_STABLES                         64
+#define TK_VGROUPS                         65
+#define TK_DROP                            66
+#define TK_DNODE                           67
+#define TK_USER                            68
+#define TK_ACCOUNT                         69
+#define TK_USE                             70
+#define TK_DESCRIBE                        71
+#define TK_ALTER                           72
+#define TK_PASS                            73
+#define TK_PRIVILEGE                       74
+#define TK_LOCAL                           75
+#define TK_IF                              76
+#define TK_EXISTS                          77
 #define TK_PPS                             78
 #define TK_TSERIES                         79
 #define TK_DBS                             80
@@ -222,7 +223,6 @@
 #define TK_INTO                           204
 #define TK_VALUES                         205
 
-
 #define TK_SPACE                          300
 #define TK_COMMENT                        301
 #define TK_ILLEGAL                        302

src/inc/twal.h

@@ -44,6 +44,7 @@ typedef void* twalh;  // WAL HANDLE
 typedef int (*FWalWrite)(void *ahandle, void *pHead, int type);
 
 twalh   walOpen(const char *path, const SWalCfg *pCfg);
+int     walAlter(twalh pWal, const SWalCfg *pCfg);
 void    walClose(twalh);
 int     walRenew(twalh);
 int     walWrite(twalh, SWalHead *);

src/mnode/src/mnodeDb.c

@@ -910,13 +910,13 @@ static SDbCfg mnodeGetAlterDbOption(SDbObj *pDb, SCMAlterDbMsg *pAlter) {
   }
 
   if (walLevel > 0 && walLevel != pDb->cfg.walLevel) {
-    mError("db:%s, can't alter walLevel option", pDb->name);
-    terrno = TSDB_CODE_MND_INVALID_DB_OPTION;
+    mDebug("db:%s, walLevel:%d change to %d", pDb->name, pDb->cfg.walLevel, walLevel);
+    newCfg.walLevel = walLevel;
   }
 
   if (fsyncPeriod >= 0 && fsyncPeriod != pDb->cfg.fsyncPeriod) {
-    mError("db:%s, can't alter fsyncPeriod option", pDb->name);
-    terrno = TSDB_CODE_MND_INVALID_DB_OPTION;
+    mDebug("db:%s, fsyncPeriod:%d change to %d", pDb->name, pDb->cfg.fsyncPeriod, fsyncPeriod);
+    newCfg.fsyncPeriod = fsyncPeriod;
   }
 
   if (replications > 0 && replications != pDb->cfg.replications) {

src/mnode/src/mnodeTable.c

@@ -1384,6 +1384,9 @@ int32_t mnodeRetrieveShowSuperTables(SShowObj *pShow, char *data, int32_t rows,
   }
 
   pShow->numOfReads += numOfRows;
+  const int32_t NUM_OF_COLUMNS = 5;
+
+  mnodeVacuumResult(data, NUM_OF_COLUMNS, numOfRows, rows, pShow);
   mnodeDecDbRef(pDb);
 
   return numOfRows;
@@ -2090,6 +2093,9 @@ static int32_t mnodeDoGetChildTableMeta(SMnodeMsg *pMsg, STableMetaMsg *pMeta) {
   pMeta->precision = pDb->cfg.precision;
   pMeta->tableType = pTable->info.type;
   tstrncpy(pMeta->tableId, pTable->info.tableId, TSDB_TABLE_FNAME_LEN);
+  if (pTable->superTable) {
+    tstrncpy(pMeta->sTableId, pTable->superTable->info.tableId, TSDB_TABLE_FNAME_LEN);
+  }
 
   if (pTable->info.type == TSDB_CHILD_TABLE) {
     pMeta->sversion     = htons(pTable->superTable->sversion);

src/mnode/src/mnodeVgroup.c

@@ -660,13 +660,13 @@ static int32_t mnodeGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *p
   for (int32_t i = 0; i < pShow->maxReplica; ++i) {
     pShow->bytes[cols] = 2;
     pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT;
-    strcpy(pSchema[cols].name, "dnode");
+    snprintf(pSchema[cols].name, TSDB_COL_NAME_LEN, "dnode%d", i + 1);
     pSchema[cols].bytes = htons(pShow->bytes[cols]);
     cols++;
 
     pShow->bytes[cols] = 9 + VARSTR_HEADER_SIZE;
     pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
-    strcpy(pSchema[cols].name, "vstatus");
+    snprintf(pSchema[cols].name, TSDB_COL_NAME_LEN, "v%dstatus", i + 1);
     pSchema[cols].bytes = htons(pShow->bytes[cols]);
     cols++;
   }

src/plugins/http/inc/httpJson.h

@@ -19,7 +19,7 @@
 #include <stdint.h>
 #include <stdbool.h>
 
-#define JSON_BUFFER_SIZE 10240
+#define JSON_BUFFER_SIZE 16384
 struct HttpContext;
 
 enum { JsonNumber, JsonString, JsonBoolean, JsonArray, JsonObject, JsonNull };

src/plugins/http/src/httpJson.c

@@ -52,12 +52,12 @@ int32_t httpWriteBufByFd(struct HttpContext* pContext, const char* buf, int32_t
     }
 
     if (len < 0) {
-      httpDebug("context:%p, fd:%d, socket write errno:%d, times:%d", pContext, pContext->fd, errno, countWait);
+      httpDebug("context:%p, fd:%d, socket write errno:%d:%s, times:%d", pContext, pContext->fd, errno, strerror(errno), countWait);
       if (++countWait > HTTP_WRITE_RETRY_TIMES) break;
       taosMsleep(HTTP_WRITE_WAIT_TIME_MS);
       continue;
     } else if (len == 0) {
-      httpDebug("context:%p, fd:%d, socket write errno:%d, connect already closed", pContext, pContext->fd, errno);
+      httpDebug("context:%p, fd:%d, socket write errno:%d:%s, connect already closed", pContext, pContext->fd, errno, strerror(errno));
       break;
     } else {
       countWait = 0;
@@ -131,14 +131,14 @@ int32_t httpWriteJsonBufBody(JsonBuf* buf, bool isTheLast) {
         httpWriteBufNoTrace(buf->pContext, sLen, len);
         remain = httpWriteBufNoTrace(buf->pContext, (const char*)compressBuf, compressBufLen);
       } else {
-        httpTrace("context:%p, fd:%d, last:%d, compress already dumped, response:\n%s", buf->pContext,
+        httpDebug("context:%p, fd:%d, last:%d, compress already dumped, response:\n%s", buf->pContext,
                   buf->pContext->fd, isTheLast, buf->buf);
-        return 0;  // there is no data to dump.
+        remain = 0;  // there is no data to dump.
       }
     } else {
       httpError("context:%p, fd:%d, failed to compress data, chunkSize:%" PRIu64 ", last:%d, error:%d, response:\n%s",
                 buf->pContext, buf->pContext->fd, srcLen, isTheLast, ret, buf->buf);
-      return 0;
+      remain = 0;
     }
   }
 

src/plugins/http/src/httpUtil.c

@@ -406,15 +406,21 @@ int32_t httpGzipCompressInit(HttpContext *pContext) {
 
 int32_t httpGzipCompress(HttpContext *pContext, char *srcData, int32_t nSrcData, char *destData, int32_t *nDestData, bool isTheLast) {
   int32_t err = 0;
+  int32_t lastTotalLen = (int32_t) (pContext->gzipStream.total_out);
   pContext->gzipStream.next_in = (Bytef *) srcData;
   pContext->gzipStream.avail_in = (uLong) nSrcData;
   pContext->gzipStream.next_out = (Bytef *) destData;
   pContext->gzipStream.avail_out = (uLong) (*nDestData);
 
-  while (pContext->gzipStream.avail_in != 0 && pContext->gzipStream.total_out < (uLong) (*nDestData)) {
+  while (pContext->gzipStream.avail_in != 0) {
     if (deflate(&pContext->gzipStream, Z_FULL_FLUSH) != Z_OK) {
       return -1;
     }
+
+    int32_t cacheLen = pContext->gzipStream.total_out - lastTotalLen;
+    if (cacheLen >= *nDestData) {
+      return -2;
+    }
   }
 
   if (pContext->gzipStream.avail_in != 0) {
@@ -427,16 +433,16 @@ int32_t httpGzipCompress(HttpContext *pContext, char *srcData, int32_t nSrcData,
         break;
       }
       if (err != Z_OK) {
-        return -2;
+        return -3;
       }
     }
 
     if (deflateEnd(&pContext->gzipStream) != Z_OK) {
-      return -3;
+      return -4;
     }
   }
 
-  *nDestData = (int32_t) (pContext->gzipStream.total_out);
+  *nDestData = (int32_t) (pContext->gzipStream.total_out) - lastTotalLen;
   return 0;
 }
 

src/query/inc/sql.y

@@ -80,6 +80,7 @@ cmd ::= SHOW GRANTS.     { setShowOptions(pInfo, TSDB_MGMT_TABLE_GRANTS, 0, 0);
 cmd ::= SHOW VNODES.                { setShowOptions(pInfo, TSDB_MGMT_TABLE_VNODES, 0, 0); }
 cmd ::= SHOW VNODES IPTOKEN(X).     { setShowOptions(pInfo, TSDB_MGMT_TABLE_VNODES, &X, 0); }
 
+
 %type dbPrefix {SStrToken}
 dbPrefix(A) ::=.                   {A.n = 0; A.type = 0;}
 dbPrefix(A) ::= ids(X) DOT.        {A = X;  }
@@ -88,6 +89,15 @@ dbPrefix(A) ::= ids(X) DOT.        {A = X;  }
 cpxName(A) ::= .             {A.n = 0;  }
 cpxName(A) ::= DOT ids(Y).   {A = Y; A.n += 1;    }
 
+cmd ::= SHOW CREATE TABLE ids(X) cpxName(Y).    {
+   X.n += Y.n;
+   setDCLSQLElems(pInfo, TSDB_SQL_SHOW_CREATE_TABLE, 1, &X);
+}    
+
+cmd ::= SHOW CREATE DATABASE ids(X). {
+  setDCLSQLElems(pInfo, TSDB_SQL_SHOW_CREATE_DATABASE, 1, &X);
+} 
+
 cmd ::= SHOW dbPrefix(X) TABLES.         {
     setShowOptions(pInfo, TSDB_MGMT_TABLE_TABLE, &X, 0);
 }

src/vnode/src/vnodeMain.c

@@ -186,6 +186,12 @@ int32_t vnodeAlter(void *param, SMDCreateVnodeMsg *pVnodeCfg) {
     return code; 
   }
 
+  code = walAlter(pVnode->wal, &pVnode->walCfg);
+  if (code != TSDB_CODE_SUCCESS) {
+    pVnode->status = TAOS_VN_STATUS_READY;
+    return code;
+  }
+
   code = syncReconfig(pVnode->sync, &pVnode->syncCfg);
   if (code != TSDB_CODE_SUCCESS) {
     pVnode->status = TAOS_VN_STATUS_READY;
@@ -390,6 +396,7 @@ void vnodeRelease(void *pVnodeRaw) {
     if (0 == tsEnableVnodeBak) {
       vInfo("vgId:%d, vnode backup not enabled", pVnode->vgId);
     } else {
+      taosRemoveDir(newDir);
       taosRename(rootDir, newDir);
     }
 

src/vnode/src/vnodeRead.c

@@ -58,7 +58,8 @@ int32_t vnodeProcessRead(void *param, SReadMsg *pReadMsg) {
     return TSDB_CODE_APP_NOT_READY;
 
   // TODO: Later, let slave to support query
-  if (pVnode->syncCfg.replica > 1 && pVnode->role != TAOS_SYNC_ROLE_MASTER) {
+  // if (pVnode->syncCfg.replica > 1 && pVnode->role != TAOS_SYNC_ROLE_MASTER) {
+  if (pVnode->role != TAOS_SYNC_ROLE_SLAVE && pVnode->role != TAOS_SYNC_ROLE_MASTER) { 
     vDebug("vgId:%d, msgType:%s not processed, replica:%d role:%d", pVnode->vgId, taosMsg[msgType], pVnode->syncCfg.replica, pVnode->role);
     return TSDB_CODE_APP_NOT_READY;
   }

src/wal/src/walMain.c

@@ -69,6 +69,13 @@ static void walModuleInitFunc() {
     wDebug("WAL module is initialized");
 }
 
+static inline bool walNeedFsyncTimer(SWal *pWal) {
+  if (pWal->fsyncPeriod > 0 && pWal->level == TAOS_WAL_FSYNC) {
+    return true;
+  }
+  return false;
+}
+
 void *walOpen(const char *path, const SWalCfg *pCfg) {
   SWal *pWal = calloc(sizeof(SWal), 1);
   if (pWal == NULL) {
@@ -95,7 +102,7 @@ void *walOpen(const char *path, const SWalCfg *pCfg) {
   tstrncpy(pWal->path, path, sizeof(pWal->path));
   pthread_mutex_init(&pWal->mutex, NULL);
 
-  if (pWal->fsyncPeriod > 0  && pWal->level == TAOS_WAL_FSYNC) {
+  if (walNeedFsyncTimer(pWal)) {
     pWal->timer = taosTmrStart(walProcessFsyncTimer, pWal->fsyncPeriod, pWal, walTmrCtrl);
     if (pWal->timer == NULL) {
       terrno = TAOS_SYSTEM_ERROR(errno);
@@ -127,6 +134,37 @@ void *walOpen(const char *path, const SWalCfg *pCfg) {
   return pWal;
 }
 
+int walAlter(twalh wal, const SWalCfg *pCfg) {
+  SWal *pWal = wal;
+  if (pWal == NULL) {
+    return TSDB_CODE_WAL_APP_ERROR;
+  }
+
+  if (pWal->level == pCfg->walLevel && pWal->fsyncPeriod == pCfg->fsyncPeriod) {
+    wDebug("wal:%s, old walLevel:%d fsync:%d, new walLevel:%d fsync:%d not change", pWal->name, pWal->level,
+           pWal->fsyncPeriod, pCfg->walLevel, pCfg->fsyncPeriod);
+    return TSDB_CODE_SUCCESS;
+  }
+
+  wInfo("wal:%s, change old walLevel:%d fsync:%d, new walLevel:%d fsync:%d", pWal->name, pWal->level, pWal->fsyncPeriod,
+        pCfg->walLevel, pCfg->fsyncPeriod);
+
+  pthread_mutex_lock(&pWal->mutex);
+  pWal->level = pCfg->walLevel;
+  pWal->fsyncPeriod = pCfg->fsyncPeriod;
+  if (walNeedFsyncTimer(pWal)) {
+    wInfo("wal:%s, reset fsync timer, walLevel:%d fsyncPeriod:%d", pWal->name, pWal->level, pWal->fsyncPeriod);
+    taosTmrReset(walProcessFsyncTimer, pWal->fsyncPeriod, pWal, &pWal->timer,walTmrCtrl);
+  } else {
+    wInfo("wal:%s, stop fsync timer, walLevel:%d fsyncPeriod:%d", pWal->name, pWal->level, pWal->fsyncPeriod);
+    taosTmrStop(pWal->timer);
+    pWal->timer = NULL;
+  }
+  pthread_mutex_unlock(&pWal->mutex);
+
+  return TSDB_CODE_SUCCESS;
+}
+
 void walClose(void *handle) {
   if (handle == NULL) return;
   
@@ -485,5 +523,11 @@ static void walProcessFsyncTimer(void *param, void *tmrId) {
     wError("wal:%s, fsync failed(%s)", pWal->name, strerror(errno));
   }
 
+  if (walNeedFsyncTimer(pWal)) {
     pWal->timer = taosTmrStart(walProcessFsyncTimer, pWal->fsyncPeriod, pWal, walTmrCtrl);
+  } else {
+    wInfo("wal:%s, stop fsync timer for walLevel:%d fsyncPeriod:%d", pWal->name, pWal->level, pWal->fsyncPeriod);
+    taosTmrStop(pWal->timer);
+    pWal->timer = NULL;
+  }
 }

tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/JDBCConnectorChecker.java

@@ -11,7 +11,6 @@ public class JDBCConnectorChecker {
     private static String tbName = "weather";
     private Connection connection;
 
-
     /**
      * get connection
      **/
@@ -170,5 +169,4 @@ public class JDBCConnectorChecker {
         checker.close();
     }
 
-
 }

tests/gotest/batchtest.sh

+#!/bin/bash
+
+bash ./case001/case001.sh
+#bash ./case002/case002.sh
+#bash ./case003/case003.sh

tests/gotest/case001/case001.sh

+#!/bin/bash
+
+##################################################
+# 
+# Do go test 
+#
+##################################################
+
+set +e
+#set -x
+
+script_dir="$(dirname $(readlink -f $0))"
+#echo "pwd: $script_dir, para0: $0"
+
+execName=$0
+execName=`echo ${execName##*/}`
+goName=`echo ${execName%.*}`
+
+###### step 1: start one taosd
+scriptDir=$script_dir/../../script/sh
+bash $scriptDir/stop_dnodes.sh
+bash $scriptDir/deploy.sh -n dnode1 -i 1
+bash $scriptDir/cfg.sh -n dnode1 -c walLevel -v 0
+bash $scriptDir/exec.sh -n dnode1 -s start
+
+###### step 2: set config item
+TAOS_CFG=/etc/taos/taos.cfg
+HOSTNAME=`hostname -f`
+
+if [ ! -f ${TAOS_CFG} ]; then
+  touch -f $TAOS_CFG
+fi 
+
+echo " "                                           > $TAOS_CFG
+echo "firstEp            ${HOSTNAME}:7100"        >> $TAOS_CFG
+echo "secondEp           ${HOSTNAME}:7200"        >> $TAOS_CFG
+echo "serverPort         7100"                    >> $TAOS_CFG
+#echo "dataDir            $DATA_DIR"              >> $TAOS_CFG
+#echo "logDir             $LOG_DIR"               >> $TAOS_CFG
+#echo "scriptDir          ${CODE_DIR}/../script"  >> $TAOS_CFG
+echo "numOfLogLines      100000000"               >> $TAOS_CFG
+echo "dDebugFlag         135"                     >> $TAOS_CFG
+echo "mDebugFlag         135"                     >> $TAOS_CFG
+echo "sdbDebugFlag       135"                     >> $TAOS_CFG
+echo "rpcDebugFlag       135"                     >> $TAOS_CFG
+echo "tmrDebugFlag       131"                     >> $TAOS_CFG
+echo "cDebugFlag         135"                     >> $TAOS_CFG
+echo "httpDebugFlag      135"                     >> $TAOS_CFG
+echo "monitorDebugFlag   135"                     >> $TAOS_CFG
+echo "udebugFlag         135"                     >> $TAOS_CFG
+echo "tablemetakeeptimer 5"                       >> $TAOS_CFG
+echo "wal                0"                       >> $TAOS_CFG
+echo "asyncLog           0"                       >> $TAOS_CFG
+echo "locale             en_US.UTF-8"             >> $TAOS_CFG
+echo "enableCoreFile     1"                       >> $TAOS_CFG
+echo " "                                          >> $TAOS_CFG  
+
+ulimit -n 600000
+ulimit -c unlimited
+#
+##sudo sysctl -w kernel.core_pattern=$TOP_DIR/core.%p.%e
+#
+
+###### step 3: start build
+cd $script_dir
+rm -f go.*
+go mod init $goName
+go build 
+sleep  1s
+sudo ./$goName

tests/gotest/test.sh

+#!/bin/bash
+
+##################################################
+# 
+# Do go test 
+#
+##################################################
+
+set +e
+#set -x
+
+FILE_NAME=
+RELEASE=0
+while getopts "f:" arg
+do
+  case $arg in
+    f)
+      FILE_NAME=$OPTARG
+      echo "input file: $FILE_NAME"
+      ;;
+    ?)
+      echo "unknow argument"
+      ;;
+  esac
+done
+
+# start one taosd
+bash ../script/sh/stop_dnodes.sh
+bash ../script/sh/deploy.sh -n dnode1 -i 1
+bash ../script/sh/cfg.sh -n dnode1 -c walLevel -v 0
+bash ../script/sh/exec.sh -n dnode1 -s start
+
+# start build test go file
+caseDir=`echo ${FILE_NAME%/*}`
+echo "caseDir: $caseDir"
+cd $caseDir
+rm go.*
+go mod init $caseDir
+go build 
+sleep 1s
+./$caseDir
+

tests/perftest-scripts/coverage_test.sh

@@ -53,7 +53,7 @@ function buildTDengine {
 function runGeneralCaseOneByOne {
 	while read -r line; do
 		if [[ $line =~ ^./test.sh* ]]; then
-			case=`echo $line | grep -w "general\|unique\/mnode\/mgmt33.sim\|unique\/stable\/dnode3.sim\|unique\/cluster\/balance3.sim\|unique\/arbitrator\/offline_replica2_alterTable_online.sim"|awk '{print $NF}'`
+			case=`echo $line | grep sim$ |awk '{print $NF}'`
 
 			if [ -n "$case" ]; then
 				./test.sh -f $case > /dev/null 2>&1 && \

tests/pytest/fulltest.sh

@@ -16,6 +16,7 @@ python3 ./test.py -f insert/nchar.py
 python3 ./test.py -f insert/nchar-unicode.py
 python3 ./test.py -f insert/multi.py
 python3 ./test.py -f insert/randomNullCommit.py
+python3 insert/retentionpolicy.py
 
 python3 ./test.py -f table/column_name.py
 python3 ./test.py -f table/column_num.py
@@ -154,6 +155,7 @@ python3 ./test.py -f stream/new.py
 python3 ./test.py -f stream/stream1.py
 python3 ./test.py -f stream/stream2.py
 python3 ./test.py -f stream/parser.py
+python3 ./test.py -f stream/history.py
 
 #alter table
 python3 ./test.py -f alter/alter_table_crash.py
@@ -192,3 +194,8 @@ python3 test.py -f query/queryInterval.py
 
 # tools
 python3 test.py -f tools/taosdemo.py
+
+# subscribe
+python3 test.py -f subscribe/singlemeter.py
+#python3 test.py -f subscribe/stability.py
+python3 test.py -f subscribe/supertable.py
\ No newline at end of file

tests/pytest/stream/history.py

+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import sys
+import time
+import taos
+from util.log import tdLog
+from util.cases import tdCases
+from util.sql import tdSql
+
+
+class TDTestCase:
+    def init(self, conn, logSql):
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor(), logSql)
+
+    def run(self):
+        tdSql.prepare()
+
+        tdSql.execute("create table cars(ts timestamp, s int) tags(id int)")
+        tdSql.execute("create table car0 using cars tags(0)")
+        tdSql.execute("create table car1 using cars tags(1)")
+        tdSql.execute("create table car2 using cars tags(2)")
+        tdSql.execute("create table car3 using cars tags(3)")
+        tdSql.execute("create table car4 using cars tags(4)")
+
+        tdSql.execute("insert into car0 values('2019-01-01 00:00:00.103', 1)")
+        tdSql.execute("insert into car1 values('2019-01-01 00:00:00.234', 1)")
+        tdSql.execute("insert into car0 values('2019-01-01 00:00:01.012', 1)")
+        tdSql.execute("insert into car0 values('2019-01-01 00:00:02.003', 1)")
+        tdSql.execute("insert into car2 values('2019-01-01 00:00:02.328', 1)")
+        tdSql.execute("insert into car0 values('2019-01-01 00:00:03.139', 1)")
+        tdSql.execute("insert into car0 values('2019-01-01 00:00:04.348', 1)")
+        tdSql.execute("insert into car0 values('2019-01-01 00:00:05.783', 1)")
+        tdSql.execute("insert into car1 values('2019-01-01 00:00:01.893', 1)")
+        tdSql.execute("insert into car1 values('2019-01-01 00:00:02.712', 1)")
+        tdSql.execute("insert into car1 values('2019-01-01 00:00:03.982', 1)")
+        tdSql.execute("insert into car3 values('2019-01-01 00:00:01.389', 1)")
+        tdSql.execute("insert into car4 values('2019-01-01 00:00:01.829', 1)")
+
+        tdSql.execute("create table strm as select count(*) from cars interval(4s)")
+        tdSql.waitedQuery("select * from strm", 2, 100)
+        tdSql.checkData(0, 1, 11)
+        tdSql.checkData(1, 1, 2)
+
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())

tests/pytest/tools/taosdemo.py

@@ -16,6 +16,7 @@ import os
 from util.log import *
 from util.cases import *
 from util.sql import *
+from util.dnodes import *
 
 
 class TDTestCase:
@@ -25,11 +26,30 @@ class TDTestCase:
         
         self.numberOfTables = 10000
         self.numberOfRecords = 100
+    def getBuildPath(self):
+        selfPath = os.path.dirname(os.path.realpath(__file__))
 
+        if ("community" in selfPath):
+            projPath = selfPath[:selfPath.find("community")]
+        else:
+            projPath = selfPath[:selfPath.find("tests")]
+
+        for root, dirs, files in os.walk(projPath):
+            if ("taosd" in files):
+                rootRealPath = os.path.dirname(os.path.realpath(root))
+                if ("packaging" not in rootRealPath):
+                    buildPath = root[:len(root)-len("/build/bin")]
+                    break
+        return buildPath
     def run(self):
         tdSql.prepare()
-        
-        os.system("yes | taosdemo -t %d -n %d" % (self.numberOfTables, self.numberOfRecords))
+        buildPath = self.getBuildPath()
+        if (buildPath == ""):
+            tdLog.exit("taosd not found!")
+        else:
+            tdLog.info("taosd found in %s" % buildPath)
+        binPath = buildPath+ "/build/bin/"
+        os.system("yes | %staosdemo -t %d -n %d" % (binPath,self.numberOfTables, self.numberOfRecords))
 
         tdSql.execute("use test")
         tdSql.query("select count(*) from meters")

tests/script/fullGeneralSuite.sim

@@ -220,3 +220,5 @@ run general/stream/table_del.sim
 run general/stream/metrics_del.sim
 run general/stream/table_replica1_vnoden.sim
 run general/stream/metrics_replica1_vnoden.sim
+run general/db/show_create_db.sim
+run general/db/show_create_table.sim