Merge branch 'develop' into feature/2.0tsdb

src/common/src/tglobal.c

@@ -129,7 +129,7 @@ int32_t tsMnodeEqualVnodeNum = 4;
 int32_t  tsEnableHttpModule = 1;
 int32_t  tsRestRowLimit = 10240;
 uint16_t tsHttpPort = 6020;  // only tcp, range tcp[6020]
-int32_t  tsHttpCacheSessions = 100;
+int32_t  tsHttpCacheSessions = 1000;
 int32_t  tsHttpSessionExpire = 36000;
 int32_t  tsHttpMaxThreads = 2;
 int32_t  tsHttpEnableCompress = 0;

src/dnode/inc/dnodeMgmt.h

@@ -22,6 +22,8 @@ extern "C" {
 
 int32_t dnodeInitMgmt();
 void    dnodeCleanupMgmt();
+int32_t dnodeInitMgmtTimer();
+void    dnodeCleanupMgmtTimer();
 void    dnodeDispatchToMgmtQueue(SRpcMsg *rpcMsg);
 
 void*   dnodeGetVnode(int32_t vgId);

src/dnode/src/dnodeMain.c

@@ -57,6 +57,7 @@ static const SDnodeComponent tsDnodeComponents[] = {
   {"server",  dnodeInitServer,     dnodeCleanupServer},
   {"mgmt",    dnodeInitMgmt,       dnodeCleanupMgmt},
   {"modules", dnodeInitModules,    dnodeCleanupModules},
+  {"mgmt-tmr",dnodeInitMgmtTimer,  dnodeCleanupMgmtTimer},
   {"shell",   dnodeInitShell,      dnodeCleanupShell}
 };
 

src/dnode/src/dnodeMgmt.c

@@ -147,6 +147,12 @@ int32_t dnodeInitMgmt() {
     return -1;
   }
 
+  dInfo("dnode mgmt is initialized");
+ 
+  return TSDB_CODE_SUCCESS;
+}
+
+int32_t dnodeInitMgmtTimer() {
   tsDnodeTmr = taosTmrInit(100, 200, 60000, "DND-DM");
   if (tsDnodeTmr == NULL) {
     dError("failed to init dnode timer");
@@ -155,13 +161,11 @@ int32_t dnodeInitMgmt() {
   }
 
   taosTmrReset(dnodeSendStatusMsg, 500, NULL, tsDnodeTmr, &tsStatusTimer);
-  
-  dInfo("dnode mgmt is initialized");
- 
+  dInfo("dnode mgmt timer is initialized");
   return TSDB_CODE_SUCCESS;
 }
 
-void dnodeCleanupMgmt() {
+void dnodeCleanupMgmtTimer() {
   if (tsStatusTimer != NULL) {
     taosTmrStopA(&tsStatusTimer);
     tsStatusTimer = NULL;
@@ -171,7 +175,10 @@ void dnodeCleanupMgmt() {
     taosTmrCleanUp(tsDnodeTmr);
     tsDnodeTmr = NULL;
   }
+}
 
+void dnodeCleanupMgmt() {
+  dnodeCleanupMgmtTimer();
   dnodeCloseVnodes();
 
   if (tsMgmtQset) taosQsetThreadResume(tsMgmtQset);

src/mnode/src/mnodeMain.c

@@ -41,7 +41,7 @@ typedef struct {
   void              (*cleanup)();
 } SMnodeComponent;
 
-void *tsMnodeTmr;
+void *tsMnodeTmr = NULL;
 static bool tsMgmtIsRunning = false;
 
 static const SMnodeComponent tsMnodeComponents[] = {

src/plugins/http/inc/httpInt.h

@@ -206,7 +206,7 @@ typedef struct HttpThread {
   pthread_mutex_t threadMutex;
   bool            stop;
   int             pollFd;
-  int             numOfFds;
+  int             numOfContexts;
   int             threadId;
   char            label[HTTP_LABEL_SIZE];
   bool (*processData)(HttpContext *pContext);

src/plugins/http/src/httpContext.c

@@ -44,7 +44,7 @@ static void httpDestroyContext(void *data) {
   HttpThread *pThread = pContext->pThread;
   httpRemoveContextFromEpoll(pContext);
   httpReleaseSession(pContext);
-  atomic_sub_fetch_32(&pThread->numOfFds, 1);
+  atomic_sub_fetch_32(&pThread->numOfContexts, 1);
   
   pContext->pThread = 0;
   pContext->state = HTTP_CONTEXT_STATE_CLOSED;
@@ -171,38 +171,39 @@ bool httpInitContext(HttpContext *pContext) {
 
 void httpCloseContextByApp(HttpContext *pContext) {
   pContext->parsed = false;
-
   bool keepAlive = true;
+
   if (pContext->httpVersion == HTTP_VERSION_10 && pContext->httpKeepAlive != HTTP_KEEPALIVE_ENABLE) {
     keepAlive = false;
   } else if (pContext->httpVersion != HTTP_VERSION_10 && pContext->httpKeepAlive == HTTP_KEEPALIVE_DISABLE) {
     keepAlive = false;
-  } else {}
+  } else {
+  }
 
   if (keepAlive) {
     if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_HANDLING, HTTP_CONTEXT_STATE_READY)) {
-      httpDebug("context:%p, fd:%d, ip:%s, last state:handling, keepAlive:true, reuse connect",
-              pContext, pContext->fd, pContext->ipstr);
+      httpDebug("context:%p, fd:%d, ip:%s, last state:handling, keepAlive:true, reuse context", pContext, pContext->fd,
+                pContext->ipstr);
     } else if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_DROPPING, HTTP_CONTEXT_STATE_CLOSED)) {
       httpRemoveContextFromEpoll(pContext);
-      httpDebug("context:%p, fd:%d, ip:%s, last state:dropping, keepAlive:true, close connect",
-              pContext, pContext->fd, pContext->ipstr);
+      httpDebug("context:%p, fd:%d, ip:%s, last state:dropping, keepAlive:true, close connect", pContext, pContext->fd,
+                pContext->ipstr);
     } else if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_READY, HTTP_CONTEXT_STATE_READY)) {
-      httpDebug("context:%p, fd:%d, ip:%s, last state:ready, keepAlive:true, reuse connect",
-              pContext, pContext->fd, pContext->ipstr);
+      httpDebug("context:%p, fd:%d, ip:%s, last state:ready, keepAlive:true, reuse context", pContext, pContext->fd,
+                pContext->ipstr);
     } else if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_CLOSED, HTTP_CONTEXT_STATE_CLOSED)) {
       httpRemoveContextFromEpoll(pContext);
-      httpDebug("context:%p, fd:%d, ip:%s, last state:ready, keepAlive:true, close connect",
-                pContext, pContext->fd, pContext->ipstr);
+      httpDebug("context:%p, fd:%d, ip:%s, last state:ready, keepAlive:true, close connect", pContext, pContext->fd,
+                pContext->ipstr);
     } else {
       httpRemoveContextFromEpoll(pContext);
-      httpError("context:%p, fd:%d, ip:%s, last state:%s:%d, keepAlive:true, close connect",
-              pContext, pContext->fd, pContext->ipstr, httpContextStateStr(pContext->state), pContext->state);
+      httpError("context:%p, fd:%d, ip:%s, last state:%s:%d, keepAlive:true, close connect", pContext, pContext->fd,
+                pContext->ipstr, httpContextStateStr(pContext->state), pContext->state);
     }
   } else {
     httpRemoveContextFromEpoll(pContext);
-    httpDebug("context:%p, fd:%d, ip:%s, last state:%s:%d, keepAlive:false, close connect",
-              pContext, pContext->fd, pContext->ipstr, httpContextStateStr(pContext->state), pContext->state);
+    httpDebug("context:%p, fd:%d, ip:%s, last state:%s:%d, keepAlive:false, close context", pContext, pContext->fd,
+              pContext->ipstr, httpContextStateStr(pContext->state), pContext->state);
   }
 
   httpReleaseContext(pContext);
@@ -214,7 +215,7 @@ void httpCloseContextByServer(HttpContext *pContext) {
   } else if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_DROPPING, HTTP_CONTEXT_STATE_DROPPING)) {
     httpDebug("context:%p, fd:%d, ip:%s, epoll already finished, wait app finished", pContext, pContext->fd, pContext->ipstr);
   } else if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_READY, HTTP_CONTEXT_STATE_CLOSED)) {
-    httpDebug("context:%p, fd:%d, ip:%s, epoll finished, close context", pContext, pContext->fd, pContext->ipstr);
+    httpDebug("context:%p, fd:%d, ip:%s, epoll finished, close connect", pContext, pContext->fd, pContext->ipstr);
   } else if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_CLOSED, HTTP_CONTEXT_STATE_CLOSED)) {
     httpDebug("context:%p, fd:%d, ip:%s, epoll finished, will be closed soon", pContext, pContext->fd, pContext->ipstr);
   } else {

src/plugins/http/src/httpHandle.c

@@ -313,9 +313,9 @@ bool httpParseRequest(HttpContext* pContext) {
     return true;
   }
 
-  httpTraceL("context:%p, fd:%d, ip:%s, thread:%s, numOfFds:%d, read size:%d, raw data:\n%s", pContext, pContext->fd,
-            pContext->ipstr, pContext->pThread->label, pContext->pThread->numOfFds, pContext->parser.bufsize,
-            pContext->parser.buffer);
+  httpTraceL("context:%p, fd:%d, ip:%s, thread:%s, numOfContexts:%d, read size:%d, raw data:\n%s", pContext,
+             pContext->fd, pContext->ipstr, pContext->pThread->label, pContext->pThread->numOfContexts,
+             pContext->parser.bufsize, pContext->parser.buffer);
 
   if (!httpGetHttpMethod(pContext)) {
     return false;

src/plugins/http/src/httpServer.c

@@ -293,7 +293,7 @@ static void *httpAcceptHttpConnection(void *arg) {
 
     totalFds = 1;
     for (int i = 0; i < pServer->numOfThreads; ++i) {
-      totalFds += pServer->pThreads[i].numOfFds;
+      totalFds += pServer->pThreads[i].numOfContexts;
     }
 
     if (totalFds > tsHttpCacheSessions * 100) {
@@ -332,9 +332,9 @@ static void *httpAcceptHttpConnection(void *arg) {
     }
 
     // notify the data process, add into the FdObj list
-    atomic_add_fetch_32(&pThread->numOfFds, 1);
-    httpDebug("context:%p, fd:%d, ip:%s, thread:%s numOfFds:%d totalFds:%d, accept a new connection", pContext, connFd,
-              pContext->ipstr, pThread->label, pThread->numOfFds, totalFds);
+    atomic_add_fetch_32(&pThread->numOfContexts, 1);
+    httpDebug("context:%p, fd:%d, ip:%s, thread:%s numOfContexts:%d totalFds:%d, accept a new connection", pContext,
+              connFd, pContext->ipstr, pThread->label, pThread->numOfContexts, totalFds);
 
     // pick up next thread for next connection
     threadId++;

src/wal/src/walMain.c

@@ -28,6 +28,7 @@
 #include "taoserror.h"
 #include "twal.h"
 #include "tqueue.h"
+#include "tfile.h"
 
 #define walPrefix "wal"
 
@@ -180,7 +181,7 @@ int walWrite(void *handle, SWalHead *pHead) {
   taosCalcChecksumAppend(0, (uint8_t *)pHead, sizeof(SWalHead));
   int contLen = pHead->len + sizeof(SWalHead);
 
-  if(write(pWal->fd, pHead, contLen) != contLen) {
+  if(twrite(pWal->fd, pHead, contLen) != contLen) {
     wError("wal:%s, failed to write(%s)", pWal->name, strerror(errno));
     terrno = TAOS_SYSTEM_ERROR(errno);
   } else {
@@ -325,7 +326,7 @@ static int walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp) {
   wDebug("wal:%s, start to restore", name);
 
   while (1) {
-    int ret = read(fd, pHead, sizeof(SWalHead));
+    int ret = tread(fd, pHead, sizeof(SWalHead));
     if ( ret == 0)  break;  
 
     if (ret != sizeof(SWalHead)) {
@@ -340,7 +341,7 @@ static int walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp) {
       break;
     } 
 
-    ret = read(fd, pHead->cont, pHead->len);
+    ret = tread(fd, pHead->cont, pHead->len);
     if ( ret != pHead->len) {
       wWarn("wal:%s, failed to read body, skip, len:%d ret:%d", name, pHead->len, ret);
       terrno = TAOS_SYSTEM_ERROR(errno);

tests/script/jenkins/basic.txt

@@ -117,8 +117,6 @@ cd ../../../debug; make
 ./test.sh -f general/parser/import_commit3.sim
 ./test.sh -f general/parser/insert_tb.sim
 ./test.sh -f general/parser/first_last.sim
-# dyh is processing this script
-#./test.sh -f general/parser/import_file.sim
 ./test.sh -f general/parser/lastrow.sim
 ./test.sh -f general/parser/nchar.sim
 ./test.sh -f general/parser/null_char.sim
@@ -145,7 +143,6 @@ cd ../../../debug; make
 ./test.sh -f general/parser/groupby.sim
 ./test.sh -f general/parser/set_tag_vals.sim
 #./test.sh -f general/parser/sliding.sim
-./test.sh -f general/parser/tags_dynamically_specifiy.sim
 ./test.sh -f general/parser/tags_filter.sim
 ./test.sh -f general/parser/slimit_alter_tags.sim
 ./test.sh -f general/parser/join.sim

tests/script/sh/deploy.sh

@@ -125,7 +125,6 @@ echo "mqttDebugFlag       131"            >> $TAOS_CFG
 echo "qdebugFlag          135"            >> $TAOS_CFG
 echo "rpcDebugFlag        135"            >> $TAOS_CFG
 echo "tmrDebugFlag        131"            >> $TAOS_CFG
-echo "cDebugFlag          135"            >> $TAOS_CFG
 echo "udebugFlag          135"            >> $TAOS_CFG
 echo "sdebugFlag          135"            >> $TAOS_CFG
 echo "wdebugFlag          135"            >> $TAOS_CFG