[jira none]

src/client/src/tscParseInsert.c

@@ -56,6 +56,7 @@ static int32_t tscToInteger(SSQLToken *pToken, int64_t *value, char **endPtr) {
     radix = 2;
   }
 
+  errno = 0;
   *value = strtoll(pToken->z, endPtr, radix);
 
   return numType;
@@ -66,6 +67,8 @@ static int32_t tscToDouble(SSQLToken *pToken, double *value, char **endPtr) {
   if (TK_ILLEGAL == numType) {
     return numType;
   }
+
+  errno = 0;
   *value = strtod(pToken->z, endPtr);
   return numType;
 }

src/client/src/tscSQLParser.c

@@ -2546,6 +2546,10 @@ int32_t setShowInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
       }
     }
   }else if (type == SHOW_VNODES) {
+    if (NULL == pInfo->pDCLInfo) {
+      return invalidSqlErrMsg(pCmd, "No specified ip of dnode");
+    }
+
     // show vnodes may be ip addr of dnode in payload
     if (pInfo->pDCLInfo->nTokens > 0) {
       SSQLToken* pDnodeIp = &pInfo->pDCLInfo->a[0];

src/client/src/tscSql.c

@@ -64,8 +64,8 @@ TAOS *taos_connect_imp(const char *ip, const char *user, const char *pass, const
 
 #ifdef CLUSTER
   if (ip && ip[0]) {
-    strcpy(tscMgmtIpList.ipstr[0], ip);
-    tscMgmtIpList.ip[0] = inet_addr(ip);
+    strcpy(tscMgmtIpList.ipstr[1], ip);
+    tscMgmtIpList.ip[1] = inet_addr(ip);
   }
 #else
   if (ip && ip[0]) {

src/modules/http/inc/httpHandle.h

@@ -68,6 +68,8 @@
 #define HTTP_COMPRESS_IDENTITY      0
 #define HTTP_COMPRESS_GZIP          2
 
+#define HTTP_SESSION_ID_LEN         (TSDB_USER_LEN * 2 + 1)
+
 typedef enum {
     HTTP_CONTEXT_STATE_READY,
     HTTP_CONTEXT_STATE_HANDLING,
@@ -83,7 +85,7 @@ typedef struct {
   int   expire;
   int   access;
   void *taos;
-  char  id[TSDB_USER_LEN];
+  char  id[HTTP_SESSION_ID_LEN + 1];
 } HttpSession;
 
 typedef enum {

src/modules/http/src/httpAuth.c

@@ -50,6 +50,7 @@ bool httpParseBasicAuthToken(HttpContext *pContext, char *token, int len) {
     return false;
   }
   strncpy(pContext->user, base64, (size_t)user_len);
+  pContext->user[user_len] = 0;
 
   char *password = user + 1;
   int   pass_len = (int)((base64 + outlen) - password);
@@ -60,6 +61,7 @@ bool httpParseBasicAuthToken(HttpContext *pContext, char *token, int len) {
     return false;
   }
   strncpy(pContext->pass, password, (size_t)pass_len);
+  pContext->pass[pass_len] = 0;
 
   free(base64);
   httpTrace("context:%p, fd:%d, ip:%s, basic token parsed success, user:%s", pContext, pContext->fd, pContext->ipstr,

src/modules/http/src/httpCode.c

@@ -69,7 +69,7 @@ char* httpMsg[] = {
     "field value type should be number or string",
     "field value is null",                     // 51
     "parse basic auth token error",
-    "parse taosd auth token error",
+    "parse http auth token error",
     "host type should be string",
 
     // grafana

src/modules/http/src/httpSession.c

@@ -41,8 +41,8 @@ void httpCreateSession(HttpContext *pContext, void *taos) {
   pthread_mutex_lock(&server->serverMutex);
 
   if (pContext->session != NULL && pContext->session == pContext->session->signature) {
-    httpTrace("context:%p, fd:%d, ip:%s, user:%s, set exist session:%p:%s:%p expired", pContext, pContext->fd,
-              pContext->ipstr, pContext->user, pContext->session, pContext->session->id, pContext->session->taos);
+    httpTrace("context:%p, fd:%d, ip:%s, user:%s, set exist session:%p:%p expired", pContext, pContext->fd,
+              pContext->ipstr, pContext->user, pContext->session, pContext->session->taos);
     pContext->session->expire = 0;
     pContext->session->access--;
   }
@@ -51,7 +51,7 @@ void httpCreateSession(HttpContext *pContext, void *taos) {
   session.taos = taos;
   session.expire = (int)taosGetTimestampSec() + server->sessionExpire;
   session.access = 1;
-  strcpy(session.id, pContext->user);
+  snprintf(session.id, HTTP_SESSION_ID_LEN, "%s.%s", pContext->user, pContext->pass);
   pContext->session = (HttpSession *)taosAddStrHash(server->pSessionHash, session.id, (char *)(&session));
   if (pContext->session == NULL) {
     httpError("context:%p, fd:%d, ip:%s, user:%s, error:%s", pContext, pContext->fd, pContext->ipstr, pContext->user,
@@ -62,20 +62,23 @@ void httpCreateSession(HttpContext *pContext, void *taos) {
   }
 
   pContext->session->signature = pContext->session;
-  httpTrace("context:%p, fd:%d, ip:%s, user:%s, create a new session:%p:%s:%p", pContext, pContext->fd, pContext->ipstr,
-            pContext->user, pContext->session, pContext->session->id, pContext->session->taos);
+  httpTrace("context:%p, fd:%d, ip:%s, user:%s, create a new session:%p:%p", pContext, pContext->fd, pContext->ipstr,
+            pContext->user, pContext->session, pContext->session->taos);
   pthread_mutex_unlock(&server->serverMutex);
 }
 
-void httpFetchSession(HttpContext *pContext) {
+void httpFetchSessionImp(HttpContext *pContext) {
   HttpServer *server = pContext->pThread->pServer;
   pthread_mutex_lock(&server->serverMutex);
 
-  pContext->session = (HttpSession *)taosGetStrHashData(server->pSessionHash, pContext->user);
+  char sessionId[HTTP_SESSION_ID_LEN];
+  snprintf(sessionId, HTTP_SESSION_ID_LEN, "%s.%s", pContext->user, pContext->pass);
+
+  pContext->session = (HttpSession *)taosGetStrHashData(server->pSessionHash, sessionId);
   if (pContext->session != NULL && pContext->session == pContext->session->signature) {
     pContext->session->access++;
-    httpTrace("context:%p, fd:%d, ip:%s, user:%s, find an exist session:%p:%s:%p, access:%d, expire:%d",
-              pContext, pContext->fd, pContext->ipstr, pContext->user, pContext->session, pContext->session->id,
+    httpTrace("context:%p, fd:%d, ip:%s, user:%s, find an exist session:%p:%p, access:%d, expire:%d",
+              pContext, pContext->fd, pContext->ipstr, pContext->user, pContext->session,
               pContext->session->taos, pContext->session->access, pContext->session->expire);
     pContext->session->expire = (int)taosGetTimestampSec() + server->sessionExpire;
   } else {
@@ -86,6 +89,20 @@ void httpFetchSession(HttpContext *pContext) {
   pthread_mutex_unlock(&server->serverMutex);
 }
 
+void httpFetchSession(HttpContext *pContext) {
+  if (pContext->session == NULL) {
+    httpFetchSessionImp(pContext);
+  } else {
+    char sessionId[HTTP_SESSION_ID_LEN];
+    snprintf(sessionId, HTTP_SESSION_ID_LEN, "%s.%s", pContext->user, pContext->pass);
+    if (strcmp(pContext->session->id, sessionId) != 0) {
+      httpError("context:%p, fd:%d, ip:%s, user:%s, password may be changed", pContext, pContext->fd, pContext->ipstr, pContext->user);
+      httpRestoreSession(pContext);
+      httpFetchSessionImp(pContext);
+    }
+  }
+}
+
 void httpRestoreSession(HttpContext *pContext) {
   HttpServer * server = pContext->pThread->pServer;
 
@@ -97,15 +114,16 @@ void httpRestoreSession(HttpContext *pContext) {
     return;
   }
   session->access--;
-  httpTrace("context:%p, ip:%s, user:%s, restore session:%p:%s:%p, access:%d, expire:%d",
-            pContext, pContext->ipstr, pContext->user, session, session->id, session->taos,
+  httpTrace("context:%p, ip:%s, user:%s, restore session:%p:%p, access:%d, expire:%d",
+            pContext, pContext->ipstr, pContext->user, session, session->taos,
             session->access, pContext->session->expire);
+  pContext->session = NULL;
   pthread_mutex_unlock(&server->serverMutex);
 }
 
 void httpResetSession(char *session) {
   HttpSession *pSession = (HttpSession *)session;
-  httpTrace("close session:%p:%s:%p", pSession, pSession->id, pSession->taos);
+  httpTrace("close session:%p:%p", pSession, pSession->taos);
   if (pSession->taos != NULL) {
     taos_close(pSession->taos);
     pSession->taos = NULL;
@@ -144,12 +162,12 @@ int httpSessionExpired(char *session) {
       return 0;  // un-expired, so return false
     }
     if (pSession->access > 0) {
-      httpTrace("session:%p:%s:%p is expired, but still access:%d", pSession, pSession->id, pSession->taos,
+      httpTrace("session:%p:%p is expired, but still access:%d", pSession, pSession->taos,
                 pSession->access);
       return 0;  // still used, so return false
     }
-    httpTrace("need close session:%p:%s:%p for it expired, cur:%d, expire:%d, invertal:%d",
-              pSession, pSession->id, pSession->taos, cur, pSession->expire, cur - pSession->expire);
+    httpTrace("need close session:%p:%p for it expired, cur:%d, expire:%d, invertal:%d",
+              pSession, pSession->taos, cur, pSession->expire, cur - pSession->expire);
   }
 
   return 1;

src/modules/http/src/httpSql.c

@@ -378,9 +378,7 @@ void httpProcessRequestCb(void *param, TAOS_RES *result, int code) {
 }
 
 void httpProcessRequest(HttpContext *pContext) {
-  if (pContext->session == NULL) {
-    httpFetchSession(pContext);
-  }
+  httpFetchSession(pContext);
 
   if (pContext->session == NULL || pContext->session != pContext->session->signature ||
       pContext->reqType == HTTP_REQTYPE_LOGIN) {

src/system/detail/src/mgmtDnode.c

@@ -476,6 +476,8 @@ int mgmtRetrieveVnodes(SShowObj *pShow, char *data, int rows, SConnObj *pConn) {
           continue;
         }
         
+        cols = 0;
+        
         pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
         *(uint32_t *)pWrite = pVnode->vnode;
         cols++;

src/system/detail/src/mgmtMeter.c

@@ -660,7 +660,7 @@ int mgmtCreateMeter(SDbObj *pDb, SCreateTableMsg *pCreate) {
     pMeter->uid = (((uint64_t)pMeter->gid.vgId) << 40) + ((((uint64_t)pMeter->gid.sid) & ((1ul << 24) - 1ul)) << 16) +
                   ((uint64_t)sdbVersion & ((1ul << 16) - 1ul));
 
-    mTrace("table:%s, create table in vgroup, vgId:%d sid:%d vnode:%d uid:%ld db:%s",
+    mTrace("table:%s, create table in vgroup, vgId:%d sid:%d vnode:%d uid:%llu db:%s",
            pMeter->meterId, pVgroup->vgId, sid, pVgroup->vnodeGid[0].vnode, pMeter->uid, pDb->name);
   } else {
     pMeter->uid = (((uint64_t)pMeter->createdTime) << 16) + ((uint64_t)sdbVersion & ((1ul << 16) - 1ul));

src/system/detail/src/vnodeCache.c

@@ -372,13 +372,60 @@ void vnodeCancelCommit(SVnodeObj *pVnode) {
   taosTmrReset(vnodeProcessCommitTimer, pVnode->cfg.commitTime * 1000, pVnode, vnodeTmrCtrl, &pVnode->commitTimer);
 }
 
+/* The vnode cache lock should be hold before calling this interface
+ */
+SCacheBlock *vnodeGetFreeCacheBlock(SVnodeObj *pVnode) {
+  SCachePool  *pPool = (SCachePool *)(pVnode->pCachePool);
+  SVnodeCfg   *pCfg = &(pVnode->cfg);
+  SCacheBlock *pCacheBlock = NULL;
+  int skipped = 0;
+
+  while (1) {
+    pCacheBlock = (SCacheBlock *)(pPool->pMem[((int64_t)pPool->freeSlot)]);
+    if (pCacheBlock->blockId == 0) break;
+
+    if (pCacheBlock->notFree) {
+      pPool->freeSlot++;
+      pPool->freeSlot = pPool->freeSlot % pCfg->cacheNumOfBlocks.totalBlocks;
+      skipped++;
+      if (skipped > pPool->threshold) {
+        vnodeCreateCommitThread(pVnode);
+        pthread_mutex_unlock(&pPool->vmutex);
+        dError("vid:%d committing process is too slow, notFreeSlots:%d....", pVnode->vnode, pPool->notFreeSlots);
+        return NULL;
+      }
+    } else {
+      SMeterObj * pRelObj = pCacheBlock->pMeterObj;
+      SCacheInfo *pRelInfo = (SCacheInfo *)pRelObj->pCache;
+      int firstSlot = (pRelInfo->currentSlot - pRelInfo->numOfBlocks + 1 + pRelInfo->maxBlocks) % pRelInfo->maxBlocks;
+      pCacheBlock = pRelInfo->cacheBlocks[firstSlot];
+      if (pCacheBlock) {
+        pPool->freeSlot = pCacheBlock->index;
+        vnodeFreeCacheBlock(pCacheBlock);
+        break;
+      } else {
+        pPool->freeSlot = (pPool->freeSlot + 1) % pCfg->cacheNumOfBlocks.totalBlocks;
+        skipped++;
+      }
+    }
+  }
+
+  pCacheBlock = (SCacheBlock *)(pPool->pMem[pPool->freeSlot]);
+  pCacheBlock->index = pPool->freeSlot;
+  pCacheBlock->notFree = 1;
+  pPool->freeSlot = (pPool->freeSlot + 1) % pCfg->cacheNumOfBlocks.totalBlocks;
+  pPool->notFreeSlots++;
+
+  return pCacheBlock;
+}
+
 int vnodeAllocateCacheBlock(SMeterObj *pObj) {
   int          index;
   SCachePool * pPool;
   SCacheBlock *pCacheBlock;
   SCacheInfo * pInfo;
   SVnodeObj *  pVnode;
-  int          skipped = 0, commit = 0;
+  int          commit = 0;
 
   pVnode = vnodeList + pObj->vnode;
   pPool = (SCachePool *)pVnode->pCachePool;
@@ -406,45 +453,10 @@ int vnodeAllocateCacheBlock(SMeterObj *pObj) {
     return -1;
   }
 
-  while (1) {
-    pCacheBlock = (SCacheBlock *)(pPool->pMem[((int64_t)pPool->freeSlot)]);
-    if (pCacheBlock->blockId == 0) break;
-
-    if (pCacheBlock->notFree) {
-      pPool->freeSlot++;
-      pPool->freeSlot = pPool->freeSlot % pCfg->cacheNumOfBlocks.totalBlocks;
-      skipped++;
-      if (skipped > pPool->threshold) {
-        vnodeCreateCommitThread(pVnode);
-        pthread_mutex_unlock(&pPool->vmutex);
-        dError("vid:%d sid:%d id:%s, committing process is too slow, notFreeSlots:%d....",
-               pObj->vnode, pObj->sid, pObj->meterId, pPool->notFreeSlots);
-        return -1;
-      }
-    } else {
-      SMeterObj  *pRelObj = pCacheBlock->pMeterObj;
-      SCacheInfo *pRelInfo = (SCacheInfo *)pRelObj->pCache;
-      int firstSlot = (pRelInfo->currentSlot - pRelInfo->numOfBlocks + 1 + pRelInfo->maxBlocks) % pRelInfo->maxBlocks;
-      pCacheBlock = pRelInfo->cacheBlocks[firstSlot];
-      if (pCacheBlock) {
-        pPool->freeSlot = pCacheBlock->index;
-        vnodeFreeCacheBlock(pCacheBlock);
-        break;
-      } else {
-        pPool->freeSlot = (pPool->freeSlot + 1) % pCfg->cacheNumOfBlocks.totalBlocks;
-        skipped++;
-      }
-    }
-  }
-
-  index = pPool->freeSlot;
-  pPool->freeSlot++;
-  pPool->freeSlot = pPool->freeSlot % pCfg->cacheNumOfBlocks.totalBlocks;
-  pPool->notFreeSlots++;
+  if ((pCacheBlock = vnodeGetFreeCacheBlock(pVnode)) == NULL) return -1;
 
+  index = pCacheBlock->index;
   pCacheBlock->pMeterObj = pObj;
-  pCacheBlock->notFree = 1;
-  pCacheBlock->index = index;
 
   pCacheBlock->offset[0] = ((char *)(pCacheBlock)) + sizeof(SCacheBlock) + pObj->numOfColumns * sizeof(char *);
   for (int col = 1; col < pObj->numOfColumns; ++col)

src/system/detail/src/vnodeFile.c

@@ -95,8 +95,8 @@ void vnodeGetDnameFromLname(char *lhead, char *ldata, char *llast, char *dhead,
 }
 
 void vnodeGetHeadTname(char *nHeadName, char *nLastName, int vnode, int fileId) {
-  sprintf(nHeadName, "%s/vnode%d/db/v%df%d.t", tsDirectory, vnode, vnode, fileId);
-  sprintf(nLastName, "%s/vnode%d/db/v%df%d.l", tsDirectory, vnode, vnode, fileId);
+  if (nHeadName != NULL) sprintf(nHeadName, "%s/vnode%d/db/v%df%d.t", tsDirectory, vnode, vnode, fileId);
+  if (nLastName != NULL) sprintf(nLastName, "%s/vnode%d/db/v%df%d.l", tsDirectory, vnode, vnode, fileId);
 }
 
 void vnodeCreateDataDirIfNeeded(int vnode, char *path) {
@@ -180,29 +180,24 @@ int vnodeCreateEmptyCompFile(int vnode, int fileId) {
   return 0;
 }
 
-int vnodeOpenCommitFiles(SVnodeObj *pVnode, int noTempLast) {
-  char        name[TSDB_FILENAME_LEN];
-  char        dHeadName[TSDB_FILENAME_LEN] = "\0";
-  char        dLastName[TSDB_FILENAME_LEN] = "\0";
-  int         len = 0;
-  struct stat filestat;
-  int         vnode = pVnode->vnode;
-  int         fileId, numOfFiles, filesAdded = 0;
-  SVnodeCfg * pCfg = &pVnode->cfg;
+int vnodeCreateNeccessaryFiles(SVnodeObj *pVnode) {
+  int        numOfFiles = 0, fileId, filesAdded = 0;
+  int        vnode = pVnode->vnode;
+  SVnodeCfg *pCfg = &(pVnode->cfg);
 
   if (pVnode->lastKeyOnFile == 0) {
     if (pCfg->daysPerFile == 0) pCfg->daysPerFile = 10;
     pVnode->fileId = pVnode->firstKey / tsMsPerDay[pVnode->cfg.precision] / pCfg->daysPerFile;
     pVnode->lastKeyOnFile = (int64_t)(pVnode->fileId + 1) * pCfg->daysPerFile * tsMsPerDay[pVnode->cfg.precision] - 1;
     pVnode->numOfFiles = 1;
-    vnodeCreateEmptyCompFile(vnode, pVnode->fileId);
+    if (vnodeCreateEmptyCompFile(vnode, pVnode->fileId) < 0) return -1;
   }
 
   numOfFiles = (pVnode->lastKeyOnFile - pVnode->commitFirstKey) / tsMsPerDay[pVnode->cfg.precision] / pCfg->daysPerFile;
   if (pVnode->commitFirstKey > pVnode->lastKeyOnFile) numOfFiles = -1;
 
-  dTrace("vid:%d, commitFirstKey:%ld lastKeyOnFile:%ld numOfFiles:%d fileId:%d vnodeNumOfFiles:%d",
-      vnode, pVnode->commitFirstKey, pVnode->lastKeyOnFile, numOfFiles, pVnode->fileId, pVnode->numOfFiles);
+  dTrace("vid:%d, commitFirstKey:%ld lastKeyOnFile:%ld numOfFiles:%d fileId:%d vnodeNumOfFiles:%d", pVnode->vnode,
+         pVnode->commitFirstKey, pVnode->lastKeyOnFile, numOfFiles, pVnode->fileId, pVnode->numOfFiles);
 
   if (numOfFiles >= pVnode->numOfFiles) {
     // create empty header files backward
@@ -214,7 +209,7 @@ int vnodeOpenCommitFiles(SVnodeObj *pVnode, int noTempLast) {
 #ifdef CLUSTER	  
 	    return vnodeRecoverFromPeer(pVnode, fileId);
 #else
-        return -1;
+      return -1;
 #endif				
     }
   } else if (numOfFiles < 0) {
@@ -224,7 +219,7 @@ int vnodeOpenCommitFiles(SVnodeObj *pVnode, int noTempLast) {
 #ifdef CLUSTER	  
 	    return vnodeRecoverFromPeer(pVnode, pVnode->fileId);
 #else
-        return -1;
+      return -1;
 #endif
     pVnode->lastKeyOnFile += (int64_t)tsMsPerDay[pVnode->cfg.precision] * pCfg->daysPerFile;
     filesAdded = 1;
@@ -238,6 +233,24 @@ int vnodeOpenCommitFiles(SVnodeObj *pVnode, int noTempLast) {
   pVnode->commitFileId = fileId;
   pVnode->numOfFiles = pVnode->numOfFiles + filesAdded;
 
+  return 0;
+}
+
+
+int vnodeOpenCommitFiles(SVnodeObj *pVnode, int noTempLast) {
+  char        name[TSDB_FILENAME_LEN];
+  char        dHeadName[TSDB_FILENAME_LEN] = "\0";
+  char        dLastName[TSDB_FILENAME_LEN] = "\0";
+  int         len = 0;
+  struct stat filestat;
+  int         vnode = pVnode->vnode;
+  int         fileId, numOfFiles, filesAdded = 0;
+  SVnodeCfg * pCfg = &pVnode->cfg;
+
+  if (vnodeCreateNeccessaryFiles(pVnode) < 0) return -1;
+
+  fileId = pVnode->commitFileId;
+
   dTrace("vid:%d, commit fileId:%d, commitLastKey:%ld, vnodeLastKey:%ld, lastKeyOnFile:%ld numOfFiles:%d",
       vnode, fileId, pVnode->commitLastKey, pVnode->lastKey, pVnode->lastKeyOnFile, pVnode->numOfFiles);
 
@@ -1817,7 +1830,15 @@ int vnodeInitFile(int vnode) {
   pVnode->fmagic = (uint64_t *)calloc(pVnode->maxFiles + 1, sizeof(uint64_t));
   int fileId = pVnode->fileId;
 
-  for (int i = 0; i < pVnode->numOfFiles; ++i) {
+  /*
+   * The actual files will far exceed the files that need to exist
+   */
+  if (pVnode->numOfFiles > pVnode->maxFiles) {
+    dError("vid:%d numOfFiles:%d should not larger than maxFiles:%d", vnode, pVnode->numOfFiles, pVnode->maxFiles);
+  }
+
+  int numOfFiles = MIN(pVnode->numOfFiles, pVnode->maxFiles);
+  for (int i = 0; i < numOfFiles; ++i) {
     if (vnodeUpdateFileMagic(vnode, fileId) < 0) {
       if (pVnode->cfg.replications > 1) {
         pVnode->badFileId = fileId;

src/system/detail/src/vnodeMeter.c

@@ -584,12 +584,12 @@ int vnodeInsertPoints(SMeterObj *pObj, char *cont, int contLen, char source, voi
   if (pVnode->cfg.commitLog && source != TSDB_DATA_SOURCE_LOG) {
     if (pVnode->logFd < 0) return TSDB_CODE_INVALID_COMMIT_LOG;
     code = vnodeWriteToCommitLog(pObj, TSDB_ACTION_INSERT, cont, contLen, sversion);
-    if (code != 0) return code;
+    if (code != TSDB_CODE_SUCCESS) return code;
   }
 
   if (source == TSDB_DATA_SOURCE_SHELL && pVnode->cfg.replications > 1) {
     code = vnodeForwardToPeer(pObj, cont, contLen, TSDB_ACTION_INSERT, sversion);
-    if (code != 0) return code;
+    if (code != TSDB_CODE_SUCCESS) return code;
   }
 
   if (pObj->sversion < sversion) {
@@ -601,11 +601,11 @@ int vnodeInsertPoints(SMeterObj *pObj, char *cont, int contLen, char source, voi
   }
 
   pData = pSubmit->payLoad;
-  code = TSDB_CODE_SUCCESS;
 
   TSKEY firstKey = *((TSKEY *)pData);
   TSKEY lastKey = *((TSKEY *)(pData + pObj->bytesPerPoint * (numOfPoints - 1)));
   int cfid = now/pVnode->cfg.daysPerFile/tsMsPerDay[pVnode->cfg.precision];
+  
   TSKEY minAllowedKey = (cfid - pVnode->maxFiles + 1)*pVnode->cfg.daysPerFile*tsMsPerDay[pVnode->cfg.precision];
   TSKEY maxAllowedKey = (cfid + 2)*pVnode->cfg.daysPerFile*tsMsPerDay[pVnode->cfg.precision] - 2;
   if (firstKey < minAllowedKey || firstKey > maxAllowedKey || lastKey < minAllowedKey || lastKey > maxAllowedKey) {
@@ -619,7 +619,7 @@ int vnodeInsertPoints(SMeterObj *pObj, char *cont, int contLen, char source, voi
   }
   
   for (i = 0; i < numOfPoints; ++i) { // meter will be dropped, abort current insertion
-    if (pObj->state >= TSDB_METER_STATE_DELETING) {
+    if (vnodeIsMeterState(pObj, TSDB_METER_STATE_DELETING)) {
       dWarn("vid:%d sid:%d id:%s, meter is dropped, abort insert, state:%d", pObj->vnode, pObj->sid, pObj->meterId,
             pObj->state);
 
@@ -648,6 +648,7 @@ int vnodeInsertPoints(SMeterObj *pObj, char *cont, int contLen, char source, voi
     pData += pObj->bytesPerPoint;
     points++;
   }
+  
   atomic_fetch_add_64(&(pVnode->vnodeStatistic.pointsWritten), points * (pObj->numOfColumns - 1));
   atomic_fetch_add_64(&(pVnode->vnodeStatistic.totalStorage), points * pObj->bytesPerPoint);
 
@@ -660,6 +661,7 @@ int vnodeInsertPoints(SMeterObj *pObj, char *cont, int contLen, char source, voi
   pVnode->version++;
 
   pthread_mutex_unlock(&(pVnode->vmutex));
+  
   vnodeClearMeterState(pObj, TSDB_METER_STATE_INSERT);
 
 _over:

src/system/detail/src/vnodeShell.c

@@ -39,10 +39,21 @@ SShellObj **shellList = NULL;
 int vnodeProcessRetrieveRequest(char *pMsg, int msgLen, SShellObj *pObj);
 int vnodeProcessQueryRequest(char *pMsg, int msgLen, SShellObj *pObj);
 int vnodeProcessShellSubmitRequest(char *pMsg, int msgLen, SShellObj *pObj);
+static void vnodeProcessBatchSubmitTimer(void *param, void *tmrId);
 
 int vnodeSelectReqNum = 0;
 int vnodeInsertReqNum = 0;
 
+typedef struct {
+  int32_t import;
+  int32_t vnode;
+  int32_t numOfSid;
+  int32_t ssid;   // Start sid
+  SShellObj *pObj;
+  int64_t offset; // offset relative the blks
+  char    blks[];
+} SBatchSubmitInfo;
+
 void *vnodeProcessMsgFromShell(char *msg, void *ahandle, void *thandle) {
   int        sid, vnode;
   SShellObj *pObj = (SShellObj *)ahandle;
@@ -249,6 +260,7 @@ int vnodeSendShellSubmitRspMsg(SShellObj *pObj, int code, int numOfPoints) {
   char *pMsg, *pStart;
   int   msgLen;
 
+  dTrace("code:%d numOfTotalPoints:%d", code, numOfPoints);
   pStart = taosBuildRspMsgWithSize(pObj->thandle, TSDB_MSG_TYPE_SUBMIT_RSP, 128);
   if (pStart == NULL) return -1;
   pMsg = pStart;
@@ -280,6 +292,7 @@ int vnodeProcessQueryRequest(char *pMsg, int msgLen, SShellObj *pObj) {
   }
 
   if (pQueryMsg->numOfSids <= 0) {
+    dError("Invalid number of meters to query, numOfSids:%d", pQueryMsg->numOfSids);
     code = TSDB_CODE_INVALID_QUERY_MSG;
     goto _query_over;
   }
@@ -485,10 +498,83 @@ int vnodeProcessRetrieveRequest(char *pMsg, int msgLen, SShellObj *pObj) {
   return msgLen;
 }
 
+static int vnodeCheckSubmitBlockContext(SShellSubmitBlock *pBlocks, SVnodeObj *pVnode) {
+  int32_t  sid = htonl(pBlocks->sid);
+  uint64_t uid = htobe64(pBlocks->uid);
+
+  if (sid >= pVnode->cfg.maxSessions || sid <= 0) {
+    dError("vid:%d sid:%d, sid is out of range", sid);
+    return TSDB_CODE_INVALID_TABLE_ID;
+  }
+
+  SMeterObj *pMeterObj = pVnode->meterList[sid];
+  if (pMeterObj == NULL) {
+    dError("vid:%d sid:%d, not active table", pVnode->vnode, sid);
+    vnodeSendMeterCfgMsg(pVnode->vnode, sid);
+    return TSDB_CODE_NOT_ACTIVE_TABLE;
+  }
+
+  if (pMeterObj->uid != uid) {
+    dError("vid:%d sid:%d id:%s, uid:%lld, uid in msg:%lld, uid mismatch", pVnode->vnode, sid, pMeterObj->meterId,
+           pMeterObj->uid, uid);
+    return TSDB_CODE_INVALID_SUBMIT_MSG;
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+static int vnodeDoSubmitJob(SVnodeObj *pVnode, int import, int32_t *ssid, int32_t esid, SShellSubmitBlock **ppBlocks,
+                            TSKEY now, SShellObj *pObj) {
+  SShellSubmitBlock *pBlocks = *ppBlocks;
+  int code = TSDB_CODE_SUCCESS;
+  int32_t numOfPoints = 0;
+  int32_t i = 0;
+
+  for (i = *ssid; i < esid; i++) {
+    numOfPoints = 0;
+
+    code = vnodeCheckSubmitBlockContext(pBlocks, pVnode);
+    if (code != TSDB_CODE_SUCCESS) break;
+
+    SMeterObj *pMeterObj = (SMeterObj *)(pVnode->meterList[htonl(pBlocks->sid)]);
+
+    // dont include sid, vid
+    int32_t subMsgLen = sizeof(pBlocks->numOfRows) + htons(pBlocks->numOfRows) * pMeterObj->bytesPerPoint;
+    int32_t sversion = htonl(pBlocks->sversion);
+
+    if (import) {
+      code = vnodeImportPoints(pMeterObj, (char *)&(pBlocks->numOfRows), subMsgLen, TSDB_DATA_SOURCE_SHELL, pObj,
+                               sversion, &numOfPoints, now);
+      pObj->numOfTotalPoints += numOfPoints;
+
+      // records for one table should be consecutive located in the payload buffer, which is guaranteed by client
+      if (code == TSDB_CODE_SUCCESS) {
+        pObj->count--;
+      }
+    } else {
+      code = vnodeInsertPoints(pMeterObj, (char *)&(pBlocks->numOfRows), subMsgLen, TSDB_DATA_SOURCE_SHELL, NULL,
+                               sversion, &numOfPoints, now);
+      pObj->numOfTotalPoints += numOfPoints;
+    }
+
+    if (code != TSDB_CODE_SUCCESS) break;
+
+    pBlocks = (SShellSubmitBlock *)((char *)pBlocks + sizeof(SShellSubmitBlock) +
+                                    htons(pBlocks->numOfRows) * pMeterObj->bytesPerPoint);
+  }
+
+  *ssid = i;
+  *ppBlocks = pBlocks;
+
+  return code;
+}
+
 int vnodeProcessShellSubmitRequest(char *pMsg, int msgLen, SShellObj *pObj) {
   int              code = 0, ret = 0;
+  int32_t          i = 0;
   SShellSubmitMsg  shellSubmit = *(SShellSubmitMsg *)pMsg;
   SShellSubmitMsg *pSubmit = &shellSubmit;
+  SShellSubmitBlock *pBlocks = NULL;
 
   pSubmit->vnode = htons(pSubmit->vnode);
   pSubmit->numOfSid = htonl(pSubmit->numOfSid);
@@ -526,67 +612,69 @@ int vnodeProcessShellSubmitRequest(char *pMsg, int msgLen, SShellObj *pObj) {
 
   pObj->count = pSubmit->numOfSid;  // for import
   pObj->code = 0;                   // for import
-  pObj->numOfTotalPoints = 0;       // for import
-  SShellSubmitBlock *pBlocks = (SShellSubmitBlock *)(pMsg + sizeof(SShellSubmitMsg));
+  pObj->numOfTotalPoints = 0;
 
-  int32_t numOfPoints = 0;
-  int32_t numOfTotalPoints = 0;
-  // We take current time here to avoid it in the for loop.
   TSKEY   now = taosGetTimestamp(pVnode->cfg.precision);
 
-  for (int32_t i = 0; i < pSubmit->numOfSid; ++i) {
-    numOfPoints = 0;
-
-    pBlocks->sid = htonl(pBlocks->sid);
-    pBlocks->uid = htobe64(pBlocks->uid);
+  pBlocks = (SShellSubmitBlock *)(pMsg + sizeof(SShellSubmitMsg));
+  i = 0;
+  code = vnodeDoSubmitJob(pVnode, pSubmit->import, &i, pSubmit->numOfSid, &pBlocks, now, pObj);
 
-    if (pBlocks->sid >= pVnode->cfg.maxSessions || pBlocks->sid <= 0) {
-      dTrace("sid:%d is out of range", pBlocks->sid);
-      code = TSDB_CODE_INVALID_TABLE_ID;
-      goto _submit_over;
+_submit_over:
+  ret = 0;
+  if (pSubmit->import) {  // Import case
+    if (code == TSDB_CODE_ACTION_IN_PROGRESS) {
+
+      SBatchSubmitInfo *pSubmitInfo =
+          (SBatchSubmitInfo *)calloc(1, sizeof(SBatchSubmitInfo) + msgLen - sizeof(SShellSubmitMsg));
+      if (pSubmitInfo == NULL) {
+        code = TSDB_CODE_SERV_OUT_OF_MEMORY;
+        ret = vnodeSendShellSubmitRspMsg(pObj, code, pObj->numOfTotalPoints);
+      } else { // Start a timer to process the next part of request
+        pSubmitInfo->import = 1;
+        pSubmitInfo->vnode = pSubmit->vnode;
+        pSubmitInfo->numOfSid = pSubmit->numOfSid;
+        pSubmitInfo->ssid = i;   // start from this position, not the initial position
+        pSubmitInfo->pObj = pObj;
+        pSubmitInfo->offset = ((char *)pBlocks) - (pMsg + sizeof(SShellSubmitMsg));
+        assert(pSubmitInfo->offset >= 0);
+        memcpy((void *)(pSubmitInfo->blks), (void *)(pMsg + sizeof(SShellSubmitMsg)), msgLen - sizeof(SShellSubmitMsg));
+        taosTmrStart(vnodeProcessBatchSubmitTimer, 10, (void *)pSubmitInfo, vnodeTmrCtrl);
+      }
+    } else {
+      if (code == TSDB_CODE_SUCCESS) assert(pObj->count == 0);
+      ret = vnodeSendShellSubmitRspMsg(pObj, code, pObj->numOfTotalPoints);
     }
+  } else {  // Insert case
+    ret = vnodeSendShellSubmitRspMsg(pObj, code, pObj->numOfTotalPoints);
+  }
 
-    int vnode = pSubmit->vnode;
-    int sid = pBlocks->sid;
-
-    SMeterObj *pMeterObj = vnodeList[vnode].meterList[sid];
-    if (pMeterObj == NULL) {
-      dError("vid:%d sid:%d, not active table", vnode, sid);
-      vnodeSendMeterCfgMsg(vnode, sid);
-      code = TSDB_CODE_NOT_ACTIVE_TABLE;
-      goto _submit_over;
-    }
+  atomic_fetch_add_32(&vnodeInsertReqNum, 1);
+  return ret;
+}
 
-    if (pMeterObj->uid != pBlocks->uid) {
-      dError("vid:%d sid:%d, meterId:%s, uid:%lld, uid in msg:%lld, uid mismatch", vnode, sid, pMeterObj->meterId,
-             pMeterObj->uid, pBlocks->uid);
-      code = TSDB_CODE_INVALID_SUBMIT_MSG;
-      goto _submit_over;
-    }
+static void vnodeProcessBatchSubmitTimer(void *param, void *tmrId) {
+  SBatchSubmitInfo *pSubmitInfo = (SBatchSubmitInfo *)param;
+  assert(pSubmitInfo != NULL && pSubmitInfo->import);
 
-    // dont include sid, vid
-    int subMsgLen = sizeof(pBlocks->numOfRows) + htons(pBlocks->numOfRows) * pMeterObj->bytesPerPoint;
-    int sversion = htonl(pBlocks->sversion);
+  int32_t i = 0;
+  int32_t code = TSDB_CODE_SUCCESS;
 
-    if (pSubmit->import) {
-      code = vnodeImportPoints(pMeterObj, (char *) &(pBlocks->numOfRows), subMsgLen, TSDB_DATA_SOURCE_SHELL, pObj,
-                               sversion, &numOfPoints, now);
-    } else {
-      code = vnodeInsertPoints(pMeterObj, (char *) &(pBlocks->numOfRows), subMsgLen, TSDB_DATA_SOURCE_SHELL, NULL,
-                               sversion, &numOfPoints, now);
-    }
+  SShellObj *        pShell = pSubmitInfo->pObj;
+  SVnodeObj *        pVnode = &vnodeList[pSubmitInfo->vnode];
+  SShellSubmitBlock *pBlocks = (SShellSubmitBlock *)(pSubmitInfo->blks + pSubmitInfo->offset);
+  TSKEY   now = taosGetTimestamp(pVnode->cfg.precision);
+  i = pSubmitInfo->ssid;
 
-    if (code != TSDB_CODE_SUCCESS) {break;}
+  code = vnodeDoSubmitJob(pVnode, pSubmitInfo->import, &i, pSubmitInfo->numOfSid, &pBlocks, now, pShell);
 
-    numOfTotalPoints += numOfPoints;
-    pBlocks = (SShellSubmitBlock *)((char *)pBlocks + sizeof(SShellSubmitBlock) +
-                                    htons(pBlocks->numOfRows) * pMeterObj->bytesPerPoint);
+  if (code == TSDB_CODE_ACTION_IN_PROGRESS) {
+    pSubmitInfo->ssid = i;
+    pSubmitInfo->offset = ((char *)pBlocks) - pSubmitInfo->blks;
+    taosTmrStart(vnodeProcessBatchSubmitTimer, 10, (void *)pSubmitInfo, vnodeTmrCtrl);
+  } else {
+    if (code == TSDB_CODE_SUCCESS) assert(pShell->count == 0);
+    tfree(param);
+    vnodeSendShellSubmitRspMsg(pShell, code, pShell->numOfTotalPoints);
   }
-
-_submit_over:
-  // for import, send the submit response only when return code is not zero
-  if (pSubmit->import == 0 || code != 0) ret = vnodeSendShellSubmitRspMsg(pObj, code, numOfTotalPoints);
-
-  atomic_fetch_add_32(&vnodeInsertReqNum, 1);
-  return ret;
 }

src/util/src/textbuffer.c

@@ -21,6 +21,7 @@
 #include "tsqlfunction.h"
 #include "ttime.h"
 #include "ttypes.h"
+#include "tutil.h"
 
 #pragma GCC diagnostic ignored "-Wformat"
 
@@ -46,8 +47,7 @@ void getTmpfilePath(const char *fileNamePrefix, char *dstPath) {
   strcpy(tmpPath, tmpDir);
   strcat(tmpPath, tdengineTmpFileNamePrefix);
   strcat(tmpPath, fileNamePrefix);
-  strcat(tmpPath, "-%u-%u");
-
+  strcat(tmpPath, "-%llu-%u");
   snprintf(dstPath, MAX_TMPFILE_PATH_LENGTH, tmpPath, taosGetPthreadId(), atomic_add_fetch_32(&tmpFileSerialNum, 1));
 }
 
@@ -431,7 +431,8 @@ void tBucketIntHash(tMemBucket *pBucket, void *value, int16_t *segIdx, int16_t *
 }
 
 void tBucketDoubleHash(tMemBucket *pBucket, void *value, int16_t *segIdx, int16_t *slotIdx) {
-  double v = *(double *)value;
+  //double v = *(double *)value;
+  double v = GET_DOUBLE_VAL(value);
 
   if (pBucket->nRange.dMinVal == DBL_MAX) {
     /*
@@ -675,7 +676,8 @@ void tMemBucketUpdateBoundingBox(MinMaxEntry *r, char *data, int32_t dataType) {
       break;
     };
     case TSDB_DATA_TYPE_DOUBLE: {
-      double val = *(double *)data;
+      //double val = *(double *)data;
+      double val = GET_DOUBLE_VAL(data);
       if (r->dMinVal > val) {
         r->dMinVal = val;
       }
@@ -686,7 +688,8 @@ void tMemBucketUpdateBoundingBox(MinMaxEntry *r, char *data, int32_t dataType) {
       break;
     };
     case TSDB_DATA_TYPE_FLOAT: {
-      double val = *(float *)data;
+      //double val = *(float *)data;
+      double val = GET_FLOAT_VAL(data);
 
       if (r->dMinVal > val) {
         r->dMinVal = val;
@@ -734,12 +737,14 @@ void tMemBucketPut(tMemBucket *pBucket, void *data, int32_t numOfRows) {
         break;
       }
       case TSDB_DATA_TYPE_DOUBLE: {
-        double val = *(double *)d;
+        //double val = *(double *)d;
+        double val = GET_DOUBLE_VAL(d);
         (pBucket->HashFunc)(pBucket, &val, &segIdx, &slotIdx);
         break;
       }
       case TSDB_DATA_TYPE_FLOAT: {
-        double val = *(float *)d;
+        //double val = *(float *)d;
+        double val = GET_FLOAT_VAL(d);
         (pBucket->HashFunc)(pBucket, &val, &segIdx, &slotIdx);
         break;
       }
@@ -840,16 +845,20 @@ static FORCE_INLINE int32_t columnValueAscendingComparator(char *f1, char *f2, i
       return (first < second) ? -1 : 1;
     };
     case TSDB_DATA_TYPE_DOUBLE: {
-      double first = *(double *)f1;
-      double second = *(double *)f2;
+      //double first = *(double *)f1;
+      double first = GET_DOUBLE_VAL(f1);
+      //double second = *(double *)f2;
+      double second = GET_DOUBLE_VAL(f2);
       if (first == second) {
         return 0;
       }
       return (first < second) ? -1 : 1;
     };
     case TSDB_DATA_TYPE_FLOAT: {
-      float first = *(float *)f1;
-      float second = *(float *)f2;
+      //float first = *(float *)f1;
+      //float second = *(float *)f2;
+      float first = GET_FLOAT_VAL(f1);
+      float second = GET_FLOAT_VAL(f2);
       if (first == second) {
         return 0;
       }
@@ -1298,10 +1307,16 @@ double findOnlyResult(tMemBucket *pMemBucket) {
                 return *(int8_t *)pPage->data;
               case TSDB_DATA_TYPE_BIGINT:
                 return (double)(*(int64_t *)pPage->data);
-              case TSDB_DATA_TYPE_DOUBLE:
-                return *(double *)pPage->data;
-              case TSDB_DATA_TYPE_FLOAT:
-                return *(float *)pPage->data;
+              case TSDB_DATA_TYPE_DOUBLE: {
+		double dv = GET_DOUBLE_VAL(pPage->data);				  
+                //return *(double *)pPage->data;
+		return dv;
+	      }
+              case TSDB_DATA_TYPE_FLOAT: {
+		float fv = GET_FLOAT_VAL(pPage->data);
+                //return *(float *)pPage->data;
+		return fv;
+	      }
               default:
                 return 0;
             }
@@ -1788,13 +1803,17 @@ double getPercentileImpl(tMemBucket *pMemBucket, int32_t count, double fraction)
               break;
             };
             case TSDB_DATA_TYPE_FLOAT: {
-              td = *(float *)thisVal;
-              nd = *(float *)nextVal;
+              //td = *(float *)thisVal;
+              //nd = *(float *)nextVal;
+              td = GET_FLOAT_VAL(thisVal);
+              nd = GET_FLOAT_VAL(nextVal);
               break;
             }
             case TSDB_DATA_TYPE_DOUBLE: {
-              td = *(double *)thisVal;
-              nd = *(double *)nextVal;
+              //td = *(double *)thisVal;
+              td = GET_DOUBLE_VAL(thisVal);
+              //nd = *(double *)nextVal;
+              nd = GET_DOUBLE_VAL(nextVal);
               break;
             }
             case TSDB_DATA_TYPE_BIGINT: {
@@ -1831,15 +1850,17 @@ double getPercentileImpl(tMemBucket *pMemBucket, int32_t count, double fraction)
                 break;
               };
               case TSDB_DATA_TYPE_FLOAT: {
-                finalResult = *(float *)thisVal;
+                //finalResult = *(float *)thisVal;
+                finalResult = GET_FLOAT_VAL(thisVal);
                 break;
               }
               case TSDB_DATA_TYPE_DOUBLE: {
-                finalResult = *(double *)thisVal;
+                //finalResult = *(double *)thisVal;
+                finalResult = GET_DOUBLE_VAL(thisVal);
                 break;
               }
               case TSDB_DATA_TYPE_BIGINT: {
-                finalResult = (double)*(int64_t *)thisVal;
+                finalResult = (double)(*(int64_t *)thisVal);
                 break;
               }
             }