Merge pull request #1061 from taosdata/feature/lihui

[#1059 TBASE-1436]

src/client/src/tscParseInsert.c

@@ -36,7 +36,7 @@ enum {
   TSDB_USE_CLI_TS = 1,
 };
 
-static int32_t tscAllocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize);
+static int32_t tscAllocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int32_t * numOfRows);
 
 static int32_t tscToInteger(SSQLToken *pToken, int64_t *value, char **endPtr) {
   int32_t numType = isValidNumber(pToken);
@@ -522,14 +522,15 @@ int tsParseValues(char **str, STableDataBlocks *pDataBlock, SMeterMeta *pMeterMe
 
     *str += index;
     if (numOfRows >= maxRows || pDataBlock->size + pMeterMeta->rowSize >= pDataBlock->nAllocSize) {
-      int32_t tSize = tscAllocateMemIfNeed(pDataBlock, pMeterMeta->rowSize);
-      if (0 == tSize) {  //TODO pass the correct error code to client
+      int32_t tSize;
+      int32_t retcode = tscAllocateMemIfNeed(pDataBlock, pMeterMeta->rowSize, &tSize);
+      if (retcode != TSDB_CODE_SUCCESS) {  //TODO pass the correct error code to client
         strcpy(error, "client out of memory");
-        *code = TSDB_CODE_CLI_OUT_OF_MEMORY;
+        *code = retcode;
         return -1;
       }
-      
-      maxRows += tSize;
+      ASSERT(tSize > maxRows);
+      maxRows = tSize;
     }
 
     int32_t len = tsParseOneRowData(str, pDataBlock, pSchema, spd, error, precision, code, tmpTokenBuf);
@@ -574,7 +575,7 @@ static void tscSetAssignedColumnInfo(SParsedDataColInfo *spd, SSchema *pSchema,
   }
 }
 
-int32_t tscAllocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize) {
+int32_t tscAllocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int32_t * numOfRows) {
   size_t    remain = pDataBlock->nAllocSize - pDataBlock->size;
   const int factor = 5;
   uint32_t nAllocSizeOld = pDataBlock->nAllocSize;
@@ -594,11 +595,13 @@ int32_t tscAllocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize) {
       //assert(false);
       // do nothing
       pDataBlock->nAllocSize = nAllocSizeOld;
-      return 0;
+      *numOfRows = (int32_t)(pDataBlock->nAllocSize) / rowSize;
+      return TSDB_CODE_CLI_OUT_OF_MEMORY;
     }
   }
 
-  return (int32_t)(pDataBlock->nAllocSize - pDataBlock->size) / rowSize;
+  *numOfRows = (int32_t)(pDataBlock->nAllocSize) / rowSize;
+  return TSDB_CODE_SUCCESS;
 }
 
 static void tsSetBlockInfo(SShellSubmitBlock *pBlocks, const SMeterMeta *pMeterMeta, int32_t numOfRows) {
@@ -664,8 +667,9 @@ static int32_t doParseInsertStatement(SSqlObj *pSql, void *pTableHashList, char
     return ret;
   }
   
-  int32_t maxNumOfRows = tscAllocateMemIfNeed(dataBuf, pMeterMeta->rowSize);
-  if (0 == maxNumOfRows) {
+  int32_t maxNumOfRows;
+  ret = tscAllocateMemIfNeed(dataBuf, pMeterMeta->rowSize, &maxNumOfRows);
+  if (TSDB_CODE_SUCCESS != ret) {
     return TSDB_CODE_CLI_OUT_OF_MEMORY;
   }
   
@@ -1326,7 +1330,7 @@ static int tscInsertDataFromFile(SSqlObj *pSql, FILE *fp, char *tmpTokenBuf) {
   char *          line = NULL;
   size_t          n = 0;
   int             len = 0;
-  uint32_t        maxRows = 0;
+  int32_t         maxRows = 0;
   SSqlCmd *       pCmd = &pSql->cmd;
   int             numOfRows = 0;
   int32_t         code = 0;
@@ -1345,8 +1349,8 @@ static int tscInsertDataFromFile(SSqlObj *pSql, FILE *fp, char *tmpTokenBuf) {
   
   tscAppendDataBlock(pCmd->pDataBlocks, pTableDataBlock);
 
-  maxRows = tscAllocateMemIfNeed(pTableDataBlock, rowSize);
-  if (maxRows < 1) return -1;
+  code = tscAllocateMemIfNeed(pTableDataBlock, rowSize, &maxRows);
+  if (TSDB_CODE_SUCCESS != code) return -1;
 
   int                count = 0;
   SParsedDataColInfo spd = {.numOfCols = pMeterMetaInfo->pMeterMeta->numOfColumns};
@@ -1362,12 +1366,6 @@ static int tscInsertDataFromFile(SSqlObj *pSql, FILE *fp, char *tmpTokenBuf) {
     char *lineptr = line;
     strtolower(line, line);
 
-    if (numOfRows >= maxRows || pTableDataBlock->size + pMeterMeta->rowSize >= pTableDataBlock->nAllocSize) {
-      uint32_t tSize = tscAllocateMemIfNeed(pTableDataBlock, pMeterMeta->rowSize);
-      if (0 == tSize) return (-TSDB_CODE_CLI_OUT_OF_MEMORY);
-      maxRows += tSize;    
-    }
-
     len = tsParseOneRowData(&lineptr, pTableDataBlock, pSchema, &spd, pCmd->payload, pMeterMeta->precision, &code, tmpTokenBuf);
     if (len <= 0 || pTableDataBlock->numOfParams > 0) {
       pSql->res.code = code;

src/system/detail/src/vnodeMeter.c

@@ -563,8 +563,8 @@ int vnodeInsertPoints(SMeterObj *pObj, char *cont, int contLen, char source, voi
 
   if (numOfPoints >= (pVnode->cfg.blocksPerMeter - 2) * pObj->pointsPerBlock) {
     code = TSDB_CODE_BATCH_SIZE_TOO_BIG;
-    dError("vid:%d sid:%d id:%s, batch size too big, it shall be smaller than:%d", pObj->vnode, pObj->sid,
-           pObj->meterId, (pVnode->cfg.blocksPerMeter - 2) * pObj->pointsPerBlock);
+    dError("vid:%d sid:%d id:%s, batch size too big, insert points:%d, it shall be smaller than:%d", pObj->vnode, pObj->sid,
+           pObj->meterId, numOfPoints, (pVnode->cfg.blocksPerMeter - 2) * pObj->pointsPerBlock);
     return code;
   }