diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 1902e3aa25a31a9a2f71bd489004e17c4fe04d6d..53c43f189258ab76f0de2beb6ad1246e24a5579e 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -44,12 +44,6 @@ static int32_t parseBoundColumns(SInsertStatementParam *pInsertParam, SParsedDat int initMemRowBuilder(SMemRowBuilder *pBuilder, uint32_t nRows, uint32_t nCols, uint32_t nBoundCols, int32_t allNullLen) { ASSERT(nRows >= 0 && nCols > 0 && (nBoundCols <= nCols)); - - if (tsForceDataRow) { - pBuilder->memRowType = SMEM_ROW_DATA; - return TSDB_CODE_SUCCESS; - } - if (nRows > 0) { // already init(bind multiple rows by single column) if (pBuilder->compareStat == ROW_COMPARE_NEED && (pBuilder->rowInfo != NULL)) { @@ -452,13 +446,13 @@ int32_t tsCheckTimestamp(STableDataBlocks *pDataBlocks, const char *start) { return TSDB_CODE_SUCCESS; } -int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, int16_t timePrec, int32_t *len, char *tmpTokenBuf, - SInsertStatementParam *pInsertParam) { +int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, int16_t timePrec, bool *isConverted, int32_t rowSize, + char *tmpTokenBuf, SInsertStatementParam *pInsertParam) { int32_t tsc_index = 0; SStrToken sToken = {0}; - char *row = pDataBlocks->pData + pDataBlocks->size; // skip the SSubmitBlk header - + char *row = pDataBlocks->pData + + ((*isConverted) ? (pDataBlocks->size - rowSize) : pDataBlocks->size); // skip the SSubmitBlk header SParsedDataColInfo *spd = &pDataBlocks->boundColumnInfo; STableMeta * pTableMeta = pDataBlocks->pTableMeta; SSchema * schema = tscGetTableSchema(pTableMeta); @@ -467,6 +461,7 @@ int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, int16_t timePrec, i int32_t kvLen = pBuilder->kvRowInitLen; bool isParseBindParam = false; + *isConverted = false; initSMemRow(row, pBuilder->memRowType, pDataBlocks, spd->numOfBound); // 1. set the parsed value from sql string @@ -571,9 +566,15 @@ int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, int16_t timePrec, i } } } + + // 4. perform the convert + if (isNeedConvertRow(row)) { + convertSMemRow(row + rowSize, row, pDataBlocks); + *isConverted = true; + } } - *len = getExtendedRowSize(pDataBlocks); + // *len = getExtendedRowSize(pDataBlocks); return TSDB_CODE_SUCCESS; } @@ -631,13 +632,15 @@ int32_t tsParseValues(char **str, STableDataBlocks *pDataBlock, int maxRows, SIn pDataBlock->boundColumnInfo.allNullLen))) { return code; } + bool isConverted = false; while (1) { tsc_index = 0; sToken = tStrGetToken(*str, &tsc_index, false); if (sToken.n == 0 || sToken.type != TK_LP) break; *str += tsc_index; - if ((*numOfRows) >= maxRows || pDataBlock->size + extendedRowSize >= pDataBlock->nAllocSize) { + // allocate 1 more row size to facilitate the SDataRow/SKVRow convert + if ((*numOfRows + 1) >= maxRows || pDataBlock->size + extendedRowSize >= pDataBlock->nAllocSize) { int32_t tSize; code = tscAllocateMemIfNeed(pDataBlock, extendedRowSize, &tSize); if (code != TSDB_CODE_SUCCESS) { //TODO pass the correct error code to client @@ -645,17 +648,18 @@ int32_t tsParseValues(char **str, STableDataBlocks *pDataBlock, int maxRows, SIn return TSDB_CODE_TSC_OUT_OF_MEMORY; } - ASSERT(tSize >= maxRows); + ASSERT(tSize > maxRows); // 1 more row allocated maxRows = tSize; } - int32_t len = 0; - code = tsParseOneRow(str, pDataBlock, precision, &len, tmpTokenBuf, pInsertParam); + // int32_t len = 0; + + code = tsParseOneRow(str, pDataBlock, precision, &isConverted, extendedRowSize, tmpTokenBuf, pInsertParam); if (code != TSDB_CODE_SUCCESS) { // error message has been set in tsParseOneRow, return directly return TSDB_CODE_TSC_SQL_SYNTAX_ERROR; } - pDataBlock->size += len; + pDataBlock->size += extendedRowSize; tsc_index = 0; sToken = tStrGetToken(*str, &tsc_index, false); @@ -667,6 +671,10 @@ int32_t tsParseValues(char **str, STableDataBlocks *pDataBlock, int maxRows, SIn (*numOfRows)++; } + if (isConverted) { + void *convertedSMemRow = pDataBlock->pData + pDataBlock->size; + memcpy(convertedSMemRow - extendedRowSize, convertedSMemRow, extendedRowSize); + } if ((*numOfRows) <= 0) { strcpy(pInsertParam->msg, "no any data points"); @@ -1728,13 +1736,19 @@ static void parseFileSendDataBlock(void *param, TAOS_RES *tres, int32_t numOfRow goto _error; } - tscAllocateMemIfNeed(pTableDataBlock, getExtendedRowSize(pTableDataBlock), &maxRows); + int32_t extendedRowSize = getExtendedRowSize(pTableDataBlock); + + tscAllocateMemIfNeed(pTableDataBlock, extendedRowSize, &maxRows); tokenBuf = calloc(1, TSDB_MAX_BYTES_PER_ROW); if (tokenBuf == NULL) { code = TSDB_CODE_TSC_OUT_OF_MEMORY; goto _error; } + --maxRows; // 1 more row needed to facilitate the SDataRow/SKVRow convert + ASSERT(maxRows > 0); + + bool isConverted = false; while ((readLen = tgetline(&line, &n, fp)) != -1) { if (('\r' == line[readLen - 1]) || ('\n' == line[readLen - 1])) { line[--readLen] = 0; @@ -1747,19 +1761,25 @@ static void parseFileSendDataBlock(void *param, TAOS_RES *tres, int32_t numOfRow char *lineptr = line; strtolower(line, line); - int32_t len = 0; - code = tsParseOneRow(&lineptr, pTableDataBlock, tinfo.precision, &len, tokenBuf, pInsertParam); + // int32_t len = 0; + + code = tsParseOneRow(&lineptr, pTableDataBlock, tinfo.precision, &isConverted, extendedRowSize, tokenBuf, + pInsertParam); if (code != TSDB_CODE_SUCCESS || pTableDataBlock->numOfParams > 0) { pSql->res.code = code; break; } - pTableDataBlock->size += len; + pTableDataBlock->size += extendedRowSize; if (++count >= maxRows) { break; } } + if (isConverted) { + void *convertedSMemRow = pTableDataBlock->pData + pTableDataBlock->size; + memcpy(convertedSMemRow - extendedRowSize, convertedSMemRow, extendedRowSize); + } tfree(tokenBuf); tfree(line); diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 5986a886ad47fa14abba8328c3c3a2f9b38d75e5..b47faa32d9100ff33139137b0ba05e2594469a9f 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1943,18 +1943,11 @@ static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, SI } } else { for (int32_t i = 0; i < numOfRows; ++i) { - char* payload = (blkKeyTuple + i)->payloadAddr; - if (isNeedConvertRow(payload)) { - convertSMemRow(pDataBlock, payload, pTableDataBlock); - TDRowTLenT rowTLen = memRowTLen(pDataBlock); - pDataBlock = POINTER_SHIFT(pDataBlock, rowTLen); - pBlock->dataLen += rowTLen; - } else { - TDRowTLenT rowTLen = memRowTLen(payload); - memcpy(pDataBlock, payload, rowTLen); - pDataBlock = POINTER_SHIFT(pDataBlock, rowTLen); - pBlock->dataLen += rowTLen; - } + char* payload = (blkKeyTuple + i)->payloadAddr; + TDRowTLenT rowTLen = memRowTLen(payload); + memcpy(pDataBlock, payload, rowTLen); + pDataBlock = POINTER_SHIFT(pDataBlock, rowTLen); + pBlock->dataLen += rowTLen; } } diff --git a/src/common/inc/tglobal.h b/src/common/inc/tglobal.h index b2fed24aaf0a89dbea2db11401191a3bf08e7a95..ffa89b8c39bea339e7916cc027232e88bfbaa44b 100644 --- a/src/common/inc/tglobal.h +++ b/src/common/inc/tglobal.h @@ -46,7 +46,6 @@ extern int32_t tsDnodeId; extern int tsRpcTimer; extern int tsRpcMaxTime; extern int tsRpcForceTcp; // all commands go to tcp protocol if this is enabled -extern int32_t tsForceDataRow; // use SDataRow forcibly extern int32_t tsMaxConnections; extern int32_t tsMaxShellConns; extern int32_t tsShellActivityTimer; diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index 7d6db5fde46b3ed58bec4c59cf7091c0568926cf..50d3fcd1a914bcb6af2629334a3f94daf7d26101 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -54,7 +54,6 @@ int32_t tsDnodeId = 0; int32_t tsRpcTimer = 300; int32_t tsRpcMaxTime = 600; // seconds; int32_t tsRpcForceTcp = 0; // disable this, means query, show command use udp protocol as default -int32_t tsForceDataRow = 0; // disable this at default int32_t tsMaxShellConns = 50000; int32_t tsMaxConnections = 5000; int32_t tsShellActivityTimer = 3; // second @@ -692,16 +691,6 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); - cfg.option = "forceDataRow"; - cfg.ptr = &tsForceDataRow; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 0; - cfg.maxValue = 1; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosInitConfigOption(cfg); - cfg.option = "statusInterval"; cfg.ptr = &tsStatusInterval; cfg.valType = TAOS_CFG_VTYPE_INT32; diff --git a/src/util/inc/tconfig.h b/src/util/inc/tconfig.h index b3d7fcd0a094f809e2cad7d65c6399ce1bab6c00..e2165ec1a32c3b47a47a4b22243d1ac178d58876 100644 --- a/src/util/inc/tconfig.h +++ b/src/util/inc/tconfig.h @@ -20,7 +20,7 @@ extern "C" { #endif -#define TSDB_CFG_MAX_NUM 123 +#define TSDB_CFG_MAX_NUM 122 #define TSDB_CFG_PRINT_LEN 23 #define TSDB_CFG_OPTION_LEN 24 #define TSDB_CFG_VALUE_LEN 41