diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 7cb9c513e5e504212ac2ec47e3b1d93a21a6279f..203ba90c5f687709fe173fc7352e15844293bc60 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -280,6 +280,7 @@ typedef struct { SResRec * pGroupRec; char * data; void ** tsrow; + int32_t* length; // length for each field for current row char ** buffer; // Buffer used to put multibytes encoded using unicode (wchar_t) SColumnIndex * pColumnIndex; struct SLocalReducer *pLocalReducer; @@ -421,7 +422,7 @@ int32_t tscInvalidSQLErrMsg(char *msg, const char *additionalInfo, const char *s void tscQueueAsyncFreeResult(SSqlObj *pSql); int32_t tscToSQLCmd(SSqlObj *pSql, struct SSqlInfo *pInfo); -char * tscGetResultColumnChr(SSqlRes *pRes, SQueryInfo *pQueryInfo, int32_t column, int16_t bytes); +void tscGetResultColumnChr(SSqlRes *pRes, SFieldInfo* pFieldInfo, int32_t column); extern void * pVnodeConn; extern void * tscCacheHandle; diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index b64c1ed8c0446c26276f90846ed5883063f047f0..9f207936df234ec87a5f8e9c74b6aa2e3ef0db3e 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -317,7 +317,7 @@ void tscProcessFetchRow(SSchedMsg *pMsg) { SFieldSupInfo* pSup = taosArrayGet(pQueryInfo->fieldsInfo.pSupportInfo, i); if (pSup->pSqlExpr != NULL) { - pRes->tsrow[i] = tscGetResultColumnChr(pRes, pQueryInfo, i, pSup->pSqlExpr->resBytes); + tscGetResultColumnChr(pRes, &pQueryInfo->fieldsInfo, i); } else { // todo add } diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 8f40cf6fb9fbbfeb191e720f946fcc34890df5ee..2ce3748dd1d9ae102c7d6a43ba25ceb84ac5f76e 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -310,7 +310,7 @@ int32_t tsParseOneColumnData(SSchema *pSchema, SSQLToken *pToken, char *payload, *payload = TSDB_DATA_BINARY_NULL; } else { // too long values will return invalid sql, not be truncated automatically - if (pToken->n > pSchema->bytes) { + if (pToken->n + VARSTR_HEADER_SIZE > pSchema->bytes) { return tscInvalidSQLErrMsg(msg, "string data overflow", pToken->z); } @@ -328,7 +328,7 @@ int32_t tsParseOneColumnData(SSchema *pSchema, SSQLToken *pToken, char *payload, } else { // if the converted output len is over than pColumnModel->bytes, return error: 'Argument list too long' int32_t resLen = -1; - if (!taosMbsToUcs4(pToken->z, pToken->n, payload + VARSTR_HEADER_SIZE, pSchema->bytes, &resLen)) { + if (!taosMbsToUcs4(pToken->z, pToken->n, payload + VARSTR_HEADER_SIZE, pSchema->bytes - VARSTR_HEADER_SIZE, &resLen)) { char buf[512] = {0}; snprintf(buf, 512, "%s", strerror(errno)); diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index aff4ad525b4c203010adebe5422427b89da28a5c..87247d874bf358e0764a942693b02924fe69c53f 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -425,7 +425,7 @@ int taos_fetch_block_impl(TAOS_RES *res, TAOS_ROW *rows) { assert(0); for (int i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { - pRes->tsrow[i] = tscGetResultColumnChr(pRes, pQueryInfo, i, 0); + tscGetResultColumnChr(pRes, &pQueryInfo->fieldsInfo, i); } *rows = pRes->tsrow; @@ -725,6 +725,15 @@ char *taos_get_server_info(TAOS *taos) { return pObj->sversion; } +int* taos_fetch_lengths(TAOS_RES *res) { + SSqlObj* pSql = (SSqlObj* ) res; + if (pSql == NULL || pSql->signature != pSql) { + return NULL; + } + + return pSql->res.length; +} + char *taos_get_client_info() { return version; } void taos_stop_query(TAOS_RES *res) { @@ -796,7 +805,7 @@ int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields) case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: { size_t xlen = 0; - for (xlen = 0; xlen <= fields[i].bytes; xlen++) { + for (xlen = 0; xlen < fields[i].bytes - VARSTR_HEADER_SIZE; xlen++) { char c = ((char *)row[i])[xlen]; if (c == 0) break; str[len++] = c; diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 9aceecd377cd682442a63f6a6f51826dca723e74..c3d6c0f5fc3b1a59b7caa879e0d4445ed1b10bf6 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -1849,6 +1849,7 @@ void tscBuildResFromSubqueries(SSqlObj *pSql) { if (pRes->tsrow == NULL) { pRes->tsrow = calloc(numOfExprs, POINTER_BYTES); + pRes->length = calloc(numOfExprs, sizeof(int32_t)); } bool success = false; @@ -1967,7 +1968,7 @@ void **doSetResultRowData(SSqlObj *pSql, bool finalResult) { for (int i = 0; i < tscNumOfFields(pQueryInfo); ++i) { SFieldSupInfo* pSup = tscFieldInfoGetSupp(&pQueryInfo->fieldsInfo, i); if (pSup->pSqlExpr != NULL) { - pRes->tsrow[i] = tscGetResultColumnChr(pRes, pQueryInfo, i, pSup->pSqlExpr->resBytes); + tscGetResultColumnChr(pRes, &pQueryInfo->fieldsInfo, i); } // primary key column cannot be null in interval query, no need to check diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 4844ab3db86bdaa25700a5b759cf2d4ea61527d9..d49fff1583094448e0c9dd6ae7f2346e447c14a8 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -210,7 +210,7 @@ bool tscNonOrderedProjectionQueryOnSTable(SQueryInfo* pQueryInfo, int32_t tableI return false; } - // order by column exists, not a non-ordered projection query + // order by columnIndex exists, not a non-ordered projection query return pQueryInfo->order.orderColId < 0; } @@ -219,7 +219,7 @@ bool tscOrderedProjectionQueryOnSTable(SQueryInfo* pQueryInfo, int32_t tableInde return false; } - // order by column exists, a non-ordered projection query + // order by columnIndex exists, a non-ordered projection query return pQueryInfo->order.orderColId >= 0; } @@ -286,13 +286,15 @@ int32_t tscCreateResPointerInfo(SSqlRes* pRes, SQueryInfo* pQueryInfo) { int32_t numOfOutput = pQueryInfo->fieldsInfo.numOfOutput; pRes->numOfCols = numOfOutput; - pRes->tsrow = calloc(POINTER_BYTES, numOfOutput); - pRes->buffer = calloc(POINTER_BYTES, numOfOutput); + pRes->tsrow = calloc(numOfOutput, POINTER_BYTES); + pRes->length = calloc(numOfOutput, sizeof(int32_t)); // todo refactor + pRes->buffer = calloc(numOfOutput, POINTER_BYTES); // not enough memory if (pRes->tsrow == NULL || (pRes->buffer == NULL && pRes->numOfCols > 0)) { tfree(pRes->tsrow); tfree(pRes->buffer); + tfree(pRes->length); pRes->code = TSDB_CODE_CLI_OUT_OF_MEMORY; return pRes->code; @@ -312,6 +314,7 @@ void tscDestroyResPointerInfo(SSqlRes* pRes) { tfree(pRes->pRsp); tfree(pRes->tsrow); + tfree(pRes->length); tfree(pRes->pGroupRec); tfree(pRes->pColumnIndex); @@ -592,7 +595,7 @@ int32_t tscGetDataBlockFromList(void* pHashList, SDataBlockList* pDataBlockList, } static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock) { - // TODO: optimize this function + // TODO: optimize this function, handle the case while binary is not presented int len = 0; STableMeta* pTableMeta = pTableDataBlock->pTableMeta; @@ -924,7 +927,7 @@ static SSqlExpr* doBuildSqlExpr(SQueryInfo* pQueryInfo, int16_t functionId, SCol SSqlExpr* pExpr = calloc(1, sizeof(SSqlExpr)); pExpr->functionId = functionId; - // set the correct column index + // set the correct columnIndex index if (pColIndex->columnIndex == TSDB_TBNAME_COLUMN_INDEX) { pExpr->colInfo.colId = TSDB_TBNAME_COLUMN_INDEX; } else { @@ -1063,7 +1066,7 @@ void tscSqlExprCopy(SArray* dst, const SArray* src, uint64_t uid, bool deepcopy) } SColumn* tscColumnListInsert(SArray* pColumnList, SColumnIndex* pColIndex) { - // ignore the tbname column to be inserted into source list + // ignore the tbname columnIndex to be inserted into source list if (pColIndex->columnIndex < 0) { return NULL; } @@ -2124,22 +2127,30 @@ void tscTryQueryNextClause(SSqlObj* pSql, void (*queryFp)()) { } } -char* tscGetResultColumnChr(SSqlRes* pRes, SQueryInfo* pQueryInfo, int32_t column, int16_t bytes) { - SFieldInfo* pFieldInfo = &pQueryInfo->fieldsInfo; - SFieldSupInfo* pInfo = tscFieldInfoGetSupp(pFieldInfo, column); - +void tscGetResultColumnChr(SSqlRes* pRes, SFieldInfo* pFieldInfo, int32_t columnIndex) { + SFieldSupInfo* pInfo = tscFieldInfoGetSupp(pFieldInfo, columnIndex); + assert(pInfo->pSqlExpr != NULL); + int32_t type = pInfo->pSqlExpr->resType; + int32_t bytes = pInfo->pSqlExpr->resBytes; + char* pData = ((char*) pRes->data) + pInfo->pSqlExpr->offset * pRes->numOfRows + bytes * pRes->row; if (type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_BINARY) { int32_t realLen = varDataLen(pData); + assert(realLen <= bytes - VARSTR_HEADER_SIZE); + if (realLen < pInfo->pSqlExpr->resBytes - VARSTR_HEADER_SIZE) { // todo refactor *(char*) (pData + realLen + VARSTR_HEADER_SIZE) = 0; } - return pData + VARSTR_HEADER_SIZE; // head is the length of binary/nchar data + pRes->tsrow[columnIndex] = pData + VARSTR_HEADER_SIZE; + pRes->length[columnIndex] = realLen; } else { - return pData; + assert(bytes == tDataTypeDesc[type].nSize); + + pRes->tsrow[columnIndex] = pData; + pRes->length[columnIndex] = bytes; } } diff --git a/src/common/inc/tdataformat.h b/src/common/inc/tdataformat.h index 78b6cb73b2bc0e960fc4ec963a55d82e2d70c852..10ca75f785f4ab4c0f88d54ff17ab41fa21bcd8a 100644 --- a/src/common/inc/tdataformat.h +++ b/src/common/inc/tdataformat.h @@ -32,7 +32,7 @@ extern "C" { #define STR_WITH_MAXSIZE_TO_VARSTR(x, str, _maxs) do {\ char* _e = stpncpy((char*)(x) + VARSTR_HEADER_SIZE, (str), (_maxs));\ - *(VarDataLenT*)(x) = _e - (x);\ + *(VarDataLenT*)(x) = (_e - (x) - VARSTR_HEADER_SIZE);\ } while(0) #define STR_WITH_SIZE_TO_VARSTR(x, str, _size) do {\ diff --git a/src/dnode/src/dnodeWrite.c b/src/dnode/src/dnodeWrite.c index babbcf4ae8aa2bcaa1f6f3dc647c5cede2149567..39757c690f6d460fab2378c249f9ce9786ac3e22 100644 --- a/src/dnode/src/dnodeWrite.c +++ b/src/dnode/src/dnodeWrite.c @@ -228,7 +228,7 @@ static void dnodeHandleIdleWorker(SWriteWorker *pWorker) { int32_t num = taosGetQueueNumber(pWorker->qset); if (num > 0) { - usleep(30); + usleep(30000); sched_yield(); } else { taosFreeQall(pWorker->qall); diff --git a/src/inc/taos.h b/src/inc/taos.h index d4f1b8f48c9b31c32acd630b8419338bbc54aed2..2f23b10a614f9ade159889d8f99e7d40092844c7 100644 --- a/src/inc/taos.h +++ b/src/inc/taos.h @@ -53,9 +53,9 @@ typedef enum { } TSDB_OPTION; typedef struct taosField { - char name[64]; - short bytes; - char type; + char name[64]; + short bytes; + uint8_t type; } TAOS_FIELD; #ifdef _TD_GO_DLL_ @@ -104,6 +104,8 @@ DLL_EXPORT void taos_stop_query(TAOS_RES *res); int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows); int taos_validate_sql(TAOS *taos, const char *sql); +int* taos_fetch_lengths(TAOS_RES *res); + // TAOS_RES *taos_list_tables(TAOS *mysql, const char *wild); // TAOS_RES *taos_list_dbs(TAOS *mysql, const char *wild); diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index aa4889ec9d7e22c15b58420356096351327c2cc8..e60124cf0a1f180249e1178646a6d5daf3da97fd 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -36,14 +36,17 @@ extern "C" { typedef int32_t VarDataOffsetT; typedef int16_t VarDataLenT; +#define VARSTR_HEADER_SIZE sizeof(VarDataLenT) + #define varDataLen(v) ((VarDataLenT *)(v))[0] #define varDataTLen(v) (sizeof(VarDataLenT) + varDataLen(v)) #define varDataVal(v) ((void *)((char *)v + sizeof(VarDataLenT))) #define varDataCopy(dst, v) memcpy((dst), (void*) (v), varDataTLen(v)) +#define varDataLenByData(v) (*(VarDataLenT *)(((char*)(v)) - VARSTR_HEADER_SIZE)) // this data type is internally used only in 'in' query to hold the values #define TSDB_DATA_TYPE_ARRAY (TSDB_DATA_TYPE_NCHAR + 1) -#define VARSTR_HEADER_SIZE sizeof(VarDataLenT) + // Bytes for each type. extern const int32_t TYPE_BYTES[11]; diff --git a/src/kit/shell/src/shellEngine.c b/src/kit/shell/src/shellEngine.c index eeaeec83f2a4d437d310f02a0753ff4dedbd8e4e..f9ea5f4777f7f1f97221e67ff1a3741347645d82 100644 --- a/src/kit/shell/src/shellEngine.c +++ b/src/kit/shell/src/shellEngine.c @@ -350,6 +350,8 @@ int shellDumpResult(TAOS *con, char *fname, int *error_no, bool printMode) { TAOS_FIELD *fields = taos_fetch_fields(result); row = taos_fetch_row(result); + int32_t* length = taos_fetch_lengths(result); + char t_str[TSDB_MAX_BYTES_PER_ROW] = "\0"; int l[TSDB_MAX_COLUMNS] = {0}; int maxLenColumnName = 0; @@ -457,7 +459,7 @@ int shellDumpResult(TAOS *con, char *fname, int *error_no, bool printMode) { case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: memset(t_str, 0, TSDB_MAX_BYTES_PER_ROW); - memcpy(t_str, row[i], fields[i].bytes); + memcpy(t_str, row[i], length[i]); /* printf("%-*s|",max(fields[i].bytes, strlen(fields[i].name)), * t_str); */ /* printf("%-*s|", l[i], t_str); */ @@ -532,7 +534,8 @@ int shellDumpResult(TAOS *con, char *fname, int *error_no, bool printMode) { case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: memset(t_str, 0, TSDB_MAX_BYTES_PER_ROW); - memcpy(t_str, row[i], fields[i].bytes); + memcpy(t_str, row[i], length[i]); + l[i] = MAX(fields[i].bytes, strlen(fields[i].name)); shellPrintNChar(t_str, l[i], printMode); break; @@ -610,7 +613,7 @@ int shellDumpResult(TAOS *con, char *fname, int *error_no, bool printMode) { case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: memset(t_str, 0, TSDB_MAX_BYTES_PER_ROW); - memcpy(t_str, row[i], fields[i].bytes); + memcpy(t_str, row[i], length[i]); fprintf(fp, "\'%s\'", t_str); break; case TSDB_DATA_TYPE_TIMESTAMP: diff --git a/src/mnode/src/mgmtDb.c b/src/mnode/src/mgmtDb.c index 9ddf647d74a0c55b00801a1510c37dca9ceed561..9eb8b8dce197b603d6f73b0b9c04afbae2392e44 100644 --- a/src/mnode/src/mgmtDb.c +++ b/src/mnode/src/mgmtDb.c @@ -672,7 +672,7 @@ static int32_t mgmtRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void * pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; char *prec = (pDb->cfg.precision == TSDB_TIME_PRECISION_MILLI) ? TSDB_TIME_PRECISION_MILLI_STR : TSDB_TIME_PRECISION_MICRO_STR; - strcpy(pWrite, prec); + STR_WITH_SIZE_TO_VARSTR(pWrite, prec, 2); cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; diff --git a/tests/tsim/src/simExe.c b/tests/tsim/src/simExe.c index aea295c563cef60b82444d31b38872a9839ce190..d0d798343a433b93f157ce60671d7c65488dbc75 100644 --- a/tests/tsim/src/simExe.c +++ b/tests/tsim/src/simExe.c @@ -676,6 +676,8 @@ bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) { while ((row = taos_fetch_row(result))) { if (numOfRows < MAX_QUERY_ROW_NUM) { TAOS_FIELD *fields = taos_fetch_fields(result); + int* length = taos_fetch_lengths(result); + for (int i = 0; i < num_fields; i++) { char *value = NULL; if (i < MAX_QUERY_COL_NUM) { @@ -734,8 +736,8 @@ bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) { case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: memset(value, 0, MAX_QUERY_VALUE_LEN); - memcpy(value, row[i], fields[i].bytes); - value[fields[i].bytes] = 0; + memcpy(value, row[i], length[i]); + value[length[i]] = 0; // snprintf(value, fields[i].bytes, "%s", (char *)row[i]); break; case TSDB_DATA_TYPE_TIMESTAMP: