From f6067e1b31ab5eebedb3b89bb339aa198365f228 Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Tue, 5 May 2020 16:02:58 +0800 Subject: [PATCH] [td-168] fix bug in var string handling --- src/client/inc/tsclient.h | 3 ++- src/client/src/tscAsync.c | 2 +- src/client/src/tscSql.c | 13 ++++++++++-- src/client/src/tscSubquery.c | 3 ++- src/client/src/tscUtil.c | 35 +++++++++++++++++++++------------ src/common/inc/tdataformat.h | 2 +- src/dnode/src/dnodeWrite.c | 2 +- src/inc/taos.h | 8 +++++--- src/inc/taosdef.h | 5 ++++- src/kit/shell/src/shellEngine.c | 7 +++++-- 10 files changed, 54 insertions(+), 26 deletions(-) diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 7cb9c513e5..203ba90c5f 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -280,6 +280,7 @@ typedef struct { SResRec * pGroupRec; char * data; void ** tsrow; + int32_t* length; // length for each field for current row char ** buffer; // Buffer used to put multibytes encoded using unicode (wchar_t) SColumnIndex * pColumnIndex; struct SLocalReducer *pLocalReducer; @@ -421,7 +422,7 @@ int32_t tscInvalidSQLErrMsg(char *msg, const char *additionalInfo, const char *s void tscQueueAsyncFreeResult(SSqlObj *pSql); int32_t tscToSQLCmd(SSqlObj *pSql, struct SSqlInfo *pInfo); -char * tscGetResultColumnChr(SSqlRes *pRes, SQueryInfo *pQueryInfo, int32_t column, int16_t bytes); +void tscGetResultColumnChr(SSqlRes *pRes, SFieldInfo* pFieldInfo, int32_t column); extern void * pVnodeConn; extern void * tscCacheHandle; diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index b64c1ed8c0..9f207936df 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -317,7 +317,7 @@ void tscProcessFetchRow(SSchedMsg *pMsg) { SFieldSupInfo* pSup = taosArrayGet(pQueryInfo->fieldsInfo.pSupportInfo, i); if (pSup->pSqlExpr != NULL) { - pRes->tsrow[i] = tscGetResultColumnChr(pRes, pQueryInfo, i, pSup->pSqlExpr->resBytes); + tscGetResultColumnChr(pRes, &pQueryInfo->fieldsInfo, i); } else { // todo add } diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index aff4ad525b..87247d874b 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -425,7 +425,7 @@ int taos_fetch_block_impl(TAOS_RES *res, TAOS_ROW *rows) { assert(0); for (int i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { - pRes->tsrow[i] = tscGetResultColumnChr(pRes, pQueryInfo, i, 0); + tscGetResultColumnChr(pRes, &pQueryInfo->fieldsInfo, i); } *rows = pRes->tsrow; @@ -725,6 +725,15 @@ char *taos_get_server_info(TAOS *taos) { return pObj->sversion; } +int* taos_fetch_lengths(TAOS_RES *res) { + SSqlObj* pSql = (SSqlObj* ) res; + if (pSql == NULL || pSql->signature != pSql) { + return NULL; + } + + return pSql->res.length; +} + char *taos_get_client_info() { return version; } void taos_stop_query(TAOS_RES *res) { @@ -796,7 +805,7 @@ int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields) case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: { size_t xlen = 0; - for (xlen = 0; xlen <= fields[i].bytes; xlen++) { + for (xlen = 0; xlen < fields[i].bytes - VARSTR_HEADER_SIZE; xlen++) { char c = ((char *)row[i])[xlen]; if (c == 0) break; str[len++] = c; diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 9aceecd377..c3d6c0f5fc 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -1849,6 +1849,7 @@ void tscBuildResFromSubqueries(SSqlObj *pSql) { if (pRes->tsrow == NULL) { pRes->tsrow = calloc(numOfExprs, POINTER_BYTES); + pRes->length = calloc(numOfExprs, sizeof(int32_t)); } bool success = false; @@ -1967,7 +1968,7 @@ void **doSetResultRowData(SSqlObj *pSql, bool finalResult) { for (int i = 0; i < tscNumOfFields(pQueryInfo); ++i) { SFieldSupInfo* pSup = tscFieldInfoGetSupp(&pQueryInfo->fieldsInfo, i); if (pSup->pSqlExpr != NULL) { - pRes->tsrow[i] = tscGetResultColumnChr(pRes, pQueryInfo, i, pSup->pSqlExpr->resBytes); + tscGetResultColumnChr(pRes, &pQueryInfo->fieldsInfo, i); } // primary key column cannot be null in interval query, no need to check diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 4844ab3db8..76afe3002d 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -210,7 +210,7 @@ bool tscNonOrderedProjectionQueryOnSTable(SQueryInfo* pQueryInfo, int32_t tableI return false; } - // order by column exists, not a non-ordered projection query + // order by columnIndex exists, not a non-ordered projection query return pQueryInfo->order.orderColId < 0; } @@ -219,7 +219,7 @@ bool tscOrderedProjectionQueryOnSTable(SQueryInfo* pQueryInfo, int32_t tableInde return false; } - // order by column exists, a non-ordered projection query + // order by columnIndex exists, a non-ordered projection query return pQueryInfo->order.orderColId >= 0; } @@ -286,13 +286,15 @@ int32_t tscCreateResPointerInfo(SSqlRes* pRes, SQueryInfo* pQueryInfo) { int32_t numOfOutput = pQueryInfo->fieldsInfo.numOfOutput; pRes->numOfCols = numOfOutput; - pRes->tsrow = calloc(POINTER_BYTES, numOfOutput); - pRes->buffer = calloc(POINTER_BYTES, numOfOutput); + pRes->tsrow = calloc(numOfOutput, POINTER_BYTES); + pRes->length = calloc(numOfOutput, sizeof(int32_t)); // todo refactor + pRes->buffer = calloc(numOfOutput, POINTER_BYTES); // not enough memory if (pRes->tsrow == NULL || (pRes->buffer == NULL && pRes->numOfCols > 0)) { tfree(pRes->tsrow); tfree(pRes->buffer); + tfree(pRes->length); pRes->code = TSDB_CODE_CLI_OUT_OF_MEMORY; return pRes->code; @@ -312,6 +314,7 @@ void tscDestroyResPointerInfo(SSqlRes* pRes) { tfree(pRes->pRsp); tfree(pRes->tsrow); + tfree(pRes->length); tfree(pRes->pGroupRec); tfree(pRes->pColumnIndex); @@ -592,7 +595,7 @@ int32_t tscGetDataBlockFromList(void* pHashList, SDataBlockList* pDataBlockList, } static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock) { - // TODO: optimize this function + // TODO: optimize this function, handle the case while binary is not presented int len = 0; STableMeta* pTableMeta = pTableDataBlock->pTableMeta; @@ -924,7 +927,7 @@ static SSqlExpr* doBuildSqlExpr(SQueryInfo* pQueryInfo, int16_t functionId, SCol SSqlExpr* pExpr = calloc(1, sizeof(SSqlExpr)); pExpr->functionId = functionId; - // set the correct column index + // set the correct columnIndex index if (pColIndex->columnIndex == TSDB_TBNAME_COLUMN_INDEX) { pExpr->colInfo.colId = TSDB_TBNAME_COLUMN_INDEX; } else { @@ -1063,7 +1066,7 @@ void tscSqlExprCopy(SArray* dst, const SArray* src, uint64_t uid, bool deepcopy) } SColumn* tscColumnListInsert(SArray* pColumnList, SColumnIndex* pColIndex) { - // ignore the tbname column to be inserted into source list + // ignore the tbname columnIndex to be inserted into source list if (pColIndex->columnIndex < 0) { return NULL; } @@ -2124,11 +2127,13 @@ void tscTryQueryNextClause(SSqlObj* pSql, void (*queryFp)()) { } } -char* tscGetResultColumnChr(SSqlRes* pRes, SQueryInfo* pQueryInfo, int32_t column, int16_t bytes) { - SFieldInfo* pFieldInfo = &pQueryInfo->fieldsInfo; - SFieldSupInfo* pInfo = tscFieldInfoGetSupp(pFieldInfo, column); - +void tscGetResultColumnChr(SSqlRes* pRes, SFieldInfo* pFieldInfo, int32_t columnIndex) { + SFieldSupInfo* pInfo = tscFieldInfoGetSupp(pFieldInfo, columnIndex); + assert(pInfo->pSqlExpr != NULL); + int32_t type = pInfo->pSqlExpr->resType; + int32_t bytes = pInfo->pSqlExpr->resBytes; + char* pData = ((char*) pRes->data) + pInfo->pSqlExpr->offset * pRes->numOfRows + bytes * pRes->row; if (type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_BINARY) { @@ -2137,9 +2142,13 @@ char* tscGetResultColumnChr(SSqlRes* pRes, SQueryInfo* pQueryInfo, int32_t colum *(char*) (pData + realLen + VARSTR_HEADER_SIZE) = 0; } - return pData + VARSTR_HEADER_SIZE; // head is the length of binary/nchar data + pRes->tsrow[columnIndex] = pData + VARSTR_HEADER_SIZE; + pRes->length[columnIndex] = realLen; } else { - return pData; + assert(bytes == tDataTypeDesc[type].nSize); + + pRes->tsrow[columnIndex] = pData; + pRes->length[columnIndex] = bytes; } } diff --git a/src/common/inc/tdataformat.h b/src/common/inc/tdataformat.h index 78b6cb73b2..10ca75f785 100644 --- a/src/common/inc/tdataformat.h +++ b/src/common/inc/tdataformat.h @@ -32,7 +32,7 @@ extern "C" { #define STR_WITH_MAXSIZE_TO_VARSTR(x, str, _maxs) do {\ char* _e = stpncpy((char*)(x) + VARSTR_HEADER_SIZE, (str), (_maxs));\ - *(VarDataLenT*)(x) = _e - (x);\ + *(VarDataLenT*)(x) = (_e - (x) - VARSTR_HEADER_SIZE);\ } while(0) #define STR_WITH_SIZE_TO_VARSTR(x, str, _size) do {\ diff --git a/src/dnode/src/dnodeWrite.c b/src/dnode/src/dnodeWrite.c index babbcf4ae8..39757c690f 100644 --- a/src/dnode/src/dnodeWrite.c +++ b/src/dnode/src/dnodeWrite.c @@ -228,7 +228,7 @@ static void dnodeHandleIdleWorker(SWriteWorker *pWorker) { int32_t num = taosGetQueueNumber(pWorker->qset); if (num > 0) { - usleep(30); + usleep(30000); sched_yield(); } else { taosFreeQall(pWorker->qall); diff --git a/src/inc/taos.h b/src/inc/taos.h index d4f1b8f48c..2f23b10a61 100644 --- a/src/inc/taos.h +++ b/src/inc/taos.h @@ -53,9 +53,9 @@ typedef enum { } TSDB_OPTION; typedef struct taosField { - char name[64]; - short bytes; - char type; + char name[64]; + short bytes; + uint8_t type; } TAOS_FIELD; #ifdef _TD_GO_DLL_ @@ -104,6 +104,8 @@ DLL_EXPORT void taos_stop_query(TAOS_RES *res); int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows); int taos_validate_sql(TAOS *taos, const char *sql); +int* taos_fetch_lengths(TAOS_RES *res); + // TAOS_RES *taos_list_tables(TAOS *mysql, const char *wild); // TAOS_RES *taos_list_dbs(TAOS *mysql, const char *wild); diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index aa4889ec9d..e60124cf0a 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -36,14 +36,17 @@ extern "C" { typedef int32_t VarDataOffsetT; typedef int16_t VarDataLenT; +#define VARSTR_HEADER_SIZE sizeof(VarDataLenT) + #define varDataLen(v) ((VarDataLenT *)(v))[0] #define varDataTLen(v) (sizeof(VarDataLenT) + varDataLen(v)) #define varDataVal(v) ((void *)((char *)v + sizeof(VarDataLenT))) #define varDataCopy(dst, v) memcpy((dst), (void*) (v), varDataTLen(v)) +#define varDataLenByData(v) (*(VarDataLenT *)(((char*)(v)) - VARSTR_HEADER_SIZE)) // this data type is internally used only in 'in' query to hold the values #define TSDB_DATA_TYPE_ARRAY (TSDB_DATA_TYPE_NCHAR + 1) -#define VARSTR_HEADER_SIZE sizeof(VarDataLenT) + // Bytes for each type. extern const int32_t TYPE_BYTES[11]; diff --git a/src/kit/shell/src/shellEngine.c b/src/kit/shell/src/shellEngine.c index eeaeec83f2..d34588661c 100644 --- a/src/kit/shell/src/shellEngine.c +++ b/src/kit/shell/src/shellEngine.c @@ -350,6 +350,8 @@ int shellDumpResult(TAOS *con, char *fname, int *error_no, bool printMode) { TAOS_FIELD *fields = taos_fetch_fields(result); row = taos_fetch_row(result); + int32_t* length = taos_fetch_lengths(result); + char t_str[TSDB_MAX_BYTES_PER_ROW] = "\0"; int l[TSDB_MAX_COLUMNS] = {0}; int maxLenColumnName = 0; @@ -457,7 +459,7 @@ int shellDumpResult(TAOS *con, char *fname, int *error_no, bool printMode) { case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: memset(t_str, 0, TSDB_MAX_BYTES_PER_ROW); - memcpy(t_str, row[i], fields[i].bytes); + memcpy(t_str, row[i], length[i]); /* printf("%-*s|",max(fields[i].bytes, strlen(fields[i].name)), * t_str); */ /* printf("%-*s|", l[i], t_str); */ @@ -532,7 +534,8 @@ int shellDumpResult(TAOS *con, char *fname, int *error_no, bool printMode) { case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: memset(t_str, 0, TSDB_MAX_BYTES_PER_ROW); - memcpy(t_str, row[i], fields[i].bytes); + memcpy(t_str, row[i], length[i]); + l[i] = MAX(fields[i].bytes, strlen(fields[i].name)); shellPrintNChar(t_str, l[i], printMode); break; -- GitLab