From d5afeae6115313bea2aaa8f08d45229b5861749c Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Sat, 18 Jun 2022 17:04:32 +0800 Subject: [PATCH] fix:raw data for json --- source/client/inc/clientInt.h | 2 + source/client/src/clientEnv.c | 1 + source/client/src/clientImpl.c | 167 ++++++++++++++++++++++----------- 3 files changed, 114 insertions(+), 56 deletions(-) diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 69a24e20cd..05e255c99d 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -173,6 +173,7 @@ typedef struct SReqResultInfo { int32_t precision; bool convertUcs4; int32_t payloadLen; + char* convertJson; } SReqResultInfo; typedef struct SRequestSendRecvBody { @@ -252,6 +253,7 @@ static FORCE_INLINE SReqResultInfo* tmqGetNextResInfo(TAOS_RES* res, bool conver taosMemoryFreeClear(msg->resInfo.pCol); taosMemoryFreeClear(msg->resInfo.length); taosMemoryFreeClear(msg->resInfo.convertBuf); + taosMemoryFreeClear(msg->resInfo.convertJson); } setQueryResultFromRsp(&msg->resInfo, pRetrieve, convertUcs4, false); return &msg->resInfo; diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index bb60624145..6ef71c85f7 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -211,6 +211,7 @@ void doFreeReqResultInfo(SReqResultInfo *pResInfo) { taosMemoryFreeClear(pResInfo->pCol); taosMemoryFreeClear(pResInfo->fields); taosMemoryFreeClear(pResInfo->userFields); + taosMemoryFreeClear(pResInfo->convertJson); if (pResInfo->convertBuf != NULL) { for (int32_t i = 0; i < pResInfo->numOfCols; ++i) { diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 18bcbf4be2..be19fdbe1f 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1297,12 +1297,12 @@ end: return string; } -static int32_t doConvert(SReqResultInfo* pResultInfo, int32_t numOfRows, int32_t numOfCols, int32_t* colLength, bool convertUCS4) { +static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int32_t numOfCols, int32_t* colLength) { for (int32_t i = 0; i < numOfCols; ++i) { int32_t type = pResultInfo->fields[i].type; int32_t bytes = pResultInfo->fields[i].bytes; - if (type == TSDB_DATA_TYPE_NCHAR && colLength[i] > 0 && convertUCS4) { + if (type == TSDB_DATA_TYPE_NCHAR && colLength[i] > 0) { char* p = taosMemoryRealloc(pResultInfo->convertBuf[i], colLength[i]); if (p == NULL) { return TSDB_CODE_OUT_OF_MEMORY; @@ -1327,70 +1327,119 @@ static int32_t doConvert(SReqResultInfo* pResultInfo, int32_t numOfRows, int32_t pResultInfo->pCol[i].pData = pResultInfo->convertBuf[i]; pResultInfo->row[i] = pResultInfo->pCol[i].pData; - } else if (type == TSDB_DATA_TYPE_JSON && colLength[i] > 0) { - char* p = taosMemoryRealloc(pResultInfo->convertBuf[i], colLength[i]); - if (p == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } + } + } - pResultInfo->convertBuf[i] = p; - int32_t len = 0; - SResultColumn* pCol = &pResultInfo->pCol[i]; - for (int32_t j = 0; j < numOfRows; ++j) { - if (pCol->offset[j] != -1) { - char* pStart = pCol->offset[j] + pCol->pData; + return TSDB_CODE_SUCCESS; +} - int32_t jsonInnerType = *pStart; - char* jsonInnerData = pStart + CHAR_BYTES; - char dst[TSDB_MAX_JSON_TAG_LEN] = {0}; - if (jsonInnerType == TSDB_DATA_TYPE_NULL) { - sprintf(varDataVal(dst), "%s", TSDB_DATA_NULL_STR_L); - varDataSetLen(dst, strlen(varDataVal(dst))); - } else if (jsonInnerType == TD_TAG_JSON) { - char* jsonString = parseTagDatatoJson(pStart); - STR_TO_VARSTR(dst, jsonString); - taosMemoryFree(jsonString); - } else if (jsonInnerType == TSDB_DATA_TYPE_NCHAR) { // value -> "value" - *(char*)varDataVal(dst) = '\"'; - int32_t length = taosUcs4ToMbs((TdUcs4*)varDataVal(jsonInnerData), varDataLen(jsonInnerData), - varDataVal(dst) + CHAR_BYTES); - if (length <= 0) { - tscError("charset:%s to %s. convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset); - length = 0; - } - varDataSetLen(dst, length + CHAR_BYTES * 2); - *(char*)POINTER_SHIFT(varDataVal(dst), length + CHAR_BYTES) = '\"'; - } else if (jsonInnerType == TSDB_DATA_TYPE_DOUBLE) { - double jsonVd = *(double*)(jsonInnerData); - sprintf(varDataVal(dst), "%.9lf", jsonVd); - varDataSetLen(dst, strlen(varDataVal(dst))); - } else if (jsonInnerType == TSDB_DATA_TYPE_BOOL) { - sprintf(varDataVal(dst), "%s", (*((char*)jsonInnerData) == 1) ? "true" : "false"); - varDataSetLen(dst, strlen(varDataVal(dst))); - } else { - ASSERT(0); - } +static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int32_t numOfRows) { + bool needConvert = false; + for (int32_t i = 0; i < numOfCols; ++i) { + if (pResultInfo->fields[i].type == TSDB_DATA_TYPE_JSON) { + needConvert = true; + break; + } + } + if(!needConvert) return TSDB_CODE_SUCCESS; - if (len + varDataTLen(dst) > colLength[i]) { - p = taosMemoryRealloc(pResultInfo->convertBuf[i], len + varDataTLen(dst)); - if (p == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } + char* p = (char*)pResultInfo->pData; + int32_t dataLen = *(int32_t*)p; + pResultInfo->convertJson = taosMemoryCalloc(1, dataLen); + if(pResultInfo->convertJson == NULL) return TSDB_CODE_OUT_OF_MEMORY; + char* p1 = pResultInfo->convertJson; + + int32_t len = sizeof(int32_t) + sizeof(uint64_t) + numOfCols *(sizeof(int16_t) + sizeof(int32_t)); + memcpy(p1, p, len); - pResultInfo->convertBuf[i] = p; + p += len; + p1 += len; + + len = sizeof(int32_t) * numOfCols; + int32_t* colLength = (int32_t*)p; + int32_t* colLength1 = (int32_t*)p1; + memcpy(p1, p, len); + p += len; + p1 += len; + + char* pStart = p; + char* pStart1 = p1; + for (int32_t i = 0; i < numOfCols; ++i) { + int32_t colLen = htonl(colLength[i]); + int32_t colLen1 = htonl(colLength1[i]); + ASSERT(colLen < dataLen); + + if (pResultInfo->fields[i].type == TSDB_DATA_TYPE_JSON) { + int32_t* offset = (int32_t*)pStart; + int32_t* offset1 = (int32_t*)pStart1; + len = numOfRows * sizeof(int32_t); + memcpy(pStart1, pStart, len); + pStart += len; + pStart1 += len; + + len = 0; + for (int32_t j = 0; j < numOfRows; ++j) { + if (offset[j] == -1) { + continue; + } + char* data = offset[j] + pStart; + + int32_t jsonInnerType = *data; + char* jsonInnerData = data + CHAR_BYTES; + char dst[TSDB_MAX_JSON_TAG_LEN] = {0}; + if (jsonInnerType == TSDB_DATA_TYPE_NULL) { + sprintf(varDataVal(dst), "%s", TSDB_DATA_NULL_STR_L); + varDataSetLen(dst, strlen(varDataVal(dst))); + } else if (jsonInnerType == TD_TAG_JSON) { + char* jsonString = parseTagDatatoJson(data); + STR_TO_VARSTR(dst, jsonString); + taosMemoryFree(jsonString); + } else if (jsonInnerType == TSDB_DATA_TYPE_NCHAR) { // value -> "value" + *(char*)varDataVal(dst) = '\"'; + int32_t length = taosUcs4ToMbs((TdUcs4*)varDataVal(jsonInnerData), varDataLen(jsonInnerData), + varDataVal(dst) + CHAR_BYTES); + if (length <= 0) { + tscError("charset:%s to %s. convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset); + length = 0; } - p = pResultInfo->convertBuf[i] + len; - memcpy(p, dst, varDataTLen(dst)); - pCol->offset[j] = len; - len += varDataTLen(dst); + varDataSetLen(dst, length + CHAR_BYTES * 2); + *(char*)POINTER_SHIFT(varDataVal(dst), length + CHAR_BYTES) = '\"'; + } else if (jsonInnerType == TSDB_DATA_TYPE_DOUBLE) { + double jsonVd = *(double*)(jsonInnerData); + sprintf(varDataVal(dst), "%.9lf", jsonVd); + varDataSetLen(dst, strlen(varDataVal(dst))); + } else if (jsonInnerType == TSDB_DATA_TYPE_BOOL) { + sprintf(varDataVal(dst), "%s", (*((char*)jsonInnerData) == 1) ? "true" : "false"); + varDataSetLen(dst, strlen(varDataVal(dst))); + } else { + ASSERT(0); } + + offset1[j]= len; + memcpy(pStart1 + len, dst, varDataTLen(dst)); + len += varDataTLen(dst); } + colLen1 = len; + colLength1[i] = htonl(len); + } else if (IS_VAR_DATA_TYPE(pResultInfo->fields[i].type)) { + len = numOfRows * sizeof(int32_t); + memcpy(pStart1, pStart, len); + pStart += len; + pStart1 += len; + memcpy(pStart1, pStart, colLen); + } else { + len = BitmapLen(pResultInfo->numOfRows); + memcpy(pStart1, pStart, len); + pStart += len; + pStart1 += len; + memcpy(pStart1, pStart, colLen); - pResultInfo->pCol[i].pData = pResultInfo->convertBuf[i]; - pResultInfo->row[i] = pResultInfo->pCol[i].pData; } + pStart += colLen; + pStart1 += colLen1; } + pResultInfo->pData = pResultInfo->convertJson; return TSDB_CODE_SUCCESS; } @@ -1405,6 +1454,10 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32 if (code != TSDB_CODE_SUCCESS) { return code; } + code = doConvertJson(pResultInfo, numOfCols, numOfRows); + if (code != TSDB_CODE_SUCCESS) { + return code; + } char* p = (char*)pResultInfo->pData; @@ -1448,7 +1501,9 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32 pStart += colLength[i]; } - code = doConvert(pResultInfo, numOfRows, numOfCols, colLength, convertUcs4); + if(convertUcs4){ + code = doConvertUCS4(pResultInfo, numOfRows, numOfCols, colLength); + } return code; } -- GitLab