From 91dc9e9089418530c75144095b426be593225af6 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 19 May 2022 17:46:28 +0800 Subject: [PATCH] fea:add select json logic --- include/util/tdef.h | 2 +- source/client/src/clientImpl.c | 104 +++++++++++------------- source/common/src/tdatablock.c | 29 +++---- source/libs/executor/src/scanoperator.c | 10 ++- source/libs/parser/src/parUtil.c | 14 ++-- source/libs/scalar/src/sclvector.c | 2 +- 6 files changed, 79 insertions(+), 82 deletions(-) diff --git a/include/util/tdef.h b/include/util/tdef.h index 70f90a8ddd..f95d96be56 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -232,7 +232,7 @@ typedef enum ELogicConditionType { #define TSDB_MAX_TAGS 128 #define TSDB_MAX_TAG_CONDITIONS 1024 -#define TSDB_MAX_JSON_TAG_LEN (16384 + VARSTR_HEADER_SIZE) +#define TSDB_MAX_JSON_TAG_LEN 16384 #define TSDB_AUTH_LEN 16 #define TSDB_PASSWORD_LEN 32 diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 2640d11272..21960c5854 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -23,6 +23,8 @@ #include "tmsgtype.h" #include "tpagedbuf.h" #include "tref.h" +#include "cJSON.h" +#include "tdataformat.h" static int32_t initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* pEpSet); static SMsgSendInfo* buildConnectMsg(SRequestObj* pRequest); @@ -745,7 +747,6 @@ static int32_t doPrepareResPtr(SReqResultInfo* pResInfo) { return TSDB_CODE_SUCCESS; } -#include "cJSON.h" static char* parseTagDatatoJson(void *p){ char* string = NULL; cJSON *json = cJSON_CreateObject(); @@ -815,14 +816,14 @@ static char* parseTagDatatoJson(void *p){ goto end; } cJSON_AddItemToObject(json, tagJsonKey, value); - }else if(type == TSDB_DATA_TYPE_BIGINT){ - int64_t jsonVd = *(int64_t*)(realData); - cJSON* value = cJSON_CreateNumber((double)jsonVd); - if (value == NULL) - { - goto end; - } - cJSON_AddItemToObject(json, tagJsonKey, value); +// }else if(type == TSDB_DATA_TYPE_BIGINT){ +// int64_t jsonVd = *(int64_t*)(realData); +// cJSON* value = cJSON_CreateNumber((double)jsonVd); +// if (value == NULL) +// { +// goto end; +// } +// cJSON_AddItemToObject(json, tagJsonKey, value); }else if (type == TSDB_DATA_TYPE_BOOL) { char jsonVd = *(char*)(realData); cJSON* value = cJSON_CreateBool(jsonVd); @@ -833,7 +834,7 @@ static char* parseTagDatatoJson(void *p){ cJSON_AddItemToObject(json, tagJsonKey, value); } else{ - tscError("unsupportted json value"); + ASSERT(0); } } @@ -842,7 +843,6 @@ end: cJSON_Delete(json); return string; } -#include "tdataformat.h" static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int32_t numOfCols, int32_t* colLength) { for (int32_t i = 0; i < numOfCols; ++i) { @@ -886,51 +886,43 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int for (int32_t j = 0; j < numOfRows; ++j) { if (pCol->offset[j] != -1) { char* pStart = pCol->offset[j] + pCol->pData; - char dst[TSDB_MAX_JSON_TAG_LEN] = {0}; - char *jsonString = parseTagDatatoJson(pStart); - STR_TO_VARSTR(dst, jsonString); - taosMemoryFree(jsonString); -// int32_t jsonInnerType = *pStart; -// char* jsonInnerData = pStart + CHAR_BYTES; -// char dst[TSDB_MAX_JSON_TAG_LEN] = {0}; -// if (jsonInnerType == TSDB_DATA_TYPE_NULL) { -// sprintf(varDataVal(dst), "%s", TSDB_DATA_NULL_STR_L); -// varDataSetLen(dst, strlen(varDataVal(dst))); -// } else if (jsonInnerType == TSDB_DATA_TYPE_JSON) { -// int32_t length = -// taosUcs4ToMbs((TdUcs4*)varDataVal(jsonInnerData), varDataLen(jsonInnerData), varDataVal(dst)); -// -// if (length <= 0) { -// tscError("charset:%s to %s. val:%s convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, -// varDataVal(jsonInnerData)); -// length = 0; -// } -// varDataSetLen(dst, length); -// } else if (jsonInnerType == TSDB_DATA_TYPE_NCHAR) { // value -> "value" -// *(char*)varDataVal(dst) = '\"'; -// int32_t length = taosUcs4ToMbs((TdUcs4*)varDataVal(jsonInnerData), varDataLen(jsonInnerData), -// varDataVal(dst) + CHAR_BYTES); -// if (length <= 0) { -// tscError("charset:%s to %s. val:%s convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, -// varDataVal(jsonInnerData)); -// length = 0; -// } -// varDataSetLen(dst, length + CHAR_BYTES * 2); -// *(char*)(varDataVal(dst), length + CHAR_BYTES) = '\"'; -// } else if (jsonInnerType == TSDB_DATA_TYPE_DOUBLE) { -// double jsonVd = *(double*)(jsonInnerData); -// sprintf(varDataVal(dst), "%.9lf", jsonVd); -// varDataSetLen(dst, strlen(varDataVal(dst))); -// } else if (jsonInnerType == TSDB_DATA_TYPE_BIGINT) { -// int64_t jsonVd = *(int64_t*)(jsonInnerData); -// sprintf(varDataVal(dst), "%" PRId64, jsonVd); -// varDataSetLen(dst, strlen(varDataVal(dst))); -// } else if (jsonInnerType == TSDB_DATA_TYPE_BOOL) { -// sprintf(varDataVal(dst), "%s", (*((char*)jsonInnerData) == 1) ? "true" : "false"); -// varDataSetLen(dst, strlen(varDataVal(dst))); -// } else { -// ASSERT(0); -// } + + + int32_t jsonInnerType = *pStart; + char* jsonInnerData = pStart + CHAR_BYTES; + char dst[TSDB_MAX_JSON_TAG_LEN] = {0}; + if (jsonInnerType == TSDB_DATA_TYPE_NULL) { + sprintf(varDataVal(dst), "%s", TSDB_DATA_NULL_STR_L); + varDataSetLen(dst, strlen(varDataVal(dst))); + } else if (jsonInnerType == TSDB_DATA_TYPE_JSON) { + char *jsonString = parseTagDatatoJson(jsonInnerData); + STR_TO_VARSTR(dst, jsonString); + taosMemoryFree(jsonString); + } else if (jsonInnerType == TSDB_DATA_TYPE_NCHAR) { // value -> "value" + *(char*)varDataVal(dst) = '\"'; + int32_t length = taosUcs4ToMbs((TdUcs4*)varDataVal(jsonInnerData), varDataLen(jsonInnerData), + varDataVal(dst) + CHAR_BYTES); + if (length <= 0) { + tscError("charset:%s to %s. val:%s convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, + varDataVal(jsonInnerData)); + length = 0; + } + varDataSetLen(dst, length + CHAR_BYTES * 2); + *(char*)(varDataVal(dst), length + CHAR_BYTES) = '\"'; + } else if (jsonInnerType == TSDB_DATA_TYPE_DOUBLE) { + double jsonVd = *(double*)(jsonInnerData); + sprintf(varDataVal(dst), "%.9lf", jsonVd); + varDataSetLen(dst, strlen(varDataVal(dst))); + } else if (jsonInnerType == TSDB_DATA_TYPE_BIGINT) { + int64_t jsonVd = *(int64_t*)(jsonInnerData); + sprintf(varDataVal(dst), "%" PRId64, jsonVd); + varDataSetLen(dst, strlen(varDataVal(dst))); + } else if (jsonInnerType == TSDB_DATA_TYPE_BOOL) { + sprintf(varDataVal(dst), "%s", (*((char*)jsonInnerData) == 1) ? "true" : "false"); + varDataSetLen(dst, strlen(varDataVal(dst))); + } else { + ASSERT(0); + } if (len + varDataTLen(dst) > colLength[i]) { p = taosMemoryRealloc(pResultInfo->convertBuf[i], len + varDataTLen(dst)); diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index f0e3c782b7..736cb98549 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -116,21 +116,22 @@ int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, con int32_t type = pColumnInfoData->info.type; if (IS_VAR_DATA_TYPE(type)) { - int32_t dataLen = 0; + int32_t dataLen = varDataTLen(pData); if (type == TSDB_DATA_TYPE_JSON) { -// if (*pData == TSDB_DATA_TYPE_NULL) { -// dataLen = 0; -// } else if (*pData == TSDB_DATA_TYPE_NCHAR) { -// dataLen = varDataTLen(pData + CHAR_BYTES); -// } else if (*pData == TSDB_DATA_TYPE_BIGINT || *pData == TSDB_DATA_TYPE_DOUBLE) { -// dataLen = LONG_BYTES; -// } else if (*pData == TSDB_DATA_TYPE_BOOL) { -// dataLen = CHAR_BYTES; -// } -// dataLen += CHAR_BYTES; - dataLen = kvRowLen(pData); - }else { - dataLen = varDataTLen(pData); + if (*pData == TSDB_DATA_TYPE_NULL) { + dataLen = 0; + } else if (*pData == TSDB_DATA_TYPE_NCHAR) { + dataLen = varDataTLen(pData + CHAR_BYTES); + } else if (*pData == TSDB_DATA_TYPE_DOUBLE) { + dataLen = DOUBLE_BYTES; + } else if (*pData == TSDB_DATA_TYPE_BOOL) { + dataLen = CHAR_BYTES; + } else if (*pData == TSDB_DATA_TYPE_JSON) { + dataLen = kvRowLen(pData + CHAR_BYTES); + } else { + ASSERT(0); + } + dataLen += CHAR_BYTES; } SVarColAttr* pAttr = &pColumnInfoData->varmeta; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index ed3e368b54..0a815b2092 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1469,7 +1469,15 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) { } else { // it is a tag value const char* p = NULL; if(pDst->info.type == TSDB_DATA_TYPE_JSON){ - p = mr.me.ctbEntry.pTags; + const uint8_t *tmp = mr.me.ctbEntry.pTags; + char *data = taosMemoryCalloc(kvRowLen(tmp) + 1, 1); + if(data == NULL){ + qError("doTagScan calloc error:%d", kvRowLen(tmp) + 1); + return NULL; + } + *data = TSDB_DATA_TYPE_JSON; + memcpy(data+1, tmp, kvRowLen(tmp)); + p = data; }else{ p = metaGetTableTagVal(&mr.me, pExprInfo[j].base.pParam[0].pCol->colId); } diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index cb2d1b7c07..67e1f41c80 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -356,8 +356,8 @@ int parseJsontoTagData(const char* json, SKVRowBuilder* kvRowBuilder, SMsgBuf* p if (keyLen == 0 || taosHashGet(keyHash, jsonKey, keyLen) != NULL) { continue; } - // key: keyLen + VARSTR_HEADER_SIZE, value type: CHAR_BYTES, value reserved: LONG_BYTES - tagKV = taosMemoryCalloc(keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES + LONG_BYTES, 1); + // key: keyLen + VARSTR_HEADER_SIZE, value type: CHAR_BYTES, value reserved: DOUBLE_BYTES + tagKV = taosMemoryCalloc(keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES + DOUBLE_BYTES, 1); if (!tagKV) { retCode = TSDB_CODE_TSC_OUT_OF_MEMORY; goto end; @@ -402,13 +402,9 @@ int parseJsontoTagData(const char* json, SKVRowBuilder* kvRowBuilder, SMsgBuf* p } char* valueType = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE); char* valueData = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES); - *valueType = - (item->valuedouble - (int64_t)(item->valuedouble) == 0) ? TSDB_DATA_TYPE_BIGINT : TSDB_DATA_TYPE_DOUBLE; - if (*valueType == TSDB_DATA_TYPE_DOUBLE) - *((double*)valueData) = item->valuedouble; - else if (*valueType == TSDB_DATA_TYPE_BIGINT) - *((int64_t*)valueData) = item->valueint; - tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagKV, keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES + LONG_BYTES); + *valueType = TSDB_DATA_TYPE_DOUBLE; + *((double*)valueData) = item->valuedouble; + tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagKV, keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES + DOUBLE_BYTES); } else if (item->type == cJSON_True || item->type == cJSON_False) { char* valueType = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE); char* valueData = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES); diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index c9fcaeb32e..b01724a7b7 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -922,7 +922,7 @@ static void doReleaseVec(SColumnInfoData* pCol, int32_t type) { } } -char *getJsonValue(char *json, char *key){ //todo +char *getJsonValue(char *json, char *key){ int16_t cols = kvRowNCols(json); for (int i = 0; i < cols; ++i) { SColIdx *pColIdx = kvRowColIdxAt(json, i); -- GitLab