diff --git a/include/util/tdef.h b/include/util/tdef.h index f95d96be56d40a24cb227820058807eac9e7f051..70f90a8ddda9f03e488b120ef95b453a0495fc8a 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -232,7 +232,7 @@ typedef enum ELogicConditionType { #define TSDB_MAX_TAGS 128 #define TSDB_MAX_TAG_CONDITIONS 1024 -#define TSDB_MAX_JSON_TAG_LEN 16384 +#define TSDB_MAX_JSON_TAG_LEN (16384 + VARSTR_HEADER_SIZE) #define TSDB_AUTH_LEN 16 #define TSDB_PASSWORD_LEN 32 diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 23787651b21945b19e549a74e9bc470eb27879ba..2640d1127276897a06cae91d92b9685bb688d114 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -268,7 +268,7 @@ void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t if (pSchema[i].type == TSDB_DATA_TYPE_VARCHAR) { pResInfo->userFields[i].bytes -= VARSTR_HEADER_SIZE; - } else if (pSchema[i].type == TSDB_DATA_TYPE_NCHAR) { + } else if (pSchema[i].type == TSDB_DATA_TYPE_NCHAR || pSchema[i].type == TSDB_DATA_TYPE_JSON) { pResInfo->userFields[i].bytes = (pResInfo->userFields[i].bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; } @@ -745,6 +745,105 @@ static int32_t doPrepareResPtr(SReqResultInfo* pResInfo) { return TSDB_CODE_SUCCESS; } +#include "cJSON.h" +static char* parseTagDatatoJson(void *p){ + char* string = NULL; + cJSON *json = cJSON_CreateObject(); + if (json == NULL) + { + goto end; + } + + int16_t nCols = kvRowNCols(p); + char tagJsonKey[256] = {0}; + for (int j = 0; j < nCols; ++j) { + SColIdx * pColIdx = kvRowColIdxAt(p, j); + void* val = (kvRowColVal(p, pColIdx)); + if (j == 0){ + if(*(char*)val == TSDB_DATA_TYPE_NULL){ + string = taosMemoryCalloc(1, 8); + sprintf(varDataVal(string), "%s", TSDB_DATA_NULL_STR_L); + varDataSetLen(string, strlen(varDataVal(string))); + goto end; + } + continue; + } + + // json key encode by binary + memset(tagJsonKey, 0, sizeof(tagJsonKey)); + memcpy(tagJsonKey, varDataVal(val), varDataLen(val)); + // json value + val += varDataTLen(val); + char* realData = POINTER_SHIFT(val, CHAR_BYTES); + char type = *(char*)val; + if(type == TSDB_DATA_TYPE_NULL) { + cJSON* value = cJSON_CreateNull(); + if (value == NULL) + { + goto end; + } + cJSON_AddItemToObject(json, tagJsonKey, value); + }else if(type == TSDB_DATA_TYPE_NCHAR) { + cJSON* value = NULL; + if (varDataLen(realData) > 0){ + char *tagJsonValue = taosMemoryCalloc(varDataLen(realData), 1); + int32_t length = taosUcs4ToMbs((TdUcs4 *)varDataVal(realData), varDataLen(realData), tagJsonValue); + if (length < 0) { + tscError("charset:%s to %s. val:%s convert json value failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, + (char*)val); + taosMemoryFree(tagJsonValue); + goto end; + } + value = cJSON_CreateString(tagJsonValue); + taosMemoryFree(tagJsonValue); + if (value == NULL) + { + goto end; + } + }else if(varDataLen(realData) == 0){ + value = cJSON_CreateString(""); + }else{ + ASSERT(0); + } + + cJSON_AddItemToObject(json, tagJsonKey, value); + }else if(type == TSDB_DATA_TYPE_DOUBLE){ + double jsonVd = *(double*)(realData); + cJSON* value = cJSON_CreateNumber(jsonVd); + if (value == NULL) + { + goto end; + } + cJSON_AddItemToObject(json, tagJsonKey, value); + }else if(type == TSDB_DATA_TYPE_BIGINT){ + int64_t jsonVd = *(int64_t*)(realData); + cJSON* value = cJSON_CreateNumber((double)jsonVd); + if (value == NULL) + { + goto end; + } + cJSON_AddItemToObject(json, tagJsonKey, value); + }else if (type == TSDB_DATA_TYPE_BOOL) { + char jsonVd = *(char*)(realData); + cJSON* value = cJSON_CreateBool(jsonVd); + if (value == NULL) + { + goto end; + } + cJSON_AddItemToObject(json, tagJsonKey, value); + } + else{ + tscError("unsupportted json value"); + } + + } + string = cJSON_PrintUnformatted(json); +end: + cJSON_Delete(json); + return string; +} +#include "tdataformat.h" + static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int32_t numOfCols, int32_t* colLength) { for (int32_t i = 0; i < numOfCols; ++i) { int32_t type = pResultInfo->fields[i].type; @@ -775,9 +874,7 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int pResultInfo->pCol[i].pData = pResultInfo->convertBuf[i]; pResultInfo->row[i] = pResultInfo->pCol[i].pData; - } - - if (type == TSDB_DATA_TYPE_JSON) { + }else if (type == TSDB_DATA_TYPE_JSON && colLength[i] > 0) { char* p = taosMemoryRealloc(pResultInfo->convertBuf[i], colLength[i]); if (p == NULL) { return TSDB_CODE_OUT_OF_MEMORY; @@ -789,48 +886,51 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int for (int32_t j = 0; j < numOfRows; ++j) { if (pCol->offset[j] != -1) { char* pStart = pCol->offset[j] + pCol->pData; - - int32_t jsonInnerType = *pStart; - char* jsonInnerData = pStart + CHAR_BYTES; - char dst[TSDB_MAX_JSON_TAG_LEN] = {0}; - if (jsonInnerType == TSDB_DATA_TYPE_NULL) { - sprintf(varDataVal(dst), "%s", TSDB_DATA_NULL_STR_L); - varDataSetLen(dst, strlen(varDataVal(dst))); - } else if (jsonInnerType == TSDB_DATA_TYPE_JSON) { - int32_t length = - taosUcs4ToMbs((TdUcs4*)varDataVal(jsonInnerData), varDataLen(jsonInnerData), varDataVal(dst)); - - if (length <= 0) { - tscError("charset:%s to %s. val:%s convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, - varDataVal(jsonInnerData)); - length = 0; - } - varDataSetLen(dst, length); - } else if (jsonInnerType == TSDB_DATA_TYPE_NCHAR) { // value -> "value" - *(char*)varDataVal(dst) = '\"'; - int32_t length = taosUcs4ToMbs((TdUcs4*)varDataVal(jsonInnerData), varDataLen(jsonInnerData), - varDataVal(dst) + CHAR_BYTES); - if (length <= 0) { - tscError("charset:%s to %s. val:%s convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, - varDataVal(jsonInnerData)); - length = 0; - } - varDataSetLen(dst, length + CHAR_BYTES * 2); - *(char*)(varDataVal(dst), length + CHAR_BYTES) = '\"'; - } else if (jsonInnerType == TSDB_DATA_TYPE_DOUBLE) { - double jsonVd = *(double*)(jsonInnerData); - sprintf(varDataVal(dst), "%.9lf", jsonVd); - varDataSetLen(dst, strlen(varDataVal(dst))); - } else if (jsonInnerType == TSDB_DATA_TYPE_BIGINT) { - int64_t jsonVd = *(int64_t*)(jsonInnerData); - sprintf(varDataVal(dst), "%" PRId64, jsonVd); - varDataSetLen(dst, strlen(varDataVal(dst))); - } else if (jsonInnerType == TSDB_DATA_TYPE_BOOL) { - sprintf(varDataVal(dst), "%s", (*((char*)jsonInnerData) == 1) ? "true" : "false"); - varDataSetLen(dst, strlen(varDataVal(dst))); - } else { - ASSERT(0); - } + char dst[TSDB_MAX_JSON_TAG_LEN] = {0}; + char *jsonString = parseTagDatatoJson(pStart); + STR_TO_VARSTR(dst, jsonString); + taosMemoryFree(jsonString); +// int32_t jsonInnerType = *pStart; +// char* jsonInnerData = pStart + CHAR_BYTES; +// char dst[TSDB_MAX_JSON_TAG_LEN] = {0}; +// if (jsonInnerType == TSDB_DATA_TYPE_NULL) { +// sprintf(varDataVal(dst), "%s", TSDB_DATA_NULL_STR_L); +// varDataSetLen(dst, strlen(varDataVal(dst))); +// } else if (jsonInnerType == TSDB_DATA_TYPE_JSON) { +// int32_t length = +// taosUcs4ToMbs((TdUcs4*)varDataVal(jsonInnerData), varDataLen(jsonInnerData), varDataVal(dst)); +// +// if (length <= 0) { +// tscError("charset:%s to %s. val:%s convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, +// varDataVal(jsonInnerData)); +// length = 0; +// } +// varDataSetLen(dst, length); +// } else if (jsonInnerType == TSDB_DATA_TYPE_NCHAR) { // value -> "value" +// *(char*)varDataVal(dst) = '\"'; +// int32_t length = taosUcs4ToMbs((TdUcs4*)varDataVal(jsonInnerData), varDataLen(jsonInnerData), +// varDataVal(dst) + CHAR_BYTES); +// if (length <= 0) { +// tscError("charset:%s to %s. val:%s convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, +// varDataVal(jsonInnerData)); +// length = 0; +// } +// varDataSetLen(dst, length + CHAR_BYTES * 2); +// *(char*)(varDataVal(dst), length + CHAR_BYTES) = '\"'; +// } else if (jsonInnerType == TSDB_DATA_TYPE_DOUBLE) { +// double jsonVd = *(double*)(jsonInnerData); +// sprintf(varDataVal(dst), "%.9lf", jsonVd); +// varDataSetLen(dst, strlen(varDataVal(dst))); +// } else if (jsonInnerType == TSDB_DATA_TYPE_BIGINT) { +// int64_t jsonVd = *(int64_t*)(jsonInnerData); +// sprintf(varDataVal(dst), "%" PRId64, jsonVd); +// varDataSetLen(dst, strlen(varDataVal(dst))); +// } else if (jsonInnerType == TSDB_DATA_TYPE_BOOL) { +// sprintf(varDataVal(dst), "%s", (*((char*)jsonInnerData) == 1) ? "true" : "false"); +// varDataSetLen(dst, strlen(varDataVal(dst))); +// } else { +// ASSERT(0); +// } if (len + varDataTLen(dst) > colLength[i]) { p = taosMemoryRealloc(pResultInfo->convertBuf[i], len + varDataTLen(dst)); diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 4d77f4eb71e3a2e59b6d3fca3996507e12eac8c5..f0e3c782b730eb97ee7cac601078f0dae0ecc6d3 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -116,18 +116,21 @@ int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, con int32_t type = pColumnInfoData->info.type; if (IS_VAR_DATA_TYPE(type)) { - int32_t dataLen = varDataTLen(pData); + int32_t dataLen = 0; if (type == TSDB_DATA_TYPE_JSON) { - if (*pData == TSDB_DATA_TYPE_NULL) { - dataLen = 0; - } else if (*pData == TSDB_DATA_TYPE_NCHAR) { - dataLen = varDataTLen(pData + CHAR_BYTES); - } else if (*pData == TSDB_DATA_TYPE_BIGINT || *pData == TSDB_DATA_TYPE_DOUBLE) { - dataLen = LONG_BYTES; - } else if (*pData == TSDB_DATA_TYPE_BOOL) { - dataLen = CHAR_BYTES; - } - dataLen += CHAR_BYTES; +// if (*pData == TSDB_DATA_TYPE_NULL) { +// dataLen = 0; +// } else if (*pData == TSDB_DATA_TYPE_NCHAR) { +// dataLen = varDataTLen(pData + CHAR_BYTES); +// } else if (*pData == TSDB_DATA_TYPE_BIGINT || *pData == TSDB_DATA_TYPE_DOUBLE) { +// dataLen = LONG_BYTES; +// } else if (*pData == TSDB_DATA_TYPE_BOOL) { +// dataLen = CHAR_BYTES; +// } +// dataLen += CHAR_BYTES; + dataLen = kvRowLen(pData); + }else { + dataLen = varDataTLen(pData); } SVarColAttr* pAttr = &pColumnInfoData->varmeta; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index eaa1197a84aa9e1f2883a6395312621ad94ec561..ed3e368b544a1bf3198418bbf3aa452c567d1fe7 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1467,7 +1467,12 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) { STR_TO_VARSTR(str, mr.me.name); colDataAppend(pDst, count, str, false); } else { // it is a tag value - const char* p = metaGetTableTagVal(&mr.me, pExprInfo[j].base.pParam[0].pCol->colId); + const char* p = NULL; + if(pDst->info.type == TSDB_DATA_TYPE_JSON){ + p = mr.me.ctbEntry.pTags; + }else{ + p = metaGetTableTagVal(&mr.me, pExprInfo[j].base.pParam[0].pCol->colId); + } colDataAppend(pDst, count, p, (p == NULL)); } } diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 9f9c8821b04760e5c49d0edd87d58899e284a1fb..a013fe8d890e62b27cb170a15a6d8f842fbca4c2 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -315,6 +315,7 @@ void shellDumpFieldToFile(TdFilePtr pFile, const char *val, TAOS_FIELD *field, i break; case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: + case TSDB_DATA_TYPE_JSON: memcpy(buf, val, length); buf[length] = 0; taosFprintfFile(pFile, "\'%s\'", buf); @@ -384,19 +385,25 @@ void shellPrintNChar(const char *str, int32_t length, int32_t width) { while (pos < length) { TdWchar wc; int32_t bytes = taosMbToWchar(&wc, str + pos, MB_CUR_MAX); - if (bytes == 0) { + if (bytes <= 0) { break; } - pos += bytes; - if (pos > length) { + + if (pos + bytes > length) { break; } - + int w = 0; #ifdef WINDOWS - int32_t w = bytes; + w = bytes; #else - int32_t w = taosWcharWidth(wc); + if(*(str + pos) == '\t' || *(str + pos) == '\n' || *(str + pos) == '\r'){ + w = bytes; + }else{ + w = taosWcharWidth(wc); + } #endif + pos += bytes; + if (w <= 0) { continue; } @@ -496,6 +503,7 @@ void shellPrintField(const char *val, TAOS_FIELD *field, int32_t width, int32_t break; case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: + case TSDB_DATA_TYPE_JSON: shellPrintNChar(val, length, width); break; case TSDB_DATA_TYPE_TIMESTAMP: @@ -604,7 +612,6 @@ int32_t shellCalcColWidth(TAOS_FIELD *field, int32_t precision) { case TSDB_DATA_TYPE_DOUBLE: return TMAX(25, width); - case TSDB_DATA_TYPE_JSON: case TSDB_DATA_TYPE_BINARY: if (field->bytes > shell.args.displayWidth) { return TMAX(shell.args.displayWidth, width); @@ -612,7 +619,8 @@ int32_t shellCalcColWidth(TAOS_FIELD *field, int32_t precision) { return TMAX(field->bytes, width); } - case TSDB_DATA_TYPE_NCHAR: { + case TSDB_DATA_TYPE_NCHAR: + case TSDB_DATA_TYPE_JSON: { int16_t bytes = field->bytes * TSDB_NCHAR_SIZE; if (bytes > shell.args.displayWidth) { return TMAX(shell.args.displayWidth, width);