From 94263fd1b1303d0dbe7af6ed87e4cce3f15b4022 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 14 Oct 2021 16:36:12 +0800 Subject: [PATCH] TD-6129 move select json->'key' decode logic to client --- src/client/inc/tscUtil.h | 3 +- src/client/src/tscLocal.c | 5 +++- src/client/src/tscParseInsert.c | 5 ++++ src/client/src/tscSQLParser.c | 17 ++++++++---- src/client/src/tscUtil.c | 49 ++++++++++++++++++++++----------- src/common/src/ttypes.c | 8 ++++-- src/common/src/tvariant.c | 11 ++++++-- src/inc/taosdef.h | 3 +- src/inc/ttype.h | 1 + src/query/src/qExecutor.c | 17 +++++------- src/query/src/qSqlParser.c | 2 +- 11 files changed, 81 insertions(+), 40 deletions(-) diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index d305ddb374..2b84846d3c 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -378,7 +378,8 @@ char* cloneCurrentDBName(SSqlObj* pSql); int parseJsontoTagData(char* json, SKVRowBuilder* kvRowBuilder, char* errMsg, int16_t startColId); char* parseTagDatatoJson(void *p); -void findTagValue(STable* data, char* key, int32_t keyLen, char* out, int16_t len); +void findTagValue(STable* data, char* key, int32_t keyLen, char* out, int* len); +void parseTagValue2Dst(char* result, char* dst); int8_t jsonType2DbType(double data, int jsonType); void* getJsonTagValue(STable* pTable, char* key, int32_t keyLen, int16_t* colId); diff --git a/src/client/src/tscLocal.c b/src/client/src/tscLocal.c index 32f43b030d..76177f8019 100644 --- a/src/client/src/tscLocal.c +++ b/src/client/src/tscLocal.c @@ -91,10 +91,13 @@ static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) { if (pSchema[i].type == TSDB_DATA_TYPE_BINARY){ bytes -= VARSTR_HEADER_SIZE; } - else if(pSchema[i].type == TSDB_DATA_TYPE_NCHAR || pSchema[i].type == TSDB_DATA_TYPE_JSON) { + else if(pSchema[i].type == TSDB_DATA_TYPE_NCHAR) { bytes -= VARSTR_HEADER_SIZE; bytes = bytes / TSDB_NCHAR_SIZE; } + else if(pSchema[i].type == TSDB_DATA_TYPE_JSON) { + bytes = bytes / TSDB_NCHAR_SIZE; + } pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 2); *(int32_t *)(pRes->data + tscFieldInfoGetOffset(pQueryInfo, 2) * totalNumOfRows + pField->bytes * i) = bytes; diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 00bc928a3d..0515c3e255 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -1089,6 +1089,11 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql, char** boundC // tscDestroyBoundColumnInfo(&spd); // return tscSQLSyntaxErrMsg(pInsertParam->msg, "json type error, should be string", NULL); // } + if(sToken.n > TSDB_MAX_TAGS_LEN/TSDB_NCHAR_SIZE){ + tdDestroyKVRowBuilder(&kvRowBuilder); + tscDestroyBoundColumnInfo(&spd); + return tscSQLSyntaxErrMsg(pInsertParam->msg, "json tag too long", NULL); + } code = parseJsontoTagData(sToken.z, &kvRowBuilder, pInsertParam->msg, pTagSchema[spd.boundedColumns[0]].colId); if (code != TSDB_CODE_SUCCESS) { tdDestroyKVRowBuilder(&kvRowBuilder); diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index d7e7f82532..06be1aa721 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -6159,6 +6159,11 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { tdDestroyKVRowBuilder(&kvRowBuilder); return invalidOperationMsg(pMsg, msg25); } + if(pItem->pVar.nType > TSDB_MAX_TAGS_LEN/TSDB_NCHAR_SIZE){ + tscError("json tag too long"); + tdDestroyKVRowBuilder(&kvRowBuilder); + return invalidOperationMsg(pMsg, msg14); + } int8_t tagVal = TSDB_DATA_BINARY_PLACEHOLDER; tdAddColToKVRow(&kvRowBuilder, pTagsSchema->colId, pTagsSchema->type, &tagVal, false); @@ -7763,11 +7768,8 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); } } - if(pSchema->type == TSDB_DATA_TYPE_JSON){ - *((int8_t *)tagVal) = TSDB_DATA_BINARY_PLACEHOLDER; - }else { - ret = tVariantDump(&(pItem->pVar), tagVal, pSchema->type, true); - } + ret = tVariantDump(&(pItem->pVar), tagVal, pSchema->type, true); + // check again after the convert since it may be converted from binary to nchar. if (pSchema->type == TSDB_DATA_TYPE_BINARY || pSchema->type == TSDB_DATA_TYPE_NCHAR) { int16_t len = varDataTLen(tagVal); @@ -7798,6 +7800,11 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) { tdDestroyKVRowBuilder(&kvRowBuilder); return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); } + if(pItem->pVar.nLen > TSDB_MAX_TAGS_LEN/TSDB_NCHAR_SIZE){ + tscError("json tag too long"); + tdDestroyKVRowBuilder(&kvRowBuilder); + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); + } ret = parseJsontoTagData(pItem->pVar.pz, &kvRowBuilder, tscGetErrorMsgPayload(pCmd), pTagSchema[0].colId); if (ret != TSDB_CODE_SUCCESS) { tdDestroyKVRowBuilder(&kvRowBuilder); diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index cb917f505d..a1469b5d71 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -736,7 +736,7 @@ static void setResRawPtrImpl(SSqlRes* pRes, SInternalField* pInfo, int32_t i, bo tfree(json); } }else if (*p == SELECT_ELEMENT_JSON_TAG){ - memcpy(dst, realData, varDataTLen(realData)); + parseTagValue2Dst(realData, dst); }else{ tscError("construct json error"); } @@ -5176,13 +5176,32 @@ char* cloneCurrentDBName(SSqlObj* pSql) { return p; } -void findTagValue(STable* data, char* key, int32_t keyLen, char* out, int16_t len){ +void findTagValue(STable* data, char* key, int32_t keyLen, char* out, int* len){ void* result = getJsonTagValue(data, key, keyLen, NULL); if (result == NULL){ // json key no result return; } + + out = result; char* realData = POINTER_SHIFT(result, CHAR_BYTES); + if(*(char*)result == TSDB_DATA_TYPE_NCHAR) { + *len = varDataTLen(realData) + CHAR_BYTES; + }else if (*(char*)result == TSDB_DATA_TYPE_DOUBLE) { + *len = DOUBLE_BYTES + CHAR_BYTES; + }else if (*(char*)result == TSDB_DATA_TYPE_BIGINT) { + *len = LONG_BYTES + CHAR_BYTES; + }else if (*(char*)result == TSDB_DATA_TYPE_BOOL) { + *len = CHAR_BYTES + CHAR_BYTES; + }else if (*(char*)result == TSDB_DATA_TYPE_NULL) { + *len = CHAR_BYTES + CHAR_BYTES; + }else { + tscError("unsupportted json value"); + return; + } +} +void parseTagValue2Dst(char* result, char* dst){ + char* realData = POINTER_SHIFT(result, CHAR_BYTES); if(*(char*)result == TSDB_DATA_TYPE_NCHAR) { char tagJsonValue[TSDB_MAX_TAGS_LEN] = {0}; int32_t length = taosUcs4ToMbs(varDataVal(realData), @@ -5192,22 +5211,20 @@ void findTagValue(STable* data, char* key, int32_t keyLen, char* out, int16_t le (char*)result); return; } - assert(length <= len); - varDataSetLen(out, length); - memcpy(varDataVal(out), tagJsonValue, length); + varDataSetLen(dst, length); + memcpy(varDataVal(dst), tagJsonValue, length); }else if (*(char*)result == TSDB_DATA_TYPE_DOUBLE) { double jsonVd = *(double*)(realData); - sprintf(varDataVal(out), "%.9lf", jsonVd); - assert(strlen(varDataVal(out)) <= len); - varDataSetLen(out, strlen(varDataVal(out))); + sprintf(varDataVal(dst), "%.9lf", jsonVd); + varDataSetLen(dst, strlen(varDataVal(dst))); }else if (*(char*)result == TSDB_DATA_TYPE_BIGINT) { int64_t jsonVd = *(int64_t*)(realData); - sprintf(varDataVal(out), "%" PRId64, jsonVd); - assert(strlen(varDataVal(out)) <= len); - varDataSetLen(out, strlen(varDataVal(out))); - } - else { - tscError("unsupportted json value"); + sprintf(varDataVal(dst), "%" PRId64, jsonVd); + varDataSetLen(dst, strlen(varDataVal(dst))); + }else if (*(char*)result == TSDB_DATA_TYPE_BOOL) { + + }else if (*(char*)result == TSDB_DATA_TYPE_NULL) { + } } @@ -5303,9 +5320,9 @@ int parseJsontoTagData(char* json, SKVRowBuilder* kvRowBuilder, char* errMsg, in retCode = tscSQLSyntaxErrMsg(errMsg, "json inner error", NULL); goto end; } - if(strlen(item->string) >= TSDB_MAX_JSON_KEY_LEN){ + if(strlen(item->string) > TSDB_MAX_JSON_KEY_LEN){ tscError("json key too long error"); - retCode = tscSQLSyntaxErrMsg(errMsg, "json key too long, exceed 256", NULL); + retCode = tscSQLSyntaxErrMsg(errMsg, "json key too long, more than 256", NULL); goto end; } diff --git a/src/common/src/ttypes.c b/src/common/src/ttypes.c index 96d5a5bb9f..e074b43b07 100644 --- a/src/common/src/ttypes.c +++ b/src/common/src/ttypes.c @@ -18,7 +18,7 @@ #include "ttokendef.h" #include "tscompression.h" -const int32_t TYPE_BYTES[17] = { +const int32_t TYPE_BYTES[16] = { -1, // TSDB_DATA_TYPE_NULL sizeof(int8_t), // TSDB_DATA_TYPE_BOOL sizeof(int8_t), // TSDB_DATA_TYPE_TINYINT @@ -384,7 +384,7 @@ tDataTypeDescriptor tDataTypes[16] = { {TSDB_DATA_TYPE_USMALLINT, 17, SHORT_BYTES, "SMALLINT UNSIGNED", 0, UINT16_MAX, tsCompressSmallint, tsDecompressSmallint, getStatics_u16}, {TSDB_DATA_TYPE_UINT, 12, INT_BYTES, "INT UNSIGNED", 0, UINT32_MAX, tsCompressInt, tsDecompressInt, getStatics_u32}, {TSDB_DATA_TYPE_UBIGINT, 15, LONG_BYTES, "BIGINT UNSIGNED", 0, UINT64_MAX, tsCompressBigint, tsDecompressBigint, getStatics_u64}, - {TSDB_DATA_TYPE_JSON,4, 0, "JSON", 0, 0, tsCompressString, tsDecompressString, getStatics_nchr}, + {TSDB_DATA_TYPE_JSON,4, JSON_BYTES, "JSON", 0, 0, tsCompressString, tsDecompressString, getStatics_nchr}, }; char tTokenTypeSwitcher[13] = { @@ -440,6 +440,9 @@ void setVardataNull(void* val, int32_t type) { } else if (type == TSDB_DATA_TYPE_NCHAR) { varDataSetLen(val, sizeof(int32_t)); *(uint32_t*) varDataVal(val) = TSDB_DATA_NCHAR_NULL; + } else if (type == TSDB_DATA_TYPE_JSON) { + varDataSetLen(val, sizeof(int32_t)); + *(uint32_t*) varDataVal(val) = TSDB_DATA_NCHAR_NULL; } else { assert(0); } @@ -507,6 +510,7 @@ void setNullN(void *val, int32_t type, int32_t bytes, int32_t numOfElems) { break; case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_JSON: for (int32_t i = 0; i < numOfElems; ++i) { setVardataNull(POINTER_SHIFT(val, i * bytes), type); } diff --git a/src/common/src/tvariant.c b/src/common/src/tvariant.c index 4e2b7039e6..e2d10be6bf 100644 --- a/src/common/src/tvariant.c +++ b/src/common/src/tvariant.c @@ -158,8 +158,7 @@ void tVariantCreateFromBinary(tVariant *pVar, const char *pz, size_t len, uint32 pVar->dKey = GET_FLOAT_VAL(pz); break; } - case TSDB_DATA_TYPE_NCHAR: - case TSDB_DATA_TYPE_JSON:{ // here we get the nchar length from raw binary bits length + case TSDB_DATA_TYPE_NCHAR:{ // here we get the nchar length from raw binary bits length size_t lenInwchar = len / TSDB_NCHAR_SIZE; pVar->wpz = calloc(1, (lenInwchar + 1) * TSDB_NCHAR_SIZE); @@ -168,6 +167,12 @@ void tVariantCreateFromBinary(tVariant *pVar, const char *pz, size_t len, uint32 break; } + case TSDB_DATA_TYPE_JSON:{ + pVar->pz = calloc(len + 2, sizeof(char)); + memcpy(pVar->pz, pz, len); + pVar->nLen = (int32_t)len; + break; + } case TSDB_DATA_TYPE_BINARY:{ pVar->pz = calloc(len + 1, sizeof(char)); memcpy(pVar->pz, pz, len); @@ -923,7 +928,7 @@ int32_t tVariantDumpEx(tVariant *pVariant, char *payload, int16_t type, bool inc if (pVariant->nType == TSDB_DATA_TYPE_NULL) { //*(int8_t *)payload = TSDB_DATA_TINYINT_NULL; } else if (pVariant->nType == TSDB_DATA_TYPE_BINARY){ - //*((int8_t *)payload) = TSDB_DATA_BINARY_PLACEHOLDER; + *((int8_t *)payload) = TSDB_DATA_BINARY_PLACEHOLDER; } else if (pVariant->nType == TSDB_DATA_TYPE_JSON){ // select * from stable, set tag type to json，from setTagValue/tag_project_function memcpy(payload, pVariant->pz, pVariant->nLen); }else { diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index c510b291f8..94ba2cc7d7 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -39,7 +39,7 @@ extern "C" { #define TSKEY_INITIAL_VAL INT64_MIN // Bytes for each type. -extern const int32_t TYPE_BYTES[17]; +extern const int32_t TYPE_BYTES[16]; // TODO: replace and remove code below #define CHAR_BYTES sizeof(char) @@ -50,6 +50,7 @@ extern const int32_t TYPE_BYTES[17]; #define DOUBLE_BYTES sizeof(double) #define POINTER_BYTES sizeof(void *) // 8 by default assert(sizeof(ptrdiff_t) == sizseof(void*) +#define JSON_BYTES (TSDB_MAX_TAGS_LEN) #define TSDB_KEYSIZE sizeof(TSKEY) #if LINUX diff --git a/src/inc/ttype.h b/src/inc/ttype.h index e73e4499c8..f9dca4099f 100644 --- a/src/inc/ttype.h +++ b/src/inc/ttype.h @@ -160,6 +160,7 @@ static FORCE_INLINE bool isNull(const void *val, int32_t type) { case TSDB_DATA_TYPE_DOUBLE: return *(uint64_t *)val == TSDB_DATA_DOUBLE_NULL; case TSDB_DATA_TYPE_NCHAR: + case TSDB_DATA_TYPE_JSON: return varDataLen(val) == sizeof(int32_t) && *(uint32_t*) varDataVal(val) == TSDB_DATA_NCHAR_NULL; case TSDB_DATA_TYPE_BINARY: return varDataLen(val) == sizeof(int8_t) && *(uint8_t *) varDataVal(val) == TSDB_DATA_BINARY_NULL; diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 8c9b5b0c9c..4bbbfe8d07 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -3359,7 +3359,7 @@ static void doSetTagValueInParam(void* pTable, int32_t tagColId, tVariant *tag, //tVariantCreateFromBinary(tag, varDataVal(val), varDataLen(val), type); } else if(type == TSDB_DATA_TYPE_JSON){ assert(kvRowLen(val) < bytes); - tVariantCreateFromBinary(tag, val, bytes, type); + tVariantCreateFromBinary(tag, val, kvRowLen(val), type); memcpy(tag->pz + 1, tag->pz, bytes - 1); // move back 1 byte for select type *(tag->pz) = SELECT_ALL_JSON_TAG; tag->nLen++; @@ -7189,18 +7189,15 @@ static SSDataBlock* doTagScan(void* param, bool* newgroup) { data = tsdbGetTableTagVal(item->pTable, pExprInfo[j].base.colInfo.colId, type, bytes); if(type == TSDB_DATA_TYPE_JSON){ if(pExprInfo[j].base.numOfParams > 0){ // tag-> operation - char* tagJsonElementData = calloc(bytes, 1); char keyMd5[TSDB_MAX_JSON_KEY_MD5_LEN] = {0}; jsonKeyMd5(pExprInfo[j].base.param[0].pz, pExprInfo[j].base.param[0].nLen, keyMd5); - findTagValue(item->pTable, keyMd5, TSDB_MAX_JSON_KEY_MD5_LEN, tagJsonElementData, bytes); - *dst = SELECT_ELEMENT_JSON_TAG; // select tag->element - dst++; - assert(varDataTLen(tagJsonElementData) < bytes); - doSetTagValueToResultBuf(dst, tagJsonElementData, type, bytes - CHAR_BYTES); - tfree(tagJsonElementData); + char* strStr = NULL; + int len = 0; + findTagValue(item->pTable, keyMd5, TSDB_MAX_JSON_KEY_MD5_LEN, strStr, &len); + *dst++ = SELECT_ELEMENT_JSON_TAG; // select tag->element + doSetTagValueToResultBuf(dst, strStr, type, len); }else{ - *dst = SELECT_ALL_JSON_TAG; // select tag - dst++; + *dst++ = SELECT_ALL_JSON_TAG; // select tag assert(kvRowLen(data) < bytes); doSetTagValueToResultBuf(dst, data, type, bytes - CHAR_BYTES); } diff --git a/src/query/src/qSqlParser.c b/src/query/src/qSqlParser.c index ad4b3bf4f2..69249e6707 100644 --- a/src/query/src/qSqlParser.c +++ b/src/query/src/qSqlParser.c @@ -716,7 +716,7 @@ void tSetColumnType(TAOS_FIELD *pField, SStrToken *type) { pField->type = i; pField->bytes = tDataTypes[i].bytes; - if (i == TSDB_DATA_TYPE_NCHAR || i == TSDB_DATA_TYPE_JSON) { + if (i == TSDB_DATA_TYPE_NCHAR) { /* * for nchar, the TOKENTYPE is the number of character, so the length is the * number of bytes in UCS-4 format, which is 4 times larger than the number of characters -- GitLab