From a91392a931c63d5cf355b67c87a6e6d8dec06c40 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 13 Oct 2021 18:20:00 +0800 Subject: [PATCH] TD-6129 delete json binary, only support json nchar --- src/client/inc/tscUtil.h | 10 +- src/client/src/tscLocal.c | 4 +- src/client/src/tscParseInsert.c | 9 +- src/client/src/tscPrepare.c | 2 +- src/client/src/tscSQLParser.c | 36 ++++---- src/client/src/tscSql.c | 4 +- src/client/src/tscSubquery.c | 2 +- src/client/src/tscUtil.c | 121 +++++++++---------------- src/common/src/tname.c | 6 +- src/common/src/ttypes.c | 10 +- src/common/src/tvariant.c | 10 +- src/connector/python/taos/bind.py | 27 +----- src/connector/python/taos/constants.py | 3 +- src/connector/python/taos/cursor.py | 7 +- src/connector/python/taos/field.py | 6 +- src/inc/taos.h | 3 +- src/inc/taosdef.h | 2 + src/inc/ttype.h | 3 +- src/kit/shell/src/shellEngine.c | 6 +- src/query/inc/qFilter.h | 2 +- src/query/src/qExecutor.c | 10 +- src/query/src/qFilter.c | 33 +++---- src/query/src/qSqlParser.c | 16 +--- src/tsdb/src/tsdbMeta.c | 21 +++-- src/tsdb/src/tsdbRead.c | 23 ++--- src/util/inc/tutil.h | 1 + src/util/src/tutil.c | 10 ++ 27 files changed, 158 insertions(+), 229 deletions(-) diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index 0cec4d8f23..d305ddb374 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -376,12 +376,12 @@ void tscRemoveCachedTableMeta(STableMetaInfo* pTableMetaInfo, uint64_t id); char* cloneCurrentDBName(SSqlObj* pSql); -int parseJsontoTagData(char* json, SKVRowBuilder* kvRowBuilder, char* errMsg, int16_t startColId, uint8_t jsonType); -char* parseTagDatatoJson(void *p, uint8_t jsonType); -void findTagValue(STable* data, char* key, int32_t keyLen, char* out, int16_t len, uint8_t jsonType); +int parseJsontoTagData(char* json, SKVRowBuilder* kvRowBuilder, char* errMsg, int16_t startColId); +char* parseTagDatatoJson(void *p); +void findTagValue(STable* data, char* key, int32_t keyLen, char* out, int16_t len); -int8_t jsonType2DbType(double data, int jsonType, uint8_t type); -void* getJsonTagValue(STable* pTable, char* key, int32_t keyLen, uint8_t jsonType, int16_t* colId); +int8_t jsonType2DbType(double data, int jsonType); +void* getJsonTagValue(STable* pTable, char* key, int32_t keyLen, int16_t* colId); #ifdef __cplusplus } diff --git a/src/client/src/tscLocal.c b/src/client/src/tscLocal.c index ee5f63bdc4..32f43b030d 100644 --- a/src/client/src/tscLocal.c +++ b/src/client/src/tscLocal.c @@ -88,10 +88,10 @@ static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) { STR_WITH_MAXSIZE_TO_VARSTR(dst, type, pField->bytes); int32_t bytes = pSchema[i].bytes; - if (pSchema[i].type == TSDB_DATA_TYPE_BINARY || pSchema[i].type == TSDB_DATA_TYPE_JSON_BINARY){ + if (pSchema[i].type == TSDB_DATA_TYPE_BINARY){ bytes -= VARSTR_HEADER_SIZE; } - else if(pSchema[i].type == TSDB_DATA_TYPE_NCHAR || pSchema[i].type == TSDB_DATA_TYPE_JSON_NCHAR) { + else if(pSchema[i].type == TSDB_DATA_TYPE_NCHAR || pSchema[i].type == TSDB_DATA_TYPE_JSON) { bytes -= VARSTR_HEADER_SIZE; bytes = bytes / TSDB_NCHAR_SIZE; } diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index fba74c68aa..00bc928a3d 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -387,8 +387,7 @@ int32_t tsParseOneColumn(SSchema *pSchema, SStrToken *pToken, char *payload, cha } break; - case TSDB_DATA_TYPE_JSON_BINARY: - case TSDB_DATA_TYPE_JSON_NCHAR: + case TSDB_DATA_TYPE_JSON: if (pToken->n >= pSchema->bytes) { // reserve 1 byte for select return tscInvalidOperationMsg(msg, "json tag length too long", pToken->z); } @@ -1082,7 +1081,7 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql, char** boundC } // encode json tag string - if(spd.numOfBound == 1 && IS_JSON_DATA_TYPE(pTagSchema[spd.boundedColumns[0]].type)){ + if(spd.numOfBound == 1 && pTagSchema[spd.boundedColumns[0]].type == TSDB_DATA_TYPE_JSON){ char tmp = sToken.z[sToken.n]; sToken.z[sToken.n] = 0; // if(sToken.type != TK_STRING) { @@ -1090,7 +1089,7 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql, char** boundC // tscDestroyBoundColumnInfo(&spd); // return tscSQLSyntaxErrMsg(pInsertParam->msg, "json type error, should be string", NULL); // } - code = parseJsontoTagData(sToken.z, &kvRowBuilder, pInsertParam->msg, pTagSchema[spd.boundedColumns[0]].colId, pTagSchema[spd.boundedColumns[0]].type); + code = parseJsontoTagData(sToken.z, &kvRowBuilder, pInsertParam->msg, pTagSchema[spd.boundedColumns[0]].colId); if (code != TSDB_CODE_SUCCESS) { tdDestroyKVRowBuilder(&kvRowBuilder); tscDestroyBoundColumnInfo(&spd); @@ -1113,7 +1112,7 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql, char** boundC return tscSQLSyntaxErrMsg(pInsertParam->msg, "tag value expected", NULL); } // encode json tag string - if(spd.numOfBound == 1 && IS_JSON_DATA_TYPE(pTagSchema[spd.boundedColumns[0]].type)){ + if(spd.numOfBound == 1 && pTagSchema[spd.boundedColumns[0]].type == TSDB_DATA_TYPE_JSON){ if(kvRowLen(row) >= pTagSchema[spd.boundedColumns[0]].bytes){ // reserve 1 byte for select char tmp[128]= {0}; sprintf(tmp, "tag value is too small, can not contain encoded json tag:%d|%d", kvRowLen(row), pTagSchema[spd.boundedColumns[0]].bytes); diff --git a/src/client/src/tscPrepare.c b/src/client/src/tscPrepare.c index 5123b75f2e..186fc6286e 100644 --- a/src/client/src/tscPrepare.c +++ b/src/client/src/tscPrepare.c @@ -905,7 +905,7 @@ static int doBindBatchParam(STableDataBlocks* pBlock, SParamInfo* param, TAOS_MU } varDataSetLen(data + param->offset, output); - } else if (IS_JSON_DATA_TYPE(param->type)) { // todo json + } else if (param->type == TSDB_DATA_TYPE_JSON) { // todo json } } diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 37c31ad868..d7e7f82532 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -162,7 +162,7 @@ bool serializeExprListToVariant(SArray* pList, tVariant **dst, int16_t colType, tSqlExpr* item = ((tSqlExprItem*)(taosArrayGet(pList, 0)))->pNode; int32_t firstVarType = item->value.nType; - if(IS_JSON_DATA_TYPE(colType)) colType = firstVarType; + if(colType == TSDB_DATA_TYPE_JSON) colType = firstVarType; SBufferWriter bw = tbufInitWriter( NULL, false); tbufEnsureCapacity(&bw, 512); @@ -1440,7 +1440,7 @@ static bool validateTableColumnInfo(SArray* pFieldList, SSqlCmd* pCmd) { int32_t nLen = 0; for (int32_t i = 0; i < numOfCols; ++i) { pField = taosArrayGet(pFieldList, i); - if (!isValidDataType(pField->type) || IS_JSON_DATA_TYPE(pField->type)) { + if (!isValidDataType(pField->type) || pField->type == TSDB_DATA_TYPE_JSON) { invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); return false; } @@ -1505,7 +1505,7 @@ static bool validateTagParams(SArray* pTagsList, SArray* pFieldList, SSqlCmd* pC return false; } - if (IS_JSON_DATA_TYPE(p->type) && numOfTags != 1) { + if (p->type == TSDB_DATA_TYPE_JSON && numOfTags != 1) { invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg8); return false; } @@ -1592,7 +1592,7 @@ int32_t validateOneTag(SSqlCmd* pCmd, TAOS_FIELD* pTagField) { // invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); // return false; //} - if (IS_JSON_DATA_TYPE(pTagField->type)) { + if (pTagField->type == TSDB_DATA_TYPE_JSON) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg8); } @@ -1603,7 +1603,7 @@ int32_t validateOneTag(SSqlCmd* pCmd, TAOS_FIELD* pTagField) { SSchema* pTagSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta); int32_t nLen = 0; - if (numOfTags == 1 && IS_JSON_DATA_TYPE(pTagSchema[0].type)){ + if (numOfTags == 1 && pTagSchema[0].type == TSDB_DATA_TYPE_JSON){ return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg8); } @@ -2319,7 +2319,7 @@ int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, t } SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, index.columnIndex); - if (tokenId == TK_ARROW && !IS_JSON_DATA_TYPE(pSchema->type)) { + if (tokenId == TK_ARROW && pSchema->type != TSDB_DATA_TYPE_JSON) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4); } @@ -4471,7 +4471,7 @@ static int32_t validateMatchExpr(tSqlExpr* pExpr, STableMeta* pTableMeta, int32_ return invalidOperationMsg(msgBuf, msg2); } - if(pLeft->tokenId == TK_ARROW && pSchema[index].type == TSDB_DATA_TYPE_JSON_NCHAR){ + if(pLeft->tokenId == TK_ARROW && pSchema[index].type == TSDB_DATA_TYPE_JSON){ return invalidOperationMsg(msgBuf, msg4); } @@ -4648,7 +4648,7 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSql } // check for json tag operation -> and ? - if (IS_JSON_DATA_TYPE(pSchema->type)){ + if (pSchema->type == TSDB_DATA_TYPE_JSON){ code = validateJsonTagExpr(*pExpr, tscGetErrorMsgPayload(pCmd)); if (code != TSDB_CODE_SUCCESS) { return code; @@ -6149,7 +6149,7 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { d += sizeof(STColumn); } - if (IS_JSON_DATA_TYPE(pTagsSchema->type)){ + if (pTagsSchema->type == TSDB_DATA_TYPE_JSON){ SKVRowBuilder kvRowBuilder = {0}; if (tdInitKVRowBuilder(&kvRowBuilder) < 0) { return TSDB_CODE_TSC_OUT_OF_MEMORY; @@ -6163,7 +6163,7 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { int8_t tagVal = TSDB_DATA_BINARY_PLACEHOLDER; tdAddColToKVRow(&kvRowBuilder, pTagsSchema->colId, pTagsSchema->type, &tagVal, false); - code = parseJsontoTagData(pItem->pVar.pz, &kvRowBuilder, pMsg, pTagsSchema->colId, pTagsSchema->type); + code = parseJsontoTagData(pItem->pVar.pz, &kvRowBuilder, pMsg, pTagsSchema->colId); if (code != TSDB_CODE_SUCCESS) { tdDestroyKVRowBuilder(&kvRowBuilder); return code; @@ -6191,7 +6191,7 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { } int32_t len = 0; - if(IS_JSON_DATA_TYPE(pTagsSchema->type)){ + if(pTagsSchema->type == TSDB_DATA_TYPE_JSON){ len = kvRowLen(pUpdateMsg->data + schemaLen); }else if (!IS_VAR_DATA_TYPE(pTagsSchema->type)) { len = tDataTypes[pTagsSchema->type].bytes; @@ -7695,13 +7695,13 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) { } else if (pItem->pVar.nType == TSDB_DATA_TYPE_TIMESTAMP) { pItem->pVar.i64 = convertTimePrecision(pItem->pVar.i64, TSDB_TIME_PRECISION_NANO, tinfo.precision); } - } else if (IS_JSON_DATA_TYPE(pSchema->type)) { + } else if (pSchema->type == TSDB_DATA_TYPE_JSON) { if (pItem->pVar.nLen > TSDB_MAX_TAGS_LEN) { tdDestroyKVRowBuilder(&kvRowBuilder); return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); } } - if(IS_JSON_DATA_TYPE(pSchema->type)){ + if(pSchema->type == TSDB_DATA_TYPE_JSON){ *((int8_t *)tagVal) = TSDB_DATA_BINARY_PLACEHOLDER; }else { ret = tVariantDump(&(pItem->pVar), tagVal, pSchema->type, true); @@ -7757,13 +7757,13 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) { } else if (pItem->pVar.nType == TSDB_DATA_TYPE_TIMESTAMP) { pItem->pVar.i64 = convertTimePrecision(pItem->pVar.i64, TSDB_TIME_PRECISION_NANO, tinfo.precision); } - } else if (IS_JSON_DATA_TYPE(pSchema->type)) { + } else if (pSchema->type == TSDB_DATA_TYPE_JSON) { if (pItem->pVar.nLen > pSchema->bytes) { tdDestroyKVRowBuilder(&kvRowBuilder); return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); } } - if(IS_JSON_DATA_TYPE(pSchema->type)){ + if(pSchema->type == TSDB_DATA_TYPE_JSON){ *((int8_t *)tagVal) = TSDB_DATA_BINARY_PLACEHOLDER; }else { ret = tVariantDump(&(pItem->pVar), tagVal, pSchema->type, true); @@ -7787,7 +7787,7 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) { } // encode json tag string - if(schemaSize == 1 && IS_JSON_DATA_TYPE(pTagSchema[0].type)){ + if(schemaSize == 1 && pTagSchema[0].type == TSDB_DATA_TYPE_JSON){ if (valSize != schemaSize) { tdDestroyKVRowBuilder(&kvRowBuilder); return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); @@ -7798,7 +7798,7 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) { tdDestroyKVRowBuilder(&kvRowBuilder); return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); } - ret = parseJsontoTagData(pItem->pVar.pz, &kvRowBuilder, tscGetErrorMsgPayload(pCmd), pTagSchema[0].colId, pTagSchema[0].type); + ret = parseJsontoTagData(pItem->pVar.pz, &kvRowBuilder, tscGetErrorMsgPayload(pCmd), pTagSchema[0].colId); if (ret != TSDB_CODE_SUCCESS) { tdDestroyKVRowBuilder(&kvRowBuilder); return ret; @@ -7812,7 +7812,7 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) { } tdSortKVRowByColIdx(row); pTag->dataLen = kvRowLen(row); - if(schemaSize == 1 && IS_JSON_DATA_TYPE(pTagSchema[0].type)){ + if(schemaSize == 1 && pTagSchema[0].type == TSDB_DATA_TYPE_JSON){ if(kvRowLen(row) >= pTagSchema[0].bytes){ // reserve 1 byte for select char tmp[128]= {0}; sprintf(tmp, "tag value is too small, can not contain encoded json tag:%d|%d", kvRowLen(row), pTagSchema[0].bytes); diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index ec05432940..54f8394ceb 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -439,9 +439,9 @@ TAOS_FIELD *taos_fetch_fields(TAOS_RES *res) { f[j] = pField->field; // revise the length for binary and nchar fields - if (f[j].type == TSDB_DATA_TYPE_BINARY || f[j].type == TSDB_DATA_TYPE_JSON_BINARY) { + if (f[j].type == TSDB_DATA_TYPE_BINARY) { f[j].bytes -= VARSTR_HEADER_SIZE; - } else if (f[j].type == TSDB_DATA_TYPE_NCHAR || f[j].type == TSDB_DATA_TYPE_JSON_NCHAR) { + } else if (f[j].type == TSDB_DATA_TYPE_NCHAR || f[j].type == TSDB_DATA_TYPE_JSON) { f[j].bytes = (f[j].bytes - VARSTR_HEADER_SIZE)/TSDB_NCHAR_SIZE; } diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 010865cbfd..c9512e514c 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -3552,7 +3552,7 @@ TAOS_ROW doSetResultRowData(SSqlObj *pSql) { int32_t type = pInfo->field.type; int32_t bytes = pInfo->field.bytes; - if (!IS_VAR_DATA_TYPE(type) && !IS_JSON_DATA_TYPE(type)) { + if (!IS_VAR_DATA_TYPE(type) && type != TSDB_DATA_TYPE_JSON) { pRes->tsrow[j] = isNull(pRes->urow[i], type) ? NULL : pRes->urow[i]; } else { pRes->tsrow[j] = isNull(pRes->urow[i], type) ? NULL : varDataVal(pRes->urow[i]); diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index f77ff019a5..086dcfb29b 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -73,8 +73,7 @@ int32_t converToStr(char *str, int type, void *buf, int32_t bufSize, int32_t *le case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: - case TSDB_DATA_TYPE_JSON_BINARY: - case TSDB_DATA_TYPE_JSON_NCHAR: + case TSDB_DATA_TYPE_JSON: if (bufSize < 0) { tscError("invalid buf size"); return TSDB_CODE_TSC_INVALID_VALUE; @@ -715,7 +714,7 @@ static void setResRawPtrImpl(SSqlRes* pRes, SInternalField* pInfo, int32_t i, bo } memcpy(pRes->urow[i], pRes->buffer[i], pInfo->field.bytes * pRes->numOfRows); - }else if (IS_JSON_DATA_TYPE(pInfo->field.type)) { + }else if (pInfo->field.type == TSDB_DATA_TYPE_JSON) { // convert unicode to native code in a temporary buffer extra one byte for terminated symbol char* buffer = realloc(pRes->buffer[i], pInfo->field.bytes * pRes->numOfRows); if(buffer == NULL) @@ -729,11 +728,11 @@ static void setResRawPtrImpl(SSqlRes* pRes, SInternalField* pInfo, int32_t i, bo char* dst = pRes->buffer[i] + k * pInfo->field.bytes; char* realData = p + CHAR_BYTES; if (*p == SELECT_ALL_JSON_TAG){ - char* json = parseTagDatatoJson(realData, pInfo->field.type); + char* json = parseTagDatatoJson(realData); if(json) { memcpy(varDataVal(dst), json, strlen(json)); varDataSetLen(dst, strlen(json)); - assert(varDataTLen(dst) <= pInfo->field.bytes); + assert(varDataTLen(dst) <= pInfo->field.type); tfree(json); } }else if (*p == SELECT_ELEMENT_JSON_TAG){ @@ -5177,18 +5176,14 @@ char* cloneCurrentDBName(SSqlObj* pSql) { return p; } -void findTagValue(STable* data, char* key, int32_t keyLen, char* out, int16_t len, uint8_t jsonType){ - void* result = getJsonTagValue(data, key, keyLen, jsonType, NULL); +void findTagValue(STable* data, char* key, int32_t keyLen, char* out, int16_t len){ + void* result = getJsonTagValue(data, key, keyLen, NULL); if (result == NULL){ // json key no result return; } char* realData = POINTER_SHIFT(result, CHAR_BYTES); - if (*(char*)result == TSDB_DATA_TYPE_BINARY){ - assert(varDataLen(realData) <= len); - memcpy(varDataVal(out), varDataVal(realData), varDataLen(realData)); - varDataSetLen(out, varDataLen(realData)); - } else if(*(char*)result == TSDB_DATA_TYPE_NCHAR) { + if(*(char*)result == TSDB_DATA_TYPE_NCHAR) { char tagJsonValue[TSDB_MAX_TAGS_LEN] = {0}; int32_t length = taosUcs4ToMbs(varDataVal(realData), varDataLen(realData), tagJsonValue); @@ -5216,7 +5211,7 @@ void findTagValue(STable* data, char* key, int32_t keyLen, char* out, int16_t le } } -char* parseTagDatatoJson(void *p, uint8_t tagType){ +char* parseTagDatatoJson(void *p){ char* string = NULL; cJSON *json = cJSON_CreateObject(); if (json == NULL) @@ -5237,30 +5232,23 @@ char* parseTagDatatoJson(void *p, uint8_t tagType){ } if (j%2 != 0) { // json key memset(tagJsonKey, 0, TSDB_MAX_TAGS_LEN); - if (tagType == TSDB_DATA_TYPE_JSON_BINARY){ - strncpy(tagJsonKey, varDataVal(val), varDataLen(val)); - } else if(tagType == TSDB_DATA_TYPE_JSON_NCHAR){ - int32_t length = taosUcs4ToMbs(varDataVal(val), varDataLen(val), tagJsonKey); - if (length == 0) { - tscError("charset:%s to %s. val:%s convert json key failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, (char*)val); - goto end; - } + int32_t length = taosUcs4ToMbs(varDataVal(val), varDataLen(val), tagJsonKey); + if (length == 0) { + tscError("charset:%s to %s. val:%s convert json key failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, (char*)val); + goto end; } }else{ // json value char tagJsonValue[TSDB_MAX_TAGS_LEN] = {0}; char* realData = POINTER_SHIFT(val, CHAR_BYTES); char type = *(char*)val; - if(IS_VAR_DATA_TYPE(type)){ - if (type == TSDB_DATA_TYPE_BINARY){ - strncpy(tagJsonValue, varDataVal(realData), varDataLen(realData)); - } else if(type == TSDB_DATA_TYPE_NCHAR) { - int32_t length = taosUcs4ToMbs(varDataVal(realData), varDataLen(realData), tagJsonValue); - if (length == 0) { - tscError("charset:%s to %s. val:%s convert json value failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, - (char*)val); - goto end; - } + if(type == TSDB_DATA_TYPE_NCHAR) { + int32_t length = taosUcs4ToMbs(varDataVal(realData), varDataLen(realData), tagJsonValue); + if (length == 0) { + tscError("charset:%s to %s. val:%s convert json value failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, + (char*)val); + goto end; } + cJSON* value = cJSON_CreateString(tagJsonValue); if (value == NULL) { @@ -5295,7 +5283,7 @@ end: return string; } -int parseJsontoTagData(char* json, SKVRowBuilder* kvRowBuilder, char* errMsg, int16_t startColId, uint8_t type){ +int parseJsontoTagData(char* json, SKVRowBuilder* kvRowBuilder, char* errMsg, int16_t startColId){ cJSON *root = cJSON_Parse(json); if (root == NULL){ tscError("json parse error"); @@ -5318,49 +5306,38 @@ int parseJsontoTagData(char* json, SKVRowBuilder* kvRowBuilder, char* errMsg, in retCode = tscSQLSyntaxErrMsg(errMsg, "json inner error", NULL); goto end; } - if(strlen(item->string) >= TSDB_COL_NAME_LEN){ + if(strlen(item->string) >= TSDB_MAX_JSON_KEY_LEN){ tscError("json key too long error"); - retCode = tscSQLSyntaxErrMsg(errMsg, "json key too long, exceed 64", NULL); + retCode = tscSQLSyntaxErrMsg(errMsg, "json key too long, exceed 256", NULL); goto end; } - char tagVal[TSDB_MAX_TAGS_LEN] = {0}; + // json key encode by binary + char tagKey[TSDB_MAX_JSON_KEY_LEN + VARSTR_HEADER_SIZE] = {0}; int32_t outLen = 0; - if (type == TSDB_DATA_TYPE_JSON_BINARY){ - strncpy(varDataVal(tagVal), item->string, strlen(item->string)); - outLen = strlen(item->string); - }else if(type == TSDB_DATA_TYPE_JSON_NCHAR){ - if (!taosMbsToUcs4(item->string, strlen(item->string), varDataVal(tagVal), TSDB_MAX_TAGS_LEN - VARSTR_HEADER_SIZE, &outLen)) { - tscError("json string error:%s|%s", strerror(errno), item->string); - retCode = tscSQLSyntaxErrMsg(errMsg, "serizelize json error", NULL); - goto end; - } - } + strncpy(varDataVal(tagKey), item->string, strlen(item->string)); + outLen = strlen(item->string); - varDataSetLen(tagVal, outLen); - tdAddColToKVRow(kvRowBuilder, jsonIndex++, TSDB_DATA_TYPE_NCHAR, tagVal, false); // add json key + varDataSetLen(tagKey, outLen); + tdAddColToKVRow(kvRowBuilder, jsonIndex++, TSDB_DATA_TYPE_NCHAR, tagKey, false); // add json key - memset(tagVal, 0, TSDB_MAX_TAGS_LEN); if(item->type == cJSON_String){ // add json value format: type|data outLen = 0; - *tagVal = jsonType2DbType(0, item->type, type); // type + char tagVal[TSDB_MAX_TAGS_LEN + VARSTR_HEADER_SIZE + CHAR_BYTES] = {0}; + *tagVal = jsonType2DbType(0, item->type); // type char* tagData = POINTER_SHIFT(tagVal,CHAR_BYTES); - if (type == TSDB_DATA_TYPE_JSON_BINARY){ - strncpy(varDataVal(tagData), item->valuestring, strlen(item->valuestring)); - outLen = strlen(item->valuestring); - }else if(type == TSDB_DATA_TYPE_JSON_NCHAR) { - if (!taosMbsToUcs4(item->valuestring, strlen(item->valuestring), varDataVal(tagData), - TSDB_MAX_TAGS_LEN - VARSTR_HEADER_SIZE, &outLen)) { - tscError("json string error:%s|%s", strerror(errno), item->string); - retCode = tscSQLSyntaxErrMsg(errMsg, "serizelize json error", NULL); - goto end; - } + if (!taosMbsToUcs4(item->valuestring, strlen(item->valuestring), varDataVal(tagData), + TSDB_MAX_TAGS_LEN, &outLen)) { + tscError("json string error:%s|%s", strerror(errno), item->string); + retCode = tscSQLSyntaxErrMsg(errMsg, "serizelize json error", NULL); + goto end; } varDataSetLen(tagData, outLen); tdAddColToKVRow(kvRowBuilder, jsonIndex++, TSDB_DATA_TYPE_NCHAR, tagVal, true); }else if(item->type == cJSON_Number){ - *tagVal = jsonType2DbType(item->valuedouble, item->type, type); // type + char tagVal[LONG_BYTES + CHAR_BYTES] = {0}; + *tagVal = jsonType2DbType(item->valuedouble, item->type); // type char* tagData = POINTER_SHIFT(tagVal,CHAR_BYTES); if(*tagVal == TSDB_DATA_TYPE_DOUBLE) *((double *)tagData) = item->valuedouble; else if(*tagVal == TSDB_DATA_TYPE_BIGINT) *((int64_t *)tagData) = item->valueint; @@ -5376,12 +5353,12 @@ end: return retCode; } -int8_t jsonType2DbType(double data, int jsonType, uint8_t type){ +int8_t jsonType2DbType(double data, int jsonType){ switch(jsonType){ case cJSON_Number: if (data - (int64_t)data > 0) return TSDB_DATA_TYPE_DOUBLE; else return TSDB_DATA_TYPE_BIGINT; case cJSON_String: - if (type == TSDB_DATA_TYPE_JSON_NCHAR) return TSDB_DATA_TYPE_NCHAR; else return TSDB_DATA_TYPE_BINARY; + return TSDB_DATA_TYPE_NCHAR; case cJSON_True: case cJSON_False: return TSDB_DATA_TYPE_BOOL; @@ -5389,21 +5366,10 @@ int8_t jsonType2DbType(double data, int jsonType, uint8_t type){ return TSDB_DATA_TYPE_NULL; } -void* getJsonTagValue(STable* pTable, char* key, int32_t keyLen, uint8_t jsonType, int16_t* retColId){ - int32_t outLen = 0; - if(jsonType == TSDB_DATA_TYPE_JSON_NCHAR){ - char tagKey[256] = {0}; - if (!taosMbsToUcs4(key, keyLen, tagKey, 256, &outLen)) { - tscError("json key to ucs4 error:%s|%s", strerror(errno), key); - return NULL; - } - key = tagKey; - }else{ - outLen = keyLen; - } +void* getJsonTagValue(STable* pTable, char* key, int32_t keyLen, int16_t* retColId){ if(TABLE_TYPE(pTable) == TSDB_CHILD_TABLE){ STable* superTable= pTable->pSuper; - SArray** data = (SArray**)taosHashGet(superTable->jsonKeyMap, key, outLen); + SArray** data = (SArray**)taosHashGet(superTable->jsonKeyMap, key, keyLen); if(data == NULL) return NULL; JsonMapValue jmvalue = {pTable, 0}; JsonMapValue* p = taosArraySearch(*data, &jmvalue, tsdbCompareJsonMapValue, TD_EQ); @@ -5412,7 +5378,7 @@ void* getJsonTagValue(STable* pTable, char* key, int32_t keyLen, uint8_t jsonTyp if(retColId) *retColId = p->colId; return tdGetKVRowValOfCol(pTable->tagVal, colId); }else if(TABLE_TYPE(pTable) == TSDB_SUPER_TABLE){ - SArray** data = (SArray**)taosHashGet(pTable->jsonKeyMap, key, outLen); + SArray** data = (SArray**)taosHashGet(pTable->jsonKeyMap, key, keyLen); if(data == NULL) return NULL; if(taosArrayGetSize(*data) == 0) return NULL; JsonMapValue* p = taosArrayGet(*data, 0); @@ -5421,5 +5387,4 @@ void* getJsonTagValue(STable* pTable, char* key, int32_t keyLen, uint8_t jsonTyp return tdGetKVRowValOfCol(((STable*)(p->table))->tagVal, colId); } return NULL; -} - +} \ No newline at end of file diff --git a/src/common/src/tname.c b/src/common/src/tname.c index 0e96a1523c..884875d50e 100644 --- a/src/common/src/tname.c +++ b/src/common/src/tname.c @@ -190,7 +190,7 @@ static bool doValidateSchema(SSchema* pSchema, int32_t numOfCols, int32_t maxLen int32_t rowLen = 0; for (int32_t i = 0; i < numOfCols; ++i) { - if (IS_JSON_DATA_TYPE(pSchema[i].type) && numOfCols != 1){ + if (pSchema[i].type == TSDB_DATA_TYPE_JSON && numOfCols != 1){ return false; } // 1. valid types @@ -199,11 +199,11 @@ static bool doValidateSchema(SSchema* pSchema, int32_t numOfCols, int32_t maxLen } // 2. valid length for each type - if (pSchema[i].type == TSDB_DATA_TYPE_BINARY || pSchema[i].type == TSDB_DATA_TYPE_JSON_BINARY) { + if (pSchema[i].type == TSDB_DATA_TYPE_BINARY) { if (pSchema[i].bytes > TSDB_MAX_BINARY_LEN) { return false; } - } else if (pSchema[i].type == TSDB_DATA_TYPE_NCHAR || pSchema[i].type == TSDB_DATA_TYPE_JSON_NCHAR) { + } else if (pSchema[i].type == TSDB_DATA_TYPE_NCHAR || pSchema[i].type == TSDB_DATA_TYPE_JSON) { if (pSchema[i].bytes > TSDB_MAX_NCHAR_LEN) { return false; } diff --git a/src/common/src/ttypes.c b/src/common/src/ttypes.c index 29822c32a4..96d5a5bb9f 100644 --- a/src/common/src/ttypes.c +++ b/src/common/src/ttypes.c @@ -34,8 +34,7 @@ const int32_t TYPE_BYTES[17] = { sizeof(uint16_t), // TSDB_DATA_TYPE_USMALLINT sizeof(uint32_t), // TSDB_DATA_TYPE_UINT sizeof(uint64_t), // TSDB_DATA_TYPE_UBIGINT - sizeof(int8_t), // TSDB_DATA_TYPE_JSON_BINARY - sizeof(int8_t), // TSDB_DATA_TYPE_JSON_NCHAR + sizeof(int8_t), // TSDB_DATA_TYPE_JSON }; #define DO_STATICS(__sum, __min, __max, __minIndex, __maxIndex, _list, _index) \ @@ -369,7 +368,7 @@ static void getStatics_nchr(const void *pData, int32_t numOfRow, int64_t *min, i *maxIndex = 0; } -tDataTypeDescriptor tDataTypes[17] = { +tDataTypeDescriptor tDataTypes[16] = { {TSDB_DATA_TYPE_NULL, 6, 1, "NOTYPE", 0, 0, NULL, NULL, NULL}, {TSDB_DATA_TYPE_BOOL, 4, CHAR_BYTES, "BOOL", false, true, tsCompressBool, tsDecompressBool, getStatics_bool}, {TSDB_DATA_TYPE_TINYINT, 7, CHAR_BYTES, "TINYINT", INT8_MIN, INT8_MAX, tsCompressTinyint, tsDecompressTinyint, getStatics_i8}, @@ -385,8 +384,7 @@ tDataTypeDescriptor tDataTypes[17] = { {TSDB_DATA_TYPE_USMALLINT, 17, SHORT_BYTES, "SMALLINT UNSIGNED", 0, UINT16_MAX, tsCompressSmallint, tsDecompressSmallint, getStatics_u16}, {TSDB_DATA_TYPE_UINT, 12, INT_BYTES, "INT UNSIGNED", 0, UINT32_MAX, tsCompressInt, tsDecompressInt, getStatics_u32}, {TSDB_DATA_TYPE_UBIGINT, 15, LONG_BYTES, "BIGINT UNSIGNED", 0, UINT64_MAX, tsCompressBigint, tsDecompressBigint, getStatics_u64}, - {TSDB_DATA_TYPE_JSON_BINARY,11, 0, "JSON BINARY", 0, 0, tsCompressString, tsDecompressString, getStatics_bin}, - {TSDB_DATA_TYPE_JSON_NCHAR, 10, 0, "JSON NCHAR", 0, 0, tsCompressString, tsDecompressString, getStatics_nchr}, + {TSDB_DATA_TYPE_JSON,4, 0, "JSON", 0, 0, tsCompressString, tsDecompressString, getStatics_nchr}, }; char tTokenTypeSwitcher[13] = { @@ -432,7 +430,7 @@ FORCE_INLINE void* getDataMax(int32_t type) { bool isValidDataType(int32_t type) { - return type >= TSDB_DATA_TYPE_NULL && type <= TSDB_DATA_TYPE_JSON_NCHAR; + return type >= TSDB_DATA_TYPE_NULL && type <= TSDB_DATA_TYPE_JSON; } void setVardataNull(void* val, int32_t type) { diff --git a/src/common/src/tvariant.c b/src/common/src/tvariant.c index 6cfc53bafe..4e2b7039e6 100644 --- a/src/common/src/tvariant.c +++ b/src/common/src/tvariant.c @@ -159,7 +159,7 @@ void tVariantCreateFromBinary(tVariant *pVar, const char *pz, size_t len, uint32 break; } case TSDB_DATA_TYPE_NCHAR: - case TSDB_DATA_TYPE_JSON_NCHAR:{ // here we get the nchar length from raw binary bits length + case TSDB_DATA_TYPE_JSON:{ // here we get the nchar length from raw binary bits length size_t lenInwchar = len / TSDB_NCHAR_SIZE; pVar->wpz = calloc(1, (lenInwchar + 1) * TSDB_NCHAR_SIZE); @@ -168,8 +168,7 @@ void tVariantCreateFromBinary(tVariant *pVar, const char *pz, size_t len, uint32 break; } - case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_JSON_BINARY:{ + case TSDB_DATA_TYPE_BINARY:{ pVar->pz = calloc(len + 1, sizeof(char)); memcpy(pVar->pz, pz, len); pVar->nLen = (int32_t)len; @@ -920,13 +919,12 @@ int32_t tVariantDumpEx(tVariant *pVariant, char *payload, int16_t type, bool inc break; } - case TSDB_DATA_TYPE_JSON_BINARY: - case TSDB_DATA_TYPE_JSON_NCHAR:{ + case TSDB_DATA_TYPE_JSON:{ if (pVariant->nType == TSDB_DATA_TYPE_NULL) { //*(int8_t *)payload = TSDB_DATA_TINYINT_NULL; } else if (pVariant->nType == TSDB_DATA_TYPE_BINARY){ //*((int8_t *)payload) = TSDB_DATA_BINARY_PLACEHOLDER; - } else if (IS_JSON_DATA_TYPE(pVariant->nType)){ // select * from stable, set tag type to json，from setTagValue/tag_project_function + } else if (pVariant->nType == TSDB_DATA_TYPE_JSON){ // select * from stable, set tag type to json，from setTagValue/tag_project_function memcpy(payload, pVariant->pz, pVariant->nLen); }else { return -1; diff --git a/src/connector/python/taos/bind.py b/src/connector/python/taos/bind.py index 23f797dc87..05659714ef 100644 --- a/src/connector/python/taos/bind.py +++ b/src/connector/python/taos/bind.py @@ -124,7 +124,7 @@ class TaosBind(ctypes.Structure): self.buffer_length = length self.length = pointer(c_size_t(self.buffer_length)) - def json_binary(self, value): + def json(self, value): buffer = None length = 0 if isinstance(value, str): @@ -134,22 +134,7 @@ class TaosBind(ctypes.Structure): else: buffer = value length = len(value) - self.buffer_type = FieldType.C_JSON_BINARY - self.buffer = cast(buffer, c_void_p) - self.buffer_length = length - self.length = pointer(c_size_t(self.buffer_length)) - - def json_nchar(self, value): - buffer = None - length = 0 - if isinstance(value, str): - bytes = value.encode("utf-8") - buffer = create_string_buffer(bytes) - length = len(bytes) - else: - buffer = value - length = len(value) - self.buffer_type = FieldType.C_JSON_NCHAR + self.buffer_type = FieldType.C_JSON self.buffer = cast(buffer, c_void_p) self.buffer_length = length self.length = pointer(c_size_t(self.buffer_length)) @@ -386,13 +371,9 @@ class TaosMultiBind(ctypes.Structure): self.buffer_type = FieldType.C_NCHAR self._str_to_buffer(values) - def json_binary(self, values): - self.buffer_type = FieldType.C_JSON_BINARY - self._str_to_buffer(values) - - def json_nchar(self, values): + def json(self, values): # type: (list[str]) -> None - self.buffer_type = FieldType.C_JSON_NCHAR + self.buffer_type = FieldType.C_JSON self._str_to_buffer(values) def tinyint_unsigned(self, values): diff --git a/src/connector/python/taos/constants.py b/src/connector/python/taos/constants.py index 4d9591e0f3..34044a15fc 100644 --- a/src/connector/python/taos/constants.py +++ b/src/connector/python/taos/constants.py @@ -25,8 +25,7 @@ class FieldType(object): C_SMALLINT_UNSIGNED = 12 C_INT_UNSIGNED = 13 C_BIGINT_UNSIGNED = 14 - C_JSON_BINARY = 15 - C_JSON_NCHAR = 16 + C_JSON = 15 # NULL value definition # NOTE: These values should change according to C definition in tsdb.h C_BOOL_NULL = 0x02 diff --git a/src/connector/python/taos/cursor.py b/src/connector/python/taos/cursor.py index 87d625fdc3..a8d82bea2e 100644 --- a/src/connector/python/taos/cursor.py +++ b/src/connector/python/taos/cursor.py @@ -188,11 +188,8 @@ class TaosCursor(object): if dataType.upper() == "NCHAR": if self._description[col][1] == FieldType.C_NCHAR: return True - if dataType.upper() == "JSON BINARY": - if self._description[col][1] == FieldType.C_JSON_BINARY: - return True - if dataType.upper() == "JSON NCHAR": - if self._description[col][1] == FieldType.C_JSON_NCHAR: + if dataType.upper() == "JSON": + if self._description[col][1] == FieldType.C_JSON: return True return False diff --git a/src/connector/python/taos/field.py b/src/connector/python/taos/field.py index 9743c44b1d..9d1799d367 100644 --- a/src/connector/python/taos/field.py +++ b/src/connector/python/taos/field.py @@ -207,8 +207,7 @@ CONVERT_FUNC = { FieldType.C_SMALLINT_UNSIGNED: _crow_smallint_unsigned_to_python, FieldType.C_INT_UNSIGNED: _crow_int_unsigned_to_python, FieldType.C_BIGINT_UNSIGNED: _crow_bigint_unsigned_to_python, - FieldType.C_JSON_NCHAR: _crow_nchar_to_python, - FieldType.C_JSON_BINARY: _crow_binary_to_python, + FieldType.C_JSON: _crow_nchar_to_python, } CONVERT_FUNC_BLOCK = { @@ -226,8 +225,7 @@ CONVERT_FUNC_BLOCK = { FieldType.C_SMALLINT_UNSIGNED: _crow_smallint_unsigned_to_python, FieldType.C_INT_UNSIGNED: _crow_int_unsigned_to_python, FieldType.C_BIGINT_UNSIGNED: _crow_bigint_unsigned_to_python, - FieldType.C_JSON_NCHAR: _crow_nchar_to_python_block, - FieldType.C_JSON_BINARY: _crow_binary_to_python_block, + FieldType.C_JSON: _crow_nchar_to_python_block, } # Corresponding TAOS_FIELD structure in C diff --git a/src/inc/taos.h b/src/inc/taos.h index 6c3885aec4..0ed2a04d8d 100644 --- a/src/inc/taos.h +++ b/src/inc/taos.h @@ -46,8 +46,7 @@ typedef void **TAOS_ROW; #define TSDB_DATA_TYPE_USMALLINT 12 // 2 bytes #define TSDB_DATA_TYPE_UINT 13 // 4 bytes #define TSDB_DATA_TYPE_UBIGINT 14 // 8 bytes -#define TSDB_DATA_TYPE_JSON_BINARY 15 // json string -#define TSDB_DATA_TYPE_JSON_NCHAR 16 // json unicode string +#define TSDB_DATA_TYPE_JSON 15 // json string typedef enum { TSDB_OPTION_LOCALE, diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index 807ed02edd..c510b291f8 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -221,6 +221,8 @@ do { \ #define TSDB_MAX_TAGS_LEN 16384 #define TSDB_MAX_TAGS 128 #define TSDB_MAX_TAG_CONDITIONS 1024 +#define TSDB_MAX_JSON_KEY_LEN 256 +#define TSDB_MAX_JSON_KEY_MD5_LEN 16 #define TSDB_AUTH_LEN 16 #define TSDB_KEY_LEN 16 diff --git a/src/inc/ttype.h b/src/inc/ttype.h index 769527291d..e73e4499c8 100644 --- a/src/inc/ttype.h +++ b/src/inc/ttype.h @@ -41,7 +41,6 @@ typedef struct { #define varDataLenByData(v) (*(VarDataLenT *)(((char*)(v)) - VARSTR_HEADER_SIZE)) #define varDataSetLen(v, _len) (((VarDataLenT *)(v))[0] = (VarDataLenT) (_len)) #define IS_VAR_DATA_TYPE(t) (((t) == TSDB_DATA_TYPE_BINARY) || ((t) == TSDB_DATA_TYPE_NCHAR)) -#define IS_JSON_DATA_TYPE(t) (((t) == TSDB_DATA_TYPE_JSON_BINARY) || ((t) == TSDB_DATA_TYPE_JSON_NCHAR)) #define varDataNetLen(v) (htons(((VarDataLenT *)(v))[0])) #define varDataNetTLen(v) (sizeof(VarDataLenT) + varDataNetLen(v)) @@ -193,7 +192,7 @@ typedef struct tDataTypeDescriptor { int16_t *minindex, int16_t *maxindex, int16_t *numofnull); } tDataTypeDescriptor; -extern tDataTypeDescriptor tDataTypes[17]; +extern tDataTypeDescriptor tDataTypes[16]; bool isValidDataType(int32_t type); diff --git a/src/kit/shell/src/shellEngine.c b/src/kit/shell/src/shellEngine.c index cc2abb8208..26e760de9a 100644 --- a/src/kit/shell/src/shellEngine.c +++ b/src/kit/shell/src/shellEngine.c @@ -683,8 +683,7 @@ static void printField(const char* val, TAOS_FIELD* field, int width, int32_t le break; case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: - case TSDB_DATA_TYPE_JSON_BINARY: - case TSDB_DATA_TYPE_JSON_NCHAR: + case TSDB_DATA_TYPE_JSON: shellPrintNChar(val, length, width); break; case TSDB_DATA_TYPE_TIMESTAMP: @@ -792,7 +791,6 @@ static int calcColWidth(TAOS_FIELD* field, int precision) { return MAX(25, width); case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_JSON_BINARY: if (field->bytes > tsMaxBinaryDisplayWidth) { return MAX(tsMaxBinaryDisplayWidth, width); } else { @@ -800,7 +798,7 @@ static int calcColWidth(TAOS_FIELD* field, int precision) { } case TSDB_DATA_TYPE_NCHAR: - case TSDB_DATA_TYPE_JSON_NCHAR:{ + case TSDB_DATA_TYPE_JSON:{ int16_t bytes = field->bytes * TSDB_NCHAR_SIZE; if (bytes > tsMaxBinaryDisplayWidth) { return MAX(tsMaxBinaryDisplayWidth, width); diff --git a/src/query/inc/qFilter.h b/src/query/inc/qFilter.h index 2a522c9a6b..9b04886b04 100644 --- a/src/query/inc/qFilter.h +++ b/src/query/inc/qFilter.h @@ -252,7 +252,7 @@ typedef struct SFilterInfo { #define COL_FIELD_SIZE (sizeof(SFilterField) + 2 * sizeof(int64_t)) -#define FILTER_NO_MERGE_DATA_TYPE(t) ((t) == TSDB_DATA_TYPE_BINARY || (t) == TSDB_DATA_TYPE_NCHAR || (t) == TSDB_DATA_TYPE_JSON_BINARY || (t) == TSDB_DATA_TYPE_JSON_NCHAR) +#define FILTER_NO_MERGE_DATA_TYPE(t) ((t) == TSDB_DATA_TYPE_BINARY || (t) == TSDB_DATA_TYPE_NCHAR || (t) == TSDB_DATA_TYPE_JSON) #define FILTER_NO_MERGE_OPTR(o) ((o) == TSDB_RELATION_ISNULL || (o) == TSDB_RELATION_NOTNULL || (o) == FILTER_DUMMY_EMPTY_OPTR) #define MR_EMPTY_RES(ctx) (ctx->rs == NULL) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index c98e287eda..8c9b5b0c9c 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -3357,7 +3357,7 @@ static void doSetTagValueInParam(void* pTable, int32_t tagColId, tVariant *tag, int32_t len = (varDataLen(val) > maxLen)? maxLen:varDataLen(val); tVariantCreateFromBinary(tag, varDataVal(val), len, type); //tVariantCreateFromBinary(tag, varDataVal(val), varDataLen(val), type); - } else if(IS_JSON_DATA_TYPE(type)){ + } else if(type == TSDB_DATA_TYPE_JSON){ assert(kvRowLen(val) < bytes); tVariantCreateFromBinary(tag, val, bytes, type); memcpy(tag->pz + 1, tag->pz, bytes - 1); // move back 1 byte for select type @@ -4335,7 +4335,7 @@ static void doCopyQueryResultToMsg(SQInfo *pQInfo, int32_t numOfRows, char *data *compLen += compSizes[col]; compSizes[col] = htonl(compSizes[col]); } else { - if(IS_JSON_DATA_TYPE(pColRes->info.type)){ // todo json + if(pColRes->info.type == TSDB_DATA_TYPE_JSON){ // todo json //pColRes->info.bytes = } @@ -7187,10 +7187,12 @@ static SSDataBlock* doTagScan(void* param, bool* newgroup) { data = tsdbGetTableName(item->pTable); } else { data = tsdbGetTableTagVal(item->pTable, pExprInfo[j].base.colInfo.colId, type, bytes); - if(IS_JSON_DATA_TYPE(type)){ + if(type == TSDB_DATA_TYPE_JSON){ if(pExprInfo[j].base.numOfParams > 0){ // tag-> operation char* tagJsonElementData = calloc(bytes, 1); - findTagValue(item->pTable, pExprInfo[j].base.param[0].pz, pExprInfo[j].base.param[0].nLen, tagJsonElementData, bytes, type); + char keyMd5[TSDB_MAX_JSON_KEY_MD5_LEN] = {0}; + jsonKeyMd5(pExprInfo[j].base.param[0].pz, pExprInfo[j].base.param[0].nLen, keyMd5); + findTagValue(item->pTable, keyMd5, TSDB_MAX_JSON_KEY_MD5_LEN, tagJsonElementData, bytes); *dst = SELECT_ELEMENT_JSON_TAG; // select tag->element dst++; assert(varDataTLen(tagJsonElementData) < bytes); diff --git a/src/query/src/qFilter.c b/src/query/src/qFilter.c index 0f71f0fd8c..c15a8c779d 100644 --- a/src/query/src/qFilter.c +++ b/src/query/src/qFilter.c @@ -199,8 +199,7 @@ int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) { case TSDB_DATA_TYPE_TIMESTAMP: comparFn = 3; break; case TSDB_DATA_TYPE_FLOAT: comparFn = 4; break; case TSDB_DATA_TYPE_DOUBLE: comparFn = 5; break; - case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_JSON_BINARY:{ + case TSDB_DATA_TYPE_BINARY:{ if (optr == TSDB_RELATION_MATCH) { comparFn = 19; } else if (optr == TSDB_RELATION_NMATCH) { @@ -219,7 +218,7 @@ int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) { } case TSDB_DATA_TYPE_NCHAR: - case TSDB_DATA_TYPE_JSON_NCHAR:{ + case TSDB_DATA_TYPE_JSON:{ if (optr == TSDB_RELATION_MATCH) { comparFn = 19; } else if (optr == TSDB_RELATION_NMATCH) { @@ -1162,24 +1161,26 @@ int32_t filterAddGroupUnitFromNode(SFilterInfo *info, tExprNode* tree, SArray *g if(pLeft->nodeType == TSQL_NODE_EXPR && pLeft->_node.optr == TSDB_RELATION_ARROW){ // json tag -> operation assert(info->pTable != NULL); SSchema* schema = pLeft->_node.pLeft->pSchema; - if(pLeft->_node.pRight->pVal->nLen >= TSDB_COL_NAME_LEN) return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; - memset(schema->name, 0, TSDB_COL_NAME_LEN); - strncpy(schema->name, pLeft->_node.pRight->pVal->pz, pLeft->_node.pRight->pVal->nLen); + if(pLeft->_node.pRight->pVal->nLen >= TSDB_MAX_JSON_KEY_LEN) return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; + char keyMd5[TSDB_MAX_JSON_KEY_MD5_LEN] = {0}; + jsonKeyMd5(pLeft->_node.pRight->pVal->pz, pLeft->_node.pRight->pVal->nLen, keyMd5); + strncpy(schema->name, keyMd5, TSDB_MAX_JSON_KEY_MD5_LEN); - void* data = getJsonTagValue(info->pTable, schema->name, strlen(schema->name), schema->type, &schema->colId); + void* data = getJsonTagValue(info->pTable, schema->name, strlen(schema->name), &schema->colId); if(data == NULL) return TSDB_CODE_QRY_JSON_KEY_NOT_EXIST; schema->type = *(char*)data; // if exist json tag-> operation get type so that can set data if (tree->_node.optr == TSDB_RELATION_IN_IN) the next and set value in filterInitValFieldData - assert(schema->type > TSDB_DATA_TYPE_NULL && schema->type < TSDB_DATA_TYPE_JSON_BINARY); - if((tree->_node.optr == TSDB_RELATION_MATCH || tree->_node.optr == TSDB_RELATION_NMATCH) - && schema->type != TSDB_DATA_TYPE_BINARY) - return TSDB_CODE_QRY_JSON_KEY_NOT_STR_ERROR; + assert(schema->type > TSDB_DATA_TYPE_NULL && schema->type < TSDB_DATA_TYPE_JSON); +// if((tree->_node.optr == TSDB_RELATION_MATCH || tree->_node.optr == TSDB_RELATION_NMATCH) +// && schema->type != TSDB_DATA_TYPE_BINARY) +// return TSDB_CODE_QRY_JSON_KEY_NOT_STR_ERROR; pLeft = pLeft->_node.pLeft; // -> operation use left as input }else if(tree->_node.optr == TSDB_RELATION_QUESTION){ SSchema* schema = pLeft->pSchema; - if(tree->_node.pRight->pVal->nLen >= TSDB_COL_NAME_LEN) return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; - memset(schema->name, 0, TSDB_COL_NAME_LEN); - strncpy(schema->name, tree->_node.pRight->pVal->pz, tree->_node.pRight->pVal->nLen); + if(tree->_node.pRight->pVal->nLen >= TSDB_MAX_JSON_KEY_LEN) return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; + char keyMd5[TSDB_MAX_JSON_KEY_MD5_LEN] = {0}; + jsonKeyMd5(tree->_node.pRight->pVal->pz, tree->_node.pRight->pVal->nLen, keyMd5); + strncpy(schema->name, keyMd5, TSDB_MAX_JSON_KEY_MD5_LEN); } SFilterFieldId left = {0}, right = {0}; @@ -1835,10 +1836,10 @@ int32_t filterInitValFieldData(SFilterInfo *info) { continue; } - if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_JSON_BINARY) { + if (type == TSDB_DATA_TYPE_BINARY) { size_t len = (var->nType == TSDB_DATA_TYPE_BINARY || var->nType == TSDB_DATA_TYPE_NCHAR) ? var->nLen : MAX_NUM_STR_SIZE; fi->data = calloc(1, len + 1 + VARSTR_HEADER_SIZE); - } else if (type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_JSON_NCHAR) { + } else if (type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_JSON) { size_t len = (var->nType == TSDB_DATA_TYPE_BINARY || var->nType == TSDB_DATA_TYPE_NCHAR) ? var->nLen : MAX_NUM_STR_SIZE; fi->data = calloc(1, (len + 1) * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE); } else { diff --git a/src/query/src/qSqlParser.c b/src/query/src/qSqlParser.c index 34bac367da..ad4b3bf4f2 100644 --- a/src/query/src/qSqlParser.c +++ b/src/query/src/qSqlParser.c @@ -702,18 +702,6 @@ void tSetColumnType(TAOS_FIELD *pField, SStrToken *type) { break; } - if((type->n == 4) && - (strncasecmp(type->z, "json", type->n) == 0) && JSON_TYPE_NCHAR){ - i = TSDB_DATA_TYPE_JSON_NCHAR; - break; - } - - if((type->n == 4) && - (strncasecmp(type->z, "json", type->n) == 0) && JSON_TYPE_BINARY){ - i = TSDB_DATA_TYPE_JSON_BINARY; - break; - } - i += 1; } @@ -728,7 +716,7 @@ void tSetColumnType(TAOS_FIELD *pField, SStrToken *type) { pField->type = i; pField->bytes = tDataTypes[i].bytes; - if (i == TSDB_DATA_TYPE_NCHAR || i == TSDB_DATA_TYPE_JSON_NCHAR) { + if (i == TSDB_DATA_TYPE_NCHAR || i == TSDB_DATA_TYPE_JSON) { /* * for nchar, the TOKENTYPE is the number of character, so the length is the * number of bytes in UCS-4 format, which is 4 times larger than the number of characters @@ -745,7 +733,7 @@ void tSetColumnType(TAOS_FIELD *pField, SStrToken *type) { } pField->bytes = (int16_t)bytes; } - } else if (i == TSDB_DATA_TYPE_BINARY || i == TSDB_DATA_TYPE_JSON_BINARY) { + } else if (i == TSDB_DATA_TYPE_BINARY) { /* for binary, the TOKENTYPE is the length of binary */ if (type->type == 0) { pField->bytes = 0; diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index e75b118e29..5816ba6b45 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -14,6 +14,7 @@ */ #include "tsdbint.h" #include "tcompare.h" +#include "tutil.h" #define TSDB_SUPER_TABLE_SL_LEVEL 5 #define DEFAULT_TAG_INDEX_COLUMN 0 @@ -211,7 +212,7 @@ void *tsdbGetTableTagVal(const void* pTable, int32_t colId, int16_t type, int16_ } char *val = NULL; - if (IS_JSON_DATA_TYPE(pCol->type)){ + if (pCol->type == TSDB_DATA_TYPE_JSON){ val = ((STable*)pTable)->tagVal; }else{ val = tdGetKVRowValOfCol(((STable*)pTable)->tagVal, colId); @@ -394,7 +395,7 @@ int tsdbUpdateTableTagValue(STsdbRepo *repo, SUpdateTableTagValMsg *pMsg) { } bool isChangeIndexCol = (pMsg->colId == colColId(schemaColAt(pTable->pSuper->tagSchema, 0))) - || IS_JSON_DATA_TYPE(pMsg->type); + || pMsg->type == TSDB_DATA_TYPE_JSON; // STColumn *pCol = bsearch(&(pMsg->colId), pMsg->data, pMsg->numOfTags, sizeof(STColumn), colIdCompar); // ASSERT(pCol != NULL); @@ -403,7 +404,7 @@ int tsdbUpdateTableTagValue(STsdbRepo *repo, SUpdateTableTagValMsg *pMsg) { tsdbRemoveTableFromIndex(pMeta, pTable); } TSDB_WLOCK_TABLE(pTable); - if (IS_JSON_DATA_TYPE(pMsg->type)){ + if (pMsg->type == TSDB_DATA_TYPE_JSON){ kvRowFree(pTable->tagVal); pTable->tagVal = tdKVRowDup(POINTER_SHIFT(pMsg->data, pMsg->schemaLen)); }else{ @@ -857,7 +858,7 @@ static STable *tsdbCreateTableFromCfg(STableCfg *pCfg, bool isSuper, STable *pST } pTable->tagVal = NULL; STColumn *pCol = schemaColAt(pTable->tagSchema, DEFAULT_TAG_INDEX_COLUMN); - if(IS_JSON_DATA_TYPE(pCol->type)){ + if(pCol->type == TSDB_DATA_TYPE_JSON){ assert(pTable->tagSchema->numOfCols == 1); pTable->jsonKeyMap = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); if (pTable->jsonKeyMap == NULL) { @@ -1096,7 +1097,7 @@ static int tsdbAddTableIntoIndex(STsdbMeta *pMeta, STable *pTable, bool refSuper pTable->pSuper = pSTable; - if(IS_JSON_DATA_TYPE(pSTable->tagSchema->columns[0].type)){ + if(pSTable->tagSchema->columns[0].type == TSDB_DATA_TYPE_JSON){ ASSERT(pSTable->tagSchema->numOfCols == 1); int16_t nCols = kvRowNCols(pTable->tagVal); ASSERT(nCols%2 == 1); @@ -1110,8 +1111,10 @@ static int tsdbAddTableIntoIndex(STsdbMeta *pMeta, STable *pTable, bool refSuper continue; } + char keyMd5[TSDB_MAX_JSON_KEY_MD5_LEN] = {0}; + jsonKeyMd5(varDataVal(val), varDataLen(val), keyMd5); SArray* tablistNew = NULL; - SArray** tablist = (SArray**)taosHashGet(pSTable->jsonKeyMap, varDataVal(val), varDataLen(val)); + SArray** tablist = (SArray**)taosHashGet(pSTable->jsonKeyMap, keyMd5, TSDB_MAX_JSON_KEY_MD5_LEN); if(tablist == NULL) { tablistNew = taosArrayInit(8, sizeof(JsonMapValue)); if(tablistNew == NULL){ @@ -1119,7 +1122,7 @@ static int tsdbAddTableIntoIndex(STsdbMeta *pMeta, STable *pTable, bool refSuper tsdbError("out of memory when alloc json tag array"); return -1; } - if(taosHashPut(pSTable->jsonKeyMap, varDataVal(val) ,varDataLen(val), &tablistNew, sizeof(void*)) < 0){ + if(taosHashPut(pSTable->jsonKeyMap, keyMd5, TSDB_MAX_JSON_KEY_MD5_LEN, &tablistNew, sizeof(void*)) < 0){ terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; tsdbError("out of memory when put json tag array"); return -1; @@ -1167,7 +1170,7 @@ static int tsdbRemoveTableFromIndex(STsdbMeta *pMeta, STable *pTable) { STable *pSTable = pTable->pSuper; ASSERT(pSTable != NULL); - if(IS_JSON_DATA_TYPE(pSTable->tagSchema->columns[0].type)){ + if(pSTable->tagSchema->columns[0].type == TSDB_DATA_TYPE_JSON){ ASSERT(pSTable->tagSchema->numOfCols == 1); int16_t nCols = kvRowNCols(pTable->tagVal); ASSERT(nCols%2 == 1); @@ -1450,7 +1453,7 @@ static void *tsdbDecodeTable(void *buf, STable **pRTable) { if (TABLE_TYPE(pTable) == TSDB_SUPER_TABLE) { buf = tdDecodeSchema(buf, &(pTable->tagSchema)); STColumn *pCol = schemaColAt(pTable->tagSchema, DEFAULT_TAG_INDEX_COLUMN); - if(IS_JSON_DATA_TYPE(pCol->type)){ + if(pCol->type == TSDB_DATA_TYPE_JSON){ assert(pTable->tagSchema->numOfCols == 1); pTable->jsonKeyMap = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); if (pTable->jsonKeyMap == NULL) { diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 5549f83594..60f81292c5 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -2678,7 +2678,7 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int static int32_t getAllTableList(STable* pSuperTable, SArray* list) { STSchema* pTagSchema = tsdbGetTableTagSchema(pSuperTable); - if(pTagSchema && pTagSchema->numOfCols == 1 && IS_JSON_DATA_TYPE(pTagSchema->columns[0].type)){ + if(pTagSchema && pTagSchema->numOfCols == 1 && pTagSchema->columns[0].type == TSDB_DATA_TYPE_JSON){ SArray** pRecord = taosHashIterate(pSuperTable->jsonKeyMap, NULL); SArray* tablist = taosArrayInit(32, sizeof(JsonMapValue)); @@ -3662,7 +3662,7 @@ SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pC // //int32_t dealWithTree(STable* pTable, tExprNode* expr){ // STSchema* pTagSchema = tsdbGetTableTagSchema(pTable); -// if(!IS_JSON_DATA_TYPE(pTagSchema->columns->type)){ +// if(pTagSchema->columns->type != TSDB_DATA_TYPE_JSON){ // return TSDB_CODE_SUCCESS; // } // @@ -4081,7 +4081,7 @@ static FORCE_INLINE int32_t tsdbGetJsonTagDataFromId(void *param, int32_t id, ch if (id == TSDB_TBNAME_COLUMN_INDEX) { *data = TABLE_NAME(pTable); } else { - void* jsonData = getJsonTagValue(pTable, name, strlen(name), pTable->pSuper->tagSchema->columns->type, NULL); + void* jsonData = getJsonTagValue(pTable, name, strlen(name), NULL); if (jsonData != NULL) jsonData += CHAR_BYTES; // jump type *data = jsonData; } @@ -4109,18 +4109,9 @@ static void queryByJsonTag(STable* pTable, void* filterInfo, SArray* res){ SFilterField* fi = &info->fields[FLD_TYPE_COLUMN].fields[i]; SSchema* sch = fi->desc; int32_t outLen = 0; - char* key = NULL; - if(pTable->tagSchema->columns->type == TSDB_DATA_TYPE_JSON_NCHAR){ - char tagKey[256] = {0}; - if (!taosMbsToUcs4(sch->name, strlen(sch->name), tagKey, 256, &outLen)) { - tsdbError("json key to ucs4 error:%s|%s", strerror(errno), sch->name); - return; - } - key = tagKey; - }else{ - key = sch->name; - outLen = strlen(sch->name); - } + char* key = sch->name; + outLen = strlen(sch->name); + SArray** data = (SArray**)taosHashGet(pTable->jsonKeyMap, key, outLen); if(data == NULL) continue; if(tabList == NULL) { @@ -4165,7 +4156,7 @@ static void queryByJsonTag(STable* pTable, void* filterInfo, SArray* res){ static int32_t tsdbQueryTableList(STable* pTable, SArray* pRes, void* filterInfo) { STSchema* pTSSchema = pTable->tagSchema; - if(IS_JSON_DATA_TYPE(pTSSchema->columns->type)){ + if(pTSSchema->columns->type == TSDB_DATA_TYPE_JSON){ queryByJsonTag(pTable, filterInfo, pRes); }else{ bool indexQuery = false; diff --git a/src/util/inc/tutil.h b/src/util/inc/tutil.h index 6bcfb5de29..968b46bfd3 100644 --- a/src/util/inc/tutil.h +++ b/src/util/inc/tutil.h @@ -45,6 +45,7 @@ int taosCheckVersion(char *input_client_version, char *input_server_version, in char * taosIpStr(uint32_t ipInt); uint32_t ip2uint(const char *const ip_addr); +void jsonKeyMd5(char *pMsg, int msgLen, void *pKey); static FORCE_INLINE void taosEncryptPass(uint8_t *inBuf, size_t inLen, char *target) { MD5_CTX context; diff --git a/src/util/src/tutil.c b/src/util/src/tutil.c index 5798bc6537..8da771f001 100644 --- a/src/util/src/tutil.c +++ b/src/util/src/tutil.c @@ -456,6 +456,16 @@ char *taosIpStr(uint32_t ipInt) { return ipStr; } +void jsonKeyMd5(char *pMsg, int msgLen, void *pKey) { + MD5_CTX context; + + MD5Init(&context); + MD5Update(&context, (uint8_t *)pMsg, msgLen); + MD5Final(&context); + + memcpy(pKey, context.digest, sizeof(context.digest)); +} + FORCE_INLINE float taos_align_get_float(const char* pBuf) { #if __STDC_VERSION__ >= 201112L static_assert(sizeof(float) == sizeof(uint32_t), "sizeof(float) must equal to sizeof(uint32_t)"); -- GitLab