diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index d47453d0cac1945fb75e51adc1bfab6d83a40e3f..e3c890e372cf640906dd5cf733d7021899919567 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -378,8 +378,8 @@ char* cloneCurrentDBName(SSqlObj* pSql); int parseJsontoTagData(char* json, SKVRowBuilder* kvRowBuilder, char* errMsg, int16_t startColId); char* parseTagDatatoJson(void *p); -void findTagValue(STable* data, char* key, int32_t keyLen, char** out, int* len); -void parseTagValue2Dst(char* result, char* dst); +void getJsonTagValueElment(STable* data, char* key, int32_t keyLen, char* out, int16_t bytes); +void getJsonTagValueAll(void* data, void* dst, int16_t bytes); int8_t jsonType2DbType(double data, int jsonType); void* getJsonTagValue(STable* pTable, char* key, int32_t keyLen, int16_t* colId); diff --git a/src/client/src/tscPrepare.c b/src/client/src/tscPrepare.c index 186fc6286efba135ead881d3ee84ec5653efd52a..d0ac0ccf4ee4bfa381a78090409a761717ceb4b0 100644 --- a/src/client/src/tscPrepare.c +++ b/src/client/src/tscPrepare.c @@ -905,8 +905,6 @@ static int doBindBatchParam(STableDataBlocks* pBlock, SParamInfo* param, TAOS_MU } varDataSetLen(data + param->offset, output); - } else if (param->type == TSDB_DATA_TYPE_JSON) { // todo json - } } diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index d0121f0639146cef11a1d80d313b2d5d7bc18869..1776cc9a0566756ae18befbaf2c03adf5d68c79e 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -684,7 +684,7 @@ static void setResRawPtrImpl(SSqlRes* pRes, SInternalField* pInfo, int32_t i, bo } } - } else if (convertNchar && pInfo->field.type == TSDB_DATA_TYPE_NCHAR) { + } else if (convertNchar && (pInfo->field.type == TSDB_DATA_TYPE_NCHAR || pInfo->field.type == TSDB_DATA_TYPE_JSON)) { // convert unicode to native code in a temporary buffer extra one byte for terminated symbol char* buffer = realloc(pRes->buffer[i], pInfo->field.bytes * pRes->numOfRows); if(buffer == NULL) @@ -697,9 +697,11 @@ static void setResRawPtrImpl(SSqlRes* pRes, SInternalField* pInfo, int32_t i, bo for (int32_t k = 0; k < pRes->numOfRows; ++k) { char* dst = pRes->buffer[i] + k * pInfo->field.bytes; - if (isNull(p, TSDB_DATA_TYPE_NCHAR)) { + if (isNull(p, TSDB_DATA_TYPE_NCHAR) && pInfo->field.type == TSDB_DATA_TYPE_NCHAR) { memcpy(dst, p, varDataTLen(p)); - } else if (varDataLen(p) > 0) { + } else if(isNull(p, TSDB_DATA_TYPE_JSON && pInfo->field.type == TSDB_DATA_TYPE_JSON)) { + memcpy(dst, p, varDataTLen(p)); + }else if (varDataLen(p) > 0) { int32_t length = taosUcs4ToMbs(varDataVal(p), varDataLen(p), varDataVal(dst)); varDataSetLen(dst, length); @@ -713,42 +715,6 @@ static void setResRawPtrImpl(SSqlRes* pRes, SInternalField* pInfo, int32_t i, bo p += pInfo->field.bytes; } - memcpy(pRes->urow[i], pRes->buffer[i], pInfo->field.bytes * pRes->numOfRows); - }else if (pInfo->field.type == TSDB_DATA_TYPE_JSON) { - // convert unicode to native code in a temporary buffer extra one byte for terminated symbol - char* buffer = realloc(pRes->buffer[i], pInfo->field.bytes * pRes->numOfRows); - if(buffer == NULL) - return ; - pRes->buffer[i] = buffer; - // string terminated char for binary data - memset(pRes->buffer[i], 0, pInfo->field.bytes * pRes->numOfRows); - - char* p = pRes->urow[i]; - for (int32_t k = 0; k < pRes->numOfRows; ++k) { - char* dst = pRes->buffer[i] + k * pInfo->field.bytes; - char* realData = p + CHAR_BYTES; - if (*p == SELECT_ALL_JSON_TAG){ - char* json = parseTagDatatoJson(realData); - if(json) { - memcpy(varDataVal(dst), json, strlen(json)); - varDataSetLen(dst, strlen(json)); - assert(varDataTLen(dst) <= pInfo->field.bytes); - tfree(json); - }else{ - setNull(dst, TSDB_DATA_TYPE_JSON, 0); - } - }else if (*p == SELECT_ELEMENT_JSON_TAG){ - if (isNull(realData, TSDB_DATA_TYPE_JSON)) { - memcpy(dst, realData, varDataTLen(realData)); - }else{ - parseTagValue2Dst(realData, dst); - } - }else{ - tscError("construct json error"); - } - p += pInfo->field.bytes; - } - memcpy(pRes->urow[i], pRes->buffer[i], pInfo->field.bytes * pRes->numOfRows); } } @@ -5182,53 +5148,53 @@ char* cloneCurrentDBName(SSqlObj* pSql) { return p; } -void findTagValue(STable* data, char* key, int32_t keyLen, char** out, int* len){ - void* result = getJsonTagValue(data, key, keyLen, NULL); +void getJsonTagValueElment(STable* data, char* key, int32_t keyLen, char* dst, int16_t bytes){ + char keyMd5[TSDB_MAX_JSON_KEY_MD5_LEN] = {0}; + jsonKeyMd5(key, keyLen, keyMd5); + + void* result = getJsonTagValue(data, keyMd5, TSDB_MAX_JSON_KEY_MD5_LEN, NULL); if (result == NULL){ // json key no result + setNull(dst, TSDB_DATA_TYPE_JSON, 0); return; } - *out = result; + char out[TSDB_MAX_TAGS_LEN] = {0}; char* realData = POINTER_SHIFT(result, CHAR_BYTES); if(*(char*)result == TSDB_DATA_TYPE_NCHAR) { - *len = varDataTLen(realData) + CHAR_BYTES; + assert(varDataTLen(result) <= TSDB_MAX_TAGS_LEN); + varDataCopy(out, result); }else if (*(char*)result == TSDB_DATA_TYPE_DOUBLE) { - *len = DOUBLE_BYTES + CHAR_BYTES; + double jsonVd = *(double*)(realData); + sprintf(varDataVal(out), "%.9lf", jsonVd); + varDataSetLen(out, strlen(varDataVal(out))); }else if (*(char*)result == TSDB_DATA_TYPE_BIGINT) { - *len = LONG_BYTES + CHAR_BYTES; + int64_t jsonVd = *(int64_t*)(realData); + sprintf(varDataVal(out), "%" PRId64, jsonVd); + varDataSetLen(out, strlen(varDataVal(out))); }else if (*(char*)result == TSDB_DATA_TYPE_BOOL) { - *len = CHAR_BYTES + CHAR_BYTES; + sprintf(varDataVal(out), "%s", (*((char *)realData) == 1) ? "true" : "false"); + varDataSetLen(out, strlen(varDataVal(out))); }else { - tscError("unsupportted json value"); - return; + assert(0); } + + int32_t length = 0; + taosMbsToUcs4(varDataVal(out), varDataLen(out), varDataVal(dst), bytes - VARSTR_HEADER_SIZE, &length); + varDataSetLen(dst, length); } -void parseTagValue2Dst(char* result, char* dst){ - char* realData = POINTER_SHIFT(result, CHAR_BYTES); - if(*(char*)result == TSDB_DATA_TYPE_NCHAR) { - char tagJsonValue[TSDB_MAX_TAGS_LEN] = {0}; - int32_t length = taosUcs4ToMbs(varDataVal(realData), - varDataLen(realData), tagJsonValue); - if (length < 0) { - tscError("charset:%s to %s. val:%s convert json value failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, - (char*)result); - return; - } - varDataSetLen(dst, length); - memcpy(varDataVal(dst), tagJsonValue, length); - }else if (*(char*)result == TSDB_DATA_TYPE_DOUBLE) { - double jsonVd = *(double*)(realData); - sprintf(varDataVal(dst), "%.9lf", jsonVd); - varDataSetLen(dst, strlen(varDataVal(dst))); - }else if (*(char*)result == TSDB_DATA_TYPE_BIGINT) { - int64_t jsonVd = *(int64_t*)(realData); - sprintf(varDataVal(dst), "%" PRId64, jsonVd); - varDataSetLen(dst, strlen(varDataVal(dst))); - }else if (*(char*)result == TSDB_DATA_TYPE_BOOL) { - sprintf(varDataVal(dst), "%s", (*((char *)realData) == 1) ? "true" : "false"); - varDataSetLen(dst, strlen(varDataVal(dst))); +void getJsonTagValueAll(void* data, void* dst, int16_t bytes) { + char* json = parseTagDatatoJson(data); + if(json == NULL){ + setNull(dst, TSDB_DATA_TYPE_JSON, 0); + return; } + + assert(strlen(json) <= bytes); + int32_t length = 0; + taosMbsToUcs4(json, strlen(json), varDataVal(dst), bytes - VARSTR_HEADER_SIZE, &length); + varDataSetLen(dst, length); + tfree(json); } char* parseTagDatatoJson(void *p){ diff --git a/src/common/src/ttypes.c b/src/common/src/ttypes.c index 574ff6a9bb9abc21f2620d56ca2e6bd418c677c3..090da68079a48be5b85c86c96e60e5ea53bf272c 100644 --- a/src/common/src/ttypes.c +++ b/src/common/src/ttypes.c @@ -510,13 +510,9 @@ void setNullN(void *val, int32_t type, int32_t bytes, int32_t numOfElems) { break; case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_BINARY: - for (int32_t i = 0; i < numOfElems; ++i) { - setVardataNull(POINTER_SHIFT(val, i * bytes), type); - } - break; case TSDB_DATA_TYPE_JSON: for (int32_t i = 0; i < numOfElems; ++i) { - *(uint8_t *)(POINTER_SHIFT(val, i * tDataTypes[type].bytes)) = TSDB_DATA_JSON_NULL; + setVardataNull(POINTER_SHIFT(val, i * bytes), type); } break; default: { diff --git a/src/inc/ttype.h b/src/inc/ttype.h index bb47bad6cb9a3268b17ec56a735e4de4e03caab3..9d3cf6d2ce498a33dc6e6fb6474a7ae26a52a78c 100644 --- a/src/inc/ttype.h +++ b/src/inc/ttype.h @@ -160,7 +160,7 @@ static FORCE_INLINE bool isNull(const void *val, int32_t type) { case TSDB_DATA_TYPE_DOUBLE: return *(uint64_t *)val == TSDB_DATA_DOUBLE_NULL; case TSDB_DATA_TYPE_JSON: - return *(uint8_t *)val == TSDB_DATA_JSON_NULL; + return varDataLen(val) == sizeof(int8_t) && *(uint8_t *) varDataVal(val) == TSDB_DATA_JSON_NULL; case TSDB_DATA_TYPE_NCHAR: return varDataLen(val) == sizeof(int32_t) && *(uint32_t*) varDataVal(val) == TSDB_DATA_NCHAR_NULL; case TSDB_DATA_TYPE_BINARY: diff --git a/src/query/inc/qUtil.h b/src/query/inc/qUtil.h index b4484fff01562796975ad81e5d9a1e4420c79945..0882df77c2a8bc38560269ce093568fd96467dae 100644 --- a/src/query/inc/qUtil.h +++ b/src/query/inc/qUtil.h @@ -17,9 +17,6 @@ #include "tbuffer.h" -#define SELECT_ALL_JSON_TAG 1 -#define SELECT_ELEMENT_JSON_TAG 2 - #define SET_RES_WINDOW_KEY(_k, _ori, _len, _uid) \ do { \ assert(sizeof(_uid) == sizeof(uint64_t)); \ diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 9252969a5648772bb3e235551b3dd75193c53e67..e5b31d620908357d4ad1280f908e2eecb71643c6 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -3358,11 +3358,9 @@ static void doSetTagValueInParam(void* pTable, int32_t tagColId, tVariant *tag, tVariantCreateFromBinary(tag, varDataVal(val), len, type); //tVariantCreateFromBinary(tag, varDataVal(val), varDataLen(val), type); } else if(type == TSDB_DATA_TYPE_JSON){ - assert(kvRowLen(val) < bytes); - tVariantCreateFromBinary(tag, val, kvRowLen(val), type); - memcpy(tag->pz + 1, tag->pz, kvRowLen(val) - 1); // move back 1 byte for select type - *(tag->pz) = SELECT_ALL_JSON_TAG; - tag->nLen++; + char jsonVal[TSDB_MAX_TAGS_LEN] = {0}; + getJsonTagValueAll(val, jsonVal, TSDB_MAX_TAGS_LEN); + tVariantCreateFromBinary(tag, jsonVal, varDataTLen(jsonVal), type); } else { tVariantCreateFromBinary(tag, val, bytes, type); @@ -4322,7 +4320,7 @@ static void doCopyQueryResultToMsg(SQInfo *pQInfo, int32_t numOfRows, char *data *compLen += compSizes[col]; compSizes[col] = htonl(compSizes[col]); } else { - memmove(data, pColRes->pData, pColRes->info.bytes * pRes->info.rows); // todo json + memmove(data, pColRes->pData, pColRes->info.bytes * pRes->info.rows); data += pColRes->info.bytes * pRes->info.rows; } } @@ -4335,10 +4333,6 @@ static void doCopyQueryResultToMsg(SQInfo *pQInfo, int32_t numOfRows, char *data *compLen += compSizes[col]; compSizes[col] = htonl(compSizes[col]); } else { - if(pColRes->info.type == TSDB_DATA_TYPE_JSON){ // todo json - //pColRes->info.bytes = - - } memmove(data, pColRes->pData, pColRes->info.bytes * numOfRows); data += pColRes->info.bytes * numOfRows; } @@ -7146,7 +7140,7 @@ static SSDataBlock* doTagScan(void* param, bool* newgroup) { if (pExprInfo->base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) { data = tsdbGetTableName(item->pTable); } else { - data = tsdbGetTableTagVal(item->pTable, pExprInfo->base.colInfo.colId, type, bytes); //todo json + data = tsdbGetTableTagVal(item->pTable, pExprInfo->base.colInfo.colId, type, bytes); } doSetTagValueToResultBuf(output, data, type, bytes); @@ -7189,17 +7183,9 @@ static SSDataBlock* doTagScan(void* param, bool* newgroup) { data = tsdbGetTableTagVal(item->pTable, pExprInfo[j].base.colInfo.colId, type, bytes); if(type == TSDB_DATA_TYPE_JSON){ if(pExprInfo[j].base.numOfParams > 0){ // tag-> operation - char keyMd5[TSDB_MAX_JSON_KEY_MD5_LEN] = {0}; - jsonKeyMd5(pExprInfo[j].base.param[0].pz, pExprInfo[j].base.param[0].nLen, keyMd5); - char* strStr = NULL; - int len = 0; - findTagValue(item->pTable, keyMd5, TSDB_MAX_JSON_KEY_MD5_LEN, &strStr, &len); - *dst++ = SELECT_ELEMENT_JSON_TAG; // select tag->element - doSetTagValueToResultBuf(dst, strStr, type, len); + getJsonTagValueElment(item->pTable, pExprInfo[j].base.param[0].pz, pExprInfo[j].base.param[0].nLen, dst, bytes); }else{ - *dst++ = SELECT_ALL_JSON_TAG; // select tag - assert(kvRowLen(data) < bytes); - doSetTagValueToResultBuf(dst, data, type, bytes - CHAR_BYTES); + getJsonTagValueAll(data, dst, bytes); } continue; } diff --git a/src/util/src/tutil.c b/src/util/src/tutil.c index 17a21759ff7ff596757429080d95b2e286929a6e..2def2be8a44dd57938683dc459ae3c03c929e629 100644 --- a/src/util/src/tutil.c +++ b/src/util/src/tutil.c @@ -95,7 +95,7 @@ size_t strtrim(char *z) { int32_t j = 0; int32_t delta = 0; - while (z[j] == ' ') { + while (isspace(z[j])) { ++j; } @@ -108,9 +108,9 @@ size_t strtrim(char *z) { int32_t stop = 0; while (z[j] != 0) { - if (z[j] == ' ' && stop == 0) { + if (isspace(z[j]) && stop == 0) { stop = j; - } else if (z[j] != ' ' && stop != 0) { + } else if (!isspace(z[j]) && stop != 0) { stop = 0; } diff --git a/tests/pytest/stable/json_tag.py b/tests/pytest/stable/json_tag.py index 249f444524c6c1d3a08d8860e1a687e2c087c516..72528e2d6885ae25cb352ea28d6bf839ab57fa9b 100644 --- a/tests/pytest/stable/json_tag.py +++ b/tests/pytest/stable/json_tag.py @@ -173,7 +173,7 @@ class TDTestCase: # test json string parse tdSql.error("CREATE TABLE if not exists db_json_tag_test.jsons1_5 using db_json_tag_test.jsons1 tags('efwewf')") - tdSql.error("CREATE TABLE if not exists db_json_tag_test.jsons1_5 using db_json_tag_test.jsons1 tags('\t')") + tdSql.execute("CREATE TABLE if not exists db_json_tag_test.jsons1_5 using db_json_tag_test.jsons1 tags('\t')") tdSql.execute("CREATE TABLE if not exists db_json_tag_test.jsons1_6 using db_json_tag_test.jsons1 tags('')") tdSql.query("select jtag from db_json_tag_test.jsons1_6") tdSql.checkData(0, 0, "") @@ -204,7 +204,7 @@ class TDTestCase: tdSql.checkRows(1) tdSql.query("select jtag from db_json_tag_test.jsons1 where jtag is null") - tdSql.checkRows(4) + tdSql.checkRows(5) tdSql.query("select jtag from db_json_tag_test.jsons1 where jtag is not null") tdSql.checkRows(5) @@ -213,7 +213,7 @@ class TDTestCase: tdSql.checkRows(3) tdSql.query("select tbname,jtag from db_json_tag_test.jsons1 where jtag->'location' is null") - tdSql.checkRows(6) + tdSql.checkRows(7) tdSql.query("select * from db_json_tag_test.jsons1 where jtag->'num' is not null") tdSql.checkRows(2)