diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index 887c51f10c5a6c10e213bb893725a837a0be77cd..9a675a61bf32bbd93d26d2caa5664d5e378b1033 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -269,6 +269,8 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TDB_MESSED_MSG TAOS_DEF_ERROR_CODE(0, 0x0614) //"TSDB messed message") #define TSDB_CODE_TDB_IVLD_TAG_VAL TAOS_DEF_ERROR_CODE(0, 0x0615) //"TSDB invalid tag value") #define TSDB_CODE_TDB_NO_CACHE_LAST_ROW TAOS_DEF_ERROR_CODE(0, 0x0616) //"TSDB no cache last row data") +#define TSDB_CODE_TDB_IVLD_SAME_JSON_VALUE TAOS_DEF_ERROR_CODE(0, 0x0617) //"TSDB invalid same json value") +#define TSDB_CODE_TDB_NO_JSON_TAG_KEY TAOS_DEF_ERROR_CODE(0, 0x0618) //"TSDB no tag json key") // query #define TSDB_CODE_QRY_INVALID_QHANDLE TAOS_DEF_ERROR_CODE(0, 0x0700) //"Invalid handle") diff --git a/src/query/inc/qFilter.h b/src/query/inc/qFilter.h index 2ad4b43c349a6428aed88a77a11a2dcab1033ec0..994265129ce4a01070e76581ecdb5a0cc5193e78 100644 --- a/src/query/inc/qFilter.h +++ b/src/query/inc/qFilter.h @@ -113,6 +113,7 @@ typedef bool (*rangeCompFunc) (const void *, const void *, const void *, const v typedef int32_t(*filter_desc_compare_func)(const void *, const void *); typedef bool(*filter_exec_func)(void *, int32_t, int8_t**, SDataStatis *, int16_t); typedef int32_t (*filer_get_col_from_id)(void *, int32_t, void **); +typedef int32_t (*filer_get_col_from_name)(void *, int32_t, char*, void **); typedef struct SFilterRangeCompare { int64_t s; @@ -333,6 +334,7 @@ typedef struct SFilterInfo { extern int32_t filterInitFromTree(tExprNode* tree, void **pinfo, uint32_t options); extern bool filterExecute(SFilterInfo *info, int32_t numOfRows, int8_t** p, SDataStatis *statis, int16_t numOfCols); extern int32_t filterSetColFieldData(SFilterInfo *info, void *param, filer_get_col_from_id fp); +extern int32_t filterSetJsonColFieldData(SFilterInfo *info, void *param, filer_get_col_from_name fp); extern int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow *win); extern int32_t filterConverNcharColumns(SFilterInfo* pFilterInfo, int32_t rows, bool *gotNchar); extern int32_t filterFreeNcharColumns(SFilterInfo* pFilterInfo); diff --git a/src/query/src/qFilter.c b/src/query/src/qFilter.c index 231e7b18873b45ff2ca3d61e82f4cffe6dea6585..471c6a9167117a0b2eca5726043508d3984a1bc7 100644 --- a/src/query/src/qFilter.c +++ b/src/query/src/qFilter.c @@ -1542,7 +1542,7 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) if (unit->right.type == FLD_TYPE_VALUE && FILTER_UNIT_OPTR(unit) != TSDB_RELATION_IN) { SFilterField *right = FILTER_UNIT_RIGHT_FIELD(info, unit); char *data = right->data; - if (IS_VAR_DATA_TYPE(type) { + if (IS_VAR_DATA_TYPE(type)) { tlen = varDataLen(data); data += VARSTR_HEADER_SIZE; } @@ -3152,6 +3152,43 @@ int32_t filterSetColFieldData(SFilterInfo *info, void *param, filer_get_col_from return TSDB_CODE_SUCCESS; } +int32_t filterSetJsonColFieldData(SFilterInfo *info, void *param, filer_get_col_from_name fp) { + CHK_LRET(info == NULL, TSDB_CODE_QRY_APP_ERROR, "info NULL"); + CHK_LRET(info->fields[FLD_TYPE_COLUMN].num <= 0, TSDB_CODE_QRY_APP_ERROR, "no column fileds"); + + if (FILTER_ALL_RES(info) || FILTER_EMPTY_RES(info)) { + return TSDB_CODE_SUCCESS; + } + + for (uint16_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) { + SFilterField* fi = &info->fields[FLD_TYPE_COLUMN].fields[i]; + SSchema* sch = fi->desc; + + (*fp)(param, sch->colId, sch->name, &fi->data); + } + + filterUpdateComUnits(info); + + return TSDB_CODE_SUCCESS; +} + +// convert json type for next compare and so on +void filterJsonTypeConvert(SFilterInfo* info) { + uint8_t type = 0; + if(JSON_TYPE_NCHAR){ type = TSDB_DATA_TYPE_NCHAR;} else {type = TSDB_DATA_TYPE_BINARY;} + for(int i = 0; i < info->unitNum; i++){ + if(info->units[info->unitNum].compare.type == TSDB_DATA_TYPE_JSON){ + info->units[info->unitNum].compare.type= type; + } + } + + for(int i = 0; i < info->fields[FLD_TYPE_COLUMN].num; i++) { + SSchema* schema = info->fields[FLD_TYPE_COLUMN].fields[i].desc; + if(schema->type == TSDB_DATA_TYPE_JSON){ + schema->type = type; + } + } +} int32_t filterInitFromTree(tExprNode* tree, void **pinfo, uint32_t options) { int32_t code = TSDB_CODE_SUCCESS; @@ -3172,8 +3209,8 @@ int32_t filterInitFromTree(tExprNode* tree, void **pinfo, uint32_t options) { filterInitUnitsFields(info); code = filterTreeToGroup(tree, info, group); - ERR_JRET(code); + filterJsonTypeConvert(info); filterConvertGroupFromArray(info, group); diff --git a/src/query/src/qUtil.c b/src/query/src/qUtil.c index bc27e094db3dcb85ffa73810e922d73cd42ab3a0..75053dd7f95434d0e2b1f7b0aaa0ddcef9191161 100644 --- a/src/query/src/qUtil.c +++ b/src/query/src/qUtil.c @@ -23,6 +23,8 @@ #include "tlosertree.h" #include "queryLog.h" #include "tscompression.h" +#include "tsdbMeta.h" +#include "tscUtil.h" typedef struct SCompSupporter { STableQueryInfo **pTableQueryInfo; @@ -725,4 +727,3 @@ void blockDistInfoFromBinary(const char* data, int32_t len, STableBlockDist* pDi tfree(outputBuf); } } - diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index 773f76ee222aced3aacc34ad22f114a67ec930d3..4c8a7af7e38be20d36895f1412e9b50fc50981b4 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -1127,12 +1127,30 @@ static int tsdbAddTableIntoIndex(STsdbMeta *pMeta, STable *pTable, bool refSuper }else{ tablistNew = *tablist; } + + if(taosArrayGetSize(tablistNew) > 0){ + // validate type + JsonMapValue* tmp = taosArrayGet(tablistNew, 0); + void* data1 = tdGetKVRowValOfCol(((STable *)(tmp->table))->tagVal, tmp->colId + 1); + SColIdx * pInsertColIdx = kvRowColIdxAt(pTable->tagVal, j + 1); + void* data2 = (kvRowColVal(pTable->tagVal, pInsertColIdx)); + if(*(char*)data1 != *(char*)data2){ + terrno = TSDB_CODE_TDB_IVLD_SAME_JSON_VALUE; + tsdbError("invalidate same json tag value"); + return -1; + } + } JsonMapValue jmvalue = {pTable, pColIdx->colId}; void* p = taosArraySearch(tablistNew, &jmvalue, tscCompareJsonMapValue, TD_EQ); if (p == NULL) { - taosArrayPush(tablistNew, &jmvalue); + p = taosArraySearch(tablistNew, &jmvalue, tscCompareJsonMapValue, TD_GE); + if(p == NULL){ + taosArrayPush(tablistNew, &jmvalue); + }else{ + taosArrayInsert(tablistNew, TARRAY_ELEM_IDX(tablistNew, p), &jmvalue); + } }else{ - taosArrayInsert(tablistNew, TARRAY_ELEM_IDX(tablistNew, p), &jmvalue); + tsdbError("insert dumplicate"); } } }else{ diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 2da457bd3489a1f7b17305a524d81c91107eecfe..f5cc4499fe4ee2005c8b6b024f8cdc74cb28aa54 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -4064,33 +4064,90 @@ static void queryIndexlessColumn(SSkipList* pSkipList, void* filterInfo, SArray* tSkipListDestroyIter(iter); } -static FORCE_INLINE int32_t tsdbGetJsonTagDataFromId(void *param, int32_t id, void **data) { +void* getJsonTagValue(STable* pTable, char* key){ + assert(TABLE_TYPE(pTable) == TSDB_CHILD_TABLE); + int32_t outLen = 0; + if(JSON_TYPE_NCHAR){ + char tagKey[256] = {0}; + if (!taosMbsToUcs4(key, strlen(key), tagKey, 256, &outLen)) { + tsdbError("json key to ucs4 error:%s|%s", strerror(errno), key); + return NULL; + } + key = tagKey; + }else{ + outLen = strlen(key); + } + STable* superTable = pTable->pSuper; + SArray** data = (SArray**)taosHashGet(superTable->jsonKeyMap, key, outLen); + if(data == NULL) return NULL; + JsonMapValue jmvalue = {pTable, 0}; + JsonMapValue* p = taosArraySearch(*data, &jmvalue, tscCompareJsonMapValue, TD_EQ); + if (p == NULL) return NULL; + int16_t valId = p->colId + 1; + return POINTER_SHIFT(kvRowValues(pTable->tagVal), valId); +} + + +static FORCE_INLINE int32_t tsdbGetJsonTagDataFromId(void *param, int32_t id, char* name, void **data) { JsonMapValue* jsonMapV = (JsonMapValue*)(param); STable* pTable = (STable*)(jsonMapV->table); if (id == TSDB_TBNAME_COLUMN_INDEX) { *data = TABLE_NAME(pTable); } else { - *data = tdGetKVRowValOfCol(pTable->tagVal, jsonMapV->colId + 1); + *data = getJsonTagValue(pTable, name); } return TSDB_CODE_SUCCESS; } -static void queryByJsonTag(SArray* pTableList, void* filterInfo, SArray* res){ - int32_t size = taosArrayGetSize(pTableList); +static void queryByJsonTag(STable* pTable, void* filterInfo, SArray* res){ + // get all table in fields, and dumplicate it + SArray* tabList = NULL; + SFilterInfo* info = (SFilterInfo*)filterInfo; + for (uint16_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) { + SFilterField* fi = &info->fields[FLD_TYPE_COLUMN].fields[i]; + SSchema* sch = fi->desc; + if (sch-> colId == TSDB_TBNAME_COLUMN_INDEX) continue; + SArray** data = (SArray**)taosHashGet(pTable->jsonKeyMap, sch->name, strlen(sch->name)); + if(data == NULL) continue; + if(tabList == NULL) { + tabList = taosArrayDup(*data); + }else{ + for(int j = 0; j < taosArrayGetSize(*data); j++){ + void* element = taosArrayGet(*data, j); + void* p = taosArraySearch(tabList, element, tscCompareJsonMapValue, TD_EQ); + if (p == NULL) { + p = taosArraySearch(tabList, element, tscCompareJsonMapValue, TD_GE); + if(p == NULL){ + taosArrayPush(tabList, tabList); + }else{ + taosArrayInsert(tabList, TARRAY_ELEM_IDX(tabList, p), tabList); + } + } + } + } + } + if(tabList == NULL || taosArrayGetSize(tabList) == 0){ + tsdbError("json key not exist"); + terrno = TSDB_CODE_TDB_NO_JSON_TAG_KEY; + taosArrayDestroy(tabList); + return; + } + int32_t size = taosArrayGetSize(tabList); int8_t *addToResult = NULL; for(int i = 0; i < size; i++){ - JsonMapValue* data = taosArrayGet(pTableList, i); - filterSetColFieldData(filterInfo, data, tsdbGetJsonTagDataFromId); + JsonMapValue* data = taosArrayGet(tabList, i); + filterSetJsonColFieldData(filterInfo, data, tsdbGetJsonTagDataFromId); bool all = filterExecute(filterInfo, 1, &addToResult, NULL, 0); if (all || (addToResult && *addToResult)) { - STableKeyInfo info = {.pTable = (void*)(data->table), .lastKey = TSKEY_INITIAL_VAL}; - taosArrayPush(res, &info); + STableKeyInfo kInfo = {.pTable = (void*)(data->table), .lastKey = TSKEY_INITIAL_VAL}; + taosArrayPush(res, &kInfo); } } tfree(addToResult); + taosArrayDestroy(tabList); } static int32_t tsdbQueryTableList(STable* pTable, SArray* pRes, void* filterInfo) {