From e0340e5664d3f04c247ef7254fdb8cb76ad29765 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 9 Nov 2021 19:34:27 +0800 Subject: [PATCH] add json compare function support --- src/client/inc/tscUtil.h | 2 +- src/client/src/tscSubquery.c | 5 ++++- src/client/src/tscUtil.c | 15 ++++++++++----- src/inc/tsdb.h | 2 +- src/query/src/qExecutor.c | 6 +++--- src/query/src/qExtbuffer.c | 9 +++++++-- src/tsdb/src/tsdbMeta.c | 5 +---- src/tsdb/src/tsdbRead.c | 9 +++++++-- src/util/src/tcompare.c | 9 +++++++-- tests/pytest/stable/json_tag.py | 9 +++++++++ 10 files changed, 50 insertions(+), 21 deletions(-) diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index 902b99fea2..abaec1bfb7 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -391,7 +391,7 @@ char* cloneCurrentDBName(SSqlObj* pSql); int parseJsontoTagData(char* json, SKVRowBuilder* kvRowBuilder, char* errMsg, int16_t startColId); char* parseTagDatatoJson(void *p); -void getJsonTagValueElment(STable* data, char* key, int32_t keyLen, char* out, int16_t bytes); +void* getJsonTagValueElment(STable* data, char* key, int32_t keyLen, char* out, int16_t bytes); void getJsonTagValueAll(void* data, void* dst, int16_t bytes); int8_t jsonType2DbType(double data, int jsonType); diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index efde9a2ff7..1c2404a313 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -721,8 +721,11 @@ int32_t tagValCompar(const void* p1, const void* p2) { const STidTags* t1 = (const STidTags*) varDataVal(p1); const STidTags* t2 = (const STidTags*) varDataVal(p2); + if (t1->padding == TSDB_DATA_TYPE_JSON){ + __compar_fn_t func = getComparFunc(t1->tag[0], 0); + return func(t1->tag + CHAR_BYTES, t2->tag + CHAR_BYTES); + } __compar_fn_t func = getComparFunc(t1->padding, 0); - return func(t1->tag, t2->tag); } diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index c672794267..5c95bd94b8 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -5372,31 +5372,36 @@ char* cloneCurrentDBName(SSqlObj* pSql) { return p; } -void getJsonTagValueElment(STable* data, char* key, int32_t keyLen, char* dst, int16_t bytes){ +void* getJsonTagValueElment(STable* data, char* key, int32_t keyLen, char* dst, int16_t bytes){ char keyMd5[TSDB_MAX_JSON_KEY_MD5_LEN] = {0}; jsonKeyMd5(key, keyLen, keyMd5); void* result = getJsonTagValue(data, keyMd5, TSDB_MAX_JSON_KEY_MD5_LEN, NULL); if (result == NULL){ // json key no result + if(!dst) return NULL; *(char*)dst = TSDB_DATA_TYPE_NCHAR; setNull(dst + CHAR_BYTES, TSDB_DATA_TYPE_JSON, 0); - return; + return dst; } char* realData = POINTER_SHIFT(result, CHAR_BYTES); if(*(char*)result == TSDB_DATA_TYPE_NCHAR || *(char*)result == TSDB_DATA_TYPE_BINARY) { assert(varDataTLen(realData) < bytes); + if(!dst) return realData; memcpy(dst, result, CHAR_BYTES + varDataTLen(realData)); - return; + return dst; }else if (*(char*)result == TSDB_DATA_TYPE_DOUBLE || *(char*)result == TSDB_DATA_TYPE_BIGINT) { + if(!dst) return realData; memcpy(dst, result, CHAR_BYTES + LONG_BYTES); - return; + return dst; }else if (*(char*)result == TSDB_DATA_TYPE_BOOL) { + if(!dst) return realData; memcpy(dst, result, CHAR_BYTES + CHAR_BYTES); - return; + return dst; }else { assert(0); } + return realData; } void getJsonTagValueAll(void* data, void* dst, int16_t bytes) { diff --git a/src/inc/tsdb.h b/src/inc/tsdb.h index 63d0ff8ac0..a90780b202 100644 --- a/src/inc/tsdb.h +++ b/src/inc/tsdb.h @@ -118,7 +118,7 @@ typedef struct { void tsdbClearTableCfg(STableCfg *config); -void *tsdbGetTableTagVal(const void *pTable, int32_t colId, int16_t type, int16_t bytes); +void *tsdbGetTableTagVal(const void *pTable, int32_t colId, int16_t type); char *tsdbGetTableName(void *pTable); #define TSDB_TABLEID(_table) ((STableId*) (_table)) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 38c9dd531b..56b64b4862 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -3381,7 +3381,7 @@ static void doSetTagValueInParam(void* pTable, char* param, int32_t paramLen, in val = tsdbGetTableName(pTable); assert(val != NULL); } else { - val = tsdbGetTableTagVal(pTable, tagColId, type, bytes); + val = tsdbGetTableTagVal(pTable, tagColId, type); } if (val == NULL || isNull(val, type)) { @@ -7218,7 +7218,7 @@ static SSDataBlock* doTagScan(void* param, bool* newgroup) { if (pExprInfo->base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) { data = tsdbGetTableName(item->pTable); } else { - data = tsdbGetTableTagVal(item->pTable, pExprInfo->base.colInfo.colId, type, bytes); + data = tsdbGetTableTagVal(item->pTable, pExprInfo->base.colInfo.colId, type); if(type == TSDB_DATA_TYPE_JSON){ if(pExprInfo->base.numOfParams > 0){ // tag-> operation getJsonTagValueElment(item->pTable, pExprInfo->base.param[0].pz, pExprInfo->base.param[0].nLen, output, bytes); @@ -7267,7 +7267,7 @@ static SSDataBlock* doTagScan(void* param, bool* newgroup) { if (pExprInfo[j].base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) { data = tsdbGetTableName(item->pTable); } else { - data = tsdbGetTableTagVal(item->pTable, pExprInfo[j].base.colInfo.colId, type, bytes); + data = tsdbGetTableTagVal(item->pTable, pExprInfo[j].base.colInfo.colId, type); if(type == TSDB_DATA_TYPE_JSON){ if(pExprInfo[j].base.numOfParams > 0){ // tag-> operation getJsonTagValueElment(item->pTable, pExprInfo[j].base.param[0].pz, pExprInfo[j].base.param[0].nLen, dst, bytes); diff --git a/src/query/src/qExtbuffer.c b/src/query/src/qExtbuffer.c index 85ac9b72aa..2efdcf7795 100644 --- a/src/query/src/qExtbuffer.c +++ b/src/query/src/qExtbuffer.c @@ -367,6 +367,12 @@ static int32_t tsCompareFunc(TSKEY k1, TSKEY k2, int32_t order) { } int32_t columnValueAscendingComparator(char *f1, char *f2, int32_t type, int32_t bytes) { + if (type == TSDB_DATA_TYPE_JSON){ + assert(*f1 == *f2); + type = *f1; + f1 = POINTER_SHIFT(f1, CHAR_BYTES); + f2 = POINTER_SHIFT(f2, CHAR_BYTES); + } switch (type) { case TSDB_DATA_TYPE_INT: DEFAULT_COMP(GET_INT32_VAL(f1), GET_INT32_VAL(f2)); case TSDB_DATA_TYPE_DOUBLE: DEFAULT_DOUBLE_COMP(GET_DOUBLE_VAL(f1), GET_DOUBLE_VAL(f2)); @@ -391,8 +397,7 @@ int32_t columnValueAscendingComparator(char *f1, char *f2, int32_t type, int32_t } }; - case TSDB_DATA_TYPE_NCHAR: - case TSDB_DATA_TYPE_JSON: { // todo handle the var string compare + case TSDB_DATA_TYPE_NCHAR: { // todo handle the var string compare int32_t len1 = varDataLen(f1); int32_t len2 = varDataLen(f2); diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index 28347f1cbd..17abb2c3b8 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -204,7 +204,7 @@ _err: return -1; } -void *tsdbGetTableTagVal(const void* pTable, int32_t colId, int16_t type, int16_t bytes) { +void *tsdbGetTableTagVal(const void* pTable, int32_t colId, int16_t type) { // TODO: this function should be changed also STSchema *pSchema = tsdbGetTableTagSchema((STable*) pTable); @@ -220,9 +220,6 @@ void *tsdbGetTableTagVal(const void* pTable, int32_t colId, int16_t type, int16_ val = tdGetKVRowValOfCol(((STable*)pTable)->tagVal, colId); assert(type == pCol->type); } - // if (val != NULL && IS_VAR_DATA_TYPE(type)) { - // assert(varDataLen(val) < pCol->bytes); - // } return val; } diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index cd561cdb64..820d78294d 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -3515,8 +3515,13 @@ static int32_t tableGroupComparFn(const void *p1, const void *p2, const void *pa STColumn* pCol = schemaColAt(pTableGroupSupp->pTagSchema, colIndex); bytes = pCol->bytes; type = pCol->type; - f1 = tdGetKVRowValOfCol(pTable1->tagVal, pCol->colId); - f2 = tdGetKVRowValOfCol(pTable2->tagVal, pCol->colId); + if (type == TSDB_DATA_TYPE_JSON){ + f1 = getJsonTagValueElment(pTable1, pColIndex->name, strlen(pColIndex->name), NULL, TSDB_MAX_JSON_TAGS_LEN); + f2 = getJsonTagValueElment(pTable2, pColIndex->name, strlen(pColIndex->name), NULL, TSDB_MAX_JSON_TAGS_LEN); + }else{ + f1 = tdGetKVRowValOfCol(pTable1->tagVal, pCol->colId); + f2 = tdGetKVRowValOfCol(pTable2->tagVal, pCol->colId); + } } } diff --git a/src/util/src/tcompare.c b/src/util/src/tcompare.c index df39673899..a238c20624 100644 --- a/src/util/src/tcompare.c +++ b/src/util/src/tcompare.c @@ -576,6 +576,12 @@ __compar_fn_t getKeyComparFunc(int32_t keyType, int32_t order) { } int32_t doCompare(const char* f1, const char* f2, int32_t type, size_t size) { + if (type == TSDB_DATA_TYPE_JSON){ + assert(*f1 == *f2); + type = *f1; + f1 = POINTER_SHIFT(f1, CHAR_BYTES); + f2 = POINTER_SHIFT(f2, CHAR_BYTES); + } switch (type) { case TSDB_DATA_TYPE_INT: DEFAULT_COMP(GET_INT32_VAL(f1), GET_INT32_VAL(f2)); case TSDB_DATA_TYPE_DOUBLE: DEFAULT_DOUBLE_COMP(GET_DOUBLE_VAL(f1), GET_DOUBLE_VAL(f2)); @@ -588,8 +594,7 @@ int32_t doCompare(const char* f1, const char* f2, int32_t type, size_t size) { case TSDB_DATA_TYPE_USMALLINT: DEFAULT_COMP(GET_UINT16_VAL(f1), GET_UINT16_VAL(f2)); case TSDB_DATA_TYPE_UINT: DEFAULT_COMP(GET_UINT32_VAL(f1), GET_UINT32_VAL(f2)); case TSDB_DATA_TYPE_UBIGINT: DEFAULT_COMP(GET_UINT64_VAL(f1), GET_UINT64_VAL(f2)); - case TSDB_DATA_TYPE_NCHAR: - case TSDB_DATA_TYPE_JSON:{ + case TSDB_DATA_TYPE_NCHAR:{ tstr* t1 = (tstr*) f1; tstr* t2 = (tstr*) f2; diff --git a/tests/pytest/stable/json_tag.py b/tests/pytest/stable/json_tag.py index 19938dfb8a..e10c72d1e8 100644 --- a/tests/pytest/stable/json_tag.py +++ b/tests/pytest/stable/json_tag.py @@ -278,9 +278,18 @@ class TDTestCase: tdSql.execute("INSERT INTO db_json_tag_test.jsons1_20 using db_json_tag_test.jsons1 tags('{\"tagint\":1}') values(now, 1, false, \"你就会\")") tdSql.execute("INSERT INTO db_json_tag_test.jsons1_21 using db_json_tag_test.jsons1 tags('{\"tagint\":11}') values(now, 11, false, \"你就会\")") tdSql.execute("INSERT INTO db_json_tag_test.jsons1_22 using db_json_tag_test.jsons1 tags('{\"tagint\":2}') values(now, 2, false, \"你就会\")") + tdSql.query("select avg(dataint),count(*) from db_json_tag_test.jsons1 group by jtag->'tagint' order by jtag->'tagint' desc") + //tdSql.checkData(1, 0, 2.5) # test json->'key'=null tdSql.execute("insert into db_json_tag_test.jsons1_9 values('2020-04-17 15:20:00.000', 5, false, 'json19')") + tdSql.query("select * from db_json_tag_test.jsons1") + tdSql.checkRows(9) + tdSql.query("select * from db_json_tag_test.jsons1 where jtag->'time' is null") + tdSql.checkRows(8) + tdSql.query("select * from db_json_tag_test.jsons1 where jtag->'time'=null") + tdSql.checkRows(1) + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) -- GitLab