From 3df259f77ec2d2a61657ed1f67b67727fad90b77 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 20 Oct 2021 14:31:10 +0800 Subject: [PATCH] feature/TD-6129 add table delete logic for json tag table --- src/client/src/tscUtil.c | 6 +++--- src/tsdb/src/tsdbMeta.c | 28 ++++++---------------------- src/tsdb/src/tsdbRead.c | 29 +++++++---------------------- src/util/src/tutil.c | 3 +-- tests/pytest/stable/json_tag.py | 11 +++++++++++ 5 files changed, 28 insertions(+), 49 deletions(-) diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index f963a1e6a2..b1e2224167 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -5296,12 +5296,12 @@ int parseJsontoTagData(char* json, SKVRowBuilder* kvRowBuilder, char* errMsg, in tdAddColToKVRow(kvRowBuilder, jsonIndex++, TSDB_DATA_TYPE_NCHAR, nullTypeKey, false); // add json null type if (strtrim(json) == 0 || strcasecmp(json, "null") == 0){ *(uint8_t*)(varDataVal(nullTypeVal + CHAR_BYTES)) = jsonNULL; - tdAddColToKVRow(kvRowBuilder, jsonIndex++, TSDB_DATA_TYPE_NCHAR, &nullTypeVal, true); // add json null value + tdAddColToKVRow(kvRowBuilder, jsonIndex++, TSDB_DATA_TYPE_NCHAR, nullTypeVal, true); // add json null value return TSDB_CODE_SUCCESS; } int8_t jsonNotNull = TSDB_DATA_JSON_NOT_NULL; *(uint8_t*)(varDataVal(nullTypeVal + CHAR_BYTES)) = jsonNotNull; - tdAddColToKVRow(kvRowBuilder, jsonIndex++, TSDB_DATA_TYPE_NCHAR, &nullTypeVal, true); // add json type + tdAddColToKVRow(kvRowBuilder, jsonIndex++, TSDB_DATA_TYPE_NCHAR, nullTypeVal, true); // add json type // set json real data cJSON *root = cJSON_Parse(json); @@ -5393,7 +5393,7 @@ int parseJsontoTagData(char* json, SKVRowBuilder* kvRowBuilder, char* errMsg, in if(taosHashGetSize(keyHash) == 0){ // set json NULL true *(uint8_t*)(varDataVal(nullTypeVal + CHAR_BYTES)) = jsonNULL; - memcpy(POINTER_SHIFT(kvRowBuilder->buf, kvRowBuilder->pColIdx[2].offset), &nullTypeVal, CHAR_BYTES + VARSTR_HEADER_SIZE + CHAR_BYTES); + memcpy(POINTER_SHIFT(kvRowBuilder->buf, kvRowBuilder->pColIdx[2].offset), nullTypeVal, CHAR_BYTES + VARSTR_HEADER_SIZE + CHAR_BYTES); } end: diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index e6512f24c3..5ba02d373e 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -1494,26 +1494,12 @@ static void *tsdbDecodeTable(void *buf, STable **pRTable) { } static SArray* getJsonTagTableList(STable *pTable){ - SArray** pRecord = taosHashIterate(pTable->jsonKeyMap, NULL); - SArray* tablist = taosArrayInit(32, sizeof(JsonMapValue)); - - while(pRecord){ - SArray* tallistOld = *pRecord; - for (int i = 0; i < taosArrayGetSize(tallistOld); ++i) { // sort to elimate dumplicate - void* element = taosArrayGet(tallistOld, i); - void* pFind = taosArraySearch(tablist, element, tsdbCompareJsonMapValue, TD_EQ); - if(pFind == NULL){ - void* p = taosArraySearch(tablist, element, tsdbCompareJsonMapValue, TD_GE); - if(p == NULL){ - taosArrayPush(tablist, element); - }else{ - taosArrayInsert(tablist, TARRAY_ELEM_IDX(tablist, p), element); - } - } - } - pRecord = taosHashIterate(pTable->jsonKeyMap, pRecord); - } - return tablist; + uint8_t key = TSDB_DATA_JSON_NULL; + char keyMd5[TSDB_MAX_JSON_KEY_MD5_LEN] = {0}; + jsonKeyMd5(&key, 1, keyMd5); + SArray** tablist = (SArray**)taosHashGet(pTable->jsonKeyMap, keyMd5, TSDB_MAX_JSON_KEY_MD5_LEN); + + return *tablist; } static int tsdbGetTableEncodeSize(int8_t act, STable *pTable) { @@ -1576,7 +1562,6 @@ static int tsdbRemoveTableFromStore(STsdbRepo *pRepo, STable *pTable) { ASSERT(TABLE_TYPE((STable *)(p->table)) == TSDB_CHILD_TABLE); pBuf = tsdbInsertTableAct(pRepo, TSDB_DROP_META, pBuf, p->table); } - taosArrayDestroy(tablist); }else { SSkipListIterator *pIter = tSkipListCreateIter(pTable->pIndex); if (pIter == NULL) { @@ -1609,7 +1594,6 @@ static int tsdbRmTableFromMeta(STsdbRepo *pRepo, STable *pTable) { JsonMapValue* p = taosArrayGet(tablist, i); tsdbRemoveTableFromMeta(pRepo, p->table, false, false); } - taosArrayDestroy(tablist); }else{ SSkipListIterator *pIter = tSkipListCreateIter(pTable->pIndex); if (pIter == NULL) { diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index c65270ddbd..6dde8ec545 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -2679,31 +2679,16 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int static int32_t getAllTableList(STable* pSuperTable, SArray* list) { STSchema* pTagSchema = tsdbGetTableTagSchema(pSuperTable); if(pTagSchema && pTagSchema->numOfCols == 1 && pTagSchema->columns[0].type == TSDB_DATA_TYPE_JSON){ - SArray** pRecord = taosHashIterate(pSuperTable->jsonKeyMap, NULL); - SArray* tablist = taosArrayInit(32, sizeof(JsonMapValue)); - - while(pRecord){ - SArray* tallistOld = *pRecord; - for (int i = 0; i < taosArrayGetSize(tallistOld); ++i) { // sort to elimate dumplicate - void* element = taosArrayGet(tallistOld, i); - void* pFind = taosArraySearch(tablist, element, tsdbCompareJsonMapValue, TD_EQ); - if(pFind == NULL){ - void* p = taosArraySearch(tablist, element, tsdbCompareJsonMapValue, TD_GE); - if(p == NULL){ - taosArrayPush(tablist, element); - }else{ - taosArrayInsert(tablist, TARRAY_ELEM_IDX(tablist, p), element); - } - } - } - pRecord = taosHashIterate(pSuperTable->jsonKeyMap, pRecord); - } - for (int i = 0; i < taosArrayGetSize(tablist); ++i) { - JsonMapValue* p = taosArrayGet(tablist, i); + uint8_t key = TSDB_DATA_JSON_NULL; + char keyMd5[TSDB_MAX_JSON_KEY_MD5_LEN] = {0}; + jsonKeyMd5(&key, 1, keyMd5); + SArray** tablist = (SArray**)taosHashGet(pSuperTable->jsonKeyMap, keyMd5, TSDB_MAX_JSON_KEY_MD5_LEN); + + for (int i = 0; i < taosArrayGetSize(*tablist); ++i) { + JsonMapValue* p = taosArrayGet(*tablist, i); STableKeyInfo info = {.pTable = p->table, .lastKey = TSKEY_INITIAL_VAL}; taosArrayPush(list, &info); } - taosArrayDestroy(tablist); }else{ SSkipListIterator* iter = tSkipListCreateIter(pSuperTable->pIndex); while (tSkipListIterNext(iter)) { diff --git a/src/util/src/tutil.c b/src/util/src/tutil.c index 4845da626b..dc242ea72c 100644 --- a/src/util/src/tutil.c +++ b/src/util/src/tutil.c @@ -470,9 +470,8 @@ bool isValidateTag(char *input) { if (!input) return false; int len = strlen(input); if (len == 0) return false; - if (input[0] != '_' && isalpha(input[0]) == 0) return false; for (int i = 1; i < len; ++i) { - if (input[i] != '_' && isalnum(input[i]) == 0) return false; + if (isprint(input[i]) == 0) return false; } return true; } diff --git a/tests/pytest/stable/json_tag.py b/tests/pytest/stable/json_tag.py index e48f340d59..2151f5f0d5 100644 --- a/tests/pytest/stable/json_tag.py +++ b/tests/pytest/stable/json_tag.py @@ -243,6 +243,17 @@ class TDTestCase: tdSql.query("select tbname,jtag from db_json_tag_test.jsons1 where jtag->'k1'='中国'") tdSql.checkRows(1) + #test dumplicate key with normal colomn + tdSql.execute("INSERT INTO db_json_tag_test.jsons1_12 using db_json_tag_test.jsons1 tags('{\"tbname\":\"tt\",\"databool\":true,\"dataStr\":\"是是是\"}') values(now, 4, false, \"你就会\")") + + tdSql.query("select *,tbname,jtag from db_json_tag_test.jsons1 where jtag->'dataStr' match '是'") + tdSql.checkRows(1) + + tdSql.query("select tbname,jtag->'tbname' from db_json_tag_test.jsons1 where jtag->'tbname'='tt'") + tdSql.checkRows(1) + + tdSql.query("select *,tbname,jtag from db_json_tag_test.jsons1 where dataBool=true") + tdSql.checkRows(2) def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) -- GitLab