From 72e12675c0a53d3cea00478a34361d40fa2109c1 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 8 Dec 2021 12:34:06 +0800 Subject: [PATCH] modify complie parameters --- src/client/CMakeLists.txt | 1 - src/client/inc/tscUtil.h | 7 -- src/client/src/tscUtil.c | 158 -------------------------------------- src/common/CMakeLists.txt | 2 - src/inc/tsdb.h | 5 ++ src/tsdb/CMakeLists.txt | 1 - src/tsdb/inc/tsdbMeta.h | 1 + src/tsdb/src/tsdbMeta.c | 14 ++++ src/tsdb/src/tsdbRead.c | 148 ++++++++++++++++++++++++++++++++++- 9 files changed, 166 insertions(+), 171 deletions(-) diff --git a/src/client/CMakeLists.txt b/src/client/CMakeLists.txt index 45f9d90264..48d3fe8675 100644 --- a/src/client/CMakeLists.txt +++ b/src/client/CMakeLists.txt @@ -3,7 +3,6 @@ PROJECT(TDengine) INCLUDE_DIRECTORIES(inc) INCLUDE_DIRECTORIES(jni) -INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/tsdb/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/zlib-1.2.11/inc) diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index f87ef189c2..7baa1bd088 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -28,8 +28,6 @@ extern "C" { #include "tscGlobalmerge.h" #include "tsched.h" #include "tsclient.h" -#include "tglobal.h" -#include "tsdbMeta.h" #define UTIL_TABLE_IS_SUPER_TABLE(metaInfo) \ (((metaInfo)->pTableMeta != NULL) && ((metaInfo)->pTableMeta->tableType == TSDB_SUPER_TABLE)) @@ -397,12 +395,7 @@ void tscRemoveCachedTableMeta(STableMetaInfo* pTableMetaInfo, uint64_t id); char* cloneCurrentDBName(SSqlObj* pSql); int parseJsontoTagData(char* json, SKVRowBuilder* kvRowBuilder, char* errMsg, int16_t startColId); -char* parseTagDatatoJson(void *p); -void* getJsonTagValueElment(STable* data, char* key, int32_t keyLen, char* out, int16_t bytes); -void getJsonTagValueAll(void* data, void* dst, int16_t bytes); - int8_t jsonType2DbType(double data, int jsonType); -void* getJsonTagValue(STable* pTable, char* key, int32_t keyLen, int16_t* colId); void getJsonKey(SStrToken *t0); char* cloneCurrentDBName(SSqlObj* pSql); diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 9332f1045c..92debee6a4 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -5418,151 +5418,6 @@ char* cloneCurrentDBName(SSqlObj* pSql) { return p; } -void* getJsonTagValueElment(STable* data, char* key, int32_t keyLen, char* dst, int16_t bytes){ - char keyMd5[TSDB_MAX_JSON_KEY_MD5_LEN] = {0}; - jsonKeyMd5(key, keyLen, keyMd5); - - void* result = getJsonTagValue(data, keyMd5, TSDB_MAX_JSON_KEY_MD5_LEN, NULL); - if (result == NULL){ // json key no result - if(!dst) return NULL; - *(char*)dst = TSDB_DATA_TYPE_JSON; - setNull(dst + CHAR_BYTES, TSDB_DATA_TYPE_JSON, 0); - return dst; - } - - char* realData = POINTER_SHIFT(result, CHAR_BYTES); - if(*(char*)result == TSDB_DATA_TYPE_NCHAR || *(char*)result == TSDB_DATA_TYPE_BINARY) { - assert(varDataTLen(realData) < bytes); - if(!dst) return result; - memcpy(dst, result, CHAR_BYTES + varDataTLen(realData)); - return dst; - }else if (*(char*)result == TSDB_DATA_TYPE_DOUBLE || *(char*)result == TSDB_DATA_TYPE_BIGINT) { - if(!dst) return result; - memcpy(dst, result, CHAR_BYTES + LONG_BYTES); - return dst; - }else if (*(char*)result == TSDB_DATA_TYPE_BOOL) { - if(!dst) return result; - memcpy(dst, result, CHAR_BYTES + CHAR_BYTES); - return dst; - }else { - assert(0); - } - return result; -} - -void getJsonTagValueAll(void* data, void* dst, int16_t bytes) { - char* json = parseTagDatatoJson(data); - char* tagData = dst + CHAR_BYTES; - *(char*)dst = TSDB_DATA_TYPE_JSON; - if(json == NULL){ - setNull(tagData, TSDB_DATA_TYPE_JSON, 0); - return; - } - - int32_t length = 0; - if(!taosMbsToUcs4(json, strlen(json), varDataVal(tagData), bytes - VARSTR_HEADER_SIZE - CHAR_BYTES, &length)){ - tscError("getJsonTagValueAll mbstoucs4 error! length:%d", length); - } - varDataSetLen(tagData, length); - assert(varDataTLen(tagData) <= bytes); - tfree(json); -} - -char* parseTagDatatoJson(void *p){ - char* string = NULL; - cJSON *json = cJSON_CreateObject(); - if (json == NULL) - { - goto end; - } - - int16_t nCols = kvRowNCols(p); - ASSERT(nCols%2 == 1); - char tagJsonKey[TSDB_MAX_JSON_KEY_LEN + 1] = {0}; - for (int j = 0; j < nCols; ++j) { - SColIdx * pColIdx = kvRowColIdxAt(p, j); - void* val = (kvRowColVal(p, pColIdx)); - if (j == 0){ - int8_t jsonPlaceHolder = *(int8_t*)val; - ASSERT(jsonPlaceHolder == TSDB_DATA_JSON_PLACEHOLDER); - continue; - } - if(j == 1){ - uint32_t jsonNULL = *(uint32_t*)(varDataVal(val)); - ASSERT(jsonNULL == TSDB_DATA_JSON_NULL); - continue; - } - if (j == 2){ - if(*(uint32_t*)(varDataVal(val + CHAR_BYTES)) == TSDB_DATA_JSON_NULL) goto end; - continue; - } - if (j%2 == 1) { // json key encode by binary - ASSERT(varDataLen(val) <= TSDB_MAX_JSON_KEY_LEN); - memset(tagJsonKey, 0, sizeof(tagJsonKey)); - memcpy(tagJsonKey, varDataVal(val), varDataLen(val)); - }else{ // json value - char tagJsonValue[TSDB_MAX_JSON_TAGS_LEN] = {0}; - char* realData = POINTER_SHIFT(val, CHAR_BYTES); - char type = *(char*)val; - if(type == TSDB_DATA_TYPE_BINARY) { - assert(*(uint32_t*)varDataVal(realData) == TSDB_DATA_JSON_null); // json null value - assert(varDataLen(realData) == INT_BYTES); - cJSON* value = cJSON_CreateNull(); - if (value == NULL) - { - goto end; - } - cJSON_AddItemToObject(json, tagJsonKey, value); - }else if(type == TSDB_DATA_TYPE_NCHAR) { - int32_t length = taosUcs4ToMbs(varDataVal(realData), varDataLen(realData), tagJsonValue); - if (length < 0) { - tscError("charset:%s to %s. val:%s convert json value failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, - (char*)val); - goto end; - } - cJSON* value = cJSON_CreateString(tagJsonValue); - - if (value == NULL) - { - goto end; - } - cJSON_AddItemToObject(json, tagJsonKey, value); - }else if(type == TSDB_DATA_TYPE_DOUBLE){ - double jsonVd = *(double*)(realData); - cJSON* value = cJSON_CreateNumber(jsonVd); - if (value == NULL) - { - goto end; - } - cJSON_AddItemToObject(json, tagJsonKey, value); - }else if(type == TSDB_DATA_TYPE_BIGINT){ - int64_t jsonVd = *(int64_t*)(realData); - cJSON* value = cJSON_CreateNumber(jsonVd); - if (value == NULL) - { - goto end; - } - cJSON_AddItemToObject(json, tagJsonKey, value); - }else if (type == TSDB_DATA_TYPE_BOOL) { - char jsonVd = *(char*)(realData); - cJSON* value = cJSON_CreateBool(jsonVd); - if (value == NULL) - { - goto end; - } - cJSON_AddItemToObject(json, tagJsonKey, value); - } - else{ - tscError("unsupportted json value"); - } - } - } - string = cJSON_PrintUnformatted(json); -end: - cJSON_Delete(json); - return string; -} - int parseJsontoTagData(char* json, SKVRowBuilder* kvRowBuilder, char* errMsg, int16_t startColId){ // set json NULL data uint8_t nullTypeVal[CHAR_BYTES + VARSTR_HEADER_SIZE + INT_BYTES] = {0}; @@ -5699,19 +5554,6 @@ int8_t jsonType2DbType(double data, int jsonType){ return TSDB_DATA_TYPE_NULL; } -void* getJsonTagValue(STable* pTable, char* key, int32_t keyLen, int16_t* retColId){ - assert(TABLE_TYPE(pTable) == TSDB_CHILD_TABLE); - STable* superTable= pTable->pSuper; - SArray** data = (SArray**)taosHashGet(superTable->jsonKeyMap, key, keyLen); - if(data == NULL) return NULL; - JsonMapValue jmvalue = {pTable, 0}; - JsonMapValue* p = taosArraySearch(*data, &jmvalue, tsdbCompareJsonMapValue, TD_EQ); - if (p == NULL) return NULL; - int16_t colId = p->colId + 1; - if(retColId) *retColId = p->colId; - return tdGetKVRowValOfCol(pTable->tagVal, colId); -} - // get key from json->'key' void getJsonKey(SStrToken *t0){ while(true){ diff --git a/src/common/CMakeLists.txt b/src/common/CMakeLists.txt index 0cd5059226..14f6733e51 100644 --- a/src/common/CMakeLists.txt +++ b/src/common/CMakeLists.txt @@ -2,8 +2,6 @@ CMAKE_MINIMUM_REQUIRED(VERSION 3.0...3.20) PROJECT(TDengine) INCLUDE_DIRECTORIES(inc) -INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc) -INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc) AUX_SOURCE_DIRECTORY(src SRC) ADD_LIBRARY(common ${SRC}) diff --git a/src/inc/tsdb.h b/src/inc/tsdb.h index a3d7b4a184..a44e958be4 100644 --- a/src/inc/tsdb.h +++ b/src/inc/tsdb.h @@ -421,6 +421,11 @@ bool tsdbNoProblem(STsdbRepo* pRepo); // unit of walSize: MB int tsdbCheckWal(STsdbRepo *pRepo, uint32_t walSize); +// for json tag +void* getJsonTagValueElment(void* data, char* key, int32_t keyLen, char* out, int16_t bytes); +void getJsonTagValueAll(void* data, void* dst, int16_t bytes); +char* parseTagDatatoJson(void *p); + #ifdef __cplusplus } #endif diff --git a/src/tsdb/CMakeLists.txt b/src/tsdb/CMakeLists.txt index b94d0c381d..0f472cfbfc 100644 --- a/src/tsdb/CMakeLists.txt +++ b/src/tsdb/CMakeLists.txt @@ -2,7 +2,6 @@ CMAKE_MINIMUM_REQUIRED(VERSION 3.0...3.20) PROJECT(TDengine) INCLUDE_DIRECTORIES(inc) -INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc) AUX_SOURCE_DIRECTORY(src SRC) ADD_LIBRARY(tsdb ${SRC}) diff --git a/src/tsdb/inc/tsdbMeta.h b/src/tsdb/inc/tsdbMeta.h index 1f9ba44755..9cdb8a83aa 100644 --- a/src/tsdb/inc/tsdbMeta.h +++ b/src/tsdb/inc/tsdbMeta.h @@ -99,6 +99,7 @@ int tsdbUpdateLastColSchema(STable *pTable, STSchema *pNewSchema); STSchema* tsdbGetTableLatestSchema(STable *pTable); void tsdbFreeLastColumns(STable* pTable); int tsdbCompareJsonMapValue(const void* a, const void* b); +void* tsdbGetJsonTagValue(STable* pTable, char* key, int32_t keyLen, int16_t* colId); static FORCE_INLINE int tsdbCompareSchemaVersion(const void *key1, const void *key2) { if (*(int16_t *)key1 < schemaVersion(*(STSchema **)key2)) { diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index 04af98a017..18225ab70c 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -1071,6 +1071,19 @@ static void tsdbRemoveTableFromMeta(STsdbRepo *pRepo, STable *pTable, bool rmFro tsdbUnRefTable(pTable); } +void* tsdbGetJsonTagValue(STable* pTable, char* key, int32_t keyLen, int16_t* retColId){ + assert(TABLE_TYPE(pTable) == TSDB_CHILD_TABLE); + STable* superTable= pTable->pSuper; + SArray** data = (SArray**)taosHashGet(superTable->jsonKeyMap, key, keyLen); + if(data == NULL) return NULL; + JsonMapValue jmvalue = {pTable, 0}; + JsonMapValue* p = taosArraySearch(*data, &jmvalue, tsdbCompareJsonMapValue, TD_EQ); + if (p == NULL) return NULL; + int16_t colId = p->colId + 1; + if(retColId) *retColId = p->colId; + return tdGetKVRowValOfCol(pTable->tagVal, colId); +} + int tsdbCompareJsonMapValue(const void* a, const void* b) { const JsonMapValue* x = (const JsonMapValue*)a; const JsonMapValue* y = (const JsonMapValue*)b; @@ -1670,3 +1683,4 @@ static void tsdbFreeTableSchema(STable *pTable) { taosArrayDestroy(pTable->schema); } } + diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index e63c6a0c8f..e05289304f 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -26,7 +26,7 @@ #include "tsdbint.h" #include "texpr.h" #include "qFilter.h" -#include "tscUtil.h" +#include "cJSON.h" #define EXTRA_BYTES 2 #define ASCENDING_TRAVERSE(o) (o == TSDB_ORDER_ASC) @@ -4047,7 +4047,7 @@ static FORCE_INLINE int32_t tsdbGetJsonTagDataFromId(void *param, int32_t id, ch if (id == TSDB_TBNAME_COLUMN_INDEX) { *data = TABLE_NAME(pTable); } else { - void* jsonData = getJsonTagValue(pTable, name, TSDB_MAX_JSON_KEY_MD5_LEN, NULL); + void* jsonData = tsdbGetJsonTagValue(pTable, name, TSDB_MAX_JSON_KEY_MD5_LEN, NULL); // jsonData == NULL for ? operation // if(jsonData != NULL) jsonData += CHAR_BYTES; // jump type *data = jsonData; @@ -4148,5 +4148,149 @@ static int32_t tsdbQueryTableList(STable* pTable, SArray* pRes, void* filterInfo return TSDB_CODE_SUCCESS; } +void* getJsonTagValueElment(void* data, char* key, int32_t keyLen, char* dst, int16_t bytes){ + char keyMd5[TSDB_MAX_JSON_KEY_MD5_LEN] = {0}; + jsonKeyMd5(key, keyLen, keyMd5); + + void* result = tsdbGetJsonTagValue(data, keyMd5, TSDB_MAX_JSON_KEY_MD5_LEN, NULL); + if (result == NULL){ // json key no result + if(!dst) return NULL; + *(char*)dst = TSDB_DATA_TYPE_JSON; + setNull(dst + CHAR_BYTES, TSDB_DATA_TYPE_JSON, 0); + return dst; + } + + char* realData = POINTER_SHIFT(result, CHAR_BYTES); + if(*(char*)result == TSDB_DATA_TYPE_NCHAR || *(char*)result == TSDB_DATA_TYPE_BINARY) { + assert(varDataTLen(realData) < bytes); + if(!dst) return result; + memcpy(dst, result, CHAR_BYTES + varDataTLen(realData)); + return dst; + }else if (*(char*)result == TSDB_DATA_TYPE_DOUBLE || *(char*)result == TSDB_DATA_TYPE_BIGINT) { + if(!dst) return result; + memcpy(dst, result, CHAR_BYTES + LONG_BYTES); + return dst; + }else if (*(char*)result == TSDB_DATA_TYPE_BOOL) { + if(!dst) return result; + memcpy(dst, result, CHAR_BYTES + CHAR_BYTES); + return dst; + }else { + assert(0); + } + return result; +} + +void getJsonTagValueAll(void* data, void* dst, int16_t bytes) { + char* json = parseTagDatatoJson(data); + char* tagData = dst + CHAR_BYTES; + *(char*)dst = TSDB_DATA_TYPE_JSON; + if(json == NULL){ + setNull(tagData, TSDB_DATA_TYPE_JSON, 0); + return; + } + + int32_t length = 0; + if(!taosMbsToUcs4(json, strlen(json), varDataVal(tagData), bytes - VARSTR_HEADER_SIZE - CHAR_BYTES, &length)){ + tsdbError("getJsonTagValueAll mbstoucs4 error! length:%d", length); + } + varDataSetLen(tagData, length); + assert(varDataTLen(tagData) <= bytes); + tfree(json); +} + +char* parseTagDatatoJson(void *p){ + char* string = NULL; + cJSON *json = cJSON_CreateObject(); + if (json == NULL) + { + goto end; + } + + int16_t nCols = kvRowNCols(p); + ASSERT(nCols%2 == 1); + char tagJsonKey[TSDB_MAX_JSON_KEY_LEN + 1] = {0}; + for (int j = 0; j < nCols; ++j) { + SColIdx * pColIdx = kvRowColIdxAt(p, j); + void* val = (kvRowColVal(p, pColIdx)); + if (j == 0){ + int8_t jsonPlaceHolder = *(int8_t*)val; + ASSERT(jsonPlaceHolder == TSDB_DATA_JSON_PLACEHOLDER); + continue; + } + if(j == 1){ + uint32_t jsonNULL = *(uint32_t*)(varDataVal(val)); + ASSERT(jsonNULL == TSDB_DATA_JSON_NULL); + continue; + } + if (j == 2){ + if(*(uint32_t*)(varDataVal(val + CHAR_BYTES)) == TSDB_DATA_JSON_NULL) goto end; + continue; + } + if (j%2 == 1) { // json key encode by binary + ASSERT(varDataLen(val) <= TSDB_MAX_JSON_KEY_LEN); + memset(tagJsonKey, 0, sizeof(tagJsonKey)); + memcpy(tagJsonKey, varDataVal(val), varDataLen(val)); + }else{ // json value + char tagJsonValue[TSDB_MAX_JSON_TAGS_LEN] = {0}; + char* realData = POINTER_SHIFT(val, CHAR_BYTES); + char type = *(char*)val; + if(type == TSDB_DATA_TYPE_BINARY) { + assert(*(uint32_t*)varDataVal(realData) == TSDB_DATA_JSON_null); // json null value + assert(varDataLen(realData) == INT_BYTES); + cJSON* value = cJSON_CreateNull(); + if (value == NULL) + { + goto end; + } + cJSON_AddItemToObject(json, tagJsonKey, value); + }else if(type == TSDB_DATA_TYPE_NCHAR) { + int32_t length = taosUcs4ToMbs(varDataVal(realData), varDataLen(realData), tagJsonValue); + if (length < 0) { + tsdbError("charset:%s to %s. val:%s convert json value failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, + (char*)val); + goto end; + } + cJSON* value = cJSON_CreateString(tagJsonValue); + + if (value == NULL) + { + goto end; + } + cJSON_AddItemToObject(json, tagJsonKey, value); + }else if(type == TSDB_DATA_TYPE_DOUBLE){ + double jsonVd = *(double*)(realData); + cJSON* value = cJSON_CreateNumber(jsonVd); + if (value == NULL) + { + goto end; + } + cJSON_AddItemToObject(json, tagJsonKey, value); + }else if(type == TSDB_DATA_TYPE_BIGINT){ + int64_t jsonVd = *(int64_t*)(realData); + cJSON* value = cJSON_CreateNumber(jsonVd); + if (value == NULL) + { + goto end; + } + cJSON_AddItemToObject(json, tagJsonKey, value); + }else if (type == TSDB_DATA_TYPE_BOOL) { + char jsonVd = *(char*)(realData); + cJSON* value = cJSON_CreateBool(jsonVd); + if (value == NULL) + { + goto end; + } + cJSON_AddItemToObject(json, tagJsonKey, value); + } + else{ + tsdbError("unsupportted json value"); + } + } + } + string = cJSON_PrintUnformatted(json); +end: + cJSON_Delete(json); + return string; +} -- GitLab