Skip to content

T
TDengine

taosdata / TDengine
1 年多 前同步成功

通知 1185
Star 22016
Fork 4786
T
TDengine
提交 a91392a9 编写于 作者: wmmhello's avatar wmmhello
浏览文件
操作

TD-6129<feature> delete json binary, only support json nchar

上级 91c750e2
隐藏空白更改
内联 并排
Showing with 158 addition and 229 deletion
src/client/inc/tscUtil.h
浏览文件 @ a91392a9
......@@ -376,12 +376,12 @@ void tscRemoveCachedTableMeta(STableMetaInfo* pTableMetaInfo, uint64_t id);
char* cloneCurrentDBName(SSqlObj* pSql);
int parseJsontoTagData(char* json, SKVRowBuilder* kvRowBuilder, char* errMsg, int16_t startColId, uint8_t jsonType);
char* parseTagDatatoJson(void *p, uint8_t jsonType);
void findTagValue(STable* data, char* key, int32_t keyLen, char* out, int16_t len, uint8_t jsonType);
int parseJsontoTagData(char* json, SKVRowBuilder* kvRowBuilder, char* errMsg, int16_t startColId);
char* parseTagDatatoJson(void *p);
void findTagValue(STable* data, char* key, int32_t keyLen, char* out, int16_t len);
int8_t jsonType2DbType(double data, int jsonType, uint8_t type);
void* getJsonTagValue(STable* pTable, char* key, int32_t keyLen, uint8_t jsonType, int16_t* colId);
int8_t jsonType2DbType(double data, int jsonType);
void* getJsonTagValue(STable* pTable, char* key, int32_t keyLen, int16_t* colId);
#ifdef __cplusplus
}
......
src/client/src/tscLocal.c
浏览文件 @ a91392a9
......@@ -88,10 +88,10 @@ static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) {
STR_WITH_MAXSIZE_TO_VARSTR(dst, type, pField->bytes);
int32_t bytes = pSchema[i].bytes;
if (pSchema[i].type == TSDB_DATA_TYPE_BINARY || pSchema[i].type == TSDB_DATA_TYPE_JSON_BINARY){
if (pSchema[i].type == TSDB_DATA_TYPE_BINARY){
bytes -= VARSTR_HEADER_SIZE;
}
else if(pSchema[i].type == TSDB_DATA_TYPE_NCHAR || pSchema[i].type == TSDB_DATA_TYPE_JSON_NCHAR) {
else if(pSchema[i].type == TSDB_DATA_TYPE_NCHAR || pSchema[i].type == TSDB_DATA_TYPE_JSON) {
bytes -= VARSTR_HEADER_SIZE;
bytes = bytes / TSDB_NCHAR_SIZE;
}
......
src/client/src/tscParseInsert.c
浏览文件 @ a91392a9
......@@ -387,8 +387,7 @@ int32_t tsParseOneColumn(SSchema *pSchema, SStrToken *pToken, char *payload, cha
}
break;
case TSDB_DATA_TYPE_JSON_BINARY:
case TSDB_DATA_TYPE_JSON_NCHAR:
case TSDB_DATA_TYPE_JSON:
if (pToken->n >= pSchema->bytes) { // reserve 1 byte for select
return tscInvalidOperationMsg(msg, "json tag length too long", pToken->z);
}
......@@ -1082,7 +1081,7 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql, char** boundC
}
// encode json tag string
if(spd.numOfBound == 1 && IS_JSON_DATA_TYPE(pTagSchema[spd.boundedColumns[0]].type)){
if(spd.numOfBound == 1 && pTagSchema[spd.boundedColumns[0]].type == TSDB_DATA_TYPE_JSON){
char tmp = sToken.z[sToken.n];
sToken.z[sToken.n] = 0;
// if(sToken.type != TK_STRING) {
......@@ -1090,7 +1089,7 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql, char** boundC
// tscDestroyBoundColumnInfo(&spd);
// return tscSQLSyntaxErrMsg(pInsertParam->msg, "json type error, should be string", NULL);
// }
code = parseJsontoTagData(sToken.z, &kvRowBuilder, pInsertParam->msg, pTagSchema[spd.boundedColumns[0]].colId, pTagSchema[spd.boundedColumns[0]].type);
code = parseJsontoTagData(sToken.z, &kvRowBuilder, pInsertParam->msg, pTagSchema[spd.boundedColumns[0]].colId);
if (code != TSDB_CODE_SUCCESS) {
tdDestroyKVRowBuilder(&kvRowBuilder);
tscDestroyBoundColumnInfo(&spd);
......@@ -1113,7 +1112,7 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql, char** boundC
return tscSQLSyntaxErrMsg(pInsertParam->msg, "tag value expected", NULL);
}
// encode json tag string
if(spd.numOfBound == 1 && IS_JSON_DATA_TYPE(pTagSchema[spd.boundedColumns[0]].type)){
if(spd.numOfBound == 1 && pTagSchema[spd.boundedColumns[0]].type == TSDB_DATA_TYPE_JSON){
if(kvRowLen(row) >= pTagSchema[spd.boundedColumns[0]].bytes){ // reserve 1 byte for select
char tmp[128]= {0};
sprintf(tmp, "tag value is too small, can not contain encoded json tag:%d|%d", kvRowLen(row), pTagSchema[spd.boundedColumns[0]].bytes);
......
src/client/src/tscPrepare.c
浏览文件 @ a91392a9
......@@ -905,7 +905,7 @@ static int doBindBatchParam(STableDataBlocks* pBlock, SParamInfo* param, TAOS_MU
}
varDataSetLen(data + param->offset, output);
} else if (IS_JSON_DATA_TYPE(param->type)) { // todo json
} else if (param->type == TSDB_DATA_TYPE_JSON) { // todo json
}
}
......
src/client/src/tscSQLParser.c
浏览文件 @ a91392a9
......@@ -162,7 +162,7 @@ bool serializeExprListToVariant(SArray* pList, tVariant **dst, int16_t colType,
tSqlExpr* item = ((tSqlExprItem*)(taosArrayGet(pList, 0)))->pNode;
int32_t firstVarType = item->value.nType;
if(IS_JSON_DATA_TYPE(colType)) colType = firstVarType;
if(colType == TSDB_DATA_TYPE_JSON) colType = firstVarType;
SBufferWriter bw = tbufInitWriter( NULL, false);
tbufEnsureCapacity(&bw, 512);
......@@ -1440,7 +1440,7 @@ static bool validateTableColumnInfo(SArray* pFieldList, SSqlCmd* pCmd) {
int32_t nLen = 0;
for (int32_t i = 0; i < numOfCols; ++i) {
pField = taosArrayGet(pFieldList, i);
if (!isValidDataType(pField->type) || IS_JSON_DATA_TYPE(pField->type)) {
if (!isValidDataType(pField->type) || pField->type == TSDB_DATA_TYPE_JSON) {
invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4);
return false;
}
......@@ -1505,7 +1505,7 @@ static bool validateTagParams(SArray* pTagsList, SArray* pFieldList, SSqlCmd* pC
return false;
}
if (IS_JSON_DATA_TYPE(p->type) && numOfTags != 1) {
if (p->type == TSDB_DATA_TYPE_JSON && numOfTags != 1) {
invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg8);
return false;
}
......@@ -1592,7 +1592,7 @@ int32_t validateOneTag(SSqlCmd* pCmd, TAOS_FIELD* pTagField) {
// invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1);
// return false;
//}
if (IS_JSON_DATA_TYPE(pTagField->type)) {
if (pTagField->type == TSDB_DATA_TYPE_JSON) {
return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg8);
}
......@@ -1603,7 +1603,7 @@ int32_t validateOneTag(SSqlCmd* pCmd, TAOS_FIELD* pTagField) {
SSchema* pTagSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta);
int32_t nLen = 0;
if (numOfTags == 1 && IS_JSON_DATA_TYPE(pTagSchema[0].type)){
if (numOfTags == 1 && pTagSchema[0].type == TSDB_DATA_TYPE_JSON){
return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg8);
}
......@@ -2319,7 +2319,7 @@ int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, t
}
SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, index.columnIndex);
if (tokenId == TK_ARROW && !IS_JSON_DATA_TYPE(pSchema->type)) {
if (tokenId == TK_ARROW && pSchema->type != TSDB_DATA_TYPE_JSON) {
return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4);
}
......@@ -4471,7 +4471,7 @@ static int32_t validateMatchExpr(tSqlExpr* pExpr, STableMeta* pTableMeta, int32_
return invalidOperationMsg(msgBuf, msg2);
}
if(pLeft->tokenId == TK_ARROW && pSchema[index].type == TSDB_DATA_TYPE_JSON_NCHAR){
if(pLeft->tokenId == TK_ARROW && pSchema[index].type == TSDB_DATA_TYPE_JSON){
return invalidOperationMsg(msgBuf, msg4);
}
......@@ -4648,7 +4648,7 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSql
}
// check for json tag operation -> and ?
if (IS_JSON_DATA_TYPE(pSchema->type)){
if (pSchema->type == TSDB_DATA_TYPE_JSON){
code = validateJsonTagExpr(*pExpr, tscGetErrorMsgPayload(pCmd));
if (code != TSDB_CODE_SUCCESS) {
return code;
......@@ -6149,7 +6149,7 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
d += sizeof(STColumn);
}
if (IS_JSON_DATA_TYPE(pTagsSchema->type)){
if (pTagsSchema->type == TSDB_DATA_TYPE_JSON){
SKVRowBuilder kvRowBuilder = {0};
if (tdInitKVRowBuilder(&kvRowBuilder) < 0) {
return TSDB_CODE_TSC_OUT_OF_MEMORY;
......@@ -6163,7 +6163,7 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
int8_t tagVal = TSDB_DATA_BINARY_PLACEHOLDER;
tdAddColToKVRow(&kvRowBuilder, pTagsSchema->colId, pTagsSchema->type, &tagVal, false);
code = parseJsontoTagData(pItem->pVar.pz, &kvRowBuilder, pMsg, pTagsSchema->colId, pTagsSchema->type);
code = parseJsontoTagData(pItem->pVar.pz, &kvRowBuilder, pMsg, pTagsSchema->colId);
if (code != TSDB_CODE_SUCCESS) {
tdDestroyKVRowBuilder(&kvRowBuilder);
return code;
......@@ -6191,7 +6191,7 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
}
int32_t len = 0;
if(IS_JSON_DATA_TYPE(pTagsSchema->type)){
if(pTagsSchema->type == TSDB_DATA_TYPE_JSON){
len = kvRowLen(pUpdateMsg->data + schemaLen);
}else if (!IS_VAR_DATA_TYPE(pTagsSchema->type)) {
len = tDataTypes[pTagsSchema->type].bytes;
......@@ -7695,13 +7695,13 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) {
} else if (pItem->pVar.nType == TSDB_DATA_TYPE_TIMESTAMP) {
pItem->pVar.i64 = convertTimePrecision(pItem->pVar.i64, TSDB_TIME_PRECISION_NANO, tinfo.precision);
}
} else if (IS_JSON_DATA_TYPE(pSchema->type)) {
} else if (pSchema->type == TSDB_DATA_TYPE_JSON) {
if (pItem->pVar.nLen > TSDB_MAX_TAGS_LEN) {
tdDestroyKVRowBuilder(&kvRowBuilder);
return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3);
}
}
if(IS_JSON_DATA_TYPE(pSchema->type)){
if(pSchema->type == TSDB_DATA_TYPE_JSON){
*((int8_t *)tagVal) = TSDB_DATA_BINARY_PLACEHOLDER;
}else {
ret = tVariantDump(&(pItem->pVar), tagVal, pSchema->type, true);
......@@ -7757,13 +7757,13 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) {
} else if (pItem->pVar.nType == TSDB_DATA_TYPE_TIMESTAMP) {
pItem->pVar.i64 = convertTimePrecision(pItem->pVar.i64, TSDB_TIME_PRECISION_NANO, tinfo.precision);
}
} else if (IS_JSON_DATA_TYPE(pSchema->type)) {
} else if (pSchema->type == TSDB_DATA_TYPE_JSON) {
if (pItem->pVar.nLen > pSchema->bytes) {
tdDestroyKVRowBuilder(&kvRowBuilder);
return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3);
}
}
if(IS_JSON_DATA_TYPE(pSchema->type)){
if(pSchema->type == TSDB_DATA_TYPE_JSON){
*((int8_t *)tagVal) = TSDB_DATA_BINARY_PLACEHOLDER;
}else {
ret = tVariantDump(&(pItem->pVar), tagVal, pSchema->type, true);
......@@ -7787,7 +7787,7 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) {
}
// encode json tag string
if(schemaSize == 1 && IS_JSON_DATA_TYPE(pTagSchema[0].type)){
if(schemaSize == 1 && pTagSchema[0].type == TSDB_DATA_TYPE_JSON){
if (valSize != schemaSize) {
tdDestroyKVRowBuilder(&kvRowBuilder);
return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5);
......@@ -7798,7 +7798,7 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) {
tdDestroyKVRowBuilder(&kvRowBuilder);
return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6);
}
ret = parseJsontoTagData(pItem->pVar.pz, &kvRowBuilder, tscGetErrorMsgPayload(pCmd), pTagSchema[0].colId, pTagSchema[0].type);
ret = parseJsontoTagData(pItem->pVar.pz, &kvRowBuilder, tscGetErrorMsgPayload(pCmd), pTagSchema[0].colId);
if (ret != TSDB_CODE_SUCCESS) {
tdDestroyKVRowBuilder(&kvRowBuilder);
return ret;
......@@ -7812,7 +7812,7 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) {
}
tdSortKVRowByColIdx(row);
pTag->dataLen = kvRowLen(row);
if(schemaSize == 1 && IS_JSON_DATA_TYPE(pTagSchema[0].type)){
if(schemaSize == 1 && pTagSchema[0].type == TSDB_DATA_TYPE_JSON){
if(kvRowLen(row) >= pTagSchema[0].bytes){ // reserve 1 byte for select
char tmp[128]= {0};
sprintf(tmp, "tag value is too small, can not contain encoded json tag:%d|%d", kvRowLen(row), pTagSchema[0].bytes);
......
src/client/src/tscSql.c
浏览文件 @ a91392a9
......@@ -439,9 +439,9 @@ TAOS_FIELD *taos_fetch_fields(TAOS_RES *res) {
f[j] = pField->field;
// revise the length for binary and nchar fields
if (f[j].type == TSDB_DATA_TYPE_BINARY || f[j].type == TSDB_DATA_TYPE_JSON_BINARY) {
if (f[j].type == TSDB_DATA_TYPE_BINARY) {
f[j].bytes -= VARSTR_HEADER_SIZE;
} else if (f[j].type == TSDB_DATA_TYPE_NCHAR || f[j].type == TSDB_DATA_TYPE_JSON_NCHAR) {
} else if (f[j].type == TSDB_DATA_TYPE_NCHAR || f[j].type == TSDB_DATA_TYPE_JSON) {
f[j].bytes = (f[j].bytes - VARSTR_HEADER_SIZE)/TSDB_NCHAR_SIZE;
}
......
src/client/src/tscSubquery.c
浏览文件 @ a91392a9
......@@ -3552,7 +3552,7 @@ TAOS_ROW doSetResultRowData(SSqlObj *pSql) {
int32_t type = pInfo->field.type;
int32_t bytes = pInfo->field.bytes;
if (!IS_VAR_DATA_TYPE(type) && !IS_JSON_DATA_TYPE(type)) {
if (!IS_VAR_DATA_TYPE(type) && type != TSDB_DATA_TYPE_JSON) {
pRes->tsrow[j] = isNull(pRes->urow[i], type) ? NULL : pRes->urow[i];
} else {
pRes->tsrow[j] = isNull(pRes->urow[i], type) ? NULL : varDataVal(pRes->urow[i]);
......
src/client/src/tscUtil.c
浏览文件 @ a91392a9
......@@ -73,8 +73,7 @@ int32_t converToStr(char *str, int type, void *buf, int32_t bufSize, int32_t *le
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR:
case TSDB_DATA_TYPE_JSON_BINARY:
case TSDB_DATA_TYPE_JSON_NCHAR:
case TSDB_DATA_TYPE_JSON:
if (bufSize < 0) {
tscError("invalid buf size");
return TSDB_CODE_TSC_INVALID_VALUE;
......@@ -715,7 +714,7 @@ static void setResRawPtrImpl(SSqlRes* pRes, SInternalField* pInfo, int32_t i, bo
}
memcpy(pRes->urow[i], pRes->buffer[i], pInfo->field.bytes * pRes->numOfRows);
}else if (IS_JSON_DATA_TYPE(pInfo->field.type)) {
}else if (pInfo->field.type == TSDB_DATA_TYPE_JSON) {
// convert unicode to native code in a temporary buffer extra one byte for terminated symbol
char* buffer = realloc(pRes->buffer[i], pInfo->field.bytes * pRes->numOfRows);
if(buffer == NULL)
......@@ -729,11 +728,11 @@ static void setResRawPtrImpl(SSqlRes* pRes, SInternalField* pInfo, int32_t i, bo
char* dst = pRes->buffer[i] + k * pInfo->field.bytes;
char* realData = p + CHAR_BYTES;
if (*p == SELECT_ALL_JSON_TAG){
char* json = parseTagDatatoJson(realData, pInfo->field.type);
char* json = parseTagDatatoJson(realData);
if(json) {
memcpy(varDataVal(dst), json, strlen(json));
varDataSetLen(dst, strlen(json));
assert(varDataTLen(dst) <= pInfo->field.bytes);
assert(varDataTLen(dst) <= pInfo->field.type);
tfree(json);
}
}else if (*p == SELECT_ELEMENT_JSON_TAG){
......@@ -5177,18 +5176,14 @@ char* cloneCurrentDBName(SSqlObj* pSql) {
return p;
}
void findTagValue(STable* data, char* key, int32_t keyLen, char* out, int16_t len, uint8_t jsonType){
void* result = getJsonTagValue(data, key, keyLen, jsonType, NULL);
void findTagValue(STable* data, char* key, int32_t keyLen, char* out, int16_t len){
void* result = getJsonTagValue(data, key, keyLen, NULL);
if (result == NULL){ // json key no result
return;
}
char* realData = POINTER_SHIFT(result, CHAR_BYTES);
if (*(char*)result == TSDB_DATA_TYPE_BINARY){
assert(varDataLen(realData) <= len);
memcpy(varDataVal(out), varDataVal(realData), varDataLen(realData));
varDataSetLen(out, varDataLen(realData));
} else if(*(char*)result == TSDB_DATA_TYPE_NCHAR) {
if(*(char*)result == TSDB_DATA_TYPE_NCHAR) {
char tagJsonValue[TSDB_MAX_TAGS_LEN] = {0};
int32_t length = taosUcs4ToMbs(varDataVal(realData),
varDataLen(realData), tagJsonValue);
......@@ -5216,7 +5211,7 @@ void findTagValue(STable* data, char* key, int32_t keyLen, char* out, int16_t le
}
}
char* parseTagDatatoJson(void *p, uint8_t tagType){
char* parseTagDatatoJson(void *p){
char* string = NULL;
cJSON *json = cJSON_CreateObject();
if (json == NULL)
......@@ -5237,30 +5232,23 @@ char* parseTagDatatoJson(void *p, uint8_t tagType){
}
if (j%2 != 0) { // json key
memset(tagJsonKey, 0, TSDB_MAX_TAGS_LEN);
if (tagType == TSDB_DATA_TYPE_JSON_BINARY){
strncpy(tagJsonKey, varDataVal(val), varDataLen(val));
} else if(tagType == TSDB_DATA_TYPE_JSON_NCHAR){
int32_t length = taosUcs4ToMbs(varDataVal(val), varDataLen(val), tagJsonKey);
if (length == 0) {
tscError("charset:%s to %s. val:%s convert json key failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, (char*)val);
goto end;
}
int32_t length = taosUcs4ToMbs(varDataVal(val), varDataLen(val), tagJsonKey);
if (length == 0) {
tscError("charset:%s to %s. val:%s convert json key failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, (char*)val);
goto end;
}
}else{ // json value
char tagJsonValue[TSDB_MAX_TAGS_LEN] = {0};
char* realData = POINTER_SHIFT(val, CHAR_BYTES);
char type = *(char*)val;
if(IS_VAR_DATA_TYPE(type)){
if (type == TSDB_DATA_TYPE_BINARY){
strncpy(tagJsonValue, varDataVal(realData), varDataLen(realData));
} else if(type == TSDB_DATA_TYPE_NCHAR) {
int32_t length = taosUcs4ToMbs(varDataVal(realData), varDataLen(realData), tagJsonValue);
if (length == 0) {
tscError("charset:%s to %s. val:%s convert json value failed.", DEFAULT_UNICODE_ENCODEC, tsCharset,
(char*)val);
goto end;
}
if(type == TSDB_DATA_TYPE_NCHAR) {
int32_t length = taosUcs4ToMbs(varDataVal(realData), varDataLen(realData), tagJsonValue);
if (length == 0) {
tscError("charset:%s to %s. val:%s convert json value failed.", DEFAULT_UNICODE_ENCODEC, tsCharset,
(char*)val);
goto end;
}
cJSON* value = cJSON_CreateString(tagJsonValue);
if (value == NULL)
{
......@@ -5295,7 +5283,7 @@ end:
return string;
}
int parseJsontoTagData(char* json, SKVRowBuilder* kvRowBuilder, char* errMsg, int16_t startColId, uint8_t type){
int parseJsontoTagData(char* json, SKVRowBuilder* kvRowBuilder, char* errMsg, int16_t startColId){
cJSON *root = cJSON_Parse(json);
if (root == NULL){
tscError("json parse error");
......@@ -5318,49 +5306,38 @@ int parseJsontoTagData(char* json, SKVRowBuilder* kvRowBuilder, char* errMsg, in
retCode = tscSQLSyntaxErrMsg(errMsg, "json inner error", NULL);
goto end;
}
if(strlen(item->string) >= TSDB_COL_NAME_LEN){
if(strlen(item->string) >= TSDB_MAX_JSON_KEY_LEN){
tscError("json key too long error");
retCode = tscSQLSyntaxErrMsg(errMsg, "json key too long, exceed 64", NULL);
retCode = tscSQLSyntaxErrMsg(errMsg, "json key too long, exceed 256", NULL);
goto end;
}
char tagVal[TSDB_MAX_TAGS_LEN] = {0};
// json key encode by binary
char tagKey[TSDB_MAX_JSON_KEY_LEN + VARSTR_HEADER_SIZE] = {0};
int32_t outLen = 0;
if (type == TSDB_DATA_TYPE_JSON_BINARY){
strncpy(varDataVal(tagVal), item->string, strlen(item->string));
outLen = strlen(item->string);
}else if(type == TSDB_DATA_TYPE_JSON_NCHAR){
if (!taosMbsToUcs4(item->string, strlen(item->string), varDataVal(tagVal), TSDB_MAX_TAGS_LEN - VARSTR_HEADER_SIZE, &outLen)) {
tscError("json string error:%s|%s", strerror(errno), item->string);
retCode = tscSQLSyntaxErrMsg(errMsg, "serizelize json error", NULL);
goto end;
}
}
strncpy(varDataVal(tagKey), item->string, strlen(item->string));
outLen = strlen(item->string);
varDataSetLen(tagVal, outLen);
tdAddColToKVRow(kvRowBuilder, jsonIndex++, TSDB_DATA_TYPE_NCHAR, tagVal, false); // add json key
varDataSetLen(tagKey, outLen);
tdAddColToKVRow(kvRowBuilder, jsonIndex++, TSDB_DATA_TYPE_NCHAR, tagKey, false); // add json key
memset(tagVal, 0, TSDB_MAX_TAGS_LEN);
if(item->type == cJSON_String){ // add json value format: type|data
outLen = 0;
*tagVal = jsonType2DbType(0, item->type, type); // type
char tagVal[TSDB_MAX_TAGS_LEN + VARSTR_HEADER_SIZE + CHAR_BYTES] = {0};
*tagVal = jsonType2DbType(0, item->type); // type
char* tagData = POINTER_SHIFT(tagVal,CHAR_BYTES);
if (type == TSDB_DATA_TYPE_JSON_BINARY){
strncpy(varDataVal(tagData), item->valuestring, strlen(item->valuestring));
outLen = strlen(item->valuestring);
}else if(type == TSDB_DATA_TYPE_JSON_NCHAR) {
if (!taosMbsToUcs4(item->valuestring, strlen(item->valuestring), varDataVal(tagData),
TSDB_MAX_TAGS_LEN - VARSTR_HEADER_SIZE, &outLen)) {
tscError("json string error:%s|%s", strerror(errno), item->string);
retCode = tscSQLSyntaxErrMsg(errMsg, "serizelize json error", NULL);
goto end;
}
if (!taosMbsToUcs4(item->valuestring, strlen(item->valuestring), varDataVal(tagData),
TSDB_MAX_TAGS_LEN, &outLen)) {
tscError("json string error:%s|%s", strerror(errno), item->string);
retCode = tscSQLSyntaxErrMsg(errMsg, "serizelize json error", NULL);
goto end;
}
varDataSetLen(tagData, outLen);
tdAddColToKVRow(kvRowBuilder, jsonIndex++, TSDB_DATA_TYPE_NCHAR, tagVal, true);
}else if(item->type == cJSON_Number){
*tagVal = jsonType2DbType(item->valuedouble, item->type, type); // type
char tagVal[LONG_BYTES + CHAR_BYTES] = {0};
*tagVal = jsonType2DbType(item->valuedouble, item->type); // type
char* tagData = POINTER_SHIFT(tagVal,CHAR_BYTES);
if(*tagVal == TSDB_DATA_TYPE_DOUBLE) *((double *)tagData) = item->valuedouble;
else if(*tagVal == TSDB_DATA_TYPE_BIGINT) *((int64_t *)tagData) = item->valueint;
......@@ -5376,12 +5353,12 @@ end:
return retCode;
}
int8_t jsonType2DbType(double data, int jsonType, uint8_t type){
int8_t jsonType2DbType(double data, int jsonType){
switch(jsonType){
case cJSON_Number:
if (data - (int64_t)data > 0) return TSDB_DATA_TYPE_DOUBLE; else return TSDB_DATA_TYPE_BIGINT;
case cJSON_String:
if (type == TSDB_DATA_TYPE_JSON_NCHAR) return TSDB_DATA_TYPE_NCHAR; else return TSDB_DATA_TYPE_BINARY;
return TSDB_DATA_TYPE_NCHAR;
case cJSON_True:
case cJSON_False:
return TSDB_DATA_TYPE_BOOL;
......@@ -5389,21 +5366,10 @@ int8_t jsonType2DbType(double data, int jsonType, uint8_t type){
return TSDB_DATA_TYPE_NULL;
}
void* getJsonTagValue(STable* pTable, char* key, int32_t keyLen, uint8_t jsonType, int16_t* retColId){
int32_t outLen = 0;
if(jsonType == TSDB_DATA_TYPE_JSON_NCHAR){
char tagKey[256] = {0};
if (!taosMbsToUcs4(key, keyLen, tagKey, 256, &outLen)) {
tscError("json key to ucs4 error:%s|%s", strerror(errno), key);
return NULL;
}
key = tagKey;
}else{
outLen = keyLen;
}
void* getJsonTagValue(STable* pTable, char* key, int32_t keyLen, int16_t* retColId){
if(TABLE_TYPE(pTable) == TSDB_CHILD_TABLE){
STable* superTable= pTable->pSuper;
SArray** data = (SArray**)taosHashGet(superTable->jsonKeyMap, key, outLen);
SArray** data = (SArray**)taosHashGet(superTable->jsonKeyMap, key, keyLen);
if(data == NULL) return NULL;
JsonMapValue jmvalue = {pTable, 0};
JsonMapValue* p = taosArraySearch(*data, &jmvalue, tsdbCompareJsonMapValue, TD_EQ);
......@@ -5412,7 +5378,7 @@ void* getJsonTagValue(STable* pTable, char* key, int32_t keyLen, uint8_t jsonTyp
if(retColId) *retColId = p->colId;
return tdGetKVRowValOfCol(pTable->tagVal, colId);
}else if(TABLE_TYPE(pTable) == TSDB_SUPER_TABLE){
SArray** data = (SArray**)taosHashGet(pTable->jsonKeyMap, key, outLen);
SArray** data = (SArray**)taosHashGet(pTable->jsonKeyMap, key, keyLen);
if(data == NULL) return NULL;
if(taosArrayGetSize(*data) == 0) return NULL;
JsonMapValue* p = taosArrayGet(*data, 0);
......@@ -5421,5 +5387,4 @@ void* getJsonTagValue(STable* pTable, char* key, int32_t keyLen, uint8_t jsonTyp
return tdGetKVRowValOfCol(((STable*)(p->table))->tagVal, colId);
}
return NULL;
}
}
\ No newline at end of file
src/common/src/tname.c
浏览文件 @ a91392a9
......@@ -190,7 +190,7 @@ static bool doValidateSchema(SSchema* pSchema, int32_t numOfCols, int32_t maxLen
int32_t rowLen = 0;
for (int32_t i = 0; i < numOfCols; ++i) {
if (IS_JSON_DATA_TYPE(pSchema[i].type) && numOfCols != 1){
if (pSchema[i].type == TSDB_DATA_TYPE_JSON && numOfCols != 1){
return false;
}
// 1. valid types
......@@ -199,11 +199,11 @@ static bool doValidateSchema(SSchema* pSchema, int32_t numOfCols, int32_t maxLen
}
// 2. valid length for each type
if (pSchema[i].type == TSDB_DATA_TYPE_BINARY || pSchema[i].type == TSDB_DATA_TYPE_JSON_BINARY) {
if (pSchema[i].type == TSDB_DATA_TYPE_BINARY) {
if (pSchema[i].bytes > TSDB_MAX_BINARY_LEN) {
return false;
}
} else if (pSchema[i].type == TSDB_DATA_TYPE_NCHAR || pSchema[i].type == TSDB_DATA_TYPE_JSON_NCHAR) {
} else if (pSchema[i].type == TSDB_DATA_TYPE_NCHAR || pSchema[i].type == TSDB_DATA_TYPE_JSON) {
if (pSchema[i].bytes > TSDB_MAX_NCHAR_LEN) {
return false;
}
......
src/common/src/ttypes.c
浏览文件 @ a91392a9
......@@ -34,8 +34,7 @@ const int32_t TYPE_BYTES[17] = {
sizeof(uint16_t), // TSDB_DATA_TYPE_USMALLINT
sizeof(uint32_t), // TSDB_DATA_TYPE_UINT
sizeof(uint64_t), // TSDB_DATA_TYPE_UBIGINT
sizeof(int8_t), // TSDB_DATA_TYPE_JSON_BINARY
sizeof(int8_t), // TSDB_DATA_TYPE_JSON_NCHAR
sizeof(int8_t), // TSDB_DATA_TYPE_JSON
};
#define DO_STATICS(__sum, __min, __max, __minIndex, __maxIndex, _list, _index) \
......@@ -369,7 +368,7 @@ static void getStatics_nchr(const void *pData, int32_t numOfRow, int64_t *min, i
*maxIndex = 0;
}
tDataTypeDescriptor tDataTypes[17] = {
tDataTypeDescriptor tDataTypes[16] = {
{TSDB_DATA_TYPE_NULL, 6, 1, "NOTYPE", 0, 0, NULL, NULL, NULL},
{TSDB_DATA_TYPE_BOOL, 4, CHAR_BYTES, "BOOL", false, true, tsCompressBool, tsDecompressBool, getStatics_bool},
{TSDB_DATA_TYPE_TINYINT, 7, CHAR_BYTES, "TINYINT", INT8_MIN, INT8_MAX, tsCompressTinyint, tsDecompressTinyint, getStatics_i8},
......@@ -385,8 +384,7 @@ tDataTypeDescriptor tDataTypes[17] = {
{TSDB_DATA_TYPE_USMALLINT, 17, SHORT_BYTES, "SMALLINT UNSIGNED", 0, UINT16_MAX, tsCompressSmallint, tsDecompressSmallint, getStatics_u16},
{TSDB_DATA_TYPE_UINT, 12, INT_BYTES, "INT UNSIGNED", 0, UINT32_MAX, tsCompressInt, tsDecompressInt, getStatics_u32},
{TSDB_DATA_TYPE_UBIGINT, 15, LONG_BYTES, "BIGINT UNSIGNED", 0, UINT64_MAX, tsCompressBigint, tsDecompressBigint, getStatics_u64},
{TSDB_DATA_TYPE_JSON_BINARY,11, 0, "JSON BINARY", 0, 0, tsCompressString, tsDecompressString, getStatics_bin},
{TSDB_DATA_TYPE_JSON_NCHAR, 10, 0, "JSON NCHAR", 0, 0, tsCompressString, tsDecompressString, getStatics_nchr},
{TSDB_DATA_TYPE_JSON,4, 0, "JSON", 0, 0, tsCompressString, tsDecompressString, getStatics_nchr},
};
char tTokenTypeSwitcher[13] = {
......@@ -432,7 +430,7 @@ FORCE_INLINE void* getDataMax(int32_t type) {
bool isValidDataType(int32_t type) {
return type >= TSDB_DATA_TYPE_NULL && type <= TSDB_DATA_TYPE_JSON_NCHAR;
return type >= TSDB_DATA_TYPE_NULL && type <= TSDB_DATA_TYPE_JSON;
}
void setVardataNull(void* val, int32_t type) {
......
src/common/src/tvariant.c
浏览文件 @ a91392a9
......@@ -159,7 +159,7 @@ void tVariantCreateFromBinary(tVariant *pVar, const char *pz, size_t len, uint32
break;
}
case TSDB_DATA_TYPE_NCHAR:
case TSDB_DATA_TYPE_JSON_NCHAR:{ // here we get the nchar length from raw binary bits length
case TSDB_DATA_TYPE_JSON:{ // here we get the nchar length from raw binary bits length
size_t lenInwchar = len / TSDB_NCHAR_SIZE;
pVar->wpz = calloc(1, (lenInwchar + 1) * TSDB_NCHAR_SIZE);
......@@ -168,8 +168,7 @@ void tVariantCreateFromBinary(tVariant *pVar, const char *pz, size_t len, uint32
break;
}
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_JSON_BINARY:{
case TSDB_DATA_TYPE_BINARY:{
pVar->pz = calloc(len + 1, sizeof(char));
memcpy(pVar->pz, pz, len);
pVar->nLen = (int32_t)len;
......@@ -920,13 +919,12 @@ int32_t tVariantDumpEx(tVariant *pVariant, char *payload, int16_t type, bool inc
break;
}
case TSDB_DATA_TYPE_JSON_BINARY:
case TSDB_DATA_TYPE_JSON_NCHAR:{
case TSDB_DATA_TYPE_JSON:{
if (pVariant->nType == TSDB_DATA_TYPE_NULL) {
//*(int8_t *)payload = TSDB_DATA_TINYINT_NULL;
} else if (pVariant->nType == TSDB_DATA_TYPE_BINARY){
//*((int8_t *)payload) = TSDB_DATA_BINARY_PLACEHOLDER;
} else if (IS_JSON_DATA_TYPE(pVariant->nType)){ // select * from stable, set tag type to json，from setTagValue/tag_project_function
} else if (pVariant->nType == TSDB_DATA_TYPE_JSON){ // select * from stable, set tag type to json，from setTagValue/tag_project_function
memcpy(payload, pVariant->pz, pVariant->nLen);
}else {
return -1;
......
src/connector/python/taos/bind.py
浏览文件 @ a91392a9
......@@ -124,7 +124,7 @@ class TaosBind(ctypes.Structure):
self.buffer_length = length
self.length = pointer(c_size_t(self.buffer_length))
def json_binary(self, value):
def json(self, value):
buffer = None
length = 0
if isinstance(value, str):
......@@ -134,22 +134,7 @@ class TaosBind(ctypes.Structure):
else:
buffer = value
length = len(value)
self.buffer_type = FieldType.C_JSON_BINARY
self.buffer = cast(buffer, c_void_p)
self.buffer_length = length
self.length = pointer(c_size_t(self.buffer_length))
def json_nchar(self, value):
buffer = None
length = 0
if isinstance(value, str):
bytes = value.encode("utf-8")
buffer = create_string_buffer(bytes)
length = len(bytes)
else:
buffer = value
length = len(value)
self.buffer_type = FieldType.C_JSON_NCHAR
self.buffer_type = FieldType.C_JSON
self.buffer = cast(buffer, c_void_p)
self.buffer_length = length
self.length = pointer(c_size_t(self.buffer_length))
......@@ -386,13 +371,9 @@ class TaosMultiBind(ctypes.Structure):
self.buffer_type = FieldType.C_NCHAR
self._str_to_buffer(values)
def json_binary(self, values):
self.buffer_type = FieldType.C_JSON_BINARY
self._str_to_buffer(values)
def json_nchar(self, values):
def json(self, values):
# type: (list[str]) -> None
self.buffer_type = FieldType.C_JSON_NCHAR
self.buffer_type = FieldType.C_JSON
self._str_to_buffer(values)
def tinyint_unsigned(self, values):
......
src/connector/python/taos/constants.py
浏览文件 @ a91392a9
......@@ -25,8 +25,7 @@ class FieldType(object):
C_SMALLINT_UNSIGNED = 12
C_INT_UNSIGNED = 13
C_BIGINT_UNSIGNED = 14
C_JSON_BINARY = 15
C_JSON_NCHAR = 16
C_JSON = 15
# NULL value definition
# NOTE: These values should change according to C definition in tsdb.h
C_BOOL_NULL = 0x02
......
src/connector/python/taos/cursor.py
浏览文件 @ a91392a9
......@@ -188,11 +188,8 @@ class TaosCursor(object):
if dataType.upper() == "NCHAR":
if self._description[col][1] == FieldType.C_NCHAR:
return True
if dataType.upper() == "JSON BINARY":
if self._description[col][1] == FieldType.C_JSON_BINARY:
return True
if dataType.upper() == "JSON NCHAR":
if self._description[col][1] == FieldType.C_JSON_NCHAR:
if dataType.upper() == "JSON":
if self._description[col][1] == FieldType.C_JSON:
return True
return False
......
src/connector/python/taos/field.py
浏览文件 @ a91392a9
......@@ -207,8 +207,7 @@ CONVERT_FUNC = {
FieldType.C_SMALLINT_UNSIGNED: _crow_smallint_unsigned_to_python,
FieldType.C_INT_UNSIGNED: _crow_int_unsigned_to_python,
FieldType.C_BIGINT_UNSIGNED: _crow_bigint_unsigned_to_python,
FieldType.C_JSON_NCHAR: _crow_nchar_to_python,
FieldType.C_JSON_BINARY: _crow_binary_to_python,
FieldType.C_JSON: _crow_nchar_to_python,
}
CONVERT_FUNC_BLOCK = {
......@@ -226,8 +225,7 @@ CONVERT_FUNC_BLOCK = {
FieldType.C_SMALLINT_UNSIGNED: _crow_smallint_unsigned_to_python,
FieldType.C_INT_UNSIGNED: _crow_int_unsigned_to_python,
FieldType.C_BIGINT_UNSIGNED: _crow_bigint_unsigned_to_python,
FieldType.C_JSON_NCHAR: _crow_nchar_to_python_block,
FieldType.C_JSON_BINARY: _crow_binary_to_python_block,
FieldType.C_JSON: _crow_nchar_to_python_block,
}
# Corresponding TAOS_FIELD structure in C
......
src/inc/taos.h
浏览文件 @ a91392a9
......@@ -46,8 +46,7 @@ typedef void **TAOS_ROW;
#define TSDB_DATA_TYPE_USMALLINT 12 // 2 bytes
#define TSDB_DATA_TYPE_UINT 13 // 4 bytes
#define TSDB_DATA_TYPE_UBIGINT 14 // 8 bytes
#define TSDB_DATA_TYPE_JSON_BINARY 15 // json string
#define TSDB_DATA_TYPE_JSON_NCHAR 16 // json unicode string
#define TSDB_DATA_TYPE_JSON 15 // json string
typedef enum {
TSDB_OPTION_LOCALE,
......
src/inc/taosdef.h
浏览文件 @ a91392a9
......@@ -221,6 +221,8 @@ do { \
#define TSDB_MAX_TAGS_LEN 16384
#define TSDB_MAX_TAGS 128
#define TSDB_MAX_TAG_CONDITIONS 1024
#define TSDB_MAX_JSON_KEY_LEN 256
#define TSDB_MAX_JSON_KEY_MD5_LEN 16
#define TSDB_AUTH_LEN 16
#define TSDB_KEY_LEN 16
......
src/inc/ttype.h
浏览文件 @ a91392a9
......@@ -41,7 +41,6 @@ typedef struct {
#define varDataLenByData(v) (*(VarDataLenT *)(((char*)(v)) - VARSTR_HEADER_SIZE))
#define varDataSetLen(v, _len) (((VarDataLenT *)(v))[0] = (VarDataLenT) (_len))
#define IS_VAR_DATA_TYPE(t) (((t) == TSDB_DATA_TYPE_BINARY) || ((t) == TSDB_DATA_TYPE_NCHAR))
#define IS_JSON_DATA_TYPE(t) (((t) == TSDB_DATA_TYPE_JSON_BINARY) || ((t) == TSDB_DATA_TYPE_JSON_NCHAR))
#define varDataNetLen(v) (htons(((VarDataLenT *)(v))[0]))
#define varDataNetTLen(v) (sizeof(VarDataLenT) + varDataNetLen(v))
......@@ -193,7 +192,7 @@ typedef struct tDataTypeDescriptor {
int16_t *minindex, int16_t *maxindex, int16_t *numofnull);
} tDataTypeDescriptor;
extern tDataTypeDescriptor tDataTypes[17];
extern tDataTypeDescriptor tDataTypes[16];
bool isValidDataType(int32_t type);
......
src/kit/shell/src/shellEngine.c
浏览文件 @ a91392a9
......@@ -683,8 +683,7 @@ static void printField(const char* val, TAOS_FIELD* field, int width, int32_t le
break;
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR:
case TSDB_DATA_TYPE_JSON_BINARY:
case TSDB_DATA_TYPE_JSON_NCHAR:
case TSDB_DATA_TYPE_JSON:
shellPrintNChar(val, length, width);
break;
case TSDB_DATA_TYPE_TIMESTAMP:
......@@ -792,7 +791,6 @@ static int calcColWidth(TAOS_FIELD* field, int precision) {
return MAX(25, width);
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_JSON_BINARY:
if (field->bytes > tsMaxBinaryDisplayWidth) {
return MAX(tsMaxBinaryDisplayWidth, width);
} else {
......@@ -800,7 +798,7 @@ static int calcColWidth(TAOS_FIELD* field, int precision) {
}
case TSDB_DATA_TYPE_NCHAR:
case TSDB_DATA_TYPE_JSON_NCHAR:{
case TSDB_DATA_TYPE_JSON:{
int16_t bytes = field->bytes * TSDB_NCHAR_SIZE;
if (bytes > tsMaxBinaryDisplayWidth) {
return MAX(tsMaxBinaryDisplayWidth, width);
......
src/query/inc/qFilter.h
浏览文件 @ a91392a9
......@@ -252,7 +252,7 @@ typedef struct SFilterInfo {
#define COL_FIELD_SIZE (sizeof(SFilterField) + 2 * sizeof(int64_t))
#define FILTER_NO_MERGE_DATA_TYPE(t) ((t) == TSDB_DATA_TYPE_BINARY || (t) == TSDB_DATA_TYPE_NCHAR || (t) == TSDB_DATA_TYPE_JSON_BINARY || (t) == TSDB_DATA_TYPE_JSON_NCHAR)
#define FILTER_NO_MERGE_DATA_TYPE(t) ((t) == TSDB_DATA_TYPE_BINARY || (t) == TSDB_DATA_TYPE_NCHAR || (t) == TSDB_DATA_TYPE_JSON)
#define FILTER_NO_MERGE_OPTR(o) ((o) == TSDB_RELATION_ISNULL || (o) == TSDB_RELATION_NOTNULL || (o) == FILTER_DUMMY_EMPTY_OPTR)
#define MR_EMPTY_RES(ctx) (ctx->rs == NULL)
......
src/query/src/qExecutor.c
浏览文件 @ a91392a9
......@@ -3357,7 +3357,7 @@ static void doSetTagValueInParam(void* pTable, int32_t tagColId, tVariant *tag,
int32_t len = (varDataLen(val) > maxLen)? maxLen:varDataLen(val);
tVariantCreateFromBinary(tag, varDataVal(val), len, type);
//tVariantCreateFromBinary(tag, varDataVal(val), varDataLen(val), type);
} else if(IS_JSON_DATA_TYPE(type)){
} else if(type == TSDB_DATA_TYPE_JSON){
assert(kvRowLen(val) < bytes);
tVariantCreateFromBinary(tag, val, bytes, type);
memcpy(tag->pz + 1, tag->pz, bytes - 1); // move back 1 byte for select type
......@@ -4335,7 +4335,7 @@ static void doCopyQueryResultToMsg(SQInfo *pQInfo, int32_t numOfRows, char *data
*compLen += compSizes[col];
compSizes[col] = htonl(compSizes[col]);
} else {
if(IS_JSON_DATA_TYPE(pColRes->info.type)){ // todo json
if(pColRes->info.type == TSDB_DATA_TYPE_JSON){ // todo json
//pColRes->info.bytes =
}
......@@ -7187,10 +7187,12 @@ static SSDataBlock* doTagScan(void* param, bool* newgroup) {
data = tsdbGetTableName(item->pTable);
} else {
data = tsdbGetTableTagVal(item->pTable, pExprInfo[j].base.colInfo.colId, type, bytes);
if(IS_JSON_DATA_TYPE(type)){
if(type == TSDB_DATA_TYPE_JSON){
if(pExprInfo[j].base.numOfParams > 0){ // tag-> operation
char* tagJsonElementData = calloc(bytes, 1);
findTagValue(item->pTable, pExprInfo[j].base.param[0].pz, pExprInfo[j].base.param[0].nLen, tagJsonElementData, bytes, type);
char keyMd5[TSDB_MAX_JSON_KEY_MD5_LEN] = {0};
jsonKeyMd5(pExprInfo[j].base.param[0].pz, pExprInfo[j].base.param[0].nLen, keyMd5);
findTagValue(item->pTable, keyMd5, TSDB_MAX_JSON_KEY_MD5_LEN, tagJsonElementData, bytes);
*dst = SELECT_ELEMENT_JSON_TAG; // select tag->element
dst++;
assert(varDataTLen(tagJsonElementData) < bytes);
......
src/query/src/qFilter.c
浏览文件 @ a91392a9
......@@ -199,8 +199,7 @@ int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) {
case TSDB_DATA_TYPE_TIMESTAMP: comparFn = 3; break;
case TSDB_DATA_TYPE_FLOAT: comparFn = 4; break;
case TSDB_DATA_TYPE_DOUBLE: comparFn = 5; break;
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_JSON_BINARY:{
case TSDB_DATA_TYPE_BINARY:{
if (optr == TSDB_RELATION_MATCH) {
comparFn = 19;
} else if (optr == TSDB_RELATION_NMATCH) {
......@@ -219,7 +218,7 @@ int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) {
}
case TSDB_DATA_TYPE_NCHAR:
case TSDB_DATA_TYPE_JSON_NCHAR:{
case TSDB_DATA_TYPE_JSON:{
if (optr == TSDB_RELATION_MATCH) {
comparFn = 19;
} else if (optr == TSDB_RELATION_NMATCH) {
......@@ -1162,24 +1161,26 @@ int32_t filterAddGroupUnitFromNode(SFilterInfo *info, tExprNode* tree, SArray *g
if(pLeft->nodeType == TSQL_NODE_EXPR && pLeft->_node.optr == TSDB_RELATION_ARROW){ // json tag -> operation
assert(info->pTable != NULL);
SSchema* schema = pLeft->_node.pLeft->pSchema;
if(pLeft->_node.pRight->pVal->nLen >= TSDB_COL_NAME_LEN) return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH;
memset(schema->name, 0, TSDB_COL_NAME_LEN);
strncpy(schema->name, pLeft->_node.pRight->pVal->pz, pLeft->_node.pRight->pVal->nLen);
if(pLeft->_node.pRight->pVal->nLen >= TSDB_MAX_JSON_KEY_LEN) return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH;
char keyMd5[TSDB_MAX_JSON_KEY_MD5_LEN] = {0};
jsonKeyMd5(pLeft->_node.pRight->pVal->pz, pLeft->_node.pRight->pVal->nLen, keyMd5);
strncpy(schema->name, keyMd5, TSDB_MAX_JSON_KEY_MD5_LEN);
void* data = getJsonTagValue(info->pTable, schema->name, strlen(schema->name), schema->type, &schema->colId);
void* data = getJsonTagValue(info->pTable, schema->name, strlen(schema->name), &schema->colId);
if(data == NULL) return TSDB_CODE_QRY_JSON_KEY_NOT_EXIST;
schema->type = *(char*)data; // if exist json tag-> operation get type so that can set data if (tree->_node.optr == TSDB_RELATION_IN_IN) the next and set value in filterInitValFieldData
assert(schema->type > TSDB_DATA_TYPE_NULL && schema->type < TSDB_DATA_TYPE_JSON_BINARY);
if((tree->_node.optr == TSDB_RELATION_MATCH || tree->_node.optr == TSDB_RELATION_NMATCH)
&& schema->type != TSDB_DATA_TYPE_BINARY)
return TSDB_CODE_QRY_JSON_KEY_NOT_STR_ERROR;
assert(schema->type > TSDB_DATA_TYPE_NULL && schema->type < TSDB_DATA_TYPE_JSON);
// if((tree->_node.optr == TSDB_RELATION_MATCH || tree->_node.optr == TSDB_RELATION_NMATCH)
// && schema->type != TSDB_DATA_TYPE_BINARY)
// return TSDB_CODE_QRY_JSON_KEY_NOT_STR_ERROR;
pLeft = pLeft->_node.pLeft; // -> operation use left as input
}else if(tree->_node.optr == TSDB_RELATION_QUESTION){
SSchema* schema = pLeft->pSchema;
if(tree->_node.pRight->pVal->nLen >= TSDB_COL_NAME_LEN) return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH;
memset(schema->name, 0, TSDB_COL_NAME_LEN);
strncpy(schema->name, tree->_node.pRight->pVal->pz, tree->_node.pRight->pVal->nLen);
if(tree->_node.pRight->pVal->nLen >= TSDB_MAX_JSON_KEY_LEN) return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH;
char keyMd5[TSDB_MAX_JSON_KEY_MD5_LEN] = {0};
jsonKeyMd5(tree->_node.pRight->pVal->pz, tree->_node.pRight->pVal->nLen, keyMd5);
strncpy(schema->name, keyMd5, TSDB_MAX_JSON_KEY_MD5_LEN);
}
SFilterFieldId left = {0}, right = {0};
......@@ -1835,10 +1836,10 @@ int32_t filterInitValFieldData(SFilterInfo *info) {
continue;
}
if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_JSON_BINARY) {
if (type == TSDB_DATA_TYPE_BINARY) {
size_t len = (var->nType == TSDB_DATA_TYPE_BINARY || var->nType == TSDB_DATA_TYPE_NCHAR) ? var->nLen : MAX_NUM_STR_SIZE;
fi->data = calloc(1, len + 1 + VARSTR_HEADER_SIZE);
} else if (type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_JSON_NCHAR) {
} else if (type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_JSON) {
size_t len = (var->nType == TSDB_DATA_TYPE_BINARY || var->nType == TSDB_DATA_TYPE_NCHAR) ? var->nLen : MAX_NUM_STR_SIZE;
fi->data = calloc(1, (len + 1) * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE);
} else {
......
src/query/src/qSqlParser.c
浏览文件 @ a91392a9
......@@ -702,18 +702,6 @@ void tSetColumnType(TAOS_FIELD *pField, SStrToken *type) {
break;
}
if((type->n == 4) &&
(strncasecmp(type->z, "json", type->n) == 0) && JSON_TYPE_NCHAR){
i = TSDB_DATA_TYPE_JSON_NCHAR;
break;
}
if((type->n == 4) &&
(strncasecmp(type->z, "json", type->n) == 0) && JSON_TYPE_BINARY){
i = TSDB_DATA_TYPE_JSON_BINARY;
break;
}
i += 1;
}
......@@ -728,7 +716,7 @@ void tSetColumnType(TAOS_FIELD *pField, SStrToken *type) {
pField->type = i;
pField->bytes = tDataTypes[i].bytes;
if (i == TSDB_DATA_TYPE_NCHAR || i == TSDB_DATA_TYPE_JSON_NCHAR) {
if (i == TSDB_DATA_TYPE_NCHAR || i == TSDB_DATA_TYPE_JSON) {
/*
* for nchar, the TOKENTYPE is the number of character, so the length is the
* number of bytes in UCS-4 format, which is 4 times larger than the number of characters
......@@ -745,7 +733,7 @@ void tSetColumnType(TAOS_FIELD *pField, SStrToken *type) {
}
pField->bytes = (int16_t)bytes;
}
} else if (i == TSDB_DATA_TYPE_BINARY || i == TSDB_DATA_TYPE_JSON_BINARY) {
} else if (i == TSDB_DATA_TYPE_BINARY) {
/* for binary, the TOKENTYPE is the length of binary */
if (type->type == 0) {
pField->bytes = 0;
......
src/tsdb/src/tsdbMeta.c
浏览文件 @ a91392a9
......@@ -14,6 +14,7 @@
*/
#include "tsdbint.h"
#include "tcompare.h"
#include "tutil.h"
#define TSDB_SUPER_TABLE_SL_LEVEL 5
#define DEFAULT_TAG_INDEX_COLUMN 0
......@@ -211,7 +212,7 @@ void *tsdbGetTableTagVal(const void* pTable, int32_t colId, int16_t type, int16_
}
char *val = NULL;
if (IS_JSON_DATA_TYPE(pCol->type)){
if (pCol->type == TSDB_DATA_TYPE_JSON){
val = ((STable*)pTable)->tagVal;
}else{
val = tdGetKVRowValOfCol(((STable*)pTable)->tagVal, colId);
......@@ -394,7 +395,7 @@ int tsdbUpdateTableTagValue(STsdbRepo *repo, SUpdateTableTagValMsg *pMsg) {
}
bool isChangeIndexCol = (pMsg->colId == colColId(schemaColAt(pTable->pSuper->tagSchema, 0)))
|| IS_JSON_DATA_TYPE(pMsg->type);
|| pMsg->type == TSDB_DATA_TYPE_JSON;
// STColumn *pCol = bsearch(&(pMsg->colId), pMsg->data, pMsg->numOfTags, sizeof(STColumn), colIdCompar);
// ASSERT(pCol != NULL);
......@@ -403,7 +404,7 @@ int tsdbUpdateTableTagValue(STsdbRepo *repo, SUpdateTableTagValMsg *pMsg) {
tsdbRemoveTableFromIndex(pMeta, pTable);
}
TSDB_WLOCK_TABLE(pTable);
if (IS_JSON_DATA_TYPE(pMsg->type)){
if (pMsg->type == TSDB_DATA_TYPE_JSON){
kvRowFree(pTable->tagVal);
pTable->tagVal = tdKVRowDup(POINTER_SHIFT(pMsg->data, pMsg->schemaLen));
}else{
......@@ -857,7 +858,7 @@ static STable *tsdbCreateTableFromCfg(STableCfg *pCfg, bool isSuper, STable *pST
}
pTable->tagVal = NULL;
STColumn *pCol = schemaColAt(pTable->tagSchema, DEFAULT_TAG_INDEX_COLUMN);
if(IS_JSON_DATA_TYPE(pCol->type)){
if(pCol->type == TSDB_DATA_TYPE_JSON){
assert(pTable->tagSchema->numOfCols == 1);
pTable->jsonKeyMap = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
if (pTable->jsonKeyMap == NULL) {
......@@ -1096,7 +1097,7 @@ static int tsdbAddTableIntoIndex(STsdbMeta *pMeta, STable *pTable, bool refSuper
pTable->pSuper = pSTable;
if(IS_JSON_DATA_TYPE(pSTable->tagSchema->columns[0].type)){
if(pSTable->tagSchema->columns[0].type == TSDB_DATA_TYPE_JSON){
ASSERT(pSTable->tagSchema->numOfCols == 1);
int16_t nCols = kvRowNCols(pTable->tagVal);
ASSERT(nCols%2 == 1);
......@@ -1110,8 +1111,10 @@ static int tsdbAddTableIntoIndex(STsdbMeta *pMeta, STable *pTable, bool refSuper
continue;
}
char keyMd5[TSDB_MAX_JSON_KEY_MD5_LEN] = {0};
jsonKeyMd5(varDataVal(val), varDataLen(val), keyMd5);
SArray* tablistNew = NULL;
SArray** tablist = (SArray**)taosHashGet(pSTable->jsonKeyMap, varDataVal(val), varDataLen(val));
SArray** tablist = (SArray**)taosHashGet(pSTable->jsonKeyMap, keyMd5, TSDB_MAX_JSON_KEY_MD5_LEN);
if(tablist == NULL) {
tablistNew = taosArrayInit(8, sizeof(JsonMapValue));
if(tablistNew == NULL){
......@@ -1119,7 +1122,7 @@ static int tsdbAddTableIntoIndex(STsdbMeta *pMeta, STable *pTable, bool refSuper
tsdbError("out of memory when alloc json tag array");
return -1;
}
if(taosHashPut(pSTable->jsonKeyMap, varDataVal(val) ,varDataLen(val), &tablistNew, sizeof(void*)) < 0){
if(taosHashPut(pSTable->jsonKeyMap, keyMd5, TSDB_MAX_JSON_KEY_MD5_LEN, &tablistNew, sizeof(void*)) < 0){
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
tsdbError("out of memory when put json tag array");
return -1;
......@@ -1167,7 +1170,7 @@ static int tsdbRemoveTableFromIndex(STsdbMeta *pMeta, STable *pTable) {
STable *pSTable = pTable->pSuper;
ASSERT(pSTable != NULL);
if(IS_JSON_DATA_TYPE(pSTable->tagSchema->columns[0].type)){
if(pSTable->tagSchema->columns[0].type == TSDB_DATA_TYPE_JSON){
ASSERT(pSTable->tagSchema->numOfCols == 1);
int16_t nCols = kvRowNCols(pTable->tagVal);
ASSERT(nCols%2 == 1);
......@@ -1450,7 +1453,7 @@ static void *tsdbDecodeTable(void *buf, STable **pRTable) {
if (TABLE_TYPE(pTable) == TSDB_SUPER_TABLE) {
buf = tdDecodeSchema(buf, &(pTable->tagSchema));
STColumn *pCol = schemaColAt(pTable->tagSchema, DEFAULT_TAG_INDEX_COLUMN);
if(IS_JSON_DATA_TYPE(pCol->type)){
if(pCol->type == TSDB_DATA_TYPE_JSON){
assert(pTable->tagSchema->numOfCols == 1);
pTable->jsonKeyMap = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
if (pTable->jsonKeyMap == NULL) {
......
src/tsdb/src/tsdbRead.c
浏览文件 @ a91392a9
......@@ -2678,7 +2678,7 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int
static int32_t getAllTableList(STable* pSuperTable, SArray* list) {
STSchema* pTagSchema = tsdbGetTableTagSchema(pSuperTable);
if(pTagSchema && pTagSchema->numOfCols == 1 && IS_JSON_DATA_TYPE(pTagSchema->columns[0].type)){
if(pTagSchema && pTagSchema->numOfCols == 1 && pTagSchema->columns[0].type == TSDB_DATA_TYPE_JSON){
SArray** pRecord = taosHashIterate(pSuperTable->jsonKeyMap, NULL);
SArray* tablist = taosArrayInit(32, sizeof(JsonMapValue));
......@@ -3662,7 +3662,7 @@ SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pC
//
//int32_t dealWithTree(STable* pTable, tExprNode* expr){
// STSchema* pTagSchema = tsdbGetTableTagSchema(pTable);
// if(!IS_JSON_DATA_TYPE(pTagSchema->columns->type)){
// if(pTagSchema->columns->type != TSDB_DATA_TYPE_JSON){
// return TSDB_CODE_SUCCESS;
// }
//
......@@ -4081,7 +4081,7 @@ static FORCE_INLINE int32_t tsdbGetJsonTagDataFromId(void *param, int32_t id, ch
if (id == TSDB_TBNAME_COLUMN_INDEX) {
*data = TABLE_NAME(pTable);
} else {
void* jsonData = getJsonTagValue(pTable, name, strlen(name), pTable->pSuper->tagSchema->columns->type, NULL);
void* jsonData = getJsonTagValue(pTable, name, strlen(name), NULL);
if (jsonData != NULL) jsonData += CHAR_BYTES; // jump type
*data = jsonData;
}
......@@ -4109,18 +4109,9 @@ static void queryByJsonTag(STable* pTable, void* filterInfo, SArray* res){
SFilterField* fi = &info->fields[FLD_TYPE_COLUMN].fields[i];
SSchema* sch = fi->desc;
int32_t outLen = 0;
char* key = NULL;
if(pTable->tagSchema->columns->type == TSDB_DATA_TYPE_JSON_NCHAR){
char tagKey[256] = {0};
if (!taosMbsToUcs4(sch->name, strlen(sch->name), tagKey, 256, &outLen)) {
tsdbError("json key to ucs4 error:%s|%s", strerror(errno), sch->name);
return;
}
key = tagKey;
}else{
key = sch->name;
outLen = strlen(sch->name);
}
char* key = sch->name;
outLen = strlen(sch->name);
SArray** data = (SArray**)taosHashGet(pTable->jsonKeyMap, key, outLen);
if(data == NULL) continue;
if(tabList == NULL) {
......@@ -4165,7 +4156,7 @@ static void queryByJsonTag(STable* pTable, void* filterInfo, SArray* res){
static int32_t tsdbQueryTableList(STable* pTable, SArray* pRes, void* filterInfo) {
STSchema* pTSSchema = pTable->tagSchema;
if(IS_JSON_DATA_TYPE(pTSSchema->columns->type)){
if(pTSSchema->columns->type == TSDB_DATA_TYPE_JSON){
queryByJsonTag(pTable, filterInfo, pRes);
}else{
bool indexQuery = false;
......
src/util/inc/tutil.h
浏览文件 @ a91392a9
......@@ -45,6 +45,7 @@ int taosCheckVersion(char *input_client_version, char *input_server_version, in
char * taosIpStr(uint32_t ipInt);
uint32_t ip2uint(const char *const ip_addr);
void jsonKeyMd5(char *pMsg, int msgLen, void *pKey);
static FORCE_INLINE void taosEncryptPass(uint8_t *inBuf, size_t inLen, char *target) {
MD5_CTX context;
......
src/util/src/tutil.c
浏览文件 @ a91392a9
......@@ -456,6 +456,16 @@ char *taosIpStr(uint32_t ipInt) {
return ipStr;
}
void jsonKeyMd5(char *pMsg, int msgLen, void *pKey) {
MD5_CTX context;
MD5Init(&context);
MD5Update(&context, (uint8_t *)pMsg, msgLen);
MD5Final(&context);
memcpy(pKey, context.digest, sizeof(context.digest));
}
FORCE_INLINE float taos_align_get_float(const char* pBuf) {
#if __STDC_VERSION__ >= 201112L
static_assert(sizeof(float) == sizeof(uint32_t), "sizeof(float) must equal to sizeof(uint32_t)");
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或