提交 b7bd654c 编写于 作者: wmmhello's avatar wmmhello

refactor: add compare logic for json value

上级 73c34da2
......@@ -49,7 +49,7 @@ typedef struct {
#define varDataCopy(dst, v) memcpy((dst), (void *)(v), varDataTLen(v))
#define varDataLenByData(v) (*(VarDataLenT *)(((char *)(v)) - VARSTR_HEADER_SIZE))
#define varDataSetLen(v, _len) (((VarDataLenT *)(v))[0] = (VarDataLenT)(_len))
#define IS_VAR_DATA_TYPE(t) (((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_NCHAR))
#define IS_VAR_DATA_TYPE(t) (((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_NCHAR) || ((t) == TSDB_DATA_TYPE_JSON))
#define varDataNetLen(v) (htons(((VarDataLenT *)(v))[0]))
#define varDataNetTLen(v) (sizeof(VarDataLenT) + varDataNetLen(v))
......
......@@ -105,6 +105,8 @@ int32_t compareStrPatternNotMatch(const void *pLeft, const void *pRight);
int32_t compareWStrPatternMatch(const void *pLeft, const void *pRight);
int32_t compareWStrPatternNotMatch(const void *pLeft, const void *pRight);
int32_t compareJsonContainsKey(const void *pLeft, const void *pRight);
__compar_fn_t getComparFunc(int32_t type, int32_t optr);
__compar_fn_t getKeyComparFunc(int32_t keyType, int32_t order);
int32_t doCompare(const char *a, const char *b, int32_t type, size_t size);
......
......@@ -274,7 +274,6 @@ typedef enum ELogicConditionType {
#define TSDB_MAX_JSON_TAG_LEN 16384
#define TSDB_JSON_PLACEHOLDER 0x7F
#define TSDB_JSON_null 0x00
#define TSDB_JSON_KEY_NULL 0x00
#define TSDB_JSON_NOT_NULL 0x01
#define TSDB_JSON_NULL 0x00
......
......@@ -1006,6 +1006,7 @@ bool nodesIsComparisonOp(const SOperatorNode* pOp) {
case OP_TYPE_NOT_LIKE:
case OP_TYPE_MATCH:
case OP_TYPE_NMATCH:
case OP_TYPE_JSON_CONTAINS:
case OP_TYPE_IS_NULL:
case OP_TYPE_IS_NOT_NULL:
case OP_TYPE_IS_TRUE:
......@@ -1024,7 +1025,6 @@ bool nodesIsComparisonOp(const SOperatorNode* pOp) {
bool nodesIsJsonOp(const SOperatorNode* pOp) {
switch (pOp->opType) {
case OP_TYPE_JSON_GET_VALUE:
case OP_TYPE_JSON_CONTAINS:
return true;
default:
break;
......
......@@ -469,14 +469,18 @@ static EDealRes translateOperator(STranslateContext* pCxt, SOperatorNode* pOp) {
pOp->node.resType.type = TSDB_DATA_TYPE_DOUBLE;
pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes;
} else if (nodesIsComparisonOp(pOp)) {
if (TSDB_DATA_TYPE_JSON == ldt.type || TSDB_DATA_TYPE_BLOB == ldt.type || TSDB_DATA_TYPE_JSON == rdt.type ||
if (TSDB_DATA_TYPE_BLOB == ldt.type || TSDB_DATA_TYPE_JSON == rdt.type ||
TSDB_DATA_TYPE_BLOB == rdt.type) {
return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName);
}
pOp->node.resType.type = TSDB_DATA_TYPE_BOOL;
pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes;
} else {
// todo json operator
} else if (nodesIsJsonOp(pOp)){
if (TSDB_DATA_TYPE_JSON != ldt.type || TSDB_DATA_TYPE_BINARY != rdt.type) {
return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName);
}
pOp->node.resType.type = TSDB_DATA_TYPE_JSON;
pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_JSON].bytes;
}
return DEAL_RES_CONTINUE;
}
......@@ -2814,25 +2818,25 @@ static void addCreateTbReqIntoVgroup(int32_t acctId, SHashObj* pVgroupHashmap, c
static int32_t addValToKVRow(STranslateContext* pCxt, SValueNode* pVal, const SSchema* pSchema,
SKVRowBuilder* pBuilder) {
if (DEAL_RES_ERROR == translateValue(pCxt, pVal)) {
return pCxt->errCode;
}
SVariant var;
valueNodeToVariant(pVal, &var);
if(pSchema->type == TSDB_DATA_TYPE_JSON){
if(var.nLen > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE){
return buildSyntaxErrMsg(&pCxt->msgBuf, "json string too long than 4095", var.pz);
if(pVal->literal && strlen(pVal->literal) > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE){
return buildSyntaxErrMsg(&pCxt->msgBuf, "json string too long than 4095", pVal->literal);
}
return parseJsontoTagData(var.pz, pBuilder, &pCxt->msgBuf, pSchema->colId);
return parseJsontoTagData(pVal->literal, pBuilder, &pCxt->msgBuf, pSchema->colId);
}
char tagVal[TSDB_MAX_TAGS_LEN] = {0};
int32_t code = taosVariantDump(&var, tagVal, pSchema->type, true);
if (TSDB_CODE_SUCCESS == code) {
tdAddColToKVRow(pBuilder, pSchema->colId, tagVal, IS_VAR_DATA_TYPE(pSchema->type) ? varDataTLen(tagVal) : TYPE_BYTES[pSchema->type]);
if (DEAL_RES_ERROR == translateValue(pCxt, pVal)) {
return pCxt->errCode;
}
return code;
if(pVal->node.resType.type == TSDB_DATA_TYPE_NULL){
// todo
}else{
tdAddColToKVRow(pBuilder, pSchema->colId, &(pVal->datum.p), IS_VAR_DATA_TYPE(pSchema->type) ? varDataTLen(pVal->datum.p) : TYPE_BYTES[pSchema->type]);
}
return TSDB_CODE_SUCCESS;
}
static int32_t buildKVRowForBindTags(STranslateContext* pCxt, SCreateSubTableClause* pStmt, STableMeta* pSuperTableMeta,
......
......@@ -252,12 +252,10 @@ static bool isValidateTag(char *input) {
int parseJsontoTagData(const char* json, SKVRowBuilder* kvRowBuilder, SMsgBuf* pMsgBuf, int16_t startColId){
// set json NULL data
uint8_t jsonKeyNULL = TSDB_JSON_KEY_NULL;
uint8_t jsonNULL = TSDB_JSON_NULL;
int jsonIndex = startColId + 1;
tdAddColToKVRow(kvRowBuilder, jsonIndex++, &jsonKeyNULL, CHAR_BYTES); // add json null type
if (!json || strcasecmp(json, TSDB_DATA_NULL_STR_L) == 0){
tdAddColToKVRow(kvRowBuilder, jsonIndex++, &jsonNULL, CHAR_BYTES); // add json null value
tdAddColToKVRow(kvRowBuilder, jsonIndex, &jsonNULL, CHAR_BYTES);
return TSDB_CODE_SUCCESS;
}
......@@ -273,6 +271,7 @@ int parseJsontoTagData(const char* json, SKVRowBuilder* kvRowBuilder, SMsgBuf* p
}
int retCode = 0;
char *tagKV = NULL;
SHashObj* keyHash = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false);
for(int i = 0; i < size; i++) {
cJSON* item = cJSON_GetArrayItem(root, i);
......@@ -296,66 +295,64 @@ int parseJsontoTagData(const char* json, SKVRowBuilder* kvRowBuilder, SMsgBuf* p
if(keyLen == 0 || taosHashGet(keyHash, jsonKey, keyLen) != NULL){
continue;
}
// key: keyLen + VARSTR_HEADER_SIZE, value type: CHAR_BYTES, value reserved: LONG_BYTES
tagKV = taosMemoryCalloc(keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES + LONG_BYTES, 1);
if(!tagKV) {
retCode = TSDB_CODE_TSC_OUT_OF_MEMORY;
goto end;
}
strncpy(varDataVal(tagKV), jsonKey, keyLen);
varDataSetLen(tagKV, keyLen);
if(taosHashGetSize(keyHash) == 0){
uint8_t jsonNotNULL = TSDB_JSON_NOT_NULL;
tdAddColToKVRow(kvRowBuilder, jsonIndex++, &jsonNotNULL, CHAR_BYTES); // add json type
}
// json key encode by binary
void *tagKey = taosMemoryCalloc(keyLen + VARSTR_HEADER_SIZE, 1);
if(!tagKey) {
retCode = TSDB_CODE_TSC_OUT_OF_MEMORY;
goto end;
}
strncpy(varDataVal(tagKey), jsonKey, keyLen);
taosHashPut(keyHash, jsonKey, keyLen, &keyLen, CHAR_BYTES); // add key to hash to remove dumplicate, value is useless
varDataSetLen(tagKey, keyLen);
tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagKey, varDataTLen(tagKey)); // add json key
taosMemoryFree(tagKey);
if(item->type == cJSON_String){ // add json value format: type|data
char *jsonValue = item->valuestring;
int32_t valLen = (int32_t)strlen(jsonValue);
char *tagVal = taosMemoryCalloc(valLen * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE + CHAR_BYTES, 1);
if(!tagVal) {
int32_t totalLen = keyLen + VARSTR_HEADER_SIZE + valLen * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE + CHAR_BYTES;
char *tmp = taosMemoryRealloc(tagKV, totalLen);
if(!tmp) {
retCode = TSDB_CODE_TSC_OUT_OF_MEMORY;
goto end;
} ;
tagVal[0] = TSDB_DATA_TYPE_NCHAR;
char* tagData = POINTER_SHIFT(tagVal, CHAR_BYTES);
if (valLen > 0 && !taosMbsToUcs4(jsonValue, valLen, (TdUcs4*)varDataVal(tagData),
}
tagKV = tmp;
char* valueType = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE);
char* valueData = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES);
*valueType = TSDB_DATA_TYPE_NCHAR;
if (valLen > 0 && !taosMbsToUcs4(jsonValue, valLen, (TdUcs4*)varDataVal(valueData),
(int32_t)(valLen * TSDB_NCHAR_SIZE), &valLen)) {
qError("charset:%s to %s. val:%s, errno:%s, convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, jsonValue, strerror(errno));
retCode = buildSyntaxErrMsg(pMsgBuf, "charset convert json error", jsonValue);
taosMemoryFree(tagVal);
goto end;
}
varDataSetLen(tagData, valLen);
tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagVal, CHAR_BYTES + varDataTLen(tagData));
taosMemoryFree(tagVal);
varDataSetLen(valueData, valLen);
tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagKV, totalLen);
}else if(item->type == cJSON_Number){
if(!isfinite(item->valuedouble)){
qError("json value is invalidate");
retCode = buildSyntaxErrMsg(pMsgBuf, "json value number is illegal", json);
goto end;
}
char tagVal[LONG_BYTES + CHAR_BYTES] = {0};
tagVal[0] = (item->valuedouble - (int64_t)(item->valuedouble) == 0) ? TSDB_DATA_TYPE_BIGINT
: TSDB_DATA_TYPE_DOUBLE;
char* tagData = POINTER_SHIFT(tagVal,CHAR_BYTES);
if(tagVal[0]== TSDB_DATA_TYPE_DOUBLE) *((double *)tagData) = item->valuedouble;
else if(tagVal[0] == TSDB_DATA_TYPE_BIGINT) *((int64_t *)tagData) = item->valueint;
tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagVal, LONG_BYTES + CHAR_BYTES);
char* valueType = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE);
char* valueData = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES);
*valueType = (item->valuedouble - (int64_t)(item->valuedouble) == 0) ? TSDB_DATA_TYPE_BIGINT : TSDB_DATA_TYPE_DOUBLE;
if(*valueType== TSDB_DATA_TYPE_DOUBLE) *((double *)valueData) = item->valuedouble;
else if(*valueType == TSDB_DATA_TYPE_BIGINT) *((int64_t *)valueData) = item->valueint;
tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagKV, keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES +LONG_BYTES);
}else if(item->type == cJSON_True || item->type == cJSON_False){
char tagVal[CHAR_BYTES + CHAR_BYTES] = {0};
tagVal[0] = TSDB_DATA_TYPE_BOOL;
tagVal[1] = (char)(item->valueint);
tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagVal, CHAR_BYTES + CHAR_BYTES);
char* valueType = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE);
char* valueData = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES);
*valueType = TSDB_DATA_TYPE_BOOL;
*valueData = (char)(item->valueint);
tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagKV, keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES + CHAR_BYTES);
}else if(item->type == cJSON_NULL){
char tagVal[CHAR_BYTES] = {TSDB_DATA_TYPE_NULL};
tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagVal, CHAR_BYTES);
char* valueType = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE);
*valueType = TSDB_DATA_TYPE_NULL;
tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagKV, keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES);
}
else{
retCode = buildSyntaxErrMsg(pMsgBuf, "invalidate json value", json);
......@@ -364,10 +361,11 @@ int parseJsontoTagData(const char* json, SKVRowBuilder* kvRowBuilder, SMsgBuf* p
}
if(taosHashGetSize(keyHash) == 0){ // set json NULL true
tdAddColToKVRow(kvRowBuilder, jsonIndex++, &jsonNULL, CHAR_BYTES);
tdAddColToKVRow(kvRowBuilder, jsonIndex, &jsonNULL, CHAR_BYTES);
}
end:
taosMemoryFree(tagKV);
taosHashCleanup(keyHash);
cJSON_Delete(root);
return retCode;
......
......@@ -53,6 +53,8 @@ static FORCE_INLINE double getVectorDoubleValue_DOUBLE(void *src, int32_t index)
return (double)*((double *)src + index);
}
double getVectorDoubleValue_JSON(void *src, int32_t index);
static FORCE_INLINE _getDoubleValue_fn_t getVectorDoubleValueFn(int32_t srcType) {
_getDoubleValue_fn_t p = NULL;
if (srcType == TSDB_DATA_TYPE_TINYINT) {
......@@ -77,6 +79,8 @@ static FORCE_INLINE _getDoubleValue_fn_t getVectorDoubleValueFn(int32_t srcType)
p = getVectorDoubleValue_DOUBLE;
} else if (srcType == TSDB_DATA_TYPE_TIMESTAMP) {
p = getVectorDoubleValue_BIGINT;
} else if (srcType == TSDB_DATA_TYPE_JSON) {
p = getVectorDoubleValue_JSON;
} else {
assert(0);
}
......
......@@ -170,7 +170,7 @@ __compar_fn_t gDataCompare[] = {compareInt32Val, compareInt8Val, compareInt16Val
compareLenPrefixedWStr, compareUint8Val, compareUint16Val, compareUint32Val, compareUint64Val,
setChkInBytes1, setChkInBytes2, setChkInBytes4, setChkInBytes8, compareStrRegexCompMatch,
compareStrRegexCompNMatch, setChkNotInBytes1, setChkNotInBytes2, setChkNotInBytes4, setChkNotInBytes8,
compareChkNotInString, compareStrPatternNotMatch, compareWStrPatternNotMatch
compareChkNotInString, compareStrPatternNotMatch, compareWStrPatternNotMatch, compareJsonContainsKey
};
int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) {
......@@ -221,7 +221,12 @@ int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) {
assert(0);
}
}
if (optr == OP_TYPE_JSON_CONTAINS && type == TSDB_DATA_TYPE_JSON) {
return 28;
}
switch (type) {
case TSDB_DATA_TYPE_BOOL:
case TSDB_DATA_TYPE_TINYINT: comparFn = 1; break;
......
......@@ -600,44 +600,39 @@ EDealRes sclWalkOperator(SNode* pNode, SScalarCtx *ctx) {
return DEAL_RES_CONTINUE;
}
EDealRes sclWalkTarget(SNode* pNode, SScalarCtx *ctx) {
STargetNode *target = (STargetNode *)pNode;
if (target->dataBlockId >= taosArrayGetSize(ctx->pBlockList)) {
sclError("target tupleId is too big, tupleId:%d, dataBlockNum:%d", target->dataBlockId, (int32_t)taosArrayGetSize(ctx->pBlockList));
ctx->code = TSDB_CODE_QRY_INVALID_INPUT;
return DEAL_RES_ERROR;
}
SSDataBlock *block = *(SSDataBlock **)taosArrayGet(ctx->pBlockList, target->dataBlockId);
if (target->slotId >= taosArrayGetSize(block->pDataBlock)) {
sclError("target slot not exist, dataBlockId:%d, slotId:%d, dataBlockNum:%d", target->dataBlockId, target->slotId, (int32_t)taosArrayGetSize(block->pDataBlock));
ctx->code = TSDB_CODE_QRY_INVALID_INPUT;
return DEAL_RES_ERROR;
}
SColumnInfoData *col = taosArrayGet(block->pDataBlock, target->slotId);
SScalarParam *res = (SScalarParam *)taosHashGet(ctx->pRes, (void *)&target->pExpr, POINTER_BYTES);
if (NULL == res) {
sclError("no valid res in hash, node:%p, type:%d", target->pExpr, nodeType(target->pExpr));
ctx->code = TSDB_CODE_QRY_APP_ERROR;
return DEAL_RES_ERROR;
}
for (int32_t i = 0; i < res->numOfRows; ++i) {
if (colDataIsNull(res->columnData, res->numOfRows, i, NULL)) {
colDataAppend(col, i, NULL, true);
} else {
char *p = colDataGetData(res->columnData, i);
colDataAppend(col, i, p, false);
}
}
sclFreeParam(res);
taosHashRemove(ctx->pRes, (void *)&target->pExpr, POINTER_BYTES);
return DEAL_RES_CONTINUE;
}
//EDealRes sclWalkTarget(SNode* pNode, SScalarCtx *ctx) {
// STargetNode *target = (STargetNode *)pNode;
//
// if (target->dataBlockId >= taosArrayGetSize(ctx->pBlockList)) {
// sclError("target tupleId is too big, tupleId:%d, dataBlockNum:%d", target->dataBlockId, (int32_t)taosArrayGetSize(ctx->pBlockList));
// ctx->code = TSDB_CODE_QRY_INVALID_INPUT;
// return DEAL_RES_ERROR;
// }
//
// SSDataBlock *block = *(SSDataBlock **)taosArrayGet(ctx->pBlockList, target->dataBlockId);
// if (target->slotId >= taosArrayGetSize(block->pDataBlock)) {
// sclError("target slot not exist, dataBlockId:%d, slotId:%d, dataBlockNum:%d", target->dataBlockId, target->slotId, (int32_t)taosArrayGetSize(block->pDataBlock));
// ctx->code = TSDB_CODE_QRY_INVALID_INPUT;
// return DEAL_RES_ERROR;
// }
//
// // if block->pDataBlock is not enough, there are problems if target->slotId bigger than the size of block->pDataBlock,
// SColumnInfoData *col = taosArrayGet(block->pDataBlock, target->slotId);
//
// SScalarParam *res = (SScalarParam *)taosHashGet(ctx->pRes, (void *)&target->pExpr, POINTER_BYTES);
// if (NULL == res) {
// sclError("no valid res in hash, node:%p, type:%d", target->pExpr, nodeType(target->pExpr));
// ctx->code = TSDB_CODE_QRY_APP_ERROR;
// return DEAL_RES_ERROR;
// }
//
// colDataAssign(col, res->columnData, res->numOfRows);
// block->info.rows = res->numOfRows;
//
// sclFreeParam(res);
// taosHashRemove(ctx->pRes, (void *)&target->pExpr, POINTER_BYTES);
// return DEAL_RES_CONTINUE;
//}
EDealRes sclCalcWalker(SNode* pNode, void* pContext) {
if (QUERY_NODE_VALUE == nodeType(pNode) || QUERY_NODE_NODE_LIST == nodeType(pNode) || QUERY_NODE_COLUMN == nodeType(pNode)) {
......@@ -657,9 +652,9 @@ EDealRes sclCalcWalker(SNode* pNode, void* pContext) {
return sclWalkOperator(pNode, ctx);
}
if (QUERY_NODE_TARGET == nodeType(pNode)) {
return sclWalkTarget(pNode, ctx);
}
// if (QUERY_NODE_TARGET == nodeType(pNode)) {
// return sclWalkTarget(pNode, ctx);
// }
sclError("invalid node type for scalar calculating, type:%d", nodeType(pNode));
ctx->code = TSDB_CODE_QRY_INVALID_INPUT;
......
此差异已折叠。
......@@ -222,6 +222,11 @@ int32_t compareLenPrefixedWStrDesc(const void *pLeft, const void *pRight) {
return compareLenPrefixedWStr(pRight, pLeft);
}
int32_t compareJsonContainsKey(const void* pLeft, const void* pRight) {
if(pLeft) return 0;
return 1;
}
/*
* Compare two strings
* TSDB_MATCH: Match
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册