diff --git a/include/util/tcompare.h b/include/util/tcompare.h index ed07ae1c9cf5b0d06a181ec4d083d5fb8f98e06e..cc9e8ae4641138be528830e17467dab7897f0166 100644 --- a/include/util/tcompare.h +++ b/include/util/tcompare.h @@ -105,8 +105,6 @@ int32_t compareStrPatternNotMatch(const void *pLeft, const void *pRight); int32_t compareWStrPatternMatch(const void *pLeft, const void *pRight); int32_t compareWStrPatternNotMatch(const void *pLeft, const void *pRight); -int32_t compareJsonContainsKey(const void *pLeft, const void *pRight); - __compar_fn_t getComparFunc(int32_t type, int32_t optr); __compar_fn_t getKeyComparFunc(int32_t keyType, int32_t order); int32_t doCompare(const char *a, const char *b, int32_t type, size_t size); diff --git a/source/libs/index/inc/indexInt.h b/source/libs/index/inc/indexInt.h index 24a4e99970692b202ab36fd1d1a83a45a09bcaa4..e3f9cb89ecd6eeb524061a7735df6161b30e195b 100644 --- a/source/libs/index/inc/indexInt.h +++ b/source/libs/index/inc/indexInt.h @@ -35,15 +35,15 @@ extern "C" { #endif // clang-format off -#define indexFatal(...) do { if (idxDebugFlag & DEBUG_FATAL) { taosPrintLog("INDEX FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }} while (0) -#define indexError(...) do { if (idxDebugFlag & DEBUG_ERROR) { taosPrintLog("INDEX ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }} while (0) -#define indexWarn(...) do { if (idxDebugFlag & DEBUG_WARN) { taosPrintLog("INDEX WARN ", DEBUG_WARN, 255, __VA_ARGS__); }} while (0) -#define indexInfo(...) do { if (idxDebugFlag & DEBUG_INFO) { taosPrintLog("INDEX ", DEBUG_INFO, 255, __VA_ARGS__); } } while (0) -#define indexDebug(...) do { if (idxDebugFlag & DEBUG_DEBUG) { taosPrintLog("INDEX ", DEBUG_DEBUG, sDebugFlag, __VA_ARGS__);} } while (0) -#define indexTrace(...) do { if (idxDebugFlag & DEBUG_TRACE) { taosPrintLog("INDEX ", DEBUG_TRACE, sDebugFlag, __VA_ARGS__);} } while (0) +#define indexFatal(...) do { if (idxDebugFlag & DEBUG_FATAL) { taosPrintLog("IDX FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }} while (0) +#define indexError(...) do { if (idxDebugFlag & DEBUG_ERROR) { taosPrintLog("IDX ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }} while (0) +#define indexWarn(...) do { if (idxDebugFlag & DEBUG_WARN) { taosPrintLog("IDX WARN ", DEBUG_WARN, 255, __VA_ARGS__); }} while (0) +#define indexInfo(...) do { if (idxDebugFlag & DEBUG_INFO) { taosPrintLog("IDX ", DEBUG_INFO, 255, __VA_ARGS__); } } while (0) +#define indexDebug(...) do { if (idxDebugFlag & DEBUG_DEBUG) { taosPrintLog("IDX ", DEBUG_DEBUG, idxDebugFlag, __VA_ARGS__);} } while (0) +#define indexTrace(...) do { if (idxDebugFlag & DEBUG_TRACE) { taosPrintLog("IDX", DEBUG_TRACE, idxDebugFlag, __VA_ARGS__);} } while (0) // clang-format on -typedef enum { LT, LE, GT, GE } RangeType; +typedef enum { LT, LE, GT, GE, CONTAINS } RangeType; typedef enum { kTypeValue, kTypeDeletion } STermValueType; typedef struct SIndexStat { diff --git a/source/libs/index/src/indexCache.c b/source/libs/index/src/indexCache.c index 4e7be245ef7fb0a4c383a0abf0b242ebbb46522c..c1fb086fa098a03e8ad28fa0ee43f72caba3f207 100644 --- a/source/libs/index/src/indexCache.c +++ b/source/libs/index/src/indexCache.c @@ -90,7 +90,7 @@ static int32_t cacheSearchTerm(void* cache, SIndexTerm* term, SIdxTRslt* tr, STe break; } CacheTerm* c = (CacheTerm*)SL_GET_NODE_DATA(node); - if (0 == strcmp(c->colVal, pCt->colVal)) { + if (0 == strcmp(c->colVal, pCt->colVal) && strlen(pCt->colVal) == strlen(c->colVal)) { if (c->operaType == ADD_VALUE) { INDEX_MERGE_ADD_DEL(tr->del, tr->add, c->uid) // taosArrayPush(result, &c->uid); @@ -222,7 +222,7 @@ static int32_t cacheSearchTerm_JSON(void* cache, SIndexTerm* term, SIdxTRslt* tr return TSDB_CODE_SUCCESS; } static int32_t cacheSearchPrefix_JSON(void* cache, SIndexTerm* term, SIdxTRslt* tr, STermValueType* s) { - return TSDB_CODE_SUCCESS; + return cacheSearchCompareFunc_JSON(cache, term, tr, s, CONTAINS); } static int32_t cacheSearchSuffix_JSON(void* cache, SIndexTerm* term, SIdxTRslt* tr, STermValueType* s) { return TSDB_CODE_SUCCESS; @@ -242,6 +242,9 @@ static int32_t cacheSearchGreaterThan_JSON(void* cache, SIndexTerm* term, SIdxTR static int32_t cacheSearchGreaterEqual_JSON(void* cache, SIndexTerm* term, SIdxTRslt* tr, STermValueType* s) { return cacheSearchCompareFunc_JSON(cache, term, tr, s, GE); } +static int32_t cacheSearchContain_JSON(void* cache, SIndexTerm* term, SIdxTRslt* tr, STermValueType* s) { + return cacheSearchCompareFunc_JSON(cache, term, tr, s, CONTAINS); +} static int32_t cacheSearchRange_JSON(void* cache, SIndexTerm* term, SIdxTRslt* tr, STermValueType* s) { return TSDB_CODE_SUCCESS; } diff --git a/source/libs/index/src/indexComm.c b/source/libs/index/src/indexComm.c index 74b1773e877058a26bfa84afdf87cb2f73fa6aee..d30147d70f03949fa04617d747687594d1edbb89 100644 --- a/source/libs/index/src/indexComm.c +++ b/source/libs/index/src/indexComm.c @@ -97,6 +97,11 @@ static TExeCond tCompareGreaterEqual(void* a, void* b, int8_t type) { __compar_fn_t func = indexGetCompar(type); return tCompare(func, QUERY_GREATER_EQUAL, a, b, type); } + +static TExeCond tCompareContains(void* a, void* b, int8_t type) { + __compar_fn_t func = indexGetCompar(type); + return tCompare(func, QUERY_TERM, a, b, type); +} TExeCond tCompare(__compar_fn_t func, int8_t cmptype, void* a, void* b, int8_t dtype) { if (dtype == TSDB_DATA_TYPE_BINARY || dtype == TSDB_DATA_TYPE_NCHAR || dtype == TSDB_DATA_TYPE_VARBINARY) { return tDoCompare(func, cmptype, a, b); @@ -185,12 +190,14 @@ TExeCond tDoCompare(__compar_fn_t func, int8_t comparType, void* a, void* b) { case QUERY_TERM: { if (ret == 0) return MATCH; } + default: + return BREAK; } return CONTINUE; } -static TExeCond (*rangeCompare[])(void* a, void* b, int8_t type) = {tCompareLessThan, tCompareLessEqual, - tCompareGreaterThan, tCompareGreaterEqual}; +static TExeCond (*rangeCompare[])(void* a, void* b, int8_t type) = { + tCompareLessThan, tCompareLessEqual, tCompareGreaterThan, tCompareGreaterEqual, tCompareContains}; _cache_range_compare indexGetCompare(RangeType ty) { return rangeCompare[ty]; } diff --git a/source/libs/index/src/indexFilter.c b/source/libs/index/src/indexFilter.c index db0d097d29d39ba1167c605e61371fc9ff861536..fd2d6a5c8b7ace1b8859680796b29fb5069b3920 100644 --- a/source/libs/index/src/indexFilter.c +++ b/source/libs/index/src/indexFilter.c @@ -64,6 +64,8 @@ static int32_t sifGetFuncFromSql(EOperatorType src, EIndexQueryType *dst) { *dst = QUERY_TERM; } else if (src == OP_TYPE_LIKE || src == OP_TYPE_MATCH || src == OP_TYPE_NMATCH) { *dst = QUERY_REGEX; + } else if (src == OP_TYPE_JSON_CONTAINS) { + *dst = QUERY_PREFIX; } else { return TSDB_CODE_QRY_INVALID_INPUT; } @@ -186,6 +188,7 @@ static int32_t sifInitParam(SNode *node, SIFParam *param, SIFCtx *ctx) { SIF_ERR_RET(sifGetValueFromNode(node, ¶m->condValue)); param->colId = -1; param->colValType = (uint8_t)(vn->node.resType.type); + memcpy(param->colName, vn->literal, strlen(vn->literal)); break; } case QUERY_NODE_COLUMN: { @@ -237,7 +240,7 @@ static int32_t sifInitOperParams(SIFParam **params, SOperatorNode *node, SIFCtx indexError("invalid operation node, left: %p, rigth: %p", node->pLeft, node->pRight); SIF_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - if (node->opType == OP_TYPE_JSON_GET_VALUE || node->opType == OP_TYPE_JSON_CONTAINS) { + if (node->opType == OP_TYPE_JSON_GET_VALUE) { return code; } SIFParam *paramList = taosMemoryCalloc(nParam, sizeof(SIFParam)); @@ -420,8 +423,8 @@ static int32_t sifNotMatchFunc(SIFParam *left, SIFParam *right, SIFParam *output return sifDoIndex(left, right, id, output); } static int32_t sifJsonContains(SIFParam *left, SIFParam *right, SIFParam *output) { - // return 0 - return 0; + int id = OP_TYPE_JSON_CONTAINS; + return sifDoIndex(left, right, id, output); } static int32_t sifJsonGetValue(SIFParam *left, SIFParam *rigth, SIFParam *output) { // return 0 @@ -501,9 +504,11 @@ static int32_t sifExecOper(SOperatorNode *node, SIFCtx *ctx, SIFParam *output) { int32_t code = 0; int32_t nParam = sifGetOperParamNum(node->opType); if (nParam <= 1) { - SIF_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); + output->status = SFLT_NOT_INDEX; + return code; + // SIF_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - if (node->opType == OP_TYPE_JSON_GET_VALUE || node->opType == OP_TYPE_JSON_CONTAINS) { + if (node->opType == OP_TYPE_JSON_GET_VALUE) { return code; } SIFParam *params = NULL; @@ -617,11 +622,11 @@ EDealRes sifCalcWalker(SNode *node, void *context) { } if (QUERY_NODE_OPERATOR == nodeType(node)) { - indexInfo("node type for index filter, type: %d", nodeType(node)); + // indexInfo("node type for index filter, type: %d", nodeType(node)); return sifWalkOper(node, ctx); } - indexError("invalid node type for index filter calculating, type:%d", nodeType(node)); + // indexError("invalid node type for index filter calculating, type:%d", nodeType(node)); ctx->code = TSDB_CODE_QRY_INVALID_INPUT; return DEAL_RES_ERROR; } diff --git a/source/libs/index/src/indexTfile.c b/source/libs/index/src/indexTfile.c index 53dd2923ac8c1f07b62098a3663c030016b46a72..0145df4676d9431e2f2bd62b4352db7da292e8c3 100644 --- a/source/libs/index/src/indexTfile.c +++ b/source/libs/index/src/indexTfile.c @@ -425,8 +425,7 @@ static int32_t tfSearchTerm_JSON(void* reader, SIndexTerm* tem, SIdxTRslt* tr) { return TSDB_CODE_SUCCESS; } static int32_t tfSearchPrefix_JSON(void* reader, SIndexTerm* tem, SIdxTRslt* tr) { - // impl later - return TSDB_CODE_SUCCESS; + return tfSearchCompareFunc_JSON(reader, tem, tr, CONTAINS); } static int32_t tfSearchSuffix_JSON(void* reader, SIndexTerm* tem, SIdxTRslt* tr) { // impl later @@ -466,10 +465,6 @@ static int32_t tfSearchCompareFunc_JSON(void* reader, SIndexTerm* tem, SIdxTRslt AutomationCtx* ctx = automCtxCreate((void*)p, AUTOMATION_PREFIX); FstStreamBuilder* sb = fstSearch(((TFileReader*)reader)->fst, ctx); - // FstSlice h = fstSliceCreate((uint8_t*)p, skip); - // fstStreamBuilderSetRange(sb, &h, ctype); - // fstSliceDestroy(&h); - StreamWithState* st = streamBuilderIntoStream(sb); StreamWithStateResult* rt = NULL; while ((rt = streamWithStateNextWith(st, NULL)) != NULL) { diff --git a/source/libs/parser/src/parTokenizer.c b/source/libs/parser/src/parTokenizer.c index 685c299c01af44d479d4e03cf42711cab30bfc73..d13f17f8958cdc7c94f65ccbba3a7ba707c8749c 100644 --- a/source/libs/parser/src/parTokenizer.c +++ b/source/libs/parser/src/parTokenizer.c @@ -64,6 +64,7 @@ static SKeyword keywordTable[] = { {"CONSUMER", TK_CONSUMER}, {"COUNT", TK_COUNT}, {"CREATE", TK_CREATE}, + {"CONTAINS", TK_CONTAINS}, {"DATABASE", TK_DATABASE}, {"DATABASES", TK_DATABASES}, {"DAYS", TK_DAYS}, diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 6334d079c826d1b66244fbe11589833492790b96..1ba47d0d15d1d5d35829727df6d8e0fb6b230c01 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -847,8 +847,12 @@ static EDealRes translateJsonOperator(STranslateContext* pCxt, SOperatorNode* pO if (TSDB_DATA_TYPE_JSON != ldt.type || TSDB_DATA_TYPE_BINARY != rdt.type) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName); } - pOp->node.resType.type = TSDB_DATA_TYPE_JSON; - pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_JSON].bytes; + if(pOp->opType == OP_TYPE_JSON_GET_VALUE){ + pOp->node.resType.type = TSDB_DATA_TYPE_JSON; + }else if(pOp->opType == OP_TYPE_JSON_CONTAINS){ + pOp->node.resType.type = TSDB_DATA_TYPE_BOOL; + } + pOp->node.resType.bytes = tDataTypes[pOp->node.resType.type].bytes; return DEAL_RES_CONTINUE; } diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 5fdbbfe8105953784091216fce49715024ec0112..12e9ec5aea3146c1804697b97bf156415fce73c7 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -168,7 +168,7 @@ __compar_fn_t gDataCompare[] = {compareInt32Val, compareInt8Val, compareInt16Val compareLenPrefixedWStr, compareUint8Val, compareUint16Val, compareUint32Val, compareUint64Val, setChkInBytes1, setChkInBytes2, setChkInBytes4, setChkInBytes8, compareStrRegexCompMatch, compareStrRegexCompNMatch, setChkNotInBytes1, setChkNotInBytes2, setChkNotInBytes4, setChkNotInBytes8, - compareChkNotInString, compareStrPatternNotMatch, compareWStrPatternNotMatch, compareJsonContainsKey + compareChkNotInString, compareStrPatternNotMatch, compareWStrPatternNotMatch }; int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) { diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index d9f582e66946fee3c897d9c85009f79fe70de1bb..747da673d12dd0ec4d96a912d4eed33ce4ad9121 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -88,7 +88,7 @@ void convertNumberToNumber(const void *inData, void *outData, int8_t inType, int } } -void convertStringToDouble(const void *inData, void *outData, int8_t inType, int8_t outType){ +void convertNcharToDouble(const void *inData, void *outData){ char *tmp = taosMemoryMalloc(varDataTLen(inData)); int len = taosUcs4ToMbs((TdUcs4 *)varDataVal(inData), varDataLen(inData), tmp); if (len < 0) { @@ -97,13 +97,24 @@ void convertStringToDouble(const void *inData, void *outData, int8_t inType, int tmp[len] = 0; - ASSERT(outType == TSDB_DATA_TYPE_DOUBLE); double value = taosStr2Double(tmp, NULL); *((double *)outData) = value; taosMemoryFreeClear(tmp); } +void convertBinaryToDouble(const void *inData, void *outData){ + char *tmp = taosMemoryCalloc(1, varDataTLen(inData)); + if(tmp == NULL){ + *((double *)outData) = 0.; + return; + } + memcpy(tmp, varDataVal(inData), varDataLen(inData)); + double ret = taosStr2Double(tmp, NULL); + taosMemoryFree(tmp); + *((double *)outData) = ret; +} + typedef int64_t (*_getBigintValue_fn_t)(void *src, int32_t index); int64_t getVectorBigintValue_TINYINT(void *src, int32_t index) { @@ -147,7 +158,7 @@ int64_t getVectorBigintValue_JSON(void *src, int32_t index){ if (*data == TSDB_DATA_TYPE_NULL){ return 0; } else if(*data == TSDB_DATA_TYPE_NCHAR) { // json inner type can not be BINARY - convertStringToDouble(data+CHAR_BYTES, &out, *data, TSDB_DATA_TYPE_DOUBLE); + convertNcharToDouble(data+CHAR_BYTES, &out); } else { convertNumberToNumber(data+CHAR_BYTES, &out, *data, TSDB_DATA_TYPE_DOUBLE); } @@ -445,14 +456,30 @@ double getVectorDoubleValue_JSON(void *src, int32_t index){ if (*data == TSDB_DATA_TYPE_NULL){ return out; } else if(*data == TSDB_DATA_TYPE_NCHAR) { // json inner type can not be BINARY - convertStringToDouble(data+CHAR_BYTES, &out, *data, TSDB_DATA_TYPE_DOUBLE); + convertNcharToDouble(data+CHAR_BYTES, &out); } else { convertNumberToNumber(data+CHAR_BYTES, &out, *data, TSDB_DATA_TYPE_DOUBLE); } return out; } -bool convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_t typeRight, char **pLeftData, char **pRightData, void *pLeftOut, void *pRightOut, bool *isNull){ +void* ncharTobinary(void *buf){ // todo need to remove , if tobinary is nchar + int32_t inputLen = varDataLen(buf); + + void* t = taosMemoryCalloc(1, inputLen); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(buf), varDataLen(buf), varDataVal(t)); + if (len < 0) { + sclError("charset:%s to %s. val:%s convert ncharTobinary failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, + (char*)varDataVal(buf)); + taosMemoryFree(t); + return NULL; + } + varDataSetLen(t, len); + return t; +} + +bool convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_t typeRight, char **pLeftData, char **pRightData, + void *pLeftOut, void *pRightOut, bool *isNull, bool *freeLeft, bool *freeRight){ if(optr == OP_TYPE_JSON_CONTAINS) { return true; } @@ -489,21 +516,41 @@ bool convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_t t *fp = filterGetCompFunc(type, optr); - if(typeLeft == TSDB_DATA_TYPE_NCHAR) { - convertStringToDouble(*pLeftData, pLeftOut, typeLeft, type); - *pLeftData = pLeftOut; - } else if(typeLeft != type) { - convertNumberToNumber(*pLeftData, pLeftOut, typeLeft, type); - *pLeftData = pLeftOut; + if(IS_NUMERIC_TYPE(type) || IS_FLOAT_TYPE(type)){ + if(typeLeft == TSDB_DATA_TYPE_NCHAR) { + convertNcharToDouble(*pLeftData, pLeftOut); + *pLeftData = pLeftOut; + } else if(typeLeft == TSDB_DATA_TYPE_BINARY) { + convertBinaryToDouble(*pLeftData, pLeftOut); + *pLeftData = pLeftOut; + } else if(typeLeft != type) { + convertNumberToNumber(*pLeftData, pLeftOut, typeLeft, type); + *pLeftData = pLeftOut; + } + + if(typeRight == TSDB_DATA_TYPE_NCHAR) { + convertNcharToDouble(*pRightData, pRightOut); + *pRightData = pRightOut; + } else if(typeRight == TSDB_DATA_TYPE_BINARY) { + convertBinaryToDouble(*pRightData, pRightOut); + *pRightData = pRightOut; + } else if(typeRight != type) { + convertNumberToNumber(*pRightData, pRightOut, typeRight, type); + *pRightData = pRightOut; + } + }else if(type == TSDB_DATA_TYPE_BINARY){ + if(typeLeft == TSDB_DATA_TYPE_NCHAR){ + *pLeftData = ncharTobinary(*pLeftData); + *freeLeft = true; + } + if(typeRight == TSDB_DATA_TYPE_NCHAR){ + *pRightData = ncharTobinary(*pRightData); + *freeRight = true; + } + }else{ + ASSERT(0); } - if(typeRight == TSDB_DATA_TYPE_NCHAR) { - convertStringToDouble(*pRightData, pRightOut, typeRight, type); - *pRightData = pRightOut; - } else if(typeRight != type) { - convertNumberToNumber(*pRightData, pRightOut, typeRight, type); - *pRightData = pRightOut; - } return true; } @@ -935,44 +982,6 @@ static void doReleaseVec(SColumnInfoData* pCol, int32_t type) { } } -STagVal getJsonValue(char *json, char *key, bool *isExist) { - STagVal val = {.pKey = key}; - bool find = tTagGet(((const STag *)json), &val); // json value is null and not exist is different - if(isExist){ - *isExist = find; - } - return val; -} - -void vectorJsonArrow(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { - SColumnInfoData *pOutputCol = pOut->columnData; - - int32_t i = ((_ord) == TSDB_ORDER_ASC)? 0 : TMAX(pLeft->numOfRows, pRight->numOfRows) - 1; - int32_t step = ((_ord) == TSDB_ORDER_ASC)? 1 : -1; - - pOut->numOfRows = TMAX(pLeft->numOfRows, pRight->numOfRows); - - char *pRightData = colDataGetVarData(pRight->columnData, 0); - char *jsonKey = taosMemoryCalloc(1, varDataLen(pRightData) + 1); - memcpy(jsonKey, varDataVal(pRightData), varDataLen(pRightData)); - for (; i >= 0 && i < pLeft->numOfRows; i += step) { - if (colDataIsNull_var(pLeft->columnData, i)) { - colDataSetNull_var(pOutputCol, i); - pOutputCol->hasNull = true; - continue; - } - char *pLeftData = colDataGetVarData(pLeft->columnData, i); - bool isExist = false; - STagVal value = getJsonValue(pLeftData, jsonKey, &isExist); - char *data = isExist ? tTagValToData(&value, true) : NULL; - colDataAppend(pOutputCol, i, data, data == NULL); - if(isExist && IS_VAR_DATA_TYPE(value.type) && data){ - taosMemoryFree(data); - } - } - taosMemoryFree(jsonKey); -} - void vectorMathAdd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { SColumnInfoData *pOutputCol = pOut->columnData; @@ -1510,6 +1519,38 @@ void vectorBitOr(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, doReleaseVec(pRightCol, rightConvert); } +#define VEC_COM_INNER(pCol, index1, index2) \ + for (; i < pCol->numOfRows && i >= 0; i += step) {\ + if (IS_HELPER_NULL(pLeft->columnData, index1) || IS_HELPER_NULL(pRight->columnData, index2)) {\ + bool res = false;\ + colDataAppendInt8(pOut->columnData, i, (int8_t*)&res);\ + continue;\ + }\ + char *pLeftData = colDataGetData(pLeft->columnData, index1);\ + char *pRightData = colDataGetData(pRight->columnData, index2);\ + int64_t leftOut = 0;\ + int64_t rightOut = 0;\ + bool freeLeft = false;\ + bool freeRight = false;\ + bool isJsonnull = false;\ + bool result = convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight),\ + &pLeftData, &pRightData, &leftOut, &rightOut, &isJsonnull, &freeLeft, &freeRight);\ + if(isJsonnull){\ + ASSERT(0);\ + }\ + if(!pLeftData || !pRightData){\ + result = false;\ + }\ + if(!result){\ + colDataAppendInt8(pOut->columnData, i, (int8_t*)&result);\ + }else{\ + bool res = filterDoCompare(fp, optr, pLeftData, pRightData);\ + colDataAppendInt8(pOut->columnData, i, (int8_t*)&res);\ + }\ + if(freeLeft) taosMemoryFreeClear(pLeftData);\ + if(freeRight) taosMemoryFreeClear(pRightData);\ + } + void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord, int32_t optr) { int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->numOfRows, pRight->numOfRows) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; @@ -1533,79 +1574,11 @@ void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam * } if (pLeft->numOfRows == pRight->numOfRows) { - for (; i < pRight->numOfRows && i >= 0; i += step) { - if (IS_HELPER_NULL(pLeft->columnData, i) || IS_HELPER_NULL(pRight->columnData, i)) { - bool res = false; - colDataAppendInt8(pOut->columnData, i, (int8_t*)&res); - continue; // TODO set null or ignore - } - - char *pLeftData = colDataGetData(pLeft->columnData, i); - char *pRightData = colDataGetData(pRight->columnData, i); - - int64_t leftOut = 0; - int64_t rightOut = 0; - bool isJsonnull = false; - bool result = convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut, &isJsonnull); - if(isJsonnull){ - ASSERT(0); - } - if(!result){ - colDataAppendInt8(pOut->columnData, i, (int8_t*)&result); - }else{ - bool res = filterDoCompare(fp, optr, pLeftData, pRightData); - colDataAppendInt8(pOut->columnData, i, (int8_t*)&res); - } - } + VEC_COM_INNER(pLeft, i, i) } else if (pRight->numOfRows == 1) { - ASSERT(pLeft->pHashFilter == NULL); - for (; i >= 0 && i < pLeft->numOfRows; i += step) { - if (IS_HELPER_NULL(pLeft->columnData, i) || IS_HELPER_NULL(pRight->columnData, 0)) { - bool res = false; - colDataAppendInt8(pOut->columnData, i, (int8_t*)&res); - continue; - } - - char *pLeftData = colDataGetData(pLeft->columnData, i); - char *pRightData = colDataGetData(pRight->columnData, 0); - int64_t leftOut = 0; - int64_t rightOut = 0; - bool isJsonnull = false; - bool result = convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut, &isJsonnull); - if(isJsonnull){ - ASSERT(0); - } - if(!result){ - colDataAppendInt8(pOut->columnData, i, (int8_t*)&result); - }else{ - bool res = filterDoCompare(fp, optr, pLeftData, pRightData); - colDataAppendInt8(pOut->columnData, i, (int8_t*)&res); - } - } + VEC_COM_INNER(pLeft, i, 0) } else if (pLeft->numOfRows == 1) { - for (; i >= 0 && i < pRight->numOfRows; i += step) { - if (IS_HELPER_NULL(pRight->columnData, i) || IS_HELPER_NULL(pLeft->columnData, 0)) { - bool res = false; - colDataAppendInt8(pOut->columnData, i, (int8_t*)&res); - continue; - } - - char *pLeftData = colDataGetData(pLeft->columnData, 0); - char *pRightData = colDataGetData(pLeft->columnData, i); - int64_t leftOut = 0; - int64_t rightOut = 0; - bool isJsonnull = false; - bool result = convertJsonValue(&fp, optr, GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), &pLeftData, &pRightData, &leftOut, &rightOut, &isJsonnull); - if(isJsonnull){ - ASSERT(0); - } - if(!result){ - colDataAppendInt8(pOut->columnData, i, (int8_t*)&result); - }else{ - bool res = filterDoCompare(fp, optr, pLeftData, pRightData); - colDataAppendInt8(pOut->columnData, i, (int8_t*)&res); - } - } + VEC_COM_INNER(pRight, 0, i) } } @@ -1683,10 +1656,6 @@ void vectorNotMatch(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOu vectorCompare(pLeft, pRight, pOut, _ord, OP_TYPE_NMATCH); } -void vectorJsonContains(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { - vectorCompare(pLeft, pRight, pOut, _ord, OP_TYPE_JSON_CONTAINS); -} - void vectorIsNull(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { for(int32_t i = 0; i < pLeft->numOfRows; ++i) { int8_t v = IS_HELPER_NULL(pLeft->columnData, i) ? 1 : 0; @@ -1707,6 +1676,69 @@ void vectorIsTrue(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, vectorConvertImpl(pLeft, pOut); } +STagVal getJsonValue(char *json, char *key, bool *isExist) { + STagVal val = {.pKey = key}; + bool find = tTagGet(((const STag *)json), &val); // json value is null and not exist is different + if(isExist){ + *isExist = find; + } + return val; +} + +void vectorJsonContains(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { + SColumnInfoData *pOutputCol = pOut->columnData; + + int32_t i = ((_ord) == TSDB_ORDER_ASC)? 0 : TMAX(pLeft->numOfRows, pRight->numOfRows) - 1; + int32_t step = ((_ord) == TSDB_ORDER_ASC)? 1 : -1; + + pOut->numOfRows = TMAX(pLeft->numOfRows, pRight->numOfRows); + + char *pRightData = colDataGetVarData(pRight->columnData, 0); + char *jsonKey = taosMemoryCalloc(1, varDataLen(pRightData) + 1); + memcpy(jsonKey, varDataVal(pRightData), varDataLen(pRightData)); + for (; i >= 0 && i < pLeft->numOfRows; i += step) { + bool isExist = false; + + if (!colDataIsNull_var(pLeft->columnData, i)) { + char *pLeftData = colDataGetVarData(pLeft->columnData, i); + getJsonValue(pLeftData, jsonKey, &isExist); + } + + colDataAppend(pOutputCol, i, (const char*)(&isExist), false); + + } + taosMemoryFree(jsonKey); +} + +void vectorJsonArrow(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { + SColumnInfoData *pOutputCol = pOut->columnData; + + int32_t i = ((_ord) == TSDB_ORDER_ASC)? 0 : TMAX(pLeft->numOfRows, pRight->numOfRows) - 1; + int32_t step = ((_ord) == TSDB_ORDER_ASC)? 1 : -1; + + pOut->numOfRows = TMAX(pLeft->numOfRows, pRight->numOfRows); + + char *pRightData = colDataGetVarData(pRight->columnData, 0); + char *jsonKey = taosMemoryCalloc(1, varDataLen(pRightData) + 1); + memcpy(jsonKey, varDataVal(pRightData), varDataLen(pRightData)); + for (; i >= 0 && i < pLeft->numOfRows; i += step) { + if (colDataIsNull_var(pLeft->columnData, i)) { + colDataSetNull_var(pOutputCol, i); + pOutputCol->hasNull = true; + continue; + } + char *pLeftData = colDataGetVarData(pLeft->columnData, i); + bool isExist = false; + STagVal value = getJsonValue(pLeftData, jsonKey, &isExist); + char *data = isExist ? tTagValToData(&value, true) : NULL; + colDataAppend(pOutputCol, i, data, data == NULL); + if(isExist && IS_VAR_DATA_TYPE(value.type) && data){ + taosMemoryFree(data); + } + } + taosMemoryFree(jsonKey); +} + _bin_scalar_fn_t getBinScalarOperatorFn(int32_t binFunctionId) { switch (binFunctionId) { case OP_TYPE_ADD: diff --git a/source/util/src/tcompare.c b/source/util/src/tcompare.c index b6609278992d2e7cd4c9ef87bf913b94ab46735f..11a1cc1c7174ebf803274a36e42f347eef091b16 100644 --- a/source/util/src/tcompare.c +++ b/source/util/src/tcompare.c @@ -222,11 +222,6 @@ int32_t compareLenPrefixedWStrDesc(const void *pLeft, const void *pRight) { return compareLenPrefixedWStr(pRight, pLeft); } -int32_t compareJsonContainsKey(const void* pLeft, const void* pRight) { - if(pLeft) return 0; - return 1; -} - // string > number > bool > null // ref: https://dev.mysql.com/doc/refman/8.0/en/json.html#json-comparison int32_t compareJsonVal(const void *pLeft, const void *pRight) { diff --git a/tests/system-test/0-others/taosdMonitor.py b/tests/system-test/0-others/taosdMonitor.py index a219c54e5925075a2c687c9cd134ada03c09e57e..4c5a434f0cc9e352934605dd2d1fd865681b7f3a 100644 --- a/tests/system-test/0-others/taosdMonitor.py +++ b/tests/system-test/0-others/taosdMonitor.py @@ -8,6 +8,7 @@ import http.server import gzip import threading import json +import pickle from util.log import * from util.sql import * @@ -15,206 +16,203 @@ from util.cases import * from util.dnodes import * telemetryPort = '6043' +serverPort = '7080' +hostname = socket.gethostname() +class RequestHandlerImpl(http.server.BaseHTTPRequestHandler): + hostPort = hostname + ":" + serverPort -def telemetryInfoCheck(infoDict=''): + def telemetryInfoCheck(self, infoDict=''): + if "ts" not in infoDict or len(infoDict["ts"]) == 0: + tdLog.exit("ts is null!") - hostname = socket.gethostname() - serverPort = 7080 + if "dnode_id" not in infoDict or infoDict["dnode_id"] != 1: + tdLog.exit("dnode_id is null!") - if "ts" not in infoDict or len(infoDict["ts"]) == 0: - tdLog.exit("ts is null!") + if "dnode_ep" not in infoDict: + tdLog.exit("dnode_ep is null!") - if "dnode_id" not in infoDict or infoDict["dnode_id"] != 1: - tdLog.exit("dnode_id is null!") + if "cluster_id" not in infoDict: + tdLog.exit("cluster_id is null!") - if "dnode_ep" not in infoDict: - tdLog.exit("dnode_ep is null!") + if "protocol" not in infoDict or infoDict["protocol"] != 1: + tdLog.exit("protocol is null!") - if "cluster_id" not in infoDict: - tdLog.exit("cluster_id is null!") + if "cluster_info" not in infoDict : + tdLog.exit("cluster_info is null!") - if "protocol" not in infoDict or infoDict["protocol"] != 1: - tdLog.exit("protocol is null!") + # cluster_info ==================================== - if "cluster_info" not in infoDict : - tdLog.exit("cluster_info is null!") + if "first_ep" not in infoDict["cluster_info"] or infoDict["cluster_info"]["first_ep"] == None: + tdLog.exit("first_ep is null!") - # cluster_info ==================================== + if "first_ep_dnode_id" not in infoDict["cluster_info"] or infoDict["cluster_info"]["first_ep_dnode_id"] != 1: + tdLog.exit("first_ep_dnode_id is null!") - if "first_ep" not in infoDict["cluster_info"] or infoDict["cluster_info"]["first_ep"] == None: - tdLog.exit("first_ep is null!") - - if "first_ep_dnode_id" not in infoDict["cluster_info"] or infoDict["cluster_info"]["first_ep_dnode_id"] != 1: - tdLog.exit("first_ep_dnode_id is null!") - - if "version" not in infoDict["cluster_info"] or infoDict["cluster_info"]["version"] == None: - tdLog.exit("first_ep_dnode_id is null!") - - if "master_uptime" not in infoDict["cluster_info"] or infoDict["cluster_info"]["master_uptime"] == None: - tdLog.exit("master_uptime is null!") + if "version" not in infoDict["cluster_info"] or infoDict["cluster_info"]["version"] == None: + tdLog.exit("first_ep_dnode_id is null!") + + if "master_uptime" not in infoDict["cluster_info"] or infoDict["cluster_info"]["master_uptime"] == None: + tdLog.exit("master_uptime is null!") - if "monitor_interval" not in infoDict["cluster_info"] or infoDict["cluster_info"]["monitor_interval"] !=5: - tdLog.exit("monitor_interval is null!") + if "monitor_interval" not in infoDict["cluster_info"] or infoDict["cluster_info"]["monitor_interval"] !=5: + tdLog.exit("monitor_interval is null!") - if "vgroups_total" not in infoDict["cluster_info"] or infoDict["cluster_info"]["vgroups_total"] < 0: - tdLog.exit("vgroups_total is null!") + if "vgroups_total" not in infoDict["cluster_info"] or infoDict["cluster_info"]["vgroups_total"] < 0: + tdLog.exit("vgroups_total is null!") - if "vgroups_alive" not in infoDict["cluster_info"] or infoDict["cluster_info"]["vgroups_alive"] < 0: - tdLog.exit("vgroups_alive is null!") + if "vgroups_alive" not in infoDict["cluster_info"] or infoDict["cluster_info"]["vgroups_alive"] < 0: + tdLog.exit("vgroups_alive is null!") - if "connections_total" not in infoDict["cluster_info"] or infoDict["cluster_info"]["connections_total"] < 0 : - tdLog.exit("connections_total is null!") + if "connections_total" not in infoDict["cluster_info"] or infoDict["cluster_info"]["connections_total"] < 0 : + tdLog.exit("connections_total is null!") - if "dnodes" not in infoDict["cluster_info"] or infoDict["cluster_info"]["dnodes"] == None : - tdLog.exit("dnodes is null!") - - dnodes_info = { "dnode_id": 1,"dnode_ep": f"{hostname}:{serverPort}","status":"ready"} - - for k ,v in dnodes_info.items(): - if k not in infoDict["cluster_info"]["dnodes"][0] or v != infoDict["cluster_info"]["dnodes"][0][k] : - tdLog.exit("dnodes info is null!") - - mnodes_info = { "mnode_id":1, "mnode_ep":f"{hostname}:{serverPort}","role": "leader" } + if "dnodes" not in infoDict["cluster_info"] or infoDict["cluster_info"]["dnodes"] == None : + tdLog.exit("dnodes is null!") + + dnodes_info = { "dnode_id": 1,"dnode_ep": self.hostPort,"status":"ready"} + + for k ,v in dnodes_info.items(): + if k not in infoDict["cluster_info"]["dnodes"][0] or v != infoDict["cluster_info"]["dnodes"][0][k] : + tdLog.exit("dnodes info is null!") + + mnodes_info = { "mnode_id":1, "mnode_ep": self.hostPort,"role": "leader" } - for k ,v in mnodes_info.items(): - if k not in infoDict["cluster_info"]["mnodes"][0] or v != infoDict["cluster_info"]["mnodes"][0][k] : - tdLog.exit("mnodes info is null!") + for k ,v in mnodes_info.items(): + if k not in infoDict["cluster_info"]["mnodes"][0] or v != infoDict["cluster_info"]["mnodes"][0][k] : + tdLog.exit("mnodes info is null!") - # vgroup_infos ==================================== + # vgroup_infos ==================================== - if "vgroup_infos" not in infoDict or infoDict["vgroup_infos"]== None: - tdLog.exit("vgroup_infos is null!") - - vgroup_infos_nums = len(infoDict["vgroup_infos"]) - - for index in range(vgroup_infos_nums): - if "vgroup_id" not in infoDict["vgroup_infos"][index] or infoDict["vgroup_infos"][index]["vgroup_id"]<0: - tdLog.exit("vgroup_id is null!") - if "database_name" not in infoDict["vgroup_infos"][index] or len(infoDict["vgroup_infos"][index]["database_name"]) < 0: - tdLog.exit("database_name is null!") - if "tables_num" not in infoDict["vgroup_infos"][index] or infoDict["vgroup_infos"][index]["tables_num"]!= 0: - tdLog.exit("tables_num is null!") - if "status" not in infoDict["vgroup_infos"][index] or len(infoDict["vgroup_infos"][index]["status"]) < 0 : - tdLog.exit("status is null!") - if "vnodes" not in infoDict["vgroup_infos"][index] or infoDict["vgroup_infos"][index]["vnodes"] ==None : - tdLog.exit("vnodes is null!") - if "dnode_id" not in infoDict["vgroup_infos"][index]["vnodes"][0] or infoDict["vgroup_infos"][index]["vnodes"][0]["dnode_id"] < 0 : - tdLog.exit("vnodes is null!") - - # grant_info ==================================== - - if "grant_info" not in infoDict or infoDict["grant_info"]== None: - tdLog.exit("grant_info is null!") - - if "expire_time" not in infoDict["grant_info"] or not infoDict["grant_info"]["expire_time"] > 0: - tdLog.exit("expire_time is null!") - - if "timeseries_used" not in infoDict["grant_info"] or not infoDict["grant_info"]["timeseries_used"] > 0: - tdLog.exit("timeseries_used is null!") - - if "timeseries_total" not in infoDict["grant_info"] or not infoDict["grant_info"]["timeseries_total"] > 0: - tdLog.exit("timeseries_total is null!") + if "vgroup_infos" not in infoDict or infoDict["vgroup_infos"]== None: + tdLog.exit("vgroup_infos is null!") - # dnode_info ==================================== - - if "dnode_info" not in infoDict or infoDict["dnode_info"]== None: - tdLog.exit("dnode_info is null!") + vgroup_infos_nums = len(infoDict["vgroup_infos"]) + + for index in range(vgroup_infos_nums): + if "vgroup_id" not in infoDict["vgroup_infos"][index] or infoDict["vgroup_infos"][index]["vgroup_id"]<0: + tdLog.exit("vgroup_id is null!") + if "database_name" not in infoDict["vgroup_infos"][index] or len(infoDict["vgroup_infos"][index]["database_name"]) < 0: + tdLog.exit("database_name is null!") + if "tables_num" not in infoDict["vgroup_infos"][index] or infoDict["vgroup_infos"][index]["tables_num"]!= 0: + tdLog.exit("tables_num is null!") + if "status" not in infoDict["vgroup_infos"][index] or len(infoDict["vgroup_infos"][index]["status"]) < 0 : + tdLog.exit("status is null!") + if "vnodes" not in infoDict["vgroup_infos"][index] or infoDict["vgroup_infos"][index]["vnodes"] ==None : + tdLog.exit("vnodes is null!") + if "dnode_id" not in infoDict["vgroup_infos"][index]["vnodes"][0] or infoDict["vgroup_infos"][index]["vnodes"][0]["dnode_id"] < 0 : + tdLog.exit("vnodes is null!") + + # grant_info ==================================== + + if "grant_info" not in infoDict or infoDict["grant_info"]== None: + tdLog.exit("grant_info is null!") + + if "expire_time" not in infoDict["grant_info"] or not infoDict["grant_info"]["expire_time"] > 0: + tdLog.exit("expire_time is null!") + + if "timeseries_used" not in infoDict["grant_info"] or not infoDict["grant_info"]["timeseries_used"] > 0: + tdLog.exit("timeseries_used is null!") + + if "timeseries_total" not in infoDict["grant_info"] or not infoDict["grant_info"]["timeseries_total"] > 0: + tdLog.exit("timeseries_total is null!") + + # dnode_info ==================================== - dnode_infos = ['uptime', 'cpu_engine', 'cpu_system', 'cpu_cores', 'mem_engine', 'mem_system', 'mem_total', 'disk_engine', - 'disk_used', 'disk_total', 'net_in', 'net_out', 'io_read', 'io_write', 'io_read_disk', 'io_write_disk', 'req_select', - 'req_select_rate', 'req_insert', 'req_insert_success', 'req_insert_rate', 'req_insert_batch', 'req_insert_batch_success', - 'req_insert_batch_rate', 'errors', 'vnodes_num', 'masters', 'has_mnode', 'has_qnode', 'has_snode', 'has_bnode'] - for elem in dnode_infos: - if elem not in infoDict["dnode_info"] or infoDict["dnode_info"][elem] < 0: - tdLog.exit(f"{elem} is null!") + if "dnode_info" not in infoDict or infoDict["dnode_info"]== None: + tdLog.exit("dnode_info is null!") - # dnode_info ==================================== + dnode_infos = ['uptime', 'cpu_engine', 'cpu_system', 'cpu_cores', 'mem_engine', 'mem_system', 'mem_total', 'disk_engine', + 'disk_used', 'disk_total', 'net_in', 'net_out', 'io_read', 'io_write', 'io_read_disk', 'io_write_disk', 'req_select', + 'req_select_rate', 'req_insert', 'req_insert_success', 'req_insert_rate', 'req_insert_batch', 'req_insert_batch_success', + 'req_insert_batch_rate', 'errors', 'vnodes_num', 'masters', 'has_mnode', 'has_qnode', 'has_snode', 'has_bnode'] + for elem in dnode_infos: + if elem not in infoDict["dnode_info"] or infoDict["dnode_info"][elem] < 0: + tdLog.exit(f"{elem} is null!") - if "disk_infos" not in infoDict or infoDict["disk_infos"]== None: - tdLog.exit("disk_infos is null!") - - # bug for data_dir - if "datadir" not in infoDict["disk_infos"] or len(infoDict["disk_infos"]["datadir"]) <=0 : - tdLog.exit("datadir is null!") + # dnode_info ==================================== - if "name" not in infoDict["disk_infos"]["datadir"][0] or len(infoDict["disk_infos"]["datadir"][0]["name"]) <= 0: - tdLog.exit("name is null!") + if "disk_infos" not in infoDict or infoDict["disk_infos"]== None: + tdLog.exit("disk_infos is null!") + + # bug for data_dir + if "datadir" not in infoDict["disk_infos"] or len(infoDict["disk_infos"]["datadir"]) <=0 : + tdLog.exit("datadir is null!") - if "level" not in infoDict["disk_infos"]["datadir"][0] or infoDict["disk_infos"]["datadir"][0]["level"] < 0: - tdLog.exit("level is null!") + if "name" not in infoDict["disk_infos"]["datadir"][0] or len(infoDict["disk_infos"]["datadir"][0]["name"]) <= 0: + tdLog.exit("name is null!") - if "avail" not in infoDict["disk_infos"]["datadir"][0] or infoDict["disk_infos"]["datadir"][0]["avail"] <= 0: - tdLog.exit("avail is null!") + if "level" not in infoDict["disk_infos"]["datadir"][0] or infoDict["disk_infos"]["datadir"][0]["level"] < 0: + tdLog.exit("level is null!") - if "used" not in infoDict["disk_infos"]["datadir"][0] or infoDict["disk_infos"]["datadir"][0]["used"] <= 0: - tdLog.exit("used is null!") + if "avail" not in infoDict["disk_infos"]["datadir"][0] or infoDict["disk_infos"]["datadir"][0]["avail"] <= 0: + tdLog.exit("avail is null!") - if "total" not in infoDict["disk_infos"]["datadir"][0] or infoDict["disk_infos"]["datadir"][0]["total"] <= 0: - tdLog.exit("total is null!") + if "used" not in infoDict["disk_infos"]["datadir"][0] or infoDict["disk_infos"]["datadir"][0]["used"] <= 0: + tdLog.exit("used is null!") + if "total" not in infoDict["disk_infos"]["datadir"][0] or infoDict["disk_infos"]["datadir"][0]["total"] <= 0: + tdLog.exit("total is null!") - if "logdir" not in infoDict["disk_infos"] or infoDict["disk_infos"]["logdir"]== None: - tdLog.exit("logdir is null!") - if "name" not in infoDict["disk_infos"]["logdir"] or len(infoDict["disk_infos"]["logdir"]["name"]) <= 0: - tdLog.exit("name is null!") + if "logdir" not in infoDict["disk_infos"] or infoDict["disk_infos"]["logdir"]== None: + tdLog.exit("logdir is null!") - if "avail" not in infoDict["disk_infos"]["logdir"] or infoDict["disk_infos"]["logdir"]["avail"] <= 0: - tdLog.exit("avail is null!") + if "name" not in infoDict["disk_infos"]["logdir"] or len(infoDict["disk_infos"]["logdir"]["name"]) <= 0: + tdLog.exit("name is null!") - if "used" not in infoDict["disk_infos"]["logdir"] or infoDict["disk_infos"]["logdir"]["used"] <= 0: - tdLog.exit("used is null!") + if "avail" not in infoDict["disk_infos"]["logdir"] or infoDict["disk_infos"]["logdir"]["avail"] <= 0: + tdLog.exit("avail is null!") - if "total" not in infoDict["disk_infos"]["logdir"] or infoDict["disk_infos"]["logdir"]["total"] <= 0: - tdLog.exit("total is null!") + if "used" not in infoDict["disk_infos"]["logdir"] or infoDict["disk_infos"]["logdir"]["used"] <= 0: + tdLog.exit("used is null!") - + if "total" not in infoDict["disk_infos"]["logdir"] or infoDict["disk_infos"]["logdir"]["total"] <= 0: + tdLog.exit("total is null!") - if "tempdir" not in infoDict["disk_infos"] or infoDict["disk_infos"]["tempdir"]== None: - tdLog.exit("tempdir is null!") + if "tempdir" not in infoDict["disk_infos"] or infoDict["disk_infos"]["tempdir"]== None: + tdLog.exit("tempdir is null!") - if "name" not in infoDict["disk_infos"]["tempdir"] or len(infoDict["disk_infos"]["tempdir"]["name"]) <= 0: - tdLog.exit("name is null!") + if "name" not in infoDict["disk_infos"]["tempdir"] or len(infoDict["disk_infos"]["tempdir"]["name"]) <= 0: + tdLog.exit("name is null!") - if "avail" not in infoDict["disk_infos"]["tempdir"] or infoDict["disk_infos"]["tempdir"]["avail"] <= 0: - tdLog.exit("avail is null!") + if "avail" not in infoDict["disk_infos"]["tempdir"] or infoDict["disk_infos"]["tempdir"]["avail"] <= 0: + tdLog.exit("avail is null!") - if "used" not in infoDict["disk_infos"]["tempdir"] or infoDict["disk_infos"]["tempdir"]["used"] <= 0: - tdLog.exit("used is null!") + if "used" not in infoDict["disk_infos"]["tempdir"] or infoDict["disk_infos"]["tempdir"]["used"] <= 0: + tdLog.exit("used is null!") - if "total" not in infoDict["disk_infos"]["tempdir"] or infoDict["disk_infos"]["tempdir"]["total"] <= 0: - tdLog.exit("total is null!") + if "total" not in infoDict["disk_infos"]["tempdir"] or infoDict["disk_infos"]["tempdir"]["total"] <= 0: + tdLog.exit("total is null!") - # log_infos ==================================== - - if "log_infos" not in infoDict or infoDict["log_infos"]== None: - tdLog.exit("log_infos is null!") + # log_infos ==================================== + + if "log_infos" not in infoDict or infoDict["log_infos"]== None: + tdLog.exit("log_infos is null!") - if "logs" not in infoDict["log_infos"] or len(infoDict["log_infos"]["logs"])!= 10: - tdLog.exit("logs is null!") + if "logs" not in infoDict["log_infos"] or len(infoDict["log_infos"]["logs"])!= 10: + tdLog.exit("logs is null!") - if "ts" not in infoDict["log_infos"]["logs"][0] or len(infoDict["log_infos"]["logs"][0]["ts"]) <= 10: - tdLog.exit("ts is null!") + if "ts" not in infoDict["log_infos"]["logs"][0] or len(infoDict["log_infos"]["logs"][0]["ts"]) <= 10: + tdLog.exit("ts is null!") - if "level" not in infoDict["log_infos"]["logs"][0] or infoDict["log_infos"]["logs"][0]["level"] not in ["error" ,"info" , "debug" ,"trace"]: - tdLog.exit("level is null!") + if "level" not in infoDict["log_infos"]["logs"][0] or infoDict["log_infos"]["logs"][0]["level"] not in ["error" ,"info" , "debug" ,"trace"]: + tdLog.exit("level is null!") - if "content" not in infoDict["log_infos"]["logs"][0] or len(infoDict["log_infos"]["logs"][0]["ts"]) <= 1: - tdLog.exit("content is null!") + if "content" not in infoDict["log_infos"]["logs"][0] or len(infoDict["log_infos"]["logs"][0]["ts"]) <= 1: + tdLog.exit("content is null!") - if "summary" not in infoDict["log_infos"] or len(infoDict["log_infos"]["summary"])!= 4: - tdLog.exit("summary is null!") + if "summary" not in infoDict["log_infos"] or len(infoDict["log_infos"]["summary"])!= 4: + tdLog.exit("summary is null!") - - if "total" not in infoDict["log_infos"]["summary"][0] or infoDict["log_infos"]["summary"][0]["total"] < 0 : - tdLog.exit("total is null!") + + if "total" not in infoDict["log_infos"]["summary"][0] or infoDict["log_infos"]["summary"][0]["total"] < 0 : + tdLog.exit("total is null!") - if "level" not in infoDict["log_infos"]["summary"][0] or infoDict["log_infos"]["summary"][0]["level"] not in ["error" ,"info" , "debug" ,"trace"]: - tdLog.exit("level is null!") - -class RequestHandlerImpl(http.server.BaseHTTPRequestHandler): + if "level" not in infoDict["log_infos"]["summary"][0] or infoDict["log_infos"]["summary"][0]["level"] not in ["error" ,"info" , "debug" ,"trace"]: + tdLog.exit("level is null!") + def do_GET(self): """ process GET request @@ -245,17 +243,23 @@ class RequestHandlerImpl(http.server.BaseHTTPRequestHandler): infoDict = json.loads(plainText) #print("================") # print(infoDict) - telemetryInfoCheck(infoDict) + self.telemetryInfoCheck(infoDict) # 4. shutdown the server and exit case - assassin = threading.Thread(target=httpServer.shutdown) + assassin = threading.Thread(target=self.server.shutdown) assassin.daemon = True assassin.start() print ("==== shutdown http server ====") class TDTestCase: - hostname = socket.gethostname() - serverPort = '7080' + global hostname + global serverPort + if (platform.system().lower() == 'windows' and not tdDnodes.dnodes[0].remoteIP == ""): + try: + config = eval(tdDnodes.dnodes[0].remoteIP ) + hostname = config["host"] + except Exception: + hostname = tdDnodes.dnodes[0].remoteIP rpcDebugFlagVal = '143' clientCfgDict = {'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''} clientCfgDict["serverPort"] = serverPort @@ -291,21 +295,19 @@ class TDTestCase: sql = "create database db3 vgroups " + vgroups tdSql.query(sql) - # loop to wait request - httpServer.serve_forever() + # create http server: bing ip/port , and request processor + if (platform.system().lower() == 'windows' and not tdDnodes.dnodes[0].remoteIP == ""): + RequestHandlerImplStr = base64.b64encode(pickle.dumps(RequestHandlerImpl)).decode() + cmdStr = "import pickle\nimport http\nRequestHandlerImpl=pickle.loads(base64.b64decode(\"%s\".encode()))\nclass NewRequestHandlerImpl(RequestHandlerImpl):\n hostPort = \'%s\'\nhttp.server.HTTPServer((\"\", %s), NewRequestHandlerImpl).serve_forever()"%(RequestHandlerImplStr,hostname+":"+serverPort,telemetryPort) + tdDnodes.dnodes[0].remoteExec({}, cmdStr) + else: + serverAddress = ("", int(telemetryPort)) + http.server.HTTPServer(serverAddress, RequestHandlerImpl).serve_forever() def stop(self): tdSql.close() tdLog.success(f"{__file__} successfully executed") -# create http server: bing ip/port , and request processor -serverAddress = ("", int(telemetryPort)) -httpServer = http.server.HTTPServer(serverAddress, RequestHandlerImpl) - tdCases.addLinux(__file__, TDTestCase()) tdCases.addWindows(__file__, TDTestCase()) - - - - diff --git a/tests/system-test/2-query/json_tag.py b/tests/system-test/2-query/json_tag.py index f6f09d189daac3b5a105b705a543d410f76f4784..7838b5c3e47b7c08a1b267b3ffda38dc503b4a9f 100644 --- a/tests/system-test/2-query/json_tag.py +++ b/tests/system-test/2-query/json_tag.py @@ -118,7 +118,7 @@ class TDTestCase: tdSql.error("select jtag->location from jsons1") tdSql.error("select jtag contains location from jsons1") tdSql.error("select * from jsons1 where jtag contains location") - tdSql.error("select * from jsons1 where jtag contains''") + #tdSql.error("select * from jsons1 where jtag contains''") tdSql.error("select * from jsons1 where jtag contains 'location'='beijing'") # # # test function error @@ -129,6 +129,41 @@ class TDTestCase: tdSql.error("select ceil(jtag->'tag1') from jsons1") tdSql.error("select ceil(jtag) from jsons1") # + + #test scalar operation + tdSql.query("select jtag contains 'tag1',jtag->'tag1' from jsons1 order by jtag->'tag1'") + tdSql.checkRows(13) + tdSql.checkData(0, 0, False) + tdSql.checkData(5, 0, True) + tdSql.checkData(12, 0, True) + tdSql.query("select jtag->'tag1' like 'fe%',jtag->'tag1' from jsons1 order by jtag->'tag1'") + tdSql.checkRows(13) + tdSql.checkData(10, 0, False) + tdSql.checkData(11, 0, False) + tdSql.checkData(12, 0, True) + tdSql.query("select jtag->'tag1' not like 'fe%',jtag->'tag1' from jsons1 order by jtag->'tag1'") + tdSql.checkRows(13) + tdSql.checkData(10, 0, False) + tdSql.checkData(11, 0, True) + tdSql.checkData(12, 0, False) + tdSql.query("select jtag->'tag1' match 'fe',jtag->'tag1' from jsons1 order by jtag->'tag1'") + tdSql.checkRows(13) + tdSql.checkData(10, 0, False) + tdSql.checkData(11, 0, False) + tdSql.checkData(12, 0, True) + tdSql.query("select jtag->'tag1' nmatch 'fe',jtag->'tag1' from jsons1 order by jtag->'tag1'") + tdSql.checkRows(13) + tdSql.checkData(10, 0, False) + tdSql.checkData(11, 0, True) + tdSql.checkData(12, 0, False) + tdSql.query("select jtag->'tag1',jtag->'tag1'>='a' from jsons1 order by jtag->'tag1'") + tdSql.checkRows(13) + tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, False) + tdSql.checkData(7, 0, "false") + tdSql.checkData(7, 1, True) + tdSql.checkData(12, 1, True) + # # test select normal column tdSql.query("select dataint from jsons1 order by dataint") tdSql.checkRows(9) diff --git a/tests/system-test/fulltest.bat b/tests/system-test/fulltest.bat index c4eaeed2adb354e0d33b2e7f7f0a35fde12a2dbe..6ef66c58c2b2e5b06677a59ce2ee3e4f60d27304 100644 --- a/tests/system-test/fulltest.bat +++ b/tests/system-test/fulltest.bat @@ -1,9 +1,9 @@ -@REM python3 .\test.py -f 0-others\taosShell.py +python3 .\test.py -f 0-others\taosShell.py python3 .\test.py -f 0-others\taosShellError.py python3 .\test.py -f 0-others\taosShellNetChk.py python3 .\test.py -f 0-others\telemetry.py -@REM python3 .\test.py -f 0-others\taosdMonitor.py +python3 .\test.py -f 0-others\taosdMonitor.py python3 .\test.py -f 0-others\udfTest.py python3 .\test.py -f 0-others\udf_create.py python3 .\test.py -f 0-others\udf_restart_taosd.py diff --git a/tests/system-test/test.py b/tests/system-test/test.py index a11085708c42ec63672129d37636e30fb9140598..d9ae792901b1d2a24f16650917fd8a360cd6b982 100644 --- a/tests/system-test/test.py +++ b/tests/system-test/test.py @@ -114,6 +114,7 @@ if __name__ == "__main__": if not execCmd == "": tdDnodes.init(deployPath) + print(execCmd) exec(execCmd) quit()