diff --git a/.travis.yml b/.travis.yml index db6ce79703bab01e55227ad11a73395883a0ba1f..39fddc20c9f1c5c2b1369f248a3859c2bf6165cb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -64,7 +64,7 @@ matrix: memError=`grep -m 1 'ERROR SUMMARY' mem-error-out.txt | awk '{print $4}'` if [ -n "$memError" ]; then - if [ "$memError" -gt 16 ] && [ "$defiMemError" -gt 0 ]; then + if [ "$memError" -gt 16 ] || [ "$defiMemError" -gt 0 ]; then echo -e "${RED} ## Memory errors number valgrind reports is $memError.\ Definitely lost is $defiMemError. More than our threshold! ## ${NC}" travis_terminate $memError diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 8a2724198628d3d860b3a5f1a79b1b4e459090e6..4682e4306a8b4b8eba7fbd3cb8a86bbdc6effe8c 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -3544,7 +3544,7 @@ static int32_t setTableCondForSTableQuery(SQueryInfo* pQueryInfo, const char* ac return TSDB_CODE_SUCCESS; } - SStringBuilder sb1; + SStringBuilder sb1 = { 0 }; taosStringBuilderAppendStringLen(&sb1, QUERY_COND_REL_PREFIX_IN, QUERY_COND_REL_PREFIX_IN_LEN); char db[TSDB_TABLE_ID_LEN] = {0}; diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 48935480addf00d34e0c021c0fd016bb222e010f..ce941086c09caf20c597328d5ec4b509b50b6670 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -697,7 +697,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pQueryMsg->slidingTimeUnit = pQueryInfo->slidingTimeUnit; pQueryMsg->numOfGroupCols = htons(pQueryInfo->groupbyExpr.numOfGroupCols); pQueryMsg->numOfTags = htonl(numOfTags); - + pQueryMsg->tagNameRelType = htons(pQueryInfo->tagCond.relType); pQueryMsg->queryType = htons(pQueryInfo->type); size_t numOfOutput = tscSqlExprNumOfExprs(pQueryInfo); @@ -891,6 +891,14 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { } } + if (pQueryInfo->tagCond.tbnameCond.cond == NULL) { + *pMsg = 0; + pMsg++; + } else { + strcpy(pMsg, pQueryInfo->tagCond.tbnameCond.cond); + pMsg += strlen(pQueryInfo->tagCond.tbnameCond.cond) + 1; + } + // tbname in/like query expression should be sent to mgmt node msgLen = pMsg - pStart; diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index 216de9059881257060c16c2a2204822baedc761e..043e58cb35e34d4b6d5791322dcb9736f6250bf4 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -32,6 +32,9 @@ extern "C" { #define TSKEY int64_t #endif +// this data type is internally used only in 'in' query to hold the values +#define TSDB_DATA_TYPE_ARRAY (TSDB_DATA_TYPE_NCHAR + 1) + // Bytes for each type. extern const int32_t TYPE_BYTES[11]; // TODO: replace and remove code below @@ -141,16 +144,17 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size); #define TSDB_RELATION_GREATER_EQUAL 5 #define TSDB_RELATION_NOT_EQUAL 6 #define TSDB_RELATION_LIKE 7 +#define TSDB_RELATION_IN 8 -#define TSDB_RELATION_AND 8 -#define TSDB_RELATION_OR 9 -#define TSDB_RELATION_NOT 10 +#define TSDB_RELATION_AND 9 +#define TSDB_RELATION_OR 10 +#define TSDB_RELATION_NOT 11 -#define TSDB_BINARY_OP_ADD 11 -#define TSDB_BINARY_OP_SUBTRACT 12 -#define TSDB_BINARY_OP_MULTIPLY 13 -#define TSDB_BINARY_OP_DIVIDE 14 -#define TSDB_BINARY_OP_REMAINDER 15 +#define TSDB_BINARY_OP_ADD 12 +#define TSDB_BINARY_OP_SUBTRACT 13 +#define TSDB_BINARY_OP_MULTIPLY 14 +#define TSDB_BINARY_OP_DIVIDE 15 +#define TSDB_BINARY_OP_REMAINDER 16 #define TSDB_USERID_LEN 9 #define TS_PATH_DELIMITER_LEN 1 diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index b89f8b871f5551c3656102f063fbed0b669c5b61..6ff522ae439660a25e1cc522e58b3b03b2f3febf 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -456,6 +456,7 @@ typedef struct { int64_t offset; uint16_t queryType; // denote another query process int16_t numOfOutput; // final output columns numbers + int16_t tagNameRelType; // relation of tag criteria and tbname criteria int16_t interpoType; // interpolate type uint64_t defaultVal; // default value array list diff --git a/src/inc/tsdb.h b/src/inc/tsdb.h index 481e05c1022ede7dea9660a9f0cab81a88b0f2cd..e5c884c81931f4b1977cc9413f1186dbaf25cdd3 100644 --- a/src/inc/tsdb.h +++ b/src/inc/tsdb.h @@ -279,8 +279,17 @@ SArray *tsdbGetTableList(TsdbQueryHandleT *pQueryHandle); * @param pTagCond. tag query condition * */ -int32_t tsdbQueryByTagsCond(TsdbRepoT *tsdb, int64_t uid, const char *pTagCond, size_t len, STableGroupInfo *pGroupList, - SColIndex *pColIndex, int32_t numOfCols); +int32_t tsdbQueryByTagsCond( + TsdbRepoT *tsdb, + int64_t uid, + const char *pTagCond, + size_t len, + int16_t tagNameRelType, + const char* tbnameCond, + STableGroupInfo *pGroupList, + SColIndex *pColIndex, + int32_t numOfCols + ); int32_t tsdbGetOneTableGroup(TsdbRepoT *tsdb, int64_t uid, STableGroupInfo *pGroupInfo); diff --git a/src/query/inc/qast.h b/src/query/inc/qast.h index b7b553dae77f2042b69736e9651c9f86c3fb5cb3..903d54a18f7370b4a3d896a5c071eb9618cb78c4 100644 --- a/src/query/inc/qast.h +++ b/src/query/inc/qast.h @@ -92,7 +92,8 @@ uint8_t getBinaryExprOptr(SSQLToken *pToken); SBuffer exprTreeToBinary(tExprNode* pExprTree); -int32_t exprTreeFromBinary(const void* pBuf, size_t size, tExprNode** pExprNode); +tExprNode* exprTreeFromBinary(const void* pBuf, size_t size); +tExprNode* exprTreeFromTableName(const char* tbnameCond); #ifdef __cplusplus } diff --git a/src/query/inc/tvariant.h b/src/query/inc/tvariant.h index 05c7358d5fa97dfdcf08fef110b55c25592a527c..c235f5317dd08e5ab309db68d2abb10c6c687c96 100644 --- a/src/query/inc/tvariant.h +++ b/src/query/inc/tvariant.h @@ -17,6 +17,7 @@ #define TDENGINE_TVARIANT_H #include "tstoken.h" +#include "tarray.h" #ifdef __cplusplus extern "C" { @@ -31,6 +32,7 @@ typedef struct tVariant { double dKey; char * pz; wchar_t *wpz; + SArray *arr; // only for 'in' query to hold value list, not value for a field }; } tVariant; diff --git a/src/query/src/qast.c b/src/query/src/qast.c index d436d96d8868f2a7fc7879683b1352e73a2325d8..fdcbeeeac0315ba29466a9c1d8a81969d4713a31 100644 --- a/src/query/src/qast.c +++ b/src/query/src/qast.c @@ -30,6 +30,7 @@ #include "tarray.h" #include "tskiplist.h" #include "queryLog.h" +#include "tsdbMain.h" /* * @@ -521,38 +522,48 @@ static int32_t setQueryCond(tQueryInfo *queryColInfo, SQueryCond* pCond) { if (optr == TSDB_RELATION_GREATER || optr == TSDB_RELATION_GREATER_EQUAL || optr == TSDB_RELATION_EQUAL || optr == TSDB_RELATION_NOT_EQUAL) { pCond->start = calloc(1, sizeof(tVariant)); - tVariantAssign(&pCond->start->v, &queryColInfo->q); pCond->start->optr = queryColInfo->optr; + } else if (optr == TSDB_RELATION_LESS || optr == TSDB_RELATION_LESS_EQUAL) { pCond->end = calloc(1, sizeof(tVariant)); - tVariantAssign(&pCond->end->v, &queryColInfo->q); pCond->end->optr = queryColInfo->optr; + + } else if (optr == TSDB_RELATION_IN) { + printf("relation is in\n"); + + } else if (optr == TSDB_RELATION_LIKE) { + printf("relation is like\n"); + } return TSDB_CODE_SUCCESS; } -static void tQueryOnSkipList(SSkipList* pSkipList, SQueryCond* pCond, int32_t type, SArray* result) { +static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArray* result) { SSkipListIterator* iter = NULL; - - if (pCond->start != NULL) { - iter = tSkipListCreateIterFromVal(pSkipList, (char*) &pCond->start->v.i64Key, type, TSDB_ORDER_ASC); + int32_t type = pQueryInfo->q.nType; + + SQueryCond cond = { 0 }; + setQueryCond(pQueryInfo, &cond); + + if (cond.start != NULL) { + iter = tSkipListCreateIterFromVal(pSkipList, (char*) &cond.start->v.i64Key, type, TSDB_ORDER_ASC); } else { - iter = tSkipListCreateIterFromVal(pSkipList, (char*) &pCond->end->v.i64Key, type, TSDB_ORDER_DESC); + iter = tSkipListCreateIterFromVal(pSkipList, (char*) &cond.end->v.i64Key, type, TSDB_ORDER_DESC); } __compar_fn_t func = getComparFunc(pSkipList->keyInfo.type, type, 0); - if (pCond->start != NULL) { - int32_t optr = pCond->start->optr; + if (cond.start != NULL) { + int32_t optr = cond.start->optr; if (optr == TSDB_RELATION_EQUAL) { while(tSkipListIterNext(iter)) { SSkipListNode* pNode = tSkipListIterGet(iter); - int32_t ret = func(SL_GET_NODE_KEY(pSkipList, pNode), &pCond->start->v.i64Key); + int32_t ret = func(SL_GET_NODE_KEY(pSkipList, pNode), &cond.start->v.i64Key); if (ret == 0) { taosArrayPush(result, SL_GET_NODE_DATA(pNode)); } else { @@ -567,7 +578,7 @@ static void tQueryOnSkipList(SSkipList* pSkipList, SQueryCond* pCond, int32_t ty SSkipListNode* pNode = tSkipListIterGet(iter); if (comp) { - ret = func(SL_GET_NODE_KEY(pSkipList, pNode), &pCond->start->v.i64Key); + ret = func(SL_GET_NODE_KEY(pSkipList, pNode), &cond.start->v.i64Key); assert(ret >= 0); } @@ -584,7 +595,7 @@ static void tQueryOnSkipList(SSkipList* pSkipList, SQueryCond* pCond, int32_t ty assert(0); } } else { - int32_t optr = pCond->end->optr; + int32_t optr = cond.end->optr; if (optr == TSDB_RELATION_LESS || optr == TSDB_RELATION_LESS_EQUAL) { bool comp = true; @@ -594,7 +605,7 @@ static void tQueryOnSkipList(SSkipList* pSkipList, SQueryCond* pCond, int32_t ty SSkipListNode* pNode = tSkipListIterGet(iter); if (comp) { - ret = func(SL_GET_NODE_KEY(pSkipList, pNode), &pCond->end->v.i64Key); + ret = func(SL_GET_NODE_KEY(pSkipList, pNode), &cond.end->v.i64Key); assert(ret <= 0); } @@ -759,19 +770,55 @@ static void exprTreeTraverseImpl(tExprNode *pExpr, SArray *pResult, SBinaryFilte taosArrayCopy(pResult, array); } -static void tSQLBinaryTraverseOnSkipList(tExprNode *pExpr, SArray *pResult, SSkipList *pSkipList, - SBinaryFilterSupp *param) { + +static void tSQLBinaryTraverseOnSkipList( + tExprNode *pExpr, + SArray *pResult, + SSkipList *pSkipList, + SBinaryFilterSupp *param +) { SSkipListIterator* iter = tSkipListCreateIter(pSkipList); while (tSkipListIterNext(iter)) { SSkipListNode *pNode = tSkipListIterGet(iter); - if (filterItem(pExpr, pNode, param)) { taosArrayPush(pResult, SL_GET_NODE_DATA(pNode)); } } + tSkipListDestroyIter(iter); } + + +static void tQueryIndexlessColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArray* result) { + SSkipListIterator* iter = tSkipListCreateIter(pSkipList); + + while (tSkipListIterNext(iter)) { + bool addToResult = false; + + SSkipListNode *pNode = tSkipListIterGet(iter); + STable* table = *(STable**) SL_GET_NODE_DATA(pNode); + + if (pQueryInfo->colIndex == TSDB_TBNAME_COLUMN_INDEX) { + if (pQueryInfo->optr == TSDB_RELATION_IN) { + addToResult = pQueryInfo->compare(table->name, pQueryInfo->q.arr); + } else if(pQueryInfo->optr == TSDB_RELATION_LIKE) { + addToResult = !pQueryInfo->compare(table->name, pQueryInfo->q.pz); + } + } else { + // TODO: other columns + } + + if (addToResult) { + taosArrayPush(result, (void*)&table); + } + } + + tSkipListDestroyIter(iter); +} + + + // post-root order traverse syntax tree void tExprTreeTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, SBinaryFilterSupp *param) { if (pExpr == NULL) { @@ -781,95 +828,99 @@ void tExprTreeTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, S tExprNode *pLeft = pExpr->_node.pLeft; tExprNode *pRight = pExpr->_node.pRight; - // recursive traverse left child branch - if (pLeft->nodeType == TSQL_NODE_EXPR || pRight->nodeType == TSQL_NODE_EXPR) { - uint8_t weight = pLeft->_node.hasPK + pRight->_node.hasPK; + // column project + if (pLeft->nodeType != TSQL_NODE_EXPR && pRight->nodeType != TSQL_NODE_EXPR) { + assert(pLeft->nodeType == TSQL_NODE_COL && pRight->nodeType == TSQL_NODE_VALUE); - if (weight == 0 && taosArrayGetSize(result) > 0 && pSkipList == NULL) { - /** - * Perform the filter operation based on the initial filter result, which is obtained from filtering from index. - * Since no index presented, the filter operation is done by scan all elements in the result set. - * - * if the query is a high selectivity filter, only small portion of meters are retrieved. - */ + param->setupInfoFn(pExpr, param->pExtInfo); + if (pSkipList == NULL) { + tSQLListTraverseOnResult(pExpr, param->fp, result); + return; + } + + tQueryInfo *pQueryInfo = pExpr->_node.info; + if (pQueryInfo->colIndex == 0 && pQueryInfo->optr != TSDB_RELATION_LIKE) { + tQueryIndexColumn(pSkipList, pQueryInfo, result); + } else { + tQueryIndexlessColumn(pSkipList, pQueryInfo, result); + } + + return; + } + + // recursive traverse left child branch + uint8_t weight = pLeft->_node.hasPK + pRight->_node.hasPK; + + if (weight == 0 ) { + if (taosArrayGetSize(result) > 0 && pSkipList == NULL) { + /** + * Perform the filter operation based on the initial filter result, which is obtained from filtering from index. + * Since no index presented, the filter operation is done by scan all elements in the result set. + * + * if the query is a high selectivity filter, only small portion of meters are retrieved. + */ exprTreeTraverseImpl(pExpr, result, param); - } else if (weight == 0) { + } else { /** * apply the hierarchical expression to every node in skiplist for find the qualified nodes */ assert(taosArrayGetSize(result) == 0); tSQLBinaryTraverseOnSkipList(pExpr, result, pSkipList, param); - } else if (weight == 2 || (weight == 1 && pExpr->_node.optr == TSDB_RELATION_OR)) { - SArray* rLeft = taosArrayInit(10, POINTER_BYTES); - SArray* rRight = taosArrayInit(10, POINTER_BYTES); - - tExprTreeTraverse(pLeft, pSkipList, rLeft, param); - tExprTreeTraverse(pRight, pSkipList, rRight, param); - - if (pExpr->_node.optr == TSDB_RELATION_AND) { // CROSS - intersect(rLeft, rRight, result); - } else if (pExpr->_node.optr == TSDB_RELATION_OR) { // or - merge(rLeft, rRight, result); - } else { - assert(false); - } + } + return; + } + + if (weight == 2 || (weight == 1 && pExpr->_node.optr == TSDB_RELATION_OR)) { + SArray* rLeft = taosArrayInit(10, POINTER_BYTES); + SArray* rRight = taosArrayInit(10, POINTER_BYTES); - taosArrayDestroy(rLeft); - taosArrayDestroy(rRight); + tExprTreeTraverse(pLeft, pSkipList, rLeft, param); + tExprTreeTraverse(pRight, pSkipList, rRight, param); + + if (pExpr->_node.optr == TSDB_RELATION_AND) { // CROSS + intersect(rLeft, rRight, result); + } else if (pExpr->_node.optr == TSDB_RELATION_OR) { // or + merge(rLeft, rRight, result); } else { - /* - * (weight == 1 && pExpr->nSQLBinaryOptr == TSDB_RELATION_AND) is handled here - * - * first, we filter results based on the skiplist index, which is the initial filter stage, - * then, we conduct the secondary filter operation based on the result from the initial filter stage. - */ - assert(pExpr->_node.optr == TSDB_RELATION_AND); - - tExprNode *pFirst = NULL; - tExprNode *pSecond = NULL; - if (pLeft->_node.hasPK == 1) { - pFirst = pLeft; - pSecond = pRight; - } else { - pFirst = pRight; - pSecond = pLeft; - } + assert(false); + } - assert(pFirst != pSecond && pFirst != NULL && pSecond != NULL); + taosArrayDestroy(rLeft); + taosArrayDestroy(rRight); + return; + } - // we filter the result based on the skiplist index in the first place - tExprTreeTraverse(pFirst, pSkipList, result, param); + /* + * (weight == 1 && pExpr->nSQLBinaryOptr == TSDB_RELATION_AND) is handled here + * + * first, we filter results based on the skiplist index, which is the initial filter stage, + * then, we conduct the secondary filter operation based on the result from the initial filter stage. + */ + assert(pExpr->_node.optr == TSDB_RELATION_AND); + + tExprNode *pFirst = NULL; + tExprNode *pSecond = NULL; + if (pLeft->_node.hasPK == 1) { + pFirst = pLeft; + pSecond = pRight; + } else { + pFirst = pRight; + pSecond = pLeft; + } - /* - * recursively perform the filter operation based on the initial results, - * So, we do not set the skip list index as a parameter - */ - tExprTreeTraverse(pSecond, NULL, result, param); - } - } else { // column project - assert(pLeft->nodeType == TSQL_NODE_COL && pRight->nodeType == TSQL_NODE_VALUE); + assert(pFirst != pSecond && pFirst != NULL && pSecond != NULL); - param->setupInfoFn(pExpr, param->pExtInfo); - if (pSkipList == NULL) { - tSQLListTraverseOnResult(pExpr, param->fp, result); - } else { - tQueryInfo *pQueryInfo = pExpr->_node.info; - - if (pQueryInfo->colIndex == 0 && pQueryInfo->optr != TSDB_RELATION_LIKE) { - SQueryCond cond = {0}; - /*int32_t ret = */ setQueryCond(pQueryInfo, &cond); - tQueryOnSkipList(pSkipList, &cond, pQueryInfo->q.nType, result); - } else { - /* Brutal force scan the whole skip list to find the appropriate result, - * since the filter is not applied to indexed column. - */ - assert(0); -// result->num = tSkipListIterateList(pSkipList, (tSkipListNode ***)&result->pRes, fp, queryColInfo); - } - } - } + // we filter the result based on the skiplist index in the first place + tExprTreeTraverse(pFirst, pSkipList, result, param); + + /* + * recursively perform the filter operation based on the initial results, + * So, we do not set the skip list index as a parameter + */ + tExprTreeTraverse(pSecond, NULL, result, param); } + void tExprTreeCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, void *param, int32_t order, char *(*getSourceDataBlock)(void *, const char*, int32_t)) { if (pExprs == NULL) { @@ -1019,49 +1070,143 @@ SBuffer exprTreeToBinary(tExprNode* pExprTree) { return buf; } -static void exprTreeFromBinaryImpl(tExprNode** pExprTree, SBuffer* pBuf) { +static tExprNode* exprTreeFromBinaryImpl(SBuffer* pBuf) { tExprNode* pExpr = calloc(1, sizeof(tExprNode)); - tbufReadToBuffer(pBuf, &pExpr->nodeType, sizeof(pExpr->nodeType)); + pExpr->nodeType = tbufReadUint8(pBuf); if (pExpr->nodeType == TSQL_NODE_VALUE) { tVariant* pVal = calloc(1, sizeof(tVariant)); + if (pVal == NULL) { + // TODO: + } + pExpr->pVal = pVal; - tbufReadToBuffer(pBuf, &pVal->nType, sizeof(pVal->nType)); + pVal->nType = tbufReadUint32(pBuf); if (pVal->nType == TSDB_DATA_TYPE_BINARY) { tbufReadToBuffer(pBuf, &pVal->nLen, sizeof(pVal->nLen)); pVal->pz = calloc(1, pVal->nLen + 1); tbufReadToBuffer(pBuf, pVal->pz, pVal->nLen); } else { - tbufReadToBuffer(pBuf, &pVal->pz, sizeof(pVal->i64Key)); + pVal->i64Key = tbufReadInt64(pBuf); } - pExpr->pVal = pVal; } else if (pExpr->nodeType == TSQL_NODE_COL) { SSchema* pSchema = calloc(1, sizeof(SSchema)); - tbufReadToBuffer(pBuf, &pSchema->colId, sizeof(pSchema->colId)); - tbufReadToBuffer(pBuf, &pSchema->bytes, sizeof(pSchema->bytes)); - tbufReadToBuffer(pBuf, &pSchema->type, sizeof(pSchema->type)); - + if (pSchema == NULL) { + // TODO: + } + pExpr->pSchema = pSchema; + + pSchema->colId = tbufReadInt16(pBuf); + pSchema->bytes = tbufReadInt16(pBuf); + pSchema->type = tbufReadUint8(pBuf); tbufReadToString(pBuf, pSchema->name, TSDB_COL_NAME_LEN); - pExpr->pSchema = pSchema; } else if (pExpr->nodeType == TSQL_NODE_EXPR) { - tbufReadToBuffer(pBuf, &pExpr->_node.optr, sizeof(pExpr->_node.optr)); - tbufReadToBuffer(pBuf, &pExpr->_node.hasPK, sizeof(pExpr->_node.hasPK)); - - exprTreeFromBinaryImpl(&pExpr->_node.pLeft, pBuf); - exprTreeFromBinaryImpl(&pExpr->_node.pRight, pBuf); + pExpr->_node.optr = tbufReadUint8(pBuf); + pExpr->_node.hasPK = tbufReadUint8(pBuf); + pExpr->_node.pLeft = exprTreeFromBinaryImpl(pBuf); + pExpr->_node.pRight = exprTreeFromBinaryImpl(pBuf); assert(pExpr->_node.pLeft != NULL && pExpr->_node.pRight != NULL); } - *pExprTree = pExpr; + return pExpr; } -int32_t exprTreeFromBinary(const void* pBuf, size_t size, tExprNode** pExprNode) { +tExprNode* exprTreeFromBinary(const void* pBuf, size_t size) { + if (size == 0) { + return NULL; + } SBuffer rbuf = {0}; - /*int32_t code =*/ tbufBeginRead(&rbuf, pBuf, size); - exprTreeFromBinaryImpl(pExprNode, &rbuf); - return TSDB_CODE_SUCCESS; + tbufBeginRead(&rbuf, pBuf, size); + return exprTreeFromBinaryImpl(&rbuf); } +tExprNode* exprTreeFromTableName(const char* tbnameCond) { + if (!tbnameCond) { + return NULL; + } + + tExprNode* expr = calloc(1, sizeof(tExprNode)); + if (expr == NULL) { + // TODO: + } + expr->nodeType = TSQL_NODE_EXPR; + + tExprNode* left = calloc(1, sizeof(tExprNode)); + if (left == NULL) { + // TODO: + } + expr->_node.pLeft = left; + + left->nodeType = TSQL_NODE_COL; + SSchema* pSchema = calloc(1, sizeof(SSchema)); + if (pSchema == NULL) { + // TODO: + } + left->pSchema = pSchema; + + pSchema->type = TSDB_DATA_TYPE_BINARY; + pSchema->bytes = TSDB_TABLE_NAME_LEN; + strcpy(pSchema->name, TSQL_TBNAME_L); + pSchema->colId = -1; + + tExprNode* right = calloc(1, sizeof(tExprNode)); + if (right == NULL) { + // TODO + } + expr->_node.pRight = right; + + if (strncmp(tbnameCond, QUERY_COND_REL_PREFIX_LIKE, QUERY_COND_REL_PREFIX_LIKE_LEN) == 0) { + right->nodeType = TSQL_NODE_VALUE; + expr->_node.optr = TSDB_RELATION_LIKE; + tVariant* pVal = calloc(1, sizeof(tVariant)); + if (pVal == NULL) { + // TODO: + } + right->pVal = pVal; + pVal->nType = TSDB_DATA_TYPE_BINARY; + size_t len = strlen(tbnameCond + QUERY_COND_REL_PREFIX_LIKE_LEN) + 1; + pVal->pz = malloc(len); + if (pVal->pz == NULL) { + // TODO: + } + memcpy(pVal->pz, tbnameCond + QUERY_COND_REL_PREFIX_LIKE_LEN, len); + pVal->nLen = (int32_t)len; + + } else if (strncmp(tbnameCond, QUERY_COND_REL_PREFIX_IN, QUERY_COND_REL_PREFIX_IN_LEN) == 0) { + right->nodeType = TSQL_NODE_VALUE; + expr->_node.optr = TSDB_RELATION_IN; + tVariant* pVal = calloc(1, sizeof(tVariant)); + if (pVal == NULL) { + // TODO: + } + right->pVal = pVal; + pVal->nType = TSDB_DATA_TYPE_ARRAY; + pVal->arr = taosArrayInit(2, sizeof(char*)); + + const char* cond = tbnameCond + QUERY_COND_REL_PREFIX_IN_LEN; + for (const char *e = cond; *e != 0; e++) { + if (*e == TS_PATH_DELIMITER[0]) { + cond = e + 1; + } else if (*e == ',') { + size_t len = e - cond + 1; + char* p = malloc( len ); + memcpy(p, cond, len); + p[len - 1] = 0; + cond += len; + taosArrayPush(pVal->arr, &p); + } + } + + if (*cond != 0) { + char* p = strdup( cond ); + taosArrayPush(pVal->arr, &p); + } + + taosArraySortString(pVal->arr); + } + + return expr; +} \ No newline at end of file diff --git a/src/query/src/queryExecutor.c b/src/query/src/queryExecutor.c index b19e024dd985c92fd895f33d8d6419d08c5328a6..75c1b9736b3c30fca3891bc66dc94f300b3377bc 100644 --- a/src/query/src/queryExecutor.c +++ b/src/query/src/queryExecutor.c @@ -5311,7 +5311,7 @@ static char *createTableIdList(SQueryTableMsg *pQueryMsg, char *pMsg, SArray **p * @return */ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, SSqlFuncMsg ***pExpr, - char **tagCond, SColIndex **groupbyCols, SColumnInfo** tagCols) { + char **tagCond, char** tbnameCond, SColIndex **groupbyCols, SColumnInfo** tagCols) { pQueryMsg->numOfTables = htonl(pQueryMsg->numOfTables); pQueryMsg->window.skey = htobe64(pQueryMsg->window.skey); @@ -5324,6 +5324,7 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, pQueryMsg->order = htons(pQueryMsg->order); pQueryMsg->orderColId = htons(pQueryMsg->orderColId); pQueryMsg->queryType = htons(pQueryMsg->queryType); + pQueryMsg->tagNameRelType = htons(pQueryMsg->tagNameRelType); pQueryMsg->numOfCols = htons(pQueryMsg->numOfCols); pQueryMsg->numOfOutput = htons(pQueryMsg->numOfOutput); @@ -5455,7 +5456,7 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, pMsg += sizeof(int64_t) * pQueryMsg->numOfOutput; } - + if (pQueryMsg->numOfTags > 0) { (*tagCols) = calloc(1, sizeof(SColumnInfo) * pQueryMsg->numOfTags); for (int32_t i = 0; i < pQueryMsg->numOfTags; ++i) { @@ -5477,6 +5478,13 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, memcpy(*tagCond, pMsg, pQueryMsg->tagCondLen); pMsg += pQueryMsg->tagCondLen; } + + if (*pMsg != 0) { + size_t len = strlen(pMsg) + 1; + *tbnameCond = malloc(len); + strcpy(*tbnameCond, pMsg); + pMsg += len; + } qTrace("qmsg:%p query on %d table(s), qrange:%" PRId64 "-%" PRId64 ", numOfGroupbyTagCols:%d, ts order:%d, " @@ -5490,12 +5498,10 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, } static int32_t buildAirthmeticExprFromMsg(SArithExprInfo *pArithExprInfo, SQueryTableMsg *pQueryMsg) { - tExprNode* pExprNode = NULL; - qTrace("qmsg:%p create arithmetic expr from binary string", pQueryMsg, pArithExprInfo->base.arg[0].argValue.pz); - int32_t ret = exprTreeFromBinary(pArithExprInfo->base.arg[0].argValue.pz, pArithExprInfo->base.arg[0].argBytes, &pExprNode); - if (pExprNode == NULL || ret != TSDB_CODE_SUCCESS) { + tExprNode* pExprNode = exprTreeFromBinary(pArithExprInfo->base.arg[0].argValue.pz, pArithExprInfo->base.arg[0].argBytes); + if (pExprNode == NULL) { qError("qmsg:%p failed to create arithmetic expression string from:%s", pQueryMsg, pArithExprInfo->base.arg[0].argValue.pz); return TSDB_CODE_APP_ERROR; } @@ -6040,13 +6046,13 @@ int32_t qCreateQueryInfo(void *tsdb, SQueryTableMsg *pQueryMsg, qinfo_t *pQInfo) int32_t code = TSDB_CODE_SUCCESS; - char * tagCond = NULL; + char * tagCond = NULL, *tbnameCond = NULL; SArray * pTableIdList = NULL; SSqlFuncMsg **pExprMsg = NULL; SColIndex * pGroupColIndex = NULL; SColumnInfo* pTagColumnInfo = NULL; - if ((code = convertQueryMsg(pQueryMsg, &pTableIdList, &pExprMsg, &tagCond, &pGroupColIndex, &pTagColumnInfo)) != + if ((code = convertQueryMsg(pQueryMsg, &pTableIdList, &pExprMsg, &tagCond, &tbnameCond, &pGroupColIndex, &pTagColumnInfo)) != TSDB_CODE_SUCCESS) { return code; } @@ -6089,7 +6095,7 @@ int32_t qCreateQueryInfo(void *tsdb, SQueryTableMsg *pQueryMsg, qinfo_t *pQInfo) } // todo handle the error - /*int32_t ret =*/tsdbQueryByTagsCond(tsdb, id->uid, tagCond, pQueryMsg->tagCondLen, &groupInfo, pGroupColIndex, + /*int32_t ret =*/tsdbQueryByTagsCond(tsdb, id->uid, tagCond, pQueryMsg->tagCondLen, pQueryMsg->tagNameRelType, tbnameCond, &groupInfo, pGroupColIndex, numOfGroupByCols); if (groupInfo.numOfTables == 0) { // no qualified tables no need to do query code = TSDB_CODE_SUCCESS; @@ -6112,6 +6118,8 @@ int32_t qCreateQueryInfo(void *tsdb, SQueryTableMsg *pQueryMsg, qinfo_t *pQInfo) code = initQInfo(pQueryMsg, tsdb, *pQInfo, isSTableQuery); _query_over: + tfree(tagCond); + tfree(tbnameCond); taosArrayDestroy(pTableIdList); // if failed to add ref for all meters in this query, abort current query diff --git a/src/query/src/tvariant.c b/src/query/src/tvariant.c index 31f04d337d2eef21a90987fc7628c43c9f324a84..ab5e30f6c40609068496098b4846e7783c09faf3 100644 --- a/src/query/src/tvariant.c +++ b/src/query/src/tvariant.c @@ -130,6 +130,17 @@ void tVariantDestroy(tVariant *pVar) { tfree(pVar->pz); pVar->nLen = 0; } + + // NOTE: this is only for string array + if (pVar->nType == TSDB_DATA_TYPE_ARRAY) { + size_t num = taosArrayGetSize(pVar->arr); + for(size_t i = 0; i < num; i++) { + void* p = taosArrayGetP(pVar->arr, i); + free(p); + } + taosArrayDestroy(pVar->arr); + pVar->arr = NULL; + } } void tVariantAssign(tVariant *pDst, const tVariant *pSrc) { @@ -145,6 +156,18 @@ void tVariantAssign(tVariant *pDst, const tVariant *pSrc) { pDst->pz = calloc(1, len); memcpy(pDst->pz, pSrc->pz, len); + return; + } + + // this is only for string array + if (pSrc->nType == TSDB_DATA_TYPE_ARRAY) { + size_t num = taosArrayGetSize(pSrc->arr); + pDst->arr = taosArrayInit(num, sizeof(char*)); + for(size_t i = 0; i < num; i++) { + char* p = (char*)taosArrayGetP(pSrc->arr, i); + char* n = strdup(p); + taosArrayPush(pDst->arr, &n); + } } } diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 2357cc892f2ac9a836dcee177954e40fb5abfe0a..eb35be5383e1e82761a47de6fcd54c223687234b 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -1213,7 +1213,9 @@ void filterPrepare(void* expr, void* param) { pInfo->compare = getComparFunc(pSchema->type, pCond->nType, pInfo->optr); tVariantAssign(&pInfo->q, pCond); - tVariantTypeSetType(&pInfo->q, pInfo->sch.type); + if (pInfo->optr != TSDB_RELATION_IN) { + tVariantTypeSetType(&pInfo->q, pInfo->sch.type); + } } int32_t doCompare(const char* f1, const char* f2, int32_t type, size_t size) { @@ -1327,12 +1329,9 @@ SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pC } if (numOfOrderCols == 0 || size == 1) { // no group by tags clause or only one table - size_t num = taosArrayGetSize(pTableList); - - SArray* sa = taosArrayInit(num, sizeof(SPair)); - for(int32_t i = 0; i < num; ++i) { + SArray* sa = taosArrayInit(size, sizeof(SPair)); + for(int32_t i = 0; i < size; ++i) { STable* pTable = taosArrayGetP(pTableList, i); - SPair p = {.first = pTable}; taosArrayPush(sa, &p); } @@ -1358,16 +1357,26 @@ bool tSkipListNodeFilterCallback(const void* pNode, void* param) { STable* pTable = *(STable**)(SL_GET_NODE_DATA((SSkipListNode*)pNode)); - char* val = dataRowTuple(pTable->tagVal); // todo not only the first column + char* val = NULL; int8_t type = pInfo->sch.type; + + if (pInfo->colIndex == TSDB_TBNAME_COLUMN_INDEX) { + val = pTable->name; + type = TSDB_DATA_TYPE_BINARY; + } else { + val = dataRowTuple(pTable->tagVal); // todo not only the first column + } int32_t ret = 0; - if (pInfo->q.nType == TSDB_DATA_TYPE_BINARY || pInfo->q.nType == TSDB_DATA_TYPE_NCHAR) { - ret = pInfo->compare(val, pInfo->q.pz); + if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { + if (pInfo->optr == TSDB_RELATION_IN) { + ret = pInfo->compare(val, pInfo->q.arr); + } else { + ret = pInfo->compare(val, pInfo->q.pz); + } } else { tVariant t = {0}; tVariantCreateFromBinary(&t, val, (uint32_t)pInfo->sch.bytes, type); - ret = pInfo->compare(&t.i64Key, &pInfo->q.i64Key); } @@ -1393,6 +1402,9 @@ bool tSkipListNodeFilterCallback(const void* pNode, void* param) { case TSDB_RELATION_LIKE: { return ret == 0; } + case TSDB_RELATION_IN: { + return ret == 1; + } default: assert(false); @@ -1400,6 +1412,7 @@ bool tSkipListNodeFilterCallback(const void* pNode, void* param) { return true; } + static int32_t doQueryTableList(STable* pSTable, SArray* pRes, tExprNode* pExpr) { // query according to the binary expression STSchema* pSchema = pSTable->tagSchema; @@ -1422,12 +1435,23 @@ static int32_t doQueryTableList(STable* pSTable, SArray* pRes, tExprNode* pExpr) tExprTreeDestroy(&pExpr, destroyHelper); convertQueryResult(pRes, pTableList); + taosArrayDestroy(pTableList); + free(schema); return TSDB_CODE_SUCCESS; } -int32_t tsdbQueryByTagsCond(TsdbRepoT* tsdb, int64_t uid, const char* pTagCond, size_t len, STableGroupInfo* pGroupInfo, - SColIndex* pColIndex, int32_t numOfCols) { - + +int32_t tsdbQueryByTagsCond( + TsdbRepoT *tsdb, + int64_t uid, + const char *pTagCond, + size_t len, + int16_t tagNameRelType, + const char* tbnameCond, + STableGroupInfo *pGroupInfo, + SColIndex *pColIndex, + int32_t numOfCols +) { STable* pSTable = tsdbGetTableByUid(tsdbGetMeta(tsdb), uid); if (pSTable == NULL) { uError("failed to get stable, uid:%" PRIu64, uid); @@ -1437,31 +1461,35 @@ int32_t tsdbQueryByTagsCond(TsdbRepoT* tsdb, int64_t uid, const char* pTagCond, SArray* res = taosArrayInit(8, POINTER_BYTES); STSchema* pTagSchema = tsdbGetTableTagSchema(tsdbGetMeta(tsdb), pSTable); - if (pTagCond == NULL || len == 0) { // no tags condition, all tables created according to this stable are involved + // no tags and tbname condition, all child tables of this stable are involved + if (tbnameCond == NULL && (pTagCond == NULL || len == 0)) { int32_t ret = getAllTableIdList(tsdb, uid, res); - if (ret != TSDB_CODE_SUCCESS) { - taosArrayDestroy(res); - return ret; + if (ret == TSDB_CODE_SUCCESS) { + pGroupInfo->numOfTables = taosArrayGetSize(res); + pGroupInfo->pGroupList = createTableGroup(res, pTagSchema, pColIndex, numOfCols); } - - pGroupInfo->numOfTables = taosArrayGetSize(res); - pGroupInfo->pGroupList = createTableGroup(res, pTagSchema, pColIndex, numOfCols); taosArrayDestroy(res); return ret; } - tExprNode* pExprNode = NULL; int32_t ret = TSDB_CODE_SUCCESS; - // failed to build expression, no result, return immediately - if ((ret = exprTreeFromBinary(pTagCond, len, &pExprNode) != TSDB_CODE_SUCCESS) || (pExprNode == NULL)) { - uError("stable:%" PRIu64 ", failed to deserialize expression tree, error exists", uid); - taosArrayDestroy(res); - return ret; + tExprNode* expr = exprTreeFromTableName(tbnameCond); + tExprNode* tagExpr = exprTreeFromBinary(pTagCond, len); + if (tagExpr != NULL) { + if (expr == NULL) { + expr = tagExpr; + } else { + tExprNode* tbnameExpr = expr; + expr = calloc(1, sizeof(tExprNode)); + expr->nodeType = TSQL_NODE_EXPR; + expr->_node.optr = tagNameRelType; + expr->_node.pLeft = tagExpr; + expr->_node.pRight = tbnameExpr; + } } - doQueryTableList(pSTable, res, pExprNode); - + doQueryTableList(pSTable, res, expr); pGroupInfo->numOfTables = taosArrayGetSize(res); pGroupInfo->pGroupList = createTableGroup(res, pTagSchema, pColIndex, numOfCols); diff --git a/src/util/inc/tarray.h b/src/util/inc/tarray.h index 6fab24d51ac1bd8bebc2ad793c1deeccbb6894f1..dbb74edf07d7535d1de4d1485fdd3df410b82c5d 100644 --- a/src/util/inc/tarray.h +++ b/src/util/inc/tarray.h @@ -53,7 +53,7 @@ void* taosArrayPush(SArray* pArray, void* pData); * * @param pArray */ -void taosArrayPop(SArray* pArray); +void* taosArrayPop(SArray* pArray); /** * get the data from array @@ -112,6 +112,34 @@ SArray* taosArrayClone(SArray* pSrc); */ void taosArrayDestroy(SArray* pArray); +/** + * sort the array + * @param pArray + * @param compar + */ +void taosArraySort(SArray* pArray, int (*compar)(const void*, const void*)); + +/** + * sort string array + * @param pArray + */ +void taosArraySortString(SArray* pArray); + +/** + * search the array + * @param pArray + * @param compar + * @param key + */ +void* taosArraySearch(const SArray* pArray, int (*compar)(const void*, const void*), const void* key); + +/** + * search the array + * @param pArray + * @param key + */ +char* taosArraySearchString(const SArray* pArray, const char* key); + #ifdef __cplusplus } #endif diff --git a/src/util/inc/tbuffer.h b/src/util/inc/tbuffer.h index d52031fc6a63252a393189d2c4bf9aa2ff622ded..9bc0fd9f43eb9122aa50a438a3eb1053312c7d6e 100644 --- a/src/util/inc/tbuffer.h +++ b/src/util/inc/tbuffer.h @@ -120,7 +120,7 @@ void tbufWriteString(SBuffer* buf, const char* str); TBUFFER_DEFINE_FUNCTION(bool, Bool) TBUFFER_DEFINE_FUNCTION(char, Char) TBUFFER_DEFINE_FUNCTION(int8_t, Int8) -TBUFFER_DEFINE_FUNCTION(uint8_t, Unt8) +TBUFFER_DEFINE_FUNCTION(uint8_t, Uint8) TBUFFER_DEFINE_FUNCTION(int16_t, Int16) TBUFFER_DEFINE_FUNCTION(uint16_t, Uint16) TBUFFER_DEFINE_FUNCTION(int32_t, Int32) diff --git a/src/util/src/tarray.c b/src/util/src/tarray.c index c0259584388bdced6811f0aba301938e27affee8..51684e767cd517bda0d7b69e04e2fc3c3fac4acd 100755 --- a/src/util/src/tarray.c +++ b/src/util/src/tarray.c @@ -27,7 +27,7 @@ void* taosArrayInit(size_t size, size_t elemSize) { return NULL; } - pArray->pData = calloc(size, elemSize * size); + pArray->pData = calloc(size, elemSize); if (pArray->pData == NULL) { free(pArray); return NULL; @@ -76,12 +76,14 @@ void* taosArrayPush(SArray* pArray, void* pData) { return dst; } -void taosArrayPop(SArray* pArray) { - if (pArray == NULL || pArray->size == 0) { - return; - } +void* taosArrayPop(SArray* pArray) { + assert( pArray != NULL ); + if (pArray->size == 0) { + return NULL; + } pArray->size -= 1; + return TARRAY_GET_ELEM(pArray, pArray->size); } void* taosArrayGet(const SArray* pArray, size_t index) { @@ -183,3 +185,40 @@ void taosArrayDestroy(SArray* pArray) { free(pArray->pData); free(pArray); } + +void taosArraySort(SArray* pArray, int (*compar)(const void*, const void*)) { + assert(pArray != NULL); + assert(compar != NULL); + + qsort(pArray->pData, pArray->size, pArray->elemSize, compar); +} + +void* taosArraySearch(const SArray* pArray, int (*compar)(const void*, const void*), const void* key) { + assert(pArray != NULL); + assert(compar != NULL); + assert(key != NULL); + + return bsearch(key, pArray->pData, pArray->size, pArray->elemSize, compar); +} + +static int taosArrayCompareString(const void* a, const void* b) { + const char* x = *(const char**)a; + const char* y = *(const char**)b; + return strcmp(x, y); +} + +void taosArraySortString(SArray* pArray) { + assert(pArray != NULL); + qsort(pArray->pData, pArray->size, pArray->elemSize, taosArrayCompareString); +} + +char* taosArraySearchString(const SArray* pArray, const char* key) { + assert(pArray != NULL); + assert(key != NULL); + + void* p = bsearch(&key, pArray->pData, pArray->size, pArray->elemSize, taosArrayCompareString); + if (p == NULL) { + return NULL; + } + return *(char**)p; +} \ No newline at end of file diff --git a/src/util/src/tcompare.c b/src/util/src/tcompare.c index 4505ea533a2725869e159c061b9dbb0baa78e050..4d78a8cf0adada9841bd70b006d5ceb588faa4b6 100644 --- a/src/util/src/tcompare.c +++ b/src/util/src/tcompare.c @@ -1,42 +1,34 @@ #include "taosdef.h" #include "tcompare.h" - +#include #include "tutil.h" int32_t compareInt32Val(const void *pLeft, const void *pRight) { - int32_t ret = GET_INT32_VAL(pLeft) - GET_INT32_VAL(pRight); - if (ret == 0) { - return 0; - } else { - return ret > 0 ? 1 : -1; - } + int32_t left = GET_INT32_VAL(pLeft), right = GET_INT32_VAL(pRight); + if (left > right) return 1; + if (left < right) return -1; + return 0; } int32_t compareInt64Val(const void *pLeft, const void *pRight) { - int64_t ret = GET_INT64_VAL(pLeft) - GET_INT64_VAL(pRight); - if (ret == 0) { - return 0; - } else { - return ret > 0 ? 1 : -1; - } + int64_t left = GET_INT64_VAL(pLeft), right = GET_INT64_VAL(pRight); + if (left > right) return 1; + if (left < right) return -1; + return 0; } int32_t compareInt16Val(const void *pLeft, const void *pRight) { - int32_t ret = GET_INT16_VAL(pLeft) - GET_INT16_VAL(pRight); - if (ret == 0) { - return 0; - } else { - return ret > 0 ? 1 : -1; - } + int16_t left = GET_INT16_VAL(pLeft), right = GET_INT16_VAL(pRight); + if (left > right) return 1; + if (left < right) return -1; + return 0; } int32_t compareInt8Val(const void *pLeft, const void *pRight) { - int32_t ret = GET_INT8_VAL(pLeft) - GET_INT8_VAL(pRight); - if (ret == 0) { - return 0; - } else { - return ret > 0 ? 1 : -1; - } + int8_t left = GET_INT8_VAL(pLeft), right = GET_INT8_VAL(pRight); + if (left > right) return 1; + if (left < right) return -1; + return 0; } int32_t compareIntDoubleVal(const void *pLeft, const void *pRight) { @@ -69,12 +61,7 @@ int32_t compareDoubleVal(const void *pLeft, const void *pRight) { } int32_t compareStrVal(const void *pLeft, const void *pRight) { - int32_t ret = strcmp(pLeft, pRight); - if (ret == 0) { - return 0; - } else { - return ret > 0 ? 1 : -1; - } + return (int32_t)strcmp(pLeft, pRight); } int32_t compareWStrVal(const void *pLeft, const void *pRight) { @@ -228,6 +215,11 @@ static UNUSED_FUNC int32_t compareStrPatternComp(const void* pLeft, const void* return (ret == TSDB_PATTERN_MATCH) ? 0 : 1; } +static int32_t compareFindStrInArray(const void* pLeft, const void* pRight) { + const SArray* arr = (const SArray*)pRight; + return taosArraySearchString(arr, pLeft) == NULL ? 0 : 1; +} + static UNUSED_FUNC int32_t compareWStrPatternComp(const void* pLeft, const void* pRight) { SPatternCompareInfo pInfo = {'%', '_'}; @@ -250,7 +242,6 @@ __compar_fn_t getComparFunc(int32_t type, int32_t filterDataType, int32_t optr) case TSDB_DATA_TYPE_BIGINT: case TSDB_DATA_TYPE_TIMESTAMP: { // assert(type == filterDataType); - if (filterDataType == TSDB_DATA_TYPE_BIGINT || filterDataType == TSDB_DATA_TYPE_TIMESTAMP) { comparFn = compareInt64Val; } else if (filterDataType >= TSDB_DATA_TYPE_FLOAT && filterDataType <= TSDB_DATA_TYPE_DOUBLE) { @@ -259,6 +250,7 @@ __compar_fn_t getComparFunc(int32_t type, int32_t filterDataType, int32_t optr) break; } + case TSDB_DATA_TYPE_BOOL: { if (filterDataType >= TSDB_DATA_TYPE_BOOL && filterDataType <= TSDB_DATA_TYPE_BIGINT) { comparFn = compareInt32Val; @@ -267,6 +259,7 @@ __compar_fn_t getComparFunc(int32_t type, int32_t filterDataType, int32_t optr) } break; } + case TSDB_DATA_TYPE_FLOAT: case TSDB_DATA_TYPE_DOUBLE: { if (filterDataType >= TSDB_DATA_TYPE_BOOL && filterDataType <= TSDB_DATA_TYPE_BIGINT) { @@ -276,12 +269,18 @@ __compar_fn_t getComparFunc(int32_t type, int32_t filterDataType, int32_t optr) } break; } + case TSDB_DATA_TYPE_BINARY: { - assert(filterDataType == TSDB_DATA_TYPE_BINARY); - if (optr == TSDB_RELATION_LIKE) { /* wildcard query using like operator */ + assert(filterDataType == TSDB_DATA_TYPE_BINARY); comparFn = compareStrPatternComp; + + } else if (optr == TSDB_RELATION_IN) { + assert(filterDataType == TSDB_DATA_TYPE_ARRAY); + comparFn = compareFindStrInArray; + } else { /* normal relational comparFn */ + assert(filterDataType == TSDB_DATA_TYPE_BINARY); comparFn = compareStrVal; } diff --git a/tests/pytest/insert/float.py b/tests/pytest/insert/float.py new file mode 100644 index 0000000000000000000000000000000000000000..7c4474f7ad0be25da8ec0622061654ea620e0219 --- /dev/null +++ b/tests/pytest/insert/float.py @@ -0,0 +1,147 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import datetime + +import taos + +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + tdSql.prepare() + + tdLog.info("=============== step1") + cmd = 'create table tb (ts timestamp, speed float)' + tdLog.info(cmd) + tdSql.execute(cmd) + cmd = 'insert into tb values (now, -3.40E+38)' + tdLog.info(cmd) + tdSql.execute(cmd) + + tdLog.info("=============== step2") + cmd = 'insert into tb values (now+1a, 3.40E+308)' + tdLog.info(cmd) + try: + tdSql.execute(cmd) + tdLog.exit( + "This test failed: insert wrong data error _not_ catched") + except Exception as e: + tdLog.info(repr(e)) + tdLog.notice("insert wrong data error catched") + + cmd = 'select * from tb order by ts desc' + tdLog.info(cmd) + tdSql.query(cmd) + tdSql.checkRows(1) + + tdLog.info("=============== step3") + cmd = "insert into tb values (now+2a, 2.85)" + tdLog.info(cmd) + tdSql.execute(cmd) + cmd = "select * from tb order by ts desc" + tdLog.info(cmd) + ret = tdSql.query(cmd) + tdSql.checkRows(2) + + if ((abs(tdSql.getData(0, 1) - 2.850000)) > 1.0e-7): + tdLog.exit("data is not 2.850000") + + tdLog.info("=============== step4") + cmd = "insert into tb values (now+3a, 3.4)" + tdLog.info(cmd) + tdSql.execute(cmd) + cmd = "select * from tb order by ts desc" + tdLog.info(cmd) + tdSql.query(cmd) + tdSql.checkRows(3) + if (abs(tdSql.getData(0, 1) - 3.400000) > 1.0e-7): + tdLog.exit("data is not 3.400000") + + tdLog.info("=============== step5") + cmd = "insert into tb values (now+4a, a2)" + tdLog.info(cmd) + try: + tdSql.execute(cmd) + tdLog.exit("This test failed: \ + insert wrong data error _not_ catched") + except Exception as e: + tdLog.info(repr(e)) + tdLog.notice("insert wrong data error catched") + + cmd = "insert into tb values (now+4a, 0)" + tdLog.info(cmd) + tdSql.execute(cmd) + cmd = "select * from tb order by ts desc" + tdLog.info(cmd) + tdSql.query(cmd) + tdSql.checkRows(4) + if (abs(tdSql.getData(0, 1) - 0.000000) != 0): + tdLog.exit("data is not 0.000000") + + tdLog.info("=============== step6") + cmd = "insert into tb values (now+5a, 2a)" + tdLog.info(cmd) + try: + tdSql.execute(cmd) + tdLog.exit( + "This test failed: insert wrong data error _not_ catched") + except Exception as e: + tdLog.info(repr(e)) + tdLog.notice("insert wrong data error catched") + + cmd = "insert into tb values (now+5a, 2)" + tdLog.info(cmd) + tdSql.execute(cmd) + cmd = "select * from tb order by ts desc" + tdLog.info(cmd) + ret = tdSql.query(cmd) + tdSql.checkRows(5) + if (abs(tdSql.getData(0, 1) - 2.000000) > 1.0e-7): + tdLog.info("data is not 2.000000") + + tdLog.info("=============== step7") + cmd = "insert into tb values (now+6a, 2a'1)" + tdLog.info(cmd) + try: + tdSql.execute(cmd) + tdLog.exit( + "This test failed: insert wrong data error _not_ catched") + except Exception as e: + tdLog.info(repr(e)) + tdLog.notice("insert wrong data error catched") + + cmd = "insert into tb values (now+6a, 2)" + tdLog.info(cmd) + tdSql.execute(cmd) + cmd = "select * from tb order by ts desc" + tdLog.info(cmd) + tdSql.query(cmd) + if (abs(tdSql.getData(0, 1) - 2.000000) > 1.0e-7): + tdLog.exit("data is not 2.000000") + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/insert/int.py b/tests/pytest/insert/int.py new file mode 100644 index 0000000000000000000000000000000000000000..8d38337576ccb78bd22155697b5d19c3655c11a1 --- /dev/null +++ b/tests/pytest/insert/int.py @@ -0,0 +1,184 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +import datetime + +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + tdSql.prepare() + + tdLog.info("=============== step1") + tdSql.execute('create table tb (ts timestamp, speed int)') + + cmd = 'insert into tb values (now, NULL)' + tdLog.info(cmd) + tdSql.execute(cmd) + tdSql.query('select * from tb order by ts desc') + tdSql.checkRows(1) + if(tdSql.getData(0, 1) is not None): + tdLog.exit("data is not NULL") + + tdLog.info("=============== step2") + cmd = 'insert into tb values (now+1m, -2147483648)' + tdLog.info(cmd) + try: + tdSql.execute(cmd) + tdLog.exit( + "This test failed: INT data overflow error _not_ catched") + except Exception as e: + tdLog.info(repr(e)) + tdLog.notice("INT data overflow error catched") + + cmd = 'insert into tb values (now+1m, NULL)' + tdLog.info(cmd) + tdSql.execute(cmd) + tdSql.query('select * from tb order by ts desc') + tdSql.checkRows(2) + + if(tdSql.getData(0, 1) is not None): + tdLog.exit("data is not NULL") + + tdLog.info("=============== step3") + cmd = 'insert into tb values (now+2m, 2147483647)' + tdLog.info(cmd) + tdSql.execute(cmd) + tdSql.query('select * from tb order by ts desc') + tdSql.checkRows(3) + if(tdSql.getData(0, 1) != 2147483647): + tdLog.exit("data is not 2147483647") + + tdLog.info("=============== step4") + cmd = 'insert into tb values (now+3m, 2147483648)' + tdLog.info(cmd) + try: + tdSql.execute(cmd) + tdLog.exit( + "This test failed: INT data overflow error _not_ catched") + except Exception as e: + tdLog.info(repr(e)) + tdLog.notice("INT data overflow error catched") + + cmd = 'insert into tb values (now+3m, NULL)' + tdLog.info(cmd) + tdSql.execute(cmd) + tdSql.query('select * from tb order by ts desc') + tdSql.checkRows(4) + + if(tdSql.getData(0, 1) is not None): + tdLog.exit("data is not NULL") + + tdLog.info("=============== step5") + cmd = 'insert into tb values (now+4m, a2)' + tdLog.info(cmd) + try: + tdSql.execute(cmd) + tdLog.exit( + "This test failed: insert wrong data error _not_ catched") + except Exception as e: + tdLog.info(repr(e)) + tdLog.notice("insert wrong data error catched") + + cmd = 'insert into tb values (now+4m, 0)' + tdLog.info(cmd) + tdSql.execute(cmd) + tdSql.query('select * from tb order by ts desc') + tdSql.checkRows(5) + + if(tdSql.getData(0, 1) != 0): + tdLog.exit("data is not 0") + + tdLog.info("=============== step6") + cmd = 'insert into tb values (now+5m, 2a)' + tdLog.info(cmd) + try: + tdSql.execute(cmd) + tdLog.exit( + "This test failed: insert wrong data error _not_ catched") + except Exception as e: + tdLog.info(repr(e)) + tdLog.notice("insert wrong data error catched") + + cmd = 'insert into tb values (now+5m, 2)' + tdLog.info(cmd) + tdSql.execute(cmd) + tdSql.query('select * from tb order by ts desc') + tdSql.checkRows(6) + if (tdSql.getData(0, 1) != 2): + tdLog.exit("data is not 2") + + tdLog.info("=============== step7") + cmd = "insert into tb values (now+6m, 2a'1)" + tdLog.info(cmd) + try: + tdSql.execute(cmd) + tdLog.exit( + "This test failed: insert wrong data error _not_ catched") + except Exception as e: + tdLog.info(repr(e)) + tdLog.notice("insert wrong data error catched") + + cmd = 'insert into tb values (now+6m, 2)' + tdLog.info(cmd) + tdSql.execute(cmd) + tdSql.query('select * from tb order by ts desc') + tdSql.checkRows(7) + if (tdSql.getData(0, 1) != 2): + tdLog.exit("data is not 2") + + tdLog.info("=============== step8") + cmd = 'insert into tb values (now+8m, "null")' + tdLog.info(cmd) + tdSql.execute(cmd) + tdSql.query('select * from tb order by ts desc') + tdSql.checkRows(8) + + if (tdSql.getData(0, 1) is not None): + tdLog.exit("data is not null") + + tdLog.info("=============== step9") + cmd = "insert into tb values (now+9m, 'null')" + tdLog.info(cmd) + tdSql.execute(cmd) + tdSql.query('select * from tb order by ts desc') + tdSql.checkRows(9) + if (tdSql.getData(0, 1) is not None): + tdLog.exit("data is not null") + + tdLog.info("=============== step10") + cmd = 'insert into tb values (now+10m, -123)' + tdLog.info(cmd) + tdSql.execute(cmd) + tdSql.query('select * from tb order by ts desc') + tdSql.checkRows(10) + + if (tdSql.getData(0, 1) != -123): + tdLog.exit("data is not -123") + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/query/__init__.py b/tests/pytest/query/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/tests/pytest/query/tbname.py b/tests/pytest/query/tbname.py new file mode 100644 index 0000000000000000000000000000000000000000..d2799efa2501c0a22d79497756c2adb87e3089f0 --- /dev/null +++ b/tests/pytest/query/tbname.py @@ -0,0 +1,75 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + tdSql.prepare() + + tdSql.execute('create table cars (ts timestamp, speed int) tags(id int)') + tdSql.execute("create table carzero using cars tags(0)") + tdSql.execute("create table carone using cars tags(1)") + tdSql.execute("create table cartwo using cars tags(2)") + + tdSql.execute("insert into carzero values(now, 100) carone values(now, 110)") + + tdSql.query("select * from cars where tbname in ('carzero', 'carone')") + tdSql.checkRows(2) + + tdSql.query("select * from cars where tbname in ('carzero', 'cartwo')") + tdSql.checkRows(1) + + tdSql.query("select * from cars where id=1 or tbname in ('carzero', 'cartwo')") + tdSql.checkRows(2) + + tdSql.query("select * from cars where id=1 and tbname in ('carzero', 'cartwo')") + tdSql.checkRows(0) + + tdSql.query("select * from cars where id=0 and tbname in ('carzero', 'cartwo')") + tdSql.checkRows(1) + + """ + tdSql.query("select * from cars where tbname like 'car%'") + tdSql.checkRows(2) + + tdSql.cursor.execute("use db") + tdSql.query("select * from cars where tbname like '%o'") + tdSql.checkRows(1) + + tdSql.query("select * from cars where id=1 and tbname like 'car%') + tdSql.checkRows(1) + + tdSql.query("select * from cars where id = 1 and tbname like '%o') + tdSql.checkRows(0) + + tdSql.query("select * from cars where id = 1 or tbname like '%o') + tdSql.checkRows(2) + """ + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/simpletest.sh b/tests/pytest/simpletest.sh index b77c1aa142413b74256b125f6c4a84485be66cff..58345e13ee8387c824fadb670f8fc99cd9e7f8a2 100755 --- a/tests/pytest/simpletest.sh +++ b/tests/pytest/simpletest.sh @@ -1,3 +1,9 @@ #!/bin/bash python3 ./test.py -f insert/basic.py $1 python3 ./test.py -s $1 +sleep 1 +python3 ./test.py -f insert/int.py $1 +python3 ./test.py -s $1 +sleep 1 +python3 ./test.py -f insert/float.py $1 +python3 ./test.py -s $1 diff --git a/tests/pytest/test.py b/tests/pytest/test.py index 55ec46e526c89fc1c6e99671a63670a167c58aa8..479406a00b728c1ea7cbb28255db2e6db11166b2 100644 --- a/tests/pytest/test.py +++ b/tests/pytest/test.py @@ -100,7 +100,7 @@ if __name__ == "__main__": tdDnodes.deploy(1) tdDnodes.start(1) conn = taos.connect( - host='192.168.0.1', + host='127.0.0.1', config=tdDnodes.getSimCfgPath()) if fileName == "all": tdCases.runAllLinux(conn) diff --git a/tests/pytest/util/cases.py b/tests/pytest/util/cases.py index 4bd1b1e9d1a0d734852584450d02b4ff91124d0f..1771c23620dbe547733e8c2ce317bd38961a87d3 100644 --- a/tests/pytest/util/cases.py +++ b/tests/pytest/util/cases.py @@ -67,12 +67,16 @@ class TDCases: if tmp.name.find(fileName) != -1: case = testModule.TDTestCase() case.init(conn) - case.run() + try: + case.run() + except Exception as e: + tdLog.notice(repr(e)) + tdLog.notice("%s failed: %s" % (__file__, fileName)) case.stop() runNum += 1 continue - tdLog.notice("total %d Linux test case(s) executed" % (runNum)) + tdLog.success("total %d Linux test case(s) executed" % (runNum)) def runAllWindows(self, conn): # TODO: load all Windows cases here diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index 6a9c2607e63f1d145af387198e1ac1fb077135ca..51eed3d25a6d8e7f116b43d3d006198f695170e8 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -19,12 +19,19 @@ from util.log import * class TDSimClient: + def __init__(self): + self.testCluster = False + def init(self, path): + self.__init__() self.path = path def getCfgDir(self): return self.cfgDir + def setTestCluster(self, value): + self.testCluster = value + def cfg(self, option, value): cmd = "echo '%s %s' >> %s" % (option, value, self.cfgPath) if os.system(cmd) != 0: @@ -55,8 +62,9 @@ class TDSimClient: if os.system(cmd) != 0: tdLog.exit(cmd) - self.cfg("masterIp", "192.168.0.1") - self.cfg("secondIp", "192.168.0.2") + if self.testCluster: + self.cfg("masterIp", "192.168.0.1") + self.cfg("secondIp", "192.168.0.2") self.cfg("logDir", self.logDir) self.cfg("numOfLogLines", "100000000") self.cfg("numOfThreadsPerCore", "2.0") @@ -128,11 +136,12 @@ class TDDnode: if self.testCluster: self.startIP() - self.cfg("masterIp", "192.168.0.1") - self.cfg("secondIp", "192.168.0.2") - self.cfg("publicIp", "192.168.0.%d" % (self.index)) - self.cfg("internalIp", "192.168.0.%d" % (self.index)) - self.cfg("privateIp", "192.168.0.%d" % (self.index)) + if self.testCluster: + self.cfg("masterIp", "192.168.0.1") + self.cfg("secondIp", "192.168.0.2") + self.cfg("publicIp", "192.168.0.%d" % (self.index)) + self.cfg("internalIp", "192.168.0.%d" % (self.index)) + self.cfg("privateIp", "192.168.0.%d" % (self.index)) self.cfg("dataDir", self.dataDir) self.cfg("logDir", self.logDir) self.cfg("numOfLogLines", "100000000") @@ -291,10 +300,6 @@ class TDDnodes: for i in range(len(self.dnodes)): self.dnodes[i].init(self.path) - self.sim = TDSimClient() - self.sim.init(self.path) - self.sim.deploy() - def setTestCluster(self, value): self.testCluster = value @@ -302,6 +307,11 @@ class TDDnodes: self.valgrind = value def deploy(self, index): + self.sim = TDSimClient() + self.sim.init(self.path) + self.sim.setTestCluster(self.testCluster) + self.sim.deploy() + self.check(index) self.dnodes[index - 1].setTestCluster(self.testCluster) self.dnodes[index - 1].setValgrind(self.valgrind) diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py index b4ac845bc899b575bcd8597d191cb8b6354fa4d7..00084681306da30e0a46333030a45c891c7780e4 100644 --- a/tests/pytest/util/sql.py +++ b/tests/pytest/util/sql.py @@ -63,7 +63,7 @@ class TDSql: def checkRows(self, expectRows): if self.queryRows != expectRows: tdLog.exit( - "sql:%.40s, queryRows:%d != expect:%d" % + "failed: sql:%.40s, queryRows:%d != expect:%d" % (self.sql, self.queryRows, expectRows)) tdLog.info("sql:%.40s, queryRows:%d == expect:%d" % (self.sql, self.queryRows, expectRows)) diff --git a/tests/test-all.sh b/tests/test-all.sh index dee89b9dc57da7e6a8292ee0aca82b329b8865f2..6943dd47a780c1de5cf431e9f45789af19412c09 100755 --- a/tests/test-all.sh +++ b/tests/test-all.sh @@ -25,15 +25,17 @@ if [ "$totalFailed" -ne "0" ]; then fi cd ../pytest -./simpletest.sh 2>&1 | grep 'successfully executed\|failed' | tee pytest-out.txt +./simpletest.sh 2>&1 | tee pytest-out.txt totalPySuccess=`grep 'successfully executed' pytest-out.txt | wc -l` if [ "$totalPySuccess" -gt "0" ]; then + grep 'successfully executed' pytest-out.txt echo -e "${GREEN} ### Total $totalPySuccess python case(s) succeed! ### ${NC}" fi -totalPyFailed=`grep 'failed' pytest-out.txt | wc -l` +totalPyFailed=`grep 'failed\|fault' pytest-out.txt | wc -l` if [ "$totalPyFailed" -ne "0" ]; then + cat pytest-out.txt echo -e "${RED} ### Total $totalPyFailed python case(s) failed! ### ${NC}" exit $totalPyFailed fi