From 01846142fa43d4fdd0d858e55e93fff2e48221f6 Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Mon, 11 May 2020 23:47:28 +0800 Subject: [PATCH] [td-225] fix bugs in tags filter query --- cmake/platform.inc | 2 +- src/client/src/tscUtil.c | 14 +++--- src/query/inc/qast.h | 4 +- src/query/src/qast.c | 26 +++++------ src/query/src/queryExecutor.c | 69 +++++++++++++---------------- src/query/src/queryFilterFunc.c | 20 ++++----- src/tsdb/src/tsdbMeta.c | 2 +- src/tsdb/src/tsdbRead.c | 4 +- src/util/src/tcompare.c | 17 +++++-- tests/script/general/tag/create.sim | 6 ++- 10 files changed, 84 insertions(+), 80 deletions(-) diff --git a/cmake/platform.inc b/cmake/platform.inc index 2e0e2d6af0..1772a83fb2 100755 --- a/cmake/platform.inc +++ b/cmake/platform.inc @@ -62,7 +62,7 @@ IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux") IF (${CMAKE_SIZEOF_VOID_P} MATCHES 8) SET(TD_LINUX_64 TRUE) SET(TD_OS_DIR ${TD_COMMUNITY_DIR}/src/os/linux) - ADD_DEFINITIONS(-D_M_X64) + ADD_DEFINITIONS(-D_M_X64 -D_DEBUG_VIEW) MESSAGE(STATUS "The current platform is Linux 64-bit") ELSEIF (${CMAKE_SIZEOF_VOID_P} MATCHES 4) IF (TD_ARM) diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 00afb977fd..c25e91a669 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1144,23 +1144,21 @@ SColumn* tscColumnListInsert(SArray* pColumnList, SColumnIndex* pColIndex) { } SColumnFilterInfo* tscFilterInfoClone(const SColumnFilterInfo* src, int32_t numOfFilters) { - SColumnFilterInfo* pFilter = NULL; - if (numOfFilters > 0) { - pFilter = calloc(1, numOfFilters * sizeof(SColumnFilterInfo)); - } else { + if (numOfFilters == 0) { assert(src == NULL); return NULL; } + SColumnFilterInfo* pFilter = calloc(1, numOfFilters * sizeof(SColumnFilterInfo)); + memcpy(pFilter, src, sizeof(SColumnFilterInfo) * numOfFilters); for (int32_t j = 0; j < numOfFilters; ++j) { + if (pFilter[j].filterstr) { size_t len = (size_t) pFilter[j].len + 1; - - char* pTmp = calloc(1, len); - pFilter[j].pz = (int64_t) pTmp; + pFilter[j].pz = (int64_t) calloc(1, len); - memcpy((char*)pFilter[j].pz, (char*)src->pz, (size_t)len); + memcpy((char*)pFilter[j].pz, (char*)src[j].pz, (size_t)len); } } diff --git a/src/query/inc/qast.h b/src/query/inc/qast.h index 9bc36413de..410b2ac9d2 100644 --- a/src/query/inc/qast.h +++ b/src/query/inc/qast.h @@ -50,11 +50,11 @@ typedef struct tQueryInfo { SSchema sch; // schema of tags char* q; __compar_fn_t compare; // filter function - void* param; // STSchema, + void* param; // STSchema } tQueryInfo; typedef struct SExprTraverseSupp { - __result_filter_fn_t fp; + __result_filter_fn_t nodeFilterFn; __do_filter_suppl_fn_t setupInfoFn; void * pExtInfo; } SExprTraverseSupp; diff --git a/src/query/src/qast.c b/src/query/src/qast.c index 43bdf57ed7..f727acb720 100644 --- a/src/query/src/qast.c +++ b/src/query/src/qast.c @@ -760,7 +760,7 @@ static bool filterItem(tExprNode *pExpr, const void *pItem, SExprTraverseSupp *p assert(pLeft->nodeType == TSQL_NODE_COL && pRight->nodeType == TSQL_NODE_VALUE); param->setupInfoFn(pExpr, param->pExtInfo); - return param->fp(pItem, pExpr->_node.info); + return param->nodeFilterFn(pItem, pExpr->_node.info); } /** @@ -801,33 +801,33 @@ static void tSQLBinaryTraverseOnSkipList(tExprNode *pExpr, SArray *pResult, SSki tSkipListDestroyIter(iter); } -static void tQueryIndexlessColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArray* result) { +static void tQueryIndexlessColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArray* res, __result_filter_fn_t filterFp) { SSkipListIterator* iter = tSkipListCreateIter(pSkipList); while (tSkipListIterNext(iter)) { bool addToResult = false; SSkipListNode *pNode = tSkipListIterGet(iter); - char* pTable = SL_GET_NODE_DATA(pNode); - - //todo refactor: - tstr* name = ((STableIndexElem*) pTable)->pTable->name; -// char* name = NULL; -// tsdbGetTableName(tsdb, pTable, &name); + char * pData = SL_GET_NODE_DATA(pNode); + + // todo refactor: + tstr *name = ((STableIndexElem *)pData)->pTable->name; + // char* name = NULL; +// tsdbGetTableName(pQueryInfo->, pTable, &name); // todo speed up by using hash if (pQueryInfo->colIndex == TSDB_TBNAME_COLUMN_INDEX) { if (pQueryInfo->optr == TSDB_RELATION_IN) { addToResult = pQueryInfo->compare(name, pQueryInfo->q); - } else if(pQueryInfo->optr == TSDB_RELATION_LIKE) { + } else if (pQueryInfo->optr == TSDB_RELATION_LIKE) { addToResult = !pQueryInfo->compare(name, pQueryInfo->q); } } else { - // TODO: other columns + addToResult = filterFp(pNode, pQueryInfo); } if (addToResult) { - taosArrayPush(result, pTable); + taosArrayPush(res, pData); } } @@ -851,7 +851,7 @@ void tExprTreeTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, S param->setupInfoFn(pExpr, param->pExtInfo); if (pSkipList == NULL) { - tArrayTraverse(pExpr, param->fp, result); + tArrayTraverse(pExpr, param->nodeFilterFn, result); return; } @@ -859,7 +859,7 @@ void tExprTreeTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, S if (pQueryInfo->colIndex == 0 && pQueryInfo->optr != TSDB_RELATION_LIKE) { tQueryIndexColumn(pSkipList, pQueryInfo, result); } else { - tQueryIndexlessColumn(pSkipList, pQueryInfo, result); + tQueryIndexlessColumn(pSkipList, pQueryInfo, result, param->nodeFilterFn); } return; diff --git a/src/query/src/queryExecutor.c b/src/query/src/queryExecutor.c index 52cfa582d0..5977b8bea7 100644 --- a/src/query/src/queryExecutor.c +++ b/src/query/src/queryExecutor.c @@ -2641,26 +2641,21 @@ static UNUSED_FUNC void printBinaryData(int32_t functionId, char *data, int32_t } } -void UNUSED_FUNC displayInterResult(SData **pdata, SQuery *pQuery, int32_t numOfRows) { -#if 0 +void UNUSED_FUNC displayInterResult(SData **pdata, SQueryRuntimeEnv* pRuntimeEnv, int32_t numOfRows) { + SQuery* pQuery = pRuntimeEnv->pQuery; int32_t numOfCols = pQuery->numOfOutput; printf("super table query intermediate result, total:%d\n", numOfRows); - SQInfo * pQInfo = (SQInfo *)(GET_QINFO_ADDR(pQuery)); - SMeterObj *pMeterObj = pQInfo->pObj; - for (int32_t j = 0; j < numOfRows; ++j) { for (int32_t i = 0; i < numOfCols; ++i) { + switch (pQuery->pSelectExpr[i].type) { case TSDB_DATA_TYPE_BINARY: { - int32_t colIndex = pQuery->pSelectExpr[i].base.colInfo.colIndex; - int32_t type = 0; - - if (TSDB_COL_IS_TAG(pQuery->pSelectExpr[i].base.colInfo.flag)) { - type = pQuery->pSelectExpr[i].type; - } else { - type = pMeterObj->schema[colIndex].type; - } +// int32_t colIndex = pQuery->pSelectExpr[i].base.colInfo.colIndex; + int32_t type = pQuery->pSelectExpr[i].type; +// } else { +// type = pMeterObj->schema[colIndex].type; +// } printBinaryData(pQuery->pSelectExpr[i].base.functionId, pdata[i]->data + pQuery->pSelectExpr[i].bytes * j, type); break; @@ -2682,7 +2677,6 @@ void UNUSED_FUNC displayInterResult(SData **pdata, SQuery *pQuery, int32_t numOf } printf("\n"); } -#endif } typedef struct SCompSupporter { @@ -2950,7 +2944,7 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { int64_t endt = taosGetTimestampMs(); #ifdef _DEBUG_VIEW - displayInterResult(pQuery->sdata, pQuery, pQuery->sdata[0]->num); + displayInterResult(pQuery->sdata, pRuntimeEnv, pQuery->sdata[0]->num); #endif qTrace("QInfo:%p result merge completed, elapsed time:%" PRId64 " ms", GET_QINFO_ADDR(pQuery), endt - startt); @@ -3588,7 +3582,8 @@ void setIntervalQueryRange(SQInfo *pQInfo, TSKEY key) { if (pTableQueryInfo->queryRangeSet) { pTableQueryInfo->lastKey = key; } else { - pQuery->window.skey = key; +// pQuery->window.skey = key; + pTableQueryInfo->win.skey = key; STimeWindow win = {.skey = key, .ekey = pQuery->window.ekey}; // for too small query range, no data in this interval. @@ -3608,7 +3603,7 @@ void setIntervalQueryRange(SQInfo *pQInfo, TSKEY key) { SWindowResInfo *pWindowResInfo = &pTableQueryInfo->windowResInfo; getAlignQueryTimeWindow(pQuery, win.skey, win.skey, win.ekey, &skey1, &ekey1, &w); - pWindowResInfo->startTime = pQuery->window.skey; // windowSKey may be 0 in case of 1970 timestamp + pWindowResInfo->startTime = pTableQueryInfo->win.skey; // windowSKey may be 0 in case of 1970 timestamp if (pWindowResInfo->prevSKey == 0) { if (QUERY_IS_ASC_QUERY(pQuery)) { @@ -3620,8 +3615,8 @@ void setIntervalQueryRange(SQInfo *pQInfo, TSKEY key) { } pTableQueryInfo->queryRangeSet = 1; - pTableQueryInfo->lastKey = pQuery->window.skey; - pTableQueryInfo->win.skey = pQuery->window.skey; + pTableQueryInfo->lastKey = pTableQueryInfo->win.skey; + pTableQueryInfo->win.skey = pTableQueryInfo->win.skey; } } @@ -3726,7 +3721,7 @@ static int32_t doCopyToSData(SQInfo *pQInfo, SWindowResult *result, int32_t orde qTrace("QInfo:%p copy data to query buf completed", pQInfo); #ifdef _DEBUG_VIEW - displayInterResult(pQuery->sdata, pQuery, numOfResult); + displayInterResult(pQuery->sdata, pRuntimeEnv, numOfResult); #endif return numOfResult; } @@ -4766,7 +4761,7 @@ static void multiTableQueryProcess(SQInfo *pQInfo) { copyResToQueryResultBuf(pQInfo, pQuery); #ifdef _DEBUG_VIEW - displayInterResult(pQuery->sdata, pQuery, pQuery->sdata[0]->num); + displayInterResult(pQuery->sdata, pRuntimeEnv, pQuery->sdata[0]->num); #endif } else { copyFromWindowResToSData(pQInfo, pRuntimeEnv->windowResInfo.pResult); @@ -4819,7 +4814,7 @@ static void multiTableQueryProcess(SQInfo *pQInfo) { copyResToQueryResultBuf(pQInfo, pQuery); #ifdef _DEBUG_VIEW - displayInterResult(pQuery->sdata, pQuery, pQuery->sdata[0]->num); + displayInterResult(pQuery->sdata, pRuntimeEnv, pQuery->sdata[0]->num); #endif } } else { // not a interval query @@ -5276,26 +5271,26 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, } for (int32_t f = 0; f < numOfFilters; ++f) { - SColumnFilterInfo *pFilterInfo = (SColumnFilterInfo *)pMsg; - SColumnFilterInfo *pDestFilterInfo = &pColInfo->filters[f]; - - pDestFilterInfo->filterstr = htons(pFilterInfo->filterstr); + SColumnFilterInfo *pFilterMsg = (SColumnFilterInfo *)pMsg; + + SColumnFilterInfo *pColFilter = &pColInfo->filters[f]; + pColFilter->filterstr = htons(pFilterMsg->filterstr); pMsg += sizeof(SColumnFilterInfo); - if (pDestFilterInfo->filterstr) { - pDestFilterInfo->len = htobe64(pFilterInfo->len); + if (pColFilter->filterstr) { + pColFilter->len = htobe64(pFilterMsg->len); - pDestFilterInfo->pz = (int64_t) calloc(1, pDestFilterInfo->len); - memcpy((void *)pDestFilterInfo->pz, pMsg, pDestFilterInfo->len); - pMsg += (pDestFilterInfo->len); + pColFilter->pz = (int64_t) calloc(1, pColFilter->len); + memcpy((void *)pColFilter->pz, pMsg, pColFilter->len); + pMsg += (pColFilter->len + 1); } else { - pDestFilterInfo->lowerBndi = htobe64(pFilterInfo->lowerBndi); - pDestFilterInfo->upperBndi = htobe64(pFilterInfo->upperBndi); + pColFilter->lowerBndi = htobe64(pFilterMsg->lowerBndi); + pColFilter->upperBndi = htobe64(pFilterMsg->upperBndi); } - pDestFilterInfo->lowerRelOptr = htons(pFilterInfo->lowerRelOptr); - pDestFilterInfo->upperRelOptr = htons(pFilterInfo->upperRelOptr); + pColFilter->lowerRelOptr = htons(pFilterMsg->lowerRelOptr); + pColFilter->upperRelOptr = htons(pFilterMsg->upperRelOptr); } } @@ -5692,9 +5687,7 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou for (int16_t i = 0; i < numOfCols; ++i) { pQuery->colList[i] = pQueryMsg->colList[i]; - - SColumnInfo *pColInfo = &pQuery->colList[i]; - pColInfo->filters = tscFilterInfoClone(pQueryMsg->colList[i].filters, pColInfo->numOfFilters); + pQuery->colList[i].filters = tscFilterInfoClone(pQueryMsg->colList[i].filters, pQuery->colList[i].numOfFilters); } pQuery->tagColList = pTagCols; diff --git a/src/query/src/queryFilterFunc.c b/src/query/src/queryFilterFunc.c index 3218b26179..41a888e92d 100644 --- a/src/query/src/queryFilterFunc.c +++ b/src/query/src/queryFilterFunc.c @@ -183,33 +183,33 @@ bool equal_dd(SColumnFilterElem *pFilter, char *minval, char *maxval) { bool equal_str(SColumnFilterElem *pFilter, char *minval, char *maxval) { // query condition string is greater than the max length of string, not qualified data - if (pFilter->filterInfo.len > pFilter->bytes) { + if (pFilter->filterInfo.len != varDataLen(minval)) { return false; } - return strncmp((char *)pFilter->filterInfo.pz, minval, pFilter->bytes) == 0; + return strncmp((char *)pFilter->filterInfo.pz, varDataVal(minval), varDataLen(minval)) == 0; } bool equal_nchar(SColumnFilterElem *pFilter, char *minval, char *maxval) { // query condition string is greater than the max length of string, not qualified data - if (pFilter->filterInfo.len > pFilter->bytes) { + if (pFilter->filterInfo.len != varDataLen(minval)) { return false; } - return wcsncmp((wchar_t *)pFilter->filterInfo.pz, (wchar_t*) minval, pFilter->bytes/TSDB_NCHAR_SIZE) == 0; + return wcsncmp((wchar_t *)pFilter->filterInfo.pz, varDataVal(minval), varDataLen(minval)/TSDB_NCHAR_SIZE) == 0; } //////////////////////////////////////////////////////////////// bool like_str(SColumnFilterElem *pFilter, char *minval, char *maxval) { SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER; - return patternMatch((char *)pFilter->filterInfo.pz, minval, pFilter->bytes, &info) == TSDB_PATTERN_MATCH; + return patternMatch((char *)pFilter->filterInfo.pz, varDataVal(minval), varDataLen(minval), &info) == TSDB_PATTERN_MATCH; } bool like_nchar(SColumnFilterElem* pFilter, char* minval, char *maxval) { SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER; - - return WCSPatternMatch((wchar_t*) pFilter->filterInfo.pz, (wchar_t*) minval, pFilter->bytes/TSDB_NCHAR_SIZE, &info) == TSDB_PATTERN_MATCH; + + return WCSPatternMatch((wchar_t*) pFilter->filterInfo.pz, varDataVal(minval), varDataLen(minval)/TSDB_NCHAR_SIZE, &info) == TSDB_PATTERN_MATCH; } //////////////////////////////////////////////////////////////// @@ -270,11 +270,11 @@ bool nequal_dd(SColumnFilterElem *pFilter, char *minval, char *maxval) { } bool nequal_str(SColumnFilterElem *pFilter, char *minval, char *maxval) { - if (pFilter->filterInfo.len > pFilter->bytes) { + if (pFilter->filterInfo.len != varDataLen(minval)) { return true; } - return strncmp((char *)pFilter->filterInfo.pz, minval, pFilter->bytes) != 0; + return strncmp((char *)pFilter->filterInfo.pz, varDataVal(minval), varDataLen(minval)) != 0; } bool nequal_nchar(SColumnFilterElem *pFilter, char* minval, char *maxval) { @@ -282,7 +282,7 @@ bool nequal_nchar(SColumnFilterElem *pFilter, char* minval, char *maxval) { return true; } - return wcsncmp((wchar_t *)pFilter->filterInfo.pz, (wchar_t*)minval, pFilter->bytes/TSDB_NCHAR_SIZE) != 0; + return wcsncmp((wchar_t *)pFilter->filterInfo.pz, varDataVal(minval), varDataLen(minval)/TSDB_NCHAR_SIZE) != 0; } //////////////////////////////////////////////////////////////// diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index c7e260ae50..7ae7890695 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -234,7 +234,6 @@ STSchema * tsdbGetTableTagSchema(STsdbMeta *pMeta, STable *pTable) { } } -// todo refactor table name definition int32_t tsdbGetTableTagVal(TsdbRepoT* repo, STableId* id, int32_t colId, int16_t* type, int16_t* bytes, char** val) { STsdbMeta* pMeta = tsdbGetMeta(repo); STable* pTable = tsdbGetTableByUid(pMeta, id->uid); @@ -242,6 +241,7 @@ int32_t tsdbGetTableTagVal(TsdbRepoT* repo, STableId* id, int32_t colId, int16_t STSchema* pSchema = tsdbGetTableTagSchema(pMeta, pTable); STColumn* pCol = NULL; + // todo binary search for(int32_t col = 0; col < schemaNCols(pSchema); ++col) { STColumn* p = schemaColAt(pSchema, col); if (p->colId == colId) { diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 49b6d2984f..84615ff8c2 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -1432,7 +1432,7 @@ SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pC return pTableGroup; } -bool tSkipListNodeFilterCallback(const void* pNode, void* param) { +bool indexedNodeFilterFp(const void* pNode, void* param) { tQueryInfo* pInfo = (tQueryInfo*) param; STableIndexElem* elem = (STableIndexElem*)(SL_GET_NODE_DATA((SSkipListNode*)pNode)); @@ -1497,7 +1497,7 @@ bool tSkipListNodeFilterCallback(const void* pNode, void* param) { static int32_t doQueryTableList(STable* pSTable, SArray* pRes, tExprNode* pExpr) { // query according to the expression tree SExprTraverseSupp supp = { - .fp = (__result_filter_fn_t) tSkipListNodeFilterCallback, + .nodeFilterFn = (__result_filter_fn_t) indexedNodeFilterFp, .setupInfoFn = filterPrepare, .pExtInfo = pSTable->tagSchema, }; diff --git a/src/util/src/tcompare.c b/src/util/src/tcompare.c index 0abf1e6be3..43caca3a7c 100644 --- a/src/util/src/tcompare.c +++ b/src/util/src/tcompare.c @@ -51,12 +51,21 @@ int32_t compareDoubleIntVal(const void *pLeft, const void *pRight) { } } +int32_t compareFloatVal(const void *pLeft, const void *pRight) { + float ret = GET_FLOAT_VAL(pLeft) - GET_FLOAT_VAL(pRight); + if (fabs(ret) < FLT_EPSILON) { + return 0; + } else { + return ret > 0? 1 : -1; + } +} + int32_t compareDoubleVal(const void *pLeft, const void *pRight) { double ret = GET_DOUBLE_VAL(pLeft) - GET_DOUBLE_VAL(pRight); if (fabs(ret) < FLT_EPSILON) { return 0; } else { - return ret > 0 ? 1 : -1; + return ret > 0? 1 : -1; } } @@ -175,7 +184,7 @@ int WCSPatternMatch(const wchar_t *patterStr, const wchar_t *str, size_t size, c wchar_t accept[3] = {towupper(c), towlower(c), 0}; while (1) { - size_t n = wcsspn(str, accept); + size_t n = wcscspn(str, accept); str += n; if (str[0] == 0 || (n >= size - 1)) { @@ -257,7 +266,7 @@ __compar_fn_t getComparFunc(int32_t type, int32_t optr) { } case TSDB_DATA_TYPE_FLOAT: { - comparFn = compareDoubleVal; break; + comparFn = compareFloatVal; break; } case TSDB_DATA_TYPE_DOUBLE: { @@ -313,6 +322,8 @@ __compar_fn_t getKeyComparFunc(int32_t keyType) { comparFn = compareInt64Val; break; case TSDB_DATA_TYPE_FLOAT: + comparFn = compareFloatVal; + break; case TSDB_DATA_TYPE_DOUBLE: comparFn = compareDoubleVal; break; diff --git a/tests/script/general/tag/create.sim b/tests/script/general/tag/create.sim index c0cdbe87dc..3e16810f91 100644 --- a/tests/script/general/tag/create.sim +++ b/tests/script/general/tag/create.sim @@ -134,7 +134,8 @@ if $data01 != 1 then return -1 endi sql select * from $mt where tgcol = 0 -if $rows != 0 then +if $rows != 0 then + print expect 0, actual: $rows return -1 endi @@ -184,7 +185,8 @@ sql create table $mt (ts timestamp, tbcol int) TAGS(tgcol bool, tgcol2 bool) sql create table $tb using $mt tags( 1, 2 ) sql insert into $tb values(now, 1) sql select * from $mt where tgcol2 = 2 -if $rows != 1 then +if $rows != 1 then + print expect 1, actual: $rows return -1 endi if $data01 != 1 then -- GitLab