diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 125e28f17778baead6b2be8d8409f2259d7d77e5..0750abdf6ff991d8c19eb63b77c5ba9ff9ddfb03 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -762,35 +762,32 @@ void tscSortRemoveDataBlockDupRowsRaw(STableDataBlocks *dataBuf) { if (!dataBuf->ordered) { char *pBlockData = pBlocks->data; qsort(pBlockData, pBlocks->numOfRows, dataBuf->rowSize, rowDataCompar); + dataBuf->ordered = true; - int32_t i = 0; - int32_t j = 1; - - while (j < pBlocks->numOfRows) { - TSKEY ti = *(TSKEY *)(pBlockData + dataBuf->rowSize * i); - TSKEY tj = *(TSKEY *)(pBlockData + dataBuf->rowSize * j); + if(tsClientMerge) { + int32_t i = 0; + int32_t j = 1; + while (j < pBlocks->numOfRows) { + TSKEY ti = *(TSKEY *)(pBlockData + dataBuf->rowSize * i); + TSKEY tj = *(TSKEY *)(pBlockData + dataBuf->rowSize * j); - if (ti == tj) { - if (dataBuf->pTableMeta && dataBuf->pTableMeta->tableInfo.update != TD_ROW_DISCARD_UPDATE) { - memmove(pBlockData + dataBuf->rowSize * i, pBlockData + dataBuf->rowSize * j, dataBuf->rowSize); + if (ti == tj) { + if (dataBuf->pTableMeta && dataBuf->pTableMeta->tableInfo.update != TD_ROW_DISCARD_UPDATE) { + memmove(pBlockData + dataBuf->rowSize * i, pBlockData + dataBuf->rowSize * j, dataBuf->rowSize); + } + ++j; + continue; } + int32_t nextPos = (++i); + if (nextPos != j) { + memmove(pBlockData + dataBuf->rowSize * nextPos, pBlockData + dataBuf->rowSize * j, dataBuf->rowSize); + } ++j; - continue; - } - - int32_t nextPos = (++i); - if (nextPos != j) { - memmove(pBlockData + dataBuf->rowSize * nextPos, pBlockData + dataBuf->rowSize * j, dataBuf->rowSize); - } - - ++j; + } + pBlocks->numOfRows = i + 1; + dataBuf->size = sizeof(SSubmitBlk) + dataBuf->rowSize * pBlocks->numOfRows; } - - dataBuf->ordered = true; - - pBlocks->numOfRows = i + 1; - dataBuf->size = sizeof(SSubmitBlk) + dataBuf->rowSize * pBlocks->numOfRows; } dataBuf->prevTS = INT64_MIN; @@ -836,32 +833,33 @@ int tscSortRemoveDataBlockDupRows(STableDataBlocks *dataBuf, SBlockKeyInfo *pBlk if (!dataBuf->ordered) { pBlkKeyTuple = pBlkKeyInfo->pKeyTuple; qsort(pBlkKeyTuple, nRows, sizeof(SBlockKeyTuple), rowDataCompar); + dataBuf->ordered = true; - pBlkKeyTuple = pBlkKeyInfo->pKeyTuple; - int32_t i = 0; - int32_t j = 1; - while (j < nRows) { - TSKEY ti = (pBlkKeyTuple + i)->skey; - TSKEY tj = (pBlkKeyTuple + j)->skey; - - if (ti == tj) { - if (dataBuf->pTableMeta && dataBuf->pTableMeta->tableInfo.update != TD_ROW_DISCARD_UPDATE) { - memmove(pBlkKeyTuple + i, pBlkKeyTuple + j, sizeof(SBlockKeyTuple)); + if(tsClientMerge) { + pBlkKeyTuple = pBlkKeyInfo->pKeyTuple; + int32_t i = 0; + int32_t j = 1; + while (j < nRows) { + TSKEY ti = (pBlkKeyTuple + i)->skey; + TSKEY tj = (pBlkKeyTuple + j)->skey; + + if (ti == tj) { + if (dataBuf->pTableMeta && dataBuf->pTableMeta->tableInfo.update != TD_ROW_DISCARD_UPDATE) { + memmove(pBlkKeyTuple + i, pBlkKeyTuple + j, sizeof(SBlockKeyTuple)); + } + + ++j; + continue; } + int32_t nextPos = (++i); + if (nextPos != j) { + memmove(pBlkKeyTuple + nextPos, pBlkKeyTuple + j, sizeof(SBlockKeyTuple)); + } ++j; - continue; - } - - int32_t nextPos = (++i); - if (nextPos != j) { - memmove(pBlkKeyTuple + nextPos, pBlkKeyTuple + j, sizeof(SBlockKeyTuple)); } - ++j; - } - - dataBuf->ordered = true; - pBlocks->numOfRows = i + 1; + pBlocks->numOfRows = i + 1; + } } dataBuf->size = sizeof(SSubmitBlk) + pBlocks->numOfRows * extendedRowSize; diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 2103c5c8ededaee59b22f9a024b71d57ff241703..4cfec94f704d2821e9400861a181369743067185 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -5423,10 +5423,14 @@ int32_t validateFillNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSqlNo const char* msg4 = "illegal value or data overflow"; const char* msg5 = "fill only available for interval query"; const char* msg6 = "not supported function now"; + const char* msg7 = "join query not supported fill operation"; if ((!isTimeWindowQuery(pQueryInfo)) && (!tscIsPointInterpQuery(pQueryInfo))) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); } + if(QUERY_IS_JOIN_QUERY(pQueryInfo->type)) { + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg7); + } /* * fill options are set at the end position, when all columns are set properly diff --git a/src/common/inc/tglobal.h b/src/common/inc/tglobal.h index a384cf6e70485b8f7d1b06b7f7e86ba92776b547..8a6cc9ddb5a4425e3d7bd10bb7dd5dda0712b0c3 100644 --- a/src/common/inc/tglobal.h +++ b/src/common/inc/tglobal.h @@ -209,6 +209,8 @@ extern int32_t wDebugFlag; extern int32_t cqDebugFlag; extern int32_t debugFlag; +extern int8_t tsClientMerge; + #ifdef TD_TSZ // lossy extern char lossyColumns[]; @@ -219,6 +221,7 @@ extern uint32_t curRange; extern char Compressor[]; #endif + typedef struct { char dir[TSDB_FILENAME_LEN]; int level; diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index 876eb4fa92bada1d22e0e43b6d6532d31a0b913c..f1a7e4f7fd12a85ed1b8b503db35fba7211fc788 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -254,6 +254,8 @@ int32_t tsdbDebugFlag = 131; int32_t cqDebugFlag = 131; int32_t fsDebugFlag = 135; +int8_t tsClientMerge = 0; + #ifdef TD_TSZ // // lossy compress 6 @@ -1581,6 +1583,16 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); + cfg.option = "clientMerge"; + cfg.ptr = &tsClientMerge; + cfg.valType = TAOS_CFG_VTYPE_INT8; + cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; + cfg.minValue = 0; + cfg.maxValue = 1; + cfg.ptrLength = 1; + cfg.unitType = TAOS_CFG_UTYPE_NONE; + taosInitConfigOption(cfg); + #ifdef TD_TSZ // lossy compress cfg.option = "lossyColumns"; diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index ae4db6bdb8a7887e7adc7bf15354a104f1a903fa..1d90fcf1ac2d021e5d6063ea5d5b1aa03c8525e2 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -1296,9 +1296,6 @@ static void projectApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx if (pCtx[k].currentStage == MERGE_STAGE) { pCtx[k].order = TSDB_ORDER_ASC; } - - pCtx[k].startTs = pQueryAttr->window.skey; - if (pCtx[k].functionId < 0) { // load the script and exec SUdfInfo* pUdfInfo = pRuntimeEnv->pUdfInfo; diff --git a/src/util/inc/tconfig.h b/src/util/inc/tconfig.h index 5bd69aece2ffa2166c268b437549c95a30608383..d6a1cb113a43874ee77b096e75959e02e4aa6747 100644 --- a/src/util/inc/tconfig.h +++ b/src/util/inc/tconfig.h @@ -20,7 +20,7 @@ extern "C" { #endif -#define TSDB_CFG_MAX_NUM 119 +#define TSDB_CFG_MAX_NUM 120 #define TSDB_CFG_PRINT_LEN 23 #define TSDB_CFG_OPTION_LEN 24 #define TSDB_CFG_VALUE_LEN 41 diff --git a/src/util/src/tskiplist.c b/src/util/src/tskiplist.c index 98fd9c094cba3e779c9f203fdacc548a3bda5ef4..2bdaeb07a16a638d75c1c5a8b20763e5c660a05f 100644 --- a/src/util/src/tskiplist.c +++ b/src/util/src/tskiplist.c @@ -144,7 +144,6 @@ void tSkipListPutBatchByIter(SSkipList *pSkipList, void *iter, iter_next_fn_t it // backward to put the first data hasDup = tSkipListGetPosToPut(pSkipList, backward, pData); - tSkipListPutImpl(pSkipList, pData, backward, false, hasDup); for (int level = 0; level < pSkipList->maxLevel; level++) { @@ -163,7 +162,12 @@ void tSkipListPutBatchByIter(SSkipList *pSkipList, void *iter, iter_next_fn_t it for (int i = 0; i < pSkipList->maxLevel; i++) { forward[i] = SL_NODE_GET_BACKWARD_POINTER(pSkipList->pTail, i); } + } else if(compare == 0) { + // same need special deal + forward[0] = SL_NODE_GET_BACKWARD_POINTER(SL_NODE_GET_BACKWARD_POINTER(pSkipList->pTail,0),0); + hasDup = true; } else { + SSkipListNode *p = NULL; SSkipListNode *px = pSkipList->pHead; for (int i = pSkipList->maxLevel - 1; i >= 0; --i) { if (i < pSkipList->level) { @@ -175,19 +179,29 @@ void tSkipListPutBatchByIter(SSkipList *pSkipList, void *iter, iter_next_fn_t it } } - SSkipListNode *p = SL_NODE_GET_FORWARD_POINTER(px, i); + // if px not head , must compare with px + if(px == pSkipList->pHead) { + p = SL_NODE_GET_FORWARD_POINTER(px, i); + } else { + p = px; + } while (p != pSkipList->pTail) { pKey = SL_GET_NODE_KEY(pSkipList, p); compare = pSkipList->comparFn(pKey, pDataKey); if (compare >= 0) { - if (compare == 0 && !hasDup) hasDup = true; + if (compare == 0) { + hasDup = true; + forward[0] = SL_NODE_GET_BACKWARD_POINTER(p, 0); + } break; } else { px = p; p = SL_NODE_GET_FORWARD_POINTER(px, i); } } + // if found duplicate, immediately break, needn't continue to loop set rest forward[i] value + if(hasDup) break; } forward[i] = px; diff --git a/tests/pytest/query/nestedQuery/nestedQuery.py b/tests/pytest/query/nestedQuery/nestedQuery.py index 453ee8f53975509c318486242c634d3b60de4992..545f6429e825c468bdb07524329d6ea49944e379 100755 --- a/tests/pytest/query/nestedQuery/nestedQuery.py +++ b/tests/pytest/query/nestedQuery/nestedQuery.py @@ -1714,7 +1714,6 @@ class TDTestCase: sql += " from regular_table_1 t1, regular_table_2 t2 where t1.ts = t2.ts and " sql += "%s " % random.choice(q_u_where) sql += "%s " % random.choice(session_u_where) - sql += "%s " % random.choice(fill_where) sql += "%s " % random.choice(order_u_where) sql += "%s " % random.choice(limit_u_where) sql += ") " @@ -1731,7 +1730,6 @@ class TDTestCase: sql += " from regular_table_1 t1, regular_table_2 t2 where t1.ts = t2.ts and " sql += "%s " % random.choice(q_u_or_where) sql += "%s " % random.choice(session_u_where) - sql += "%s " % random.choice(fill_where) sql += "%s " % random.choice(order_u_where) sql += "%s " % random.choice(limit_u_where) sql += ") " @@ -1767,7 +1765,6 @@ class TDTestCase: sql += " from table_1 t1, regular_table_2 t2 where t1.ts = t2.ts and " sql += "%s " % random.choice(q_u_where) sql += "%s " % random.choice(session_u_where) - sql += "%s " % random.choice(fill_where) sql += "%s " % random.choice(order_u_where) sql += "%s " % random.choice(limit_u_where) sql += ") " @@ -1784,7 +1781,6 @@ class TDTestCase: sql += " from table_1 t1, regular_table_2 t2 where t1.ts = t2.ts and " sql += "%s " % random.choice(q_u_or_where) sql += "%s " % random.choice(session_u_where) - sql += "%s " % random.choice(fill_where) sql += "%s " % random.choice(order_u_where) sql += "%s " % random.choice(limit_u_where) sql += ") " @@ -1818,7 +1814,6 @@ class TDTestCase: sql += " from stable_1 t1, stable_2 t2 where t1.ts = t2.ts and " sql += "%s " % random.choice(t_join_where) sql += "%s " % random.choice(session_u_where) - sql += "%s " % random.choice(fill_where) sql += "%s " % random.choice(order_u_where) sql += "%s " % random.choice(limit_u_where) sql += ") " @@ -1835,7 +1830,6 @@ class TDTestCase: sql += " from stable_1 t1, stable_2 t2 where t1.ts = t2.ts and " sql += "%s " % random.choice(qt_u_or_where) sql += "%s " % random.choice(session_u_where) - sql += "%s " % random.choice(fill_where) sql += "%s " % random.choice(order_u_where) sql += "%s " % random.choice(limit_u_where) sql += ") " @@ -2015,7 +2009,6 @@ class TDTestCase: sql += " from stable_1 t1 , stable_2 t2 where t1.ts = t2.ts and " sql += "%s and " % random.choice(t_join_where) sql += "%s " % random.choice(interp_where_j) - sql += "%s " % random.choice(fill_where) sql += "%s " % random.choice(order_u_where) sql += "%s " % random.choice(limit_u_where) sql += ") " @@ -2032,7 +2025,6 @@ class TDTestCase: sql += " from stable_1 t1 , stable_2 t2 where t1.ts = t2.ts and " sql += "%s and " % random.choice(qt_u_or_where) sql += "%s " % random.choice(interp_where_j) - sql += "%s " % random.choice(fill_where) sql += "%s " % random.choice(order_u_where) sql += "%s " % random.choice(limit_u_where) sql += ") " @@ -2065,7 +2057,6 @@ class TDTestCase: sql += " from table_0 t1, table_1 t2 where t1.ts = t2.ts and " #sql += "%s and " % random.choice(t_join_where) sql += "%s " % interp_where_j[random.randint(0,5)] - sql += "%s " % random.choice(fill_where) sql += "%s " % random.choice(order_u_where) sql += "%s " % random.choice(limit_u_where) sql += ") " @@ -2116,7 +2107,6 @@ class TDTestCase: sql += " from regular_table_1 t1, regular_table_2 t2 where t1.ts = t2.ts and " #sql += "%s " % random.choice(interp_where_j) sql += "%s " % interp_where_j[random.randint(0,5)] - sql += "%s " % random.choice(fill_where) sql += "%s " % random.choice(order_u_where) sql += "%s " % random.choice(limit_u_where) sql += ") " diff --git a/tests/script/general/parser/join.sim b/tests/script/general/parser/join.sim index e2132589bd3a54d42e683094f184b3a4a4932f71..36deea0371486394125355cc92a9785764610569 100644 --- a/tests/script/general/parser/join.sim +++ b/tests/script/general/parser/join.sim @@ -445,7 +445,7 @@ if $rows != $val then endi print ================>TD-5600 -sql select first(join_tb0.c8),first(join_tb0.c9) from join_tb1 , join_tb0 where join_tb1.ts = join_tb0.ts and join_tb1.ts <= 100002 and join_tb1.ts>=100000 interval(1s) fill(linear); +sql select first(join_tb0.c8),first(join_tb0.c9) from join_tb1 , join_tb0 where join_tb1.ts = join_tb0.ts and join_tb1.ts <= 100002 and join_tb1.ts>=100000 interval(1s); #===============================================================