diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index e36fb0d86a7f1bbd24af22a9f1a92522ab60ba57..0a0b7cb8aeabf32aa8278968445b3672ce4dff8b 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -571,7 +571,6 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char pTableIdInfo->key = htobe64(tscGetSubscriptionProgress(pSql->pSubscription, pTableMeta->uid, dfltKey)); pQueryMsg->numOfTables = htonl(1); // set the number of tables - pMsg += sizeof(STableIdInfo); } else { int32_t index = pTableMetaInfo->vgroupIndex; @@ -601,8 +600,8 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char } } - tscTrace("%p vgId:%d, query on table:%s, uid:%" PRIu64, pSql, htonl(pQueryMsg->head.vgId), pTableMetaInfo->name, - pTableMeta->uid); + tscTrace("%p vgId:%d, query on table:%s, tid:%d, uid:%" PRIu64, pSql, htonl(pQueryMsg->head.vgId), pTableMetaInfo->name, + pTableMeta->sid, pTableMeta->uid); return pMsg; } @@ -1863,6 +1862,7 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) { } free(pTableMeta); + tscTrace("%p recv table meta: %"PRId64 ", tid:%d, name:%s", pSql, pTableMeta->uid, pTableMeta->sid, pTableMetaInfo->name); return TSDB_CODE_SUCCESS; } diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index e148bcb58e924b55033640265f2ca8fe27112948..e853a548ee03e242580488a9aaeb2530230fb3df 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -6041,6 +6041,8 @@ int32_t qCreateQueryInfo(void *tsdb, int32_t vgId, SQueryTableMsg *pQueryMsg, qi isSTableQuery = TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_MULTITABLE_QUERY); STableIdInfo *id = taosArrayGet(pTableIdList, 0); + qTrace("qmsg:%p query table, uid:%"PRId64", tid:%d", pQueryMsg, id->uid, id->tid); + if ((code = tsdbGetOneTableGroup(tsdb, id->uid, &groupInfo)) != TSDB_CODE_SUCCESS) { goto _over; } diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 2a269eb04a8596abb6ceb0806f6b802e143335ee..89f290184d7c0b37c7d577ad9d4f16d6fa73e65c 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -485,6 +485,8 @@ static SArray* getDefaultLoadColumns(STsdbQueryHandle* pQueryHandle, bool loadTS static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SCompBlock* pBlock, SArray* sa); static int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order); +static int tsdbReadRowsFromCache(SSkipListIterator* pIter, STable* pTable, TSKEY maxKey, int maxRowsToRead, TSKEY* skey, TSKEY* ekey, + STsdbQueryHandle* pQueryHandle); static bool doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock, STableCheckInfo* pCheckInfo) { STsdbRepo *pRepo = pQueryHandle->pTsdb; @@ -562,11 +564,33 @@ static void handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SCompBlock* cur->pos = ASCENDING_TRAVERSE(pQueryHandle->order)? 0:(binfo.rows-1); - // todo refactor: - if ((ASCENDING_TRAVERSE(pQueryHandle->order) && ((k1 != TSKEY_INITIAL_VAL && k1 <= binfo.window.ekey && k1 >= binfo.window.skey) || - (k2 != TSKEY_INITIAL_VAL && k2 <= binfo.window.ekey && k2 >= binfo.window.skey))) || - (!ASCENDING_TRAVERSE(pQueryHandle->order) && ((k1 != TSKEY_INITIAL_VAL && k1 <= binfo.window.ekey && k1 >= binfo.window.skey) || - (k2 != TSKEY_INITIAL_VAL && k2 <= binfo.window.ekey && k2 >= binfo.window.skey)))) { + if ((ASCENDING_TRAVERSE(pQueryHandle->order) && + ((k1 != TSKEY_INITIAL_VAL && k1 <= binfo.window.ekey) || (k2 != TSKEY_INITIAL_VAL && k2 <= binfo.window.ekey))) || + (!ASCENDING_TRAVERSE(pQueryHandle->order) && + ((k1 != TSKEY_INITIAL_VAL && k1 >= binfo.window.skey) || (k2 != TSKEY_INITIAL_VAL && k2 >= binfo.window.skey)))) { + + if ((ASCENDING_TRAVERSE(pQueryHandle->order) && + ((k1 != TSKEY_INITIAL_VAL && k1 < binfo.window.skey) || (k2 != TSKEY_INITIAL_VAL && k2 < binfo.window.skey))) || + (!ASCENDING_TRAVERSE(pQueryHandle->order) && + (((k1 != TSKEY_INITIAL_VAL && k1 > binfo.window.skey) || (k2 != TSKEY_INITIAL_VAL && k2 > binfo.window.skey))))) { + // do not load file block into buffer + int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order) ? 1 : -1; + + cur->rows = tsdbReadRowsFromCache(pCheckInfo->iter, pCheckInfo->pTableObj, binfo.window.skey - step, + pQueryHandle->outputCapacity, &cur->win.skey, &cur->win.ekey, pQueryHandle); + pQueryHandle->realNumOfRows = cur->rows; + + // update the last key value + pCheckInfo->lastKey = cur->win.ekey + step; + if (!ASCENDING_TRAVERSE(pQueryHandle->order)) { + SWAP(cur->win.skey, cur->win.ekey, TSKEY); + } + + cur->mixBlock = true; + cur->blockCompleted = false; + return; + } + doLoadFileDataBlock(pQueryHandle, pBlock, pCheckInfo); doMergeTwoLevelData(pQueryHandle, pCheckInfo, pBlock, sa); } else { @@ -917,6 +941,10 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* } while (numOfRows < pQueryHandle->outputCapacity); if (numOfRows < pQueryHandle->outputCapacity) { + /** + * if cache is empty, load remain file block data. In contrast, if there are remain data in cache, do NOT + * copy them all to result buffer, since it may be overlapped with file data block. + */ if (node == NULL || ((dataRowKey(SL_GET_NODE_DATA(node)) > pQueryHandle->window.ekey) && ASCENDING_TRAVERSE(pQueryHandle->order)) || ((dataRowKey(SL_GET_NODE_DATA(node)) < pQueryHandle->window.ekey) && !ASCENDING_TRAVERSE(pQueryHandle->order))) { @@ -949,28 +977,6 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* numOfRows = copyDataFromFileBlock(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, start, end); pos += (end - start + 1) * step; - } else { - - while(numOfRows < pQueryHandle->outputCapacity && node != NULL && - (((dataRowKey(SL_GET_NODE_DATA(node)) <= pQueryHandle->window.ekey) && ASCENDING_TRAVERSE(pQueryHandle->order)) || - ((dataRowKey(SL_GET_NODE_DATA(node)) >= pQueryHandle->window.ekey) && !ASCENDING_TRAVERSE(pQueryHandle->order)))) { - SDataRow row = SL_GET_NODE_DATA(node); - TSKEY key = dataRowKey(row); - - copyOneRowFromMem(pQueryHandle, pCheckInfo, pQueryHandle->outputCapacity, numOfRows, row, pSchema); - numOfRows += 1; - - if (cur->win.skey == TSKEY_INITIAL_VAL) { - cur->win.skey = key; - } - - cur->win.ekey = key; - cur->lastKey = key + step; - cur->mixBlock = true; - - tSkipListIterNext(pCheckInfo->iter); - node = tSkipListIterGet(pCheckInfo->iter); - } } } } @@ -1275,7 +1281,8 @@ static bool getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle) { (cur->slot == 0 && !ASCENDING_TRAVERSE(pQueryHandle->order))) { // all data blocks in current file has been checked already, try next file if exists return getDataBlocksInFilesImpl(pQueryHandle); - } else { // next block of the same file + } else { + // next block of the same file int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order) ? 1 : -1; cur->slot += step; @@ -1286,9 +1293,8 @@ static bool getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle) { return loadFileDataBlock(pQueryHandle, pNext->pBlock.compBlock, pNext->pTableCheckInfo); } } else { - SArray* sa = getDefaultLoadColumns(pQueryHandle, true); - doMergeTwoLevelData(pQueryHandle, pCheckInfo, pBlockInfo->pBlock.compBlock, sa); - return true; + handleDataMergeIfNeeded(pQueryHandle, pBlockInfo->pBlock.compBlock, pCheckInfo); + return pQueryHandle->realNumOfRows > 0; } } } @@ -1472,17 +1478,14 @@ static int tsdbReadRowsFromCache(SSkipListIterator* pIter, STable* pTable, TSKEY SDataBlockInfo tsdbRetrieveDataBlockInfo(TsdbQueryHandleT* pQueryHandle) { STsdbQueryHandle* pHandle = (STsdbQueryHandle*)pQueryHandle; - STable* pTable = NULL; - int32_t rows = 0; - int32_t step = ASCENDING_TRAVERSE(pHandle->order)? 1:-1; // there are data in file if (pHandle->cur.fid >= 0) { STableBlockInfo* pBlockInfo = &pHandle->pDataBlockInfo[pHandle->cur.slot]; STableCheckInfo* pCheckInfo = pBlockInfo->pTableCheckInfo; - - pTable = pCheckInfo->pTableObj; + + STable* pTable = pCheckInfo->pTableObj; if (pHandle->cur.mixBlock) { SDataBlockInfo blockInfo = { @@ -1494,19 +1497,17 @@ SDataBlockInfo tsdbRetrieveDataBlockInfo(TsdbQueryHandleT* pQueryHandle) { return blockInfo; } else { - SDataBlockInfo binfo = getTrueDataBlockInfo(pCheckInfo, pBlockInfo->pBlock.compBlock); - return binfo; + return getTrueDataBlockInfo(pCheckInfo, pBlockInfo->pBlock.compBlock); } } else { STableCheckInfo* pCheckInfo = taosArrayGet(pHandle->pTableCheckInfo, pHandle->activeIndex); - pTable = pCheckInfo->pTableObj; - - if (pTable->mem != NULL) { - // create mem table iterator if it is not created yet + + STable* pTable = pCheckInfo->pTableObj; + if (pTable->mem != NULL) { // create mem table iterator if it is not created yet assert(pCheckInfo->iter != NULL); STimeWindow* win = &pHandle->cur.win; - rows = tsdbReadRowsFromCache(pCheckInfo->iter, pCheckInfo->pTableObj, pHandle->window.ekey, + pHandle->cur.rows = tsdbReadRowsFromCache(pCheckInfo->iter, pCheckInfo->pTableObj, pHandle->window.ekey, pHandle->outputCapacity, &win->skey, &win->ekey, pHandle); // todo refactor API // update the last key value @@ -1520,7 +1521,7 @@ SDataBlockInfo tsdbRetrieveDataBlockInfo(TsdbQueryHandleT* pQueryHandle) { SDataBlockInfo blockInfo = { .uid = pTable->tableId.uid, .tid = pTable->tableId.tid, - .rows = rows, + .rows = pHandle->cur.rows, .window = pHandle->cur.win, }; diff --git a/tests/script/general/import/replica1.sim b/tests/script/general/import/replica1.sim index ad5a3faed5d00941d56ed3fa18af5340a40240d3..61f563ba8e9489e2ad0ffe217b37603ffb975319 100644 --- a/tests/script/general/import/replica1.sim +++ b/tests/script/general/import/replica1.sim @@ -32,10 +32,10 @@ sql connect sql create database ir1db days 7 sql use ir1db -sql create table tb(ts timestamp, i int) +sql create table tb(ts timestamp, i bigint) print ================= step1 -sql import into tb values(1520000010000, 10000) +sql import into tb values(1520000010000, 1520000010000) sql select * from tb; print $rows if $rows != 1 then @@ -43,7 +43,7 @@ if $rows != 1 then endi print ================= step2 -sql insert into tb values(1520000008000, 8000) +sql insert into tb values(1520000008000, 1520000008000) print $rows sql select * from tb; if $rows != 2 then @@ -51,7 +51,7 @@ if $rows != 2 then endi print ================= step3 -sql insert into tb values(1520000020000, 20000) +sql insert into tb values(1520000020000, 1520000020000) sql select * from tb; print $rows if $rows != 3 then @@ -59,9 +59,9 @@ if $rows != 3 then endi print ================= step4 -sql import into tb values(1520000009000, 9000) -sql import into tb values(1520000015000, 15000) -sql import into tb values(1520000030000, 30000) +sql import into tb values(1520000009000, 1520000009000) +sql import into tb values(1520000015000, 1520000015000) +sql import into tb values(1520000030000, 1520000030000) sql select * from tb; print $rows if $rows != 6 then @@ -69,10 +69,10 @@ if $rows != 6 then endi print ================= step5 -sql insert into tb values(1520000008000, 8000) -sql insert into tb values(1520000014000, 14000) -sql insert into tb values(1520000025000, 25000) -sql insert into tb values(1520000040000, 40000) +sql insert into tb values(1520000008000, 1520000008000) +sql insert into tb values(1520000014000, 1520000014000) +sql insert into tb values(1520000025000, 1520000025000) +sql insert into tb values(1520000040000, 1520000040000) sql select * from tb; print $rows if $rows != 9 then @@ -80,11 +80,11 @@ if $rows != 9 then endi print ================= step6 -sql import into tb values(1520000007000, 7000) -sql import into tb values(1520000012000, 12000) -sql import into tb values(1520000023000, 23000) -sql import into tb values(1520000034000, 34000) -sql import into tb values(1520000050000, 50000) +sql import into tb values(1520000007000, 1520000007000) +sql import into tb values(1520000012000, 1520000012000) +sql import into tb values(1520000023000, 1520000023000) +sql import into tb values(1520000034000, 1520000034000) +sql import into tb values(1520000050000, 1520000050000) sql select * from tb; print $rows if $rows != 14 then @@ -104,11 +104,11 @@ if $rows != 14 then endi print ================= step7 -sql import into tb values(1520000007001, 7001) -sql import into tb values(1520000012001, 12001) -sql import into tb values(1520000023001, 23001) -sql import into tb values(1520000034001, 34001) -sql import into tb values(1520000050001, 50001) +sql import into tb values(1520000007001, 1520000007001) +sql import into tb values(1520000012001, 1520000012001) +sql import into tb values(1520000023001, 1520000023001) +sql import into tb values(1520000034001, 1520000034001) +sql import into tb values(1520000050001, 1520000050001) sql select * from tb; print $rows if $rows != 19 then @@ -117,10 +117,10 @@ if $rows != 19 then endi print ================= step8 -sql insert into tb values(1520000008002, 8002) -sql insert into tb values(1520000014002, 14002) -sql insert into tb values(1520000025002, 25002) -sql insert into tb values(1520000060000, 60000) +sql insert into tb values(1520000008002, 1520000008002) +sql insert into tb values(1520000014002, 1520000014002) +sql insert into tb values(1520000025002, 1520000025002) +sql insert into tb values(1520000060000, 1520000060000) sql select * from tb; print $rows if $rows != 23 then @@ -142,18 +142,18 @@ print ================= step9 #sql import into tb values(now+14d, 50001) #sql import into tb values(now+16d, 500051) -sql import into tb values(1517408000000, 7003) -sql import into tb values(1518272000000, 34003) -sql import into tb values(1519136000000, 34003) -sql import into tb values(1519568000000, 34003) -sql import into tb values(1519654400000, 50001) -sql import into tb values(1519827200000, 50001) -sql import into tb values(1520345600000, 50001) -sql import into tb values(1520691200000, 50002) -sql import into tb values(1520864000000, 50003) -sql import into tb values(1521900800000, 50004) -sql import into tb values(1523110400000, 50001) -sql import into tb values(1521382400000, 500051) +sql import into tb values(1517408000000, 1517408000000) +sql import into tb values(1518272000000, 1518272000000) +sql import into tb values(1519136000000, 1519136000000) +sql import into tb values(1519568000000, 1519568000000) +sql import into tb values(1519654400000, 1519654400000) +sql import into tb values(1519827200000, 1519827200000) +sql import into tb values(1520345600000, 1520345600000) +sql import into tb values(1520691200000, 1520691200000) +sql import into tb values(1520864000000, 1520864000000) +sql import into tb values(1521900800000, 1521900800000) +sql import into tb values(1523110400000, 1523110400000) +sql import into tb values(1521382400000, 1521382400000) sql select * from tb; print $rows if $rows != 35 then @@ -176,7 +176,7 @@ endi print ================= step11 #sql import into tb values(now-50d, 7003) (now-48d, 7003) (now-46d, 7003) (now-44d, 7003) (now-42d, 7003) -sql import into tb values(1515680000000, 7003) (1515852800000, 7003) (1516025600000, 7003) (1516198400000, 7003) (1516371200000, 7003) +sql import into tb values(1515680000000, 1) (1515852800000, 2) (1516025600000, 3) (1516198400000, 4) (1516371200000, 5) sql select * from tb; if $rows != 40 then return -1 @@ -184,8 +184,8 @@ endi print ================= step12 #1520000000000 -#sql import into tb values(now-19d, 7003) (now-18d, 7003) (now-17d, 7003) (now-16d, 7003) (now-15d, 7003) (now-14d, 7003) (now-13d, 7003) (now-12d, 7003) (now-11d, 7003) -sql import into tb values(1518358400000, 7003) (1518444800000, 7003) (1518531200000, 7003) (1518617600000, 7003) (1518704000000, 7003) (1518790400000, 7003) (1518876800000, 7003) (1518963200000, 7003) (1519049600000, 7003) +#sql import into tb values(now-19d, -19) (now-18d, -18) (now-17d, -17) (now-16d, -16) (now-15d, -15) (now-14d, -14) (now-13d, -13) (now-12d, -12) (now-11d, -11) +sql import into tb values(1518358400000, 6) (1518444800000, 7) (1518531200000, 8) (1518617600000, 9) (1518704000000, 10) (1518790400000, 11) (1518876800000, 12) (1518963200000, 13) (1519049600000, 14) sql select * from tb; print $rows if $rows != 49 then @@ -195,13 +195,13 @@ endi print ================= step14 #1520000000000 -#sql import into tb values(now-48d, 34003) -#sql import into tb values(now-38d, 50001) -#sql import into tb values(now-28d, 50001) +#sql import into tb values(now-48d, -48) +#sql import into tb values(now-38d, -38) +#sql import into tb values(now-28d, -28) -sql import into tb values(1515852800001, 34003) -sql import into tb values(1516716800000, 50001) -sql import into tb values(1517580800000, 50001) +sql import into tb values(1515852800001, -48) +sql import into tb values(1516716800000, -38) +sql import into tb values(1517580800000, -28) sql select * from tb; if $rows != 52 then diff --git a/tests/script/general/parser/limit1.sim b/tests/script/general/parser/limit1.sim index 7971cb81c6eec09530b4bcaf264e5c5b154b388a..48fb6aaedeef38ed26fc0cde92d5b7869afcb3ba 100644 --- a/tests/script/general/parser/limit1.sim +++ b/tests/script/general/parser/limit1.sim @@ -65,7 +65,7 @@ sleep 2000 system sh/exec.sh -n dnode1 -s start print ================== server restart completed -#run general/parser/limit1_tb.sim +run general/parser/limit1_tb.sim run general/parser/limit1_stb.sim system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/sh/deploy.sh b/tests/script/sh/deploy.sh index bd85ce9704945c26ea75ba663a66a735038bb971..83ecea555a8c85eebadfd19fade5fd17340b237e 100755 --- a/tests/script/sh/deploy.sh +++ b/tests/script/sh/deploy.sh @@ -107,6 +107,7 @@ echo "monitorDebugFlag 131" >> $TAOS_CFG echo "udebugFlag 131" >> $TAOS_CFG echo "jnidebugFlag 131" >> $TAOS_CFG echo "sdebugFlag 135" >> $TAOS_CFG +echo "qdebugFlag 135" >> $TAOS_CFG echo "monitor 0" >> $TAOS_CFG echo "monitorInterval 1" >> $TAOS_CFG echo "http 0" >> $TAOS_CFG