[td-225] fix bugs.

src/client/src/tscServer.c

@@ -571,7 +571,6 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char
     pTableIdInfo->key = htobe64(tscGetSubscriptionProgress(pSql->pSubscription, pTableMeta->uid, dfltKey));
 
     pQueryMsg->numOfTables = htonl(1);  // set the number of tables
-
     pMsg += sizeof(STableIdInfo);
   } else {
     int32_t index = pTableMetaInfo->vgroupIndex;
@@ -601,8 +600,8 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char
     }
   }
   
-  tscTrace("%p vgId:%d, query on table:%s, uid:%" PRIu64, pSql, htonl(pQueryMsg->head.vgId), pTableMetaInfo->name,
-      pTableMeta->uid);
+  tscTrace("%p vgId:%d, query on table:%s, tid:%d, uid:%" PRIu64, pSql, htonl(pQueryMsg->head.vgId), pTableMetaInfo->name,
+      pTableMeta->sid, pTableMeta->uid);
   
   return pMsg;
 }
@@ -1863,6 +1862,7 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) {
   }
 
   free(pTableMeta);
+  tscTrace("%p recv table meta: %"PRId64 ", tid:%d, name:%s", pSql, pTableMeta->uid, pTableMeta->sid, pTableMetaInfo->name);
   
   return TSDB_CODE_SUCCESS;
 }

src/query/src/qExecutor.c

@@ -6041,6 +6041,8 @@ int32_t qCreateQueryInfo(void *tsdb, int32_t vgId, SQueryTableMsg *pQueryMsg, qi
     isSTableQuery = TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_MULTITABLE_QUERY);
     
     STableIdInfo *id = taosArrayGet(pTableIdList, 0);
+    qTrace("qmsg:%p query table, uid:%"PRId64", tid:%d", pQueryMsg, id->uid, id->tid);
+    
     if ((code = tsdbGetOneTableGroup(tsdb, id->uid, &groupInfo)) != TSDB_CODE_SUCCESS) {
       goto _over;
     }

src/tsdb/src/tsdbRead.c

@@ -485,6 +485,8 @@ static SArray* getDefaultLoadColumns(STsdbQueryHandle* pQueryHandle, bool loadTS
 static void    doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SCompBlock* pBlock,
                                      SArray* sa);
 static int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order);
+static int tsdbReadRowsFromCache(SSkipListIterator* pIter, STable* pTable, TSKEY maxKey, int maxRowsToRead, TSKEY* skey, TSKEY* ekey,
+                                 STsdbQueryHandle* pQueryHandle);
 
 static bool doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock, STableCheckInfo* pCheckInfo) {
   STsdbRepo *pRepo = pQueryHandle->pTsdb;
@@ -562,11 +564,33 @@ static void handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SCompBlock*
   
   cur->pos = ASCENDING_TRAVERSE(pQueryHandle->order)? 0:(binfo.rows-1);
   
-  // todo refactor:
-  if ((ASCENDING_TRAVERSE(pQueryHandle->order) && ((k1 != TSKEY_INITIAL_VAL && k1 <= binfo.window.ekey && k1 >= binfo.window.skey) ||
-  (k2 != TSKEY_INITIAL_VAL && k2 <= binfo.window.ekey && k2 >= binfo.window.skey))) ||
-      (!ASCENDING_TRAVERSE(pQueryHandle->order) && ((k1 != TSKEY_INITIAL_VAL && k1 <= binfo.window.ekey && k1 >= binfo.window.skey) ||
-          (k2 != TSKEY_INITIAL_VAL && k2 <= binfo.window.ekey && k2 >= binfo.window.skey)))) {
+  if ((ASCENDING_TRAVERSE(pQueryHandle->order) &&
+      ((k1 != TSKEY_INITIAL_VAL && k1 <= binfo.window.ekey) || (k2 != TSKEY_INITIAL_VAL && k2 <= binfo.window.ekey))) ||
+      (!ASCENDING_TRAVERSE(pQueryHandle->order) &&
+      ((k1 != TSKEY_INITIAL_VAL && k1 >= binfo.window.skey) || (k2 != TSKEY_INITIAL_VAL && k2 >= binfo.window.skey)))) {
+    
+    if ((ASCENDING_TRAVERSE(pQueryHandle->order) &&
+    ((k1 != TSKEY_INITIAL_VAL && k1 < binfo.window.skey) || (k2 != TSKEY_INITIAL_VAL && k2 < binfo.window.skey))) ||
+        (!ASCENDING_TRAVERSE(pQueryHandle->order) &&
+         (((k1 != TSKEY_INITIAL_VAL && k1 > binfo.window.skey) || (k2 != TSKEY_INITIAL_VAL && k2 > binfo.window.skey))))) {
+      // do not load file block into buffer
+      int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order) ? 1 : -1;
+
+      cur->rows = tsdbReadRowsFromCache(pCheckInfo->iter, pCheckInfo->pTableObj, binfo.window.skey - step,
+                                        pQueryHandle->outputCapacity, &cur->win.skey, &cur->win.ekey, pQueryHandle);
+      pQueryHandle->realNumOfRows = cur->rows;
+
+      // update the last key value
+      pCheckInfo->lastKey = cur->win.ekey + step;
+      if (!ASCENDING_TRAVERSE(pQueryHandle->order)) {
+        SWAP(cur->win.skey, cur->win.ekey, TSKEY);
+      }
+      
+      cur->mixBlock = true;
+      cur->blockCompleted = false;
+      return;
+    }
+    
     doLoadFileDataBlock(pQueryHandle, pBlock, pCheckInfo);
     doMergeTwoLevelData(pQueryHandle, pCheckInfo, pBlock, sa);
   } else {
@@ -917,6 +941,10 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
     } while (numOfRows < pQueryHandle->outputCapacity);
     
     if (numOfRows < pQueryHandle->outputCapacity) {
+      /**
+       * if cache is empty, load remain file block data. In contrast, if there are remain data in cache, do NOT
+       * copy them all to result buffer, since it may be overlapped with file data block.
+       */
       if (node == NULL ||
           ((dataRowKey(SL_GET_NODE_DATA(node)) > pQueryHandle->window.ekey) && ASCENDING_TRAVERSE(pQueryHandle->order)) ||
           ((dataRowKey(SL_GET_NODE_DATA(node)) < pQueryHandle->window.ekey) && !ASCENDING_TRAVERSE(pQueryHandle->order))) {
@@ -949,28 +977,6 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
 
         numOfRows = copyDataFromFileBlock(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, start, end);
         pos += (end - start + 1) * step;
-      } else {
-        
-        while(numOfRows < pQueryHandle->outputCapacity && node != NULL &&
-            (((dataRowKey(SL_GET_NODE_DATA(node)) <= pQueryHandle->window.ekey) && ASCENDING_TRAVERSE(pQueryHandle->order)) ||
-            ((dataRowKey(SL_GET_NODE_DATA(node)) >= pQueryHandle->window.ekey) && !ASCENDING_TRAVERSE(pQueryHandle->order)))) {
-          SDataRow row = SL_GET_NODE_DATA(node);
-          TSKEY key = dataRowKey(row);
-          
-          copyOneRowFromMem(pQueryHandle, pCheckInfo, pQueryHandle->outputCapacity, numOfRows, row, pSchema);
-          numOfRows += 1;
-          
-          if (cur->win.skey == TSKEY_INITIAL_VAL) {
-            cur->win.skey = key;
-          }
-  
-          cur->win.ekey = key;
-          cur->lastKey  = key + step;
-          cur->mixBlock = true;
-  
-          tSkipListIterNext(pCheckInfo->iter);
-          node = tSkipListIterGet(pCheckInfo->iter);
-        }
       }
     }
   }
@@ -1275,7 +1281,8 @@ static bool getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle) {
           (cur->slot == 0 && !ASCENDING_TRAVERSE(pQueryHandle->order))) {
         // all data blocks in current file has been checked already, try next file if exists
         return getDataBlocksInFilesImpl(pQueryHandle);
-      } else {  // next block of the same file
+      } else {
+        // next block of the same file
         int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order) ? 1 : -1;
         cur->slot += step;
         
@@ -1286,9 +1293,8 @@ static bool getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle) {
         return loadFileDataBlock(pQueryHandle, pNext->pBlock.compBlock, pNext->pTableCheckInfo);
       }
     } else {
-      SArray* sa = getDefaultLoadColumns(pQueryHandle, true);
-      doMergeTwoLevelData(pQueryHandle, pCheckInfo, pBlockInfo->pBlock.compBlock, sa);
-      return true;
+      handleDataMergeIfNeeded(pQueryHandle, pBlockInfo->pBlock.compBlock, pCheckInfo);
+      return pQueryHandle->realNumOfRows > 0;
     }
   }
 }
@@ -1472,9 +1478,6 @@ static int tsdbReadRowsFromCache(SSkipListIterator* pIter, STable* pTable, TSKEY
 SDataBlockInfo tsdbRetrieveDataBlockInfo(TsdbQueryHandleT* pQueryHandle) {
   STsdbQueryHandle* pHandle = (STsdbQueryHandle*)pQueryHandle;
 
-  STable* pTable = NULL;
-  int32_t rows = 0;
-
   int32_t step = ASCENDING_TRAVERSE(pHandle->order)? 1:-1;
   
   // there are data in file
@@ -1482,7 +1485,7 @@ SDataBlockInfo tsdbRetrieveDataBlockInfo(TsdbQueryHandleT* pQueryHandle) {
     STableBlockInfo* pBlockInfo = &pHandle->pDataBlockInfo[pHandle->cur.slot];
     STableCheckInfo* pCheckInfo = pBlockInfo->pTableCheckInfo;
   
-    pTable = pCheckInfo->pTableObj;
+    STable* pTable = pCheckInfo->pTableObj;
 
     if (pHandle->cur.mixBlock) {
       SDataBlockInfo blockInfo = {
@@ -1494,19 +1497,17 @@ SDataBlockInfo tsdbRetrieveDataBlockInfo(TsdbQueryHandleT* pQueryHandle) {
   
       return blockInfo;
     } else {
-      SDataBlockInfo binfo = getTrueDataBlockInfo(pCheckInfo, pBlockInfo->pBlock.compBlock);
-      return binfo;
+      return getTrueDataBlockInfo(pCheckInfo, pBlockInfo->pBlock.compBlock);
     }
   } else {
     STableCheckInfo* pCheckInfo = taosArrayGet(pHandle->pTableCheckInfo, pHandle->activeIndex);
-    pTable = pCheckInfo->pTableObj;
     
-    if (pTable->mem != NULL) {
-      // create mem table iterator if it is not created yet
+    STable* pTable = pCheckInfo->pTableObj;
+    if (pTable->mem != NULL) { // create mem table iterator if it is not created yet
       assert(pCheckInfo->iter != NULL);
       STimeWindow* win = &pHandle->cur.win;
       
-      rows = tsdbReadRowsFromCache(pCheckInfo->iter, pCheckInfo->pTableObj, pHandle->window.ekey,
+      pHandle->cur.rows = tsdbReadRowsFromCache(pCheckInfo->iter, pCheckInfo->pTableObj, pHandle->window.ekey,
           pHandle->outputCapacity, &win->skey, &win->ekey, pHandle);  // todo refactor API
 
       // update the last key value
@@ -1520,7 +1521,7 @@ SDataBlockInfo tsdbRetrieveDataBlockInfo(TsdbQueryHandleT* pQueryHandle) {
     SDataBlockInfo blockInfo = {
         .uid = pTable->tableId.uid,
         .tid = pTable->tableId.tid,
-        .rows = rows,
+        .rows = pHandle->cur.rows,
         .window = pHandle->cur.win,
     };
   

tests/script/general/import/replica1.sim

@@ -32,10 +32,10 @@ sql connect
 
 sql create database ir1db days 7
 sql use ir1db
-sql create table tb(ts timestamp, i int)
+sql create table tb(ts timestamp, i bigint)
 
 print ================= step1
-sql import into tb values(1520000010000, 10000)
+sql import into tb values(1520000010000, 1520000010000)
 sql select * from tb;
 print $rows
 if $rows != 1 then 
@@ -43,7 +43,7 @@ if $rows != 1 then
 endi
 
 print ================= step2
-sql insert into tb values(1520000008000, 8000)
+sql insert into tb values(1520000008000, 1520000008000)
 print $rows
 sql select * from tb;
 if $rows != 2 then 
@@ -51,7 +51,7 @@ if $rows != 2 then
 endi
 
 print ================= step3
-sql insert into tb values(1520000020000, 20000)
+sql insert into tb values(1520000020000, 1520000020000)
 sql select * from tb;
 print $rows
 if $rows != 3 then 
@@ -59,9 +59,9 @@ if $rows != 3 then
 endi
 
 print ================= step4
-sql import into tb values(1520000009000, 9000)
-sql import into tb values(1520000015000, 15000)
-sql import into tb values(1520000030000, 30000)
+sql import into tb values(1520000009000, 1520000009000)
+sql import into tb values(1520000015000, 1520000015000)
+sql import into tb values(1520000030000, 1520000030000)
 sql select * from tb;
 print $rows
 if $rows != 6 then 
@@ -69,10 +69,10 @@ if $rows != 6 then
 endi
 
 print ================= step5
-sql insert into tb values(1520000008000, 8000)
-sql insert into tb values(1520000014000, 14000)
-sql insert into tb values(1520000025000, 25000)
-sql insert into tb values(1520000040000, 40000)
+sql insert into tb values(1520000008000, 1520000008000)
+sql insert into tb values(1520000014000, 1520000014000)
+sql insert into tb values(1520000025000, 1520000025000)
+sql insert into tb values(1520000040000, 1520000040000)
 sql select * from tb;
 print $rows
 if $rows != 9 then 
@@ -80,11 +80,11 @@ if $rows != 9 then
 endi
 
 print ================= step6
-sql import into tb values(1520000007000, 7000)
-sql import into tb values(1520000012000, 12000)
-sql import into tb values(1520000023000, 23000)
-sql import into tb values(1520000034000, 34000)
-sql import into tb values(1520000050000, 50000)
+sql import into tb values(1520000007000, 1520000007000)
+sql import into tb values(1520000012000, 1520000012000)
+sql import into tb values(1520000023000, 1520000023000)
+sql import into tb values(1520000034000, 1520000034000)
+sql import into tb values(1520000050000, 1520000050000)
 sql select * from tb;
 print $rows
 if $rows != 14 then 
@@ -104,11 +104,11 @@ if $rows != 14 then
 endi
 
 print ================= step7
-sql import into tb values(1520000007001, 7001)
-sql import into tb values(1520000012001, 12001)
-sql import into tb values(1520000023001, 23001)
-sql import into tb values(1520000034001, 34001)
-sql import into tb values(1520000050001, 50001)
+sql import into tb values(1520000007001, 1520000007001)
+sql import into tb values(1520000012001, 1520000012001)
+sql import into tb values(1520000023001, 1520000023001)
+sql import into tb values(1520000034001, 1520000034001)
+sql import into tb values(1520000050001, 1520000050001)
 sql select * from tb;
 print $rows
 if $rows != 19 then
@@ -117,10 +117,10 @@ if $rows != 19 then
 endi
 
 print ================= step8
-sql insert into tb values(1520000008002, 8002)
-sql insert into tb values(1520000014002, 14002)
-sql insert into tb values(1520000025002, 25002)
-sql insert into tb values(1520000060000, 60000)
+sql insert into tb values(1520000008002, 1520000008002)
+sql insert into tb values(1520000014002, 1520000014002)
+sql insert into tb values(1520000025002, 1520000025002)
+sql insert into tb values(1520000060000, 1520000060000)
 sql select * from tb;
 print $rows
 if $rows != 23 then 
@@ -142,18 +142,18 @@ print ================= step9
 #sql import into tb values(now+14d, 50001)
 #sql import into tb values(now+16d, 500051)
 
-sql import into tb values(1517408000000, 7003)
-sql import into tb values(1518272000000, 34003)
-sql import into tb values(1519136000000, 34003)
-sql import into tb values(1519568000000, 34003)
-sql import into tb values(1519654400000, 50001)
-sql import into tb values(1519827200000, 50001)
-sql import into tb values(1520345600000, 50001)
-sql import into tb values(1520691200000, 50002)
-sql import into tb values(1520864000000, 50003)
-sql import into tb values(1521900800000, 50004)
-sql import into tb values(1523110400000, 50001)
-sql import into tb values(1521382400000, 500051)
+sql import into tb values(1517408000000, 1517408000000)
+sql import into tb values(1518272000000, 1518272000000)
+sql import into tb values(1519136000000, 1519136000000)
+sql import into tb values(1519568000000, 1519568000000)
+sql import into tb values(1519654400000, 1519654400000)
+sql import into tb values(1519827200000, 1519827200000)
+sql import into tb values(1520345600000, 1520345600000)
+sql import into tb values(1520691200000, 1520691200000)
+sql import into tb values(1520864000000, 1520864000000)
+sql import into tb values(1521900800000, 1521900800000)
+sql import into tb values(1523110400000, 1523110400000)
+sql import into tb values(1521382400000, 1521382400000)
 sql select * from tb;
 print $rows
 if $rows != 35 then 
@@ -176,7 +176,7 @@ endi
 print ================= step11
 
 #sql import into tb values(now-50d, 7003) (now-48d, 7003) (now-46d, 7003) (now-44d, 7003) (now-42d, 7003)
-sql import into tb values(1515680000000, 7003) (1515852800000, 7003) (1516025600000, 7003) (1516198400000, 7003) (1516371200000, 7003)
+sql import into tb values(1515680000000, 1) (1515852800000, 2) (1516025600000, 3) (1516198400000, 4) (1516371200000, 5)
 sql select * from tb;
 if $rows != 40 then 
   return -1
@@ -184,8 +184,8 @@ endi
 
 print ================= step12
 #1520000000000
-#sql import into tb values(now-19d, 7003) (now-18d, 7003) (now-17d, 7003) (now-16d, 7003) (now-15d, 7003) (now-14d, 7003) (now-13d, 7003) (now-12d, 7003) (now-11d, 7003)
-sql import into tb values(1518358400000, 7003) (1518444800000, 7003) (1518531200000, 7003) (1518617600000, 7003) (1518704000000, 7003) (1518790400000, 7003) (1518876800000, 7003) (1518963200000, 7003) (1519049600000, 7003)
+#sql import into tb values(now-19d, -19) (now-18d, -18) (now-17d, -17) (now-16d, -16) (now-15d, -15) (now-14d, -14) (now-13d, -13) (now-12d, -12) (now-11d, -11)
+sql import into tb values(1518358400000, 6) (1518444800000, 7) (1518531200000, 8) (1518617600000, 9) (1518704000000, 10) (1518790400000, 11) (1518876800000, 12) (1518963200000, 13) (1519049600000, 14)
 sql select * from tb;
 print $rows
 if $rows != 49 then 
@@ -195,13 +195,13 @@ endi
 
 print ================= step14
 #1520000000000
-#sql import into tb values(now-48d, 34003)
-#sql import into tb values(now-38d, 50001)
-#sql import into tb values(now-28d, 50001)
+#sql import into tb values(now-48d, -48)
+#sql import into tb values(now-38d, -38)
+#sql import into tb values(now-28d, -28)
 
-sql import into tb values(1515852800001, 34003)
-sql import into tb values(1516716800000, 50001)
-sql import into tb values(1517580800000, 50001)
+sql import into tb values(1515852800001, -48)
+sql import into tb values(1516716800000, -38)
+sql import into tb values(1517580800000, -28)
 
 sql select * from tb;
 if $rows != 52 then 

tests/script/general/parser/limit1.sim

@@ -65,7 +65,7 @@ sleep 2000
 system sh/exec.sh -n dnode1 -s start
 print ================== server restart completed
 
-#run general/parser/limit1_tb.sim
+run general/parser/limit1_tb.sim
 run general/parser/limit1_stb.sim
 
 system sh/exec.sh -n dnode1 -s stop -x SIGINT
\ No newline at end of file

tests/script/sh/deploy.sh

@@ -107,6 +107,7 @@ echo "monitorDebugFlag    131"            >> $TAOS_CFG
 echo "udebugFlag          131"            >> $TAOS_CFG
 echo "jnidebugFlag        131"            >> $TAOS_CFG
 echo "sdebugFlag          135"            >> $TAOS_CFG
+echo "qdebugFlag          135"            >> $TAOS_CFG
 echo "monitor             0"              >> $TAOS_CFG
 echo "monitorInterval     1"              >> $TAOS_CFG
 echo "http                0"              >> $TAOS_CFG