Merge remote-tracking branch 'origin/develop' into feature/sim

packaging/cfg/taos.cfg

@@ -39,10 +39,10 @@
 # number of management nodes in the system
 # numOfMnodes               3
 
-# enable/disable backuping vnode directory when removing dnode
+# enable/disable backuping vnode directory when removing vnode
 # vnodeBak                  1
 
-# if report installation / use information
+# enable/disable installation / usage report
 # telemetryReporting        1
 
 # enable/disable load balancing
@@ -81,7 +81,7 @@
 # minimum time window, milli-second
 # minIntervalTime           10
 
-# maximum delay before launching a stream compution, milli-second
+# maximum delay before launching a stream computation, milli-second
 # maxStreamCompDelay        20000
 
 # maximum delay before launching a stream computation for the first time, milli-second
@@ -156,7 +156,7 @@
 # max number of connections allowed in dnode
 # maxShellConns         5000
 
-# max numerber of connections allowed in client 
+# max number of connections allowed in client
 # maxConnections        5000
 
 # stop writing logs when the disk size of the log folder is less than this value
@@ -187,7 +187,7 @@
 # restfulRowLimit       10240
 
 # The following parameter is used to limit the maximum number of lines in log files.
-# max number of rows per log filters
+# max number of lines per log filters
 # numOfLogLines         10000000
 
 # enable/disable async log
@@ -199,7 +199,9 @@
 
 # The following parameters are used for debug purpose only.
 # debugFlag 8 bits mask: FILE-SCREEN-UNUSED-HeartBeat-DUMP-TRACE_WARN-ERROR
-# 131: output warning and error, 135: output debug, warning and error, 143 : output trace, debug, warning and error to log.
+# 131: output warning and error 
+# 135: output debug, warning and error
+# 143: output trace, debug, warning and error to log
 # 199: output debug, warning and error to both screen and file
 # 207: output trace, debug, warning and error to both screen and file
 
@@ -231,10 +233,10 @@
 # cDebugFlag            131
 
 # debug flag for JNI
-# jniDebugflag          131
+# jniDebugFlag          131
 
 # debug flag for storage
-# uDebugflag            131
+# uDebugFlag            131
 
 # debug flag for http server
 # httpDebugFlag         131
@@ -243,12 +245,12 @@
 # monDebugFlag          131
 
 # debug flag for query
-# qDebugflag            131
+# qDebugFlag            131
 
 # debug flag for vnode
-# vDebugflag            131
+# vDebugFlag            131
 
-# debug flag for http server
+# debug flag for TSDB
 # tsdbDebugFlag         131
 
 # debug flag for continue query

packaging/tools/post.sh

@@ -265,8 +265,14 @@ function install_config() {
         [ -f ${cfg_dir}/taos.cfg ] && ${csudo} cp ${cfg_dir}/taos.cfg ${cfg_install_dir}
         ${csudo} chmod 644 ${cfg_install_dir}/*
     fi
+    
+    # Save standard input to 6 and open / dev / TTY on standard input
+    exec 6<&0 0</dev/tty 
 
     local_fqdn_check
+    
+    # restore the backup standard input, and turn off 6
+    exec 0<&6 6<&-
 
     ${csudo} mv ${cfg_dir}/taos.cfg ${cfg_dir}/taos.cfg.org
     ${csudo} ln -s ${cfg_install_dir}/taos.cfg ${cfg_dir}
@@ -422,7 +428,7 @@ function install_service() {
 }
 
 function install_TDengine() {
-    echo -e "${GREEN}Start to install TDEngine...${NC}"
+    echo -e "${GREEN}Start to install TDengine...${NC}"
 
     #install log and data dir , then ln to /usr/local/taos
     ${csudo} mkdir -p ${log_dir} && ${csudo} chmod 777 ${log_dir}

packaging/tools/preun.sh

@@ -119,4 +119,4 @@ if ((${service_mod}==2)); then
     kill_taosd
 fi
 
-echo -e "${GREEN}TDEngine is removed successfully!${NC}"
+echo -e "${GREEN}TDengine is removed successfully!${NC}"

src/client/src/tscAsync.c

@@ -388,10 +388,10 @@ void tscQueueAsyncRes(SSqlObj *pSql) {
     return;
   }
 
+  assert(pSql->res.code != TSDB_CODE_SUCCESS);
   tscError("%p add into queued async res, code:%s", pSql, tstrerror(pSql->res.code));
 
   SSqlRes *pRes = &pSql->res;
-
   if (pSql->fp == NULL || pSql->fetchFp == NULL){
     return;
   }

src/client/src/tscFunctionImpl.c

@@ -2597,14 +2597,23 @@ static void percentile_next_step(SQLFunctionCtx *pCtx) {
 }
 
 //////////////////////////////////////////////////////////////////////////////////
+static void buildHistogramInfo(SAPercentileInfo* pInfo) {
+  pInfo->pHisto = (SHistogramInfo*) ((char*) pInfo + sizeof(SAPercentileInfo));
+  pInfo->pHisto->elems = (SHistBin*) ((char*)pInfo->pHisto + sizeof(SHistogramInfo));
+}
+
 static SAPercentileInfo *getAPerctInfo(SQLFunctionCtx *pCtx) {
   SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
-  
+  SAPercentileInfo* pInfo = NULL;
+
   if (pCtx->stableQuery && pCtx->currentStage != SECONDARY_STAGE_MERGE) {
-    return (SAPercentileInfo*) pCtx->aOutputBuf;
+    pInfo = (SAPercentileInfo*) pCtx->aOutputBuf;
   } else {
-    return GET_ROWCELL_INTERBUF(pResInfo);
+    pInfo = GET_ROWCELL_INTERBUF(pResInfo);
   }
+
+  buildHistogramInfo(pInfo);
+  return pInfo;
 }
 
 static bool apercentile_function_setup(SQLFunctionCtx *pCtx) {
@@ -2616,6 +2625,7 @@ static bool apercentile_function_setup(SQLFunctionCtx *pCtx) {
   
   char *tmp = (char *)pInfo + sizeof(SAPercentileInfo);
   pInfo->pHisto = tHistogramCreateFrom(tmp, MAX_HISTOGRAM_BIN);
+  printf("%p, %p\n", pInfo->pHisto, pInfo->pHisto->elems);
   return true;
 }
 
@@ -2624,6 +2634,8 @@ static void apercentile_function(SQLFunctionCtx *pCtx) {
   
   SResultRowCellInfo *     pResInfo = GET_RES_INFO(pCtx);
   SAPercentileInfo *pInfo = getAPerctInfo(pCtx);
+
+  assert(pInfo->pHisto->elems != NULL);
   
   for (int32_t i = 0; i < pCtx->size; ++i) {
     char *data = GET_INPUT_CHAR_INDEX(pCtx, i);

src/client/src/tscLocalMerge.c

@@ -911,6 +911,13 @@ static void genFinalResWithoutFill(SSqlRes* pRes, SLocalReducer *pLocalReducer,
     }
   }
 
+  if (pRes->numOfRowsGroup >= pQueryInfo->limit.limit && pQueryInfo->limit.limit > 0) {
+    pRes->numOfRows = 0;
+    pBeforeFillData->num = 0;
+    pLocalReducer->discard = true;
+    return;
+  }
+
   pRes->numOfRowsGroup += pRes->numOfRows;
 
   // impose the limitation of output rows on the final result

src/client/src/tscSql.c

@@ -284,16 +284,18 @@ void taos_close(TAOS *taos) {
     return;
   }
 
-  SSqlObj* pHb = (SSqlObj*)taosAcquireRef(tscObjRef, pObj->hbrid);
-  if (pHb != NULL) {
-    if (pHb->rpcRid > 0) {  // wait for rsp from dnode
-      rpcCancelRequest(pHb->rpcRid);
-      pHb->rpcRid = -1;
-    }
+  if (RID_VALID(pObj->hbrid)) {
+    SSqlObj* pHb = (SSqlObj*)taosAcquireRef(tscObjRef, pObj->hbrid);
+    if (pHb != NULL) {
+      if (RID_VALID(pHb->rpcRid)) {  // wait for rsp from dnode
+        rpcCancelRequest(pHb->rpcRid);
+        pHb->rpcRid = -1;
+      }
 
-    tscDebug("%p HB is freed", pHb);
-    taosReleaseRef(tscObjRef, pHb->self);
-    taos_free_result(pHb);
+      tscDebug("%p HB is freed", pHb);
+      taosReleaseRef(tscObjRef, pHb->self);
+      taos_free_result(pHb);
+    }
   }
 
   tscDebug("%p all sqlObj are freed, free tscObj and close dnodeConn:%p", pObj, pObj->pDnodeConn);

src/common/src/tglobal.c

@@ -222,7 +222,7 @@ int32_t uDebugFlag = 131;
 int32_t debugFlag = 0;
 int32_t sDebugFlag = 135;
 int32_t wDebugFlag = 135;
-int32_t tsdbDebugFlag = 131;
+uint32_t tsdbDebugFlag = 131;
 int32_t cqDebugFlag = 131;
 
 int32_t (*monStartSystemFp)() = NULL;

grafanaplugin @ ec77d904

-Subproject commit 32e2c97a4cf7bedaa99f5d6dd8cb036e7f4470df
+Subproject commit ec77d9049a719dabfd1a7c1122a209e201861944

src/connector/nodejs/nodetaos/cinterface.js

@@ -349,11 +349,12 @@ CTaosInterface.prototype.useResult = function useResult(result) {
   return fields;
 }
 CTaosInterface.prototype.fetchBlock = function fetchBlock(result, fields) {
-  let pblock = ref.ref(ref.ref(ref.NULL)); // equal to our raw data
-  let num_of_rows = this.libtaos.taos_fetch_block(result, pblock)
-  if (num_of_rows == 0) {
+  //let pblock = ref.ref(ref.ref(ref.NULL)); // equal to our raw data
+  let pblock = this.libtaos.taos_fetch_row(result);
+  if (pblock == null) {
     return {block:null, num_of_rows:0};
   }
+
   var fieldL = this.libtaos.taos_fetch_lengths(result);
 
   let isMicro = (this.libtaos.taos_result_precision(result) == FieldTypes.C_TIMESTAMP_MICRO);
@@ -361,7 +362,6 @@ CTaosInterface.prototype.fetchBlock = function fetchBlock(result, fields) {
   var fieldlens = [];
   
   if (ref.isNull(fieldL) == false) {
-    
     for (let i = 0; i < fields.length; i ++) {
       let plen = ref.reinterpret(fieldL, 4, i*4);
       let len = plen.readInt32LE(0);

src/inc/ttype.h

@@ -28,7 +28,7 @@ extern "C" {
     default:                                         \
       (_v) = (_finalType)GET_INT32_VAL(_data);       \
       break;                                         \
-  };
+  }
 
 #ifdef __cplusplus
 }

src/plugins/http/src/httpResp.c

@@ -136,7 +136,7 @@ void httpSendErrorResp(HttpContext *pContext, int32_t errNo) {
   else
     httpCode = 400;
 
-  if (pContext->parser->httpCode != 0) {
+  if (pContext->parser && pContext->parser->httpCode != 0) {
     httpCode = pContext->parser->httpCode;
   }
 

src/query/inc/qExecutor.h

@@ -33,13 +33,6 @@ struct SColumnFilterElem;
 typedef bool (*__filter_func_t)(struct SColumnFilterElem* pFilter, char* val1, char* val2);
 typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int32_t order);
 
-typedef struct SGroupResInfo {
-  int32_t  groupId;
-  int32_t  numOfDataPages;
-  int32_t  pageId;
-  int32_t  rowId;
-} SGroupResInfo;
-
 typedef struct SResultRowPool {
   int32_t elemSize;
   int32_t blockSize;
@@ -72,6 +65,12 @@ typedef struct SResultRow {
   union {STimeWindow win; char* key;};  // start key of current time window
 } SResultRow;
 
+typedef struct SGroupResInfo {
+  int32_t rowId;
+  int32_t index;
+  SArray* pRows;      // SArray<SResultRow*>
+} SGroupResInfo;
+
 /**
  * If the number of generated results is greater than this value,
  * query query will be halt and return results to client immediate.
@@ -89,7 +88,6 @@ typedef struct SResultRowInfo {
   int32_t      size:24;    // number of result set
   int32_t      capacity;   // max capacity
   int32_t      curIndex;   // current start active index
-  int64_t      startTime;  // start time of the first time window for sliding query
   int64_t      prevSKey;   // previous (not completed) sliding window start key
 } SResultRowInfo;
 

src/query/inc/qHistogram.h

@@ -67,7 +67,7 @@ void tHistogramDestroy(SHistogramInfo** pHisto);
 
 void tHistogramPrint(SHistogramInfo* pHisto);
 
-int32_t vnodeHistobinarySearch(SHistBin* pEntry, int32_t len, double val);
+//int32_t histoBinarySearch(SHistBin* pEntry, int32_t len, double val);
 
 SHeapEntry* tHeapCreate(int32_t numOfEntries);
 void tHeapSort(SHeapEntry* pEntry, int32_t len);

src/query/inc/qUtil.h

@@ -77,7 +77,6 @@ void* destroyResultRowPool(SResultRowPool* p);
 int32_t getNumOfAllocatedResultRows(SResultRowPool* p);
 int32_t getNumOfUsedResultRows(SResultRowPool* p);
 
-uint64_t getResultInfoUId(SQueryRuntimeEnv* pRuntimeEnv);
 bool isPointInterpoQuery(SQuery *pQuery);
 
 

src/query/src/qHistogram.c

@@ -120,6 +120,7 @@
 //}
 
 static int32_t histogramCreateBin(SHistogramInfo* pHisto, int32_t index, double val);
+static int32_t histoBinarySearch(SHistBin* pEntry, int32_t len, double val);
 
 SHistogramInfo* tHistogramCreate(int32_t numOfEntries) {
   /* need one redundant slot */
@@ -158,8 +159,8 @@ int32_t tHistogramAdd(SHistogramInfo** pHisto, double val) {
   }
 
 #if defined(USE_ARRAYLIST)
-  int32_t idx = vnodeHistobinarySearch((*pHisto)->elems, (*pHisto)->numOfEntries, val);
-  assert(idx >= 0 && idx <= (*pHisto)->maxEntries);
+  int32_t idx = histoBinarySearch((*pHisto)->elems, (*pHisto)->numOfEntries, val);
+  assert(idx >= 0 && idx <= (*pHisto)->maxEntries && (*pHisto)->elems != NULL);
 
   if ((*pHisto)->elems[idx].val == val && idx >= 0) {
     (*pHisto)->elems[idx].num += 1;
@@ -356,7 +357,7 @@ int32_t tHistogramAdd(SHistogramInfo** pHisto, double val) {
   return 0;
 }
 
-int32_t vnodeHistobinarySearch(SHistBin* pEntry, int32_t len, double val) {
+int32_t histoBinarySearch(SHistBin* pEntry, int32_t len, double val) {
   int32_t end = len - 1;
   int32_t start = 0;
 
@@ -466,7 +467,7 @@ void tHistogramPrint(SHistogramInfo* pHisto) {
  */
 int64_t tHistogramSum(SHistogramInfo* pHisto, double v) {
 #if defined(USE_ARRAYLIST)
-  int32_t slotIdx = vnodeHistobinarySearch(pHisto->elems, pHisto->numOfEntries, v);
+  int32_t slotIdx = histoBinarySearch(pHisto->elems, pHisto->numOfEntries, v);
   if (pHisto->elems[slotIdx].val != v) {
     slotIdx -= 1;
 

src/query/src/qUtil.c

@@ -96,8 +96,6 @@ void resetResultRowInfo(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRo
   
   pResultRowInfo->curIndex = -1;
   pResultRowInfo->size = 0;
-  
-  pResultRowInfo->startTime = TSKEY_INITIAL_VAL;
   pResultRowInfo->prevSKey = TSKEY_INITIAL_VAL;
 }
 
@@ -110,7 +108,7 @@ void popFrontResultRow(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRow
   assert(num >= 0 && num <= numOfClosed);
 
   int16_t type = pResultRowInfo->type;
-  int64_t uid = getResultInfoUId(pRuntimeEnv);
+  int64_t uid  = 0;
 
   char    *key  = NULL;
   int16_t  bytes = -1;
@@ -181,11 +179,12 @@ void closeAllResultRows(SResultRowInfo *pResultRowInfo) {
   assert(pResultRowInfo->size >= 0 && pResultRowInfo->capacity >= pResultRowInfo->size);
   
   for (int32_t i = 0; i < pResultRowInfo->size; ++i) {
-    if (pResultRowInfo->pResult[i]->closed) {
+    SResultRow* pRow = pResultRowInfo->pResult[i];
+    if (pRow->closed) {
       continue;
     }
     
-    pResultRowInfo->pResult[i]->closed = true;
+    pRow->closed = true;
   }
 }
 
@@ -383,18 +382,4 @@ void* destroyResultRowPool(SResultRowPool* p) {
 
   tfree(p);
   return NULL;
-}
-
-uint64_t getResultInfoUId(SQueryRuntimeEnv* pRuntimeEnv) {
-  if (!pRuntimeEnv->stableQuery) {
-    return 0;  // for simple table query, the uid is always set to be 0;
-  }
-
-  SQuery* pQuery = pRuntimeEnv->pQuery;
-  if (pQuery->interval.interval == 0 || isPointInterpoQuery(pQuery) || pRuntimeEnv->groupbyNormalCol) {
-    return 0;
-  }
-
-  STableId* id = TSDB_TABLEID(pRuntimeEnv->pQuery->current->pTable);
-  return id->uid;
 }
\ No newline at end of file

src/tsdb/inc/tsdbMain.h

@@ -31,19 +31,19 @@
 extern "C" {
 #endif
 
-extern int tsdbDebugFlag;
+extern uint32_t tsdbDebugFlag;
 
-#define tsdbFatal(...) { if (tsdbDebugFlag & DEBUG_FATAL) { taosPrintLog("TDB FATAL ", 255, __VA_ARGS__); }}
-#define tsdbError(...) { if (tsdbDebugFlag & DEBUG_ERROR) { taosPrintLog("TDB ERROR ", 255, __VA_ARGS__); }}
-#define tsdbWarn(...)  { if (tsdbDebugFlag & DEBUG_WARN)  { taosPrintLog("TDB WARN ", 255, __VA_ARGS__); }}
-#define tsdbInfo(...)  { if (tsdbDebugFlag & DEBUG_INFO)  { taosPrintLog("TDB ", 255, __VA_ARGS__); }}
-#define tsdbDebug(...) { if (tsdbDebugFlag & DEBUG_DEBUG) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }}
-#define tsdbTrace(...) { if (tsdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }}
+#define tsdbFatal(...) do { if (tsdbDebugFlag & DEBUG_FATAL) { taosPrintLog("TDB FATAL ", 255, __VA_ARGS__); }}     while(0)
+#define tsdbError(...) do { if (tsdbDebugFlag & DEBUG_ERROR) { taosPrintLog("TDB ERROR ", 255, __VA_ARGS__); }}     while(0)
+#define tsdbWarn(...)  do { if (tsdbDebugFlag & DEBUG_WARN)  { taosPrintLog("TDB WARN ", 255, __VA_ARGS__); }}      while(0)
+#define tsdbInfo(...)  do { if (tsdbDebugFlag & DEBUG_INFO)  { taosPrintLog("TDB ", 255, __VA_ARGS__); }}           while(0)
+#define tsdbDebug(...) do { if (tsdbDebugFlag & DEBUG_DEBUG) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }} while(0)
+#define tsdbTrace(...) do { if (tsdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }} while(0)
 
 #define TSDB_MAX_TABLE_SCHEMAS 16
-#define TSDB_FILE_HEAD_SIZE 512
-#define TSDB_FILE_DELIMITER 0xF00AFA0F
-#define TSDB_FILE_INIT_MAGIC 0xFFFFFFFF
+#define TSDB_FILE_HEAD_SIZE    512
+#define TSDB_FILE_DELIMITER    0xF00AFA0F
+#define TSDB_FILE_INIT_MAGIC   0xFFFFFFFF
 
 #define TAOS_IN_RANGE(key, keyMin, keyLast) (((key) >= (keyMin)) && ((key) <= (keyMax)))
 

src/tsdb/src/tsdbRead.c

@@ -956,9 +956,9 @@ static int32_t loadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBl
         return code;
       }
 
-      SDataCols* pTSCol = pQueryHandle->rhelper.pDataCols[0];
+      SDataCols* pTsCol = pQueryHandle->rhelper.pDataCols[0];
       if (pCheckInfo->lastKey < pBlock->keyLast) {
-        cur->pos = binarySearchForKey(pTSCol->cols[0].pData, pBlock->numOfRows, pCheckInfo->lastKey, pQueryHandle->order);
+        cur->pos = binarySearchForKey(pTsCol->cols[0].pData, pBlock->numOfRows, pCheckInfo->lastKey, pQueryHandle->order);
       } else {
         cur->pos = pBlock->numOfRows - 1;
       }
@@ -1704,7 +1704,32 @@ static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numO
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t getDataBlocksInFilesImpl(STsdbQueryHandle* pQueryHandle, bool* exists) {
+static int32_t getFirstFileDataBlock(STsdbQueryHandle* pQueryHandle, bool* exists);
+
+static int32_t getDataBlockRv(STsdbQueryHandle* pQueryHandle, STableBlockInfo* pNext, bool *exists) {
+  int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1 : -1;
+  SQueryFilePos* cur = &pQueryHandle->cur;
+
+  while(1) {
+    int32_t code = loadFileDataBlock(pQueryHandle, pNext->compBlock, pNext->pTableCheckInfo, exists);
+    if (code != TSDB_CODE_SUCCESS || *exists) {
+      return code;
+    }
+
+    if ((cur->slot == pQueryHandle->numOfBlocks - 1 && ASCENDING_TRAVERSE(pQueryHandle->order)) ||
+        (cur->slot == 0 && !ASCENDING_TRAVERSE(pQueryHandle->order))) {
+      // all data blocks in current file has been checked already, try next file if exists
+      return getFirstFileDataBlock(pQueryHandle, exists);
+    } else {  // next block of the same file
+      cur->slot += step;
+      cur->mixBlock = false;
+      cur->blockCompleted = false;
+      pNext = &pQueryHandle->pDataBlockInfo[cur->slot];
+    }
+  }
+}
+
+static int32_t getFirstFileDataBlock(STsdbQueryHandle* pQueryHandle, bool* exists) {
   pQueryHandle->numOfBlocks = 0;
   SQueryFilePos* cur = &pQueryHandle->cur;
 
@@ -1789,7 +1814,23 @@ static int32_t getDataBlocksInFilesImpl(STsdbQueryHandle* pQueryHandle, bool* ex
   cur->fid = pQueryHandle->pFileGroup->fileId;
 
   STableBlockInfo* pBlockInfo = &pQueryHandle->pDataBlockInfo[cur->slot];
-  return loadFileDataBlock(pQueryHandle, pBlockInfo->compBlock, pBlockInfo->pTableCheckInfo, exists);
+  return getDataBlockRv(pQueryHandle, pBlockInfo, exists);
+}
+
+static bool isEndFileDataBlock(SQueryFilePos* cur, int32_t numOfBlocks, bool ascTrav) {
+  assert(cur != NULL && numOfBlocks > 0);
+  return (cur->slot == numOfBlocks - 1 && ascTrav) || (cur->slot == 0 && !ascTrav);
+}
+
+static void moveToNextDataBlockInCurrentFile(STsdbQueryHandle* pQueryHandle) {
+  int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1 : -1;
+
+  SQueryFilePos* cur = &pQueryHandle->cur;
+  assert(cur->slot < pQueryHandle->numOfBlocks && cur->slot >= 0);
+
+  cur->slot += step;
+  cur->mixBlock       = false;
+  cur->blockCompleted = false;
 }
 
 static int32_t getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle, bool* exists) {
@@ -1800,14 +1841,14 @@ static int32_t getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle, bool* exists
   if (!pQueryHandle->locateStart) {
     pQueryHandle->locateStart = true;
     STsdbCfg* pCfg = &pQueryHandle->pTsdb->config;
-    int32_t fid = getFileIdFromKey(pQueryHandle->window.skey, pCfg->daysPerFile, pCfg->precision);
+    int32_t   fid = getFileIdFromKey(pQueryHandle->window.skey, pCfg->daysPerFile, pCfg->precision);
 
     pthread_rwlock_rdlock(&pQueryHandle->pTsdb->tsdbFileH->fhlock);
     tsdbInitFileGroupIter(pFileHandle, &pQueryHandle->fileIter, pQueryHandle->order);
     tsdbSeekFileGroupIter(&pQueryHandle->fileIter, fid);
     pthread_rwlock_unlock(&pQueryHandle->pTsdb->tsdbFileH->fhlock);
 
-    return getDataBlocksInFilesImpl(pQueryHandle, exists);
+    return getFirstFileDataBlock(pQueryHandle, exists);
   } else {
     // check if current file block is all consumed
     STableBlockInfo* pBlockInfo = &pQueryHandle->pDataBlockInfo[cur->slot];
@@ -1815,27 +1856,26 @@ static int32_t getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle, bool* exists
 
     // current block is done, try next
     if ((!cur->mixBlock) || cur->blockCompleted) {
-      if ((cur->slot == pQueryHandle->numOfBlocks - 1 && ASCENDING_TRAVERSE(pQueryHandle->order)) ||
-          (cur->slot == 0 && !ASCENDING_TRAVERSE(pQueryHandle->order))) {
-        // all data blocks in current file has been checked already, try next file if exists
-        return getDataBlocksInFilesImpl(pQueryHandle, exists);
-      } else {
-        // next block of the same file
-        int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order) ? 1 : -1;
-        cur->slot += step;
-
-        cur->mixBlock = false;
-        cur->blockCompleted = false;
-
-        STableBlockInfo* pNext = &pQueryHandle->pDataBlockInfo[cur->slot];
-        return loadFileDataBlock(pQueryHandle, pNext->compBlock, pNext->pTableCheckInfo, exists);
-      }
+      // all data blocks in current file has been checked already, try next file if exists
     } else {
-      tsdbDebug("%p continue in current data block, index:%d, pos:%d, %p", pQueryHandle, cur->slot, cur->pos, pQueryHandle->qinfo);
+      tsdbDebug("%p continue in current data block, index:%d, pos:%d, %p", pQueryHandle, cur->slot, cur->pos,
+                pQueryHandle->qinfo);
       int32_t code = handleDataMergeIfNeeded(pQueryHandle, pBlockInfo->compBlock, pCheckInfo);
-      *exists = pQueryHandle->realNumOfRows > 0;
+      *exists = (pQueryHandle->realNumOfRows > 0);
 
-      return code;
+      if (code != TSDB_CODE_SUCCESS || *exists) {
+        return code;
+      }
+    }
+
+    // current block is empty, try next block in file
+    // all data blocks in current file has been checked already, try next file if exists
+    if (isEndFileDataBlock(cur, pQueryHandle->numOfBlocks, ASCENDING_TRAVERSE(pQueryHandle->order))) {
+      return getFirstFileDataBlock(pQueryHandle, exists);
+    } else {
+      moveToNextDataBlockInCurrentFile(pQueryHandle);
+      STableBlockInfo* pNext = &pQueryHandle->pDataBlockInfo[cur->slot];
+      return getDataBlockRv(pQueryHandle, pNext, exists);
     }
   }
 }

src/util/inc/tref.h

@@ -52,6 +52,8 @@ void *taosIterateRef(int rsetId, int64_t rid);
 // return the number of references in system
 int  taosListRef();  
 
+#define RID_VALID(x) ((x) > 0)
+
 /* sample code to iterate the refs 
 
 void demoIterateRefs(int rsetId) {

src/util/src/tcache.c

@@ -547,13 +547,14 @@ void taosAddToTrashcan(SCacheObj *pCacheObj, SCacheDataNode *pNode) {
     return;
   }
 
+  __cache_wr_lock(pCacheObj);
   STrashElem *pElem = calloc(1, sizeof(STrashElem));
   pElem->pData = pNode;
-  pElem->prev  = NULL;
+  pElem->prev = NULL;
+  pElem->next = NULL;
   pNode->inTrashcan = true;
   pNode->pTNodeHeader = pElem;
 
-  __cache_wr_lock(pCacheObj);
   pElem->next = pCacheObj->pTrash;
   if (pCacheObj->pTrash) {
     pCacheObj->pTrash->prev = pElem;
@@ -563,8 +564,8 @@ void taosAddToTrashcan(SCacheObj *pCacheObj, SCacheDataNode *pNode) {
   pCacheObj->numOfElemsInTrash++;
   __cache_unlock(pCacheObj);
 
-  uDebug("cache:%s key:%p, %p move to trashcan, pTrashElem:%p, numOfElem in trashcan:%d", pCacheObj->name,
-      pNode->key, pNode->data, pElem, pCacheObj->numOfElemsInTrash);
+  uDebug("cache:%s key:%p, %p move to trashcan, pTrashElem:%p, numOfElem in trashcan:%d", pCacheObj->name, pNode->key,
+         pNode->data, pElem, pCacheObj->numOfElemsInTrash);
 }
 
 void taosTrashcanEmpty(SCacheObj *pCacheObj, bool force) {

src/util/src/tskiplist.c

@@ -280,10 +280,13 @@ bool tSkipListIterNext(SSkipListIterator *iter) {
   tSkipListRLock(pSkipList);
 
   if (iter->order == TSDB_ORDER_ASC) {
-    if (iter->cur == pSkipList->pTail) {
+    // no data in the skip list
+    if (iter->cur == pSkipList->pTail || iter->next == NULL) {
+      iter->cur = pSkipList->pTail;
       tSkipListUnlock(pSkipList);
       return false;
     }
+
     iter->cur = SL_NODE_GET_FORWARD_POINTER(iter->cur, 0);
 
     // a new node is inserted into between iter->cur and iter->next, ignore it
@@ -295,9 +298,11 @@ bool tSkipListIterNext(SSkipListIterator *iter) {
     iter->step++;
   } else {
     if (iter->cur == pSkipList->pHead) {
+      iter->cur = pSkipList->pHead;
       tSkipListUnlock(pSkipList);
       return false;
     }
+
     iter->cur = SL_NODE_GET_BACKWARD_POINTER(iter->cur, 0);
 
     // a new node is inserted into between iter->cur and iter->next, ignore it

src/vnode/src/vnodeRead.c

@@ -307,8 +307,11 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pRead) {
         // NOTE: set return code to be TSDB_CODE_QRY_HAS_RSP to notify dnode to return msg to client
         code = TSDB_CODE_QRY_HAS_RSP;
       } else {
-        void *h1 = qGetResultRetrieveMsg(*qhandle);
-        assert(h1 == NULL);
+        //void *h1 = qGetResultRetrieveMsg(*qhandle);
+
+        /* remove this assert, one possible case that will cause h1 not NULL: query thread unlock pQInfo->lock, and then FETCH thread execute twice before query thread reach here */
+        //assert(h1 == NULL);
+
         freehandle = qQueryCompleted(*qhandle);
       }
 

tests/Jenkinsfile

@@ -50,12 +50,7 @@ pipeline {
           agent{label 'master'}
           steps {
             pre_test()
-            catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
-                sh '''
-                cd ${WKC}/tests/pytest
-                python3 concurrent_inquiry.py -c 1
-                '''
-            }
+            
             sh '''
             cd ${WKC}/tests
             ./test-all.sh b1
@@ -82,53 +77,26 @@ pipeline {
                 ./handle_crash_gen_val_log.sh
                 '''
             }
-            sh '''
-            cd ${WKC}/tests
-            ./test-all.sh b2
-            date
-            '''
-          }
-        }
-
-        stage('test_valgrind') {
-          agent{label "186"}
-
-          steps {
-            pre_test()
-            sh '''
-            cd ${WKC}/tests/pytest
-            ./valgrind-test.sh 2>&1 > mem-error-out.log
-            ./handle_val_log.sh
-          
-            date
-            cd ${WKC}/tests
-            ./test-all.sh b3
-            date'''
-          }
-        }
-       stage('connector'){
-         agent{label "release"}
-         steps{
             sh'''
-            cd ${WORKSPACE}
-            git checkout develop
+            systemctl start taosd 
+            sleep 10
             '''
             catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
                 sh '''
-                cd ${WORKSPACE}/tests/gotest
+                cd ${WKC}/tests/gotest
                 bash batchtest.sh
                 '''
             }
             catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
                 sh '''
-                cd ${WORKSPACE}/tests/examples/python/PYTHONConnectorChecker
+                cd ${WKC}/tests/examples/python/PYTHONConnectorChecker
                 python3 PythonChecker.py
                 '''
             }
             catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
                 sh '''
-                cd ${WORKSPACE}/tests/examples/JDBC/JDBCDemo/
-                mvn clean package assembly:single >/dev/null 
+                cd ${WKC}/tests/examples/JDBC/JDBCDemo/
+                mvn clean package assembly:single -DskipTests >/dev/null 
                 java -jar target/jdbcChecker-SNAPSHOT-jar-with-dependencies.jar -host 127.0.0.1
                 '''
             }
@@ -138,9 +106,41 @@ pipeline {
                 dotnet run
                 '''
             }
+            sh '''
+            systemctl stop taosd 
+            cd ${WKC}/tests
+            ./test-all.sh b2
+            date
+            '''
+          }
+        }
+
+        stage('test_valgrind') {
+          agent{label "186"}
+
+          steps {
+            pre_test()
+            catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
+                sh '''
+                cd ${WKC}/tests/pytest
+                nohup taosd >/dev/null &
+                sleep 10
+                python3 concurrent_inquiry.py -c 1
+                
+                '''
+            }
+            sh '''
+            cd ${WKC}/tests/pytest
+            ./valgrind-test.sh 2>&1 > mem-error-out.log
+            ./handle_val_log.sh
           
-         }
-       }
+            date
+            cd ${WKC}/tests
+            ./test-all.sh b3
+            date'''
+          }
+        }
+       
        stage('arm64_build'){
                 agent{label 'arm64'}
                 steps{

tests/pytest/concurrent_inquiry.py

@@ -388,7 +388,9 @@ class ConcurrentInquiry:
                     print(
                 "Failure thread%d, sql: %s \nexception: %s" %
                 (threadID, str(sql),str(e)))
-                    #exit(-1)
+                    err_uec='Unable to establish connection'
+                    if err_uec in str(e) and loop >0:
+                        exit(-1)
                 loop -= 1
                 if loop == 0: break
                     
@@ -415,7 +417,9 @@ class ConcurrentInquiry:
                 print(
             "Failure thread%d, sql: %s \nexception: %s" %
             (threadID, str(sql),str(e)))
-                #exit(-1)
+                err_uec='Unable to establish connection'
+                if err_uec in str(e) and loop >0:
+                    exit(-1)
             loop -= 1    
             if loop == 0: break
                 

tests/pytest/handle_crash_gen_val_log.sh

@@ -5,9 +5,10 @@ GREEN='\033[1;32m'
 GREEN_DARK='\033[0;32m'
 GREEN_UNDERLINE='\033[4;32m'
 NC='\033[0m'
-nohup /var/lib/jenkins/workspace/TDinternal/debug/build/bin/taosd -c /var/lib/jenkins/workspace/TDinternal/community/sim/dnode1/cfg >/dev/null &
+#nohup /var/lib/jenkins/workspace/TDinternal/debug/build/bin/taosd -c /var/lib/jenkins/workspace/TDinternal/community/sim/dnode1/cfg >/dev/null &
+nohup /root/TDinternal/debug/build/bin/taosd -c /root/TDinternal/community/sim/dnode1/cfg >/dev/null &
 ./crash_gen.sh --valgrind -p -t 10 -s 250 -b 4
-pidof taosd|xargs kill  
+pidof taosd|xargs kill -9
 grep 'start to execute\|ERROR SUMMARY' valgrind.err|grep -v 'grep'|uniq|tee crash_gen_mem_err.log
 
 for memError in `grep 'ERROR SUMMARY' crash_gen_mem_err.log | awk '{print $4}'`
@@ -31,4 +32,4 @@ if [ -n "$defiMemError" ]; then
     exit 8
     fi
 fi
-done
\ No newline at end of file
+done

tests/pytest/table/alter_wal0.py

@@ -43,7 +43,8 @@ class TDTestCase:
 
         print("==============step2")
         tdDnodes.stopAll()
-        filename = '/var/lib/taos/mnode/wal/wal0'
+        path = tdDnodes.getDnodesRootDir()        
+        filename = path + '/dnode1/data/mnode/wal/wal0'
 
         with open(filename, 'rb') as f1:
             temp = f1.read()
@@ -57,12 +58,12 @@ class TDTestCase:
         print("==============step3")
         tdSql.execute("use demo;")
         tdSql.query('show tables;')
-        tdSql.checkRows(10)
+        tdSql.checkRows(9)
         for i in range(11,21):
             tdSql.execute("CREATE table if not exists test{num} using meters tags({num});".format(num=i))
 
         tdSql.query('show tables;')
-        tdSql.checkRows(20)
+        tdSql.checkRows(19)
         print("==============check table numbers and create 10 tables")
 
 

tests/script/general/parser/mixed_blocks.sim

@@ -144,4 +144,20 @@ if $data03 != 319 then
   return -1
 endi
 
-system sh/exec.sh -n dnode1 -s stop -x SIGINT
\ No newline at end of file
+print ===================> TD-2488
+sql create table m1(ts timestamp, k int) tags(a int);
+sql create table t1 using m1 tags(1);
+sql create table t2 using m1 tags(2);
+sql insert into t1 values('2020-1-1 1:1:1', 1);
+sql insert into t1 values('2020-1-1 1:10:1', 2);
+sql insert into t2 values('2020-1-1 1:5:1', 99);
+
+print ================== restart server to commit data into disk
+system sh/exec.sh -n dnode1 -s stop -x SIGINT
+sleep 3000
+system sh/exec.sh -n dnode1 -s start
+print ================== server restart completed
+sql select ts from m1 where ts='2020-1-1 1:5:1'
+if $rows != 1 then
+  return -1
+endi
\ No newline at end of file

tests/test-all.sh

@@ -31,11 +31,17 @@ function runSimCaseOneByOnefq {
 			case=`echo $line | grep sim$ |awk '{print $NF}'`
 
       start_time=`date +%s`
-      ./test.sh -f $case > /dev/null 2>&1 && \
+      IN_TDINTERNAL="community"
+      if [[ "$tests_dir" == *"$IN_TDINTERNAL"* ]]; then
+        ./test.sh -f $case > /dev/null 2>&1 && \
         echo -e "${GREEN}$case success${NC}" | tee -a out.log || \
         ( grep 'script.*success.*m$' ../../../sim/tsim/log/taoslog0.0 && echo -e "${GREEN}$case success${NC}" | tee -a out.log ) || echo -e "${RED}$case failed${NC}" | tee -a out.log
-        
-
+      else
+        ./test.sh -f $case > /dev/null 2>&1 && \
+        echo -e "${GREEN}$case success${NC}" | tee -a out.log || \
+        ( grep 'script.*success.*m$' ../../sim/tsim/log/taoslog0.0 && echo -e "${GREEN}$case success${NC}" | tee -a out.log ) || echo -e "${RED}$case failed${NC}" | tee -a out.log
+      fi
+      
       out_log=`tail -1 out.log  `
       if [[ $out_log =~ 'failed' ]];then
         exit 8