[td-225] update the sim script

src/query/src/qExecutor.c

@@ -110,7 +110,7 @@ static void resetMergeResultBuf(SQuery *pQuery, SQLFunctionCtx *pCtx, SResultInf
 static bool functionNeedToExecute(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx, int32_t functionId);
 static void getNextTimeWindow(SQuery *pQuery, STimeWindow *pTimeWindow);
 
-static void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void *inputData, TSKEY *tsCol, int32_t size,
+static void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void* pData, TSKEY *tsCol, SDataBlockInfo* pBlockInfo,
                           int32_t functionId, SDataStatis *pStatis, bool hasNull, void *param, int32_t scanFlag);
 static void initCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv);
 static void destroyTableQueryInfo(STableQueryInfo *pTableQueryInfo, int32_t numOfCols);
@@ -733,7 +733,7 @@ static int32_t getNumOfRowsInTimeWindow(SQuery *pQuery, SDataBlockInfo *pDataBlo
 }
 
 static void doBlockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SWindowStatus *pStatus, STimeWindow *pWin,
-                                      int32_t startPos, int32_t forwardStep, TSKEY *tsBuf) {
+                                      int32_t offset, int32_t forwardStep, TSKEY *tsBuf, int32_t numOfTotal) {
   SQuery *        pQuery = pRuntimeEnv->pQuery;
   SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx;
 
@@ -743,12 +743,17 @@ static void doBlockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SWindowStat
 
       pCtx[k].nStartQueryTimestamp = pWin->skey;
       pCtx[k].size = forwardStep;
-      pCtx[k].startOffset = (QUERY_IS_ASC_QUERY(pQuery)) ? startPos : startPos - (forwardStep - 1);
+      pCtx[k].startOffset = (QUERY_IS_ASC_QUERY(pQuery)) ? offset : offset - (forwardStep - 1);
 
       if ((aAggs[functionId].nStatus & TSDB_FUNCSTATE_SELECTIVITY) != 0) {
-        pCtx[k].ptsList = &tsBuf[pCtx[k].startOffset];
+        pCtx[k].ptsList = tsBuf;
       }
 
+      // not a whole block involved in query processing, statistics data can not be used
+      if (forwardStep != numOfTotal) {
+        pCtx[k].preAggVals.isSet = false;
+      }
+      
       if (functionNeedToExecute(pRuntimeEnv, &pCtx[k], functionId)) {
         aAggs[functionId].xFunction(&pCtx[k]);
       }
@@ -890,7 +895,7 @@ static char *getDataBlock(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas
       }
       
       assert(dataBlock != NULL);
-      sas->data[i] = dataBlock + pCtx->startOffset * pQuery->colList[i].bytes;  // start from the offset
+      sas->data[i] = dataBlock/* + pQuery->colList[i].bytes*/;  // start from the offset
     }
 
   } else {  // other type of query function
@@ -933,14 +938,15 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *
   
   for (int32_t k = 0; k < pQuery->numOfOutput; ++k) {
     int32_t functionId = pQuery->pSelectExpr[k].base.functionId;
+    int32_t colId = pQuery->pSelectExpr[k].base.colInfo.colId;
 
     SDataStatis *tpField = NULL;
 
     bool  hasNull = hasNullValue(pQuery, k, pDataBlockInfo->numOfCols, pStatis, &tpField);
     char *dataBlock = getDataBlock(pRuntimeEnv, &sasArray[k], k, pDataBlockInfo->rows, pDataBlock);
 
-    setExecParams(pQuery, &pCtx[k], dataBlock, primaryKeyCol, pDataBlockInfo->rows, functionId, tpField, hasNull,
-                  &sasArray[k], pRuntimeEnv->scanFlag);
+    setExecParams(pQuery, &pCtx[k], dataBlock, primaryKeyCol, pDataBlockInfo, functionId, tpField, hasNull,
+                  &sasArray[k], colId);
   }
 
   int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order);
@@ -958,7 +964,7 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *
         getNumOfRowsInTimeWindow(pQuery, pDataBlockInfo, primaryKeyCol, pQuery->pos, ekey, searchFn, true);
 
     SWindowStatus *pStatus = getTimeWindowResStatus(pWindowResInfo, curTimeWindow(pWindowResInfo));
-    doBlockwiseApplyFunctions(pRuntimeEnv, pStatus, &win, pQuery->pos, forwardStep, primaryKeyCol);
+    doBlockwiseApplyFunctions(pRuntimeEnv, pStatus, &win, pQuery->pos, forwardStep, primaryKeyCol, pDataBlockInfo->rows);
 
     int32_t     index = pWindowResInfo->curIndex;
     STimeWindow nextWin = win;
@@ -978,7 +984,7 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *
       forwardStep = getNumOfRowsInTimeWindow(pQuery, pDataBlockInfo, primaryKeyCol, startPos, ekey, searchFn, true);
 
       pStatus = getTimeWindowResStatus(pWindowResInfo, curTimeWindow(pWindowResInfo));
-      doBlockwiseApplyFunctions(pRuntimeEnv, pStatus, &nextWin, startPos, forwardStep, primaryKeyCol);
+      doBlockwiseApplyFunctions(pRuntimeEnv, pStatus, &nextWin, startPos, forwardStep, primaryKeyCol, pDataBlockInfo->rows);
     }
 
     pWindowResInfo->curIndex = index;
@@ -1154,14 +1160,15 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS
 
   for (int32_t k = 0; k < pQuery->numOfOutput; ++k) {
     int32_t functionId = pQuery->pSelectExpr[k].base.functionId;
-
+    int32_t colId = pQuery->pSelectExpr[k].base.colInfo.colId;
+    
     SDataStatis *pColStatis = NULL;
 
     bool  hasNull = hasNullValue(pQuery, k, pDataBlockInfo->numOfCols, pStatis, &pColStatis);
     char *dataBlock = getDataBlock(pRuntimeEnv, &sasArray[k], k, pDataBlockInfo->rows, pDataBlock);
 
-    setExecParams(pQuery, &pCtx[k], dataBlock, primaryKeyCol, pDataBlockInfo->rows, functionId, pColStatis, hasNull,
-                  &sasArray[k], pRuntimeEnv->scanFlag);
+    setExecParams(pQuery, &pCtx[k], dataBlock, primaryKeyCol, pDataBlockInfo, functionId, pColStatis, hasNull,
+                  &sasArray[k], colId);
   }
 
   // set the input column data
@@ -1214,7 +1221,7 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS
       }
 
       // all startOffset are identical
-      offset -= pCtx[0].startOffset;
+//      offset -= pCtx[0].startOffset;
 
       SWindowStatus *pStatus = getTimeWindowResStatus(pWindowResInfo, curTimeWindow(pWindowResInfo));
       doRowwiseApplyFunctions(pRuntimeEnv, pStatus, &win, offset);
@@ -1255,9 +1262,6 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS
         }
       }
 
-      // all startOffset are identical
-      offset -= pCtx[0].startOffset;
-
       for (int32_t k = 0; k < pQuery->numOfOutput; ++k) {
         int32_t functionId = pQuery->pSelectExpr[k].base.functionId;
         if (functionNeedToExecute(pRuntimeEnv, &pCtx[k], functionId)) {
@@ -1333,26 +1337,28 @@ static int32_t tableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBl
   return numOfRes;
 }
 
-void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void *inputData, TSKEY *tsCol, int32_t size,
-                   int32_t functionId, SDataStatis *pStatis, bool hasNull, void *param, int32_t scanFlag) {
-  pCtx->scanFlag = scanFlag;
-
+void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void* inputData, TSKEY *tsCol, SDataBlockInfo* pBlockInfo,
+                   int32_t functionId, SDataStatis *pStatis, bool hasNull, void *param, int32_t colId) {
+  pCtx->hasNull  = hasNull;
   pCtx->aInputElemBuf = inputData;
-  pCtx->hasNull = hasNull;
 
   if (pStatis != NULL) {
-    pCtx->preAggVals.isSet = true;
+    pCtx->preAggVals.isSet  = true;
     pCtx->preAggVals.statis = *pStatis;
+    if (pCtx->preAggVals.statis.numOfNull == -1) {
+      pCtx->preAggVals.statis.numOfNull = pBlockInfo->rows;  // todo :can not be -1
+    }
   } else {
     pCtx->preAggVals.isSet = false;
   }
 
-  pCtx->startOffset = QUERY_IS_ASC_QUERY(pQuery) ? pQuery->pos : 0;
-  pCtx->size = size;
+  // limit/offset query will affect this value
+  pCtx->startOffset = QUERY_IS_ASC_QUERY(pQuery) ? pQuery->pos:0;
+  pCtx->size = QUERY_IS_ASC_QUERY(pQuery) ? pBlockInfo->rows - pQuery->pos : pQuery->pos + 1;
 
   uint32_t status = aAggs[functionId].nStatus;
   if (((status & (TSDB_FUNCSTATE_SELECTIVITY | TSDB_FUNCSTATE_NEED_TS)) != 0) && (tsCol != NULL)) {
-    pCtx->ptsList = &tsCol[pCtx->startOffset];
+    pCtx->ptsList = tsCol;
   }
 
   if (functionId >= TSDB_FUNC_FIRST_DST && functionId <= TSDB_FUNC_LAST_DST) {
@@ -1362,7 +1368,7 @@ void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void *inputData, TSKEY
   } else if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_TWA ||
              functionId == TSDB_FUNC_DIFF || (functionId >= TSDB_FUNC_RATE && functionId <= TSDB_FUNC_AVG_IRATE)) {
     /*
-     * leastsquares function needs two columns of input, currently, the x value of linear equation is set to
+     * least squares function needs two columns of input, currently, the x value of linear equation is set to
      * timestamp column, and the y-value is the column specified in pQuery->pSelectExpr[i].colIdxInBuffer
      *
      * top/bottom function needs timestamp to indicate when the
@@ -1376,8 +1382,12 @@ void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void *inputData, TSKEY
 
   } else if (functionId == TSDB_FUNC_ARITHM) {
     pCtx->param[1].pz = param;
-  } else if (functionId == TSDB_FUNC_SPREAD) {
-    pCtx->
+  } else if (functionId == TSDB_FUNC_SPREAD) {  // set the statistics data for primary time stamp column
+    if (colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) {
+      pCtx->preAggVals.isSet  = true;
+      pCtx->preAggVals.statis.min = pBlockInfo->window.skey;
+      pCtx->preAggVals.statis.max = pBlockInfo->window.ekey;
+    }
   }
 
 #if defined(_DEBUG_VIEW)
@@ -3634,12 +3644,14 @@ void setIntervalQueryRange(SQInfo *pQInfo, TSKEY key) {
     STimeWindow     w = {0};
     SWindowResInfo *pWindowResInfo = &pTableQueryInfo->windowResInfo;
 
-    getAlignQueryTimeWindow(pQuery, win.skey, win.skey, win.ekey, &skey1, &ekey1, &w);
+    TSKEY sk = MIN(win.skey, win.ekey);
+    TSKEY ek = MAX(win.skey, win.ekey);
+    getAlignQueryTimeWindow(pQuery, win.skey, sk, ek, &skey1, &ekey1, &w);
     pWindowResInfo->startTime = pTableQueryInfo->win.skey;  // windowSKey may be 0 in case of 1970 timestamp
 
     if (pWindowResInfo->prevSKey == TSKEY_INITIAL_VAL) {
       if (!QUERY_IS_ASC_QUERY(pQuery)) {
-        assert(win.ekey == pQuery->window.skey);
+        assert(win.ekey == pQuery->window.ekey);
       }
       
       pWindowResInfo->prevSKey = w.skey;
@@ -3674,10 +3686,6 @@ bool needPrimaryTimestampCol(SQuery *pQuery, SDataBlockInfo *pDataBlockInfo) {
   return loadPrimaryTS;
 }
 
-bool onDemandLoadDatablock(SQuery *pQuery, int16_t queryRangeSet) {
-  return (pQuery->intervalTime == 0) || ((queryRangeSet == 1) && (isIntervalQuery(pQuery)));
-}
-
 static int32_t getNumOfSubset(SQInfo *pQInfo) {
   SQuery *pQuery = pQInfo->runtimeEnv.pQuery;
 

tests/script/general/parser/create_db.sim

@@ -104,14 +104,14 @@ $replica = 1 # max=3
 $days = 10
 $keep = 365
 $rows_db = 1000
-$cache = 4096 # 4 kb
+$cache = 16 # 16MB
 $ablocks = 100 
 $tblocks = 32 # max=512, automatically trimmed when exceeding
 $ctime = 36000 # 10 hours
 $wal = 0 # valid value is 0, 1, 2
 $comp = 1 # max=32, automatically trimmed when exceeding
 
-sql create database $db replica $replica days $days keep $keep maxrows $rows_db cache $cache ablocks $ablocks tblocks $tblocks ctime $ctime wal $wal comp $comp
+sql create database $db replica $replica days $days keep $keep maxrows $rows_db cache $cache ctime $ctime wal $wal comp $comp
 sql show databases
 if $rows != 1 then
   return -1

tests/script/general/parser/first_last_query.sim

@@ -27,32 +27,25 @@ endi
 if $data00 != @18-09-17 08:59:00.000@ then
   return -1
 endi
-#if $data01 != NULL then
 if $data01 != 0 then
   return -1   
 endi  
-#if $data02 != NULL then
 if $data02 != 0 then
   return -1  
 endi 
-#if $data03 != NULL then
 print data03 = $data03
 if $data03 != 0.00000 then
   return -1  
 endi    
-#if $data04 != NULL then
 if $data04 != 0.000000000 then
   return -1
 endi
-#if $data05 != NULL then
 if $data05 != 0 then
   return -1
 endi 
-#if $data06 != NULL then
 if $data06 != 0 then
   return -1
 endi
-#if $data07 != NULL then
 if $data07 != 1 then
   return -1
 endi

tests/script/general/parser/null_char.sim

@@ -177,7 +177,7 @@ sql_error insert into st34 using mt3 tags ('NULL', '123aBc', 105, NULL)   values
 #### case 3: set tag value
 sql create table mt4 (ts timestamp, c1 int) tags (tag_binary binary(16), tag_nchar nchar(16), tag_int int, tag_bool bool, tag_float float, tag_double double)
 sql create table st41 using mt4 tags ("beijing", 'nchar_tag', 100, false, 9.12345, 7.123456789)
-sql select tag_binary, tag_nchar, tag_int, tag_bool, tag_float, tag_double st41
+sql select tag_binary, tag_nchar, tag_int, tag_bool, tag_float, tag_double from st41
 if $rows != 1 then
   return -1
 endi
@@ -190,13 +190,17 @@ endi
 if $data02 != 100 then
   return -1
 endi
-if $data03 != false then
+if $data03 != 0 then
   return -1
 endi
-if $dat04 != 9.123450 then
+
+if $data04 != 9.12345 then
+  print expect 9.12345 , actual: $data04
   return -1
 endi
-if $data05 != 7.123457 then
+
+if $data05 != 7.123456789 then
+  print expect 7.123456789 , actual: $data05
   return -1
 endi
 

tests/script/general/parser/projection_limit_offset.sim

@@ -344,11 +344,11 @@ if $data11 != 1 then
    return -1
 endi
 
-if $data12 != null then
+if $data12 != NULL then
    return -1
 endi
 
-if $data13 != null then
+if $data13 != NULL then
    return -1
 endi
 
@@ -358,6 +358,15 @@ if $row != 8 then
    return -1
 endi
 
+sql select diff(k) from tm0
+if $row != 3 then
+   return -1
+endi
+
+if $data21 != -1 then
+   return -1
+endi
+
 #error sql
 sql_error select * from 1;
 sql_error select 1;