Merge pull request #4299 from taosdata/release/s106

Release/s106

src/client/src/tscFunctionImpl.c

@@ -681,7 +681,7 @@ static int32_t firstFuncRequired(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, i
   }
   
   // no result for first query, data block is required
-  if (GET_RES_INFO(pCtx)->numOfRes <= 0) {
+  if (GET_RES_INFO(pCtx) == NULL || GET_RES_INFO(pCtx)->numOfRes <= 0) {
     return BLK_DATA_ALL_NEEDED;
   } else {
     return BLK_DATA_NO_NEEDED;
@@ -693,7 +693,7 @@ static int32_t lastFuncRequired(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, in
     return BLK_DATA_NO_NEEDED;
   }
   
-  if (GET_RES_INFO(pCtx)->numOfRes <= 0) {
+  if (GET_RES_INFO(pCtx) == NULL || GET_RES_INFO(pCtx)->numOfRes <= 0) {
     return BLK_DATA_ALL_NEEDED;
   } else {
     return BLK_DATA_NO_NEEDED;

src/client/src/tscSQLParser.c

@@ -5509,7 +5509,9 @@ void doAddGroupColumnForSubquery(SQueryInfo* pQueryInfo, int32_t tagIndex) {
 
   tscAddSpecialColumnForSelect(pQueryInfo, (int32_t)size, TSDB_FUNC_PRJ, &colIndex, pSchema, TSDB_COL_NORMAL);
 
-  SInternalField* pInfo = tscFieldInfoGetInternalField(&pQueryInfo->fieldsInfo, (int32_t)size);
+  int32_t numOfFields = tscNumOfFields(pQueryInfo);
+  SInternalField* pInfo = tscFieldInfoGetInternalField(&pQueryInfo->fieldsInfo, numOfFields - 1);
+
   doLimitOutputNormalColOfGroupby(pInfo->pSqlExpr);
   pInfo->visible = false;
 }

src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java

@@ -89,7 +89,7 @@ public class TSDBDriver extends AbstractTaosDriver {
     /**
      * fetch data from native function in a batch model
      */
-    public static final String PROPERTY_KEY_BATCH_LOAD = "batch";
+    public static final String PROPERTY_KEY_BATCH_LOAD = "batchfetch";
     
     private TSDBDatabaseMetaData dbMetaData = null;
 

src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetBlockData.java

@@ -31,8 +31,6 @@ import java.util.List;
 
 public class TSDBResultSetBlockData {
 	private int numOfRows = 0;
-	private int numOfCols = 0;
-
 	private int rowIndex = 0;
 
 	private List<ColumnMetaData> columnMetaDataList;
@@ -40,22 +38,20 @@ public class TSDBResultSetBlockData {
 
 	public TSDBResultSetBlockData(List<ColumnMetaData> colMeta, int numOfCols) {
 		this.columnMetaDataList = colMeta;
-		this.setNumOfCols(numOfCols);
+		this.colData = new ArrayList<Object>(numOfCols);
 	}
 
 	public TSDBResultSetBlockData() {
 		this.colData = new ArrayList<Object>();
-		this.setNumOfCols(0);
 	}
 
 	public void clear() {
+		int size = this.colData.size();
 		if (this.colData != null) {
 			this.colData.clear();
 		}
-
-		if (this.numOfCols == 0) {
-			return;
-		}
+		
+		setNumOfCols(size);
 	}
 
 	public int getNumOfRows() {
@@ -67,12 +63,12 @@ public class TSDBResultSetBlockData {
 	}
 
 	public int getNumOfCols() {
-		return numOfCols;
+		return this.colData.size();
 	}
 
 	public void setNumOfCols(int numOfCols) {
-		this.numOfCols = numOfCols;
-		this.clear();
+		this.colData = new ArrayList<Object>(numOfCols);
+		this.colData.addAll(Collections.nCopies(numOfCols, null));
 	}
 
 	public boolean hasMore() {

src/query/src/qExecutor.c

@@ -5533,10 +5533,12 @@ static void tableIntervalProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) {
   TSKEY newStartKey = TSKEY_INITIAL_VAL;
 
   // skip blocks without load the actual data block from file if no filter condition present
-  skipTimeInterval(pRuntimeEnv, &newStartKey);
-  if (pQuery->limit.offset > 0 && pQuery->numOfFilterCols == 0 && pRuntimeEnv->pFillInfo == NULL) {
-    setQueryStatus(pQuery, QUERY_COMPLETED);
-    return;
+  if (!pRuntimeEnv->groupbyNormalCol) {
+    skipTimeInterval(pRuntimeEnv, &newStartKey);
+    if (pQuery->limit.offset > 0 && pQuery->numOfFilterCols == 0 && pRuntimeEnv->pFillInfo == NULL) {
+      setQueryStatus(pQuery, QUERY_COMPLETED);
+      return;
+    }
   }
 
   while (1) {
@@ -5551,7 +5553,7 @@ static void tableIntervalProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) {
     }
 
     // no result generated, abort
-    if (pQuery->rec.rows == 0) {
+    if (pQuery->rec.rows == 0 || pRuntimeEnv->groupbyNormalCol) {
       break;
     }
 
@@ -5579,10 +5581,21 @@ static void tableIntervalProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) {
 
   // all data scanned, the group by normal column can return
   if (pRuntimeEnv->groupbyNormalCol) {  // todo refactor with merge interval time result
-    pQInfo->groupIndex = 0;
-    pQuery->rec.rows = 0;
-    copyFromWindowResToSData(pQInfo, &pRuntimeEnv->windowResInfo);
-    clearFirstNTimeWindow(pRuntimeEnv, pQInfo->groupIndex);
+    // maxOutput <= 0, means current query does not generate any results
+    int32_t numOfClosed = numOfClosedTimeWindow(&pRuntimeEnv->windowResInfo);
+
+    if ((pQuery->limit.offset > 0 && pQuery->limit.offset < numOfClosed) || pQuery->limit.offset == 0) {
+      // skip offset result rows
+      clearFirstNTimeWindow(pRuntimeEnv, (int32_t) pQuery->limit.offset);
+
+      pQuery->rec.rows   = 0;
+      pQInfo->groupIndex = 0;
+      copyFromWindowResToSData(pQInfo, &pRuntimeEnv->windowResInfo);
+      clearFirstNTimeWindow(pRuntimeEnv, pQInfo->groupIndex);
+
+      doSecondaryArithmeticProcess(pQuery);
+      limitResults(pRuntimeEnv);
+    }
   }
 }
 

src/query/src/qUtil.c

@@ -53,7 +53,7 @@ void cleanupTimeWindowInfo(SWindowResInfo *pWindowResInfo) {
     return;
   }
   if (pWindowResInfo->capacity == 0) {
-    assert(/*pWindowResInfo->hashList == NULL && */pWindowResInfo->pResult == NULL);
+    assert(pWindowResInfo->pResult == NULL);
     return;
   }
   
@@ -88,6 +88,11 @@ void clearFirstNTimeWindow(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) {
 
   int16_t  type = pWindowResInfo->type;
   STableId* id  = TSDB_TABLEID(pRuntimeEnv->pQuery->current->pTable); // uid is always set to be 0.
+  int64_t uid = id->uid;
+  if (pRuntimeEnv->groupbyNormalCol) {
+    uid = 0;
+  }
+
   char    *key  = NULL;
   int16_t  bytes = -1;
 
@@ -97,14 +102,14 @@ void clearFirstNTimeWindow(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) {
 
       // todo refactor
       if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) {
-        key = varDataVal(pResult->key);
+        key   = varDataVal(pResult->key);
         bytes = varDataLen(pResult->key);
       } else {
         key = (char*) &pResult->win.skey;
         bytes = tDataTypeDesc[pWindowResInfo->type].nSize;
       }
 
-      SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, key, bytes, id->uid);
+      SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, key, bytes, uid);
       taosHashRemove(pRuntimeEnv->pResultRowHashTable, (const char *)pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes));
     } else {
       break;
@@ -137,14 +142,14 @@ void clearFirstNTimeWindow(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) {
       bytes = tDataTypeDesc[pWindowResInfo->type].nSize;
     }
 
-    SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, key, bytes, id->uid);
+    SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, key, bytes, uid);
     int32_t *p = (int32_t *)taosHashGet(pRuntimeEnv->pResultRowHashTable, (const char *)pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes));
     assert(p != NULL); 
 
     int32_t  v = (*p - num);
     assert(v >= 0 && v <= pWindowResInfo->size);
 
-    SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, key, bytes, id->uid);
+    SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, key, bytes, uid);
     taosHashPut(pRuntimeEnv->pResultRowHashTable, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes), (char *)&v, sizeof(int32_t));
   }
   

tests/script/general/parser/groupby.sim

@@ -33,12 +33,19 @@ sql create database if not exists $db keep 36500
 sql use $db
 sql create table $mt (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9)) TAGS(t1 int, t2 binary(12))
 
+$half = $tbNum / 2
+
 $i = 0
-while $i < $tbNum
+while $i < $half
   $tb = $tbPrefix . $i
   $tg2 = ' . abc
   $tg2 = $tg2 . '
+
+  $nextSuffix = $i + $half
+  $tb1 = $tbPrefix . $nextSuffix
+
   sql create table $tb using $mt tags( $i , $tg2 )
+  sql create table $tb1 using $mt tags( $nextSuffix , $tg2 )
 
   $x = 0
   while $x < $rowNum
@@ -55,7 +62,7 @@ while $i < $tbNum
     $nchar = $nchar . $c
     $nchar = $nchar . '
 
-    sql insert into $tb values ($tstart , $c , $c , $c , $c , $c , $c , $c , $binary , $nchar )
+    sql insert into $tb values ($tstart , $c , $c , $x , $x , $c , $c , $c , $binary , $nchar ) $tb1 values ($tstart , $c , $c , $c , $c , $c , $c , $c , $binary , $nchar )
     $tstart = $tstart + 1
     $x = $x + 1
   endw
@@ -423,7 +430,108 @@ if $data97 != @group_tb0@ then
   return -1
 endi
 
+sql select count(*),first(ts),last(ts),min(c3) from group_tb1 group by c4;
+if $rows != 10000 then
+  return -1
+endi
+
+if $data00 != 1 then
+  return -1
+endi
+
+if $data01 != @70-01-01 08:01:40.000@ then
+  return -1
+endi
+
+if $data02 !=  @70-01-01 08:01:40.000@ then
+  return -1
+endi
+
+if $data03 != 0 then
+  return -1
+endi
+
+sql select count(*),first(ts),last(ts),min(c3) from group_tb1 group by c4 limit 1;
+if $rows != 1 then
+  return -1
+endi
+
+sql select count(*),first(ts),last(ts),min(c3) from group_tb1 group by c4 limit 20 offset 9990;
+if $rows != 10 then
+  return -1
+endi
+
+sql select count(*),first(ts),last(ts),min(c3),max(c3),sum(c3),avg(c3),sum(c4)/count(c4) from group_tb1 group by c4;
+if $rows != 10000 then
+  return -1
+endi
+
 print ---------------------------------> group by binary|nchar data add cases
+sql select count(*) from group_tb1 group by c8;
+if $rows != 100 then
+  return -1
+endi
+
+sql select count(*),sum(c4), count(c4), sum(c4)/count(c4) from group_tb1 group by c8
+if $rows != 100 then
+  return -1
+endi
+
+if $data00 != 100 then
+  return -1
+endi
+
+if $data01 != 495000 then
+  return -1
+endi
+
+if $data02 != 100 then
+  return -1
+endi
+
+if $data03 != 4950.000000000 then
+  print expect 4950.000000000 , acutal $data03
+  return -1
+endi
+
+if $data10 != 100 then
+  return -1
+endi
+
+if $data11 != 495100 then
+  return -1
+endi
+
+if $data13 != 4951.000000000 then
+  return -1
+endi
+
+print ====================> group by normal column + slimit + soffset
+sql select count(*), c8 from group_mt0 group by c8 limit 1 offset 0;
+if $rows != 100 then
+  return -1
+endi
+
+sql select sum(c2),c8,avg(c2), sum(c2)/count(*) from group_mt0 group by c8 slimit 2 soffset 99
+if $rows != 1 then
+  return -1
+endi
+
+if $data00 != 79200.000000000 then
+  return -1
+endi
+
+if $data01 != @binary99@ then
+  return -1
+endi
+
+if $data02 != 99.000000000 then
+  return -1
+endi
+
+if $data03 != 99.000000000 then
+  return -1
+endi
 
 
 #=========================== group by multi tags ======================

tests/script/general/parser/interp.sim

@@ -47,8 +47,7 @@ while $i < $halfNum
     $binary = $binary . '
     $nchar = 'nchar . $c
     $nchar = $nchar . '
-    sql insert into $tb values ( $ts , $c , $c , $c , $c , $c , $c , true, $binary , $nchar ) 
-    sql insert into $tb1 values ( $ts , $c , NULL , $c , NULL , $c , $c , true, $binary , $nchar ) 
+    sql insert into $tb values ( $ts , $c , $c , $c , $c , $c , $c , true, $binary , $nchar )  $tb1 values ( $ts , $c , NULL , $c , NULL , $c , $c , true, $binary , $nchar )
     $x = $x + 1
   endw 
 

tests/script/general/parser/tbnameIn.sim

@@ -55,8 +55,7 @@ while $i < $halfNum
     $binary = $binary . '
     $nchar = 'nchar . $c 
     $nchar = $nchar . '
-    sql insert into $tb values ( $ts , $c , $c , $c , $c , $c , $c , true, $binary , $nchar ) 
-    sql insert into $tb1 values ( $ts , $c , NULL , $c , NULL , $c , $c , true, $binary , $nchar ) 
+    sql insert into $tb values ( $ts , $c , $c , $c , $c , $c , $c , true, $binary , $nchar )  $tb1 values ( $ts , $c , NULL , $c , NULL , $c , $c , true, $binary , $nchar )
     $x = $x + 1
   endw 
 

tests/script/general/parser/testSuite.sim

-#sleep 500
-#run general/parser/alter.sim
-#sleep 500
-#run general/parser/alter1.sim
-#sleep 500
-#run general/parser/alter_stable.sim
-#sleep 500
-#run general/parser/auto_create_tb.sim
-#sleep 500
-#run general/parser/auto_create_tb_drop_tb.sim
-#sleep 500
-#run general/parser/col_arithmetic_operation.sim
-#sleep 500
-#run general/parser/columnValue.sim
-#sleep 500
-#run general/parser/commit.sim
-#sleep 500
-#run general/parser/create_db.sim
-#sleep 500
-#run general/parser/create_mt.sim
-#sleep 500
-#run general/parser/create_tb.sim
-#sleep 500
-#run general/parser/dbtbnameValidate.sim
-#sleep 500
-#run general/parser/fill.sim
-#sleep 500
-#run general/parser/fill_stb.sim
-#sleep 500
-##run general/parser/fill_us.sim               #
-#sleep 500
-#run general/parser/first_last.sim
-#sleep 500
-#run general/parser/import_commit1.sim
-#sleep 500
-#run general/parser/import_commit2.sim
-#sleep 500
-#run general/parser/import_commit3.sim
-#sleep 500
-##run general/parser/import_file.sim
-#sleep 500
-#run general/parser/insert_tb.sim
-#sleep 500
-#run general/parser/tags_dynamically_specifiy.sim
-#sleep 500
-#run general/parser/interp.sim
-#sleep 500
-#run general/parser/lastrow.sim
-#sleep 500
-#run general/parser/limit.sim
-#sleep 500
-#run general/parser/limit1.sim
-#sleep 500
-#run general/parser/limit1_tblocks100.sim
-#sleep 500
-#run general/parser/limit2.sim
-#sleep 500
-#run general/parser/mixed_blocks.sim
-#sleep 500
-#run general/parser/nchar.sim
-#sleep 500
-#run general/parser/null_char.sim
-#sleep 500
-#run general/parser/selectResNum.sim
+sleep 500
+run general/parser/alter.sim
+sleep 500
+run general/parser/alter1.sim
+sleep 500
+run general/parser/alter_stable.sim
+sleep 500
+run general/parser/auto_create_tb.sim
+sleep 500
+run general/parser/auto_create_tb_drop_tb.sim
+sleep 500
+run general/parser/col_arithmetic_operation.sim
+sleep 500
+run general/parser/columnValue.sim
+sleep 500
+run general/parser/commit.sim
+sleep 500
+run general/parser/create_db.sim
+sleep 500
+run general/parser/create_mt.sim
+sleep 500
+run general/parser/create_tb.sim
+sleep 500
+run general/parser/dbtbnameValidate.sim
+sleep 500
+run general/parser/fill.sim
+sleep 500
+run general/parser/fill_stb.sim
+sleep 500
+#run general/parser/fill_us.sim               #
+sleep 500
+run general/parser/first_last.sim
+sleep 500
+run general/parser/import_commit1.sim
+sleep 500
+run general/parser/import_commit2.sim
+sleep 500
+run general/parser/import_commit3.sim
+sleep 500
+#run general/parser/import_file.sim
+sleep 500
+run general/parser/insert_tb.sim
+sleep 500
+run general/parser/tags_dynamically_specifiy.sim
+sleep 500
+run general/parser/interp.sim
+sleep 500
+run general/parser/lastrow.sim
+sleep 500
+run general/parser/limit.sim
+sleep 500
+run general/parser/limit1.sim
+sleep 500
+run general/parser/limit1_tblocks100.sim
+sleep 500
+run general/parser/limit2.sim
+sleep 500
+run general/parser/mixed_blocks.sim
+sleep 500
+run general/parser/nchar.sim
+sleep 500
+run general/parser/null_char.sim
+sleep 500
+run general/parser/selectResNum.sim
 sleep 500
 run general/parser/select_across_vnodes.sim
 sleep 500

tests/script/general/parser/union.sim

@@ -36,9 +36,16 @@ sql create table $mt (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5
 $i = 0
 $t = 1578203484000
 
-while $i < $tbNum
+$half = $tbNum / 2
+
+while $i < $half
   $tb = $tbPrefix . $i
+
+  $nextSuffix = $i + $half
+  $tb1 = $tbPrefix . $nextSuffix
+
   sql create table $tb using $mt tags( $i )
+  sql create table $tb1 using $mt tags( $nextSuffix )
 
   $x = 0
   while $x < $rowNum
@@ -54,7 +61,7 @@ while $i < $tbNum
     $nchar = $nchar . '
 
     $t1 = $t + $ms
-    sql insert into $tb values ($t1 , $c , $c , $c , $c , $c , $c , $c , $binary , $nchar )
+    sql insert into $tb values ($t1 , $c , $c , $c , $c , $c , $c , $c , $binary , $nchar )  $tb1 values ($t1 , $c , $c , $c , $c , $c , $c , $c , $binary , $nchar )
     $x = $x + 1
   endw
 

tests/script/general/parser/where.sim

@@ -26,10 +26,17 @@ sql create database if not exists $db
 sql use $db
 sql create table $mt (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9)) TAGS(t1 int)
 
+$half = $tbNum / 2
+
 $i = 0
-while $i < $tbNum
+while $i < $half
   $tb = $tbPrefix . $i
+
+  $nextSuffix = $i + $half
+  $tb1 = $tbPrefix . $nextSuffix
+
   sql create table $tb using $mt tags( $i )
+  sql create table $tb1 using $mt tags( $nextSuffix )
 
   $x = 0
   while $x < $rowNum
@@ -42,7 +49,7 @@ while $i < $tbNum
     $binary = $binary . '
     $nchar = 'nchar . $c
     $nchar = $nchar . '
-    sql insert into $tb values ($ms , $c , $c , $c , $c , $c , $c , $c , $binary , $nchar )
+    sql insert into $tb values ($ms , $c , $c , $c , $c , $c , $c , $c , $binary , $nchar )  $tb1 values ($ms , $c , $c , $c , $c , $c , $c , $c , $binary , $nchar )
     $x = $x + 1
   endw