diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index 35dc94f37bccf46ba95cee119faef3921a63705e..8c11913a664730aae176e789b6bbe4c3e656de43 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -681,7 +681,7 @@ static int32_t firstFuncRequired(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, i } // no result for first query, data block is required - if (GET_RES_INFO(pCtx)->numOfRes <= 0) { + if (GET_RES_INFO(pCtx) == NULL || GET_RES_INFO(pCtx)->numOfRes <= 0) { return BLK_DATA_ALL_NEEDED; } else { return BLK_DATA_NO_NEEDED; @@ -693,7 +693,7 @@ static int32_t lastFuncRequired(SQLFunctionCtx *pCtx, TSKEY start, TSKEY end, in return BLK_DATA_NO_NEEDED; } - if (GET_RES_INFO(pCtx)->numOfRes <= 0) { + if (GET_RES_INFO(pCtx) == NULL || GET_RES_INFO(pCtx)->numOfRes <= 0) { return BLK_DATA_ALL_NEEDED; } else { return BLK_DATA_NO_NEEDED; diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 069f7303d67461580e08e0ca79c05c720d2043f1..a149b277e048ec3ab81c28b630663420d31a29e4 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -5509,7 +5509,9 @@ void doAddGroupColumnForSubquery(SQueryInfo* pQueryInfo, int32_t tagIndex) { tscAddSpecialColumnForSelect(pQueryInfo, (int32_t)size, TSDB_FUNC_PRJ, &colIndex, pSchema, TSDB_COL_NORMAL); - SInternalField* pInfo = tscFieldInfoGetInternalField(&pQueryInfo->fieldsInfo, (int32_t)size); + int32_t numOfFields = tscNumOfFields(pQueryInfo); + SInternalField* pInfo = tscFieldInfoGetInternalField(&pQueryInfo->fieldsInfo, numOfFields - 1); + doLimitOutputNormalColOfGroupby(pInfo->pSqlExpr); pInfo->visible = false; } diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java index 4dc48c84b30f363c382a641cb705893c4aca8118..06f88cebfaa8aa90cc81506a98374ec8076ad82e 100755 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java @@ -89,7 +89,7 @@ public class TSDBDriver extends AbstractTaosDriver { /** * fetch data from native function in a batch model */ - public static final String PROPERTY_KEY_BATCH_LOAD = "batch"; + public static final String PROPERTY_KEY_BATCH_LOAD = "batchfetch"; private TSDBDatabaseMetaData dbMetaData = null; diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetBlockData.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetBlockData.java index 7373890428ae0e02e4ea621595333f13824072a3..9352cf525350ff57525680f405d61c6b00c0cf55 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetBlockData.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetBlockData.java @@ -31,8 +31,6 @@ import java.util.List; public class TSDBResultSetBlockData { private int numOfRows = 0; - private int numOfCols = 0; - private int rowIndex = 0; private List columnMetaDataList; @@ -40,22 +38,20 @@ public class TSDBResultSetBlockData { public TSDBResultSetBlockData(List colMeta, int numOfCols) { this.columnMetaDataList = colMeta; - this.setNumOfCols(numOfCols); + this.colData = new ArrayList(numOfCols); } public TSDBResultSetBlockData() { this.colData = new ArrayList(); - this.setNumOfCols(0); } public void clear() { + int size = this.colData.size(); if (this.colData != null) { this.colData.clear(); } - - if (this.numOfCols == 0) { - return; - } + + setNumOfCols(size); } public int getNumOfRows() { @@ -67,12 +63,12 @@ public class TSDBResultSetBlockData { } public int getNumOfCols() { - return numOfCols; + return this.colData.size(); } public void setNumOfCols(int numOfCols) { - this.numOfCols = numOfCols; - this.clear(); + this.colData = new ArrayList(numOfCols); + this.colData.addAll(Collections.nCopies(numOfCols, null)); } public boolean hasMore() { diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 2096cf4766843b81014baa972f0d3cc1d5733dc0..6bd4c868b3fc98b6da6b29f93a9be9630043769f 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -5533,10 +5533,12 @@ static void tableIntervalProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) { TSKEY newStartKey = TSKEY_INITIAL_VAL; // skip blocks without load the actual data block from file if no filter condition present - skipTimeInterval(pRuntimeEnv, &newStartKey); - if (pQuery->limit.offset > 0 && pQuery->numOfFilterCols == 0 && pRuntimeEnv->pFillInfo == NULL) { - setQueryStatus(pQuery, QUERY_COMPLETED); - return; + if (!pRuntimeEnv->groupbyNormalCol) { + skipTimeInterval(pRuntimeEnv, &newStartKey); + if (pQuery->limit.offset > 0 && pQuery->numOfFilterCols == 0 && pRuntimeEnv->pFillInfo == NULL) { + setQueryStatus(pQuery, QUERY_COMPLETED); + return; + } } while (1) { @@ -5551,7 +5553,7 @@ static void tableIntervalProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) { } // no result generated, abort - if (pQuery->rec.rows == 0) { + if (pQuery->rec.rows == 0 || pRuntimeEnv->groupbyNormalCol) { break; } @@ -5579,10 +5581,21 @@ static void tableIntervalProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) { // all data scanned, the group by normal column can return if (pRuntimeEnv->groupbyNormalCol) { // todo refactor with merge interval time result - pQInfo->groupIndex = 0; - pQuery->rec.rows = 0; - copyFromWindowResToSData(pQInfo, &pRuntimeEnv->windowResInfo); - clearFirstNTimeWindow(pRuntimeEnv, pQInfo->groupIndex); + // maxOutput <= 0, means current query does not generate any results + int32_t numOfClosed = numOfClosedTimeWindow(&pRuntimeEnv->windowResInfo); + + if ((pQuery->limit.offset > 0 && pQuery->limit.offset < numOfClosed) || pQuery->limit.offset == 0) { + // skip offset result rows + clearFirstNTimeWindow(pRuntimeEnv, (int32_t) pQuery->limit.offset); + + pQuery->rec.rows = 0; + pQInfo->groupIndex = 0; + copyFromWindowResToSData(pQInfo, &pRuntimeEnv->windowResInfo); + clearFirstNTimeWindow(pRuntimeEnv, pQInfo->groupIndex); + + doSecondaryArithmeticProcess(pQuery); + limitResults(pRuntimeEnv); + } } } diff --git a/src/query/src/qUtil.c b/src/query/src/qUtil.c index f4f89a8709a16d0d84ff8333cc6ef3d4463b6c64..61d080bb6c2c973e4a5e4b04c4bf8acff201c31f 100644 --- a/src/query/src/qUtil.c +++ b/src/query/src/qUtil.c @@ -53,7 +53,7 @@ void cleanupTimeWindowInfo(SWindowResInfo *pWindowResInfo) { return; } if (pWindowResInfo->capacity == 0) { - assert(/*pWindowResInfo->hashList == NULL && */pWindowResInfo->pResult == NULL); + assert(pWindowResInfo->pResult == NULL); return; } @@ -88,6 +88,11 @@ void clearFirstNTimeWindow(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) { int16_t type = pWindowResInfo->type; STableId* id = TSDB_TABLEID(pRuntimeEnv->pQuery->current->pTable); // uid is always set to be 0. + int64_t uid = id->uid; + if (pRuntimeEnv->groupbyNormalCol) { + uid = 0; + } + char *key = NULL; int16_t bytes = -1; @@ -97,14 +102,14 @@ void clearFirstNTimeWindow(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) { // todo refactor if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { - key = varDataVal(pResult->key); + key = varDataVal(pResult->key); bytes = varDataLen(pResult->key); } else { key = (char*) &pResult->win.skey; bytes = tDataTypeDesc[pWindowResInfo->type].nSize; } - SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, key, bytes, id->uid); + SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, key, bytes, uid); taosHashRemove(pRuntimeEnv->pResultRowHashTable, (const char *)pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); } else { break; @@ -137,14 +142,14 @@ void clearFirstNTimeWindow(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) { bytes = tDataTypeDesc[pWindowResInfo->type].nSize; } - SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, key, bytes, id->uid); + SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, key, bytes, uid); int32_t *p = (int32_t *)taosHashGet(pRuntimeEnv->pResultRowHashTable, (const char *)pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); assert(p != NULL); int32_t v = (*p - num); assert(v >= 0 && v <= pWindowResInfo->size); - SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, key, bytes, id->uid); + SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, key, bytes, uid); taosHashPut(pRuntimeEnv->pResultRowHashTable, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes), (char *)&v, sizeof(int32_t)); } diff --git a/tests/script/general/parser/groupby.sim b/tests/script/general/parser/groupby.sim index c0a6709fe4b4369a5415860b87d949351a85c7ca..48298733ec72ecfd9ffdf0cba80aee732ee99b4c 100644 --- a/tests/script/general/parser/groupby.sim +++ b/tests/script/general/parser/groupby.sim @@ -33,12 +33,19 @@ sql create database if not exists $db keep 36500 sql use $db sql create table $mt (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9)) TAGS(t1 int, t2 binary(12)) +$half = $tbNum / 2 + $i = 0 -while $i < $tbNum +while $i < $half $tb = $tbPrefix . $i $tg2 = ' . abc $tg2 = $tg2 . ' + + $nextSuffix = $i + $half + $tb1 = $tbPrefix . $nextSuffix + sql create table $tb using $mt tags( $i , $tg2 ) + sql create table $tb1 using $mt tags( $nextSuffix , $tg2 ) $x = 0 while $x < $rowNum @@ -55,7 +62,7 @@ while $i < $tbNum $nchar = $nchar . $c $nchar = $nchar . ' - sql insert into $tb values ($tstart , $c , $c , $c , $c , $c , $c , $c , $binary , $nchar ) + sql insert into $tb values ($tstart , $c , $c , $x , $x , $c , $c , $c , $binary , $nchar ) $tb1 values ($tstart , $c , $c , $c , $c , $c , $c , $c , $binary , $nchar ) $tstart = $tstart + 1 $x = $x + 1 endw @@ -423,7 +430,108 @@ if $data97 != @group_tb0@ then return -1 endi +sql select count(*),first(ts),last(ts),min(c3) from group_tb1 group by c4; +if $rows != 10000 then + return -1 +endi + +if $data00 != 1 then + return -1 +endi + +if $data01 != @70-01-01 08:01:40.000@ then + return -1 +endi + +if $data02 != @70-01-01 08:01:40.000@ then + return -1 +endi + +if $data03 != 0 then + return -1 +endi + +sql select count(*),first(ts),last(ts),min(c3) from group_tb1 group by c4 limit 1; +if $rows != 1 then + return -1 +endi + +sql select count(*),first(ts),last(ts),min(c3) from group_tb1 group by c4 limit 20 offset 9990; +if $rows != 10 then + return -1 +endi + +sql select count(*),first(ts),last(ts),min(c3),max(c3),sum(c3),avg(c3),sum(c4)/count(c4) from group_tb1 group by c4; +if $rows != 10000 then + return -1 +endi + print ---------------------------------> group by binary|nchar data add cases +sql select count(*) from group_tb1 group by c8; +if $rows != 100 then + return -1 +endi + +sql select count(*),sum(c4), count(c4), sum(c4)/count(c4) from group_tb1 group by c8 +if $rows != 100 then + return -1 +endi + +if $data00 != 100 then + return -1 +endi + +if $data01 != 495000 then + return -1 +endi + +if $data02 != 100 then + return -1 +endi + +if $data03 != 4950.000000000 then + print expect 4950.000000000 , acutal $data03 + return -1 +endi + +if $data10 != 100 then + return -1 +endi + +if $data11 != 495100 then + return -1 +endi + +if $data13 != 4951.000000000 then + return -1 +endi + +print ====================> group by normal column + slimit + soffset +sql select count(*), c8 from group_mt0 group by c8 limit 1 offset 0; +if $rows != 100 then + return -1 +endi + +sql select sum(c2),c8,avg(c2), sum(c2)/count(*) from group_mt0 group by c8 slimit 2 soffset 99 +if $rows != 1 then + return -1 +endi + +if $data00 != 79200.000000000 then + return -1 +endi + +if $data01 != @binary99@ then + return -1 +endi + +if $data02 != 99.000000000 then + return -1 +endi + +if $data03 != 99.000000000 then + return -1 +endi #=========================== group by multi tags ====================== diff --git a/tests/script/general/parser/interp.sim b/tests/script/general/parser/interp.sim index 8777c3ed9bd4cb22b72253b6503e8c78b96bb703..0d5c1804dd6cd4e929aaaa4de2e6775cc5b71153 100644 --- a/tests/script/general/parser/interp.sim +++ b/tests/script/general/parser/interp.sim @@ -47,8 +47,7 @@ while $i < $halfNum $binary = $binary . ' $nchar = 'nchar . $c $nchar = $nchar . ' - sql insert into $tb values ( $ts , $c , $c , $c , $c , $c , $c , true, $binary , $nchar ) - sql insert into $tb1 values ( $ts , $c , NULL , $c , NULL , $c , $c , true, $binary , $nchar ) + sql insert into $tb values ( $ts , $c , $c , $c , $c , $c , $c , true, $binary , $nchar ) $tb1 values ( $ts , $c , NULL , $c , NULL , $c , $c , true, $binary , $nchar ) $x = $x + 1 endw diff --git a/tests/script/general/parser/tbnameIn.sim b/tests/script/general/parser/tbnameIn.sim index 87c48717926a535f77e73e69f02464b947a42ee6..d0f74ae53d5c696cb598b471dfae32307dfd0f73 100644 --- a/tests/script/general/parser/tbnameIn.sim +++ b/tests/script/general/parser/tbnameIn.sim @@ -55,8 +55,7 @@ while $i < $halfNum $binary = $binary . ' $nchar = 'nchar . $c $nchar = $nchar . ' - sql insert into $tb values ( $ts , $c , $c , $c , $c , $c , $c , true, $binary , $nchar ) - sql insert into $tb1 values ( $ts , $c , NULL , $c , NULL , $c , $c , true, $binary , $nchar ) + sql insert into $tb values ( $ts , $c , $c , $c , $c , $c , $c , true, $binary , $nchar ) $tb1 values ( $ts , $c , NULL , $c , NULL , $c , $c , true, $binary , $nchar ) $x = $x + 1 endw diff --git a/tests/script/general/parser/testSuite.sim b/tests/script/general/parser/testSuite.sim index 0e123796597c3d5009860aa85d5c2a73084a9f72..7c9dd2b6b5f0192bc29fd076acfd0a3a71f9ab76 100644 --- a/tests/script/general/parser/testSuite.sim +++ b/tests/script/general/parser/testSuite.sim @@ -1,67 +1,67 @@ -#sleep 500 -#run general/parser/alter.sim -#sleep 500 -#run general/parser/alter1.sim -#sleep 500 -#run general/parser/alter_stable.sim -#sleep 500 -#run general/parser/auto_create_tb.sim -#sleep 500 -#run general/parser/auto_create_tb_drop_tb.sim -#sleep 500 -#run general/parser/col_arithmetic_operation.sim -#sleep 500 -#run general/parser/columnValue.sim -#sleep 500 -#run general/parser/commit.sim -#sleep 500 -#run general/parser/create_db.sim -#sleep 500 -#run general/parser/create_mt.sim -#sleep 500 -#run general/parser/create_tb.sim -#sleep 500 -#run general/parser/dbtbnameValidate.sim -#sleep 500 -#run general/parser/fill.sim -#sleep 500 -#run general/parser/fill_stb.sim -#sleep 500 -##run general/parser/fill_us.sim # -#sleep 500 -#run general/parser/first_last.sim -#sleep 500 -#run general/parser/import_commit1.sim -#sleep 500 -#run general/parser/import_commit2.sim -#sleep 500 -#run general/parser/import_commit3.sim -#sleep 500 -##run general/parser/import_file.sim -#sleep 500 -#run general/parser/insert_tb.sim -#sleep 500 -#run general/parser/tags_dynamically_specifiy.sim -#sleep 500 -#run general/parser/interp.sim -#sleep 500 -#run general/parser/lastrow.sim -#sleep 500 -#run general/parser/limit.sim -#sleep 500 -#run general/parser/limit1.sim -#sleep 500 -#run general/parser/limit1_tblocks100.sim -#sleep 500 -#run general/parser/limit2.sim -#sleep 500 -#run general/parser/mixed_blocks.sim -#sleep 500 -#run general/parser/nchar.sim -#sleep 500 -#run general/parser/null_char.sim -#sleep 500 -#run general/parser/selectResNum.sim +sleep 500 +run general/parser/alter.sim +sleep 500 +run general/parser/alter1.sim +sleep 500 +run general/parser/alter_stable.sim +sleep 500 +run general/parser/auto_create_tb.sim +sleep 500 +run general/parser/auto_create_tb_drop_tb.sim +sleep 500 +run general/parser/col_arithmetic_operation.sim +sleep 500 +run general/parser/columnValue.sim +sleep 500 +run general/parser/commit.sim +sleep 500 +run general/parser/create_db.sim +sleep 500 +run general/parser/create_mt.sim +sleep 500 +run general/parser/create_tb.sim +sleep 500 +run general/parser/dbtbnameValidate.sim +sleep 500 +run general/parser/fill.sim +sleep 500 +run general/parser/fill_stb.sim +sleep 500 +#run general/parser/fill_us.sim # +sleep 500 +run general/parser/first_last.sim +sleep 500 +run general/parser/import_commit1.sim +sleep 500 +run general/parser/import_commit2.sim +sleep 500 +run general/parser/import_commit3.sim +sleep 500 +#run general/parser/import_file.sim +sleep 500 +run general/parser/insert_tb.sim +sleep 500 +run general/parser/tags_dynamically_specifiy.sim +sleep 500 +run general/parser/interp.sim +sleep 500 +run general/parser/lastrow.sim +sleep 500 +run general/parser/limit.sim +sleep 500 +run general/parser/limit1.sim +sleep 500 +run general/parser/limit1_tblocks100.sim +sleep 500 +run general/parser/limit2.sim +sleep 500 +run general/parser/mixed_blocks.sim +sleep 500 +run general/parser/nchar.sim +sleep 500 +run general/parser/null_char.sim +sleep 500 +run general/parser/selectResNum.sim sleep 500 run general/parser/select_across_vnodes.sim sleep 500 diff --git a/tests/script/general/parser/union.sim b/tests/script/general/parser/union.sim index d779d757310336b3b97c6748301e1eab24fe331f..cb46ac6b0d1a6d381d33974660b6584e12faff44 100644 --- a/tests/script/general/parser/union.sim +++ b/tests/script/general/parser/union.sim @@ -36,9 +36,16 @@ sql create table $mt (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 $i = 0 $t = 1578203484000 -while $i < $tbNum +$half = $tbNum / 2 + +while $i < $half $tb = $tbPrefix . $i + + $nextSuffix = $i + $half + $tb1 = $tbPrefix . $nextSuffix + sql create table $tb using $mt tags( $i ) + sql create table $tb1 using $mt tags( $nextSuffix ) $x = 0 while $x < $rowNum @@ -54,7 +61,7 @@ while $i < $tbNum $nchar = $nchar . ' $t1 = $t + $ms - sql insert into $tb values ($t1 , $c , $c , $c , $c , $c , $c , $c , $binary , $nchar ) + sql insert into $tb values ($t1 , $c , $c , $c , $c , $c , $c , $c , $binary , $nchar ) $tb1 values ($t1 , $c , $c , $c , $c , $c , $c , $c , $binary , $nchar ) $x = $x + 1 endw diff --git a/tests/script/general/parser/where.sim b/tests/script/general/parser/where.sim index 4a38e3c68c64252ab9c53926c2ff33cfddbee2ae..8e17220b5b38f995c6dc7e662130f7250157aa44 100644 --- a/tests/script/general/parser/where.sim +++ b/tests/script/general/parser/where.sim @@ -26,10 +26,17 @@ sql create database if not exists $db sql use $db sql create table $mt (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9)) TAGS(t1 int) +$half = $tbNum / 2 + $i = 0 -while $i < $tbNum +while $i < $half $tb = $tbPrefix . $i + + $nextSuffix = $i + $half + $tb1 = $tbPrefix . $nextSuffix + sql create table $tb using $mt tags( $i ) + sql create table $tb1 using $mt tags( $nextSuffix ) $x = 0 while $x < $rowNum @@ -42,7 +49,7 @@ while $i < $tbNum $binary = $binary . ' $nchar = 'nchar . $c $nchar = $nchar . ' - sql insert into $tb values ($ms , $c , $c , $c , $c , $c , $c , $c , $binary , $nchar ) + sql insert into $tb values ($ms , $c , $c , $c , $c , $c , $c , $c , $binary , $nchar ) $tb1 values ($ms , $c , $c , $c , $c , $c , $c , $c , $binary , $nchar ) $x = $x + 1 endw