Merge pull request #7367 from taosdata/feature/TD-2581

[TD-2581]improve filter condition performance

src/client/src/tscUtil.c

@@ -821,17 +821,22 @@ static void doSetupSDataBlock(SSqlRes* pRes, SSDataBlock* pBlock, SFilterInfo* p
   // filter data if needed
   if (pFilterInfo) {
     //doSetFilterColumnInfo(pFilterInfo, numOfFilterCols, pBlock); 
-    doSetFilterColInfo(pFilterInfo, pBlock);
+    filterSetColFieldData(pFilterInfo, pBlock->info.numOfCols, pBlock->pDataBlock);
     bool gotNchar = false;
     filterConverNcharColumns(pFilterInfo, pBlock->info.rows, &gotNchar);
-    int8_t* p = calloc(pBlock->info.rows, sizeof(int8_t));
+    int8_t* p = NULL;
     //bool all = doFilterDataBlock(pFilterInfo, numOfFilterCols, pBlock->info.rows, p);
-    bool all = filterExecute(pFilterInfo, pBlock->info.rows, p);
+    bool all = filterExecute(pFilterInfo, pBlock->info.rows, &p, NULL, 0);
     if (gotNchar) {
       filterFreeNcharColumns(pFilterInfo);
     }
     if (!all) {
-      doCompactSDataBlock(pBlock, pBlock->info.rows, p);
+      if (p) {
+        doCompactSDataBlock(pBlock, pBlock->info.rows, p);
+      } else {
+        pBlock->info.rows = 0;
+        pBlock->pBlockStatis = NULL;  // clean the block statistics info
+      }
     }
 
     tfree(p);

src/query/inc/qExecutor.h

@@ -598,7 +598,6 @@ SSDataBlock* doSLimit(void* param, bool* newgroup);
 
 int32_t doCreateFilterInfo(SColumnInfo* pCols, int32_t numOfCols, int32_t numOfFilterCols, SSingleColumnFilterInfo** pFilterInfo, uint64_t qId);
 void doSetFilterColumnInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols, SSDataBlock* pBlock);
-void doSetFilterColInfo(SFilterInfo     *pFilters, SSDataBlock* pBlock);
 bool doFilterDataBlock(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols, int32_t numOfRows, int8_t* p);
 void doCompactSDataBlock(SSDataBlock* pBlock, int32_t numOfRows, int8_t* p);
 

src/query/inc/qFilter.h

@@ -31,10 +31,11 @@ extern "C" {
 #define FILTER_DEFAULT_GROUP_UNIT_SIZE 2
 
 #define FILTER_DUMMY_EMPTY_OPTR  127
-#define FILTER_DUMMY_RANGE_OPTR  126
 
 #define MAX_NUM_STR_SIZE 40
 
+#define FILTER_RM_UNIT_MIN_ROWS 100
+
 enum {
   FLD_TYPE_COLUMN = 1,
   FLD_TYPE_VALUE = 2,  
@@ -70,6 +71,12 @@ enum {
   FI_STATUS_CLONED = 8,
 };
 
+enum {
+  FI_STATUS_BLK_ALL = 1,
+  FI_STATUS_BLK_EMPTY = 2,
+  FI_STATUS_BLK_ACTIVE = 4,
+};
+
 enum {
   RANGE_TYPE_UNIT = 1,
   RANGE_TYPE_VAR_HASH = 2,
@@ -98,7 +105,7 @@ typedef struct SFilterColRange {
 
 typedef bool (*rangeCompFunc) (const void *, const void *, const void *, const void *, __compar_fn_t);
 typedef int32_t(*filter_desc_compare_func)(const void *, const void *);
-typedef bool(*filter_exec_func)(void *, int32_t, int8_t*);
+typedef bool(*filter_exec_func)(void *, int32_t, int8_t**, SDataStatis *, int16_t);
 
 typedef struct SFilterRangeCompare {
   int64_t s;
@@ -197,6 +204,7 @@ typedef struct SFilterComUnit {
   void *colData;
   void *valData;
   void *valData2;
+  uint16_t colId;
   uint16_t dataSize;
   uint8_t dataType;
   uint8_t optr;
@@ -224,7 +232,11 @@ typedef struct SFilterInfo {
   uint8_t          *unitRes;    // result
   uint8_t          *unitFlags;  // got result
   SFilterRangeCtx **colRange;
-  filter_exec_func  func;
+  filter_exec_func  func;          
+  uint8_t           blkFlag;
+  uint16_t          blkGroupNum;
+  uint16_t         *blkUnits;
+  int8_t           *blkUnitRes;
   
   SFilterPCtx       pctx;
 } SFilterInfo;
@@ -265,12 +277,13 @@ typedef struct SFilterInfo {
 #define CHK_RET(c, r) do { if (c) { return r; } } while (0)
 #define CHK_JMP(c) do { if (c) { goto _return; } } while (0)
 #define CHK_LRETV(c,...) do { if (c) { qError(__VA_ARGS__); return; } } while (0)
-#define CHK_LRET(c, r,...) do { if (c) { qError(__VA_ARGS__); return r; } } while (0)
+#define CHK_LRET(c, r,...) do { if (c) { if (r) {qError(__VA_ARGS__); } else { qDebug(__VA_ARGS__); } return r; } } while (0)
 
 #define FILTER_GET_FIELD(i, id) (&((i)->fields[(id).type].fields[(id).idx]))
 #define FILTER_GET_COL_FIELD(i, idx) (&((i)->fields[FLD_TYPE_COLUMN].fields[idx]))
 #define FILTER_GET_COL_FIELD_TYPE(fi) (((SSchema *)((fi)->desc))->type)
 #define FILTER_GET_COL_FIELD_SIZE(fi) (((SSchema *)((fi)->desc))->bytes)
+#define FILTER_GET_COL_FIELD_ID(fi) (((SSchema *)((fi)->desc))->colId)
 #define FILTER_GET_COL_FIELD_DESC(fi) ((SSchema *)((fi)->desc))
 #define FILTER_GET_COL_FIELD_DATA(fi, ri) ((char *)(fi)->data + ((SSchema *)((fi)->desc))->bytes * (ri))
 #define FILTER_GET_VAL_FIELD_TYPE(fi) (((tVariant *)((fi)->desc))->nType)
@@ -280,10 +293,12 @@ typedef struct SFilterInfo {
 #define FILTER_GROUP_UNIT(i, g, uid) ((i)->units + (g)->unitIdxs[uid])
 #define FILTER_UNIT_LEFT_FIELD(i, u) FILTER_GET_FIELD(i, (u)->left)
 #define FILTER_UNIT_RIGHT_FIELD(i, u) FILTER_GET_FIELD(i, (u)->right)
+#define FILTER_UNIT_RIGHT2_FIELD(i, u) FILTER_GET_FIELD(i, (u)->right2)
 #define FILTER_UNIT_DATA_TYPE(u) ((u)->compare.type)
 #define FILTER_UNIT_COL_DESC(i, u) FILTER_GET_COL_FIELD_DESC(FILTER_UNIT_LEFT_FIELD(i, u))
 #define FILTER_UNIT_COL_DATA(i, u, ri) FILTER_GET_COL_FIELD_DATA(FILTER_UNIT_LEFT_FIELD(i, u), ri)
 #define FILTER_UNIT_COL_SIZE(i, u) FILTER_GET_COL_FIELD_SIZE(FILTER_UNIT_LEFT_FIELD(i, u))
+#define FILTER_UNIT_COL_ID(i, u) FILTER_GET_COL_FIELD_ID(FILTER_UNIT_LEFT_FIELD(i, u))
 #define FILTER_UNIT_VAL_DATA(i, u) FILTER_GET_VAL_FIELD_DATA(FILTER_UNIT_RIGHT_FIELD(i, u))
 #define FILTER_UNIT_COL_IDX(u) ((u)->left.idx)
 #define FILTER_UNIT_OPTR(u) ((u)->compare.optr)
@@ -309,8 +324,8 @@ typedef struct SFilterInfo {
 
 
 extern int32_t filterInitFromTree(tExprNode* tree, SFilterInfo **pinfo, uint32_t options);
-extern bool filterExecute(SFilterInfo *info, int32_t numOfRows, int8_t* p);
-extern int32_t filterSetColFieldData(SFilterInfo *info, int16_t colId, void *data);
+extern bool filterExecute(SFilterInfo *info, int32_t numOfRows, int8_t** p, SDataStatis *statis, int16_t numOfCols);
+extern int32_t filterSetColFieldData(SFilterInfo *info, int32_t numOfCols, SArray* pDataBlock);
 extern int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow *win);
 extern int32_t filterConverNcharColumns(SFilterInfo* pFilterInfo, int32_t rows, bool *gotNchar);
 extern int32_t filterFreeNcharColumns(SFilterInfo* pFilterInfo);

src/query/src/qExecutor.c

@@ -2951,12 +2951,13 @@ void filterRowsInDataBlock(SQueryRuntimeEnv* pRuntimeEnv, SSingleColumnFilterInf
 void filterColRowsInDataBlock(SQueryRuntimeEnv* pRuntimeEnv, SSDataBlock* pBlock, bool ascQuery) {
  int32_t numOfRows = pBlock->info.rows;
 
- int8_t *p = calloc(numOfRows, sizeof(int8_t));
+ int8_t *p = NULL;
  bool    all = true;
 
  if (pRuntimeEnv->pTsBuf != NULL) {
-   SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, 0);
-
+   SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, 0);   
+   p = calloc(numOfRows, sizeof(int8_t));
+   
    TSKEY* k = (TSKEY*) pColInfoData->pData;
    for (int32_t i = 0; i < numOfRows; ++i) {
      int32_t offset = ascQuery? i:(numOfRows - i - 1);
@@ -2979,11 +2980,16 @@ void filterColRowsInDataBlock(SQueryRuntimeEnv* pRuntimeEnv, SSDataBlock* pBlock
    // save the cursor status
    pRuntimeEnv->current->cur = tsBufGetCursor(pRuntimeEnv->pTsBuf);
  } else {
-   all = filterExecute(pRuntimeEnv->pQueryAttr->pFilters, numOfRows, p);
+   all = filterExecute(pRuntimeEnv->pQueryAttr->pFilters, numOfRows, &p, pBlock->pBlockStatis, pRuntimeEnv->pQueryAttr->numOfCols);
  }
 
  if (!all) {
-   doCompactSDataBlock(pBlock, numOfRows, p);
+   if (p) {
+     doCompactSDataBlock(pBlock, numOfRows, p);
+   } else {
+     pBlock->info.rows = 0;
+     pBlock->pBlockStatis = NULL;  // clean the block statistics info
+   }
  }
 
  tfree(p);
@@ -3032,15 +3038,6 @@ void doSetFilterColumnInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFi
   }
 }
 
-
-void doSetFilterColInfo(SFilterInfo     * pFilters, SSDataBlock* pBlock) {
-  for (int32_t j = 0; j < pBlock->info.numOfCols; ++j) {
-    SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, j);
-
-    filterSetColFieldData(pFilters, pColInfo->info.colId, pColInfo->pData);
-  }
-}
-
 int32_t loadDataBlockOnDemand(SQueryRuntimeEnv* pRuntimeEnv, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock,
                               uint32_t* status) {
   *status = BLK_DATA_NO_NEEDED;
@@ -3195,7 +3192,7 @@ int32_t loadDataBlockOnDemand(SQueryRuntimeEnv* pRuntimeEnv, STableScanInfo* pTa
     }
 
     if (pQueryAttr->pFilters != NULL) {
-      doSetFilterColInfo(pQueryAttr->pFilters, pBlock);
+      filterSetColFieldData(pQueryAttr->pFilters, pBlock->info.numOfCols, pBlock->pDataBlock);
     }
     
     if (pQueryAttr->pFilters != NULL || pRuntimeEnv->pTsBuf != NULL) {

tests/script/general/parser/condition.sim

@@ -3,6 +3,7 @@ system sh/stop_dnodes.sh
 system sh/deploy.sh -n dnode1 -i 1
 system sh/cfg.sh -n dnode1 -c walLevel -v 1
 system sh/cfg.sh -n dnode1 -c maxtablespervnode -v 4
+system sh/cfg.sh -n dnode1 -c cache -v 1
 system sh/exec.sh -n dnode1 -s start
 
 sleep 100
@@ -93,6 +94,47 @@ sql insert into tb3_2 values ('2021-05-06 18:19:28',15,NULL,15,NULL,15,NULL,true
 sql insert into tb3_2 values ('2021-06-06 18:19:28',NULL,16.0,NULL,16,NULL,16.0,NULL,'16',NULL)
 sql insert into tb3_2 values ('2021-07-06 18:19:28',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL)
 
+
+sql create table stb4 (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9),c10 binary(16300)) TAGS(t1 int, t2 binary(10), t3 double)
+
+sql create table tb4_0 using stb4 tags(0,'0',0.0)
+sql create table tb4_1 using stb4 tags(1,'1',1.0)
+sql create table tb4_2 using stb4 tags(2,'2',2.0)
+sql create table tb4_3 using stb4 tags(3,'3',3.0)
+sql create table tb4_4 using stb4 tags(4,'4',4.0)
+
+$i = 0
+$ts0 = 1625850000000
+$blockNum = 5
+$delta = 0
+$tbname0 = tb4_
+$a = 0
+$b = 200
+$c = 400
+while $i < $blockNum
+  $x = 0
+  $rowNum = 1200
+  while $x < $rowNum
+    $ts = $ts0 + $x
+    $a = $a + 1
+    $b = $b + 1
+    $c = $c + 1
+    $d = $x / 10
+    $tin = $rowNum
+    $binary = 'binary . $c
+    $binary = $binary . '
+    $nchar = 'nchar . $c
+    $nchar = $nchar . '
+    $tbname = 'tb4_ . $i
+    $tbname = $tbname . '
+    sql insert into $tbname values ( $ts , $a , $b , $c , $d , $d , $c , true, $binary , $nchar , $binary )
+    $x = $x + 1
+  endw
+
+  $i = $i + 1
+  $ts0 = $ts0 + 259200000
+endw
+
 sleep 100
 
 sql connect

tests/script/general/parser/condition_query.sim

@@ -1959,6 +1959,117 @@ if $rows != 14 then
   return -1
 endi
 
+sql select ts,c1 from stb4 where c1 = 200;
+if $rows != 1 then
+  return -1
+endi
+if $data00 != @21-07-10 01:00:00.199@ then
+  return -1
+endi
+
+sql select ts,c1 from stb4 where c1 != 200;
+if $rows != 5999 then
+  return -1
+endi
+
+
+
+sql select ts,c1,c2,c3,c4 from stb4 where c1 >= 200 and c2 > 500 and c3 < 800 and c4 between 33 and 37 and c4 != 35 and c2 < 555 and c1 < 339 and c1 in (331,333,335);
+if $rows != 3 then
+  return -1
+endi
+if $data00 != @21-07-10 01:00:00.330@ then
+  return -1
+endi
+if $data10 != @21-07-10 01:00:00.332@ then
+  return -1
+endi
+if $data20 != @21-07-10 01:00:00.334@ then
+  return -1
+endi
+
+sql select ts,c1,c2,c3,c4 from stb4 where c1 > -3 and c1 < 5;
+if $rows != 4 then
+  return -1
+endi
+if $data00 != @21-07-10 01:00:00.000@ then
+  return -1
+endi
+if $data10 != @21-07-10 01:00:00.001@ then
+  return -1
+endi
+if $data20 != @21-07-10 01:00:00.002@ then
+  return -1
+endi
+if $data30 != @21-07-10 01:00:00.003@ then
+  return -1
+endi
+
+sql select ts,c1,c2,c3,c4 from stb4 where c1 >= 2 and c1 < 5;
+if $rows != 3 then
+  return -1
+endi
+if $data00 != @21-07-10 01:00:00.001@ then
+  return -1
+endi
+if $data10 != @21-07-10 01:00:00.002@ then
+  return -1
+endi
+if $data20 != @21-07-10 01:00:00.003@ then
+  return -1
+endi
+
+sql select ts,c1,c2,c3,c4 from stb4 where c1 >= -3 and c1 < 1300;
+if $rows != 1299 then
+  return -1
+endi
+
+sql select ts,c1,c2,c3,c4 from stb4 where c1 >= 1298 and c1 < 1300 or c2 > 210 and c2 < 213;
+if $rows != 4 then
+  return -1
+endi
+if $data00 != @21-07-10 01:00:00.010@ then
+  return -1
+endi
+if $data10 != @21-07-10 01:00:00.011@ then
+  return -1
+endi
+if $data20 != @21-07-13 01:00:00.097@ then
+  return -1
+endi
+if $data30 != @21-07-13 01:00:00.098@ then
+  return -1
+endi
+
+sql select ts,c1,c2,c3,c4 from stb4 where c1 >= -3;
+if $rows != 6000 then
+  return -1
+endi
+
+sql select ts,c1,c2,c3,c4 from stb4 where c1 < 1400;
+if $rows != 1399 then
+  return -1
+endi
+
+sql select ts,c1,c2,c3,c4 from stb4 where c1 < 1100;
+if $rows != 1099 then
+  return -1
+endi
+
+
+sql select ts,c1,c2,c3,c4 from stb4 where c1 in(10,100, 1100,3300) and c1 != 10;
+if $rows != 3 then
+  return -1
+endi
+if $data00 != @21-07-10 01:00:00.099@ then
+  return -1
+endi
+if $data10 != @21-07-10 01:00:01.099@ then
+  return -1
+endi
+if $data20 != @21-07-16 01:00:00.899@ then
+  return -1
+endi
 
 
 print "ts test"