提交 cc4db891 编写于 作者: W wpan

improve performance

上级 03063121
......@@ -821,17 +821,22 @@ static void doSetupSDataBlock(SSqlRes* pRes, SSDataBlock* pBlock, SFilterInfo* p
// filter data if needed
if (pFilterInfo) {
//doSetFilterColumnInfo(pFilterInfo, numOfFilterCols, pBlock);
doSetFilterColInfo(pFilterInfo, pBlock);
filterSetColFieldData(pFilterInfo, pBlock->info.numOfCols, pBlock->pDataBlock);
bool gotNchar = false;
filterConverNcharColumns(pFilterInfo, pBlock->info.rows, &gotNchar);
int8_t* p = calloc(pBlock->info.rows, sizeof(int8_t));
int8_t* p = NULL;
//bool all = doFilterDataBlock(pFilterInfo, numOfFilterCols, pBlock->info.rows, p);
bool all = filterExecute(pFilterInfo, pBlock->info.rows, p);
bool all = filterExecute(pFilterInfo, pBlock->info.rows, &p, NULL, 0);
if (gotNchar) {
filterFreeNcharColumns(pFilterInfo);
}
if (!all) {
doCompactSDataBlock(pBlock, pBlock->info.rows, p);
if (p) {
doCompactSDataBlock(pBlock, pBlock->info.rows, p);
} else {
pBlock->info.rows = 0;
pBlock->pBlockStatis = NULL; // clean the block statistics info
}
}
tfree(p);
......
......@@ -578,7 +578,6 @@ SSDataBlock* doSLimit(void* param, bool* newgroup);
int32_t doCreateFilterInfo(SColumnInfo* pCols, int32_t numOfCols, int32_t numOfFilterCols, SSingleColumnFilterInfo** pFilterInfo, uint64_t qId);
void doSetFilterColumnInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols, SSDataBlock* pBlock);
void doSetFilterColInfo(SFilterInfo *pFilters, SSDataBlock* pBlock);
bool doFilterDataBlock(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols, int32_t numOfRows, int8_t* p);
void doCompactSDataBlock(SSDataBlock* pBlock, int32_t numOfRows, int8_t* p);
......
......@@ -31,11 +31,10 @@ extern "C" {
#define FILTER_DEFAULT_GROUP_UNIT_SIZE 2
#define FILTER_DUMMY_EMPTY_OPTR 127
#define FILTER_DUMMY_RANGE_OPTR 126
#define MAX_NUM_STR_SIZE 40
#define FILTER_RM_UNIT_MIN_ROWS 1024
#define FILTER_RM_UNIT_MIN_ROWS 100
enum {
FLD_TYPE_COLUMN = 1,
......@@ -106,7 +105,7 @@ typedef struct SFilterColRange {
typedef bool (*rangeCompFunc) (const void *, const void *, const void *, const void *, __compar_fn_t);
typedef int32_t(*filter_desc_compare_func)(const void *, const void *);
typedef bool(*filter_exec_func)(void *, int32_t, int8_t*);
typedef bool(*filter_exec_func)(void *, int32_t, int8_t**, SDataStatis *, int16_t);
typedef struct SFilterRangeCompare {
int64_t s;
......@@ -278,12 +277,13 @@ typedef struct SFilterInfo {
#define CHK_RET(c, r) do { if (c) { return r; } } while (0)
#define CHK_JMP(c) do { if (c) { goto _return; } } while (0)
#define CHK_LRETV(c,...) do { if (c) { qError(__VA_ARGS__); return; } } while (0)
#define CHK_LRET(c, r,...) do { if (c) { qError(__VA_ARGS__); return r; } } while (0)
#define CHK_LRET(c, r,...) do { if (c) { if (r) {qError(__VA_ARGS__); } else { qDebug(__VA_ARGS__); } return r; } } while (0)
#define FILTER_GET_FIELD(i, id) (&((i)->fields[(id).type].fields[(id).idx]))
#define FILTER_GET_COL_FIELD(i, idx) (&((i)->fields[FLD_TYPE_COLUMN].fields[idx]))
#define FILTER_GET_COL_FIELD_TYPE(fi) (((SSchema *)((fi)->desc))->type)
#define FILTER_GET_COL_FIELD_SIZE(fi) (((SSchema *)((fi)->desc))->bytes)
#define FILTER_GET_COL_FIELD_ID(fi) (((SSchema *)((fi)->desc))->colId)
#define FILTER_GET_COL_FIELD_DESC(fi) ((SSchema *)((fi)->desc))
#define FILTER_GET_COL_FIELD_DATA(fi, ri) ((char *)(fi)->data + ((SSchema *)((fi)->desc))->bytes * (ri))
#define FILTER_GET_VAL_FIELD_TYPE(fi) (((tVariant *)((fi)->desc))->nType)
......@@ -293,10 +293,12 @@ typedef struct SFilterInfo {
#define FILTER_GROUP_UNIT(i, g, uid) ((i)->units + (g)->unitIdxs[uid])
#define FILTER_UNIT_LEFT_FIELD(i, u) FILTER_GET_FIELD(i, (u)->left)
#define FILTER_UNIT_RIGHT_FIELD(i, u) FILTER_GET_FIELD(i, (u)->right)
#define FILTER_UNIT_RIGHT2_FIELD(i, u) FILTER_GET_FIELD(i, (u)->right2)
#define FILTER_UNIT_DATA_TYPE(u) ((u)->compare.type)
#define FILTER_UNIT_COL_DESC(i, u) FILTER_GET_COL_FIELD_DESC(FILTER_UNIT_LEFT_FIELD(i, u))
#define FILTER_UNIT_COL_DATA(i, u, ri) FILTER_GET_COL_FIELD_DATA(FILTER_UNIT_LEFT_FIELD(i, u), ri)
#define FILTER_UNIT_COL_SIZE(i, u) FILTER_GET_COL_FIELD_SIZE(FILTER_UNIT_LEFT_FIELD(i, u))
#define FILTER_UNIT_COL_ID(i, u) FILTER_GET_COL_FIELD_ID(FILTER_UNIT_LEFT_FIELD(i, u))
#define FILTER_UNIT_VAL_DATA(i, u) FILTER_GET_VAL_FIELD_DATA(FILTER_UNIT_RIGHT_FIELD(i, u))
#define FILTER_UNIT_COL_IDX(u) ((u)->left.idx)
#define FILTER_UNIT_OPTR(u) ((u)->compare.optr)
......@@ -322,8 +324,8 @@ typedef struct SFilterInfo {
extern int32_t filterInitFromTree(tExprNode* tree, SFilterInfo **pinfo, uint32_t options);
extern bool filterExecute(SFilterInfo *info, int32_t numOfRows, int8_t* p);
extern int32_t filterSetColFieldData(SFilterInfo *info, int16_t colId, void *data);
extern bool filterExecute(SFilterInfo *info, int32_t numOfRows, int8_t** p, SDataStatis *statis, int16_t numOfCols);
extern int32_t filterSetColFieldData(SFilterInfo *info, int32_t numOfCols, SArray* pDataBlock);
extern int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow *win);
extern int32_t filterConverNcharColumns(SFilterInfo* pFilterInfo, int32_t rows, bool *gotNchar);
extern int32_t filterFreeNcharColumns(SFilterInfo* pFilterInfo);
......
......@@ -2774,12 +2774,13 @@ void filterRowsInDataBlock(SQueryRuntimeEnv* pRuntimeEnv, SSingleColumnFilterInf
void filterColRowsInDataBlock(SQueryRuntimeEnv* pRuntimeEnv, SSDataBlock* pBlock, bool ascQuery) {
int32_t numOfRows = pBlock->info.rows;
int8_t *p = calloc(numOfRows, sizeof(int8_t));
int8_t *p = NULL;
bool all = true;
if (pRuntimeEnv->pTsBuf != NULL) {
SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, 0);
SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, 0);
p = calloc(numOfRows, sizeof(int8_t));
TSKEY* k = (TSKEY*) pColInfoData->pData;
for (int32_t i = 0; i < numOfRows; ++i) {
int32_t offset = ascQuery? i:(numOfRows - i - 1);
......@@ -2802,11 +2803,16 @@ void filterColRowsInDataBlock(SQueryRuntimeEnv* pRuntimeEnv, SSDataBlock* pBlock
// save the cursor status
pRuntimeEnv->current->cur = tsBufGetCursor(pRuntimeEnv->pTsBuf);
} else {
all = filterExecute(pRuntimeEnv->pQueryAttr->pFilters, numOfRows, p);
all = filterExecute(pRuntimeEnv->pQueryAttr->pFilters, numOfRows, &p, pBlock->pBlockStatis, pRuntimeEnv->pQueryAttr->numOfCols);
}
if (!all) {
doCompactSDataBlock(pBlock, numOfRows, p);
if (p) {
doCompactSDataBlock(pBlock, numOfRows, p);
} else {
pBlock->info.rows = 0;
pBlock->pBlockStatis = NULL; // clean the block statistics info
}
}
tfree(p);
......@@ -2855,15 +2861,6 @@ void doSetFilterColumnInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFi
}
}
void doSetFilterColInfo(SFilterInfo * pFilters, SSDataBlock* pBlock) {
for (int32_t j = 0; j < pBlock->info.numOfCols; ++j) {
SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, j);
filterSetColFieldData(pFilters, pColInfo->info.colId, pColInfo->pData);
}
}
int32_t loadDataBlockOnDemand(SQueryRuntimeEnv* pRuntimeEnv, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock,
uint32_t* status) {
*status = BLK_DATA_NO_NEEDED;
......@@ -3003,7 +3000,7 @@ int32_t loadDataBlockOnDemand(SQueryRuntimeEnv* pRuntimeEnv, STableScanInfo* pTa
}
if (pQueryAttr->pFilters != NULL) {
doSetFilterColInfo(pQueryAttr->pFilters, pBlock);
filterSetColFieldData(pQueryAttr->pFilters, pBlock->info.numOfCols, pBlock->pDataBlock);
}
if (pQueryAttr->pFilters != NULL || pRuntimeEnv->pTsBuf != NULL) {
......
此差异已折叠。
......@@ -3,6 +3,7 @@ system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/cfg.sh -n dnode1 -c walLevel -v 1
system sh/cfg.sh -n dnode1 -c maxtablespervnode -v 4
system sh/cfg.sh -n dnode1 -c cache -v 1
system sh/exec.sh -n dnode1 -s start
sleep 100
......@@ -93,6 +94,47 @@ sql insert into tb3_2 values ('2021-05-06 18:19:28',15,NULL,15,NULL,15,NULL,true
sql insert into tb3_2 values ('2021-06-06 18:19:28',NULL,16.0,NULL,16,NULL,16.0,NULL,'16',NULL)
sql insert into tb3_2 values ('2021-07-06 18:19:28',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL)
sql create table stb4 (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9),c10 binary(16300)) TAGS(t1 int, t2 binary(10), t3 double)
sql create table tb4_0 using stb4 tags(0,'0',0.0)
sql create table tb4_1 using stb4 tags(1,'1',1.0)
sql create table tb4_2 using stb4 tags(2,'2',2.0)
sql create table tb4_3 using stb4 tags(3,'3',3.0)
sql create table tb4_4 using stb4 tags(4,'4',4.0)
$i = 0
$ts0 = 1625850000000
$blockNum = 5
$delta = 0
$tbname0 = tb4_
$a = 0
$b = 200
$c = 400
while $i < $blockNum
$x = 0
$rowNum = 1200
while $x < $rowNum
$ts = $ts0 + $x
$a = $a + 1
$b = $b + 1
$c = $c + 1
$d = $x / 10
$tin = $rowNum
$binary = 'binary . $c
$binary = $binary . '
$nchar = 'nchar . $c
$nchar = $nchar . '
$tbname = 'tb4_ . $i
$tbname = $tbname . '
sql insert into $tbname values ( $ts , $a , $b , $c , $d , $d , $c , true, $binary , $nchar , $binary )
$x = $x + 1
endw
$i = $i + 1
$ts0 = $ts0 + 259200000
endw
sleep 100
sql connect
......
......@@ -1959,6 +1959,117 @@ if $rows != 14 then
return -1
endi
sql select ts,c1 from stb4 where c1 = 200;
if $rows != 1 then
return -1
endi
if $data00 != @21-07-10 01:00:00.199@ then
return -1
endi
sql select ts,c1 from stb4 where c1 != 200;
if $rows != 5999 then
return -1
endi
sql select ts,c1,c2,c3,c4 from stb4 where c1 >= 200 and c2 > 500 and c3 < 800 and c4 between 33 and 37 and c4 != 35 and c2 < 555 and c1 < 339 and c1 in (331,333,335);
if $rows != 3 then
return -1
endi
if $data00 != @21-07-10 01:00:00.330@ then
return -1
endi
if $data10 != @21-07-10 01:00:00.332@ then
return -1
endi
if $data20 != @21-07-10 01:00:00.334@ then
return -1
endi
sql select ts,c1,c2,c3,c4 from stb4 where c1 > -3 and c1 < 5;
if $rows != 4 then
return -1
endi
if $data00 != @21-07-10 01:00:00.000@ then
return -1
endi
if $data10 != @21-07-10 01:00:00.001@ then
return -1
endi
if $data20 != @21-07-10 01:00:00.002@ then
return -1
endi
if $data30 != @21-07-10 01:00:00.003@ then
return -1
endi
sql select ts,c1,c2,c3,c4 from stb4 where c1 >= 2 and c1 < 5;
if $rows != 3 then
return -1
endi
if $data00 != @21-07-10 01:00:00.001@ then
return -1
endi
if $data10 != @21-07-10 01:00:00.002@ then
return -1
endi
if $data20 != @21-07-10 01:00:00.003@ then
return -1
endi
sql select ts,c1,c2,c3,c4 from stb4 where c1 >= -3 and c1 < 1300;
if $rows != 1299 then
return -1
endi
sql select ts,c1,c2,c3,c4 from stb4 where c1 >= 1298 and c1 < 1300 or c2 > 210 and c2 < 213;
if $rows != 4 then
return -1
endi
if $data00 != @21-07-10 01:00:00.010@ then
return -1
endi
if $data10 != @21-07-10 01:00:00.011@ then
return -1
endi
if $data20 != @21-07-13 01:00:00.097@ then
return -1
endi
if $data30 != @21-07-13 01:00:00.098@ then
return -1
endi
sql select ts,c1,c2,c3,c4 from stb4 where c1 >= -3;
if $rows != 6000 then
return -1
endi
sql select ts,c1,c2,c3,c4 from stb4 where c1 < 1400;
if $rows != 1399 then
return -1
endi
sql select ts,c1,c2,c3,c4 from stb4 where c1 < 1100;
if $rows != 1099 then
return -1
endi
sql select ts,c1,c2,c3,c4 from stb4 where c1 in(10,100, 1100,3300) and c1 != 10;
if $rows != 3 then
return -1
endi
if $data00 != @21-07-10 01:00:00.099@ then
return -1
endi
if $data10 != @21-07-10 01:00:01.099@ then
return -1
endi
if $data20 != @21-07-16 01:00:00.899@ then
return -1
endi
print "ts test"
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册