Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
lzh_me
TDengine
提交
aa8b12e2
TDengine
项目概览
lzh_me
/
TDengine
与 Fork 源项目一致
Fork自
taosdata / TDengine
通知
1
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
TDengine
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
aa8b12e2
编写于
8月 06, 2021
作者:
Y
yihaoDeng
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
[TD-5797]<feature> support multi distinct
上级
bc07960f
变更
3
隐藏空白更改
内联
并排
Showing
3 changed file
with
101 addition
and
58 deletion
+101
-58
src/client/src/tscSQLParser.c
src/client/src/tscSQLParser.c
+4
-4
src/query/inc/qExecutor.h
src/query/inc/qExecutor.h
+9
-1
src/query/src/qExecutor.c
src/query/src/qExecutor.c
+88
-53
未找到文件。
src/client/src/tscSQLParser.c
浏览文件 @
aa8b12e2
...
...
@@ -1948,10 +1948,10 @@ bool isValidDistinctSql(SQueryInfo* pQueryInfo) {
&&
(
pQueryInfo
->
type
&
TSDB_QUERY_TYPE_TABLE_QUERY
)
!=
TSDB_QUERY_TYPE_TABLE_QUERY
)
{
return
false
;
}
if
(
tscNumOfExprs
(
pQueryInfo
)
==
1
){
return
true
;
}
return
fals
e
;
//
if (tscNumOfExprs(pQueryInfo) == 1){
//
return true;
//
}
return
tru
e
;
}
static
bool
hasNoneUserDefineExpr
(
SQueryInfo
*
pQueryInfo
)
{
...
...
src/query/inc/qExecutor.h
浏览文件 @
aa8b12e2
...
...
@@ -508,13 +508,21 @@ typedef struct SStateWindowOperatorInfo {
bool
reptScan
;
}
SStateWindowOperatorInfo
;
typedef
struct
SDistinctDataInfo
{
int32_t
index
;
int32_t
type
;
int32_t
bytes
;
}
SDistinctDataInfo
;
typedef
struct
SDistinctOperatorInfo
{
SHashObj
*
pSet
;
SSDataBlock
*
pRes
;
bool
recordNullVal
;
//has already record the null value, no need to try again
int64_t
threshold
;
int64_t
outputCapacity
;
int32_t
colIndex
;
int32_t
totalBytes
;
char
*
buf
;
SArray
*
pDistinctDataInfo
;
}
SDistinctOperatorInfo
;
struct
SGlobalMerger
;
...
...
src/query/src/qExecutor.c
浏览文件 @
aa8b12e2
...
...
@@ -44,6 +44,10 @@
#define SDATA_BLOCK_INITIALIZER (SDataBlockInfo) {{0}, 0}
#define MULTI_KEY_DELIM "-"
#define HASH_CAPACITY_LIMIT 10000000
#define TIME_WINDOW_COPY(_dst, _src) do {\
(_dst).skey = (_src).skey;\
(_dst).ekey = (_src).ekey;\
...
...
@@ -6109,6 +6113,8 @@ static void destroyConditionOperatorInfo(void* param, int32_t numOfOutput) {
static
void
destroyDistinctOperatorInfo
(
void
*
param
,
int32_t
numOfOutput
)
{
SDistinctOperatorInfo
*
pInfo
=
(
SDistinctOperatorInfo
*
)
param
;
taosHashCleanup
(
pInfo
->
pSet
);
tfree
(
pInfo
->
buf
);
taosArrayDestroy
(
pInfo
->
pDistinctDataInfo
);
pInfo
->
pRes
=
destroyOutputBuf
(
pInfo
->
pRes
);
}
...
...
@@ -6600,20 +6606,65 @@ SOperatorInfo* createTagScanOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SExprInf
return
pOperator
;
}
static
bool
initMultiDistinctInfo
(
SDistinctOperatorInfo
*
pInfo
,
SOperatorInfo
*
pOperator
,
SSDataBlock
*
pBlock
)
{
if
(
taosArrayGetSize
(
pInfo
->
pDistinctDataInfo
)
==
pOperator
->
numOfOutput
)
{
// distinct info already inited
return
true
;
}
for
(
int
i
=
0
;
i
<
pOperator
->
numOfOutput
;
i
++
)
{
pInfo
->
totalBytes
+=
pOperator
->
pExpr
[
i
].
base
.
colBytes
;
}
for
(
int
i
=
0
;
i
<
pOperator
->
numOfOutput
;
i
++
)
{
int
numOfBlock
=
taosArrayGetSize
(
pBlock
->
pDataBlock
);
assert
(
i
<
numOfBlock
);
for
(
int
j
=
0
;
j
<
numOfBlock
;
j
++
)
{
SColumnInfoData
*
pColDataInfo
=
taosArrayGet
(
pBlock
->
pDataBlock
,
j
);
if
(
pColDataInfo
->
info
.
colId
==
pOperator
->
pExpr
[
i
].
base
.
resColId
)
{
SDistinctDataInfo
item
=
{.
index
=
j
,
.
type
=
pColDataInfo
->
info
.
type
,
.
bytes
=
pColDataInfo
->
info
.
bytes
};
taosArrayInsert
(
pInfo
->
pDistinctDataInfo
,
i
,
&
item
);
}
}
}
pInfo
->
totalBytes
+=
strlen
(
MULTI_KEY_DELIM
)
*
(
pOperator
->
numOfOutput
);
pInfo
->
buf
=
calloc
(
1
,
pInfo
->
totalBytes
);
return
taosArrayGetSize
(
pInfo
->
pDistinctDataInfo
)
==
pOperator
->
numOfOutput
?
true
:
false
;
}
static
void
buildMultiDistinctKey
(
SDistinctOperatorInfo
*
pInfo
,
SSDataBlock
*
pBlock
,
int32_t
rowId
)
{
char
*
p
=
pInfo
->
buf
;
memset
(
p
,
0
,
pInfo
->
totalBytes
);
for
(
int
i
=
0
;
i
<
taosArrayGetSize
(
pInfo
->
pDistinctDataInfo
);
i
++
)
{
SDistinctDataInfo
*
pDistDataInfo
=
(
SDistinctDataInfo
*
)
taosArrayGet
(
pInfo
->
pDistinctDataInfo
,
i
);
SColumnInfoData
*
pColDataInfo
=
taosArrayGet
(
pBlock
->
pDataBlock
,
pDistDataInfo
->
index
);
char
*
val
=
((
char
*
)
pColDataInfo
->
pData
)
+
pColDataInfo
->
info
.
bytes
*
rowId
;
if
(
isNull
(
val
,
pDistDataInfo
->
type
))
{
p
+=
pDistDataInfo
->
bytes
;
continue
;
}
if
(
IS_VAR_DATA_TYPE
(
pDistDataInfo
->
type
))
{
memcpy
(
p
,
varDataVal
(
val
),
varDataLen
(
val
));
p
+=
varDataLen
(
val
);
}
else
{
memcpy
(
p
,
val
,
pDistDataInfo
->
bytes
);
p
+=
pDistDataInfo
->
bytes
;
}
memcpy
(
p
,
MULTI_KEY_DELIM
,
strlen
(
MULTI_KEY_DELIM
));
p
+=
strlen
(
MULTI_KEY_DELIM
);
}
}
static
SSDataBlock
*
hashDistinct
(
void
*
param
,
bool
*
newgroup
)
{
SOperatorInfo
*
pOperator
=
(
SOperatorInfo
*
)
param
;
if
(
pOperator
->
status
==
OP_EXEC_DONE
)
{
return
NULL
;
}
SDistinctOperatorInfo
*
pInfo
=
pOperator
->
info
;
SSDataBlock
*
pRes
=
pInfo
->
pRes
;
pRes
->
info
.
rows
=
0
;
SSDataBlock
*
pBlock
=
NULL
;
while
(
1
)
{
publishOperatorProfEvent
(
pOperator
->
upstream
[
0
],
QUERY_PROF_BEFORE_OPERATOR_EXEC
);
pBlock
=
pOperator
->
upstream
[
0
]
->
exec
(
pOperator
->
upstream
[
0
],
newgroup
);
...
...
@@ -6624,63 +6675,44 @@ static SSDataBlock* hashDistinct(void* param, bool* newgroup) {
pOperator
->
status
=
OP_EXEC_DONE
;
break
;
}
if
(
pInfo
->
colIndex
==
-
1
)
{
for
(
int
i
=
0
;
i
<
taosArrayGetSize
(
pBlock
->
pDataBlock
);
i
++
)
{
SColumnInfoData
*
pColDataInfo
=
taosArrayGet
(
pBlock
->
pDataBlock
,
i
);
if
(
pColDataInfo
->
info
.
colId
==
pOperator
->
pExpr
[
0
].
base
.
resColId
)
{
pInfo
->
colIndex
=
i
;
break
;
}
}
}
if
(
pInfo
->
colIndex
==
-
1
)
{
if
(
!
initMultiDistinctInfo
(
pInfo
,
pOperator
,
pBlock
))
{
setQueryStatus
(
pOperator
->
pRuntimeEnv
,
QUERY_COMPLETED
);
pOperator
->
status
=
OP_EXEC_DONE
;
return
NULL
;
break
;
}
SColumnInfoData
*
pColInfoData
=
taosArrayGet
(
pBlock
->
pDataBlock
,
pInfo
->
colIndex
);
int16_t
bytes
=
pColInfoData
->
info
.
bytes
;
int16_t
type
=
pColInfoData
->
info
.
type
;
// ensure the output buffer size
SColumnInfoData
*
pResultColInfoData
=
taosArrayGet
(
pRes
->
pDataBlock
,
0
);
// ensure result output buf
if
(
pRes
->
info
.
rows
+
pBlock
->
info
.
rows
>
pInfo
->
outputCapacity
)
{
int32_t
newSize
=
pRes
->
info
.
rows
+
pBlock
->
info
.
rows
;
char
*
tmp
=
realloc
(
pResultColInfoData
->
pData
,
newSize
*
bytes
);
if
(
tmp
==
NULL
)
{
return
NULL
;
}
else
{
pResultColInfoData
->
pData
=
tmp
;
pInfo
->
outputCapacity
=
newSize
;
for
(
int
i
=
0
;
i
<
taosArrayGetSize
(
pRes
->
pDataBlock
);
i
++
)
{
SColumnInfoData
*
pResultColInfoData
=
taosArrayGet
(
pRes
->
pDataBlock
,
i
);
SDistinctDataInfo
*
pDistDataInfo
=
taosArrayGet
(
pInfo
->
pDistinctDataInfo
,
i
);
char
*
tmp
=
realloc
(
pResultColInfoData
->
pData
,
newSize
*
pDistDataInfo
->
bytes
);
if
(
tmp
==
NULL
)
{
return
NULL
;
}
else
{
pResultColInfoData
->
pData
=
tmp
;
}
}
pInfo
->
outputCapacity
=
newSize
;
}
for
(
int32_t
i
=
0
;
i
<
pBlock
->
info
.
rows
;
++
i
)
{
char
*
val
=
((
char
*
)
pColInfoData
->
pData
)
+
bytes
*
i
;
if
(
isNull
(
val
,
type
))
{
continue
;
}
char
*
p
=
val
;
size_t
keyLen
=
0
;
if
(
IS_VAR_DATA_TYPE
(
pOperator
->
pExpr
->
base
.
colType
))
{
tstr
*
var
=
(
tstr
*
)(
val
);
p
=
var
->
data
;
keyLen
=
varDataLen
(
var
);
}
else
{
keyLen
=
bytes
;
}
for
(
int32_t
i
=
0
;
i
<
pBlock
->
info
.
rows
;
i
++
)
{
buildMultiDistinctKey
(
pInfo
,
pBlock
,
i
);
if
(
taosHashGet
(
pInfo
->
pSet
,
pInfo
->
buf
,
pInfo
->
totalBytes
)
==
NULL
)
{
int32_t
dummy
;
taosHashPut
(
pInfo
->
pSet
,
pInfo
->
buf
,
pInfo
->
totalBytes
,
&
dummy
,
sizeof
(
dummy
));
for
(
int
j
=
0
;
j
<
taosArrayGetSize
(
pRes
->
pDataBlock
);
j
++
)
{
SDistinctDataInfo
*
pDistDataInfo
=
taosArrayGet
(
pInfo
->
pDistinctDataInfo
,
j
);
// distinct meta info
SColumnInfoData
*
pColInfoData
=
taosArrayGet
(
pBlock
->
pDataBlock
,
pDistDataInfo
->
index
);
//src
SColumnInfoData
*
pResultColInfoData
=
taosArrayGet
(
pRes
->
pDataBlock
,
j
);
// dist
int
dummy
;
void
*
res
=
taosHashGet
(
pInfo
->
pSet
,
p
,
keyLen
);
if
(
res
==
NULL
)
{
taosHashPut
(
pInfo
->
pSet
,
p
,
keyLen
,
&
dummy
,
sizeof
(
dummy
));
char
*
start
=
pResultColInfoData
->
pData
+
bytes
*
pInfo
->
pRes
->
info
.
rows
;
memcpy
(
start
,
val
,
bytes
);
char
*
val
=
((
char
*
)
pColInfoData
->
pData
)
+
pDistDataInfo
->
bytes
*
i
;
char
*
start
=
pResultColInfoData
->
pData
+
pDistDataInfo
->
bytes
*
pInfo
->
pRes
->
info
.
rows
;
memcpy
(
start
,
val
,
pDistDataInfo
->
bytes
);
}
pRes
->
info
.
rows
+=
1
;
}
}
}
if
(
pRes
->
info
.
rows
>=
pInfo
->
threshold
)
{
break
;
}
...
...
@@ -6691,11 +6723,14 @@ static SSDataBlock* hashDistinct(void* param, bool* newgroup) {
SOperatorInfo
*
createDistinctOperatorInfo
(
SQueryRuntimeEnv
*
pRuntimeEnv
,
SOperatorInfo
*
upstream
,
SExprInfo
*
pExpr
,
int32_t
numOfOutput
)
{
SDistinctOperatorInfo
*
pInfo
=
calloc
(
1
,
sizeof
(
SDistinctOperatorInfo
));
pInfo
->
colIndex
=
-
1
;
pInfo
->
threshold
=
10000000
;
// distinct result threshold
pInfo
->
outputCapacity
=
4096
;
pInfo
->
pSet
=
taosHashInit
(
64
,
taosGetDefaultHashFunction
(
pExpr
->
base
.
colType
),
false
,
HASH_NO_LOCK
);
pInfo
->
totalBytes
=
0
;
pInfo
->
buf
=
NULL
;
pInfo
->
threshold
=
HASH_CAPACITY_LIMIT
;
// distinct result threshold
pInfo
->
outputCapacity
=
4096
;
pInfo
->
pDistinctDataInfo
=
taosArrayInit
(
numOfOutput
,
sizeof
(
SDistinctDataInfo
));
pInfo
->
pSet
=
taosHashInit
(
64
,
taosGetDefaultHashFunction
(
TSDB_DATA_TYPE_BINARY
),
false
,
HASH_NO_LOCK
);
pInfo
->
pRes
=
createOutputBuf
(
pExpr
,
numOfOutput
,
(
int32_t
)
pInfo
->
outputCapacity
);
SOperatorInfo
*
pOperator
=
calloc
(
1
,
sizeof
(
SOperatorInfo
));
pOperator
->
name
=
"DistinctOperator"
;
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录