Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
taosdata
TDengine
提交
d7f9857a
T
TDengine
项目概览
taosdata
/
TDengine
1 年多 前同步成功
通知
1185
Star
22016
Fork
4786
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
1
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
T
TDengine
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
1
Issue
1
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
未验证
提交
d7f9857a
编写于
6月 08, 2023
作者:
H
Haojun Liao
提交者:
GitHub
6月 08, 2023
浏览文件
操作
浏览文件
下载
差异文件
Merge pull request #21632 from taosdata/fix/TD-24008
opt stream block dispatch
上级
958160be
1e9c4d5f
变更
6
隐藏空白更改
内联
并排
Showing
6 changed file
with
68 addition
and
25 deletion
+68
-25
include/common/tdatablock.h
include/common/tdatablock.h
+1
-0
include/libs/stream/tstream.h
include/libs/stream/tstream.h
+1
-0
source/common/src/tdatablock.c
source/common/src/tdatablock.c
+18
-6
source/dnode/vnode/src/tq/tqSink.c
source/dnode/vnode/src/tq/tqSink.c
+2
-4
source/libs/stream/src/streamDispatch.c
source/libs/stream/src/streamDispatch.c
+42
-15
source/libs/stream/src/streamTask.c
source/libs/stream/src/streamTask.c
+4
-0
未找到文件。
include/common/tdatablock.h
浏览文件 @
d7f9857a
...
...
@@ -248,6 +248,7 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq2** pReq, const SSDataBlock* pData
tb_uid_t
suid
);
char
*
buildCtbNameByGroupId
(
const
char
*
stbName
,
uint64_t
groupId
);
int32_t
buildCtbNameByGroupIdImpl
(
const
char
*
stbName
,
uint64_t
groupId
,
char
*
pBuf
);
static
FORCE_INLINE
int32_t
blockGetEncodeSize
(
const
SSDataBlock
*
pBlock
)
{
return
blockDataGetSerialMetaSize
(
taosArrayGetSize
(
pBlock
->
pDataBlock
))
+
blockDataGetSize
(
pBlock
);
...
...
include/libs/stream/tstream.h
浏览文件 @
d7f9857a
...
...
@@ -327,6 +327,7 @@ struct SStreamTask {
int64_t
checkpointingId
;
int32_t
checkpointAlignCnt
;
struct
SStreamMeta
*
pMeta
;
SSHashObj
*
pNameMap
;
};
// meta
...
...
source/common/src/tdatablock.c
浏览文件 @
d7f9857a
...
...
@@ -2465,19 +2465,31 @@ _end:
}
char
*
buildCtbNameByGroupId
(
const
char
*
stbFullName
,
uint64_t
groupId
)
{
if
(
stbFullName
[
0
]
==
0
)
{
char
*
pBuf
=
taosMemoryCalloc
(
1
,
TSDB_TABLE_NAME_LEN
+
1
);
if
(
!
pBuf
)
{
return
NULL
;
}
int32_t
code
=
buildCtbNameByGroupIdImpl
(
stbFullName
,
groupId
,
pBuf
);
if
(
code
!=
TSDB_CODE_SUCCESS
)
{
taosMemoryFree
(
pBuf
);
return
NULL
;
}
return
pBuf
;
}
int32_t
buildCtbNameByGroupIdImpl
(
const
char
*
stbFullName
,
uint64_t
groupId
,
char
*
cname
)
{
if
(
stbFullName
[
0
]
==
0
)
{
return
TSDB_CODE_FAILED
;
}
SArray
*
tags
=
taosArrayInit
(
0
,
sizeof
(
SSmlKv
));
if
(
tags
==
NULL
)
{
return
NULL
;
return
TSDB_CODE_FAILED
;
}
void
*
cname
=
taosMemoryCalloc
(
1
,
TSDB_TABLE_NAME_LEN
+
1
);
if
(
cname
==
NULL
)
{
taosArrayDestroy
(
tags
);
return
NULL
;
return
TSDB_CODE_FAILED
;
}
SSmlKv
pTag
=
{.
key
=
"group_id"
,
...
...
@@ -2499,9 +2511,9 @@ char* buildCtbNameByGroupId(const char* stbFullName, uint64_t groupId) {
taosArrayDestroy
(
tags
);
if
((
rname
.
ctbShortName
&&
rname
.
ctbShortName
[
0
])
==
0
)
{
return
NULL
;
return
TSDB_CODE_FAILED
;
}
return
rname
.
ctbShortName
;
return
TSDB_CODE_SUCCESS
;
}
int32_t
blockEncode
(
const
SSDataBlock
*
pBlock
,
char
*
data
,
int32_t
numOfCols
)
{
...
...
source/dnode/vnode/src/tq/tqSink.c
浏览文件 @
d7f9857a
...
...
@@ -298,10 +298,8 @@ void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, int64_t ver, void* d
if
(
res
==
TSDB_CODE_SUCCESS
)
{
memcpy
(
ctbName
,
pTableSinkInfo
->
tbName
,
strlen
(
pTableSinkInfo
->
tbName
));
}
else
{
char
*
tmp
=
buildCtbNameByGroupId
(
stbFullName
,
pDataBlock
->
info
.
id
.
groupId
);
memcpy
(
ctbName
,
tmp
,
strlen
(
tmp
));
memcpy
(
pTableSinkInfo
->
tbName
,
tmp
,
strlen
(
tmp
));
taosMemoryFree
(
tmp
);
buildCtbNameByGroupIdImpl
(
stbFullName
,
pDataBlock
->
info
.
id
.
groupId
,
ctbName
);
memcpy
(
pTableSinkInfo
->
tbName
,
ctbName
,
strlen
(
ctbName
));
tqDebug
(
"vgId:%d, gropuId:%"
PRIu64
" datablock table name is null"
,
TD_VID
(
pVnode
),
pDataBlock
->
info
.
id
.
groupId
);
}
...
...
source/libs/stream/src/streamDispatch.c
浏览文件 @
d7f9857a
...
...
@@ -15,6 +15,13 @@
#include "streamInc.h"
#define MAX_BLOCK_NAME_NUM 1024
typedef
struct
SBlockName
{
uint32_t
hashValue
;
char
parTbName
[
TSDB_TABLE_NAME_LEN
];
}
SBlockName
;
int32_t
tEncodeStreamDispatchReq
(
SEncoder
*
pEncoder
,
const
SStreamDispatchReq
*
pReq
)
{
if
(
tStartEncode
(
pEncoder
)
<
0
)
return
-
1
;
if
(
tEncodeI64
(
pEncoder
,
pReq
->
streamId
)
<
0
)
return
-
1
;
...
...
@@ -331,26 +338,46 @@ FAIL:
int32_t
streamSearchAndAddBlock
(
SStreamTask
*
pTask
,
SStreamDispatchReq
*
pReqs
,
SSDataBlock
*
pDataBlock
,
int32_t
vgSz
,
int64_t
groupId
)
{
char
*
ctbName
=
taosMemoryCalloc
(
1
,
TSDB_TABLE_FNAME_LEN
);
if
(
ctbName
==
NULL
)
{
return
-
1
;
uint32_t
hashValue
=
0
;
SArray
*
vgInfo
=
pTask
->
shuffleDispatcher
.
dbInfo
.
pVgroupInfos
;
if
(
pTask
->
pNameMap
==
NULL
)
{
pTask
->
pNameMap
=
tSimpleHashInit
(
1024
,
taosGetDefaultHashFunction
(
TSDB_DATA_TYPE_BIGINT
));
}
if
(
pDataBlock
->
info
.
parTbName
[
0
])
{
snprintf
(
ctbName
,
TSDB_TABLE_NAME_LEN
,
"%s.%s"
,
pTask
->
shuffleDispatcher
.
dbInfo
.
db
,
pDataBlock
->
info
.
parTbName
);
void
*
pVal
=
tSimpleHashGet
(
pTask
->
pNameMap
,
&
groupId
,
sizeof
(
int64_t
));
if
(
pVal
)
{
SBlockName
*
pBln
=
(
SBlockName
*
)
pVal
;
hashValue
=
pBln
->
hashValue
;
if
(
!
pDataBlock
->
info
.
parTbName
[
0
])
{
memcpy
(
pDataBlock
->
info
.
parTbName
,
pBln
->
parTbName
,
strlen
(
pBln
->
parTbName
));
}
}
else
{
char
*
ctb
ShortName
=
buildCtbNameByGroupId
(
pTask
->
shuffleDispatcher
.
stbFullName
,
groupId
);
snprintf
(
ctbName
,
TSDB_TABLE_NAME_LEN
,
"%s.%s"
,
pTask
->
shuffleDispatcher
.
dbInfo
.
db
,
ctbShortName
);
taosMemoryFree
(
ctbShortName
)
;
}
char
*
ctb
Name
=
taosMemoryCalloc
(
1
,
TSDB_TABLE_FNAME_LEN
);
if
(
ctbName
==
NULL
)
{
return
-
1
;
}
SArray
*
vgInfo
=
pTask
->
shuffleDispatcher
.
dbInfo
.
pVgroupInfos
;
if
(
pDataBlock
->
info
.
parTbName
[
0
])
{
snprintf
(
ctbName
,
TSDB_TABLE_NAME_LEN
,
"%s.%s"
,
pTask
->
shuffleDispatcher
.
dbInfo
.
db
,
pDataBlock
->
info
.
parTbName
);
}
else
{
buildCtbNameByGroupIdImpl
(
pTask
->
shuffleDispatcher
.
stbFullName
,
groupId
,
pDataBlock
->
info
.
parTbName
);
snprintf
(
ctbName
,
TSDB_TABLE_NAME_LEN
,
"%s.%s"
,
pTask
->
shuffleDispatcher
.
dbInfo
.
db
,
pDataBlock
->
info
.
parTbName
);
}
/*uint32_t hashValue = MurmurHash3_32(ctbName, strlen(ctbName));*/
SUseDbRsp
*
pDbInfo
=
&
pTask
->
shuffleDispatcher
.
dbInfo
;
uint32_t
hashValue
=
taosGetTbHashVal
(
ctbName
,
strlen
(
ctbName
),
pDbInfo
->
hashMethod
,
pDbInfo
->
hashPrefix
,
pDbInfo
->
hashSuffix
);
taosMemoryFree
(
ctbName
);
SArray
*
vgInfo
=
pTask
->
shuffleDispatcher
.
dbInfo
.
pVgroupInfos
;
/*uint32_t hashValue = MurmurHash3_32(ctbName, strlen(ctbName));*/
SUseDbRsp
*
pDbInfo
=
&
pTask
->
shuffleDispatcher
.
dbInfo
;
hashValue
=
taosGetTbHashVal
(
ctbName
,
strlen
(
ctbName
),
pDbInfo
->
hashMethod
,
pDbInfo
->
hashPrefix
,
pDbInfo
->
hashSuffix
);
taosMemoryFree
(
ctbName
);
SBlockName
bln
=
{
0
};
bln
.
hashValue
=
hashValue
;
memcpy
(
bln
.
parTbName
,
pDataBlock
->
info
.
parTbName
,
strlen
(
pDataBlock
->
info
.
parTbName
));
if
(
tSimpleHashGetSize
(
pTask
->
pNameMap
)
<
MAX_BLOCK_NAME_NUM
)
{
tSimpleHashPut
(
pTask
->
pNameMap
,
&
groupId
,
sizeof
(
int64_t
),
&
bln
,
sizeof
(
SBlockName
));
}
}
bool
found
=
false
;
// TODO: optimize search
...
...
source/libs/stream/src/streamTask.c
浏览文件 @
d7f9857a
...
...
@@ -224,5 +224,9 @@ void tFreeStreamTask(SStreamTask* pTask) {
taosMemoryFree
((
void
*
)
pTask
->
id
.
idStr
);
}
if
(
pTask
->
pNameMap
)
{
tSimpleHashCleanup
(
pTask
->
pNameMap
);
}
taosMemoryFree
(
pTask
);
}
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录