Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
taosdata
TDengine
提交
5d06474f
T
TDengine
项目概览
taosdata
/
TDengine
大约 1 年 前同步成功
通知
1184
Star
22015
Fork
4786
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
1
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
T
TDengine
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
1
Issue
1
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
5d06474f
编写于
3月 24, 2023
作者:
wmmhello
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
fix:send data batch if consume wal where subscribe db
上级
dcaab41a
变更
5
展开全部
隐藏空白更改
内联
并排
Showing
5 changed file
with
324 addition
and
394 deletion
+324
-394
source/common/src/tdataformat.c
source/common/src/tdataformat.c
+4
-0
source/dnode/vnode/src/tq/tq.c
source/dnode/vnode/src/tq/tq.c
+10
-0
source/dnode/vnode/src/tq/tqExec.c
source/dnode/vnode/src/tq/tqExec.c
+1
-1
source/libs/parser/src/parInsertUtil.c
source/libs/parser/src/parInsertUtil.c
+31
-115
utils/test/c/tmq_taosx_ci.c
utils/test/c/tmq_taosx_ci.c
+278
-278
未找到文件。
source/common/src/tdataformat.c
浏览文件 @
5d06474f
...
...
@@ -2439,6 +2439,10 @@ _exit:
int32_t
tColDataAddValueByDataBlock
(
SColData
*
pColData
,
int8_t
type
,
int32_t
bytes
,
int32_t
nRows
,
char
*
lengthOrbitmap
,
char
*
data
)
{
int32_t
code
=
0
;
if
(
data
==
NULL
){
code
=
tColDataAppendValueImpl
[
pColData
->
flag
][
CV_FLAG_NONE
](
pColData
,
NULL
,
0
);
goto
_exit
;
}
if
(
IS_VAR_DATA_TYPE
(
type
))
{
// var-length data type
for
(
int32_t
i
=
0
;
i
<
nRows
;
++
i
)
{
...
...
source/dnode/vnode/src/tq/tq.c
浏览文件 @
5d06474f
...
...
@@ -629,6 +629,14 @@ static int32_t extractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq*
// process meta
if
(
pHead
->
msgType
!=
TDMT_VND_SUBMIT
)
{
if
(
totalRows
>
0
)
{
tqOffsetResetToLog
(
&
taosxRsp
.
rspOffset
,
fetchVer
-
1
);
code
=
tqSendDataRsp
(
pTq
,
pMsg
,
pRequest
,
(
SMqDataRsp
*
)
&
taosxRsp
,
TMQ_MSG_TYPE__TAOSX_RSP
);
tDeleteSTaosxRsp
(
&
taosxRsp
);
taosMemoryFreeClear
(
pCkHead
);
return
code
;
}
tqDebug
(
"fetch meta msg, ver:%"
PRId64
", type:%s"
,
pHead
->
version
,
TMSG_INFO
(
pHead
->
msgType
));
tqOffsetResetToLog
(
&
metaRsp
.
rspOffset
,
fetchVer
);
metaRsp
.
resMsgType
=
pHead
->
msgType
;
...
...
@@ -656,6 +664,8 @@ static int32_t extractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq*
if
(
tqTaosxScanLog
(
pTq
,
pHandle
,
submit
,
&
taosxRsp
,
&
totalRows
)
<
0
)
{
tqError
(
"tmq poll: tqTaosxScanLog error %"
PRId64
", in vgId:%d, subkey %s"
,
consumerId
,
vgId
,
pRequest
->
subKey
);
taosMemoryFreeClear
(
pCkHead
);
tDeleteSTaosxRsp
(
&
taosxRsp
);
return
-
1
;
}
...
...
source/dnode/vnode/src/tq/tqExec.c
浏览文件 @
5d06474f
...
...
@@ -346,7 +346,7 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR
SSDataBlock
*
pBlock
=
taosArrayGet
(
pBlocks
,
i
);
tqAddBlockDataToRsp
(
pBlock
,
(
SMqDataRsp
*
)
pRsp
,
taosArrayGetSize
(
pBlock
->
pDataBlock
),
pTq
->
pVnode
->
config
.
tsdbCfg
.
precision
);
totalRows
+=
pBlock
->
info
.
rows
;
*
totalRows
+=
pBlock
->
info
.
rows
;
blockDataFreeRes
(
pBlock
);
SSchemaWrapper
*
pSW
=
taosArrayGetP
(
pSchemas
,
i
);
taosArrayPush
(
pRsp
->
blockSchema
,
&
pSW
);
...
...
source/libs/parser/src/parInsertUtil.c
浏览文件 @
5d06474f
...
...
@@ -566,75 +566,14 @@ int32_t insBuildVgDataBlocks(SHashObj* pVgroupsHashObj, SArray* pVgDataCxtList,
return
code
;
}
static
int
bindFileds
(
SBoundColInfo
*
pBoundInfo
,
SSchema
*
pSchema
,
TAOS_FIELD
*
fields
,
int
numFields
)
{
bool
*
pUseCols
=
taosMemoryCalloc
(
pBoundInfo
->
numOfCols
,
sizeof
(
bool
));
if
(
NULL
==
pUseCols
)
{
return
TSDB_CODE_OUT_OF_MEMORY
;
}
pBoundInfo
->
numOfBound
=
0
;
int16_t
lastColIdx
=
-
1
;
// last column found
int32_t
code
=
TSDB_CODE_SUCCESS
;
for
(
int
i
=
0
;
i
<
numFields
;
i
++
)
{
SToken
token
;
token
.
z
=
fields
[
i
].
name
;
token
.
n
=
strlen
(
fields
[
i
].
name
);
int16_t
t
=
lastColIdx
+
1
;
int16_t
index
=
insFindCol
(
&
token
,
t
,
pBoundInfo
->
numOfCols
,
pSchema
);
if
(
index
<
0
&&
t
>
0
)
{
index
=
insFindCol
(
&
token
,
0
,
t
,
pSchema
);
}
if
(
index
<
0
)
{
uError
(
"can not find column name:%s"
,
token
.
z
);
code
=
TSDB_CODE_PAR_INVALID_COLUMN
;
break
;
}
else
if
(
pUseCols
[
index
])
{
code
=
TSDB_CODE_PAR_INVALID_COLUMN
;
uError
(
"duplicated column name:%s"
,
token
.
z
);
break
;
}
else
{
lastColIdx
=
index
;
pUseCols
[
index
]
=
true
;
pBoundInfo
->
pColIndex
[
pBoundInfo
->
numOfBound
]
=
index
;
++
pBoundInfo
->
numOfBound
;
}
}
if
(
TSDB_CODE_SUCCESS
==
code
&&
!
pUseCols
[
0
])
{
uError
(
"primary timestamp column can not be null:"
);
code
=
TSDB_CODE_PAR_INVALID_COLUMN
;
}
taosMemoryFree
(
pUseCols
);
return
code
;
}
static
bool
isSameBindFileds
(
SBoundColInfo
*
pBoundInfo
,
SSchema
*
pSchema
,
TAOS_FIELD
*
fields
,
int
numFields
)
{
int16_t
lastColIdx
=
-
1
;
// last column found
static
bool
findFileds
(
SSchema
*
pSchema
,
TAOS_FIELD
*
fields
,
int
numFields
)
{
for
(
int
i
=
0
;
i
<
numFields
;
i
++
)
{
SToken
token
;
token
.
z
=
fields
[
i
].
name
;
token
.
n
=
strlen
(
fields
[
i
].
name
);
int16_t
t
=
lastColIdx
+
1
;
int16_t
index
=
insFindCol
(
&
token
,
t
,
pBoundInfo
->
numOfCols
,
pSchema
);
if
(
index
<
0
&&
t
>
0
)
{
index
=
insFindCol
(
&
token
,
0
,
t
,
pSchema
);
}
if
(
index
<
0
)
{
uError
(
"can not find column name:%s"
,
token
.
z
);
return
false
;
}
else
{
lastColIdx
=
index
;
if
(
pBoundInfo
->
pColIndex
[
i
]
!=
index
){
return
false
;
}
if
(
strcmp
(
pSchema
->
name
,
fields
[
i
].
name
)
==
0
){
return
true
;
}
}
return
tru
e
;
return
fals
e
;
}
int
rawBlockBindData
(
SQuery
*
query
,
STableMeta
*
pTableMeta
,
void
*
data
,
SVCreateTbReq
*
pCreateTb
,
TAOS_FIELD
*
tFields
,
...
...
@@ -648,39 +587,13 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate
goto
end
;
}
do
{
if
(
tmp
!=
NULL
){
if
(
!
isSameBindFileds
(
&
pTableCxt
->
boundColsInfo
,
getTableColumnSchema
(
pTableMeta
),
tFields
,
numFields
)){
char
*
fieldNames
=
(
char
*
)
taosMemoryCalloc
(
numFields
,
sizeof
(
tFields
[
0
].
name
));
for
(
int
i
=
0
;
i
<
numFields
;
i
++
){
memcpy
(
fieldNames
+
i
*
sizeof
(
tFields
[
0
].
name
),
tFields
[
i
].
name
,
sizeof
(
tFields
[
0
].
name
));
}
ret
=
insGetTableDataCxt
(((
SVnodeModifyOpStmt
*
)(
query
->
pRoot
))
->
pTableBlockHashObj
,
fieldNames
,
numFields
*
sizeof
(
tFields
[
0
].
name
),
pTableMeta
,
&
pCreateTb
,
&
pTableCxt
,
true
);
taosMemoryFree
(
fieldNames
);
if
(
ret
!=
TSDB_CODE_SUCCESS
)
{
uError
(
"insGetTableDataCxt inner error"
);
goto
end
;
}
}
else
{
break
;
}
}
if
(
tFields
!=
NULL
)
{
ret
=
bindFileds
(
&
pTableCxt
->
boundColsInfo
,
getTableColumnSchema
(
pTableMeta
),
tFields
,
numFields
);
if
(
ret
!=
TSDB_CODE_SUCCESS
)
{
uError
(
"bindFileds error"
);
goto
end
;
}
}
// no need to bind, because select * get all fields
if
(
tmp
==
NULL
){
ret
=
initTableColSubmitData
(
pTableCxt
);
if
(
ret
!=
TSDB_CODE_SUCCESS
)
{
uError
(
"initTableColSubmitData error"
);
goto
end
;
}
}
while
(
0
);
}
char
*
p
=
(
char
*
)
data
;
// | version | total length | total rows | total columns | flag seg| block group id | column schema | each column
...
...
@@ -708,35 +621,38 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate
SSchema
*
pSchema
=
getTableColumnSchema
(
pTableCxt
->
pMeta
);
SBoundColInfo
*
boundInfo
=
&
pTableCxt
->
boundColsInfo
;
if
(
boundInfo
->
numOfBound
!=
numOfCols
)
{
uError
(
"
boundInfo->numOfBound:%d != numOfCols:%d"
,
boundInfo
->
numOfBound
,
numOfCols
);
if
(
numFields
!=
numOfCols
)
{
uError
(
"
numFields:%d != numOfCols:%d"
,
numFields
,
numOfCols
);
ret
=
TSDB_CODE_INVALID_PARA
;
goto
end
;
}
for
(
int
c
=
0
;
c
<
boundInfo
->
numOfBound
;
++
c
)
{
SSchema
*
pColSchema
=
&
pSchema
[
boundInfo
->
pColIndex
[
c
]
];
SSchema
*
pColSchema
=
&
pSchema
[
c
];
SColData
*
pCol
=
taosArrayGet
(
pTableCxt
->
pData
->
aCol
,
c
);
if
(
findFileds
(
pColSchema
,
tFields
,
numFields
)){
if
(
*
fields
!=
pColSchema
->
type
&&
*
(
int32_t
*
)(
fields
+
sizeof
(
int8_t
))
!=
pColSchema
->
bytes
)
{
uError
(
"type or bytes not equal"
);
ret
=
TSDB_CODE_INVALID_PARA
;
goto
end
;
}
if
(
*
fields
!=
pColSchema
->
type
&&
*
(
int32_t
*
)(
fields
+
sizeof
(
int8_t
))
!=
pColSchema
->
bytes
)
{
uError
(
"type or bytes not equal"
);
ret
=
TSDB_CODE_INVALID_PARA
;
goto
end
;
}
int8_t
*
offset
=
pStart
;
if
(
IS_VAR_DATA_TYPE
(
pColSchema
->
type
))
{
pStart
+=
numOfRows
*
sizeof
(
int32_t
);
}
else
{
pStart
+=
BitmapLen
(
numOfRows
);
}
char
*
pData
=
pStart
;
int8_t
*
offset
=
pStart
;
if
(
IS_VAR_DATA_TYPE
(
pColSchema
->
type
))
{
pStart
+=
numOfRows
*
sizeof
(
int32_t
);
}
else
{
pStart
+=
BitmapLen
(
numOfRows
);
}
char
*
pData
=
pStart
;
tColDataAddValueByDataBlock
(
pCol
,
pColSchema
->
type
,
pColSchema
->
bytes
,
numOfRows
,
offset
,
pData
);
fields
+=
sizeof
(
int8_t
)
+
sizeof
(
int32_t
);
if
(
needChangeLength
)
{
pStart
+=
htonl
(
colLength
[
c
]);
}
else
{
pStart
+=
colLength
[
c
];
tColDataAddValueByDataBlock
(
pCol
,
pColSchema
->
type
,
pColSchema
->
bytes
,
numOfRows
,
offset
,
pData
);
fields
+=
sizeof
(
int8_t
)
+
sizeof
(
int32_t
);
if
(
needChangeLength
)
{
pStart
+=
htonl
(
colLength
[
c
]);
}
else
{
pStart
+=
colLength
[
c
];
}
}
else
{
tColDataAddValueByDataBlock
(
pCol
,
pColSchema
->
type
,
pColSchema
->
bytes
,
numOfRows
,
NULL
,
NULL
);
}
}
...
...
utils/test/c/tmq_taosx_ci.c
浏览文件 @
5d06474f
此差异已折叠。
点击以展开。
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录