Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
lijianghuflute
canal
提交
474063af
canal
项目概览
lijianghuflute
/
canal
与 Fork 源项目一致
从无法访问的项目Fork
通知
3
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
canal
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
前往新版Gitcode,体验更适合开发者的 AI 搜索 >>
提交
474063af
编写于
4月 28, 2021
作者:
A
agapple
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
fixed issue #3494 , polardb-x 2.0 cdc support implicit_id/varaibles
上级
5cc85ff2
变更
5
隐藏空白更改
内联
并排
Showing
5 changed file
with
36 addition
and
16 deletion
+36
-16
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java
...baba/otter/canal/parse/inbound/mysql/MysqlConnection.java
+2
-2
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java
...aba/otter/canal/parse/inbound/mysql/MysqlEventParser.java
+10
-8
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java
...ter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java
+4
-1
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/TableMetaCache.java
...tter/canal/parse/inbound/mysql/dbsync/TableMetaCache.java
+17
-1
parse/src/test/java/com/alibaba/otter/canal/parse/DirectLogFetcherTest.java
...a/com/alibaba/otter/canal/parse/DirectLogFetcherTest.java
+3
-4
未找到文件。
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlConnection.java
浏览文件 @
474063af
...
...
@@ -21,7 +21,6 @@ import com.alibaba.otter.canal.parse.driver.mysql.MysqlQueryExecutor;
import
com.alibaba.otter.canal.parse.driver.mysql.MysqlUpdateExecutor
;
import
com.alibaba.otter.canal.parse.driver.mysql.packets.GTIDSet
;
import
com.alibaba.otter.canal.parse.driver.mysql.packets.HeaderPacket
;
import
com.alibaba.otter.canal.parse.driver.mysql.packets.MysqlGTIDSet
;
import
com.alibaba.otter.canal.parse.driver.mysql.packets.client.BinlogDumpCommandPacket
;
import
com.alibaba.otter.canal.parse.driver.mysql.packets.client.BinlogDumpGTIDCommandPacket
;
import
com.alibaba.otter.canal.parse.driver.mysql.packets.client.RegisterSlaveCommandPacket
;
...
...
@@ -444,7 +443,8 @@ public class MysqlConnection implements ErosaConnection {
// mysql5.6需要设置slave_uuid避免被server kill链接
update
(
"set @slave_uuid=uuid()"
);
}
catch
(
Exception
e
)
{
if
(!
StringUtils
.
contains
(
e
.
getMessage
(),
"Unknown system variable"
))
{
if
(!
StringUtils
.
contains
(
e
.
getMessage
(),
"Unknown system variable"
)
&&
!
StringUtils
.
contains
(
e
.
getMessage
(),
"slave_uuid can't be set"
))
{
logger
.
warn
(
"update slave_uuid failed"
,
e
);
}
}
...
...
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/MysqlEventParser.java
浏览文件 @
474063af
...
...
@@ -69,7 +69,8 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE
private
int
dumpErrorCount
=
0
;
// binlogDump失败异常计数
private
int
dumpErrorCountThreshold
=
2
;
// binlogDump失败异常计数阀值
private
boolean
rdsOssMode
=
false
;
private
boolean
autoResetLatestPosMode
=
false
;
// true: binlog被删除之后,自动按最新的数据订阅
private
boolean
autoResetLatestPosMode
=
false
;
// true:
// binlog被删除之后,自动按最新的数据订阅
protected
ErosaConnection
buildErosaConnection
()
{
return
buildMysqlConnection
(
this
.
runningInfo
);
...
...
@@ -347,7 +348,7 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE
if
(
StringUtils
.
isNotEmpty
(
logPosition
.
getPostion
().
getGtid
()))
{
return
logPosition
.
getPostion
();
}
}
else
{
}
else
{
if
(
masterPosition
!=
null
&&
StringUtils
.
isNotEmpty
(
masterPosition
.
getGtid
()))
{
return
masterPosition
;
}
...
...
@@ -401,7 +402,7 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE
fixedPosition
.
getJournalName
(),
true
);
if
(
entryPosition
==
null
)
{
throw
new
CanalParseException
(
"[fixed timestamp] can't found begin/commit position before with fixed position"
throw
new
CanalParseException
(
"[fixed timestamp] can't found begin/commit position before with fixed position
"
+
fixedPosition
.
getJournalName
()
+
":"
+
fixedPosition
.
getPosition
());
}
return
entryPosition
;
...
...
@@ -486,7 +487,8 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE
return
findPosition
;
}
// 处理 binlog 位点被删除的情况,提供自动重置到当前位点的功能
// 应用场景: 测试环境不稳定,位点经常被删。强烈不建议在正式环境中开启此控制参数,因为binlog 丢失调到最新位点也即意味着数据丢失
// 应用场景: 测试环境不稳定,位点经常被删。强烈不建议在正式环境中开启此控制参数,因为binlog
// 丢失调到最新位点也即意味着数据丢失
if
(
isAutoResetLatestPosMode
())
{
dumpErrorCount
=
0
;
return
findEndPosition
(
mysqlConnection
);
...
...
@@ -497,9 +499,9 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE
return
null
;
}
}
else
if
(
StringUtils
.
isBlank
(
logPosition
.
getPostion
().
getJournalName
())
&&
logPosition
.
getPostion
().
getPosition
()
<=
0
&&
logPosition
.
getPostion
().
getTimestamp
()
>
0
)
{
return
fallbackFindByStartTimestamp
(
logPosition
,
mysqlConnection
);
&&
logPosition
.
getPostion
().
getPosition
()
<=
0
&&
logPosition
.
getPostion
().
getTimestamp
()
>
0
)
{
return
fallbackFindByStartTimestamp
(
logPosition
,
mysqlConnection
);
}
// 其余情况
logger
.
warn
(
"prepare to find start position just last position\n {}"
,
...
...
@@ -522,7 +524,7 @@ public class MysqlEventParser extends AbstractMysqlEventParser implements CanalE
* @param mysqlConnection
* @return
*/
protected
EntryPosition
fallbackFindByStartTimestamp
(
LogPosition
logPosition
,
MysqlConnection
mysqlConnection
)
{
protected
EntryPosition
fallbackFindByStartTimestamp
(
LogPosition
logPosition
,
MysqlConnection
mysqlConnection
)
{
long
timestamp
=
logPosition
.
getPostion
().
getTimestamp
();
long
newStartTimestamp
=
timestamp
-
fallbackIntervalInSeconds
*
1000
;
logger
.
warn
(
"prepare to find start position by last position {}:{}:{}"
,
new
Object
[]
{
""
,
""
,
...
...
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/LogEventConvert.java
浏览文件 @
474063af
...
...
@@ -627,7 +627,7 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
}
if
(
tableMeta
!=
null
&&
columnInfo
.
length
>
tableMeta
.
getFields
().
size
())
{
if
(
tableMetaCache
.
isOnRDS
())
{
if
(
tableMetaCache
.
isOnRDS
()
||
tableMetaCache
.
isOnPolarX
()
)
{
// 特殊处理下RDS的场景
List
<
FieldMeta
>
primaryKeys
=
tableMeta
.
getPrimaryFields
();
if
(
primaryKeys
==
null
||
primaryKeys
.
isEmpty
())
{
...
...
@@ -680,6 +680,9 @@ public class LogEventConvert extends AbstractCanalLifeCycle implements BinlogPar
if
(
existRDSNoPrimaryKey
&&
i
==
columnCnt
-
1
&&
info
.
type
==
LogEvent
.
MYSQL_TYPE_LONGLONG
)
{
// 不解析最后一列
String
rdsRowIdColumnName
=
"__#alibaba_rds_row_id#__"
;
if
(
tableMetaCache
.
isOnPolarX
())
{
rdsRowIdColumnName
=
"_drds_implicit_id_"
;
}
buffer
.
nextValue
(
rdsRowIdColumnName
,
i
,
info
.
type
,
info
.
meta
,
false
);
Column
.
Builder
columnBuilder
=
Column
.
newBuilder
();
columnBuilder
.
setName
(
rdsRowIdColumnName
);
...
...
parse/src/main/java/com/alibaba/otter/canal/parse/inbound/mysql/dbsync/TableMetaCache.java
浏览文件 @
474063af
...
...
@@ -39,6 +39,7 @@ public class TableMetaCache {
public
static
final
String
EXTRA
=
"EXTRA"
;
private
MysqlConnection
connection
;
private
boolean
isOnRDS
=
false
;
private
boolean
isOnPolarX
=
false
;
private
boolean
isOnTSDB
=
false
;
private
TableMetaTSDB
tableMetaTSDB
;
...
...
@@ -79,6 +80,14 @@ public class TableMetaCache {
}
}
catch
(
IOException
e
)
{
}
try
{
ResultSetPacket
packet
=
connection
.
query
(
"show global variables like 'polarx\\_%'"
);
if
(
packet
.
getFieldValues
().
size
()
>
0
)
{
isOnPolarX
=
true
;
}
}
catch
(
IOException
e
)
{
}
}
private
synchronized
TableMeta
getTableMetaByDB
(
String
fullname
)
throws
IOException
{
...
...
@@ -254,7 +263,6 @@ public class TableMetaCache {
.
toString
();
}
public
boolean
isOnTSDB
()
{
return
isOnTSDB
;
}
...
...
@@ -271,4 +279,12 @@ public class TableMetaCache {
this
.
isOnRDS
=
isOnRDS
;
}
public
boolean
isOnPolarX
()
{
return
isOnPolarX
;
}
public
void
setOnPolarX
(
boolean
isOnPolarX
)
{
this
.
isOnPolarX
=
isOnPolarX
;
}
}
parse/src/test/java/com/alibaba/otter/canal/parse/DirectLogFetcherTest.java
浏览文件 @
474063af
...
...
@@ -13,7 +13,6 @@ import java.util.concurrent.TimeUnit;
import
org.apache.commons.lang.StringUtils
;
import
org.junit.Assert
;
import
org.junit.Ignore
;
import
org.junit.Test
;
import
org.slf4j.Logger
;
import
org.slf4j.LoggerFactory
;
...
...
@@ -46,7 +45,6 @@ import com.taobao.tddl.dbsync.binlog.event.WriteRowsLogEvent;
import
com.taobao.tddl.dbsync.binlog.event.XidLogEvent
;
import
com.taobao.tddl.dbsync.binlog.event.mariadb.AnnotateRowsEvent
;
@Ignore
public
class
DirectLogFetcherTest
{
protected
final
Logger
logger
=
LoggerFactory
.
getLogger
(
this
.
getClass
());
...
...
@@ -58,7 +56,7 @@ public class DirectLogFetcherTest {
public
void
testSimple
()
{
DirectLogFetcher
fetcher
=
new
DirectLogFetcher
();
try
{
MysqlConnector
connector
=
new
MysqlConnector
(
new
InetSocketAddress
(
"127.0.0.1"
,
3306
),
"
root"
,
"hello
"
);
MysqlConnector
connector
=
new
MysqlConnector
(
new
InetSocketAddress
(
"127.0.0.1"
,
3306
),
"
canal"
,
"canal
"
);
connector
.
connect
();
updateSettings
(
connector
);
loadBinlogChecksum
(
connector
);
...
...
@@ -210,7 +208,8 @@ public class DirectLogFetcherTest {
// mysql5.6需要设置slave_uuid避免被server kill链接
update
(
"set @slave_uuid=uuid()"
,
connector
);
}
catch
(
Exception
e
)
{
if
(!
StringUtils
.
contains
(
e
.
getMessage
(),
"Unknown system variable"
))
{
if
(!
StringUtils
.
contains
(
e
.
getMessage
(),
"Unknown system variable"
)
&&
!
StringUtils
.
contains
(
e
.
getMessage
(),
"slave_uuid can't be set"
))
{
logger
.
warn
(
"update slave_uuid failed"
,
e
);
}
}
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录