fix(tianmu): fix problems related to master slave synchronization (#819)

fix 1:#818 Master slave synchronization - There will be too many tuples problem

Cause:
Tianmu::dbhandler::TianmuHandler::current_position This variable is not initialized. In some cases,
a large value may cause too many tuples problem.
Solution: Initialize the variable

fix 2:#819 Master slave synchronization - Primary key conflict problem

Solution: Modify the modification logic of master slave synchronization, so that the delete and update operations do not follow the primary key logic

Supplement

sql/sql_insert.cc
Fix the problem that the insert statement will not generate binlog in the delayed insert mode
storage/tianmu/handler/tianmu_handler.cpp
Fix the binlog error of the line format generated by the tianmu engine

mysql-test/suite/tianmu/r/issue819.result

+include/master-slave.inc
+[connection master]
+#
+# Test the master-slave function of innodb
+# 
+# connection master
+use test;
+create table ttt(id int primary key,name varchar(5))engine=innodb;
+insert into ttt values(1,'AAA');
+delete from ttt where id=1;
+insert into ttt values(1,'aaa');
+select * from ttt;
+id	name
+1	aaa
+# connection slave
+include/sync_slave_sql_with_master.inc
+select * from ttt;
+id	name
+1	aaa
+# connection master
+drop table ttt;
+include/sync_slave_sql_with_master.inc
+# connection master
+CREATE TABLE t1 (a int not null,b int not null)engine=innodb;
+CREATE TABLE t2 (a int not null, b int not null, primary key (a,b))engine=innodb;
+CREATE TABLE t3 (a int not null, b int not null, primary key (a,b))engine=innodb;
+insert into t1 values (1,1),(2,1),(1,3);
+insert into t2 values (1,1),(2,2),(3,3);
+insert into t3 values (1,1),(2,1),(1,3);
+delete t2.*,t3.* from t1,t2,t3 where t1.a=t2.a AND t2.b=t3.a and t1.b=t3.b;
+select * from t1;
+a	b
+1	1
+2	1
+1	3
+select * from t2;
+a	b
+3	3
+select * from t3;
+a	b
+# connection slave
+include/sync_slave_sql_with_master.inc
+select * from t1;
+a	b
+1	1
+2	1
+1	3
+select * from t2;
+a	b
+3	3
+select * from t3;
+a	b
+# connection master
+drop table t1,t2,t3;
+include/sync_slave_sql_with_master.inc
+# connection master
+CREATE TABLE t1
+(
+place_id int (10) unsigned NOT NULL,
+shows int(10) unsigned DEFAULT '0' NOT NULL,
+ishows int(10) unsigned DEFAULT '0' NOT NULL,
+ushows int(10) unsigned DEFAULT '0' NOT NULL,
+clicks int(10) unsigned DEFAULT '0' NOT NULL,
+iclicks int(10) unsigned DEFAULT '0' NOT NULL,
+uclicks int(10) unsigned DEFAULT '0' NOT NULL,
+ts timestamp,
+PRIMARY KEY (place_id,ts)
+)engine=innodb;
+INSERT INTO t1 (place_id,shows,ishows,ushows,clicks,iclicks,uclicks,ts)
+VALUES (1,0,0,0,0,0,0,20000928174434);
+UPDATE t1 SET shows=shows+1,ishows=ishows+1,ushows=ushows+1,clicks=clicks+1,iclicks=iclicks+1,uclicks=uclicks+1 WHERE place_id=1 AND ts>="2000-09-28 00:00:00";
+select place_id,shows from t1;
+place_id	shows
+1	1
+# connection slave
+include/sync_slave_sql_with_master.inc
+select place_id,shows from t1;
+place_id	shows
+1	1
+# connection master
+drop table t1;
+include/sync_slave_sql_with_master.inc
+#
+# Test the master-slave function of tianmu
+# 
+# connection master
+use test;
+create table ttt(id int primary key,name varchar(5))engine=tianmu;
+insert into ttt values(1,'AAA');
+update ttt set name='hhhh' where id=1;
+select * from ttt;
+id	name
+1	hhhh
+# connection slave
+include/sync_slave_sql_with_master.inc
+select * from ttt;
+id	name
+1	hhhh
+# connection master
+drop table ttt;
+include/sync_slave_sql_with_master.inc
+# connection master
+create table t1(id int primary key,name varchar(5))engine=tianmu;
+insert into t1 values(1,'AAA');
+delete from t1 where id=1;
+insert into t1 values(1,'aaa');
+select * from t1;
+id	name
+1	aaa
+# connection slave
+include/sync_slave_sql_with_master.inc
+select * from t1;
+id	name
+1	aaa
+# connection master
+drop table t1;
+CREATE TABLE t1 (a int not null,b int not null)engine=tianmu;
+CREATE TABLE t2 (a int not null, b int not null, primary key (a,b))engine=tianmu;
+CREATE TABLE t3 (a int not null, b int not null, primary key (a,b))engine=tianmu;
+insert into t1 values (1,1),(2,1),(1,3);
+insert into t2 values (1,1),(2,2),(3,3);
+insert into t3 values (1,1),(2,1),(1,3);
+delete t2.*,t3.* from t1,t2,t3 where t1.a=t2.a AND t2.b=t3.a and t1.b=t3.b;
+select * from t1;
+a	b
+1	1
+2	1
+1	3
+select * from t2;
+a	b
+3	3
+select * from t3;
+a	b
+# connection slave
+include/sync_slave_sql_with_master.inc
+select * from t1;
+a	b
+1	1
+2	1
+1	3
+select * from t2;
+a	b
+3	3
+select * from t3;
+a	b
+# connection master
+drop table t1,t2,t3;
+include/sync_slave_sql_with_master.inc
+# connection master
+CREATE TABLE t1
+(
+place_id int (10),
+shows int(10),
+ishows int(10),
+ushows int(10),
+clicks int(10),
+iclicks int(10),
+uclicks int(10),
+ts timestamp,
+PRIMARY KEY (place_id,ts)
+)engine=tianmu;
+INSERT INTO t1 (place_id,shows,ishows,ushows,clicks,iclicks,uclicks,ts)
+VALUES (1,0,0,0,0,0,0,20000928174434);
+UPDATE t1 SET shows=shows+1,ishows=ishows+1,ushows=ushows+1,clicks=clicks+1,iclicks=iclicks+1,uclicks=uclicks+1 WHERE place_id=1 AND ts>="2000-09-28 00:00:00";
+select place_id,shows from t1;
+place_id	shows
+1	1
+# connection slave
+include/sync_slave_sql_with_master.inc
+select place_id,shows from t1;
+place_id	shows
+1	1
+# connection master
+drop table t1;
+create table t1 (s1 int);
+create table t2 (s2 int);
+insert into t1 values (1), (2);
+insert into t2 values (2), (3);
+create view v1 as select * from t1,t2 union all select * from t1,t2;
+select * from v1;
+s1	s2
+1	2
+2	2
+1	3
+2	3
+1	2
+2	2
+1	3
+2	3
+# connection slave
+include/sync_slave_sql_with_master.inc
+select * from v1;
+s1	s2
+1	2
+2	2
+1	3
+2	3
+1	2
+2	2
+1	3
+2	3
+# connection master
+drop view v1;
+drop tables t1, t2;
+create table t1 (col1 int);
+insert into t1 values (1);
+create view v1 as select count(*) from t1;
+insert into t1 values (null);
+select * from v1;
+count(*)
+2
+# connection slave
+include/sync_slave_sql_with_master.inc
+select * from v1;
+count(*)
+2
+# connection master
+drop view v1;
+drop table t1;
+create table t1 (a int);
+create table t2 (a int);
+create view v1 as select a from t1;
+create view v2 as select a from t2 where a in (select a from v1);
+show create view v2;
+View	Create View	character_set_client	collation_connection
+v2	CREATE ALGORITHM=UNDEFINED DEFINER=`root`@`localhost` SQL SECURITY DEFINER VIEW `v2` AS select `t2`.`a` AS `a` from `t2` where `t2`.`a` in (select `v1`.`a` from `v1`)	latin1	latin1_swedish_ci
+# connection slave
+include/sync_slave_sql_with_master.inc
+show create view v2;
+View	Create View	character_set_client	collation_connection
+v2	CREATE ALGORITHM=UNDEFINED DEFINER=`root`@`localhost` SQL SECURITY DEFINER VIEW `v2` AS select `t2`.`a` AS `a` from `t2` where `t2`.`a` in (select `v1`.`a` from `v1`)	latin1	latin1_swedish_ci
+# connection master
+drop view v2, v1;
+drop table t1, t2;

mysql-test/suite/tianmu/t/issue819.test

+-- source include/have_tianmu.inc
+--disable_warnings
+-- source include/master-slave.inc
+--enable_warnings
+--echo #
+--echo # Test the master-slave function of innodb
+--echo # 
+
+--echo # connection master
+connection master;
+use test;
+create table ttt(id int primary key,name varchar(5))engine=innodb;
+insert into ttt values(1,'AAA');
+delete from ttt where id=1;
+insert into ttt values(1,'aaa');
+select * from ttt;
+
+--echo # connection slave
+--source include/sync_slave_sql_with_master.inc
+
+select * from ttt;
+
+--echo # connection master
+connection master;
+drop table ttt;
+--source include/sync_slave_sql_with_master.inc
+
+--echo # connection master
+connection master;
+CREATE TABLE t1 (a int not null,b int not null)engine=innodb;
+CREATE TABLE t2 (a int not null, b int not null, primary key (a,b))engine=innodb;
+CREATE TABLE t3 (a int not null, b int not null, primary key (a,b))engine=innodb;
+insert into t1 values (1,1),(2,1),(1,3);
+insert into t2 values (1,1),(2,2),(3,3);
+insert into t3 values (1,1),(2,1),(1,3);
+delete t2.*,t3.* from t1,t2,t3 where t1.a=t2.a AND t2.b=t3.a and t1.b=t3.b;
+
+select * from t1;
+select * from t2;
+select * from t3;
+
+--echo # connection slave
+--source include/sync_slave_sql_with_master.inc
+
+select * from t1;
+select * from t2;
+select * from t3;
+
+--echo # connection master
+connection master;
+drop table t1,t2,t3;
+--source include/sync_slave_sql_with_master.inc
+
+--echo # connection master
+connection master;
+CREATE TABLE t1
+ (
+ place_id int (10) unsigned NOT NULL,
+ shows int(10) unsigned DEFAULT '0' NOT NULL,
+ ishows int(10) unsigned DEFAULT '0' NOT NULL,
+ ushows int(10) unsigned DEFAULT '0' NOT NULL,
+ clicks int(10) unsigned DEFAULT '0' NOT NULL,
+ iclicks int(10) unsigned DEFAULT '0' NOT NULL,
+ uclicks int(10) unsigned DEFAULT '0' NOT NULL,
+ ts timestamp,
+ PRIMARY KEY (place_id,ts)
+ )engine=innodb;
+
+INSERT INTO t1 (place_id,shows,ishows,ushows,clicks,iclicks,uclicks,ts)
+VALUES (1,0,0,0,0,0,0,20000928174434);
+UPDATE t1 SET shows=shows+1,ishows=ishows+1,ushows=ushows+1,clicks=clicks+1,iclicks=iclicks+1,uclicks=uclicks+1 WHERE place_id=1 AND ts>="2000-09-28 00:00:00";
+select place_id,shows from t1;
+
+--echo # connection slave
+--source include/sync_slave_sql_with_master.inc
+select place_id,shows from t1;
+
+--echo # connection master
+connection master;
+drop table t1;
+--source include/sync_slave_sql_with_master.inc
+
+--echo #
+--echo # Test the master-slave function of tianmu
+--echo # 
+
+--echo # connection master
+connection master;
+use test;
+create table ttt(id int primary key,name varchar(5))engine=tianmu;
+insert into ttt values(1,'AAA');
+update ttt set name='hhhh' where id=1;
+select * from ttt;
+
+--echo # connection slave
+--source include/sync_slave_sql_with_master.inc
+
+select * from ttt;
+
+--echo # connection master
+connection master;
+drop table ttt;
+--source include/sync_slave_sql_with_master.inc
+
+--echo # connection master
+connection master;
+create table t1(id int primary key,name varchar(5))engine=tianmu;
+insert into t1 values(1,'AAA');
+delete from t1 where id=1;
+insert into t1 values(1,'aaa');
+select * from t1;
+
+--echo # connection slave
+--source include/sync_slave_sql_with_master.inc
+select * from t1;
+
+--echo # connection master
+connection master;
+drop table t1;
+
+CREATE TABLE t1 (a int not null,b int not null)engine=tianmu;
+CREATE TABLE t2 (a int not null, b int not null, primary key (a,b))engine=tianmu;
+CREATE TABLE t3 (a int not null, b int not null, primary key (a,b))engine=tianmu;
+insert into t1 values (1,1),(2,1),(1,3);
+insert into t2 values (1,1),(2,2),(3,3);
+insert into t3 values (1,1),(2,1),(1,3);
+delete t2.*,t3.* from t1,t2,t3 where t1.a=t2.a AND t2.b=t3.a and t1.b=t3.b;
+
+select * from t1;
+select * from t2;
+select * from t3;
+
+--echo # connection slave
+--source include/sync_slave_sql_with_master.inc
+
+select * from t1;
+select * from t2;
+select * from t3;
+
+--echo # connection master
+connection master;
+drop table t1,t2,t3;
+--source include/sync_slave_sql_with_master.inc
+
+--echo # connection master
+connection master;
+CREATE TABLE t1
+ (
+ place_id int (10),
+ shows int(10),
+ ishows int(10),
+ ushows int(10),
+ clicks int(10),
+ iclicks int(10),
+ uclicks int(10),
+ ts timestamp,
+ PRIMARY KEY (place_id,ts)
+ )engine=tianmu;
+
+INSERT INTO t1 (place_id,shows,ishows,ushows,clicks,iclicks,uclicks,ts)
+VALUES (1,0,0,0,0,0,0,20000928174434);
+UPDATE t1 SET shows=shows+1,ishows=ishows+1,ushows=ushows+1,clicks=clicks+1,iclicks=iclicks+1,uclicks=uclicks+1 WHERE place_id=1 AND ts>="2000-09-28 00:00:00";
+select place_id,shows from t1;
+
+--echo # connection slave
+--source include/sync_slave_sql_with_master.inc
+select place_id,shows from t1;
+
+--echo # connection master
+connection master;
+drop table t1;
+
+#
+# VIEW built over UNION
+#
+create table t1 (s1 int);
+create table t2 (s2 int);
+insert into t1 values (1), (2);
+insert into t2 values (2), (3);
+create view v1 as select * from t1,t2 union all select * from t1,t2;
+select * from v1;
+--echo # connection slave
+--source include/sync_slave_sql_with_master.inc
+select * from v1;
+--echo # connection master
+connection master;
+drop view v1;
+drop tables t1, t2;
+
+#
+# Aggregate functions in view list
+#
+create table t1 (col1 int);
+insert into t1 values (1);
+create view v1 as select count(*) from t1;
+insert into t1 values (null);
+select * from v1;
+--echo # connection slave
+--source include/sync_slave_sql_with_master.inc
+select * from v1;
+--echo # connection master
+connection master;
+drop view v1;
+drop table t1;
+
+#
+# Showing VIEW with VIEWs in subquery
+#
+create table t1 (a int);
+create table t2 (a int);
+create view v1 as select a from t1;
+create view v2 as select a from t2 where a in (select a from v1);
+show create view v2;
+--echo # connection slave
+--source include/sync_slave_sql_with_master.inc
+show create view v2;
+--echo # connection master
+connection master;
+drop view v2, v1;
+drop table t1, t2;

sql/log_event.cc

@@ -9681,6 +9681,19 @@ search_key_in_table(TABLE *table, MY_BITMAP *bi_cols, uint key_type)
   KEY *keyinfo;
   uint res= MAX_KEY;
   uint key;
+  /*
+    PK has bugs, not support
+  */
+  bool check_if_tianmu_engine = table && table->s && 
+                      (table->s->db_type() ? (table->s->db_type()->db_type == DB_TYPE_TIANMU): false);
+  enum_sql_command sql_command = SQLCOM_END;
+  if(table->in_use && table->in_use->lex) sql_command = table->in_use->lex->sql_command;
+  if (check_if_tianmu_engine && ((sql_command == SQLCOM_DELETE) ||
+                        (sql_command == SQLCOM_DELETE_MULTI) ||
+                        (sql_command == SQLCOM_UPDATE) ||
+                        (sql_command == SQLCOM_UPDATE_MULTI))){
+    DBUG_RETURN(res);
+  }
 
   if (key_type & PRI_KEY_FLAG &&
       (table->s->primary_key < MAX_KEY))

sql/sql_insert.cc

@@ -48,6 +48,8 @@
 #include "partition_info.h"           // partition_info
 #include "probes_mysql.h"             // MYSQL_INSERT_START
 
+#include "../storage/tianmu/system/configuration.h"
+
 static bool check_view_insertability(THD *thd, TABLE_LIST *view,
                                      const TABLE_LIST *insert_table_ref);
 
@@ -830,30 +832,30 @@ bool Sql_cmd_insert::mysql_insert(THD *thd,TABLE_LIST *table_list)
     if (error <= 0 || thd->get_transaction()->cannot_safely_rollback(
         Transaction_ctx::STMT))
     {
-//TIANMU UPGRADE BEGIN
-#if defined(TIANMU)
       bool tianmu_engine = insert_table->s->db_type() ? insert_table->s->db_type()->db_type == DB_TYPE_TIANMU: false;
-      if (mysql_bin_log.is_open()&& !tianmu_engine)
-#else
-      if (mysql_bin_log.is_open())
-#endif
-//END
+      /*
+        In the delayed insert mode, tianmu will actively write data to the binlog cache, 
+        but there is no non delayed insert. 
+        You need to release the logic written to the binlog cache when the sql layer 
+        is in the non delayed insert mode.
+      */
+      if (mysql_bin_log.is_open()&& !(tianmu_engine && tianmu_sysvar_insert_delayed))
       {
         int errcode= 0;
-	if (error <= 0)
+	      if (error <= 0)
         {
-	  /*
-	    [Guilhem wrote] Temporary errors may have filled
-	    thd->net.last_error/errno.  For example if there has
-	    been a disk full error when writing the row, and it was
-	    MyISAM, then thd->net.last_error/errno will be set to
-            "disk full"... and the mysql_file_pwrite() will wait until free
-	    space appears, and so when it finishes then the
-	    write_row() was entirely successful
-	  */
-	  /* todo: consider removing */
-	  thd->clear_error();
-	}
+          /*
+            [Guilhem wrote] Temporary errors may have filled
+            thd->net.last_error/errno.  For example if there has
+            been a disk full error when writing the row, and it was
+            MyISAM, then thd->net.last_error/errno will be set to
+                  "disk full"... and the mysql_file_pwrite() will wait until free
+            space appears, and so when it finishes then the
+            write_row() was entirely successful
+          */
+          /* todo: consider removing */
+          thd->clear_error();
+	      }
         else
           errcode= query_error_code(thd, thd->killed == THD::NOT_KILLED);
         

sql/sql_optimizer.cc

@@ -5133,16 +5133,16 @@ bool JOIN::make_join_plan()
       The following codes can be deleted after subsequent support
     */
     TABLE *const table= join_tab->table();
-    bool tianmu_engine = table && table->s && 
-                        (table->s->db_type() ? table->s->db_type()->db_type == DB_TYPE_TIANMU: false);
-    enum_sql_command sqlCommand = SQLCOM_END;
-    if(thd->lex) sqlCommand = thd->lex->sql_command;
-    bool tianmuDeleteOrUpdate = (tianmu_engine && (sqlCommand == SQLCOM_DELETE ||
-                                          sqlCommand == SQLCOM_DELETE_MULTI ||
-                                          sqlCommand == SQLCOM_UPDATE ||
-                                          sqlCommand == SQLCOM_UPDATE_MULTI));
-
-    if (!tianmuDeleteOrUpdate && update_ref_and_keys(thd, &keyuse_array, join_tab, tables, where_cond,
+    bool check_if_tianmu_engine = table && table->s && 
+                        (table->s->db_type() ? (table->s->db_type()->db_type == DB_TYPE_TIANMU): false);
+    enum_sql_command sql_command = SQLCOM_END;
+    if(thd->lex) sql_command = thd->lex->sql_command;
+    bool check_tianmu_delete_or_update = (check_if_tianmu_engine && ((sql_command == SQLCOM_DELETE) ||
+                                          (sql_command == SQLCOM_DELETE_MULTI) ||
+                                          (sql_command == SQLCOM_UPDATE) ||
+                                          (sql_command == SQLCOM_UPDATE_MULTI)));
+
+    if (!check_tianmu_delete_or_update && update_ref_and_keys(thd, &keyuse_array, join_tab, tables, where_cond,
                             cond_equal, ~select_lex->outer_join, select_lex,
                             &sargables))
       DBUG_RETURN(true);

storage/tianmu/handler/tianmu_handler.cpp

@@ -170,7 +170,23 @@ namespace {
 std::vector<bool> GetAttrsUseIndicator(TABLE *table) {
   int col_id = 0;
   std::vector<bool> attr_uses;
+  enum_sql_command sql_command = SQLCOM_END;
+  if (table->in_use && table->in_use->lex)
+    sql_command = table->in_use->lex->sql_command;
+  bool check_tianmu_delete_or_update = (sql_command == SQLCOM_DELETE) || (sql_command == SQLCOM_DELETE_MULTI) ||
+                                       (sql_command == SQLCOM_UPDATE) || (sql_command == SQLCOM_UPDATE_MULTI);
+
   for (Field **field = table->field; *field; ++field, ++col_id) {
+    /*
+      The binlog in row format will record the information in each column of the currently modified row and generate a
+      change log, The information of each column in the current row is obtained from the engine layer, so when the
+      current statement is delete or update, you need to fill in data for each column.
+      Here, should set each column to be valid.
+    */
+    if (check_tianmu_delete_or_update) {
+      attr_uses.push_back(true);
+      continue;
+    }
     if (bitmap_is_set(table->read_set, col_id) || bitmap_is_set(table->write_set, col_id))
       attr_uses.push_back(true);
     else