MDEV-33850 : For Galera, create sequence with low cache got signal 6 …

…error: [ERROR] WSREP: FSM: no such a transition REPLICATING -> COMMITTED

Wsrep transaction was BF aborted but it must replay because certification
succeeded. The transaction must not be written into binlog yet, it will
be done during commit after the replay.

mysql-test/suite/galera/disabled.def

@@ -10,4 +10,3 @@
 #
 ##############################################################################
 
-galera_sequences : MDEV-35934/MDEV-33850 For Galera, create sequence with low cache got signal 6 error: [ERROR] WSREP: FSM: no such a transition REPLICATING -> COMMITTED

mysql-test/suite/galera/r/galera_sequences.result

@@ -47,6 +47,9 @@ select NEXT VALUE FOR Seq1_1;
 NEXT VALUE FOR Seq1_1
 4
 connection node_1;
+SHOW CREATE SEQUENCE Seq1_1;
+Table	Create Table
+Seq1_1	CREATE SEQUENCE `Seq1_1` start with 1 minvalue 1 maxvalue 9223372036854775806 increment by 1 nocache nocycle ENGINE=InnoDB
 DROP SEQUENCE Seq1_1;
 connection node_1;
 CREATE TABLE t2 (d CHAR(1)KEY);

mysql-test/suite/galera/r/galera_sequences_bf_kill.result

@@ -0,0 +1,79 @@
+connection node_2;
+connection node_1;
+connection node_1;
+CREATE SEQUENCE s INCREMENT=0 CACHE=10 ENGINE=InnoDB;
+CREATE TABLE t1 (f1 INT PRIMARY KEY, f2 INT) ENGINE=InnoDB;
+INSERT INTO t1 VALUES (1, 0), (3, 0);
+connection node_1;
+START TRANSACTION;
+INSERT INTO t1 VALUES (4, next value for s);
+INSERT INTO t1 VALUES (5, next value for s);
+INSERT INTO t1 VALUES (6, next value for s);
+INSERT INTO t1 VALUES (7, next value for s);
+INSERT INTO t1 VALUES (8, next value for s);
+INSERT INTO t1 VALUES (9, next value for s);
+INSERT INTO t1 VALUES (10, next value for s);
+INSERT INTO t1 VALUES (11, next value for s);
+INSERT INTO t1 VALUES (12, next value for s);
+INSERT INTO t1 VALUES (13, next value for s);
+INSERT INTO t1 VALUES (14, next value for s);
+SELECT * FROM t1 WHERE f1 > 0 FOR UPDATE;
+f1	f2
+1	0
+3	0
+4	1
+5	3
+6	5
+7	7
+8	9
+9	11
+10	13
+11	15
+12	17
+13	19
+14	21
+connect node_1a, 127.0.0.1, root, , test, $NODE_MYPORT_1;
+SET SESSION wsrep_sync_wait=0;
+SET GLOBAL wsrep_provider_options = 'dbug=d,apply_monitor_slave_enter_sync';
+connection node_2;
+INSERT INTO t1 VALUES (2, 2);
+connection node_1a;
+SET SESSION wsrep_on = 0;
+SET SESSION wsrep_on = 1;
+SET GLOBAL wsrep_provider_options = 'dbug=';
+SET GLOBAL wsrep_provider_options = 'dbug=d,commit_monitor_master_enter_sync';
+connection node_1;
+COMMIT;
+connection node_1a;
+SET SESSION wsrep_on = 0;
+SET SESSION wsrep_on = 1;
+SET GLOBAL wsrep_provider_options = 'dbug=';
+SET GLOBAL wsrep_provider_options = 'dbug=d,abort_trx_end';
+SET GLOBAL wsrep_provider_options = 'signal=apply_monitor_slave_enter_sync';
+SET SESSION wsrep_on = 0;
+SET SESSION wsrep_on = 1;
+SET GLOBAL wsrep_provider_options = 'dbug=';
+SET GLOBAL wsrep_provider_options = 'signal=abort_trx_end';
+SET GLOBAL wsrep_provider_options = 'signal=commit_monitor_master_enter_sync';
+connection node_1;
+wsrep_local_replays
+1
+connection node_1;
+SELECT * FROM t1;
+f1	f2
+1	0
+2	2
+3	0
+4	1
+5	3
+6	5
+7	7
+8	9
+9	11
+10	13
+11	15
+12	17
+13	19
+14	21
+DROP SEQUENCE s;
+DROP TABLE t1;

mysql-test/suite/galera/t/galera_sequences.test

@@ -45,6 +45,7 @@ SHOW CREATE SEQUENCE Seq1_1;
 select NEXT VALUE FOR Seq1_1;
 
 --connection node_1
+SHOW CREATE SEQUENCE Seq1_1;
 DROP SEQUENCE Seq1_1;
 
 #

mysql-test/suite/galera/t/galera_sequences_bf_kill.cnf

@@ -0,0 +1,5 @@
+!include ../galera_2nodes.cnf
+
+[mysqld]
+log-bin
+log-slave-updates

mysql-test/suite/galera/t/galera_sequences_bf_kill.test

@@ -0,0 +1,100 @@
+--source include/galera_cluster.inc
+--source include/have_innodb.inc
+--source include/have_debug.inc
+--source include/have_debug_sync.inc
+--source include/galera_have_debug_sync.inc
+
+
+#
+# We create InnoDB seqeuence with small cache that is then
+# used as default value for column in table.
+#
+--connection node_1
+--let $wsrep_local_replays_old = `SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_local_replays'`
+CREATE SEQUENCE s INCREMENT=0 CACHE=10 ENGINE=InnoDB;
+CREATE TABLE t1 (f1 INT PRIMARY KEY, f2 INT) ENGINE=InnoDB;
+INSERT INTO t1 VALUES (1, 0), (3, 0);
+--connection node_1
+START TRANSACTION;
+INSERT INTO t1 VALUES (4, next value for s); # No conflict in cert
+INSERT INTO t1 VALUES (5, next value for s); # No conflict in cert
+INSERT INTO t1 VALUES (6, next value for s); # No conflict in cert
+INSERT INTO t1 VALUES (7, next value for s); # No conflict in cert
+INSERT INTO t1 VALUES (8, next value for s); # No conflict in cert
+INSERT INTO t1 VALUES (9, next value for s); # No conflict in cert
+INSERT INTO t1 VALUES (10, next value for s); # No conflict in cert
+INSERT INTO t1 VALUES (11, next value for s); # No conflict in cert
+INSERT INTO t1 VALUES (12, next value for s); # No conflict in cert
+INSERT INTO t1 VALUES (13, next value for s); # No conflict in cert
+INSERT INTO t1 VALUES (14, next value for s); # No conflict in cert
+SELECT * FROM t1 WHERE f1 > 0 FOR UPDATE; # Should cause GAP lock between 1 and 3
+
+--connect node_1a, 127.0.0.1, root, , test, $NODE_MYPORT_1
+SET SESSION wsrep_sync_wait=0;
+# Block the applier on node #1 and issue a conflicting update on node #2
+--let $galera_sync_point = apply_monitor_slave_enter_sync
+--source include/galera_set_sync_point.inc
+
+#
+# Send conflicting INSERT
+#
+--connection node_2
+INSERT INTO t1 VALUES (2, 2); # This should BF abort because of GAP lock
+
+--connection node_1a
+--source include/galera_wait_sync_point.inc
+--source include/galera_clear_sync_point.inc
+
+# Block the commit, send the COMMIT and wait until it gets blocked
+--let $galera_sync_point = commit_monitor_master_enter_sync
+--source include/galera_set_sync_point.inc
+
+--connection node_1
+--send COMMIT
+
+--connection node_1a
+
+--let $galera_sync_point = apply_monitor_slave_enter_sync commit_monitor_master_enter_sync
+--source include/galera_wait_sync_point.inc
+--source include/galera_clear_sync_point.inc
+
+--let $galera_sync_point = abort_trx_end
+--source include/galera_set_sync_point.inc
+--let $galera_sync_point = apply_monitor_slave_enter_sync
+--source include/galera_signal_sync_point.inc
+--let $galera_sync_point = abort_trx_end commit_monitor_master_enter_sync
+--source include/galera_wait_sync_point.inc
+
+# Let the transactions proceed
+--source include/galera_clear_sync_point.inc
+--let $galera_sync_point = abort_trx_end
+--source include/galera_signal_sync_point.inc
+--let $galera_sync_point = commit_monitor_master_enter_sync
+--source include/galera_signal_sync_point.inc
+
+# Commit succeeds
+--connection node_1
+--reap
+
+# wsrep_local_replays has increased by 1
+--let $wsrep_local_replays_new = `SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_local_replays'`
+--disable_query_log
+--eval SELECT $wsrep_local_replays_new - $wsrep_local_replays_old = 1 AS wsrep_local_replays;
+--enable_query_log
+
+--connection node_1
+SELECT * FROM t1;
+SELECT LASTVAL(s);
+
+--connection node_2
+SELECT * FROM t1;
+SELECT LASTVAL(s);
+
+--connection node_1
+SELECT NEXTVAL(s);
+
+--connection node_2
+SELECT NEXTVAL(s);
+
+DROP SEQUENCE s;
+DROP TABLE t1;

sql/log.cc

@@ -1812,6 +1812,16 @@ binlog_flush_cache(THD *thd, binlog_cache_mngr *cache_mngr,
     if (using_trx && thd->binlog_flush_pending_rows_event(TRUE, TRUE))
       DBUG_RETURN(1);
 
+#ifdef WITH_WSREP
+    /* Wsrep transaction was BF aborted but it must replay because certification
+       succeeded. The transaction must not be written into binlog yet, it will
+       be done during commit after the replay. */
+    if (WSREP(thd) && wsrep_must_replay(thd))
+    {
+      DBUG_RETURN(0);
+    }
+#endif /* WITH_WSREP */
+
     /*
       Doing a commit or a rollback including non-transactional tables,
       i.e., ending a transaction where we might write the transaction

sql/wsrep_client_service.cc

@@ -304,6 +304,13 @@ enum wsrep::provider::status Wsrep_client_service::replay()
     replayer_service.replay_status(ret);
   }
 
+  // Original thd binlog stmt cache could contain
+  // changes on non-transactional objects like sequences
+  // In Galera we allow only InnoDB sequences, thus
+  // sequence table updates are in writeset.
+  // Binlog cache needs reset so that binlog_close
+  binlog_reset_cache(m_thd);
+
   replayer_thd->main_security_ctx = old_ctx;
   delete replayer_thd;
   DBUG_RETURN(ret);