From 152b27ba53b819cbacfa1e358993a2c636654c88 Mon Sep 17 00:00:00 2001 From: Vincent Lee Date: Wed, 1 Jan 2020 23:34:44 +0800 Subject: [PATCH 1/6] check last index continue --- node/node.go | 3 --- node/raft.go | 15 +++++++++++++++ raft/node.go | 50 +++++++++++++++++++++++++++++++++++++------------- 3 files changed, 52 insertions(+), 16 deletions(-) diff --git a/node/node.go b/node/node.go index c104f78a..c3fa56a0 100644 --- a/node/node.go +++ b/node/node.go @@ -170,7 +170,6 @@ type customProposeData struct { // a key-value node backed by raft type KVNode struct { - reqProposeC *entryQueue readyC chan struct{} rn *raftNode store *KVStore @@ -234,7 +233,6 @@ func NewKVNode(kvopts *KVOptions, config *RaftConfig, return nil, err } s := &KVNode{ - reqProposeC: newEntryQueue(proposeQueueLen, 1), readyC: make(chan struct{}, 1), stopChan: stopChan, stopDone: make(chan struct{}), @@ -326,7 +324,6 @@ func (nd *KVNode) Stop() { } defer close(nd.stopDone) close(nd.stopChan) - nd.reqProposeC.close() nd.expireHandler.Stop() nd.wg.Wait() nd.rn.StopNode() diff --git a/node/raft.go b/node/raft.go index 9ab154c1..3145423f 100644 --- a/node/raft.go +++ b/node/raft.go @@ -126,6 +126,7 @@ type raftNode struct { replayRunning int32 busySnapshot int32 loopServering int32 + lastPublished uint64 } // newRaftNode initiates a raft instance and returns a committed log entry @@ -925,6 +926,20 @@ func (rc *raftNode) processReady(rd raft.Ready) { } processedMsgs, hasRequestSnapMsg := rc.processMessages(rd.Messages) if len(rd.CommittedEntries) > 0 || !raft.IsEmptySnap(rd.Snapshot) || hasRequestSnapMsg { + var newPublished uint64 + if !raft.IsEmptySnap(rd.Snapshot) { + newPublished = rd.Snapshot.Metadata.Index + } + if len(rd.CommittedEntries) > 0 { + newPublished = rd.CommittedEntries[len(rd.CommittedEntries)-1].Index + firsti := rd.CommittedEntries[0].Index + if rc.lastPublished != 0 && firsti > rc.lastPublished+1 { + e := fmt.Sprintf("%v first index of committed entry[%d] should <= last published[%d] + 1", rc.Descrp(), firsti, rc.lastPublished) + rc.Errorf("%s", e) + panic(e) + } + } + rc.lastPublished = newPublished rc.publishEntries(rd.CommittedEntries, rd.Snapshot, applySnapshotTransferResult, raftDone, applyWaitDone) } if !raft.IsEmptySnap(rd.Snapshot) { diff --git a/raft/node.go b/raft/node.go index 947279a4..7ea990bb 100644 --- a/raft/node.go +++ b/raft/node.go @@ -16,6 +16,7 @@ package raft import ( "errors" + "fmt" "time" pb "github.com/youzan/ZanRedisDB/raft/raftpb" @@ -255,6 +256,8 @@ func StartNode(c *Config, peers []Peer, isLearner bool) Node { n.logger = c.Logger n.r = r n.prevS = newPrevState(r) + off := max(r.raftLog.applied+1, r.raftLog.firstIndex()) + n.lastSteppedIndex = off n.NotifyEventCh() return &n } @@ -270,25 +273,28 @@ func RestartNode(c *Config) Node { n.logger = c.Logger n.r = r n.prevS = newPrevState(r) + off := max(r.raftLog.applied+1, r.raftLog.firstIndex()) + n.lastSteppedIndex = off n.NotifyEventCh() return &n } // node is the canonical implementation of the Node interface type node struct { - propQ *ProposalQueue - msgQ *MessageQueue - confc chan pb.ConfChange - confstatec chan pb.ConfState - tickc chan struct{} - done chan struct{} - stop chan struct{} - status chan chan Status - eventNotifyCh chan bool - r *raft - prevS *prevState - newReadyFunc func(*raft, *SoftState, pb.HardState, bool) Ready - needAdvance bool + propQ *ProposalQueue + msgQ *MessageQueue + confc chan pb.ConfChange + confstatec chan pb.ConfState + tickc chan struct{} + done chan struct{} + stop chan struct{} + status chan chan Status + eventNotifyCh chan bool + r *raft + prevS *prevState + newReadyFunc func(*raft, *SoftState, pb.HardState, bool) Ready + needAdvance bool + lastSteppedIndex uint64 logger Logger } @@ -361,6 +367,24 @@ func (n *node) StepNode(moreEntriesToApply bool, busySnap bool) (Ready, bool) { rd := n.newReadyFunc(n.r, n.prevS.prevSoftSt, n.prevS.prevHardSt, moreEntriesToApply) if rd.containsUpdates() { n.needAdvance = true + var stepIndex uint64 + if !IsEmptySnap(rd.Snapshot) { + stepIndex = rd.Snapshot.Metadata.Index + } + if len(rd.CommittedEntries) > 0 { + fi := rd.CommittedEntries[0].Index + if n.lastSteppedIndex != 0 && fi > n.lastSteppedIndex+1 { + ents := n.r.raftLog.allEntries() + e := fmt.Sprintf("raft.node: %x(%v) index not continued: %v, %v, prev: %v, logs: %v, %v ", + n.r.id, n.r.group, fi, n.lastSteppedIndex, n.prevS, len(ents), + n.r.raftLog.String()) + n.logger.Error(e) + n.logger.Errorf("all entries: %v", ents) + panic(e) + } + stepIndex = rd.CommittedEntries[len(rd.CommittedEntries)-1].Index + } + n.lastSteppedIndex = stepIndex return rd, true } return Ready{}, false From 19fd2f58a08006c5febb10dd144fbaa844a5cd93 Mon Sep 17 00:00:00 2001 From: Vincent Lee Date: Thu, 2 Jan 2020 14:03:30 +0800 Subject: [PATCH 2/6] more debug logs for unexpected committed index --- node/raft.go | 9 +++++++-- node/raft_test.go | 1 + raft/log.go | 4 ++++ raft/node.go | 20 +++++++++++++++++--- 4 files changed, 29 insertions(+), 5 deletions(-) diff --git a/node/raft.go b/node/raft.go index 3145423f..c1df3731 100644 --- a/node/raft.go +++ b/node/raft.go @@ -871,6 +871,9 @@ func (rc *raftNode) serveChannels() { continue } rc.processReady(rd) + if rd.MoreCommittedEntries { + rc.node.NotifyEventCh() + } } } } @@ -931,13 +934,15 @@ func (rc *raftNode) processReady(rd raft.Ready) { newPublished = rd.Snapshot.Metadata.Index } if len(rd.CommittedEntries) > 0 { - newPublished = rd.CommittedEntries[len(rd.CommittedEntries)-1].Index firsti := rd.CommittedEntries[0].Index if rc.lastPublished != 0 && firsti > rc.lastPublished+1 { - e := fmt.Sprintf("%v first index of committed entry[%d] should <= last published[%d] + 1", rc.Descrp(), firsti, rc.lastPublished) + e := fmt.Sprintf("%v first index of committed entry[%d] should <= last published[%d] + 1, snap: %v", + rc.Descrp(), firsti, rc.lastPublished, rd.Snapshot.Metadata.String()) rc.Errorf("%s", e) + rc.Errorf("raft node status: %v", rc.node.DebugString()) panic(e) } + newPublished = rd.CommittedEntries[len(rd.CommittedEntries)-1].Index } rc.lastPublished = newPublished rc.publishEntries(rd.CommittedEntries, rd.Snapshot, applySnapshotTransferResult, raftDone, applyWaitDone) diff --git a/node/raft_test.go b/node/raft_test.go index fdad7efc..a7a32825 100644 --- a/node/raft_test.go +++ b/node/raft_test.go @@ -80,6 +80,7 @@ func (n *nodeRecorder) ApplyConfChange(conf raftpb.ConfChange) *raftpb.ConfState func (n *nodeRecorder) Stop() { n.Record(testutil.Action{Name: "Stop"}) } +func (n *nodeRecorder) DebugString() string { return "" } func (n *nodeRecorder) ReportUnreachable(id uint64, g raftpb.Group) {} diff --git a/raft/log.go b/raft/log.go index 5e8175cc..67d1bbb8 100644 --- a/raft/log.go +++ b/raft/log.go @@ -155,6 +155,10 @@ func (l *raftLog) hasNextEnts() bool { return l.committed+1 > off } +func (l *raftLog) hasMoreNextEnts(appliedTo uint64) bool { + return l.committed > appliedTo +} + func (l *raftLog) snapshot() (pb.Snapshot, error) { if l.unstable.snapshot != nil { return *l.unstable.snapshot, nil diff --git a/raft/node.go b/raft/node.go index 7ea990bb..302d899e 100644 --- a/raft/node.go +++ b/raft/node.go @@ -84,7 +84,8 @@ type Ready struct { // store/state-machine. These have previously been committed to stable // store. CommittedEntries []pb.Entry - + // Whether there are more committed entries ready to be applied. + MoreCommittedEntries bool // Messages specifies outbound messages to be sent AFTER Entries are // committed to stable storage. // If it contains a MsgSnap message, the application MUST report back to raft @@ -186,6 +187,7 @@ type Node interface { ReportSnapshot(id uint64, group pb.Group, status SnapshotStatus) // Stop performs any necessary termination of the Node. Stop() + DebugString() string } type Peer struct { @@ -375,8 +377,8 @@ func (n *node) StepNode(moreEntriesToApply bool, busySnap bool) (Ready, bool) { fi := rd.CommittedEntries[0].Index if n.lastSteppedIndex != 0 && fi > n.lastSteppedIndex+1 { ents := n.r.raftLog.allEntries() - e := fmt.Sprintf("raft.node: %x(%v) index not continued: %v, %v, prev: %v, logs: %v, %v ", - n.r.id, n.r.group, fi, n.lastSteppedIndex, n.prevS, len(ents), + e := fmt.Sprintf("raft.node: %x(%v) index not continued: %v, %v, %v, snap:%v, prev: %v, logs: %v, %v ", + n.r.id, n.r.group, fi, n.lastSteppedIndex, stepIndex, rd.Snapshot.Metadata.String(), n.prevS, len(ents), n.r.raftLog.String()) n.logger.Error(e) n.logger.Errorf("all entries: %v", ents) @@ -390,6 +392,14 @@ func (n *node) StepNode(moreEntriesToApply bool, busySnap bool) (Ready, bool) { return Ready{}, false } +func (n *node) DebugString() string { + ents := n.r.raftLog.allEntries() + e := fmt.Sprintf("raft.node: %x(%v) index not continued: %v, prev: %v, logs: %v, %v ", + n.r.id, n.r.group, n.lastSteppedIndex, n.prevS, len(ents), + n.r.raftLog.String()) + return e +} + func (n *node) handleLeaderUpdate(r *raft) bool { lead := n.prevS.prevLead needHandleProposal := lead != None @@ -746,6 +756,10 @@ func newReady(r *raft, prevSoftSt *SoftState, prevHardSt pb.HardState, moreEntri if moreEntriesToApply { rd.CommittedEntries = r.raftLog.nextEnts() } + if len(rd.CommittedEntries) > 0 { + lastIndex := rd.CommittedEntries[len(rd.CommittedEntries)-1].Index + rd.MoreCommittedEntries = r.raftLog.hasMoreNextEnts(lastIndex) + } if softSt := r.softState(); !softSt.equal(prevSoftSt) { rd.SoftState = softSt } From ef2975424849f5721e5023e92580c111659ab65d Mon Sep 17 00:00:00 2001 From: Vincent Lee Date: Thu, 2 Jan 2020 21:02:06 +0800 Subject: [PATCH 3/6] more logs for unmatched committed index --- raft/node.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/raft/node.go b/raft/node.go index 302d899e..50dd69d2 100644 --- a/raft/node.go +++ b/raft/node.go @@ -385,6 +385,16 @@ func (n *node) StepNode(moreEntriesToApply bool, busySnap bool) (Ready, bool) { panic(e) } stepIndex = rd.CommittedEntries[len(rd.CommittedEntries)-1].Index + if rd.HardState.Commit != stepIndex { + ents := n.r.raftLog.allEntries() + e := fmt.Sprintf("raft.node: %x(%v) index not continued: %v, %v, %v, snap:%v, prev: %v, logs: %v, %v ", + n.r.id, n.r.group, fi, n.lastSteppedIndex, stepIndex, rd.Snapshot.Metadata.String(), n.prevS, len(ents), + n.r.raftLog.String()) + n.logger.Error(e) + n.logger.Errorf("all entries: %v", ents) + n.logger.Errorf("unmatched commit index: %v, %v, snap: %v", rd.HardState, stepIndex, rd.Snapshot.Metadata) + panic("unmatched hard state and committed entries") + } } n.lastSteppedIndex = stepIndex return rd, true From 486ff7fd16aedea8ecd65ab53f0f5fd49c4f0936 Mon Sep 17 00:00:00 2001 From: Vincent Lee Date: Fri, 3 Jan 2020 00:16:52 +0800 Subject: [PATCH 4/6] fix raft commit index continue while the channel buffer is full --- raft/node.go | 20 +++++++++-- raft/node_test.go | 79 ++++++++++++++++++++++++++++++++++++++++++++ raft/rawnode.go | 17 ++++------ raft/rawnode_test.go | 79 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 182 insertions(+), 13 deletions(-) diff --git a/raft/node.go b/raft/node.go index 50dd69d2..41c56d69 100644 --- a/raft/node.go +++ b/raft/node.go @@ -117,6 +117,19 @@ func (rd Ready) containsUpdates() bool { len(rd.CommittedEntries) > 0 || len(rd.Messages) > 0 || len(rd.ReadStates) != 0 } +// appliedCursor extracts from the Ready the highest index the client has +// applied (once the Ready is confirmed via Advance). If no information is +// contained in the Ready, returns zero. +func (rd Ready) appliedCursor() uint64 { + if n := len(rd.CommittedEntries); n > 0 { + return rd.CommittedEntries[n-1].Index + } + if index := rd.Snapshot.Metadata.Index; index > 0 { + return index + } + return 0 +} + type msgWithDrop struct { m pb.Message dropCB context.CancelFunc @@ -492,8 +505,11 @@ func (n *node) Advance(rd Ready) { n.r.msgs = nil n.r.readStates = nil - if n.prevS.prevHardSt.Commit != 0 { - n.r.raftLog.appliedTo(n.prevS.prevHardSt.Commit) + appliedI := rd.appliedCursor() + if appliedI != 0 { + // since the committed entries may less than the hard commit index due to the + // limit for buffer len, we should not use the hard state commit index. + n.r.raftLog.appliedTo(appliedI) } if n.prevS.havePrevLastUnstablei { n.r.raftLog.stableTo(n.prevS.prevLastUnstablei, n.prevS.prevLastUnstablet) diff --git a/raft/node_test.go b/raft/node_test.go index b58b07df..0b7a87ee 100644 --- a/raft/node_test.go +++ b/raft/node_test.go @@ -16,6 +16,7 @@ package raft import ( "bytes" + "math" "reflect" "testing" "time" @@ -25,6 +26,14 @@ import ( "golang.org/x/net/context" ) +type ignoreSizeHintMemStorage struct { + *MemoryStorage +} + +func (s *ignoreSizeHintMemStorage) Entries(lo, hi uint64, maxSize uint64) ([]raftpb.Entry, error) { + return s.MemoryStorage.Entries(lo, hi, math.MaxUint64) +} + // TestNodeStep ensures that node.Step sends msgProp to propc chan // and other kinds of messages to recvc chan. func TestNodeStep(t *testing.T) { @@ -935,3 +944,73 @@ func TestNodeProposeAddLearnerNode(t *testing.T) { close(stop) <-done } + +// TestNodeCommitPaginationAfterRestart regression tests a scenario in which the +// Storage's Entries size limitation is slightly more permissive than Raft's +// internal one. The original bug was the following: +// +// - node learns that index 11 (or 100, doesn't matter) is committed +// - nextEnts returns index 1..10 in CommittedEntries due to size limiting. However, +// index 10 already exceeds maxBytes, due to a user-provided impl of Entries. +// - Commit index gets bumped to 10 +// - the node persists the HardState, but crashes before applying the entries +// - upon restart, the storage returns the same entries, but `slice` takes a different code path +// (since it is now called with an upper bound of 10) and removes the last entry. +// - Raft emits a HardState with a regressing commit index. +// +// A simpler version of this test would have the storage return a lot less entries than dictated +// by maxSize (for example, exactly one entry) after the restart, resulting in a larger regression. +// This wouldn't need to exploit anything about Raft-internal code paths to fail. +func TestNodeCommitPaginationAfterRestart(t *testing.T) { + s := &ignoreSizeHintMemStorage{ + MemoryStorage: NewRealMemoryStorage(), + } + persistedHardState := raftpb.HardState{ + Term: 1, + Vote: 1, + Commit: 10, + } + + s.hardState = persistedHardState + s.ents = make([]raftpb.Entry, 10) + var size uint64 + for i := range s.ents { + ent := raftpb.Entry{ + Term: 1, + Index: uint64(i + 1), + Type: raftpb.EntryNormal, + Data: []byte("a"), + } + + s.ents[i] = ent + size += uint64(ent.Size()) + } + + cfg := newTestConfig(1, []uint64{1}, 10, 1, s) + // Set a MaxSizePerMsg that would suggest to Raft that the last committed entry should + // not be included in the initial rd.CommittedEntries. However, our storage will ignore + // this and *will* return it (which is how the Commit index ended up being 10 initially). + cfg.MaxSizePerMsg = size - uint64(s.ents[len(s.ents)-1].Size()) - 1 + + r := newRaft(cfg) + n := newNode() + defer s.Close() + n.r = r + n.prevS = newPrevState(r) + n.NotifyEventCh() + n.Campaign(context.TODO()) + + defer n.Stop() + + rd, _ := n.StepNode(true, false) + if !IsEmptyHardState(rd.HardState) && rd.HardState.Commit < persistedHardState.Commit { + t.Errorf("HardState regressed: Commit %d -> %d\nCommitting:\n", + persistedHardState.Commit, rd.HardState.Commit, + ) + } +} + +// TestNodeCommitEntriesTooMuch check the commit index will be continued even +// if the apply commit channel is full +func TestNodeCommitEntriesTooMuch(t *testing.T) { +} diff --git a/raft/rawnode.go b/raft/rawnode.go index f9c2c204..b98d6aef 100644 --- a/raft/rawnode.go +++ b/raft/rawnode.go @@ -47,17 +47,12 @@ func (rn *RawNode) commitReady(rd Ready) { if !IsEmptyHardState(rd.HardState) { rn.prevHardSt = rd.HardState } - if rn.prevHardSt.Commit != 0 { - // In most cases, prevHardSt and rd.HardState will be the same - // because when there are new entries to apply we just sent a - // HardState with an updated Commit value. However, on initial - // startup the two are different because we don't send a HardState - // until something changes, but we do send any un-applied but - // committed entries (and previously-committed entries may be - // incorporated into the snapshot, even if rd.CommittedEntries is - // empty). Therefore we mark all committed entries as applied - // whether they were included in rd.HardState or not. - rn.raft.raftLog.appliedTo(rn.prevHardSt.Commit) + // If entries were applied (or a snapshot), update our cursor for + // the next Ready. Note that if the current HardState contains a + // new Commit index, this does not mean that we're also applying + // all of the new entries due to commit pagination by size. + if index := rd.appliedCursor(); index > 0 { + rn.raft.raftLog.appliedTo(index) } if len(rd.Entries) > 0 { e := rd.Entries[len(rd.Entries)-1] diff --git a/raft/rawnode_test.go b/raft/rawnode_test.go index e103957e..3b478fa6 100644 --- a/raft/rawnode_test.go +++ b/raft/rawnode_test.go @@ -435,6 +435,85 @@ func TestRawNodeStatus(t *testing.T) { } } +// TestRawNodeCommitPaginationAfterRestart is the RawNode version of +// TestNodeCommitPaginationAfterRestart. The anomaly here was even worse as the +// Raft group would forget to apply entries: +// +// - node learns that index 11 is committed +// - nextEnts returns index 1..10 in CommittedEntries (but index 10 already +// exceeds maxBytes), which isn't noticed internally by Raft +// - Commit index gets bumped to 10 +// - the node persists the HardState, but crashes before applying the entries +// - upon restart, the storage returns the same entries, but `slice` takes a +// different code path and removes the last entry. +// - Raft does not emit a HardState, but when the app calls Advance(), it bumps +// its internal applied index cursor to 10 (when it should be 9) +// - the next Ready asks the app to apply index 11 (omitting index 10), losing a +// write. +func TestRawNodeCommitPaginationAfterRestart(t *testing.T) { + s := &ignoreSizeHintMemStorage{ + MemoryStorage: NewRealMemoryStorage(), + } + persistedHardState := raftpb.HardState{ + Term: 1, + Vote: 1, + Commit: 10, + } + + s.hardState = persistedHardState + s.ents = make([]raftpb.Entry, 10) + var size uint64 + for i := range s.ents { + ent := raftpb.Entry{ + Term: 1, + Index: uint64(i + 1), + Type: raftpb.EntryNormal, + Data: []byte("a"), + } + + s.ents[i] = ent + size += uint64(ent.Size()) + } + + cfg := newTestConfig(1, []uint64{1}, 10, 1, s) + // Set a MaxSizePerMsg that would suggest to Raft that the last committed entry should + // not be included in the initial rd.CommittedEntries. However, our storage will ignore + // this and *will* return it (which is how the Commit index ended up being 10 initially). + cfg.MaxSizePerMsg = size - uint64(s.ents[len(s.ents)-1].Size()) - 1 + + s.ents = append(s.ents, raftpb.Entry{ + Term: 1, + Index: uint64(11), + Type: raftpb.EntryNormal, + Data: []byte("boom"), + }) + + rawNode, err := NewRawNode(cfg, []Peer{{NodeID: 1, ReplicaID: 1}}) + if err != nil { + t.Fatal(err) + } + + for highestApplied := uint64(0); highestApplied != 11; { + rd := rawNode.Ready() + n := len(rd.CommittedEntries) + if n == 0 { + t.Fatalf("stopped applying entries at index %d", highestApplied) + } + if next := rd.CommittedEntries[0].Index; highestApplied != 0 && highestApplied+1 != next { + t.Fatalf("attempting to apply index %d after index %d, leaving a gap", next, highestApplied) + } + highestApplied = rd.CommittedEntries[n-1].Index + rawNode.Advance(rd) + rawNode.Step(raftpb.Message{ + Type: raftpb.MsgHeartbeat, + To: 1, + From: 1, // illegal, but we get away with it + Term: 1, + Commit: 11, + }) + } +} + func BenchmarkStatusProgress(b *testing.B) { setup := func(members int) *RawNode { peers := make([]uint64, members) From f1911c0a96c942cd261a2e75b9ced96421d3a976 Mon Sep 17 00:00:00 2001 From: Vincent Lee Date: Fri, 3 Jan 2020 00:36:14 +0800 Subject: [PATCH 5/6] remove hard state commit check --- node/raft.go | 1 - raft/node.go | 17 ++--------------- 2 files changed, 2 insertions(+), 16 deletions(-) diff --git a/node/raft.go b/node/raft.go index c1df3731..52e895a9 100644 --- a/node/raft.go +++ b/node/raft.go @@ -940,7 +940,6 @@ func (rc *raftNode) processReady(rd raft.Ready) { rc.Descrp(), firsti, rc.lastPublished, rd.Snapshot.Metadata.String()) rc.Errorf("%s", e) rc.Errorf("raft node status: %v", rc.node.DebugString()) - panic(e) } newPublished = rd.CommittedEntries[len(rd.CommittedEntries)-1].Index } diff --git a/raft/node.go b/raft/node.go index 41c56d69..f467a3cd 100644 --- a/raft/node.go +++ b/raft/node.go @@ -389,25 +389,12 @@ func (n *node) StepNode(moreEntriesToApply bool, busySnap bool) (Ready, bool) { if len(rd.CommittedEntries) > 0 { fi := rd.CommittedEntries[0].Index if n.lastSteppedIndex != 0 && fi > n.lastSteppedIndex+1 { - ents := n.r.raftLog.allEntries() - e := fmt.Sprintf("raft.node: %x(%v) index not continued: %v, %v, %v, snap:%v, prev: %v, logs: %v, %v ", - n.r.id, n.r.group, fi, n.lastSteppedIndex, stepIndex, rd.Snapshot.Metadata.String(), n.prevS, len(ents), + e := fmt.Sprintf("raft.node: %x(%v) index not continued: %v, %v, %v, snap:%v, prev: %v, logs: %v ", + n.r.id, n.r.group, fi, n.lastSteppedIndex, stepIndex, rd.Snapshot.Metadata.String(), n.prevS, n.r.raftLog.String()) n.logger.Error(e) - n.logger.Errorf("all entries: %v", ents) - panic(e) } stepIndex = rd.CommittedEntries[len(rd.CommittedEntries)-1].Index - if rd.HardState.Commit != stepIndex { - ents := n.r.raftLog.allEntries() - e := fmt.Sprintf("raft.node: %x(%v) index not continued: %v, %v, %v, snap:%v, prev: %v, logs: %v, %v ", - n.r.id, n.r.group, fi, n.lastSteppedIndex, stepIndex, rd.Snapshot.Metadata.String(), n.prevS, len(ents), - n.r.raftLog.String()) - n.logger.Error(e) - n.logger.Errorf("all entries: %v", ents) - n.logger.Errorf("unmatched commit index: %v, %v, snap: %v", rd.HardState, stepIndex, rd.Snapshot.Metadata) - panic("unmatched hard state and committed entries") - } } n.lastSteppedIndex = stepIndex return rd, true From 9f658aefdbee569df4cdcde887a3a3efc118c0b0 Mon Sep 17 00:00:00 2001 From: Vincent Lee Date: Fri, 3 Jan 2020 13:32:47 +0800 Subject: [PATCH 6/6] bump version --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 3fd15388..06e6db9e 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ PREFIX=/usr/local DESTDIR= BINDIR=${PREFIX}/bin PROJECT?=github.com/youzan/ZanRedisDB -VERBINARY?= 0.7.1 +VERBINARY?= 0.7.2 COMMIT?=$(shell git rev-parse --short HEAD) BUILD_TIME?=$(shell date '+%Y-%m-%d_%H:%M:%S-%Z') GOFLAGS=-ldflags "-X ${PROJECT}/common.VerBinary=${VERBINARY} -X ${PROJECT}/common.Commit=${COMMIT} -X ${PROJECT}/common.BuildTime=${BUILD_TIME}"