From d9c4b3582a42571b7e5bb9216a1a54707ccc3b82 Mon Sep 17 00:00:00 2001 From: wangm Date: Thu, 28 Sep 2023 15:24:00 +0800 Subject: [PATCH] fix panic on commit index regression Signed-off-by: wangm --- log.go | 6 +++++- log_test.go | 12 +++--------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/log.go b/log.go index 84826c3a..edb4a933 100644 --- a/log.go +++ b/log.go @@ -319,7 +319,11 @@ func (l *raftLog) commitTo(tocommit uint64) { // never decrease commit if l.committed < tocommit { if l.lastIndex() < tocommit { - l.logger.Panicf("tocommit(%d) is out of range [lastIndex(%d)]. Was the raft log corrupted, truncated, or lost?", tocommit, l.lastIndex()) + // this can happen when the follower crashed before the commit index persistent + // after the leader received its ack (it is too expensive to do fsync on every write) + l.logger.Warningf("tocommit(%d) is out of range [lastIndex(%d)]. Was the raft log corrupted, truncated, or lost?", tocommit, l.lastIndex()) + l.committed = l.lastIndex() + return } l.committed = tocommit } diff --git a/log_test.go b/log_test.go index 2711ff9c..b961a6bf 100644 --- a/log_test.go +++ b/log_test.go @@ -615,19 +615,13 @@ func TestCommitTo(t *testing.T) { tests := []struct { commit uint64 wcommit uint64 - wpanic bool }{ - {3, 3, false}, - {1, 2, false}, // never decrease - {4, 0, true}, // commit out of range -> panic + {3, 3}, + {1, 2}, // never decrease + {4, 3}, // commit out of range -> recover } for i, tt := range tests { t.Run(fmt.Sprint(i), func(t *testing.T) { - defer func() { - if r := recover(); r != nil { - require.True(t, tt.wpanic) - } - }() raftLog := newLog(NewMemoryStorage(), raftLogger) raftLog.append(previousEnts...) raftLog.committed = commit