NRG: Campaign early unless already candidate (#6511)

If an outdated server wants to become leader, other servers can
recognize they have a more up-to-date and start campaigning early to
speed up the leader election process.

However, if two servers would both become candidate at the same time due
to this, and one of the candidate's log is slightly more ahead it would
again start campaigning early. This can essentially short-circuit the
voting process, starting a new leader election/term while that wouldn't
be necessary to do. That ahead candidate could just reject the vote for
the other candidate, and wait for successful votes for itself to come
in.

Signed-off-by: Maurice van Veen <[email protected]>

Author: derekcollison

server/raft.go

@@ -4138,11 +4138,10 @@ func (n *raft) processVoteRequest(vr *voteRequest) error {
 		n.vote = vr.candidate
 		n.writeTermVote()
 		n.resetElectionTimeout()
-	} else {
-		if vr.term >= n.term && n.vote == noVote {
-			n.term = vr.term
-			n.resetElect(randCampaignTimeout())
-		}
+	} else if n.vote == noVote && n.State() != Candidate {
+		// We have a more up-to-date log, and haven't voted yet.
+		// Start campaigning earlier, but only if not candidate already, as that would short-circuit us.
+		n.resetElect(randCampaignTimeout())
 	}
 
 	// Term might have changed, make sure response has the most current

server/raft_test.go

@@ -526,6 +526,60 @@ func TestNRGUnsuccessfulVoteRequestDoesntResetElectionTimer(t *testing.T) {
 	require_True(t, followerEqual)
 }
 
+func TestNRGUnsuccessfulVoteRequestCampaignEarly(t *testing.T) {
+	n, cleanup := initSingleMemRaftNode(t)
+	defer cleanup()
+
+	nats0 := "S1Nunr6R" // "nats-0"
+	n.etlr = time.Time{}
+
+	// Simple case: we are follower and vote for a candidate.
+	require_NoError(t, n.processVoteRequest(&voteRequest{term: 1, lastTerm: 0, lastIndex: 0, candidate: nats0}))
+	require_Equal(t, n.term, 1)
+	require_Equal(t, n.vote, nats0)
+	require_NotEqual(t, n.etlr, time.Time{}) // Resets election timer as it voted.
+	n.etlr = time.Time{}
+
+	// We are follower and deny vote for outdated candidate.
+	n.pterm, n.pindex = 1, 100
+	require_NoError(t, n.processVoteRequest(&voteRequest{term: 2, lastTerm: 1, lastIndex: 2, candidate: nats0}))
+	require_Equal(t, n.term, 2)
+	require_Equal(t, n.vote, noVote)
+	require_NotEqual(t, n.etlr, time.Time{}) // Resets election timer as it starts campaigning.
+	n.etlr = time.Time{}
+
+	// Switch to candidate.
+	n.pterm, n.pindex = 2, 200
+	n.switchToCandidate()
+	require_Equal(t, n.term, 3)
+	require_Equal(t, n.State(), Candidate)
+	require_NotEqual(t, n.etlr, time.Time{}) // Resets election timer as part of switching state.
+	n.etlr = time.Time{}
+
+	// We are candidate and deny vote for outdated candidate. But they were on a more recent term, restart campaign.
+	require_NoError(t, n.processVoteRequest(&voteRequest{term: 4, lastTerm: 1, lastIndex: 2, candidate: nats0}))
+	require_Equal(t, n.term, 4)
+	require_Equal(t, n.vote, noVote)
+	require_NotEqual(t, n.etlr, time.Time{}) // Resets election timer as it restarts campaigning.
+	n.etlr = time.Time{}
+
+	// Switch to candidate.
+	n.pterm, n.pindex = 4, 400
+	n.switchToCandidate()
+	require_Equal(t, n.term, 5)
+	require_Equal(t, n.State(), Candidate)
+	require_NotEqual(t, n.etlr, time.Time{}) // Resets election timer as part of switching state.
+	n.etlr = time.Time{}
+
+	// We are candidate and deny vote for outdated candidate. Don't start campaigning early.
+	require_NoError(t, n.processVoteRequest(&voteRequest{term: 5, lastTerm: 1, lastIndex: 2, candidate: nats0}))
+	require_Equal(t, n.term, 5)
+	require_Equal(t, n.vote, noVote)
+	// Election timer must NOT be updated as that would mean another candidate that we don't vote
+	// for can short-circuit us by making us restart elections, denying us the ability to become leader.
+	require_Equal(t, n.etlr, time.Time{})
+}
+
 func TestNRGInvalidTAVDoesntPanic(t *testing.T) {
 	c := createJetStreamClusterExplicit(t, "R3S", 3)
 	defer c.shutdown()