Skip to content

Commit 344d919

Browse files
authored
[op-conductor] make health check more robust (ethereum-optimism#10325)
1 parent a7ff230 commit 344d919

File tree

2 files changed

+18
-5
lines changed

2 files changed

+18
-5
lines changed

op-conductor/health/monitor.go

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -134,7 +134,7 @@ func (hm *SequencerHealthMonitor) healthCheck() error {
134134

135135
var timeDiff, blockDiff, expectedBlocks uint64
136136
if hm.lastSeenUnsafeNum != 0 {
137-
timeDiff = now - hm.lastSeenUnsafeTime
137+
timeDiff = calculateTimeDiff(now, hm.lastSeenUnsafeTime)
138138
blockDiff = status.UnsafeL2.Number - hm.lastSeenUnsafeNum
139139
// how many blocks do we expect to see, minus 1 to account for edge case with respect to time.
140140
// for example, if diff = 2.001s and block time = 2s, expecting to see 1 block could potentially cause sequencer to be considered unhealthy.
@@ -160,7 +160,7 @@ func (hm *SequencerHealthMonitor) healthCheck() error {
160160
return ErrSequencerNotHealthy
161161
}
162162

163-
if now-status.UnsafeL2.Time > hm.unsafeInterval {
163+
if calculateTimeDiff(now, status.UnsafeL2.Time) > hm.unsafeInterval {
164164
hm.log.Error(
165165
"unsafe head is not progressing as expected",
166166
"now", now,
@@ -171,7 +171,7 @@ func (hm *SequencerHealthMonitor) healthCheck() error {
171171
return ErrSequencerNotHealthy
172172
}
173173

174-
if hm.safeEnabled && now-status.SafeL2.Time > hm.safeInterval {
174+
if hm.safeEnabled && calculateTimeDiff(now, status.SafeL2.Time) > hm.safeInterval {
175175
hm.log.Error(
176176
"safe head is not progressing as expected",
177177
"now", now,
@@ -196,6 +196,13 @@ func (hm *SequencerHealthMonitor) healthCheck() error {
196196
return nil
197197
}
198198

199+
func calculateTimeDiff(now, then uint64) uint64 {
200+
if now < then {
201+
return 0
202+
}
203+
return now - then
204+
}
205+
199206
func currentTimeProvicer() uint64 {
200207
return uint64(time.Now().Unix())
201208
}

op-conductor/health/monitor_test.go

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,7 @@ func (s *HealthMonitorTestSuite) TestUnhealthyUnsafeHeadNotProgressing() {
103103

104104
rc := &testutils.MockRollupClient{}
105105
ss1 := mockSyncStatus(now, 5, now-8, 1)
106-
for i := 0; i < 6; i++ {
106+
for i := 0; i < 5; i++ {
107107
rc.ExpectSyncStatus(ss1, nil)
108108
}
109109

@@ -168,7 +168,8 @@ func (s *HealthMonitorTestSuite) TestHealthyWithUnsafeLag() {
168168
rc.ExpectSyncStatus(mockSyncStatus(now-10, 1, now, 1), nil)
169169
rc.ExpectSyncStatus(mockSyncStatus(now-10, 1, now, 1), nil)
170170
rc.ExpectSyncStatus(mockSyncStatus(now-8, 2, now, 1), nil)
171-
rc.ExpectSyncStatus(mockSyncStatus(now-8, 2, now, 1), nil)
171+
// in this case now time is behind unsafe head time, this should still be considered healthy.
172+
rc.ExpectSyncStatus(mockSyncStatus(now+5, 2, now, 1), nil)
172173

173174
monitor := s.SetupMonitor(now, 60, 60, rc, nil)
174175
healthUpdateCh := monitor.Subscribe()
@@ -194,6 +195,11 @@ func (s *HealthMonitorTestSuite) TestHealthyWithUnsafeLag() {
194195
s.Equal(lastSeenUnsafeTime+2, monitor.lastSeenUnsafeTime)
195196
s.Equal(uint64(2), monitor.lastSeenUnsafeNum)
196197

198+
healthy = <-healthUpdateCh
199+
s.Nil(healthy)
200+
s.Equal(lastSeenUnsafeTime+2, monitor.lastSeenUnsafeTime)
201+
s.Equal(uint64(2), monitor.lastSeenUnsafeNum)
202+
197203
s.NoError(monitor.Stop())
198204
}
199205

0 commit comments

Comments
 (0)