Skip to content

Commit 619220d

Browse files
alex-plekhanovalex-plekhanov
committed
IGNITE-20697 Store crash recovery data to checkpoint recovery files
1 parent 871db13 commit 619220d

File tree

12 files changed

+160
-46
lines changed

12 files changed

+160
-46
lines changed

‎modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1576,13 +1576,13 @@ private long requiredOffheap() {
15761576
for (DataRegionConfiguration dataReg : dataRegions) {
15771577
res += dataReg.getMaxSize();
15781578

1579-
res += U.checkpointBufferSize(dataReg);
1579+
res += U.checkpointBufferSize(memCfg, dataReg);
15801580
}
15811581
}
15821582

15831583
res += memCfg.getDefaultDataRegionConfiguration().getMaxSize();
15841584

1585-
res += U.checkpointBufferSize(memCfg.getDefaultDataRegionConfiguration());
1585+
res += U.checkpointBufferSize(memCfg, memCfg.getDefaultDataRegionConfiguration());
15861586

15871587
return res;
15881588
}

‎modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1192,7 +1192,7 @@ private long[] calculateFragmentSizes(String regionName, int concLvl, long cache
11921192
long cacheSize = regCfg.getMaxSize();
11931193

11941194
// Checkpoint buffer size can not be greater than cache size, it does not make sense.
1195-
long chpBufSize = checkpointBufferSize(regCfg);
1195+
long chpBufSize = checkpointBufferSize(dsCfg, regCfg);
11961196

11971197
if (chpBufSize > cacheSize) {
11981198
U.quietAndInfo(log,

‎modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/checkpoint/CheckpointProgressImpl.java

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -215,8 +215,10 @@ public void reason(String reason) {
215215
@Override public void updateEvictedPages(int delta) {
216216
A.ensure(delta > 0, "param must be positive");
217217

218-
if (evictedPagesCounter() != null)
219-
evictedPagesCounter().addAndGet(delta);
218+
AtomicInteger cntr = evictedPagesCounter();
219+
220+
if (cntr != null)
221+
cntr.addAndGet(delta);
220222
}
221223

222224
/** {@inheritDoc} */

‎modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/CheckpointBufferOverflowWatchdog.java

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,11 +40,18 @@ class CheckpointBufferOverflowWatchdog {
4040
* {@link PagesWriteThrottlePolicy#CP_BUF_FILL_THRESHOLD} of the buffer is filled) and, hence, writer threads need
4141
* to be throttled.
4242
*
43-
* @return {@code true} iff Checkpoint Buffer is in danger zone
43+
* @return {@code true} if Checkpoint Buffer is in danger zone
4444
*/
4545
boolean isInDangerZone() {
4646
int checkpointBufLimit = (int)(pageMemory.checkpointBufferPagesSize() * CP_BUF_FILL_THRESHOLD);
4747

4848
return pageMemory.checkpointBufferPagesCount() > checkpointBufLimit;
4949
}
50+
51+
/**
52+
* @return Checkpoint Buffer fill rate.
53+
*/
54+
double fillRate() {
55+
return (double)pageMemory.checkpointBufferPagesCount() / pageMemory.checkpointBufferPagesSize();
56+
}
5057
}

‎modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/ExponentialBackoffThrottlingStrategy.java

Lines changed: 5 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
/**
2121
* Logic used to protect memory (mainly, Checkpoint Buffer) from exhaustion using exponential backoff.
2222
*/
23-
class ExponentialBackoffThrottlingStrategy {
23+
class ExponentialBackoffThrottlingStrategy implements ThrottlingStrategy {
2424
/**
2525
* Starting throttle time. Limits write speed to 1000 MB/s.
2626
*/
@@ -36,21 +36,13 @@ class ExponentialBackoffThrottlingStrategy {
3636
*/
3737
private final ExponentialBackoff backoff = new ExponentialBackoff(STARTING_THROTTLE_NANOS, BACKOFF_RATIO);
3838

39-
/**
40-
* Computes next duration (in nanos) to throttle a thread to protect Checkpoint Buffer.
41-
*
42-
* @return park time in nanos
43-
*/
44-
long protectionParkTime() {
39+
/** {@inheritDoc} */
40+
@Override public long protectionParkTime() {
4541
return backoff.nextDuration();
4642
}
4743

48-
/**
49-
* Resets the backoff counter. Invoked when no throttling is needed anymore.
50-
*
51-
* @return {@code true} iff the backoff was not already in a reset state
52-
*/
53-
boolean resetBackoff() {
44+
/** {@inheritDoc} */
45+
@Override public boolean reset() {
5446
return backoff.reset();
5547
}
5648
}

‎modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/FillRateBasedThrottlingStrategy.java

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.ignite.internal.processors.cache.persistence.pagemem;
19+
20+
import java.util.concurrent.atomic.AtomicBoolean;
21+
22+
import static org.apache.ignite.internal.processors.cache.persistence.pagemem.PagesWriteThrottlePolicy.CP_BUF_FILL_THRESHOLD;
23+
24+
/**
25+
* Logic used to protect memory (Checkpoint Buffer) from exhaustion using throttling duration based on storage fill rate.
26+
*/
27+
class FillRateBasedThrottlingStrategy implements ThrottlingStrategy {
28+
/**
29+
* Minimum throttle time. 10 microseconds.
30+
*/
31+
private static final long MIN_THROTTLE_NANOS = 10_000L;
32+
33+
/**
34+
* Maximum throttle time. 1 second.
35+
*/
36+
private static final long MAX_THROTTLE_NANOS = 1_000_000_000L;
37+
38+
/**
39+
* The exponent to calculate park time.
40+
*/
41+
private static final double POW = Math.log((double)MAX_THROTTLE_NANOS / MIN_THROTTLE_NANOS);
42+
43+
/** */
44+
private final CheckpointBufferOverflowWatchdog cpBufOverflowWatchdog;
45+
46+
/** */
47+
private final AtomicBoolean throttlingStarted = new AtomicBoolean();
48+
49+
/** */
50+
FillRateBasedThrottlingStrategy(CheckpointBufferOverflowWatchdog watchdog) {
51+
cpBufOverflowWatchdog = watchdog;
52+
}
53+
54+
/** {@inheritDoc} */
55+
@Override public long protectionParkTime() {
56+
double fillRate = cpBufOverflowWatchdog.fillRate();
57+
58+
if (fillRate < CP_BUF_FILL_THRESHOLD)
59+
return 0;
60+
61+
throttlingStarted.set(true);
62+
63+
return (long)(Math.exp(POW * (fillRate - CP_BUF_FILL_THRESHOLD) / (1 - CP_BUF_FILL_THRESHOLD)) * MIN_THROTTLE_NANOS);
64+
}
65+
66+
/** {@inheritDoc} */
67+
@Override public boolean reset() {
68+
return throttlingStarted.compareAndSet(true, false);
69+
}
70+
}

‎modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -427,12 +427,19 @@ public PageMemoryImpl(
427427
* Resolves instance of {@link PagesWriteThrottlePolicy} according to chosen throttle policy.
428428
*/
429429
private void initWriteThrottle() {
430+
boolean fillRateBasedCpBufProtection = ctx.kernalContext().config().getDataStorageConfiguration()
431+
.isWriteRecoveryDataOnCheckpoint();
432+
430433
if (throttlingPlc == ThrottlingPolicy.SPEED_BASED)
431434
writeThrottle = new PagesWriteSpeedBasedThrottle(this, cpProgressProvider, stateChecker, log);
432-
else if (throttlingPlc == ThrottlingPolicy.TARGET_RATIO_BASED)
433-
writeThrottle = new PagesWriteThrottle(this, cpProgressProvider, stateChecker, false, log);
434-
else if (throttlingPlc == ThrottlingPolicy.CHECKPOINT_BUFFER_ONLY)
435-
writeThrottle = new PagesWriteThrottle(this, null, stateChecker, true, log);
435+
else if (throttlingPlc == ThrottlingPolicy.TARGET_RATIO_BASED) {
436+
writeThrottle = new PagesWriteThrottle(this, cpProgressProvider, stateChecker,
437+
false, fillRateBasedCpBufProtection, log);
438+
}
439+
else if (throttlingPlc == ThrottlingPolicy.CHECKPOINT_BUFFER_ONLY) {
440+
writeThrottle = new PagesWriteThrottle(this, null, stateChecker,
441+
true, fillRateBasedCpBufProtection, log);
442+
}
436443
}
437444

438445
/** {@inheritDoc} */

‎modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteSpeedBasedThrottle.java

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -135,7 +135,7 @@ private long computeThrottlingParkTime(boolean isPageInCheckpoint, long curNanoT
135135
if (isPageInCheckpoint) {
136136
// The fact that we are here means that we checked whether CP Buffer is in danger zone and found that
137137
// it is ok, so its protector may relax, hence we reset it.
138-
cpBufferProtector.resetBackoff();
138+
cpBufferProtector.reset();
139139
}
140140
return cleanPagesProtector.protectionParkTime(curNanoTime);
141141
}
@@ -230,7 +230,7 @@ long getCleanPagesProtectionParkTime(
230230

231231
/** {@inheritDoc} */
232232
@Override public void onFinishCheckpoint() {
233-
cpBufferProtector.resetBackoff();
233+
cpBufferProtector.reset();
234234

235235
cleanPagesProtector.finish();
236236
markSpeedAndAvgParkTime.finishInterval();
@@ -306,7 +306,7 @@ public double throttleWeight() {
306306
/** {@inheritDoc} */
307307
@Override public void wakeupThrottledThreads() {
308308
if (!isCpBufferOverflowThresholdExceeded()) {
309-
cpBufferProtector.resetBackoff();
309+
cpBufferProtector.reset();
310310

311311
unparkParkedThreads();
312312
}

‎modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteThrottle.java

Lines changed: 17 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -43,8 +43,7 @@ public class PagesWriteThrottle implements PagesWriteThrottlePolicy {
4343
private final CheckpointLockStateChecker stateChecker;
4444

4545
/** In-checkpoint protection logic. */
46-
private final ExponentialBackoffThrottlingStrategy inCheckpointProtection
47-
= new ExponentialBackoffThrottlingStrategy();
46+
private final ThrottlingStrategy inCheckpointProtection;
4847

4948
/** Not-in-checkpoint protection logic. */
5049
private final ExponentialBackoffThrottlingStrategy notInCheckpointProtection
@@ -64,12 +63,15 @@ public class PagesWriteThrottle implements PagesWriteThrottlePolicy {
6463
* @param cpProgress Database manager.
6564
* @param stateChecker checkpoint lock state checker.
6665
* @param throttleOnlyPagesInCheckpoint If true, throttle will only protect from checkpoint buffer overflow.
66+
* @param fillRateBasedCpBufProtection If true, fill rate based throttling will be used to protect from
67+
* checkpoint buffer overflow.
6768
* @param log Logger.
6869
*/
6970
public PagesWriteThrottle(PageMemoryImpl pageMemory,
7071
IgniteOutClosure<CheckpointProgress> cpProgress,
7172
CheckpointLockStateChecker stateChecker,
7273
boolean throttleOnlyPagesInCheckpoint,
74+
boolean fillRateBasedCpBufProtection,
7375
IgniteLogger log
7476
) {
7577
this.pageMemory = pageMemory;
@@ -78,6 +80,8 @@ public PagesWriteThrottle(PageMemoryImpl pageMemory,
7880
this.throttleOnlyPagesInCheckpoint = throttleOnlyPagesInCheckpoint;
7981
cpBufferWatchdog = new CheckpointBufferOverflowWatchdog(pageMemory);
8082
this.log = log;
83+
inCheckpointProtection = fillRateBasedCpBufProtection ? new FillRateBasedThrottlingStrategy(cpBufferWatchdog) :
84+
new ExponentialBackoffThrottlingStrategy();
8185

8286
assert throttleOnlyPagesInCheckpoint || cpProgress != null
8387
: "cpProgress must be not null if ratio based throttling mode is used";
@@ -95,11 +99,13 @@ public PagesWriteThrottle(PageMemoryImpl pageMemory,
9599
if (!shouldThrottle && !throttleOnlyPagesInCheckpoint) {
96100
CheckpointProgress progress = cpProgress.apply();
97101

98-
AtomicInteger writtenPagesCntr = progress == null ? null : cpProgress.apply().writtenPagesCounter();
102+
AtomicInteger writtenPagesCntr = progress == null ? null : progress.writtenPagesCounter();
103+
AtomicInteger writtenRecoveryPagesCntr = progress == null ? null : progress.writtenRecoveryPagesCounter();
99104

100-
if (progress == null || writtenPagesCntr == null)
105+
if (progress == null || writtenPagesCntr == null || writtenRecoveryPagesCntr == null)
101106
return; // Don't throttle if checkpoint is not running.
102107

108+
int cpWrittenRecoveryPages = writtenRecoveryPagesCntr.get();
103109
int cpWrittenPages = writtenPagesCntr.get();
104110

105111
int cpTotalPages = progress.currentCheckpointPagesCount();
@@ -109,7 +115,8 @@ public PagesWriteThrottle(PageMemoryImpl pageMemory,
109115
shouldThrottle = pageMemory.shouldThrottle(3.0 / 4);
110116
}
111117
else {
112-
double dirtyRatioThreshold = ((double)cpWrittenPages) / cpTotalPages;
118+
double dirtyRatioThreshold = cpWrittenRecoveryPages == 0 ? ((double)cpWrittenPages) / cpTotalPages :
119+
(cpWrittenRecoveryPages + cpWrittenPages) / 2d / cpTotalPages;
113120

114121
// Starting with 0.05 to avoid throttle right after checkpoint start
115122
// 7/12 is maximum ratio of dirty pages
@@ -119,8 +126,7 @@ public PagesWriteThrottle(PageMemoryImpl pageMemory,
119126
}
120127
}
121128

122-
ExponentialBackoffThrottlingStrategy exponentialThrottle = isPageInCheckpoint
123-
? inCheckpointProtection : notInCheckpointProtection;
129+
ThrottlingStrategy exponentialThrottle = isPageInCheckpoint ? inCheckpointProtection : notInCheckpointProtection;
124130

125131
if (shouldThrottle) {
126132
long throttleParkTimeNs = exponentialThrottle.protectionParkTime();
@@ -155,7 +161,7 @@ public PagesWriteThrottle(PageMemoryImpl pageMemory,
155161
pageMemory.metrics().addThrottlingTime(U.currentTimeMillis() - startTime);
156162
}
157163
else {
158-
boolean backoffWasAlreadyStarted = exponentialThrottle.resetBackoff();
164+
boolean backoffWasAlreadyStarted = exponentialThrottle.reset();
159165

160166
if (isPageInCheckpoint && backoffWasAlreadyStarted)
161167
unparkParkedThreads();
@@ -165,7 +171,7 @@ public PagesWriteThrottle(PageMemoryImpl pageMemory,
165171
/** {@inheritDoc} */
166172
@Override public void wakeupThrottledThreads() {
167173
if (!isCpBufferOverflowThresholdExceeded()) {
168-
inCheckpointProtection.resetBackoff();
174+
inCheckpointProtection.reset();
169175

170176
unparkParkedThreads();
171177
}
@@ -184,8 +190,8 @@ private void unparkParkedThreads() {
184190

185191
/** {@inheritDoc} */
186192
@Override public void onFinishCheckpoint() {
187-
inCheckpointProtection.resetBackoff();
188-
notInCheckpointProtection.resetBackoff();
193+
inCheckpointProtection.reset();
194+
notInCheckpointProtection.reset();
189195
}
190196

191197
/** {@inheritDoc} */

‎modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/ThrottlingStrategy.java

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
package org.apache.ignite.internal.processors.cache.persistence.pagemem;
2+
3+
/**
4+
* Strategy used to protect memory from exhaustion.
5+
*/
6+
public interface ThrottlingStrategy {
7+
/**
8+
* Computes next duration (in nanos) to throttle a thread.
9+
*
10+
* @return park time in nanos.
11+
*/
12+
public long protectionParkTime();
13+
14+
/**
15+
* Resets the state. Invoked when no throttling is needed anymore.
16+
*
17+
* @return {@code true} if the instance was not already in a reset state
18+
*/
19+
public boolean reset();
20+
}

‎modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java

Lines changed: 17 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -342,8 +342,15 @@ public abstract class IgniteUtils {
342342
/** Minimum checkpointing page buffer size (may be adjusted by Ignite). */
343343
public static final Long DFLT_MIN_CHECKPOINTING_PAGE_BUFFER_SIZE = GB / 4;
344344

345-
/** Default minimum checkpointing page buffer size (may be adjusted by Ignite). */
346-
public static final Long DFLT_MAX_CHECKPOINTING_PAGE_BUFFER_SIZE = 2 * GB;
345+
/** Default maximum checkpointing page buffer size (when recovery data stored in WAL). */
346+
public static final Long DFLT_MAX_CHECKPOINTING_PAGE_BUFFER_SIZE_WAL_RECOVERY = 2 * GB;
347+
348+
/**
349+
* Default maximum checkpointing page buffer size (when recovery data stored on checkpoint).
350+
* In this mode checkpoint duration can be twice as long as for mode with storing recovery data to WAL.
351+
* Also, checkpoint buffer pages can't be released during write recovery data phase, so we need larger buffer size.
352+
*/
353+
public static final Long DFLT_MAX_CHECKPOINTING_PAGE_BUFFER_SIZE_CP_RECOVERY = 5 * GB;
347354

348355
/** @see IgniteSystemProperties#IGNITE_MBEAN_APPEND_CLASS_LOADER_ID */
349356
public static final boolean DFLT_MBEAN_APPEND_CLASS_LOADER_ID = true;
@@ -11160,19 +11167,21 @@ public static <T> T fromBytes(byte[] data) {
1116011167
* @param regCfg Configuration.
1116111168
* @return Checkpoint buffer size.
1116211169
*/
11163-
public static long checkpointBufferSize(DataRegionConfiguration regCfg) {
11170+
public static long checkpointBufferSize(DataStorageConfiguration dsCfg, DataRegionConfiguration regCfg) {
1116411171
if (!regCfg.isPersistenceEnabled())
1116511172
return 0L;
1116611173

1116711174
long res = regCfg.getCheckpointPageBufferSize();
1116811175

1116911176
if (res == 0L) {
11177+
long maxCpPageBufSize = dsCfg.isWriteRecoveryDataOnCheckpoint() ?
11178+
DFLT_MAX_CHECKPOINTING_PAGE_BUFFER_SIZE_CP_RECOVERY :
11179+
DFLT_MAX_CHECKPOINTING_PAGE_BUFFER_SIZE_WAL_RECOVERY;
11180+
1117011181
if (regCfg.getMaxSize() < GB)
1117111182
res = Math.min(DFLT_MIN_CHECKPOINTING_PAGE_BUFFER_SIZE, regCfg.getMaxSize());
11172-
else if (regCfg.getMaxSize() < 8 * GB)
11173-
res = regCfg.getMaxSize() / 4;
1117411183
else
11175-
res = DFLT_MAX_CHECKPOINTING_PAGE_BUFFER_SIZE;
11184+
res = Math.min(regCfg.getMaxSize() / 4, maxCpPageBufSize);
1117611185
}
1117711186

1117811187
return res;
@@ -11194,7 +11203,7 @@ public static long adjustedWalHistorySize(DataStorageConfiguration dsCfg, @Nulla
1119411203

1119511204
if (dsCfg.getDataRegionConfigurations() != null) {
1119611205
for (DataRegionConfiguration regCfg : dsCfg.getDataRegionConfigurations()) {
11197-
long cpBufSize = checkpointBufferSize(regCfg);
11206+
long cpBufSize = checkpointBufferSize(dsCfg, regCfg);
1119811207

1119911208
if (cpBufSize > regCfg.getMaxSize())
1120011209
cpBufSize = regCfg.getMaxSize();
@@ -11207,7 +11216,7 @@ public static long adjustedWalHistorySize(DataStorageConfiguration dsCfg, @Nulla
1120711216
{
1120811217
DataRegionConfiguration regCfg = dsCfg.getDefaultDataRegionConfiguration();
1120911218

11210-
long cpBufSize = checkpointBufferSize(regCfg);
11219+
long cpBufSize = checkpointBufferSize(dsCfg, regCfg);
1121111220

1121211221
if (cpBufSize > regCfg.getMaxSize())
1121311222
cpBufSize = regCfg.getMaxSize();

0 commit comments

Comments
 (0)