From af7f1b394aa3935f178252851e74c35b3a742269 Mon Sep 17 00:00:00 2001 From: Dmitry Kropachev Date: Tue, 25 Mar 2025 19:38:22 -0400 Subject: [PATCH] Introduce exponential retry backoff policy Make driver wait before retry. It is needed to mitigate retry storms that can happen in certain cases. --- .../api/core/config/DefaultDriverOption.java | 15 +- .../driver/api/core/config/OptionsMap.java | 4 + .../api/core/config/TypedDriverOption.java | 14 ++ .../api/core/context/DriverContext.java | 21 ++ .../api/core/retry/BackoffRetryPolicy.java | 185 +++++++++++++++++ .../api/core/retry/BackoffRetryVerdict.java | 32 +++ .../core/context/DefaultDriverContext.java | 18 ++ .../driver/internal/core/cql/Conversions.java | 6 + .../internal/core/cql/CqlRequestHandler.java | 147 +++++++++++-- .../retry/DefaultBackoffRetryVerdict.java | 42 ++++ .../core/retry/ExponentialBackoffPolicy.java | 193 ++++++++++++++++++ .../internal/core/retry/NoBackoffPolicy.java | 109 ++++++++++ core/src/main/resources/reference.conf | 21 ++ .../core/config/TypedDriverOptionTest.java | 1 + .../map/MapBasedDriverConfigLoaderTest.java | 2 + .../core/cql/RequestHandlerTestHarness.java | 2 + .../core/config/MapBasedConfigLoaderIT.java | 5 + manual/core/retries/README.md | 25 +++ 18 files changed, 824 insertions(+), 18 deletions(-) create mode 100644 core/src/main/java/com/datastax/oss/driver/api/core/retry/BackoffRetryPolicy.java create mode 100644 core/src/main/java/com/datastax/oss/driver/api/core/retry/BackoffRetryVerdict.java create mode 100644 core/src/main/java/com/datastax/oss/driver/internal/core/retry/DefaultBackoffRetryVerdict.java create mode 100644 core/src/main/java/com/datastax/oss/driver/internal/core/retry/ExponentialBackoffPolicy.java create mode 100644 core/src/main/java/com/datastax/oss/driver/internal/core/retry/NoBackoffPolicy.java diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/config/DefaultDriverOption.java b/core/src/main/java/com/datastax/oss/driver/api/core/config/DefaultDriverOption.java index 7ff3b2719ba..dca7a59d6a2 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/config/DefaultDriverOption.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/config/DefaultDriverOption.java @@ -190,6 +190,20 @@ public enum DefaultDriverOption implements DriverOption { */ RETRY_POLICY_CLASS("advanced.retry-policy.class"), + // BACKOFF_RETRY_POLICY is a collection of sub-properties + BACKOFF_RETRY_POLICY("advanced.backoff-retry-policy"), + + /** + * The class of the backoff retry policy. + * + *

Value-type: {@link String} + */ + BACKOFF_RETRY_POLICY_CLASS("advanced.backoff-retry-policy.class"), + + BACKOFF_RETRY_MAX_BACKOFF_MS("advanced.backoff-retry-policy.max-backoff-ms"), + BACKOFF_RETRY_BASE_BACKOFF_MS("advanced.backoff-retry-policy.base-backoff-ms"), + BACKOFF_RETRY_JITTER_RATIO("advanced.backoff-retry-policy.jitter-ratio"), + // SPECULATIVE_EXECUTION_POLICY is a collection of sub-properties SPECULATIVE_EXECUTION_POLICY("advanced.speculative-execution-policy"), /** @@ -537,7 +551,6 @@ public enum DefaultDriverOption implements DriverOption { *

Value-type: {@link java.time.Duration Duration} */ METRICS_NODE_CQL_MESSAGES_INTERVAL("advanced.metrics.node.cql-messages.refresh-interval"), - /** * Whether or not to disable the Nagle algorithm. * diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/config/OptionsMap.java b/core/src/main/java/com/datastax/oss/driver/api/core/config/OptionsMap.java index 64665709028..7953154509d 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/config/OptionsMap.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/config/OptionsMap.java @@ -281,6 +281,10 @@ protected static void fillWithDriverDefaults(OptionsMap map) { map.put(TypedDriverOption.RECONNECTION_BASE_DELAY, Duration.ofSeconds(1)); map.put(TypedDriverOption.RECONNECTION_MAX_DELAY, Duration.ofSeconds(60)); map.put(TypedDriverOption.RETRY_POLICY_CLASS, "DefaultRetryPolicy"); + map.put(TypedDriverOption.BACKOFF_RETRY_POLICY_CLASS, "NoBackoffPolicy"); + map.put(TypedDriverOption.BACKOFF_RETRY_BASE_BACKOFF_MS, 100); + map.put(TypedDriverOption.BACKOFF_RETRY_MAX_BACKOFF_MS, 10000); + map.put(TypedDriverOption.BACKOFF_RETRY_JITTER_RATIO, 0.1); map.put(TypedDriverOption.SPECULATIVE_EXECUTION_POLICY_CLASS, "NoSpeculativeExecutionPolicy"); map.put(TypedDriverOption.TIMESTAMP_GENERATOR_CLASS, "AtomicTimestampGenerator"); map.put(TypedDriverOption.TIMESTAMP_GENERATOR_DRIFT_WARNING_THRESHOLD, Duration.ofSeconds(1)); diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/config/TypedDriverOption.java b/core/src/main/java/com/datastax/oss/driver/api/core/config/TypedDriverOption.java index b6051749c71..c2e9c12cb6f 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/config/TypedDriverOption.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/config/TypedDriverOption.java @@ -199,6 +199,20 @@ public String toString() { /** The class of the retry policy. */ public static final TypedDriverOption RETRY_POLICY_CLASS = new TypedDriverOption<>(DefaultDriverOption.RETRY_POLICY_CLASS, GenericType.STRING); + /** The class of the retry policy. */ + public static final TypedDriverOption BACKOFF_RETRY_POLICY_CLASS = + new TypedDriverOption<>(DefaultDriverOption.BACKOFF_RETRY_POLICY_CLASS, GenericType.STRING); + /** The class of the retry policy. */ + public static final TypedDriverOption BACKOFF_RETRY_BASE_BACKOFF_MS = + new TypedDriverOption<>( + DefaultDriverOption.BACKOFF_RETRY_BASE_BACKOFF_MS, GenericType.INTEGER); + /** The class of the retry policy. */ + public static final TypedDriverOption BACKOFF_RETRY_MAX_BACKOFF_MS = + new TypedDriverOption<>( + DefaultDriverOption.BACKOFF_RETRY_MAX_BACKOFF_MS, GenericType.INTEGER); + /** The class of the retry policy. */ + public static final TypedDriverOption BACKOFF_RETRY_JITTER_RATIO = + new TypedDriverOption<>(DefaultDriverOption.BACKOFF_RETRY_JITTER_RATIO, GenericType.DOUBLE); /** The class of the speculative execution policy. */ public static final TypedDriverOption SPECULATIVE_EXECUTION_POLICY_CLASS = new TypedDriverOption<>( diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/context/DriverContext.java b/core/src/main/java/com/datastax/oss/driver/api/core/context/DriverContext.java index 5b32389e362..d6e246361f6 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/context/DriverContext.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/context/DriverContext.java @@ -27,6 +27,7 @@ import com.datastax.oss.driver.api.core.loadbalancing.LoadBalancingPolicy; import com.datastax.oss.driver.api.core.metadata.NodeStateListener; import com.datastax.oss.driver.api.core.metadata.schema.SchemaChangeListener; +import com.datastax.oss.driver.api.core.retry.BackoffRetryPolicy; import com.datastax.oss.driver.api.core.retry.RetryPolicy; import com.datastax.oss.driver.api.core.session.Session; import com.datastax.oss.driver.api.core.session.throttling.RequestThrottler; @@ -95,6 +96,26 @@ default RetryPolicy getRetryPolicy(@NonNull String profileName) { return (policy != null) ? policy : getRetryPolicies().get(DriverExecutionProfile.DEFAULT_NAME); } + /** + * @return The driver's retry policies, keyed by profile name; the returned map is guaranteed to + * never be {@code null} and to always contain an entry for the {@value + * DriverExecutionProfile#DEFAULT_NAME} profile. + */ + @NonNull + Map getBackoffRetryPolicies(); + + /** + * @param profileName the profile name; never {@code null}. + * @return The driver's retry policy for the given profile; never {@code null}. + */ + @NonNull + default BackoffRetryPolicy getBackoffRetryPolicy(@NonNull String profileName) { + BackoffRetryPolicy policy = getBackoffRetryPolicies().get(profileName); + return (policy != null) + ? policy + : getBackoffRetryPolicies().get(DriverExecutionProfile.DEFAULT_NAME); + } + /** * @return The driver's speculative execution policies, keyed by profile name; the returned map is * guaranteed to never be {@code null} and to always contain an entry for the {@value diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/retry/BackoffRetryPolicy.java b/core/src/main/java/com/datastax/oss/driver/api/core/retry/BackoffRetryPolicy.java new file mode 100644 index 00000000000..c874f36d199 --- /dev/null +++ b/core/src/main/java/com/datastax/oss/driver/api/core/retry/BackoffRetryPolicy.java @@ -0,0 +1,185 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.api.core.retry; + +import com.datastax.oss.driver.api.core.ConsistencyLevel; +import com.datastax.oss.driver.api.core.connection.ClosedConnectionException; +import com.datastax.oss.driver.api.core.connection.HeartbeatException; +import com.datastax.oss.driver.api.core.loadbalancing.LoadBalancingPolicy; +import com.datastax.oss.driver.api.core.servererrors.BootstrappingException; +import com.datastax.oss.driver.api.core.servererrors.CoordinatorException; +import com.datastax.oss.driver.api.core.servererrors.FunctionFailureException; +import com.datastax.oss.driver.api.core.servererrors.OverloadedException; +import com.datastax.oss.driver.api.core.servererrors.ProtocolError; +import com.datastax.oss.driver.api.core.servererrors.QueryValidationException; +import com.datastax.oss.driver.api.core.servererrors.ReadFailureException; +import com.datastax.oss.driver.api.core.servererrors.ReadTimeoutException; +import com.datastax.oss.driver.api.core.servererrors.ServerError; +import com.datastax.oss.driver.api.core.servererrors.TruncateException; +import com.datastax.oss.driver.api.core.servererrors.WriteFailureException; +import com.datastax.oss.driver.api.core.servererrors.WriteType; +import com.datastax.oss.driver.api.core.session.Request; +import edu.umd.cs.findbugs.annotations.NonNull; + +/** + * Defines the behavior to adopt when a request fails. + * + *

For each request, the driver gets a "query plan" (a list of coordinators to try) from the + * {@link LoadBalancingPolicy}, and tries each node in sequence. This policy is invoked if the + * request to that node fails. + * + *

The methods of this interface are invoked on I/O threads, therefore implementations should + * never block. In particular, don't call {@link Thread#sleep(long)} to retry after a delay: + * this would prevent asynchronous processing of other requests, and very negatively impact + * throughput. If the application needs to back off and retry later, this should be implemented in + * client code, not in this policy. + */ +public interface BackoffRetryPolicy extends AutoCloseable { + /** + * Whether to retry when the server replied with a {@code READ_TIMEOUT} error; this indicates a + * server-side timeout during a read query, i.e. some replicas did not reply to the + * coordinator in time. + * + * @param request the request that timed out. + * @param cl the requested consistency level. + * @param blockFor the minimum number of replica acknowledgements/responses that were required to + * fulfill the operation. + * @param received the number of replica that had acknowledged/responded to the operation before + * it failed. + * @param dataPresent whether the actual data was amongst the received replica responses. See + * {@link ReadTimeoutException#wasDataPresent()}. + * @param retryCount how many times the retry policy has been invoked already for this request + * (not counting the current invocation). + */ + int onReadTimeoutBackoffMs( + @NonNull Request request, + @NonNull ConsistencyLevel cl, + int blockFor, + int received, + boolean dataPresent, + int retryCount, + RetryVerdict verdict); + + /** + * Whether to retry when the server replied with a {@code WRITE_TIMEOUT} error; this indicates a + * server-side timeout during a write query, i.e. some replicas did not reply to the + * coordinator in time. + * + *

Note that this method will only be invoked for {@link Request#isIdempotent()} idempotent} + * requests: when a write times out, it is impossible to determine with 100% certainty whether the + * mutation was applied or not, so the write is never safe to retry; the driver will rethrow the + * error directly, without invoking the retry policy. + * + * @param request the request that timed out. + * @param cl the requested consistency level. + * @param writeType the type of the write for which the timeout was raised. + * @param blockFor the minimum number of replica acknowledgements/responses that were required to + * fulfill the operation. + * @param received the number of replica that had acknowledged/responded to the operation before + * it failed. + * @param retryCount how many times the retry policy has been invoked already for this request + * (not counting the current invocation). + */ + int onWriteTimeoutBackoffMs( + @NonNull Request request, + @NonNull ConsistencyLevel cl, + @NonNull WriteType writeType, + int blockFor, + int received, + int retryCount, + RetryVerdict verdict); + + /** + * Whether to retry when the server replied with an {@code UNAVAILABLE} error; this indicates that + * the coordinator determined that there were not enough replicas alive to perform a query with + * the requested consistency level. + * + * @param request the request that timed out. + * @param cl the requested consistency level. + * @param required the number of replica acknowledgements/responses required to perform the + * operation (with its required consistency level). + * @param alive the number of replicas that were known to be alive by the coordinator node when it + * tried to execute the operation. + * @param retryCount how many times the retry policy has been invoked already for this request + * (not counting the current invocation). + */ + int onUnavailableBackoffMs( + @NonNull Request request, + @NonNull ConsistencyLevel cl, + int required, + int alive, + int retryCount, + RetryVerdict verdict); + + /** + * Whether to retry when a request was aborted before we could get a response from the server. + * + *

This can happen in two cases: if the connection was closed due to an external event (this + * will manifest as a {@link ClosedConnectionException}, or {@link HeartbeatException} for a + * heartbeat failure); or if there was an unexpected error while decoding the response (this can + * only be a driver bug). + * + *

Note that this method will only be invoked for {@linkplain Request#isIdempotent() + * idempotent} requests: when execution was aborted before getting a response, it is impossible to + * determine with 100% certainty whether a mutation was applied or not, so a write is never safe + * to retry; the driver will rethrow the error directly, without invoking the retry policy. + * + * @param request the request that was aborted. + * @param error the error. + * @param retryCount how many times the retry policy has been invoked already for this request + * (not counting the current invocation). + */ + int onRequestAbortedBackoffMs( + @NonNull Request request, @NonNull Throwable error, int retryCount, RetryVerdict verdict); + + /** + * Whether to retry when the server replied with a recoverable error (other than {@code + * READ_TIMEOUT}, {@code WRITE_TIMEOUT} or {@code UNAVAILABLE}). + * + *

This can happen for the following errors: {@link OverloadedException}, {@link ServerError}, + * {@link TruncateException}, {@link ReadFailureException}, {@link WriteFailureException}. + * + *

The following errors are handled internally by the driver, and therefore will never + * be encountered in this method: + * + *

+ * + *

Note that this method will only be invoked for {@link Request#isIdempotent()} idempotent} + * requests: when execution was aborted before getting a response, it is impossible to determine + * with 100% certainty whether a mutation was applied or not, so a write is never safe to retry; + * the driver will rethrow the error directly, without invoking the retry policy. + * + * @param request the request that failed. + * @param error the error. + * @param retryCount how many times the retry policy has been invoked already for this request + * (not counting the current invocation). + */ + int onErrorResponseBackoff( + @NonNull Request request, + @NonNull CoordinatorException error, + int retryCount, + RetryVerdict verdict); + + /** Called when the cluster that this policy is associated with closes. */ + @Override + void close(); +} diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/retry/BackoffRetryVerdict.java b/core/src/main/java/com/datastax/oss/driver/api/core/retry/BackoffRetryVerdict.java new file mode 100644 index 00000000000..f95688d0319 --- /dev/null +++ b/core/src/main/java/com/datastax/oss/driver/api/core/retry/BackoffRetryVerdict.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.api.core.retry; + +import java.time.Duration; + +/** + * The verdict returned by a {@link RetryPolicy} determining what to do when a request failed. A + * verdict contains a {@link RetryDecision} indicating if a retry should be attempted at all and + * where, with what delay, and a method that allows the original request to be modified before the + * retry. + */ +public interface BackoffRetryVerdict extends RetryVerdict { + + /** @return a delay that request needs to take before retrying. */ + Duration getRetryBackoff(); +} diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/context/DefaultDriverContext.java b/core/src/main/java/com/datastax/oss/driver/internal/core/context/DefaultDriverContext.java index dd59a76f7d8..53d49b228e9 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/context/DefaultDriverContext.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/context/DefaultDriverContext.java @@ -38,6 +38,7 @@ import com.datastax.oss.driver.api.core.metadata.Node; import com.datastax.oss.driver.api.core.metadata.NodeStateListener; import com.datastax.oss.driver.api.core.metadata.schema.SchemaChangeListener; +import com.datastax.oss.driver.api.core.retry.BackoffRetryPolicy; import com.datastax.oss.driver.api.core.retry.RetryPolicy; import com.datastax.oss.driver.api.core.session.ProgrammaticArguments; import com.datastax.oss.driver.api.core.session.throttling.RequestThrottler; @@ -150,6 +151,8 @@ public class DefaultDriverContext implements InternalDriverContext { new LazyReference<>("reconnectionPolicy", this::buildReconnectionPolicy, cycleDetector); private final LazyReference> retryPoliciesRef = new LazyReference<>("retryPolicies", this::buildRetryPolicies, cycleDetector); + private final LazyReference> backoffRetryPoliciesRef = + new LazyReference<>("backoffRetryPolicies", this::buildBackoffRetryPolicies, cycleDetector); private final LazyReference> speculativeExecutionPoliciesRef = new LazyReference<>( @@ -367,6 +370,15 @@ protected Map buildRetryPolicies() { "com.datastax.oss.driver.internal.core.retry"); } + protected Map buildBackoffRetryPolicies() { + return Reflection.buildFromConfigProfiles( + this, + DefaultDriverOption.BACKOFF_RETRY_POLICY_CLASS, + DefaultDriverOption.BACKOFF_RETRY_POLICY, + BackoffRetryPolicy.class, + "com.datastax.oss.driver.internal.core.retry"); + } + protected Map buildSpeculativeExecutionPolicies() { return Reflection.buildFromConfigProfiles( this, @@ -768,6 +780,12 @@ public Map getRetryPolicies() { return retryPoliciesRef.get(); } + @NonNull + @Override + public Map getBackoffRetryPolicies() { + return backoffRetryPoliciesRef.get(); + } + @NonNull @Override public Map getSpeculativeExecutionPolicies() { diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/cql/Conversions.java b/core/src/main/java/com/datastax/oss/driver/internal/core/cql/Conversions.java index 88f35eb75a0..4e4c04b212a 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/cql/Conversions.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/cql/Conversions.java @@ -46,6 +46,7 @@ import com.datastax.oss.driver.api.core.metadata.schema.ColumnMetadata; import com.datastax.oss.driver.api.core.metadata.schema.RelationMetadata; import com.datastax.oss.driver.api.core.metadata.token.Partitioner; +import com.datastax.oss.driver.api.core.retry.BackoffRetryPolicy; import com.datastax.oss.driver.api.core.retry.RetryPolicy; import com.datastax.oss.driver.api.core.servererrors.AlreadyExistsException; import com.datastax.oss.driver.api.core.servererrors.BootstrappingException; @@ -606,6 +607,11 @@ public static RetryPolicy resolveRetryPolicy( return context.getRetryPolicy(executionProfile.getName()); } + public static BackoffRetryPolicy resolveBackoffRetryPolicy( + InternalDriverContext context, DriverExecutionProfile executionProfile) { + return context.getBackoffRetryPolicy(executionProfile.getName()); + } + /** * Use {@link #resolveSpeculativeExecutionPolicy(InternalDriverContext, DriverExecutionProfile)} * instead. diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/cql/CqlRequestHandler.java b/core/src/main/java/com/datastax/oss/driver/internal/core/cql/CqlRequestHandler.java index 99297f4266a..bb1e3d09914 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/cql/CqlRequestHandler.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/cql/CqlRequestHandler.java @@ -43,6 +43,7 @@ import com.datastax.oss.driver.api.core.metadata.token.Token; import com.datastax.oss.driver.api.core.metrics.DefaultNodeMetric; import com.datastax.oss.driver.api.core.metrics.DefaultSessionMetric; +import com.datastax.oss.driver.api.core.retry.BackoffRetryPolicy; import com.datastax.oss.driver.api.core.retry.RetryPolicy; import com.datastax.oss.driver.api.core.retry.RetryVerdict; import com.datastax.oss.driver.api.core.servererrors.BootstrappingException; @@ -434,6 +435,61 @@ private void cancelScheduledTasks() { } } + /** + * Schedules the request to the next available node. + * + * @param statement The statement to execute. + * @param retriedNode if not null, it will be attempted first before the rest of the query plan. + * @param queryPlan the list of nodes to try (shared with all other executions) + * @param currentExecutionIndex 0 for the initial execution, 1 for the first speculative one, etc. + * @param retryCount the number of times that the retry policy was invoked for this execution + * already (note that some internal retries don't go through the policy, and therefore don't + * increment this counter) + * @param scheduleNextExecution whether to schedule the next speculative execution + */ + private void scheduleRequest( + Statement statement, + Node retriedNode, + Queue queryPlan, + int currentExecutionIndex, + int retryCount, + boolean scheduleNextExecution, + int backoffMs) { + try { + scheduledExecutions.add( + timer.newTimeout( + (Timeout timeout1) -> { + if (!result.isDone()) { + LOG.trace( + "[{}] Starting delayed (by {}) execution {}", + CqlRequestHandler.this.logPrefix, + backoffMs, + currentExecutionIndex); + activeExecutionsCount.incrementAndGet(); + startedSpeculativeExecutionsCount.incrementAndGet(); + // Note that `node` is the first node of the execution, it might not be the + // "slow" one if there were retries, but in practice retries are rare. + sendRequest( + statement, + retriedNode, + queryPlan, + currentExecutionIndex, + retryCount, + scheduleNextExecution); + } + }, + backoffMs, + TimeUnit.MILLISECONDS)); + } catch (IllegalStateException e) { + // If we're racing with session shutdown, the timer might be stopped already. We don't want + // to schedule more executions anyway, so swallow the error. + if (!"cannot be started once stopped".equals(e.getMessage())) { + Loggers.warnWithException( + LOG, "[{}] Error while scheduling delayed execution", logPrefix, e); + } + } + } + private void setFinalResult( Result resultMessage, Frame responseFrame, @@ -887,7 +943,10 @@ private void processErrorResponse(Error errorMessage) { setFinalError(statement, error, node, execution); } else { RetryPolicy retryPolicy = Conversions.resolveRetryPolicy(context, executionProfile); + BackoffRetryPolicy backoffPolicy = + Conversions.resolveBackoffRetryPolicy(context, executionProfile); RetryVerdict verdict; + int backoff; if (error instanceof ReadTimeoutException) { ReadTimeoutException readTimeout = (ReadTimeoutException) error; verdict = @@ -898,6 +957,15 @@ private void processErrorResponse(Error errorMessage) { readTimeout.getReceived(), readTimeout.wasDataPresent(), retryCount); + backoff = + backoffPolicy.onReadTimeoutBackoffMs( + statement, + readTimeout.getConsistencyLevel(), + readTimeout.getBlockFor(), + readTimeout.getReceived(), + readTimeout.wasDataPresent(), + retryCount, + verdict); updateErrorMetrics( metricUpdater, verdict, @@ -916,6 +984,15 @@ private void processErrorResponse(Error errorMessage) { writeTimeout.getReceived(), retryCount) : RetryVerdict.RETHROW; + backoff = + backoffPolicy.onWriteTimeoutBackoffMs( + statement, + writeTimeout.getConsistencyLevel(), + writeTimeout.getWriteType(), + writeTimeout.getBlockFor(), + writeTimeout.getReceived(), + retryCount, + verdict); updateErrorMetrics( metricUpdater, verdict, @@ -931,6 +1008,14 @@ private void processErrorResponse(Error errorMessage) { unavailable.getRequired(), unavailable.getAlive(), retryCount); + backoff = + backoffPolicy.onUnavailableBackoffMs( + statement, + unavailable.getConsistencyLevel(), + unavailable.getRequired(), + unavailable.getAlive(), + retryCount, + verdict); updateErrorMetrics( metricUpdater, verdict, @@ -942,6 +1027,7 @@ private void processErrorResponse(Error errorMessage) { Conversions.resolveIdempotence(statement, executionProfile) ? retryPolicy.onErrorResponseVerdict(statement, error, retryCount) : RetryVerdict.RETHROW; + backoff = backoffPolicy.onErrorResponseBackoff(statement, error, retryCount, verdict); updateErrorMetrics( metricUpdater, verdict, @@ -949,34 +1035,56 @@ private void processErrorResponse(Error errorMessage) { DefaultNodeMetric.RETRIES_ON_OTHER_ERROR, DefaultNodeMetric.IGNORES_ON_OTHER_ERROR); } - processRetryVerdict(verdict, error); + processRetryVerdict(verdict, error, backoff); } } - private void processRetryVerdict(RetryVerdict verdict, Throwable error) { + private void processRetryVerdict(RetryVerdict verdict, Throwable error, int backoff) { LOG.trace("[{}] Processing retry decision {}", logPrefix, verdict); switch (verdict.getRetryDecision()) { case RETRY_SAME: recordError(node, error); trackNodeError(node, error, NANOTIME_NOT_MEASURED_YET); - sendRequest( - verdict.getRetryRequest(statement), - node, - queryPlan, - execution, - retryCount + 1, - false); + if (backoff > 0) { + scheduleRequest( + verdict.getRetryRequest(statement), + node, + queryPlan, + execution, + retryCount + 1, + false, + backoff); + } else { + sendRequest( + verdict.getRetryRequest(statement), + node, + queryPlan, + execution, + retryCount + 1, + false); + } break; case RETRY_NEXT: recordError(node, error); trackNodeError(node, error, NANOTIME_NOT_MEASURED_YET); - sendRequest( - verdict.getRetryRequest(statement), - null, - queryPlan, - execution, - retryCount + 1, - false); + if (backoff > 0) { + scheduleRequest( + verdict.getRetryRequest(statement), + null, + queryPlan, + execution, + retryCount + 1, + false, + backoff); + } else { + sendRequest( + verdict.getRetryRequest(statement), + null, + queryPlan, + execution, + retryCount + 1, + false); + } break; case RETHROW: trackNodeError(node, error, NANOTIME_NOT_MEASURED_YET); @@ -1018,13 +1126,18 @@ public void onFailure(Throwable error) { } LOG.trace("[{}] Request failure, processing: {}", logPrefix, error); RetryVerdict verdict; + int backoff; if (!Conversions.resolveIdempotence(statement, executionProfile) || error instanceof FrameTooLongException) { verdict = RetryVerdict.RETHROW; + backoff = 0; } else { try { RetryPolicy retryPolicy = Conversions.resolveRetryPolicy(context, executionProfile); + BackoffRetryPolicy backoffPolicy = + Conversions.resolveBackoffRetryPolicy(context, executionProfile); verdict = retryPolicy.onRequestAbortedVerdict(statement, error, retryCount); + backoff = backoffPolicy.onRequestAbortedBackoffMs(statement, error, retryCount, verdict); } catch (Throwable cause) { setFinalError( statement, @@ -1034,7 +1147,7 @@ public void onFailure(Throwable error) { return; } } - processRetryVerdict(verdict, error); + processRetryVerdict(verdict, error, backoff); updateErrorMetrics( ((DefaultNode) node).getMetricUpdater(), verdict, diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/retry/DefaultBackoffRetryVerdict.java b/core/src/main/java/com/datastax/oss/driver/internal/core/retry/DefaultBackoffRetryVerdict.java new file mode 100644 index 00000000000..ef9fcf61ef8 --- /dev/null +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/retry/DefaultBackoffRetryVerdict.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.internal.core.retry; + +import com.datastax.oss.driver.api.core.retry.BackoffRetryVerdict; +import com.datastax.oss.driver.api.core.retry.RetryDecision; +import edu.umd.cs.findbugs.annotations.NonNull; +import java.time.Duration; + +public class DefaultBackoffRetryVerdict extends DefaultRetryVerdict implements BackoffRetryVerdict { + private final Duration delay; + + public DefaultBackoffRetryVerdict(@NonNull RetryDecision decision, Duration delay) { + super(decision); + this.delay = delay; + } + + @Override + public String toString() { + return String.format("%s(%s)", getRetryDecision().name(), delay); + } + + @Override + public Duration getRetryBackoff() { + return delay; + } +} diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/retry/ExponentialBackoffPolicy.java b/core/src/main/java/com/datastax/oss/driver/internal/core/retry/ExponentialBackoffPolicy.java new file mode 100644 index 00000000000..1526ecb1925 --- /dev/null +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/retry/ExponentialBackoffPolicy.java @@ -0,0 +1,193 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.internal.core.retry; + +import com.datastax.oss.driver.api.core.ConsistencyLevel; +import com.datastax.oss.driver.api.core.config.DefaultDriverOption; +import com.datastax.oss.driver.api.core.config.DriverExecutionProfile; +import com.datastax.oss.driver.api.core.context.DriverContext; +import com.datastax.oss.driver.api.core.retry.BackoffRetryPolicy; +import com.datastax.oss.driver.api.core.retry.RetryVerdict; +import com.datastax.oss.driver.api.core.servererrors.CoordinatorException; +import com.datastax.oss.driver.api.core.servererrors.WriteType; +import com.datastax.oss.driver.api.core.session.Request; +import com.datastax.oss.driver.shaded.guava.common.annotations.VisibleForTesting; +import edu.umd.cs.findbugs.annotations.NonNull; +import java.util.SplittableRandom; +import net.jcip.annotations.ThreadSafe; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * The default retry policy. + * + *

This is a very conservative implementation: it triggers a maximum of one retry per request, + * and only in cases that have a high chance of success (see the method javadocs for detailed + * explanations of each case). + * + *

To activate this policy, modify the {@code advanced.retry-policy} section in the driver + * configuration, for example: + * + * 

+ * datastax-java-driver {
+ *   advanced.retry-policy {
+ *     class = DefaultRetryPolicy
+ *   }
+ * }
+ *
+ * + *

See {@code reference.conf} (in the manual or core driver JAR) for more details. + */ +@ThreadSafe +public class ExponentialBackoffPolicy implements BackoffRetryPolicy { + private static final SplittableRandom random = new SplittableRandom(); + + private static final Logger LOG = LoggerFactory.getLogger(ExponentialBackoffPolicy.class); + + @VisibleForTesting + public static final String BACKOFF_ON_READ_TIMEOUT = + "[{}] Delaying retry on read timeout for {}ms (consistency: {}, required responses: {}, " + + "received responses: {}, data retrieved: {}, retries: {})"; + + @VisibleForTesting + public static final String BACKOFF_ON_WRITE_TIMEOUT = + "[{}] Delaying retry on write timeout for {}ms (consistency: {}, write type: {}, " + + "required acknowledgments: {}, received acknowledgments: {}, retries: {})"; + + @VisibleForTesting + public static final String BACKOFF_ON_UNAVAILABLE = + "[{}] Delaying retry on unavailable exception for {}ms (consistency: {}, " + + "required replica: {}, alive replica: {}, retries: {})"; + + @VisibleForTesting + public static final String BACKOFF_ON_ABORTED = + "[{}] Delaying retry on aborted request for {}ms (retries: {})"; + + @VisibleForTesting + public static final String BACKOFF_ON_ERROR = + "[{}] Delaying on node error for {}ms (retries: {})"; + + private final String logPrefix; + private final int baseDelayMs; + private final int maxDelayMs; + private final double jitterRatio; + + public ExponentialBackoffPolicy(DriverContext context, String profileName) { + DriverExecutionProfile profile = context.getConfig().getProfile(profileName); + this.logPrefix = context.getSessionName() + "|" + profileName; + this.baseDelayMs = profile.getInt(DefaultDriverOption.BACKOFF_RETRY_BASE_BACKOFF_MS); + this.maxDelayMs = profile.getInt(DefaultDriverOption.BACKOFF_RETRY_MAX_BACKOFF_MS); + this.jitterRatio = profile.getDouble(DefaultDriverOption.BACKOFF_RETRY_JITTER_RATIO); + } + + public ExponentialBackoffPolicy( + String logPrefix, int baseDelayMs, int maxDelayMs, double jitterRatio) { + this.logPrefix = logPrefix; + this.baseDelayMs = baseDelayMs; + this.maxDelayMs = maxDelayMs; + this.jitterRatio = jitterRatio; + } + + @Override + public int onReadTimeoutBackoffMs( + @NonNull Request request, + @NonNull ConsistencyLevel cl, + int blockFor, + int received, + boolean dataPresent, + int retryCount, + RetryVerdict verdict) { + int backoffMs = calculateBackoffMs(retryCount); + if (LOG.isTraceEnabled() && backoffMs != 0) { + LOG.trace( + BACKOFF_ON_READ_TIMEOUT, logPrefix, backoffMs, cl, blockFor, received, false, retryCount); + } + return backoffMs; + } + + @Override + public int onWriteTimeoutBackoffMs( + @NonNull Request request, + @NonNull ConsistencyLevel cl, + @NonNull WriteType writeType, + int blockFor, + int received, + int retryCount, + RetryVerdict verdict) { + int backoffMs = calculateBackoffMs(retryCount); + if (LOG.isTraceEnabled() && backoffMs != 0) { + LOG.trace( + BACKOFF_ON_WRITE_TIMEOUT, + logPrefix, + backoffMs, + cl, + blockFor, + received, + false, + retryCount); + } + return backoffMs; + } + + @Override + public int onUnavailableBackoffMs( + @NonNull Request request, + @NonNull ConsistencyLevel cl, + int required, + int alive, + int retryCount, + RetryVerdict verdict) { + int backoffMs = calculateBackoffMs(retryCount); + if (LOG.isTraceEnabled() && backoffMs != 0) { + LOG.trace(BACKOFF_ON_UNAVAILABLE, logPrefix, backoffMs, cl, required, alive, retryCount); + } + return backoffMs; + } + + @Override + public int onRequestAbortedBackoffMs( + @NonNull Request request, @NonNull Throwable error, int retryCount, RetryVerdict verdict) { + int backoffMs = calculateBackoffMs(retryCount); + if (LOG.isTraceEnabled() && backoffMs != 0) { + LOG.trace(BACKOFF_ON_ABORTED, logPrefix, backoffMs, retryCount, error); + } + return backoffMs; + } + + @Override + public int onErrorResponseBackoff( + @NonNull Request request, + @NonNull CoordinatorException error, + int retryCount, + RetryVerdict verdict) { + int backoffMs = calculateBackoffMs(retryCount); + if (LOG.isTraceEnabled() && backoffMs != 0) { + LOG.trace(BACKOFF_ON_ERROR, logPrefix, backoffMs, retryCount, error); + } + return backoffMs; + } + + private int calculateBackoffMs(int attempt) { + int expDelay = (int) (baseDelayMs * Math.pow(2, attempt - 1)); + int jitter = random.nextInt((int) (jitterRatio * expDelay)); + return Math.min(expDelay + jitter, maxDelayMs); + } + + @Override + public void close() {} +} diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/retry/NoBackoffPolicy.java b/core/src/main/java/com/datastax/oss/driver/internal/core/retry/NoBackoffPolicy.java new file mode 100644 index 00000000000..d6e18ec3032 --- /dev/null +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/retry/NoBackoffPolicy.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.internal.core.retry; + +import com.datastax.oss.driver.api.core.ConsistencyLevel; +import com.datastax.oss.driver.api.core.context.DriverContext; +import com.datastax.oss.driver.api.core.retry.BackoffRetryPolicy; +import com.datastax.oss.driver.api.core.retry.RetryVerdict; +import com.datastax.oss.driver.api.core.servererrors.CoordinatorException; +import com.datastax.oss.driver.api.core.servererrors.WriteType; +import com.datastax.oss.driver.api.core.session.Request; +import edu.umd.cs.findbugs.annotations.NonNull; +import net.jcip.annotations.ThreadSafe; + +/** + * The default retry policy. + * + *

This is a very conservative implementation: it triggers a maximum of one retry per request, + * and only in cases that have a high chance of success (see the method javadocs for detailed + * explanations of each case). + * + *

To activate this policy, modify the {@code advanced.retry-policy} section in the driver + * configuration, for example: + * + * 

+ * datastax-java-driver {
+ *   advanced.retry-policy {
+ *     class = DefaultRetryPolicy
+ *   }
+ * }
+ *
+ * + * See {@code reference.conf} (in the manual or core driver JAR) for more details. + */ +@ThreadSafe +public class NoBackoffPolicy implements BackoffRetryPolicy { + + public NoBackoffPolicy(DriverContext context, String profileName) {} + + public NoBackoffPolicy() {} + + @Override + public int onReadTimeoutBackoffMs( + @NonNull Request request, + @NonNull ConsistencyLevel cl, + int blockFor, + int received, + boolean dataPresent, + int retryCount, + RetryVerdict verdict) { + return 0; + } + + @Override + public int onWriteTimeoutBackoffMs( + @NonNull Request request, + @NonNull ConsistencyLevel cl, + @NonNull WriteType writeType, + int blockFor, + int received, + int retryCount, + RetryVerdict verdict) { + return 0; + } + + @Override + public int onUnavailableBackoffMs( + @NonNull Request request, + @NonNull ConsistencyLevel cl, + int required, + int alive, + int retryCount, + RetryVerdict verdict) { + return 0; + } + + @Override + public int onRequestAbortedBackoffMs( + @NonNull Request request, @NonNull Throwable error, int retryCount, RetryVerdict verdict) { + return 0; + } + + @Override + public int onErrorResponseBackoff( + @NonNull Request request, + @NonNull CoordinatorException error, + int retryCount, + RetryVerdict verdict) { + return 0; + } + + @Override + public void close() {} +} diff --git a/core/src/main/resources/reference.conf b/core/src/main/resources/reference.conf index 1ac37d14132..84c44e3240c 100644 --- a/core/src/main/resources/reference.conf +++ b/core/src/main/resources/reference.conf @@ -649,6 +649,27 @@ datastax-java-driver { class = DefaultRetryPolicy } + advanced.backoff-retry-policy { + # The class of the policy. If it is not qualified, the driver assumes that it resides in the + # package com.datastax.oss.driver.internal.core.retry. + # + # The driver provides two implementations out of the box: + # + # - DefaultRetryPolicy: the default policy, should almost always be the right choice. + # - ConsistencyDowngradingRetryPolicy: an alternative policy that weakens consistency guarantees + # as a trade-off to maximize the chance of success when retrying. Use with caution. + # + # Refer to the manual to understand how these policies work. + # + # You can also specify a custom class that implements RetryPolicy and has a public constructor + # with two arguments: the DriverContext and a String representing the profile name. + class = NoBackoffPolicy + + max-backoff-ms = 10000 + base-backoff-ms = 100 + jitter-ratio = 0.1 + } + # The policy that controls if the driver pre-emptively tries other nodes if a node takes too long # to respond. # diff --git a/core/src/test/java/com/datastax/oss/driver/api/core/config/TypedDriverOptionTest.java b/core/src/test/java/com/datastax/oss/driver/api/core/config/TypedDriverOptionTest.java index eee4000a459..9e5a1a24f51 100644 --- a/core/src/test/java/com/datastax/oss/driver/api/core/config/TypedDriverOptionTest.java +++ b/core/src/test/java/com/datastax/oss/driver/api/core/config/TypedDriverOptionTest.java @@ -44,6 +44,7 @@ public void should_have_equivalents_for_all_builtin_untyped_options() { ImmutableSet.of( DefaultDriverOption.LOAD_BALANCING_POLICY, DefaultDriverOption.RETRY_POLICY, + DefaultDriverOption.BACKOFF_RETRY_POLICY, DefaultDriverOption.SPECULATIVE_EXECUTION_POLICY); for (DriverOption option : diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/config/map/MapBasedDriverConfigLoaderTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/config/map/MapBasedDriverConfigLoaderTest.java index 93f6b274826..76966b9eefe 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/config/map/MapBasedDriverConfigLoaderTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/config/map/MapBasedDriverConfigLoaderTest.java @@ -103,6 +103,8 @@ private Optional get(DriverExecutionProfile config, TypedDriverOption value = config.getDuration(option); } else if (type.equals(GenericType.INTEGER)) { value = config.getInt(option); + } else if (type.equals(GenericType.DOUBLE)) { + value = config.getDouble(option); } else if (type.equals(GenericType.BOOLEAN)) { value = config.getBoolean(option); } else if (type.equals(GenericType.LONG)) { diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/cql/RequestHandlerTestHarness.java b/core/src/test/java/com/datastax/oss/driver/internal/core/cql/RequestHandlerTestHarness.java index 3ec30286755..324fd396d66 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/cql/RequestHandlerTestHarness.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/cql/RequestHandlerTestHarness.java @@ -56,6 +56,7 @@ import com.datastax.oss.driver.internal.core.metadata.MetadataManager; import com.datastax.oss.driver.internal.core.metrics.SessionMetricUpdater; import com.datastax.oss.driver.internal.core.pool.ChannelPool; +import com.datastax.oss.driver.internal.core.retry.NoBackoffPolicy; import com.datastax.oss.driver.internal.core.servererrors.DefaultWriteTypeRegistry; import com.datastax.oss.driver.internal.core.session.DefaultSession; import com.datastax.oss.driver.internal.core.session.throttling.PassThroughRequestThrottler; @@ -135,6 +136,7 @@ protected RequestHandlerTestHarness(Builder builder) { when(context.getLoadBalancingPolicyWrapper()).thenReturn(loadBalancingPolicyWrapper); when(context.getRetryPolicy(anyString())).thenReturn(retryPolicy); + when(context.getBackoffRetryPolicy(anyString())).thenReturn(new NoBackoffPolicy()); // Disable speculative executions by default when(speculativeExecutionPolicy.nextExecution( diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/core/config/MapBasedConfigLoaderIT.java b/integration-tests/src/test/java/com/datastax/oss/driver/core/config/MapBasedConfigLoaderIT.java index b8a6accce69..27aa1617e15 100644 --- a/integration-tests/src/test/java/com/datastax/oss/driver/core/config/MapBasedConfigLoaderIT.java +++ b/integration-tests/src/test/java/com/datastax/oss/driver/core/config/MapBasedConfigLoaderIT.java @@ -43,6 +43,7 @@ import com.datastax.oss.driver.api.testinfra.simulacron.SimulacronRule; import com.datastax.oss.driver.categories.ParallelizableTests; import com.datastax.oss.driver.internal.core.config.ConfigChangeEvent; +import com.datastax.oss.driver.internal.core.retry.NoBackoffPolicy; import com.datastax.oss.simulacron.common.cluster.ClusterSpec; import edu.umd.cs.findbugs.annotations.NonNull; import java.util.List; @@ -106,6 +107,10 @@ public void should_create_policies_per_profile() { String alternateProfile = "profile1"; optionsMap.put( alternateProfile, TypedDriverOption.RETRY_POLICY_CLASS, IgnoreAllPolicy.class.getName()); + optionsMap.put( + alternateProfile, + TypedDriverOption.BACKOFF_RETRY_POLICY_CLASS, + NoBackoffPolicy.class.getName()); try (CqlSession session = CqlSession.builder() diff --git a/manual/core/retries/README.md b/manual/core/retries/README.md index e92f8e214aa..5156b11d1ce 100644 --- a/manual/core/retries/README.md +++ b/manual/core/retries/README.md @@ -69,6 +69,31 @@ datastax-java-driver.advanced.retry-policy.class = ConsistencyDowngradingRetryPo You can also use your own policy by specifying for the above option the fully-qualified name of a class that implements [RetryPolicy]. +### Built-in backoff retry policies + +The driver ships with two retry policies: `NoBackoffPolicy` –– the default –– and +`ExponentialBackoffPolicy`. + +The default backoff retry policy makes driver have no delay between retries. + +`ExponentialBackoffPolicy` is provided for cases where the application needs to slow down +to avoid overwhelming cluster with retry requests. +The following needs to be added to the [configuration](../configuration/): + +``` +datastax-java-driver.advanced.backoff-retry-policy.class = ExponentialBackoffPolicy +``` + +You can also control following options for it: +``` +datastax-java-driver.advanced.backoff-retry-policy.max-backoff-ms = 10000 +datastax-java-driver.advanced.backoff-retry-policy.base-backoff-ms = 100 +datastax-java-driver.advanced.backoff-retry-policy.jitter-ratio = 0.1 +``` + +You can also use your own policy by specifying for the above option the fully-qualified name of a +class that implements [BackoffRetryPolicy]. + ### Behavior The behavior of both policies will be detailed in the sections below.