Integrate RPC failover into NetworkController

When configuring an Infura network, a list of failover endpoint URLs can
now be supplied. If Infura is perceived to be down (after making 15
attempts to make a request, all of which either throw an error, return a
non-200 response, or return a non-parseable response) then the request
will be forwarded automatically to the first failover URL (using
subsequent URLs as failovers for previous URLs). By default, all Infura
networks will use Quicknode as a failover.

Under the hood, the list of failover URLs are loaded into an
RpcServiceChain (added in a previous commit) and this is then passed to
the Infura middleware.

Author: mcmire

packages/assets-controllers/src/AssetsContractController.test.ts

@@ -13,6 +13,7 @@ import type {
   NetworkClientId,
   NetworkControllerActions,
   NetworkControllerEvents,
+  InfuraNetworkClientConfiguration,
 } from '@metamask/network-controller';
 import {
   NetworkController,
@@ -69,13 +70,14 @@ async function setupAssetContractControllers({
   useNetworkControllerProvider?: boolean;
   infuraProjectId?: string;
 } = {}) {
-  const networkClientConfiguration = {
+  const networkClientConfiguration: InfuraNetworkClientConfiguration = {
     type: NetworkClientType.Infura,
     network: NetworkType.mainnet,
+    failoverEndpointUrls: [],
     infuraProjectId,
     chainId: BUILT_IN_NETWORKS.mainnet.chainId,
     ticker: BUILT_IN_NETWORKS.mainnet.ticker,
-  } as const;
+  };
   let provider: Provider;
 
   const messenger = new Messenger<
@@ -1091,6 +1093,7 @@ describe('AssetsContractController', () => {
         ticker: BUILT_IN_NETWORKS.sepolia.ticker,
         type: NetworkClientType.Infura,
         network: 'sepolia',
+        failoverEndpointUrls: [],
         infuraProjectId: networkClientConfiguration.infuraProjectId,
       },
       mocks: [

packages/assets-controllers/src/AssetsContractControllerWithNetworkClientId.test.ts

@@ -757,6 +757,7 @@ describe('AssetsContractController with NetworkClientId', () => {
         ticker: BUILT_IN_NETWORKS.sepolia.ticker,
         type: NetworkClientType.Infura,
         network: 'sepolia',
+        failoverEndpointUrls: [],
         infuraProjectId: networkClientConfiguration.infuraProjectId,
       },
       mocks: [

packages/network-controller/CHANGELOG.md

@@ -17,11 +17,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
     - The request returns a non-200 response
 - Use exponential backoff / jitter when retrying requests to Infura and custom RPC endpoints ([#5290](https://github.com/MetaMask/core/pull/5290))
   - As requests are retried, the delay between retries will increase exponentially (using random variance to prevent bursts)
+- Add support for automatic failover when Infura is down ([#5630](https://github.com/MetaMask/core/pull/5630))
+  - An Infura RPC endpoint can now be configured with a list of failover URLs (see "Changed" for details).
+  - If, after many attempts, an Infura network is perceived to be down, the list of failover URLs will be tried in turn.
+  - All Infura RPC endpoints are configured to use Quicknode by default.
 
 ### Changed
 
 - **BREAKING:** `NetworkController` constructor now takes two required options, `fetch` and `btoa` ([#5290](https://github.com/MetaMask/core/pull/5290))
   - These are passed along to functions that create the JSON-RPC middleware
+- **BREAKING:** Add required property `failoverEndpointUrls` to `InfuraRpcEndpoint` ([#5630](https://github.com/MetaMask/core/pull/5630))
+- **BREAKING:** Add required property `failoverEndpointUrls` to `InfuraNetworkClientConfiguration` ([#5630](https://github.com/MetaMask/core/pull/5630))
 - Synchronize retry logic and error handling behavior between Infura and custom RPC endpoints ([#5290](https://github.com/MetaMask/core/pull/5290))
   - A request to a custom endpoint that returns a 418 response will no longer return a JSON-RPC response with the error "Request is being rate limited"
   - A request to a custom endpoint that returns a 429 response now returns a JSON-RPC response with the error "Request is being rate limited"

packages/network-controller/src/NetworkController.ts

@@ -28,7 +28,11 @@ import { createSelector } from 'reselect';
 import * as URI from 'uri-js';
 import { v4 as uuidV4 } from 'uuid';
 
-import { INFURA_BLOCKED_KEY, NetworkStatus } from './constants';
+import {
+  DEFAULT_INFURA_FAILOVER_URL,
+  INFURA_BLOCKED_KEY,
+  NetworkStatus,
+} from './constants';
 import type {
   AutoManagedNetworkClient,
   ProxyWithAccessibleTarget,
@@ -95,6 +99,10 @@ export enum RpcEndpointType {
  * separate type.
  */
 export type InfuraRpcEndpoint = {
+  /**
+   * Alternate RPC endpoints to use when this endpoint is down.
+   */
+  failoverUrls: string[];
   /**
    * The optional user-facing nickname of the endpoint.
    */
@@ -569,6 +577,7 @@ function getDefaultNetworkConfigurationsByChainId(): Record<
       nativeCurrency: NetworksTicker[infuraNetworkType],
       rpcEndpoints: [
         {
+          failoverUrls: [DEFAULT_INFURA_FAILOVER_URL],
           networkClientId: infuraNetworkType,
           type: RpcEndpointType.Infura,
           url: rpcEndpointUrl,
@@ -2446,6 +2455,7 @@ export class NetworkController extends BaseController<
             type: NetworkClientType.Infura,
             chainId: networkFields.chainId,
             network: addedRpcEndpoint.networkClientId,
+            failoverEndpointUrls: addedRpcEndpoint.failoverUrls,
             infuraProjectId: this.#infuraProjectId,
             ticker: networkFields.nativeCurrency,
           },
@@ -2617,6 +2627,7 @@ export class NetworkController extends BaseController<
               networkClientConfiguration: {
                 type: NetworkClientType.Infura,
                 network: infuraNetworkName,
+                failoverEndpointUrls: rpcEndpoint.failoverUrls,
                 infuraProjectId: this.#infuraProjectId,
                 chainId: networkConfiguration.chainId,
                 ticker: networkConfiguration.nativeCurrency,

packages/network-controller/src/constants.ts

@@ -26,3 +26,8 @@ export enum NetworkStatus {
 }
 
 export const INFURA_BLOCKED_KEY = 'countryBlocked';
+
+/**
+ * The default endpoint URL to hit when Infura is down.
+ */
+export const DEFAULT_INFURA_FAILOVER_URL = 'https://quicknode.com';

packages/network-controller/src/create-auto-managed-network-client.test.ts

@@ -19,14 +19,15 @@ describe('createAutoManagedNetworkClient', () => {
       rpcUrl: 'https://test.chain',
       chainId: '0x1337',
       ticker: 'ETH',
-    } as const,
+    },
     {
       type: NetworkClientType.Infura,
       network: NetworkType.mainnet,
       chainId: BUILT_IN_NETWORKS[NetworkType.mainnet].chainId,
       infuraProjectId: 'some-infura-project-id',
       ticker: BUILT_IN_NETWORKS[NetworkType.mainnet].ticker,
-    } as const,
+      failoverEndpointUrls: [],
+    },
   ];
   for (const networkClientConfiguration of networkClientConfigurations) {
     describe(`given configuration for a ${networkClientConfiguration.type} network client`, () => {

packages/network-controller/src/create-network-client.ts

@@ -25,7 +25,7 @@ import {
 import type { JsonRpcMiddleware } from '@metamask/json-rpc-engine';
 import type { Hex, Json, JsonRpcParams } from '@metamask/utils';
 
-import { RpcService } from './rpc-service/rpc-service';
+import { RpcServiceChain } from './rpc-service/rpc-service-chain';
 import type {
   BlockTracker,
   NetworkClientConfiguration,
@@ -66,18 +66,21 @@ export function createNetworkClient({
   fetch: typeof fetch;
   btoa: typeof btoa;
 }): NetworkClient {
-  const rpcService =
+  const primaryEndpointUrl =
     configuration.type === NetworkClientType.Infura
-      ? new RpcService({
-          fetch: givenFetch,
-          btoa: givenBtoa,
-          endpointUrl: `https://${configuration.network}.infura.io/v3/${configuration.infuraProjectId}`,
-        })
-      : new RpcService({
-          fetch: givenFetch,
-          btoa: givenBtoa,
-          endpointUrl: configuration.rpcUrl,
-        });
+      ? `https://${configuration.network}.infura.io/v3/${configuration.infuraProjectId}`
+      : configuration.rpcUrl;
+  const failoverEndpointUrls =
+    configuration.type === NetworkClientType.Infura
+      ? configuration.failoverEndpointUrls
+      : [];
+  const rpcService = new RpcServiceChain({
+    fetch: givenFetch,
+    btoa: givenBtoa,
+    serviceConfigurations: [primaryEndpointUrl, ...failoverEndpointUrls].map(
+      (endpointUrl) => ({ endpointUrl }),
+    ),
+  });
 
   const rpcApiMiddleware =
     configuration.type === NetworkClientType.Infura

packages/network-controller/src/types.ts

@@ -18,27 +18,40 @@ export enum NetworkClientType {
 }
 
 /**
- * A configuration object that can be used to create a client for a custom
- * network.
+ * A configuration object that can be used to create a client for a network.
  */
-export type CustomNetworkClientConfiguration = {
+type CommonNetworkClientConfiguration = {
+  /**
+   * The chain ID of the network.
+   */
   chainId: Hex;
-  rpcUrl: string;
+  /**
+   * The name of the currency to use for the chain.
+   */
   ticker: string;
-  type: NetworkClientType.Custom;
 };
 
+/**
+ * A configuration object that can be used to create a client for a custom
+ * network.
+ */
+export type CustomNetworkClientConfiguration =
+  CommonNetworkClientConfiguration & {
+    rpcUrl: string;
+    type: NetworkClientType.Custom;
+  };
+
 /**
  * A configuration object that can be used to create a client for an Infura
  * network.
  */
-export type InfuraNetworkClientConfiguration = {
-  chainId: Hex;
-  network: InfuraNetworkType;
-  infuraProjectId: string;
-  ticker: string;
-  type: NetworkClientType.Infura;
-};
+export type InfuraNetworkClientConfiguration =
+  CommonNetworkClientConfiguration & {
+    network: InfuraNetworkType;
+    infuraProjectId: string;
+    failoverEndpointUrls: string[];
+    type: NetworkClientType.Infura;
+  };
 
 /**
  * A configuration object that can be used to create a client for a network.