Improve reliability of deployment cancellation and refunds

Author: jmrossy

src/features/deployerWallet/refund.ts

@@ -4,7 +4,7 @@ import {
   TypedTransaction,
   TypedTransactionReceipt,
 } from '@hyperlane-xyz/sdk';
-import { assert, ProtocolType } from '@hyperlane-xyz/utils';
+import { assert, ProtocolType, retryAsync } from '@hyperlane-xyz/utils';
 import { AccountInfo, getAccountAddressForChain, useAccounts } from '@hyperlane-xyz/widgets';
 import { useMutation } from '@tanstack/react-query';
 import BigNumber from 'bignumber.js';
@@ -91,7 +91,11 @@ async function transferBalances(
 
           const tx = await getTransferTx(recipient, adjustedAmount, token, multiProvider);
 
-          const txReceipt = await sendTxFromWallet(deployer, tx, chainName, multiProvider);
+          const txReceipt = await retryAsync(
+            () => sendTxFromWallet(deployer, tx, chainName, multiProvider),
+            3,
+            2000,
+          );
           logger.debug('Transfer tx confirmed on chain', chainName, txReceipt.receipt);
           return txReceipt;
         } catch (error) {

src/features/deployerWallet/transactions.ts

@@ -1,14 +1,17 @@
 import {
   getChainIdNumber,
   MultiProtocolProvider,
+  MultiProvider,
   ProviderType,
   Token,
+  TOKEN_STANDARD_TO_PROVIDER_TYPE,
   TypedTransaction,
   TypedTransactionReceipt,
   WarpTxCategory,
   WarpTypedTransaction,
 } from '@hyperlane-xyz/sdk';
 import { ProtocolType } from '@hyperlane-xyz/utils';
+import { logger } from '../../utils/logger';
 import { TypedWallet } from './types';
 
 // TODO edit Widgets lib to default to TypedTransaction instead of WarpTypedTransaction?
@@ -31,9 +34,7 @@ export async function getTransferTx(
   }
 
   return {
-    // TODO use TOKEN_STANDARD_TO_PROVIDER_TYPE here when it's exported from the SDK
-    // type: TOKEN_STANDARD_TO_PROVIDER_TYPE[token.standard],
-    type: ProviderType.EthersV5,
+    type: TOKEN_STANDARD_TO_PROVIDER_TYPE[token.standard],
     transaction: txParams,
     category: WarpTxCategory.Transfer,
   } as WarpTypedTransaction;
@@ -60,3 +61,35 @@ export async function sendTxFromWallet(
     throw new Error(`Unsupported provider type for sending txs: ${typedWallet.type}`);
   }
 }
+
+// TODO multi-protocol support
+export async function hasPendingTx(
+  typedWallet: TypedWallet,
+  chainName: ChainName,
+  multiProvider: MultiProvider,
+) {
+  const { type, address } = typedWallet;
+  let hasPending = false;
+  if (type === ProviderType.EthersV5) {
+    const provider = multiProvider.getProvider(chainName);
+    // Based on https://github.com/ethers-io/ethers.js/discussions/3470
+    // This is an imperfect way to check for pending txs but it's probably
+    // the best approximation we can do. The eth_pendingTransactions may also
+    // help but it's not supported by all providers.
+    const [currentNonce, pendingNonce] = await Promise.all([
+      provider.getTransactionCount(address),
+      provider.getTransactionCount(address, 'pending'),
+    ]);
+    hasPending = currentNonce !== pendingNonce;
+  } else {
+    throw new Error(`Unsupported provider type for sending txs: ${typedWallet.type}`);
+  }
+
+  if (hasPending) {
+    logger.debug(`Pending tx found for for ${address} on ${chainName}`);
+    return true;
+  } else {
+    logger.debug(`No pending tx found for for ${address} on ${chainName}`);
+    return false;
+  }
+}

src/features/deployment/warp/WarpDeploymentDeploy.tsx

@@ -1,5 +1,5 @@
 import { MultiProtocolProvider, WarpCoreConfig } from '@hyperlane-xyz/sdk';
-import { errorToString, sleep } from '@hyperlane-xyz/utils';
+import { errorToString } from '@hyperlane-xyz/utils';
 import { Button, SpinnerIcon, useModal } from '@hyperlane-xyz/widgets';
 import clsx from 'clsx';
 import { useCallback, useMemo, useState } from 'react';
@@ -26,8 +26,6 @@ import { DeploymentStatus, DeploymentType, WarpDeploymentConfig } from '../types
 import { useWarpDeployment } from './deploy';
 import { WarpDeploymentProgressIndicator } from './WarpDeploymentProgressIndicator';
 
-const CANCEL_SLEEP_DELAY = 10_000;
-
 enum DeployStep {
   FundDeployer,
   ExecuteDeploy,
@@ -49,7 +47,7 @@ export function WarpDeploymentDeploy() {
   const { refundAsync } = useRefundDeployerAccounts();
 
   const onFailure = useCallback(
-    (error: Error) => {
+    async (error: Error) => {
       const errMsg = errorToString(error, 5000);
       failDeployment(currentIndex, errMsg);
       refundAsync().finally(() => setPage(CardPage.WarpFailure));
@@ -71,7 +69,6 @@ export function WarpDeploymentDeploy() {
   const {
     deploy,
     isIdle: isDeploymentIdle,
-    isPending: isDeploymentPending,
     cancel: cancelDeployment,
   } = useWarpDeployment(deploymentConfig, onDeploymentSuccess, onFailure);
 
@@ -81,25 +78,11 @@ export function WarpDeploymentDeploy() {
   }, [isDeploymentIdle, deploy, setStep]);
 
   const onCancel = useCallback(async () => {
-    if (isDeploymentPending) cancelDeployment();
     updateDeploymentStatus(currentIndex, DeploymentStatus.Cancelled);
     setStep(DeployStep.CancelDeploy);
-
-    // A delay is required to ensure that pending txs have a chance to settle
-    // before the refunder attempts to send new ones
-    // This is imperfect but users can always run it again from DeployerRecoveryModal
-    // TODO consider replacing with logic to check for pending txs from any deployer wallets
-    await sleep(CANCEL_SLEEP_DELAY);
-
+    await cancelDeployment();
     refundAsync().finally(() => setPage(CardPage.WarpForm));
-  }, [
-    isDeploymentPending,
-    currentIndex,
-    cancelDeployment,
-    refundAsync,
-    setPage,
-    updateDeploymentStatus,
-  ]);
+  }, [currentIndex, cancelDeployment, refundAsync, setPage, updateDeploymentStatus]);
 
   if (!deploymentConfig) throw new Error('Deployment config is required');
 
@@ -189,7 +172,17 @@ function FundDeployerAccounts({
         className="px-4 py-1.5 text-md"
         onClick={onClickFund}
         disabled={isTxPending || isDeployerLoading}
-      >{`Fund on ${currentChainDisplay} (Chain ${currentChainIndex + 1} / ${numChains})`}</SolidButton>
+      >
+        {isTxPending ? (
+          <>
+            <span>Funding on {currentChainDisplay}</span>
+            <SpinnerIcon width={18} height={18} color={Color.gray['600']} className="ml-3" />
+          </>
+        ) : (
+          `Fund on ${currentChainDisplay} (Chain ${currentChainIndex + 1} / ${numChains})`
+        )}
+        {}
+      </SolidButton>
     </div>
   );
 }

src/features/deployment/warp/deploy.ts

@@ -30,8 +30,11 @@ import {
   isCollateralTokenConfig,
   isTokenMetadata,
 } from '@hyperlane-xyz/sdk';
-import { Address, ProtocolType, assert, objMap, promiseObjAll } from '@hyperlane-xyz/utils';
+import { Address, ProtocolType, assert, objMap, promiseObjAll, sleep } from '@hyperlane-xyz/utils';
 import { useCallback, useMemo, useState } from 'react';
+import { hasPendingTx } from '../../deployerWallet/transactions';
+
+const NUM_SECONDS_FOR_TX_WAIT = 10;
 
 export function useWarpDeployment(
   deploymentConfig?: WarpDeploymentConfig,
@@ -54,7 +57,8 @@ export function useWarpDeployment(
       return executeDeploy(multiProvider, wallets, deploymentConfig);
     },
     retry: false,
-    onError: (e: Error) => {
+    onError: async (e: Error) => {
+      await haltDeployment(multiProvider, wallets, deploymentConfig?.chains);
       if (!isCancelled) onFailure?.(e);
     },
     onSuccess: (r: WarpCoreConfig) => {
@@ -64,14 +68,11 @@ export function useWarpDeployment(
 
   useToastError(!isCancelled && error, 'Error deploying warp route.');
 
-  const cancel = useCallback(() => {
+  const cancel = useCallback(async () => {
     if (!isPending) return;
     setIsCancelled(true);
     logger.debug('Cancelling deployment');
-    // Clear signers from multiProvider to force a failure of the
-    // next tx from the deployers
-    multiProvider.setSharedSigner(null);
-
+    await haltDeployment(multiProvider, wallets, deploymentConfig?.chains);
     // multiProvider is intentionally excluded from the deps array
     // to ensure that this cancel callback remains bound to the
     // one used in the active deployment
@@ -354,3 +355,46 @@ function fullyConnectTokens(warpCoreConfig: WarpCoreConfig): void {
     }
   }
 }
+
+/**
+ * This attempts to halt active deployments by disabling the signers
+ * in the multiProvider. This works to prevent new txs initiated from
+ * the multiProvider but cannot stop txs that are 1) already in flight or
+ * 2) will be sent from a signer other than the one in the multiProvider (e.g
+ * an ethers Factory created a copy of the signer and uses that to send txs).
+ * After several hours of experiments, I suspect this is the best we can do
+ * without restructuring the SDK's deployers.
+ *
+ * In the event where this misses a tx or two, there's a chance the refund tx
+ * will fail. Assuming the retries fail as well, a user would need to manually
+ * initiate a refund later via the DeployerRecoveryModal.
+ */
+async function haltDeployment(
+  multiProvider: MultiProvider,
+  wallets: DeployerWallets,
+  chains: ChainName[] = [],
+) {
+  logger.debug('Clearing signers from multiProvider');
+  multiProvider.setSharedSigner(null);
+  logger.debug('Waiting for pending txs to settle');
+  await sleep(5000);
+  // Wait up to NUM_SECONDS_FOR_TX_WAIT for tx to settle
+  for (let i = 0; i < NUM_SECONDS_FOR_TX_WAIT; i++) {
+    const results = await Promise.all(
+      chains.map(async (chainName) => {
+        try {
+          const protocol = multiProvider.getProtocol(chainName);
+          const deployer = wallets[protocol];
+          if (!deployer) return false;
+          return hasPendingTx(deployer, chainName, multiProvider);
+        } catch (error) {
+          logger.error(`Error checking pending txs on ${chainName}`, error);
+          return true;
+        }
+      }),
+    );
+    if (results.some((r) => r)) sleep(1000);
+    else return;
+  }
+  logger.warn('Timed out waiting for pending txs to settle');
+}

src/features/logs/useSdkLogs.ts

@@ -36,7 +36,7 @@ export function useSdkLogWatcher() {
 
     return () => {
       // Replace global rootLogger with new one
-      configureRootLogger(LogFormat.JSON, LogLevelWithOff.Debug);
+      configureRootLogger(LogFormat.JSON, LogLevelWithOff.Info);
     };
   }, []);
 }