Subgraph Service: Integration tests

packages/horizon/tasks/test/fixtures/indexers.ts

@@ -38,7 +38,7 @@ const INDEXER_TWO_SECOND_ALLOCATION_PRIVATE_KEY = '0xab6cb9dbb3646a856e6cac2c0e2
 
 // Indexer three data
 const INDEXER_THREE_ADDRESS = '0x28a8746e75304c0780E011BEd21C72cD78cd535E' // Hardhat account #6
-
+const INDEXER_THREE_REWARDS_DESTINATION = '0xA3D22DDf431A8745888804F520D4eA51Cb43A458'
 // Subgraph deployment IDs
 const SUBGRAPH_DEPLOYMENT_ID_ONE = '0x02cd85012c1f075fd58fad178fd23ab841d3b5ddcf5cd3377c30118da97cb2a4'
 const SUBGRAPH_DEPLOYMENT_ID_TWO = '0x03ca89485a59894f1acfa34660c69024b6b90ce45171dece7662b0886bc375c7'
@@ -47,7 +47,7 @@ const SUBGRAPH_DEPLOYMENT_ID_THREE = '0x0472e8c46f728adb65a22187c6740532f82c2eba
 export const indexers: Indexer[] = [
   {
     address: INDEXER_ONE_ADDRESS,
-    stake: parseEther('1000000'),
+    stake: parseEther('1100000'),
     tokensToUnstake: parseEther('10000'),
     indexingRewardCut: 900000, // 90%
     queryFeeCut: 900000, // 90%
@@ -74,7 +74,7 @@ export const indexers: Indexer[] = [
   },
   {
     address: INDEXER_TWO_ADDRESS,
-    stake: parseEther('1000000'),
+    stake: parseEther('1100000'),
     tokensToUnstake: parseEther('1000000'),
     indexingRewardCut: 850000, // 85%
     queryFeeCut: 850000, // 85%
@@ -96,9 +96,10 @@ export const indexers: Indexer[] = [
   },
   {
     address: INDEXER_THREE_ADDRESS,
-    stake: parseEther('1000000'),
+    stake: parseEther('1100000'),
     indexingRewardCut: 800000, // 80%
     queryFeeCut: 800000, // 80%
+    rewardsDestination: INDEXER_THREE_REWARDS_DESTINATION,
     allocations: [],
   },
 ]

packages/subgraph-service/ignition/configs/migrate.integration-test.json5

@@ -0,0 +1,33 @@
+{
+  "$global": {
+    // Accounts for new deployment - derived from local network mnemonic
+    "governor": "0xFFcf8FDEE72ac11b5c542428B35EEF5769C409f0",
+    "arbitrator": "0x22d491Bde2303f2f43325b2108D26f1eAbA1e32b",
+    "pauseGuardian": "0xE11BA2b4D45Eaed5996Cd0823791E0C93114882d",
+
+    // Addresses for contracts deployed in the original Graph Protocol - Arbitrum Sepolia values
+    "controllerAddress": "0x9DB3ee191681f092607035d9BDA6e59FbEaCa695",
+    "curationProxyAddress": "0xDe761f075200E75485F4358978FB4d1dC8644FD5",
+    "curationImplementationAddress": "0xd90022aB67920212D0F902F5c427DE82732DE136",
+
+    // Must be set for step 2 of the deployment
+    "disputeManagerProxyAddress": "",
+    "disputeManagerProxyAdminAddress": "",
+    "subgraphServiceProxyAddress": "",
+    "subgraphServiceProxyAdminAddress": "",
+    "graphTallyCollectorAddress": ""
+  },
+  "DisputeManager": {
+    "disputePeriod": 2419200,
+    "disputeDeposit": "10000000000000000000000n",
+    "fishermanRewardCut": 500000,
+    "maxSlashingCut": 1000000,
+  },
+  "SubgraphService": {
+    "minimumProvisionTokens": "100000000000000000000000n",
+    "maximumDelegationRatio": 16,
+    "stakeToFeesRatio": 2,
+    "maxPOIStaleness": 2419200, // 28 days = 2419200 seconds
+    "curationCut": 100000,
+  }
+}

packages/subgraph-service/package.json

@@ -20,7 +20,8 @@
     "clean": "rm -rf build dist cache cache_forge typechain-types",
     "build": "hardhat compile",
     "test": "forge test",
-    "test:deployment": "SECURE_ACCOUNTS_DISABLE_PROVIDER=true hardhat test"
+    "test:deployment": "SECURE_ACCOUNTS_DISABLE_PROVIDER=true hardhat test",
+    "test:integration": "./scripts/test/integration"
   },
   "devDependencies": {
     "@defi-wonderland/natspec-smells": "^1.1.6",
@@ -47,6 +48,7 @@
     "eslint": "^8.56.0",
     "eslint-graph-config": "workspace:^0.0.1",
     "ethers": "^6.13.4",
+    "glob": "^11.0.1",
     "hardhat": "^2.22.18",
     "hardhat-contract-sizer": "^2.10.0",
     "hardhat-gas-reporter": "^1.0.8",

packages/subgraph-service/scripts/test/integration

@@ -0,0 +1,137 @@
+#!/bin/bash
+
+set -eo pipefail
+
+NON_INTERACTIVE=${NON_INTERACTIVE:-false}
+
+# Set environment variables for this script
+export SECURE_ACCOUNTS_DISABLE_PROVIDER=true
+export FORK_FROM_CHAIN_ID=${FORK_FROM_CHAIN_ID:-421614}
+
+# Function to cleanup resources
+cleanup() {
+    # Kill hardhat node only if we started it
+    if [ ! -z "$NODE_PID" ] && [ "$STARTED_NODE" = true ]; then
+        echo "Cleaning up node process..."
+        kill $NODE_PID 2>/dev/null || true
+    fi
+}
+
+# Set trap to call cleanup function on script exit (normal or error)
+trap cleanup EXIT
+
+# Check if any deployment folders exist
+SUBGRAPH_DEPLOYMENT_EXISTS=false
+HORIZON_DEPLOYMENT_EXISTS=false
+
+if [ -d "ignition/deployments/subgraph-service-localhost" ]; then
+    SUBGRAPH_DEPLOYMENT_EXISTS=true
+fi
+
+if [ -d "../horizon/ignition/deployments/horizon-localhost" ]; then
+    HORIZON_DEPLOYMENT_EXISTS=true
+fi
+
+# If any deployment exists, ask once for confirmation
+if [ "$SUBGRAPH_DEPLOYMENT_EXISTS" = true ] || [ "$HORIZON_DEPLOYMENT_EXISTS" = true ]; then
+    echo "The following deployment files already exist and must be removed for the tests to work properly:"
+    if [ "$SUBGRAPH_DEPLOYMENT_EXISTS" = true ]; then
+        echo "- Subgraph Service: ignition/deployments/subgraph-service-localhost"
+    fi
+    if [ "$HORIZON_DEPLOYMENT_EXISTS" = true ]; then
+        echo "- Horizon: ../horizon/ignition/deployments/horizon-localhost"
+    fi
+
+    read -p "Remove these deployment files? (y/n) [y]: " confirm
+    confirm=${confirm:-y}
+    if [[ $confirm == [yY] || $confirm == [yY][eE][sS] ]]; then
+        if [ "$SUBGRAPH_DEPLOYMENT_EXISTS" = true ]; then
+            echo "Removing Subgraph Service deployment files..."
+            rm -rf ignition/deployments/subgraph-service-localhost
+        fi
+        if [ "$HORIZON_DEPLOYMENT_EXISTS" = true ]; then
+            echo "Removing Horizon deployment files..."
+            rm -rf ../horizon/ignition/deployments/horizon-localhost
+        fi
+    else
+        echo "Cannot continue with existing deployment files. Exiting."
+        exit 1
+    fi
+fi
+
+# Check required env variables
+BLOCKCHAIN_RPC=${BLOCKCHAIN_RPC:-$(npx hardhat vars get ARBITRUM_SEPOLIA_RPC)}
+if [ -z "$BLOCKCHAIN_RPC" ]; then
+    echo "BLOCKCHAIN_RPC environment variable is required"
+    exit 1
+fi
+
+echo "Starting integration tests..."
+
+# Check if hardhat node is already running on port 8545
+STARTED_NODE=false
+if lsof -i:8545 > /dev/null 2>&1; then
+    echo "Hardhat node already running on port 8545, using existing node"
+    # Get the PID of the process using port 8545
+    NODE_PID=$(lsof -t -i:8545)
+else
+    # Start local hardhat node forked from Arbitrum Sepolia
+    echo "Starting local hardhat node..."
+    npx hardhat node --fork $BLOCKCHAIN_RPC > node.log 2>&1 &
+    NODE_PID=$!
+    STARTED_NODE=true
+
+    # Wait for node to start
+    sleep 10
+fi
+
+# Setup subgraph service address book
+jq '{"31337": ."'"$FORK_FROM_CHAIN_ID"'"}' addresses.json > addresses-localhost.json
+
+# Run Horizon pre-upgrade steps
+cd ../horizon
+
+# Setup pre horizon migration state needed for the e2e tests
+npx hardhat test:seed --network localhost
+
+# Transfer ownership of protocol to hardhat signer 1
+npx hardhat test:transfer-ownership --network localhost
+
+# Run Horizon steps 1 deployment
+npx hardhat deploy:migrate --network localhost --horizon-config e2e-test --step 1 --account-index 0 --patch-config
+
+# Run Subgraph Service steps 1 deployment
+cd ../subgraph-service
+npx hardhat deploy:migrate --network localhost --step 1 --subgraph-service-config integration-test --patch-config --account-index 0 --hide-banner
+
+# Run Horizon deployment steps 2 and 3
+cd ../horizon
+npx hardhat deploy:migrate --network localhost --horizon-config e2e-test --step 2 --patch-config --account-index 1 --hide-banner
+npx hardhat deploy:migrate --network localhost --horizon-config e2e-test --step 3 --patch-config --account-index 0 --hide-banner
+
+# Run Subgraph Service deployment step 2
+cd ../subgraph-service
+npx hardhat deploy:migrate --network localhost --step 2 --subgraph-service-config integration-test --patch-config --account-index 0 --hide-banner
+
+# Run Horizon deployment steps 4
+cd ../horizon
+npx hardhat deploy:migrate --network localhost --horizon-config e2e-test --step 4 --patch-config --account-index 1 --hide-banner
+
+# Run Subgraph Service seed steps
+cd ../subgraph-service
+npx hardhat test:seed --network localhost
+
+# Run integration tests - During transition period
+npx hardhat test:integration --phase during-transition-period --network localhost
+
+# Clear thawing period
+cd ../horizon
+npx hardhat transition:clear-thawing --network localhost --governor-index 1
+
+# Run integration tests - After transition period
+cd ../subgraph-service
+npx hardhat test:integration --phase after-transition-period --network localhost
+
+echo ""
+echo "🎉 ✨ 🚀 ✅ E2E tests completed successfully! 🎉 ✨ 🚀 ✅"
+echo ""

packages/subgraph-service/tasks/deploy.ts

@@ -93,14 +93,19 @@ task('deploy:protocol', 'Deploy a new version of the Graph Protocol Horizon cont
 task('deploy:migrate', 'Deploy the Subgraph Service on an existing Horizon deployment')
   .addOptionalParam('step', 'Migration step to run (1, 2)', undefined, types.int)
   .addOptionalParam('subgraphServiceConfig', 'Name of the Subgraph Service configuration file to use. Format is "migrate.<name>.json5", file must be in the "ignition/configs/" directory. Defaults to network name.', undefined, types.string)
+  .addOptionalParam('accountIndex', 'Derivation path index for the account to use', 0, types.int)
   .addFlag('patchConfig', 'Patch configuration file using address book values - does not save changes')
+  .addFlag('hideBanner', 'Hide the banner display')
   .setAction(async (args, hre: HardhatRuntimeEnvironment) => {
     // Task parameters
     const step: number = args.step ?? 0
     const patchConfig: boolean = args.patchConfig ?? false
 
     const graph = hre.graph()
-    printHorizonBanner()
+
+    if (!args.hideBanner) {
+      printHorizonBanner()
+    }
 
     // Migration step to run
     console.log('\n========== 🏗️ Migration steps ==========')
@@ -119,7 +124,7 @@ task('deploy:migrate', 'Deploy the Subgraph Service on an existing Horizon deplo
 
     // Display the deployer -- this also triggers the secure accounts prompt if being used
     console.log('\n========== 🔑 Deployer account ==========')
-    const deployer = await graph.accounts.getDeployer(args.deployerIndex)
+    const deployer = await graph.accounts.getDeployer(args.accountIndex)
     console.log('Using deployer account:', deployer.address)
     const balance = await hre.ethers.provider.getBalance(deployer.address)
     console.log('Deployer balance:', hre.ethers.formatEther(balance), 'ETH')

packages/subgraph-service/tasks/test/fixtures/indexers.ts

@@ -0,0 +1,101 @@
+import { indexers as horizonIndexers } from '../../../../horizon/tasks/test/fixtures/indexers'
+import { parseEther } from 'ethers'
+
+// Allocation interface
+export interface Allocation {
+  allocationID: string
+  subgraphDeploymentID: string
+  allocationPrivateKey: string
+  tokens: bigint
+}
+
+// Indexer interface
+export interface Indexer {
+  address: string
+  url: string
+  geoHash: string
+  rewardsDestination?: string
+  provisionTokens: bigint
+  legacyAllocations: Allocation[]
+  allocations: Allocation[]
+}
+
+// Subgraph deployment IDs
+const SUBGRAPH_DEPLOYMENT_ID_ONE = '0x02cd85012c1f075fd58fad178fd23ab841d3b5ddcf5cd3377c30118da97cb2a4'
+const SUBGRAPH_DEPLOYMENT_ID_TWO = '0x03ca89485a59894f1acfa34660c69024b6b90ce45171dece7662b0886bc375c7'
+const SUBGRAPH_DEPLOYMENT_ID_THREE = '0x0472e8c46f728adb65a22187c6740532f82c2ebadaeabbbe59a2bb4a1bdde197'
+
+// Indexer one allocations
+const INDEXER_ONE_FIRST_ALLOCATION_ID = '0x097DC23d51A7800f9B1EA37919A5b223C0224eC2'
+const INDEXER_ONE_FIRST_ALLOCATION_PRIVATE_KEY = '0xec5739112bc20845cdd80b2612dfb0a75599ea6fbdd8916a1e7d5be98118c315'
+const INDEXER_ONE_SECOND_ALLOCATION_ID = '0x897E7056FB86372CB676EBAE73a360c22b21D4aD'
+const INDEXER_ONE_SECOND_ALLOCATION_PRIVATE_KEY = '0x298519bdc6a73f0d64c96e1f7c39aba3f825886a37e0349294ce7c407bd88370'
+const INDEXER_ONE_THIRD_ALLOCATION_ID = '0x02C64e54100b3Cb324ac50d9b3823402e6aA5297'
+const INDEXER_ONE_THIRD_ALLOCATION_PRIVATE_KEY = '0xb8ca0ab93098c2c478c5657da7a7bb89522bb1e3198f8b469de252dfee5469a3'
+
+// Indexer two allocations
+const INDEXER_TWO_FIRST_ALLOCATION_ID = '0xB609bBf1D5Ae3C246dA1F9a5EA327DBa66BbcB05'
+const INDEXER_TWO_FIRST_ALLOCATION_PRIVATE_KEY = '0x21dce628700b82e2d9045d756e4d0ba736f652a170655398a15fadae10b0e846'
+const INDEXER_TWO_SECOND_ALLOCATION_ID = '0x1bF6afCF9542983432B2fab15717c2537A3d3F2A'
+const INDEXER_TWO_SECOND_ALLOCATION_PRIVATE_KEY = '0x4bf454f7d52fff97701c1ea5d1e6184c81543780ca61b82cce155a5a3e35a134'
+
+// Allocations map
+const allocations = new Map<string, Allocation[]>([
+  [
+    horizonIndexers[0].address,
+    [
+      {
+        allocationID: INDEXER_ONE_FIRST_ALLOCATION_ID,
+        subgraphDeploymentID: SUBGRAPH_DEPLOYMENT_ID_ONE,
+        allocationPrivateKey: INDEXER_ONE_FIRST_ALLOCATION_PRIVATE_KEY,
+        tokens: parseEther('10000'),
+      },
+      {
+        allocationID: INDEXER_ONE_SECOND_ALLOCATION_ID,
+        subgraphDeploymentID: SUBGRAPH_DEPLOYMENT_ID_TWO,
+        allocationPrivateKey: INDEXER_ONE_SECOND_ALLOCATION_PRIVATE_KEY,
+        tokens: parseEther('8000'),
+      },
+      {
+        allocationID: INDEXER_ONE_THIRD_ALLOCATION_ID,
+        subgraphDeploymentID: SUBGRAPH_DEPLOYMENT_ID_THREE,
+        allocationPrivateKey: INDEXER_ONE_THIRD_ALLOCATION_PRIVATE_KEY,
+        tokens: parseEther('5000'),
+      },
+    ],
+  ],
+  [
+    horizonIndexers[2].address,
+    [
+      {
+        allocationID: INDEXER_TWO_FIRST_ALLOCATION_ID,
+        subgraphDeploymentID: SUBGRAPH_DEPLOYMENT_ID_ONE,
+        allocationPrivateKey: INDEXER_TWO_FIRST_ALLOCATION_PRIVATE_KEY,
+        tokens: parseEther('10000'),
+      },
+      {
+        allocationID: INDEXER_TWO_SECOND_ALLOCATION_ID,
+        subgraphDeploymentID: SUBGRAPH_DEPLOYMENT_ID_TWO,
+        allocationPrivateKey: INDEXER_TWO_SECOND_ALLOCATION_PRIVATE_KEY,
+        tokens: parseEther('8000'),
+      },
+    ],
+  ],
+])
+
+// Indexers data
+export const indexers: Indexer[] = horizonIndexers
+  .filter(indexer => !indexer.tokensToUnstake || indexer.tokensToUnstake <= parseEther('100000'))
+  .map((indexer) => {
+    // Move existing allocations to legacyAllocations
+    const legacyAllocations = indexer.allocations
+
+    return {
+      ...indexer,
+      url: 'url',
+      geoHash: 'geohash',
+      provisionTokens: parseEther('1000000'),
+      legacyAllocations,
+      allocations: allocations.get(indexer.address) || [],
+    }
+  })

packages/subgraph-service/tasks/test/integration.ts

@@ -0,0 +1,33 @@
+import { glob } from 'glob'
+import { task } from 'hardhat/config'
+import { TASK_TEST } from 'hardhat/builtin-tasks/task-names'
+
+import { printBanner } from '@graphprotocol/toolshed/utils'
+
+task('test:integration', 'Runs all integration tests')
+  .addParam(
+    'phase',
+    'Test phase to run: "during-transition-period", "after-transition-period", "after-delegation-slashing-enabled"',
+  )
+  .setAction(async (taskArgs, hre) => {
+    // Get test files for each phase
+    const duringTransitionPeriodFiles = await glob('test/integration/during-transition-period/**/*.{js,ts}')
+    const afterTransitionPeriodFiles = await glob('test/integration/after-transition-period/**/*.{js,ts}')
+
+    // Display banner for the current test phase
+    printBanner(taskArgs.phase, 'INTEGRATION TESTS: ')
+
+    // Run tests for the current phase
+    switch (taskArgs.phase) {
+      case 'during-transition-period':
+        await hre.run(TASK_TEST, { testFiles: duringTransitionPeriodFiles })
+        break
+      case 'after-transition-period':
+        await hre.run(TASK_TEST, { testFiles: afterTransitionPeriodFiles })
+        break
+      default:
+        throw new Error(
+          'Invalid phase. Must be "during-transition-period", "after-transition-period", "after-delegation-slashing-enabled", or "all"',
+        )
+    }
+  })