From 6e9f7ac38976667fc8651f069573f751ef89161b Mon Sep 17 00:00:00 2001 From: brendan-coughlan Date: Wed, 12 Feb 2025 13:39:42 -0800 Subject: [PATCH] feat(restore-snapshot): add manifest URL flags --- cmd/restoreSnapshot.go | 4 + cmd/root.go | 5 +- .../content/docs/running/getting-started.mdx | 3 +- docs/src/content/docs/running/snapshots.md | 16 +- internal/config/config.go | 3 + pkg/snapshot/snapshot.go | 197 ++++++++---- pkg/snapshot/snapshot_test.go | 301 +++++++++++++++++- 7 files changed, 449 insertions(+), 80 deletions(-) diff --git a/cmd/restoreSnapshot.go b/cmd/restoreSnapshot.go index 4ade62bd..187603ef 100644 --- a/cmd/restoreSnapshot.go +++ b/cmd/restoreSnapshot.go @@ -5,6 +5,7 @@ import ( "github.com/Layr-Labs/sidecar/internal/config" "github.com/Layr-Labs/sidecar/internal/logger" + "github.com/Layr-Labs/sidecar/internal/version" "github.com/Layr-Labs/sidecar/pkg/snapshot" "github.com/spf13/cobra" "github.com/spf13/pflag" @@ -29,12 +30,15 @@ Follow the snapshot docs if you need to convert the snapshot to a different sche svc, err := snapshot.NewSnapshotService(&snapshot.SnapshotConfig{ Input: cfg.SnapshotConfig.Input, VerifyInput: cfg.SnapshotConfig.VerifyInput, + ManifestURL: cfg.SnapshotConfig.ManifestURL, Host: cfg.DatabaseConfig.Host, Port: cfg.DatabaseConfig.Port, User: cfg.DatabaseConfig.User, Password: cfg.DatabaseConfig.Password, DbName: cfg.DatabaseConfig.DbName, SchemaName: cfg.DatabaseConfig.SchemaName, + Version: version.GetVersion(), + Chain: cfg.Chain.String(), }, l) if err != nil { return err diff --git a/cmd/root.go b/cmd/root.go index 1c733b0f..0acc7edc 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -65,8 +65,9 @@ func init() { // bind any subcommand flags createSnapshotCmd.PersistentFlags().String(config.SnapshotOutputFile, "", "Path to save the snapshot file to (required), also creates a hash file") - restoreSnapshotCmd.PersistentFlags().String(config.SnapshotInput, "", "Path to the snapshot file either a URL or a local file (required)") - restoreSnapshotCmd.PersistentFlags().Bool(config.SnapshotVerifyInput, true, "Boolean to verify the input file against its .sha256sum file, if input is a url then it downloads the file, (default is true)") + restoreSnapshotCmd.PersistentFlags().String(config.SnapshotInput, "", "Path to the snapshot file either a URL or a local file (optional), **If specified, this file is used instead of the manifest desired snapshot** ") + restoreSnapshotCmd.PersistentFlags().Bool(config.SnapshotVerifyInput, true, "Boolean to verify the input file against its .sha256sum file, if input is a url then it downloads the file") + restoreSnapshotCmd.PersistentFlags().String(config.SnapshotManifestURL, "https://sidecar.eigenlayer.xyz/snapshots/snapshots_manifest_v1.0.0.json", "URL to a manifest json. Gets the latest snapshot matching the current runtime configurations of version, chain, and schema") rpcCmd.PersistentFlags().String(config.SidecarPrimaryUrl, "", `RPC url of the "primary" Sidecar instance in an HA environment`) diff --git a/docs/src/content/docs/running/getting-started.mdx b/docs/src/content/docs/running/getting-started.mdx index d4c1c56f..850b12b0 100644 --- a/docs/src/content/docs/running/getting-started.mdx +++ b/docs/src/content/docs/running/getting-started.mdx @@ -46,8 +46,7 @@ import { Steps } from '@astrojs/starlight/components'; ```bash /usr/local/bin/sidecar restore-snapshot \ - --snapshot.input=https://eigenlayer-sidecar.s3.us-east-1.amazonaws.com/snapshots/testnet-holesky/sidecar-testnet-holesky_v3.0.0-rc.1_public_20250122.dump \ - --snapshot.verify-input=false \ + --chain="mainnet" \ --database.host= \ --database.user= \ --database.password= \ diff --git a/docs/src/content/docs/running/snapshots.md b/docs/src/content/docs/running/snapshots.md index bfc5d79b..1641a40e 100644 --- a/docs/src/content/docs/running/snapshots.md +++ b/docs/src/content/docs/running/snapshots.md @@ -6,15 +6,12 @@ description: How to use a snapshot to start or restore your Sidecar Snapshots are a quicker way to sync to tip and get started. ## Snapshot Sources +You can get the snapshots here: [https://sidecar.eigenlayer.xyz/snapshots/index.html](https://sidecar.eigenlayer.xyz/snapshots/index.html) -* Mainnet Ethereum (not yet available) -* Testnet Holesky ([2025-01-22](https://eigenlayer-sidecar.s3.us-east-1.amazonaws.com/snapshots/testnet-holesky/sidecar-testnet-holesky_v3.0.0-rc.1_public_20250122.dump)) - -## Example boot from testnet snapshot +## Example boot from mainnet snapshot ```bash ./bin/sidecar restore-snapshot \ - --snapshot.input=https://eigenlayer-sidecar.s3.us-east-1.amazonaws.com/snapshots/testnet-holesky/sidecar-testnet-holesky_v3.0.0-rc.1_public_20250122.dump \ - --snapshot.verify-input=false \ + --snapshot.chain = mainnet \ --database.host=localhost \ --database.user=sidecar \ --database.password=... \ @@ -36,9 +33,10 @@ Usage: sidecar restore-snapshot [flags] Flags: - -h, --help help for restore-snapshot - --snapshot.input string Path to the snapshot file either a URL or a local file (required) - --snapshot.verify-input Boolean to verify the input file against its .sha256sum file, if input is a url then it downloads the file, (default is true) (default true) + -h, --help help for restore-snapshot + --snapshot.input string Path to the snapshot file either a URL or a local file (optional), **If specified, this file is used instead of the manifest desired snapshot** + --snapshot.manifest-url string URL to a manifest json. Gets the latest snapshot matching the current runtime configurations of version, chain, and schema (default "https://sidecar.eigenlayer.xyz/snapshots/snapshots_manifest_v1.0.0.json") + --snapshot.verify-input Boolean to verify the input file against its .sha256sum file, if input is a url then it downloads the file (default true) Global Flags: -c, --chain string The chain to use (mainnet, holesky, preprod (default "mainnet") diff --git a/internal/config/config.go b/internal/config/config.go index f14f03f6..f83bc1a0 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -66,6 +66,7 @@ type SnapshotConfig struct { OutputFile string Input string VerifyInput bool + ManifestURL string } type RpcConfig struct { @@ -130,6 +131,7 @@ var ( SnapshotOutputFile = "snapshot.output-file" SnapshotInput = "snapshot.input" SnapshotVerifyInput = "snapshot.verify-input" + SnapshotManifestURL = "snapshot.manifest-url" RewardsValidateRewardsRoot = "rewards.validate_rewards_root" RewardsGenerateStakerOperatorsTable = "rewards.generate_staker_operators_table" @@ -175,6 +177,7 @@ func NewConfig() *Config { OutputFile: viper.GetString(normalizeFlagName(SnapshotOutputFile)), Input: viper.GetString(normalizeFlagName(SnapshotInput)), VerifyInput: viper.GetBool(normalizeFlagName(SnapshotVerifyInput)), + ManifestURL: viper.GetString(normalizeFlagName(SnapshotManifestURL)), }, RpcConfig: RpcConfig{ diff --git a/pkg/snapshot/snapshot.go b/pkg/snapshot/snapshot.go index 25e24aeb..59aeb959 100644 --- a/pkg/snapshot/snapshot.go +++ b/pkg/snapshot/snapshot.go @@ -3,6 +3,7 @@ package snapshot import ( "crypto/sha256" "encoding/hex" + "encoding/json" "fmt" "io" "net/http" @@ -23,12 +24,15 @@ type SnapshotConfig struct { OutputFile string Input string VerifyInput bool + ManifestURL string Host string Port int User string Password string DbName string SchemaName string + Version string + Chain string } // SnapshotService encapsulates the configuration and logger for snapshot operations. @@ -122,73 +126,45 @@ func (s *SnapshotService) RestoreSnapshot() error { s.tempFiles = s.tempFiles[:0] // Clear the tempFiles slice after cleanup }() - var resolvedFilePath string - if isHttpURL(s.cfg.Input) { - inputUrl := s.cfg.Input - // Check if the input URL exists - snapshotExists, err := urlExists(inputUrl) - if err != nil { - return errors.Wrap(err, fmt.Sprintf("error checking existence of snapshot URL '%s'", inputUrl)) - } - if !snapshotExists { - return errors.Wrap(fmt.Errorf("snapshot file not found at '%s'. Ensure the file exists", inputUrl), "snapshot file not found") - } - - fileName, err := getFileNameFromURL(inputUrl) + var snapshotAbsoluteFilePath string + var err error + if s.cfg.Input == "" && s.cfg.ManifestURL == "" { + return errors.Wrap(fmt.Errorf("input file or manifest URL is required"), "missing required configuration") + } else if s.cfg.Input == "" && s.cfg.ManifestURL != "" { + // Get the desired snapshot URL from the manifest + desiredURL, err := s.getDesiredURLFromManifest(s.cfg.ManifestURL) if err != nil { - return errors.Wrap(err, fmt.Sprintf("failed to extract file name from URL '%s'", inputUrl)) - } - - inputFilePath := filepath.Join(os.TempDir(), fileName) - - // Download the snapshot file hash - if s.cfg.VerifyInput { - hashFilePath := getHashName(inputFilePath) - hashUrl := getHashName(inputUrl) - hashFileExists, err := urlExists(hashUrl) - if err != nil { - return errors.Wrap(err, fmt.Sprintf("error checking existence of snapshot hash URL '%s'", hashUrl)) - } - if !hashFileExists { - return errors.Wrap(fmt.Errorf("snapshot hash file not found at '%s'. Ensure the file exists or set --verify-input=false to skip verification", hashUrl), "snapshot hash file not found") - } - - err = downloadFile(hashUrl, hashFilePath) - if err != nil { - return errors.Wrap(err, fmt.Sprintf("error downloading snapshot hash from '%s'", hashUrl)) - } - s.tempFiles = append(s.tempFiles, hashFilePath) + return err } - - // Download the snapshot file - err = downloadFile(inputUrl, inputFilePath) + // Use the desired URL as the input for downloading + snapshotAbsoluteFilePath, err = s.downloadSnapshotAndVerificationFiles(desiredURL) if err != nil { - return errors.Wrap(err, fmt.Sprintf("error downloading snapshot from '%s'", inputUrl)) + return err } - s.tempFiles = append(s.tempFiles, inputFilePath) - - resolvedFilePath, err = resolveFilePath(inputFilePath) + } else if isHttpURL(s.cfg.Input) { + inputUrl := s.cfg.Input + snapshotAbsoluteFilePath, err = s.downloadSnapshotAndVerificationFiles(inputUrl) if err != nil { - return errors.Wrap(err, fmt.Sprintf("failed to resolve input file path '%s'", inputFilePath)) + return err } } else { var err error - resolvedFilePath, err = resolveFilePath(s.cfg.Input) + snapshotAbsoluteFilePath, err = resolveFilePath(s.cfg.Input) if err != nil { return errors.Wrap(err, fmt.Sprintf("failed to resolve input file path '%s'", s.cfg.Input)) } } - s.l.Sugar().Debugw("Resolved file path", zap.String("resolvedFilePath", resolvedFilePath)) + s.l.Sugar().Debugw("Snapshot absolute file path", zap.String("snapshotAbsoluteFilePath", snapshotAbsoluteFilePath)) // validate snapshot against the hash file if s.cfg.VerifyInput { - if err := validateInputFileHash(resolvedFilePath, getHashName(resolvedFilePath)); err != nil { - return errors.Wrap(err, fmt.Sprintf("input file hash validation failed for '%s'", resolvedFilePath)) + if err := validateInputFileHash(snapshotAbsoluteFilePath, getHashName(snapshotAbsoluteFilePath)); err != nil { + return errors.Wrap(err, fmt.Sprintf("input file hash validation failed for '%s'", snapshotAbsoluteFilePath)) } s.l.Sugar().Debugw("Input file hash validated successfully", - zap.String("input", resolvedFilePath), - zap.String("inputHashFile", getHashName(resolvedFilePath)), + zap.String("input", snapshotAbsoluteFilePath), + zap.String("inputHashFile", getHashName(snapshotAbsoluteFilePath)), ) } @@ -197,7 +173,7 @@ func (s *SnapshotService) RestoreSnapshot() error { return err } - restoreExec := restore.Exec(resolvedFilePath, pgcommands.ExecOptions{StreamPrint: false}) + restoreExec := restore.Exec(snapshotAbsoluteFilePath, pgcommands.ExecOptions{StreamPrint: false}) if restoreExec.Error != nil { s.l.Sugar().Errorw("Failed to restore from snapshot", zap.Error(restoreExec.Error.Err), @@ -343,6 +319,10 @@ func downloadFile(url, downloadDestFilePath string) error { } defer resp.Body.Close() + if resp.StatusCode == http.StatusNotFound { + return fmt.Errorf("404 Not Found") + } + if resp.StatusCode != http.StatusOK { return fmt.Errorf("downloading error, received status code %d", resp.StatusCode) } @@ -389,22 +369,117 @@ func getFileNameFromURL(rawURL string) (string, error) { return path.Base(parsedURL.Path), nil } -// urlExists checks if the given URL is accessible by sending a HEAD request. -func urlExists(url string) (bool, error) { - resp, err := http.Head(url) +func (s *SnapshotService) downloadSnapshotAndVerificationFiles(inputUrl string) (string, error) { + fileName, err := getFileNameFromURL(inputUrl) if err != nil { - return false, errors.Wrap(err, "failed to send HEAD request") + return "", errors.Wrap(err, fmt.Sprintf("failed to extract file name from URL '%s'", inputUrl)) } - defer resp.Body.Close() - if resp.StatusCode >= 200 && resp.StatusCode < 400 { - return true, nil + inputFilePath := filepath.Join(os.TempDir(), fileName) + + // Download the snapshot file hash + if s.cfg.VerifyInput { + hashFilePath := getHashName(inputFilePath) + hashUrl := getHashName(inputUrl) + + err = downloadFile(hashUrl, hashFilePath) + if err != nil { + if err.Error() == "404 Not Found" { + return "", errors.Wrap(fmt.Errorf("snapshot hash file not found at '%s'. Ensure the file exists or set --verify-input=false to skip verification", hashUrl), "snapshot hash file not found") + } + return "", errors.Wrap(err, fmt.Sprintf("error downloading snapshot hash from '%s'", hashUrl)) + } + s.tempFiles = append(s.tempFiles, hashFilePath) } - // Return false for 404 without an error - if resp.StatusCode == http.StatusNotFound { - return false, nil + // Download the snapshot file + err = downloadFile(inputUrl, inputFilePath) + if err != nil { + if err.Error() == "404 Not Found" { + return "", errors.Wrap(fmt.Errorf("snapshot file not found at '%s'. Ensure the file exists", inputUrl), "snapshot file not found") + } + return "", errors.Wrap(err, fmt.Sprintf("error downloading snapshot from '%s'", inputUrl)) + } + s.tempFiles = append(s.tempFiles, inputFilePath) + + resolvedFilePath, err := resolveFilePath(inputFilePath) + if err != nil { + return "", errors.Wrap(err, fmt.Sprintf("failed to resolve input file path '%s'", inputFilePath)) } - return false, fmt.Errorf("URL not accessible, received status code %d", resp.StatusCode) + return resolvedFilePath, nil +} + +// Define a struct to match the manifest format, used in MetadataVersion v1.0.0 +type Manifest struct { + Meta Meta `json:"meta"` + Snapshots []Snapshot `json:"snapshots"` +} + +type Meta struct { + MetadataVersion string `json:"metadataVersion"` + LastRefreshed string `json:"lastRefreshed"` +} + +type Snapshot struct { + SnapshotURL string `json:"snapshotURL"` + Chain string `json:"chain"` + Version string `json:"version"` + Schema string `json:"schema"` + Timestamp string `json:"timestamp"` +} + +func (s *SnapshotService) getDesiredURLFromManifest(manifestURL string) (string, error) { + // Fetch the snapshot URL from the manifest + if !isHttpURL(manifestURL) { + return "", errors.Wrap(fmt.Errorf("manifest URL must be a network URL"), "invalid manifest URL") + } + + // Download the manifest file + manifestFileName, err := getFileNameFromURL(manifestURL) + if err != nil { + return "", errors.Wrap(err, fmt.Sprintf("failed to extract file name from manifest URL '%s'", manifestURL)) + } + + manifestFilePath := filepath.Join(os.TempDir(), manifestFileName) + + err = downloadFile(manifestURL, manifestFilePath) + if err != nil { + if err.Error() == "404 Not Found" { + return "", errors.Wrap(fmt.Errorf("manifest file not found at '%s'. Ensure the file exists", manifestURL), "manifest file not found") + } + return "", errors.Wrap(err, fmt.Sprintf("error downloading manifest from '%s'", manifestURL)) + } + + s.tempFiles = append(s.tempFiles, manifestFilePath) + s.l.Sugar().Infow("Downloaded manifest file", zap.String("manifestFilePath", manifestFilePath)) + + // Read and parse the manifest file + manifestData, err := os.ReadFile(manifestFilePath) + if err != nil { + return "", errors.Wrap(err, "failed to read manifest file") + } + + var manifest Manifest + if err := json.Unmarshal(manifestData, &manifest); err != nil { + return "", errors.Wrap(err, "failed to parse manifest JSON") + } + + // Filter snapshots by version, chain, and schema + var latestSnapshot *Snapshot + for _, snapshot := range manifest.Snapshots { + if snapshot.Version == s.cfg.Version && snapshot.Chain == s.cfg.Chain && snapshot.Schema == s.cfg.SchemaName { + if latestSnapshot == nil || snapshot.Timestamp > latestSnapshot.Timestamp { + latestSnapshot = &snapshot + } + } + } + + if latestSnapshot == nil { + return "", errors.Wrap(fmt.Errorf("no matching snapshot found for version '%s', chain '%s', and schema '%s'", s.cfg.Version, s.cfg.Chain, s.cfg.SchemaName), "no matching snapshot found") + } + + s.l.Sugar().Debugw("Selected snapshot", zap.String("snapshotURL", latestSnapshot.SnapshotURL)) + + return latestSnapshot.SnapshotURL, nil } diff --git a/pkg/snapshot/snapshot_test.go b/pkg/snapshot/snapshot_test.go index 1b20bec2..bd39f4cf 100644 --- a/pkg/snapshot/snapshot_test.go +++ b/pkg/snapshot/snapshot_test.go @@ -4,6 +4,7 @@ import ( "crypto/sha256" "encoding/hex" "fmt" + "net" "net/http" "net/http/httptest" "os" @@ -341,23 +342,183 @@ func TestCreateAndRestoreSnapshot(t *testing.T) { err = svc.RestoreSnapshot() assert.NoError(t, err, "Restoring snapshot should not fail") - // Validate the restore process + // Step 1: Validate the restore process + var countBefore int64 + dbGrm.Raw("SELECT COUNT(*) FROM migrations").Scan(&countBefore) + + // Step 2: Setup your migrator for db (the restored snapshot) and attempt running all migrations + migrator := migrations.NewMigrator(nil, dbGrm, l, cfg) + err = migrator.MigrateAll() + assert.NoError(t, err, "Expected MigrateAll to succeed on db") + + // Step 3: Count again after running migrations + var countAfter int64 + dbGrm.Raw("SELECT COUNT(*) FROM migrations").Scan(&countAfter) + + // Step 4: If countBefore == countAfter, no new migration records were created + // => meaning db was already fully up-to-date + assert.Equal(t, countBefore, countAfter, "No migrations should have been newly applied if db matches the original") + + t.Cleanup(func() { + postgres.TeardownTestDatabase(dbName, cfg, dbGrm, l) + }) + }) + + t.Run("Restore snapshot to a new database from URL", func(t *testing.T) { + // Create a test server to serve the snapshot and hash files + snapshotContent, err := os.ReadFile(dumpFile) + assert.NoError(t, err, "Reading snapshot file should not fail") + + hashContent, err := os.ReadFile(dumpFileHash) + assert.NoError(t, err, "Reading hash file should not fail") + + testServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/test_snapshot.dump.sha256sum" { + w.Write(hashContent) + } else if r.URL.Path == "/test_snapshot.dump" { + w.Write(snapshotContent) + } else { + http.NotFound(w, r) + } + })) + defer testServer.Close() + + // Append a filename to the test server URL + snapshotURL := testServer.URL + "/test_snapshot.dump" + + dbName, _, dbGrm, dbErr := postgres.GetTestPostgresDatabaseWithoutMigrations(cfg.DatabaseConfig, l) + if dbErr != nil { + t.Fatal(dbErr) + } + + snapshotCfg := &SnapshotConfig{ + Input: snapshotURL, + VerifyInput: true, + Host: cfg.DatabaseConfig.Host, + Port: cfg.DatabaseConfig.Port, + User: cfg.DatabaseConfig.User, + Password: cfg.DatabaseConfig.Password, + DbName: dbName, + SchemaName: cfg.DatabaseConfig.SchemaName, + } + svc, err := NewSnapshotService(snapshotCfg, l) + assert.NoError(t, err, "NewSnapshotService should not return an error") + err = svc.RestoreSnapshot() + assert.NoError(t, err, "Restoring snapshot should not return an error") + + // Step 1: Validate the restore process + var countBefore int64 + dbGrm.Raw("SELECT COUNT(*) FROM migrations").Scan(&countBefore) + + // Step 2: Setup your migrator for db (the restored snapshot) and attempt running all migrations + migrator := migrations.NewMigrator(nil, dbGrm, l, cfg) + err = migrator.MigrateAll() + assert.NoError(t, err, "Expected MigrateAll to succeed on db") + + // Step 3: Count again after running migrations + var countAfter int64 + dbGrm.Raw("SELECT COUNT(*) FROM migrations").Scan(&countAfter) + + // Step 4: If countBefore == countAfter, no new migration records were created + // => meaning db was already fully up-to-date + assert.Equal(t, countBefore, countAfter, "No migrations should have been newly applied if db matches the original") + + t.Cleanup(func() { + postgres.TeardownTestDatabase(dbName, cfg, dbGrm, l) + }) + }) + + t.Run("Restore snapshot to a new database from Manifest", func(t *testing.T) { + snapshotContent, err := os.ReadFile(dumpFile) + assert.NoError(t, err, "Reading snapshot file should not fail") + + hashContent, err := os.ReadFile(dumpFileHash) + assert.NoError(t, err, "Reading hash file should not fail") + + // Create an unstarted test server + testServer := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Printf("Received request for URL: %s\n", r.URL.String()) + + if strings.HasSuffix(r.URL.Path, ".sha256sum") { + fmt.Println("Serving hash content") + w.Write(hashContent) + } else if strings.HasSuffix(r.URL.Path, ".dump") { + fmt.Println("Serving snapshot content") + w.Write(snapshotContent) + } else { + mockManifest := fmt.Sprintf(`{ + "meta": { + "metadataVersion": "v1.0.0", + "lastRefreshed": "2025-01-30T14:29:04Z" + }, + "snapshots": [ + { + "snapshotURL": "http://localhost:5050/snapshots/mainnet/sidecar_mainnet_v2.0.0_public_20250130103903.dump", + "chain": "mainnet", + "version": "v2.0.0", + "schema": "public", + "timestamp": "2025-01-30T10:39:03Z" + } + ] + }`) + + fmt.Println("Serving mock manifest") + w.Write([]byte(mockManifest)) + } + })) + + // Set a fixed URL for the test server, dynamic testServer URLs break the mockManifest snapshotURL + testServer.Listener.Close() // Close the auto-assigned listener + fixedAddr := "127.0.0.1:5050" + listener, err := net.Listen("tcp", fixedAddr) + assert.NoError(t, err, "Failed to start test server on fixed address") + testServer.Listener = listener + testServer.Start() + + defer testServer.Close() + + // Use the fixed URL + manifestURL := "http://" + fixedAddr + "/mock_manifest.json" + + dbName, _, dbGrm, dbErr := postgres.GetTestPostgresDatabaseWithoutMigrations(cfg.DatabaseConfig, l) + if dbErr != nil { + t.Fatal(dbErr) + } - // 1) Count how many migration records already exist in db + snapshotCfg := &SnapshotConfig{ + ManifestURL: manifestURL, + Version: "v2.0.0", + Chain: "mainnet", + SchemaName: "public", + VerifyInput: true, + Host: cfg.DatabaseConfig.Host, + Port: cfg.DatabaseConfig.Port, + User: cfg.DatabaseConfig.User, + Password: cfg.DatabaseConfig.Password, + DbName: dbName, + } + svc, err := NewSnapshotService(snapshotCfg, l) + assert.NoError(t, err, "NewSnapshotService should not return an error") + + // Call the RestoreSnapshot function + err = svc.RestoreSnapshot() + assert.NoError(t, err, "Restoring snapshot should not return an error") + + // Step 1: Validate the restore process var countBefore int64 dbGrm.Raw("SELECT COUNT(*) FROM migrations").Scan(&countBefore) - // 2) Setup your migrator for db (the restored snapshot) and attempt running all migrations + // Step 2: Setup your migrator for db (the restored snapshot) and attempt running all migrations migrator := migrations.NewMigrator(nil, dbGrm, l, cfg) err = migrator.MigrateAll() assert.NoError(t, err, "Expected MigrateAll to succeed on db") - // 3) Count again after running migrations + // Step 3: Count again after running migrations var countAfter int64 dbGrm.Raw("SELECT COUNT(*) FROM migrations").Scan(&countAfter) - // 4) If countBefore == countAfter, no new migration records were created - // => meaning db was already fully up-to-date + // Step 4: If countBefore == countAfter, no new migration records were created + // => meaning db was already fully up-to-date assert.Equal(t, countBefore, countAfter, "No migrations should have been newly applied if db matches the original") t.Cleanup(func() { @@ -397,3 +558,131 @@ func TestIsHttpURL(t *testing.T) { }) } } + +func TestGetDesiredURLFromManifest(t *testing.T) { + // Create a mock manifest JSON + mockManifest := `{ + "meta": { + "metadataVersion": "v1.0.0", + "lastRefreshed": "2025-01-30T14:29:04Z" + }, + "snapshots": [ + { + "snapshotURL": "https://sidecar.eigenlayer.xyz/snapshots/mainnet/sidecar_mainnet_v2.0.0_public_20250130103903.dump", + "chain": "mainnet", + "version": "v2.0.0", + "schema": "public", + "timestamp": "2025-01-30T10:39:03Z" + }, + { + "snapshotURL": "https://sidecar.eigenlayer.xyz/snapshots/holesky/sidecar_holesky_v3.0.0-rc.1_sidecar_testnet_holesky_20250130140613.dump", + "chain": "testnet", + "version": "v3.0.0-rc.1", + "schema": "sidecar_testnet_holesky", + "timestamp": "2025-01-30T14:06:13Z" + } + ] + }` + + // Create a test server to serve the mock manifest + testServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, mockManifest) + })) + defer testServer.Close() + + // Append a filename to the test server URL + manifestURL := testServer.URL + "/mock_manifest.json" + + // Setup the SnapshotService with the desired configuration + cfg := &SnapshotConfig{ + Version: "v2.0.0", + Chain: "mainnet", + SchemaName: "public", + } + l, _ := zap.NewDevelopment() + svc, err := NewSnapshotService(cfg, l) + assert.NoError(t, err, "NewSnapshotService should not return an error") + + // Call the function to test + desiredURL, err := svc.getDesiredURLFromManifest(manifestURL) + assert.NoError(t, err, "getDesiredURLFromManifest should not return an error") + assert.Equal(t, "https://sidecar.eigenlayer.xyz/snapshots/mainnet/sidecar_mainnet_v2.0.0_public_20250130103903.dump", desiredURL, "The desired URL should match the expected snapshot URL") +} + +func TestDownloadSnapshotAndVerificationFilesWithVerifyInput(t *testing.T) { + // Create a test server to serve the snapshot and hash files + snapshotContent := "This is a test snapshot file." + hashContent := fmt.Sprintf("%x test_snapshot.dump", sha256.Sum256([]byte(snapshotContent))) + + testServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, ".sha256sum") { + fmt.Fprintln(w, hashContent) + } else { + fmt.Fprintln(w, snapshotContent) + } + })) + defer testServer.Close() + // Append a unique identifier to the test server URL + snapshotURL := testServer.URL + "/test_snapshot_" + fmt.Sprintf("%d", time.Now().UnixNano()) + ".dump" + + cfg := &SnapshotConfig{ + VerifyInput: true, + } + l, _ := zap.NewDevelopment() + svc, err := NewSnapshotService(cfg, l) + assert.NoError(t, err, "NewSnapshotService should not return an error") + + // Call the function to test + resolvedFilePath, err := svc.downloadSnapshotAndVerificationFiles(snapshotURL) + assert.NoError(t, err, "downloadSnapshotAndVerificationFiles should not return an error") + + // Verify the snapshot file content + content, err := os.ReadFile(resolvedFilePath) + assert.NoError(t, err, "Reading downloaded snapshot file should not fail") + assert.Contains(t, string(content), snapshotContent, "Snapshot file content should match expected content") + + // Verify the hash file content + hashFilePath := getHashName(resolvedFilePath) + hashContentRead, err := os.ReadFile(hashFilePath) + assert.NoError(t, err, "Reading hash file should not fail") + assert.Equal(t, hashContent, strings.TrimSpace(string(hashContentRead)), "Hash file content should match expected content") +} + +func TestDownloadSnapshotAndVerificationFilesWithoutVerifyInput(t *testing.T) { + // Create a test server to serve the snapshot and hash files + snapshotContent := "This is a test snapshot file." + hashContent := fmt.Sprintf("%x test_snapshot.dump", sha256.Sum256([]byte(snapshotContent))) + + testServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, ".sha256sum") { + fmt.Fprintln(w, hashContent) + } else { + fmt.Fprintln(w, snapshotContent) + } + })) + defer testServer.Close() + + // Append a filename to the test server URL + snapshotURL := testServer.URL + "/test_snapshot_" + fmt.Sprintf("%d", time.Now().UnixNano()) + ".dump" + + cfg := &SnapshotConfig{ + VerifyInput: false, + } + l, _ := zap.NewDevelopment() + svc, err := NewSnapshotService(cfg, l) + assert.NoError(t, err, "NewSnapshotService should not return an error") + + // Call the function to test + resolvedFilePath, err := svc.downloadSnapshotAndVerificationFiles(snapshotURL) + assert.NoError(t, err, "downloadSnapshotAndVerificationFiles should not return an error") + + // Verify the snapshot file content + content, err := os.ReadFile(resolvedFilePath) + assert.NoError(t, err, "Reading downloaded snapshot file should not fail") + assert.Contains(t, string(content), snapshotContent, "Snapshot file content should match expected content") + + // Ensure no hash file is created + hashFilePath := getHashName(resolvedFilePath) + _, err = os.Stat(hashFilePath) + assert.True(t, os.IsNotExist(err), "Hash file should not exist when VerifyInput is false") +}