Merge pull request #6993 from onflow/fxamacker/update-diff-states-to-…

…optionally-exclude-random-beacon-history

Add `--exclude-randombeaconhistory` to `diff-states` cmd in `util` program

cmd/util/cmd/diff-states/cmd.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/dustin/go-humanize/english"
 	"github.com/onflow/cadence/common"
+	"github.com/onflow/cadence/interpreter"
 	"github.com/rs/zerolog/log"
 	"github.com/spf13/cobra"
 	"golang.org/x/sync/errgroup"
@@ -26,17 +27,18 @@ import (
 )
 
 var (
-	flagOutputDirectory  string
-	flagPayloads1        string
-	flagPayloads2        string
-	flagState1           string
-	flagState2           string
-	flagStateCommitment1 string
-	flagStateCommitment2 string
-	flagMode             string
-	flagAlwaysDiffValues bool
-	flagNWorker          int
-	flagChain            string
+	flagOutputDirectory            string
+	flagPayloads1                  string
+	flagPayloads2                  string
+	flagState1                     string
+	flagState2                     string
+	flagStateCommitment1           string
+	flagStateCommitment2           string
+	flagMode                       string
+	flagAlwaysDiffValues           bool
+	flagExcludeRandomBeaconHistory bool
+	flagNWorker                    int
+	flagChain                      string
 )
 
 var Cmd = &cobra.Command{
@@ -139,8 +141,20 @@ func init() {
 		"Chain name",
 	)
 	_ = Cmd.MarkFlagRequired("chain")
+
+	Cmd.Flags().BoolVar(
+		&flagExcludeRandomBeaconHistory,
+		"exclude-randombeaconhistory",
+		false,
+		"exclude random beacon history",
+	)
 }
 
+const (
+	randomBeaconHistoryDomain    = common.StorageDomainContract
+	randomBeaconHistoryDomainKey = interpreter.StringStorageMapKey("RandomBeaconHistory")
+)
+
 type mode uint8
 
 const (
@@ -191,6 +205,10 @@ func run(*cobra.Command, []string) {
 		)
 	}
 
+	if flagExcludeRandomBeaconHistory {
+		log.Info().Msg("--exclude-randombeaconhistory is set to exclude random beacon history")
+	}
+
 	var acctsToSkipForCadenceValueDiff []string
 
 	// Skip EVM storage account when diffing Cadence values.
@@ -336,6 +354,7 @@ func diffAccount(
 	rw reporters.ReportWriter,
 	mode mode,
 	acctsToSkip []string,
+	serviceAccountAddress common.Address,
 ) (err error) {
 
 	diffValues := flagAlwaysDiffValues
@@ -398,6 +417,15 @@ func diffAccount(
 			accountRegisters1,
 			accountRegisters2,
 			common.AllStorageDomains,
+			func(address common.Address, domain common.StorageDomain, key any) bool {
+				if flagExcludeRandomBeaconHistory {
+					if isRandomBeaconHistory(serviceAccountAddress, address, domain, key) {
+						log.Info().Msgf("excluding random beacon history in account %s, domain %s, key %v", address, domain.Identifier(), key)
+						return false
+					}
+				}
+				return true
+			},
 		)
 	}
 
@@ -415,6 +443,8 @@ func diff(
 ) error {
 	log.Info().Msgf("Diffing %d accounts", registers1.AccountCount())
 
+	serviceAccountAddress := serviceAccountAddressForChain(chainID)
+
 	if registers1.AccountCount() < nWorkers {
 		nWorkers = registers1.AccountCount()
 	}
@@ -454,6 +484,7 @@ func diff(
 				rw,
 				mode,
 				acctsToSkip,
+				common.Address(serviceAccountAddress),
 			)
 			if err != nil {
 				log.Warn().Err(err).Msgf("failed to diff account %x", []byte(owner))
@@ -509,6 +540,7 @@ func diff(
 					rw,
 					mode,
 					acctsToSkip,
+					common.Address(serviceAccountAddress),
 				)
 
 				select {
@@ -669,3 +701,29 @@ func (e countDiff) MarshalJSON() ([]byte, error) {
 		State2: e.State2,
 	})
 }
+
+func isRandomBeaconHistory(serviceAccountAddress, address common.Address, domain common.StorageDomain, key any) bool {
+	if serviceAccountAddress.Compare(address) != 0 {
+		return false
+	}
+
+	if domain != randomBeaconHistoryDomain {
+		return false
+	}
+
+	switch key := key.(type) {
+	case interpreter.StringAtreeValue:
+		return interpreter.StringStorageMapKey(key) == randomBeaconHistoryDomainKey
+
+	case interpreter.StringStorageMapKey:
+		return key == randomBeaconHistoryDomainKey
+
+	default:
+		return false
+	}
+}
+
+func serviceAccountAddressForChain(chainID flow.ChainID) flow.Address {
+	sc := systemcontracts.SystemContractsForChain(chainID)
+	return sc.FlowServiceAccount.Address
+}

cmd/util/ledger/migrations/cadence_value_diff.go

@@ -2,12 +2,10 @@ package migrations
 
 import (
 	"fmt"
-	"time"
 
 	"github.com/onflow/cadence/common"
 	"github.com/onflow/cadence/interpreter"
 	"github.com/onflow/cadence/runtime"
-	"github.com/rs/zerolog/log"
 	"golang.org/x/sync/errgroup"
 
 	"github.com/onflow/flow-go/cmd/util/ledger/reporters"
@@ -76,8 +74,6 @@ type difference struct {
 	NewValueStaticType string `json:",omitempty"`
 }
 
-const minLargeAccountRegisterCount = 1_000_000
-
 type CadenceValueDiffReporter struct {
 	address        common.Address
 	chainID        flow.ChainID
@@ -102,7 +98,13 @@ func NewCadenceValueDiffReporter(
 	}
 }
 
-func (dr *CadenceValueDiffReporter) DiffStates(oldRegs, newRegs registers.Registers, domains []common.StorageDomain) {
+type isValueIncludedFunc func(address common.Address, domain common.StorageDomain, key any) bool
+
+func (dr *CadenceValueDiffReporter) DiffStates(
+	oldRegs, newRegs registers.Registers,
+	domains []common.StorageDomain,
+	isValueIncluded isValueIncludedFunc,
+) {
 
 	oldStorage := newReadonlyStorage(oldRegs)
 
@@ -160,14 +162,15 @@ func (dr *CadenceValueDiffReporter) DiffStates(oldRegs, newRegs registers.Regist
 	}
 
 	for _, domain := range domains {
-		dr.diffDomain(oldRuntime, newRuntime, domain)
+		dr.diffDomain(oldRuntime, newRuntime, domain, isValueIncluded)
 	}
 }
 
 func (dr *CadenceValueDiffReporter) diffDomain(
 	oldRuntime *readonlyStorageRuntime,
 	newRuntime *readonlyStorageRuntime,
 	domain common.StorageDomain,
+	isValueIncluded isValueIncludedFunc,
 ) {
 	defer func() {
 		if r := recover(); r != nil {
@@ -340,38 +343,12 @@ func (dr *CadenceValueDiffReporter) diffDomain(
 		}
 	}
 
-	// Skip goroutine overhead for non-storage domain and small accounts.
-	if domain != common.StorageDomainPathStorage ||
-		oldRuntime.PayloadCount < minLargeAccountRegisterCount ||
-		len(sharedKeys) == 1 {
-
-		for _, key := range sharedKeys {
-			oldValue, newValue, trace, canDiff := getValues(key)
-			if canDiff {
-				diffValues(
-					oldRuntime.Interpreter,
-					oldValue,
-					newRuntime.Interpreter,
-					newValue,
-					trace,
-				)
-			}
-		}
-		return
-	}
-
-	startTime := time.Now()
-
-	log.Info().Msgf(
-		"Diffing %x storage domain containing %d elements (%d payloads) ...",
-		dr.address[:],
-		len(sharedKeys),
-		oldRuntime.PayloadCount,
-	)
-
 	// Diffing storage domain
 
 	for _, key := range sharedKeys {
+		if !isValueIncluded(dr.address, domain, key) {
+			continue
+		}
 		oldValue, newValue, trace, canDiff := getValues(key)
 		if canDiff {
 			diffValues(
@@ -383,15 +360,6 @@ func (dr *CadenceValueDiffReporter) diffDomain(
 			)
 		}
 	}
-
-	log.Info().
-		Msgf(
-			"Finished diffing %x storage domain containing %d elements (%d payloads) in %s",
-			dr.address[:],
-			len(sharedKeys),
-			oldRuntime.PayloadCount,
-			time.Since(startTime),
-		)
 }
 
 func (dr *CadenceValueDiffReporter) diffValues(

cmd/util/ledger/migrations/cadence_value_diff_test.go

@@ -26,6 +26,10 @@ func TestDiffCadenceValues(t *testing.T) {
 
 	const domain = common.StorageDomainPathStorage
 
+	alwaysDiff := func(address common.Address, domain common.StorageDomain, key any) bool {
+		return true
+	}
+
 	t.Run("no diff", func(t *testing.T) {
 		t.Parallel()
 
@@ -37,6 +41,7 @@ func TestDiffCadenceValues(t *testing.T) {
 			createTestRegisters(t, address, domain),
 			createTestRegisters(t, address, domain),
 			[]common.StorageDomain{domain},
+			alwaysDiff,
 		)
 		require.NoError(t, err)
 		require.Equal(t, 0, len(writer.entries))
@@ -53,6 +58,7 @@ func TestDiffCadenceValues(t *testing.T) {
 			createTestRegisters(t, address, domain),
 			registers.NewByAccount(),
 			[]common.StorageDomain{domain},
+			alwaysDiff,
 		)
 		require.NoError(t, err)
 		require.Equal(t, 1, len(writer.entries))
@@ -74,6 +80,7 @@ func TestDiffCadenceValues(t *testing.T) {
 			createTestRegisters(t, address, domain),
 			createStorageMapRegisters(t, address, domain, []string{"unique_key"}, []interpreter.Value{interpreter.UInt64Value(0)}),
 			[]common.StorageDomain{domain},
+			alwaysDiff,
 		)
 		require.NoError(t, err)
 
@@ -101,6 +108,7 @@ func TestDiffCadenceValues(t *testing.T) {
 			createStorageMapRegisters(t, address, domain, []string{"0", "1"}, []interpreter.Value{interpreter.UInt64Value(0), interpreter.UInt64Value(0)}),
 			createStorageMapRegisters(t, address, domain, []string{"2", "0"}, []interpreter.Value{interpreter.UInt64Value(0), interpreter.UInt64Value(0)}),
 			[]common.StorageDomain{domain},
+			alwaysDiff,
 		)
 		require.NoError(t, err)
 
@@ -128,6 +136,7 @@ func TestDiffCadenceValues(t *testing.T) {
 			createStorageMapRegisters(t, address, domain, []string{"0", "1"}, []interpreter.Value{interpreter.UInt64Value(100), interpreter.UInt64Value(101)}),
 			createStorageMapRegisters(t, address, domain, []string{"0", "1"}, []interpreter.Value{interpreter.UInt64Value(111), interpreter.UInt64Value(101)}),
 			[]common.StorageDomain{domain},
+			alwaysDiff,
 		)
 		require.NoError(t, err)
 
@@ -155,6 +164,7 @@ func TestDiffCadenceValues(t *testing.T) {
 			createStorageMapRegisters(t, address, domain, []string{"0", "1"}, []interpreter.Value{interpreter.UInt64Value(100), interpreter.UInt64Value(101)}),
 			createStorageMapRegisters(t, address, domain, []string{"0", "1"}, []interpreter.Value{interpreter.UInt64Value(111), interpreter.UInt64Value(102)}),
 			[]common.StorageDomain{domain},
+			alwaysDiff,
 		)
 		require.NoError(t, err)
 
@@ -261,6 +271,7 @@ func TestDiffCadenceValues(t *testing.T) {
 				interpreter.UInt64Value(5),
 			}),
 			[]common.StorageDomain{domain},
+			alwaysDiff,
 		)
 		require.NoError(t, err)
 
@@ -385,6 +396,7 @@ func TestDiffCadenceValues(t *testing.T) {
 					interpreter.UInt64Value(3),
 				}),
 			[]common.StorageDomain{domain},
+			alwaysDiff,
 		)
 		require.NoError(t, err)
 
@@ -516,6 +528,7 @@ func TestDiffCadenceValues(t *testing.T) {
 					interpreter.UInt64Value(3),
 				}),
 			[]common.StorageDomain{domain},
+			alwaysDiff,
 		)
 		require.NoError(t, err)
 
@@ -647,6 +660,7 @@ func TestDiffCadenceValues(t *testing.T) {
 					interpreter.UInt64Value(3),
 				}),
 			[]common.StorageDomain{domain},
+			alwaysDiff,
 		)
 		require.NoError(t, err)