Skip to content

Commit b6e1078

Browse files
fxamackerfxamacker
committed
Add --exclude-randombeaconhistory to diff-stats cmd
This commit adds "--exclude-randombeaconhistory" flag to util's "diff-states" command. This flag can be used to reduce noise when comparing states. Without this flag, the diff-states report (JSONL) can sometimes contain lines that are 30+ GB each, depending on the states being compared (e.g. before/after test of zero-downtime migration).
1 parent 4ce667d commit b6e1078

File tree

3 files changed

+95
-55
lines changed

3 files changed

+95
-55
lines changed

‎cmd/util/cmd/diff-states/cmd.go

Lines changed: 69 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import (
1111

1212
"github.com/dustin/go-humanize/english"
1313
"github.com/onflow/cadence/common"
14+
"github.com/onflow/cadence/interpreter"
1415
"github.com/rs/zerolog/log"
1516
"github.com/spf13/cobra"
1617
"golang.org/x/sync/errgroup"
@@ -26,17 +27,18 @@ import (
2627
)
2728

2829
var (
29-
flagOutputDirectory string
30-
flagPayloads1 string
31-
flagPayloads2 string
32-
flagState1 string
33-
flagState2 string
34-
flagStateCommitment1 string
35-
flagStateCommitment2 string
36-
flagMode string
37-
flagAlwaysDiffValues bool
38-
flagNWorker int
39-
flagChain string
30+
flagOutputDirectory string
31+
flagPayloads1 string
32+
flagPayloads2 string
33+
flagState1 string
34+
flagState2 string
35+
flagStateCommitment1 string
36+
flagStateCommitment2 string
37+
flagMode string
38+
flagAlwaysDiffValues bool
39+
flagExcludeRandomBeaconHistory bool
40+
flagNWorker int
41+
flagChain string
4042
)
4143

4244
var Cmd = &cobra.Command{
@@ -139,8 +141,20 @@ func init() {
139141
"Chain name",
140142
)
141143
_ = Cmd.MarkFlagRequired("chain")
144+
145+
Cmd.Flags().BoolVar(
146+
&flagExcludeRandomBeaconHistory,
147+
"exclude-randombeaconhistory",
148+
false,
149+
"exclude random beacon history",
150+
)
142151
}
143152

153+
const (
154+
randomBeaconHistoryDomain = common.StorageDomainContract
155+
randomBeaconHistoryDomainKey = interpreter.StringStorageMapKey("RandomBeaconHistory")
156+
)
157+
144158
type mode uint8
145159

146160
const (
@@ -191,6 +205,10 @@ func run(*cobra.Command, []string) {
191205
)
192206
}
193207

208+
if flagExcludeRandomBeaconHistory {
209+
log.Info().Msg("--exclude-randombeaconhistory is set to exclude random beacon history")
210+
}
211+
194212
var acctsToSkipForCadenceValueDiff []string
195213

196214
// Skip EVM storage account when diffing Cadence values.
@@ -336,6 +354,7 @@ func diffAccount(
336354
rw reporters.ReportWriter,
337355
mode mode,
338356
acctsToSkip []string,
357+
serviceAccountAddress common.Address,
339358
) (err error) {
340359

341360
diffValues := flagAlwaysDiffValues
@@ -398,6 +417,15 @@ func diffAccount(
398417
accountRegisters1,
399418
accountRegisters2,
400419
common.AllStorageDomains,
420+
func(address common.Address, domain common.StorageDomain, key any) bool {
421+
if flagExcludeRandomBeaconHistory {
422+
if isRandomBeaconHistory(serviceAccountAddress, address, domain, key) {
423+
log.Info().Msgf("excluding random beacon history in account %s, domain %s, key %v", address, domain.Identifier(), key)
424+
return false
425+
}
426+
}
427+
return true
428+
},
401429
)
402430
}
403431

@@ -415,6 +443,8 @@ func diff(
415443
) error {
416444
log.Info().Msgf("Diffing %d accounts", registers1.AccountCount())
417445

446+
serviceAccountAddress := serviceAccountAddressForChain(chainID)
447+
418448
if registers1.AccountCount() < nWorkers {
419449
nWorkers = registers1.AccountCount()
420450
}
@@ -454,6 +484,7 @@ func diff(
454484
rw,
455485
mode,
456486
acctsToSkip,
487+
common.Address(serviceAccountAddress),
457488
)
458489
if err != nil {
459490
log.Warn().Err(err).Msgf("failed to diff account %x", []byte(owner))
@@ -509,6 +540,7 @@ func diff(
509540
rw,
510541
mode,
511542
acctsToSkip,
543+
common.Address(serviceAccountAddress),
512544
)
513545

514546
select {
@@ -669,3 +701,29 @@ func (e countDiff) MarshalJSON() ([]byte, error) {
669701
State2: e.State2,
670702
})
671703
}
704+
705+
func isRandomBeaconHistory(serviceAccountAddress, address common.Address, domain common.StorageDomain, key any) bool {
706+
if serviceAccountAddress.Compare(address) != 0 {
707+
return false
708+
}
709+
710+
if domain != randomBeaconHistoryDomain {
711+
return false
712+
}
713+
714+
switch key := key.(type) {
715+
case interpreter.StringAtreeValue:
716+
return interpreter.StringStorageMapKey(key) == randomBeaconHistoryDomainKey
717+
718+
case interpreter.StringStorageMapKey:
719+
return key == randomBeaconHistoryDomainKey
720+
721+
default:
722+
return false
723+
}
724+
}
725+
726+
func serviceAccountAddressForChain(chainID flow.ChainID) flow.Address {
727+
sc := systemcontracts.SystemContractsForChain(chainID)
728+
return sc.FlowServiceAccount.Address
729+
}

‎cmd/util/ledger/migrations/cadence_value_diff.go

Lines changed: 12 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -2,12 +2,10 @@ package migrations
22

33
import (
44
"fmt"
5-
"time"
65

76
"github.com/onflow/cadence/common"
87
"github.com/onflow/cadence/interpreter"
98
"github.com/onflow/cadence/runtime"
10-
"github.com/rs/zerolog/log"
119
"golang.org/x/sync/errgroup"
1210

1311
"github.com/onflow/flow-go/cmd/util/ledger/reporters"
@@ -76,8 +74,6 @@ type difference struct {
7674
NewValueStaticType string `json:",omitempty"`
7775
}
7876

79-
const minLargeAccountRegisterCount = 1_000_000
80-
8177
type CadenceValueDiffReporter struct {
8278
address common.Address
8379
chainID flow.ChainID
@@ -102,7 +98,13 @@ func NewCadenceValueDiffReporter(
10298
}
10399
}
104100

105-
func (dr *CadenceValueDiffReporter) DiffStates(oldRegs, newRegs registers.Registers, domains []common.StorageDomain) {
101+
type isValueIncludedFunc func(address common.Address, domain common.StorageDomain, key any) bool
102+
103+
func (dr *CadenceValueDiffReporter) DiffStates(
104+
oldRegs, newRegs registers.Registers,
105+
domains []common.StorageDomain,
106+
isValueIncluded isValueIncludedFunc,
107+
) {
106108

107109
oldStorage := newReadonlyStorage(oldRegs)
108110

@@ -160,14 +162,15 @@ func (dr *CadenceValueDiffReporter) DiffStates(oldRegs, newRegs registers.Regist
160162
}
161163

162164
for _, domain := range domains {
163-
dr.diffDomain(oldRuntime, newRuntime, domain)
165+
dr.diffDomain(oldRuntime, newRuntime, domain, isValueIncluded)
164166
}
165167
}
166168

167169
func (dr *CadenceValueDiffReporter) diffDomain(
168170
oldRuntime *readonlyStorageRuntime,
169171
newRuntime *readonlyStorageRuntime,
170172
domain common.StorageDomain,
173+
isValueIncluded isValueIncludedFunc,
171174
) {
172175
defer func() {
173176
if r := recover(); r != nil {
@@ -340,38 +343,12 @@ func (dr *CadenceValueDiffReporter) diffDomain(
340343
}
341344
}
342345

343-
// Skip goroutine overhead for non-storage domain and small accounts.
344-
if domain != common.StorageDomainPathStorage ||
345-
oldRuntime.PayloadCount < minLargeAccountRegisterCount ||
346-
len(sharedKeys) == 1 {
347-
348-
for _, key := range sharedKeys {
349-
oldValue, newValue, trace, canDiff := getValues(key)
350-
if canDiff {
351-
diffValues(
352-
oldRuntime.Interpreter,
353-
oldValue,
354-
newRuntime.Interpreter,
355-
newValue,
356-
trace,
357-
)
358-
}
359-
}
360-
return
361-
}
362-
363-
startTime := time.Now()
364-
365-
log.Info().Msgf(
366-
"Diffing %x storage domain containing %d elements (%d payloads) ...",
367-
dr.address[:],
368-
len(sharedKeys),
369-
oldRuntime.PayloadCount,
370-
)
371-
372346
// Diffing storage domain
373347

374348
for _, key := range sharedKeys {
349+
if !isValueIncluded(dr.address, domain, key) {
350+
continue
351+
}
375352
oldValue, newValue, trace, canDiff := getValues(key)
376353
if canDiff {
377354
diffValues(
@@ -383,15 +360,6 @@ func (dr *CadenceValueDiffReporter) diffDomain(
383360
)
384361
}
385362
}
386-
387-
log.Info().
388-
Msgf(
389-
"Finished diffing %x storage domain containing %d elements (%d payloads) in %s",
390-
dr.address[:],
391-
len(sharedKeys),
392-
oldRuntime.PayloadCount,
393-
time.Since(startTime),
394-
)
395363
}
396364

397365
func (dr *CadenceValueDiffReporter) diffValues(

‎cmd/util/ledger/migrations/cadence_value_diff_test.go

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,10 @@ func TestDiffCadenceValues(t *testing.T) {
2626

2727
const domain = common.StorageDomainPathStorage
2828

29+
alwaysDiff := func(address common.Address, domain common.StorageDomain, key any) bool {
30+
return true
31+
}
32+
2933
t.Run("no diff", func(t *testing.T) {
3034
t.Parallel()
3135

@@ -37,6 +41,7 @@ func TestDiffCadenceValues(t *testing.T) {
3741
createTestRegisters(t, address, domain),
3842
createTestRegisters(t, address, domain),
3943
[]common.StorageDomain{domain},
44+
alwaysDiff,
4045
)
4146
require.NoError(t, err)
4247
require.Equal(t, 0, len(writer.entries))
@@ -53,6 +58,7 @@ func TestDiffCadenceValues(t *testing.T) {
5358
createTestRegisters(t, address, domain),
5459
registers.NewByAccount(),
5560
[]common.StorageDomain{domain},
61+
alwaysDiff,
5662
)
5763
require.NoError(t, err)
5864
require.Equal(t, 1, len(writer.entries))
@@ -74,6 +80,7 @@ func TestDiffCadenceValues(t *testing.T) {
7480
createTestRegisters(t, address, domain),
7581
createStorageMapRegisters(t, address, domain, []string{"unique_key"}, []interpreter.Value{interpreter.UInt64Value(0)}),
7682
[]common.StorageDomain{domain},
83+
alwaysDiff,
7784
)
7885
require.NoError(t, err)
7986

@@ -101,6 +108,7 @@ func TestDiffCadenceValues(t *testing.T) {
101108
createStorageMapRegisters(t, address, domain, []string{"0", "1"}, []interpreter.Value{interpreter.UInt64Value(0), interpreter.UInt64Value(0)}),
102109
createStorageMapRegisters(t, address, domain, []string{"2", "0"}, []interpreter.Value{interpreter.UInt64Value(0), interpreter.UInt64Value(0)}),
103110
[]common.StorageDomain{domain},
111+
alwaysDiff,
104112
)
105113
require.NoError(t, err)
106114

@@ -128,6 +136,7 @@ func TestDiffCadenceValues(t *testing.T) {
128136
createStorageMapRegisters(t, address, domain, []string{"0", "1"}, []interpreter.Value{interpreter.UInt64Value(100), interpreter.UInt64Value(101)}),
129137
createStorageMapRegisters(t, address, domain, []string{"0", "1"}, []interpreter.Value{interpreter.UInt64Value(111), interpreter.UInt64Value(101)}),
130138
[]common.StorageDomain{domain},
139+
alwaysDiff,
131140
)
132141
require.NoError(t, err)
133142

@@ -155,6 +164,7 @@ func TestDiffCadenceValues(t *testing.T) {
155164
createStorageMapRegisters(t, address, domain, []string{"0", "1"}, []interpreter.Value{interpreter.UInt64Value(100), interpreter.UInt64Value(101)}),
156165
createStorageMapRegisters(t, address, domain, []string{"0", "1"}, []interpreter.Value{interpreter.UInt64Value(111), interpreter.UInt64Value(102)}),
157166
[]common.StorageDomain{domain},
167+
alwaysDiff,
158168
)
159169
require.NoError(t, err)
160170

@@ -261,6 +271,7 @@ func TestDiffCadenceValues(t *testing.T) {
261271
interpreter.UInt64Value(5),
262272
}),
263273
[]common.StorageDomain{domain},
274+
alwaysDiff,
264275
)
265276
require.NoError(t, err)
266277

@@ -385,6 +396,7 @@ func TestDiffCadenceValues(t *testing.T) {
385396
interpreter.UInt64Value(3),
386397
}),
387398
[]common.StorageDomain{domain},
399+
alwaysDiff,
388400
)
389401
require.NoError(t, err)
390402

@@ -516,6 +528,7 @@ func TestDiffCadenceValues(t *testing.T) {
516528
interpreter.UInt64Value(3),
517529
}),
518530
[]common.StorageDomain{domain},
531+
alwaysDiff,
519532
)
520533
require.NoError(t, err)
521534

@@ -647,6 +660,7 @@ func TestDiffCadenceValues(t *testing.T) {
647660
interpreter.UInt64Value(3),
648661
}),
649662
[]common.StorageDomain{domain},
663+
alwaysDiff,
650664
)
651665
require.NoError(t, err)
652666

0 commit comments

Comments
 (0)