From 0da27c1a74a96c8eafa0ca9c62adfe363d52f1ab Mon Sep 17 00:00:00 2001 From: vishal <1117327+vishalchangrani@users.noreply.github.com> Date: Mon, 9 Sep 2024 23:23:54 -0700 Subject: [PATCH 01/72] update to transit script pull command to download file if md5 of remote file is not known --- cmd/bootstrap/transit/cmd/pull.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/bootstrap/transit/cmd/pull.go b/cmd/bootstrap/transit/cmd/pull.go index a3a0c35eae7..cd5360fef83 100644 --- a/cmd/bootstrap/transit/cmd/pull.go +++ b/cmd/bootstrap/transit/cmd/pull.go @@ -97,7 +97,7 @@ func pull(cmd *cobra.Command, args []string) { fullOutpath := filepath.Join(flagBootDir, "public-root-information", filepath.Base(file.Name)) fmd5 := utils.CalcMd5(fullOutpath) // only skip files that have an MD5 hash - if file.MD5 != nil && bytes.Equal(fmd5, file.MD5) { + if len(file.MD5) > 0 && bytes.Equal(fmd5, file.MD5) { log.Info().Str("source", file.Name).Str("dest", fullOutpath).Msgf("skipping existing file from transit servers") return } From a4913f8ade1bc28eeb515e4247ed969d28fecd15 Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Tue, 17 Sep 2024 11:03:31 -0700 Subject: [PATCH 02/72] remove internal partners from root block creation --- cmd/bootstrap/cmd/rootblock.go | 6 +++++- cmd/bootstrap/cmd/rootblock_test.go | 3 +++ cmd/util/cmd/common/node_info.go | 16 ++++++++++++++++ 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/cmd/bootstrap/cmd/rootblock.go b/cmd/bootstrap/cmd/rootblock.go index 505d3239024..7d9f612ee10 100644 --- a/cmd/bootstrap/cmd/rootblock.go +++ b/cmd/bootstrap/cmd/rootblock.go @@ -146,7 +146,7 @@ func rootBlock(cmd *cobra.Command, args []string) { } log.Info().Msg("collecting partner network and staking keys") - partnerNodes, err := common.ReadFullPartnerNodeInfos(log, flagPartnerWeights, flagPartnerNodeInfoDir) + rawPartnerNodes, err := common.ReadFullPartnerNodeInfos(log, flagPartnerWeights, flagPartnerNodeInfoDir) if err != nil { log.Fatal().Err(err).Msg("failed to read full partner node infos") } @@ -160,6 +160,10 @@ func rootBlock(cmd *cobra.Command, args []string) { log.Info().Msg("") + log.Info().Msg("remove internal partner nodes") + partnerNodes := common.FilterInternalPartners(rawPartnerNodes, internalNodes) + log.Info().Msgf("removed %d internal partner nodes", len(rawPartnerNodes)-len(partnerNodes)) + log.Info().Msg("checking constraints on consensus nodes") checkConstraints(partnerNodes, internalNodes) log.Info().Msg("") diff --git a/cmd/bootstrap/cmd/rootblock_test.go b/cmd/bootstrap/cmd/rootblock_test.go index 01222c0c476..8c3080d0b11 100644 --- a/cmd/bootstrap/cmd/rootblock_test.go +++ b/cmd/bootstrap/cmd/rootblock_test.go @@ -21,7 +21,10 @@ const rootBlockHappyPathLogs = "collecting partner network and staking keys" + `read \d+ internal private node-info files` + `read internal node configurations` + `read \d+ weights for internal nodes` + + `remove internal partner nodes` + + `removed 0 internal partner nodes` + `checking constraints on consensus nodes` + + `assembling network and staking keys` + `wrote file \S+/node-infos.pub.json` + `running DKG for consensus nodes` + diff --git a/cmd/util/cmd/common/node_info.go b/cmd/util/cmd/common/node_info.go index 061741d0955..5f9ebb79cd4 100644 --- a/cmd/util/cmd/common/node_info.go +++ b/cmd/util/cmd/common/node_info.go @@ -224,3 +224,19 @@ func internalWeightsByAddress(log zerolog.Logger, config string) map[string]uint return weights } + +// Reject any partner nodes that are in the internal node list. +func FilterInternalPartners(partners []bootstrap.NodeInfo, internal []bootstrap.NodeInfo) []bootstrap.NodeInfo { + lookup := make(map[flow.Identifier]struct{}) + for _, node := range internal { + lookup[node.NodeID] = struct{}{} + } + + var filtered []bootstrap.NodeInfo + for _, node := range partners { + if _, ok := lookup[node.NodeID]; !ok { + filtered = append(filtered, node) + } + } + return filtered +} From 20982132258df82bc4f98e6909a804e06d12cdf6 Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Tue, 17 Sep 2024 13:06:27 -0700 Subject: [PATCH 03/72] print nodeID when missing partner info --- cmd/util/cmd/common/node_info.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/util/cmd/common/node_info.go b/cmd/util/cmd/common/node_info.go index 5f9ebb79cd4..6f2a93a4917 100644 --- a/cmd/util/cmd/common/node_info.go +++ b/cmd/util/cmd/common/node_info.go @@ -50,7 +50,7 @@ func ReadFullPartnerNodeInfos(log zerolog.Logger, partnerWeightsPath, partnerNod weight := weights[partner.NodeID] if valid := ValidateWeight(weight); !valid { - return nil, fmt.Errorf(fmt.Sprintf("invalid partner weight: %d", weight)) + return nil, fmt.Errorf(fmt.Sprintf("invalid partner weight %v: %d", partner.NodeID, weight)) } if weight != flow.DefaultInitialWeight { @@ -148,7 +148,7 @@ func ReadFullInternalNodeInfos(log zerolog.Logger, internalNodePrivInfoDir, inte weight := weights[internal.Address] if valid := ValidateWeight(weight); !valid { - return nil, fmt.Errorf(fmt.Sprintf("invalid partner weight: %d", weight)) + return nil, fmt.Errorf(fmt.Sprintf("invalid partner weight %v: %d", internal.NodeID, weight)) } if weight != flow.DefaultInitialWeight { log.Warn().Msgf("internal node (id=%x) has non-default weight (%d != %d)", internal.NodeID, weight, flow.DefaultInitialWeight) From f18670fb2873bbd48fef5b7a848c01a32d019eb3 Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Wed, 18 Sep 2024 10:05:16 -0700 Subject: [PATCH 04/72] include mismatch number in error message --- cmd/bootstrap/run/qc.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/bootstrap/run/qc.go b/cmd/bootstrap/run/qc.go index 4a31a125ff3..6f1c4dcdf29 100644 --- a/cmd/bootstrap/run/qc.go +++ b/cmd/bootstrap/run/qc.go @@ -169,7 +169,7 @@ func GenerateQCParticipantData(allNodes, internalNodes []bootstrap.NodeInfo, dkg // length of DKG participants needs to match stakingNodes, since we run DKG for external and internal validators if len(allNodes) != len(dkgData.PrivKeyShares) { - return nil, fmt.Errorf("need exactly the same number of staking public keys as DKG private participants") + return nil, fmt.Errorf("need exactly the same number of staking public keys as DKG private participants, (all=%d, dkg=%d)", len(allNodes), len(dkgData.PrivKeyShares)) } qcData := &ParticipantData{} From 4cca67be442cc51c0db9ce9158cc5bbea659d760 Mon Sep 17 00:00:00 2001 From: ramtinms Date: Tue, 17 Sep 2024 09:46:07 -0700 Subject: [PATCH 05/72] remove unsupported checks --- fvm/evm/emulator/emulator_test.go | 24 +++++------------------- 1 file changed, 5 insertions(+), 19 deletions(-) diff --git a/fvm/evm/emulator/emulator_test.go b/fvm/evm/emulator/emulator_test.go index 4e268a28902..3ac96e9e7df 100644 --- a/fvm/evm/emulator/emulator_test.go +++ b/fvm/evm/emulator/emulator_test.go @@ -1194,7 +1194,6 @@ func TestTransactionTracing(t *testing.T) { blk, uploader, tracer := blockWithTracer(t, emu) var txID gethCommon.Hash - var trace json.RawMessage blockID := flow.Identifier{0x01} uploaded := make(chan struct{}) @@ -1202,7 +1201,7 @@ func TestTransactionTracing(t *testing.T) { uploader.UploadFunc = func(id string, message json.RawMessage) error { uploaded <- struct{}{} require.Equal(t, debug.TraceID(txID, blockID), id) - require.Equal(t, trace, message) + require.NotEmpty(t, message) require.Greater(t, len(message), 0) return nil } @@ -1225,8 +1224,6 @@ func TestTransactionTracing(t *testing.T) { require.NoError(t, res.VMError) txID = res.TxHash tracer.Collect(txID) - trace = tracer.GetResultByTxHash(txID) - require.NotEmpty(t, trace) testAccount.SetNonce(testAccount.Nonce() + 1) require.Eventuallyf(t, func() bool { @@ -1241,7 +1238,6 @@ func TestTransactionTracing(t *testing.T) { blk, uploader, tracer := blockWithTracer(t, emu) var txID gethCommon.Hash - var trace json.RawMessage uploaded := make(chan struct{}) blockID := flow.Identifier{0x01} @@ -1249,7 +1245,7 @@ func TestTransactionTracing(t *testing.T) { uploader.UploadFunc = func(id string, message json.RawMessage) error { uploaded <- struct{}{} require.Equal(t, debug.TraceID(txID, blockID), id) - require.Equal(t, trace, message) + require.NotEmpty(t, message) require.Greater(t, len(message), 0) return nil } @@ -1269,8 +1265,6 @@ func TestTransactionTracing(t *testing.T) { txID = res.TxHash tracer.WithBlockID(blockID) tracer.Collect(txID) - trace = tracer.GetResultByTxHash(txID) - require.NotEmpty(t, trace) testAccount.SetNonce(testAccount.Nonce() + 1) require.Eventuallyf(t, func() bool { @@ -1285,7 +1279,6 @@ func TestTransactionTracing(t *testing.T) { blk, uploader, tracer := blockWithTracer(t, emu) var txID gethCommon.Hash - var trace json.RawMessage blockID := flow.Identifier{0x02} uploaded := make(chan struct{}) @@ -1293,7 +1286,7 @@ func TestTransactionTracing(t *testing.T) { uploader.UploadFunc = func(id string, message json.RawMessage) error { uploaded <- struct{}{} require.Equal(t, debug.TraceID(txID, blockID), id) - require.Equal(t, trace, message) + require.NotEmpty(t, message) require.Greater(t, len(message), 0) return nil } @@ -1313,8 +1306,6 @@ func TestTransactionTracing(t *testing.T) { txID = res.TxHash tracer.WithBlockID(blockID) tracer.Collect(txID) - trace = tracer.GetResultByTxHash(txID) - require.NotEmpty(t, trace) testAccount.SetNonce(testAccount.Nonce() + 1) require.Eventuallyf(t, func() bool { @@ -1362,7 +1353,6 @@ func TestTransactionTracing(t *testing.T) { blk, uploader, tracer := blockWithTracer(t, emu) var txID gethCommon.Hash - var trace json.RawMessage blockID := flow.Identifier{0x02} uploaded := make(chan struct{}) @@ -1370,7 +1360,7 @@ func TestTransactionTracing(t *testing.T) { uploader.UploadFunc = func(id string, message json.RawMessage) error { uploaded <- struct{}{} require.Equal(t, debug.TraceID(txID, blockID), id) - require.Equal(t, trace, message) + require.NotEmpty(t, message) require.Greater(t, len(message), 0) return nil } @@ -1391,8 +1381,6 @@ func TestTransactionTracing(t *testing.T) { txID = res.TxHash tracer.WithBlockID(blockID) tracer.Collect(txID) - trace = tracer.GetResultByTxHash(txID) - require.NotEmpty(t, trace) testAccount.SetNonce(testAccount.Nonce() + 1) require.Eventuallyf(t, func() bool { @@ -1455,7 +1443,6 @@ func TestTransactionTracing(t *testing.T) { blk, uploader, tracer := blockWithTracer(t, emu) var txID gethCommon.Hash - var trace json.RawMessage blockID := flow.Identifier{0x02} uploaded := make(chan struct{}) @@ -1463,7 +1450,7 @@ func TestTransactionTracing(t *testing.T) { uploader.UploadFunc = func(id string, message json.RawMessage) error { uploaded <- struct{}{} require.Equal(t, debug.TraceID(txID, blockID), id) - require.Equal(t, trace, message) + require.NotEmpty(t, message) require.Greater(t, len(message), 0) return nil } @@ -1484,7 +1471,6 @@ func TestTransactionTracing(t *testing.T) { txID = results[0].TxHash tracer.WithBlockID(blockID) tracer.Collect(txID) - trace = tracer.GetResultByTxHash(txID) require.Eventuallyf(t, func() bool { <-uploaded From a2c95f8e92b410d190441f7b5a6062fae8ae1d72 Mon Sep 17 00:00:00 2001 From: ramtinms Date: Tue, 17 Sep 2024 10:45:17 -0700 Subject: [PATCH 06/72] fix tests --- fvm/evm/emulator/emulator_test.go | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/fvm/evm/emulator/emulator_test.go b/fvm/evm/emulator/emulator_test.go index 3ac96e9e7df..a89ca945c1a 100644 --- a/fvm/evm/emulator/emulator_test.go +++ b/fvm/evm/emulator/emulator_test.go @@ -1103,7 +1103,8 @@ func TestCallingExtraPrecompiles(t *testing.T) { AddressFunc: func() types.Address { return addr }, - RequiredGasFunc: func(input []byte) uint64 { + RequiredGasFunc: func(inp []byte) uint64 { + require.Equal(t, input, inp) return uint64(10) }, RunFunc: func(inp []byte) ([]byte, error) { @@ -1194,6 +1195,7 @@ func TestTransactionTracing(t *testing.T) { blk, uploader, tracer := blockWithTracer(t, emu) var txID gethCommon.Hash + var trace json.RawMessage blockID := flow.Identifier{0x01} uploaded := make(chan struct{}) @@ -1201,7 +1203,7 @@ func TestTransactionTracing(t *testing.T) { uploader.UploadFunc = func(id string, message json.RawMessage) error { uploaded <- struct{}{} require.Equal(t, debug.TraceID(txID, blockID), id) - require.NotEmpty(t, message) + require.Equal(t, trace, message) require.Greater(t, len(message), 0) return nil } @@ -1223,6 +1225,8 @@ func TestTransactionTracing(t *testing.T) { require.NoError(t, res.ValidationError) require.NoError(t, res.VMError) txID = res.TxHash + trace = tracer.GetResultByTxHash(txID) + require.NotEmpty(t, trace) tracer.Collect(txID) testAccount.SetNonce(testAccount.Nonce() + 1) @@ -1238,6 +1242,7 @@ func TestTransactionTracing(t *testing.T) { blk, uploader, tracer := blockWithTracer(t, emu) var txID gethCommon.Hash + var trace json.RawMessage uploaded := make(chan struct{}) blockID := flow.Identifier{0x01} @@ -1245,7 +1250,7 @@ func TestTransactionTracing(t *testing.T) { uploader.UploadFunc = func(id string, message json.RawMessage) error { uploaded <- struct{}{} require.Equal(t, debug.TraceID(txID, blockID), id) - require.NotEmpty(t, message) + require.Equal(t, trace, message) require.Greater(t, len(message), 0) return nil } @@ -1263,6 +1268,8 @@ func TestTransactionTracing(t *testing.T) { ) requireSuccessfulExecution(t, err, res) txID = res.TxHash + trace = tracer.GetResultByTxHash(txID) + require.NotEmpty(t, trace) tracer.WithBlockID(blockID) tracer.Collect(txID) @@ -1279,6 +1286,7 @@ func TestTransactionTracing(t *testing.T) { blk, uploader, tracer := blockWithTracer(t, emu) var txID gethCommon.Hash + var trace json.RawMessage blockID := flow.Identifier{0x02} uploaded := make(chan struct{}) @@ -1286,7 +1294,7 @@ func TestTransactionTracing(t *testing.T) { uploader.UploadFunc = func(id string, message json.RawMessage) error { uploaded <- struct{}{} require.Equal(t, debug.TraceID(txID, blockID), id) - require.NotEmpty(t, message) + require.Equal(t, trace, message) require.Greater(t, len(message), 0) return nil } @@ -1304,6 +1312,8 @@ func TestTransactionTracing(t *testing.T) { res, err := blk.RunTransaction(tx) requireSuccessfulExecution(t, err, res) txID = res.TxHash + trace = tracer.GetResultByTxHash(txID) + require.NotEmpty(t, trace) tracer.WithBlockID(blockID) tracer.Collect(txID) @@ -1353,6 +1363,7 @@ func TestTransactionTracing(t *testing.T) { blk, uploader, tracer := blockWithTracer(t, emu) var txID gethCommon.Hash + var trace json.RawMessage blockID := flow.Identifier{0x02} uploaded := make(chan struct{}) @@ -1360,7 +1371,7 @@ func TestTransactionTracing(t *testing.T) { uploader.UploadFunc = func(id string, message json.RawMessage) error { uploaded <- struct{}{} require.Equal(t, debug.TraceID(txID, blockID), id) - require.NotEmpty(t, message) + require.Equal(t, trace, message) require.Greater(t, len(message), 0) return nil } @@ -1380,6 +1391,8 @@ func TestTransactionTracing(t *testing.T) { requireSuccessfulExecution(t, err, res) txID = res.TxHash tracer.WithBlockID(blockID) + trace = tracer.GetResultByTxHash(txID) + require.NotEmpty(t, trace) tracer.Collect(txID) testAccount.SetNonce(testAccount.Nonce() + 1) @@ -1443,6 +1456,7 @@ func TestTransactionTracing(t *testing.T) { blk, uploader, tracer := blockWithTracer(t, emu) var txID gethCommon.Hash + var trace json.RawMessage blockID := flow.Identifier{0x02} uploaded := make(chan struct{}) @@ -1450,7 +1464,7 @@ func TestTransactionTracing(t *testing.T) { uploader.UploadFunc = func(id string, message json.RawMessage) error { uploaded <- struct{}{} require.Equal(t, debug.TraceID(txID, blockID), id) - require.NotEmpty(t, message) + require.Equal(t, trace, message) require.Greater(t, len(message), 0) return nil } @@ -1469,6 +1483,7 @@ func TestTransactionTracing(t *testing.T) { require.NoError(t, err) require.Len(t, results, 1) txID = results[0].TxHash + trace = tracer.GetResultByTxHash(txID) tracer.WithBlockID(blockID) tracer.Collect(txID) From f581a2080507b3ee2762e657c71dccbfe0698a1e Mon Sep 17 00:00:00 2001 From: ramtinms Date: Mon, 16 Sep 2024 22:13:04 -0700 Subject: [PATCH 07/72] move map update outside of concurrent path --- fvm/evm/debug/tracer.go | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/fvm/evm/debug/tracer.go b/fvm/evm/debug/tracer.go index 0a047aed3df..52b4a7e5236 100644 --- a/fvm/evm/debug/tracer.go +++ b/fvm/evm/debug/tracer.go @@ -76,16 +76,25 @@ func (t *CallTracer) GetResultByTxHash(txID gethCommon.Hash) json.RawMessage { } func (t *CallTracer) Collect(txID gethCommon.Hash) { + l := t.logger.With(). + Str("tx-id", txID.String()). + Str("block-id", t.blockID.String()). + Logger() + + // collect the trace result + res, found := t.resultsByTxID[txID] + if !found { + l.Error().Msg("trace result not found") + return + } + // remove the result + delete(t.resultsByTxID, txID) + // upload is concurrent and it doesn't produce any errors, as the // client doesn't expect it, we don't want to break execution flow, // in case there are errors we retry, and if we fail after retries // we log them and continue. go func() { - l := t.logger.With(). - Str("tx-id", txID.String()). - Str("block-id", t.blockID.String()). - Logger() - defer func() { if r := recover(); r != nil { err, ok := r.(error) @@ -98,20 +107,12 @@ func (t *CallTracer) Collect(txID gethCommon.Hash) { Msg("failed to collect EVM traces") } }() - - res, found := t.resultsByTxID[txID] - if !found { - l.Error().Msg("trace result not found") - return - } if err := t.uploader.Upload(TraceID(txID, t.blockID), res); err != nil { l.Error().Err(err). Str("traces", string(res)). Msg("failed to upload trace results, no more retries") return } - // remove the result - delete(t.resultsByTxID, txID) l.Debug().Msg("evm traces uploaded successfully") }() From b6fce94ca5b1f98d40541ebc68647cdfe1d7cf01 Mon Sep 17 00:00:00 2001 From: Vishal <1117327+vishalchangrani@users.noreply.github.com> Date: Wed, 18 Sep 2024 16:16:31 -0700 Subject: [PATCH 08/72] switching off peer-scoring by default (#6473) Co-authored-by: Khalil Claybon --- config/default-config.yml | 2 +- .../rpc_inspector/validation_inspector_test.go | 10 ++++++++++ .../test/gossipsub/scoring/scoring_test.go | 1 + integration/tests/bft/base_suite.go | 16 ++++++++++------ network/p2p/scoring/scoring_test.go | 1 + 5 files changed, 23 insertions(+), 7 deletions(-) diff --git a/config/default-config.yml b/config/default-config.yml index 2e8422b4529..b50804a0230 100644 --- a/config/default-config.yml +++ b/config/default-config.yml @@ -277,7 +277,7 @@ network-config: # instead, the counter will be set to 0. This is to prevent the counter from becoming a large number over time. skip-decay-threshold: 0.1 # Peer scoring is the default value for enabling peer scoring - peer-scoring-enabled: true + peer-scoring-enabled: false scoring-parameters: peer-scoring: internal: diff --git a/insecure/integration/functional/test/gossipsub/rpc_inspector/validation_inspector_test.go b/insecure/integration/functional/test/gossipsub/rpc_inspector/validation_inspector_test.go index 1c43a9999e7..6237a12d81a 100644 --- a/insecure/integration/functional/test/gossipsub/rpc_inspector/validation_inspector_test.go +++ b/insecure/integration/functional/test/gossipsub/rpc_inspector/validation_inspector_test.go @@ -44,6 +44,7 @@ func TestValidationInspector_InvalidTopicId_Detection(t *testing.T) { sporkID := unittest.IdentifierFixture() flowConfig, err := config.DefaultConfig() require.NoError(t, err) + flowConfig.NetworkConfig.GossipSub.PeerScoringEnabled = true // default is false inspectorConfig := flowConfig.NetworkConfig.GossipSub.RpcInspector.Validation messageCount := 100 @@ -173,6 +174,7 @@ func TestValidationInspector_DuplicateTopicId_Detection(t *testing.T) { sporkID := unittest.IdentifierFixture() flowConfig, err := config.DefaultConfig() require.NoError(t, err) + flowConfig.NetworkConfig.GossipSub.PeerScoringEnabled = true // default is false inspectorConfig := flowConfig.NetworkConfig.GossipSub.RpcInspector.Validation inspectorConfig.InspectionQueue.NumberOfWorkers = 1 @@ -282,6 +284,7 @@ func TestValidationInspector_IHaveDuplicateMessageId_Detection(t *testing.T) { sporkID := unittest.IdentifierFixture() flowConfig, err := config.DefaultConfig() require.NoError(t, err) + flowConfig.NetworkConfig.GossipSub.PeerScoringEnabled = true // default is false inspectorConfig := flowConfig.NetworkConfig.GossipSub.RpcInspector.Validation inspectorConfig.InspectionQueue.NumberOfWorkers = 1 @@ -379,6 +382,7 @@ func TestValidationInspector_UnknownClusterId_Detection(t *testing.T) { sporkID := unittest.IdentifierFixture() flowConfig, err := config.DefaultConfig() require.NoError(t, err) + flowConfig.NetworkConfig.GossipSub.PeerScoringEnabled = true // default is false inspectorConfig := flowConfig.NetworkConfig.GossipSub.RpcInspector.Validation // set hard threshold to 0 so that in the case of invalid cluster ID // we force the inspector to return an error @@ -489,6 +493,7 @@ func TestValidationInspector_ActiveClusterIdsNotSet_Graft_Detection(t *testing.T sporkID := unittest.IdentifierFixture() flowConfig, err := config.DefaultConfig() require.NoError(t, err) + flowConfig.NetworkConfig.GossipSub.PeerScoringEnabled = true // default is false inspectorConfig := flowConfig.NetworkConfig.GossipSub.RpcInspector.Validation inspectorConfig.GraftPrune.InvalidTopicIdThreshold = 0 inspectorConfig.ClusterPrefixedMessage.HardThreshold = 5 @@ -570,6 +575,7 @@ func TestValidationInspector_ActiveClusterIdsNotSet_Prune_Detection(t *testing.T sporkID := unittest.IdentifierFixture() flowConfig, err := config.DefaultConfig() require.NoError(t, err) + flowConfig.NetworkConfig.GossipSub.PeerScoringEnabled = true // default is false inspectorConfig := flowConfig.NetworkConfig.GossipSub.RpcInspector.Validation inspectorConfig.GraftPrune.InvalidTopicIdThreshold = 0 inspectorConfig.ClusterPrefixedMessage.HardThreshold = 5 @@ -649,6 +655,7 @@ func TestValidationInspector_UnstakedNode_Detection(t *testing.T) { sporkID := unittest.IdentifierFixture() flowConfig, err := config.DefaultConfig() require.NoError(t, err) + flowConfig.NetworkConfig.GossipSub.PeerScoringEnabled = true // default is false inspectorConfig := flowConfig.NetworkConfig.GossipSub.RpcInspector.Validation inspectorConfig.InspectionQueue.NumberOfWorkers = 1 controlMessageCount := int64(1) @@ -741,6 +748,7 @@ func TestValidationInspector_InspectIWants_CacheMissThreshold(t *testing.T) { // create our RPC validation inspector flowConfig, err := config.DefaultConfig() require.NoError(t, err) + flowConfig.NetworkConfig.GossipSub.PeerScoringEnabled = true // default is false inspectorConfig := flowConfig.NetworkConfig.GossipSub.RpcInspector.Validation inspectorConfig.InspectionQueue.NumberOfWorkers = 1 inspectorConfig.IWant.CacheMissThreshold = 10 @@ -836,6 +844,7 @@ func TestValidationInspector_InspectRpcPublishMessages(t *testing.T) { // create our RPC validation inspector flowConfig, err := config.DefaultConfig() require.NoError(t, err) + flowConfig.NetworkConfig.GossipSub.PeerScoringEnabled = true // default is false inspectorConfig := flowConfig.NetworkConfig.GossipSub.RpcInspector.Validation inspectorConfig.InspectionQueue.NumberOfWorkers = 1 @@ -987,6 +996,7 @@ func testGossipSubSpamMitigationIntegration(t *testing.T, msgType p2pmsg.Control cfg, err := config.DefaultConfig() require.NoError(t, err) + cfg.NetworkConfig.GossipSub.PeerScoringEnabled = true // default is false // set the scoring parameters to be more aggressive to speed up the test cfg.NetworkConfig.GossipSub.RpcTracer.ScoreTracerInterval = 100 * time.Millisecond cfg.NetworkConfig.GossipSub.ScoringParameters.ScoringRegistryParameters.AppSpecificScore.ScoreTTL = 100 * time.Millisecond diff --git a/insecure/integration/functional/test/gossipsub/scoring/scoring_test.go b/insecure/integration/functional/test/gossipsub/scoring/scoring_test.go index 184f365f60c..54af4740ede 100644 --- a/insecure/integration/functional/test/gossipsub/scoring/scoring_test.go +++ b/insecure/integration/functional/test/gossipsub/scoring/scoring_test.go @@ -105,6 +105,7 @@ func testGossipSubInvalidMessageDeliveryScoring(t *testing.T, spamMsgFactory fun cfg, err := config.DefaultConfig() require.NoError(t, err) + cfg.NetworkConfig.GossipSub.PeerScoringEnabled = true // default is false // we override the decay interval to 1 second so that the score is updated within 1 second intervals. cfg.NetworkConfig.GossipSub.RpcTracer.ScoreTracerInterval = 1 * time.Second cfg.NetworkConfig.GossipSub.ScoringParameters.PeerScoring.Internal.TopicParameters.InvalidMessageDeliveriesDecay = .99 diff --git a/integration/tests/bft/base_suite.go b/integration/tests/bft/base_suite.go index 2e6e74de881..8f897c5fd8e 100644 --- a/integration/tests/bft/base_suite.go +++ b/integration/tests/bft/base_suite.go @@ -67,7 +67,9 @@ func (b *BaseSuite) SetupSuite() { // setup single access node b.NodeConfigs = append(b.NodeConfigs, - testnet.NewNodeConfig(flow.RoleAccess, testnet.WithLogLevel(zerolog.FatalLevel)), + testnet.NewNodeConfig(flow.RoleAccess, + testnet.WithLogLevel(zerolog.FatalLevel), + testnet.WithAdditionalFlag("--gossipsub-peer-scoring-enabled=true")), // currently, peer scoring is off by default ) // setup consensus nodes @@ -83,25 +85,27 @@ func (b *BaseSuite) SetupSuite() { // TODO: fix the access integration test logic to function without slowing down // the block rate testnet.WithAdditionalFlag("--cruise-ctl-fallback-proposal-duration=250ms"), + testnet.WithAdditionalFlag("--gossipsub-peer-scoring-enabled=true"), ) b.NodeConfigs = append(b.NodeConfigs, nodeConfig) } // setup single verification node b.NodeConfigs = append(b.NodeConfigs, - testnet.NewNodeConfig(flow.RoleVerification, testnet.WithLogLevel(zerolog.FatalLevel)), + testnet.NewNodeConfig(flow.RoleVerification, testnet.WithLogLevel(zerolog.FatalLevel), + testnet.WithAdditionalFlag("--gossipsub-peer-scoring-enabled=true")), ) // setup execution nodes b.NodeConfigs = append(b.NodeConfigs, - testnet.NewNodeConfig(flow.RoleExecution, testnet.WithLogLevel(zerolog.FatalLevel)), - testnet.NewNodeConfig(flow.RoleExecution, testnet.WithLogLevel(zerolog.FatalLevel)), + testnet.NewNodeConfig(flow.RoleExecution, testnet.WithLogLevel(zerolog.FatalLevel), testnet.WithAdditionalFlag("--gossipsub-peer-scoring-enabled=true")), + testnet.NewNodeConfig(flow.RoleExecution, testnet.WithLogLevel(zerolog.FatalLevel), testnet.WithAdditionalFlag("--gossipsub-peer-scoring-enabled=true")), ) // setup collection nodes b.NodeConfigs = append(b.NodeConfigs, - testnet.NewNodeConfig(flow.RoleCollection, testnet.WithLogLevel(zerolog.FatalLevel), testnet.WithAdditionalFlag("--hotstuff-proposal-duration=1ms")), - testnet.NewNodeConfig(flow.RoleCollection, testnet.WithLogLevel(zerolog.FatalLevel), testnet.WithAdditionalFlag("--hotstuff-proposal-duration=1ms")), + testnet.NewNodeConfig(flow.RoleCollection, testnet.WithLogLevel(zerolog.FatalLevel), testnet.WithAdditionalFlag("--hotstuff-proposal-duration=1ms"), testnet.WithAdditionalFlag("--gossipsub-peer-scoring-enabled=true")), + testnet.NewNodeConfig(flow.RoleCollection, testnet.WithLogLevel(zerolog.FatalLevel), testnet.WithAdditionalFlag("--hotstuff-proposal-duration=1ms"), testnet.WithAdditionalFlag("--gossipsub-peer-scoring-enabled=true")), ) // Ghost Node diff --git a/network/p2p/scoring/scoring_test.go b/network/p2p/scoring/scoring_test.go index f448cd271bd..d37c7864173 100644 --- a/network/p2p/scoring/scoring_test.go +++ b/network/p2p/scoring/scoring_test.go @@ -43,6 +43,7 @@ func TestInvalidCtrlMsgScoringIntegration(t *testing.T) { cfg, err := config.DefaultConfig() require.NoError(t, err) + cfg.NetworkConfig.GossipSub.PeerScoringEnabled = true // default is false cfg.NetworkConfig.GossipSub.ScoringParameters.ScoringRegistryParameters.AppSpecificScore.ScoreTTL = 10 * time.Millisecond // speed up the test var notificationConsumer p2p.GossipSubInvCtrlMsgNotifConsumer From a80a13987664ac4c9400cf30254b83957035c90e Mon Sep 17 00:00:00 2001 From: ramtinms Date: Thu, 19 Sep 2024 13:16:42 -0700 Subject: [PATCH 09/72] Handle UInt,Int types for ABI encoding/decoding --- fvm/evm/impl/abi.go | 44 ++++++++++++ fvm/evm/stdlib/contract_test.go | 115 +++++++++++++++++++++++++++++++- 2 files changed, 157 insertions(+), 2 deletions(-) diff --git a/fvm/evm/impl/abi.go b/fvm/evm/impl/abi.go index 812e00ba2cb..5b88da53845 100644 --- a/fvm/evm/impl/abi.go +++ b/fvm/evm/impl/abi.go @@ -95,12 +95,14 @@ func reportABIEncodingComputation( reportComputation(computation) case interpreter.BoolValue, + interpreter.UIntValue, interpreter.UInt8Value, interpreter.UInt16Value, interpreter.UInt32Value, interpreter.UInt64Value, interpreter.UInt128Value, interpreter.UInt256Value, + interpreter.IntValue, interpreter.Int8Value, interpreter.Int16Value, interpreter.Int32Value, @@ -227,6 +229,8 @@ var gethTypeString = gethABI.Type{T: gethABI.StringTy} var gethTypeBool = gethABI.Type{T: gethABI.BoolTy} +var gethTypeUint = gethABI.Type{T: gethABI.UintTy, Size: 256} + var gethTypeUint8 = gethABI.Type{T: gethABI.UintTy, Size: 8} var gethTypeUint16 = gethABI.Type{T: gethABI.UintTy, Size: 16} @@ -239,6 +243,8 @@ var gethTypeUint128 = gethABI.Type{T: gethABI.UintTy, Size: 128} var gethTypeUint256 = gethABI.Type{T: gethABI.UintTy, Size: 256} +var gethTypeInt = gethABI.Type{T: gethABI.IntTy, Size: 256} + var gethTypeInt8 = gethABI.Type{T: gethABI.IntTy, Size: 8} var gethTypeInt16 = gethABI.Type{T: gethABI.IntTy, Size: 16} @@ -259,6 +265,8 @@ func gethABIType(staticType interpreter.StaticType, evmAddressTypeID common.Type return gethTypeString, true case interpreter.PrimitiveStaticTypeBool: return gethTypeBool, true + case interpreter.PrimitiveStaticTypeUInt: + return gethTypeUint, true case interpreter.PrimitiveStaticTypeUInt8: return gethTypeUint8, true case interpreter.PrimitiveStaticTypeUInt16: @@ -271,6 +279,8 @@ func gethABIType(staticType interpreter.StaticType, evmAddressTypeID common.Type return gethTypeUint128, true case interpreter.PrimitiveStaticTypeUInt256: return gethTypeUint256, true + case interpreter.PrimitiveStaticTypeInt: + return gethTypeInt, true case interpreter.PrimitiveStaticTypeInt8: return gethTypeInt8, true case interpreter.PrimitiveStaticTypeInt16: @@ -338,6 +348,8 @@ func goType( return reflect.TypeOf(""), true case interpreter.PrimitiveStaticTypeBool: return reflect.TypeOf(true), true + case interpreter.PrimitiveStaticTypeUInt: + return reflect.TypeOf((*big.Int)(nil)), true case interpreter.PrimitiveStaticTypeUInt8: return reflect.TypeOf(uint8(0)), true case interpreter.PrimitiveStaticTypeUInt16: @@ -350,6 +362,8 @@ func goType( return reflect.TypeOf((*big.Int)(nil)), true case interpreter.PrimitiveStaticTypeUInt256: return reflect.TypeOf((*big.Int)(nil)), true + case interpreter.PrimitiveStaticTypeInt: + return reflect.TypeOf((*big.Int)(nil)), true case interpreter.PrimitiveStaticTypeInt8: return reflect.TypeOf(int8(0)), true case interpreter.PrimitiveStaticTypeInt16: @@ -414,6 +428,11 @@ func encodeABI( return bool(value), gethTypeBool, nil } + case interpreter.UIntValue: + if staticType == interpreter.PrimitiveStaticTypeUInt { + return value.BigInt, gethTypeUint, nil + } + case interpreter.UInt8Value: if staticType == interpreter.PrimitiveStaticTypeUInt8 { return uint8(value), gethTypeUint8, nil @@ -444,6 +463,11 @@ func encodeABI( return value.BigInt, gethTypeUint256, nil } + case interpreter.IntValue: + if staticType == interpreter.PrimitiveStaticTypeInt { + return value.BigInt, gethTypeInt, nil + } + case interpreter.Int8Value: if staticType == interpreter.PrimitiveStaticTypeInt8 { return int8(value), gethTypeInt8, nil @@ -712,6 +736,16 @@ func decodeABI( } return interpreter.BoolValue(value), nil + case interpreter.PrimitiveStaticTypeUInt: + value, ok := value.(*big.Int) + if !ok { + break + } + memoryUsage := common.NewBigIntMemoryUsage( + common.BigIntByteLength(value), + ) + return interpreter.NewUIntValueFromBigInt(inter, memoryUsage, func() *big.Int { return value }), nil + case interpreter.PrimitiveStaticTypeUInt8: value, ok := value.(uint8) if !ok { @@ -754,6 +788,16 @@ func decodeABI( } return interpreter.NewUInt256ValueFromBigInt(inter, func() *big.Int { return value }), nil + case interpreter.PrimitiveStaticTypeInt: + value, ok := value.(*big.Int) + if !ok { + break + } + memoryUsage := common.NewBigIntMemoryUsage( + common.BigIntByteLength(value), + ) + return interpreter.NewIntValueFromBigInt(inter, memoryUsage, func() *big.Int { return value }), nil + case interpreter.PrimitiveStaticTypeInt8: value, ok := value.(int8) if !ok { diff --git a/fvm/evm/stdlib/contract_test.go b/fvm/evm/stdlib/contract_test.go index 88f05e63983..7156cc44be3 100644 --- a/fvm/evm/stdlib/contract_test.go +++ b/fvm/evm/stdlib/contract_test.go @@ -989,6 +989,111 @@ func TestEVMDecodeABIComputation(t *testing.T) { assert.Equal(t, computation, uint(len(cdcBytes.Values))) } +func TestEVMEncodeDecodeABIRoundtripForUintIntTypes(t *testing.T) { + + t.Parallel() + + handler := &testContractHandler{} + + contractsAddress := flow.BytesToAddress([]byte{0x1}) + + transactionEnvironment := newEVMTransactionEnvironment(handler, contractsAddress) + scriptEnvironment := newEVMScriptEnvironment(handler, contractsAddress) + + rt := runtime.NewInterpreterRuntime(runtime.Config{}) + + script := []byte(` + import EVM from 0x1 + + access(all) + fun main(): Bool { + // Check UInt*/Int* encode/decode + let amount: UInt256 = 18446744073709551615 + let minBalance: Int256 = -18446744073709551615 + let data = EVM.encodeABIWithSignature( + "withdraw(uint,int)", + [UInt(amount), Int(minBalance)] + ) + let values = EVM.decodeABIWithSignature( + "withdraw(uint,int)", + types: [Type(), Type()], + data: data + ) + assert((values[0] as! UInt) == UInt(amount)) + assert((values[1] as! Int) == Int(minBalance)) + + return true + } + `) + + accountCodes := map[common.Location][]byte{} + var events []cadence.Event + + runtimeInterface := &TestRuntimeInterface{ + Storage: NewTestLedger(nil, nil), + OnGetSigningAccounts: func() ([]runtime.Address, error) { + return []runtime.Address{runtime.Address(contractsAddress)}, nil + }, + OnResolveLocation: LocationResolver, + OnUpdateAccountContractCode: func(location common.AddressLocation, code []byte) error { + accountCodes[location] = code + return nil + }, + OnGetAccountContractCode: func(location common.AddressLocation) (code []byte, err error) { + code = accountCodes[location] + return code, nil + }, + OnEmitEvent: func(event cadence.Event) error { + events = append(events, event) + return nil + }, + OnDecodeArgument: func(b []byte, t cadence.Type) (cadence.Value, error) { + return json.Decode(nil, b) + }, + OnHash: func( + data []byte, + tag string, + hashAlgorithm runtime.HashAlgorithm, + ) ([]byte, error) { + return crypto.Keccak256(data), nil + }, + } + + nextTransactionLocation := NewTransactionLocationGenerator() + nextScriptLocation := NewScriptLocationGenerator() + + // Deploy contracts + + deployContracts( + t, + rt, + contractsAddress, + runtimeInterface, + transactionEnvironment, + nextTransactionLocation, + ) + + // Run script + + result, err := rt.ExecuteScript( + runtime.Script{ + Source: script, + Arguments: [][]byte{}, + }, + runtime.Context{ + Interface: runtimeInterface, + Environment: scriptEnvironment, + Location: nextScriptLocation(), + }, + ) + require.NoError(t, err) + + assert.Equal(t, + cadence.Bool(true), + result, + ) +} + func TestEVMEncodeDecodeABIRoundtrip(t *testing.T) { t.Parallel() @@ -1043,7 +1148,9 @@ func TestEVMEncodeDecodeABIRoundtrip(t *testing.T) { Int32(-33), Int64(-33), Int128(-33), - Int256(-33) + Int256(-33), + UInt(33), + Int(-33) ]) values = EVM.decodeABI( types: [ @@ -1058,7 +1165,9 @@ func TestEVMEncodeDecodeABIRoundtrip(t *testing.T) { Type(), Type(), Type(), - Type() + Type(), + Type(), + Type() ], data: data ) @@ -1074,6 +1183,8 @@ func TestEVMEncodeDecodeABIRoundtrip(t *testing.T) { assert((values[9] as! Int64) == -33) assert((values[10] as! Int128) == -33) assert((values[11] as! Int256) == -33) + assert((values[12] as! UInt) == 33) + assert((values[13] as! Int) == -33) // Check variable-size array of leaf types encode/decode data = EVM.encodeABI([ From c245fd9d047fa9d53d54e7362200fe9f042a3a20 Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Thu, 19 Sep 2024 14:40:16 -0700 Subject: [PATCH 10/72] fix root block finalization --- cmd/bootstrap/cmd/final_list.go | 7 +++++-- cmd/bootstrap/cmd/finalize.go | 15 ++++++++++----- cmd/bootstrap/cmd/finalize_test.go | 27 +++++++++++++++++++++++++++ cmd/bootstrap/cmd/rootblock.go | 5 ++++- 4 files changed, 46 insertions(+), 8 deletions(-) diff --git a/cmd/bootstrap/cmd/final_list.go b/cmd/bootstrap/cmd/final_list.go index ca34739de2a..cc41c741881 100644 --- a/cmd/bootstrap/cmd/final_list.go +++ b/cmd/bootstrap/cmd/final_list.go @@ -62,13 +62,16 @@ func finalList(cmd *cobra.Command, args []string) { registeredNodes := readStakingContractDetails() // merge internal and partner node infos (from local files) - localNodes := mergeNodeInfos(internalNodes, partnerNodes) + localNodes, err := mergeNodeInfos(internalNodes, partnerNodes) + if err != nil { + log.Fatal().Err(err).Msg("failed to merge node infos") + } // reconcile nodes from staking contract nodes validateNodes(localNodes, registeredNodes) // write node-config.json with the new list of nodes to be used for the `finalize` command - err := common.WriteJSON(model.PathFinallist, flagOutdir, model.ToPublicNodeInfoList(localNodes)) + err = common.WriteJSON(model.PathFinallist, flagOutdir, model.ToPublicNodeInfoList(localNodes)) if err != nil { log.Fatal().Err(err).Msg("failed to write json") } diff --git a/cmd/bootstrap/cmd/finalize.go b/cmd/bootstrap/cmd/finalize.go index 0578a3dcebc..b935a2ac86f 100644 --- a/cmd/bootstrap/cmd/finalize.go +++ b/cmd/bootstrap/cmd/finalize.go @@ -132,7 +132,10 @@ func finalize(cmd *cobra.Command, args []string) { log.Info().Msg("") log.Info().Msg("assembling network and staking keys") - stakingNodes := mergeNodeInfos(internalNodes, partnerNodes) + stakingNodes, err := mergeNodeInfos(internalNodes, partnerNodes) + if err != nil { + log.Fatal().Err(err).Msgf("failed to merge internal and partner nodes: %v", err) + } log.Info().Msg("") // create flow.IdentityList representation of participant set @@ -312,29 +315,31 @@ func readRootBlockVotes() []*hotstuff.Vote { // // IMPORTANT: node infos are returned in the canonical ordering, meaning this // is safe to use as the input to the DKG and protocol state. -func mergeNodeInfos(internalNodes, partnerNodes []model.NodeInfo) []model.NodeInfo { +func mergeNodeInfos(internalNodes, partnerNodes []model.NodeInfo) ([]model.NodeInfo, error) { nodes := append(internalNodes, partnerNodes...) // test for duplicate Addresses addressLookup := make(map[string]struct{}) for _, node := range nodes { if _, ok := addressLookup[node.Address]; ok { - log.Fatal().Str("address", node.Address).Msg("duplicate node address") + return nil, fmt.Errorf("duplicate node address: %v", node.Address) } + addressLookup[node.Address] = struct{}{} } // test for duplicate node IDs idLookup := make(map[flow.Identifier]struct{}) for _, node := range nodes { if _, ok := idLookup[node.NodeID]; ok { - log.Fatal().Str("NodeID", node.NodeID.String()).Msg("duplicate node ID") + return nil, fmt.Errorf("duplicate node ID: %v", node.NodeID.String()) } + idLookup[node.NodeID] = struct{}{} } // sort nodes using the canonical ordering nodes = model.Sort(nodes, flow.Canonical[flow.Identity]) - return nodes + return nodes, nil } // readRootBlock reads root block data from disc, this file needs to be prepared with diff --git a/cmd/bootstrap/cmd/finalize_test.go b/cmd/bootstrap/cmd/finalize_test.go index 1f7fee3f2c0..1edb039e2a3 100644 --- a/cmd/bootstrap/cmd/finalize_test.go +++ b/cmd/bootstrap/cmd/finalize_test.go @@ -190,3 +190,30 @@ func checkClusterConstraint(clusters flow.ClusterList, partnersInfo []model.Node } return true } + +func TestMergeNodeInfos(t *testing.T) { + partnersLen := 7 + internalLen := 22 + partners := unittest.NodeInfosFixture(partnersLen, unittest.WithRole(flow.RoleCollection)) + internals := unittest.NodeInfosFixture(internalLen, unittest.WithRole(flow.RoleCollection)) + + // Check if there is no overlap, then should pass + merged, err := mergeNodeInfos(partners, internals) + require.NoError(t, err) + require.Len(t, merged, partnersLen+internalLen) + + // Check if internals and partners have overlap, then should fail + internalAndPartnersHaveOverlap := append(partners, internals[0]) + _, err = mergeNodeInfos(internalAndPartnersHaveOverlap, internals) + require.Error(t, err) + + // Check if partners have overlap, then should fail + partnersHaveOverlap := append(partners, partners[0]) + _, err = mergeNodeInfos(partnersHaveOverlap, internals) + require.Error(t, err) + + // Check if internals have overlap, then should fail + internalsHaveOverlap := append(internals, internals[0]) + _, err = mergeNodeInfos(partners, internalsHaveOverlap) + require.Error(t, err) +} diff --git a/cmd/bootstrap/cmd/rootblock.go b/cmd/bootstrap/cmd/rootblock.go index 7d9f612ee10..8d24f2b34f1 100644 --- a/cmd/bootstrap/cmd/rootblock.go +++ b/cmd/bootstrap/cmd/rootblock.go @@ -169,7 +169,10 @@ func rootBlock(cmd *cobra.Command, args []string) { log.Info().Msg("") log.Info().Msg("assembling network and staking keys") - stakingNodes := mergeNodeInfos(internalNodes, partnerNodes) + stakingNodes, err := mergeNodeInfos(internalNodes, partnerNodes) + if err != nil { + log.Fatal().Err(err).Msgf("failed to merge node infos") + } err = common.WriteJSON(model.PathNodeInfosPub, flagOutdir, model.ToPublicNodeInfoList(stakingNodes)) if err != nil { log.Fatal().Err(err).Msg("failed to write json") From 6465e2bea5e95e1927cb39b7d4449284cd54fb36 Mon Sep 17 00:00:00 2001 From: Ardit Marku Date: Fri, 20 Sep 2024 14:57:38 +0300 Subject: [PATCH 11/72] Add boundary checks for ABI encoding values of type UInt & Int --- fvm/evm/impl/abi.go | 22 ++- fvm/evm/stdlib/contract_test.go | 243 ++++++++++++++++++++++++++------ 2 files changed, 220 insertions(+), 45 deletions(-) diff --git a/fvm/evm/impl/abi.go b/fvm/evm/impl/abi.go index 5b88da53845..871d523abef 100644 --- a/fvm/evm/impl/abi.go +++ b/fvm/evm/impl/abi.go @@ -9,6 +9,7 @@ import ( "github.com/onflow/cadence/runtime/common" "github.com/onflow/cadence/runtime/errors" "github.com/onflow/cadence/runtime/interpreter" + "github.com/onflow/cadence/runtime/sema" gethABI "github.com/onflow/go-ethereum/accounts/abi" gethCommon "github.com/onflow/go-ethereum/common" @@ -21,7 +22,8 @@ const abiEncodingByteSize = 32 // abiEncodingError type abiEncodingError struct { - Type interpreter.StaticType + Type interpreter.StaticType + Message string } var _ errors.UserError = abiEncodingError{} @@ -38,6 +40,12 @@ func (e abiEncodingError) Error() string { b.WriteString(ty.String()) } + message := e.Message + if message != "" { + b.WriteString(": ") + b.WriteString(message) + } + return b.String() } @@ -430,6 +438,12 @@ func encodeABI( case interpreter.UIntValue: if staticType == interpreter.PrimitiveStaticTypeUInt { + if value.BigInt.Cmp(sema.UInt256TypeMaxIntBig) > 0 || value.BigInt.Cmp(sema.UInt256TypeMinIntBig) < 0 { + return nil, gethABI.Type{}, abiEncodingError{ + Type: value.StaticType(inter), + Message: "value outside the boundaries of uint256", + } + } return value.BigInt, gethTypeUint, nil } @@ -465,6 +479,12 @@ func encodeABI( case interpreter.IntValue: if staticType == interpreter.PrimitiveStaticTypeInt { + if value.BigInt.Cmp(sema.Int256TypeMaxIntBig) > 0 || value.BigInt.Cmp(sema.Int256TypeMinIntBig) < 0 { + return nil, gethABI.Type{}, abiEncodingError{ + Type: value.StaticType(inter), + Message: "value outside the boundaries of int256", + } + } return value.BigInt, gethTypeInt, nil } diff --git a/fvm/evm/stdlib/contract_test.go b/fvm/evm/stdlib/contract_test.go index 7156cc44be3..f64963f59db 100644 --- a/fvm/evm/stdlib/contract_test.go +++ b/fvm/evm/stdlib/contract_test.go @@ -994,38 +994,11 @@ func TestEVMEncodeDecodeABIRoundtripForUintIntTypes(t *testing.T) { t.Parallel() handler := &testContractHandler{} - contractsAddress := flow.BytesToAddress([]byte{0x1}) - transactionEnvironment := newEVMTransactionEnvironment(handler, contractsAddress) scriptEnvironment := newEVMScriptEnvironment(handler, contractsAddress) - rt := runtime.NewInterpreterRuntime(runtime.Config{}) - script := []byte(` - import EVM from 0x1 - - access(all) - fun main(): Bool { - // Check UInt*/Int* encode/decode - let amount: UInt256 = 18446744073709551615 - let minBalance: Int256 = -18446744073709551615 - let data = EVM.encodeABIWithSignature( - "withdraw(uint,int)", - [UInt(amount), Int(minBalance)] - ) - let values = EVM.decodeABIWithSignature( - "withdraw(uint,int)", - types: [Type(), Type()], - data: data - ) - assert((values[0] as! UInt) == UInt(amount)) - assert((values[1] as! Int) == Int(minBalance)) - - return true - } - `) - accountCodes := map[common.Location][]byte{} var events []cadence.Event @@ -1073,25 +1046,207 @@ func TestEVMEncodeDecodeABIRoundtripForUintIntTypes(t *testing.T) { nextTransactionLocation, ) - // Run script + t.Run("with values between the boundaries", func(t *testing.T) { - result, err := rt.ExecuteScript( - runtime.Script{ - Source: script, - Arguments: [][]byte{}, - }, - runtime.Context{ - Interface: runtimeInterface, - Environment: scriptEnvironment, - Location: nextScriptLocation(), - }, - ) - require.NoError(t, err) + script := []byte(` + import EVM from 0x1 + + access(all) + fun main(): Bool { + // Check UInt/Int encode/decode + let amount: UInt256 = 18446744073709551615 + let minBalance: Int256 = -18446744073709551615 + let data = EVM.encodeABIWithSignature( + "withdraw(uint,int)", + [UInt(amount), Int(minBalance)] + ) + let values = EVM.decodeABIWithSignature( + "withdraw(uint,int)", + types: [Type(), Type()], + data: data + ) + assert((values[0] as! UInt) == UInt(amount)) + assert((values[1] as! Int) == Int(minBalance)) - assert.Equal(t, - cadence.Bool(true), - result, - ) + return true + } + `) + + // Run script + + result, err := rt.ExecuteScript( + runtime.Script{ + Source: script, + Arguments: [][]byte{}, + }, + runtime.Context{ + Interface: runtimeInterface, + Environment: scriptEnvironment, + Location: nextScriptLocation(), + }, + ) + require.NoError(t, err) + + assert.Equal(t, cadence.Bool(true), result) + }) + + t.Run("with values at the boundaries", func(t *testing.T) { + + script := []byte(` + import EVM from 0x1 + + access(all) + fun main(): Bool { + // Check UInt*/Int* encode/decode + let data = EVM.encodeABIWithSignature( + "withdraw(uint,int,uint,int)", + [UInt(UInt256.max), Int(Int256.max),UInt(UInt256.min), Int(Int256.min)] + ) + let values = EVM.decodeABIWithSignature( + "withdraw(uint,int,uint,int)", + types: [Type(), Type(),Type(), Type()], + data: data + ) + assert((values[0] as! UInt) == UInt(UInt256.max)) + assert((values[1] as! Int) == Int(Int256.max)) + assert((values[2] as! UInt) == UInt(UInt256.min)) + assert((values[3] as! Int) == Int(Int256.min)) + + return true + } + `) + + // Run script + + result, err := rt.ExecuteScript( + runtime.Script{ + Source: script, + Arguments: [][]byte{}, + }, + runtime.Context{ + Interface: runtimeInterface, + Environment: scriptEnvironment, + Location: nextScriptLocation(), + }, + ) + require.NoError(t, err) + + assert.Equal(t, cadence.Bool(true), result) + }) + + t.Run("with UInt values outside the boundaries", func(t *testing.T) { + + script := []byte(` + import EVM from 0x1 + + access(all) + fun main(): Bool { + let data = EVM.encodeABIWithSignature( + "withdraw(uint)", + [UInt(UInt256.max)+10] + ) + + return true + } + `) + + // Run script + + _, err := rt.ExecuteScript( + runtime.Script{ + Source: script, + Arguments: [][]byte{}, + }, + runtime.Context{ + Interface: runtimeInterface, + Environment: scriptEnvironment, + Location: nextScriptLocation(), + }, + ) + require.Error(t, err) + + assert.ErrorContains( + t, + err, + "failed to ABI encode value of type UInt: value outside the boundaries of uint256", + ) + }) + + t.Run("with Int values outside the max boundary", func(t *testing.T) { + + script := []byte(` + import EVM from 0x1 + + access(all) + fun main(): Bool { + let data = EVM.encodeABIWithSignature( + "withdraw(int)", + [Int(Int256.max)+10] + ) + + return true + } + `) + + // Run script + + _, err := rt.ExecuteScript( + runtime.Script{ + Source: script, + Arguments: [][]byte{}, + }, + runtime.Context{ + Interface: runtimeInterface, + Environment: scriptEnvironment, + Location: nextScriptLocation(), + }, + ) + require.Error(t, err) + + assert.ErrorContains( + t, + err, + "failed to ABI encode value of type Int: value outside the boundaries of int256", + ) + }) + + t.Run("with Int values outside the min boundary", func(t *testing.T) { + + script := []byte(` + import EVM from 0x1 + + access(all) + fun main(): Bool { + let data = EVM.encodeABIWithSignature( + "withdraw(int)", + [Int(Int256.min)-10] + ) + + return true + } + `) + + // Run script + + _, err := rt.ExecuteScript( + runtime.Script{ + Source: script, + Arguments: [][]byte{}, + }, + runtime.Context{ + Interface: runtimeInterface, + Environment: scriptEnvironment, + Location: nextScriptLocation(), + }, + ) + require.Error(t, err) + + assert.ErrorContains( + t, + err, + "failed to ABI encode value of type Int: value outside the boundaries of int256", + ) + }) } func TestEVMEncodeDecodeABIRoundtrip(t *testing.T) { From 3a7f490709a2ecb510410b93bf0c156b5d7eef28 Mon Sep 17 00:00:00 2001 From: Peter Argue <89119817+peterargue@users.noreply.github.com> Date: Mon, 23 Sep 2024 20:17:46 +0000 Subject: [PATCH 12/72] Merge pull request #6479 from onflow/khalil/libp2p-peer-gater Add Libp2p Gossipsub Peer Gater --- config/default-config.yml | 8 +++++ .../corruptlibp2p/pubsub_adapter_config.go | 6 ++++ network/netconf/flags.go | 14 +++++++++ .../p2p/builder/gossipsub/gossipSubBuilder.go | 8 +++++ network/p2p/config/gossipsub.go | 30 +++++++++++++++++++ .../p2p/config/gossipsub_rpc_inspectors.go | 1 + network/p2p/mock/pub_sub_adapter_config.go | 7 +++++ network/p2p/node/gossipSubAdapterConfig.go | 12 ++++++++ network/p2p/pubsub.go | 1 + 9 files changed, 87 insertions(+) diff --git a/config/default-config.yml b/config/default-config.yml index b50804a0230..8706f697dad 100644 --- a/config/default-config.yml +++ b/config/default-config.yml @@ -628,6 +628,14 @@ network-config: # keep the entire network's size. Otherwise, the local node's view of the network will be incomplete due to cache eviction. # Recommended size is 10x the number of peers in the network. cache-size: 10000 + # Enables or disables the libp2p peer gater. + peer-gater-enabled: false + # The per IP decay for all counters tracked by the peer gater for a peer. + peer-gater-source-decay: 10m + # The priority topic delivery weights. + peer-gater-topic-delivery-weights-override: | + consensus-committee: 1.5, sync-committee: .75 + # Application layer spam prevention alsp-spam-record-cache-size: 1000 alsp-spam-report-queue-size: 10_000 diff --git a/insecure/corruptlibp2p/pubsub_adapter_config.go b/insecure/corruptlibp2p/pubsub_adapter_config.go index adc3337d629..e3343f2dce8 100644 --- a/insecure/corruptlibp2p/pubsub_adapter_config.go +++ b/insecure/corruptlibp2p/pubsub_adapter_config.go @@ -1,6 +1,8 @@ package corruptlibp2p import ( + "time" + pb "github.com/libp2p/go-libp2p-pubsub/pb" "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/routing" @@ -157,6 +159,10 @@ func (c *CorruptPubSubAdapterConfig) WithRpcInspector(_ p2p.GossipSubRPCInspecto // CorruptPubSub does not support inspector suite. This is a no-op. } +func (c *CorruptPubSubAdapterConfig) WithPeerGater(_ map[string]float64, _ time.Duration) { + // CorruptPubSub does not need peer gater. This is a no-op. +} + func (c *CorruptPubSubAdapterConfig) Build() []corrupt.Option { return c.options } diff --git a/network/netconf/flags.go b/network/netconf/flags.go index 8b3cb8a5427..db99cfcdaa6 100644 --- a/network/netconf/flags.go +++ b/network/netconf/flags.go @@ -200,6 +200,10 @@ func AllFlagNames() []string { BuildFlagName(gossipsubKey, p2pconfig.ScoreParamsKey, p2pconfig.ScoringRegistryKey, p2pconfig.MisbehaviourPenaltiesKey, p2pconfig.IWantKey), BuildFlagName(gossipsubKey, p2pconfig.ScoreParamsKey, p2pconfig.ScoringRegistryKey, p2pconfig.MisbehaviourPenaltiesKey, p2pconfig.PublishKey), BuildFlagName(gossipsubKey, p2pconfig.ScoreParamsKey, p2pconfig.ScoringRegistryKey, p2pconfig.MisbehaviourPenaltiesKey, p2pconfig.ClusterPrefixedReductionFactorKey), + + BuildFlagName(gossipsubKey, p2pconfig.PeerGaterKey, p2pconfig.EnabledKey), + BuildFlagName(gossipsubKey, p2pconfig.PeerGaterKey, p2pconfig.SourceDecayKey), + BuildFlagName(gossipsubKey, p2pconfig.PeerGaterKey, p2pconfig.TopicDeliveryWeightsKey), } for _, scope := range []string{systemScope, transientScope, protocolScope, peerScope, peerProtocolScope} { @@ -597,6 +601,15 @@ func InitializeNetworkFlags(flags *pflag.FlagSet, config *Config) { config.GossipSub.ScoringParameters.ScoringRegistryParameters.MisbehaviourPenalties.ClusterPrefixedReductionFactor, "the factor used to reduce the penalty for control message misbehaviours on cluster prefixed topics") + flags.Bool(BuildFlagName(gossipsubKey, p2pconfig.PeerGaterKey, p2pconfig.EnabledKey), + config.GossipSub.PeerGaterEnabled, + "enable the libp2p peer gater") + flags.Duration(BuildFlagName(gossipsubKey, p2pconfig.PeerGaterKey, p2pconfig.SourceDecayKey), + config.GossipSub.PeerGaterSourceDecay, + "the per IP decay for all counters tracked by the peer gater for a peer") + flags.String(BuildFlagName(gossipsubKey, p2pconfig.PeerGaterKey, p2pconfig.TopicDeliveryWeightsKey), + config.GossipSub.PeerGaterTopicDeliveryWeightsOverride, + "topic delivery weights override, this is a comma separated with the format topic_1:2.2,topic_2:3.2,topic_3:1.7 these will be used to override the default topic weight of 1.0 for the specified topic.") } // LoadLibP2PResourceManagerFlags loads all CLI flags for the libp2p resource manager configuration on the provided pflag set. @@ -666,6 +679,7 @@ func SetAliases(conf *viper.Viper) error { // mapping should be from network-p2pconfig.key1.key2.key3... to network-config-key1-key2-key3... m[strings.Join(s[1:], "-")] = key } + // each flag name should correspond to exactly one key in our config store after it is loaded with the default config for _, flagName := range AllFlagNames() { fullKey, ok := m[flagName] diff --git a/network/p2p/builder/gossipsub/gossipSubBuilder.go b/network/p2p/builder/gossipsub/gossipSubBuilder.go index da3b74943b5..7877f7d8e15 100644 --- a/network/p2p/builder/gossipsub/gossipSubBuilder.go +++ b/network/p2p/builder/gossipsub/gossipSubBuilder.go @@ -269,6 +269,14 @@ func (g *Builder) Build(ctx irrecoverable.SignalerContext) (p2p.PubSubAdapter, e }) gossipSubConfigs.WithMessageIdFunction(utils.MessageID) + if g.gossipSubCfg.PeerGaterEnabled { + topicDeliveryWeights, err := g.gossipSubCfg.PeerGaterTopicDeliveryWeights() + if err != nil { + return nil, fmt.Errorf("failed to add peer gater option: %w", err) + } + gossipSubConfigs.WithPeerGater(topicDeliveryWeights, g.gossipSubCfg.PeerGaterSourceDecay) + } + if g.routingSystem != nil { gossipSubConfigs.WithRoutingDiscovery(g.routingSystem) } diff --git a/network/p2p/config/gossipsub.go b/network/p2p/config/gossipsub.go index 16770b7efd0..a8ea16ce604 100644 --- a/network/p2p/config/gossipsub.go +++ b/network/p2p/config/gossipsub.go @@ -1,6 +1,8 @@ package p2pconfig import ( + "strconv" + "strings" "time" ) @@ -61,6 +63,9 @@ const ( PeerScoringEnabledKey = "peer-scoring-enabled" ScoreParamsKey = "scoring-parameters" SubscriptionProviderKey = "subscription-provider" + PeerGaterKey = "peer-gater" + SourceDecayKey = "source-decay" + TopicDeliveryWeightsKey = "topic-delivery-weights-override" ) // GossipSubParameters is the configuration for the GossipSub pubsub implementation. @@ -76,6 +81,15 @@ type GossipSubParameters struct { PeerScoringEnabled bool `mapstructure:"peer-scoring-enabled"` SubscriptionProvider SubscriptionProviderParameters `mapstructure:"subscription-provider"` ScoringParameters ScoringParameters `mapstructure:"scoring-parameters"` + + // PeerGaterEnabled enables the peer gater. + PeerGaterEnabled bool `mapstructure:"peer-gater-enabled"` + // PeerGaterSourceDecay the per IP decay for all counters tracked by the peer gater for a peer. + PeerGaterSourceDecay time.Duration `mapstructure:"peer-gater-source-decay"` + // PeerGaterTopicDeliveryWeightsOverride topic delivery weights that will override the default value for the specified channel. + // This is a comma separated list "channel:weight, channel2:weight, channel3:weight". + // i.e: consensus-committee: 1.5, sync-committee: .75 + PeerGaterTopicDeliveryWeightsOverride string `mapstructure:"peer-gater-topic-delivery-weights-override"` } const ( @@ -89,6 +103,22 @@ type ScoringParameters struct { ScoringRegistryParameters ScoringRegistryParameters `validate:"required" mapstructure:"scoring-registry"` } +// PeerGaterTopicDeliveryWeights returns the topic delivery weights configured on this struct as a map[string]float64 . +// Note: When new topic delivery weights are added to the struct this func should be updated. +func (g *GossipSubParameters) PeerGaterTopicDeliveryWeights() (map[string]float64, error) { + m := make(map[string]float64) + for _, weightConfig := range strings.Split(g.PeerGaterTopicDeliveryWeightsOverride, ",") { + wc := strings.Split(weightConfig, ":") + f, err := strconv.ParseFloat(strings.TrimSpace(wc[1]), 64) + if err != nil { + return nil, err + } + m[strings.TrimSpace(wc[0])] = f + } + + return m, nil +} + // SubscriptionProviderParameters keys. const ( UpdateIntervalKey = "update-interval" diff --git a/network/p2p/config/gossipsub_rpc_inspectors.go b/network/p2p/config/gossipsub_rpc_inspectors.go index d18953d3240..4144ff2843c 100644 --- a/network/p2p/config/gossipsub_rpc_inspectors.go +++ b/network/p2p/config/gossipsub_rpc_inspectors.go @@ -48,6 +48,7 @@ const ( InspectionKey = "inspection" TruncationKey = "truncation" EnableKey = "enable" + EnabledKey = "enabled" DisabledKey = "disabled" MessageIDKey = "message-id" RejectUnstakedPeers = "reject-unstaked-peers" diff --git a/network/p2p/mock/pub_sub_adapter_config.go b/network/p2p/mock/pub_sub_adapter_config.go index 466ca754211..ac339a4683b 100644 --- a/network/p2p/mock/pub_sub_adapter_config.go +++ b/network/p2p/mock/pub_sub_adapter_config.go @@ -7,6 +7,8 @@ import ( mock "github.com/stretchr/testify/mock" routing "github.com/libp2p/go-libp2p/core/routing" + + time "time" ) // PubSubAdapterConfig is an autogenerated mock type for the PubSubAdapterConfig type @@ -19,6 +21,11 @@ func (_m *PubSubAdapterConfig) WithMessageIdFunction(f func([]byte) string) { _m.Called(f) } +// WithPeerGater provides a mock function with given fields: topicDeliveryWeights, sourceDecay +func (_m *PubSubAdapterConfig) WithPeerGater(topicDeliveryWeights map[string]float64, sourceDecay time.Duration) { + _m.Called(topicDeliveryWeights, sourceDecay) +} + // WithRoutingDiscovery provides a mock function with given fields: _a0 func (_m *PubSubAdapterConfig) WithRoutingDiscovery(_a0 routing.ContentRouting) { _m.Called(_a0) diff --git a/network/p2p/node/gossipSubAdapterConfig.go b/network/p2p/node/gossipSubAdapterConfig.go index e9b102a6e81..20d7d94b709 100644 --- a/network/p2p/node/gossipSubAdapterConfig.go +++ b/network/p2p/node/gossipSubAdapterConfig.go @@ -1,6 +1,8 @@ package p2pnode import ( + "time" + pubsub "github.com/libp2p/go-libp2p-pubsub" pb "github.com/libp2p/go-libp2p-pubsub/pb" "github.com/libp2p/go-libp2p/core/peer" @@ -97,6 +99,16 @@ func (g *GossipSubAdapterConfig) WithTracer(tracer p2p.PubSubTracer) { g.options = append(g.options, pubsub.WithRawTracer(tracer)) } +// WithPeerGater adds a peer gater option to the config. +// Args: +// - params: the topic delivery weights to use +// Returns: +// -None +func (g *GossipSubAdapterConfig) WithPeerGater(topicDeliveryWeights map[string]float64, sourceDecay time.Duration) { + peerGaterParams := pubsub.NewPeerGaterParams(pubsub.DefaultPeerGaterThreshold, pubsub.DefaultPeerGaterGlobalDecay, pubsub.ScoreParameterDecay(sourceDecay)).WithTopicDeliveryWeights(topicDeliveryWeights) + g.options = append(g.options, pubsub.WithPeerGater(peerGaterParams)) +} + // ScoreTracer returns the tracer for the peer score. // Args: // - None diff --git a/network/p2p/pubsub.go b/network/p2p/pubsub.go index 97741d0820e..acc860c6dec 100644 --- a/network/p2p/pubsub.go +++ b/network/p2p/pubsub.go @@ -84,6 +84,7 @@ type PubSubAdapterConfig interface { // This is used to expose the local scoring table of the GossipSub node to its higher level components. WithScoreTracer(tracer PeerScoreTracer) WithRpcInspector(GossipSubRPCInspector) + WithPeerGater(topicDeliveryWeights map[string]float64, sourceDecay time.Duration) } // GossipSubRPCInspector abstracts the general behavior of an app specific RPC inspector specifically From d5206ed903509917af48ca77cc4d70cdc6267342 Mon Sep 17 00:00:00 2001 From: Vishal <1117327+vishalchangrani@users.noreply.github.com> Date: Mon, 23 Sep 2024 16:30:18 -0700 Subject: [PATCH 13/72] adding an option to boostrapcmd to pull candidate access nodes in addition to proposed nodes (#6489) * adding an option to boostrapcmd to pull candidate access nodes in addition to proposed nodes * fixing log message --- cmd/bootstrap/cmd/partner_infos.go | 38 ++++++++++++++++++++++++++--- cmd/util/cmd/common/transactions.go | 28 +++++++++++++++++++++ 2 files changed, 62 insertions(+), 4 deletions(-) diff --git a/cmd/bootstrap/cmd/partner_infos.go b/cmd/bootstrap/cmd/partner_infos.go index f6f463941b3..033fc22437e 100644 --- a/cmd/bootstrap/cmd/partner_infos.go +++ b/cmd/bootstrap/cmd/partner_infos.go @@ -35,9 +35,10 @@ const ( ) var ( - flagANAddress string - flagANNetworkKey string - flagNetworkEnv string + flagANAddress string + flagANNetworkKey string + flagNetworkEnv string + flagIncludeAccessNodes bool ) // populatePartnerInfos represents the `populate-partner-infos` command which will read the proposed node @@ -55,6 +56,7 @@ func init() { populatePartnerInfosCMD.Flags().StringVar(&flagANAddress, "access-address", "", "the address of the access node used for client connections") populatePartnerInfosCMD.Flags().StringVar(&flagANNetworkKey, "access-network-key", "", "the network key of the access node used for client connections in hex string format") populatePartnerInfosCMD.Flags().StringVar(&flagNetworkEnv, "network", "mainnet", "the network string, expecting one of ( mainnet | testnet | emulator )") + populatePartnerInfosCMD.Flags().BoolVar(&flagIncludeAccessNodes, "include-candidate-access-nodes", false, "whether to include the candidate access nodes") cmd.MarkFlagRequired(populatePartnerInfosCMD, "access-address") } @@ -76,13 +78,26 @@ func populatePartnerInfosRun(_ *cobra.Command, _ []string) { flow.RoleAccess: 0, } totalNumOfPartnerNodes := 0 + var allNodes []cadence.Value nodeInfos, err := executeGetProposedNodesInfosScript(ctx, flowClient) if err != nil { log.Fatal().Err(err).Msg("could not get node info for nodes in the proposed table") } + allNodes = nodeInfos.(cadence.Array).Values[:] - for _, info := range nodeInfos.(cadence.Array).Values { + log.Info().Int("total_proposed_nodes", len(allNodes)).Msg("total nodes in proposed table") + + if flagIncludeAccessNodes { + candidateNodeInfos, err := executeGetCandidateAccessNodesInfosScript(ctx, flowClient) + if err != nil { + log.Fatal().Err(err).Msg("could not get node info for nodes in the candidate table") + } + log.Info().Int("total_candidate_access_nodes", len(candidateNodeInfos.(cadence.Array).Values)).Msg("total access nodes in candidate table") + allNodes = append(allNodes, candidateNodeInfos.(cadence.Array).Values[:]...) + } + + for _, info := range allNodes { nodePubInfo, err := parseNodeInfo(info) if err != nil { log.Fatal().Err(err).Msg("could not parse node info from cadence script") @@ -146,6 +161,21 @@ func executeGetProposedNodesInfosScript(ctx context.Context, client *client.Clie return infos, nil } +// GetNodeInfoForCandidateNodesScript executes the get node info for each Access node ID in the candidate table +func executeGetCandidateAccessNodesInfosScript(ctx context.Context, client *client.Client) (cadence.Value, error) { + script, err := common.GetNodeInfoForCandidateNodesScript(flagNetworkEnv) + if err != nil { + return nil, fmt.Errorf("failed to get cadence script: %w", err) + } + + infos, err := client.ExecuteScriptAtLatestBlock(ctx, script, []cadence.Value{}) + if err != nil { + return nil, fmt.Errorf("failed to execute the get node info script: %w", err) + } + + return infos, nil +} + // parseNodeInfo convert node info retrieved from cadence script func parseNodeInfo(info cadence.Value) (*bootstrap.NodeInfoPub, error) { fields := cadence.FieldsMappedByName(info.(cadence.Struct)) diff --git a/cmd/util/cmd/common/transactions.go b/cmd/util/cmd/common/transactions.go index 00952749e36..75082ef5462 100644 --- a/cmd/util/cmd/common/transactions.go +++ b/cmd/util/cmd/common/transactions.go @@ -23,6 +23,21 @@ const ( return infos }` + + getInfoForCandidateAccessNodesScript = ` + import FlowIDTableStaking from "FlowIDTableStaking" + access(all) fun main(): [FlowIDTableStaking.NodeInfo] { + let candidateNodes = FlowIDTableStaking.getCandidateNodeList() + let candidateAccessNodes = candidateNodes[UInt8(5)]! + + let nodeInfos: [FlowIDTableStaking.NodeInfo] = [] + for nodeID in candidateAccessNodes.keys { + let nodeInfo = FlowIDTableStaking.NodeInfo(nodeID: nodeID) + nodeInfos.append(nodeInfo) + } + + return nodeInfos + }` ) // GetNodeInfoForProposedNodesScript returns a script that will return an array of FlowIDTableStaking.NodeInfo for each @@ -37,3 +52,16 @@ func GetNodeInfoForProposedNodesScript(network string) ([]byte, error) { ), ), nil } + +// GetNodeInfoForCandidateNodesScript returns a script that will return an array of FlowIDTableStaking.NodeInfo for each +// node in the candidate table (nodes which have staked but not yet chosen by the network). +func GetNodeInfoForCandidateNodesScript(network string) ([]byte, error) { + contracts := systemcontracts.SystemContractsForChain(flow.ChainID(fmt.Sprintf("flow-%s", network))) + + return []byte( + templates.ReplaceAddresses( + getInfoForCandidateAccessNodesScript, + contracts.AsTemplateEnv(), + ), + ), nil +} From 85dea97c494c97f26244a2750a07ebd10b7aefbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Wed, 4 Sep 2024 20:10:40 -0700 Subject: [PATCH 14/72] reject entitled Account.capabilities.get/borrow --- fvm/environment/facade_env.go | 20 +++++++++++++ fvm/environment/mock/environment.go | 28 ++++++++++++++++++ fvm/fvm_test.go | 44 +++++++++++++++++++++++++++++ go.mod | 2 ++ go.sum | 12 ++++++-- insecure/go.mod | 2 ++ insecure/go.sum | 12 ++++++-- integration/go.mod | 2 ++ integration/go.sum | 12 ++++++-- 9 files changed, 125 insertions(+), 9 deletions(-) diff --git a/fvm/environment/facade_env.go b/fvm/environment/facade_env.go index a9e558c5106..a47d4b124ca 100644 --- a/fvm/environment/facade_env.go +++ b/fvm/environment/facade_env.go @@ -6,6 +6,7 @@ import ( "github.com/onflow/cadence/runtime/ast" "github.com/onflow/cadence/runtime/common" "github.com/onflow/cadence/runtime/interpreter" + "github.com/onflow/cadence/runtime/sema" "github.com/onflow/flow-go/fvm/storage" "github.com/onflow/flow-go/fvm/storage/snapshot" @@ -343,3 +344,22 @@ func (env *facadeEnvironment) RecoverProgram(program *ast.Program, location comm location, ) } + +func (env *facadeEnvironment) ValidateAccountCapabilitiesGet( + _ *interpreter.Interpreter, + _ interpreter.LocationRange, + _ interpreter.AddressValue, + _ interpreter.PathValue, + wantedBorrowType *sema.ReferenceType, + _ *sema.ReferenceType, +) (bool, error) { + _, hasEntitlements := wantedBorrowType.Authorization.(sema.EntitlementSetAccess) + if hasEntitlements { + // TODO: maybe abort + //return false, interpreter.GetCapabilityError{ + // LocationRange: locationRange, + //} + return false, nil + } + return true, nil +} diff --git a/fvm/environment/mock/environment.go b/fvm/environment/mock/environment.go index 967095141bc..a01112bc42f 100644 --- a/fvm/environment/mock/environment.go +++ b/fvm/environment/mock/environment.go @@ -1724,6 +1724,34 @@ func (_m *Environment) UpdateAccountContractCode(location common.AddressLocation return r0 } +// ValidateAccountCapabilitiesGet provides a mock function with given fields: inter, locationRange, address, path, wantedBorrowType, capabilityBorrowType +func (_m *Environment) ValidateAccountCapabilitiesGet(inter *interpreter.Interpreter, locationRange interpreter.LocationRange, address interpreter.AddressValue, path interpreter.PathValue, wantedBorrowType *sema.ReferenceType, capabilityBorrowType *sema.ReferenceType) (bool, error) { + ret := _m.Called(inter, locationRange, address, path, wantedBorrowType, capabilityBorrowType) + + if len(ret) == 0 { + panic("no return value specified for ValidateAccountCapabilitiesGet") + } + + var r0 bool + var r1 error + if rf, ok := ret.Get(0).(func(*interpreter.Interpreter, interpreter.LocationRange, interpreter.AddressValue, interpreter.PathValue, *sema.ReferenceType, *sema.ReferenceType) (bool, error)); ok { + return rf(inter, locationRange, address, path, wantedBorrowType, capabilityBorrowType) + } + if rf, ok := ret.Get(0).(func(*interpreter.Interpreter, interpreter.LocationRange, interpreter.AddressValue, interpreter.PathValue, *sema.ReferenceType, *sema.ReferenceType) bool); ok { + r0 = rf(inter, locationRange, address, path, wantedBorrowType, capabilityBorrowType) + } else { + r0 = ret.Get(0).(bool) + } + + if rf, ok := ret.Get(1).(func(*interpreter.Interpreter, interpreter.LocationRange, interpreter.AddressValue, interpreter.PathValue, *sema.ReferenceType, *sema.ReferenceType) error); ok { + r1 = rf(inter, locationRange, address, path, wantedBorrowType, capabilityBorrowType) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // ValidatePublicKey provides a mock function with given fields: key func (_m *Environment) ValidatePublicKey(key *stdlib.PublicKey) error { ret := _m.Called(key) diff --git a/fvm/fvm_test.go b/fvm/fvm_test.go index b48fedebe22..17f23139407 100644 --- a/fvm/fvm_test.go +++ b/fvm/fvm_test.go @@ -3086,3 +3086,47 @@ func TestEVM(t *testing.T) { }), ) } + +func TestAccountCapabilitiesGetEntitledRejection(t *testing.T) { + + t.Run("successful transaction", newVMTest(). + run(func( + t *testing.T, + vm fvm.VM, + chain flow.Chain, + ctx fvm.Context, + snapshotTree snapshot.SnapshotTree, + ) { + + serviceAddress := chain.ServiceAddress() + txBody := flow.NewTransactionBody(). + SetScript([]byte(` + transaction { + prepare(signer: auth(Capabilities, Storage) &Account) { + signer.storage.save(42, to: /storage/number) + let cap = signer.capabilities.storage.issue(/storage/number) + signer.capabilities.publish(cap, at: /public/number) + + let number = signer.capabilities.borrow(/public/number) + assert(number == nil) + } + } + `)). + AddAuthorizer(serviceAddress). + SetProposalKey(serviceAddress, 0, 0). + SetPayer(serviceAddress) + + err := testutil.SignTransactionAsServiceAccount(txBody, 0, chain) + require.NoError(t, err) + + _, output, err := vm.Run( + ctx, + fvm.Transaction(txBody, 0), + snapshotTree) + + require.NoError(t, err) + require.NoError(t, output.Err) + }), + ) + +} diff --git a/go.mod b/go.mod index 15d4a7cd71e..9c6f79a3828 100644 --- a/go.mod +++ b/go.mod @@ -334,3 +334,5 @@ replace github.com/ipfs/boxo => github.com/onflow/boxo v0.0.0-20240201202436-f24 // TODO: remove it when https://github.com/ipfs/go-ds-pebble/pull/36 merged replace github.com/ipfs/go-ds-pebble v0.3.1 => github.com/onflow/go-ds-pebble v0.0.0-20240731130313-f186539f382c + +replace github.com/onflow/cadence => github.com/dapperlabs/cadence-internal v1.0.0-preview.52-patch.1 diff --git a/go.sum b/go.sum index c0ac54f6600..8230135af5d 100644 --- a/go.sum +++ b/go.sum @@ -981,6 +981,7 @@ github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8= github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= +github.com/PuerkitoBio/goquery v1.5.0/go.mod h1:qD2PgZ9lccMbQlc7eEOjaeRlFQON7xY8kdmcsrnKqMg= github.com/SaveTheRbtz/mph v0.1.1-0.20240117162131-4166ec7869bc h1:DCHzPQOcU/7gwDTWbFQZc5qHMPS1g0xTO56k8NXsv9M= github.com/SaveTheRbtz/mph v0.1.1-0.20240117162131-4166ec7869bc/go.mod h1:LJM5a3zcIJ/8TmZwlUczvROEJT8ntOdhdG9jjcR1B0I= github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0= @@ -1012,6 +1013,7 @@ github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= github.com/antihax/optional v1.0.0 h1:xK2lYat7ZLaVVcIuj82J8kIro4V6kDe0AUDFboUCwcg= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= @@ -1248,6 +1250,8 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3 github.com/cskr/pubsub v1.0.2 h1:vlOzMhl6PFn60gRlTQQsIfVwaPB/B/8MziK8FhEPt/0= github.com/cskr/pubsub v1.0.2/go.mod h1:/8MzYXk/NJAz782G8RPkFzXTZVu63VotefPnR9TIRis= github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= +github.com/dapperlabs/cadence-internal v1.0.0-preview.52-patch.1 h1:iD5d8WZpPcFa3iWcuX2/3gA0NZwToXsc4Gjh804QxOA= +github.com/dapperlabs/cadence-internal v1.0.0-preview.52-patch.1/go.mod h1:7wvvecnAZtYOspLOS3Lh+FuAmMeSrXhAWiycC3kQ1UU= github.com/dave/astrid v0.0.0-20170323122508-8c2895878b14/go.mod h1:Sth2QfxfATb/nW4EsrSi2KyJmbcniZ8TgTaji17D6ms= github.com/dave/brenda v1.1.0/go.mod h1:4wCUr6gSlu5/1Tk7akE5X7UorwiQ8Rij0SKH3/BGMOM= github.com/dave/courtney v0.3.0/go.mod h1:BAv3hA06AYfNUjfjQr+5gc6vxeBVOupLqrColj+QSD8= @@ -1838,6 +1842,8 @@ github.com/iris-contrib/blackfriday v2.0.0+incompatible/go.mod h1:UzZ2bDEoaSGPbk github.com/iris-contrib/go.uuid v2.0.0+incompatible/go.mod h1:iz2lgM/1UnEf1kP0L/+fafWORmlnuysV2EMP8MW+qe0= github.com/iris-contrib/i18n v0.0.0-20171121225848-987a633949d0/go.mod h1:pMCz62A0xJL6I+umB2YTlFRwWXaDFA0jy+5HzGiJjqI= github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw= +github.com/itchyny/gojq v0.12.14/go.mod h1:y1G7oO7XkcR1LPZO59KyoCRy08T3j9vDYRV0GgYSS+s= +github.com/itchyny/timefmt-go v0.1.5/go.mod h1:nEP7L+2YmAbT2kZ2HfSs1d8Xtw9LY8D2stDBckWakZ8= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jbenet/go-cienv v0.1.0/go.mod h1:TqNnHUmJgXau0nCzC7kXWeotg3J9W34CUv5Djy1+FlA= @@ -1917,6 +1923,7 @@ github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/4 github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6/go.mod h1:+ZoRqAPRLkC4NPOvfYeR5KNOrY6TD+/sAC3HXPZgDYg= github.com/klauspost/pgzip v1.0.2-0.20170402124221-0bf5dcad4ada/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= +github.com/kodova/html-to-markdown v1.0.1/go.mod h1:NhDrT7QdSrdpezFg/0EQx9zeobCHR5oAguzrKrC6mVU= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/koron/go-ssdp v0.0.4 h1:1IDwrghSKYM7yLf7XCzbByg2sJ/JcNOZRXS2jczTwz0= @@ -2167,9 +2174,6 @@ github.com/onflow/atree v0.8.0-rc.6 h1:GWgaylK24b5ta2Hq+TvyOF7X5tZLiLzMMn7lEt59f github.com/onflow/atree v0.8.0-rc.6/go.mod h1:yccR+LR7xc1Jdic0mrjocbHvUD7lnVvg8/Ct1AA5zBo= github.com/onflow/boxo v0.0.0-20240201202436-f2477b92f483 h1:LpiQhTAfM9CAmNVEs0n//cBBgCg+vJSiIxTHYUklZ84= github.com/onflow/boxo v0.0.0-20240201202436-f2477b92f483/go.mod h1:pIZgTWdm3k3pLF9Uq6MB8JEcW07UDwNJjlXW1HELW80= -github.com/onflow/cadence v1.0.0-M3/go.mod h1:odXGZZ/wGNA5mwT8bC9v8u8EXACHllB2ABSZK65TGL8= -github.com/onflow/cadence v1.0.0-preview.52 h1:hZ92e6lL2+PQa3C1i5jJh0zZYFdW89+X1MS0Bkd6Ayo= -github.com/onflow/cadence v1.0.0-preview.52/go.mod h1:7wvvecnAZtYOspLOS3Lh+FuAmMeSrXhAWiycC3kQ1UU= github.com/onflow/crypto v0.25.0/go.mod h1:C8FbaX0x8y+FxWjbkHy0Q4EASCDR9bSPWZqlpCLYyVI= github.com/onflow/crypto v0.25.2 h1:GjHunqVt+vPcdqhxxhAXiMIF3YiLX7gTuTR5O+VG2ns= github.com/onflow/crypto v0.25.2/go.mod h1:fY7eLqUdMKV8EGOw301unP8h7PvLVy8/6gVR++/g0BY= @@ -2769,6 +2773,7 @@ golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -2796,6 +2801,7 @@ golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191109021931-daa7c04131f5/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= diff --git a/insecure/go.mod b/insecure/go.mod index 0c627ca59a0..be5e68ca85a 100644 --- a/insecure/go.mod +++ b/insecure/go.mod @@ -312,3 +312,5 @@ replace github.com/onflow/flow-go => ../ // TODO: remove it when https://github.com/ipfs/go-ds-pebble/pull/36 merged replace github.com/ipfs/go-ds-pebble v0.3.1 => github.com/onflow/go-ds-pebble v0.0.0-20240731130313-f186539f382c + +replace github.com/onflow/cadence => github.com/dapperlabs/cadence-internal v1.0.0-preview.52-patch.1 diff --git a/insecure/go.sum b/insecure/go.sum index 39c9676ca6e..6c16ff2eaa4 100644 --- a/insecure/go.sum +++ b/insecure/go.sum @@ -981,6 +981,7 @@ github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8= github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= +github.com/PuerkitoBio/goquery v1.5.0/go.mod h1:qD2PgZ9lccMbQlc7eEOjaeRlFQON7xY8kdmcsrnKqMg= github.com/SaveTheRbtz/mph v0.1.1-0.20240117162131-4166ec7869bc h1:DCHzPQOcU/7gwDTWbFQZc5qHMPS1g0xTO56k8NXsv9M= github.com/SaveTheRbtz/mph v0.1.1-0.20240117162131-4166ec7869bc/go.mod h1:LJM5a3zcIJ/8TmZwlUczvROEJT8ntOdhdG9jjcR1B0I= github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0= @@ -1012,6 +1013,7 @@ github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/apache/arrow/go/arrow v0.0.0-20191024131854-af6fa24be0db/go.mod h1:VTxUBvSJ3s3eHAg65PNgrsn5BtqCRPdmyXh6rAfdxN0= @@ -1247,6 +1249,8 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3 github.com/cskr/pubsub v1.0.2 h1:vlOzMhl6PFn60gRlTQQsIfVwaPB/B/8MziK8FhEPt/0= github.com/cskr/pubsub v1.0.2/go.mod h1:/8MzYXk/NJAz782G8RPkFzXTZVu63VotefPnR9TIRis= github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= +github.com/dapperlabs/cadence-internal v1.0.0-preview.52-patch.1 h1:iD5d8WZpPcFa3iWcuX2/3gA0NZwToXsc4Gjh804QxOA= +github.com/dapperlabs/cadence-internal v1.0.0-preview.52-patch.1/go.mod h1:7wvvecnAZtYOspLOS3Lh+FuAmMeSrXhAWiycC3kQ1UU= github.com/dave/astrid v0.0.0-20170323122508-8c2895878b14/go.mod h1:Sth2QfxfATb/nW4EsrSi2KyJmbcniZ8TgTaji17D6ms= github.com/dave/brenda v1.1.0/go.mod h1:4wCUr6gSlu5/1Tk7akE5X7UorwiQ8Rij0SKH3/BGMOM= github.com/dave/courtney v0.3.0/go.mod h1:BAv3hA06AYfNUjfjQr+5gc6vxeBVOupLqrColj+QSD8= @@ -1833,6 +1837,8 @@ github.com/iris-contrib/blackfriday v2.0.0+incompatible/go.mod h1:UzZ2bDEoaSGPbk github.com/iris-contrib/go.uuid v2.0.0+incompatible/go.mod h1:iz2lgM/1UnEf1kP0L/+fafWORmlnuysV2EMP8MW+qe0= github.com/iris-contrib/i18n v0.0.0-20171121225848-987a633949d0/go.mod h1:pMCz62A0xJL6I+umB2YTlFRwWXaDFA0jy+5HzGiJjqI= github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw= +github.com/itchyny/gojq v0.12.14/go.mod h1:y1G7oO7XkcR1LPZO59KyoCRy08T3j9vDYRV0GgYSS+s= +github.com/itchyny/timefmt-go v0.1.5/go.mod h1:nEP7L+2YmAbT2kZ2HfSs1d8Xtw9LY8D2stDBckWakZ8= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jbenet/go-cienv v0.1.0/go.mod h1:TqNnHUmJgXau0nCzC7kXWeotg3J9W34CUv5Djy1+FlA= @@ -1912,6 +1918,7 @@ github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/4 github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6/go.mod h1:+ZoRqAPRLkC4NPOvfYeR5KNOrY6TD+/sAC3HXPZgDYg= github.com/klauspost/pgzip v1.0.2-0.20170402124221-0bf5dcad4ada/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= +github.com/kodova/html-to-markdown v1.0.1/go.mod h1:NhDrT7QdSrdpezFg/0EQx9zeobCHR5oAguzrKrC6mVU= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/koron/go-ssdp v0.0.4 h1:1IDwrghSKYM7yLf7XCzbByg2sJ/JcNOZRXS2jczTwz0= @@ -2157,9 +2164,6 @@ github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6 github.com/onflow/atree v0.6.1-0.20230711151834-86040b30171f/go.mod h1:xvP61FoOs95K7IYdIYRnNcYQGf4nbF/uuJ0tHf4DRuM= github.com/onflow/atree v0.8.0-rc.6 h1:GWgaylK24b5ta2Hq+TvyOF7X5tZLiLzMMn7lEt59fsA= github.com/onflow/atree v0.8.0-rc.6/go.mod h1:yccR+LR7xc1Jdic0mrjocbHvUD7lnVvg8/Ct1AA5zBo= -github.com/onflow/cadence v1.0.0-M3/go.mod h1:odXGZZ/wGNA5mwT8bC9v8u8EXACHllB2ABSZK65TGL8= -github.com/onflow/cadence v1.0.0-preview.52 h1:hZ92e6lL2+PQa3C1i5jJh0zZYFdW89+X1MS0Bkd6Ayo= -github.com/onflow/cadence v1.0.0-preview.52/go.mod h1:7wvvecnAZtYOspLOS3Lh+FuAmMeSrXhAWiycC3kQ1UU= github.com/onflow/crypto v0.25.0/go.mod h1:C8FbaX0x8y+FxWjbkHy0Q4EASCDR9bSPWZqlpCLYyVI= github.com/onflow/crypto v0.25.2 h1:GjHunqVt+vPcdqhxxhAXiMIF3YiLX7gTuTR5O+VG2ns= github.com/onflow/crypto v0.25.2/go.mod h1:fY7eLqUdMKV8EGOw301unP8h7PvLVy8/6gVR++/g0BY= @@ -2755,6 +2759,7 @@ golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -2782,6 +2787,7 @@ golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191109021931-daa7c04131f5/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= diff --git a/integration/go.mod b/integration/go.mod index e61eec29247..facf3fd1532 100644 --- a/integration/go.mod +++ b/integration/go.mod @@ -362,3 +362,5 @@ replace github.com/onflow/flow-go/insecure => ../insecure // TODO: remove it when https://github.com/ipfs/go-ds-pebble/pull/36 merged replace github.com/ipfs/go-ds-pebble v0.3.1 => github.com/onflow/go-ds-pebble v0.0.0-20240731130313-f186539f382c + +replace github.com/onflow/cadence => github.com/dapperlabs/cadence-internal v1.0.0-preview.52-patch.1 diff --git a/integration/go.sum b/integration/go.sum index 1b425524376..aaaec013681 100644 --- a/integration/go.sum +++ b/integration/go.sum @@ -993,6 +993,7 @@ github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8 github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371 h1:kkhsdkhsCvIsutKu5zLMgWtgh9YxGCNAw8Ad8hjwfYg= github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= +github.com/PuerkitoBio/goquery v1.5.0/go.mod h1:qD2PgZ9lccMbQlc7eEOjaeRlFQON7xY8kdmcsrnKqMg= github.com/SaveTheRbtz/mph v0.1.1-0.20240117162131-4166ec7869bc h1:DCHzPQOcU/7gwDTWbFQZc5qHMPS1g0xTO56k8NXsv9M= github.com/SaveTheRbtz/mph v0.1.1-0.20240117162131-4166ec7869bc/go.mod h1:LJM5a3zcIJ/8TmZwlUczvROEJT8ntOdhdG9jjcR1B0I= github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0= @@ -1022,6 +1023,7 @@ github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= @@ -1255,6 +1257,8 @@ github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg= github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= github.com/danieljoos/wincred v1.2.0/go.mod h1:FzQLLMKBFdvu+osBrnFODiv32YGwCfx0SkRa/eYHgec= +github.com/dapperlabs/cadence-internal v1.0.0-preview.52-patch.1 h1:iD5d8WZpPcFa3iWcuX2/3gA0NZwToXsc4Gjh804QxOA= +github.com/dapperlabs/cadence-internal v1.0.0-preview.52-patch.1/go.mod h1:7wvvecnAZtYOspLOS3Lh+FuAmMeSrXhAWiycC3kQ1UU= github.com/dapperlabs/testingdock v0.4.5-0.20231020233342-a2853fe18724 h1:zOOpPLu5VvH8ixyoDWHnQHWoEHtryT1ne31vwz0G7Fo= github.com/dapperlabs/testingdock v0.4.5-0.20231020233342-a2853fe18724/go.mod h1:U0cEcbf9hAwPSuuoPVqXKhcWV+IU4CStK75cJ52f2/A= github.com/dave/astrid v0.0.0-20170323122508-8c2895878b14/go.mod h1:Sth2QfxfATb/nW4EsrSi2KyJmbcniZ8TgTaji17D6ms= @@ -1823,6 +1827,8 @@ github.com/iris-contrib/blackfriday v2.0.0+incompatible/go.mod h1:UzZ2bDEoaSGPbk github.com/iris-contrib/go.uuid v2.0.0+incompatible/go.mod h1:iz2lgM/1UnEf1kP0L/+fafWORmlnuysV2EMP8MW+qe0= github.com/iris-contrib/i18n v0.0.0-20171121225848-987a633949d0/go.mod h1:pMCz62A0xJL6I+umB2YTlFRwWXaDFA0jy+5HzGiJjqI= github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw= +github.com/itchyny/gojq v0.12.14/go.mod h1:y1G7oO7XkcR1LPZO59KyoCRy08T3j9vDYRV0GgYSS+s= +github.com/itchyny/timefmt-go v0.1.5/go.mod h1:nEP7L+2YmAbT2kZ2HfSs1d8Xtw9LY8D2stDBckWakZ8= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jbenet/go-cienv v0.1.0/go.mod h1:TqNnHUmJgXau0nCzC7kXWeotg3J9W34CUv5Djy1+FlA= @@ -1907,6 +1913,7 @@ github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/4 github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6/go.mod h1:+ZoRqAPRLkC4NPOvfYeR5KNOrY6TD+/sAC3HXPZgDYg= github.com/klauspost/pgzip v1.0.2-0.20170402124221-0bf5dcad4ada/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= +github.com/kodova/html-to-markdown v1.0.1/go.mod h1:NhDrT7QdSrdpezFg/0EQx9zeobCHR5oAguzrKrC6mVU= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/koron/go-ssdp v0.0.4 h1:1IDwrghSKYM7yLf7XCzbByg2sJ/JcNOZRXS2jczTwz0= @@ -2141,9 +2148,6 @@ github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6 github.com/onflow/atree v0.6.1-0.20230711151834-86040b30171f/go.mod h1:xvP61FoOs95K7IYdIYRnNcYQGf4nbF/uuJ0tHf4DRuM= github.com/onflow/atree v0.8.0-rc.6 h1:GWgaylK24b5ta2Hq+TvyOF7X5tZLiLzMMn7lEt59fsA= github.com/onflow/atree v0.8.0-rc.6/go.mod h1:yccR+LR7xc1Jdic0mrjocbHvUD7lnVvg8/Ct1AA5zBo= -github.com/onflow/cadence v1.0.0-M3/go.mod h1:odXGZZ/wGNA5mwT8bC9v8u8EXACHllB2ABSZK65TGL8= -github.com/onflow/cadence v1.0.0-preview.52 h1:hZ92e6lL2+PQa3C1i5jJh0zZYFdW89+X1MS0Bkd6Ayo= -github.com/onflow/cadence v1.0.0-preview.52/go.mod h1:7wvvecnAZtYOspLOS3Lh+FuAmMeSrXhAWiycC3kQ1UU= github.com/onflow/crypto v0.25.0/go.mod h1:C8FbaX0x8y+FxWjbkHy0Q4EASCDR9bSPWZqlpCLYyVI= github.com/onflow/crypto v0.25.2 h1:GjHunqVt+vPcdqhxxhAXiMIF3YiLX7gTuTR5O+VG2ns= github.com/onflow/crypto v0.25.2/go.mod h1:fY7eLqUdMKV8EGOw301unP8h7PvLVy8/6gVR++/g0BY= @@ -2721,6 +2725,7 @@ golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0= golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -2745,6 +2750,7 @@ golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191007182048-72f939374954/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191109021931-daa7c04131f5/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= From b43743e22bf78875c8c5c9af1d07e4335fda8e88 Mon Sep 17 00:00:00 2001 From: Supun Setunga Date: Wed, 11 Sep 2024 17:16:17 -0700 Subject: [PATCH 15/72] Reject publishing entitled capabilities --- fvm/environment/facade_env.go | 15 ++++++ fvm/environment/mock/environment.go | 24 +++++++++ fvm/fvm_test.go | 77 +++++++++++++++++++++++++++++ go.mod | 2 +- go.sum | 4 +- insecure/go.mod | 2 +- insecure/go.sum | 4 +- integration/go.mod | 2 +- integration/go.sum | 4 +- 9 files changed, 125 insertions(+), 9 deletions(-) diff --git a/fvm/environment/facade_env.go b/fvm/environment/facade_env.go index a47d4b124ca..ba1f2d21189 100644 --- a/fvm/environment/facade_env.go +++ b/fvm/environment/facade_env.go @@ -363,3 +363,18 @@ func (env *facadeEnvironment) ValidateAccountCapabilitiesGet( } return true, nil } + +func (env *facadeEnvironment) ValidateAccountCapabilitiesPublish( + _ *interpreter.Interpreter, + _ interpreter.LocationRange, + _ interpreter.AddressValue, + _ interpreter.PathValue, + capabilityBorrowType *interpreter.ReferenceStaticType, +) (bool, error) { + _, isEntitledCapability := capabilityBorrowType.Authorization.(interpreter.EntitlementSetAuthorization) + if isEntitledCapability { + // TODO: maybe abort + return false, nil + } + return true, nil +} diff --git a/fvm/environment/mock/environment.go b/fvm/environment/mock/environment.go index a01112bc42f..24ec42f7e4b 100644 --- a/fvm/environment/mock/environment.go +++ b/fvm/environment/mock/environment.go @@ -1752,6 +1752,30 @@ func (_m *Environment) ValidateAccountCapabilitiesGet(inter *interpreter.Interpr return r0, r1 } +// ValidateAccountCapabilitiesPublish provides a mock function with given fields: inter, locationRange, address, path, capabilityBorrowType +func (_m *Environment) ValidateAccountCapabilitiesPublish(inter *interpreter.Interpreter, locationRange interpreter.LocationRange, address interpreter.AddressValue, path interpreter.PathValue, capabilityBorrowType *interpreter.ReferenceStaticType) (bool, error) { + ret := _m.Called(inter, locationRange, address, path, capabilityBorrowType) + + var r0 bool + var r1 error + if rf, ok := ret.Get(0).(func(*interpreter.Interpreter, interpreter.LocationRange, interpreter.AddressValue, interpreter.PathValue, *interpreter.ReferenceStaticType) (bool, error)); ok { + return rf(inter, locationRange, address, path, capabilityBorrowType) + } + if rf, ok := ret.Get(0).(func(*interpreter.Interpreter, interpreter.LocationRange, interpreter.AddressValue, interpreter.PathValue, *interpreter.ReferenceStaticType) bool); ok { + r0 = rf(inter, locationRange, address, path, capabilityBorrowType) + } else { + r0 = ret.Get(0).(bool) + } + + if rf, ok := ret.Get(1).(func(*interpreter.Interpreter, interpreter.LocationRange, interpreter.AddressValue, interpreter.PathValue, *interpreter.ReferenceStaticType) error); ok { + r1 = rf(inter, locationRange, address, path, capabilityBorrowType) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // ValidatePublicKey provides a mock function with given fields: key func (_m *Environment) ValidatePublicKey(key *stdlib.PublicKey) error { ret := _m.Called(key) diff --git a/fvm/fvm_test.go b/fvm/fvm_test.go index 17f23139407..e0155a0decb 100644 --- a/fvm/fvm_test.go +++ b/fvm/fvm_test.go @@ -19,6 +19,7 @@ import ( "github.com/onflow/cadence/runtime" "github.com/onflow/cadence/runtime/common" cadenceErrors "github.com/onflow/cadence/runtime/errors" + "github.com/onflow/cadence/runtime/interpreter" "github.com/onflow/cadence/runtime/tests/utils" "github.com/onflow/crypto" "github.com/stretchr/testify/assert" @@ -3128,5 +3129,81 @@ func TestAccountCapabilitiesGetEntitledRejection(t *testing.T) { require.NoError(t, output.Err) }), ) +} + +func TestAccountCapabilitiesPublishEntitledRejection(t *testing.T) { + + t.Run("entitled capability", newVMTest(). + run(func( + t *testing.T, + vm fvm.VM, + chain flow.Chain, + ctx fvm.Context, + snapshotTree snapshot.SnapshotTree, + ) { + + serviceAddress := chain.ServiceAddress() + txBody := flow.NewTransactionBody(). + SetScript([]byte(` + transaction { + prepare(signer: auth(Capabilities, Storage) &Account) { + signer.storage.save(42, to: /storage/number) + let cap = signer.capabilities.storage.issue(/storage/number) + signer.capabilities.publish(cap, at: /public/number) + } + } + `)). + AddAuthorizer(serviceAddress). + SetProposalKey(serviceAddress, 0, 0). + SetPayer(serviceAddress) + + err := testutil.SignTransactionAsServiceAccount(txBody, 0, chain) + require.NoError(t, err) + _, output, err := vm.Run( + ctx, + fvm.Transaction(txBody, 0), + snapshotTree) + + require.NoError(t, err) + require.ErrorAs(t, output.Err, &interpreter.EntitledCapabilityPublishingError{}) + }), + ) + + t.Run("non entitled capability", newVMTest(). + run(func( + t *testing.T, + vm fvm.VM, + chain flow.Chain, + ctx fvm.Context, + snapshotTree snapshot.SnapshotTree, + ) { + + serviceAddress := chain.ServiceAddress() + txBody := flow.NewTransactionBody(). + SetScript([]byte(` + transaction { + prepare(signer: auth(Capabilities, Storage) &Account) { + signer.storage.save(42, to: /storage/number) + let cap = signer.capabilities.storage.issue<&Int>(/storage/number) + signer.capabilities.publish(cap, at: /public/number) + } + } + `)). + AddAuthorizer(serviceAddress). + SetProposalKey(serviceAddress, 0, 0). + SetPayer(serviceAddress) + + err := testutil.SignTransactionAsServiceAccount(txBody, 0, chain) + require.NoError(t, err) + + _, output, err := vm.Run( + ctx, + fvm.Transaction(txBody, 0), + snapshotTree) + + require.NoError(t, err) + require.NoError(t, output.Err) + }), + ) } diff --git a/go.mod b/go.mod index 9c6f79a3828..bd62293b756 100644 --- a/go.mod +++ b/go.mod @@ -335,4 +335,4 @@ replace github.com/ipfs/boxo => github.com/onflow/boxo v0.0.0-20240201202436-f24 // TODO: remove it when https://github.com/ipfs/go-ds-pebble/pull/36 merged replace github.com/ipfs/go-ds-pebble v0.3.1 => github.com/onflow/go-ds-pebble v0.0.0-20240731130313-f186539f382c -replace github.com/onflow/cadence => github.com/dapperlabs/cadence-internal v1.0.0-preview.52-patch.1 +replace github.com/onflow/cadence => github.com/dapperlabs/cadence-internal v1.0.0-preview.52-patch.2 diff --git a/go.sum b/go.sum index 8230135af5d..35a8059aeea 100644 --- a/go.sum +++ b/go.sum @@ -1250,8 +1250,8 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3 github.com/cskr/pubsub v1.0.2 h1:vlOzMhl6PFn60gRlTQQsIfVwaPB/B/8MziK8FhEPt/0= github.com/cskr/pubsub v1.0.2/go.mod h1:/8MzYXk/NJAz782G8RPkFzXTZVu63VotefPnR9TIRis= github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= -github.com/dapperlabs/cadence-internal v1.0.0-preview.52-patch.1 h1:iD5d8WZpPcFa3iWcuX2/3gA0NZwToXsc4Gjh804QxOA= -github.com/dapperlabs/cadence-internal v1.0.0-preview.52-patch.1/go.mod h1:7wvvecnAZtYOspLOS3Lh+FuAmMeSrXhAWiycC3kQ1UU= +github.com/dapperlabs/cadence-internal v1.0.0-preview.52-patch.2 h1:HOAq/2Xiq+zO9/sMXCQjhjF3Ej/URqTqCPtQaPrn2oY= +github.com/dapperlabs/cadence-internal v1.0.0-preview.52-patch.2/go.mod h1:7wvvecnAZtYOspLOS3Lh+FuAmMeSrXhAWiycC3kQ1UU= github.com/dave/astrid v0.0.0-20170323122508-8c2895878b14/go.mod h1:Sth2QfxfATb/nW4EsrSi2KyJmbcniZ8TgTaji17D6ms= github.com/dave/brenda v1.1.0/go.mod h1:4wCUr6gSlu5/1Tk7akE5X7UorwiQ8Rij0SKH3/BGMOM= github.com/dave/courtney v0.3.0/go.mod h1:BAv3hA06AYfNUjfjQr+5gc6vxeBVOupLqrColj+QSD8= diff --git a/insecure/go.mod b/insecure/go.mod index be5e68ca85a..1c4df0a4351 100644 --- a/insecure/go.mod +++ b/insecure/go.mod @@ -313,4 +313,4 @@ replace github.com/onflow/flow-go => ../ // TODO: remove it when https://github.com/ipfs/go-ds-pebble/pull/36 merged replace github.com/ipfs/go-ds-pebble v0.3.1 => github.com/onflow/go-ds-pebble v0.0.0-20240731130313-f186539f382c -replace github.com/onflow/cadence => github.com/dapperlabs/cadence-internal v1.0.0-preview.52-patch.1 +replace github.com/onflow/cadence => github.com/dapperlabs/cadence-internal v1.0.0-preview.52-patch.2 diff --git a/insecure/go.sum b/insecure/go.sum index 6c16ff2eaa4..77ed1f1be15 100644 --- a/insecure/go.sum +++ b/insecure/go.sum @@ -1249,8 +1249,8 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3 github.com/cskr/pubsub v1.0.2 h1:vlOzMhl6PFn60gRlTQQsIfVwaPB/B/8MziK8FhEPt/0= github.com/cskr/pubsub v1.0.2/go.mod h1:/8MzYXk/NJAz782G8RPkFzXTZVu63VotefPnR9TIRis= github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= -github.com/dapperlabs/cadence-internal v1.0.0-preview.52-patch.1 h1:iD5d8WZpPcFa3iWcuX2/3gA0NZwToXsc4Gjh804QxOA= -github.com/dapperlabs/cadence-internal v1.0.0-preview.52-patch.1/go.mod h1:7wvvecnAZtYOspLOS3Lh+FuAmMeSrXhAWiycC3kQ1UU= +github.com/dapperlabs/cadence-internal v1.0.0-preview.52-patch.2 h1:HOAq/2Xiq+zO9/sMXCQjhjF3Ej/URqTqCPtQaPrn2oY= +github.com/dapperlabs/cadence-internal v1.0.0-preview.52-patch.2/go.mod h1:7wvvecnAZtYOspLOS3Lh+FuAmMeSrXhAWiycC3kQ1UU= github.com/dave/astrid v0.0.0-20170323122508-8c2895878b14/go.mod h1:Sth2QfxfATb/nW4EsrSi2KyJmbcniZ8TgTaji17D6ms= github.com/dave/brenda v1.1.0/go.mod h1:4wCUr6gSlu5/1Tk7akE5X7UorwiQ8Rij0SKH3/BGMOM= github.com/dave/courtney v0.3.0/go.mod h1:BAv3hA06AYfNUjfjQr+5gc6vxeBVOupLqrColj+QSD8= diff --git a/integration/go.mod b/integration/go.mod index facf3fd1532..8ca3f831805 100644 --- a/integration/go.mod +++ b/integration/go.mod @@ -363,4 +363,4 @@ replace github.com/onflow/flow-go/insecure => ../insecure // TODO: remove it when https://github.com/ipfs/go-ds-pebble/pull/36 merged replace github.com/ipfs/go-ds-pebble v0.3.1 => github.com/onflow/go-ds-pebble v0.0.0-20240731130313-f186539f382c -replace github.com/onflow/cadence => github.com/dapperlabs/cadence-internal v1.0.0-preview.52-patch.1 +replace github.com/onflow/cadence => github.com/dapperlabs/cadence-internal v1.0.0-preview.52-patch.2 diff --git a/integration/go.sum b/integration/go.sum index aaaec013681..a445109cf06 100644 --- a/integration/go.sum +++ b/integration/go.sum @@ -1257,8 +1257,8 @@ github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg= github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= github.com/danieljoos/wincred v1.2.0/go.mod h1:FzQLLMKBFdvu+osBrnFODiv32YGwCfx0SkRa/eYHgec= -github.com/dapperlabs/cadence-internal v1.0.0-preview.52-patch.1 h1:iD5d8WZpPcFa3iWcuX2/3gA0NZwToXsc4Gjh804QxOA= -github.com/dapperlabs/cadence-internal v1.0.0-preview.52-patch.1/go.mod h1:7wvvecnAZtYOspLOS3Lh+FuAmMeSrXhAWiycC3kQ1UU= +github.com/dapperlabs/cadence-internal v1.0.0-preview.52-patch.2 h1:HOAq/2Xiq+zO9/sMXCQjhjF3Ej/URqTqCPtQaPrn2oY= +github.com/dapperlabs/cadence-internal v1.0.0-preview.52-patch.2/go.mod h1:7wvvecnAZtYOspLOS3Lh+FuAmMeSrXhAWiycC3kQ1UU= github.com/dapperlabs/testingdock v0.4.5-0.20231020233342-a2853fe18724 h1:zOOpPLu5VvH8ixyoDWHnQHWoEHtryT1ne31vwz0G7Fo= github.com/dapperlabs/testingdock v0.4.5-0.20231020233342-a2853fe18724/go.mod h1:U0cEcbf9hAwPSuuoPVqXKhcWV+IU4CStK75cJ52f2/A= github.com/dave/astrid v0.0.0-20170323122508-8c2895878b14/go.mod h1:Sth2QfxfATb/nW4EsrSi2KyJmbcniZ8TgTaji17D6ms= From 32430dbc90d3768336a24089be9898a8b87f1f14 Mon Sep 17 00:00:00 2001 From: Supun Setunga Date: Thu, 19 Sep 2024 16:15:40 -0700 Subject: [PATCH 16/72] Update capability get validation test --- fvm/fvm_test.go | 90 ++++++++++++++++++++++++++++++------------------- 1 file changed, 56 insertions(+), 34 deletions(-) diff --git a/fvm/fvm_test.go b/fvm/fvm_test.go index e0155a0decb..7526f5b3837 100644 --- a/fvm/fvm_test.go +++ b/fvm/fvm_test.go @@ -1,6 +1,7 @@ package fvm_test import ( + "context" "crypto/rand" "encoding/hex" "fmt" @@ -20,6 +21,7 @@ import ( "github.com/onflow/cadence/runtime/common" cadenceErrors "github.com/onflow/cadence/runtime/errors" "github.com/onflow/cadence/runtime/interpreter" + "github.com/onflow/cadence/runtime/sema" "github.com/onflow/cadence/runtime/tests/utils" "github.com/onflow/crypto" "github.com/stretchr/testify/assert" @@ -40,6 +42,7 @@ import ( "github.com/onflow/flow-go/fvm/storage/snapshot/mock" "github.com/onflow/flow-go/fvm/storage/testutils" "github.com/onflow/flow-go/fvm/systemcontracts" + "github.com/onflow/flow-go/fvm/tracing" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/utils/unittest" ) @@ -3090,45 +3093,64 @@ func TestEVM(t *testing.T) { func TestAccountCapabilitiesGetEntitledRejection(t *testing.T) { - t.Run("successful transaction", newVMTest(). - run(func( - t *testing.T, - vm fvm.VM, - chain flow.Chain, - ctx fvm.Context, - snapshotTree snapshot.SnapshotTree, - ) { + // Note: This cannot be tested anymore using a transaction, + // because publish method also aborts when trying to publish an entitled capability. + // Therefore, test the functionality of the `ValidateAccountCapabilitiesGet` function. - serviceAddress := chain.ServiceAddress() - txBody := flow.NewTransactionBody(). - SetScript([]byte(` - transaction { - prepare(signer: auth(Capabilities, Storage) &Account) { - signer.storage.save(42, to: /storage/number) - let cap = signer.capabilities.storage.issue(/storage/number) - signer.capabilities.publish(cap, at: /public/number) + t.Run("entitled capability", func(t *testing.T) { - let number = signer.capabilities.borrow(/public/number) - assert(number == nil) - } - } - `)). - AddAuthorizer(serviceAddress). - SetProposalKey(serviceAddress, 0, 0). - SetPayer(serviceAddress) + env := environment.NewScriptEnv( + context.TODO(), + tracing.NewMockTracerSpan(), + environment.DefaultEnvironmentParams(), + nil, + ) - err := testutil.SignTransactionAsServiceAccount(txBody, 0, chain) - require.NoError(t, err) + valid, err := env.ValidateAccountCapabilitiesGet( + nil, + interpreter.EmptyLocationRange, + interpreter.AddressValue(common.ZeroAddress), + interpreter.NewUnmeteredPathValue(common.PathDomainPublic, "dummy_value"), + sema.NewReferenceType( + nil, + sema.NewEntitlementSetAccess( + []*sema.EntitlementType{ + sema.MutateType, + }, + sema.Conjunction, + ), + sema.IntType, + ), + nil, + ) + assert.NoError(t, err) + assert.False(t, valid) + }) - _, output, err := vm.Run( - ctx, - fvm.Transaction(txBody, 0), - snapshotTree) + t.Run("non-entitled capability", func(t *testing.T) { - require.NoError(t, err) - require.NoError(t, output.Err) - }), - ) + env := environment.NewScriptEnv( + context.TODO(), + tracing.NewMockTracerSpan(), + environment.DefaultEnvironmentParams(), + nil, + ) + + valid, err := env.ValidateAccountCapabilitiesGet( + nil, + interpreter.EmptyLocationRange, + interpreter.AddressValue(common.ZeroAddress), + interpreter.NewUnmeteredPathValue(common.PathDomainPublic, "dummy_value"), + sema.NewReferenceType( + nil, + sema.UnauthorizedAccess, + sema.IntType, + ), + nil, + ) + assert.NoError(t, err) + assert.True(t, valid) + }) } func TestAccountCapabilitiesPublishEntitledRejection(t *testing.T) { From cd2105e3b9a91923b9f8d946c705408348fabb6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Wed, 25 Sep 2024 14:51:45 -0700 Subject: [PATCH 17/72] remove internal dependency --- go.mod | 2 -- go.sum | 12 +++--------- insecure/go.mod | 2 -- insecure/go.sum | 12 +++--------- integration/go.mod | 2 -- integration/go.sum | 12 +++--------- 6 files changed, 9 insertions(+), 33 deletions(-) diff --git a/go.mod b/go.mod index bd62293b756..15d4a7cd71e 100644 --- a/go.mod +++ b/go.mod @@ -334,5 +334,3 @@ replace github.com/ipfs/boxo => github.com/onflow/boxo v0.0.0-20240201202436-f24 // TODO: remove it when https://github.com/ipfs/go-ds-pebble/pull/36 merged replace github.com/ipfs/go-ds-pebble v0.3.1 => github.com/onflow/go-ds-pebble v0.0.0-20240731130313-f186539f382c - -replace github.com/onflow/cadence => github.com/dapperlabs/cadence-internal v1.0.0-preview.52-patch.2 diff --git a/go.sum b/go.sum index 35a8059aeea..c0ac54f6600 100644 --- a/go.sum +++ b/go.sum @@ -981,7 +981,6 @@ github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8= github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= -github.com/PuerkitoBio/goquery v1.5.0/go.mod h1:qD2PgZ9lccMbQlc7eEOjaeRlFQON7xY8kdmcsrnKqMg= github.com/SaveTheRbtz/mph v0.1.1-0.20240117162131-4166ec7869bc h1:DCHzPQOcU/7gwDTWbFQZc5qHMPS1g0xTO56k8NXsv9M= github.com/SaveTheRbtz/mph v0.1.1-0.20240117162131-4166ec7869bc/go.mod h1:LJM5a3zcIJ/8TmZwlUczvROEJT8ntOdhdG9jjcR1B0I= github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0= @@ -1013,7 +1012,6 @@ github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= -github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= github.com/antihax/optional v1.0.0 h1:xK2lYat7ZLaVVcIuj82J8kIro4V6kDe0AUDFboUCwcg= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= @@ -1250,8 +1248,6 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3 github.com/cskr/pubsub v1.0.2 h1:vlOzMhl6PFn60gRlTQQsIfVwaPB/B/8MziK8FhEPt/0= github.com/cskr/pubsub v1.0.2/go.mod h1:/8MzYXk/NJAz782G8RPkFzXTZVu63VotefPnR9TIRis= github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= -github.com/dapperlabs/cadence-internal v1.0.0-preview.52-patch.2 h1:HOAq/2Xiq+zO9/sMXCQjhjF3Ej/URqTqCPtQaPrn2oY= -github.com/dapperlabs/cadence-internal v1.0.0-preview.52-patch.2/go.mod h1:7wvvecnAZtYOspLOS3Lh+FuAmMeSrXhAWiycC3kQ1UU= github.com/dave/astrid v0.0.0-20170323122508-8c2895878b14/go.mod h1:Sth2QfxfATb/nW4EsrSi2KyJmbcniZ8TgTaji17D6ms= github.com/dave/brenda v1.1.0/go.mod h1:4wCUr6gSlu5/1Tk7akE5X7UorwiQ8Rij0SKH3/BGMOM= github.com/dave/courtney v0.3.0/go.mod h1:BAv3hA06AYfNUjfjQr+5gc6vxeBVOupLqrColj+QSD8= @@ -1842,8 +1838,6 @@ github.com/iris-contrib/blackfriday v2.0.0+incompatible/go.mod h1:UzZ2bDEoaSGPbk github.com/iris-contrib/go.uuid v2.0.0+incompatible/go.mod h1:iz2lgM/1UnEf1kP0L/+fafWORmlnuysV2EMP8MW+qe0= github.com/iris-contrib/i18n v0.0.0-20171121225848-987a633949d0/go.mod h1:pMCz62A0xJL6I+umB2YTlFRwWXaDFA0jy+5HzGiJjqI= github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw= -github.com/itchyny/gojq v0.12.14/go.mod h1:y1G7oO7XkcR1LPZO59KyoCRy08T3j9vDYRV0GgYSS+s= -github.com/itchyny/timefmt-go v0.1.5/go.mod h1:nEP7L+2YmAbT2kZ2HfSs1d8Xtw9LY8D2stDBckWakZ8= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jbenet/go-cienv v0.1.0/go.mod h1:TqNnHUmJgXau0nCzC7kXWeotg3J9W34CUv5Djy1+FlA= @@ -1923,7 +1917,6 @@ github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/4 github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6/go.mod h1:+ZoRqAPRLkC4NPOvfYeR5KNOrY6TD+/sAC3HXPZgDYg= github.com/klauspost/pgzip v1.0.2-0.20170402124221-0bf5dcad4ada/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= -github.com/kodova/html-to-markdown v1.0.1/go.mod h1:NhDrT7QdSrdpezFg/0EQx9zeobCHR5oAguzrKrC6mVU= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/koron/go-ssdp v0.0.4 h1:1IDwrghSKYM7yLf7XCzbByg2sJ/JcNOZRXS2jczTwz0= @@ -2174,6 +2167,9 @@ github.com/onflow/atree v0.8.0-rc.6 h1:GWgaylK24b5ta2Hq+TvyOF7X5tZLiLzMMn7lEt59f github.com/onflow/atree v0.8.0-rc.6/go.mod h1:yccR+LR7xc1Jdic0mrjocbHvUD7lnVvg8/Ct1AA5zBo= github.com/onflow/boxo v0.0.0-20240201202436-f2477b92f483 h1:LpiQhTAfM9CAmNVEs0n//cBBgCg+vJSiIxTHYUklZ84= github.com/onflow/boxo v0.0.0-20240201202436-f2477b92f483/go.mod h1:pIZgTWdm3k3pLF9Uq6MB8JEcW07UDwNJjlXW1HELW80= +github.com/onflow/cadence v1.0.0-M3/go.mod h1:odXGZZ/wGNA5mwT8bC9v8u8EXACHllB2ABSZK65TGL8= +github.com/onflow/cadence v1.0.0-preview.52 h1:hZ92e6lL2+PQa3C1i5jJh0zZYFdW89+X1MS0Bkd6Ayo= +github.com/onflow/cadence v1.0.0-preview.52/go.mod h1:7wvvecnAZtYOspLOS3Lh+FuAmMeSrXhAWiycC3kQ1UU= github.com/onflow/crypto v0.25.0/go.mod h1:C8FbaX0x8y+FxWjbkHy0Q4EASCDR9bSPWZqlpCLYyVI= github.com/onflow/crypto v0.25.2 h1:GjHunqVt+vPcdqhxxhAXiMIF3YiLX7gTuTR5O+VG2ns= github.com/onflow/crypto v0.25.2/go.mod h1:fY7eLqUdMKV8EGOw301unP8h7PvLVy8/6gVR++/g0BY= @@ -2773,7 +2769,6 @@ golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -2801,7 +2796,6 @@ golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191109021931-daa7c04131f5/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= diff --git a/insecure/go.mod b/insecure/go.mod index 1c4df0a4351..0c627ca59a0 100644 --- a/insecure/go.mod +++ b/insecure/go.mod @@ -312,5 +312,3 @@ replace github.com/onflow/flow-go => ../ // TODO: remove it when https://github.com/ipfs/go-ds-pebble/pull/36 merged replace github.com/ipfs/go-ds-pebble v0.3.1 => github.com/onflow/go-ds-pebble v0.0.0-20240731130313-f186539f382c - -replace github.com/onflow/cadence => github.com/dapperlabs/cadence-internal v1.0.0-preview.52-patch.2 diff --git a/insecure/go.sum b/insecure/go.sum index 77ed1f1be15..39c9676ca6e 100644 --- a/insecure/go.sum +++ b/insecure/go.sum @@ -981,7 +981,6 @@ github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8= github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= -github.com/PuerkitoBio/goquery v1.5.0/go.mod h1:qD2PgZ9lccMbQlc7eEOjaeRlFQON7xY8kdmcsrnKqMg= github.com/SaveTheRbtz/mph v0.1.1-0.20240117162131-4166ec7869bc h1:DCHzPQOcU/7gwDTWbFQZc5qHMPS1g0xTO56k8NXsv9M= github.com/SaveTheRbtz/mph v0.1.1-0.20240117162131-4166ec7869bc/go.mod h1:LJM5a3zcIJ/8TmZwlUczvROEJT8ntOdhdG9jjcR1B0I= github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0= @@ -1013,7 +1012,6 @@ github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= -github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/apache/arrow/go/arrow v0.0.0-20191024131854-af6fa24be0db/go.mod h1:VTxUBvSJ3s3eHAg65PNgrsn5BtqCRPdmyXh6rAfdxN0= @@ -1249,8 +1247,6 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3 github.com/cskr/pubsub v1.0.2 h1:vlOzMhl6PFn60gRlTQQsIfVwaPB/B/8MziK8FhEPt/0= github.com/cskr/pubsub v1.0.2/go.mod h1:/8MzYXk/NJAz782G8RPkFzXTZVu63VotefPnR9TIRis= github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= -github.com/dapperlabs/cadence-internal v1.0.0-preview.52-patch.2 h1:HOAq/2Xiq+zO9/sMXCQjhjF3Ej/URqTqCPtQaPrn2oY= -github.com/dapperlabs/cadence-internal v1.0.0-preview.52-patch.2/go.mod h1:7wvvecnAZtYOspLOS3Lh+FuAmMeSrXhAWiycC3kQ1UU= github.com/dave/astrid v0.0.0-20170323122508-8c2895878b14/go.mod h1:Sth2QfxfATb/nW4EsrSi2KyJmbcniZ8TgTaji17D6ms= github.com/dave/brenda v1.1.0/go.mod h1:4wCUr6gSlu5/1Tk7akE5X7UorwiQ8Rij0SKH3/BGMOM= github.com/dave/courtney v0.3.0/go.mod h1:BAv3hA06AYfNUjfjQr+5gc6vxeBVOupLqrColj+QSD8= @@ -1837,8 +1833,6 @@ github.com/iris-contrib/blackfriday v2.0.0+incompatible/go.mod h1:UzZ2bDEoaSGPbk github.com/iris-contrib/go.uuid v2.0.0+incompatible/go.mod h1:iz2lgM/1UnEf1kP0L/+fafWORmlnuysV2EMP8MW+qe0= github.com/iris-contrib/i18n v0.0.0-20171121225848-987a633949d0/go.mod h1:pMCz62A0xJL6I+umB2YTlFRwWXaDFA0jy+5HzGiJjqI= github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw= -github.com/itchyny/gojq v0.12.14/go.mod h1:y1G7oO7XkcR1LPZO59KyoCRy08T3j9vDYRV0GgYSS+s= -github.com/itchyny/timefmt-go v0.1.5/go.mod h1:nEP7L+2YmAbT2kZ2HfSs1d8Xtw9LY8D2stDBckWakZ8= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jbenet/go-cienv v0.1.0/go.mod h1:TqNnHUmJgXau0nCzC7kXWeotg3J9W34CUv5Djy1+FlA= @@ -1918,7 +1912,6 @@ github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/4 github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6/go.mod h1:+ZoRqAPRLkC4NPOvfYeR5KNOrY6TD+/sAC3HXPZgDYg= github.com/klauspost/pgzip v1.0.2-0.20170402124221-0bf5dcad4ada/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= -github.com/kodova/html-to-markdown v1.0.1/go.mod h1:NhDrT7QdSrdpezFg/0EQx9zeobCHR5oAguzrKrC6mVU= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/koron/go-ssdp v0.0.4 h1:1IDwrghSKYM7yLf7XCzbByg2sJ/JcNOZRXS2jczTwz0= @@ -2164,6 +2157,9 @@ github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6 github.com/onflow/atree v0.6.1-0.20230711151834-86040b30171f/go.mod h1:xvP61FoOs95K7IYdIYRnNcYQGf4nbF/uuJ0tHf4DRuM= github.com/onflow/atree v0.8.0-rc.6 h1:GWgaylK24b5ta2Hq+TvyOF7X5tZLiLzMMn7lEt59fsA= github.com/onflow/atree v0.8.0-rc.6/go.mod h1:yccR+LR7xc1Jdic0mrjocbHvUD7lnVvg8/Ct1AA5zBo= +github.com/onflow/cadence v1.0.0-M3/go.mod h1:odXGZZ/wGNA5mwT8bC9v8u8EXACHllB2ABSZK65TGL8= +github.com/onflow/cadence v1.0.0-preview.52 h1:hZ92e6lL2+PQa3C1i5jJh0zZYFdW89+X1MS0Bkd6Ayo= +github.com/onflow/cadence v1.0.0-preview.52/go.mod h1:7wvvecnAZtYOspLOS3Lh+FuAmMeSrXhAWiycC3kQ1UU= github.com/onflow/crypto v0.25.0/go.mod h1:C8FbaX0x8y+FxWjbkHy0Q4EASCDR9bSPWZqlpCLYyVI= github.com/onflow/crypto v0.25.2 h1:GjHunqVt+vPcdqhxxhAXiMIF3YiLX7gTuTR5O+VG2ns= github.com/onflow/crypto v0.25.2/go.mod h1:fY7eLqUdMKV8EGOw301unP8h7PvLVy8/6gVR++/g0BY= @@ -2759,7 +2755,6 @@ golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -2787,7 +2782,6 @@ golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191109021931-daa7c04131f5/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= diff --git a/integration/go.mod b/integration/go.mod index 8ca3f831805..e61eec29247 100644 --- a/integration/go.mod +++ b/integration/go.mod @@ -362,5 +362,3 @@ replace github.com/onflow/flow-go/insecure => ../insecure // TODO: remove it when https://github.com/ipfs/go-ds-pebble/pull/36 merged replace github.com/ipfs/go-ds-pebble v0.3.1 => github.com/onflow/go-ds-pebble v0.0.0-20240731130313-f186539f382c - -replace github.com/onflow/cadence => github.com/dapperlabs/cadence-internal v1.0.0-preview.52-patch.2 diff --git a/integration/go.sum b/integration/go.sum index a445109cf06..1b425524376 100644 --- a/integration/go.sum +++ b/integration/go.sum @@ -993,7 +993,6 @@ github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8 github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371 h1:kkhsdkhsCvIsutKu5zLMgWtgh9YxGCNAw8Ad8hjwfYg= github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= -github.com/PuerkitoBio/goquery v1.5.0/go.mod h1:qD2PgZ9lccMbQlc7eEOjaeRlFQON7xY8kdmcsrnKqMg= github.com/SaveTheRbtz/mph v0.1.1-0.20240117162131-4166ec7869bc h1:DCHzPQOcU/7gwDTWbFQZc5qHMPS1g0xTO56k8NXsv9M= github.com/SaveTheRbtz/mph v0.1.1-0.20240117162131-4166ec7869bc/go.mod h1:LJM5a3zcIJ/8TmZwlUczvROEJT8ntOdhdG9jjcR1B0I= github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0= @@ -1023,7 +1022,6 @@ github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= -github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= @@ -1257,8 +1255,6 @@ github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg= github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= github.com/danieljoos/wincred v1.2.0/go.mod h1:FzQLLMKBFdvu+osBrnFODiv32YGwCfx0SkRa/eYHgec= -github.com/dapperlabs/cadence-internal v1.0.0-preview.52-patch.2 h1:HOAq/2Xiq+zO9/sMXCQjhjF3Ej/URqTqCPtQaPrn2oY= -github.com/dapperlabs/cadence-internal v1.0.0-preview.52-patch.2/go.mod h1:7wvvecnAZtYOspLOS3Lh+FuAmMeSrXhAWiycC3kQ1UU= github.com/dapperlabs/testingdock v0.4.5-0.20231020233342-a2853fe18724 h1:zOOpPLu5VvH8ixyoDWHnQHWoEHtryT1ne31vwz0G7Fo= github.com/dapperlabs/testingdock v0.4.5-0.20231020233342-a2853fe18724/go.mod h1:U0cEcbf9hAwPSuuoPVqXKhcWV+IU4CStK75cJ52f2/A= github.com/dave/astrid v0.0.0-20170323122508-8c2895878b14/go.mod h1:Sth2QfxfATb/nW4EsrSi2KyJmbcniZ8TgTaji17D6ms= @@ -1827,8 +1823,6 @@ github.com/iris-contrib/blackfriday v2.0.0+incompatible/go.mod h1:UzZ2bDEoaSGPbk github.com/iris-contrib/go.uuid v2.0.0+incompatible/go.mod h1:iz2lgM/1UnEf1kP0L/+fafWORmlnuysV2EMP8MW+qe0= github.com/iris-contrib/i18n v0.0.0-20171121225848-987a633949d0/go.mod h1:pMCz62A0xJL6I+umB2YTlFRwWXaDFA0jy+5HzGiJjqI= github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw= -github.com/itchyny/gojq v0.12.14/go.mod h1:y1G7oO7XkcR1LPZO59KyoCRy08T3j9vDYRV0GgYSS+s= -github.com/itchyny/timefmt-go v0.1.5/go.mod h1:nEP7L+2YmAbT2kZ2HfSs1d8Xtw9LY8D2stDBckWakZ8= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jbenet/go-cienv v0.1.0/go.mod h1:TqNnHUmJgXau0nCzC7kXWeotg3J9W34CUv5Djy1+FlA= @@ -1913,7 +1907,6 @@ github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/4 github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6/go.mod h1:+ZoRqAPRLkC4NPOvfYeR5KNOrY6TD+/sAC3HXPZgDYg= github.com/klauspost/pgzip v1.0.2-0.20170402124221-0bf5dcad4ada/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= -github.com/kodova/html-to-markdown v1.0.1/go.mod h1:NhDrT7QdSrdpezFg/0EQx9zeobCHR5oAguzrKrC6mVU= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/koron/go-ssdp v0.0.4 h1:1IDwrghSKYM7yLf7XCzbByg2sJ/JcNOZRXS2jczTwz0= @@ -2148,6 +2141,9 @@ github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6 github.com/onflow/atree v0.6.1-0.20230711151834-86040b30171f/go.mod h1:xvP61FoOs95K7IYdIYRnNcYQGf4nbF/uuJ0tHf4DRuM= github.com/onflow/atree v0.8.0-rc.6 h1:GWgaylK24b5ta2Hq+TvyOF7X5tZLiLzMMn7lEt59fsA= github.com/onflow/atree v0.8.0-rc.6/go.mod h1:yccR+LR7xc1Jdic0mrjocbHvUD7lnVvg8/Ct1AA5zBo= +github.com/onflow/cadence v1.0.0-M3/go.mod h1:odXGZZ/wGNA5mwT8bC9v8u8EXACHllB2ABSZK65TGL8= +github.com/onflow/cadence v1.0.0-preview.52 h1:hZ92e6lL2+PQa3C1i5jJh0zZYFdW89+X1MS0Bkd6Ayo= +github.com/onflow/cadence v1.0.0-preview.52/go.mod h1:7wvvecnAZtYOspLOS3Lh+FuAmMeSrXhAWiycC3kQ1UU= github.com/onflow/crypto v0.25.0/go.mod h1:C8FbaX0x8y+FxWjbkHy0Q4EASCDR9bSPWZqlpCLYyVI= github.com/onflow/crypto v0.25.2 h1:GjHunqVt+vPcdqhxxhAXiMIF3YiLX7gTuTR5O+VG2ns= github.com/onflow/crypto v0.25.2/go.mod h1:fY7eLqUdMKV8EGOw301unP8h7PvLVy8/6gVR++/g0BY= @@ -2725,7 +2721,6 @@ golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0= golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -2750,7 +2745,6 @@ golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191007182048-72f939374954/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191109021931-daa7c04131f5/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= From 056a53c27f34d2730a9fd7a776cde6bb04f14ae6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Wed, 4 Sep 2024 22:33:20 -0700 Subject: [PATCH 18/72] start work on new public entitlements migration --- .../public_entitlements_migration.go | 279 ++++++++++++++++++ .../public_entitlements_migration_test.go | 67 +++++ 2 files changed, 346 insertions(+) create mode 100644 cmd/util/ledger/migrations/public_entitlements_migration.go create mode 100644 cmd/util/ledger/migrations/public_entitlements_migration_test.go diff --git a/cmd/util/ledger/migrations/public_entitlements_migration.go b/cmd/util/ledger/migrations/public_entitlements_migration.go new file mode 100644 index 00000000000..50af97a3e87 --- /dev/null +++ b/cmd/util/ledger/migrations/public_entitlements_migration.go @@ -0,0 +1,279 @@ +package migrations + +import ( + "errors" + "fmt" + + "github.com/onflow/cadence/migrations" + "github.com/onflow/cadence/runtime" + "github.com/onflow/cadence/runtime/common" + cadenceErrors "github.com/onflow/cadence/runtime/errors" + "github.com/onflow/cadence/runtime/interpreter" + "github.com/onflow/cadence/runtime/sema" + "github.com/rs/zerolog" + + "github.com/onflow/flow-go/cmd/util/ledger/reporters" + "github.com/onflow/flow-go/fvm/environment" + "github.com/onflow/flow-go/model/flow" +) + +type PublicEntitlementsMigrationReporter interface { + MigratedCapability(key interpreter.StorageKey, value *interpreter.IDCapabilityValue) + MigratedCapabilityController(key interpreter.StorageKey, value *interpreter.StorageCapabilityControllerValue) +} + +type PublicEntitlementsMigration struct { + Reporter PublicEntitlementsMigrationReporter +} + +var _ migrations.ValueMigration = &PublicEntitlementsMigration{} + +func (*PublicEntitlementsMigration) Name() string { + return "PublicEntitlementsMigration" +} + +func (*PublicEntitlementsMigration) Domains() map[string]struct{} { + return nil +} + +func (m *PublicEntitlementsMigration) Migrate( + storageKey interpreter.StorageKey, + _ interpreter.StorageMapKey, + value interpreter.Value, + _ *interpreter.Interpreter, + _ migrations.ValueMigrationPosition, +) ( + interpreter.Value, + error, +) { + switch value := value.(type) { + case *interpreter.IDCapabilityValue: + // TODO: + m.Reporter.MigratedCapability(storageKey, value) + + case *interpreter.StorageCapabilityControllerValue: + // TODO: + m.Reporter.MigratedCapabilityController(storageKey, value) + } + + return nil, nil +} + +func (*PublicEntitlementsMigration) CanSkip(valueType interpreter.StaticType) bool { + return CanSkipPublicEntitlementsMigration(valueType) +} + +func CanSkipPublicEntitlementsMigration(valueType interpreter.StaticType) bool { + switch valueType := valueType.(type) { + case *interpreter.DictionaryStaticType: + return CanSkipPublicEntitlementsMigration(valueType.KeyType) && + CanSkipPublicEntitlementsMigration(valueType.ValueType) + + case interpreter.ArrayStaticType: + return CanSkipPublicEntitlementsMigration(valueType.ElementType()) + + case *interpreter.OptionalStaticType: + return CanSkipPublicEntitlementsMigration(valueType.Type) + + case *interpreter.CapabilityStaticType: + return false + + case interpreter.PrimitiveStaticType: + + switch valueType { + case interpreter.PrimitiveStaticTypeCapability, + interpreter.PrimitiveStaticTypeStorageCapabilityController: + return false + + case interpreter.PrimitiveStaticTypeBool, + interpreter.PrimitiveStaticTypeVoid, + interpreter.PrimitiveStaticTypeAddress, + interpreter.PrimitiveStaticTypeMetaType, + interpreter.PrimitiveStaticTypeBlock, + interpreter.PrimitiveStaticTypeString, + interpreter.PrimitiveStaticTypeCharacter: + + return true + } + + if !valueType.IsDeprecated() { //nolint:staticcheck + semaType := valueType.SemaType() + + if sema.IsSubType(semaType, sema.NumberType) || + sema.IsSubType(semaType, sema.PathType) { + + return true + } + } + } + + return false +} + +type PublicEntitlementsMigrationOptions struct { + ChainID flow.ChainID + NWorker int + VerboseErrorOutput bool + LogVerboseDiff bool + DiffMigrations bool + CheckStorageHealthBeforeMigration bool +} + +func NewPublicEntitlementsValueMigration( + rwf reporters.ReportWriterFactory, + errorMessageHandler *errorMessageHandler, + programs map[runtime.Location]*interpreter.Program, + opts PublicEntitlementsMigrationOptions, +) *CadenceBaseMigration { + var diffReporter reporters.ReportWriter + if opts.DiffMigrations { + diffReporter = rwf.ReportWriter("public-entitlements-migration-diff") + } + + reporter := rwf.ReportWriter("public-entitlements-migration") + + return &CadenceBaseMigration{ + name: "public_entitlements_migration", + reporter: reporter, + diffReporter: diffReporter, + logVerboseDiff: opts.LogVerboseDiff, + verboseErrorOutput: opts.VerboseErrorOutput, + checkStorageHealthBeforeMigration: opts.CheckStorageHealthBeforeMigration, + valueMigrations: func( + _ *interpreter.Interpreter, + _ environment.Accounts, + _ *cadenceValueMigrationReporter, + ) []migrations.ValueMigration { + + return []migrations.ValueMigration{ + &PublicEntitlementsMigration{ + Reporter: &publicEntitlementsMigrationReporter{ + reportWriter: reporter, + errorMessageHandler: errorMessageHandler, + verboseErrorOutput: opts.VerboseErrorOutput, + }, + }, + } + }, + errorMessageHandler: errorMessageHandler, + programs: programs, + chainID: opts.ChainID, + } +} + +type publicEntitlementsMigrationReporter struct { + reportWriter reporters.ReportWriter + errorMessageHandler *errorMessageHandler + verboseErrorOutput bool +} + +var _ PublicEntitlementsMigrationReporter = &publicEntitlementsMigrationReporter{} +var _ migrations.Reporter = &publicEntitlementsMigrationReporter{} + +func (r *publicEntitlementsMigrationReporter) Migrated( + storageKey interpreter.StorageKey, + storageMapKey interpreter.StorageMapKey, + migration string, +) { + r.reportWriter.Write(cadenceValueMigrationEntry{ + StorageKey: storageKey, + StorageMapKey: storageMapKey, + Migration: migration, + }) +} + +func (r *publicEntitlementsMigrationReporter) Error(err error) { + + var migrationErr migrations.StorageMigrationError + + if !errors.As(err, &migrationErr) { + panic(cadenceErrors.NewUnreachableError()) + } + + message, showStack := r.errorMessageHandler.FormatError(migrationErr.Err) + + storageKey := migrationErr.StorageKey + storageMapKey := migrationErr.StorageMapKey + migration := migrationErr.Migration + + if showStack && len(migrationErr.Stack) > 0 { + message = fmt.Sprintf("%s\n%s", message, migrationErr.Stack) + } + + if r.verboseErrorOutput { + r.reportWriter.Write(cadenceValueMigrationFailureEntry{ + StorageKey: storageKey, + StorageMapKey: storageMapKey, + Migration: migration, + Message: message, + }) + } +} + +func (r *publicEntitlementsMigrationReporter) DictionaryKeyConflict(accountAddressPath interpreter.AddressPath) { + r.reportWriter.Write(dictionaryKeyConflictEntry{ + AddressPath: accountAddressPath, + }) +} + +func (r *publicEntitlementsMigrationReporter) MigratedCapability( + key interpreter.StorageKey, + value *interpreter.IDCapabilityValue, +) { + // TODO: +} + +func (r *publicEntitlementsMigrationReporter) MigratedCapabilityController( + key interpreter.StorageKey, + value *interpreter.StorageCapabilityControllerValue, +) { + // TODO: +} + +func NewPublicEntitlementsMigrations( + log zerolog.Logger, + rwf reporters.ReportWriterFactory, + opts PublicEntitlementsMigrationOptions, +) []NamedMigration { + + errorMessageHandler := &errorMessageHandler{} + + // The value migrations are run as account-based migrations, + // i.e. the migrations are only given the payloads for the account to be migrated. + // However, the migrations need to be able to get the code for contracts of any account. + // + // To achieve this, the contracts are extracted from the payloads once, + // before the value migrations are run. + + programs := make(map[common.Location]*interpreter.Program, 1000) + + return []NamedMigration{ + { + Name: "check-contracts", + Migrate: NewContractCheckingMigration( + log, + rwf, + opts.ChainID, + opts.VerboseErrorOutput, + // TODO: what are the important locations? + map[common.AddressLocation]struct{}{}, + programs, + ), + }, + { + Name: "fix-public-entitlements", + Migrate: NewAccountBasedMigration( + log, + opts.NWorker, + []AccountBasedMigration{ + NewPublicEntitlementsValueMigration( + rwf, + errorMessageHandler, + programs, + opts, + ), + }, + ), + }, + } +} diff --git a/cmd/util/ledger/migrations/public_entitlements_migration_test.go b/cmd/util/ledger/migrations/public_entitlements_migration_test.go new file mode 100644 index 00000000000..a7b48cad7bd --- /dev/null +++ b/cmd/util/ledger/migrations/public_entitlements_migration_test.go @@ -0,0 +1,67 @@ +package migrations + +import ( + "testing" + + "github.com/rs/zerolog" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/cmd/util/ledger/util/registers" + "github.com/onflow/flow-go/model/flow" +) + +func TestPublicEntitlementMigration(t *testing.T) { + t.Parallel() + + const chainID = flow.Emulator + chain := chainID.Chain() + + const nWorker = 2 + + log := zerolog.New(zerolog.NewTestWriter(t)) + + bootstrapPayloads, err := newBootstrapPayloads(chainID) + require.NoError(t, err) + + registersByAccount, err := registers.NewByAccountFromPayloads(bootstrapPayloads) + require.NoError(t, err) + + tx := flow.NewTransactionBody(). + SetScript([]byte(` + transaction { + prepare(signer: auth(Storage, Capabilities) &Account) { + let cap = signer.capabilities.storage.issue(/storage/ints) + signer.storage.save([cap], to: /storage/caps) + } + } + `)). + AddAuthorizer(chain.ServiceAddress()) + + setupTx := NewTransactionBasedMigration( + tx, + chainID, + log, + map[flow.Address]struct{}{ + chain.ServiceAddress(): {}, + }, + ) + + err = setupTx(registersByAccount) + require.NoError(t, err) + + rwf := &testReportWriterFactory{} + + options := PublicEntitlementsMigrationOptions{ + ChainID: chainID, + NWorker: nWorker, + } + + migrations := NewPublicEntitlementsMigrations(log, rwf, options) + + for _, namedMigration := range migrations { + err = namedMigration.Migrate(registersByAccount) + require.NoError(t, err) + } + + // TODO: validate +} From d467b1f2d0998260789ca50569106056d20f058b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Thu, 5 Sep 2024 14:41:48 -0700 Subject: [PATCH 19/72] split migration into two: fix capability controllers, then capabilities --- .../migrations/fix_entitlements_migration.go | 381 ++++++++++++++++++ ....go => fix_entitlements_migration_test.go} | 4 +- .../public_entitlements_migration.go | 279 ------------- 3 files changed, 383 insertions(+), 281 deletions(-) create mode 100644 cmd/util/ledger/migrations/fix_entitlements_migration.go rename cmd/util/ledger/migrations/{public_entitlements_migration_test.go => fix_entitlements_migration_test.go} (92%) delete mode 100644 cmd/util/ledger/migrations/public_entitlements_migration.go diff --git a/cmd/util/ledger/migrations/fix_entitlements_migration.go b/cmd/util/ledger/migrations/fix_entitlements_migration.go new file mode 100644 index 00000000000..d6c3aaa40a2 --- /dev/null +++ b/cmd/util/ledger/migrations/fix_entitlements_migration.go @@ -0,0 +1,381 @@ +package migrations + +import ( + "errors" + "fmt" + + "github.com/onflow/cadence/migrations" + "github.com/onflow/cadence/runtime" + "github.com/onflow/cadence/runtime/common" + cadenceErrors "github.com/onflow/cadence/runtime/errors" + "github.com/onflow/cadence/runtime/interpreter" + "github.com/onflow/cadence/runtime/sema" + "github.com/rs/zerolog" + + "github.com/onflow/flow-go/cmd/util/ledger/reporters" + "github.com/onflow/flow-go/fvm/environment" + "github.com/onflow/flow-go/model/flow" +) + +// FixCapabilityControllerEntitlementsMigration + +type FixCapabilityControllerEntitlementsMigrationReporter interface { + MigratedCapabilityController( + key interpreter.StorageKey, + value *interpreter.StorageCapabilityControllerValue, + ) +} + +type FixCapabilityControllerEntitlementsMigration struct { + Reporter FixCapabilityControllerEntitlementsMigrationReporter +} + +var _ migrations.ValueMigration = &FixCapabilityControllerEntitlementsMigration{} + +func (*FixCapabilityControllerEntitlementsMigration) Name() string { + return "FixCapabilityControllerEntitlementsMigration" +} + +func (*FixCapabilityControllerEntitlementsMigration) Domains() map[string]struct{} { + return nil +} + +func (m *FixCapabilityControllerEntitlementsMigration) Migrate( + storageKey interpreter.StorageKey, + _ interpreter.StorageMapKey, + value interpreter.Value, + _ *interpreter.Interpreter, + _ migrations.ValueMigrationPosition, +) ( + interpreter.Value, + error, +) { + if capability, ok := value.(*interpreter.StorageCapabilityControllerValue); ok { + // TODO: + m.Reporter.MigratedCapabilityController(storageKey, capability) + } + + return nil, nil +} + +func (*FixCapabilityControllerEntitlementsMigration) CanSkip(valueType interpreter.StaticType) bool { + return CanSkipFixEntitlementsMigration(valueType) +} + +// FixCapabilityEntitlementsMigration + +type FixCapabilityEntitlementsMigrationReporter interface { + MigratedCapability( + key interpreter.StorageKey, + value *interpreter.IDCapabilityValue, + ) +} + +type FixCapabilityEntitlementsMigration struct { + Reporter FixCapabilityEntitlementsMigrationReporter +} + +var _ migrations.ValueMigration = &FixCapabilityEntitlementsMigration{} + +func (*FixCapabilityEntitlementsMigration) Name() string { + return "FixCapabilityEntitlementsMigration" +} + +func (*FixCapabilityEntitlementsMigration) Domains() map[string]struct{} { + return nil +} + +func (m *FixCapabilityEntitlementsMigration) Migrate( + storageKey interpreter.StorageKey, + _ interpreter.StorageMapKey, + value interpreter.Value, + _ *interpreter.Interpreter, + _ migrations.ValueMigrationPosition, +) ( + interpreter.Value, + error, +) { + if capability, ok := value.(*interpreter.IDCapabilityValue); ok { + // TODO: + m.Reporter.MigratedCapability(storageKey, capability) + } + + return nil, nil +} + +func (*FixCapabilityEntitlementsMigration) CanSkip(valueType interpreter.StaticType) bool { + return CanSkipFixEntitlementsMigration(valueType) +} + +func CanSkipFixEntitlementsMigration(valueType interpreter.StaticType) bool { + switch valueType := valueType.(type) { + case *interpreter.DictionaryStaticType: + return CanSkipFixEntitlementsMigration(valueType.KeyType) && + CanSkipFixEntitlementsMigration(valueType.ValueType) + + case interpreter.ArrayStaticType: + return CanSkipFixEntitlementsMigration(valueType.ElementType()) + + case *interpreter.OptionalStaticType: + return CanSkipFixEntitlementsMigration(valueType.Type) + + case *interpreter.CapabilityStaticType: + return false + + case interpreter.PrimitiveStaticType: + + switch valueType { + case interpreter.PrimitiveStaticTypeCapability, + interpreter.PrimitiveStaticTypeStorageCapabilityController: + return false + + case interpreter.PrimitiveStaticTypeBool, + interpreter.PrimitiveStaticTypeVoid, + interpreter.PrimitiveStaticTypeAddress, + interpreter.PrimitiveStaticTypeMetaType, + interpreter.PrimitiveStaticTypeBlock, + interpreter.PrimitiveStaticTypeString, + interpreter.PrimitiveStaticTypeCharacter: + + return true + } + + if !valueType.IsDeprecated() { //nolint:staticcheck + semaType := valueType.SemaType() + + if sema.IsSubType(semaType, sema.NumberType) || + sema.IsSubType(semaType, sema.PathType) { + + return true + } + } + } + + return false +} + +type FixEntitlementsMigrationOptions struct { + ChainID flow.ChainID + NWorker int + VerboseErrorOutput bool + LogVerboseDiff bool + DiffMigrations bool + CheckStorageHealthBeforeMigration bool +} + +func NewFixCapabilityControllerEntitlementsMigration( + rwf reporters.ReportWriterFactory, + errorMessageHandler *errorMessageHandler, + programs map[runtime.Location]*interpreter.Program, + opts FixEntitlementsMigrationOptions, +) *CadenceBaseMigration { + var diffReporter reporters.ReportWriter + if opts.DiffMigrations { + diffReporter = rwf.ReportWriter("fix-capability-controller-entitlements-migration-diff") + } + + reporter := rwf.ReportWriter("fix-capability-controller-entitlements-migration") + + return &CadenceBaseMigration{ + name: "fix_capability_controller_entitlements_migration", + reporter: reporter, + diffReporter: diffReporter, + logVerboseDiff: opts.LogVerboseDiff, + verboseErrorOutput: opts.VerboseErrorOutput, + checkStorageHealthBeforeMigration: opts.CheckStorageHealthBeforeMigration, + valueMigrations: func( + _ *interpreter.Interpreter, + _ environment.Accounts, + _ *cadenceValueMigrationReporter, + ) []migrations.ValueMigration { + + return []migrations.ValueMigration{ + &FixCapabilityControllerEntitlementsMigration{ + Reporter: &fixEntitlementsMigrationReporter{ + reportWriter: reporter, + errorMessageHandler: errorMessageHandler, + verboseErrorOutput: opts.VerboseErrorOutput, + }, + }, + } + }, + errorMessageHandler: errorMessageHandler, + programs: programs, + chainID: opts.ChainID, + } +} + +func NewFixCapabilityEntitlementsMigration( + rwf reporters.ReportWriterFactory, + errorMessageHandler *errorMessageHandler, + programs map[runtime.Location]*interpreter.Program, + opts FixEntitlementsMigrationOptions, +) *CadenceBaseMigration { + var diffReporter reporters.ReportWriter + if opts.DiffMigrations { + diffReporter = rwf.ReportWriter("fix-capability-entitlements-migration-diff") + } + + reporter := rwf.ReportWriter("fix-capability-entitlements-migration") + + return &CadenceBaseMigration{ + name: "fix_capability_entitlements_migration", + reporter: reporter, + diffReporter: diffReporter, + logVerboseDiff: opts.LogVerboseDiff, + verboseErrorOutput: opts.VerboseErrorOutput, + checkStorageHealthBeforeMigration: opts.CheckStorageHealthBeforeMigration, + valueMigrations: func( + _ *interpreter.Interpreter, + _ environment.Accounts, + _ *cadenceValueMigrationReporter, + ) []migrations.ValueMigration { + + return []migrations.ValueMigration{ + &FixCapabilityEntitlementsMigration{ + Reporter: &fixEntitlementsMigrationReporter{ + reportWriter: reporter, + errorMessageHandler: errorMessageHandler, + verboseErrorOutput: opts.VerboseErrorOutput, + }, + }, + } + }, + errorMessageHandler: errorMessageHandler, + programs: programs, + chainID: opts.ChainID, + } +} + +type fixEntitlementsMigrationReporter struct { + reportWriter reporters.ReportWriter + errorMessageHandler *errorMessageHandler + verboseErrorOutput bool +} + +var _ FixCapabilityEntitlementsMigrationReporter = &fixEntitlementsMigrationReporter{} +var _ FixCapabilityControllerEntitlementsMigrationReporter = &fixEntitlementsMigrationReporter{} +var _ migrations.Reporter = &fixEntitlementsMigrationReporter{} + +func (r *fixEntitlementsMigrationReporter) Migrated( + storageKey interpreter.StorageKey, + storageMapKey interpreter.StorageMapKey, + migration string, +) { + r.reportWriter.Write(cadenceValueMigrationEntry{ + StorageKey: storageKey, + StorageMapKey: storageMapKey, + Migration: migration, + }) +} + +func (r *fixEntitlementsMigrationReporter) Error(err error) { + + var migrationErr migrations.StorageMigrationError + + if !errors.As(err, &migrationErr) { + panic(cadenceErrors.NewUnreachableError()) + } + + message, showStack := r.errorMessageHandler.FormatError(migrationErr.Err) + + storageKey := migrationErr.StorageKey + storageMapKey := migrationErr.StorageMapKey + migration := migrationErr.Migration + + if showStack && len(migrationErr.Stack) > 0 { + message = fmt.Sprintf("%s\n%s", message, migrationErr.Stack) + } + + if r.verboseErrorOutput { + r.reportWriter.Write(cadenceValueMigrationFailureEntry{ + StorageKey: storageKey, + StorageMapKey: storageMapKey, + Migration: migration, + Message: message, + }) + } +} + +func (r *fixEntitlementsMigrationReporter) DictionaryKeyConflict(accountAddressPath interpreter.AddressPath) { + r.reportWriter.Write(dictionaryKeyConflictEntry{ + AddressPath: accountAddressPath, + }) +} + +func (r *fixEntitlementsMigrationReporter) MigratedCapability( + key interpreter.StorageKey, + value *interpreter.IDCapabilityValue, +) { + // TODO: +} + +func (r *fixEntitlementsMigrationReporter) MigratedCapabilityController( + key interpreter.StorageKey, + value *interpreter.StorageCapabilityControllerValue, +) { + // TODO: +} + +func NewFixEntitlementsMigrations( + log zerolog.Logger, + rwf reporters.ReportWriterFactory, + opts FixEntitlementsMigrationOptions, +) []NamedMigration { + + errorMessageHandler := &errorMessageHandler{} + + // The value migrations are run as account-based migrations, + // i.e. the migrations are only given the payloads for the account to be migrated. + // However, the migrations need to be able to get the code for contracts of any account. + // + // To achieve this, the contracts are extracted from the payloads once, + // before the value migrations are run. + + programs := make(map[common.Location]*interpreter.Program, 1000) + + return []NamedMigration{ + { + Name: "check-contracts", + Migrate: NewContractCheckingMigration( + log, + rwf, + opts.ChainID, + opts.VerboseErrorOutput, + // TODO: what are the important locations? + map[common.AddressLocation]struct{}{}, + programs, + ), + }, + { + Name: "fix-capability-controller-entitlements", + Migrate: NewAccountBasedMigration( + log, + opts.NWorker, + []AccountBasedMigration{ + NewFixCapabilityControllerEntitlementsMigration( + rwf, + errorMessageHandler, + programs, + opts, + ), + }, + ), + }, + { + Name: "fix-capability-entitlements", + Migrate: NewAccountBasedMigration( + log, + opts.NWorker, + []AccountBasedMigration{ + NewFixCapabilityEntitlementsMigration( + rwf, + errorMessageHandler, + programs, + opts, + ), + }, + ), + }, + } +} diff --git a/cmd/util/ledger/migrations/public_entitlements_migration_test.go b/cmd/util/ledger/migrations/fix_entitlements_migration_test.go similarity index 92% rename from cmd/util/ledger/migrations/public_entitlements_migration_test.go rename to cmd/util/ledger/migrations/fix_entitlements_migration_test.go index a7b48cad7bd..bc405050b40 100644 --- a/cmd/util/ledger/migrations/public_entitlements_migration_test.go +++ b/cmd/util/ledger/migrations/fix_entitlements_migration_test.go @@ -51,12 +51,12 @@ func TestPublicEntitlementMigration(t *testing.T) { rwf := &testReportWriterFactory{} - options := PublicEntitlementsMigrationOptions{ + options := FixEntitlementsMigrationOptions{ ChainID: chainID, NWorker: nWorker, } - migrations := NewPublicEntitlementsMigrations(log, rwf, options) + migrations := NewFixEntitlementsMigrations(log, rwf, options) for _, namedMigration := range migrations { err = namedMigration.Migrate(registersByAccount) diff --git a/cmd/util/ledger/migrations/public_entitlements_migration.go b/cmd/util/ledger/migrations/public_entitlements_migration.go deleted file mode 100644 index 50af97a3e87..00000000000 --- a/cmd/util/ledger/migrations/public_entitlements_migration.go +++ /dev/null @@ -1,279 +0,0 @@ -package migrations - -import ( - "errors" - "fmt" - - "github.com/onflow/cadence/migrations" - "github.com/onflow/cadence/runtime" - "github.com/onflow/cadence/runtime/common" - cadenceErrors "github.com/onflow/cadence/runtime/errors" - "github.com/onflow/cadence/runtime/interpreter" - "github.com/onflow/cadence/runtime/sema" - "github.com/rs/zerolog" - - "github.com/onflow/flow-go/cmd/util/ledger/reporters" - "github.com/onflow/flow-go/fvm/environment" - "github.com/onflow/flow-go/model/flow" -) - -type PublicEntitlementsMigrationReporter interface { - MigratedCapability(key interpreter.StorageKey, value *interpreter.IDCapabilityValue) - MigratedCapabilityController(key interpreter.StorageKey, value *interpreter.StorageCapabilityControllerValue) -} - -type PublicEntitlementsMigration struct { - Reporter PublicEntitlementsMigrationReporter -} - -var _ migrations.ValueMigration = &PublicEntitlementsMigration{} - -func (*PublicEntitlementsMigration) Name() string { - return "PublicEntitlementsMigration" -} - -func (*PublicEntitlementsMigration) Domains() map[string]struct{} { - return nil -} - -func (m *PublicEntitlementsMigration) Migrate( - storageKey interpreter.StorageKey, - _ interpreter.StorageMapKey, - value interpreter.Value, - _ *interpreter.Interpreter, - _ migrations.ValueMigrationPosition, -) ( - interpreter.Value, - error, -) { - switch value := value.(type) { - case *interpreter.IDCapabilityValue: - // TODO: - m.Reporter.MigratedCapability(storageKey, value) - - case *interpreter.StorageCapabilityControllerValue: - // TODO: - m.Reporter.MigratedCapabilityController(storageKey, value) - } - - return nil, nil -} - -func (*PublicEntitlementsMigration) CanSkip(valueType interpreter.StaticType) bool { - return CanSkipPublicEntitlementsMigration(valueType) -} - -func CanSkipPublicEntitlementsMigration(valueType interpreter.StaticType) bool { - switch valueType := valueType.(type) { - case *interpreter.DictionaryStaticType: - return CanSkipPublicEntitlementsMigration(valueType.KeyType) && - CanSkipPublicEntitlementsMigration(valueType.ValueType) - - case interpreter.ArrayStaticType: - return CanSkipPublicEntitlementsMigration(valueType.ElementType()) - - case *interpreter.OptionalStaticType: - return CanSkipPublicEntitlementsMigration(valueType.Type) - - case *interpreter.CapabilityStaticType: - return false - - case interpreter.PrimitiveStaticType: - - switch valueType { - case interpreter.PrimitiveStaticTypeCapability, - interpreter.PrimitiveStaticTypeStorageCapabilityController: - return false - - case interpreter.PrimitiveStaticTypeBool, - interpreter.PrimitiveStaticTypeVoid, - interpreter.PrimitiveStaticTypeAddress, - interpreter.PrimitiveStaticTypeMetaType, - interpreter.PrimitiveStaticTypeBlock, - interpreter.PrimitiveStaticTypeString, - interpreter.PrimitiveStaticTypeCharacter: - - return true - } - - if !valueType.IsDeprecated() { //nolint:staticcheck - semaType := valueType.SemaType() - - if sema.IsSubType(semaType, sema.NumberType) || - sema.IsSubType(semaType, sema.PathType) { - - return true - } - } - } - - return false -} - -type PublicEntitlementsMigrationOptions struct { - ChainID flow.ChainID - NWorker int - VerboseErrorOutput bool - LogVerboseDiff bool - DiffMigrations bool - CheckStorageHealthBeforeMigration bool -} - -func NewPublicEntitlementsValueMigration( - rwf reporters.ReportWriterFactory, - errorMessageHandler *errorMessageHandler, - programs map[runtime.Location]*interpreter.Program, - opts PublicEntitlementsMigrationOptions, -) *CadenceBaseMigration { - var diffReporter reporters.ReportWriter - if opts.DiffMigrations { - diffReporter = rwf.ReportWriter("public-entitlements-migration-diff") - } - - reporter := rwf.ReportWriter("public-entitlements-migration") - - return &CadenceBaseMigration{ - name: "public_entitlements_migration", - reporter: reporter, - diffReporter: diffReporter, - logVerboseDiff: opts.LogVerboseDiff, - verboseErrorOutput: opts.VerboseErrorOutput, - checkStorageHealthBeforeMigration: opts.CheckStorageHealthBeforeMigration, - valueMigrations: func( - _ *interpreter.Interpreter, - _ environment.Accounts, - _ *cadenceValueMigrationReporter, - ) []migrations.ValueMigration { - - return []migrations.ValueMigration{ - &PublicEntitlementsMigration{ - Reporter: &publicEntitlementsMigrationReporter{ - reportWriter: reporter, - errorMessageHandler: errorMessageHandler, - verboseErrorOutput: opts.VerboseErrorOutput, - }, - }, - } - }, - errorMessageHandler: errorMessageHandler, - programs: programs, - chainID: opts.ChainID, - } -} - -type publicEntitlementsMigrationReporter struct { - reportWriter reporters.ReportWriter - errorMessageHandler *errorMessageHandler - verboseErrorOutput bool -} - -var _ PublicEntitlementsMigrationReporter = &publicEntitlementsMigrationReporter{} -var _ migrations.Reporter = &publicEntitlementsMigrationReporter{} - -func (r *publicEntitlementsMigrationReporter) Migrated( - storageKey interpreter.StorageKey, - storageMapKey interpreter.StorageMapKey, - migration string, -) { - r.reportWriter.Write(cadenceValueMigrationEntry{ - StorageKey: storageKey, - StorageMapKey: storageMapKey, - Migration: migration, - }) -} - -func (r *publicEntitlementsMigrationReporter) Error(err error) { - - var migrationErr migrations.StorageMigrationError - - if !errors.As(err, &migrationErr) { - panic(cadenceErrors.NewUnreachableError()) - } - - message, showStack := r.errorMessageHandler.FormatError(migrationErr.Err) - - storageKey := migrationErr.StorageKey - storageMapKey := migrationErr.StorageMapKey - migration := migrationErr.Migration - - if showStack && len(migrationErr.Stack) > 0 { - message = fmt.Sprintf("%s\n%s", message, migrationErr.Stack) - } - - if r.verboseErrorOutput { - r.reportWriter.Write(cadenceValueMigrationFailureEntry{ - StorageKey: storageKey, - StorageMapKey: storageMapKey, - Migration: migration, - Message: message, - }) - } -} - -func (r *publicEntitlementsMigrationReporter) DictionaryKeyConflict(accountAddressPath interpreter.AddressPath) { - r.reportWriter.Write(dictionaryKeyConflictEntry{ - AddressPath: accountAddressPath, - }) -} - -func (r *publicEntitlementsMigrationReporter) MigratedCapability( - key interpreter.StorageKey, - value *interpreter.IDCapabilityValue, -) { - // TODO: -} - -func (r *publicEntitlementsMigrationReporter) MigratedCapabilityController( - key interpreter.StorageKey, - value *interpreter.StorageCapabilityControllerValue, -) { - // TODO: -} - -func NewPublicEntitlementsMigrations( - log zerolog.Logger, - rwf reporters.ReportWriterFactory, - opts PublicEntitlementsMigrationOptions, -) []NamedMigration { - - errorMessageHandler := &errorMessageHandler{} - - // The value migrations are run as account-based migrations, - // i.e. the migrations are only given the payloads for the account to be migrated. - // However, the migrations need to be able to get the code for contracts of any account. - // - // To achieve this, the contracts are extracted from the payloads once, - // before the value migrations are run. - - programs := make(map[common.Location]*interpreter.Program, 1000) - - return []NamedMigration{ - { - Name: "check-contracts", - Migrate: NewContractCheckingMigration( - log, - rwf, - opts.ChainID, - opts.VerboseErrorOutput, - // TODO: what are the important locations? - map[common.AddressLocation]struct{}{}, - programs, - ), - }, - { - Name: "fix-public-entitlements", - Migrate: NewAccountBasedMigration( - log, - opts.NWorker, - []AccountBasedMigration{ - NewPublicEntitlementsValueMigration( - rwf, - errorMessageHandler, - programs, - opts, - ), - }, - ), - }, - } -} From 4e951816090c0aa821c3ec65dfb62fe0ba215565 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Thu, 5 Sep 2024 16:41:03 -0700 Subject: [PATCH 20/72] read link migration report into capability ID to path mapping --- .../migrations/fix_entitlements_migration.go | 70 +++++++++++++++++++ .../fix_entitlements_migration_test.go | 32 ++++++++- 2 files changed, 101 insertions(+), 1 deletion(-) diff --git a/cmd/util/ledger/migrations/fix_entitlements_migration.go b/cmd/util/ledger/migrations/fix_entitlements_migration.go index d6c3aaa40a2..acbd5940e2d 100644 --- a/cmd/util/ledger/migrations/fix_entitlements_migration.go +++ b/cmd/util/ledger/migrations/fix_entitlements_migration.go @@ -1,8 +1,10 @@ package migrations import ( + "encoding/json" "errors" "fmt" + "io" "github.com/onflow/cadence/migrations" "github.com/onflow/cadence/runtime" @@ -317,6 +319,74 @@ func (r *fixEntitlementsMigrationReporter) MigratedCapabilityController( // TODO: } +type AccountCapabilityControllerID struct { + Address common.Address + CapabilityID uint64 +} + +// ReadLinkMigrationReport reads a link migration report from the given reader. +// The report is expected to be a JSON array of objects with the following structure: +// +// [ +// +// {"kind":"link-migration-success","account_address":"0x1","path":"/public/foo","capability_id":1}, +// {"kind":"link-migration-success","account_address":"0x2","path":"/private/bar","capability_id":2} +// +// ] +// +// The function returns a mapping from account capability controller IDs to paths. +func ReadLinkMigrationReport(reader io.Reader) (map[AccountCapabilityControllerID]string, error) { + mapping := make(map[AccountCapabilityControllerID]string) + + dec := json.NewDecoder(reader) + + token, err := dec.Token() + if err != nil { + return nil, fmt.Errorf("failed to read token: %w", err) + } + if token != json.Delim('[') { + return nil, fmt.Errorf("expected start of array, got %s", token) + } + + for dec.More() { + var entry struct { + Kind string `json:"kind"` + Address string `json:"account_address"` + Path string `json:"path"` + CapabilityID uint64 `json:"capability_id"` + } + err := dec.Decode(&entry) + if err != nil { + return nil, fmt.Errorf("failed to decode entry: %w", err) + } + + if entry.Kind != "link-migration-success" { + continue + } + + address, err := common.HexToAddress(entry.Address) + if err != nil { + return nil, fmt.Errorf("failed to parse address: %w", err) + } + + key := AccountCapabilityControllerID{ + Address: address, + CapabilityID: entry.CapabilityID, + } + mapping[key] = entry.Path + } + + token, err = dec.Token() + if err != nil { + return nil, fmt.Errorf("failed to read token: %w", err) + } + if token != json.Delim(']') { + return nil, fmt.Errorf("expected end of array, got %s", token) + } + + return mapping, nil +} + func NewFixEntitlementsMigrations( log zerolog.Logger, rwf reporters.ReportWriterFactory, diff --git a/cmd/util/ledger/migrations/fix_entitlements_migration_test.go b/cmd/util/ledger/migrations/fix_entitlements_migration_test.go index bc405050b40..29ea9887b9f 100644 --- a/cmd/util/ledger/migrations/fix_entitlements_migration_test.go +++ b/cmd/util/ledger/migrations/fix_entitlements_migration_test.go @@ -1,8 +1,10 @@ package migrations import ( + "strings" "testing" + "github.com/onflow/cadence/runtime/common" "github.com/rs/zerolog" "github.com/stretchr/testify/require" @@ -10,7 +12,7 @@ import ( "github.com/onflow/flow-go/model/flow" ) -func TestPublicEntitlementMigration(t *testing.T) { +func TestFixEntitlementMigrations(t *testing.T) { t.Parallel() const chainID = flow.Emulator @@ -65,3 +67,31 @@ func TestPublicEntitlementMigration(t *testing.T) { // TODO: validate } + +func TestReadLinkMigrationReport(t *testing.T) { + t.Parallel() + + reader := strings.NewReader(` + [ + {"kind":"link-migration-success","account_address":"0x1","path":"/public/foo","capability_id":1}, + {"kind":"link-migration-success","account_address":"0x2","path":"/private/bar","capability_id":2} + ] + `) + + mapping, err := ReadLinkMigrationReport(reader) + require.NoError(t, err) + + require.Equal(t, + map[AccountCapabilityControllerID]string{ + { + Address: common.MustBytesToAddress([]byte{0x1}), + CapabilityID: 1, + }: "/public/foo", + { + Address: common.MustBytesToAddress([]byte{0x2}), + CapabilityID: 2, + }: "/private/bar", + }, + mapping, + ) +} From dd2fd52746f9b143f9fe5400e743eb74b03a4bab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Thu, 5 Sep 2024 16:55:18 -0700 Subject: [PATCH 21/72] readd link report into mapping from address path to link info --- .../migrations/fix_entitlements_migration.go | 76 +++++++++++++++++-- .../fix_entitlements_migration_test.go | 41 ++++++++++ 2 files changed, 111 insertions(+), 6 deletions(-) diff --git a/cmd/util/ledger/migrations/fix_entitlements_migration.go b/cmd/util/ledger/migrations/fix_entitlements_migration.go index acbd5940e2d..803ddefed9d 100644 --- a/cmd/util/ledger/migrations/fix_entitlements_migration.go +++ b/cmd/util/ledger/migrations/fix_entitlements_migration.go @@ -327,12 +327,9 @@ type AccountCapabilityControllerID struct { // ReadLinkMigrationReport reads a link migration report from the given reader. // The report is expected to be a JSON array of objects with the following structure: // -// [ -// -// {"kind":"link-migration-success","account_address":"0x1","path":"/public/foo","capability_id":1}, -// {"kind":"link-migration-success","account_address":"0x2","path":"/private/bar","capability_id":2} -// -// ] +// [ +// {"kind":"link-migration-success","account_address":"0x1","path":"/public/foo","capability_id":1}, +// ] // // The function returns a mapping from account capability controller IDs to paths. func ReadLinkMigrationReport(reader io.Reader) (map[AccountCapabilityControllerID]string, error) { @@ -387,6 +384,73 @@ func ReadLinkMigrationReport(reader io.Reader) (map[AccountCapabilityControllerI return mapping, nil } +type LinkInfo struct { + BorrowType common.TypeID + AccessibleMembers []string +} + +// ReadLinkReport reads a link report from the given reader. +// The report is expected to be a JSON array of objects with the following structure: +// +// [ +// {"address":"0x1","identifier":"foo","linkType":"&Foo","accessibleMembers":["foo"]} +// ] +// +// The function returns a mapping from account paths to link info. +func ReadLinkReport(reader io.Reader) (map[interpreter.AddressPath]LinkInfo, error) { + mapping := make(map[interpreter.AddressPath]LinkInfo) + + dec := json.NewDecoder(reader) + + token, err := dec.Token() + if err != nil { + return nil, fmt.Errorf("failed to read token: %w", err) + } + if token != json.Delim('[') { + return nil, fmt.Errorf("expected start of array, got %s", token) + } + + for dec.More() { + var entry struct { + Address string `json:"address"` + Identifier string `json:"identifier"` + LinkTypeID string `json:"linkType"` + AccessibleMembers []string `json:"accessibleMembers"` + } + err := dec.Decode(&entry) + if err != nil { + return nil, fmt.Errorf("failed to decode entry: %w", err) + } + + address, err := common.HexToAddress(entry.Address) + if err != nil { + return nil, fmt.Errorf("failed to parse address: %w", err) + } + + key := interpreter.AddressPath{ + Address: address, + Path: interpreter.PathValue{ + Domain: common.PathDomainPublic, + Identifier: entry.Identifier, + }, + } + mapping[key] = LinkInfo{ + BorrowType: common.TypeID(entry.LinkTypeID), + AccessibleMembers: entry.AccessibleMembers, + } + } + + token, err = dec.Token() + if err != nil { + return nil, fmt.Errorf("failed to read token: %w", err) + } + if token != json.Delim(']') { + return nil, fmt.Errorf("expected end of array, got %s", token) + } + + return mapping, nil +} + func NewFixEntitlementsMigrations( log zerolog.Logger, rwf reporters.ReportWriterFactory, diff --git a/cmd/util/ledger/migrations/fix_entitlements_migration_test.go b/cmd/util/ledger/migrations/fix_entitlements_migration_test.go index 29ea9887b9f..41433bb9aac 100644 --- a/cmd/util/ledger/migrations/fix_entitlements_migration_test.go +++ b/cmd/util/ledger/migrations/fix_entitlements_migration_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/onflow/cadence/runtime/common" + "github.com/onflow/cadence/runtime/interpreter" "github.com/rs/zerolog" "github.com/stretchr/testify/require" @@ -95,3 +96,43 @@ func TestReadLinkMigrationReport(t *testing.T) { mapping, ) } + +func TestReadLinkReport(t *testing.T) { + t.Parallel() + + reader := strings.NewReader(` + [ + {"address":"0x1","identifier":"foo","linkType":"&Foo","accessibleMembers":["foo"]}, + {"address":"0x2","identifier":"bar","linkType":"&Bar","accessibleMembers":null} + ] + `) + + mapping, err := ReadLinkReport(reader) + require.NoError(t, err) + + require.Equal(t, + map[interpreter.AddressPath]LinkInfo{ + { + Address: common.MustBytesToAddress([]byte{0x1}), + Path: interpreter.PathValue{ + Domain: common.PathDomainPublic, + Identifier: "foo", + }, + }: { + BorrowType: "&Foo", + AccessibleMembers: []string{"foo"}, + }, + { + Address: common.MustBytesToAddress([]byte{0x2}), + Path: interpreter.PathValue{ + Domain: common.PathDomainPublic, + Identifier: "bar", + }, + }: { + BorrowType: "&Bar", + AccessibleMembers: nil, + }, + }, + mapping, + ) +} From 6ca4d0f30b27b2b44f8ff68dd615b5131c2e9025 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Thu, 5 Sep 2024 17:45:38 -0700 Subject: [PATCH 22/72] improve report parsing, use reports in cap con migration --- .../migrations/fix_entitlements_migration.go | 123 +++++++++++++----- .../fix_entitlements_migration_test.go | 29 ++--- 2 files changed, 100 insertions(+), 52 deletions(-) diff --git a/cmd/util/ledger/migrations/fix_entitlements_migration.go b/cmd/util/ledger/migrations/fix_entitlements_migration.go index 803ddefed9d..991e2d3acda 100644 --- a/cmd/util/ledger/migrations/fix_entitlements_migration.go +++ b/cmd/util/ledger/migrations/fix_entitlements_migration.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "io" + "strings" "github.com/onflow/cadence/migrations" "github.com/onflow/cadence/runtime" @@ -12,7 +13,9 @@ import ( cadenceErrors "github.com/onflow/cadence/runtime/errors" "github.com/onflow/cadence/runtime/interpreter" "github.com/onflow/cadence/runtime/sema" + "github.com/onflow/cadence/runtime/stdlib" "github.com/rs/zerolog" + "github.com/rs/zerolog/log" "github.com/onflow/flow-go/cmd/util/ledger/reporters" "github.com/onflow/flow-go/fvm/environment" @@ -23,13 +26,16 @@ import ( type FixCapabilityControllerEntitlementsMigrationReporter interface { MigratedCapabilityController( - key interpreter.StorageKey, - value *interpreter.StorageCapabilityControllerValue, + storageKey interpreter.StorageKey, + capabilityController *interpreter.StorageCapabilityControllerValue, + linkInfo LinkInfo, ) } type FixCapabilityControllerEntitlementsMigration struct { - Reporter FixCapabilityControllerEntitlementsMigrationReporter + Reporter FixCapabilityControllerEntitlementsMigrationReporter + LinkMigrationReport LinkMigrationReport + LinkReport PublicLinkReport } var _ migrations.ValueMigration = &FixCapabilityControllerEntitlementsMigration{} @@ -38,8 +44,12 @@ func (*FixCapabilityControllerEntitlementsMigration) Name() string { return "FixCapabilityControllerEntitlementsMigration" } +var fixCapabilityControllerEntitlementsMigrationDomains = map[string]struct{}{ + stdlib.CapabilityControllerStorageDomain: {}, +} + func (*FixCapabilityControllerEntitlementsMigration) Domains() map[string]struct{} { - return nil + return fixCapabilityControllerEntitlementsMigrationDomains } func (m *FixCapabilityControllerEntitlementsMigration) Migrate( @@ -52,14 +62,48 @@ func (m *FixCapabilityControllerEntitlementsMigration) Migrate( interpreter.Value, error, ) { - if capability, ok := value.(*interpreter.StorageCapabilityControllerValue); ok { - // TODO: - m.Reporter.MigratedCapabilityController(storageKey, capability) + if capabilityController, ok := value.(*interpreter.StorageCapabilityControllerValue); ok { + address := storageKey.Address + capabilityID := capabilityController.CapabilityID + + publicPathIdentifier := m.capabilityControllerPublicPathIdentifier(address, capabilityID) + if publicPathIdentifier == "" { + log.Warn().Msgf("missing capability controller path for account %s, capability ID %d", address, capabilityID) + return nil, nil + } + + linkInfo := m.publicPathLinkInfo(address, publicPathIdentifier) + + m.Reporter.MigratedCapabilityController( + storageKey, + capabilityController, + linkInfo, + ) } return nil, nil } +func (m *FixCapabilityControllerEntitlementsMigration) capabilityControllerPublicPathIdentifier( + address common.Address, + capabilityID interpreter.UInt64Value, +) string { + return m.LinkMigrationReport[AccountCapabilityControllerID{ + Address: address, + CapabilityID: uint64(capabilityID), + }] +} + +func (m *FixCapabilityControllerEntitlementsMigration) publicPathLinkInfo( + address common.Address, + publicPathIdentifier string, +) LinkInfo { + return m.LinkReport[AddressPublicPath{ + Address: address, + Identifier: publicPathIdentifier, + }] +} + func (*FixCapabilityControllerEntitlementsMigration) CanSkip(valueType interpreter.StaticType) bool { return CanSkipFixEntitlementsMigration(valueType) } @@ -68,8 +112,8 @@ func (*FixCapabilityControllerEntitlementsMigration) CanSkip(valueType interpret type FixCapabilityEntitlementsMigrationReporter interface { MigratedCapability( - key interpreter.StorageKey, - value *interpreter.IDCapabilityValue, + storageKey interpreter.StorageKey, + capability *interpreter.IDCapabilityValue, ) } @@ -306,15 +350,16 @@ func (r *fixEntitlementsMigrationReporter) DictionaryKeyConflict(accountAddressP } func (r *fixEntitlementsMigrationReporter) MigratedCapability( - key interpreter.StorageKey, - value *interpreter.IDCapabilityValue, + _ interpreter.StorageKey, + _ *interpreter.IDCapabilityValue, ) { // TODO: } func (r *fixEntitlementsMigrationReporter) MigratedCapabilityController( - key interpreter.StorageKey, - value *interpreter.StorageCapabilityControllerValue, + _ interpreter.StorageKey, + _ *interpreter.StorageCapabilityControllerValue, + _ LinkInfo, ) { // TODO: } @@ -324,16 +369,19 @@ type AccountCapabilityControllerID struct { CapabilityID uint64 } -// ReadLinkMigrationReport reads a link migration report from the given reader. +// LinkMigrationReport is a mapping from account capability controller IDs to path identifier. +type LinkMigrationReport map[AccountCapabilityControllerID]string + +// ReadPublicLinkMigrationReport reads a link migration report from the given reader, +// and extracts the public paths that were migrated. +// // The report is expected to be a JSON array of objects with the following structure: // // [ // {"kind":"link-migration-success","account_address":"0x1","path":"/public/foo","capability_id":1}, // ] -// -// The function returns a mapping from account capability controller IDs to paths. -func ReadLinkMigrationReport(reader io.Reader) (map[AccountCapabilityControllerID]string, error) { - mapping := make(map[AccountCapabilityControllerID]string) +func ReadPublicLinkMigrationReport(reader io.Reader) (LinkMigrationReport, error) { + mapping := LinkMigrationReport{} dec := json.NewDecoder(reader) @@ -361,6 +409,11 @@ func ReadLinkMigrationReport(reader io.Reader) (map[AccountCapabilityControllerI continue } + identifier, ok := strings.CutPrefix(entry.Path, "/public/") + if !ok { + continue + } + address, err := common.HexToAddress(entry.Address) if err != nil { return nil, fmt.Errorf("failed to parse address: %w", err) @@ -370,7 +423,7 @@ func ReadLinkMigrationReport(reader io.Reader) (map[AccountCapabilityControllerI Address: address, CapabilityID: entry.CapabilityID, } - mapping[key] = entry.Path + mapping[key] = identifier } token, err = dec.Token() @@ -389,16 +442,22 @@ type LinkInfo struct { AccessibleMembers []string } -// ReadLinkReport reads a link report from the given reader. +type AddressPublicPath struct { + Address common.Address + Identifier string +} + +// PublicLinkReport is a mapping from public account paths to link info. +type PublicLinkReport map[AddressPublicPath]LinkInfo + +// ReadPublicLinkReport reads a link report from the given reader. // The report is expected to be a JSON array of objects with the following structure: // // [ // {"address":"0x1","identifier":"foo","linkType":"&Foo","accessibleMembers":["foo"]} // ] -// -// The function returns a mapping from account paths to link info. -func ReadLinkReport(reader io.Reader) (map[interpreter.AddressPath]LinkInfo, error) { - mapping := make(map[interpreter.AddressPath]LinkInfo) +func ReadPublicLinkReport(reader io.Reader) (PublicLinkReport, error) { + report := PublicLinkReport{} dec := json.NewDecoder(reader) @@ -427,14 +486,11 @@ func ReadLinkReport(reader io.Reader) (map[interpreter.AddressPath]LinkInfo, err return nil, fmt.Errorf("failed to parse address: %w", err) } - key := interpreter.AddressPath{ - Address: address, - Path: interpreter.PathValue{ - Domain: common.PathDomainPublic, - Identifier: entry.Identifier, - }, + key := AddressPublicPath{ + Address: address, + Identifier: entry.Identifier, } - mapping[key] = LinkInfo{ + report[key] = LinkInfo{ BorrowType: common.TypeID(entry.LinkTypeID), AccessibleMembers: entry.AccessibleMembers, } @@ -448,7 +504,7 @@ func ReadLinkReport(reader io.Reader) (map[interpreter.AddressPath]LinkInfo, err return nil, fmt.Errorf("expected end of array, got %s", token) } - return mapping, nil + return report, nil } func NewFixEntitlementsMigrations( @@ -468,6 +524,9 @@ func NewFixEntitlementsMigrations( programs := make(map[common.Location]*interpreter.Program, 1000) + // TODO: + //fixedEntitlements := map[AccountCapabilityControllerID]struct{}{} + return []NamedMigration{ { Name: "check-contracts", diff --git a/cmd/util/ledger/migrations/fix_entitlements_migration_test.go b/cmd/util/ledger/migrations/fix_entitlements_migration_test.go index 41433bb9aac..9639d67e746 100644 --- a/cmd/util/ledger/migrations/fix_entitlements_migration_test.go +++ b/cmd/util/ledger/migrations/fix_entitlements_migration_test.go @@ -5,7 +5,6 @@ import ( "testing" "github.com/onflow/cadence/runtime/common" - "github.com/onflow/cadence/runtime/interpreter" "github.com/rs/zerolog" "github.com/stretchr/testify/require" @@ -79,19 +78,15 @@ func TestReadLinkMigrationReport(t *testing.T) { ] `) - mapping, err := ReadLinkMigrationReport(reader) + mapping, err := ReadPublicLinkMigrationReport(reader) require.NoError(t, err) require.Equal(t, - map[AccountCapabilityControllerID]string{ + LinkMigrationReport{ { Address: common.MustBytesToAddress([]byte{0x1}), CapabilityID: 1, - }: "/public/foo", - { - Address: common.MustBytesToAddress([]byte{0x2}), - CapabilityID: 2, - }: "/private/bar", + }: "foo", }, mapping, ) @@ -107,27 +102,21 @@ func TestReadLinkReport(t *testing.T) { ] `) - mapping, err := ReadLinkReport(reader) + mapping, err := ReadPublicLinkReport(reader) require.NoError(t, err) require.Equal(t, - map[interpreter.AddressPath]LinkInfo{ + PublicLinkReport{ { - Address: common.MustBytesToAddress([]byte{0x1}), - Path: interpreter.PathValue{ - Domain: common.PathDomainPublic, - Identifier: "foo", - }, + Address: common.MustBytesToAddress([]byte{0x1}), + Identifier: "foo", }: { BorrowType: "&Foo", AccessibleMembers: []string{"foo"}, }, { - Address: common.MustBytesToAddress([]byte{0x2}), - Path: interpreter.PathValue{ - Domain: common.PathDomainPublic, - Identifier: "bar", - }, + Address: common.MustBytesToAddress([]byte{0x2}), + Identifier: "bar", }: { BorrowType: "&Bar", AccessibleMembers: nil, From 09175c6621c70580b1e22a15afc6a353eaad05ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Thu, 5 Sep 2024 18:11:45 -0700 Subject: [PATCH 23/72] improve naming and tests --- .../migrations/fix_entitlements_migration.go | 26 ++++-- .../fix_entitlements_migration_test.go | 87 +++++++++++++++++-- 2 files changed, 95 insertions(+), 18 deletions(-) diff --git a/cmd/util/ledger/migrations/fix_entitlements_migration.go b/cmd/util/ledger/migrations/fix_entitlements_migration.go index 991e2d3acda..e4305ff6ea6 100644 --- a/cmd/util/ledger/migrations/fix_entitlements_migration.go +++ b/cmd/util/ledger/migrations/fix_entitlements_migration.go @@ -33,9 +33,9 @@ type FixCapabilityControllerEntitlementsMigrationReporter interface { } type FixCapabilityControllerEntitlementsMigration struct { - Reporter FixCapabilityControllerEntitlementsMigrationReporter - LinkMigrationReport LinkMigrationReport - LinkReport PublicLinkReport + Reporter FixCapabilityControllerEntitlementsMigrationReporter + PublicLinkReport PublicLinkReport + PublicLinkMigrationReport PublicLinkMigrationReport } var _ migrations.ValueMigration = &FixCapabilityControllerEntitlementsMigration{} @@ -88,7 +88,7 @@ func (m *FixCapabilityControllerEntitlementsMigration) capabilityControllerPubli address common.Address, capabilityID interpreter.UInt64Value, ) string { - return m.LinkMigrationReport[AccountCapabilityControllerID{ + return m.PublicLinkMigrationReport[AccountCapabilityControllerID{ Address: address, CapabilityID: uint64(capabilityID), }] @@ -98,7 +98,7 @@ func (m *FixCapabilityControllerEntitlementsMigration) publicPathLinkInfo( address common.Address, publicPathIdentifier string, ) LinkInfo { - return m.LinkReport[AddressPublicPath{ + return m.PublicLinkReport[AddressPublicPath{ Address: address, Identifier: publicPathIdentifier, }] @@ -213,6 +213,8 @@ func NewFixCapabilityControllerEntitlementsMigration( rwf reporters.ReportWriterFactory, errorMessageHandler *errorMessageHandler, programs map[runtime.Location]*interpreter.Program, + publicLinkReport PublicLinkReport, + publicLinkMigrationReport PublicLinkMigrationReport, opts FixEntitlementsMigrationOptions, ) *CadenceBaseMigration { var diffReporter reporters.ReportWriter @@ -237,6 +239,8 @@ func NewFixCapabilityControllerEntitlementsMigration( return []migrations.ValueMigration{ &FixCapabilityControllerEntitlementsMigration{ + PublicLinkReport: publicLinkReport, + PublicLinkMigrationReport: publicLinkMigrationReport, Reporter: &fixEntitlementsMigrationReporter{ reportWriter: reporter, errorMessageHandler: errorMessageHandler, @@ -369,8 +373,8 @@ type AccountCapabilityControllerID struct { CapabilityID uint64 } -// LinkMigrationReport is a mapping from account capability controller IDs to path identifier. -type LinkMigrationReport map[AccountCapabilityControllerID]string +// PublicLinkMigrationReport is a mapping from account capability controller IDs to public path identifier. +type PublicLinkMigrationReport map[AccountCapabilityControllerID]string // ReadPublicLinkMigrationReport reads a link migration report from the given reader, // and extracts the public paths that were migrated. @@ -380,8 +384,8 @@ type LinkMigrationReport map[AccountCapabilityControllerID]string // [ // {"kind":"link-migration-success","account_address":"0x1","path":"/public/foo","capability_id":1}, // ] -func ReadPublicLinkMigrationReport(reader io.Reader) (LinkMigrationReport, error) { - mapping := LinkMigrationReport{} +func ReadPublicLinkMigrationReport(reader io.Reader) (PublicLinkMigrationReport, error) { + mapping := PublicLinkMigrationReport{} dec := json.NewDecoder(reader) @@ -510,6 +514,8 @@ func ReadPublicLinkReport(reader io.Reader) (PublicLinkReport, error) { func NewFixEntitlementsMigrations( log zerolog.Logger, rwf reporters.ReportWriterFactory, + publicLinkReport PublicLinkReport, + publicLinkMigrationReport PublicLinkMigrationReport, opts FixEntitlementsMigrationOptions, ) []NamedMigration { @@ -550,6 +556,8 @@ func NewFixEntitlementsMigrations( rwf, errorMessageHandler, programs, + publicLinkReport, + publicLinkMigrationReport, opts, ), }, diff --git a/cmd/util/ledger/migrations/fix_entitlements_migration_test.go b/cmd/util/ledger/migrations/fix_entitlements_migration_test.go index 9639d67e746..4e082ee784a 100644 --- a/cmd/util/ledger/migrations/fix_entitlements_migration_test.go +++ b/cmd/util/ledger/migrations/fix_entitlements_migration_test.go @@ -20,6 +20,11 @@ func TestFixEntitlementMigrations(t *testing.T) { const nWorker = 2 + address, err := chain.AddressAtIndex(1000) + require.NoError(t, err) + + require.Equal(t, "bf519681cdb888b1", address.Hex()) + log := zerolog.New(zerolog.NewTestWriter(t)) bootstrapPayloads, err := newBootstrapPayloads(chainID) @@ -28,24 +33,45 @@ func TestFixEntitlementMigrations(t *testing.T) { registersByAccount, err := registers.NewByAccountFromPayloads(bootstrapPayloads) require.NoError(t, err) + mr := NewBasicMigrationRuntime(registersByAccount) + err = mr.Accounts.Create(nil, address) + require.NoError(t, err) + + expectedWriteAddresses := map[flow.Address]struct{}{ + address: {}, + } + + err = mr.Commit(expectedWriteAddresses, log) + require.NoError(t, err) + tx := flow.NewTransactionBody(). SetScript([]byte(` transaction { prepare(signer: auth(Storage, Capabilities) &Account) { - let cap = signer.capabilities.storage.issue(/storage/ints) - signer.storage.save([cap], to: /storage/caps) + // Capability 1 was a public, unauthorized capability. + // It should lose its entitlement + let cap1 = signer.capabilities.storage.issue(/storage/ints) + signer.capabilities.publish(cap1, at: /public/ints) + + // Capability 2 was a public, unauthorized capability, stored nested in storage. + // It should lose its entitlement + let cap2 = signer.capabilities.storage.issue(/storage/ints) + signer.storage.save([cap2], to: /storage/caps1) + + // Capability 3 was a private, authorized capability, stored nested in storage. + // It should keep its entitlement + let cap3 = signer.capabilities.storage.issue(/storage/ints) + signer.storage.save([cap3], to: /storage/caps2) } } `)). - AddAuthorizer(chain.ServiceAddress()) + AddAuthorizer(address) setupTx := NewTransactionBasedMigration( tx, chainID, log, - map[flow.Address]struct{}{ - chain.ServiceAddress(): {}, - }, + expectedWriteAddresses, ) err = setupTx(registersByAccount) @@ -58,7 +84,50 @@ func TestFixEntitlementMigrations(t *testing.T) { NWorker: nWorker, } - migrations := NewFixEntitlementsMigrations(log, rwf, options) + // Capability 1 was a public, unauthorized capability. + // It should lose its entitlement + // + // Capability 2 was a public, unauthorized capability, stored nested in storage. + // It should lose its entitlement + // + // Capability 3 was a private, authorized capability, stored nested in storage. + // It should keep its entitlement + + publicLinkReport := PublicLinkReport{ + { + Address: common.Address(address), + Identifier: "ints", + }: { + BorrowType: "&[Int]", + AccessibleMembers: []string{}, + }, + { + Address: common.Address(address), + Identifier: "ints2", + }: { + BorrowType: "&[Int]", + AccessibleMembers: []string{}, + }, + } + + publicLinkMigrationReport := PublicLinkMigrationReport{ + { + Address: common.Address(address), + CapabilityID: 1, + }: "ints", + { + Address: common.Address(address), + CapabilityID: 2, + }: "ints2", + } + + migrations := NewFixEntitlementsMigrations( + log, + rwf, + publicLinkReport, + publicLinkMigrationReport, + options, + ) for _, namedMigration := range migrations { err = namedMigration.Migrate(registersByAccount) @@ -68,7 +137,7 @@ func TestFixEntitlementMigrations(t *testing.T) { // TODO: validate } -func TestReadLinkMigrationReport(t *testing.T) { +func TestReadPublicLinkMigrationReport(t *testing.T) { t.Parallel() reader := strings.NewReader(` @@ -82,7 +151,7 @@ func TestReadLinkMigrationReport(t *testing.T) { require.NoError(t, err) require.Equal(t, - LinkMigrationReport{ + PublicLinkMigrationReport{ { Address: common.MustBytesToAddress([]byte{0x1}), CapabilityID: 1, From 425cc290398633b1db2878006ddbb3acccd7c9ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Thu, 5 Sep 2024 19:10:33 -0700 Subject: [PATCH 24/72] report fixed capability controllers --- .../migrations/fix_entitlements_migration.go | 51 ++++++++++++++++--- .../fix_entitlements_migration_test.go | 33 ++++++++++++ 2 files changed, 76 insertions(+), 8 deletions(-) diff --git a/cmd/util/ledger/migrations/fix_entitlements_migration.go b/cmd/util/ledger/migrations/fix_entitlements_migration.go index e4305ff6ea6..55240a9e17f 100644 --- a/cmd/util/ledger/migrations/fix_entitlements_migration.go +++ b/cmd/util/ledger/migrations/fix_entitlements_migration.go @@ -73,6 +73,12 @@ func (m *FixCapabilityControllerEntitlementsMigration) Migrate( } linkInfo := m.publicPathLinkInfo(address, publicPathIdentifier) + if linkInfo.BorrowType == "" { + log.Warn().Msgf("missing link info for account %s, public path %s", address, publicPathIdentifier) + return nil, nil + } + + // TODO: m.Reporter.MigratedCapabilityController( storageKey, @@ -209,6 +215,8 @@ type FixEntitlementsMigrationOptions struct { CheckStorageHealthBeforeMigration bool } +const fixCapabilityControllerEntitlementMigrationReportName = "fix-capability-controller-entitlements-migration" + func NewFixCapabilityControllerEntitlementsMigration( rwf reporters.ReportWriterFactory, errorMessageHandler *errorMessageHandler, @@ -222,7 +230,7 @@ func NewFixCapabilityControllerEntitlementsMigration( diffReporter = rwf.ReportWriter("fix-capability-controller-entitlements-migration-diff") } - reporter := rwf.ReportWriter("fix-capability-controller-entitlements-migration") + reporter := rwf.ReportWriter(fixCapabilityControllerEntitlementMigrationReportName) return &CadenceBaseMigration{ name: "fix_capability_controller_entitlements_migration", @@ -255,6 +263,8 @@ func NewFixCapabilityControllerEntitlementsMigration( } } +const fixCapabilityEntitlementsMigrationReporterName = "fix-capability-entitlements-migration" + func NewFixCapabilityEntitlementsMigration( rwf reporters.ReportWriterFactory, errorMessageHandler *errorMessageHandler, @@ -266,7 +276,7 @@ func NewFixCapabilityEntitlementsMigration( diffReporter = rwf.ReportWriter("fix-capability-entitlements-migration-diff") } - reporter := rwf.ReportWriter("fix-capability-entitlements-migration") + reporter := rwf.ReportWriter(fixCapabilityEntitlementsMigrationReporterName) return &CadenceBaseMigration{ name: "fix_capability_entitlements_migration", @@ -353,6 +363,17 @@ func (r *fixEntitlementsMigrationReporter) DictionaryKeyConflict(accountAddressP }) } +func (r *fixEntitlementsMigrationReporter) MigratedCapabilityController( + storageKey interpreter.StorageKey, + capabilityController *interpreter.StorageCapabilityControllerValue, + _ LinkInfo, +) { + r.reportWriter.Write(capabilityControllerEntitlementsFixedEntry{ + StorageKey: storageKey, + CapabilityID: uint64(capabilityController.CapabilityID), + }) +} + func (r *fixEntitlementsMigrationReporter) MigratedCapability( _ interpreter.StorageKey, _ *interpreter.IDCapabilityValue, @@ -360,12 +381,26 @@ func (r *fixEntitlementsMigrationReporter) MigratedCapability( // TODO: } -func (r *fixEntitlementsMigrationReporter) MigratedCapabilityController( - _ interpreter.StorageKey, - _ *interpreter.StorageCapabilityControllerValue, - _ LinkInfo, -) { - // TODO: +// capabilityControllerEntitlementsFixedEntry +type capabilityControllerEntitlementsFixedEntry struct { + StorageKey interpreter.StorageKey + CapabilityID uint64 +} + +var _ json.Marshaler = capabilityControllerEntitlementsFixedEntry{} + +func (e capabilityControllerEntitlementsFixedEntry) MarshalJSON() ([]byte, error) { + return json.Marshal(struct { + Kind string `json:"kind"` + AccountAddress string `json:"account_address"` + StorageDomain string `json:"domain"` + CapabilityID uint64 `json:"capability_id"` + }{ + Kind: "capability-controller-entitlements-fixed", + AccountAddress: e.StorageKey.Address.HexWithPrefix(), + StorageDomain: e.StorageKey.Key, + CapabilityID: e.CapabilityID, + }) } type AccountCapabilityControllerID struct { diff --git a/cmd/util/ledger/migrations/fix_entitlements_migration_test.go b/cmd/util/ledger/migrations/fix_entitlements_migration_test.go index 4e082ee784a..ad18039072f 100644 --- a/cmd/util/ledger/migrations/fix_entitlements_migration_test.go +++ b/cmd/util/ledger/migrations/fix_entitlements_migration_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/onflow/cadence/runtime/common" + "github.com/onflow/cadence/runtime/interpreter" "github.com/rs/zerolog" "github.com/stretchr/testify/require" @@ -134,7 +135,39 @@ func TestFixEntitlementMigrations(t *testing.T) { require.NoError(t, err) } + reporter := rwf.reportWriters[fixCapabilityControllerEntitlementMigrationReportName] + require.NotNil(t, reporter) + + var entries []any + + for _, entry := range reporter.entries { + switch entry := entry.(type) { + case capabilityControllerEntitlementsFixedEntry: + entries = append(entries, entry) + } + } + // TODO: validate + + require.ElementsMatch(t, + []any{ + capabilityControllerEntitlementsFixedEntry{ + StorageKey: interpreter.StorageKey{ + Key: "cap_con", + Address: common.Address(address), + }, + CapabilityID: 1, + }, + capabilityControllerEntitlementsFixedEntry{ + StorageKey: interpreter.StorageKey{ + Key: "cap_con", + Address: common.Address(address), + }, + CapabilityID: 2, + }, + }, + entries, + ) } func TestReadPublicLinkMigrationReport(t *testing.T) { From b770bceebc3bcf8a8f6108743f96fdf223f7fe0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Thu, 5 Sep 2024 20:13:07 -0700 Subject: [PATCH 25/72] compare old and new accessible members, ignore when old accessible members are unavailable --- .../migrations/fix_entitlements_migration.go | 111 ++++++++++++++++-- .../fix_entitlements_migration_test.go | 62 +++++++++- 2 files changed, 154 insertions(+), 19 deletions(-) diff --git a/cmd/util/ledger/migrations/fix_entitlements_migration.go b/cmd/util/ledger/migrations/fix_entitlements_migration.go index 55240a9e17f..6576eca3e3e 100644 --- a/cmd/util/ledger/migrations/fix_entitlements_migration.go +++ b/cmd/util/ledger/migrations/fix_entitlements_migration.go @@ -5,10 +5,12 @@ import ( "errors" "fmt" "io" + "sort" "strings" "github.com/onflow/cadence/migrations" "github.com/onflow/cadence/runtime" + "github.com/onflow/cadence/runtime/ast" "github.com/onflow/cadence/runtime/common" cadenceErrors "github.com/onflow/cadence/runtime/errors" "github.com/onflow/cadence/runtime/interpreter" @@ -16,6 +18,7 @@ import ( "github.com/onflow/cadence/runtime/stdlib" "github.com/rs/zerolog" "github.com/rs/zerolog/log" + "golang.org/x/exp/slices" "github.com/onflow/flow-go/cmd/util/ledger/reporters" "github.com/onflow/flow-go/fvm/environment" @@ -28,7 +31,8 @@ type FixCapabilityControllerEntitlementsMigrationReporter interface { MigratedCapabilityController( storageKey interpreter.StorageKey, capabilityController *interpreter.StorageCapabilityControllerValue, - linkInfo LinkInfo, + oldAccessibleMembers []string, + newAccessibleMembers []string, ) } @@ -56,7 +60,7 @@ func (m *FixCapabilityControllerEntitlementsMigration) Migrate( storageKey interpreter.StorageKey, _ interpreter.StorageMapKey, value interpreter.Value, - _ *interpreter.Interpreter, + inter *interpreter.Interpreter, _ migrations.ValueMigrationPosition, ) ( interpreter.Value, @@ -78,13 +82,38 @@ func (m *FixCapabilityControllerEntitlementsMigration) Migrate( return nil, nil } - // TODO: + oldAccessibleMembers := linkInfo.AccessibleMembers + if oldAccessibleMembers == nil { + log.Warn().Msgf( + "old accessible members for account %s, capability controller %s not available", + address, + capabilityController.BorrowType, + ) + return nil, nil + } - m.Reporter.MigratedCapabilityController( - storageKey, - capabilityController, - linkInfo, - ) + newAccessibleMembers, err := getAccessibleMembers(inter, capabilityController.BorrowType) + if err != nil { + log.Warn().Msgf( + "failed to get new accessible members for account %s, capability controller %s: %s", + address, + capabilityController.BorrowType, + err, + ) + return nil, nil + } + + sort.Strings(oldAccessibleMembers) + sort.Strings(newAccessibleMembers) + + if !slices.Equal(linkInfo.AccessibleMembers, newAccessibleMembers) { + m.Reporter.MigratedCapabilityController( + storageKey, + capabilityController, + oldAccessibleMembers, + newAccessibleMembers, + ) + } } return nil, nil @@ -110,6 +139,57 @@ func (m *FixCapabilityControllerEntitlementsMigration) publicPathLinkInfo( }] } +func getAccessibleMembers( + inter *interpreter.Interpreter, + staticType interpreter.StaticType, +) ( + accessibleMembers []string, + err error, +) { + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("panic: %v", r) + } + }() + + semaType, err := inter.ConvertStaticToSemaType(staticType) + if err != nil { + return nil, fmt.Errorf( + "failed to convert static type %s to semantic type: %w", + staticType.ID(), + err, + ) + } + if semaType == nil { + return nil, fmt.Errorf( + "failed to convert static type %s to semantic type", + staticType.ID(), + ) + } + + // NOTE: RestrictedType.GetMembers returns *all* members, + // including those that are not accessible, for DX purposes. + // We need to resolve the members and filter out the inaccessible members, + // using the error reported when resolving + + memberResolvers := semaType.GetMembers() + + accessibleMembers = make([]string, 0, len(memberResolvers)) + + for memberName, memberResolver := range memberResolvers { + var resolveErr error + memberResolver.Resolve(nil, memberName, ast.EmptyRange, func(err error) { + resolveErr = err + }) + if resolveErr != nil { + continue + } + accessibleMembers = append(accessibleMembers, memberName) + } + + return accessibleMembers, nil +} + func (*FixCapabilityControllerEntitlementsMigration) CanSkip(valueType interpreter.StaticType) bool { return CanSkipFixEntitlementsMigration(valueType) } @@ -366,11 +446,14 @@ func (r *fixEntitlementsMigrationReporter) DictionaryKeyConflict(accountAddressP func (r *fixEntitlementsMigrationReporter) MigratedCapabilityController( storageKey interpreter.StorageKey, capabilityController *interpreter.StorageCapabilityControllerValue, - _ LinkInfo, + oldAccessibleMembers []string, + newAccessibleMembers []string, ) { r.reportWriter.Write(capabilityControllerEntitlementsFixedEntry{ - StorageKey: storageKey, - CapabilityID: uint64(capabilityController.CapabilityID), + StorageKey: storageKey, + CapabilityID: uint64(capabilityController.CapabilityID), + OldAccessibleMembers: oldAccessibleMembers, + NewAccessibleMembers: newAccessibleMembers, }) } @@ -383,8 +466,10 @@ func (r *fixEntitlementsMigrationReporter) MigratedCapability( // capabilityControllerEntitlementsFixedEntry type capabilityControllerEntitlementsFixedEntry struct { - StorageKey interpreter.StorageKey - CapabilityID uint64 + StorageKey interpreter.StorageKey + CapabilityID uint64 + OldAccessibleMembers []string + NewAccessibleMembers []string } var _ json.Marshaler = capabilityControllerEntitlementsFixedEntry{} diff --git a/cmd/util/ledger/migrations/fix_entitlements_migration_test.go b/cmd/util/ledger/migrations/fix_entitlements_migration_test.go index ad18039072f..a192292bc02 100644 --- a/cmd/util/ledger/migrations/fix_entitlements_migration_test.go +++ b/cmd/util/ledger/migrations/fix_entitlements_migration_test.go @@ -1,6 +1,7 @@ package migrations import ( + "sort" "strings" "testing" @@ -57,12 +58,17 @@ func TestFixEntitlementMigrations(t *testing.T) { // Capability 2 was a public, unauthorized capability, stored nested in storage. // It should lose its entitlement let cap2 = signer.capabilities.storage.issue(/storage/ints) - signer.storage.save([cap2], to: /storage/caps1) + signer.storage.save([cap2], to: /storage/caps2) // Capability 3 was a private, authorized capability, stored nested in storage. // It should keep its entitlement let cap3 = signer.capabilities.storage.issue(/storage/ints) - signer.storage.save([cap3], to: /storage/caps2) + signer.storage.save([cap3], to: /storage/caps3) + + // Capability 4 was a capability with unavailable accessible members, stored nested in storage. + // It should keep its entitlement + let cap4 = signer.capabilities.storage.issue(/storage/ints) + signer.storage.save([cap4], to: /storage/caps4) } } `)). @@ -93,6 +99,35 @@ func TestFixEntitlementMigrations(t *testing.T) { // // Capability 3 was a private, authorized capability, stored nested in storage. // It should keep its entitlement + // + // Capability 4 was a capability with unavailable accessible members, stored nested in storage. + // It should keep its entitlement + + readArrayMembers := []string{ + "concat", + "contains", + "filter", + "firstIndex", + "getType", + "isInstance", + "length", + "map", + "slice", + "toConstantSized", + } + + writeArrayMembers := []string{ + "append", + "appendAll", + "insert", + "remove", + "removeFirst", + "removeLast", + "reverse", + } + + readWriteArrayMembers := common.Concat(readArrayMembers, writeArrayMembers) + sort.Strings(readWriteArrayMembers) publicLinkReport := PublicLinkReport{ { @@ -100,14 +135,21 @@ func TestFixEntitlementMigrations(t *testing.T) { Identifier: "ints", }: { BorrowType: "&[Int]", - AccessibleMembers: []string{}, + AccessibleMembers: readArrayMembers, }, { Address: common.Address(address), Identifier: "ints2", }: { BorrowType: "&[Int]", - AccessibleMembers: []string{}, + AccessibleMembers: readArrayMembers, + }, + { + Address: common.Address(address), + Identifier: "ints4", + }: { + BorrowType: "&[Int]", + AccessibleMembers: nil, }, } @@ -120,6 +162,10 @@ func TestFixEntitlementMigrations(t *testing.T) { Address: common.Address(address), CapabilityID: 2, }: "ints2", + { + Address: common.Address(address), + CapabilityID: 4, + }: "ints4", } migrations := NewFixEntitlementsMigrations( @@ -156,14 +202,18 @@ func TestFixEntitlementMigrations(t *testing.T) { Key: "cap_con", Address: common.Address(address), }, - CapabilityID: 1, + CapabilityID: 1, + OldAccessibleMembers: readArrayMembers, + NewAccessibleMembers: readWriteArrayMembers, }, capabilityControllerEntitlementsFixedEntry{ StorageKey: interpreter.StorageKey{ Key: "cap_con", Address: common.Address(address), }, - CapabilityID: 2, + CapabilityID: 2, + OldAccessibleMembers: readArrayMembers, + NewAccessibleMembers: readWriteArrayMembers, }, }, entries, From e05567c24e7d8b6a153fb91230b57ac9d1a45a3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Fri, 6 Sep 2024 11:55:17 -0700 Subject: [PATCH 26/72] refactor migration to only apply fixes, not compute them --- .../migrations/fix_entitlements_migration.go | 559 +++++------------- .../fix_entitlements_migration_test.go | 169 +----- 2 files changed, 178 insertions(+), 550 deletions(-) diff --git a/cmd/util/ledger/migrations/fix_entitlements_migration.go b/cmd/util/ledger/migrations/fix_entitlements_migration.go index 6576eca3e3e..9dd6ce83ef7 100644 --- a/cmd/util/ledger/migrations/fix_entitlements_migration.go +++ b/cmd/util/ledger/migrations/fix_entitlements_migration.go @@ -4,238 +4,143 @@ import ( "encoding/json" "errors" "fmt" - "io" - "sort" - "strings" "github.com/onflow/cadence/migrations" "github.com/onflow/cadence/runtime" - "github.com/onflow/cadence/runtime/ast" "github.com/onflow/cadence/runtime/common" cadenceErrors "github.com/onflow/cadence/runtime/errors" "github.com/onflow/cadence/runtime/interpreter" "github.com/onflow/cadence/runtime/sema" - "github.com/onflow/cadence/runtime/stdlib" "github.com/rs/zerolog" "github.com/rs/zerolog/log" - "golang.org/x/exp/slices" "github.com/onflow/flow-go/cmd/util/ledger/reporters" "github.com/onflow/flow-go/fvm/environment" "github.com/onflow/flow-go/model/flow" ) -// FixCapabilityControllerEntitlementsMigration +type AccountCapabilityControllerID struct { + Address common.Address + CapabilityID uint64 +} + +// FixEntitlementsMigration -type FixCapabilityControllerEntitlementsMigrationReporter interface { +type FixEntitlementsMigrationReporter interface { + MigratedCapability( + storageKey interpreter.StorageKey, + capabilityAddress common.Address, + capabilityID uint64, + newAuthorization interpreter.Authorization, + ) MigratedCapabilityController( storageKey interpreter.StorageKey, - capabilityController *interpreter.StorageCapabilityControllerValue, - oldAccessibleMembers []string, - newAccessibleMembers []string, + capabilityID uint64, + newAuthorization interpreter.Authorization, ) } -type FixCapabilityControllerEntitlementsMigration struct { - Reporter FixCapabilityControllerEntitlementsMigrationReporter - PublicLinkReport PublicLinkReport - PublicLinkMigrationReport PublicLinkMigrationReport +type FixEntitlementsMigration struct { + Reporter FixEntitlementsMigrationReporter + NewAuthorizations map[AccountCapabilityControllerID]interpreter.Authorization } -var _ migrations.ValueMigration = &FixCapabilityControllerEntitlementsMigration{} +var _ migrations.ValueMigration = &FixEntitlementsMigration{} -func (*FixCapabilityControllerEntitlementsMigration) Name() string { - return "FixCapabilityControllerEntitlementsMigration" +func (*FixEntitlementsMigration) Name() string { + return "FixEntitlementsMigration" } -var fixCapabilityControllerEntitlementsMigrationDomains = map[string]struct{}{ - stdlib.CapabilityControllerStorageDomain: {}, -} - -func (*FixCapabilityControllerEntitlementsMigration) Domains() map[string]struct{} { - return fixCapabilityControllerEntitlementsMigrationDomains +func (*FixEntitlementsMigration) Domains() map[string]struct{} { + return nil } -func (m *FixCapabilityControllerEntitlementsMigration) Migrate( +func (m *FixEntitlementsMigration) Migrate( storageKey interpreter.StorageKey, _ interpreter.StorageMapKey, value interpreter.Value, - inter *interpreter.Interpreter, + _ *interpreter.Interpreter, _ migrations.ValueMigrationPosition, ) ( interpreter.Value, error, ) { - if capabilityController, ok := value.(*interpreter.StorageCapabilityControllerValue); ok { - address := storageKey.Address - capabilityID := capabilityController.CapabilityID - - publicPathIdentifier := m.capabilityControllerPublicPathIdentifier(address, capabilityID) - if publicPathIdentifier == "" { - log.Warn().Msgf("missing capability controller path for account %s, capability ID %d", address, capabilityID) - return nil, nil - } - - linkInfo := m.publicPathLinkInfo(address, publicPathIdentifier) - if linkInfo.BorrowType == "" { - log.Warn().Msgf("missing link info for account %s, public path %s", address, publicPathIdentifier) + switch value := value.(type) { + case *interpreter.IDCapabilityValue: + capabilityAddress := common.Address(value.Address()) + capabilityID := uint64(value.ID) + + newAuthorization := m.NewAuthorizations[AccountCapabilityControllerID{ + Address: capabilityAddress, + CapabilityID: capabilityID, + }] + if newAuthorization == nil { + // Nothing to fix for this capability return nil, nil } - oldAccessibleMembers := linkInfo.AccessibleMembers - if oldAccessibleMembers == nil { + borrowType := value.BorrowType + if borrowType == nil { log.Warn().Msgf( - "old accessible members for account %s, capability controller %s not available", - address, - capabilityController.BorrowType, + "missing borrow type for capability with target %s#%d", + capabilityAddress.HexWithPrefix(), + capabilityID, ) - return nil, nil } - newAccessibleMembers, err := getAccessibleMembers(inter, capabilityController.BorrowType) - if err != nil { + borrowReferenceType, ok := borrowType.(*interpreter.ReferenceStaticType) + if !ok { log.Warn().Msgf( - "failed to get new accessible members for account %s, capability controller %s: %s", - address, - capabilityController.BorrowType, - err, + "invalid non-reference borrow type for capability with target %s#%d: %s", + capabilityAddress.HexWithPrefix(), + capabilityID, + borrowType, ) return nil, nil } - sort.Strings(oldAccessibleMembers) - sort.Strings(newAccessibleMembers) - - if !slices.Equal(linkInfo.AccessibleMembers, newAccessibleMembers) { - m.Reporter.MigratedCapabilityController( - storageKey, - capabilityController, - oldAccessibleMembers, - newAccessibleMembers, - ) - } - } + borrowReferenceType.Authorization = newAuthorization + value.BorrowType = borrowReferenceType - return nil, nil -} - -func (m *FixCapabilityControllerEntitlementsMigration) capabilityControllerPublicPathIdentifier( - address common.Address, - capabilityID interpreter.UInt64Value, -) string { - return m.PublicLinkMigrationReport[AccountCapabilityControllerID{ - Address: address, - CapabilityID: uint64(capabilityID), - }] -} - -func (m *FixCapabilityControllerEntitlementsMigration) publicPathLinkInfo( - address common.Address, - publicPathIdentifier string, -) LinkInfo { - return m.PublicLinkReport[AddressPublicPath{ - Address: address, - Identifier: publicPathIdentifier, - }] -} - -func getAccessibleMembers( - inter *interpreter.Interpreter, - staticType interpreter.StaticType, -) ( - accessibleMembers []string, - err error, -) { - defer func() { - if r := recover(); r != nil { - err = fmt.Errorf("panic: %v", r) - } - }() - - semaType, err := inter.ConvertStaticToSemaType(staticType) - if err != nil { - return nil, fmt.Errorf( - "failed to convert static type %s to semantic type: %w", - staticType.ID(), - err, + m.Reporter.MigratedCapability( + storageKey, + capabilityAddress, + capabilityID, + newAuthorization, ) - } - if semaType == nil { - return nil, fmt.Errorf( - "failed to convert static type %s to semantic type", - staticType.ID(), - ) - } - - // NOTE: RestrictedType.GetMembers returns *all* members, - // including those that are not accessible, for DX purposes. - // We need to resolve the members and filter out the inaccessible members, - // using the error reported when resolving - memberResolvers := semaType.GetMembers() + return value, nil - accessibleMembers = make([]string, 0, len(memberResolvers)) + case *interpreter.StorageCapabilityControllerValue: + // The capability controller's address is implicitly + // the address of the account in which it is stored + capabilityAddress := storageKey.Address + capabilityID := uint64(value.CapabilityID) - for memberName, memberResolver := range memberResolvers { - var resolveErr error - memberResolver.Resolve(nil, memberName, ast.EmptyRange, func(err error) { - resolveErr = err - }) - if resolveErr != nil { - continue + newAuthorization := m.NewAuthorizations[AccountCapabilityControllerID{ + Address: capabilityAddress, + CapabilityID: capabilityID, + }] + if newAuthorization == nil { + // Nothing to fix for this capability controller + return nil, nil } - accessibleMembers = append(accessibleMembers, memberName) - } - - return accessibleMembers, nil -} - -func (*FixCapabilityControllerEntitlementsMigration) CanSkip(valueType interpreter.StaticType) bool { - return CanSkipFixEntitlementsMigration(valueType) -} - -// FixCapabilityEntitlementsMigration -type FixCapabilityEntitlementsMigrationReporter interface { - MigratedCapability( - storageKey interpreter.StorageKey, - capability *interpreter.IDCapabilityValue, - ) -} - -type FixCapabilityEntitlementsMigration struct { - Reporter FixCapabilityEntitlementsMigrationReporter -} - -var _ migrations.ValueMigration = &FixCapabilityEntitlementsMigration{} - -func (*FixCapabilityEntitlementsMigration) Name() string { - return "FixCapabilityEntitlementsMigration" -} + value.BorrowType.Authorization = newAuthorization -func (*FixCapabilityEntitlementsMigration) Domains() map[string]struct{} { - return nil -} + m.Reporter.MigratedCapabilityController( + storageKey, + capabilityID, + newAuthorization, + ) -func (m *FixCapabilityEntitlementsMigration) Migrate( - storageKey interpreter.StorageKey, - _ interpreter.StorageMapKey, - value interpreter.Value, - _ *interpreter.Interpreter, - _ migrations.ValueMigrationPosition, -) ( - interpreter.Value, - error, -) { - if capability, ok := value.(*interpreter.IDCapabilityValue); ok { - // TODO: - m.Reporter.MigratedCapability(storageKey, capability) + return value, nil } return nil, nil } -func (*FixCapabilityEntitlementsMigration) CanSkip(valueType interpreter.StaticType) bool { +func (*FixEntitlementsMigration) CanSkip(valueType interpreter.StaticType) bool { return CanSkipFixEntitlementsMigration(valueType) } @@ -295,25 +200,24 @@ type FixEntitlementsMigrationOptions struct { CheckStorageHealthBeforeMigration bool } -const fixCapabilityControllerEntitlementMigrationReportName = "fix-capability-controller-entitlements-migration" +const fixEntitlementsMigrationReporterName = "fix-entitlements-migration" -func NewFixCapabilityControllerEntitlementsMigration( +func NewFixEntitlementsMigration( rwf reporters.ReportWriterFactory, errorMessageHandler *errorMessageHandler, programs map[runtime.Location]*interpreter.Program, - publicLinkReport PublicLinkReport, - publicLinkMigrationReport PublicLinkMigrationReport, + newAuthorizations map[AccountCapabilityControllerID]interpreter.Authorization, opts FixEntitlementsMigrationOptions, ) *CadenceBaseMigration { var diffReporter reporters.ReportWriter if opts.DiffMigrations { - diffReporter = rwf.ReportWriter("fix-capability-controller-entitlements-migration-diff") + diffReporter = rwf.ReportWriter("fix-entitlements-migration-diff") } - reporter := rwf.ReportWriter(fixCapabilityControllerEntitlementMigrationReportName) + reporter := rwf.ReportWriter(fixEntitlementsMigrationReporterName) return &CadenceBaseMigration{ - name: "fix_capability_controller_entitlements_migration", + name: "fix_entitlements_migration", reporter: reporter, diffReporter: diffReporter, logVerboseDiff: opts.LogVerboseDiff, @@ -326,53 +230,8 @@ func NewFixCapabilityControllerEntitlementsMigration( ) []migrations.ValueMigration { return []migrations.ValueMigration{ - &FixCapabilityControllerEntitlementsMigration{ - PublicLinkReport: publicLinkReport, - PublicLinkMigrationReport: publicLinkMigrationReport, - Reporter: &fixEntitlementsMigrationReporter{ - reportWriter: reporter, - errorMessageHandler: errorMessageHandler, - verboseErrorOutput: opts.VerboseErrorOutput, - }, - }, - } - }, - errorMessageHandler: errorMessageHandler, - programs: programs, - chainID: opts.ChainID, - } -} - -const fixCapabilityEntitlementsMigrationReporterName = "fix-capability-entitlements-migration" - -func NewFixCapabilityEntitlementsMigration( - rwf reporters.ReportWriterFactory, - errorMessageHandler *errorMessageHandler, - programs map[runtime.Location]*interpreter.Program, - opts FixEntitlementsMigrationOptions, -) *CadenceBaseMigration { - var diffReporter reporters.ReportWriter - if opts.DiffMigrations { - diffReporter = rwf.ReportWriter("fix-capability-entitlements-migration-diff") - } - - reporter := rwf.ReportWriter(fixCapabilityEntitlementsMigrationReporterName) - - return &CadenceBaseMigration{ - name: "fix_capability_entitlements_migration", - reporter: reporter, - diffReporter: diffReporter, - logVerboseDiff: opts.LogVerboseDiff, - verboseErrorOutput: opts.VerboseErrorOutput, - checkStorageHealthBeforeMigration: opts.CheckStorageHealthBeforeMigration, - valueMigrations: func( - _ *interpreter.Interpreter, - _ environment.Accounts, - _ *cadenceValueMigrationReporter, - ) []migrations.ValueMigration { - - return []migrations.ValueMigration{ - &FixCapabilityEntitlementsMigration{ + &FixEntitlementsMigration{ + NewAuthorizations: newAuthorizations, Reporter: &fixEntitlementsMigrationReporter{ reportWriter: reporter, errorMessageHandler: errorMessageHandler, @@ -393,8 +252,7 @@ type fixEntitlementsMigrationReporter struct { verboseErrorOutput bool } -var _ FixCapabilityEntitlementsMigrationReporter = &fixEntitlementsMigrationReporter{} -var _ FixCapabilityControllerEntitlementsMigrationReporter = &fixEntitlementsMigrationReporter{} +var _ FixEntitlementsMigrationReporter = &fixEntitlementsMigrationReporter{} var _ migrations.Reporter = &fixEntitlementsMigrationReporter{} func (r *fixEntitlementsMigrationReporter) Migrated( @@ -445,197 +303,96 @@ func (r *fixEntitlementsMigrationReporter) DictionaryKeyConflict(accountAddressP func (r *fixEntitlementsMigrationReporter) MigratedCapabilityController( storageKey interpreter.StorageKey, - capabilityController *interpreter.StorageCapabilityControllerValue, - oldAccessibleMembers []string, - newAccessibleMembers []string, + capabilityID uint64, + newAuthorization interpreter.Authorization, ) { r.reportWriter.Write(capabilityControllerEntitlementsFixedEntry{ - StorageKey: storageKey, - CapabilityID: uint64(capabilityController.CapabilityID), - OldAccessibleMembers: oldAccessibleMembers, - NewAccessibleMembers: newAccessibleMembers, + StorageKey: storageKey, + CapabilityID: capabilityID, + NewAuthorization: newAuthorization, }) } func (r *fixEntitlementsMigrationReporter) MigratedCapability( - _ interpreter.StorageKey, - _ *interpreter.IDCapabilityValue, + storageKey interpreter.StorageKey, + capabilityAddress common.Address, + capabilityID uint64, + newAuthorization interpreter.Authorization, ) { - // TODO: + r.reportWriter.Write(capabilityEntitlementsFixedEntry{ + StorageKey: storageKey, + CapabilityAddress: capabilityAddress, + CapabilityID: capabilityID, + NewAuthorization: newAuthorization, + }) +} + +func jsonEncodeAuthorization(authorization interpreter.Authorization) string { + switch authorization { + case interpreter.UnauthorizedAccess, interpreter.InaccessibleAccess: + return "" + default: + return string(authorization.ID()) + } } // capabilityControllerEntitlementsFixedEntry type capabilityControllerEntitlementsFixedEntry struct { - StorageKey interpreter.StorageKey - CapabilityID uint64 - OldAccessibleMembers []string - NewAccessibleMembers []string + StorageKey interpreter.StorageKey + CapabilityID uint64 + NewAuthorization interpreter.Authorization } var _ json.Marshaler = capabilityControllerEntitlementsFixedEntry{} func (e capabilityControllerEntitlementsFixedEntry) MarshalJSON() ([]byte, error) { return json.Marshal(struct { - Kind string `json:"kind"` - AccountAddress string `json:"account_address"` - StorageDomain string `json:"domain"` - CapabilityID uint64 `json:"capability_id"` + Kind string `json:"kind"` + AccountAddress string `json:"account_address"` + StorageDomain string `json:"domain"` + CapabilityID uint64 `json:"capability_id"` + NewAuthorization string `json:"new_authorization"` }{ - Kind: "capability-controller-entitlements-fixed", - AccountAddress: e.StorageKey.Address.HexWithPrefix(), - StorageDomain: e.StorageKey.Key, - CapabilityID: e.CapabilityID, + Kind: "capability-controller-entitlements-fixed", + AccountAddress: e.StorageKey.Address.HexWithPrefix(), + StorageDomain: e.StorageKey.Key, + CapabilityID: e.CapabilityID, + NewAuthorization: jsonEncodeAuthorization(e.NewAuthorization), }) } -type AccountCapabilityControllerID struct { - Address common.Address - CapabilityID uint64 +// capabilityEntitlementsFixedEntry +type capabilityEntitlementsFixedEntry struct { + StorageKey interpreter.StorageKey + CapabilityAddress common.Address + CapabilityID uint64 + NewAuthorization interpreter.Authorization } -// PublicLinkMigrationReport is a mapping from account capability controller IDs to public path identifier. -type PublicLinkMigrationReport map[AccountCapabilityControllerID]string - -// ReadPublicLinkMigrationReport reads a link migration report from the given reader, -// and extracts the public paths that were migrated. -// -// The report is expected to be a JSON array of objects with the following structure: -// -// [ -// {"kind":"link-migration-success","account_address":"0x1","path":"/public/foo","capability_id":1}, -// ] -func ReadPublicLinkMigrationReport(reader io.Reader) (PublicLinkMigrationReport, error) { - mapping := PublicLinkMigrationReport{} - - dec := json.NewDecoder(reader) - - token, err := dec.Token() - if err != nil { - return nil, fmt.Errorf("failed to read token: %w", err) - } - if token != json.Delim('[') { - return nil, fmt.Errorf("expected start of array, got %s", token) - } - - for dec.More() { - var entry struct { - Kind string `json:"kind"` - Address string `json:"account_address"` - Path string `json:"path"` - CapabilityID uint64 `json:"capability_id"` - } - err := dec.Decode(&entry) - if err != nil { - return nil, fmt.Errorf("failed to decode entry: %w", err) - } - - if entry.Kind != "link-migration-success" { - continue - } +var _ json.Marshaler = capabilityEntitlementsFixedEntry{} - identifier, ok := strings.CutPrefix(entry.Path, "/public/") - if !ok { - continue - } - - address, err := common.HexToAddress(entry.Address) - if err != nil { - return nil, fmt.Errorf("failed to parse address: %w", err) - } - - key := AccountCapabilityControllerID{ - Address: address, - CapabilityID: entry.CapabilityID, - } - mapping[key] = identifier - } - - token, err = dec.Token() - if err != nil { - return nil, fmt.Errorf("failed to read token: %w", err) - } - if token != json.Delim(']') { - return nil, fmt.Errorf("expected end of array, got %s", token) - } - - return mapping, nil -} - -type LinkInfo struct { - BorrowType common.TypeID - AccessibleMembers []string -} - -type AddressPublicPath struct { - Address common.Address - Identifier string -} - -// PublicLinkReport is a mapping from public account paths to link info. -type PublicLinkReport map[AddressPublicPath]LinkInfo - -// ReadPublicLinkReport reads a link report from the given reader. -// The report is expected to be a JSON array of objects with the following structure: -// -// [ -// {"address":"0x1","identifier":"foo","linkType":"&Foo","accessibleMembers":["foo"]} -// ] -func ReadPublicLinkReport(reader io.Reader) (PublicLinkReport, error) { - report := PublicLinkReport{} - - dec := json.NewDecoder(reader) - - token, err := dec.Token() - if err != nil { - return nil, fmt.Errorf("failed to read token: %w", err) - } - if token != json.Delim('[') { - return nil, fmt.Errorf("expected start of array, got %s", token) - } - - for dec.More() { - var entry struct { - Address string `json:"address"` - Identifier string `json:"identifier"` - LinkTypeID string `json:"linkType"` - AccessibleMembers []string `json:"accessibleMembers"` - } - err := dec.Decode(&entry) - if err != nil { - return nil, fmt.Errorf("failed to decode entry: %w", err) - } - - address, err := common.HexToAddress(entry.Address) - if err != nil { - return nil, fmt.Errorf("failed to parse address: %w", err) - } - - key := AddressPublicPath{ - Address: address, - Identifier: entry.Identifier, - } - report[key] = LinkInfo{ - BorrowType: common.TypeID(entry.LinkTypeID), - AccessibleMembers: entry.AccessibleMembers, - } - } - - token, err = dec.Token() - if err != nil { - return nil, fmt.Errorf("failed to read token: %w", err) - } - if token != json.Delim(']') { - return nil, fmt.Errorf("expected end of array, got %s", token) - } - - return report, nil +func (e capabilityEntitlementsFixedEntry) MarshalJSON() ([]byte, error) { + return json.Marshal(struct { + Kind string `json:"kind"` + AccountAddress string `json:"account_address"` + StorageDomain string `json:"domain"` + CapabilityAddress string `json:"capability_address"` + CapabilityID uint64 `json:"capability_id"` + NewAuthorization string `json:"new_authorization"` + }{ + Kind: "capability-entitlements-fixed", + AccountAddress: e.StorageKey.Address.HexWithPrefix(), + StorageDomain: e.StorageKey.Key, + CapabilityAddress: e.CapabilityAddress.HexWithPrefix(), + CapabilityID: e.CapabilityID, + NewAuthorization: jsonEncodeAuthorization(e.NewAuthorization), + }) } func NewFixEntitlementsMigrations( log zerolog.Logger, rwf reporters.ReportWriterFactory, - publicLinkReport PublicLinkReport, - publicLinkMigrationReport PublicLinkMigrationReport, + newAuthorizations map[AccountCapabilityControllerID]interpreter.Authorization, opts FixEntitlementsMigrationOptions, ) []NamedMigration { @@ -650,10 +407,8 @@ func NewFixEntitlementsMigrations( programs := make(map[common.Location]*interpreter.Program, 1000) - // TODO: - //fixedEntitlements := map[AccountCapabilityControllerID]struct{}{} - return []NamedMigration{ + // TODO: unnecessary? remove? { Name: "check-contracts", Migrate: NewContractCheckingMigration( @@ -667,32 +422,16 @@ func NewFixEntitlementsMigrations( ), }, { - Name: "fix-capability-controller-entitlements", - Migrate: NewAccountBasedMigration( - log, - opts.NWorker, - []AccountBasedMigration{ - NewFixCapabilityControllerEntitlementsMigration( - rwf, - errorMessageHandler, - programs, - publicLinkReport, - publicLinkMigrationReport, - opts, - ), - }, - ), - }, - { - Name: "fix-capability-entitlements", + Name: "fix-entitlements", Migrate: NewAccountBasedMigration( log, opts.NWorker, []AccountBasedMigration{ - NewFixCapabilityEntitlementsMigration( + NewFixEntitlementsMigration( rwf, errorMessageHandler, programs, + newAuthorizations, opts, ), }, diff --git a/cmd/util/ledger/migrations/fix_entitlements_migration_test.go b/cmd/util/ledger/migrations/fix_entitlements_migration_test.go index a192292bc02..578b4b6f8ce 100644 --- a/cmd/util/ledger/migrations/fix_entitlements_migration_test.go +++ b/cmd/util/ledger/migrations/fix_entitlements_migration_test.go @@ -1,8 +1,6 @@ package migrations import ( - "sort" - "strings" "testing" "github.com/onflow/cadence/runtime/common" @@ -91,88 +89,21 @@ func TestFixEntitlementMigrations(t *testing.T) { NWorker: nWorker, } - // Capability 1 was a public, unauthorized capability. - // It should lose its entitlement - // - // Capability 2 was a public, unauthorized capability, stored nested in storage. - // It should lose its entitlement - // - // Capability 3 was a private, authorized capability, stored nested in storage. - // It should keep its entitlement - // - // Capability 4 was a capability with unavailable accessible members, stored nested in storage. - // It should keep its entitlement - - readArrayMembers := []string{ - "concat", - "contains", - "filter", - "firstIndex", - "getType", - "isInstance", - "length", - "map", - "slice", - "toConstantSized", - } - - writeArrayMembers := []string{ - "append", - "appendAll", - "insert", - "remove", - "removeFirst", - "removeLast", - "reverse", - } - - readWriteArrayMembers := common.Concat(readArrayMembers, writeArrayMembers) - sort.Strings(readWriteArrayMembers) - - publicLinkReport := PublicLinkReport{ - { - Address: common.Address(address), - Identifier: "ints", - }: { - BorrowType: "&[Int]", - AccessibleMembers: readArrayMembers, - }, - { - Address: common.Address(address), - Identifier: "ints2", - }: { - BorrowType: "&[Int]", - AccessibleMembers: readArrayMembers, - }, - { - Address: common.Address(address), - Identifier: "ints4", - }: { - BorrowType: "&[Int]", - AccessibleMembers: nil, - }, - } - - publicLinkMigrationReport := PublicLinkMigrationReport{ + fixes := map[AccountCapabilityControllerID]interpreter.Authorization{ { Address: common.Address(address), CapabilityID: 1, - }: "ints", + }: interpreter.UnauthorizedAccess, { Address: common.Address(address), CapabilityID: 2, - }: "ints2", - { - Address: common.Address(address), - CapabilityID: 4, - }: "ints4", + }: interpreter.UnauthorizedAccess, } migrations := NewFixEntitlementsMigrations( log, rwf, - publicLinkReport, - publicLinkMigrationReport, + fixes, options, ) @@ -181,20 +112,20 @@ func TestFixEntitlementMigrations(t *testing.T) { require.NoError(t, err) } - reporter := rwf.reportWriters[fixCapabilityControllerEntitlementMigrationReportName] + reporter := rwf.reportWriters[fixEntitlementsMigrationReporterName] require.NotNil(t, reporter) var entries []any for _, entry := range reporter.entries { switch entry := entry.(type) { - case capabilityControllerEntitlementsFixedEntry: + case capabilityEntitlementsFixedEntry, + capabilityControllerEntitlementsFixedEntry: + entries = append(entries, entry) } } - // TODO: validate - require.ElementsMatch(t, []any{ capabilityControllerEntitlementsFixedEntry{ @@ -202,78 +133,36 @@ func TestFixEntitlementMigrations(t *testing.T) { Key: "cap_con", Address: common.Address(address), }, - CapabilityID: 1, - OldAccessibleMembers: readArrayMembers, - NewAccessibleMembers: readWriteArrayMembers, + CapabilityID: 1, + NewAuthorization: interpreter.UnauthorizedAccess, }, capabilityControllerEntitlementsFixedEntry{ StorageKey: interpreter.StorageKey{ Key: "cap_con", Address: common.Address(address), }, - CapabilityID: 2, - OldAccessibleMembers: readArrayMembers, - NewAccessibleMembers: readWriteArrayMembers, + CapabilityID: 2, + NewAuthorization: interpreter.UnauthorizedAccess, }, - }, - entries, - ) -} - -func TestReadPublicLinkMigrationReport(t *testing.T) { - t.Parallel() - - reader := strings.NewReader(` - [ - {"kind":"link-migration-success","account_address":"0x1","path":"/public/foo","capability_id":1}, - {"kind":"link-migration-success","account_address":"0x2","path":"/private/bar","capability_id":2} - ] - `) - - mapping, err := ReadPublicLinkMigrationReport(reader) - require.NoError(t, err) - - require.Equal(t, - PublicLinkMigrationReport{ - { - Address: common.MustBytesToAddress([]byte{0x1}), - CapabilityID: 1, - }: "foo", - }, - mapping, - ) -} - -func TestReadLinkReport(t *testing.T) { - t.Parallel() - - reader := strings.NewReader(` - [ - {"address":"0x1","identifier":"foo","linkType":"&Foo","accessibleMembers":["foo"]}, - {"address":"0x2","identifier":"bar","linkType":"&Bar","accessibleMembers":null} - ] - `) - - mapping, err := ReadPublicLinkReport(reader) - require.NoError(t, err) - - require.Equal(t, - PublicLinkReport{ - { - Address: common.MustBytesToAddress([]byte{0x1}), - Identifier: "foo", - }: { - BorrowType: "&Foo", - AccessibleMembers: []string{"foo"}, + capabilityEntitlementsFixedEntry{ + StorageKey: interpreter.StorageKey{ + Key: "public", + Address: common.Address(address), + }, + CapabilityAddress: common.Address(address), + CapabilityID: 1, + NewAuthorization: interpreter.UnauthorizedAccess, }, - { - Address: common.MustBytesToAddress([]byte{0x2}), - Identifier: "bar", - }: { - BorrowType: "&Bar", - AccessibleMembers: nil, + capabilityEntitlementsFixedEntry{ + StorageKey: interpreter.StorageKey{ + Key: "storage", + Address: common.Address(address), + }, + CapabilityAddress: common.Address(address), + CapabilityID: 2, + NewAuthorization: interpreter.UnauthorizedAccess, }, }, - mapping, + entries, ) } From 63b0ccd977a6b38b6730231626730b657fa581a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Fri, 6 Sep 2024 11:57:22 -0700 Subject: [PATCH 27/72] start work on command to generate list of entitlements fixes --- .../cmd/generate-entitlement-fixes/cmd.go | 153 ++++++++++++++++++ .../generate-entitlement-fixes/cmd_test.go | 67 ++++++++ 2 files changed, 220 insertions(+) create mode 100644 cmd/util/cmd/generate-entitlement-fixes/cmd.go create mode 100644 cmd/util/cmd/generate-entitlement-fixes/cmd_test.go diff --git a/cmd/util/cmd/generate-entitlement-fixes/cmd.go b/cmd/util/cmd/generate-entitlement-fixes/cmd.go new file mode 100644 index 00000000000..da1079f9317 --- /dev/null +++ b/cmd/util/cmd/generate-entitlement-fixes/cmd.go @@ -0,0 +1,153 @@ +package generate_entitlement_fixes + +import ( + "encoding/json" + "fmt" + "io" + "strings" + + "github.com/onflow/cadence/runtime/common" +) + +type AccountCapabilityControllerID struct { + Address common.Address + CapabilityID uint64 +} + +type LinkInfo struct { + BorrowType common.TypeID + AccessibleMembers []string +} + +type AddressPublicPath struct { + Address common.Address + Identifier string +} + +// PublicLinkReport is a mapping from public account paths to link info. +type PublicLinkReport map[AddressPublicPath]LinkInfo + +// ReadPublicLinkReport reads a link report from the given reader. +// The report is expected to be a JSON array of objects with the following structure: +// +// [ +// {"address":"0x1","identifier":"foo","linkType":"&Foo","accessibleMembers":["foo"]} +// ] +func ReadPublicLinkReport(reader io.Reader) (PublicLinkReport, error) { + report := PublicLinkReport{} + + dec := json.NewDecoder(reader) + + token, err := dec.Token() + if err != nil { + return nil, fmt.Errorf("failed to read token: %w", err) + } + if token != json.Delim('[') { + return nil, fmt.Errorf("expected start of array, got %s", token) + } + + for dec.More() { + var entry struct { + Address string `json:"address"` + Identifier string `json:"identifier"` + LinkTypeID string `json:"linkType"` + AccessibleMembers []string `json:"accessibleMembers"` + } + err := dec.Decode(&entry) + if err != nil { + return nil, fmt.Errorf("failed to decode entry: %w", err) + } + + address, err := common.HexToAddress(entry.Address) + if err != nil { + return nil, fmt.Errorf("failed to parse address: %w", err) + } + + key := AddressPublicPath{ + Address: address, + Identifier: entry.Identifier, + } + report[key] = LinkInfo{ + BorrowType: common.TypeID(entry.LinkTypeID), + AccessibleMembers: entry.AccessibleMembers, + } + } + + token, err = dec.Token() + if err != nil { + return nil, fmt.Errorf("failed to read token: %w", err) + } + if token != json.Delim(']') { + return nil, fmt.Errorf("expected end of array, got %s", token) + } + + return report, nil +} + +// PublicLinkMigrationReport is a mapping from account capability controller IDs to public path identifier. +type PublicLinkMigrationReport map[AccountCapabilityControllerID]string + +// ReadPublicLinkMigrationReport reads a link migration report from the given reader, +// and extracts the public paths that were migrated. +// +// The report is expected to be a JSON array of objects with the following structure: +// +// [ +// {"kind":"link-migration-success","account_address":"0x1","path":"/public/foo","capability_id":1}, +// ] +func ReadPublicLinkMigrationReport(reader io.Reader) (PublicLinkMigrationReport, error) { + mapping := PublicLinkMigrationReport{} + + dec := json.NewDecoder(reader) + + token, err := dec.Token() + if err != nil { + return nil, fmt.Errorf("failed to read token: %w", err) + } + if token != json.Delim('[') { + return nil, fmt.Errorf("expected start of array, got %s", token) + } + + for dec.More() { + var entry struct { + Kind string `json:"kind"` + Address string `json:"account_address"` + Path string `json:"path"` + CapabilityID uint64 `json:"capability_id"` + } + err := dec.Decode(&entry) + if err != nil { + return nil, fmt.Errorf("failed to decode entry: %w", err) + } + + if entry.Kind != "link-migration-success" { + continue + } + + identifier, ok := strings.CutPrefix(entry.Path, "/public/") + if !ok { + continue + } + + address, err := common.HexToAddress(entry.Address) + if err != nil { + return nil, fmt.Errorf("failed to parse address: %w", err) + } + + key := AccountCapabilityControllerID{ + Address: address, + CapabilityID: entry.CapabilityID, + } + mapping[key] = identifier + } + + token, err = dec.Token() + if err != nil { + return nil, fmt.Errorf("failed to read token: %w", err) + } + if token != json.Delim(']') { + return nil, fmt.Errorf("expected end of array, got %s", token) + } + + return mapping, nil +} diff --git a/cmd/util/cmd/generate-entitlement-fixes/cmd_test.go b/cmd/util/cmd/generate-entitlement-fixes/cmd_test.go new file mode 100644 index 00000000000..28d84e8c66b --- /dev/null +++ b/cmd/util/cmd/generate-entitlement-fixes/cmd_test.go @@ -0,0 +1,67 @@ +package generate_entitlement_fixes + +import ( + "strings" + "testing" + + "github.com/onflow/cadence/runtime/common" + "github.com/stretchr/testify/require" +) + +func TestReadPublicLinkMigrationReport(t *testing.T) { + t.Parallel() + + reader := strings.NewReader(` + [ + {"kind":"link-migration-success","account_address":"0x1","path":"/public/foo","capability_id":1}, + {"kind":"link-migration-success","account_address":"0x2","path":"/private/bar","capability_id":2} + ] + `) + + mapping, err := ReadPublicLinkMigrationReport(reader) + require.NoError(t, err) + + require.Equal(t, + PublicLinkMigrationReport{ + { + Address: common.MustBytesToAddress([]byte{0x1}), + CapabilityID: 1, + }: "foo", + }, + mapping, + ) +} + +func TestReadLinkReport(t *testing.T) { + t.Parallel() + + reader := strings.NewReader(` + [ + {"address":"0x1","identifier":"foo","linkType":"&Foo","accessibleMembers":["foo"]}, + {"address":"0x2","identifier":"bar","linkType":"&Bar","accessibleMembers":null} + ] + `) + + mapping, err := ReadPublicLinkReport(reader) + require.NoError(t, err) + + require.Equal(t, + PublicLinkReport{ + { + Address: common.MustBytesToAddress([]byte{0x1}), + Identifier: "foo", + }: { + BorrowType: "&Foo", + AccessibleMembers: []string{"foo"}, + }, + { + Address: common.MustBytesToAddress([]byte{0x2}), + Identifier: "bar", + }: { + BorrowType: "&Bar", + AccessibleMembers: nil, + }, + }, + mapping, + ) +} From b2b238c99b1ca3313169f8697d9d30663076e418 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Fri, 6 Sep 2024 12:03:10 -0700 Subject: [PATCH 28/72] allow filtering when reading reports --- .../cmd/generate-entitlement-fixes/cmd.go | 24 ++- .../generate-entitlement-fixes/cmd_test.go | 151 +++++++++++++----- 2 files changed, 134 insertions(+), 41 deletions(-) diff --git a/cmd/util/cmd/generate-entitlement-fixes/cmd.go b/cmd/util/cmd/generate-entitlement-fixes/cmd.go index da1079f9317..9c2e4713d8f 100644 --- a/cmd/util/cmd/generate-entitlement-fixes/cmd.go +++ b/cmd/util/cmd/generate-entitlement-fixes/cmd.go @@ -33,7 +33,11 @@ type PublicLinkReport map[AddressPublicPath]LinkInfo // [ // {"address":"0x1","identifier":"foo","linkType":"&Foo","accessibleMembers":["foo"]} // ] -func ReadPublicLinkReport(reader io.Reader) (PublicLinkReport, error) { +func ReadPublicLinkReport( + reader io.Reader, + filter map[common.Address]struct{}, +) (PublicLinkReport, error) { + report := PublicLinkReport{} dec := json.NewDecoder(reader) @@ -63,6 +67,12 @@ func ReadPublicLinkReport(reader io.Reader) (PublicLinkReport, error) { return nil, fmt.Errorf("failed to parse address: %w", err) } + if filter != nil { + if _, ok := filter[address]; !ok { + continue + } + } + key := AddressPublicPath{ Address: address, Identifier: entry.Identifier, @@ -95,7 +105,11 @@ type PublicLinkMigrationReport map[AccountCapabilityControllerID]string // [ // {"kind":"link-migration-success","account_address":"0x1","path":"/public/foo","capability_id":1}, // ] -func ReadPublicLinkMigrationReport(reader io.Reader) (PublicLinkMigrationReport, error) { +func ReadPublicLinkMigrationReport( + reader io.Reader, + filter map[common.Address]struct{}, +) (PublicLinkMigrationReport, error) { + mapping := PublicLinkMigrationReport{} dec := json.NewDecoder(reader) @@ -134,6 +148,12 @@ func ReadPublicLinkMigrationReport(reader io.Reader) (PublicLinkMigrationReport, return nil, fmt.Errorf("failed to parse address: %w", err) } + if filter != nil { + if _, ok := filter[address]; !ok { + continue + } + } + key := AccountCapabilityControllerID{ Address: address, CapabilityID: entry.CapabilityID, diff --git a/cmd/util/cmd/generate-entitlement-fixes/cmd_test.go b/cmd/util/cmd/generate-entitlement-fixes/cmd_test.go index 28d84e8c66b..40b6863b7b8 100644 --- a/cmd/util/cmd/generate-entitlement-fixes/cmd_test.go +++ b/cmd/util/cmd/generate-entitlement-fixes/cmd_test.go @@ -11,57 +11,130 @@ import ( func TestReadPublicLinkMigrationReport(t *testing.T) { t.Parallel() - reader := strings.NewReader(` + contents := ` [ {"kind":"link-migration-success","account_address":"0x1","path":"/public/foo","capability_id":1}, - {"kind":"link-migration-success","account_address":"0x2","path":"/private/bar","capability_id":2} + {"kind":"link-migration-success","account_address":"0x2","path":"/private/bar","capability_id":2}, + {"kind":"link-migration-success","account_address":"0x3","path":"/public/baz","capability_id":3} ] - `) - - mapping, err := ReadPublicLinkMigrationReport(reader) - require.NoError(t, err) - - require.Equal(t, - PublicLinkMigrationReport{ - { - Address: common.MustBytesToAddress([]byte{0x1}), - CapabilityID: 1, - }: "foo", - }, - mapping, - ) + ` + + t.Run("unfiltered", func(t *testing.T) { + t.Parallel() + + reader := strings.NewReader(contents) + + mapping, err := ReadPublicLinkMigrationReport(reader, nil) + require.NoError(t, err) + + require.Equal(t, + PublicLinkMigrationReport{ + { + Address: common.MustBytesToAddress([]byte{0x1}), + CapabilityID: 1, + }: "foo", + { + Address: common.MustBytesToAddress([]byte{0x3}), + CapabilityID: 3, + }: "baz", + }, + mapping, + ) + }) + + t.Run("filtered", func(t *testing.T) { + t.Parallel() + + address1 := common.MustBytesToAddress([]byte{0x1}) + + reader := strings.NewReader(contents) + + mapping, err := ReadPublicLinkMigrationReport( + reader, + map[common.Address]struct{}{ + address1: {}, + }, + ) + require.NoError(t, err) + + require.Equal(t, + PublicLinkMigrationReport{ + { + Address: address1, + CapabilityID: 1, + }: "foo", + }, + mapping, + ) + }) } func TestReadLinkReport(t *testing.T) { t.Parallel() - reader := strings.NewReader(` + contents := ` [ {"address":"0x1","identifier":"foo","linkType":"&Foo","accessibleMembers":["foo"]}, {"address":"0x2","identifier":"bar","linkType":"&Bar","accessibleMembers":null} ] - `) - - mapping, err := ReadPublicLinkReport(reader) - require.NoError(t, err) - - require.Equal(t, - PublicLinkReport{ - { - Address: common.MustBytesToAddress([]byte{0x1}), - Identifier: "foo", - }: { - BorrowType: "&Foo", - AccessibleMembers: []string{"foo"}, + ` + + t.Run("unfiltered", func(t *testing.T) { + + t.Parallel() + + reader := strings.NewReader(contents) + + mapping, err := ReadPublicLinkReport(reader, nil) + require.NoError(t, err) + + require.Equal(t, + PublicLinkReport{ + { + Address: common.MustBytesToAddress([]byte{0x1}), + Identifier: "foo", + }: { + BorrowType: "&Foo", + AccessibleMembers: []string{"foo"}, + }, + { + Address: common.MustBytesToAddress([]byte{0x2}), + Identifier: "bar", + }: { + BorrowType: "&Bar", + AccessibleMembers: nil, + }, }, - { - Address: common.MustBytesToAddress([]byte{0x2}), - Identifier: "bar", - }: { - BorrowType: "&Bar", - AccessibleMembers: nil, + mapping, + ) + }) + + t.Run("unfiltered", func(t *testing.T) { + + t.Parallel() + + address1 := common.MustBytesToAddress([]byte{0x1}) + + reader := strings.NewReader(contents) + + mapping, err := ReadPublicLinkReport( + reader, + map[common.Address]struct{}{ + address1: {}, + }) + require.NoError(t, err) + + require.Equal(t, + PublicLinkReport{ + { + Address: address1, + Identifier: "foo", + }: { + BorrowType: "&Foo", + AccessibleMembers: []string{"foo"}, + }, }, - }, - mapping, - ) + mapping, + ) + }) } From cc08fa810b025490c661f7d85e9a5f9c560d26eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Fri, 6 Sep 2024 13:32:39 -0700 Subject: [PATCH 29/72] port getAccessibleMembers from link reporter --- .../cmd/generate-entitlement-fixes/cmd.go | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/cmd/util/cmd/generate-entitlement-fixes/cmd.go b/cmd/util/cmd/generate-entitlement-fixes/cmd.go index 9c2e4713d8f..3b5574af779 100644 --- a/cmd/util/cmd/generate-entitlement-fixes/cmd.go +++ b/cmd/util/cmd/generate-entitlement-fixes/cmd.go @@ -6,7 +6,9 @@ import ( "io" "strings" + "github.com/onflow/cadence/runtime/ast" "github.com/onflow/cadence/runtime/common" + "github.com/onflow/cadence/runtime/interpreter" ) type AccountCapabilityControllerID struct { @@ -171,3 +173,55 @@ func ReadPublicLinkMigrationReport( return mapping, nil } + +func getAccessibleMembers( + inter *interpreter.Interpreter, + staticType interpreter.StaticType, +) ( + accessibleMembers []string, + err error, +) { + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("panic: %v", r) + } + }() + + semaType, err := inter.ConvertStaticToSemaType(staticType) + if err != nil { + return nil, fmt.Errorf( + "failed to convert static type %s to semantic type: %w", + staticType.ID(), + err, + ) + } + if semaType == nil { + return nil, fmt.Errorf( + "failed to convert static type %s to semantic type", + staticType.ID(), + ) + } + + // NOTE: RestrictedType.GetMembers returns *all* members, + // including those that are not accessible, for DX purposes. + // We need to resolve the members and filter out the inaccessible members, + // using the error reported when resolving + + memberResolvers := semaType.GetMembers() + + accessibleMembers = make([]string, 0, len(memberResolvers)) + + for memberName, memberResolver := range memberResolvers { + var resolveErr error + memberResolver.Resolve(nil, memberName, ast.EmptyRange, func(err error) { + resolveErr = err + }) + if resolveErr != nil { + continue + } + accessibleMembers = append(accessibleMembers, memberName) + } + + return accessibleMembers, nil +} + From c1f092456ecb7227a4f9f31be5bcdcae1724747d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Fri, 6 Sep 2024 13:52:11 -0700 Subject: [PATCH 30/72] refactor into separate files --- .../accessible_members.go | 59 ++++ .../check_contract.go | 119 +++++++ .../cmd/generate-entitlement-fixes/cmd.go | 328 ++++++++---------- .../generate-entitlement-fixes/contracts.go | 82 +++++ .../link_migration_report.go | 94 +++++ .../link_migration_report_test.go | 70 ++++ .../generate-entitlement-fixes/link_report.go | 90 +++++ .../{cmd_test.go => link_report_test.go} | 61 ---- 8 files changed, 665 insertions(+), 238 deletions(-) create mode 100644 cmd/util/cmd/generate-entitlement-fixes/accessible_members.go create mode 100644 cmd/util/cmd/generate-entitlement-fixes/check_contract.go create mode 100644 cmd/util/cmd/generate-entitlement-fixes/contracts.go create mode 100644 cmd/util/cmd/generate-entitlement-fixes/link_migration_report.go create mode 100644 cmd/util/cmd/generate-entitlement-fixes/link_migration_report_test.go create mode 100644 cmd/util/cmd/generate-entitlement-fixes/link_report.go rename cmd/util/cmd/generate-entitlement-fixes/{cmd_test.go => link_report_test.go} (53%) diff --git a/cmd/util/cmd/generate-entitlement-fixes/accessible_members.go b/cmd/util/cmd/generate-entitlement-fixes/accessible_members.go new file mode 100644 index 00000000000..52f68a8ff7c --- /dev/null +++ b/cmd/util/cmd/generate-entitlement-fixes/accessible_members.go @@ -0,0 +1,59 @@ +package generate_entitlement_fixes + +import ( + "fmt" + + "github.com/onflow/cadence/runtime/ast" + "github.com/onflow/cadence/runtime/interpreter" +) + +func getAccessibleMembers( + inter *interpreter.Interpreter, + staticType interpreter.StaticType, +) ( + accessibleMembers []string, + err error, +) { + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("panic: %v", r) + } + }() + + semaType, err := inter.ConvertStaticToSemaType(staticType) + if err != nil { + return nil, fmt.Errorf( + "failed to convert static type %s to semantic type: %w", + staticType.ID(), + err, + ) + } + if semaType == nil { + return nil, fmt.Errorf( + "failed to convert static type %s to semantic type", + staticType.ID(), + ) + } + + // NOTE: RestrictedType.GetMembers returns *all* members, + // including those that are not accessible, for DX purposes. + // We need to resolve the members and filter out the inaccessible members, + // using the error reported when resolving + + memberResolvers := semaType.GetMembers() + + accessibleMembers = make([]string, 0, len(memberResolvers)) + + for memberName, memberResolver := range memberResolvers { + var resolveErr error + memberResolver.Resolve(nil, memberName, ast.EmptyRange, func(err error) { + resolveErr = err + }) + if resolveErr != nil { + continue + } + accessibleMembers = append(accessibleMembers, memberName) + } + + return accessibleMembers, nil +} diff --git a/cmd/util/cmd/generate-entitlement-fixes/check_contract.go b/cmd/util/cmd/generate-entitlement-fixes/check_contract.go new file mode 100644 index 00000000000..4616287c572 --- /dev/null +++ b/cmd/util/cmd/generate-entitlement-fixes/check_contract.go @@ -0,0 +1,119 @@ +package generate_entitlement_fixes + +import ( + "encoding/json" + "strings" + + "github.com/onflow/cadence/runtime/common" + "github.com/onflow/cadence/runtime/interpreter" + "github.com/onflow/cadence/runtime/pretty" + "github.com/rs/zerolog/log" + + "github.com/onflow/flow-go/cmd/util/ledger/migrations" + "github.com/onflow/flow-go/cmd/util/ledger/reporters" +) + +func checkContract( + contract AddressContract, + mr *migrations.InterpreterMigrationRuntime, + contractsForPrettyPrinting map[common.Location][]byte, + reporter reporters.ReportWriter, + programs map[common.Location]*interpreter.Program, +) { + location := contract.Location + code := contract.Code + + log.Info().Msgf("checking contract %s ...", location) + + // Check contract code + const getAndSetProgram = true + program, err := mr.ContractAdditionHandler.ParseAndCheckProgram(code, location, getAndSetProgram) + if err != nil { + + // Pretty print the error + var builder strings.Builder + errorPrinter := pretty.NewErrorPrettyPrinter(&builder, false) + + printErr := errorPrinter.PrettyPrintError(err, location, contractsForPrettyPrinting) + + var errorDetails string + if printErr == nil { + errorDetails = builder.String() + } else { + errorDetails = err.Error() + } + + log.Error().Msgf( + "error checking contract %s: %s", + location, + errorDetails, + ) + + reporter.Write(contractCheckingFailure{ + AccountAddress: location.Address, + ContractName: location.Name, + Code: string(code), + Error: errorDetails, + }) + + return + } + + // Record the checked program for future use + programs[location] = program + + reporter.Write(contractCheckingSuccess{ + AccountAddress: location.Address, + ContractName: location.Name, + Code: string(code), + }) + + log.Info().Msgf("finished checking contract %s", location) +} + +type contractCheckingFailure struct { + AccountAddress common.Address + ContractName string + Code string + Error string +} + +var _ json.Marshaler = contractCheckingFailure{} + +func (e contractCheckingFailure) MarshalJSON() ([]byte, error) { + return json.Marshal(struct { + Kind string `json:"kind"` + AccountAddress string `json:"address"` + ContractName string `json:"name"` + Code string `json:"code"` + Error string `json:"error"` + }{ + Kind: "checking-failure", + AccountAddress: e.AccountAddress.HexWithPrefix(), + ContractName: e.ContractName, + Code: e.Code, + Error: e.Error, + }) +} + +type contractCheckingSuccess struct { + AccountAddress common.Address + ContractName string + Code string +} + +var _ json.Marshaler = contractCheckingSuccess{} + +func (e contractCheckingSuccess) MarshalJSON() ([]byte, error) { + return json.Marshal(struct { + Kind string `json:"kind"` + AccountAddress string `json:"address"` + ContractName string `json:"name"` + Code string `json:"code"` + }{ + Kind: "checking-success", + AccountAddress: e.AccountAddress.HexWithPrefix(), + ContractName: e.ContractName, + Code: e.Code, + }) +} diff --git a/cmd/util/cmd/generate-entitlement-fixes/cmd.go b/cmd/util/cmd/generate-entitlement-fixes/cmd.go index 3b5574af779..db12fb423a0 100644 --- a/cmd/util/cmd/generate-entitlement-fixes/cmd.go +++ b/cmd/util/cmd/generate-entitlement-fixes/cmd.go @@ -2,226 +2,200 @@ package generate_entitlement_fixes import ( "encoding/json" - "fmt" - "io" - "strings" - "github.com/onflow/cadence/runtime/ast" "github.com/onflow/cadence/runtime/common" "github.com/onflow/cadence/runtime/interpreter" + "github.com/rs/zerolog/log" + "github.com/spf13/cobra" + + "github.com/onflow/flow-go/cmd/util/ledger/migrations" + "github.com/onflow/flow-go/cmd/util/ledger/reporters" + "github.com/onflow/flow-go/cmd/util/ledger/util" + "github.com/onflow/flow-go/cmd/util/ledger/util/registers" + "github.com/onflow/flow-go/ledger" + "github.com/onflow/flow-go/model/flow" ) -type AccountCapabilityControllerID struct { - Address common.Address - CapabilityID uint64 -} +var ( + flagPayloads string + flagState string + flagStateCommitment string + flagOutputDirectory string + flagChain string +) -type LinkInfo struct { - BorrowType common.TypeID - AccessibleMembers []string +var Cmd = &cobra.Command{ + Use: "report-links", + Short: "reports links", + Run: run, } -type AddressPublicPath struct { - Address common.Address - Identifier string +func init() { + + Cmd.Flags().StringVar( + &flagPayloads, + "payloads", + "", + "Input payload file name", + ) + + Cmd.Flags().StringVar( + &flagState, + "state", + "", + "Input state file name", + ) + + Cmd.Flags().StringVar( + &flagStateCommitment, + "state-commitment", + "", + "Input state commitment", + ) + + Cmd.Flags().StringVar( + &flagOutputDirectory, + "output-directory", + "", + "Output directory", + ) + + Cmd.Flags().StringVar( + &flagChain, + "chain", + "", + "Chain name", + ) + _ = Cmd.MarkFlagRequired("chain") } -// PublicLinkReport is a mapping from public account paths to link info. -type PublicLinkReport map[AddressPublicPath]LinkInfo - -// ReadPublicLinkReport reads a link report from the given reader. -// The report is expected to be a JSON array of objects with the following structure: -// -// [ -// {"address":"0x1","identifier":"foo","linkType":"&Foo","accessibleMembers":["foo"]} -// ] -func ReadPublicLinkReport( - reader io.Reader, - filter map[common.Address]struct{}, -) (PublicLinkReport, error) { - - report := PublicLinkReport{} +const contractCountEstimate = 1000 - dec := json.NewDecoder(reader) +func run(*cobra.Command, []string) { - token, err := dec.Token() - if err != nil { - return nil, fmt.Errorf("failed to read token: %w", err) + if flagPayloads == "" && flagState == "" { + log.Fatal().Msg("Either --payloads or --state must be provided") + } else if flagPayloads != "" && flagState != "" { + log.Fatal().Msg("Only one of --payloads or --state must be provided") } - if token != json.Delim('[') { - return nil, fmt.Errorf("expected start of array, got %s", token) + if flagState != "" && flagStateCommitment == "" { + log.Fatal().Msg("--state-commitment must be provided when --state is provided") } - for dec.More() { - var entry struct { - Address string `json:"address"` - Identifier string `json:"identifier"` - LinkTypeID string `json:"linkType"` - AccessibleMembers []string `json:"accessibleMembers"` - } - err := dec.Decode(&entry) - if err != nil { - return nil, fmt.Errorf("failed to decode entry: %w", err) - } + rwf := reporters.NewReportFileWriterFactory(flagOutputDirectory, log.Logger) - address, err := common.HexToAddress(entry.Address) - if err != nil { - return nil, fmt.Errorf("failed to parse address: %w", err) - } + reporter := rwf.ReportWriter("entitlement-fixes") + defer reporter.Close() - if filter != nil { - if _, ok := filter[address]; !ok { - continue - } - } + chainID := flow.ChainID(flagChain) + // Validate chain ID + _ = chainID.Chain() - key := AddressPublicPath{ - Address: address, - Identifier: entry.Identifier, - } - report[key] = LinkInfo{ - BorrowType: common.TypeID(entry.LinkTypeID), - AccessibleMembers: entry.AccessibleMembers, - } - } - - token, err = dec.Token() - if err != nil { - return nil, fmt.Errorf("failed to read token: %w", err) - } - if token != json.Delim(']') { - return nil, fmt.Errorf("expected end of array, got %s", token) - } - - return report, nil -} - -// PublicLinkMigrationReport is a mapping from account capability controller IDs to public path identifier. -type PublicLinkMigrationReport map[AccountCapabilityControllerID]string - -// ReadPublicLinkMigrationReport reads a link migration report from the given reader, -// and extracts the public paths that were migrated. -// -// The report is expected to be a JSON array of objects with the following structure: -// -// [ -// {"kind":"link-migration-success","account_address":"0x1","path":"/public/foo","capability_id":1}, -// ] -func ReadPublicLinkMigrationReport( - reader io.Reader, - filter map[common.Address]struct{}, -) (PublicLinkMigrationReport, error) { + var payloads []*ledger.Payload + var err error - mapping := PublicLinkMigrationReport{} + // Read payloads from payload file or checkpoint file - dec := json.NewDecoder(reader) - - token, err := dec.Token() - if err != nil { - return nil, fmt.Errorf("failed to read token: %w", err) - } - if token != json.Delim('[') { - return nil, fmt.Errorf("expected start of array, got %s", token) - } + if flagPayloads != "" { + log.Info().Msgf("Reading payloads from %s", flagPayloads) - for dec.More() { - var entry struct { - Kind string `json:"kind"` - Address string `json:"account_address"` - Path string `json:"path"` - CapabilityID uint64 `json:"capability_id"` - } - err := dec.Decode(&entry) + _, payloads, err = util.ReadPayloadFile(log.Logger, flagPayloads) if err != nil { - return nil, fmt.Errorf("failed to decode entry: %w", err) - } - - if entry.Kind != "link-migration-success" { - continue + log.Fatal().Err(err).Msg("failed to read payloads") } + } else { + log.Info().Msgf("Reading trie %s", flagStateCommitment) - identifier, ok := strings.CutPrefix(entry.Path, "/public/") - if !ok { - continue - } - - address, err := common.HexToAddress(entry.Address) + stateCommitment := util.ParseStateCommitment(flagStateCommitment) + payloads, err = util.ReadTrie(flagState, stateCommitment) if err != nil { - return nil, fmt.Errorf("failed to parse address: %w", err) + log.Fatal().Err(err).Msg("failed to read state") } - - if filter != nil { - if _, ok := filter[address]; !ok { - continue - } - } - - key := AccountCapabilityControllerID{ - Address: address, - CapabilityID: entry.CapabilityID, - } - mapping[key] = identifier } - token, err = dec.Token() + log.Info().Msgf("creating registers from payloads (%d)", len(payloads)) + + registersByAccount, err := registers.NewByAccountFromPayloads(payloads) if err != nil { - return nil, fmt.Errorf("failed to read token: %w", err) + log.Fatal().Err(err) } - if token != json.Delim(']') { - return nil, fmt.Errorf("expected end of array, got %s", token) + log.Info().Msgf( + "created %d registers from payloads (%d accounts)", + registersByAccount.Count(), + registersByAccount.AccountCount(), + ) + + mr, err := migrations.NewInterpreterMigrationRuntime( + registersByAccount, + chainID, + migrations.InterpreterMigrationRuntimeConfig{}, + ) + if err != nil { + log.Fatal().Err(err) } - return mapping, nil + checkContracts(registersByAccount, mr, reporter) + } -func getAccessibleMembers( - inter *interpreter.Interpreter, - staticType interpreter.StaticType, -) ( - accessibleMembers []string, - err error, +func checkContracts( + registersByAccount *registers.ByAccount, + mr *migrations.InterpreterMigrationRuntime, + reporter reporters.ReportWriter, ) { - defer func() { - if r := recover(); r != nil { - err = fmt.Errorf("panic: %v", r) - } - }() - - semaType, err := inter.ConvertStaticToSemaType(staticType) + contracts, err := gatherContractsFromRegisters(registersByAccount) if err != nil { - return nil, fmt.Errorf( - "failed to convert static type %s to semantic type: %w", - staticType.ID(), - err, - ) + log.Fatal().Err(err) } - if semaType == nil { - return nil, fmt.Errorf( - "failed to convert static type %s to semantic type", - staticType.ID(), - ) + + programs := make(map[common.Location]*interpreter.Program, contractCountEstimate) + + contractsForPrettyPrinting := make(map[common.Location][]byte, len(contracts)) + for _, contract := range contracts { + contractsForPrettyPrinting[contract.Location] = contract.Code } - // NOTE: RestrictedType.GetMembers returns *all* members, - // including those that are not accessible, for DX purposes. - // We need to resolve the members and filter out the inaccessible members, - // using the error reported when resolving + log.Info().Msg("Checking contracts ...") - memberResolvers := semaType.GetMembers() + for _, contract := range contracts { + checkContract( + contract, + mr, + contractsForPrettyPrinting, + reporter, + programs, + ) + } - accessibleMembers = make([]string, 0, len(memberResolvers)) + log.Info().Msgf("Checked %d contracts ...", len(contracts)) +} - for memberName, memberResolver := range memberResolvers { - var resolveErr error - memberResolver.Resolve(nil, memberName, ast.EmptyRange, func(err error) { - resolveErr = err - }) - if resolveErr != nil { - continue - } - accessibleMembers = append(accessibleMembers, memberName) +func jsonEncodeAuthorization(authorization interpreter.Authorization) string { + switch authorization { + case interpreter.UnauthorizedAccess, interpreter.InaccessibleAccess: + return "" + default: + return string(authorization.ID()) } +} - return accessibleMembers, nil +type fixEntitlementsEntry struct { + AccountCapabilityID + NewAuthorization interpreter.Authorization } +var _ json.Marshaler = fixEntitlementsEntry{} + +func (e fixEntitlementsEntry) MarshalJSON() ([]byte, error) { + return json.Marshal(struct { + Kind string `json:"kind"` + CapabilityAddress string `json:"capability_address"` + CapabilityID uint64 `json:"capability_id"` + NewAuthorization string `json:"new_authorization"` + }{ + Kind: "fix-entitlements", + CapabilityAddress: e.Address.String(), + CapabilityID: e.CapabilityID, + NewAuthorization: jsonEncodeAuthorization(e.NewAuthorization), + }) +} diff --git a/cmd/util/cmd/generate-entitlement-fixes/contracts.go b/cmd/util/cmd/generate-entitlement-fixes/contracts.go new file mode 100644 index 00000000000..25a95161863 --- /dev/null +++ b/cmd/util/cmd/generate-entitlement-fixes/contracts.go @@ -0,0 +1,82 @@ +package generate_entitlement_fixes + +import ( + "bytes" + "fmt" + "sort" + + "github.com/onflow/cadence/runtime/common" + "github.com/rs/zerolog/log" + + "github.com/onflow/flow-go/cmd/util/ledger/util/registers" + "github.com/onflow/flow-go/fvm/environment" + "github.com/onflow/flow-go/model/flow" +) + +type AddressContract struct { + Location common.AddressLocation + Code []byte +} + +func gatherContractsFromRegisters(registersByAccount *registers.ByAccount) ([]AddressContract, error) { + log.Info().Msg("Gathering contracts ...") + + contracts := make([]AddressContract, 0, contractCountEstimate) + + err := registersByAccount.ForEachAccount(func(accountRegisters *registers.AccountRegisters) error { + owner := accountRegisters.Owner() + + encodedContractNames, err := accountRegisters.Get(owner, flow.ContractNamesKey) + if err != nil { + return err + } + + contractNames, err := environment.DecodeContractNames(encodedContractNames) + if err != nil { + return err + } + + for _, contractName := range contractNames { + + contractKey := flow.ContractKey(contractName) + + code, err := accountRegisters.Get(owner, contractKey) + if err != nil { + return err + } + + if len(bytes.TrimSpace(code)) == 0 { + continue + } + + address := common.Address([]byte(owner)) + location := common.AddressLocation{ + Address: address, + Name: contractName, + } + + contracts = append( + contracts, + AddressContract{ + Location: location, + Code: code, + }, + ) + } + + return nil + }) + if err != nil { + return nil, fmt.Errorf("failed to get contracts of accounts: %w", err) + } + + sort.Slice(contracts, func(i, j int) bool { + a := contracts[i] + b := contracts[j] + return a.Location.ID() < b.Location.ID() + }) + + log.Info().Msgf("Gathered all contracts (%d)", len(contracts)) + + return contracts, nil +} diff --git a/cmd/util/cmd/generate-entitlement-fixes/link_migration_report.go b/cmd/util/cmd/generate-entitlement-fixes/link_migration_report.go new file mode 100644 index 00000000000..b99284eb621 --- /dev/null +++ b/cmd/util/cmd/generate-entitlement-fixes/link_migration_report.go @@ -0,0 +1,94 @@ +package generate_entitlement_fixes + +import ( + "encoding/json" + "fmt" + "io" + "strings" + + "github.com/onflow/cadence/runtime/common" +) + +// AccountCapabilityID is a capability ID in an account. +type AccountCapabilityID struct { + Address common.Address + CapabilityID uint64 +} + +// PublicLinkMigrationReport is a mapping from account capability controller IDs to public path identifier. +type PublicLinkMigrationReport map[AccountCapabilityID]string + +// ReadPublicLinkMigrationReport reads a link migration report from the given reader, +// and extracts the public paths that were migrated. +// +// The report is expected to be a JSON array of objects with the following structure: +// +// [ +// {"kind":"link-migration-success","account_address":"0x1","path":"/public/foo","capability_id":1}, +// ] +func ReadPublicLinkMigrationReport( + reader io.Reader, + filter map[common.Address]struct{}, +) (PublicLinkMigrationReport, error) { + + mapping := PublicLinkMigrationReport{} + + dec := json.NewDecoder(reader) + + token, err := dec.Token() + if err != nil { + return nil, fmt.Errorf("failed to read token: %w", err) + } + if token != json.Delim('[') { + return nil, fmt.Errorf("expected start of array, got %s", token) + } + + for dec.More() { + var entry struct { + Kind string `json:"kind"` + Address string `json:"account_address"` + Path string `json:"path"` + CapabilityID uint64 `json:"capability_id"` + } + err := dec.Decode(&entry) + if err != nil { + return nil, fmt.Errorf("failed to decode entry: %w", err) + } + + if entry.Kind != "link-migration-success" { + continue + } + + identifier, ok := strings.CutPrefix(entry.Path, "/public/") + if !ok { + continue + } + + address, err := common.HexToAddress(entry.Address) + if err != nil { + return nil, fmt.Errorf("failed to parse address: %w", err) + } + + if filter != nil { + if _, ok := filter[address]; !ok { + continue + } + } + + key := AccountCapabilityID{ + Address: address, + CapabilityID: entry.CapabilityID, + } + mapping[key] = identifier + } + + token, err = dec.Token() + if err != nil { + return nil, fmt.Errorf("failed to read token: %w", err) + } + if token != json.Delim(']') { + return nil, fmt.Errorf("expected end of array, got %s", token) + } + + return mapping, nil +} diff --git a/cmd/util/cmd/generate-entitlement-fixes/link_migration_report_test.go b/cmd/util/cmd/generate-entitlement-fixes/link_migration_report_test.go new file mode 100644 index 00000000000..3a2b81a3952 --- /dev/null +++ b/cmd/util/cmd/generate-entitlement-fixes/link_migration_report_test.go @@ -0,0 +1,70 @@ +package generate_entitlement_fixes + +import ( + "strings" + "testing" + + "github.com/onflow/cadence/runtime/common" + "github.com/stretchr/testify/require" +) + +func TestReadPublicLinkMigrationReport(t *testing.T) { + t.Parallel() + + contents := ` + [ + {"kind":"link-migration-success","account_address":"0x1","path":"/public/foo","capability_id":1}, + {"kind":"link-migration-success","account_address":"0x2","path":"/private/bar","capability_id":2}, + {"kind":"link-migration-success","account_address":"0x3","path":"/public/baz","capability_id":3} + ] + ` + + t.Run("unfiltered", func(t *testing.T) { + t.Parallel() + + reader := strings.NewReader(contents) + + mapping, err := ReadPublicLinkMigrationReport(reader, nil) + require.NoError(t, err) + + require.Equal(t, + PublicLinkMigrationReport{ + { + Address: common.MustBytesToAddress([]byte{0x1}), + CapabilityID: 1, + }: "foo", + { + Address: common.MustBytesToAddress([]byte{0x3}), + CapabilityID: 3, + }: "baz", + }, + mapping, + ) + }) + + t.Run("filtered", func(t *testing.T) { + t.Parallel() + + address1 := common.MustBytesToAddress([]byte{0x1}) + + reader := strings.NewReader(contents) + + mapping, err := ReadPublicLinkMigrationReport( + reader, + map[common.Address]struct{}{ + address1: {}, + }, + ) + require.NoError(t, err) + + require.Equal(t, + PublicLinkMigrationReport{ + { + Address: address1, + CapabilityID: 1, + }: "foo", + }, + mapping, + ) + }) +} diff --git a/cmd/util/cmd/generate-entitlement-fixes/link_report.go b/cmd/util/cmd/generate-entitlement-fixes/link_report.go new file mode 100644 index 00000000000..306742666f8 --- /dev/null +++ b/cmd/util/cmd/generate-entitlement-fixes/link_report.go @@ -0,0 +1,90 @@ +package generate_entitlement_fixes + +import ( + "encoding/json" + "fmt" + "io" + + "github.com/onflow/cadence/runtime/common" +) + +// AddressPublicPath is a public path in an account. +type AddressPublicPath struct { + Address common.Address + Identifier string +} + +type LinkInfo struct { + BorrowType common.TypeID + AccessibleMembers []string +} + +// PublicLinkReport is a mapping from public account paths to link info. +type PublicLinkReport map[AddressPublicPath]LinkInfo + +// ReadPublicLinkReport reads a link report from the given reader. +// The report is expected to be a JSON array of objects with the following structure: +// +// [ +// {"address":"0x1","identifier":"foo","linkType":"&Foo","accessibleMembers":["foo"]} +// ] +func ReadPublicLinkReport( + reader io.Reader, + filter map[common.Address]struct{}, +) (PublicLinkReport, error) { + + report := PublicLinkReport{} + + dec := json.NewDecoder(reader) + + token, err := dec.Token() + if err != nil { + return nil, fmt.Errorf("failed to read token: %w", err) + } + if token != json.Delim('[') { + return nil, fmt.Errorf("expected start of array, got %s", token) + } + + for dec.More() { + var entry struct { + Address string `json:"address"` + Identifier string `json:"identifier"` + LinkTypeID string `json:"linkType"` + AccessibleMembers []string `json:"accessibleMembers"` + } + err := dec.Decode(&entry) + if err != nil { + return nil, fmt.Errorf("failed to decode entry: %w", err) + } + + address, err := common.HexToAddress(entry.Address) + if err != nil { + return nil, fmt.Errorf("failed to parse address: %w", err) + } + + if filter != nil { + if _, ok := filter[address]; !ok { + continue + } + } + + key := AddressPublicPath{ + Address: address, + Identifier: entry.Identifier, + } + report[key] = LinkInfo{ + BorrowType: common.TypeID(entry.LinkTypeID), + AccessibleMembers: entry.AccessibleMembers, + } + } + + token, err = dec.Token() + if err != nil { + return nil, fmt.Errorf("failed to read token: %w", err) + } + if token != json.Delim(']') { + return nil, fmt.Errorf("expected end of array, got %s", token) + } + + return report, nil +} diff --git a/cmd/util/cmd/generate-entitlement-fixes/cmd_test.go b/cmd/util/cmd/generate-entitlement-fixes/link_report_test.go similarity index 53% rename from cmd/util/cmd/generate-entitlement-fixes/cmd_test.go rename to cmd/util/cmd/generate-entitlement-fixes/link_report_test.go index 40b6863b7b8..527f0c946e1 100644 --- a/cmd/util/cmd/generate-entitlement-fixes/cmd_test.go +++ b/cmd/util/cmd/generate-entitlement-fixes/link_report_test.go @@ -8,67 +8,6 @@ import ( "github.com/stretchr/testify/require" ) -func TestReadPublicLinkMigrationReport(t *testing.T) { - t.Parallel() - - contents := ` - [ - {"kind":"link-migration-success","account_address":"0x1","path":"/public/foo","capability_id":1}, - {"kind":"link-migration-success","account_address":"0x2","path":"/private/bar","capability_id":2}, - {"kind":"link-migration-success","account_address":"0x3","path":"/public/baz","capability_id":3} - ] - ` - - t.Run("unfiltered", func(t *testing.T) { - t.Parallel() - - reader := strings.NewReader(contents) - - mapping, err := ReadPublicLinkMigrationReport(reader, nil) - require.NoError(t, err) - - require.Equal(t, - PublicLinkMigrationReport{ - { - Address: common.MustBytesToAddress([]byte{0x1}), - CapabilityID: 1, - }: "foo", - { - Address: common.MustBytesToAddress([]byte{0x3}), - CapabilityID: 3, - }: "baz", - }, - mapping, - ) - }) - - t.Run("filtered", func(t *testing.T) { - t.Parallel() - - address1 := common.MustBytesToAddress([]byte{0x1}) - - reader := strings.NewReader(contents) - - mapping, err := ReadPublicLinkMigrationReport( - reader, - map[common.Address]struct{}{ - address1: {}, - }, - ) - require.NoError(t, err) - - require.Equal(t, - PublicLinkMigrationReport{ - { - Address: address1, - CapabilityID: 1, - }: "foo", - }, - mapping, - ) - }) -} - func TestReadLinkReport(t *testing.T) { t.Parallel() From 5b98f5119dea7e694226a1c316d8f4986b0af28f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Fri, 6 Sep 2024 17:19:36 -0700 Subject: [PATCH 31/72] detect authorization mismatches for capability controllers migrated from public links --- .../accessible_members.go | 2 +- .../check_contract.go | 2 +- .../cmd/generate-authorization-fixes/cmd.go | 462 ++++++++++++++++++ .../generate-authorization-fixes/cmd_test.go | 207 ++++++++ .../contracts.go | 2 +- .../link_migration_report.go | 2 +- .../link_migration_report_test.go | 2 +- .../link_report.go | 2 +- .../link_report_test.go | 2 +- .../cmd/generate-entitlement-fixes/cmd.go | 201 -------- .../fix_entitlements_migration_test.go | 2 +- 11 files changed, 677 insertions(+), 209 deletions(-) rename cmd/util/cmd/{generate-entitlement-fixes => generate-authorization-fixes}/accessible_members.go (97%) rename cmd/util/cmd/{generate-entitlement-fixes => generate-authorization-fixes}/check_contract.go (98%) create mode 100644 cmd/util/cmd/generate-authorization-fixes/cmd.go create mode 100644 cmd/util/cmd/generate-authorization-fixes/cmd_test.go rename cmd/util/cmd/{generate-entitlement-fixes => generate-authorization-fixes}/contracts.go (97%) rename cmd/util/cmd/{generate-entitlement-fixes => generate-authorization-fixes}/link_migration_report.go (98%) rename cmd/util/cmd/{generate-entitlement-fixes => generate-authorization-fixes}/link_migration_report_test.go (97%) rename cmd/util/cmd/{generate-entitlement-fixes => generate-authorization-fixes}/link_report.go (98%) rename cmd/util/cmd/{generate-entitlement-fixes => generate-authorization-fixes}/link_report_test.go (97%) delete mode 100644 cmd/util/cmd/generate-entitlement-fixes/cmd.go diff --git a/cmd/util/cmd/generate-entitlement-fixes/accessible_members.go b/cmd/util/cmd/generate-authorization-fixes/accessible_members.go similarity index 97% rename from cmd/util/cmd/generate-entitlement-fixes/accessible_members.go rename to cmd/util/cmd/generate-authorization-fixes/accessible_members.go index 52f68a8ff7c..f4071f34a69 100644 --- a/cmd/util/cmd/generate-entitlement-fixes/accessible_members.go +++ b/cmd/util/cmd/generate-authorization-fixes/accessible_members.go @@ -1,4 +1,4 @@ -package generate_entitlement_fixes +package generate_authorization_fixes import ( "fmt" diff --git a/cmd/util/cmd/generate-entitlement-fixes/check_contract.go b/cmd/util/cmd/generate-authorization-fixes/check_contract.go similarity index 98% rename from cmd/util/cmd/generate-entitlement-fixes/check_contract.go rename to cmd/util/cmd/generate-authorization-fixes/check_contract.go index 4616287c572..6461ef6ad25 100644 --- a/cmd/util/cmd/generate-entitlement-fixes/check_contract.go +++ b/cmd/util/cmd/generate-authorization-fixes/check_contract.go @@ -1,4 +1,4 @@ -package generate_entitlement_fixes +package generate_authorization_fixes import ( "encoding/json" diff --git a/cmd/util/cmd/generate-authorization-fixes/cmd.go b/cmd/util/cmd/generate-authorization-fixes/cmd.go new file mode 100644 index 00000000000..0f88b56fca2 --- /dev/null +++ b/cmd/util/cmd/generate-authorization-fixes/cmd.go @@ -0,0 +1,462 @@ +package generate_authorization_fixes + +import ( + "encoding/json" + "os" + "sort" + "strings" + + "github.com/onflow/cadence/runtime/common" + "github.com/onflow/cadence/runtime/interpreter" + "github.com/onflow/cadence/runtime/stdlib" + "github.com/rs/zerolog/log" + "github.com/spf13/cobra" + "golang.org/x/exp/slices" + + common2 "github.com/onflow/flow-go/cmd/util/common" + "github.com/onflow/flow-go/cmd/util/ledger/migrations" + "github.com/onflow/flow-go/cmd/util/ledger/reporters" + "github.com/onflow/flow-go/cmd/util/ledger/util" + "github.com/onflow/flow-go/cmd/util/ledger/util/registers" + "github.com/onflow/flow-go/ledger" + "github.com/onflow/flow-go/model/flow" +) + +var ( + flagPayloads string + flagState string + flagStateCommitment string + flagOutputDirectory string + flagChain string + flagLinkReport string + flagLinkMigrationReport string + flagAddresses string +) + +var Cmd = &cobra.Command{ + Use: "generate-authorization-fixes", + Short: "generate authorization fixes for capability controllers", + Run: run, +} + +func init() { + + Cmd.Flags().StringVar( + &flagPayloads, + "payloads", + "", + "Input payload file name", + ) + + Cmd.Flags().StringVar( + &flagState, + "state", + "", + "Input state file name", + ) + + Cmd.Flags().StringVar( + &flagStateCommitment, + "state-commitment", + "", + "Input state commitment", + ) + + Cmd.Flags().StringVar( + &flagOutputDirectory, + "output-directory", + "", + "Output directory", + ) + + Cmd.Flags().StringVar( + &flagChain, + "chain", + "", + "Chain name", + ) + _ = Cmd.MarkFlagRequired("chain") + + Cmd.Flags().StringVar( + &flagLinkReport, + "link-report", + "", + "Input link report file name", + ) + _ = Cmd.MarkFlagRequired("link-report") + + Cmd.Flags().StringVar( + &flagLinkMigrationReport, + "link-migration-report", + "", + "Input link migration report file name", + ) + _ = Cmd.MarkFlagRequired("link-migration-report") + + Cmd.Flags().StringVar( + &flagAddresses, + "addresses", + "", + "only generate fixes for given accounts (comma-separated hex-encoded addresses)", + ) +} + +const contractCountEstimate = 1000 + +func run(*cobra.Command, []string) { + + var addressFilter map[common.Address]struct{} + + if len(flagAddresses) > 0 { + for _, hexAddr := range strings.Split(flagAddresses, ",") { + + hexAddr = strings.TrimSpace(hexAddr) + + if len(hexAddr) == 0 { + continue + } + + addr, err := common2.ParseAddress(hexAddr) + if err != nil { + log.Fatal().Err(err).Msgf("failed to parse address: %s", hexAddr) + } + + addressFilter[common.Address(addr)] = struct{}{} + } + } + + if flagPayloads == "" && flagState == "" { + log.Fatal().Msg("Either --payloads or --state must be provided") + } else if flagPayloads != "" && flagState != "" { + log.Fatal().Msg("Only one of --payloads or --state must be provided") + } + if flagState != "" && flagStateCommitment == "" { + log.Fatal().Msg("--state-commitment must be provided when --state is provided") + } + + rwf := reporters.NewReportFileWriterFactory(flagOutputDirectory, log.Logger) + + reporter := rwf.ReportWriter("entitlement-fixes") + defer reporter.Close() + + chainID := flow.ChainID(flagChain) + // Validate chain ID + _ = chainID.Chain() + + var payloads []*ledger.Payload + var err error + + // Read public link report + + linkReportFile, err := os.Open(flagLinkReport) + if err != nil { + log.Fatal().Err(err).Msgf("can't open link report: %s", flagLinkReport) + } + defer linkReportFile.Close() + + publicLinkReport, err := ReadPublicLinkReport(linkReportFile, addressFilter) + if err != nil { + log.Fatal().Err(err).Msgf("failed to read public link report %s", flagLinkReport) + } + + // Read link migration report + + linkMigrationReportFile, err := os.Open(flagLinkMigrationReport) + if err != nil { + log.Fatal().Err(err).Msgf("can't open link migration report: %s", flagLinkMigrationReport) + } + defer linkMigrationReportFile.Close() + + publicLinkMigrationReport, err := ReadPublicLinkMigrationReport(linkMigrationReportFile, addressFilter) + if err != nil { + log.Fatal().Err(err).Msgf("failed to read public link report: %s", flagLinkMigrationReport) + } + + // Read payloads from payload file or checkpoint file + + if flagPayloads != "" { + log.Info().Msgf("Reading payloads from %s", flagPayloads) + + _, payloads, err = util.ReadPayloadFile(log.Logger, flagPayloads) + if err != nil { + log.Fatal().Err(err).Msg("failed to read payloads") + } + } else { + log.Info().Msgf("Reading trie %s", flagStateCommitment) + + stateCommitment := util.ParseStateCommitment(flagStateCommitment) + payloads, err = util.ReadTrie(flagState, stateCommitment) + if err != nil { + log.Fatal().Err(err).Msg("failed to read state") + } + } + + log.Info().Msgf("creating registers from payloads (%d)", len(payloads)) + + registersByAccount, err := registers.NewByAccountFromPayloads(payloads) + if err != nil { + log.Fatal().Err(err) + } + log.Info().Msgf( + "created %d registers from payloads (%d accounts)", + registersByAccount.Count(), + registersByAccount.AccountCount(), + ) + + mr, err := migrations.NewInterpreterMigrationRuntime( + registersByAccount, + chainID, + migrations.InterpreterMigrationRuntimeConfig{}, + ) + if err != nil { + log.Fatal().Err(err) + } + + checkContracts( + registersByAccount, + mr, + reporter, + ) + + authorizationFixGenerator := &AuthorizationFixGenerator{ + registersByAccount: registersByAccount, + mr: mr, + publicLinkReport: publicLinkReport, + publicLinkMigrationReport: publicLinkMigrationReport, + reporter: reporter, + } + authorizationFixGenerator.generateFixesForAllAccounts() +} + +func checkContracts( + registersByAccount *registers.ByAccount, + mr *migrations.InterpreterMigrationRuntime, + reporter reporters.ReportWriter, +) { + contracts, err := gatherContractsFromRegisters(registersByAccount) + if err != nil { + log.Fatal().Err(err) + } + + programs := make(map[common.Location]*interpreter.Program, contractCountEstimate) + + contractsForPrettyPrinting := make(map[common.Location][]byte, len(contracts)) + for _, contract := range contracts { + contractsForPrettyPrinting[contract.Location] = contract.Code + } + + log.Info().Msg("Checking contracts ...") + + for _, contract := range contracts { + checkContract( + contract, + mr, + contractsForPrettyPrinting, + reporter, + programs, + ) + } + + log.Info().Msgf("Checked %d contracts ...", len(contracts)) +} + +func jsonEncodeAuthorization(authorization interpreter.Authorization) string { + switch authorization { + case interpreter.UnauthorizedAccess, interpreter.InaccessibleAccess: + return "" + default: + return string(authorization.ID()) + } +} + +type fixEntitlementsEntry struct { + AccountCapabilityID + NewAuthorization interpreter.Authorization +} + +var _ json.Marshaler = fixEntitlementsEntry{} + +func (e fixEntitlementsEntry) MarshalJSON() ([]byte, error) { + return json.Marshal(struct { + Kind string `json:"kind"` + CapabilityAddress string `json:"capability_address"` + CapabilityID uint64 `json:"capability_id"` + NewAuthorization string `json:"new_authorization"` + }{ + Kind: "fix-entitlements", + CapabilityAddress: e.Address.String(), + CapabilityID: e.CapabilityID, + NewAuthorization: jsonEncodeAuthorization(e.NewAuthorization), + }) +} + +type AuthorizationFixGenerator struct { + registersByAccount *registers.ByAccount + mr *migrations.InterpreterMigrationRuntime + publicLinkReport PublicLinkReport + publicLinkMigrationReport PublicLinkMigrationReport + reporter reporters.ReportWriter +} + +func (g *AuthorizationFixGenerator) generateFixesForAllAccounts() { + err := g.registersByAccount.ForEachAccount(func(accountRegisters *registers.AccountRegisters) error { + address := common.MustBytesToAddress([]byte(accountRegisters.Owner())) + g.generateFixesForAccount(address) + return nil + }) + if err != nil { + log.Fatal().Err(err) + } +} + +func (g *AuthorizationFixGenerator) generateFixesForAccount(address common.Address) { + capabilityControllerStorage := g.mr.Storage.GetStorageMap( + address, + stdlib.CapabilityControllerStorageDomain, + false, + ) + if capabilityControllerStorage == nil { + return + } + + iterator := capabilityControllerStorage.Iterator(nil) + for { + k, v := iterator.Next() + + if k == nil || v == nil { + break + } + + key, ok := k.(interpreter.Uint64AtreeValue) + if !ok { + log.Fatal().Msgf("unexpected key type: %T", k) + } + + capabilityID := uint64(key) + + value := interpreter.MustConvertUnmeteredStoredValue(v) + + capabilityController, ok := value.(*interpreter.StorageCapabilityControllerValue) + if !ok { + continue + } + + borrowType := capabilityController.BorrowType + + switch borrowType.Authorization.(type) { + case interpreter.EntitlementSetAuthorization: + g.maybeGenerateFixForCapabilityController( + address, + capabilityID, + borrowType, + ) + + case interpreter.Unauthorized: + // Already unauthorized, nothing to do + + case interpreter.Inaccessible: + log.Warn().Msgf( + "capability controller %d in account %s has borrow type with inaccessible authorization", + capabilityID, + address.HexWithPrefix(), + ) + + case interpreter.EntitlementMapAuthorization: + log.Warn().Msgf( + "capability controller %d in account %s has borrow type with entitlement map authorization", + capabilityID, + address.HexWithPrefix(), + ) + + default: + log.Warn().Msgf( + "capability controller %d in account %s has borrow type with entitlement map authorization", + capabilityID, + address.HexWithPrefix(), + ) + } + } +} + +func (g *AuthorizationFixGenerator) maybeGenerateFixForCapabilityController( + address common.Address, + capabilityID uint64, + borrowType *interpreter.ReferenceStaticType, +) { + // Only fix the entitlements if the capability controller was migrated from a public link + publicPathIdentifier := g.capabilityControllerPublicPathIdentifier(address, capabilityID) + if publicPathIdentifier == "" { + return + } + + linkInfo := g.publicPathLinkInfo(address, publicPathIdentifier) + if linkInfo.BorrowType == "" { + log.Warn().Msgf( + "missing link info for /public/%s in account %s", + publicPathIdentifier, + address.HexWithPrefix(), + ) + return + } + + // Compare previously accessible members with new accessible members. + // They should be the same. + + oldAccessibleMembers := linkInfo.AccessibleMembers + if oldAccessibleMembers == nil { + log.Warn().Msgf( + "missing old accessible members for for /public/%s in account %s", + publicPathIdentifier, + address.HexWithPrefix(), + ) + return + } + + newAccessibleMembers, err := getAccessibleMembers(g.mr.Interpreter, borrowType) + if err != nil { + log.Warn().Err(err).Msgf( + "failed to get new accessible members for capability controller %d in account %s", + capabilityID, + address.HexWithPrefix(), + ) + return + } + + sort.Strings(oldAccessibleMembers) + sort.Strings(newAccessibleMembers) + + if slices.Equal(oldAccessibleMembers, newAccessibleMembers) { + // Nothing to fix + return + } + + log.Info().Msgf( + "member mismatch for capability controller %d in account %s: expected %v, got %v", + capabilityID, + address.HexWithPrefix(), + oldAccessibleMembers, + newAccessibleMembers, + ) + + // TODO: generate and report entitlement fix +} + +func (g *AuthorizationFixGenerator) capabilityControllerPublicPathIdentifier( + address common.Address, + capabilityID uint64, +) string { + return g.publicLinkMigrationReport[AccountCapabilityID{ + Address: address, + CapabilityID: capabilityID, + }] +} + +func (g *AuthorizationFixGenerator) publicPathLinkInfo( + address common.Address, + publicPathIdentifier string, +) LinkInfo { + return g.publicLinkReport[AddressPublicPath{ + Address: address, + Identifier: publicPathIdentifier, + }] +} diff --git a/cmd/util/cmd/generate-authorization-fixes/cmd_test.go b/cmd/util/cmd/generate-authorization-fixes/cmd_test.go new file mode 100644 index 00000000000..b0413697deb --- /dev/null +++ b/cmd/util/cmd/generate-authorization-fixes/cmd_test.go @@ -0,0 +1,207 @@ +package generate_authorization_fixes + +import ( + "testing" + + "github.com/onflow/cadence/runtime/common" + "github.com/rs/zerolog" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/cmd/util/ledger/migrations" + "github.com/onflow/flow-go/cmd/util/ledger/util/registers" + "github.com/onflow/flow-go/fvm" + "github.com/onflow/flow-go/fvm/storage/snapshot" + "github.com/onflow/flow-go/ledger" + "github.com/onflow/flow-go/ledger/common/convert" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/utils/unittest" +) + +func newBootstrapPayloads( + chainID flow.ChainID, + bootstrapProcedureOptions ...fvm.BootstrapProcedureOption, +) ([]*ledger.Payload, error) { + + ctx := fvm.NewContext( + fvm.WithChain(chainID.Chain()), + ) + + vm := fvm.NewVirtualMachine() + + storageSnapshot := snapshot.MapStorageSnapshot{} + + bootstrapProcedure := fvm.Bootstrap( + unittest.ServiceAccountPublicKey, + bootstrapProcedureOptions..., + ) + + executionSnapshot, _, err := vm.Run( + ctx, + bootstrapProcedure, + storageSnapshot, + ) + if err != nil { + return nil, err + } + + payloads := make([]*ledger.Payload, 0, len(executionSnapshot.WriteSet)) + + for registerID, registerValue := range executionSnapshot.WriteSet { + payloadKey := convert.RegisterIDToLedgerKey(registerID) + payload := ledger.NewPayload(payloadKey, registerValue) + payloads = append(payloads, payload) + } + + return payloads, nil +} + +func TestFixAuthorizationsMigrations(t *testing.T) { + t.Parallel() + + const chainID = flow.Emulator + chain := chainID.Chain() + + const nWorker = 2 + + address, err := chain.AddressAtIndex(1000) + require.NoError(t, err) + + require.Equal(t, "bf519681cdb888b1", address.Hex()) + + log := zerolog.New(zerolog.NewTestWriter(t)) + + bootstrapPayloads, err := newBootstrapPayloads(chainID) + require.NoError(t, err) + + registersByAccount, err := registers.NewByAccountFromPayloads(bootstrapPayloads) + require.NoError(t, err) + + mr := migrations.NewBasicMigrationRuntime(registersByAccount) + + err = mr.Accounts.Create(nil, address) + require.NoError(t, err) + + expectedWriteAddresses := map[flow.Address]struct{}{ + address: {}, + } + + err = mr.Commit(expectedWriteAddresses, log) + require.NoError(t, err) + + tx := flow.NewTransactionBody(). + SetScript([]byte(` + transaction { + prepare(signer: auth(Storage, Capabilities) &Account) { + // Capability 1 was a public, unauthorized capability. + // It should lose its entitlement + let cap1 = signer.capabilities.storage.issue(/storage/ints) + signer.capabilities.publish(cap1, at: /public/ints) + + // Capability 2 was a public, unauthorized capability, stored nested in storage. + // It should lose its entitlement + let cap2 = signer.capabilities.storage.issue(/storage/ints) + signer.storage.save([cap2], to: /storage/caps2) + + // Capability 3 was a private, authorized capability, stored nested in storage. + // It should keep its entitlement + let cap3 = signer.capabilities.storage.issue(/storage/ints) + signer.storage.save([cap3], to: /storage/caps3) + + // Capability 4 was a capability with unavailable accessible members, stored nested in storage. + // It should keep its entitlement + let cap4 = signer.capabilities.storage.issue(/storage/ints) + signer.storage.save([cap4], to: /storage/caps4) + } + } + `)). + AddAuthorizer(address) + + setupTx := migrations.NewTransactionBasedMigration( + tx, + chainID, + log, + expectedWriteAddresses, + ) + err = setupTx(registersByAccount) + require.NoError(t, err) + + mr2, err := migrations.NewInterpreterMigrationRuntime( + registersByAccount, + chainID, + migrations.InterpreterMigrationRuntimeConfig{}, + ) + require.NoError(t, err) + + // Capability 1 was a public, unauthorized capability. + // It should lose its entitlement + // + // Capability 2 was a public, unauthorized capability, stored nested in storage. + // It should lose its entitlement + // + // Capability 3 was a private, authorized capability, stored nested in storage. + // It should keep its entitlement + // + // Capability 4 was a capability with unavailable accessible members, stored nested in storage. + // It should keep its entitlement + + readArrayMembers := []string{ + "concat", + "contains", + "filter", + "firstIndex", + "getType", + "isInstance", + "length", + "map", + "slice", + "toConstantSized", + } + + publicLinkReport := PublicLinkReport{ + { + Address: common.Address(address), + Identifier: "ints", + }: { + BorrowType: "&[Int]", + AccessibleMembers: readArrayMembers, + }, + { + Address: common.Address(address), + Identifier: "ints2", + }: { + BorrowType: "&[Int]", + AccessibleMembers: readArrayMembers, + }, + { + Address: common.Address(address), + Identifier: "ints4", + }: { + BorrowType: "&[Int]", + AccessibleMembers: nil, + }, + } + + publicLinkMigrationReport := PublicLinkMigrationReport{ + { + Address: common.Address(address), + CapabilityID: 1, + }: "ints", + { + Address: common.Address(address), + CapabilityID: 2, + }: "ints2", + { + Address: common.Address(address), + CapabilityID: 4, + }: "ints4", + } + + generator := &AuthorizationFixGenerator{ + registersByAccount: registersByAccount, + mr: mr2, + publicLinkReport: publicLinkReport, + publicLinkMigrationReport: publicLinkMigrationReport, + } + generator.generateFixesForAllAccounts() + +} diff --git a/cmd/util/cmd/generate-entitlement-fixes/contracts.go b/cmd/util/cmd/generate-authorization-fixes/contracts.go similarity index 97% rename from cmd/util/cmd/generate-entitlement-fixes/contracts.go rename to cmd/util/cmd/generate-authorization-fixes/contracts.go index 25a95161863..3e8308973d9 100644 --- a/cmd/util/cmd/generate-entitlement-fixes/contracts.go +++ b/cmd/util/cmd/generate-authorization-fixes/contracts.go @@ -1,4 +1,4 @@ -package generate_entitlement_fixes +package generate_authorization_fixes import ( "bytes" diff --git a/cmd/util/cmd/generate-entitlement-fixes/link_migration_report.go b/cmd/util/cmd/generate-authorization-fixes/link_migration_report.go similarity index 98% rename from cmd/util/cmd/generate-entitlement-fixes/link_migration_report.go rename to cmd/util/cmd/generate-authorization-fixes/link_migration_report.go index b99284eb621..1d096e8c555 100644 --- a/cmd/util/cmd/generate-entitlement-fixes/link_migration_report.go +++ b/cmd/util/cmd/generate-authorization-fixes/link_migration_report.go @@ -1,4 +1,4 @@ -package generate_entitlement_fixes +package generate_authorization_fixes import ( "encoding/json" diff --git a/cmd/util/cmd/generate-entitlement-fixes/link_migration_report_test.go b/cmd/util/cmd/generate-authorization-fixes/link_migration_report_test.go similarity index 97% rename from cmd/util/cmd/generate-entitlement-fixes/link_migration_report_test.go rename to cmd/util/cmd/generate-authorization-fixes/link_migration_report_test.go index 3a2b81a3952..de03ffb0e35 100644 --- a/cmd/util/cmd/generate-entitlement-fixes/link_migration_report_test.go +++ b/cmd/util/cmd/generate-authorization-fixes/link_migration_report_test.go @@ -1,4 +1,4 @@ -package generate_entitlement_fixes +package generate_authorization_fixes import ( "strings" diff --git a/cmd/util/cmd/generate-entitlement-fixes/link_report.go b/cmd/util/cmd/generate-authorization-fixes/link_report.go similarity index 98% rename from cmd/util/cmd/generate-entitlement-fixes/link_report.go rename to cmd/util/cmd/generate-authorization-fixes/link_report.go index 306742666f8..392fa41e519 100644 --- a/cmd/util/cmd/generate-entitlement-fixes/link_report.go +++ b/cmd/util/cmd/generate-authorization-fixes/link_report.go @@ -1,4 +1,4 @@ -package generate_entitlement_fixes +package generate_authorization_fixes import ( "encoding/json" diff --git a/cmd/util/cmd/generate-entitlement-fixes/link_report_test.go b/cmd/util/cmd/generate-authorization-fixes/link_report_test.go similarity index 97% rename from cmd/util/cmd/generate-entitlement-fixes/link_report_test.go rename to cmd/util/cmd/generate-authorization-fixes/link_report_test.go index 527f0c946e1..f7f10e2b049 100644 --- a/cmd/util/cmd/generate-entitlement-fixes/link_report_test.go +++ b/cmd/util/cmd/generate-authorization-fixes/link_report_test.go @@ -1,4 +1,4 @@ -package generate_entitlement_fixes +package generate_authorization_fixes import ( "strings" diff --git a/cmd/util/cmd/generate-entitlement-fixes/cmd.go b/cmd/util/cmd/generate-entitlement-fixes/cmd.go deleted file mode 100644 index db12fb423a0..00000000000 --- a/cmd/util/cmd/generate-entitlement-fixes/cmd.go +++ /dev/null @@ -1,201 +0,0 @@ -package generate_entitlement_fixes - -import ( - "encoding/json" - - "github.com/onflow/cadence/runtime/common" - "github.com/onflow/cadence/runtime/interpreter" - "github.com/rs/zerolog/log" - "github.com/spf13/cobra" - - "github.com/onflow/flow-go/cmd/util/ledger/migrations" - "github.com/onflow/flow-go/cmd/util/ledger/reporters" - "github.com/onflow/flow-go/cmd/util/ledger/util" - "github.com/onflow/flow-go/cmd/util/ledger/util/registers" - "github.com/onflow/flow-go/ledger" - "github.com/onflow/flow-go/model/flow" -) - -var ( - flagPayloads string - flagState string - flagStateCommitment string - flagOutputDirectory string - flagChain string -) - -var Cmd = &cobra.Command{ - Use: "report-links", - Short: "reports links", - Run: run, -} - -func init() { - - Cmd.Flags().StringVar( - &flagPayloads, - "payloads", - "", - "Input payload file name", - ) - - Cmd.Flags().StringVar( - &flagState, - "state", - "", - "Input state file name", - ) - - Cmd.Flags().StringVar( - &flagStateCommitment, - "state-commitment", - "", - "Input state commitment", - ) - - Cmd.Flags().StringVar( - &flagOutputDirectory, - "output-directory", - "", - "Output directory", - ) - - Cmd.Flags().StringVar( - &flagChain, - "chain", - "", - "Chain name", - ) - _ = Cmd.MarkFlagRequired("chain") -} - -const contractCountEstimate = 1000 - -func run(*cobra.Command, []string) { - - if flagPayloads == "" && flagState == "" { - log.Fatal().Msg("Either --payloads or --state must be provided") - } else if flagPayloads != "" && flagState != "" { - log.Fatal().Msg("Only one of --payloads or --state must be provided") - } - if flagState != "" && flagStateCommitment == "" { - log.Fatal().Msg("--state-commitment must be provided when --state is provided") - } - - rwf := reporters.NewReportFileWriterFactory(flagOutputDirectory, log.Logger) - - reporter := rwf.ReportWriter("entitlement-fixes") - defer reporter.Close() - - chainID := flow.ChainID(flagChain) - // Validate chain ID - _ = chainID.Chain() - - var payloads []*ledger.Payload - var err error - - // Read payloads from payload file or checkpoint file - - if flagPayloads != "" { - log.Info().Msgf("Reading payloads from %s", flagPayloads) - - _, payloads, err = util.ReadPayloadFile(log.Logger, flagPayloads) - if err != nil { - log.Fatal().Err(err).Msg("failed to read payloads") - } - } else { - log.Info().Msgf("Reading trie %s", flagStateCommitment) - - stateCommitment := util.ParseStateCommitment(flagStateCommitment) - payloads, err = util.ReadTrie(flagState, stateCommitment) - if err != nil { - log.Fatal().Err(err).Msg("failed to read state") - } - } - - log.Info().Msgf("creating registers from payloads (%d)", len(payloads)) - - registersByAccount, err := registers.NewByAccountFromPayloads(payloads) - if err != nil { - log.Fatal().Err(err) - } - log.Info().Msgf( - "created %d registers from payloads (%d accounts)", - registersByAccount.Count(), - registersByAccount.AccountCount(), - ) - - mr, err := migrations.NewInterpreterMigrationRuntime( - registersByAccount, - chainID, - migrations.InterpreterMigrationRuntimeConfig{}, - ) - if err != nil { - log.Fatal().Err(err) - } - - checkContracts(registersByAccount, mr, reporter) - -} - -func checkContracts( - registersByAccount *registers.ByAccount, - mr *migrations.InterpreterMigrationRuntime, - reporter reporters.ReportWriter, -) { - contracts, err := gatherContractsFromRegisters(registersByAccount) - if err != nil { - log.Fatal().Err(err) - } - - programs := make(map[common.Location]*interpreter.Program, contractCountEstimate) - - contractsForPrettyPrinting := make(map[common.Location][]byte, len(contracts)) - for _, contract := range contracts { - contractsForPrettyPrinting[contract.Location] = contract.Code - } - - log.Info().Msg("Checking contracts ...") - - for _, contract := range contracts { - checkContract( - contract, - mr, - contractsForPrettyPrinting, - reporter, - programs, - ) - } - - log.Info().Msgf("Checked %d contracts ...", len(contracts)) -} - -func jsonEncodeAuthorization(authorization interpreter.Authorization) string { - switch authorization { - case interpreter.UnauthorizedAccess, interpreter.InaccessibleAccess: - return "" - default: - return string(authorization.ID()) - } -} - -type fixEntitlementsEntry struct { - AccountCapabilityID - NewAuthorization interpreter.Authorization -} - -var _ json.Marshaler = fixEntitlementsEntry{} - -func (e fixEntitlementsEntry) MarshalJSON() ([]byte, error) { - return json.Marshal(struct { - Kind string `json:"kind"` - CapabilityAddress string `json:"capability_address"` - CapabilityID uint64 `json:"capability_id"` - NewAuthorization string `json:"new_authorization"` - }{ - Kind: "fix-entitlements", - CapabilityAddress: e.Address.String(), - CapabilityID: e.CapabilityID, - NewAuthorization: jsonEncodeAuthorization(e.NewAuthorization), - }) -} diff --git a/cmd/util/ledger/migrations/fix_entitlements_migration_test.go b/cmd/util/ledger/migrations/fix_entitlements_migration_test.go index 578b4b6f8ce..30680cad6ba 100644 --- a/cmd/util/ledger/migrations/fix_entitlements_migration_test.go +++ b/cmd/util/ledger/migrations/fix_entitlements_migration_test.go @@ -12,7 +12,7 @@ import ( "github.com/onflow/flow-go/model/flow" ) -func TestFixEntitlementMigrations(t *testing.T) { +func TestEntitlements(t *testing.T) { t.Parallel() const chainID = flow.Emulator From f4284ca010b1a8c2a5bd5188c2c509a5465e776f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Fri, 6 Sep 2024 17:49:47 -0700 Subject: [PATCH 32/72] add entitlement set generation from Supun --- .../entitlements.go | 68 ++++++++++ .../entitlements_test.go | 125 ++++++++++++++++++ 2 files changed, 193 insertions(+) create mode 100644 cmd/util/cmd/generate-authorization-fixes/entitlements.go create mode 100644 cmd/util/cmd/generate-authorization-fixes/entitlements_test.go diff --git a/cmd/util/cmd/generate-authorization-fixes/entitlements.go b/cmd/util/cmd/generate-authorization-fixes/entitlements.go new file mode 100644 index 00000000000..ba60dbdb228 --- /dev/null +++ b/cmd/util/cmd/generate-authorization-fixes/entitlements.go @@ -0,0 +1,68 @@ +package generate_authorization_fixes + +import ( + "github.com/onflow/cadence/runtime/ast" + "github.com/onflow/cadence/runtime/common" + "github.com/onflow/cadence/runtime/common/orderedmap" + "github.com/onflow/cadence/runtime/sema" + "golang.org/x/exp/slices" +) + +func findMinimalEntitlementSet( + ty sema.Type, + neededMethods []string, +) []*sema.EntitlementType { + + entitlementsToMethod := orderedmap.OrderedMap[*sema.EntitlementType, []string]{} + + // NOTE: GetMembers might return members that are actually not accessible, for DX purposes. + // We need to resolve the members and filter out the inaccessible members, + // using the error reported when resolving + + memberResolvers := ty.GetMembers() + + for memberName, memberResolver := range memberResolvers { + var resolveErr error + member := memberResolver.Resolve(nil, memberName, ast.EmptyRange, func(err error) { + resolveErr = err + }) + if resolveErr != nil { + continue + } + + if member.DeclarationKind != common.DeclarationKindFunction { + continue + } + + entitlementSetAccess, ok := member.Access.(sema.EntitlementSetAccess) + if !ok { + continue + } + + // TODO: handle SetKind + entitlementSetAccess.Entitlements.Foreach(func(entitlementType *sema.EntitlementType, _ struct{}) { + methodsForEntitlement, _ := entitlementsToMethod.Get(entitlementType) + methodsForEntitlement = append(methodsForEntitlement, memberName) + entitlementsToMethod.Set(entitlementType, methodsForEntitlement) + }) + } + + var entitlements []*sema.EntitlementType + entitlementsToMethod.Foreach(func(entitlement *sema.EntitlementType, methodsWithEntitlement []string) { + if isSubset(methodsWithEntitlement, neededMethods) { + entitlements = append(entitlements, entitlement) + } + }) + + return entitlements +} + +func isSubset(subSet, superSet []string) bool { + for _, element := range subSet { + if !slices.Contains(superSet, element) { + return false + } + } + + return true +} diff --git a/cmd/util/cmd/generate-authorization-fixes/entitlements_test.go b/cmd/util/cmd/generate-authorization-fixes/entitlements_test.go new file mode 100644 index 00000000000..5ac66ed581f --- /dev/null +++ b/cmd/util/cmd/generate-authorization-fixes/entitlements_test.go @@ -0,0 +1,125 @@ +package generate_authorization_fixes + +import ( + "testing" + + "github.com/onflow/cadence/runtime/sema" + . "github.com/onflow/cadence/runtime/tests/checker" + "github.com/stretchr/testify/require" +) + +func TestFindMinimalEntitlementsSet(t *testing.T) { + + t.Parallel() + + checker, err := ParseAndCheck(t, ` + entitlement E1 + entitlement E2 + entitlement E3 + resource interface I1 { + access(E1) fun method1() + } + resource interface I2 { + access(E1, E2) fun method2() + access(E2) fun method3() + } + resource interface I3 { + access(E3) fun method4() + } + // Assume this is the intersection-type that is left now. + // It may have some set of entitlements, but we are not interested: + // We are going to re-derive the minimal set of entitlements needed, + // depending on what methods to keep. + var intersectionType: &{I1, I2, I3}? = nil + `) + require.NoError(t, err) + + vValueType := RequireGlobalValue(t, checker.Elaboration, "intersectionType") + + require.IsType(t, &sema.OptionalType{}, vValueType) + optionalType := vValueType.(*sema.OptionalType) + + require.IsType(t, &sema.ReferenceType{}, optionalType.Type) + referenceType := optionalType.Type.(*sema.ReferenceType) + + require.IsType(t, &sema.IntersectionType{}, referenceType.Type) + intersectionType := referenceType.Type.(*sema.IntersectionType) + + e1 := RequireGlobalType(t, checker.Elaboration, "E1").(*sema.EntitlementType) + e2 := RequireGlobalType(t, checker.Elaboration, "E2").(*sema.EntitlementType) + e3 := RequireGlobalType(t, checker.Elaboration, "E3").(*sema.EntitlementType) + + t.Run("method1, method2", func(t *testing.T) { + entitlements := findMinimalEntitlementSet( + intersectionType, + []string{ + "method1", + "method2", + }, + ) + require.Equal(t, + []*sema.EntitlementType{e1}, + entitlements, + ) + }) + + t.Run("method1, method2, method3", func(t *testing.T) { + entitlements := findMinimalEntitlementSet( + intersectionType, + []string{ + "method1", + "method2", + "method3", + }, + ) + require.Equal(t, + []*sema.EntitlementType{e1, e2}, + entitlements, + ) + }) + + t.Run("method1, method2, method4", func(t *testing.T) { + entitlements := findMinimalEntitlementSet( + intersectionType, + []string{ + "method1", + "method2", + "method4", + }, + ) + require.Equal( + t, + []*sema.EntitlementType{e1, e3}, + entitlements, + ) + }) + + t.Run("method1, method2, method3, method4", func(t *testing.T) { + entitlements := findMinimalEntitlementSet( + intersectionType, + []string{ + "method1", + "method2", + "method3", + "method4", + }, + ) + require.Equal(t, + []*sema.EntitlementType{e1, e2, e3}, + entitlements, + ) + }) + + t.Run("method4", func(t *testing.T) { + entitlements := findMinimalEntitlementSet( + intersectionType, + []string{ + "method4", + }, + ) + require.Equal(t, + []*sema.EntitlementType{e3}, + entitlements, + ) + }) +} From 969dbfe0ca76219614c5808981875f826c778511 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Fri, 6 Sep 2024 18:16:13 -0700 Subject: [PATCH 33/72] report new authorization using minimal entitlement set function --- .../accessible_members.go | 42 ++-------- .../cmd/generate-authorization-fixes/cmd.go | 83 +++++++++++++++++-- .../generate-authorization-fixes/cmd_test.go | 40 +++++++++ .../entitlements.go | 10 +++ 4 files changed, 130 insertions(+), 45 deletions(-) diff --git a/cmd/util/cmd/generate-authorization-fixes/accessible_members.go b/cmd/util/cmd/generate-authorization-fixes/accessible_members.go index f4071f34a69..92dcd2c9e71 100644 --- a/cmd/util/cmd/generate-authorization-fixes/accessible_members.go +++ b/cmd/util/cmd/generate-authorization-fixes/accessible_members.go @@ -1,48 +1,18 @@ package generate_authorization_fixes import ( - "fmt" - "github.com/onflow/cadence/runtime/ast" - "github.com/onflow/cadence/runtime/interpreter" + "github.com/onflow/cadence/runtime/sema" ) -func getAccessibleMembers( - inter *interpreter.Interpreter, - staticType interpreter.StaticType, -) ( - accessibleMembers []string, - err error, -) { - defer func() { - if r := recover(); r != nil { - err = fmt.Errorf("panic: %v", r) - } - }() - - semaType, err := inter.ConvertStaticToSemaType(staticType) - if err != nil { - return nil, fmt.Errorf( - "failed to convert static type %s to semantic type: %w", - staticType.ID(), - err, - ) - } - if semaType == nil { - return nil, fmt.Errorf( - "failed to convert static type %s to semantic type", - staticType.ID(), - ) - } - - // NOTE: RestrictedType.GetMembers returns *all* members, - // including those that are not accessible, for DX purposes. +func getAccessibleMembers(ty sema.Type) []string { + // NOTE: GetMembers might return members that are actually not accessible, for DX purposes. // We need to resolve the members and filter out the inaccessible members, // using the error reported when resolving - memberResolvers := semaType.GetMembers() + memberResolvers := ty.GetMembers() - accessibleMembers = make([]string, 0, len(memberResolvers)) + accessibleMembers := make([]string, 0, len(memberResolvers)) for memberName, memberResolver := range memberResolvers { var resolveErr error @@ -55,5 +25,5 @@ func getAccessibleMembers( accessibleMembers = append(accessibleMembers, memberName) } - return accessibleMembers, nil + return accessibleMembers } diff --git a/cmd/util/cmd/generate-authorization-fixes/cmd.go b/cmd/util/cmd/generate-authorization-fixes/cmd.go index 0f88b56fca2..c26bf0cdabe 100644 --- a/cmd/util/cmd/generate-authorization-fixes/cmd.go +++ b/cmd/util/cmd/generate-authorization-fixes/cmd.go @@ -2,12 +2,14 @@ package generate_authorization_fixes import ( "encoding/json" + "fmt" "os" "sort" "strings" "github.com/onflow/cadence/runtime/common" "github.com/onflow/cadence/runtime/interpreter" + "github.com/onflow/cadence/runtime/sema" "github.com/onflow/cadence/runtime/stdlib" "github.com/rs/zerolog/log" "github.com/spf13/cobra" @@ -379,22 +381,22 @@ func (g *AuthorizationFixGenerator) generateFixesForAccount(address common.Addre } func (g *AuthorizationFixGenerator) maybeGenerateFixForCapabilityController( - address common.Address, + capabilityAddress common.Address, capabilityID uint64, borrowType *interpreter.ReferenceStaticType, ) { // Only fix the entitlements if the capability controller was migrated from a public link - publicPathIdentifier := g.capabilityControllerPublicPathIdentifier(address, capabilityID) + publicPathIdentifier := g.capabilityControllerPublicPathIdentifier(capabilityAddress, capabilityID) if publicPathIdentifier == "" { return } - linkInfo := g.publicPathLinkInfo(address, publicPathIdentifier) + linkInfo := g.publicPathLinkInfo(capabilityAddress, publicPathIdentifier) if linkInfo.BorrowType == "" { log.Warn().Msgf( "missing link info for /public/%s in account %s", publicPathIdentifier, - address.HexWithPrefix(), + capabilityAddress.HexWithPrefix(), ) return } @@ -407,21 +409,23 @@ func (g *AuthorizationFixGenerator) maybeGenerateFixForCapabilityController( log.Warn().Msgf( "missing old accessible members for for /public/%s in account %s", publicPathIdentifier, - address.HexWithPrefix(), + capabilityAddress.HexWithPrefix(), ) return } - newAccessibleMembers, err := getAccessibleMembers(g.mr.Interpreter, borrowType) + semaBorrowType, err := convertStaticToSemaType(g.mr.Interpreter, borrowType) if err != nil { log.Warn().Err(err).Msgf( "failed to get new accessible members for capability controller %d in account %s", capabilityID, - address.HexWithPrefix(), + capabilityAddress.HexWithPrefix(), ) return } + newAccessibleMembers := getAccessibleMembers(semaBorrowType) + sort.Strings(oldAccessibleMembers) sort.Strings(newAccessibleMembers) @@ -433,12 +437,73 @@ func (g *AuthorizationFixGenerator) maybeGenerateFixForCapabilityController( log.Info().Msgf( "member mismatch for capability controller %d in account %s: expected %v, got %v", capabilityID, - address.HexWithPrefix(), + capabilityAddress.HexWithPrefix(), oldAccessibleMembers, newAccessibleMembers, ) - // TODO: generate and report entitlement fix + minimalEntitlementSet := findMinimalEntitlementSet( + semaBorrowType, + oldAccessibleMembers, + ) + + newAuthorization := interpreter.UnauthorizedAccess + if len(minimalEntitlementSet) > 0 { + newAuthorization = interpreter.NewEntitlementSetAuthorization( + nil, + func() []common.TypeID { + typeIDs := make([]common.TypeID, 0, len(minimalEntitlementSet)) + for _, entitlementType := range minimalEntitlementSet { + typeIDs = append(typeIDs, entitlementType.ID()) + } + return typeIDs + }, + len(minimalEntitlementSet), + // TODO: + sema.Conjunction, + ) + } + + g.reporter.Write(fixEntitlementsEntry{ + AccountCapabilityID: AccountCapabilityID{ + Address: capabilityAddress, + CapabilityID: capabilityID, + }, + NewAuthorization: newAuthorization, + }) + +} + +func convertStaticToSemaType( + inter *interpreter.Interpreter, + staticType interpreter.StaticType, +) ( + semaType sema.Type, + err error, +) { + + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("panic: %v", r) + } + }() + + semaType, err = inter.ConvertStaticToSemaType(staticType) + if err != nil { + return nil, fmt.Errorf( + "failed to convert static type %s to semantic type: %w", + staticType.ID(), + err, + ) + } + if semaType == nil { + return nil, fmt.Errorf( + "failed to convert static type %s to semantic type", + staticType.ID(), + ) + } + + return semaType, nil } func (g *AuthorizationFixGenerator) capabilityControllerPublicPathIdentifier( diff --git a/cmd/util/cmd/generate-authorization-fixes/cmd_test.go b/cmd/util/cmd/generate-authorization-fixes/cmd_test.go index b0413697deb..d18baee746c 100644 --- a/cmd/util/cmd/generate-authorization-fixes/cmd_test.go +++ b/cmd/util/cmd/generate-authorization-fixes/cmd_test.go @@ -4,10 +4,13 @@ import ( "testing" "github.com/onflow/cadence/runtime/common" + "github.com/onflow/cadence/runtime/interpreter" "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/onflow/flow-go/cmd/util/ledger/migrations" + "github.com/onflow/flow-go/cmd/util/ledger/reporters" "github.com/onflow/flow-go/cmd/util/ledger/util/registers" "github.com/onflow/flow-go/fvm" "github.com/onflow/flow-go/fvm/storage/snapshot" @@ -55,6 +58,20 @@ func newBootstrapPayloads( return payloads, nil } +type testReportWriter struct { + entries []any +} + +func (t *testReportWriter) Write(entry interface{}) { + t.entries = append(t.entries, entry) +} + +func (*testReportWriter) Close() { + // NO-OP +} + +var _ reporters.ReportWriter = &testReportWriter{} + func TestFixAuthorizationsMigrations(t *testing.T) { t.Parallel() @@ -196,12 +213,35 @@ func TestFixAuthorizationsMigrations(t *testing.T) { }: "ints4", } + reporter := &testReportWriter{} + generator := &AuthorizationFixGenerator{ registersByAccount: registersByAccount, mr: mr2, publicLinkReport: publicLinkReport, publicLinkMigrationReport: publicLinkMigrationReport, + reporter: reporter, } generator.generateFixesForAllAccounts() + assert.Equal(t, + []any{ + fixEntitlementsEntry{ + AccountCapabilityID: AccountCapabilityID{ + Address: common.Address(address), + CapabilityID: 2, + }, + NewAuthorization: interpreter.UnauthorizedAccess, + }, + fixEntitlementsEntry{ + AccountCapabilityID: AccountCapabilityID{ + Address: common.Address(address), + CapabilityID: 1, + }, + NewAuthorization: interpreter.UnauthorizedAccess, + }, + }, + reporter.entries, + ) + } diff --git a/cmd/util/cmd/generate-authorization-fixes/entitlements.go b/cmd/util/cmd/generate-authorization-fixes/entitlements.go index ba60dbdb228..2b461b3d310 100644 --- a/cmd/util/cmd/generate-authorization-fixes/entitlements.go +++ b/cmd/util/cmd/generate-authorization-fixes/entitlements.go @@ -1,6 +1,8 @@ package generate_authorization_fixes import ( + "sort" + "github.com/onflow/cadence/runtime/ast" "github.com/onflow/cadence/runtime/common" "github.com/onflow/cadence/runtime/common/orderedmap" @@ -8,6 +10,7 @@ import ( "golang.org/x/exp/slices" ) +// TODO: handle unsatisfiable case func findMinimalEntitlementSet( ty sema.Type, neededMethods []string, @@ -54,6 +57,13 @@ func findMinimalEntitlementSet( } }) + sort.Slice( + entitlements, + func(i, j int) bool { + return entitlements[i].ID() < entitlements[j].ID() + }, + ) + return entitlements } From b8e0b5d8ed302590dd726a175ad6cbff62480b5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Fri, 6 Sep 2024 18:17:07 -0700 Subject: [PATCH 34/72] fix test name --- cmd/util/cmd/generate-authorization-fixes/link_report_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/util/cmd/generate-authorization-fixes/link_report_test.go b/cmd/util/cmd/generate-authorization-fixes/link_report_test.go index f7f10e2b049..0227685ce48 100644 --- a/cmd/util/cmd/generate-authorization-fixes/link_report_test.go +++ b/cmd/util/cmd/generate-authorization-fixes/link_report_test.go @@ -48,7 +48,7 @@ func TestReadLinkReport(t *testing.T) { ) }) - t.Run("unfiltered", func(t *testing.T) { + t.Run("filtered", func(t *testing.T) { t.Parallel() From 55a3448e7a0047baed846206278f918fee024836 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Mon, 9 Sep 2024 11:55:38 -0700 Subject: [PATCH 35/72] improve calculation of new authorization, handle edge cases --- .../entitlements.go | 151 ++++++++--- .../entitlements_test.go | 250 ++++++++++++------ 2 files changed, 288 insertions(+), 113 deletions(-) diff --git a/cmd/util/cmd/generate-authorization-fixes/entitlements.go b/cmd/util/cmd/generate-authorization-fixes/entitlements.go index 2b461b3d310..97f667a58c4 100644 --- a/cmd/util/cmd/generate-authorization-fixes/entitlements.go +++ b/cmd/util/cmd/generate-authorization-fixes/entitlements.go @@ -1,22 +1,31 @@ package generate_authorization_fixes import ( + "errors" + "fmt" "sort" "github.com/onflow/cadence/runtime/ast" - "github.com/onflow/cadence/runtime/common" "github.com/onflow/cadence/runtime/common/orderedmap" + "github.com/onflow/cadence/runtime/interpreter" "github.com/onflow/cadence/runtime/sema" - "golang.org/x/exp/slices" ) -// TODO: handle unsatisfiable case -func findMinimalEntitlementSet( +func findMinimalAuthorization( ty sema.Type, - neededMethods []string, -) []*sema.EntitlementType { + neededMembers map[string]struct{}, +) ( + authorization interpreter.Authorization, + unresolvedMembers map[string]error, +) { + entitlements := &sema.EntitlementSet{} + unresolvedMembers = map[string]error{} + resolvedMembers := make(map[string]struct{}, len(neededMembers)) - entitlementsToMethod := orderedmap.OrderedMap[*sema.EntitlementType, []string]{} + // Iterate over the members of the type, and check if they are accessible. + // This constructs the set of entitlements needed to access the members. + // If a member is accessible, the entitlements needed to access it are added to the entitlements set. + // If a member is not accessible, it is added to the unresolved members map. // NOTE: GetMembers might return members that are actually not accessible, for DX purposes. // We need to resolve the members and filter out the inaccessible members, @@ -24,7 +33,19 @@ func findMinimalEntitlementSet( memberResolvers := ty.GetMembers() - for memberName, memberResolver := range memberResolvers { + sortedMemberNames := make([]string, 0, len(memberResolvers)) + for memberName := range memberResolvers { + sortedMemberNames = append(sortedMemberNames, memberName) + } + sort.Strings(sortedMemberNames) + + for _, memberName := range sortedMemberNames { + if _, ok := neededMembers[memberName]; !ok { + continue + } + + memberResolver := memberResolvers[memberName] + var resolveErr error member := memberResolver.Resolve(nil, memberName, ast.EmptyRange, func(err error) { resolveErr = err @@ -33,46 +54,100 @@ func findMinimalEntitlementSet( continue } - if member.DeclarationKind != common.DeclarationKindFunction { - continue + switch access := member.Access.(type) { + case sema.EntitlementSetAccess: + switch access.SetKind { + case sema.Conjunction: + access.Entitlements.Foreach(func(entitlementType *sema.EntitlementType, _ struct{}) { + entitlements.Add(entitlementType) + }) + + case sema.Disjunction: + entitlements.AddDisjunction(access.Entitlements) + + default: + panic(fmt.Errorf("unsupported set kind: %v", access.SetKind)) + } + + resolvedMembers[memberName] = struct{}{} + + case *sema.EntitlementMapAccess: + unresolvedMembers[memberName] = fmt.Errorf( + "member requires entitlement map access: %s", + access.QualifiedKeyword(), + ) + + case sema.PrimitiveAccess: + if access == sema.PrimitiveAccess(ast.AccessAll) { + // member is always accessible + resolvedMembers[memberName] = struct{}{} + } else { + unresolvedMembers[memberName] = fmt.Errorf( + "member is inaccessible (%s)", + access.QualifiedKeyword(), + ) + } + + default: + panic(fmt.Errorf("unsupported access kind: %T", member.Access)) } + } - entitlementSetAccess, ok := member.Access.(sema.EntitlementSetAccess) - if !ok { + // Check if all needed members were resolved + + for memberName := range neededMembers { + if _, ok := resolvedMembers[memberName]; ok { continue } - - // TODO: handle SetKind - entitlementSetAccess.Entitlements.Foreach(func(entitlementType *sema.EntitlementType, _ struct{}) { - methodsForEntitlement, _ := entitlementsToMethod.Get(entitlementType) - methodsForEntitlement = append(methodsForEntitlement, memberName) - entitlementsToMethod.Set(entitlementType, methodsForEntitlement) - }) + if _, ok := unresolvedMembers[memberName]; ok { + continue + } + unresolvedMembers[memberName] = errors.New("member does not exist") } - var entitlements []*sema.EntitlementType - entitlementsToMethod.Foreach(func(entitlement *sema.EntitlementType, methodsWithEntitlement []string) { - if isSubset(methodsWithEntitlement, neededMethods) { - entitlements = append(entitlements, entitlement) - } - }) + return entitlementSetMinimalAuthorization(entitlements), unresolvedMembers +} - sort.Slice( - entitlements, - func(i, j int) bool { - return entitlements[i].ID() < entitlements[j].ID() - }, - ) +// entitlementSetMinimalAuthorization returns the minimal authorization required to access the entitlements in the set. +// It is similar to `EntitlementSet.Access()`, but it returns the minimal authorization, +// i.e. does not return a disjunction if there is only one disjunction in the set, +// and only grants one entitlement for each disjunction. +func entitlementSetMinimalAuthorization(s *sema.EntitlementSet) interpreter.Authorization { - return entitlements -} + s.Minimize() + + var entitlements *sema.EntitlementOrderedSet + if s.Entitlements != nil && s.Entitlements.Len() > 0 { + entitlements = orderedmap.New[sema.EntitlementOrderedSet](s.Entitlements.Len()) + entitlements.SetAll(s.Entitlements) + } -func isSubset(subSet, superSet []string) bool { - for _, element := range subSet { - if !slices.Contains(superSet, element) { - return false + if s.Disjunctions != nil && s.Disjunctions.Len() > 0 { + if entitlements == nil { + // There are no entitlements, but disjunctions. + // Allocate a new ordered map for all entitlements in the disjunctions + // (at minimum there are two entitlements in each disjunction). + entitlements = orderedmap.New[sema.EntitlementOrderedSet](s.Disjunctions.Len() * 2) } + + // Add one entitlement for each of the disjunctions to the entitlements + s.Disjunctions.Foreach(func(_ string, disjunction *sema.EntitlementOrderedSet) { + // Only add the first entitlement in the disjunction + entitlements.Set(disjunction.Oldest().Key, struct{}{}) + }) + } + + if entitlements == nil { + return interpreter.UnauthorizedAccess } - return true + entitlementTypeIDs := orderedmap.New[sema.TypeIDOrderedSet](entitlements.Len()) + entitlements.Foreach(func(entitlement *sema.EntitlementType, _ struct{}) { + entitlementTypeIDs.Set(entitlement.ID(), struct{}{}) + }) + + return interpreter.EntitlementSetAuthorization{ + Entitlements: entitlementTypeIDs, + SetKind: sema.Conjunction, + } } diff --git a/cmd/util/cmd/generate-authorization-fixes/entitlements_test.go b/cmd/util/cmd/generate-authorization-fixes/entitlements_test.go index 5ac66ed581f..a3c6a9f8c80 100644 --- a/cmd/util/cmd/generate-authorization-fixes/entitlements_test.go +++ b/cmd/util/cmd/generate-authorization-fixes/entitlements_test.go @@ -1,14 +1,36 @@ package generate_authorization_fixes import ( + "errors" "testing" + "github.com/onflow/cadence/runtime/common" + "github.com/onflow/cadence/runtime/interpreter" "github.com/onflow/cadence/runtime/sema" . "github.com/onflow/cadence/runtime/tests/checker" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -func TestFindMinimalEntitlementsSet(t *testing.T) { +func newEntitlementSetAuthorizationFromEntitlementTypes( + entitlements []*sema.EntitlementType, + kind sema.EntitlementSetKind, +) interpreter.EntitlementSetAuthorization { + return interpreter.NewEntitlementSetAuthorization( + nil, + func() []common.TypeID { + typeIDs := make([]common.TypeID, len(entitlements)) + for i, e := range entitlements { + typeIDs[i] = e.ID() + } + return typeIDs + }, + len(entitlements), + kind, + ) +} + +func TestFindMinimalAuthorization(t *testing.T) { t.Parallel() @@ -16,110 +38,188 @@ func TestFindMinimalEntitlementsSet(t *testing.T) { entitlement E1 entitlement E2 entitlement E3 - resource interface I1 { - access(E1) fun method1() - } - resource interface I2 { - access(E1, E2) fun method2() - access(E2) fun method3() - } - resource interface I3 { - access(E3) fun method4() + + struct S { + access(all) fun accessAll() {} + access(self) fun accessSelf() {} + access(contract) fun accessContract() {} + access(account) fun accessAccount() {} + + access(E1) fun accessE1() {} + access(E2) fun accessE2() {} + access(E1, E2) fun accessE1AndE2() {} + access(E1 | E2) fun accessE1OrE2() {} } - // Assume this is the intersection-type that is left now. - // It may have some set of entitlements, but we are not interested: - // We are going to re-derive the minimal set of entitlements needed, - // depending on what methods to keep. - var intersectionType: &{I1, I2, I3}? = nil `) require.NoError(t, err) - vValueType := RequireGlobalValue(t, checker.Elaboration, "intersectionType") + ty := RequireGlobalType(t, checker.Elaboration, "S") - require.IsType(t, &sema.OptionalType{}, vValueType) - optionalType := vValueType.(*sema.OptionalType) + e1 := RequireGlobalType(t, checker.Elaboration, "E1").(*sema.EntitlementType) + e2 := RequireGlobalType(t, checker.Elaboration, "E2").(*sema.EntitlementType) - require.IsType(t, &sema.ReferenceType{}, optionalType.Type) - referenceType := optionalType.Type.(*sema.ReferenceType) + t.Run("accessAll, accessSelf, accessContract, accessAccount", func(t *testing.T) { + t.Parallel() - require.IsType(t, &sema.IntersectionType{}, referenceType.Type) - intersectionType := referenceType.Type.(*sema.IntersectionType) + authorization, unresolved := findMinimalAuthorization( + ty, + map[string]struct{}{ + "accessAll": {}, + "accessSelf": {}, + "accessContract": {}, + "accessAccount": {}, + "undefined": {}, + }, + ) + assert.Equal(t, + interpreter.UnauthorizedAccess, + authorization, + ) + assert.Equal(t, + map[string]error{ + "accessSelf": errors.New("member is inaccessible (access(self))"), + "accessContract": errors.New("member is inaccessible (access(contract))"), + "accessAccount": errors.New("member is inaccessible (access(account))"), + "undefined": errors.New("member does not exist"), + }, + unresolved, + ) + }) - e1 := RequireGlobalType(t, checker.Elaboration, "E1").(*sema.EntitlementType) - e2 := RequireGlobalType(t, checker.Elaboration, "E2").(*sema.EntitlementType) - e3 := RequireGlobalType(t, checker.Elaboration, "E3").(*sema.EntitlementType) - - t.Run("method1, method2", func(t *testing.T) { - entitlements := findMinimalEntitlementSet( - intersectionType, - []string{ - "method1", - "method2", + t.Run("accessE1", func(t *testing.T) { + t.Parallel() + + authorization, unresolved := findMinimalAuthorization( + ty, + map[string]struct{}{ + "accessE1": {}, + "undefined": {}, }, ) - require.Equal(t, - []*sema.EntitlementType{e1}, - entitlements, + assert.Equal(t, + newEntitlementSetAuthorizationFromEntitlementTypes( + []*sema.EntitlementType{ + e1, + }, + sema.Conjunction, + ), + authorization, + ) + assert.Equal(t, + map[string]error{ + "undefined": errors.New("member does not exist"), + }, + unresolved, ) }) - t.Run("method1, method2, method3", func(t *testing.T) { - entitlements := findMinimalEntitlementSet( - intersectionType, - []string{ - "method1", - "method2", - "method3", + t.Run("accessE1, accessE2", func(t *testing.T) { + t.Parallel() + + authorization, unresolved := findMinimalAuthorization( + ty, + map[string]struct{}{ + "accessE1": {}, + "accessE2": {}, + "undefined": {}, }, ) - require.Equal(t, - []*sema.EntitlementType{e1, e2}, - entitlements, + assert.Equal(t, + newEntitlementSetAuthorizationFromEntitlementTypes( + []*sema.EntitlementType{ + e1, e2, + }, + sema.Conjunction, + ), + authorization, + ) + assert.Equal(t, + map[string]error{ + "undefined": errors.New("member does not exist"), + }, + unresolved, ) }) - t.Run("method1, method2, method4", func(t *testing.T) { - entitlements := findMinimalEntitlementSet( - intersectionType, - []string{ - "method1", - "method2", - "method4", + t.Run("accessE1AndE2", func(t *testing.T) { + t.Parallel() + + authorization, unresolved := findMinimalAuthorization( + ty, + map[string]struct{}{ + "accessE1AndE2": {}, + "undefined": {}, }, ) - require.Equal( - t, - []*sema.EntitlementType{e1, e3}, - entitlements, + assert.Equal(t, + newEntitlementSetAuthorizationFromEntitlementTypes( + []*sema.EntitlementType{ + e1, e2, + }, + sema.Conjunction, + ), + authorization, + ) + assert.Equal(t, + map[string]error{ + "undefined": errors.New("member does not exist"), + }, + unresolved, ) }) - t.Run("method1, method2, method3, method4", func(t *testing.T) { - entitlements := findMinimalEntitlementSet( - intersectionType, - []string{ - "method1", - "method2", - "method3", - "method4", + t.Run("accessE1OrE2", func(t *testing.T) { + t.Parallel() + + authorization, unresolved := findMinimalAuthorization( + ty, + map[string]struct{}{ + "accessE1OrE2": {}, + "undefined": {}, }, ) - require.Equal(t, - []*sema.EntitlementType{e1, e2, e3}, - entitlements, + assert.Equal(t, + newEntitlementSetAuthorizationFromEntitlementTypes( + []*sema.EntitlementType{ + e1, + }, + sema.Conjunction, + ), + authorization, + ) + assert.Equal(t, + map[string]error{ + "undefined": errors.New("member does not exist"), + }, + unresolved, ) }) - t.Run("method4", func(t *testing.T) { - entitlements := findMinimalEntitlementSet( - intersectionType, - []string{ - "method4", + t.Run("accessE1OrE2, accessE1AndE2", func(t *testing.T) { + t.Parallel() + + authorization, unresolved := findMinimalAuthorization( + ty, + map[string]struct{}{ + "accessE1OrE2": {}, + "accessE1AndE2": {}, + "undefined": {}, }, ) - require.Equal(t, - []*sema.EntitlementType{e3}, - entitlements, + assert.Equal(t, + newEntitlementSetAuthorizationFromEntitlementTypes( + []*sema.EntitlementType{ + e1, e2, + }, + sema.Conjunction, + ), + authorization, + ) + assert.Equal(t, + map[string]error{ + "undefined": errors.New("member does not exist"), + }, + unresolved, ) }) } From 4e38c27f2b059626005dd1fd6b62f8e2dfb61905 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Mon, 9 Sep 2024 11:56:05 -0700 Subject: [PATCH 36/72] warn for unresolved members --- .../cmd/generate-authorization-fixes/cmd.go | 58 +++++++++++++------ 1 file changed, 40 insertions(+), 18 deletions(-) diff --git a/cmd/util/cmd/generate-authorization-fixes/cmd.go b/cmd/util/cmd/generate-authorization-fixes/cmd.go index c26bf0cdabe..f6c9cefe6f3 100644 --- a/cmd/util/cmd/generate-authorization-fixes/cmd.go +++ b/cmd/util/cmd/generate-authorization-fixes/cmd.go @@ -273,7 +273,8 @@ func jsonEncodeAuthorization(authorization interpreter.Authorization) string { type fixEntitlementsEntry struct { AccountCapabilityID - NewAuthorization interpreter.Authorization + NewAuthorization interpreter.Authorization + UnresolvedMembers map[string]error } var _ json.Marshaler = fixEntitlementsEntry{} @@ -284,14 +285,24 @@ func (e fixEntitlementsEntry) MarshalJSON() ([]byte, error) { CapabilityAddress string `json:"capability_address"` CapabilityID uint64 `json:"capability_id"` NewAuthorization string `json:"new_authorization"` + UnresolvedMembers map[string]string }{ Kind: "fix-entitlements", CapabilityAddress: e.Address.String(), CapabilityID: e.CapabilityID, NewAuthorization: jsonEncodeAuthorization(e.NewAuthorization), + UnresolvedMembers: jsonEncodeMemberErrorMap(e.UnresolvedMembers), }) } +func jsonEncodeMemberErrorMap(m map[string]error) map[string]string { + result := make(map[string]string, len(m)) + for key, value := range m { + result[key] = value.Error() + } + return result +} + type AuthorizationFixGenerator struct { registersByAccount *registers.ByAccount mr *migrations.InterpreterMigrationRuntime @@ -414,6 +425,15 @@ func (g *AuthorizationFixGenerator) maybeGenerateFixForCapabilityController( return } + // Assume we already had access to public built-in functions. + // For example, forEachAttachment was added in Cadence 1.0, + // so we should not consider it as a new member. + + oldAccessibleMembers = append( + []string{"getType", "isInstance", "forEachAttachment"}, + oldAccessibleMembers..., + ) + semaBorrowType, err := convertStaticToSemaType(g.mr.Interpreter, borrowType) if err != nil { log.Warn().Err(err).Msgf( @@ -442,26 +462,27 @@ func (g *AuthorizationFixGenerator) maybeGenerateFixForCapabilityController( newAccessibleMembers, ) - minimalEntitlementSet := findMinimalEntitlementSet( + oldAccessibleMemberSet := make(map[string]struct{}) + for _, memberName := range oldAccessibleMembers { + oldAccessibleMemberSet[memberName] = struct{}{} + } + + newAuthorization, unresolvedMembers := findMinimalAuthorization( semaBorrowType, - oldAccessibleMembers, + oldAccessibleMemberSet, ) - newAuthorization := interpreter.UnauthorizedAccess - if len(minimalEntitlementSet) > 0 { - newAuthorization = interpreter.NewEntitlementSetAuthorization( - nil, - func() []common.TypeID { - typeIDs := make([]common.TypeID, 0, len(minimalEntitlementSet)) - for _, entitlementType := range minimalEntitlementSet { - typeIDs = append(typeIDs, entitlementType.ID()) - } - return typeIDs - }, - len(minimalEntitlementSet), - // TODO: - sema.Conjunction, + if len(unresolvedMembers) > 0 { + // TODO: format unresolved members + log.Warn().Msgf( + "failed to find minimal entitlement set for capability controller %d in account %s: unresolved members: %v", + capabilityID, + capabilityAddress.HexWithPrefix(), + unresolvedMembers, ) + + // NOTE: still continue with the fix, + // we should not leave the capability controller vulnerable } g.reporter.Write(fixEntitlementsEntry{ @@ -469,7 +490,8 @@ func (g *AuthorizationFixGenerator) maybeGenerateFixForCapabilityController( Address: capabilityAddress, CapabilityID: capabilityID, }, - NewAuthorization: newAuthorization, + NewAuthorization: newAuthorization, + UnresolvedMembers: unresolvedMembers, }) } From 968fcfc4de3a6f6ee94dfafcf2b8d15fbdf55dd8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Mon, 9 Sep 2024 11:59:40 -0700 Subject: [PATCH 37/72] make integration test more realistic, test missing member --- .../generate-authorization-fixes/cmd_test.go | 188 ++++++++++++------ 1 file changed, 124 insertions(+), 64 deletions(-) diff --git a/cmd/util/cmd/generate-authorization-fixes/cmd_test.go b/cmd/util/cmd/generate-authorization-fixes/cmd_test.go index d18baee746c..06360bc6856 100644 --- a/cmd/util/cmd/generate-authorization-fixes/cmd_test.go +++ b/cmd/util/cmd/generate-authorization-fixes/cmd_test.go @@ -1,10 +1,15 @@ package generate_authorization_fixes import ( + "errors" + "fmt" "testing" + "github.com/onflow/cadence" + jsoncdc "github.com/onflow/cadence/encoding/json" "github.com/onflow/cadence/runtime/common" "github.com/onflow/cadence/runtime/interpreter" + "github.com/onflow/cadence/runtime/sema" "github.com/rs/zerolog" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -72,6 +77,20 @@ func (*testReportWriter) Close() { var _ reporters.ReportWriter = &testReportWriter{} +func newEntitlementSetAuthorizationFromTypeIDs( + typeIDs []common.TypeID, + setKind sema.EntitlementSetKind, +) interpreter.EntitlementSetAuthorization { + return interpreter.NewEntitlementSetAuthorization( + nil, + func() []common.TypeID { + return typeIDs + }, + len(typeIDs), + setKind, + ) +} + func TestFixAuthorizationsMigrations(t *testing.T) { t.Parallel() @@ -105,41 +124,78 @@ func TestFixAuthorizationsMigrations(t *testing.T) { err = mr.Commit(expectedWriteAddresses, log) require.NoError(t, err) - tx := flow.NewTransactionBody(). + const contractCode = ` + access(all) contract Test { + access(all) entitlement E1 + access(all) entitlement E2 + + access(all) struct S { + access(E1) fun f1() {} + access(E2) fun f2() {} + access(all) fun f3() {} + } + } + ` + + deployTX := flow.NewTransactionBody(). SetScript([]byte(` - transaction { - prepare(signer: auth(Storage, Capabilities) &Account) { - // Capability 1 was a public, unauthorized capability. - // It should lose its entitlement - let cap1 = signer.capabilities.storage.issue(/storage/ints) - signer.capabilities.publish(cap1, at: /public/ints) - - // Capability 2 was a public, unauthorized capability, stored nested in storage. - // It should lose its entitlement - let cap2 = signer.capabilities.storage.issue(/storage/ints) - signer.storage.save([cap2], to: /storage/caps2) - - // Capability 3 was a private, authorized capability, stored nested in storage. - // It should keep its entitlement - let cap3 = signer.capabilities.storage.issue(/storage/ints) - signer.storage.save([cap3], to: /storage/caps3) - - // Capability 4 was a capability with unavailable accessible members, stored nested in storage. - // It should keep its entitlement - let cap4 = signer.capabilities.storage.issue(/storage/ints) - signer.storage.save([cap4], to: /storage/caps4) + transaction(code: String) { + prepare(signer: auth(Contracts) &Account) { + signer.contracts.add(name: "Test", code: code.utf8) } } `)). + AddAuthorizer(address). + AddArgument(jsoncdc.MustEncode(cadence.String(contractCode))) + + runDeployTx := migrations.NewTransactionBasedMigration( + deployTX, + chainID, + log, + expectedWriteAddresses, + ) + err = runDeployTx(registersByAccount) + require.NoError(t, err) + + setupTx := flow.NewTransactionBody(). + SetScript([]byte(fmt.Sprintf(` + import Test from %s + + transaction { + prepare(signer: auth(Storage, Capabilities) &Account) { + // Capability 1 was a public, unauthorized capability. + // It should lose its entitlement + let cap1 = signer.capabilities.storage.issue(/storage/s) + signer.capabilities.publish(cap1, at: /public/s) + + // Capability 2 was a public, unauthorized capability, stored nested in storage. + // It should lose its entitlement + let cap2 = signer.capabilities.storage.issue(/storage/s) + signer.storage.save([cap2], to: /storage/caps2) + + // Capability 3 was a private, authorized capability, stored nested in storage. + // It should keep its entitlement + let cap3 = signer.capabilities.storage.issue(/storage/s) + signer.storage.save([cap3], to: /storage/caps3) + + // Capability 4 was a capability with unavailable accessible members, stored nested in storage. + // It should keep its entitlement + let cap4 = signer.capabilities.storage.issue(/storage/s) + signer.storage.save([cap4], to: /storage/caps4) + } + } + `, + address.HexWithPrefix(), + ))). AddAuthorizer(address) - setupTx := migrations.NewTransactionBasedMigration( - tx, + runSetupTx := migrations.NewTransactionBasedMigration( + setupTx, chainID, log, expectedWriteAddresses, ) - err = setupTx(registersByAccount) + err = runSetupTx(registersByAccount) require.NoError(t, err) mr2, err := migrations.NewInterpreterMigrationRuntime( @@ -149,51 +205,38 @@ func TestFixAuthorizationsMigrations(t *testing.T) { ) require.NoError(t, err) - // Capability 1 was a public, unauthorized capability. - // It should lose its entitlement - // - // Capability 2 was a public, unauthorized capability, stored nested in storage. - // It should lose its entitlement - // - // Capability 3 was a private, authorized capability, stored nested in storage. - // It should keep its entitlement - // - // Capability 4 was a capability with unavailable accessible members, stored nested in storage. - // It should keep its entitlement - - readArrayMembers := []string{ - "concat", - "contains", - "filter", - "firstIndex", - "getType", - "isInstance", - "length", - "map", - "slice", - "toConstantSized", + oldAccessibleMembers := []string{ + "f1", + "f3", + "undefined", + } + + testContractLocation := common.AddressLocation{ + Address: common.Address(address), + Name: "Test", } + borrowTypeID := testContractLocation.TypeID(nil, "Test.S") publicLinkReport := PublicLinkReport{ { Address: common.Address(address), - Identifier: "ints", + Identifier: "s", }: { - BorrowType: "&[Int]", - AccessibleMembers: readArrayMembers, + BorrowType: borrowTypeID, + AccessibleMembers: oldAccessibleMembers, }, { Address: common.Address(address), - Identifier: "ints2", + Identifier: "s2", }: { - BorrowType: "&[Int]", - AccessibleMembers: readArrayMembers, + BorrowType: borrowTypeID, + AccessibleMembers: oldAccessibleMembers, }, { Address: common.Address(address), - Identifier: "ints4", + Identifier: "s4", }: { - BorrowType: "&[Int]", + BorrowType: borrowTypeID, AccessibleMembers: nil, }, } @@ -202,15 +245,15 @@ func TestFixAuthorizationsMigrations(t *testing.T) { { Address: common.Address(address), CapabilityID: 1, - }: "ints", + }: "s", { Address: common.Address(address), CapabilityID: 2, - }: "ints2", + }: "s2", { Address: common.Address(address), CapabilityID: 4, - }: "ints4", + }: "s4", } reporter := &testReportWriter{} @@ -224,24 +267,41 @@ func TestFixAuthorizationsMigrations(t *testing.T) { } generator.generateFixesForAllAccounts() + e1TypeID := testContractLocation.TypeID(nil, "Test.E1") + assert.Equal(t, []any{ fixEntitlementsEntry{ AccountCapabilityID: AccountCapabilityID{ Address: common.Address(address), - CapabilityID: 2, + CapabilityID: 1, + }, + NewAuthorization: newEntitlementSetAuthorizationFromTypeIDs( + []common.TypeID{ + e1TypeID, + }, + sema.Conjunction, + ), + UnresolvedMembers: map[string]error{ + "undefined": errors.New("member does not exist"), }, - NewAuthorization: interpreter.UnauthorizedAccess, }, fixEntitlementsEntry{ AccountCapabilityID: AccountCapabilityID{ Address: common.Address(address), - CapabilityID: 1, + CapabilityID: 2, + }, + NewAuthorization: newEntitlementSetAuthorizationFromTypeIDs( + []common.TypeID{ + e1TypeID, + }, + sema.Conjunction, + ), + UnresolvedMembers: map[string]error{ + "undefined": errors.New("member does not exist"), }, - NewAuthorization: interpreter.UnauthorizedAccess, }, }, reporter.entries, ) - } From 5a803a3092fc09d79922f4326cbdbe8de50e418d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Mon, 9 Sep 2024 14:48:35 -0700 Subject: [PATCH 38/72] add command to util --- cmd/util/cmd/root.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmd/util/cmd/root.go b/cmd/util/cmd/root.go index 12e50909d2c..22fd4b86ae4 100644 --- a/cmd/util/cmd/root.go +++ b/cmd/util/cmd/root.go @@ -27,6 +27,7 @@ import ( extractpayloads "github.com/onflow/flow-go/cmd/util/cmd/extract-payloads-by-address" find_inconsistent_result "github.com/onflow/flow-go/cmd/util/cmd/find-inconsistent-result" find_trie_root "github.com/onflow/flow-go/cmd/util/cmd/find-trie-root" + generate_authorization_fixes "github.com/onflow/flow-go/cmd/util/cmd/generate-authorization-fixes" read_badger "github.com/onflow/flow-go/cmd/util/cmd/read-badger/cmd" read_execution_state "github.com/onflow/flow-go/cmd/util/cmd/read-execution-state" read_hotstuff "github.com/onflow/flow-go/cmd/util/cmd/read-hotstuff/cmd" @@ -118,6 +119,7 @@ func addCommands() { rootCmd.AddCommand(run_script.Cmd) rootCmd.AddCommand(system_addresses.Cmd) rootCmd.AddCommand(check_storage.Cmd) + rootCmd.AddCommand(generate_authorization_fixes.Cmd) } func initConfig() { From 1feb3b99eb79776ff5a86fc73b849ba4913e5abc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Mon, 9 Sep 2024 14:48:49 -0700 Subject: [PATCH 39/72] add support for reading gzipped reports --- .../cmd/generate-authorization-fixes/cmd.go | 42 ++++++++++++++----- 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/cmd/util/cmd/generate-authorization-fixes/cmd.go b/cmd/util/cmd/generate-authorization-fixes/cmd.go index f6c9cefe6f3..7fc69a029dc 100644 --- a/cmd/util/cmd/generate-authorization-fixes/cmd.go +++ b/cmd/util/cmd/generate-authorization-fixes/cmd.go @@ -1,8 +1,10 @@ package generate_authorization_fixes import ( + "compress/gzip" "encoding/json" "fmt" + "io" "os" "sort" "strings" @@ -30,7 +32,7 @@ var ( flagStateCommitment string flagOutputDirectory string flagChain string - flagLinkReport string + flagPublicLinkReport string flagLinkMigrationReport string flagAddresses string ) @@ -80,10 +82,10 @@ func init() { _ = Cmd.MarkFlagRequired("chain") Cmd.Flags().StringVar( - &flagLinkReport, - "link-report", + &flagPublicLinkReport, + "public-link-report", "", - "Input link report file name", + "Input public link report file name", ) _ = Cmd.MarkFlagRequired("link-report") @@ -150,15 +152,23 @@ func run(*cobra.Command, []string) { // Read public link report - linkReportFile, err := os.Open(flagLinkReport) + publicLinkReportFile, err := os.Open(flagPublicLinkReport) if err != nil { - log.Fatal().Err(err).Msgf("can't open link report: %s", flagLinkReport) + log.Fatal().Err(err).Msgf("can't open link report: %s", flagPublicLinkReport) } - defer linkReportFile.Close() + defer publicLinkReportFile.Close() - publicLinkReport, err := ReadPublicLinkReport(linkReportFile, addressFilter) + var publicLinkReportReader io.Reader = publicLinkReportFile + if isGzip(publicLinkReportFile) { + publicLinkReportReader, err = gzip.NewReader(publicLinkReportFile) + if err != nil { + log.Fatal().Err(err).Msgf("failed to create gzip reader for %s", flagPublicLinkReport) + } + } + + publicLinkReport, err := ReadPublicLinkReport(publicLinkReportReader, addressFilter) if err != nil { - log.Fatal().Err(err).Msgf("failed to read public link report %s", flagLinkReport) + log.Fatal().Err(err).Msgf("failed to read public link report %s", flagPublicLinkReport) } // Read link migration report @@ -169,7 +179,15 @@ func run(*cobra.Command, []string) { } defer linkMigrationReportFile.Close() - publicLinkMigrationReport, err := ReadPublicLinkMigrationReport(linkMigrationReportFile, addressFilter) + var linkMigrationReportReader io.Reader = linkMigrationReportFile + if isGzip(linkMigrationReportFile) { + linkMigrationReportReader, err = gzip.NewReader(linkMigrationReportFile) + if err != nil { + log.Fatal().Err(err).Msgf("failed to create gzip reader for %s", flagLinkMigrationReport) + } + } + + publicLinkMigrationReport, err := ReadPublicLinkMigrationReport(linkMigrationReportReader, addressFilter) if err != nil { log.Fatal().Err(err).Msgf("failed to read public link report: %s", flagLinkMigrationReport) } @@ -547,3 +565,7 @@ func (g *AuthorizationFixGenerator) publicPathLinkInfo( Identifier: publicPathIdentifier, }] } + +func isGzip(file *os.File) bool { + return strings.HasSuffix(file.Name(), ".gz") +} From f00424e356d01eb23d1a68906e8035dbd8559ad1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Mon, 9 Sep 2024 15:06:53 -0700 Subject: [PATCH 40/72] use existing contract gathering and checking functions --- .../check_contract.go | 119 ------------------ .../cmd/generate-authorization-fixes/cmd.go | 9 +- .../generate-authorization-fixes/contracts.go | 82 ------------ .../migrations/contract_checking_migration.go | 8 +- .../migrations/type_requirements_extractor.go | 2 +- 5 files changed, 11 insertions(+), 209 deletions(-) delete mode 100644 cmd/util/cmd/generate-authorization-fixes/check_contract.go delete mode 100644 cmd/util/cmd/generate-authorization-fixes/contracts.go diff --git a/cmd/util/cmd/generate-authorization-fixes/check_contract.go b/cmd/util/cmd/generate-authorization-fixes/check_contract.go deleted file mode 100644 index 6461ef6ad25..00000000000 --- a/cmd/util/cmd/generate-authorization-fixes/check_contract.go +++ /dev/null @@ -1,119 +0,0 @@ -package generate_authorization_fixes - -import ( - "encoding/json" - "strings" - - "github.com/onflow/cadence/runtime/common" - "github.com/onflow/cadence/runtime/interpreter" - "github.com/onflow/cadence/runtime/pretty" - "github.com/rs/zerolog/log" - - "github.com/onflow/flow-go/cmd/util/ledger/migrations" - "github.com/onflow/flow-go/cmd/util/ledger/reporters" -) - -func checkContract( - contract AddressContract, - mr *migrations.InterpreterMigrationRuntime, - contractsForPrettyPrinting map[common.Location][]byte, - reporter reporters.ReportWriter, - programs map[common.Location]*interpreter.Program, -) { - location := contract.Location - code := contract.Code - - log.Info().Msgf("checking contract %s ...", location) - - // Check contract code - const getAndSetProgram = true - program, err := mr.ContractAdditionHandler.ParseAndCheckProgram(code, location, getAndSetProgram) - if err != nil { - - // Pretty print the error - var builder strings.Builder - errorPrinter := pretty.NewErrorPrettyPrinter(&builder, false) - - printErr := errorPrinter.PrettyPrintError(err, location, contractsForPrettyPrinting) - - var errorDetails string - if printErr == nil { - errorDetails = builder.String() - } else { - errorDetails = err.Error() - } - - log.Error().Msgf( - "error checking contract %s: %s", - location, - errorDetails, - ) - - reporter.Write(contractCheckingFailure{ - AccountAddress: location.Address, - ContractName: location.Name, - Code: string(code), - Error: errorDetails, - }) - - return - } - - // Record the checked program for future use - programs[location] = program - - reporter.Write(contractCheckingSuccess{ - AccountAddress: location.Address, - ContractName: location.Name, - Code: string(code), - }) - - log.Info().Msgf("finished checking contract %s", location) -} - -type contractCheckingFailure struct { - AccountAddress common.Address - ContractName string - Code string - Error string -} - -var _ json.Marshaler = contractCheckingFailure{} - -func (e contractCheckingFailure) MarshalJSON() ([]byte, error) { - return json.Marshal(struct { - Kind string `json:"kind"` - AccountAddress string `json:"address"` - ContractName string `json:"name"` - Code string `json:"code"` - Error string `json:"error"` - }{ - Kind: "checking-failure", - AccountAddress: e.AccountAddress.HexWithPrefix(), - ContractName: e.ContractName, - Code: e.Code, - Error: e.Error, - }) -} - -type contractCheckingSuccess struct { - AccountAddress common.Address - ContractName string - Code string -} - -var _ json.Marshaler = contractCheckingSuccess{} - -func (e contractCheckingSuccess) MarshalJSON() ([]byte, error) { - return json.Marshal(struct { - Kind string `json:"kind"` - AccountAddress string `json:"address"` - ContractName string `json:"name"` - Code string `json:"code"` - }{ - Kind: "checking-success", - AccountAddress: e.AccountAddress.HexWithPrefix(), - ContractName: e.ContractName, - Code: e.Code, - }) -} diff --git a/cmd/util/cmd/generate-authorization-fixes/cmd.go b/cmd/util/cmd/generate-authorization-fixes/cmd.go index 7fc69a029dc..38a7ffa0d5f 100644 --- a/cmd/util/cmd/generate-authorization-fixes/cmd.go +++ b/cmd/util/cmd/generate-authorization-fixes/cmd.go @@ -87,7 +87,7 @@ func init() { "", "Input public link report file name", ) - _ = Cmd.MarkFlagRequired("link-report") + _ = Cmd.MarkFlagRequired("public-link-report") Cmd.Flags().StringVar( &flagLinkMigrationReport, @@ -253,7 +253,7 @@ func checkContracts( mr *migrations.InterpreterMigrationRuntime, reporter reporters.ReportWriter, ) { - contracts, err := gatherContractsFromRegisters(registersByAccount) + contracts, err := migrations.GatherContractsFromRegisters(registersByAccount, log.Logger) if err != nil { log.Fatal().Err(err) } @@ -268,11 +268,14 @@ func checkContracts( log.Info().Msg("Checking contracts ...") for _, contract := range contracts { - checkContract( + migrations.CheckContract( contract, + log.Logger, mr, contractsForPrettyPrinting, + false, reporter, + nil, programs, ) } diff --git a/cmd/util/cmd/generate-authorization-fixes/contracts.go b/cmd/util/cmd/generate-authorization-fixes/contracts.go deleted file mode 100644 index 3e8308973d9..00000000000 --- a/cmd/util/cmd/generate-authorization-fixes/contracts.go +++ /dev/null @@ -1,82 +0,0 @@ -package generate_authorization_fixes - -import ( - "bytes" - "fmt" - "sort" - - "github.com/onflow/cadence/runtime/common" - "github.com/rs/zerolog/log" - - "github.com/onflow/flow-go/cmd/util/ledger/util/registers" - "github.com/onflow/flow-go/fvm/environment" - "github.com/onflow/flow-go/model/flow" -) - -type AddressContract struct { - Location common.AddressLocation - Code []byte -} - -func gatherContractsFromRegisters(registersByAccount *registers.ByAccount) ([]AddressContract, error) { - log.Info().Msg("Gathering contracts ...") - - contracts := make([]AddressContract, 0, contractCountEstimate) - - err := registersByAccount.ForEachAccount(func(accountRegisters *registers.AccountRegisters) error { - owner := accountRegisters.Owner() - - encodedContractNames, err := accountRegisters.Get(owner, flow.ContractNamesKey) - if err != nil { - return err - } - - contractNames, err := environment.DecodeContractNames(encodedContractNames) - if err != nil { - return err - } - - for _, contractName := range contractNames { - - contractKey := flow.ContractKey(contractName) - - code, err := accountRegisters.Get(owner, contractKey) - if err != nil { - return err - } - - if len(bytes.TrimSpace(code)) == 0 { - continue - } - - address := common.Address([]byte(owner)) - location := common.AddressLocation{ - Address: address, - Name: contractName, - } - - contracts = append( - contracts, - AddressContract{ - Location: location, - Code: code, - }, - ) - } - - return nil - }) - if err != nil { - return nil, fmt.Errorf("failed to get contracts of accounts: %w", err) - } - - sort.Slice(contracts, func(i, j int) bool { - a := contracts[i] - b := contracts[j] - return a.Location.ID() < b.Location.ID() - }) - - log.Info().Msgf("Gathered all contracts (%d)", len(contracts)) - - return contracts, nil -} diff --git a/cmd/util/ledger/migrations/contract_checking_migration.go b/cmd/util/ledger/migrations/contract_checking_migration.go index bffc05c9dfd..91aa7bbb902 100644 --- a/cmd/util/ledger/migrations/contract_checking_migration.go +++ b/cmd/util/ledger/migrations/contract_checking_migration.go @@ -51,7 +51,7 @@ func NewContractCheckingMigration( return fmt.Errorf("failed to create interpreter migration runtime: %w", err) } - contracts, err := gatherContractsFromRegisters(registersByAccount, log) + contracts, err := GatherContractsFromRegisters(registersByAccount, log) if err != nil { return err } @@ -64,7 +64,7 @@ func NewContractCheckingMigration( // Check all contracts for _, contract := range contracts { - checkContract( + CheckContract( contract, log, mr, @@ -80,7 +80,7 @@ func NewContractCheckingMigration( } } -func gatherContractsFromRegisters(registersByAccount *registers.ByAccount, log zerolog.Logger) ([]AddressContract, error) { +func GatherContractsFromRegisters(registersByAccount *registers.ByAccount, log zerolog.Logger) ([]AddressContract, error) { log.Info().Msg("Gathering contracts ...") contracts := make([]AddressContract, 0, contractCountEstimate) @@ -142,7 +142,7 @@ func gatherContractsFromRegisters(registersByAccount *registers.ByAccount, log z return contracts, nil } -func checkContract( +func CheckContract( contract AddressContract, log zerolog.Logger, mr *InterpreterMigrationRuntime, diff --git a/cmd/util/ledger/migrations/type_requirements_extractor.go b/cmd/util/ledger/migrations/type_requirements_extractor.go index 9fcfe70396d..67de5cb52c9 100644 --- a/cmd/util/ledger/migrations/type_requirements_extractor.go +++ b/cmd/util/ledger/migrations/type_requirements_extractor.go @@ -37,7 +37,7 @@ func NewTypeRequirementsExtractingMigration( // Gather all contracts - contracts, err := gatherContractsFromRegisters(registersByAccount, log) + contracts, err := GatherContractsFromRegisters(registersByAccount, log) if err != nil { return err } From 7d841dcadd80c3b1645e0016e387d15b6bd7871e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Mon, 9 Sep 2024 15:15:43 -0700 Subject: [PATCH 41/72] delete from existing map instead of allocating new map --- cmd/util/cmd/generate-authorization-fixes/entitlements.go | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/cmd/util/cmd/generate-authorization-fixes/entitlements.go b/cmd/util/cmd/generate-authorization-fixes/entitlements.go index 97f667a58c4..cc3aff355cc 100644 --- a/cmd/util/cmd/generate-authorization-fixes/entitlements.go +++ b/cmd/util/cmd/generate-authorization-fixes/entitlements.go @@ -20,7 +20,6 @@ func findMinimalAuthorization( ) { entitlements := &sema.EntitlementSet{} unresolvedMembers = map[string]error{} - resolvedMembers := make(map[string]struct{}, len(neededMembers)) // Iterate over the members of the type, and check if they are accessible. // This constructs the set of entitlements needed to access the members. @@ -69,7 +68,7 @@ func findMinimalAuthorization( panic(fmt.Errorf("unsupported set kind: %v", access.SetKind)) } - resolvedMembers[memberName] = struct{}{} + delete(neededMembers, memberName) case *sema.EntitlementMapAccess: unresolvedMembers[memberName] = fmt.Errorf( @@ -80,7 +79,7 @@ func findMinimalAuthorization( case sema.PrimitiveAccess: if access == sema.PrimitiveAccess(ast.AccessAll) { // member is always accessible - resolvedMembers[memberName] = struct{}{} + delete(neededMembers, memberName) } else { unresolvedMembers[memberName] = fmt.Errorf( "member is inaccessible (%s)", @@ -96,9 +95,6 @@ func findMinimalAuthorization( // Check if all needed members were resolved for memberName := range neededMembers { - if _, ok := resolvedMembers[memberName]; ok { - continue - } if _, ok := unresolvedMembers[memberName]; ok { continue } From e2e8227053d417d32f71d40f79886dbbd52469ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Mon, 9 Sep 2024 15:19:58 -0700 Subject: [PATCH 42/72] iterate over needed members instead of member resolvers --- .../entitlements.go | 35 ++++++------------- 1 file changed, 11 insertions(+), 24 deletions(-) diff --git a/cmd/util/cmd/generate-authorization-fixes/entitlements.go b/cmd/util/cmd/generate-authorization-fixes/entitlements.go index cc3aff355cc..5f4b20bc693 100644 --- a/cmd/util/cmd/generate-authorization-fixes/entitlements.go +++ b/cmd/util/cmd/generate-authorization-fixes/entitlements.go @@ -21,35 +21,31 @@ func findMinimalAuthorization( entitlements := &sema.EntitlementSet{} unresolvedMembers = map[string]error{} - // Iterate over the members of the type, and check if they are accessible. - // This constructs the set of entitlements needed to access the members. - // If a member is accessible, the entitlements needed to access it are added to the entitlements set. - // If a member is not accessible, it is added to the unresolved members map. - // NOTE: GetMembers might return members that are actually not accessible, for DX purposes. // We need to resolve the members and filter out the inaccessible members, // using the error reported when resolving - memberResolvers := ty.GetMembers() - - sortedMemberNames := make([]string, 0, len(memberResolvers)) - for memberName := range memberResolvers { - sortedMemberNames = append(sortedMemberNames, memberName) + sortedNeededMembers := make([]string, 0, len(neededMembers)) + for memberName := range neededMembers { + sortedNeededMembers = append(sortedNeededMembers, memberName) } - sort.Strings(sortedMemberNames) + sort.Strings(sortedNeededMembers) - for _, memberName := range sortedMemberNames { - if _, ok := neededMembers[memberName]; !ok { + memberResolvers := ty.GetMembers() + + for _, memberName := range sortedNeededMembers { + memberResolver, ok := memberResolvers[memberName] + if !ok { + unresolvedMembers[memberName] = errors.New("member does not exist") continue } - memberResolver := memberResolvers[memberName] - var resolveErr error member := memberResolver.Resolve(nil, memberName, ast.EmptyRange, func(err error) { resolveErr = err }) if resolveErr != nil { + unresolvedMembers[memberName] = resolveErr continue } @@ -92,15 +88,6 @@ func findMinimalAuthorization( } } - // Check if all needed members were resolved - - for memberName := range neededMembers { - if _, ok := unresolvedMembers[memberName]; ok { - continue - } - unresolvedMembers[memberName] = errors.New("member does not exist") - } - return entitlementSetMinimalAuthorization(entitlements), unresolvedMembers } From 5cae03dfb528c85619b1655966be9102c864ff59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Mon, 9 Sep 2024 15:22:14 -0700 Subject: [PATCH 43/72] allocate map before assigning --- cmd/util/cmd/generate-authorization-fixes/cmd.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cmd/util/cmd/generate-authorization-fixes/cmd.go b/cmd/util/cmd/generate-authorization-fixes/cmd.go index 38a7ffa0d5f..7e3d5448e97 100644 --- a/cmd/util/cmd/generate-authorization-fixes/cmd.go +++ b/cmd/util/cmd/generate-authorization-fixes/cmd.go @@ -125,6 +125,9 @@ func run(*cobra.Command, []string) { log.Fatal().Err(err).Msgf("failed to parse address: %s", hexAddr) } + if addressFilter == nil { + addressFilter = make(map[common.Address]struct{}) + } addressFilter[common.Address(addr)] = struct{}{} } } From 60b3a8a2a81ceb4df8559c146386f07658197a2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Mon, 9 Sep 2024 15:29:21 -0700 Subject: [PATCH 44/72] improve naming --- ...ion.go => fix_authorizations_migration.go} | 98 +++++++++---------- ...o => fix_authorizations_migration_test.go} | 20 ++-- 2 files changed, 59 insertions(+), 59 deletions(-) rename cmd/util/ledger/migrations/{fix_entitlements_migration.go => fix_authorizations_migration.go} (77%) rename cmd/util/ledger/migrations/{fix_entitlements_migration_test.go => fix_authorizations_migration_test.go} (89%) diff --git a/cmd/util/ledger/migrations/fix_entitlements_migration.go b/cmd/util/ledger/migrations/fix_authorizations_migration.go similarity index 77% rename from cmd/util/ledger/migrations/fix_entitlements_migration.go rename to cmd/util/ledger/migrations/fix_authorizations_migration.go index 9dd6ce83ef7..7c4f25d1a4d 100644 --- a/cmd/util/ledger/migrations/fix_entitlements_migration.go +++ b/cmd/util/ledger/migrations/fix_authorizations_migration.go @@ -24,9 +24,9 @@ type AccountCapabilityControllerID struct { CapabilityID uint64 } -// FixEntitlementsMigration +// FixAuthorizationsMigration -type FixEntitlementsMigrationReporter interface { +type FixAuthorizationsMigrationReporter interface { MigratedCapability( storageKey interpreter.StorageKey, capabilityAddress common.Address, @@ -40,22 +40,22 @@ type FixEntitlementsMigrationReporter interface { ) } -type FixEntitlementsMigration struct { - Reporter FixEntitlementsMigrationReporter +type FixAuthorizationsMigration struct { + Reporter FixAuthorizationsMigrationReporter NewAuthorizations map[AccountCapabilityControllerID]interpreter.Authorization } -var _ migrations.ValueMigration = &FixEntitlementsMigration{} +var _ migrations.ValueMigration = &FixAuthorizationsMigration{} -func (*FixEntitlementsMigration) Name() string { - return "FixEntitlementsMigration" +func (*FixAuthorizationsMigration) Name() string { + return "FixAuthorizationsMigration" } -func (*FixEntitlementsMigration) Domains() map[string]struct{} { +func (*FixAuthorizationsMigration) Domains() map[string]struct{} { return nil } -func (m *FixEntitlementsMigration) Migrate( +func (m *FixAuthorizationsMigration) Migrate( storageKey interpreter.StorageKey, _ interpreter.StorageMapKey, value interpreter.Value, @@ -140,21 +140,21 @@ func (m *FixEntitlementsMigration) Migrate( return nil, nil } -func (*FixEntitlementsMigration) CanSkip(valueType interpreter.StaticType) bool { - return CanSkipFixEntitlementsMigration(valueType) +func (*FixAuthorizationsMigration) CanSkip(valueType interpreter.StaticType) bool { + return CanSkipFixAuthorizationsMigration(valueType) } -func CanSkipFixEntitlementsMigration(valueType interpreter.StaticType) bool { +func CanSkipFixAuthorizationsMigration(valueType interpreter.StaticType) bool { switch valueType := valueType.(type) { case *interpreter.DictionaryStaticType: - return CanSkipFixEntitlementsMigration(valueType.KeyType) && - CanSkipFixEntitlementsMigration(valueType.ValueType) + return CanSkipFixAuthorizationsMigration(valueType.KeyType) && + CanSkipFixAuthorizationsMigration(valueType.ValueType) case interpreter.ArrayStaticType: - return CanSkipFixEntitlementsMigration(valueType.ElementType()) + return CanSkipFixAuthorizationsMigration(valueType.ElementType()) case *interpreter.OptionalStaticType: - return CanSkipFixEntitlementsMigration(valueType.Type) + return CanSkipFixAuthorizationsMigration(valueType.Type) case *interpreter.CapabilityStaticType: return false @@ -191,7 +191,7 @@ func CanSkipFixEntitlementsMigration(valueType interpreter.StaticType) bool { return false } -type FixEntitlementsMigrationOptions struct { +type FixAuthorizationsMigrationOptions struct { ChainID flow.ChainID NWorker int VerboseErrorOutput bool @@ -200,24 +200,24 @@ type FixEntitlementsMigrationOptions struct { CheckStorageHealthBeforeMigration bool } -const fixEntitlementsMigrationReporterName = "fix-entitlements-migration" +const fixAuthorizationsMigrationReporterName = "fix-authorizations-migration" -func NewFixEntitlementsMigration( +func NewFixAuhorizationsMigration( rwf reporters.ReportWriterFactory, errorMessageHandler *errorMessageHandler, programs map[runtime.Location]*interpreter.Program, newAuthorizations map[AccountCapabilityControllerID]interpreter.Authorization, - opts FixEntitlementsMigrationOptions, + opts FixAuthorizationsMigrationOptions, ) *CadenceBaseMigration { var diffReporter reporters.ReportWriter if opts.DiffMigrations { - diffReporter = rwf.ReportWriter("fix-entitlements-migration-diff") + diffReporter = rwf.ReportWriter("fix-authorizations-migration-diff") } - reporter := rwf.ReportWriter(fixEntitlementsMigrationReporterName) + reporter := rwf.ReportWriter(fixAuthorizationsMigrationReporterName) return &CadenceBaseMigration{ - name: "fix_entitlements_migration", + name: "fix_authorizations_migration", reporter: reporter, diffReporter: diffReporter, logVerboseDiff: opts.LogVerboseDiff, @@ -230,9 +230,9 @@ func NewFixEntitlementsMigration( ) []migrations.ValueMigration { return []migrations.ValueMigration{ - &FixEntitlementsMigration{ + &FixAuthorizationsMigration{ NewAuthorizations: newAuthorizations, - Reporter: &fixEntitlementsMigrationReporter{ + Reporter: &fixAuthorizationsMigrationReporter{ reportWriter: reporter, errorMessageHandler: errorMessageHandler, verboseErrorOutput: opts.VerboseErrorOutput, @@ -246,16 +246,16 @@ func NewFixEntitlementsMigration( } } -type fixEntitlementsMigrationReporter struct { +type fixAuthorizationsMigrationReporter struct { reportWriter reporters.ReportWriter errorMessageHandler *errorMessageHandler verboseErrorOutput bool } -var _ FixEntitlementsMigrationReporter = &fixEntitlementsMigrationReporter{} -var _ migrations.Reporter = &fixEntitlementsMigrationReporter{} +var _ FixAuthorizationsMigrationReporter = &fixAuthorizationsMigrationReporter{} +var _ migrations.Reporter = &fixAuthorizationsMigrationReporter{} -func (r *fixEntitlementsMigrationReporter) Migrated( +func (r *fixAuthorizationsMigrationReporter) Migrated( storageKey interpreter.StorageKey, storageMapKey interpreter.StorageMapKey, migration string, @@ -267,7 +267,7 @@ func (r *fixEntitlementsMigrationReporter) Migrated( }) } -func (r *fixEntitlementsMigrationReporter) Error(err error) { +func (r *fixAuthorizationsMigrationReporter) Error(err error) { var migrationErr migrations.StorageMigrationError @@ -295,31 +295,31 @@ func (r *fixEntitlementsMigrationReporter) Error(err error) { } } -func (r *fixEntitlementsMigrationReporter) DictionaryKeyConflict(accountAddressPath interpreter.AddressPath) { +func (r *fixAuthorizationsMigrationReporter) DictionaryKeyConflict(accountAddressPath interpreter.AddressPath) { r.reportWriter.Write(dictionaryKeyConflictEntry{ AddressPath: accountAddressPath, }) } -func (r *fixEntitlementsMigrationReporter) MigratedCapabilityController( +func (r *fixAuthorizationsMigrationReporter) MigratedCapabilityController( storageKey interpreter.StorageKey, capabilityID uint64, newAuthorization interpreter.Authorization, ) { - r.reportWriter.Write(capabilityControllerEntitlementsFixedEntry{ + r.reportWriter.Write(capabilityControllerAuthorizationFixedEntry{ StorageKey: storageKey, CapabilityID: capabilityID, NewAuthorization: newAuthorization, }) } -func (r *fixEntitlementsMigrationReporter) MigratedCapability( +func (r *fixAuthorizationsMigrationReporter) MigratedCapability( storageKey interpreter.StorageKey, capabilityAddress common.Address, capabilityID uint64, newAuthorization interpreter.Authorization, ) { - r.reportWriter.Write(capabilityEntitlementsFixedEntry{ + r.reportWriter.Write(capabilityAuthorizationFixedEntry{ StorageKey: storageKey, CapabilityAddress: capabilityAddress, CapabilityID: capabilityID, @@ -336,16 +336,16 @@ func jsonEncodeAuthorization(authorization interpreter.Authorization) string { } } -// capabilityControllerEntitlementsFixedEntry -type capabilityControllerEntitlementsFixedEntry struct { +// capabilityControllerAuthorizationFixedEntry +type capabilityControllerAuthorizationFixedEntry struct { StorageKey interpreter.StorageKey CapabilityID uint64 NewAuthorization interpreter.Authorization } -var _ json.Marshaler = capabilityControllerEntitlementsFixedEntry{} +var _ json.Marshaler = capabilityControllerAuthorizationFixedEntry{} -func (e capabilityControllerEntitlementsFixedEntry) MarshalJSON() ([]byte, error) { +func (e capabilityControllerAuthorizationFixedEntry) MarshalJSON() ([]byte, error) { return json.Marshal(struct { Kind string `json:"kind"` AccountAddress string `json:"account_address"` @@ -353,7 +353,7 @@ func (e capabilityControllerEntitlementsFixedEntry) MarshalJSON() ([]byte, error CapabilityID uint64 `json:"capability_id"` NewAuthorization string `json:"new_authorization"` }{ - Kind: "capability-controller-entitlements-fixed", + Kind: "capability-controller-authorizations-fixed", AccountAddress: e.StorageKey.Address.HexWithPrefix(), StorageDomain: e.StorageKey.Key, CapabilityID: e.CapabilityID, @@ -361,17 +361,17 @@ func (e capabilityControllerEntitlementsFixedEntry) MarshalJSON() ([]byte, error }) } -// capabilityEntitlementsFixedEntry -type capabilityEntitlementsFixedEntry struct { +// capabilityAuthorizationFixedEntry +type capabilityAuthorizationFixedEntry struct { StorageKey interpreter.StorageKey CapabilityAddress common.Address CapabilityID uint64 NewAuthorization interpreter.Authorization } -var _ json.Marshaler = capabilityEntitlementsFixedEntry{} +var _ json.Marshaler = capabilityAuthorizationFixedEntry{} -func (e capabilityEntitlementsFixedEntry) MarshalJSON() ([]byte, error) { +func (e capabilityAuthorizationFixedEntry) MarshalJSON() ([]byte, error) { return json.Marshal(struct { Kind string `json:"kind"` AccountAddress string `json:"account_address"` @@ -380,7 +380,7 @@ func (e capabilityEntitlementsFixedEntry) MarshalJSON() ([]byte, error) { CapabilityID uint64 `json:"capability_id"` NewAuthorization string `json:"new_authorization"` }{ - Kind: "capability-entitlements-fixed", + Kind: "capability-authorizations-fixed", AccountAddress: e.StorageKey.Address.HexWithPrefix(), StorageDomain: e.StorageKey.Key, CapabilityAddress: e.CapabilityAddress.HexWithPrefix(), @@ -389,11 +389,11 @@ func (e capabilityEntitlementsFixedEntry) MarshalJSON() ([]byte, error) { }) } -func NewFixEntitlementsMigrations( +func NewFixAuthorizationsMigrations( log zerolog.Logger, rwf reporters.ReportWriterFactory, newAuthorizations map[AccountCapabilityControllerID]interpreter.Authorization, - opts FixEntitlementsMigrationOptions, + opts FixAuthorizationsMigrationOptions, ) []NamedMigration { errorMessageHandler := &errorMessageHandler{} @@ -422,12 +422,12 @@ func NewFixEntitlementsMigrations( ), }, { - Name: "fix-entitlements", + Name: "fix-authorizations", Migrate: NewAccountBasedMigration( log, opts.NWorker, []AccountBasedMigration{ - NewFixEntitlementsMigration( + NewFixAuhorizationsMigration( rwf, errorMessageHandler, programs, diff --git a/cmd/util/ledger/migrations/fix_entitlements_migration_test.go b/cmd/util/ledger/migrations/fix_authorizations_migration_test.go similarity index 89% rename from cmd/util/ledger/migrations/fix_entitlements_migration_test.go rename to cmd/util/ledger/migrations/fix_authorizations_migration_test.go index 30680cad6ba..9f0ceac8318 100644 --- a/cmd/util/ledger/migrations/fix_entitlements_migration_test.go +++ b/cmd/util/ledger/migrations/fix_authorizations_migration_test.go @@ -12,7 +12,7 @@ import ( "github.com/onflow/flow-go/model/flow" ) -func TestEntitlements(t *testing.T) { +func TestFixAuthorizationsMigration(t *testing.T) { t.Parallel() const chainID = flow.Emulator @@ -84,7 +84,7 @@ func TestEntitlements(t *testing.T) { rwf := &testReportWriterFactory{} - options := FixEntitlementsMigrationOptions{ + options := FixAuthorizationsMigrationOptions{ ChainID: chainID, NWorker: nWorker, } @@ -100,7 +100,7 @@ func TestEntitlements(t *testing.T) { }: interpreter.UnauthorizedAccess, } - migrations := NewFixEntitlementsMigrations( + migrations := NewFixAuthorizationsMigrations( log, rwf, fixes, @@ -112,15 +112,15 @@ func TestEntitlements(t *testing.T) { require.NoError(t, err) } - reporter := rwf.reportWriters[fixEntitlementsMigrationReporterName] + reporter := rwf.reportWriters[fixAuthorizationsMigrationReporterName] require.NotNil(t, reporter) var entries []any for _, entry := range reporter.entries { switch entry := entry.(type) { - case capabilityEntitlementsFixedEntry, - capabilityControllerEntitlementsFixedEntry: + case capabilityAuthorizationFixedEntry, + capabilityControllerAuthorizationFixedEntry: entries = append(entries, entry) } @@ -128,7 +128,7 @@ func TestEntitlements(t *testing.T) { require.ElementsMatch(t, []any{ - capabilityControllerEntitlementsFixedEntry{ + capabilityControllerAuthorizationFixedEntry{ StorageKey: interpreter.StorageKey{ Key: "cap_con", Address: common.Address(address), @@ -136,7 +136,7 @@ func TestEntitlements(t *testing.T) { CapabilityID: 1, NewAuthorization: interpreter.UnauthorizedAccess, }, - capabilityControllerEntitlementsFixedEntry{ + capabilityControllerAuthorizationFixedEntry{ StorageKey: interpreter.StorageKey{ Key: "cap_con", Address: common.Address(address), @@ -144,7 +144,7 @@ func TestEntitlements(t *testing.T) { CapabilityID: 2, NewAuthorization: interpreter.UnauthorizedAccess, }, - capabilityEntitlementsFixedEntry{ + capabilityAuthorizationFixedEntry{ StorageKey: interpreter.StorageKey{ Key: "public", Address: common.Address(address), @@ -153,7 +153,7 @@ func TestEntitlements(t *testing.T) { CapabilityID: 1, NewAuthorization: interpreter.UnauthorizedAccess, }, - capabilityEntitlementsFixedEntry{ + capabilityAuthorizationFixedEntry{ StorageKey: interpreter.StorageKey{ Key: "storage", Address: common.Address(address), From 67ae379a42017887cdc722da3c1de17f9f4595be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Mon, 9 Sep 2024 15:29:34 -0700 Subject: [PATCH 45/72] remove TODO --- cmd/util/ledger/migrations/fix_authorizations_migration.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cmd/util/ledger/migrations/fix_authorizations_migration.go b/cmd/util/ledger/migrations/fix_authorizations_migration.go index 7c4f25d1a4d..ba8532863ee 100644 --- a/cmd/util/ledger/migrations/fix_authorizations_migration.go +++ b/cmd/util/ledger/migrations/fix_authorizations_migration.go @@ -408,7 +408,6 @@ func NewFixAuthorizationsMigrations( programs := make(map[common.Location]*interpreter.Program, 1000) return []NamedMigration{ - // TODO: unnecessary? remove? { Name: "check-contracts", Migrate: NewContractCheckingMigration( From 884b01bec38060dbf0d8e56914cc60710d1fd98b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Mon, 9 Sep 2024 15:37:57 -0700 Subject: [PATCH 46/72] clean up --- cmd/util/ledger/migrations/fix_authorizations_migration.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/cmd/util/ledger/migrations/fix_authorizations_migration.go b/cmd/util/ledger/migrations/fix_authorizations_migration.go index ba8532863ee..b8edef0bc4a 100644 --- a/cmd/util/ledger/migrations/fix_authorizations_migration.go +++ b/cmd/util/ledger/migrations/fix_authorizations_migration.go @@ -202,7 +202,7 @@ type FixAuthorizationsMigrationOptions struct { const fixAuthorizationsMigrationReporterName = "fix-authorizations-migration" -func NewFixAuhorizationsMigration( +func NewFixAuthorizationsMigration( rwf reporters.ReportWriterFactory, errorMessageHandler *errorMessageHandler, programs map[runtime.Location]*interpreter.Program, @@ -415,8 +415,7 @@ func NewFixAuthorizationsMigrations( rwf, opts.ChainID, opts.VerboseErrorOutput, - // TODO: what are the important locations? - map[common.AddressLocation]struct{}{}, + nil, programs, ), }, @@ -426,7 +425,7 @@ func NewFixAuthorizationsMigrations( log, opts.NWorker, []AccountBasedMigration{ - NewFixAuhorizationsMigration( + NewFixAuthorizationsMigration( rwf, errorMessageHandler, programs, From 1dfad3f9f53bbef86323d2d19de83c2012651bc0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Mon, 9 Sep 2024 15:59:24 -0700 Subject: [PATCH 47/72] return replacement values instead of mutating old values --- .../fix_authorizations_migration.go | 37 ++++++++++++++----- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/cmd/util/ledger/migrations/fix_authorizations_migration.go b/cmd/util/ledger/migrations/fix_authorizations_migration.go index b8edef0bc4a..dcc1a08b1aa 100644 --- a/cmd/util/ledger/migrations/fix_authorizations_migration.go +++ b/cmd/util/ledger/migrations/fix_authorizations_migration.go @@ -79,8 +79,8 @@ func (m *FixAuthorizationsMigration) Migrate( return nil, nil } - borrowType := value.BorrowType - if borrowType == nil { + oldBorrowType := value.BorrowType + if oldBorrowType == nil { log.Warn().Msgf( "missing borrow type for capability with target %s#%d", capabilityAddress.HexWithPrefix(), @@ -88,19 +88,27 @@ func (m *FixAuthorizationsMigration) Migrate( ) } - borrowReferenceType, ok := borrowType.(*interpreter.ReferenceStaticType) + oldBorrowReferenceType, ok := oldBorrowType.(*interpreter.ReferenceStaticType) if !ok { log.Warn().Msgf( "invalid non-reference borrow type for capability with target %s#%d: %s", capabilityAddress.HexWithPrefix(), capabilityID, - borrowType, + oldBorrowType, ) return nil, nil } - borrowReferenceType.Authorization = newAuthorization - value.BorrowType = borrowReferenceType + newBorrowType := interpreter.NewReferenceStaticType( + nil, + newAuthorization, + oldBorrowReferenceType.ReferencedType, + ) + newCapabilityValue := interpreter.NewUnmeteredCapabilityValue( + interpreter.UInt64Value(capabilityID), + interpreter.AddressValue(capabilityAddress), + newBorrowType, + ) m.Reporter.MigratedCapability( storageKey, @@ -109,7 +117,7 @@ func (m *FixAuthorizationsMigration) Migrate( newAuthorization, ) - return value, nil + return newCapabilityValue, nil case *interpreter.StorageCapabilityControllerValue: // The capability controller's address is implicitly @@ -126,7 +134,18 @@ func (m *FixAuthorizationsMigration) Migrate( return nil, nil } - value.BorrowType.Authorization = newAuthorization + oldBorrowReferenceType := value.BorrowType + + newBorrowType := interpreter.NewReferenceStaticType( + nil, + newAuthorization, + oldBorrowReferenceType.ReferencedType, + ) + newStorageCapabilityControllerValue := interpreter.NewUnmeteredStorageCapabilityControllerValue( + newBorrowType, + interpreter.UInt64Value(capabilityID), + value.TargetPath, + ) m.Reporter.MigratedCapabilityController( storageKey, @@ -134,7 +153,7 @@ func (m *FixAuthorizationsMigration) Migrate( newAuthorization, ) - return value, nil + return newStorageCapabilityControllerValue, nil } return nil, nil From 5edc9d96934d5320d65d987e5138db2424334baa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Mon, 9 Sep 2024 16:10:23 -0700 Subject: [PATCH 48/72] improve tests, align with tests in generate-authorization-fixes util command --- .../fix_authorizations_migration.go | 12 +- .../fix_authorizations_migration_test.go | 136 +++++++++++++----- 2 files changed, 108 insertions(+), 40 deletions(-) diff --git a/cmd/util/ledger/migrations/fix_authorizations_migration.go b/cmd/util/ledger/migrations/fix_authorizations_migration.go index dcc1a08b1aa..948199c623d 100644 --- a/cmd/util/ledger/migrations/fix_authorizations_migration.go +++ b/cmd/util/ledger/migrations/fix_authorizations_migration.go @@ -19,7 +19,7 @@ import ( "github.com/onflow/flow-go/model/flow" ) -type AccountCapabilityControllerID struct { +type AccountCapabilityID struct { Address common.Address CapabilityID uint64 } @@ -42,7 +42,7 @@ type FixAuthorizationsMigrationReporter interface { type FixAuthorizationsMigration struct { Reporter FixAuthorizationsMigrationReporter - NewAuthorizations map[AccountCapabilityControllerID]interpreter.Authorization + NewAuthorizations map[AccountCapabilityID]interpreter.Authorization } var _ migrations.ValueMigration = &FixAuthorizationsMigration{} @@ -70,7 +70,7 @@ func (m *FixAuthorizationsMigration) Migrate( capabilityAddress := common.Address(value.Address()) capabilityID := uint64(value.ID) - newAuthorization := m.NewAuthorizations[AccountCapabilityControllerID{ + newAuthorization := m.NewAuthorizations[AccountCapabilityID{ Address: capabilityAddress, CapabilityID: capabilityID, }] @@ -125,7 +125,7 @@ func (m *FixAuthorizationsMigration) Migrate( capabilityAddress := storageKey.Address capabilityID := uint64(value.CapabilityID) - newAuthorization := m.NewAuthorizations[AccountCapabilityControllerID{ + newAuthorization := m.NewAuthorizations[AccountCapabilityID{ Address: capabilityAddress, CapabilityID: capabilityID, }] @@ -225,7 +225,7 @@ func NewFixAuthorizationsMigration( rwf reporters.ReportWriterFactory, errorMessageHandler *errorMessageHandler, programs map[runtime.Location]*interpreter.Program, - newAuthorizations map[AccountCapabilityControllerID]interpreter.Authorization, + newAuthorizations map[AccountCapabilityID]interpreter.Authorization, opts FixAuthorizationsMigrationOptions, ) *CadenceBaseMigration { var diffReporter reporters.ReportWriter @@ -411,7 +411,7 @@ func (e capabilityAuthorizationFixedEntry) MarshalJSON() ([]byte, error) { func NewFixAuthorizationsMigrations( log zerolog.Logger, rwf reporters.ReportWriterFactory, - newAuthorizations map[AccountCapabilityControllerID]interpreter.Authorization, + newAuthorizations map[AccountCapabilityID]interpreter.Authorization, opts FixAuthorizationsMigrationOptions, ) []NamedMigration { diff --git a/cmd/util/ledger/migrations/fix_authorizations_migration_test.go b/cmd/util/ledger/migrations/fix_authorizations_migration_test.go index 9f0ceac8318..ac438c1cb67 100644 --- a/cmd/util/ledger/migrations/fix_authorizations_migration_test.go +++ b/cmd/util/ledger/migrations/fix_authorizations_migration_test.go @@ -1,10 +1,14 @@ package migrations import ( + "fmt" "testing" + "github.com/onflow/cadence" + jsoncdc "github.com/onflow/cadence/encoding/json" "github.com/onflow/cadence/runtime/common" "github.com/onflow/cadence/runtime/interpreter" + "github.com/onflow/cadence/runtime/sema" "github.com/rs/zerolog" "github.com/stretchr/testify/require" @@ -12,6 +16,20 @@ import ( "github.com/onflow/flow-go/model/flow" ) +func newEntitlementSetAuthorizationFromTypeIDs( + typeIDs []common.TypeID, + setKind sema.EntitlementSetKind, +) interpreter.EntitlementSetAuthorization { + return interpreter.NewEntitlementSetAuthorization( + nil, + func() []common.TypeID { + return typeIDs + }, + len(typeIDs), + setKind, + ) +} + func TestFixAuthorizationsMigration(t *testing.T) { t.Parallel() @@ -44,42 +62,79 @@ func TestFixAuthorizationsMigration(t *testing.T) { err = mr.Commit(expectedWriteAddresses, log) require.NoError(t, err) - tx := flow.NewTransactionBody(). + const contractCode = ` + access(all) contract Test { + access(all) entitlement E1 + access(all) entitlement E2 + + access(all) struct S { + access(E1) fun f1() {} + access(E2) fun f2() {} + access(all) fun f3() {} + } + } + ` + + deployTX := flow.NewTransactionBody(). SetScript([]byte(` - transaction { - prepare(signer: auth(Storage, Capabilities) &Account) { - // Capability 1 was a public, unauthorized capability. - // It should lose its entitlement - let cap1 = signer.capabilities.storage.issue(/storage/ints) - signer.capabilities.publish(cap1, at: /public/ints) - - // Capability 2 was a public, unauthorized capability, stored nested in storage. - // It should lose its entitlement - let cap2 = signer.capabilities.storage.issue(/storage/ints) - signer.storage.save([cap2], to: /storage/caps2) - - // Capability 3 was a private, authorized capability, stored nested in storage. - // It should keep its entitlement - let cap3 = signer.capabilities.storage.issue(/storage/ints) - signer.storage.save([cap3], to: /storage/caps3) - - // Capability 4 was a capability with unavailable accessible members, stored nested in storage. - // It should keep its entitlement - let cap4 = signer.capabilities.storage.issue(/storage/ints) - signer.storage.save([cap4], to: /storage/caps4) + transaction(code: String) { + prepare(signer: auth(Contracts) &Account) { + signer.contracts.add(name: "Test", code: code.utf8) } } `)). + AddAuthorizer(address). + AddArgument(jsoncdc.MustEncode(cadence.String(contractCode))) + + runDeployTx := NewTransactionBasedMigration( + deployTX, + chainID, + log, + expectedWriteAddresses, + ) + err = runDeployTx(registersByAccount) + require.NoError(t, err) + + setupTx := flow.NewTransactionBody(). + SetScript([]byte(fmt.Sprintf(` + import Test from %s + + transaction { + prepare(signer: auth(Storage, Capabilities) &Account) { + // Capability 1 was a public, unauthorized capability. + // It should lose its entitlement + let cap1 = signer.capabilities.storage.issue(/storage/s) + signer.capabilities.publish(cap1, at: /public/s) + + // Capability 2 was a public, unauthorized capability, stored nested in storage. + // It should lose its entitlement + let cap2 = signer.capabilities.storage.issue(/storage/s) + signer.storage.save([cap2], to: /storage/caps2) + + // Capability 3 was a private, authorized capability, stored nested in storage. + // It should keep its entitlement + let cap3 = signer.capabilities.storage.issue(/storage/s) + signer.storage.save([cap3], to: /storage/caps3) + + // Capability 4 was a capability with unavailable accessible members, stored nested in storage. + // It should keep its entitlement + let cap4 = signer.capabilities.storage.issue(/storage/s) + signer.storage.save([cap4], to: /storage/caps4) + } + } + `, + address.HexWithPrefix(), + ))). AddAuthorizer(address) - setupTx := NewTransactionBasedMigration( - tx, + runSetupTx := NewTransactionBasedMigration( + setupTx, chainID, log, expectedWriteAddresses, ) - err = setupTx(registersByAccount) + err = runSetupTx(registersByAccount) require.NoError(t, err) rwf := &testReportWriterFactory{} @@ -89,15 +144,28 @@ func TestFixAuthorizationsMigration(t *testing.T) { NWorker: nWorker, } - fixes := map[AccountCapabilityControllerID]interpreter.Authorization{ - { + testContractLocation := common.AddressLocation{ + Address: common.Address(address), + Name: "Test", + } + e1TypeID := testContractLocation.TypeID(nil, "Test.E1") + + fixedAuthorization := newEntitlementSetAuthorizationFromTypeIDs( + []common.TypeID{ + e1TypeID, + }, + sema.Conjunction, + ) + + fixes := map[AccountCapabilityID]interpreter.Authorization{ + AccountCapabilityID{ Address: common.Address(address), CapabilityID: 1, - }: interpreter.UnauthorizedAccess, - { + }: fixedAuthorization, + AccountCapabilityID{ Address: common.Address(address), CapabilityID: 2, - }: interpreter.UnauthorizedAccess, + }: fixedAuthorization, } migrations := NewFixAuthorizationsMigrations( @@ -134,7 +202,7 @@ func TestFixAuthorizationsMigration(t *testing.T) { Address: common.Address(address), }, CapabilityID: 1, - NewAuthorization: interpreter.UnauthorizedAccess, + NewAuthorization: fixedAuthorization, }, capabilityControllerAuthorizationFixedEntry{ StorageKey: interpreter.StorageKey{ @@ -142,7 +210,7 @@ func TestFixAuthorizationsMigration(t *testing.T) { Address: common.Address(address), }, CapabilityID: 2, - NewAuthorization: interpreter.UnauthorizedAccess, + NewAuthorization: fixedAuthorization, }, capabilityAuthorizationFixedEntry{ StorageKey: interpreter.StorageKey{ @@ -151,7 +219,7 @@ func TestFixAuthorizationsMigration(t *testing.T) { }, CapabilityAddress: common.Address(address), CapabilityID: 1, - NewAuthorization: interpreter.UnauthorizedAccess, + NewAuthorization: fixedAuthorization, }, capabilityAuthorizationFixedEntry{ StorageKey: interpreter.StorageKey{ @@ -160,7 +228,7 @@ func TestFixAuthorizationsMigration(t *testing.T) { }, CapabilityAddress: common.Address(address), CapabilityID: 2, - NewAuthorization: interpreter.UnauthorizedAccess, + NewAuthorization: fixedAuthorization, }, }, entries, From 3d32e4aade18f0852e0a9cb880fc9e3fdf6ee290 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Mon, 9 Sep 2024 16:25:55 -0700 Subject: [PATCH 49/72] validate migrated state, not just report --- .../fix_authorizations_migration_test.go | 52 +++++++++++++++++-- 1 file changed, 48 insertions(+), 4 deletions(-) diff --git a/cmd/util/ledger/migrations/fix_authorizations_migration_test.go b/cmd/util/ledger/migrations/fix_authorizations_migration_test.go index ac438c1cb67..2e8b66fa9b3 100644 --- a/cmd/util/ledger/migrations/fix_authorizations_migration_test.go +++ b/cmd/util/ledger/migrations/fix_authorizations_migration_test.go @@ -101,24 +101,30 @@ func TestFixAuthorizationsMigration(t *testing.T) { transaction { prepare(signer: auth(Storage, Capabilities) &Account) { + signer.storage.save(Test.S(), to: /storage/s) + // Capability 1 was a public, unauthorized capability. - // It should lose its entitlement + // It should lose entitlement E2 let cap1 = signer.capabilities.storage.issue(/storage/s) + assert(cap1.borrow() != nil) signer.capabilities.publish(cap1, at: /public/s) // Capability 2 was a public, unauthorized capability, stored nested in storage. - // It should lose its entitlement + // It should lose entitlement E2 let cap2 = signer.capabilities.storage.issue(/storage/s) + assert(cap2.borrow() != nil) signer.storage.save([cap2], to: /storage/caps2) // Capability 3 was a private, authorized capability, stored nested in storage. - // It should keep its entitlement + // It should keep entitlement E2 let cap3 = signer.capabilities.storage.issue(/storage/s) + assert(cap3.borrow() != nil) signer.storage.save([cap3], to: /storage/caps3) // Capability 4 was a capability with unavailable accessible members, stored nested in storage. - // It should keep its entitlement + // It should keep entitlement E2 let cap4 = signer.capabilities.storage.issue(/storage/s) + assert(cap4.borrow() != nil) signer.storage.save([cap4], to: /storage/caps4) } } @@ -233,4 +239,42 @@ func TestFixAuthorizationsMigration(t *testing.T) { }, entries, ) + + // Check account + + _, err = runScript( + chainID, + registersByAccount, + fmt.Sprintf( + //language=Cadence + ` + import Test from %s + + access(all) + fun main() { + let account = getAuthAccount(%[1]s) + // NOTE: capability can NOT be borrowed with E2 anymore + assert(account.capabilities.borrow(/public/s) == nil) + assert(account.capabilities.borrow(/public/s) != nil) + + let caps2 = account.storage.copy<[Capability]>(from: /storage/caps2)! + // NOTE: capability can NOT be borrowed with E2 anymore + assert(caps2[0].borrow() == nil) + assert(caps2[0].borrow() != nil) + + let caps3 = account.storage.copy<[Capability]>(from: /storage/caps3)! + // NOTE: capability can still be borrowed with E2 + assert(caps3[0].borrow() != nil) + assert(caps3[0].borrow() != nil) + + let caps4 = account.storage.copy<[Capability]>(from: /storage/caps4)! + // NOTE: capability can still be borrowed with E2 + assert(caps4[0].borrow() != nil) + assert(caps4[0].borrow() != nil) + } + `, + address.HexWithPrefix(), + ), + ) + require.NoError(t, err) } From 487b9b1c5ac3e5a125f61c64c914ad9dc3764339 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Mon, 9 Sep 2024 16:53:55 -0700 Subject: [PATCH 50/72] load reports in parallel --- .../cmd/generate-authorization-fixes/cmd.go | 147 +++++++++++------- 1 file changed, 94 insertions(+), 53 deletions(-) diff --git a/cmd/util/cmd/generate-authorization-fixes/cmd.go b/cmd/util/cmd/generate-authorization-fixes/cmd.go index 7e3d5448e97..44cb19d426b 100644 --- a/cmd/util/cmd/generate-authorization-fixes/cmd.go +++ b/cmd/util/cmd/generate-authorization-fixes/cmd.go @@ -130,6 +130,16 @@ func run(*cobra.Command, []string) { } addressFilter[common.Address(addr)] = struct{}{} } + + addresses := make([]string, 0, len(addressFilter)) + for addr := range addressFilter { + addresses = append(addresses, addr.HexWithPrefix()) + } + log.Info().Msgf( + "Only generating fixes for %d accounts: %s", + len(addressFilter), + addresses, + ) } if flagPayloads == "" && flagState == "" { @@ -150,53 +160,52 @@ func run(*cobra.Command, []string) { // Validate chain ID _ = chainID.Chain() - var payloads []*ledger.Payload - var err error - - // Read public link report - - publicLinkReportFile, err := os.Open(flagPublicLinkReport) - if err != nil { - log.Fatal().Err(err).Msgf("can't open link report: %s", flagPublicLinkReport) - } - defer publicLinkReportFile.Close() + publicLinkReportChan := make(chan PublicLinkReport, 1) + go func() { + publicLinkReportChan <- readPublicLinkReport(addressFilter) + }() - var publicLinkReportReader io.Reader = publicLinkReportFile - if isGzip(publicLinkReportFile) { - publicLinkReportReader, err = gzip.NewReader(publicLinkReportFile) - if err != nil { - log.Fatal().Err(err).Msgf("failed to create gzip reader for %s", flagPublicLinkReport) - } - } + publicLinkMigrationReportChan := make(chan PublicLinkMigrationReport, 1) + go func() { + publicLinkMigrationReportChan <- readLinkMigrationReport(addressFilter) + }() - publicLinkReport, err := ReadPublicLinkReport(publicLinkReportReader, addressFilter) - if err != nil { - log.Fatal().Err(err).Msgf("failed to read public link report %s", flagPublicLinkReport) - } + publicLinkReport := <-publicLinkReportChan + publicLinkMigrationReport := <-publicLinkMigrationReportChan - // Read link migration report + registersByAccount := loadRegistersByAccount() - linkMigrationReportFile, err := os.Open(flagLinkMigrationReport) + mr, err := migrations.NewInterpreterMigrationRuntime( + registersByAccount, + chainID, + migrations.InterpreterMigrationRuntimeConfig{}, + ) if err != nil { - log.Fatal().Err(err).Msgf("can't open link migration report: %s", flagLinkMigrationReport) + log.Fatal().Err(err) } - defer linkMigrationReportFile.Close() - var linkMigrationReportReader io.Reader = linkMigrationReportFile - if isGzip(linkMigrationReportFile) { - linkMigrationReportReader, err = gzip.NewReader(linkMigrationReportFile) - if err != nil { - log.Fatal().Err(err).Msgf("failed to create gzip reader for %s", flagLinkMigrationReport) - } - } + checkContracts( + registersByAccount, + mr, + reporter, + ) - publicLinkMigrationReport, err := ReadPublicLinkMigrationReport(linkMigrationReportReader, addressFilter) - if err != nil { - log.Fatal().Err(err).Msgf("failed to read public link report: %s", flagLinkMigrationReport) + authorizationFixGenerator := &AuthorizationFixGenerator{ + registersByAccount: registersByAccount, + mr: mr, + publicLinkReport: publicLinkReport, + publicLinkMigrationReport: publicLinkMigrationReport, + reporter: reporter, } + authorizationFixGenerator.generateFixesForAllAccounts() +} +func loadRegistersByAccount() *registers.ByAccount { // Read payloads from payload file or checkpoint file + var payloads []*ledger.Payload + var err error + if flagPayloads != "" { log.Info().Msgf("Reading payloads from %s", flagPayloads) @@ -226,29 +235,61 @@ func run(*cobra.Command, []string) { registersByAccount.AccountCount(), ) - mr, err := migrations.NewInterpreterMigrationRuntime( - registersByAccount, - chainID, - migrations.InterpreterMigrationRuntimeConfig{}, - ) + return registersByAccount +} + +func readPublicLinkReport(addressFilter map[common.Address]struct{}) PublicLinkReport { + // Read public link report + + publicLinkReportFile, err := os.Open(flagPublicLinkReport) if err != nil { - log.Fatal().Err(err) + log.Fatal().Err(err).Msgf("can't open link report: %s", flagPublicLinkReport) } + defer publicLinkReportFile.Close() - checkContracts( - registersByAccount, - mr, - reporter, - ) + var publicLinkReportReader io.Reader = publicLinkReportFile + if isGzip(publicLinkReportFile) { + publicLinkReportReader, err = gzip.NewReader(publicLinkReportFile) + if err != nil { + log.Fatal().Err(err).Msgf("failed to create gzip reader for %s", flagPublicLinkReport) + } + } - authorizationFixGenerator := &AuthorizationFixGenerator{ - registersByAccount: registersByAccount, - mr: mr, - publicLinkReport: publicLinkReport, - publicLinkMigrationReport: publicLinkMigrationReport, - reporter: reporter, + log.Info().Msgf("Reading public link report from %s ...", flagPublicLinkReport) + + publicLinkReport, err := ReadPublicLinkReport(publicLinkReportReader, addressFilter) + if err != nil { + log.Fatal().Err(err).Msgf("failed to read public link report %s", flagPublicLinkReport) } - authorizationFixGenerator.generateFixesForAllAccounts() + + return publicLinkReport +} + +func readLinkMigrationReport(addressFilter map[common.Address]struct{}) PublicLinkMigrationReport { + // Read link migration report + + linkMigrationReportFile, err := os.Open(flagLinkMigrationReport) + if err != nil { + log.Fatal().Err(err).Msgf("can't open link migration report: %s", flagLinkMigrationReport) + } + defer linkMigrationReportFile.Close() + + var linkMigrationReportReader io.Reader = linkMigrationReportFile + if isGzip(linkMigrationReportFile) { + linkMigrationReportReader, err = gzip.NewReader(linkMigrationReportFile) + if err != nil { + log.Fatal().Err(err).Msgf("failed to create gzip reader for %s", flagLinkMigrationReport) + } + } + + log.Info().Msgf("Reading link migration report from %s ...", flagLinkMigrationReport) + + publicLinkMigrationReport, err := ReadPublicLinkMigrationReport(linkMigrationReportReader, addressFilter) + if err != nil { + log.Fatal().Err(err).Msgf("failed to read public link report: %s", flagLinkMigrationReport) + } + + return publicLinkMigrationReport } func checkContracts( From a335dcad74e80d942d9553e7aa1e19ea212c7a40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Mon, 9 Sep 2024 17:07:14 -0700 Subject: [PATCH 51/72] remove unnecessary deletion of resolved members --- cmd/util/cmd/generate-authorization-fixes/entitlements.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/cmd/util/cmd/generate-authorization-fixes/entitlements.go b/cmd/util/cmd/generate-authorization-fixes/entitlements.go index 5f4b20bc693..137a0a15d60 100644 --- a/cmd/util/cmd/generate-authorization-fixes/entitlements.go +++ b/cmd/util/cmd/generate-authorization-fixes/entitlements.go @@ -64,8 +64,6 @@ func findMinimalAuthorization( panic(fmt.Errorf("unsupported set kind: %v", access.SetKind)) } - delete(neededMembers, memberName) - case *sema.EntitlementMapAccess: unresolvedMembers[memberName] = fmt.Errorf( "member requires entitlement map access: %s", @@ -73,10 +71,7 @@ func findMinimalAuthorization( ) case sema.PrimitiveAccess: - if access == sema.PrimitiveAccess(ast.AccessAll) { - // member is always accessible - delete(neededMembers, memberName) - } else { + if access != sema.PrimitiveAccess(ast.AccessAll) { unresolvedMembers[memberName] = fmt.Errorf( "member is inaccessible (%s)", access.QualifiedKeyword(), From a45bf8d3f10980f26981b2a6eb7080d20d830752 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Mon, 9 Sep 2024 17:08:16 -0700 Subject: [PATCH 52/72] test access with entitlement mapping --- .../entitlements.go | 4 ++-- .../entitlements_test.go | 18 ++++++++++++++---- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/cmd/util/cmd/generate-authorization-fixes/entitlements.go b/cmd/util/cmd/generate-authorization-fixes/entitlements.go index 137a0a15d60..c80f34b760f 100644 --- a/cmd/util/cmd/generate-authorization-fixes/entitlements.go +++ b/cmd/util/cmd/generate-authorization-fixes/entitlements.go @@ -66,14 +66,14 @@ func findMinimalAuthorization( case *sema.EntitlementMapAccess: unresolvedMembers[memberName] = fmt.Errorf( - "member requires entitlement map access: %s", + "member has entitlement map access: %s", access.QualifiedKeyword(), ) case sema.PrimitiveAccess: if access != sema.PrimitiveAccess(ast.AccessAll) { unresolvedMembers[memberName] = fmt.Errorf( - "member is inaccessible (%s)", + "member is inaccessible: %s", access.QualifiedKeyword(), ) } diff --git a/cmd/util/cmd/generate-authorization-fixes/entitlements_test.go b/cmd/util/cmd/generate-authorization-fixes/entitlements_test.go index a3c6a9f8c80..362f1ec61c1 100644 --- a/cmd/util/cmd/generate-authorization-fixes/entitlements_test.go +++ b/cmd/util/cmd/generate-authorization-fixes/entitlements_test.go @@ -39,16 +39,24 @@ func TestFindMinimalAuthorization(t *testing.T) { entitlement E2 entitlement E3 + entitlement mapping M {} + struct S { access(all) fun accessAll() {} access(self) fun accessSelf() {} access(contract) fun accessContract() {} access(account) fun accessAccount() {} + access(mapping M) let accessMapping: auth(mapping M) &Int + access(E1) fun accessE1() {} access(E2) fun accessE2() {} access(E1, E2) fun accessE1AndE2() {} access(E1 | E2) fun accessE1OrE2() {} + + init() { + self.accessMapping = &0 + } } `) require.NoError(t, err) @@ -58,7 +66,7 @@ func TestFindMinimalAuthorization(t *testing.T) { e1 := RequireGlobalType(t, checker.Elaboration, "E1").(*sema.EntitlementType) e2 := RequireGlobalType(t, checker.Elaboration, "E2").(*sema.EntitlementType) - t.Run("accessAll, accessSelf, accessContract, accessAccount", func(t *testing.T) { + t.Run("accessAll, accessSelf, accessContract, accessAccount, accessMapping", func(t *testing.T) { t.Parallel() authorization, unresolved := findMinimalAuthorization( @@ -68,6 +76,7 @@ func TestFindMinimalAuthorization(t *testing.T) { "accessSelf": {}, "accessContract": {}, "accessAccount": {}, + "accessMapping": {}, "undefined": {}, }, ) @@ -77,9 +86,10 @@ func TestFindMinimalAuthorization(t *testing.T) { ) assert.Equal(t, map[string]error{ - "accessSelf": errors.New("member is inaccessible (access(self))"), - "accessContract": errors.New("member is inaccessible (access(contract))"), - "accessAccount": errors.New("member is inaccessible (access(account))"), + "accessSelf": errors.New("member is inaccessible: access(self)"), + "accessContract": errors.New("member is inaccessible: access(contract)"), + "accessAccount": errors.New("member is inaccessible: access(account)"), + "accessMapping": errors.New("member has entitlement map access: access(mapping M)"), "undefined": errors.New("member does not exist"), }, unresolved, From 6b88034e2a64369ac0b16ebc5ebe0ee4efa49f94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Mon, 9 Sep 2024 17:12:12 -0700 Subject: [PATCH 53/72] log read report entries --- cmd/util/cmd/generate-authorization-fixes/cmd.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cmd/util/cmd/generate-authorization-fixes/cmd.go b/cmd/util/cmd/generate-authorization-fixes/cmd.go index 44cb19d426b..d2b07020a88 100644 --- a/cmd/util/cmd/generate-authorization-fixes/cmd.go +++ b/cmd/util/cmd/generate-authorization-fixes/cmd.go @@ -262,6 +262,8 @@ func readPublicLinkReport(addressFilter map[common.Address]struct{}) PublicLinkR log.Fatal().Err(err).Msgf("failed to read public link report %s", flagPublicLinkReport) } + log.Info().Msgf("Read %d public link entries", len(publicLinkReport)) + return publicLinkReport } @@ -289,6 +291,8 @@ func readLinkMigrationReport(addressFilter map[common.Address]struct{}) PublicLi log.Fatal().Err(err).Msgf("failed to read public link report: %s", flagLinkMigrationReport) } + log.Info().Msgf("Read %d public link migration entries", len(publicLinkMigrationReport)) + return publicLinkMigrationReport } From cd491e4a044dd964c3b3eb0cb9c97119f939c1b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Tue, 10 Sep 2024 16:22:01 -0700 Subject: [PATCH 54/72] only report a fix when the calculated authorization is different. ignore Insert|Mutate|Remove changes --- .../cmd/generate-authorization-fixes/cmd.go | 46 +++++++++++++++++++ .../fix_authorizations_migration_test.go | 14 ------ 2 files changed, 46 insertions(+), 14 deletions(-) diff --git a/cmd/util/cmd/generate-authorization-fixes/cmd.go b/cmd/util/cmd/generate-authorization-fixes/cmd.go index d2b07020a88..a945961b9df 100644 --- a/cmd/util/cmd/generate-authorization-fixes/cmd.go +++ b/cmd/util/cmd/generate-authorization-fixes/cmd.go @@ -460,7 +460,39 @@ func (g *AuthorizationFixGenerator) generateFixesForAccount(address common.Addre } } +func newEntitlementSetAuthorizationFromTypeIDs( + typeIDs []common.TypeID, + setKind sema.EntitlementSetKind, +) interpreter.EntitlementSetAuthorization { + return interpreter.NewEntitlementSetAuthorization( + nil, + func() []common.TypeID { + return typeIDs + }, + len(typeIDs), + setKind, + ) +} + +var insertRemoveAuthorization = newEntitlementSetAuthorizationFromTypeIDs( + []common.TypeID{ + sema.InsertType.ID(), + sema.RemoveType.ID(), + }, + sema.Conjunction, +) + +var insertMutateRemoveAuthorization = newEntitlementSetAuthorizationFromTypeIDs( + []common.TypeID{ + sema.InsertType.ID(), + sema.MutateType.ID(), + sema.RemoveType.ID(), + }, + sema.Conjunction, +) + func (g *AuthorizationFixGenerator) maybeGenerateFixForCapabilityController( + inter *interpreter.Interpreter, capabilityAddress common.Address, capabilityID uint64, borrowType *interpreter.ReferenceStaticType, @@ -554,6 +586,20 @@ func (g *AuthorizationFixGenerator) maybeGenerateFixForCapabilityController( // we should not leave the capability controller vulnerable } + // Only fix the authorization if it is different from the old one. + // If the old authorization was `Insert, Mutate, Remove`, + // the calculated minimal authorization is `Insert, Remove`, + // but we ignore the difference, and keep the Mutate entitlement. + + oldAuthorization := borrowType.Authorization + if newAuthorization.Equal(oldAuthorization) || + (oldAuthorization.Equal(insertMutateRemoveAuthorization) && + newAuthorization.Equal(insertRemoveAuthorization)) { + + // Nothing to fix + return + } + g.reporter.Write(fixEntitlementsEntry{ AccountCapabilityID: AccountCapabilityID{ Address: capabilityAddress, diff --git a/cmd/util/ledger/migrations/fix_authorizations_migration_test.go b/cmd/util/ledger/migrations/fix_authorizations_migration_test.go index 2e8b66fa9b3..8ad439a2039 100644 --- a/cmd/util/ledger/migrations/fix_authorizations_migration_test.go +++ b/cmd/util/ledger/migrations/fix_authorizations_migration_test.go @@ -16,20 +16,6 @@ import ( "github.com/onflow/flow-go/model/flow" ) -func newEntitlementSetAuthorizationFromTypeIDs( - typeIDs []common.TypeID, - setKind sema.EntitlementSetKind, -) interpreter.EntitlementSetAuthorization { - return interpreter.NewEntitlementSetAuthorization( - nil, - func() []common.TypeID { - return typeIDs - }, - len(typeIDs), - setKind, - ) -} - func TestFixAuthorizationsMigration(t *testing.T) { t.Parallel() From 78b451561e650d07928bd5b067f32d8e53e2805f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Tue, 10 Sep 2024 16:23:14 -0700 Subject: [PATCH 55/72] include additional context in fixes and logs --- .../cmd/generate-authorization-fixes/cmd.go | 82 ++++++++++++++----- 1 file changed, 62 insertions(+), 20 deletions(-) diff --git a/cmd/util/cmd/generate-authorization-fixes/cmd.go b/cmd/util/cmd/generate-authorization-fixes/cmd.go index a945961b9df..8e3ef6fd70c 100644 --- a/cmd/util/cmd/generate-authorization-fixes/cmd.go +++ b/cmd/util/cmd/generate-authorization-fixes/cmd.go @@ -342,25 +342,35 @@ func jsonEncodeAuthorization(authorization interpreter.Authorization) string { type fixEntitlementsEntry struct { AccountCapabilityID - NewAuthorization interpreter.Authorization - UnresolvedMembers map[string]error + ReferencedType interpreter.StaticType + OldAuthorization interpreter.Authorization + NewAuthorization interpreter.Authorization + OldAccessibleMembers []string + NewAccessibleMembers []string + UnresolvedMembers map[string]error } var _ json.Marshaler = fixEntitlementsEntry{} func (e fixEntitlementsEntry) MarshalJSON() ([]byte, error) { return json.Marshal(struct { - Kind string `json:"kind"` - CapabilityAddress string `json:"capability_address"` - CapabilityID uint64 `json:"capability_id"` - NewAuthorization string `json:"new_authorization"` - UnresolvedMembers map[string]string + CapabilityAddress string `json:"capability_address"` + CapabilityID uint64 `json:"capability_id"` + ReferencedType string `json:"referenced_type"` + OldAuthorization string `json:"old_authorization"` + NewAuthorization string `json:"new_authorization"` + OldAccessibleMembers []string `json:"old_members"` + NewAccessibleMembers []string `json:"new_members"` + UnresolvedMembers map[string]string `json:"unresolved_members,omitempty"` }{ - Kind: "fix-entitlements", - CapabilityAddress: e.Address.String(), - CapabilityID: e.CapabilityID, - NewAuthorization: jsonEncodeAuthorization(e.NewAuthorization), - UnresolvedMembers: jsonEncodeMemberErrorMap(e.UnresolvedMembers), + CapabilityAddress: e.Address.String(), + CapabilityID: e.CapabilityID, + ReferencedType: string(e.ReferencedType.ID()), + OldAuthorization: jsonEncodeAuthorization(e.OldAuthorization), + NewAuthorization: jsonEncodeAuthorization(e.NewAuthorization), + OldAccessibleMembers: e.OldAccessibleMembers, + NewAccessibleMembers: e.NewAccessibleMembers, + UnresolvedMembers: jsonEncodeMemberErrorMap(e.UnresolvedMembers), }) } @@ -555,19 +565,31 @@ func (g *AuthorizationFixGenerator) maybeGenerateFixForCapabilityController( return } + oldAccessibleMemberSet := make(map[string]struct{}) + for _, memberName := range oldAccessibleMembers { + oldAccessibleMemberSet[memberName] = struct{}{} + } + + newAccessibleMemberSet := make(map[string]struct{}) + for _, memberName := range newAccessibleMembers { + newAccessibleMemberSet[memberName] = struct{}{} + } + + membersAdded, membersRemoved := sortedDiffStringSets( + oldAccessibleMemberSet, + newAccessibleMemberSet, + ) + log.Info().Msgf( - "member mismatch for capability controller %d in account %s: expected %v, got %v", + "member mismatch for capability controller %d in account %s: expected %v, got %v (added: %v, removed: %v)", capabilityID, capabilityAddress.HexWithPrefix(), oldAccessibleMembers, newAccessibleMembers, + membersAdded, + membersRemoved, ) - oldAccessibleMemberSet := make(map[string]struct{}) - for _, memberName := range oldAccessibleMembers { - oldAccessibleMemberSet[memberName] = struct{}{} - } - newAuthorization, unresolvedMembers := findMinimalAuthorization( semaBorrowType, oldAccessibleMemberSet, @@ -605,8 +627,12 @@ func (g *AuthorizationFixGenerator) maybeGenerateFixForCapabilityController( Address: capabilityAddress, CapabilityID: capabilityID, }, - NewAuthorization: newAuthorization, - UnresolvedMembers: unresolvedMembers, + ReferencedType: borrowType.ReferencedType, + OldAuthorization: oldAuthorization, + NewAuthorization: newAuthorization, + OldAccessibleMembers: oldAccessibleMembers, + NewAccessibleMembers: newAccessibleMembers, + UnresolvedMembers: unresolvedMembers, }) } @@ -666,3 +692,19 @@ func (g *AuthorizationFixGenerator) publicPathLinkInfo( func isGzip(file *os.File) bool { return strings.HasSuffix(file.Name(), ".gz") } + +func sortedDiffStringSets(a, b map[string]struct{}) (added, removed []string) { + for key := range a { + if _, ok := b[key]; !ok { + removed = append(removed, key) + } + } + for key := range b { + if _, ok := a[key]; !ok { + added = append(added, key) + } + } + sort.Strings(added) + sort.Strings(removed) + return +} From 9b473d8ce7e2ee25bac8141445aa0f6d63c0f4f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Tue, 10 Sep 2024 16:24:21 -0700 Subject: [PATCH 56/72] report contract checking and fixes to separate reports --- cmd/util/cmd/generate-authorization-fixes/cmd.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/cmd/util/cmd/generate-authorization-fixes/cmd.go b/cmd/util/cmd/generate-authorization-fixes/cmd.go index 8e3ef6fd70c..9fbc795cce6 100644 --- a/cmd/util/cmd/generate-authorization-fixes/cmd.go +++ b/cmd/util/cmd/generate-authorization-fixes/cmd.go @@ -153,9 +153,6 @@ func run(*cobra.Command, []string) { rwf := reporters.NewReportFileWriterFactory(flagOutputDirectory, log.Logger) - reporter := rwf.ReportWriter("entitlement-fixes") - defer reporter.Close() - chainID := flow.ChainID(flagChain) // Validate chain ID _ = chainID.Chain() @@ -183,19 +180,24 @@ func run(*cobra.Command, []string) { if err != nil { log.Fatal().Err(err) } + checkingReporter := rwf.ReportWriter("contract-checking") + defer checkingReporter.Close() checkContracts( registersByAccount, mr, - reporter, + checkingReporter, ) + fixReporter := rwf.ReportWriter("authorization-fixes") + defer fixReporter.Close() + authorizationFixGenerator := &AuthorizationFixGenerator{ registersByAccount: registersByAccount, mr: mr, publicLinkReport: publicLinkReport, publicLinkMigrationReport: publicLinkMigrationReport, - reporter: reporter, + reporter: fixReporter, } authorizationFixGenerator.generateFixesForAllAccounts() } From 3a19094d2b581ec97d6cf859f24b92d174bfab38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Tue, 10 Sep 2024 16:24:58 -0700 Subject: [PATCH 57/72] remove built-in handling, e.g. forEachAttachment is not always available --- cmd/util/cmd/generate-authorization-fixes/cmd.go | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/cmd/util/cmd/generate-authorization-fixes/cmd.go b/cmd/util/cmd/generate-authorization-fixes/cmd.go index 9fbc795cce6..2487e454678 100644 --- a/cmd/util/cmd/generate-authorization-fixes/cmd.go +++ b/cmd/util/cmd/generate-authorization-fixes/cmd.go @@ -538,16 +538,7 @@ func (g *AuthorizationFixGenerator) maybeGenerateFixForCapabilityController( return } - // Assume we already had access to public built-in functions. - // For example, forEachAttachment was added in Cadence 1.0, - // so we should not consider it as a new member. - - oldAccessibleMembers = append( - []string{"getType", "isInstance", "forEachAttachment"}, - oldAccessibleMembers..., - ) - - semaBorrowType, err := convertStaticToSemaType(g.mr.Interpreter, borrowType) + semaBorrowType, err := convertStaticToSemaType(inter, borrowType) if err != nil { log.Warn().Err(err).Msgf( "failed to get new accessible members for capability controller %d in account %s", From a91cc8a99024609631e10f6788ba1fef52858df4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Tue, 10 Sep 2024 16:25:59 -0700 Subject: [PATCH 58/72] parallelize state loading and fix calculation per account --- .../cmd/generate-authorization-fixes/cmd.go | 83 +++++++++++++++---- 1 file changed, 66 insertions(+), 17 deletions(-) diff --git a/cmd/util/cmd/generate-authorization-fixes/cmd.go b/cmd/util/cmd/generate-authorization-fixes/cmd.go index 2487e454678..d4de80f7679 100644 --- a/cmd/util/cmd/generate-authorization-fixes/cmd.go +++ b/cmd/util/cmd/generate-authorization-fixes/cmd.go @@ -8,12 +8,14 @@ import ( "os" "sort" "strings" + "sync" "github.com/onflow/cadence/runtime/common" "github.com/onflow/cadence/runtime/interpreter" "github.com/onflow/cadence/runtime/sema" "github.com/onflow/cadence/runtime/stdlib" "github.com/rs/zerolog/log" + "github.com/schollz/progressbar/v3" "github.com/spf13/cobra" "golang.org/x/exp/slices" @@ -167,25 +169,21 @@ func run(*cobra.Command, []string) { publicLinkMigrationReportChan <- readLinkMigrationReport(addressFilter) }() + registersByAccountChan := make(chan *registers.ByAccount, 1) + go func() { + registersByAccountChan <- loadRegistersByAccount() + }() + publicLinkReport := <-publicLinkReportChan publicLinkMigrationReport := <-publicLinkMigrationReportChan + registersByAccount := <-registersByAccountChan - registersByAccount := loadRegistersByAccount() - - mr, err := migrations.NewInterpreterMigrationRuntime( - registersByAccount, - chainID, - migrations.InterpreterMigrationRuntimeConfig{}, - ) - if err != nil { - log.Fatal().Err(err) - } checkingReporter := rwf.ReportWriter("contract-checking") defer checkingReporter.Close() checkContracts( registersByAccount, - mr, + chainID, checkingReporter, ) @@ -194,12 +192,16 @@ func run(*cobra.Command, []string) { authorizationFixGenerator := &AuthorizationFixGenerator{ registersByAccount: registersByAccount, - mr: mr, + chainID: chainID, publicLinkReport: publicLinkReport, publicLinkMigrationReport: publicLinkMigrationReport, reporter: fixReporter, } - authorizationFixGenerator.generateFixesForAllAccounts() + if len(addressFilter) > 0 { + authorizationFixGenerator.generateFixesForAccounts(addressFilter) + } else { + authorizationFixGenerator.generateFixesForAllAccounts() + } } func loadRegistersByAccount() *registers.ByAccount { @@ -300,7 +302,7 @@ func readLinkMigrationReport(addressFilter map[common.Address]struct{}) PublicLi func checkContracts( registersByAccount *registers.ByAccount, - mr *migrations.InterpreterMigrationRuntime, + chainID flow.ChainID, reporter reporters.ReportWriter, ) { contracts, err := migrations.GatherContractsFromRegisters(registersByAccount, log.Logger) @@ -317,6 +319,15 @@ func checkContracts( log.Info().Msg("Checking contracts ...") + mr, err := migrations.NewInterpreterMigrationRuntime( + registersByAccount, + chainID, + migrations.InterpreterMigrationRuntimeConfig{}, + ) + if err != nil { + log.Fatal().Err(err) + } + for _, contract := range contracts { migrations.CheckContract( contract, @@ -386,25 +397,62 @@ func jsonEncodeMemberErrorMap(m map[string]error) map[string]string { type AuthorizationFixGenerator struct { registersByAccount *registers.ByAccount - mr *migrations.InterpreterMigrationRuntime + chainID flow.ChainID publicLinkReport PublicLinkReport publicLinkMigrationReport PublicLinkMigrationReport reporter reporters.ReportWriter } func (g *AuthorizationFixGenerator) generateFixesForAllAccounts() { + var wg sync.WaitGroup + progress := progressbar.Default(int64(g.registersByAccount.AccountCount()), "Processing:") + err := g.registersByAccount.ForEachAccount(func(accountRegisters *registers.AccountRegisters) error { address := common.MustBytesToAddress([]byte(accountRegisters.Owner())) - g.generateFixesForAccount(address) + wg.Add(1) + go func(address common.Address) { + defer wg.Done() + g.generateFixesForAccount(address) + progress.Add(1) + }(address) return nil }) if err != nil { log.Fatal().Err(err) } + + wg.Wait() + progress.Finish() +} + +func (g *AuthorizationFixGenerator) generateFixesForAccounts(addresses map[common.Address]struct{}) { + var wg sync.WaitGroup + progress := progressbar.Default(int64(len(addresses)), "Processing:") + + for address := range addresses { + wg.Add(1) + go func(address common.Address) { + defer wg.Done() + g.generateFixesForAccount(address) + progress.Add(1) + }(address) + } + + wg.Wait() + progress.Finish() } func (g *AuthorizationFixGenerator) generateFixesForAccount(address common.Address) { - capabilityControllerStorage := g.mr.Storage.GetStorageMap( + mr, err := migrations.NewInterpreterMigrationRuntime( + g.registersByAccount, + g.chainID, + migrations.InterpreterMigrationRuntimeConfig{}, + ) + if err != nil { + log.Fatal().Err(err) + } + + capabilityControllerStorage := mr.Storage.GetStorageMap( address, stdlib.CapabilityControllerStorageDomain, false, @@ -440,6 +488,7 @@ func (g *AuthorizationFixGenerator) generateFixesForAccount(address common.Addre switch borrowType.Authorization.(type) { case interpreter.EntitlementSetAuthorization: g.maybeGenerateFixForCapabilityController( + mr.Interpreter, address, capabilityID, borrowType, From 0800bc11517ec37beae3c6ef3ba6fcf4c542d0e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Tue, 10 Sep 2024 17:18:01 -0700 Subject: [PATCH 59/72] fix tests --- .../generate-authorization-fixes/cmd_test.go | 91 ++++++++++++++----- .../fix_authorizations_migration_test.go | 14 +++ 2 files changed, 82 insertions(+), 23 deletions(-) diff --git a/cmd/util/cmd/generate-authorization-fixes/cmd_test.go b/cmd/util/cmd/generate-authorization-fixes/cmd_test.go index 06360bc6856..bf5877414d6 100644 --- a/cmd/util/cmd/generate-authorization-fixes/cmd_test.go +++ b/cmd/util/cmd/generate-authorization-fixes/cmd_test.go @@ -77,21 +77,7 @@ func (*testReportWriter) Close() { var _ reporters.ReportWriter = &testReportWriter{} -func newEntitlementSetAuthorizationFromTypeIDs( - typeIDs []common.TypeID, - setKind sema.EntitlementSetKind, -) interpreter.EntitlementSetAuthorization { - return interpreter.NewEntitlementSetAuthorization( - nil, - func() []common.TypeID { - return typeIDs - }, - len(typeIDs), - setKind, - ) -} - -func TestFixAuthorizationsMigrations(t *testing.T) { +func TestGenerateAuthorizationFixes(t *testing.T) { t.Parallel() const chainID = flow.Emulator @@ -182,6 +168,9 @@ func TestFixAuthorizationsMigrations(t *testing.T) { // It should keep its entitlement let cap4 = signer.capabilities.storage.issue(/storage/s) signer.storage.save([cap4], to: /storage/caps4) + + let cap5 = signer.capabilities.storage.issue(/storage/dict) + signer.capabilities.publish(cap5, at: /public/dict) } } `, @@ -198,19 +187,24 @@ func TestFixAuthorizationsMigrations(t *testing.T) { err = runSetupTx(registersByAccount) require.NoError(t, err) - mr2, err := migrations.NewInterpreterMigrationRuntime( - registersByAccount, - chainID, - migrations.InterpreterMigrationRuntimeConfig{}, - ) - require.NoError(t, err) - oldAccessibleMembers := []string{ "f1", "f3", + "forEachAttachment", + "getType", + "isInstance", "undefined", } + newAccessibleMembers := []string{ + "f1", + "f2", + "f3", + "forEachAttachment", + "getType", + "isInstance", + } + testContractLocation := common.AddressLocation{ Address: common.Address(address), Name: "Test", @@ -239,6 +233,24 @@ func TestFixAuthorizationsMigrations(t *testing.T) { BorrowType: borrowTypeID, AccessibleMembers: nil, }, + { + Address: common.Address(address), + Identifier: "dict", + }: { + BorrowType: "{String:String}", + AccessibleMembers: []string{ + "containsKey", + "forEachAttachment", + "forEachKey", + "getType", + "insert", + "isInstance", + "keys", + "length", + "remove", + "values", + }, + }, } publicLinkMigrationReport := PublicLinkMigrationReport{ @@ -254,13 +266,17 @@ func TestFixAuthorizationsMigrations(t *testing.T) { Address: common.Address(address), CapabilityID: 4, }: "s4", + { + Address: common.Address(address), + CapabilityID: 5, + }: "dict", } reporter := &testReportWriter{} generator := &AuthorizationFixGenerator{ registersByAccount: registersByAccount, - mr: mr2, + chainID: chainID, publicLinkReport: publicLinkReport, publicLinkMigrationReport: publicLinkMigrationReport, reporter: reporter, @@ -268,6 +284,7 @@ func TestFixAuthorizationsMigrations(t *testing.T) { generator.generateFixesForAllAccounts() e1TypeID := testContractLocation.TypeID(nil, "Test.E1") + e2TypeID := testContractLocation.TypeID(nil, "Test.E2") assert.Equal(t, []any{ @@ -276,12 +293,26 @@ func TestFixAuthorizationsMigrations(t *testing.T) { Address: common.Address(address), CapabilityID: 1, }, + ReferencedType: interpreter.NewCompositeStaticTypeComputeTypeID( + nil, + testContractLocation, + "Test.S", + ), + OldAuthorization: newEntitlementSetAuthorizationFromTypeIDs( + []common.TypeID{ + e1TypeID, + e2TypeID, + }, + sema.Conjunction, + ), NewAuthorization: newEntitlementSetAuthorizationFromTypeIDs( []common.TypeID{ e1TypeID, }, sema.Conjunction, ), + OldAccessibleMembers: oldAccessibleMembers, + NewAccessibleMembers: newAccessibleMembers, UnresolvedMembers: map[string]error{ "undefined": errors.New("member does not exist"), }, @@ -291,12 +322,26 @@ func TestFixAuthorizationsMigrations(t *testing.T) { Address: common.Address(address), CapabilityID: 2, }, + ReferencedType: interpreter.NewCompositeStaticTypeComputeTypeID( + nil, + testContractLocation, + "Test.S", + ), + OldAuthorization: newEntitlementSetAuthorizationFromTypeIDs( + []common.TypeID{ + e1TypeID, + e2TypeID, + }, + sema.Conjunction, + ), NewAuthorization: newEntitlementSetAuthorizationFromTypeIDs( []common.TypeID{ e1TypeID, }, sema.Conjunction, ), + OldAccessibleMembers: oldAccessibleMembers, + NewAccessibleMembers: newAccessibleMembers, UnresolvedMembers: map[string]error{ "undefined": errors.New("member does not exist"), }, diff --git a/cmd/util/ledger/migrations/fix_authorizations_migration_test.go b/cmd/util/ledger/migrations/fix_authorizations_migration_test.go index 8ad439a2039..2e8b66fa9b3 100644 --- a/cmd/util/ledger/migrations/fix_authorizations_migration_test.go +++ b/cmd/util/ledger/migrations/fix_authorizations_migration_test.go @@ -16,6 +16,20 @@ import ( "github.com/onflow/flow-go/model/flow" ) +func newEntitlementSetAuthorizationFromTypeIDs( + typeIDs []common.TypeID, + setKind sema.EntitlementSetKind, +) interpreter.EntitlementSetAuthorization { + return interpreter.NewEntitlementSetAuthorization( + nil, + func() []common.TypeID { + return typeIDs + }, + len(typeIDs), + setKind, + ) +} + func TestFixAuthorizationsMigration(t *testing.T) { t.Parallel() From b48a042eb9544e581c251cb0e5ef0e4217aea82d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Wed, 11 Sep 2024 08:21:31 -0700 Subject: [PATCH 60/72] when auth(Insert|Remove) is inferred, also infer Mutate entitlement --- .../cmd/generate-authorization-fixes/cmd.go | 31 ++++++------------- 1 file changed, 10 insertions(+), 21 deletions(-) diff --git a/cmd/util/cmd/generate-authorization-fixes/cmd.go b/cmd/util/cmd/generate-authorization-fixes/cmd.go index d4de80f7679..3056af50a70 100644 --- a/cmd/util/cmd/generate-authorization-fixes/cmd.go +++ b/cmd/util/cmd/generate-authorization-fixes/cmd.go @@ -535,23 +535,6 @@ func newEntitlementSetAuthorizationFromTypeIDs( ) } -var insertRemoveAuthorization = newEntitlementSetAuthorizationFromTypeIDs( - []common.TypeID{ - sema.InsertType.ID(), - sema.RemoveType.ID(), - }, - sema.Conjunction, -) - -var insertMutateRemoveAuthorization = newEntitlementSetAuthorizationFromTypeIDs( - []common.TypeID{ - sema.InsertType.ID(), - sema.MutateType.ID(), - sema.RemoveType.ID(), - }, - sema.Conjunction, -) - func (g *AuthorizationFixGenerator) maybeGenerateFixForCapabilityController( inter *interpreter.Interpreter, capabilityAddress common.Address, @@ -650,15 +633,21 @@ func (g *AuthorizationFixGenerator) maybeGenerateFixForCapabilityController( // we should not leave the capability controller vulnerable } - // Only fix the authorization if it is different from the old one. // If the old authorization was `Insert, Mutate, Remove`, // the calculated minimal authorization is `Insert, Remove`, // but we ignore the difference, and keep the Mutate entitlement. + if newEntitlementSetAuthorization, ok := newAuthorization.(interpreter.EntitlementSetAuthorization); ok { + if newEntitlementSetAuthorization.Entitlements.Contains(sema.InsertType.ID()) && + newEntitlementSetAuthorization.Entitlements.Contains(sema.RemoveType.ID()) { + + newEntitlementSetAuthorization.Entitlements.Set(sema.MutateType.ID(), struct{}{}) + } + } + + // Only fix the authorization if it is different from the old one. oldAuthorization := borrowType.Authorization - if newAuthorization.Equal(oldAuthorization) || - (oldAuthorization.Equal(insertMutateRemoveAuthorization) && - newAuthorization.Equal(insertRemoveAuthorization)) { + if newAuthorization.Equal(oldAuthorization) { // Nothing to fix return From 8557bfc7766e408f4466d4bb0a8ea2a93a662450 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Wed, 11 Sep 2024 14:14:50 -0700 Subject: [PATCH 61/72] only mention most recent touched state if available --- cmd/util/ledger/util/state.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/cmd/util/ledger/util/state.go b/cmd/util/ledger/util/state.go index 9e56a461985..08deba733c3 100644 --- a/cmd/util/ledger/util/state.go +++ b/cmd/util/ledger/util/state.go @@ -78,10 +78,14 @@ func ReadTrie(dir string, targetHash flow.StateCommitment) ([]*ledger.Payload, e trie, err := led.Trie(ledger.RootHash(state)) if err != nil { - s, _ := led.MostRecentTouchedState() - log.Info(). - Str("hash", s.String()). - Msgf("Most recently touched state") + s, mostRecentErr := led.MostRecentTouchedState() + if mostRecentErr != nil { + log.Error().Err(mostRecentErr).Msg("cannot get most recently touched state") + } else { + log.Info(). + Str("hash", s.String()). + Msgf("Most recently touched state") + } return nil, fmt.Errorf("cannot get trie at the given state commitment: %w", err) } From 19ae07771ce207f668cf25a4c6c09a31ebbaa8bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Wed, 11 Sep 2024 14:17:57 -0700 Subject: [PATCH 62/72] integrate authorization fix migration into execution-state-extract command --- cmd/util/cmd/execution-state-extract/cmd.go | 86 ++++++++++++- .../execution_state_extract.go | 44 ++++++- .../fix_authorizations_migration.go | 117 +++++++++++++++--- .../fix_authorizations_migration_test.go | 106 +++++++++++++++- 4 files changed, 333 insertions(+), 20 deletions(-) diff --git a/cmd/util/cmd/execution-state-extract/cmd.go b/cmd/util/cmd/execution-state-extract/cmd.go index 2196b554c93..ecb814926ad 100644 --- a/cmd/util/cmd/execution-state-extract/cmd.go +++ b/cmd/util/cmd/execution-state-extract/cmd.go @@ -1,8 +1,10 @@ package extract import ( + "compress/gzip" "encoding/hex" "fmt" + "io" "os" "path" "runtime/pprof" @@ -31,6 +33,8 @@ var ( flagChain string flagNWorker int flagNoMigration bool + flagMigration string + flagAuthorizationFixes string flagNoReport bool flagValidateMigration bool flagAllowPartialStateFromPayloads bool @@ -86,6 +90,12 @@ func init() { Cmd.Flags().BoolVar(&flagNoMigration, "no-migration", false, "don't migrate the state") + Cmd.Flags().StringVar(&flagMigration, "migration", "cadence-1.0", + "migration name. 'cadence-1.0' (default) or 'fix-authorizations'") + + Cmd.Flags().StringVar(&flagAuthorizationFixes, "authorization-fixes", "", + "authorization fixes to apply. requires '--migration=fix-authorizations'") + Cmd.Flags().BoolVar(&flagNoReport, "no-report", false, "don't report the state") @@ -194,7 +204,10 @@ func run(*cobra.Command, []string) { defer pprof.StopCPUProfile() } - var stateCommitment flow.StateCommitment + err := os.MkdirAll(flagOutputDir, 0755) + if err != nil { + log.Fatal().Err(err).Msgf("cannot create output directory %s", flagOutputDir) + } if len(flagBlockHash) > 0 && len(flagStateCommitment) > 0 { log.Fatal().Msg("cannot run the command with both block hash and state commitment as inputs, only one of them should be provided") @@ -218,6 +231,21 @@ func run(*cobra.Command, []string) { log.Fatal().Msg("Both --validate and --diff are enabled, please specify only one (or none) of these") } + switch flagMigration { + case "cadence-1.0": + // valid, no-op + + case "fix-authorizations": + if flagAuthorizationFixes == "" { + log.Fatal().Msg("--migration=fix-authorizations requires --authorization-fixes") + } + + default: + log.Fatal().Msg("Invalid --migration: got %s, expected 'cadence-1.0' or 'fix-authorizations'") + } + + var stateCommitment flow.StateCommitment + if len(flagBlockHash) > 0 { blockID, err := flow.HexStringToIdentifier(flagBlockHash) if err != nil { @@ -427,9 +455,29 @@ func run(*cobra.Command, []string) { // Migrate payloads. if !flagNoMigration { - migrations := newMigrations(log.Logger, flagOutputDir, opts) + var migs []migrations.NamedMigration + + switch flagMigration { + case "cadence-1.0": + migs = newCadence1Migrations( + log.Logger, + flagOutputDir, + opts, + ) + + case "fix-authorizations": + migs = newFixAuthorizationsMigrations( + log.Logger, + flagAuthorizationFixes, + flagOutputDir, + opts, + ) + + default: + log.Fatal().Msgf("unknown migration: %s", flagMigration) + } - migration := newMigration(log.Logger, migrations, flagNWorker) + migration := newMigration(log.Logger, migs, flagNWorker) payloads, err = migration(payloads) if err != nil { @@ -507,3 +555,35 @@ func run(*cobra.Command, []string) { // // return fmt.Errorf("no checkpoint file was found, no root checkpoint file was found") // } + +func readAuthorizationFixes(path string) migrations.AuthorizationFixes { + + file, err := os.Open(path) + if err != nil { + log.Fatal().Err(err).Msgf("can't open authorization fixes: %s", path) + } + defer file.Close() + + var reader io.Reader = file + if isGzip(file) { + reader, err = gzip.NewReader(file) + if err != nil { + log.Fatal().Err(err).Msgf("failed to create gzip reader for %s", path) + } + } + + log.Info().Msgf("Reading authorization fixes from %s ...", path) + + fixes, err := migrations.ReadAuthorizationFixes(reader, nil) + if err != nil { + log.Fatal().Err(err).Msgf("failed to read authorization fixes %s", path) + } + + log.Info().Msgf("Read %d authorization fixes", len(fixes)) + + return fixes +} + +func isGzip(file *os.File) bool { + return strings.HasSuffix(file.Name(), ".gz") +} diff --git a/cmd/util/cmd/execution-state-extract/execution_state_extract.go b/cmd/util/cmd/execution-state-extract/execution_state_extract.go index f9055f2f2d0..49b1728bb69 100644 --- a/cmd/util/cmd/execution-state-extract/execution_state_extract.go +++ b/cmd/util/cmd/execution-state-extract/execution_state_extract.go @@ -358,13 +358,13 @@ func createTrieFromPayloads(logger zerolog.Logger, payloads []*ledger.Payload) ( return newTrie, nil } -func newMigrations( +func newCadence1Migrations( log zerolog.Logger, outputDir string, opts migrators.Options, ) []migrators.NamedMigration { - log.Info().Msg("initializing migrations") + log.Info().Msg("initializing Cadence 1.0 migrations ...") rwf := reporters.NewReportFileWriterFactory(outputDir, log) @@ -394,3 +394,43 @@ func newMigrations( return namedMigrations } + +func newFixAuthorizationsMigrations( + log zerolog.Logger, + authorizationFixesPath string, + outputDir string, + opts migrators.Options, +) []migrators.NamedMigration { + + log.Info().Msg("initializing authorization fix migrations ...") + + rwf := reporters.NewReportFileWriterFactory(outputDir, log) + + authorizationFixes := readAuthorizationFixes(authorizationFixesPath) + + namedMigrations := migrators.NewFixAuthorizationsMigrations( + log, + rwf, + authorizationFixes, + opts, + ) + + // At the end, fix up storage-used discrepancies + namedMigrations = append( + namedMigrations, + migrators.NamedMigration{ + Name: "account-usage-migration", + Migrate: migrators.NewAccountBasedMigration( + log, + opts.NWorker, + []migrators.AccountBasedMigration{ + migrators.NewAccountUsageMigration(rwf), + }, + ), + }, + ) + + log.Info().Msg("initialized migrations") + + return namedMigrations +} diff --git a/cmd/util/ledger/migrations/fix_authorizations_migration.go b/cmd/util/ledger/migrations/fix_authorizations_migration.go index 948199c623d..b7d5df4a3f7 100644 --- a/cmd/util/ledger/migrations/fix_authorizations_migration.go +++ b/cmd/util/ledger/migrations/fix_authorizations_migration.go @@ -4,6 +4,8 @@ import ( "encoding/json" "errors" "fmt" + "io" + "strings" "github.com/onflow/cadence/migrations" "github.com/onflow/cadence/runtime" @@ -16,7 +18,6 @@ import ( "github.com/onflow/flow-go/cmd/util/ledger/reporters" "github.com/onflow/flow-go/fvm/environment" - "github.com/onflow/flow-go/model/flow" ) type AccountCapabilityID struct { @@ -210,23 +211,14 @@ func CanSkipFixAuthorizationsMigration(valueType interpreter.StaticType) bool { return false } -type FixAuthorizationsMigrationOptions struct { - ChainID flow.ChainID - NWorker int - VerboseErrorOutput bool - LogVerboseDiff bool - DiffMigrations bool - CheckStorageHealthBeforeMigration bool -} - const fixAuthorizationsMigrationReporterName = "fix-authorizations-migration" func NewFixAuthorizationsMigration( rwf reporters.ReportWriterFactory, errorMessageHandler *errorMessageHandler, programs map[runtime.Location]*interpreter.Program, - newAuthorizations map[AccountCapabilityID]interpreter.Authorization, - opts FixAuthorizationsMigrationOptions, + newAuthorizations AuthorizationFixes, + opts Options, ) *CadenceBaseMigration { var diffReporter reporters.ReportWriter if opts.DiffMigrations { @@ -411,8 +403,8 @@ func (e capabilityAuthorizationFixedEntry) MarshalJSON() ([]byte, error) { func NewFixAuthorizationsMigrations( log zerolog.Logger, rwf reporters.ReportWriterFactory, - newAuthorizations map[AccountCapabilityID]interpreter.Authorization, - opts FixAuthorizationsMigrationOptions, + newAuthorizations AuthorizationFixes, + opts Options, ) []NamedMigration { errorMessageHandler := &errorMessageHandler{} @@ -456,3 +448,100 @@ func NewFixAuthorizationsMigrations( }, } } + +type AuthorizationFixes map[AccountCapabilityID]interpreter.Authorization + +// ReadAuthorizationFixes reads a report of authorization fixes from the given reader. +// The report is expected to be a JSON array of objects with the following structure: +// +// [ +// {"address":"0x1","identifier":"foo","linkType":"&Foo","accessibleMembers":["foo"]} +// ] +func ReadAuthorizationFixes( + reader io.Reader, + filter map[common.Address]struct{}, +) (AuthorizationFixes, error) { + + fixes := AuthorizationFixes{} + + dec := json.NewDecoder(reader) + + token, err := dec.Token() + if err != nil { + return nil, fmt.Errorf("failed to read token: %w", err) + } + if token != json.Delim('[') { + return nil, fmt.Errorf("expected start of array, got %s", token) + } + + for dec.More() { + var entry struct { + CapabilityAddress string `json:"capability_address"` + CapabilityID uint64 `json:"capability_id"` + NewAuthorization string `json:"new_authorization"` + } + err := dec.Decode(&entry) + if err != nil { + return nil, fmt.Errorf("failed to decode entry: %w", err) + } + + address, err := common.HexToAddress(entry.CapabilityAddress) + if err != nil { + return nil, fmt.Errorf("failed to parse address: %w", err) + } + + if filter != nil { + if _, ok := filter[address]; !ok { + continue + } + } + + newAuthorization, err := jsonDecodeAuthorization(entry.NewAuthorization) + if err != nil { + return nil, fmt.Errorf("failed to decode new authorization '%s': %w", entry.NewAuthorization, err) + } + + accountCapabilityID := AccountCapabilityID{ + Address: address, + CapabilityID: entry.CapabilityID, + } + + fixes[accountCapabilityID] = newAuthorization + } + + token, err = dec.Token() + if err != nil { + return nil, fmt.Errorf("failed to read token: %w", err) + } + if token != json.Delim(']') { + return nil, fmt.Errorf("expected end of array, got %s", token) + } + + return fixes, nil +} + +func jsonDecodeAuthorization(encoded string) (interpreter.Authorization, error) { + if encoded == "" { + return interpreter.UnauthorizedAccess, nil + } + + if strings.Contains(encoded, "|") { + return nil, fmt.Errorf("invalid disjunction entitlement set authorization: %s", encoded) + } + + var typeIDs []common.TypeID + for _, part := range strings.Split(encoded, ",") { + typeIDs = append(typeIDs, common.TypeID(part)) + } + + entitlementSetAuthorization := interpreter.NewEntitlementSetAuthorization( + nil, + func() []common.TypeID { + return typeIDs + }, + len(typeIDs), + sema.Conjunction, + ) + + return entitlementSetAuthorization, nil +} diff --git a/cmd/util/ledger/migrations/fix_authorizations_migration_test.go b/cmd/util/ledger/migrations/fix_authorizations_migration_test.go index 2e8b66fa9b3..69e2f250792 100644 --- a/cmd/util/ledger/migrations/fix_authorizations_migration_test.go +++ b/cmd/util/ledger/migrations/fix_authorizations_migration_test.go @@ -2,6 +2,7 @@ package migrations import ( "fmt" + "strings" "testing" "github.com/onflow/cadence" @@ -145,7 +146,7 @@ func TestFixAuthorizationsMigration(t *testing.T) { rwf := &testReportWriterFactory{} - options := FixAuthorizationsMigrationOptions{ + options := Options{ ChainID: chainID, NWorker: nWorker, } @@ -278,3 +279,106 @@ func TestFixAuthorizationsMigration(t *testing.T) { ) require.NoError(t, err) } + +func TestReadAuthorizationFixes(t *testing.T) { + t.Parallel() + + validContents := ` + [ + {"capability_address":"01","capability_id":4,"new_authorization":""}, + {"capability_address":"02","capability_id":5,"new_authorization":"A.0000000000000001.Foo.Bar"}, + {"capability_address":"03","capability_id":6,"new_authorization":"A.0000000000000001.Foo.Bar,A.0000000000000001.Foo.Baz"} + ] + ` + + t.Run("unfiltered", func(t *testing.T) { + + t.Parallel() + + reader := strings.NewReader(validContents) + + mapping, err := ReadAuthorizationFixes(reader, nil) + require.NoError(t, err) + + require.Equal(t, + AuthorizationFixes{ + { + Address: common.MustBytesToAddress([]byte{0x1}), + CapabilityID: 4, + }: interpreter.UnauthorizedAccess, + { + Address: common.MustBytesToAddress([]byte{0x2}), + CapabilityID: 5, + }: newEntitlementSetAuthorizationFromTypeIDs( + []common.TypeID{ + "A.0000000000000001.Foo.Bar", + }, + sema.Conjunction, + ), + { + Address: common.MustBytesToAddress([]byte{0x3}), + CapabilityID: 6, + }: newEntitlementSetAuthorizationFromTypeIDs( + []common.TypeID{ + "A.0000000000000001.Foo.Bar", + "A.0000000000000001.Foo.Baz", + }, + sema.Conjunction, + ), + }, + mapping, + ) + }) + + t.Run("filtered", func(t *testing.T) { + + t.Parallel() + + address1 := common.MustBytesToAddress([]byte{0x1}) + address3 := common.MustBytesToAddress([]byte{0x3}) + + addressFilter := map[common.Address]struct{}{ + address1: {}, + address3: {}, + } + + reader := strings.NewReader(validContents) + + mapping, err := ReadAuthorizationFixes(reader, addressFilter) + require.NoError(t, err) + + require.Equal(t, + AuthorizationFixes{ + { + Address: common.MustBytesToAddress([]byte{0x1}), + CapabilityID: 4, + }: interpreter.UnauthorizedAccess, + { + Address: common.MustBytesToAddress([]byte{0x3}), + CapabilityID: 6, + }: newEntitlementSetAuthorizationFromTypeIDs( + []common.TypeID{ + "A.0000000000000001.Foo.Bar", + "A.0000000000000001.Foo.Baz", + }, + sema.Conjunction, + ), + }, + mapping, + ) + }) + + t.Run("invalid disjunction entitlement set authorization", func(t *testing.T) { + + t.Parallel() + + reader := strings.NewReader(` + [ + {"capability_address":"03","capability_id":6,"new_authorization":"A.0000000000000001.Foo.Bar|A.0000000000000001.Foo.Baz"} + ] + `) + + _, err := ReadAuthorizationFixes(reader, nil) + require.ErrorContains(t, err, "invalid disjunction entitlement set authorization") + }) +} From 7f3bba29950f064b4f38c95d04ba5a2e25bb6c5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Wed, 11 Sep 2024 16:56:25 -0700 Subject: [PATCH 63/72] refactor to simply remove all authorizations from cap cons which were migration from public links --- .../accessible_members.go | 29 -- .../cmd/generate-authorization-fixes/cmd.go | 414 +++--------------- .../generate-authorization-fixes/cmd_test.go | 176 ++------ .../entitlements.go | 131 ------ .../entitlements_test.go | 235 ---------- .../link_migration_report.go | 23 +- .../link_migration_report_test.go | 14 +- .../link_report.go | 90 ---- .../link_report_test.go | 79 ---- .../fix_authorizations_migration.go | 108 ++--- .../fix_authorizations_migration_test.go | 174 +++----- 11 files changed, 211 insertions(+), 1262 deletions(-) delete mode 100644 cmd/util/cmd/generate-authorization-fixes/accessible_members.go delete mode 100644 cmd/util/cmd/generate-authorization-fixes/entitlements.go delete mode 100644 cmd/util/cmd/generate-authorization-fixes/entitlements_test.go delete mode 100644 cmd/util/cmd/generate-authorization-fixes/link_report.go delete mode 100644 cmd/util/cmd/generate-authorization-fixes/link_report_test.go diff --git a/cmd/util/cmd/generate-authorization-fixes/accessible_members.go b/cmd/util/cmd/generate-authorization-fixes/accessible_members.go deleted file mode 100644 index 92dcd2c9e71..00000000000 --- a/cmd/util/cmd/generate-authorization-fixes/accessible_members.go +++ /dev/null @@ -1,29 +0,0 @@ -package generate_authorization_fixes - -import ( - "github.com/onflow/cadence/runtime/ast" - "github.com/onflow/cadence/runtime/sema" -) - -func getAccessibleMembers(ty sema.Type) []string { - // NOTE: GetMembers might return members that are actually not accessible, for DX purposes. - // We need to resolve the members and filter out the inaccessible members, - // using the error reported when resolving - - memberResolvers := ty.GetMembers() - - accessibleMembers := make([]string, 0, len(memberResolvers)) - - for memberName, memberResolver := range memberResolvers { - var resolveErr error - memberResolver.Resolve(nil, memberName, ast.EmptyRange, func(err error) { - resolveErr = err - }) - if resolveErr != nil { - continue - } - accessibleMembers = append(accessibleMembers, memberName) - } - - return accessibleMembers -} diff --git a/cmd/util/cmd/generate-authorization-fixes/cmd.go b/cmd/util/cmd/generate-authorization-fixes/cmd.go index 3056af50a70..ce034ef3d79 100644 --- a/cmd/util/cmd/generate-authorization-fixes/cmd.go +++ b/cmd/util/cmd/generate-authorization-fixes/cmd.go @@ -3,10 +3,8 @@ package generate_authorization_fixes import ( "compress/gzip" "encoding/json" - "fmt" "io" "os" - "sort" "strings" "sync" @@ -17,7 +15,6 @@ import ( "github.com/rs/zerolog/log" "github.com/schollz/progressbar/v3" "github.com/spf13/cobra" - "golang.org/x/exp/slices" common2 "github.com/onflow/flow-go/cmd/util/common" "github.com/onflow/flow-go/cmd/util/ledger/migrations" @@ -34,7 +31,6 @@ var ( flagStateCommitment string flagOutputDirectory string flagChain string - flagPublicLinkReport string flagLinkMigrationReport string flagAddresses string ) @@ -83,14 +79,6 @@ func init() { ) _ = Cmd.MarkFlagRequired("chain") - Cmd.Flags().StringVar( - &flagPublicLinkReport, - "public-link-report", - "", - "Input public link report file name", - ) - _ = Cmd.MarkFlagRequired("public-link-report") - Cmd.Flags().StringVar( &flagLinkMigrationReport, "link-migration-report", @@ -107,8 +95,6 @@ func init() { ) } -const contractCountEstimate = 1000 - func run(*cobra.Command, []string) { var addressFilter map[common.Address]struct{} @@ -159,14 +145,12 @@ func run(*cobra.Command, []string) { // Validate chain ID _ = chainID.Chain() - publicLinkReportChan := make(chan PublicLinkReport, 1) - go func() { - publicLinkReportChan <- readPublicLinkReport(addressFilter) - }() - - publicLinkMigrationReportChan := make(chan PublicLinkMigrationReport, 1) + migratedPublicLinkSetChan := make(chan MigratedPublicLinkSet, 1) go func() { - publicLinkMigrationReportChan <- readLinkMigrationReport(addressFilter) + migratedPublicLinkSetChan <- readMigratedPublicLinkSet( + flagLinkMigrationReport, + addressFilter, + ) }() registersByAccountChan := make(chan *registers.ByAccount, 1) @@ -174,29 +158,21 @@ func run(*cobra.Command, []string) { registersByAccountChan <- loadRegistersByAccount() }() - publicLinkReport := <-publicLinkReportChan - publicLinkMigrationReport := <-publicLinkMigrationReportChan + migratedPublicLinkSet := <-migratedPublicLinkSetChan registersByAccount := <-registersByAccountChan - checkingReporter := rwf.ReportWriter("contract-checking") - defer checkingReporter.Close() - - checkContracts( - registersByAccount, - chainID, - checkingReporter, - ) - fixReporter := rwf.ReportWriter("authorization-fixes") defer fixReporter.Close() authorizationFixGenerator := &AuthorizationFixGenerator{ - registersByAccount: registersByAccount, - chainID: chainID, - publicLinkReport: publicLinkReport, - publicLinkMigrationReport: publicLinkMigrationReport, - reporter: fixReporter, + registersByAccount: registersByAccount, + chainID: chainID, + migratedPublicLinkSet: migratedPublicLinkSet, + reporter: fixReporter, } + + log.Info().Msg("Generating authorization fixes ...") + if len(addressFilter) > 0 { authorizationFixGenerator.generateFixesForAccounts(addressFilter) } else { @@ -242,106 +218,32 @@ func loadRegistersByAccount() *registers.ByAccount { return registersByAccount } -func readPublicLinkReport(addressFilter map[common.Address]struct{}) PublicLinkReport { - // Read public link report - - publicLinkReportFile, err := os.Open(flagPublicLinkReport) - if err != nil { - log.Fatal().Err(err).Msgf("can't open link report: %s", flagPublicLinkReport) - } - defer publicLinkReportFile.Close() - - var publicLinkReportReader io.Reader = publicLinkReportFile - if isGzip(publicLinkReportFile) { - publicLinkReportReader, err = gzip.NewReader(publicLinkReportFile) - if err != nil { - log.Fatal().Err(err).Msgf("failed to create gzip reader for %s", flagPublicLinkReport) - } - } - - log.Info().Msgf("Reading public link report from %s ...", flagPublicLinkReport) - - publicLinkReport, err := ReadPublicLinkReport(publicLinkReportReader, addressFilter) - if err != nil { - log.Fatal().Err(err).Msgf("failed to read public link report %s", flagPublicLinkReport) - } - - log.Info().Msgf("Read %d public link entries", len(publicLinkReport)) - - return publicLinkReport -} - -func readLinkMigrationReport(addressFilter map[common.Address]struct{}) PublicLinkMigrationReport { - // Read link migration report +func readMigratedPublicLinkSet(path string, addressFilter map[common.Address]struct{}) MigratedPublicLinkSet { - linkMigrationReportFile, err := os.Open(flagLinkMigrationReport) + file, err := os.Open(path) if err != nil { - log.Fatal().Err(err).Msgf("can't open link migration report: %s", flagLinkMigrationReport) + log.Fatal().Err(err).Msgf("can't open link migration report: %s", path) } - defer linkMigrationReportFile.Close() + defer file.Close() - var linkMigrationReportReader io.Reader = linkMigrationReportFile - if isGzip(linkMigrationReportFile) { - linkMigrationReportReader, err = gzip.NewReader(linkMigrationReportFile) + var reader io.Reader = file + if isGzip(file) { + reader, err = gzip.NewReader(file) if err != nil { - log.Fatal().Err(err).Msgf("failed to create gzip reader for %s", flagLinkMigrationReport) + log.Fatal().Err(err).Msgf("failed to create gzip reader for %s", path) } } - log.Info().Msgf("Reading link migration report from %s ...", flagLinkMigrationReport) - - publicLinkMigrationReport, err := ReadPublicLinkMigrationReport(linkMigrationReportReader, addressFilter) - if err != nil { - log.Fatal().Err(err).Msgf("failed to read public link report: %s", flagLinkMigrationReport) - } - - log.Info().Msgf("Read %d public link migration entries", len(publicLinkMigrationReport)) - - return publicLinkMigrationReport -} - -func checkContracts( - registersByAccount *registers.ByAccount, - chainID flow.ChainID, - reporter reporters.ReportWriter, -) { - contracts, err := migrations.GatherContractsFromRegisters(registersByAccount, log.Logger) - if err != nil { - log.Fatal().Err(err) - } - - programs := make(map[common.Location]*interpreter.Program, contractCountEstimate) - - contractsForPrettyPrinting := make(map[common.Location][]byte, len(contracts)) - for _, contract := range contracts { - contractsForPrettyPrinting[contract.Location] = contract.Code - } - - log.Info().Msg("Checking contracts ...") + log.Info().Msgf("Reading link migration report from %s ...", path) - mr, err := migrations.NewInterpreterMigrationRuntime( - registersByAccount, - chainID, - migrations.InterpreterMigrationRuntimeConfig{}, - ) + migratedPublicLinkSet, err := ReadMigratedPublicLinkSet(reader, addressFilter) if err != nil { - log.Fatal().Err(err) + log.Fatal().Err(err).Msgf("failed to read public link report: %s", path) } - for _, contract := range contracts { - migrations.CheckContract( - contract, - log.Logger, - mr, - contractsForPrettyPrinting, - false, - reporter, - nil, - programs, - ) - } + log.Info().Msgf("Read %d public link migration entries", len(migratedPublicLinkSet)) - log.Info().Msgf("Checked %d contracts ...", len(contracts)) + return migratedPublicLinkSet } func jsonEncodeAuthorization(authorization interpreter.Authorization) string { @@ -354,53 +256,33 @@ func jsonEncodeAuthorization(authorization interpreter.Authorization) string { } type fixEntitlementsEntry struct { - AccountCapabilityID - ReferencedType interpreter.StaticType - OldAuthorization interpreter.Authorization - NewAuthorization interpreter.Authorization - OldAccessibleMembers []string - NewAccessibleMembers []string - UnresolvedMembers map[string]error + CapabilityAddress common.Address + CapabilityID uint64 + ReferencedType interpreter.StaticType + Authorization interpreter.Authorization } var _ json.Marshaler = fixEntitlementsEntry{} func (e fixEntitlementsEntry) MarshalJSON() ([]byte, error) { return json.Marshal(struct { - CapabilityAddress string `json:"capability_address"` - CapabilityID uint64 `json:"capability_id"` - ReferencedType string `json:"referenced_type"` - OldAuthorization string `json:"old_authorization"` - NewAuthorization string `json:"new_authorization"` - OldAccessibleMembers []string `json:"old_members"` - NewAccessibleMembers []string `json:"new_members"` - UnresolvedMembers map[string]string `json:"unresolved_members,omitempty"` + CapabilityAddress string `json:"capability_address"` + CapabilityID uint64 `json:"capability_id"` + ReferencedType string `json:"referenced_type"` + Authorization string `json:"authorization"` }{ - CapabilityAddress: e.Address.String(), - CapabilityID: e.CapabilityID, - ReferencedType: string(e.ReferencedType.ID()), - OldAuthorization: jsonEncodeAuthorization(e.OldAuthorization), - NewAuthorization: jsonEncodeAuthorization(e.NewAuthorization), - OldAccessibleMembers: e.OldAccessibleMembers, - NewAccessibleMembers: e.NewAccessibleMembers, - UnresolvedMembers: jsonEncodeMemberErrorMap(e.UnresolvedMembers), + CapabilityAddress: e.CapabilityAddress.String(), + CapabilityID: e.CapabilityID, + ReferencedType: string(e.ReferencedType.ID()), + Authorization: jsonEncodeAuthorization(e.Authorization), }) } -func jsonEncodeMemberErrorMap(m map[string]error) map[string]string { - result := make(map[string]string, len(m)) - for key, value := range m { - result[key] = value.Error() - } - return result -} - type AuthorizationFixGenerator struct { - registersByAccount *registers.ByAccount - chainID flow.ChainID - publicLinkReport PublicLinkReport - publicLinkMigrationReport PublicLinkMigrationReport - reporter reporters.ReportWriter + registersByAccount *registers.ByAccount + chainID flow.ChainID + migratedPublicLinkSet MigratedPublicLinkSet + reporter reporters.ReportWriter } func (g *AuthorizationFixGenerator) generateFixesForAllAccounts() { @@ -413,7 +295,7 @@ func (g *AuthorizationFixGenerator) generateFixesForAllAccounts() { go func(address common.Address) { defer wg.Done() g.generateFixesForAccount(address) - progress.Add(1) + _ = progress.Add(1) }(address) return nil }) @@ -422,7 +304,7 @@ func (g *AuthorizationFixGenerator) generateFixesForAllAccounts() { } wg.Wait() - progress.Finish() + _ = progress.Finish() } func (g *AuthorizationFixGenerator) generateFixesForAccounts(addresses map[common.Address]struct{}) { @@ -434,12 +316,12 @@ func (g *AuthorizationFixGenerator) generateFixesForAccounts(addresses map[commo go func(address common.Address) { defer wg.Done() g.generateFixesForAccount(address) - progress.Add(1) + _ = progress.Add(1) }(address) } wg.Wait() - progress.Finish() + _ = progress.Finish() } func (g *AuthorizationFixGenerator) generateFixesForAccount(address common.Address) { @@ -487,8 +369,7 @@ func (g *AuthorizationFixGenerator) generateFixesForAccount(address common.Addre switch borrowType.Authorization.(type) { case interpreter.EntitlementSetAuthorization: - g.maybeGenerateFixForCapabilityController( - mr.Interpreter, + g.maybeGenerateFixForEntitledCapabilityController( address, capabilityID, borrowType, @@ -535,207 +416,28 @@ func newEntitlementSetAuthorizationFromTypeIDs( ) } -func (g *AuthorizationFixGenerator) maybeGenerateFixForCapabilityController( - inter *interpreter.Interpreter, +func (g *AuthorizationFixGenerator) maybeGenerateFixForEntitledCapabilityController( capabilityAddress common.Address, capabilityID uint64, borrowType *interpreter.ReferenceStaticType, ) { - // Only fix the entitlements if the capability controller was migrated from a public link - publicPathIdentifier := g.capabilityControllerPublicPathIdentifier(capabilityAddress, capabilityID) - if publicPathIdentifier == "" { - return - } - - linkInfo := g.publicPathLinkInfo(capabilityAddress, publicPathIdentifier) - if linkInfo.BorrowType == "" { - log.Warn().Msgf( - "missing link info for /public/%s in account %s", - publicPathIdentifier, - capabilityAddress.HexWithPrefix(), - ) - return - } - - // Compare previously accessible members with new accessible members. - // They should be the same. - - oldAccessibleMembers := linkInfo.AccessibleMembers - if oldAccessibleMembers == nil { - log.Warn().Msgf( - "missing old accessible members for for /public/%s in account %s", - publicPathIdentifier, - capabilityAddress.HexWithPrefix(), - ) - return - } - - semaBorrowType, err := convertStaticToSemaType(inter, borrowType) - if err != nil { - log.Warn().Err(err).Msgf( - "failed to get new accessible members for capability controller %d in account %s", - capabilityID, - capabilityAddress.HexWithPrefix(), - ) - return - } - - newAccessibleMembers := getAccessibleMembers(semaBorrowType) - - sort.Strings(oldAccessibleMembers) - sort.Strings(newAccessibleMembers) - - if slices.Equal(oldAccessibleMembers, newAccessibleMembers) { - // Nothing to fix - return - } - - oldAccessibleMemberSet := make(map[string]struct{}) - for _, memberName := range oldAccessibleMembers { - oldAccessibleMemberSet[memberName] = struct{}{} - } - - newAccessibleMemberSet := make(map[string]struct{}) - for _, memberName := range newAccessibleMembers { - newAccessibleMemberSet[memberName] = struct{}{} - } - - membersAdded, membersRemoved := sortedDiffStringSets( - oldAccessibleMemberSet, - newAccessibleMemberSet, - ) - - log.Info().Msgf( - "member mismatch for capability controller %d in account %s: expected %v, got %v (added: %v, removed: %v)", - capabilityID, - capabilityAddress.HexWithPrefix(), - oldAccessibleMembers, - newAccessibleMembers, - membersAdded, - membersRemoved, - ) - - newAuthorization, unresolvedMembers := findMinimalAuthorization( - semaBorrowType, - oldAccessibleMemberSet, - ) - - if len(unresolvedMembers) > 0 { - // TODO: format unresolved members - log.Warn().Msgf( - "failed to find minimal entitlement set for capability controller %d in account %s: unresolved members: %v", - capabilityID, - capabilityAddress.HexWithPrefix(), - unresolvedMembers, - ) - - // NOTE: still continue with the fix, - // we should not leave the capability controller vulnerable - } - - // If the old authorization was `Insert, Mutate, Remove`, - // the calculated minimal authorization is `Insert, Remove`, - // but we ignore the difference, and keep the Mutate entitlement. - - if newEntitlementSetAuthorization, ok := newAuthorization.(interpreter.EntitlementSetAuthorization); ok { - if newEntitlementSetAuthorization.Entitlements.Contains(sema.InsertType.ID()) && - newEntitlementSetAuthorization.Entitlements.Contains(sema.RemoveType.ID()) { - - newEntitlementSetAuthorization.Entitlements.Set(sema.MutateType.ID(), struct{}{}) - } - } - - // Only fix the authorization if it is different from the old one. - oldAuthorization := borrowType.Authorization - if newAuthorization.Equal(oldAuthorization) { - - // Nothing to fix + // Only remove the authorization if the capability controller was migrated from a public link + _, ok := g.migratedPublicLinkSet[AccountCapabilityID{ + Address: capabilityAddress, + CapabilityID: capabilityID, + }] + if !ok { return } g.reporter.Write(fixEntitlementsEntry{ - AccountCapabilityID: AccountCapabilityID{ - Address: capabilityAddress, - CapabilityID: capabilityID, - }, - ReferencedType: borrowType.ReferencedType, - OldAuthorization: oldAuthorization, - NewAuthorization: newAuthorization, - OldAccessibleMembers: oldAccessibleMembers, - NewAccessibleMembers: newAccessibleMembers, - UnresolvedMembers: unresolvedMembers, + CapabilityAddress: capabilityAddress, + CapabilityID: capabilityID, + ReferencedType: borrowType.ReferencedType, + Authorization: borrowType.Authorization, }) - -} - -func convertStaticToSemaType( - inter *interpreter.Interpreter, - staticType interpreter.StaticType, -) ( - semaType sema.Type, - err error, -) { - - defer func() { - if r := recover(); r != nil { - err = fmt.Errorf("panic: %v", r) - } - }() - - semaType, err = inter.ConvertStaticToSemaType(staticType) - if err != nil { - return nil, fmt.Errorf( - "failed to convert static type %s to semantic type: %w", - staticType.ID(), - err, - ) - } - if semaType == nil { - return nil, fmt.Errorf( - "failed to convert static type %s to semantic type", - staticType.ID(), - ) - } - - return semaType, nil -} - -func (g *AuthorizationFixGenerator) capabilityControllerPublicPathIdentifier( - address common.Address, - capabilityID uint64, -) string { - return g.publicLinkMigrationReport[AccountCapabilityID{ - Address: address, - CapabilityID: capabilityID, - }] -} - -func (g *AuthorizationFixGenerator) publicPathLinkInfo( - address common.Address, - publicPathIdentifier string, -) LinkInfo { - return g.publicLinkReport[AddressPublicPath{ - Address: address, - Identifier: publicPathIdentifier, - }] } func isGzip(file *os.File) bool { return strings.HasSuffix(file.Name(), ".gz") } - -func sortedDiffStringSets(a, b map[string]struct{}) (added, removed []string) { - for key := range a { - if _, ok := b[key]; !ok { - removed = append(removed, key) - } - } - for key := range b { - if _, ok := a[key]; !ok { - added = append(added, key) - } - } - sort.Strings(added) - sort.Strings(removed) - return -} diff --git a/cmd/util/cmd/generate-authorization-fixes/cmd_test.go b/cmd/util/cmd/generate-authorization-fixes/cmd_test.go index bf5877414d6..7a5f8f0f459 100644 --- a/cmd/util/cmd/generate-authorization-fixes/cmd_test.go +++ b/cmd/util/cmd/generate-authorization-fixes/cmd_test.go @@ -1,7 +1,6 @@ package generate_authorization_fixes import ( - "errors" "fmt" "testing" @@ -83,8 +82,6 @@ func TestGenerateAuthorizationFixes(t *testing.T) { const chainID = flow.Emulator chain := chainID.Chain() - const nWorker = 2 - address, err := chain.AddressAtIndex(1000) require.NoError(t, err) @@ -112,14 +109,10 @@ func TestGenerateAuthorizationFixes(t *testing.T) { const contractCode = ` access(all) contract Test { - access(all) entitlement E1 - access(all) entitlement E2 - access(all) struct S { - access(E1) fun f1() {} - access(E2) fun f2() {} - access(all) fun f3() {} - } + access(all) entitlement E + + access(all) struct S {} } ` @@ -149,28 +142,36 @@ func TestGenerateAuthorizationFixes(t *testing.T) { transaction { prepare(signer: auth(Storage, Capabilities) &Account) { - // Capability 1 was a public, unauthorized capability. + // Capability 1 was a public, unauthorized capability, which is now authorized. // It should lose its entitlement - let cap1 = signer.capabilities.storage.issue(/storage/s) - signer.capabilities.publish(cap1, at: /public/s) + let cap1 = signer.capabilities.storage.issue(/storage/s) + signer.capabilities.publish(cap1, at: /public/s1) - // Capability 2 was a public, unauthorized capability, stored nested in storage. + // Capability 2 was a public, unauthorized capability, which is now authorized. + // It is currently only stored, nested, in storage, and is not published. // It should lose its entitlement - let cap2 = signer.capabilities.storage.issue(/storage/s) + let cap2 = signer.capabilities.storage.issue(/storage/s) signer.storage.save([cap2], to: /storage/caps2) - // Capability 3 was a private, authorized capability, stored nested in storage. + // Capability 3 was a private, authorized capability. + // It is currently only stored, nested, in storage, and is not published. // It should keep its entitlement - let cap3 = signer.capabilities.storage.issue(/storage/s) + let cap3 = signer.capabilities.storage.issue(/storage/s) signer.storage.save([cap3], to: /storage/caps3) - // Capability 4 was a capability with unavailable accessible members, stored nested in storage. - // It should keep its entitlement - let cap4 = signer.capabilities.storage.issue(/storage/s) + // Capability 4 was a private, authorized capability. + // It is currently both stored, nested, in storage, and is published. + // It should keep its entitlement + let cap4 = signer.capabilities.storage.issue(/storage/s) signer.storage.save([cap4], to: /storage/caps4) - - let cap5 = signer.capabilities.storage.issue(/storage/dict) - signer.capabilities.publish(cap5, at: /public/dict) + signer.capabilities.publish(cap4, at: /public/s4) + + // Capability 5 was a public, unauthorized capability, which is still unauthorized. + // It is currently both stored, nested, in storage, and is published. + // There is no need to fix it. + let cap5 = signer.capabilities.storage.issue<&Test.S>(/storage/s) + signer.storage.save([cap5], to: /storage/caps5) + signer.capabilities.publish(cap5, at: /public/s5) } } `, @@ -187,164 +188,69 @@ func TestGenerateAuthorizationFixes(t *testing.T) { err = runSetupTx(registersByAccount) require.NoError(t, err) - oldAccessibleMembers := []string{ - "f1", - "f3", - "forEachAttachment", - "getType", - "isInstance", - "undefined", - } - - newAccessibleMembers := []string{ - "f1", - "f2", - "f3", - "forEachAttachment", - "getType", - "isInstance", - } - testContractLocation := common.AddressLocation{ Address: common.Address(address), Name: "Test", } - borrowTypeID := testContractLocation.TypeID(nil, "Test.S") - - publicLinkReport := PublicLinkReport{ - { - Address: common.Address(address), - Identifier: "s", - }: { - BorrowType: borrowTypeID, - AccessibleMembers: oldAccessibleMembers, - }, - { - Address: common.Address(address), - Identifier: "s2", - }: { - BorrowType: borrowTypeID, - AccessibleMembers: oldAccessibleMembers, - }, - { - Address: common.Address(address), - Identifier: "s4", - }: { - BorrowType: borrowTypeID, - AccessibleMembers: nil, - }, - { - Address: common.Address(address), - Identifier: "dict", - }: { - BorrowType: "{String:String}", - AccessibleMembers: []string{ - "containsKey", - "forEachAttachment", - "forEachKey", - "getType", - "insert", - "isInstance", - "keys", - "length", - "remove", - "values", - }, - }, - } - publicLinkMigrationReport := PublicLinkMigrationReport{ + migratedPublicLinkSet := MigratedPublicLinkSet{ { Address: common.Address(address), CapabilityID: 1, - }: "s", + }: {}, { Address: common.Address(address), CapabilityID: 2, - }: "s2", - { - Address: common.Address(address), - CapabilityID: 4, - }: "s4", + }: {}, { Address: common.Address(address), CapabilityID: 5, - }: "dict", + }: {}, } reporter := &testReportWriter{} generator := &AuthorizationFixGenerator{ - registersByAccount: registersByAccount, - chainID: chainID, - publicLinkReport: publicLinkReport, - publicLinkMigrationReport: publicLinkMigrationReport, - reporter: reporter, + registersByAccount: registersByAccount, + chainID: chainID, + migratedPublicLinkSet: migratedPublicLinkSet, + reporter: reporter, } generator.generateFixesForAllAccounts() - e1TypeID := testContractLocation.TypeID(nil, "Test.E1") - e2TypeID := testContractLocation.TypeID(nil, "Test.E2") + eTypeID := testContractLocation.TypeID(nil, "Test.E") assert.Equal(t, []any{ fixEntitlementsEntry{ - AccountCapabilityID: AccountCapabilityID{ - Address: common.Address(address), - CapabilityID: 1, - }, + CapabilityAddress: common.Address(address), + CapabilityID: 1, ReferencedType: interpreter.NewCompositeStaticTypeComputeTypeID( nil, testContractLocation, "Test.S", ), - OldAuthorization: newEntitlementSetAuthorizationFromTypeIDs( + Authorization: newEntitlementSetAuthorizationFromTypeIDs( []common.TypeID{ - e1TypeID, - e2TypeID, + eTypeID, }, sema.Conjunction, ), - NewAuthorization: newEntitlementSetAuthorizationFromTypeIDs( - []common.TypeID{ - e1TypeID, - }, - sema.Conjunction, - ), - OldAccessibleMembers: oldAccessibleMembers, - NewAccessibleMembers: newAccessibleMembers, - UnresolvedMembers: map[string]error{ - "undefined": errors.New("member does not exist"), - }, }, fixEntitlementsEntry{ - AccountCapabilityID: AccountCapabilityID{ - Address: common.Address(address), - CapabilityID: 2, - }, + CapabilityAddress: common.Address(address), + CapabilityID: 2, ReferencedType: interpreter.NewCompositeStaticTypeComputeTypeID( nil, testContractLocation, "Test.S", ), - OldAuthorization: newEntitlementSetAuthorizationFromTypeIDs( - []common.TypeID{ - e1TypeID, - e2TypeID, - }, - sema.Conjunction, - ), - NewAuthorization: newEntitlementSetAuthorizationFromTypeIDs( + Authorization: newEntitlementSetAuthorizationFromTypeIDs( []common.TypeID{ - e1TypeID, + eTypeID, }, sema.Conjunction, ), - OldAccessibleMembers: oldAccessibleMembers, - NewAccessibleMembers: newAccessibleMembers, - UnresolvedMembers: map[string]error{ - "undefined": errors.New("member does not exist"), - }, }, }, reporter.entries, diff --git a/cmd/util/cmd/generate-authorization-fixes/entitlements.go b/cmd/util/cmd/generate-authorization-fixes/entitlements.go deleted file mode 100644 index c80f34b760f..00000000000 --- a/cmd/util/cmd/generate-authorization-fixes/entitlements.go +++ /dev/null @@ -1,131 +0,0 @@ -package generate_authorization_fixes - -import ( - "errors" - "fmt" - "sort" - - "github.com/onflow/cadence/runtime/ast" - "github.com/onflow/cadence/runtime/common/orderedmap" - "github.com/onflow/cadence/runtime/interpreter" - "github.com/onflow/cadence/runtime/sema" -) - -func findMinimalAuthorization( - ty sema.Type, - neededMembers map[string]struct{}, -) ( - authorization interpreter.Authorization, - unresolvedMembers map[string]error, -) { - entitlements := &sema.EntitlementSet{} - unresolvedMembers = map[string]error{} - - // NOTE: GetMembers might return members that are actually not accessible, for DX purposes. - // We need to resolve the members and filter out the inaccessible members, - // using the error reported when resolving - - sortedNeededMembers := make([]string, 0, len(neededMembers)) - for memberName := range neededMembers { - sortedNeededMembers = append(sortedNeededMembers, memberName) - } - sort.Strings(sortedNeededMembers) - - memberResolvers := ty.GetMembers() - - for _, memberName := range sortedNeededMembers { - memberResolver, ok := memberResolvers[memberName] - if !ok { - unresolvedMembers[memberName] = errors.New("member does not exist") - continue - } - - var resolveErr error - member := memberResolver.Resolve(nil, memberName, ast.EmptyRange, func(err error) { - resolveErr = err - }) - if resolveErr != nil { - unresolvedMembers[memberName] = resolveErr - continue - } - - switch access := member.Access.(type) { - case sema.EntitlementSetAccess: - switch access.SetKind { - case sema.Conjunction: - access.Entitlements.Foreach(func(entitlementType *sema.EntitlementType, _ struct{}) { - entitlements.Add(entitlementType) - }) - - case sema.Disjunction: - entitlements.AddDisjunction(access.Entitlements) - - default: - panic(fmt.Errorf("unsupported set kind: %v", access.SetKind)) - } - - case *sema.EntitlementMapAccess: - unresolvedMembers[memberName] = fmt.Errorf( - "member has entitlement map access: %s", - access.QualifiedKeyword(), - ) - - case sema.PrimitiveAccess: - if access != sema.PrimitiveAccess(ast.AccessAll) { - unresolvedMembers[memberName] = fmt.Errorf( - "member is inaccessible: %s", - access.QualifiedKeyword(), - ) - } - - default: - panic(fmt.Errorf("unsupported access kind: %T", member.Access)) - } - } - - return entitlementSetMinimalAuthorization(entitlements), unresolvedMembers -} - -// entitlementSetMinimalAuthorization returns the minimal authorization required to access the entitlements in the set. -// It is similar to `EntitlementSet.Access()`, but it returns the minimal authorization, -// i.e. does not return a disjunction if there is only one disjunction in the set, -// and only grants one entitlement for each disjunction. -func entitlementSetMinimalAuthorization(s *sema.EntitlementSet) interpreter.Authorization { - - s.Minimize() - - var entitlements *sema.EntitlementOrderedSet - if s.Entitlements != nil && s.Entitlements.Len() > 0 { - entitlements = orderedmap.New[sema.EntitlementOrderedSet](s.Entitlements.Len()) - entitlements.SetAll(s.Entitlements) - } - - if s.Disjunctions != nil && s.Disjunctions.Len() > 0 { - if entitlements == nil { - // There are no entitlements, but disjunctions. - // Allocate a new ordered map for all entitlements in the disjunctions - // (at minimum there are two entitlements in each disjunction). - entitlements = orderedmap.New[sema.EntitlementOrderedSet](s.Disjunctions.Len() * 2) - } - - // Add one entitlement for each of the disjunctions to the entitlements - s.Disjunctions.Foreach(func(_ string, disjunction *sema.EntitlementOrderedSet) { - // Only add the first entitlement in the disjunction - entitlements.Set(disjunction.Oldest().Key, struct{}{}) - }) - } - - if entitlements == nil { - return interpreter.UnauthorizedAccess - } - - entitlementTypeIDs := orderedmap.New[sema.TypeIDOrderedSet](entitlements.Len()) - entitlements.Foreach(func(entitlement *sema.EntitlementType, _ struct{}) { - entitlementTypeIDs.Set(entitlement.ID(), struct{}{}) - }) - - return interpreter.EntitlementSetAuthorization{ - Entitlements: entitlementTypeIDs, - SetKind: sema.Conjunction, - } -} diff --git a/cmd/util/cmd/generate-authorization-fixes/entitlements_test.go b/cmd/util/cmd/generate-authorization-fixes/entitlements_test.go deleted file mode 100644 index 362f1ec61c1..00000000000 --- a/cmd/util/cmd/generate-authorization-fixes/entitlements_test.go +++ /dev/null @@ -1,235 +0,0 @@ -package generate_authorization_fixes - -import ( - "errors" - "testing" - - "github.com/onflow/cadence/runtime/common" - "github.com/onflow/cadence/runtime/interpreter" - "github.com/onflow/cadence/runtime/sema" - . "github.com/onflow/cadence/runtime/tests/checker" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func newEntitlementSetAuthorizationFromEntitlementTypes( - entitlements []*sema.EntitlementType, - kind sema.EntitlementSetKind, -) interpreter.EntitlementSetAuthorization { - return interpreter.NewEntitlementSetAuthorization( - nil, - func() []common.TypeID { - typeIDs := make([]common.TypeID, len(entitlements)) - for i, e := range entitlements { - typeIDs[i] = e.ID() - } - return typeIDs - }, - len(entitlements), - kind, - ) -} - -func TestFindMinimalAuthorization(t *testing.T) { - - t.Parallel() - - checker, err := ParseAndCheck(t, ` - entitlement E1 - entitlement E2 - entitlement E3 - - entitlement mapping M {} - - struct S { - access(all) fun accessAll() {} - access(self) fun accessSelf() {} - access(contract) fun accessContract() {} - access(account) fun accessAccount() {} - - access(mapping M) let accessMapping: auth(mapping M) &Int - - access(E1) fun accessE1() {} - access(E2) fun accessE2() {} - access(E1, E2) fun accessE1AndE2() {} - access(E1 | E2) fun accessE1OrE2() {} - - init() { - self.accessMapping = &0 - } - } - `) - require.NoError(t, err) - - ty := RequireGlobalType(t, checker.Elaboration, "S") - - e1 := RequireGlobalType(t, checker.Elaboration, "E1").(*sema.EntitlementType) - e2 := RequireGlobalType(t, checker.Elaboration, "E2").(*sema.EntitlementType) - - t.Run("accessAll, accessSelf, accessContract, accessAccount, accessMapping", func(t *testing.T) { - t.Parallel() - - authorization, unresolved := findMinimalAuthorization( - ty, - map[string]struct{}{ - "accessAll": {}, - "accessSelf": {}, - "accessContract": {}, - "accessAccount": {}, - "accessMapping": {}, - "undefined": {}, - }, - ) - assert.Equal(t, - interpreter.UnauthorizedAccess, - authorization, - ) - assert.Equal(t, - map[string]error{ - "accessSelf": errors.New("member is inaccessible: access(self)"), - "accessContract": errors.New("member is inaccessible: access(contract)"), - "accessAccount": errors.New("member is inaccessible: access(account)"), - "accessMapping": errors.New("member has entitlement map access: access(mapping M)"), - "undefined": errors.New("member does not exist"), - }, - unresolved, - ) - }) - - t.Run("accessE1", func(t *testing.T) { - t.Parallel() - - authorization, unresolved := findMinimalAuthorization( - ty, - map[string]struct{}{ - "accessE1": {}, - "undefined": {}, - }, - ) - assert.Equal(t, - newEntitlementSetAuthorizationFromEntitlementTypes( - []*sema.EntitlementType{ - e1, - }, - sema.Conjunction, - ), - authorization, - ) - assert.Equal(t, - map[string]error{ - "undefined": errors.New("member does not exist"), - }, - unresolved, - ) - }) - - t.Run("accessE1, accessE2", func(t *testing.T) { - t.Parallel() - - authorization, unresolved := findMinimalAuthorization( - ty, - map[string]struct{}{ - "accessE1": {}, - "accessE2": {}, - "undefined": {}, - }, - ) - assert.Equal(t, - newEntitlementSetAuthorizationFromEntitlementTypes( - []*sema.EntitlementType{ - e1, e2, - }, - sema.Conjunction, - ), - authorization, - ) - assert.Equal(t, - map[string]error{ - "undefined": errors.New("member does not exist"), - }, - unresolved, - ) - }) - - t.Run("accessE1AndE2", func(t *testing.T) { - t.Parallel() - - authorization, unresolved := findMinimalAuthorization( - ty, - map[string]struct{}{ - "accessE1AndE2": {}, - "undefined": {}, - }, - ) - assert.Equal(t, - newEntitlementSetAuthorizationFromEntitlementTypes( - []*sema.EntitlementType{ - e1, e2, - }, - sema.Conjunction, - ), - authorization, - ) - assert.Equal(t, - map[string]error{ - "undefined": errors.New("member does not exist"), - }, - unresolved, - ) - }) - - t.Run("accessE1OrE2", func(t *testing.T) { - t.Parallel() - - authorization, unresolved := findMinimalAuthorization( - ty, - map[string]struct{}{ - "accessE1OrE2": {}, - "undefined": {}, - }, - ) - assert.Equal(t, - newEntitlementSetAuthorizationFromEntitlementTypes( - []*sema.EntitlementType{ - e1, - }, - sema.Conjunction, - ), - authorization, - ) - assert.Equal(t, - map[string]error{ - "undefined": errors.New("member does not exist"), - }, - unresolved, - ) - }) - - t.Run("accessE1OrE2, accessE1AndE2", func(t *testing.T) { - t.Parallel() - - authorization, unresolved := findMinimalAuthorization( - ty, - map[string]struct{}{ - "accessE1OrE2": {}, - "accessE1AndE2": {}, - "undefined": {}, - }, - ) - assert.Equal(t, - newEntitlementSetAuthorizationFromEntitlementTypes( - []*sema.EntitlementType{ - e1, e2, - }, - sema.Conjunction, - ), - authorization, - ) - assert.Equal(t, - map[string]error{ - "undefined": errors.New("member does not exist"), - }, - unresolved, - ) - }) -} diff --git a/cmd/util/cmd/generate-authorization-fixes/link_migration_report.go b/cmd/util/cmd/generate-authorization-fixes/link_migration_report.go index 1d096e8c555..b5888d8cf92 100644 --- a/cmd/util/cmd/generate-authorization-fixes/link_migration_report.go +++ b/cmd/util/cmd/generate-authorization-fixes/link_migration_report.go @@ -15,23 +15,23 @@ type AccountCapabilityID struct { CapabilityID uint64 } -// PublicLinkMigrationReport is a mapping from account capability controller IDs to public path identifier. -type PublicLinkMigrationReport map[AccountCapabilityID]string +// MigratedPublicLinkSet is a set of capability controller IDs which were migrated from public links. +type MigratedPublicLinkSet map[AccountCapabilityID]struct{} -// ReadPublicLinkMigrationReport reads a link migration report from the given reader, -// and extracts the public paths that were migrated. +// ReadMigratedPublicLinkSet reads a link migration report from the given reader, +// and returns a set of all capability controller IDs which were migrated from public links. // // The report is expected to be a JSON array of objects with the following structure: // // [ // {"kind":"link-migration-success","account_address":"0x1","path":"/public/foo","capability_id":1}, // ] -func ReadPublicLinkMigrationReport( +func ReadMigratedPublicLinkSet( reader io.Reader, filter map[common.Address]struct{}, -) (PublicLinkMigrationReport, error) { +) (MigratedPublicLinkSet, error) { - mapping := PublicLinkMigrationReport{} + set := MigratedPublicLinkSet{} dec := json.NewDecoder(reader) @@ -59,8 +59,7 @@ func ReadPublicLinkMigrationReport( continue } - identifier, ok := strings.CutPrefix(entry.Path, "/public/") - if !ok { + if !strings.HasPrefix(entry.Path, "/public/") { continue } @@ -75,11 +74,11 @@ func ReadPublicLinkMigrationReport( } } - key := AccountCapabilityID{ + accountCapabilityID := AccountCapabilityID{ Address: address, CapabilityID: entry.CapabilityID, } - mapping[key] = identifier + set[accountCapabilityID] = struct{}{} } token, err = dec.Token() @@ -90,5 +89,5 @@ func ReadPublicLinkMigrationReport( return nil, fmt.Errorf("expected end of array, got %s", token) } - return mapping, nil + return set, nil } diff --git a/cmd/util/cmd/generate-authorization-fixes/link_migration_report_test.go b/cmd/util/cmd/generate-authorization-fixes/link_migration_report_test.go index de03ffb0e35..fd7ffac5ed9 100644 --- a/cmd/util/cmd/generate-authorization-fixes/link_migration_report_test.go +++ b/cmd/util/cmd/generate-authorization-fixes/link_migration_report_test.go @@ -24,19 +24,19 @@ func TestReadPublicLinkMigrationReport(t *testing.T) { reader := strings.NewReader(contents) - mapping, err := ReadPublicLinkMigrationReport(reader, nil) + mapping, err := ReadMigratedPublicLinkSet(reader, nil) require.NoError(t, err) require.Equal(t, - PublicLinkMigrationReport{ + MigratedPublicLinkSet{ { Address: common.MustBytesToAddress([]byte{0x1}), CapabilityID: 1, - }: "foo", + }: struct{}{}, { Address: common.MustBytesToAddress([]byte{0x3}), CapabilityID: 3, - }: "baz", + }: struct{}{}, }, mapping, ) @@ -49,7 +49,7 @@ func TestReadPublicLinkMigrationReport(t *testing.T) { reader := strings.NewReader(contents) - mapping, err := ReadPublicLinkMigrationReport( + mapping, err := ReadMigratedPublicLinkSet( reader, map[common.Address]struct{}{ address1: {}, @@ -58,11 +58,11 @@ func TestReadPublicLinkMigrationReport(t *testing.T) { require.NoError(t, err) require.Equal(t, - PublicLinkMigrationReport{ + MigratedPublicLinkSet{ { Address: address1, CapabilityID: 1, - }: "foo", + }: struct{}{}, }, mapping, ) diff --git a/cmd/util/cmd/generate-authorization-fixes/link_report.go b/cmd/util/cmd/generate-authorization-fixes/link_report.go deleted file mode 100644 index 392fa41e519..00000000000 --- a/cmd/util/cmd/generate-authorization-fixes/link_report.go +++ /dev/null @@ -1,90 +0,0 @@ -package generate_authorization_fixes - -import ( - "encoding/json" - "fmt" - "io" - - "github.com/onflow/cadence/runtime/common" -) - -// AddressPublicPath is a public path in an account. -type AddressPublicPath struct { - Address common.Address - Identifier string -} - -type LinkInfo struct { - BorrowType common.TypeID - AccessibleMembers []string -} - -// PublicLinkReport is a mapping from public account paths to link info. -type PublicLinkReport map[AddressPublicPath]LinkInfo - -// ReadPublicLinkReport reads a link report from the given reader. -// The report is expected to be a JSON array of objects with the following structure: -// -// [ -// {"address":"0x1","identifier":"foo","linkType":"&Foo","accessibleMembers":["foo"]} -// ] -func ReadPublicLinkReport( - reader io.Reader, - filter map[common.Address]struct{}, -) (PublicLinkReport, error) { - - report := PublicLinkReport{} - - dec := json.NewDecoder(reader) - - token, err := dec.Token() - if err != nil { - return nil, fmt.Errorf("failed to read token: %w", err) - } - if token != json.Delim('[') { - return nil, fmt.Errorf("expected start of array, got %s", token) - } - - for dec.More() { - var entry struct { - Address string `json:"address"` - Identifier string `json:"identifier"` - LinkTypeID string `json:"linkType"` - AccessibleMembers []string `json:"accessibleMembers"` - } - err := dec.Decode(&entry) - if err != nil { - return nil, fmt.Errorf("failed to decode entry: %w", err) - } - - address, err := common.HexToAddress(entry.Address) - if err != nil { - return nil, fmt.Errorf("failed to parse address: %w", err) - } - - if filter != nil { - if _, ok := filter[address]; !ok { - continue - } - } - - key := AddressPublicPath{ - Address: address, - Identifier: entry.Identifier, - } - report[key] = LinkInfo{ - BorrowType: common.TypeID(entry.LinkTypeID), - AccessibleMembers: entry.AccessibleMembers, - } - } - - token, err = dec.Token() - if err != nil { - return nil, fmt.Errorf("failed to read token: %w", err) - } - if token != json.Delim(']') { - return nil, fmt.Errorf("expected end of array, got %s", token) - } - - return report, nil -} diff --git a/cmd/util/cmd/generate-authorization-fixes/link_report_test.go b/cmd/util/cmd/generate-authorization-fixes/link_report_test.go deleted file mode 100644 index 0227685ce48..00000000000 --- a/cmd/util/cmd/generate-authorization-fixes/link_report_test.go +++ /dev/null @@ -1,79 +0,0 @@ -package generate_authorization_fixes - -import ( - "strings" - "testing" - - "github.com/onflow/cadence/runtime/common" - "github.com/stretchr/testify/require" -) - -func TestReadLinkReport(t *testing.T) { - t.Parallel() - - contents := ` - [ - {"address":"0x1","identifier":"foo","linkType":"&Foo","accessibleMembers":["foo"]}, - {"address":"0x2","identifier":"bar","linkType":"&Bar","accessibleMembers":null} - ] - ` - - t.Run("unfiltered", func(t *testing.T) { - - t.Parallel() - - reader := strings.NewReader(contents) - - mapping, err := ReadPublicLinkReport(reader, nil) - require.NoError(t, err) - - require.Equal(t, - PublicLinkReport{ - { - Address: common.MustBytesToAddress([]byte{0x1}), - Identifier: "foo", - }: { - BorrowType: "&Foo", - AccessibleMembers: []string{"foo"}, - }, - { - Address: common.MustBytesToAddress([]byte{0x2}), - Identifier: "bar", - }: { - BorrowType: "&Bar", - AccessibleMembers: nil, - }, - }, - mapping, - ) - }) - - t.Run("filtered", func(t *testing.T) { - - t.Parallel() - - address1 := common.MustBytesToAddress([]byte{0x1}) - - reader := strings.NewReader(contents) - - mapping, err := ReadPublicLinkReport( - reader, - map[common.Address]struct{}{ - address1: {}, - }) - require.NoError(t, err) - - require.Equal(t, - PublicLinkReport{ - { - Address: address1, - Identifier: "foo", - }: { - BorrowType: "&Foo", - AccessibleMembers: []string{"foo"}, - }, - }, - mapping, - ) - }) -} diff --git a/cmd/util/ledger/migrations/fix_authorizations_migration.go b/cmd/util/ledger/migrations/fix_authorizations_migration.go index b7d5df4a3f7..43826c87151 100644 --- a/cmd/util/ledger/migrations/fix_authorizations_migration.go +++ b/cmd/util/ledger/migrations/fix_authorizations_migration.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" "io" - "strings" "github.com/onflow/cadence/migrations" "github.com/onflow/cadence/runtime" @@ -32,18 +31,16 @@ type FixAuthorizationsMigrationReporter interface { storageKey interpreter.StorageKey, capabilityAddress common.Address, capabilityID uint64, - newAuthorization interpreter.Authorization, ) MigratedCapabilityController( storageKey interpreter.StorageKey, capabilityID uint64, - newAuthorization interpreter.Authorization, ) } type FixAuthorizationsMigration struct { - Reporter FixAuthorizationsMigrationReporter - NewAuthorizations map[AccountCapabilityID]interpreter.Authorization + Reporter FixAuthorizationsMigrationReporter + AuthorizationFixes AuthorizationFixes } var _ migrations.ValueMigration = &FixAuthorizationsMigration{} @@ -71,12 +68,12 @@ func (m *FixAuthorizationsMigration) Migrate( capabilityAddress := common.Address(value.Address()) capabilityID := uint64(value.ID) - newAuthorization := m.NewAuthorizations[AccountCapabilityID{ + _, ok := m.AuthorizationFixes[AccountCapabilityID{ Address: capabilityAddress, CapabilityID: capabilityID, }] - if newAuthorization == nil { - // Nothing to fix for this capability + if !ok { + // This capability does not need to be fixed return nil, nil } @@ -102,7 +99,7 @@ func (m *FixAuthorizationsMigration) Migrate( newBorrowType := interpreter.NewReferenceStaticType( nil, - newAuthorization, + interpreter.UnauthorizedAccess, oldBorrowReferenceType.ReferencedType, ) newCapabilityValue := interpreter.NewUnmeteredCapabilityValue( @@ -115,7 +112,6 @@ func (m *FixAuthorizationsMigration) Migrate( storageKey, capabilityAddress, capabilityID, - newAuthorization, ) return newCapabilityValue, nil @@ -126,12 +122,12 @@ func (m *FixAuthorizationsMigration) Migrate( capabilityAddress := storageKey.Address capabilityID := uint64(value.CapabilityID) - newAuthorization := m.NewAuthorizations[AccountCapabilityID{ + _, ok := m.AuthorizationFixes[AccountCapabilityID{ Address: capabilityAddress, CapabilityID: capabilityID, }] - if newAuthorization == nil { - // Nothing to fix for this capability controller + if !ok { + // This capability controller does not need to be fixed return nil, nil } @@ -139,7 +135,7 @@ func (m *FixAuthorizationsMigration) Migrate( newBorrowType := interpreter.NewReferenceStaticType( nil, - newAuthorization, + interpreter.UnauthorizedAccess, oldBorrowReferenceType.ReferencedType, ) newStorageCapabilityControllerValue := interpreter.NewUnmeteredStorageCapabilityControllerValue( @@ -151,7 +147,6 @@ func (m *FixAuthorizationsMigration) Migrate( m.Reporter.MigratedCapabilityController( storageKey, capabilityID, - newAuthorization, ) return newStorageCapabilityControllerValue, nil @@ -242,7 +237,7 @@ func NewFixAuthorizationsMigration( return []migrations.ValueMigration{ &FixAuthorizationsMigration{ - NewAuthorizations: newAuthorizations, + AuthorizationFixes: newAuthorizations, Reporter: &fixAuthorizationsMigrationReporter{ reportWriter: reporter, errorMessageHandler: errorMessageHandler, @@ -315,12 +310,10 @@ func (r *fixAuthorizationsMigrationReporter) DictionaryKeyConflict(accountAddres func (r *fixAuthorizationsMigrationReporter) MigratedCapabilityController( storageKey interpreter.StorageKey, capabilityID uint64, - newAuthorization interpreter.Authorization, ) { r.reportWriter.Write(capabilityControllerAuthorizationFixedEntry{ - StorageKey: storageKey, - CapabilityID: capabilityID, - NewAuthorization: newAuthorization, + StorageKey: storageKey, + CapabilityID: capabilityID, }) } @@ -328,47 +321,33 @@ func (r *fixAuthorizationsMigrationReporter) MigratedCapability( storageKey interpreter.StorageKey, capabilityAddress common.Address, capabilityID uint64, - newAuthorization interpreter.Authorization, ) { r.reportWriter.Write(capabilityAuthorizationFixedEntry{ StorageKey: storageKey, CapabilityAddress: capabilityAddress, CapabilityID: capabilityID, - NewAuthorization: newAuthorization, }) } -func jsonEncodeAuthorization(authorization interpreter.Authorization) string { - switch authorization { - case interpreter.UnauthorizedAccess, interpreter.InaccessibleAccess: - return "" - default: - return string(authorization.ID()) - } -} - // capabilityControllerAuthorizationFixedEntry type capabilityControllerAuthorizationFixedEntry struct { - StorageKey interpreter.StorageKey - CapabilityID uint64 - NewAuthorization interpreter.Authorization + StorageKey interpreter.StorageKey + CapabilityID uint64 } var _ json.Marshaler = capabilityControllerAuthorizationFixedEntry{} func (e capabilityControllerAuthorizationFixedEntry) MarshalJSON() ([]byte, error) { return json.Marshal(struct { - Kind string `json:"kind"` - AccountAddress string `json:"account_address"` - StorageDomain string `json:"domain"` - CapabilityID uint64 `json:"capability_id"` - NewAuthorization string `json:"new_authorization"` + Kind string `json:"kind"` + AccountAddress string `json:"account_address"` + StorageDomain string `json:"domain"` + CapabilityID uint64 `json:"capability_id"` }{ - Kind: "capability-controller-authorizations-fixed", - AccountAddress: e.StorageKey.Address.HexWithPrefix(), - StorageDomain: e.StorageKey.Key, - CapabilityID: e.CapabilityID, - NewAuthorization: jsonEncodeAuthorization(e.NewAuthorization), + Kind: "capability-controller-authorizations-fixed", + AccountAddress: e.StorageKey.Address.HexWithPrefix(), + StorageDomain: e.StorageKey.Key, + CapabilityID: e.CapabilityID, }) } @@ -377,7 +356,6 @@ type capabilityAuthorizationFixedEntry struct { StorageKey interpreter.StorageKey CapabilityAddress common.Address CapabilityID uint64 - NewAuthorization interpreter.Authorization } var _ json.Marshaler = capabilityAuthorizationFixedEntry{} @@ -389,14 +367,12 @@ func (e capabilityAuthorizationFixedEntry) MarshalJSON() ([]byte, error) { StorageDomain string `json:"domain"` CapabilityAddress string `json:"capability_address"` CapabilityID uint64 `json:"capability_id"` - NewAuthorization string `json:"new_authorization"` }{ Kind: "capability-authorizations-fixed", AccountAddress: e.StorageKey.Address.HexWithPrefix(), StorageDomain: e.StorageKey.Key, CapabilityAddress: e.CapabilityAddress.HexWithPrefix(), CapabilityID: e.CapabilityID, - NewAuthorization: jsonEncodeAuthorization(e.NewAuthorization), }) } @@ -449,13 +425,13 @@ func NewFixAuthorizationsMigrations( } } -type AuthorizationFixes map[AccountCapabilityID]interpreter.Authorization +type AuthorizationFixes map[AccountCapabilityID]struct{} // ReadAuthorizationFixes reads a report of authorization fixes from the given reader. // The report is expected to be a JSON array of objects with the following structure: // // [ -// {"address":"0x1","identifier":"foo","linkType":"&Foo","accessibleMembers":["foo"]} +// {"capability_address":"0x1","capability_id":1} // ] func ReadAuthorizationFixes( reader io.Reader, @@ -478,7 +454,6 @@ func ReadAuthorizationFixes( var entry struct { CapabilityAddress string `json:"capability_address"` CapabilityID uint64 `json:"capability_id"` - NewAuthorization string `json:"new_authorization"` } err := dec.Decode(&entry) if err != nil { @@ -496,17 +471,12 @@ func ReadAuthorizationFixes( } } - newAuthorization, err := jsonDecodeAuthorization(entry.NewAuthorization) - if err != nil { - return nil, fmt.Errorf("failed to decode new authorization '%s': %w", entry.NewAuthorization, err) - } - accountCapabilityID := AccountCapabilityID{ Address: address, CapabilityID: entry.CapabilityID, } - fixes[accountCapabilityID] = newAuthorization + fixes[accountCapabilityID] = struct{}{} } token, err = dec.Token() @@ -519,29 +489,3 @@ func ReadAuthorizationFixes( return fixes, nil } - -func jsonDecodeAuthorization(encoded string) (interpreter.Authorization, error) { - if encoded == "" { - return interpreter.UnauthorizedAccess, nil - } - - if strings.Contains(encoded, "|") { - return nil, fmt.Errorf("invalid disjunction entitlement set authorization: %s", encoded) - } - - var typeIDs []common.TypeID - for _, part := range strings.Split(encoded, ",") { - typeIDs = append(typeIDs, common.TypeID(part)) - } - - entitlementSetAuthorization := interpreter.NewEntitlementSetAuthorization( - nil, - func() []common.TypeID { - return typeIDs - }, - len(typeIDs), - sema.Conjunction, - ) - - return entitlementSetAuthorization, nil -} diff --git a/cmd/util/ledger/migrations/fix_authorizations_migration_test.go b/cmd/util/ledger/migrations/fix_authorizations_migration_test.go index 69e2f250792..37c2c06fb7f 100644 --- a/cmd/util/ledger/migrations/fix_authorizations_migration_test.go +++ b/cmd/util/ledger/migrations/fix_authorizations_migration_test.go @@ -65,14 +65,10 @@ func TestFixAuthorizationsMigration(t *testing.T) { const contractCode = ` access(all) contract Test { - access(all) entitlement E1 - access(all) entitlement E2 - access(all) struct S { - access(E1) fun f1() {} - access(E2) fun f2() {} - access(all) fun f3() {} - } + access(all) entitlement E + + access(all) struct S {} } ` @@ -104,29 +100,41 @@ func TestFixAuthorizationsMigration(t *testing.T) { prepare(signer: auth(Storage, Capabilities) &Account) { signer.storage.save(Test.S(), to: /storage/s) - // Capability 1 was a public, unauthorized capability. - // It should lose entitlement E2 - let cap1 = signer.capabilities.storage.issue(/storage/s) + // Capability 1 was a public, unauthorized capability, which is now authorized. + // It should lose its entitlement + let cap1 = signer.capabilities.storage.issue(/storage/s) assert(cap1.borrow() != nil) - signer.capabilities.publish(cap1, at: /public/s) + signer.capabilities.publish(cap1, at: /public/s1) - // Capability 2 was a public, unauthorized capability, stored nested in storage. - // It should lose entitlement E2 - let cap2 = signer.capabilities.storage.issue(/storage/s) + // Capability 2 was a public, unauthorized capability, which is now authorized. + // It is currently only stored, nested, in storage, and is not published. + // It should lose its entitlement + let cap2 = signer.capabilities.storage.issue(/storage/s) assert(cap2.borrow() != nil) signer.storage.save([cap2], to: /storage/caps2) - // Capability 3 was a private, authorized capability, stored nested in storage. - // It should keep entitlement E2 - let cap3 = signer.capabilities.storage.issue(/storage/s) + // Capability 3 was a private, authorized capability. + // It is currently only stored, nested, in storage, and is not published. + // It should keep its entitlement + let cap3 = signer.capabilities.storage.issue(/storage/s) assert(cap3.borrow() != nil) signer.storage.save([cap3], to: /storage/caps3) - // Capability 4 was a capability with unavailable accessible members, stored nested in storage. - // It should keep entitlement E2 - let cap4 = signer.capabilities.storage.issue(/storage/s) + // Capability 4 was a private, authorized capability. + // It is currently both stored, nested, in storage, and is published. + // It should keep its entitlement + let cap4 = signer.capabilities.storage.issue(/storage/s) assert(cap4.borrow() != nil) signer.storage.save([cap4], to: /storage/caps4) + signer.capabilities.publish(cap4, at: /public/s4) + + // Capability 5 was a public, unauthorized capability, which is still unauthorized. + // It is currently both stored, nested, in storage, and is published. + // There is no need to fix it. + let cap5 = signer.capabilities.storage.issue<&Test.S>(/storage/s) + assert(cap5.borrow() != nil) + signer.storage.save([cap5], to: /storage/caps5) + signer.capabilities.publish(cap5, at: /public/s5) } } `, @@ -151,28 +159,15 @@ func TestFixAuthorizationsMigration(t *testing.T) { NWorker: nWorker, } - testContractLocation := common.AddressLocation{ - Address: common.Address(address), - Name: "Test", - } - e1TypeID := testContractLocation.TypeID(nil, "Test.E1") - - fixedAuthorization := newEntitlementSetAuthorizationFromTypeIDs( - []common.TypeID{ - e1TypeID, - }, - sema.Conjunction, - ) - - fixes := map[AccountCapabilityID]interpreter.Authorization{ + fixes := AuthorizationFixes{ AccountCapabilityID{ Address: common.Address(address), CapabilityID: 1, - }: fixedAuthorization, + }: {}, AccountCapabilityID{ Address: common.Address(address), CapabilityID: 2, - }: fixedAuthorization, + }: {}, } migrations := NewFixAuthorizationsMigrations( @@ -208,16 +203,14 @@ func TestFixAuthorizationsMigration(t *testing.T) { Key: "cap_con", Address: common.Address(address), }, - CapabilityID: 1, - NewAuthorization: fixedAuthorization, + CapabilityID: 1, }, capabilityControllerAuthorizationFixedEntry{ StorageKey: interpreter.StorageKey{ Key: "cap_con", Address: common.Address(address), }, - CapabilityID: 2, - NewAuthorization: fixedAuthorization, + CapabilityID: 2, }, capabilityAuthorizationFixedEntry{ StorageKey: interpreter.StorageKey{ @@ -226,7 +219,6 @@ func TestFixAuthorizationsMigration(t *testing.T) { }, CapabilityAddress: common.Address(address), CapabilityID: 1, - NewAuthorization: fixedAuthorization, }, capabilityAuthorizationFixedEntry{ StorageKey: interpreter.StorageKey{ @@ -235,7 +227,6 @@ func TestFixAuthorizationsMigration(t *testing.T) { }, CapabilityAddress: common.Address(address), CapabilityID: 2, - NewAuthorization: fixedAuthorization, }, }, entries, @@ -249,31 +240,33 @@ func TestFixAuthorizationsMigration(t *testing.T) { fmt.Sprintf( //language=Cadence ` - import Test from %s - - access(all) - fun main() { - let account = getAuthAccount(%[1]s) - // NOTE: capability can NOT be borrowed with E2 anymore - assert(account.capabilities.borrow(/public/s) == nil) - assert(account.capabilities.borrow(/public/s) != nil) - - let caps2 = account.storage.copy<[Capability]>(from: /storage/caps2)! - // NOTE: capability can NOT be borrowed with E2 anymore - assert(caps2[0].borrow() == nil) - assert(caps2[0].borrow() != nil) - - let caps3 = account.storage.copy<[Capability]>(from: /storage/caps3)! - // NOTE: capability can still be borrowed with E2 - assert(caps3[0].borrow() != nil) - assert(caps3[0].borrow() != nil) - - let caps4 = account.storage.copy<[Capability]>(from: /storage/caps4)! - // NOTE: capability can still be borrowed with E2 - assert(caps4[0].borrow() != nil) - assert(caps4[0].borrow() != nil) - } - `, + import Test from %s + + access(all) + fun main() { + let account = getAuthAccount(%[1]s) + // NOTE: capability can NOT be borrowed with E anymore + assert(account.capabilities.borrow(/public/s1) == nil) + assert(account.capabilities.borrow<&Test.S>(/public/s1) != nil) + + let caps2 = account.storage.copy<[Capability]>(from: /storage/caps2)! + // NOTE: capability can NOT be borrowed with E anymore + assert(caps2[0].borrow() == nil) + assert(caps2[0].borrow<&Test.S>() != nil) + + let caps3 = account.storage.copy<[Capability]>(from: /storage/caps3)! + // NOTE: capability can still be borrowed with E + assert(caps3[0].borrow() != nil) + assert(caps3[0].borrow<&Test.S>() != nil) + + let caps4 = account.storage.copy<[Capability]>(from: /storage/caps4)! + // NOTE: capability can still be borrowed with E + assert(account.capabilities.borrow(/public/s4) != nil) + assert(account.capabilities.borrow<&Test.S>(/public/s4) != nil) + assert(caps4[0].borrow() != nil) + assert(caps4[0].borrow<&Test.S>() != nil) + } + `, address.HexWithPrefix(), ), ) @@ -285,9 +278,9 @@ func TestReadAuthorizationFixes(t *testing.T) { validContents := ` [ - {"capability_address":"01","capability_id":4,"new_authorization":""}, - {"capability_address":"02","capability_id":5,"new_authorization":"A.0000000000000001.Foo.Bar"}, - {"capability_address":"03","capability_id":6,"new_authorization":"A.0000000000000001.Foo.Bar,A.0000000000000001.Foo.Baz"} + {"capability_address":"01","capability_id":4}, + {"capability_address":"02","capability_id":5}, + {"capability_address":"03","capability_id":6} ] ` @@ -305,26 +298,15 @@ func TestReadAuthorizationFixes(t *testing.T) { { Address: common.MustBytesToAddress([]byte{0x1}), CapabilityID: 4, - }: interpreter.UnauthorizedAccess, + }: {}, { Address: common.MustBytesToAddress([]byte{0x2}), CapabilityID: 5, - }: newEntitlementSetAuthorizationFromTypeIDs( - []common.TypeID{ - "A.0000000000000001.Foo.Bar", - }, - sema.Conjunction, - ), + }: {}, { Address: common.MustBytesToAddress([]byte{0x3}), CapabilityID: 6, - }: newEntitlementSetAuthorizationFromTypeIDs( - []common.TypeID{ - "A.0000000000000001.Foo.Bar", - "A.0000000000000001.Foo.Baz", - }, - sema.Conjunction, - ), + }: {}, }, mapping, ) @@ -352,33 +334,13 @@ func TestReadAuthorizationFixes(t *testing.T) { { Address: common.MustBytesToAddress([]byte{0x1}), CapabilityID: 4, - }: interpreter.UnauthorizedAccess, + }: {}, { Address: common.MustBytesToAddress([]byte{0x3}), CapabilityID: 6, - }: newEntitlementSetAuthorizationFromTypeIDs( - []common.TypeID{ - "A.0000000000000001.Foo.Bar", - "A.0000000000000001.Foo.Baz", - }, - sema.Conjunction, - ), + }: {}, }, mapping, ) }) - - t.Run("invalid disjunction entitlement set authorization", func(t *testing.T) { - - t.Parallel() - - reader := strings.NewReader(` - [ - {"capability_address":"03","capability_id":6,"new_authorization":"A.0000000000000001.Foo.Bar|A.0000000000000001.Foo.Baz"} - ] - `) - - _, err := ReadAuthorizationFixes(reader, nil) - require.ErrorContains(t, err, "invalid disjunction entitlement set authorization") - }) } From 40241dd65911be90cbabc1c67baf873eeab67265 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Wed, 11 Sep 2024 18:19:03 -0700 Subject: [PATCH 64/72] remove unnecessary contract checking --- .../fix_authorizations_migration.go | 44 +++---------------- 1 file changed, 7 insertions(+), 37 deletions(-) diff --git a/cmd/util/ledger/migrations/fix_authorizations_migration.go b/cmd/util/ledger/migrations/fix_authorizations_migration.go index 43826c87151..ff4ab7898af 100644 --- a/cmd/util/ledger/migrations/fix_authorizations_migration.go +++ b/cmd/util/ledger/migrations/fix_authorizations_migration.go @@ -7,7 +7,6 @@ import ( "io" "github.com/onflow/cadence/migrations" - "github.com/onflow/cadence/runtime" "github.com/onflow/cadence/runtime/common" cadenceErrors "github.com/onflow/cadence/runtime/errors" "github.com/onflow/cadence/runtime/interpreter" @@ -210,9 +209,7 @@ const fixAuthorizationsMigrationReporterName = "fix-authorizations-migration" func NewFixAuthorizationsMigration( rwf reporters.ReportWriterFactory, - errorMessageHandler *errorMessageHandler, - programs map[runtime.Location]*interpreter.Program, - newAuthorizations AuthorizationFixes, + authorizationFixes AuthorizationFixes, opts Options, ) *CadenceBaseMigration { var diffReporter reporters.ReportWriter @@ -237,18 +234,15 @@ func NewFixAuthorizationsMigration( return []migrations.ValueMigration{ &FixAuthorizationsMigration{ - AuthorizationFixes: newAuthorizations, + AuthorizationFixes: authorizationFixes, Reporter: &fixAuthorizationsMigrationReporter{ - reportWriter: reporter, - errorMessageHandler: errorMessageHandler, - verboseErrorOutput: opts.VerboseErrorOutput, + reportWriter: reporter, + verboseErrorOutput: opts.VerboseErrorOutput, }, }, } }, - errorMessageHandler: errorMessageHandler, - programs: programs, - chainID: opts.ChainID, + chainID: opts.ChainID, } } @@ -379,33 +373,11 @@ func (e capabilityAuthorizationFixedEntry) MarshalJSON() ([]byte, error) { func NewFixAuthorizationsMigrations( log zerolog.Logger, rwf reporters.ReportWriterFactory, - newAuthorizations AuthorizationFixes, + authorizationFixes AuthorizationFixes, opts Options, ) []NamedMigration { - errorMessageHandler := &errorMessageHandler{} - - // The value migrations are run as account-based migrations, - // i.e. the migrations are only given the payloads for the account to be migrated. - // However, the migrations need to be able to get the code for contracts of any account. - // - // To achieve this, the contracts are extracted from the payloads once, - // before the value migrations are run. - - programs := make(map[common.Location]*interpreter.Program, 1000) - return []NamedMigration{ - { - Name: "check-contracts", - Migrate: NewContractCheckingMigration( - log, - rwf, - opts.ChainID, - opts.VerboseErrorOutput, - nil, - programs, - ), - }, { Name: "fix-authorizations", Migrate: NewAccountBasedMigration( @@ -414,9 +386,7 @@ func NewFixAuthorizationsMigrations( []AccountBasedMigration{ NewFixAuthorizationsMigration( rwf, - errorMessageHandler, - programs, - newAuthorizations, + authorizationFixes, opts, ), }, From e6392a31d48403c3c63d7d198dea51d9176456cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Thu, 12 Sep 2024 11:36:11 -0700 Subject: [PATCH 65/72] add REST API to run-script command --- cmd/util/cmd/run-script/cmd.go | 416 ++++++++++++++++++++++++++++++++- 1 file changed, 404 insertions(+), 12 deletions(-) diff --git a/cmd/util/cmd/run-script/cmd.go b/cmd/util/cmd/run-script/cmd.go index 7f12cef5b35..1f24d2599c2 100644 --- a/cmd/util/cmd/run-script/cmd.go +++ b/cmd/util/cmd/run-script/cmd.go @@ -1,19 +1,29 @@ package run_script import ( + "context" + "errors" + "fmt" "io" "os" jsoncdc "github.com/onflow/cadence/encoding/json" + "github.com/onflow/flow/protobuf/go/flow/entities" "github.com/rs/zerolog/log" "github.com/spf13/cobra" + "github.com/onflow/flow-go/access" "github.com/onflow/flow-go/cmd/util/ledger/util" "github.com/onflow/flow-go/cmd/util/ledger/util/registers" + "github.com/onflow/flow-go/engine/access/rest" + "github.com/onflow/flow-go/engine/access/state_stream/backend" + "github.com/onflow/flow-go/engine/access/subscription" "github.com/onflow/flow-go/engine/execution/computation" "github.com/onflow/flow-go/fvm" + "github.com/onflow/flow-go/fvm/storage/snapshot" "github.com/onflow/flow-go/ledger" "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/metrics" ) var ( @@ -21,6 +31,8 @@ var ( flagState string flagStateCommitment string flagChain string + flagServe bool + flagPort int ) var Cmd = &cobra.Command{ @@ -60,6 +72,20 @@ func init() { "Chain name", ) _ = Cmd.MarkFlagRequired("chain") + + Cmd.Flags().BoolVar( + &flagServe, + "serve", + false, + "serve with an HTTP server", + ) + + Cmd.Flags().IntVar( + &flagPort, + "port", + 8000, + "port for HTTP server", + ) } func run(*cobra.Command, []string) { @@ -75,15 +101,14 @@ func run(*cobra.Command, []string) { chainID := flow.ChainID(flagChain) // Validate chain ID - _ = chainID.Chain() + chain := chainID.Chain() - code, err := io.ReadAll(os.Stdin) - if err != nil { - log.Fatal().Msgf("failed to read script: %s", err) - } - - var payloads []*ledger.Payload + log.Info().Msg("loading state ...") + var ( + err error + payloads []*ledger.Payload + ) if flagPayloads != "" { _, payloads, err = util.ReadPayloadFile(log.Logger, flagPayloads) } else { @@ -125,23 +150,390 @@ func run(*cobra.Command, []string) { vm := fvm.NewVirtualMachine() + if flagServe { + + api := &api{ + chainID: chainID, + vm: vm, + ctx: ctx, + storageSnapshot: storageSnapshot, + } + + server, err := rest.NewServer( + api, + rest.Config{ + ListenAddress: fmt.Sprintf(":%d", flagPort), + }, + log.Logger, + chain, + metrics.NewNoopCollector(), + nil, + backend.Config{}, + ) + if err != nil { + log.Fatal().Err(err).Msg("failed to create server") + } + + log.Info().Msgf("serving on port %d", flagPort) + + err = server.ListenAndServe() + if err != nil { + log.Info().Msg("server stopped") + } + } else { + code, err := io.ReadAll(os.Stdin) + if err != nil { + log.Fatal().Msgf("failed to read script: %s", err) + } + + encodedResult, err := runScript(vm, ctx, storageSnapshot, code, nil) + if err != nil { + log.Fatal().Err(err).Msg("failed to run script") + } + + _, _ = os.Stdout.Write(encodedResult) + } +} + +func runScript( + vm *fvm.VirtualMachine, + ctx fvm.Context, + storageSnapshot snapshot.StorageSnapshot, + code []byte, + arguments [][]byte, +) ( + encodedResult []byte, + err error, +) { _, res, err := vm.Run( ctx, - fvm.Script(code), + fvm.Script(code).WithArguments(arguments...), storageSnapshot, ) if err != nil { - log.Fatal().Msgf("failed to run script: %s", err) + return nil, err } if res.Err != nil { - log.Fatal().Msgf("script failed: %s", res.Err) + return nil, res.Err } encoded, err := jsoncdc.Encode(res.Value) if err != nil { - log.Fatal().Msgf("failed to encode result: %s", err) + return nil, err } - _, _ = os.Stdout.Write(encoded) + return encoded, nil +} + +type api struct { + chainID flow.ChainID + vm *fvm.VirtualMachine + ctx fvm.Context + storageSnapshot registers.StorageSnapshot +} + +var _ access.API = &api{} + +func (*api) Ping(_ context.Context) error { + return nil +} + +func (a *api) GetNetworkParameters(_ context.Context) access.NetworkParameters { + return access.NetworkParameters{ + ChainID: a.chainID, + } +} + +func (*api) GetNodeVersionInfo(_ context.Context) (*access.NodeVersionInfo, error) { + return nil, errors.New("unimplemented") +} + +func (*api) GetLatestBlockHeader(_ context.Context, _ bool) (*flow.Header, flow.BlockStatus, error) { + return nil, flow.BlockStatusUnknown, errors.New("unimplemented") +} + +func (*api) GetBlockHeaderByHeight(_ context.Context, _ uint64) (*flow.Header, flow.BlockStatus, error) { + return nil, flow.BlockStatusUnknown, errors.New("unimplemented") +} + +func (*api) GetBlockHeaderByID(_ context.Context, _ flow.Identifier) (*flow.Header, flow.BlockStatus, error) { + return nil, flow.BlockStatusUnknown, errors.New("unimplemented") +} + +func (*api) GetLatestBlock(_ context.Context, _ bool) (*flow.Block, flow.BlockStatus, error) { + return nil, flow.BlockStatusUnknown, errors.New("unimplemented") +} + +func (*api) GetBlockByHeight(_ context.Context, _ uint64) (*flow.Block, flow.BlockStatus, error) { + return nil, flow.BlockStatusUnknown, errors.New("unimplemented") +} + +func (*api) GetBlockByID(_ context.Context, _ flow.Identifier) (*flow.Block, flow.BlockStatus, error) { + return nil, flow.BlockStatusUnknown, errors.New("unimplemented") +} + +func (*api) GetCollectionByID(_ context.Context, _ flow.Identifier) (*flow.LightCollection, error) { + return nil, errors.New("unimplemented") +} + +func (*api) GetFullCollectionByID(_ context.Context, _ flow.Identifier) (*flow.Collection, error) { + return nil, errors.New("unimplemented") +} + +func (*api) SendTransaction(_ context.Context, _ *flow.TransactionBody) error { + return errors.New("unimplemented") +} + +func (*api) GetTransaction(_ context.Context, _ flow.Identifier) (*flow.TransactionBody, error) { + return nil, errors.New("unimplemented") +} + +func (*api) GetTransactionsByBlockID(_ context.Context, _ flow.Identifier) ([]*flow.TransactionBody, error) { + return nil, errors.New("unimplemented") +} + +func (*api) GetTransactionResult( + _ context.Context, + _ flow.Identifier, + _ flow.Identifier, + _ flow.Identifier, + _ entities.EventEncodingVersion, +) (*access.TransactionResult, error) { + return nil, errors.New("unimplemented") +} + +func (*api) GetTransactionResultByIndex( + _ context.Context, + _ flow.Identifier, + _ uint32, + _ entities.EventEncodingVersion, +) (*access.TransactionResult, error) { + return nil, errors.New("unimplemented") +} + +func (*api) GetTransactionResultsByBlockID( + _ context.Context, + _ flow.Identifier, + _ entities.EventEncodingVersion, +) ([]*access.TransactionResult, error) { + return nil, errors.New("unimplemented") +} + +func (*api) GetSystemTransaction( + _ context.Context, + _ flow.Identifier, +) (*flow.TransactionBody, error) { + return nil, errors.New("unimplemented") +} + +func (*api) GetSystemTransactionResult( + _ context.Context, + _ flow.Identifier, + _ entities.EventEncodingVersion, +) (*access.TransactionResult, error) { + return nil, errors.New("unimplemented") +} + +func (*api) GetAccount(_ context.Context, _ flow.Address) (*flow.Account, error) { + return nil, errors.New("unimplemented") +} + +func (*api) GetAccountAtLatestBlock(_ context.Context, _ flow.Address) (*flow.Account, error) { + return nil, errors.New("unimplemented") +} + +func (*api) GetAccountAtBlockHeight(_ context.Context, _ flow.Address, _ uint64) (*flow.Account, error) { + return nil, errors.New("unimplemented") +} + +func (*api) GetAccountBalanceAtLatestBlock(_ context.Context, _ flow.Address) (uint64, error) { + return 0, errors.New("unimplemented") +} + +func (*api) GetAccountBalanceAtBlockHeight( + _ context.Context, + _ flow.Address, + _ uint64, +) (uint64, error) { + return 0, errors.New("unimplemented") +} + +func (*api) GetAccountKeyAtLatestBlock( + _ context.Context, + _ flow.Address, + _ uint32, +) (*flow.AccountPublicKey, error) { + return nil, errors.New("unimplemented") +} + +func (*api) GetAccountKeyAtBlockHeight( + _ context.Context, + _ flow.Address, + _ uint32, + _ uint64, +) (*flow.AccountPublicKey, error) { + return nil, errors.New("unimplemented") +} + +func (*api) GetAccountKeysAtLatestBlock( + _ context.Context, + _ flow.Address, +) ([]flow.AccountPublicKey, error) { + return nil, errors.New("unimplemented") +} + +func (*api) GetAccountKeysAtBlockHeight( + _ context.Context, + _ flow.Address, + _ uint64, +) ([]flow.AccountPublicKey, error) { + return nil, errors.New("unimplemented") +} + +func (a *api) ExecuteScriptAtLatestBlock( + _ context.Context, + script []byte, + arguments [][]byte, +) ([]byte, error) { + return runScript( + a.vm, + a.ctx, + a.storageSnapshot, + script, + arguments, + ) +} + +func (*api) ExecuteScriptAtBlockHeight( + _ context.Context, + _ uint64, + _ []byte, + _ [][]byte, +) ([]byte, error) { + return nil, errors.New("unimplemented") +} + +func (*api) ExecuteScriptAtBlockID( + _ context.Context, + _ flow.Identifier, + _ []byte, + _ [][]byte, +) ([]byte, error) { + return nil, errors.New("unimplemented") +} + +func (a *api) GetEventsForHeightRange( + _ context.Context, + _ string, + _, _ uint64, + _ entities.EventEncodingVersion, +) ([]flow.BlockEvents, error) { + return nil, errors.New("unimplemented") +} + +func (a *api) GetEventsForBlockIDs( + _ context.Context, + _ string, + _ []flow.Identifier, + _ entities.EventEncodingVersion, +) ([]flow.BlockEvents, error) { + return nil, errors.New("unimplemented") +} + +func (*api) GetLatestProtocolStateSnapshot(_ context.Context) ([]byte, error) { + return nil, errors.New("unimplemented") +} + +func (*api) GetProtocolStateSnapshotByBlockID(_ context.Context, _ flow.Identifier) ([]byte, error) { + return nil, errors.New("unimplemented") +} + +func (*api) GetProtocolStateSnapshotByHeight(_ context.Context, _ uint64) ([]byte, error) { + return nil, errors.New("unimplemented") +} + +func (*api) GetExecutionResultForBlockID(_ context.Context, _ flow.Identifier) (*flow.ExecutionResult, error) { + return nil, errors.New("unimplemented") +} + +func (*api) GetExecutionResultByID(_ context.Context, _ flow.Identifier) (*flow.ExecutionResult, error) { + return nil, errors.New("unimplemented") +} + +func (*api) SubscribeBlocksFromStartBlockID( + _ context.Context, + _ flow.Identifier, + _ flow.BlockStatus, +) subscription.Subscription { + return nil +} + +func (*api) SubscribeBlocksFromStartHeight( + _ context.Context, + _ uint64, + _ flow.BlockStatus, +) subscription.Subscription { + return nil +} + +func (*api) SubscribeBlocksFromLatest( + _ context.Context, + _ flow.BlockStatus, +) subscription.Subscription { + return nil +} + +func (*api) SubscribeBlockHeadersFromStartBlockID( + _ context.Context, + _ flow.Identifier, + _ flow.BlockStatus, +) subscription.Subscription { + return nil +} + +func (*api) SubscribeBlockHeadersFromStartHeight( + _ context.Context, + _ uint64, + _ flow.BlockStatus, +) subscription.Subscription { + return nil +} + +func (*api) SubscribeBlockHeadersFromLatest( + _ context.Context, + _ flow.BlockStatus, +) subscription.Subscription { + return nil +} + +func (*api) SubscribeBlockDigestsFromStartBlockID( + _ context.Context, + _ flow.Identifier, + _ flow.BlockStatus, +) subscription.Subscription { + return nil +} + +func (*api) SubscribeBlockDigestsFromStartHeight( + _ context.Context, + _ uint64, + _ flow.BlockStatus, +) subscription.Subscription { + return nil +} + +func (*api) SubscribeBlockDigestsFromLatest( + _ context.Context, + _ flow.BlockStatus, +) subscription.Subscription { + return nil +} + +func (*api) SubscribeTransactionStatuses( + _ context.Context, + _ *flow.TransactionBody, + _ entities.EventEncodingVersion, +) subscription.Subscription { + return nil } From b7752d9b38a4ba717da3f8fdbfc4a290c7d9b457 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Thu, 26 Sep 2024 20:38:55 -0700 Subject: [PATCH 66/72] update to Cadence v1.0.0 --- go.mod | 6 +++--- go.sum | 12 ++++++------ insecure/go.mod | 6 +++--- insecure/go.sum | 12 ++++++------ integration/go.mod | 6 +++--- integration/go.sum | 12 ++++++------ 6 files changed, 27 insertions(+), 27 deletions(-) diff --git a/go.mod b/go.mod index 15d4a7cd71e..a2a68ab22de 100644 --- a/go.mod +++ b/go.mod @@ -48,13 +48,13 @@ require ( github.com/multiformats/go-multiaddr-dns v0.3.1 github.com/multiformats/go-multihash v0.2.3 github.com/onflow/atree v0.8.0-rc.6 - github.com/onflow/cadence v1.0.0-preview.52 + github.com/onflow/cadence v1.0.0 github.com/onflow/crypto v0.25.2 github.com/onflow/flow v0.3.4 github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.1 github.com/onflow/flow-core-contracts/lib/go/templates v1.3.1 - github.com/onflow/flow-go-sdk v1.0.0-preview.54 - github.com/onflow/flow/protobuf/go/flow v0.4.5 + github.com/onflow/flow-go-sdk v1.0.0 + github.com/onflow/flow/protobuf/go/flow v0.4.7 github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 github.com/pierrec/lz4 v2.6.1+incompatible github.com/pkg/errors v0.9.1 diff --git a/go.sum b/go.sum index c0ac54f6600..5c279bad057 100644 --- a/go.sum +++ b/go.sum @@ -2168,8 +2168,8 @@ github.com/onflow/atree v0.8.0-rc.6/go.mod h1:yccR+LR7xc1Jdic0mrjocbHvUD7lnVvg8/ github.com/onflow/boxo v0.0.0-20240201202436-f2477b92f483 h1:LpiQhTAfM9CAmNVEs0n//cBBgCg+vJSiIxTHYUklZ84= github.com/onflow/boxo v0.0.0-20240201202436-f2477b92f483/go.mod h1:pIZgTWdm3k3pLF9Uq6MB8JEcW07UDwNJjlXW1HELW80= github.com/onflow/cadence v1.0.0-M3/go.mod h1:odXGZZ/wGNA5mwT8bC9v8u8EXACHllB2ABSZK65TGL8= -github.com/onflow/cadence v1.0.0-preview.52 h1:hZ92e6lL2+PQa3C1i5jJh0zZYFdW89+X1MS0Bkd6Ayo= -github.com/onflow/cadence v1.0.0-preview.52/go.mod h1:7wvvecnAZtYOspLOS3Lh+FuAmMeSrXhAWiycC3kQ1UU= +github.com/onflow/cadence v1.0.0 h1:bvT75F2LZJvDCBmmajAv7QLISK6Qp30FAKcSwqNNH+o= +github.com/onflow/cadence v1.0.0/go.mod h1:7wvvecnAZtYOspLOS3Lh+FuAmMeSrXhAWiycC3kQ1UU= github.com/onflow/crypto v0.25.0/go.mod h1:C8FbaX0x8y+FxWjbkHy0Q4EASCDR9bSPWZqlpCLYyVI= github.com/onflow/crypto v0.25.2 h1:GjHunqVt+vPcdqhxxhAXiMIF3YiLX7gTuTR5O+VG2ns= github.com/onflow/crypto v0.25.2/go.mod h1:fY7eLqUdMKV8EGOw301unP8h7PvLVy8/6gVR++/g0BY= @@ -2184,15 +2184,15 @@ github.com/onflow/flow-ft/lib/go/contracts v1.0.0/go.mod h1:PwsL8fC81cjnUnTfmyL/ github.com/onflow/flow-ft/lib/go/templates v1.0.0 h1:6cMS/lUJJ17HjKBfMO/eh0GGvnpElPgBXx7h5aoWJhs= github.com/onflow/flow-ft/lib/go/templates v1.0.0/go.mod h1:uQ8XFqmMK2jxyBSVrmyuwdWjTEb+6zGjRYotfDJ5pAE= github.com/onflow/flow-go-sdk v1.0.0-M1/go.mod h1:TDW0MNuCs4SvqYRUzkbRnRmHQL1h4X8wURsCw9P9beo= -github.com/onflow/flow-go-sdk v1.0.0-preview.54 h1:5GjCkyIyvE9KolOUUPTkGdEiV/8qOe1MGnLHOLBmthA= -github.com/onflow/flow-go-sdk v1.0.0-preview.54/go.mod h1:u9oFiS25TpnU1EW62PQlq22jzkwBAj4VEiiCBM6nhHo= +github.com/onflow/flow-go-sdk v1.0.0 h1:Ha4fQm1MMKsyaqMkQLCN3rA/yaQKG6DGwiIfx06j40c= +github.com/onflow/flow-go-sdk v1.0.0/go.mod h1:iZkW2IWieVUZKK06mQCxpjJzPDgS0VtGpTaP/rKu6J4= github.com/onflow/flow-nft/lib/go/contracts v1.2.1 h1:woAAS5z651sDpi7ihAHll8NvRS9uFXIXkL6xR+bKFZY= github.com/onflow/flow-nft/lib/go/contracts v1.2.1/go.mod h1:2gpbza+uzs1k7x31hkpBPlggIRkI53Suo0n2AyA2HcE= github.com/onflow/flow-nft/lib/go/templates v1.2.0 h1:JSQyh9rg0RC+D1930BiRXN8lrtMs+ubVMK6aQPon6Yc= github.com/onflow/flow-nft/lib/go/templates v1.2.0/go.mod h1:p+2hRvtjLUR3MW1NsoJe5Gqgr2eeH49QB6+s6ze00w0= github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20231121210617-52ee94b830c2/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= -github.com/onflow/flow/protobuf/go/flow v0.4.5 h1:6o+pgYGqwXdEhqSJxu2BdnDXkOQVOkfGAb6IiXB+NPM= -github.com/onflow/flow/protobuf/go/flow v0.4.5/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= +github.com/onflow/flow/protobuf/go/flow v0.4.7 h1:iP6DFx4wZ3ETORsyeqzHu7neFT3d1CXF6wdK+AOOjmc= +github.com/onflow/flow/protobuf/go/flow v0.4.7/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= github.com/onflow/go-ds-pebble v0.0.0-20240731130313-f186539f382c h1:T0jDCm7k7uqDo26JiiujQ5oryl30itPnlmZQywTu9ng= github.com/onflow/go-ds-pebble v0.0.0-20240731130313-f186539f382c/go.mod h1:XYnWtulwJvHVOr2B0WVA/UC3dvRgFevjp8Pn9a3E1xo= github.com/onflow/go-ethereum v1.14.7 h1:gg3awYqI02e3AypRdpJKEvNTJ6kz/OhAqRti0h54Wlc= diff --git a/insecure/go.mod b/insecure/go.mod index 0c627ca59a0..bf29608f74b 100644 --- a/insecure/go.mod +++ b/insecure/go.mod @@ -202,15 +202,15 @@ require ( github.com/multiformats/go-varint v0.0.7 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/onflow/atree v0.8.0-rc.6 // indirect - github.com/onflow/cadence v1.0.0-preview.52 // indirect + github.com/onflow/cadence v1.0.0 // indirect github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.1 // indirect github.com/onflow/flow-core-contracts/lib/go/templates v1.3.1 // indirect github.com/onflow/flow-ft/lib/go/contracts v1.0.0 // indirect github.com/onflow/flow-ft/lib/go/templates v1.0.0 // indirect - github.com/onflow/flow-go-sdk v1.0.0-preview.54 // indirect + github.com/onflow/flow-go-sdk v1.0.0 // indirect github.com/onflow/flow-nft/lib/go/contracts v1.2.1 // indirect github.com/onflow/flow-nft/lib/go/templates v1.2.0 // indirect - github.com/onflow/flow/protobuf/go/flow v0.4.5 // indirect + github.com/onflow/flow/protobuf/go/flow v0.4.7 // indirect github.com/onflow/go-ethereum v1.14.7 // indirect github.com/onflow/sdks v0.6.0-preview.1 // indirect github.com/onflow/wal v1.0.2 // indirect diff --git a/insecure/go.sum b/insecure/go.sum index 39c9676ca6e..ad9a676cd1f 100644 --- a/insecure/go.sum +++ b/insecure/go.sum @@ -2158,8 +2158,8 @@ github.com/onflow/atree v0.6.1-0.20230711151834-86040b30171f/go.mod h1:xvP61FoOs github.com/onflow/atree v0.8.0-rc.6 h1:GWgaylK24b5ta2Hq+TvyOF7X5tZLiLzMMn7lEt59fsA= github.com/onflow/atree v0.8.0-rc.6/go.mod h1:yccR+LR7xc1Jdic0mrjocbHvUD7lnVvg8/Ct1AA5zBo= github.com/onflow/cadence v1.0.0-M3/go.mod h1:odXGZZ/wGNA5mwT8bC9v8u8EXACHllB2ABSZK65TGL8= -github.com/onflow/cadence v1.0.0-preview.52 h1:hZ92e6lL2+PQa3C1i5jJh0zZYFdW89+X1MS0Bkd6Ayo= -github.com/onflow/cadence v1.0.0-preview.52/go.mod h1:7wvvecnAZtYOspLOS3Lh+FuAmMeSrXhAWiycC3kQ1UU= +github.com/onflow/cadence v1.0.0 h1:bvT75F2LZJvDCBmmajAv7QLISK6Qp30FAKcSwqNNH+o= +github.com/onflow/cadence v1.0.0/go.mod h1:7wvvecnAZtYOspLOS3Lh+FuAmMeSrXhAWiycC3kQ1UU= github.com/onflow/crypto v0.25.0/go.mod h1:C8FbaX0x8y+FxWjbkHy0Q4EASCDR9bSPWZqlpCLYyVI= github.com/onflow/crypto v0.25.2 h1:GjHunqVt+vPcdqhxxhAXiMIF3YiLX7gTuTR5O+VG2ns= github.com/onflow/crypto v0.25.2/go.mod h1:fY7eLqUdMKV8EGOw301unP8h7PvLVy8/6gVR++/g0BY= @@ -2172,15 +2172,15 @@ github.com/onflow/flow-ft/lib/go/contracts v1.0.0/go.mod h1:PwsL8fC81cjnUnTfmyL/ github.com/onflow/flow-ft/lib/go/templates v1.0.0 h1:6cMS/lUJJ17HjKBfMO/eh0GGvnpElPgBXx7h5aoWJhs= github.com/onflow/flow-ft/lib/go/templates v1.0.0/go.mod h1:uQ8XFqmMK2jxyBSVrmyuwdWjTEb+6zGjRYotfDJ5pAE= github.com/onflow/flow-go-sdk v1.0.0-M1/go.mod h1:TDW0MNuCs4SvqYRUzkbRnRmHQL1h4X8wURsCw9P9beo= -github.com/onflow/flow-go-sdk v1.0.0-preview.54 h1:5GjCkyIyvE9KolOUUPTkGdEiV/8qOe1MGnLHOLBmthA= -github.com/onflow/flow-go-sdk v1.0.0-preview.54/go.mod h1:u9oFiS25TpnU1EW62PQlq22jzkwBAj4VEiiCBM6nhHo= +github.com/onflow/flow-go-sdk v1.0.0 h1:Ha4fQm1MMKsyaqMkQLCN3rA/yaQKG6DGwiIfx06j40c= +github.com/onflow/flow-go-sdk v1.0.0/go.mod h1:iZkW2IWieVUZKK06mQCxpjJzPDgS0VtGpTaP/rKu6J4= github.com/onflow/flow-nft/lib/go/contracts v1.2.1 h1:woAAS5z651sDpi7ihAHll8NvRS9uFXIXkL6xR+bKFZY= github.com/onflow/flow-nft/lib/go/contracts v1.2.1/go.mod h1:2gpbza+uzs1k7x31hkpBPlggIRkI53Suo0n2AyA2HcE= github.com/onflow/flow-nft/lib/go/templates v1.2.0 h1:JSQyh9rg0RC+D1930BiRXN8lrtMs+ubVMK6aQPon6Yc= github.com/onflow/flow-nft/lib/go/templates v1.2.0/go.mod h1:p+2hRvtjLUR3MW1NsoJe5Gqgr2eeH49QB6+s6ze00w0= github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20231121210617-52ee94b830c2/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= -github.com/onflow/flow/protobuf/go/flow v0.4.5 h1:6o+pgYGqwXdEhqSJxu2BdnDXkOQVOkfGAb6IiXB+NPM= -github.com/onflow/flow/protobuf/go/flow v0.4.5/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= +github.com/onflow/flow/protobuf/go/flow v0.4.7 h1:iP6DFx4wZ3ETORsyeqzHu7neFT3d1CXF6wdK+AOOjmc= +github.com/onflow/flow/protobuf/go/flow v0.4.7/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= github.com/onflow/go-ds-pebble v0.0.0-20240731130313-f186539f382c h1:T0jDCm7k7uqDo26JiiujQ5oryl30itPnlmZQywTu9ng= github.com/onflow/go-ds-pebble v0.0.0-20240731130313-f186539f382c/go.mod h1:XYnWtulwJvHVOr2B0WVA/UC3dvRgFevjp8Pn9a3E1xo= github.com/onflow/go-ethereum v1.14.7 h1:gg3awYqI02e3AypRdpJKEvNTJ6kz/OhAqRti0h54Wlc= diff --git a/integration/go.mod b/integration/go.mod index e61eec29247..ca9e516c0b7 100644 --- a/integration/go.mod +++ b/integration/go.mod @@ -20,15 +20,15 @@ require ( github.com/ipfs/go-ds-badger2 v0.1.3 github.com/ipfs/go-ds-pebble v0.3.1 github.com/libp2p/go-libp2p v0.32.2 - github.com/onflow/cadence v1.0.0-preview.52 + github.com/onflow/cadence v1.0.0 github.com/onflow/crypto v0.25.2 github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.1 github.com/onflow/flow-core-contracts/lib/go/templates v1.3.1 github.com/onflow/flow-emulator v1.0.0-preview.36.0.20240729195722-d4eb1c30eb9f github.com/onflow/flow-go v0.36.8-0.20240729193633-433a32eeb0cd - github.com/onflow/flow-go-sdk v1.0.0-preview.54 + github.com/onflow/flow-go-sdk v1.0.0 github.com/onflow/flow-go/insecure v0.0.0-00010101000000-000000000000 - github.com/onflow/flow/protobuf/go/flow v0.4.5 + github.com/onflow/flow/protobuf/go/flow v0.4.7 github.com/onflow/go-ethereum v1.14.7 github.com/prometheus/client_golang v1.18.0 github.com/prometheus/client_model v0.5.0 diff --git a/integration/go.sum b/integration/go.sum index 1b425524376..8883d603d15 100644 --- a/integration/go.sum +++ b/integration/go.sum @@ -2142,8 +2142,8 @@ github.com/onflow/atree v0.6.1-0.20230711151834-86040b30171f/go.mod h1:xvP61FoOs github.com/onflow/atree v0.8.0-rc.6 h1:GWgaylK24b5ta2Hq+TvyOF7X5tZLiLzMMn7lEt59fsA= github.com/onflow/atree v0.8.0-rc.6/go.mod h1:yccR+LR7xc1Jdic0mrjocbHvUD7lnVvg8/Ct1AA5zBo= github.com/onflow/cadence v1.0.0-M3/go.mod h1:odXGZZ/wGNA5mwT8bC9v8u8EXACHllB2ABSZK65TGL8= -github.com/onflow/cadence v1.0.0-preview.52 h1:hZ92e6lL2+PQa3C1i5jJh0zZYFdW89+X1MS0Bkd6Ayo= -github.com/onflow/cadence v1.0.0-preview.52/go.mod h1:7wvvecnAZtYOspLOS3Lh+FuAmMeSrXhAWiycC3kQ1UU= +github.com/onflow/cadence v1.0.0 h1:bvT75F2LZJvDCBmmajAv7QLISK6Qp30FAKcSwqNNH+o= +github.com/onflow/cadence v1.0.0/go.mod h1:7wvvecnAZtYOspLOS3Lh+FuAmMeSrXhAWiycC3kQ1UU= github.com/onflow/crypto v0.25.0/go.mod h1:C8FbaX0x8y+FxWjbkHy0Q4EASCDR9bSPWZqlpCLYyVI= github.com/onflow/crypto v0.25.2 h1:GjHunqVt+vPcdqhxxhAXiMIF3YiLX7gTuTR5O+VG2ns= github.com/onflow/crypto v0.25.2/go.mod h1:fY7eLqUdMKV8EGOw301unP8h7PvLVy8/6gVR++/g0BY= @@ -2158,15 +2158,15 @@ github.com/onflow/flow-ft/lib/go/contracts v1.0.0/go.mod h1:PwsL8fC81cjnUnTfmyL/ github.com/onflow/flow-ft/lib/go/templates v1.0.0 h1:6cMS/lUJJ17HjKBfMO/eh0GGvnpElPgBXx7h5aoWJhs= github.com/onflow/flow-ft/lib/go/templates v1.0.0/go.mod h1:uQ8XFqmMK2jxyBSVrmyuwdWjTEb+6zGjRYotfDJ5pAE= github.com/onflow/flow-go-sdk v1.0.0-M1/go.mod h1:TDW0MNuCs4SvqYRUzkbRnRmHQL1h4X8wURsCw9P9beo= -github.com/onflow/flow-go-sdk v1.0.0-preview.54 h1:5GjCkyIyvE9KolOUUPTkGdEiV/8qOe1MGnLHOLBmthA= -github.com/onflow/flow-go-sdk v1.0.0-preview.54/go.mod h1:u9oFiS25TpnU1EW62PQlq22jzkwBAj4VEiiCBM6nhHo= +github.com/onflow/flow-go-sdk v1.0.0 h1:Ha4fQm1MMKsyaqMkQLCN3rA/yaQKG6DGwiIfx06j40c= +github.com/onflow/flow-go-sdk v1.0.0/go.mod h1:iZkW2IWieVUZKK06mQCxpjJzPDgS0VtGpTaP/rKu6J4= github.com/onflow/flow-nft/lib/go/contracts v1.2.1 h1:woAAS5z651sDpi7ihAHll8NvRS9uFXIXkL6xR+bKFZY= github.com/onflow/flow-nft/lib/go/contracts v1.2.1/go.mod h1:2gpbza+uzs1k7x31hkpBPlggIRkI53Suo0n2AyA2HcE= github.com/onflow/flow-nft/lib/go/templates v1.2.0 h1:JSQyh9rg0RC+D1930BiRXN8lrtMs+ubVMK6aQPon6Yc= github.com/onflow/flow-nft/lib/go/templates v1.2.0/go.mod h1:p+2hRvtjLUR3MW1NsoJe5Gqgr2eeH49QB6+s6ze00w0= github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20231121210617-52ee94b830c2/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= -github.com/onflow/flow/protobuf/go/flow v0.4.5 h1:6o+pgYGqwXdEhqSJxu2BdnDXkOQVOkfGAb6IiXB+NPM= -github.com/onflow/flow/protobuf/go/flow v0.4.5/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= +github.com/onflow/flow/protobuf/go/flow v0.4.7 h1:iP6DFx4wZ3ETORsyeqzHu7neFT3d1CXF6wdK+AOOjmc= +github.com/onflow/flow/protobuf/go/flow v0.4.7/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= github.com/onflow/go-ds-pebble v0.0.0-20240731130313-f186539f382c h1:T0jDCm7k7uqDo26JiiujQ5oryl30itPnlmZQywTu9ng= github.com/onflow/go-ds-pebble v0.0.0-20240731130313-f186539f382c/go.mod h1:XYnWtulwJvHVOr2B0WVA/UC3dvRgFevjp8Pn9a3E1xo= github.com/onflow/go-ethereum v1.14.7 h1:gg3awYqI02e3AypRdpJKEvNTJ6kz/OhAqRti0h54Wlc= From 725459169cfc8b46888f46b22d7c2b355c59d681 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Thu, 26 Sep 2024 20:49:33 -0700 Subject: [PATCH 67/72] update mocks --- engine/access/mock/execution_api_client.go | 37 ++++++++++++++++++++++ engine/access/mock/execution_api_server.go | 30 ++++++++++++++++++ fvm/environment/mock/environment.go | 4 +++ 3 files changed, 71 insertions(+) diff --git a/engine/access/mock/execution_api_client.go b/engine/access/mock/execution_api_client.go index fd5fc20c718..7aa04d143c7 100644 --- a/engine/access/mock/execution_api_client.go +++ b/engine/access/mock/execution_api_client.go @@ -349,6 +349,43 @@ func (_m *ExecutionAPIClient) GetTransactionErrorMessagesByBlockID(ctx context.C return r0, r1 } +// GetTransactionExecutionMetricsAfter provides a mock function with given fields: ctx, in, opts +func (_m *ExecutionAPIClient) GetTransactionExecutionMetricsAfter(ctx context.Context, in *execution.GetTransactionExecutionMetricsAfterRequest, opts ...grpc.CallOption) (*execution.GetTransactionExecutionMetricsAfterResponse, error) { + _va := make([]interface{}, len(opts)) + for _i := range opts { + _va[_i] = opts[_i] + } + var _ca []interface{} + _ca = append(_ca, ctx, in) + _ca = append(_ca, _va...) + ret := _m.Called(_ca...) + + if len(ret) == 0 { + panic("no return value specified for GetTransactionExecutionMetricsAfter") + } + + var r0 *execution.GetTransactionExecutionMetricsAfterResponse + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *execution.GetTransactionExecutionMetricsAfterRequest, ...grpc.CallOption) (*execution.GetTransactionExecutionMetricsAfterResponse, error)); ok { + return rf(ctx, in, opts...) + } + if rf, ok := ret.Get(0).(func(context.Context, *execution.GetTransactionExecutionMetricsAfterRequest, ...grpc.CallOption) *execution.GetTransactionExecutionMetricsAfterResponse); ok { + r0 = rf(ctx, in, opts...) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*execution.GetTransactionExecutionMetricsAfterResponse) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, *execution.GetTransactionExecutionMetricsAfterRequest, ...grpc.CallOption) error); ok { + r1 = rf(ctx, in, opts...) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // GetTransactionResult provides a mock function with given fields: ctx, in, opts func (_m *ExecutionAPIClient) GetTransactionResult(ctx context.Context, in *execution.GetTransactionResultRequest, opts ...grpc.CallOption) (*execution.GetTransactionResultResponse, error) { _va := make([]interface{}, len(opts)) diff --git a/engine/access/mock/execution_api_server.go b/engine/access/mock/execution_api_server.go index e61517cb617..2c3f1f3b5a7 100644 --- a/engine/access/mock/execution_api_server.go +++ b/engine/access/mock/execution_api_server.go @@ -284,6 +284,36 @@ func (_m *ExecutionAPIServer) GetTransactionErrorMessagesByBlockID(_a0 context.C return r0, r1 } +// GetTransactionExecutionMetricsAfter provides a mock function with given fields: _a0, _a1 +func (_m *ExecutionAPIServer) GetTransactionExecutionMetricsAfter(_a0 context.Context, _a1 *execution.GetTransactionExecutionMetricsAfterRequest) (*execution.GetTransactionExecutionMetricsAfterResponse, error) { + ret := _m.Called(_a0, _a1) + + if len(ret) == 0 { + panic("no return value specified for GetTransactionExecutionMetricsAfter") + } + + var r0 *execution.GetTransactionExecutionMetricsAfterResponse + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *execution.GetTransactionExecutionMetricsAfterRequest) (*execution.GetTransactionExecutionMetricsAfterResponse, error)); ok { + return rf(_a0, _a1) + } + if rf, ok := ret.Get(0).(func(context.Context, *execution.GetTransactionExecutionMetricsAfterRequest) *execution.GetTransactionExecutionMetricsAfterResponse); ok { + r0 = rf(_a0, _a1) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*execution.GetTransactionExecutionMetricsAfterResponse) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, *execution.GetTransactionExecutionMetricsAfterRequest) error); ok { + r1 = rf(_a0, _a1) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // GetTransactionResult provides a mock function with given fields: _a0, _a1 func (_m *ExecutionAPIServer) GetTransactionResult(_a0 context.Context, _a1 *execution.GetTransactionResultRequest) (*execution.GetTransactionResultResponse, error) { ret := _m.Called(_a0, _a1) diff --git a/fvm/environment/mock/environment.go b/fvm/environment/mock/environment.go index 24ec42f7e4b..ab6fd164c30 100644 --- a/fvm/environment/mock/environment.go +++ b/fvm/environment/mock/environment.go @@ -1756,6 +1756,10 @@ func (_m *Environment) ValidateAccountCapabilitiesGet(inter *interpreter.Interpr func (_m *Environment) ValidateAccountCapabilitiesPublish(inter *interpreter.Interpreter, locationRange interpreter.LocationRange, address interpreter.AddressValue, path interpreter.PathValue, capabilityBorrowType *interpreter.ReferenceStaticType) (bool, error) { ret := _m.Called(inter, locationRange, address, path, capabilityBorrowType) + if len(ret) == 0 { + panic("no return value specified for ValidateAccountCapabilitiesPublish") + } + var r0 bool var r1 error if rf, ok := ret.Get(0).(func(*interpreter.Interpreter, interpreter.LocationRange, interpreter.AddressValue, interpreter.PathValue, *interpreter.ReferenceStaticType) (bool, error)); ok { From c1441d501985a9a317958b281b9b25ffb733ca23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Fri, 27 Sep 2024 09:58:51 -0700 Subject: [PATCH 68/72] downgrade SDK --- engine/access/mock/execution_api_client.go | 37 ---------------------- engine/access/mock/execution_api_server.go | 30 ------------------ go.mod | 2 +- go.sum | 4 +-- insecure/go.mod | 2 +- insecure/go.sum | 4 +-- integration/go.mod | 2 +- integration/go.sum | 4 +-- 8 files changed, 9 insertions(+), 76 deletions(-) diff --git a/engine/access/mock/execution_api_client.go b/engine/access/mock/execution_api_client.go index 7aa04d143c7..fd5fc20c718 100644 --- a/engine/access/mock/execution_api_client.go +++ b/engine/access/mock/execution_api_client.go @@ -349,43 +349,6 @@ func (_m *ExecutionAPIClient) GetTransactionErrorMessagesByBlockID(ctx context.C return r0, r1 } -// GetTransactionExecutionMetricsAfter provides a mock function with given fields: ctx, in, opts -func (_m *ExecutionAPIClient) GetTransactionExecutionMetricsAfter(ctx context.Context, in *execution.GetTransactionExecutionMetricsAfterRequest, opts ...grpc.CallOption) (*execution.GetTransactionExecutionMetricsAfterResponse, error) { - _va := make([]interface{}, len(opts)) - for _i := range opts { - _va[_i] = opts[_i] - } - var _ca []interface{} - _ca = append(_ca, ctx, in) - _ca = append(_ca, _va...) - ret := _m.Called(_ca...) - - if len(ret) == 0 { - panic("no return value specified for GetTransactionExecutionMetricsAfter") - } - - var r0 *execution.GetTransactionExecutionMetricsAfterResponse - var r1 error - if rf, ok := ret.Get(0).(func(context.Context, *execution.GetTransactionExecutionMetricsAfterRequest, ...grpc.CallOption) (*execution.GetTransactionExecutionMetricsAfterResponse, error)); ok { - return rf(ctx, in, opts...) - } - if rf, ok := ret.Get(0).(func(context.Context, *execution.GetTransactionExecutionMetricsAfterRequest, ...grpc.CallOption) *execution.GetTransactionExecutionMetricsAfterResponse); ok { - r0 = rf(ctx, in, opts...) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(*execution.GetTransactionExecutionMetricsAfterResponse) - } - } - - if rf, ok := ret.Get(1).(func(context.Context, *execution.GetTransactionExecutionMetricsAfterRequest, ...grpc.CallOption) error); ok { - r1 = rf(ctx, in, opts...) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - // GetTransactionResult provides a mock function with given fields: ctx, in, opts func (_m *ExecutionAPIClient) GetTransactionResult(ctx context.Context, in *execution.GetTransactionResultRequest, opts ...grpc.CallOption) (*execution.GetTransactionResultResponse, error) { _va := make([]interface{}, len(opts)) diff --git a/engine/access/mock/execution_api_server.go b/engine/access/mock/execution_api_server.go index 2c3f1f3b5a7..e61517cb617 100644 --- a/engine/access/mock/execution_api_server.go +++ b/engine/access/mock/execution_api_server.go @@ -284,36 +284,6 @@ func (_m *ExecutionAPIServer) GetTransactionErrorMessagesByBlockID(_a0 context.C return r0, r1 } -// GetTransactionExecutionMetricsAfter provides a mock function with given fields: _a0, _a1 -func (_m *ExecutionAPIServer) GetTransactionExecutionMetricsAfter(_a0 context.Context, _a1 *execution.GetTransactionExecutionMetricsAfterRequest) (*execution.GetTransactionExecutionMetricsAfterResponse, error) { - ret := _m.Called(_a0, _a1) - - if len(ret) == 0 { - panic("no return value specified for GetTransactionExecutionMetricsAfter") - } - - var r0 *execution.GetTransactionExecutionMetricsAfterResponse - var r1 error - if rf, ok := ret.Get(0).(func(context.Context, *execution.GetTransactionExecutionMetricsAfterRequest) (*execution.GetTransactionExecutionMetricsAfterResponse, error)); ok { - return rf(_a0, _a1) - } - if rf, ok := ret.Get(0).(func(context.Context, *execution.GetTransactionExecutionMetricsAfterRequest) *execution.GetTransactionExecutionMetricsAfterResponse); ok { - r0 = rf(_a0, _a1) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(*execution.GetTransactionExecutionMetricsAfterResponse) - } - } - - if rf, ok := ret.Get(1).(func(context.Context, *execution.GetTransactionExecutionMetricsAfterRequest) error); ok { - r1 = rf(_a0, _a1) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - // GetTransactionResult provides a mock function with given fields: _a0, _a1 func (_m *ExecutionAPIServer) GetTransactionResult(_a0 context.Context, _a1 *execution.GetTransactionResultRequest) (*execution.GetTransactionResultResponse, error) { ret := _m.Called(_a0, _a1) diff --git a/go.mod b/go.mod index a2a68ab22de..669ff4522fd 100644 --- a/go.mod +++ b/go.mod @@ -53,7 +53,7 @@ require ( github.com/onflow/flow v0.3.4 github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.1 github.com/onflow/flow-core-contracts/lib/go/templates v1.3.1 - github.com/onflow/flow-go-sdk v1.0.0 + github.com/onflow/flow-go-sdk v1.0.0-preview.54 github.com/onflow/flow/protobuf/go/flow v0.4.7 github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 github.com/pierrec/lz4 v2.6.1+incompatible diff --git a/go.sum b/go.sum index 5c279bad057..aae8074fda1 100644 --- a/go.sum +++ b/go.sum @@ -2184,8 +2184,8 @@ github.com/onflow/flow-ft/lib/go/contracts v1.0.0/go.mod h1:PwsL8fC81cjnUnTfmyL/ github.com/onflow/flow-ft/lib/go/templates v1.0.0 h1:6cMS/lUJJ17HjKBfMO/eh0GGvnpElPgBXx7h5aoWJhs= github.com/onflow/flow-ft/lib/go/templates v1.0.0/go.mod h1:uQ8XFqmMK2jxyBSVrmyuwdWjTEb+6zGjRYotfDJ5pAE= github.com/onflow/flow-go-sdk v1.0.0-M1/go.mod h1:TDW0MNuCs4SvqYRUzkbRnRmHQL1h4X8wURsCw9P9beo= -github.com/onflow/flow-go-sdk v1.0.0 h1:Ha4fQm1MMKsyaqMkQLCN3rA/yaQKG6DGwiIfx06j40c= -github.com/onflow/flow-go-sdk v1.0.0/go.mod h1:iZkW2IWieVUZKK06mQCxpjJzPDgS0VtGpTaP/rKu6J4= +github.com/onflow/flow-go-sdk v1.0.0-preview.54 h1:5GjCkyIyvE9KolOUUPTkGdEiV/8qOe1MGnLHOLBmthA= +github.com/onflow/flow-go-sdk v1.0.0-preview.54/go.mod h1:u9oFiS25TpnU1EW62PQlq22jzkwBAj4VEiiCBM6nhHo= github.com/onflow/flow-nft/lib/go/contracts v1.2.1 h1:woAAS5z651sDpi7ihAHll8NvRS9uFXIXkL6xR+bKFZY= github.com/onflow/flow-nft/lib/go/contracts v1.2.1/go.mod h1:2gpbza+uzs1k7x31hkpBPlggIRkI53Suo0n2AyA2HcE= github.com/onflow/flow-nft/lib/go/templates v1.2.0 h1:JSQyh9rg0RC+D1930BiRXN8lrtMs+ubVMK6aQPon6Yc= diff --git a/insecure/go.mod b/insecure/go.mod index bf29608f74b..ae146a04d89 100644 --- a/insecure/go.mod +++ b/insecure/go.mod @@ -207,7 +207,7 @@ require ( github.com/onflow/flow-core-contracts/lib/go/templates v1.3.1 // indirect github.com/onflow/flow-ft/lib/go/contracts v1.0.0 // indirect github.com/onflow/flow-ft/lib/go/templates v1.0.0 // indirect - github.com/onflow/flow-go-sdk v1.0.0 // indirect + github.com/onflow/flow-go-sdk v1.0.0-preview.54 // indirect github.com/onflow/flow-nft/lib/go/contracts v1.2.1 // indirect github.com/onflow/flow-nft/lib/go/templates v1.2.0 // indirect github.com/onflow/flow/protobuf/go/flow v0.4.7 // indirect diff --git a/insecure/go.sum b/insecure/go.sum index ad9a676cd1f..cf2b7ae3549 100644 --- a/insecure/go.sum +++ b/insecure/go.sum @@ -2172,8 +2172,8 @@ github.com/onflow/flow-ft/lib/go/contracts v1.0.0/go.mod h1:PwsL8fC81cjnUnTfmyL/ github.com/onflow/flow-ft/lib/go/templates v1.0.0 h1:6cMS/lUJJ17HjKBfMO/eh0GGvnpElPgBXx7h5aoWJhs= github.com/onflow/flow-ft/lib/go/templates v1.0.0/go.mod h1:uQ8XFqmMK2jxyBSVrmyuwdWjTEb+6zGjRYotfDJ5pAE= github.com/onflow/flow-go-sdk v1.0.0-M1/go.mod h1:TDW0MNuCs4SvqYRUzkbRnRmHQL1h4X8wURsCw9P9beo= -github.com/onflow/flow-go-sdk v1.0.0 h1:Ha4fQm1MMKsyaqMkQLCN3rA/yaQKG6DGwiIfx06j40c= -github.com/onflow/flow-go-sdk v1.0.0/go.mod h1:iZkW2IWieVUZKK06mQCxpjJzPDgS0VtGpTaP/rKu6J4= +github.com/onflow/flow-go-sdk v1.0.0-preview.54 h1:5GjCkyIyvE9KolOUUPTkGdEiV/8qOe1MGnLHOLBmthA= +github.com/onflow/flow-go-sdk v1.0.0-preview.54/go.mod h1:u9oFiS25TpnU1EW62PQlq22jzkwBAj4VEiiCBM6nhHo= github.com/onflow/flow-nft/lib/go/contracts v1.2.1 h1:woAAS5z651sDpi7ihAHll8NvRS9uFXIXkL6xR+bKFZY= github.com/onflow/flow-nft/lib/go/contracts v1.2.1/go.mod h1:2gpbza+uzs1k7x31hkpBPlggIRkI53Suo0n2AyA2HcE= github.com/onflow/flow-nft/lib/go/templates v1.2.0 h1:JSQyh9rg0RC+D1930BiRXN8lrtMs+ubVMK6aQPon6Yc= diff --git a/integration/go.mod b/integration/go.mod index ca9e516c0b7..e23e08a5075 100644 --- a/integration/go.mod +++ b/integration/go.mod @@ -26,7 +26,7 @@ require ( github.com/onflow/flow-core-contracts/lib/go/templates v1.3.1 github.com/onflow/flow-emulator v1.0.0-preview.36.0.20240729195722-d4eb1c30eb9f github.com/onflow/flow-go v0.36.8-0.20240729193633-433a32eeb0cd - github.com/onflow/flow-go-sdk v1.0.0 + github.com/onflow/flow-go-sdk v1.0.0-preview.54 github.com/onflow/flow-go/insecure v0.0.0-00010101000000-000000000000 github.com/onflow/flow/protobuf/go/flow v0.4.7 github.com/onflow/go-ethereum v1.14.7 diff --git a/integration/go.sum b/integration/go.sum index 8883d603d15..da7ace5501a 100644 --- a/integration/go.sum +++ b/integration/go.sum @@ -2158,8 +2158,8 @@ github.com/onflow/flow-ft/lib/go/contracts v1.0.0/go.mod h1:PwsL8fC81cjnUnTfmyL/ github.com/onflow/flow-ft/lib/go/templates v1.0.0 h1:6cMS/lUJJ17HjKBfMO/eh0GGvnpElPgBXx7h5aoWJhs= github.com/onflow/flow-ft/lib/go/templates v1.0.0/go.mod h1:uQ8XFqmMK2jxyBSVrmyuwdWjTEb+6zGjRYotfDJ5pAE= github.com/onflow/flow-go-sdk v1.0.0-M1/go.mod h1:TDW0MNuCs4SvqYRUzkbRnRmHQL1h4X8wURsCw9P9beo= -github.com/onflow/flow-go-sdk v1.0.0 h1:Ha4fQm1MMKsyaqMkQLCN3rA/yaQKG6DGwiIfx06j40c= -github.com/onflow/flow-go-sdk v1.0.0/go.mod h1:iZkW2IWieVUZKK06mQCxpjJzPDgS0VtGpTaP/rKu6J4= +github.com/onflow/flow-go-sdk v1.0.0-preview.54 h1:5GjCkyIyvE9KolOUUPTkGdEiV/8qOe1MGnLHOLBmthA= +github.com/onflow/flow-go-sdk v1.0.0-preview.54/go.mod h1:u9oFiS25TpnU1EW62PQlq22jzkwBAj4VEiiCBM6nhHo= github.com/onflow/flow-nft/lib/go/contracts v1.2.1 h1:woAAS5z651sDpi7ihAHll8NvRS9uFXIXkL6xR+bKFZY= github.com/onflow/flow-nft/lib/go/contracts v1.2.1/go.mod h1:2gpbza+uzs1k7x31hkpBPlggIRkI53Suo0n2AyA2HcE= github.com/onflow/flow-nft/lib/go/templates v1.2.0 h1:JSQyh9rg0RC+D1930BiRXN8lrtMs+ubVMK6aQPon6Yc= From e0a8ceb6e09b02052d636d34ce574807f932a55a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Fri, 27 Sep 2024 10:01:15 -0700 Subject: [PATCH 69/72] downgrade protobuf again --- go.mod | 2 +- go.sum | 4 ++-- insecure/go.mod | 2 +- insecure/go.sum | 4 ++-- integration/go.mod | 2 +- integration/go.sum | 4 ++-- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 669ff4522fd..080c5d7bad4 100644 --- a/go.mod +++ b/go.mod @@ -54,7 +54,7 @@ require ( github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.1 github.com/onflow/flow-core-contracts/lib/go/templates v1.3.1 github.com/onflow/flow-go-sdk v1.0.0-preview.54 - github.com/onflow/flow/protobuf/go/flow v0.4.7 + github.com/onflow/flow/protobuf/go/flow v0.4.5 github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 github.com/pierrec/lz4 v2.6.1+incompatible github.com/pkg/errors v0.9.1 diff --git a/go.sum b/go.sum index aae8074fda1..b79dd58976d 100644 --- a/go.sum +++ b/go.sum @@ -2191,8 +2191,8 @@ github.com/onflow/flow-nft/lib/go/contracts v1.2.1/go.mod h1:2gpbza+uzs1k7x31hkp github.com/onflow/flow-nft/lib/go/templates v1.2.0 h1:JSQyh9rg0RC+D1930BiRXN8lrtMs+ubVMK6aQPon6Yc= github.com/onflow/flow-nft/lib/go/templates v1.2.0/go.mod h1:p+2hRvtjLUR3MW1NsoJe5Gqgr2eeH49QB6+s6ze00w0= github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20231121210617-52ee94b830c2/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= -github.com/onflow/flow/protobuf/go/flow v0.4.7 h1:iP6DFx4wZ3ETORsyeqzHu7neFT3d1CXF6wdK+AOOjmc= -github.com/onflow/flow/protobuf/go/flow v0.4.7/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= +github.com/onflow/flow/protobuf/go/flow v0.4.5 h1:6o+pgYGqwXdEhqSJxu2BdnDXkOQVOkfGAb6IiXB+NPM= +github.com/onflow/flow/protobuf/go/flow v0.4.5/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= github.com/onflow/go-ds-pebble v0.0.0-20240731130313-f186539f382c h1:T0jDCm7k7uqDo26JiiujQ5oryl30itPnlmZQywTu9ng= github.com/onflow/go-ds-pebble v0.0.0-20240731130313-f186539f382c/go.mod h1:XYnWtulwJvHVOr2B0WVA/UC3dvRgFevjp8Pn9a3E1xo= github.com/onflow/go-ethereum v1.14.7 h1:gg3awYqI02e3AypRdpJKEvNTJ6kz/OhAqRti0h54Wlc= diff --git a/insecure/go.mod b/insecure/go.mod index ae146a04d89..42544d16120 100644 --- a/insecure/go.mod +++ b/insecure/go.mod @@ -210,7 +210,7 @@ require ( github.com/onflow/flow-go-sdk v1.0.0-preview.54 // indirect github.com/onflow/flow-nft/lib/go/contracts v1.2.1 // indirect github.com/onflow/flow-nft/lib/go/templates v1.2.0 // indirect - github.com/onflow/flow/protobuf/go/flow v0.4.7 // indirect + github.com/onflow/flow/protobuf/go/flow v0.4.5 // indirect github.com/onflow/go-ethereum v1.14.7 // indirect github.com/onflow/sdks v0.6.0-preview.1 // indirect github.com/onflow/wal v1.0.2 // indirect diff --git a/insecure/go.sum b/insecure/go.sum index cf2b7ae3549..8c2e2b0e30b 100644 --- a/insecure/go.sum +++ b/insecure/go.sum @@ -2179,8 +2179,8 @@ github.com/onflow/flow-nft/lib/go/contracts v1.2.1/go.mod h1:2gpbza+uzs1k7x31hkp github.com/onflow/flow-nft/lib/go/templates v1.2.0 h1:JSQyh9rg0RC+D1930BiRXN8lrtMs+ubVMK6aQPon6Yc= github.com/onflow/flow-nft/lib/go/templates v1.2.0/go.mod h1:p+2hRvtjLUR3MW1NsoJe5Gqgr2eeH49QB6+s6ze00w0= github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20231121210617-52ee94b830c2/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= -github.com/onflow/flow/protobuf/go/flow v0.4.7 h1:iP6DFx4wZ3ETORsyeqzHu7neFT3d1CXF6wdK+AOOjmc= -github.com/onflow/flow/protobuf/go/flow v0.4.7/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= +github.com/onflow/flow/protobuf/go/flow v0.4.5 h1:6o+pgYGqwXdEhqSJxu2BdnDXkOQVOkfGAb6IiXB+NPM= +github.com/onflow/flow/protobuf/go/flow v0.4.5/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= github.com/onflow/go-ds-pebble v0.0.0-20240731130313-f186539f382c h1:T0jDCm7k7uqDo26JiiujQ5oryl30itPnlmZQywTu9ng= github.com/onflow/go-ds-pebble v0.0.0-20240731130313-f186539f382c/go.mod h1:XYnWtulwJvHVOr2B0WVA/UC3dvRgFevjp8Pn9a3E1xo= github.com/onflow/go-ethereum v1.14.7 h1:gg3awYqI02e3AypRdpJKEvNTJ6kz/OhAqRti0h54Wlc= diff --git a/integration/go.mod b/integration/go.mod index e23e08a5075..9d440f9d18f 100644 --- a/integration/go.mod +++ b/integration/go.mod @@ -28,7 +28,7 @@ require ( github.com/onflow/flow-go v0.36.8-0.20240729193633-433a32eeb0cd github.com/onflow/flow-go-sdk v1.0.0-preview.54 github.com/onflow/flow-go/insecure v0.0.0-00010101000000-000000000000 - github.com/onflow/flow/protobuf/go/flow v0.4.7 + github.com/onflow/flow/protobuf/go/flow v0.4.5 github.com/onflow/go-ethereum v1.14.7 github.com/prometheus/client_golang v1.18.0 github.com/prometheus/client_model v0.5.0 diff --git a/integration/go.sum b/integration/go.sum index da7ace5501a..c41095c8789 100644 --- a/integration/go.sum +++ b/integration/go.sum @@ -2165,8 +2165,8 @@ github.com/onflow/flow-nft/lib/go/contracts v1.2.1/go.mod h1:2gpbza+uzs1k7x31hkp github.com/onflow/flow-nft/lib/go/templates v1.2.0 h1:JSQyh9rg0RC+D1930BiRXN8lrtMs+ubVMK6aQPon6Yc= github.com/onflow/flow-nft/lib/go/templates v1.2.0/go.mod h1:p+2hRvtjLUR3MW1NsoJe5Gqgr2eeH49QB6+s6ze00w0= github.com/onflow/flow/protobuf/go/flow v0.3.2-0.20231121210617-52ee94b830c2/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= -github.com/onflow/flow/protobuf/go/flow v0.4.7 h1:iP6DFx4wZ3ETORsyeqzHu7neFT3d1CXF6wdK+AOOjmc= -github.com/onflow/flow/protobuf/go/flow v0.4.7/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= +github.com/onflow/flow/protobuf/go/flow v0.4.5 h1:6o+pgYGqwXdEhqSJxu2BdnDXkOQVOkfGAb6IiXB+NPM= +github.com/onflow/flow/protobuf/go/flow v0.4.5/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk= github.com/onflow/go-ds-pebble v0.0.0-20240731130313-f186539f382c h1:T0jDCm7k7uqDo26JiiujQ5oryl30itPnlmZQywTu9ng= github.com/onflow/go-ds-pebble v0.0.0-20240731130313-f186539f382c/go.mod h1:XYnWtulwJvHVOr2B0WVA/UC3dvRgFevjp8Pn9a3E1xo= github.com/onflow/go-ethereum v1.14.7 h1:gg3awYqI02e3AypRdpJKEvNTJ6kz/OhAqRti0h54Wlc= From b75c08bf363b261c00cf6eeeb3083b3ac8dd1ce7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Fri, 27 Sep 2024 10:22:33 -0700 Subject: [PATCH 70/72] skip test --- cmd/util/cmd/generate-authorization-fixes/cmd_test.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cmd/util/cmd/generate-authorization-fixes/cmd_test.go b/cmd/util/cmd/generate-authorization-fixes/cmd_test.go index 7a5f8f0f459..ed015ad986f 100644 --- a/cmd/util/cmd/generate-authorization-fixes/cmd_test.go +++ b/cmd/util/cmd/generate-authorization-fixes/cmd_test.go @@ -79,6 +79,11 @@ var _ reporters.ReportWriter = &testReportWriter{} func TestGenerateAuthorizationFixes(t *testing.T) { t.Parallel() + // This test no longer works because publishing authorized capabilities is no longer allowed. + // The migration and test are kept for historical reasons. + + t.Skip() + const chainID = flow.Emulator chain := chainID.Chain() From 55352fb7dec9cdf89055ebda509e135aaf1ae50e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Fri, 27 Sep 2024 10:23:05 -0700 Subject: [PATCH 71/72] skip test --- .../ledger/migrations/fix_authorizations_migration_test.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cmd/util/ledger/migrations/fix_authorizations_migration_test.go b/cmd/util/ledger/migrations/fix_authorizations_migration_test.go index 37c2c06fb7f..2fcf1fd00f5 100644 --- a/cmd/util/ledger/migrations/fix_authorizations_migration_test.go +++ b/cmd/util/ledger/migrations/fix_authorizations_migration_test.go @@ -34,6 +34,11 @@ func newEntitlementSetAuthorizationFromTypeIDs( func TestFixAuthorizationsMigration(t *testing.T) { t.Parallel() + // This test no longer works because publishing authorized capabilities is no longer allowed. + // The migration and test are kept for historical reasons. + + t.Skip() + const chainID = flow.Emulator chain := chainID.Chain() From fd23598923d0382ab1945e13ebfa7eafcf416f7f Mon Sep 17 00:00:00 2001 From: Vishal <1117327+vishalchangrani@users.noreply.github.com> Date: Thu, 3 Oct 2024 15:59:34 -0700 Subject: [PATCH 72/72] cherrypick commits from PR: https://github.com/onflow/flow-go/pull/6513 (#6530) * add getIDs to program recover * fix formatting --------- Co-authored-by: Josh Hannan --- fvm/environment/program_recovery.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/fvm/environment/program_recovery.go b/fvm/environment/program_recovery.go index a4a41f733d7..930db764fbe 100644 --- a/fvm/environment/program_recovery.go +++ b/fvm/environment/program_recovery.go @@ -201,6 +201,11 @@ func RecoveredNonFungibleTokenCode(nonFungibleTokenAddress common.Address, contr %[2]s.recoveryPanic("Collection.deposit") } + access(all) + view fun getIDs(): [UInt64] { + return self.ownedNFTs.keys + } + access(all) view fun getSupportedNFTTypes(): {Type: Bool} { %[2]s.recoveryPanic("Collection.getSupportedNFTTypes")