Merge branch 'v0.37' into ramtin/port-6456

cmd/bootstrap/cmd/final_list.go

@@ -62,13 +62,16 @@ func finalList(cmd *cobra.Command, args []string) {
 	registeredNodes := readStakingContractDetails()
 
 	// merge internal and partner node infos (from local files)
-	localNodes := mergeNodeInfos(internalNodes, partnerNodes)
+	localNodes, err := mergeNodeInfos(internalNodes, partnerNodes)
+	if err != nil {
+		log.Fatal().Err(err).Msg("failed to merge node infos")
+	}
 
 	// reconcile nodes from staking contract nodes
 	validateNodes(localNodes, registeredNodes)
 
 	// write node-config.json with the new list of nodes to be used for the `finalize` command
-	err := common.WriteJSON(model.PathFinallist, flagOutdir, model.ToPublicNodeInfoList(localNodes))
+	err = common.WriteJSON(model.PathFinallist, flagOutdir, model.ToPublicNodeInfoList(localNodes))
 	if err != nil {
 		log.Fatal().Err(err).Msg("failed to write json")
 	}

cmd/bootstrap/cmd/finalize.go

@@ -132,7 +132,10 @@ func finalize(cmd *cobra.Command, args []string) {
 	log.Info().Msg("")
 
 	log.Info().Msg("assembling network and staking keys")
-	stakingNodes := mergeNodeInfos(internalNodes, partnerNodes)
+	stakingNodes, err := mergeNodeInfos(internalNodes, partnerNodes)
+	if err != nil {
+		log.Fatal().Err(err).Msgf("failed to merge internal and partner nodes: %v", err)
+	}
 	log.Info().Msg("")
 
 	// create flow.IdentityList representation of participant set
@@ -312,29 +315,31 @@ func readRootBlockVotes() []*hotstuff.Vote {
 //
 // IMPORTANT: node infos are returned in the canonical ordering, meaning this
 // is safe to use as the input to the DKG and protocol state.
-func mergeNodeInfos(internalNodes, partnerNodes []model.NodeInfo) []model.NodeInfo {
+func mergeNodeInfos(internalNodes, partnerNodes []model.NodeInfo) ([]model.NodeInfo, error) {
 	nodes := append(internalNodes, partnerNodes...)
 
 	// test for duplicate Addresses
 	addressLookup := make(map[string]struct{})
 	for _, node := range nodes {
 		if _, ok := addressLookup[node.Address]; ok {
-			log.Fatal().Str("address", node.Address).Msg("duplicate node address")
+			return nil, fmt.Errorf("duplicate node address: %v", node.Address)
 		}
+		addressLookup[node.Address] = struct{}{}
 	}
 
 	// test for duplicate node IDs
 	idLookup := make(map[flow.Identifier]struct{})
 	for _, node := range nodes {
 		if _, ok := idLookup[node.NodeID]; ok {
-			log.Fatal().Str("NodeID", node.NodeID.String()).Msg("duplicate node ID")
+			return nil, fmt.Errorf("duplicate node ID: %v", node.NodeID.String())
 		}
+		idLookup[node.NodeID] = struct{}{}
 	}
 
 	// sort nodes using the canonical ordering
 	nodes = model.Sort(nodes, flow.Canonical[flow.Identity])
 
-	return nodes
+	return nodes, nil
 }
 
 // readRootBlock reads root block data from disc, this file needs to be prepared with

cmd/bootstrap/cmd/finalize_test.go

@@ -190,3 +190,30 @@ func checkClusterConstraint(clusters flow.ClusterList, partnersInfo []model.Node
 	}
 	return true
 }
+
+func TestMergeNodeInfos(t *testing.T) {
+	partnersLen := 7
+	internalLen := 22
+	partners := unittest.NodeInfosFixture(partnersLen, unittest.WithRole(flow.RoleCollection))
+	internals := unittest.NodeInfosFixture(internalLen, unittest.WithRole(flow.RoleCollection))
+
+	// Check if there is no overlap, then should pass
+	merged, err := mergeNodeInfos(partners, internals)
+	require.NoError(t, err)
+	require.Len(t, merged, partnersLen+internalLen)
+
+	// Check if internals and partners have overlap, then should fail
+	internalAndPartnersHaveOverlap := append(partners, internals[0])
+	_, err = mergeNodeInfos(internalAndPartnersHaveOverlap, internals)
+	require.Error(t, err)
+
+	// Check if partners have overlap, then should fail
+	partnersHaveOverlap := append(partners, partners[0])
+	_, err = mergeNodeInfos(partnersHaveOverlap, internals)
+	require.Error(t, err)
+
+	// Check if internals have overlap, then should fail
+	internalsHaveOverlap := append(internals, internals[0])
+	_, err = mergeNodeInfos(partners, internalsHaveOverlap)
+	require.Error(t, err)
+}

cmd/bootstrap/cmd/partner_infos.go

@@ -35,9 +35,10 @@ const (
 )
 
 var (
-	flagANAddress    string
-	flagANNetworkKey string
-	flagNetworkEnv   string
+	flagANAddress          string
+	flagANNetworkKey       string
+	flagNetworkEnv         string
+	flagIncludeAccessNodes bool
 )
 
 // populatePartnerInfos represents the `populate-partner-infos` command which will read the proposed node
@@ -55,6 +56,7 @@ func init() {
 	populatePartnerInfosCMD.Flags().StringVar(&flagANAddress, "access-address", "", "the address of the access node used for client connections")
 	populatePartnerInfosCMD.Flags().StringVar(&flagANNetworkKey, "access-network-key", "", "the network key of the access node used for client connections in hex string format")
 	populatePartnerInfosCMD.Flags().StringVar(&flagNetworkEnv, "network", "mainnet", "the network string, expecting one of ( mainnet | testnet | emulator )")
+	populatePartnerInfosCMD.Flags().BoolVar(&flagIncludeAccessNodes, "include-candidate-access-nodes", false, "whether to include the candidate access nodes")
 
 	cmd.MarkFlagRequired(populatePartnerInfosCMD, "access-address")
 }
@@ -76,13 +78,26 @@ func populatePartnerInfosRun(_ *cobra.Command, _ []string) {
 		flow.RoleAccess:       0,
 	}
 	totalNumOfPartnerNodes := 0
+	var allNodes []cadence.Value
 
 	nodeInfos, err := executeGetProposedNodesInfosScript(ctx, flowClient)
 	if err != nil {
 		log.Fatal().Err(err).Msg("could not get node info for nodes in the proposed table")
 	}
+	allNodes = nodeInfos.(cadence.Array).Values[:]
 
-	for _, info := range nodeInfos.(cadence.Array).Values {
+	log.Info().Int("total_proposed_nodes", len(allNodes)).Msg("total nodes in proposed table")
+
+	if flagIncludeAccessNodes {
+		candidateNodeInfos, err := executeGetCandidateAccessNodesInfosScript(ctx, flowClient)
+		if err != nil {
+			log.Fatal().Err(err).Msg("could not get node info for nodes in the candidate table")
+		}
+		log.Info().Int("total_candidate_access_nodes", len(candidateNodeInfos.(cadence.Array).Values)).Msg("total access nodes in candidate table")
+		allNodes = append(allNodes, candidateNodeInfos.(cadence.Array).Values[:]...)
+	}
+
+	for _, info := range allNodes {
 		nodePubInfo, err := parseNodeInfo(info)
 		if err != nil {
 			log.Fatal().Err(err).Msg("could not parse node info from cadence script")
@@ -146,6 +161,21 @@ func executeGetProposedNodesInfosScript(ctx context.Context, client *client.Clie
 	return infos, nil
 }
 
+// GetNodeInfoForCandidateNodesScript executes the get node info for each Access node ID in the candidate table
+func executeGetCandidateAccessNodesInfosScript(ctx context.Context, client *client.Client) (cadence.Value, error) {
+	script, err := common.GetNodeInfoForCandidateNodesScript(flagNetworkEnv)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get cadence script: %w", err)
+	}
+
+	infos, err := client.ExecuteScriptAtLatestBlock(ctx, script, []cadence.Value{})
+	if err != nil {
+		return nil, fmt.Errorf("failed to execute the get node info script: %w", err)
+	}
+
+	return infos, nil
+}
+
 // parseNodeInfo convert node info retrieved from cadence script
 func parseNodeInfo(info cadence.Value) (*bootstrap.NodeInfoPub, error) {
 	fields := cadence.FieldsMappedByName(info.(cadence.Struct))

cmd/bootstrap/cmd/rootblock.go

@@ -146,7 +146,7 @@ func rootBlock(cmd *cobra.Command, args []string) {
 	}
 
 	log.Info().Msg("collecting partner network and staking keys")
-	partnerNodes, err := common.ReadFullPartnerNodeInfos(log, flagPartnerWeights, flagPartnerNodeInfoDir)
+	rawPartnerNodes, err := common.ReadFullPartnerNodeInfos(log, flagPartnerWeights, flagPartnerNodeInfoDir)
 	if err != nil {
 		log.Fatal().Err(err).Msg("failed to read full partner node infos")
 	}
@@ -160,12 +160,19 @@ func rootBlock(cmd *cobra.Command, args []string) {
 
 	log.Info().Msg("")
 
+	log.Info().Msg("remove internal partner nodes")
+	partnerNodes := common.FilterInternalPartners(rawPartnerNodes, internalNodes)
+	log.Info().Msgf("removed %d internal partner nodes", len(rawPartnerNodes)-len(partnerNodes))
+
 	log.Info().Msg("checking constraints on consensus nodes")
 	checkConstraints(partnerNodes, internalNodes)
 	log.Info().Msg("")
 
 	log.Info().Msg("assembling network and staking keys")
-	stakingNodes := mergeNodeInfos(internalNodes, partnerNodes)
+	stakingNodes, err := mergeNodeInfos(internalNodes, partnerNodes)
+	if err != nil {
+		log.Fatal().Err(err).Msgf("failed to merge node infos")
+	}
 	err = common.WriteJSON(model.PathNodeInfosPub, flagOutdir, model.ToPublicNodeInfoList(stakingNodes))
 	if err != nil {
 		log.Fatal().Err(err).Msg("failed to write json")

cmd/bootstrap/cmd/rootblock_test.go

@@ -21,7 +21,10 @@ const rootBlockHappyPathLogs = "collecting partner network and staking keys" +
 	`read \d+ internal private node-info files` +
 	`read internal node configurations` +
 	`read \d+ weights for internal nodes` +
+	`remove internal partner nodes` +
+	`removed 0 internal partner nodes` +
 	`checking constraints on consensus nodes` +
+
 	`assembling network and staking keys` +
 	`wrote file \S+/node-infos.pub.json` +
 	`running DKG for consensus nodes` +

cmd/bootstrap/run/qc.go

@@ -169,7 +169,7 @@ func GenerateQCParticipantData(allNodes, internalNodes []bootstrap.NodeInfo, dkg
 
 	// length of DKG participants needs to match stakingNodes, since we run DKG for external and internal validators
 	if len(allNodes) != len(dkgData.PrivKeyShares) {
-		return nil, fmt.Errorf("need exactly the same number of staking public keys as DKG private participants")
+		return nil, fmt.Errorf("need exactly the same number of staking public keys as DKG private participants, (all=%d, dkg=%d)", len(allNodes), len(dkgData.PrivKeyShares))
 	}
 
 	qcData := &ParticipantData{}

cmd/bootstrap/transit/cmd/pull.go

@@ -97,7 +97,7 @@ func pull(cmd *cobra.Command, args []string) {
 			fullOutpath := filepath.Join(flagBootDir, "public-root-information", filepath.Base(file.Name))
 			fmd5 := utils.CalcMd5(fullOutpath)
 			// only skip files that have an MD5 hash
-			if file.MD5 != nil && bytes.Equal(fmd5, file.MD5) {
+			if len(file.MD5) > 0 && bytes.Equal(fmd5, file.MD5) {
 				log.Info().Str("source", file.Name).Str("dest", fullOutpath).Msgf("skipping existing file from transit servers")
 				return
 			}

cmd/util/cmd/common/node_info.go

@@ -50,7 +50,7 @@ func ReadFullPartnerNodeInfos(log zerolog.Logger, partnerWeightsPath, partnerNod
 
 		weight := weights[partner.NodeID]
 		if valid := ValidateWeight(weight); !valid {
-			return nil, fmt.Errorf(fmt.Sprintf("invalid partner weight: %d", weight))
+			return nil, fmt.Errorf(fmt.Sprintf("invalid partner weight %v: %d", partner.NodeID, weight))
 		}
 
 		if weight != flow.DefaultInitialWeight {
@@ -148,7 +148,7 @@ func ReadFullInternalNodeInfos(log zerolog.Logger, internalNodePrivInfoDir, inte
 		weight := weights[internal.Address]
 
 		if valid := ValidateWeight(weight); !valid {
-			return nil, fmt.Errorf(fmt.Sprintf("invalid partner weight: %d", weight))
+			return nil, fmt.Errorf(fmt.Sprintf("invalid partner weight %v: %d", internal.NodeID, weight))
 		}
 		if weight != flow.DefaultInitialWeight {
 			log.Warn().Msgf("internal node (id=%x) has non-default weight (%d != %d)", internal.NodeID, weight, flow.DefaultInitialWeight)
@@ -224,3 +224,19 @@ func internalWeightsByAddress(log zerolog.Logger, config string) map[string]uint
 
 	return weights
 }
+
+// Reject any partner nodes that are in the internal node list.
+func FilterInternalPartners(partners []bootstrap.NodeInfo, internal []bootstrap.NodeInfo) []bootstrap.NodeInfo {
+	lookup := make(map[flow.Identifier]struct{})
+	for _, node := range internal {
+		lookup[node.NodeID] = struct{}{}
+	}
+
+	var filtered []bootstrap.NodeInfo
+	for _, node := range partners {
+		if _, ok := lookup[node.NodeID]; !ok {
+			filtered = append(filtered, node)
+		}
+	}
+	return filtered
+}

cmd/util/cmd/common/transactions.go

@@ -23,6 +23,21 @@ const (
 		
 			return infos
 	}`
+
+	getInfoForCandidateAccessNodesScript = `
+		import FlowIDTableStaking from "FlowIDTableStaking"
+		access(all) fun main(): [FlowIDTableStaking.NodeInfo] {
+		let candidateNodes = FlowIDTableStaking.getCandidateNodeList()
+		let candidateAccessNodes = candidateNodes[UInt8(5)]!
+
+        let nodeInfos: [FlowIDTableStaking.NodeInfo] = []
+        for nodeID in candidateAccessNodes.keys {
+            let nodeInfo = FlowIDTableStaking.NodeInfo(nodeID: nodeID)
+            nodeInfos.append(nodeInfo)
+        }
+
+        return nodeInfos
+	}`
 )
 
 // GetNodeInfoForProposedNodesScript returns a script that will return an array of FlowIDTableStaking.NodeInfo for each
@@ -37,3 +52,16 @@ func GetNodeInfoForProposedNodesScript(network string) ([]byte, error) {
 		),
 	), nil
 }
+
+// GetNodeInfoForCandidateNodesScript returns a script that will return an array of FlowIDTableStaking.NodeInfo for each
+// node in the candidate table (nodes which have staked but not yet chosen by the network).
+func GetNodeInfoForCandidateNodesScript(network string) ([]byte, error) {
+	contracts := systemcontracts.SystemContractsForChain(flow.ChainID(fmt.Sprintf("flow-%s", network)))
+
+	return []byte(
+		templates.ReplaceAddresses(
+			getInfoForCandidateAccessNodesScript,
+			contracts.AsTemplateEnv(),
+		),
+	), nil
+}