feat(consensus)!: implements state merkle root

Author: sdbondi

.license.ignore

@@ -6,4 +6,4 @@
 ./dan_layer/storage_sqlite/src/global/schema.rs
 ./dan_layer/state_store_sqlite/src/schema.rs
 ./dan_layer/wallet/storage_sqlite/src/schema.rs
-./scripts/env_sample
+./scripts/env_sample

Cargo.lock

@@ -26,6 +26,7 @@ members = [
     "dan_layer/p2p",
     "dan_layer/rpc_state_sync",
     "dan_layer/state_store_sqlite",
+    "dan_layer/state_tree",
     "dan_layer/storage_lmdb",
     "dan_layer/storage_sqlite",
     "dan_layer/storage",
@@ -87,6 +88,7 @@ tari_networking = { path = "networking/core" }
 tari_rpc_framework = { path = "networking/rpc_framework" }
 tari_rpc_macros = { path = "networking/rpc_macros" }
 tari_state_store_sqlite = { path = "dan_layer/state_store_sqlite" }
+tari_state_tree = { path = "dan_layer/state_tree" }
 tari_swarm = { path = "networking/swarm" }
 tari_template_abi = { version = "0.3.0", path = "dan_layer/template_abi" }
 tari_template_builtin = { path = "dan_layer/template_builtin" }

Cargo.toml

@@ -1,3 +1,8 @@
+/*
+ * //  Copyright 2024 The Tari Project
+ * //  SPDX-License-Identifier: BSD-3-Clause
+ */
+
 // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
 import type { SubstateId } from "./SubstateId";
 

bindings/src/types/Account.ts

@@ -1,3 +1,8 @@
+/*
+ * //  Copyright 2024 The Tari Project
+ * //  SPDX-License-Identifier: BSD-3-Clause
+ */
+
 // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
 import type { JrpcPermissions } from "./JrpcPermissions";
 

bindings/src/types/Claims.ts

@@ -1,3 +1,8 @@
+/*
+ * //  Copyright 2024 The Tari Project
+ * //  SPDX-License-Identifier: BSD-3-Clause
+ */
+
 // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
 import type { ComponentAddress } from "./ComponentAddress";
 import type { ResourceAddress } from "./ResourceAddress";

bindings/src/types/JrpcPermission.ts

@@ -1,3 +1,8 @@
+/*
+ * //  Copyright 2024 The Tari Project
+ * //  SPDX-License-Identifier: BSD-3-Clause
+ */
+
 // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
 import type { JrpcPermission } from "./JrpcPermission";
 

bindings/src/types/JrpcPermissions.ts

@@ -1,3 +1,8 @@
+/*
+ * //  Copyright 2024 The Tari Project
+ * //  SPDX-License-Identifier: BSD-3-Clause
+ */
+
 // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
 
 export type TransactionStatus =

bindings/src/types/TransactionStatus.ts

@@ -45,8 +45,11 @@ impl TariHasher {
     }
 
     pub fn result(self) -> FixedHash {
-        let hash: [u8; 32] = self.hasher.finalize().into();
-        hash.into()
+        self.finalize_into_array().into()
+    }
+
+    pub fn finalize_into_array(self) -> [u8; 32] {
+        self.hasher.finalize().into()
     }
 
     fn hash_writer(&mut self) -> impl Write + '_ {

dan_layer/common_types/src/hasher.rs

@@ -10,8 +10,10 @@ license.workspace = true
 [dependencies]
 tari_dan_common_types = { workspace = true }
 tari_dan_storage = { workspace = true }
+tari_engine_types = { workspace = true }
 tari_transaction = { workspace = true }
 tari_epoch_manager = { workspace = true }
+tari_state_tree = { workspace = true }
 
 # Used for PublicKey and Signature and Network enum
 tari_common = { workspace = true }

dan_layer/consensus/Cargo.toml

@@ -3,8 +3,22 @@
 
 use log::*;
 use tari_common::configuration::Network;
+use tari_common_types::types::FixedHash;
 use tari_dan_common_types::{committee::Committee, Epoch, NodeAddressable, NodeHeight};
-use tari_dan_storage::consensus_models::{Block, QuorumCertificate};
+use tari_dan_storage::consensus_models::{Block, LeafBlock, PendingStateTreeDiff, QuorumCertificate};
+use tari_engine_types::{
+    hashing::substate_value_hasher32,
+    substate::{Substate, SubstateDiff},
+};
+use tari_state_tree::{
+    Hash,
+    StagedTreeStore,
+    StateHashTreeDiff,
+    StateTreeError,
+    SubstateChange,
+    TreeStoreReader,
+    Version,
+};
 
 use crate::traits::LeaderStrategy;
 
@@ -15,14 +29,17 @@ const LOG_TARGET: &str = "tari::dan::consensus::hotstuff::common";
 /// TODO: exhaust > 0
 pub const EXHAUST_DIVISOR: u64 = 0;
 
-pub fn calculate_dummy_blocks<TAddr: NodeAddressable, TLeaderStrategy: LeaderStrategy<TAddr>>(
+/// Calculates the dummy block required to reach the new height and returns the last dummy block (parent for next
+/// proposal).
+pub fn calculate_last_dummy_block<TAddr: NodeAddressable, TLeaderStrategy: LeaderStrategy<TAddr>>(
     network: Network,
     epoch: Epoch,
     high_qc: &QuorumCertificate,
+    parent_merkle_root: FixedHash,
     new_height: NodeHeight,
     leader_strategy: &TLeaderStrategy,
     local_committee: &Committee<TAddr>,
-) -> Vec<Block> {
+) -> Option<LeafBlock> {
     let mut parent_block = high_qc.as_leaf_block();
     let mut current_height = high_qc.block_height() + NodeHeight(1);
     if current_height > new_height {
@@ -32,7 +49,7 @@ pub fn calculate_dummy_blocks<TAddr: NodeAddressable, TLeaderStrategy: LeaderStr
             current_height,
             new_height,
         );
-        return Vec::new();
+        return None;
     }
 
     debug!(
@@ -41,8 +58,6 @@ pub fn calculate_dummy_blocks<TAddr: NodeAddressable, TLeaderStrategy: LeaderStr
         current_height,
         new_height,
     );
-    let num_blocks = new_height.saturating_sub(current_height).as_u64() as usize;
-    let mut blocks = Vec::with_capacity(num_blocks);
     loop {
         let leader = leader_strategy.get_leader_public_key(local_committee, current_height);
         let dummy_block = Block::dummy_block(
@@ -52,20 +67,56 @@ pub fn calculate_dummy_blocks<TAddr: NodeAddressable, TLeaderStrategy: LeaderStr
             current_height,
             high_qc.clone(),
             epoch,
+            parent_merkle_root,
         );
         debug!(
             target: LOG_TARGET,
             "🍼 new dummy block: {}",
             dummy_block,
         );
         parent_block = dummy_block.as_leaf_block();
-        blocks.push(dummy_block);
 
         if current_height == new_height {
             break;
         }
         current_height += NodeHeight(1);
     }
 
-    blocks
+    Some(parent_block)
+}
+
+pub fn diff_to_substate_changes(diff: &SubstateDiff) -> impl Iterator<Item = SubstateChange> + '_ {
+    diff.down_iter()
+        .map(|(substate_id, _version)| SubstateChange::Down {
+            id: substate_id.clone(),
+        })
+        .chain(diff.up_iter().map(move |(substate_id, value)| SubstateChange::Up {
+            id: substate_id.clone(),
+            value_hash: hash_substate(value),
+        }))
+}
+
+pub fn hash_substate(substate: &Substate) -> FixedHash {
+    substate_value_hasher32().chain(substate).result().into_array().into()
+}
+
+pub fn calculate_state_merkle_diff<TTx: TreeStoreReader<Version>, I: IntoIterator<Item = SubstateChange>>(
+    tx: &TTx,
+    current_version: Version,
+    next_version: Version,
+    pending_tree_updates: Vec<PendingStateTreeDiff>,
+    substate_changes: I,
+) -> Result<(Hash, StateHashTreeDiff), StateTreeError> {
+    debug!(
+        target: LOG_TARGET,
+        "Calculating state merkle diff from version {} to {} with {} update(s)",
+        current_version,
+        next_version,
+        pending_tree_updates.len(),
+    );
+    let mut store = StagedTreeStore::new(tx);
+    store.apply_ordered_diffs(pending_tree_updates.into_iter().map(|diff| diff.diff));
+    let mut state_tree = tari_state_tree::SpreadPrefixStateTree::new(&mut store);
+    let state_root = state_tree.put_substate_changes(current_version, next_version, substate_changes)?;
+    Ok((state_root, store.into_diff()))
 }

dan_layer/consensus/src/hotstuff/common.rs

@@ -1,13 +1,15 @@
 //   Copyright 2023 The Tari Project
 //   SPDX-License-Identifier: BSD-3-Clause
 
+use tari_common_types::types::FixedHash;
 use tari_dan_common_types::{Epoch, NodeHeight};
 use tari_dan_storage::{
     consensus_models::{BlockId, LeafBlock, LockedBlock, QuorumCertificate, TransactionPoolError},
     StorageError,
 };
 use tari_epoch_manager::EpochManagerError;
 use tari_mmr::BalancedBinaryMerkleProofError;
+use tari_state_tree::StateTreeError;
 use tari_transaction::TransactionId;
 
 use crate::traits::{InboundMessagingError, OutboundMessagingError};
@@ -16,6 +18,8 @@ use crate::traits::{InboundMessagingError, OutboundMessagingError};
 pub enum HotStuffError {
     #[error("Storage error: {0}")]
     StorageError(#[from] StorageError),
+    #[error("State tree error: {0}")]
+    StateTreeError(#[from] StateTreeError),
     #[error("Internal channel send error when {context}")]
     InternalChannelClosed { context: &'static str },
     #[error("Inbound messaging error: {0}")]
@@ -176,4 +180,10 @@ pub enum ProposalValidationError {
         block_network: String,
         block_id: BlockId,
     },
+    #[error("Invalid state merkle root for block {block_id}: calculated {calculated} but block has {from_block}")]
+    InvalidStateMerkleRoot {
+        block_id: BlockId,
+        calculated: FixedHash,
+        from_block: FixedHash,
+    },
 }