feat(consensus)!: implements state merkle root (#924)

Description
---
Implements Merkle root committing to substate state within a shard

Motivation and Context
---
This PR adds a correct Merkle root within block proposals and implements
validation against the state to be committed in the proposed block. The
state Merkle has is also validated when syncing. The tree is an
implementation of [Jellyfish Merkle
tree](https://diem-developers-components.netlify.app/papers/jellyfish-merkle-tree/2021-01-14.pdf)
adapted from Scrypto.

The tree is not currently pruned although each node is marked to be
removed. A state tree diff is produced for each uncommitted block and
applied and removed once the block is committed (3-chain).

How Has This Been Tested?
---
New unit tests, existing cucumbers, existing consensus tests, manually
(swap bench, stress test, sync fresh node, sync after offline)

When running the swap bench, clearing and re-syncing a node, there was
an MR mismatch for one very large transaction (creating hundreds of
accounts in 1 tx). I have not been able to track down the cause, but I
believe that this is a pre-existing determinism issue with sync/substate
store. Other than that, this appears to be working as expected.

What process can a PR reviewer use to test or verify this change?
---
Nodes should function as before

Breaking Changes
---

- [ ] None
- [x] Requires data directory to be deleted
- [ ] Other - Please specify

.license.ignore

@@ -6,4 +6,4 @@
 ./dan_layer/storage_sqlite/src/global/schema.rs
 ./dan_layer/state_store_sqlite/src/schema.rs
 ./dan_layer/wallet/storage_sqlite/src/schema.rs
-./scripts/env_sample
+./scripts/env_sample

Cargo.toml

@@ -26,6 +26,7 @@ members = [
     "dan_layer/p2p",
     "dan_layer/rpc_state_sync",
     "dan_layer/state_store_sqlite",
+    "dan_layer/state_tree",
     "dan_layer/storage_lmdb",
     "dan_layer/storage_sqlite",
     "dan_layer/storage",
@@ -87,6 +88,7 @@ tari_networking = { path = "networking/core" }
 tari_rpc_framework = { path = "networking/rpc_framework" }
 tari_rpc_macros = { path = "networking/rpc_macros" }
 tari_state_store_sqlite = { path = "dan_layer/state_store_sqlite" }
+tari_state_tree = { path = "dan_layer/state_tree" }
 tari_swarm = { path = "networking/swarm" }
 tari_template_abi = { version = "0.3.0", path = "dan_layer/template_abi" }
 tari_template_builtin = { path = "dan_layer/template_builtin" }

bindings/src/types/Account.ts

@@ -1,3 +1,8 @@
+/*
+ * //  Copyright 2024 The Tari Project
+ * //  SPDX-License-Identifier: BSD-3-Clause
+ */
+
 // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
 import type { SubstateId } from "./SubstateId";
 

bindings/src/types/Claims.ts

@@ -1,3 +1,8 @@
+/*
+ * //  Copyright 2024 The Tari Project
+ * //  SPDX-License-Identifier: BSD-3-Clause
+ */
+
 // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
 import type { JrpcPermissions } from "./JrpcPermissions";
 

bindings/src/types/JrpcPermission.ts

@@ -1,3 +1,8 @@
+/*
+ * //  Copyright 2024 The Tari Project
+ * //  SPDX-License-Identifier: BSD-3-Clause
+ */
+
 // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
 import type { ComponentAddress } from "./ComponentAddress";
 import type { ResourceAddress } from "./ResourceAddress";

bindings/src/types/JrpcPermissions.ts

@@ -1,3 +1,8 @@
+/*
+ * //  Copyright 2024 The Tari Project
+ * //  SPDX-License-Identifier: BSD-3-Clause
+ */
+
 // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
 import type { JrpcPermission } from "./JrpcPermission";
 

bindings/src/types/TransactionStatus.ts

@@ -1,3 +1,8 @@
+/*
+ * //  Copyright 2024 The Tari Project
+ * //  SPDX-License-Identifier: BSD-3-Clause
+ */
+
 // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
 
 export type TransactionStatus =

dan_layer/common_types/src/hasher.rs

@@ -45,8 +45,11 @@ impl TariHasher {
     }
 
     pub fn result(self) -> FixedHash {
-        let hash: [u8; 32] = self.hasher.finalize().into();
-        hash.into()
+        self.finalize_into_array().into()
+    }
+
+    pub fn finalize_into_array(self) -> [u8; 32] {
+        self.hasher.finalize().into()
     }
 
     fn hash_writer(&mut self) -> impl Write + '_ {

dan_layer/consensus/Cargo.toml

@@ -10,8 +10,10 @@ license.workspace = true
 [dependencies]
 tari_dan_common_types = { workspace = true }
 tari_dan_storage = { workspace = true }
+tari_engine_types = { workspace = true }
 tari_transaction = { workspace = true }
 tari_epoch_manager = { workspace = true }
+tari_state_tree = { workspace = true }
 
 # Used for PublicKey and Signature and Network enum
 tari_common = { workspace = true }

dan_layer/consensus/src/hotstuff/common.rs

@@ -3,8 +3,22 @@
 
 use log::*;
 use tari_common::configuration::Network;
+use tari_common_types::types::FixedHash;
 use tari_dan_common_types::{committee::Committee, Epoch, NodeAddressable, NodeHeight};
-use tari_dan_storage::consensus_models::{Block, QuorumCertificate};
+use tari_dan_storage::consensus_models::{Block, LeafBlock, PendingStateTreeDiff, QuorumCertificate};
+use tari_engine_types::{
+    hashing::substate_value_hasher32,
+    substate::{Substate, SubstateDiff},
+};
+use tari_state_tree::{
+    Hash,
+    StagedTreeStore,
+    StateHashTreeDiff,
+    StateTreeError,
+    SubstateChange,
+    TreeStoreReader,
+    Version,
+};
 
 use crate::traits::LeaderStrategy;
 
@@ -15,14 +29,17 @@ const LOG_TARGET: &str = "tari::dan::consensus::hotstuff::common";
 /// TODO: exhaust > 0
 pub const EXHAUST_DIVISOR: u64 = 0;
 
-pub fn calculate_dummy_blocks<TAddr: NodeAddressable, TLeaderStrategy: LeaderStrategy<TAddr>>(
+/// Calculates the dummy block required to reach the new height and returns the last dummy block (parent for next
+/// proposal).
+pub fn calculate_last_dummy_block<TAddr: NodeAddressable, TLeaderStrategy: LeaderStrategy<TAddr>>(
     network: Network,
     epoch: Epoch,
     high_qc: &QuorumCertificate,
+    parent_merkle_root: FixedHash,
     new_height: NodeHeight,
     leader_strategy: &TLeaderStrategy,
     local_committee: &Committee<TAddr>,
-) -> Vec<Block> {
+) -> Option<LeafBlock> {
     let mut parent_block = high_qc.as_leaf_block();
     let mut current_height = high_qc.block_height() + NodeHeight(1);
     if current_height > new_height {
@@ -32,7 +49,7 @@ pub fn calculate_dummy_blocks<TAddr: NodeAddressable, TLeaderStrategy: LeaderStr
             current_height,
             new_height,
         );
-        return Vec::new();
+        return None;
     }
 
     debug!(
@@ -41,8 +58,6 @@ pub fn calculate_dummy_blocks<TAddr: NodeAddressable, TLeaderStrategy: LeaderStr
         current_height,
         new_height,
     );
-    let num_blocks = new_height.saturating_sub(current_height).as_u64() as usize;
-    let mut blocks = Vec::with_capacity(num_blocks);
     loop {
         let leader = leader_strategy.get_leader_public_key(local_committee, current_height);
         let dummy_block = Block::dummy_block(
@@ -52,20 +67,56 @@ pub fn calculate_dummy_blocks<TAddr: NodeAddressable, TLeaderStrategy: LeaderStr
             current_height,
             high_qc.clone(),
             epoch,
+            parent_merkle_root,
         );
         debug!(
             target: LOG_TARGET,
             "🍼 new dummy block: {}",
             dummy_block,
         );
         parent_block = dummy_block.as_leaf_block();
-        blocks.push(dummy_block);
 
         if current_height == new_height {
             break;
         }
         current_height += NodeHeight(1);
     }
 
-    blocks
+    Some(parent_block)
+}
+
+pub fn diff_to_substate_changes(diff: &SubstateDiff) -> impl Iterator<Item = SubstateChange> + '_ {
+    diff.down_iter()
+        .map(|(substate_id, _version)| SubstateChange::Down {
+            id: substate_id.clone(),
+        })
+        .chain(diff.up_iter().map(move |(substate_id, value)| SubstateChange::Up {
+            id: substate_id.clone(),
+            value_hash: hash_substate(value),
+        }))
+}
+
+pub fn hash_substate(substate: &Substate) -> FixedHash {
+    substate_value_hasher32().chain(substate).result().into_array().into()
+}
+
+pub fn calculate_state_merkle_diff<TTx: TreeStoreReader<Version>, I: IntoIterator<Item = SubstateChange>>(
+    tx: &TTx,
+    current_version: Version,
+    next_version: Version,
+    pending_tree_updates: Vec<PendingStateTreeDiff>,
+    substate_changes: I,
+) -> Result<(Hash, StateHashTreeDiff), StateTreeError> {
+    debug!(
+        target: LOG_TARGET,
+        "Calculating state merkle diff from version {} to {} with {} update(s)",
+        current_version,
+        next_version,
+        pending_tree_updates.len(),
+    );
+    let mut store = StagedTreeStore::new(tx);
+    store.apply_ordered_diffs(pending_tree_updates.into_iter().map(|diff| diff.diff));
+    let mut state_tree = tari_state_tree::SpreadPrefixStateTree::new(&mut store);
+    let state_root = state_tree.put_substate_changes(current_version, next_version, substate_changes)?;
+    Ok((state_root, store.into_diff()))
 }

dan_layer/consensus/src/hotstuff/error.rs

@@ -1,13 +1,15 @@
 //   Copyright 2023 The Tari Project
 //   SPDX-License-Identifier: BSD-3-Clause
 
+use tari_common_types::types::FixedHash;
 use tari_dan_common_types::{Epoch, NodeHeight};
 use tari_dan_storage::{
     consensus_models::{BlockId, LeafBlock, LockedBlock, QuorumCertificate, TransactionPoolError},
     StorageError,
 };
 use tari_epoch_manager::EpochManagerError;
 use tari_mmr::BalancedBinaryMerkleProofError;
+use tari_state_tree::StateTreeError;
 use tari_transaction::TransactionId;
 
 use crate::traits::{InboundMessagingError, OutboundMessagingError};
@@ -16,6 +18,8 @@ use crate::traits::{InboundMessagingError, OutboundMessagingError};
 pub enum HotStuffError {
     #[error("Storage error: {0}")]
     StorageError(#[from] StorageError),
+    #[error("State tree error: {0}")]
+    StateTreeError(#[from] StateTreeError),
     #[error("Internal channel send error when {context}")]
     InternalChannelClosed { context: &'static str },
     #[error("Inbound messaging error: {0}")]
@@ -178,4 +182,10 @@ pub enum ProposalValidationError {
         block_network: String,
         block_id: BlockId,
     },
+    #[error("Invalid state merkle root for block {block_id}: calculated {calculated} but block has {from_block}")]
+    InvalidStateMerkleRoot {
+        block_id: BlockId,
+        calculated: FixedHash,
+        from_block: FixedHash,
+    },
 }