From 1a33149697c48feb420d8ed778febca953a67222 Mon Sep 17 00:00:00 2001 From: Stan Bondi Date: Fri, 9 Feb 2024 11:42:39 +0400 Subject: [PATCH] feat(consensus)!: implements state merkle root (#924) Description --- Implements Merkle root committing to substate state within a shard Motivation and Context --- This PR adds a correct Merkle root within block proposals and implements validation against the state to be committed in the proposed block. The state Merkle has is also validated when syncing. The tree is an implementation of [Jellyfish Merkle tree](https://diem-developers-components.netlify.app/papers/jellyfish-merkle-tree/2021-01-14.pdf) adapted from Scrypto. The tree is not currently pruned although each node is marked to be removed. A state tree diff is produced for each uncommitted block and applied and removed once the block is committed (3-chain). How Has This Been Tested? --- New unit tests, existing cucumbers, existing consensus tests, manually (swap bench, stress test, sync fresh node, sync after offline) When running the swap bench, clearing and re-syncing a node, there was an MR mismatch for one very large transaction (creating hundreds of accounts in 1 tx). I have not been able to track down the cause, but I believe that this is a pre-existing determinism issue with sync/substate store. Other than that, this appears to be working as expected. What process can a PR reviewer use to test or verify this change? --- Nodes should function as before Breaking Changes --- - [ ] None - [x] Requires data directory to be deleted - [ ] Other - Please specify --- .license.ignore | 2 +- Cargo.lock | 23 + Cargo.toml | 2 + bindings/src/types/Account.ts | 5 + bindings/src/types/Claims.ts | 5 + bindings/src/types/JrpcPermission.ts | 5 + bindings/src/types/JrpcPermissions.ts | 5 + bindings/src/types/TransactionStatus.ts | 5 + dan_layer/common_types/src/hasher.rs | 7 +- dan_layer/consensus/Cargo.toml | 2 + dan_layer/consensus/src/hotstuff/common.rs | 67 +- dan_layer/consensus/src/hotstuff/error.rs | 10 + .../consensus/src/hotstuff/on_propose.rs | 52 +- .../on_ready_to_vote_on_local_block.rs | 23 +- .../src/hotstuff/on_receive_local_proposal.rs | 60 +- .../src/hotstuff/on_receive_new_view.rs | 12 +- dan_layer/consensus/src/hotstuff/worker.rs | 64 +- dan_layer/consensus/src/traits/sync.rs | 2 +- dan_layer/consensus_tests/Cargo.toml | 1 + dan_layer/consensus_tests/src/consensus.rs | 3 +- dan_layer/consensus_tests/src/support/sync.rs | 2 +- .../src/support/transaction.rs | 26 +- dan_layer/engine/src/runtime/tracker.rs | 4 - dan_layer/engine_types/src/commit_result.rs | 16 + dan_layer/engine_types/src/hashing.rs | 6 + dan_layer/p2p/proto/rpc.proto | 7 +- dan_layer/p2p/src/conversions/consensus.rs | 1 + dan_layer/p2p/src/conversions/rpc.rs | 50 +- dan_layer/rpc_state_sync/Cargo.toml | 1 + dan_layer/rpc_state_sync/src/error.rs | 2 + dan_layer/rpc_state_sync/src/manager.rs | 161 ++- dan_layer/state_store_sqlite/Cargo.toml | 5 +- .../up.sql | 28 + dan_layer/state_store_sqlite/src/lib.rs | 1 + dan_layer/state_store_sqlite/src/reader.rs | 356 +++-- dan_layer/state_store_sqlite/src/schema.rs | 23 + .../src/sql_models/block.rs | 4 + .../state_store_sqlite/src/sql_models/mod.rs | 2 + .../src/sql_models/pending_state_tree_diff.rs | 28 + .../src/sqlite_transaction.rs | 13 +- .../state_store_sqlite/src/tree_store.rs | 70 + dan_layer/state_store_sqlite/src/writer.rs | 320 +++-- dan_layer/state_store_sqlite/tests/tests.rs | 1 + dan_layer/state_tree/Cargo.toml | 23 + dan_layer/state_tree/src/error.rs | 10 + dan_layer/state_tree/src/jellyfish/hash.rs | 134 ++ dan_layer/state_tree/src/jellyfish/mod.rs | 14 + dan_layer/state_tree/src/jellyfish/store.rs | 79 ++ dan_layer/state_tree/src/jellyfish/tree.rs | 733 ++++++++++ dan_layer/state_tree/src/jellyfish/types.rs | 1249 +++++++++++++++++ dan_layer/state_tree/src/key_mapper.rs | 22 + dan_layer/state_tree/src/lib.rs | 16 + dan_layer/state_tree/src/memory_store.rs | 75 + dan_layer/state_tree/src/staged_store.rs | 92 ++ dan_layer/state_tree/src/tree.rs | 165 +++ dan_layer/state_tree/tests/support.rs | 79 ++ dan_layer/state_tree/tests/test.rs | 161 +++ dan_layer/storage/Cargo.toml | 1 + .../storage/src/consensus_models/block.rs | 75 +- dan_layer/storage/src/consensus_models/mod.rs | 2 + .../src/consensus_models/state_tree_diff.rs | 60 + .../storage/src/consensus_models/substate.rs | 36 +- .../src/consensus_models/transaction_pool.rs | 18 +- .../src/consensus_models/validated_block.rs | 6 +- dan_layer/storage/src/shard_store.rs | 256 ---- dan_layer/storage/src/state_store/mod.rs | 24 +- 66 files changed, 4086 insertions(+), 726 deletions(-) create mode 100644 dan_layer/state_store_sqlite/src/sql_models/pending_state_tree_diff.rs create mode 100644 dan_layer/state_store_sqlite/src/tree_store.rs create mode 100644 dan_layer/state_tree/Cargo.toml create mode 100644 dan_layer/state_tree/src/error.rs create mode 100644 dan_layer/state_tree/src/jellyfish/hash.rs create mode 100644 dan_layer/state_tree/src/jellyfish/mod.rs create mode 100644 dan_layer/state_tree/src/jellyfish/store.rs create mode 100644 dan_layer/state_tree/src/jellyfish/tree.rs create mode 100644 dan_layer/state_tree/src/jellyfish/types.rs create mode 100644 dan_layer/state_tree/src/key_mapper.rs create mode 100644 dan_layer/state_tree/src/lib.rs create mode 100644 dan_layer/state_tree/src/memory_store.rs create mode 100644 dan_layer/state_tree/src/staged_store.rs create mode 100644 dan_layer/state_tree/src/tree.rs create mode 100644 dan_layer/state_tree/tests/support.rs create mode 100644 dan_layer/state_tree/tests/test.rs create mode 100644 dan_layer/storage/src/consensus_models/state_tree_diff.rs delete mode 100644 dan_layer/storage/src/shard_store.rs diff --git a/.license.ignore b/.license.ignore index d4cc847e2b..a2cc470eb3 100644 --- a/.license.ignore +++ b/.license.ignore @@ -6,4 +6,4 @@ ./dan_layer/storage_sqlite/src/global/schema.rs ./dan_layer/state_store_sqlite/src/schema.rs ./dan_layer/wallet/storage_sqlite/src/schema.rs -./scripts/env_sample +./scripts/env_sample \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index 8171179171..2fb2312529 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1673,6 +1673,7 @@ dependencies = [ "log", "rand", "serde", + "tari_bor", "tari_common_types", "tari_consensus", "tari_crypto", @@ -8905,9 +8906,11 @@ dependencies = [ "tari_common_types", "tari_dan_common_types", "tari_dan_storage", + "tari_engine_types", "tari_epoch_manager", "tari_mmr", "tari_shutdown", + "tari_state_tree", "tari_transaction", "thiserror", "tokio", @@ -9173,6 +9176,7 @@ dependencies = [ "tari_dan_common_types", "tari_engine_types", "tari_mmr", + "tari_state_tree", "tari_transaction", "thiserror", "time", @@ -9662,6 +9666,7 @@ dependencies = [ "tari_dan_storage", "tari_epoch_manager", "tari_rpc_framework", + "tari_state_tree", "tari_transaction", "tari_validator_node_rpc", "thiserror", @@ -9758,12 +9763,30 @@ dependencies = [ "tari_dan_common_types", "tari_dan_storage", "tari_engine_types", + "tari_state_tree", "tari_transaction", "tari_utilities", "thiserror", "time", ] +[[package]] +name = "tari_state_tree" +version = "0.3.0" +dependencies = [ + "hex", + "indexmap 2.1.0", + "itertools 0.11.0", + "log", + "serde", + "tari_common_types", + "tari_crypto", + "tari_dan_common_types", + "tari_engine_types", + "tari_template_lib", + "thiserror", +] + [[package]] name = "tari_storage" version = "1.0.0-dan.5" diff --git a/Cargo.toml b/Cargo.toml index 7647d1318b..fc215f92aa 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,6 +26,7 @@ members = [ "dan_layer/p2p", "dan_layer/rpc_state_sync", "dan_layer/state_store_sqlite", + "dan_layer/state_tree", "dan_layer/storage_lmdb", "dan_layer/storage_sqlite", "dan_layer/storage", @@ -87,6 +88,7 @@ tari_networking = { path = "networking/core" } tari_rpc_framework = { path = "networking/rpc_framework" } tari_rpc_macros = { path = "networking/rpc_macros" } tari_state_store_sqlite = { path = "dan_layer/state_store_sqlite" } +tari_state_tree = { path = "dan_layer/state_tree" } tari_swarm = { path = "networking/swarm" } tari_template_abi = { version = "0.3.0", path = "dan_layer/template_abi" } tari_template_builtin = { path = "dan_layer/template_builtin" } diff --git a/bindings/src/types/Account.ts b/bindings/src/types/Account.ts index ab5adb79bb..4895dbf4a1 100644 --- a/bindings/src/types/Account.ts +++ b/bindings/src/types/Account.ts @@ -1,3 +1,8 @@ +/* + * // Copyright 2024 The Tari Project + * // SPDX-License-Identifier: BSD-3-Clause + */ + // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. import type { SubstateId } from "./SubstateId"; diff --git a/bindings/src/types/Claims.ts b/bindings/src/types/Claims.ts index 03c6b98d7d..59f53a7c44 100644 --- a/bindings/src/types/Claims.ts +++ b/bindings/src/types/Claims.ts @@ -1,3 +1,8 @@ +/* + * // Copyright 2024 The Tari Project + * // SPDX-License-Identifier: BSD-3-Clause + */ + // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. import type { JrpcPermissions } from "./JrpcPermissions"; diff --git a/bindings/src/types/JrpcPermission.ts b/bindings/src/types/JrpcPermission.ts index c8ca7afe08..735a39e0bb 100644 --- a/bindings/src/types/JrpcPermission.ts +++ b/bindings/src/types/JrpcPermission.ts @@ -1,3 +1,8 @@ +/* + * // Copyright 2024 The Tari Project + * // SPDX-License-Identifier: BSD-3-Clause + */ + // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. import type { ComponentAddress } from "./ComponentAddress"; import type { ResourceAddress } from "./ResourceAddress"; diff --git a/bindings/src/types/JrpcPermissions.ts b/bindings/src/types/JrpcPermissions.ts index 9c212ab4eb..90158ef867 100644 --- a/bindings/src/types/JrpcPermissions.ts +++ b/bindings/src/types/JrpcPermissions.ts @@ -1,3 +1,8 @@ +/* + * // Copyright 2024 The Tari Project + * // SPDX-License-Identifier: BSD-3-Clause + */ + // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. import type { JrpcPermission } from "./JrpcPermission"; diff --git a/bindings/src/types/TransactionStatus.ts b/bindings/src/types/TransactionStatus.ts index ab9356ff73..cb3b43f711 100644 --- a/bindings/src/types/TransactionStatus.ts +++ b/bindings/src/types/TransactionStatus.ts @@ -1,3 +1,8 @@ +/* + * // Copyright 2024 The Tari Project + * // SPDX-License-Identifier: BSD-3-Clause + */ + // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. export type TransactionStatus = diff --git a/dan_layer/common_types/src/hasher.rs b/dan_layer/common_types/src/hasher.rs index b8dc8aca4e..2bca823594 100644 --- a/dan_layer/common_types/src/hasher.rs +++ b/dan_layer/common_types/src/hasher.rs @@ -45,8 +45,11 @@ impl TariHasher { } pub fn result(self) -> FixedHash { - let hash: [u8; 32] = self.hasher.finalize().into(); - hash.into() + self.finalize_into_array().into() + } + + pub fn finalize_into_array(self) -> [u8; 32] { + self.hasher.finalize().into() } fn hash_writer(&mut self) -> impl Write + '_ { diff --git a/dan_layer/consensus/Cargo.toml b/dan_layer/consensus/Cargo.toml index 7fdaef8642..827db23bf1 100644 --- a/dan_layer/consensus/Cargo.toml +++ b/dan_layer/consensus/Cargo.toml @@ -10,8 +10,10 @@ license.workspace = true [dependencies] tari_dan_common_types = { workspace = true } tari_dan_storage = { workspace = true } +tari_engine_types = { workspace = true } tari_transaction = { workspace = true } tari_epoch_manager = { workspace = true } +tari_state_tree = { workspace = true } # Used for PublicKey and Signature and Network enum tari_common = { workspace = true } diff --git a/dan_layer/consensus/src/hotstuff/common.rs b/dan_layer/consensus/src/hotstuff/common.rs index dde7131cea..fb0e88323c 100644 --- a/dan_layer/consensus/src/hotstuff/common.rs +++ b/dan_layer/consensus/src/hotstuff/common.rs @@ -3,8 +3,22 @@ use log::*; use tari_common::configuration::Network; +use tari_common_types::types::FixedHash; use tari_dan_common_types::{committee::Committee, Epoch, NodeAddressable, NodeHeight}; -use tari_dan_storage::consensus_models::{Block, QuorumCertificate}; +use tari_dan_storage::consensus_models::{Block, LeafBlock, PendingStateTreeDiff, QuorumCertificate}; +use tari_engine_types::{ + hashing::substate_value_hasher32, + substate::{Substate, SubstateDiff}, +}; +use tari_state_tree::{ + Hash, + StagedTreeStore, + StateHashTreeDiff, + StateTreeError, + SubstateChange, + TreeStoreReader, + Version, +}; use crate::traits::LeaderStrategy; @@ -15,14 +29,17 @@ const LOG_TARGET: &str = "tari::dan::consensus::hotstuff::common"; /// TODO: exhaust > 0 pub const EXHAUST_DIVISOR: u64 = 0; -pub fn calculate_dummy_blocks>( +/// Calculates the dummy block required to reach the new height and returns the last dummy block (parent for next +/// proposal). +pub fn calculate_last_dummy_block>( network: Network, epoch: Epoch, high_qc: &QuorumCertificate, + parent_merkle_root: FixedHash, new_height: NodeHeight, leader_strategy: &TLeaderStrategy, local_committee: &Committee, -) -> Vec { +) -> Option { let mut parent_block = high_qc.as_leaf_block(); let mut current_height = high_qc.block_height() + NodeHeight(1); if current_height > new_height { @@ -32,7 +49,7 @@ pub fn calculate_dummy_blocks impl Iterator + '_ { + diff.down_iter() + .map(|(substate_id, _version)| SubstateChange::Down { + id: substate_id.clone(), + }) + .chain(diff.up_iter().map(move |(substate_id, value)| SubstateChange::Up { + id: substate_id.clone(), + value_hash: hash_substate(value), + })) +} + +pub fn hash_substate(substate: &Substate) -> FixedHash { + substate_value_hasher32().chain(substate).result().into_array().into() +} + +pub fn calculate_state_merkle_diff, I: IntoIterator>( + tx: &TTx, + current_version: Version, + next_version: Version, + pending_tree_updates: Vec, + substate_changes: I, +) -> Result<(Hash, StateHashTreeDiff), StateTreeError> { + debug!( + target: LOG_TARGET, + "Calculating state merkle diff from version {} to {} with {} update(s)", + current_version, + next_version, + pending_tree_updates.len(), + ); + let mut store = StagedTreeStore::new(tx); + store.apply_ordered_diffs(pending_tree_updates.into_iter().map(|diff| diff.diff)); + let mut state_tree = tari_state_tree::SpreadPrefixStateTree::new(&mut store); + let state_root = state_tree.put_substate_changes(current_version, next_version, substate_changes)?; + Ok((state_root, store.into_diff())) } diff --git a/dan_layer/consensus/src/hotstuff/error.rs b/dan_layer/consensus/src/hotstuff/error.rs index 20374b5f72..1dcc2bdf84 100644 --- a/dan_layer/consensus/src/hotstuff/error.rs +++ b/dan_layer/consensus/src/hotstuff/error.rs @@ -1,6 +1,7 @@ // Copyright 2023 The Tari Project // SPDX-License-Identifier: BSD-3-Clause +use tari_common_types::types::FixedHash; use tari_dan_common_types::{Epoch, NodeHeight}; use tari_dan_storage::{ consensus_models::{BlockId, LeafBlock, LockedBlock, QuorumCertificate, TransactionPoolError}, @@ -8,6 +9,7 @@ use tari_dan_storage::{ }; use tari_epoch_manager::EpochManagerError; use tari_mmr::BalancedBinaryMerkleProofError; +use tari_state_tree::StateTreeError; use tari_transaction::TransactionId; use crate::traits::{InboundMessagingError, OutboundMessagingError}; @@ -16,6 +18,8 @@ use crate::traits::{InboundMessagingError, OutboundMessagingError}; pub enum HotStuffError { #[error("Storage error: {0}")] StorageError(#[from] StorageError), + #[error("State tree error: {0}")] + StateTreeError(#[from] StateTreeError), #[error("Internal channel send error when {context}")] InternalChannelClosed { context: &'static str }, #[error("Inbound messaging error: {0}")] @@ -178,4 +182,10 @@ pub enum ProposalValidationError { block_network: String, block_id: BlockId, }, + #[error("Invalid state merkle root for block {block_id}: calculated {calculated} but block has {from_block}")] + InvalidStateMerkleRoot { + block_id: BlockId, + calculated: FixedHash, + from_block: FixedHash, + }, } diff --git a/dan_layer/consensus/src/hotstuff/on_propose.rs b/dan_layer/consensus/src/hotstuff/on_propose.rs index 6502280303..bf48aaa699 100644 --- a/dan_layer/consensus/src/hotstuff/on_propose.rs +++ b/dan_layer/consensus/src/hotstuff/on_propose.rs @@ -1,7 +1,11 @@ // Copyright 2023 The Tari Project // SPDX-License-Identifier: BSD-3-Clause -use std::{collections::BTreeSet, num::NonZeroU64, ops::DerefMut}; +use std::{ + collections::BTreeSet, + num::NonZeroU64, + ops::{Deref, DerefMut}, +}; use indexmap::IndexMap; use log::*; @@ -23,6 +27,7 @@ use tari_dan_storage::{ LastProposed, LeafBlock, LockedBlock, + PendingStateTreeDiff, QuorumCertificate, TransactionPool, TransactionPoolStage, @@ -32,7 +37,13 @@ use tari_dan_storage::{ use tari_epoch_manager::EpochManagerReader; use crate::{ - hotstuff::{common::EXHAUST_DIVISOR, error::HotStuffError, proposer}, + hotstuff::{ + calculate_state_merkle_diff, + common::EXHAUST_DIVISOR, + diff_to_substate_changes, + error::HotStuffError, + proposer, + }, messages::{HotstuffMessage, ProposalMessage}, traits::{ConsensusSpec, OutboundMessaging, ValidatorSignatureService}, }; @@ -171,6 +182,7 @@ where TConsensusSpec: ConsensusSpec Ok(()) } + #[allow(clippy::too_many_lines)] fn build_next_block( &self, tx: &mut ::ReadTransaction<'_>, @@ -188,8 +200,11 @@ where TConsensusSpec: ConsensusSpec } else { self.transaction_pool.get_batch_for_next_block(tx, TARGET_BLOCK_SIZE)? }; + let current_version = high_qc.block_height().as_u64(); // parent_block.height.as_u64(); + let next_height = parent_block.height() + NodeHeight(1); let mut total_leader_fee = 0; + let mut substate_changes = vec![]; let locked_block = LockedBlock::get(tx)?; let pending_proposals = ForeignProposal::get_all_pending(tx, locked_block.block_id(), parent_block.block_id())?; let commands = ForeignProposal::get_all_new(tx)? @@ -224,7 +239,25 @@ where TConsensusSpec: ConsensusSpec })?; let leader_fee = t.calculate_leader_fee(involved, EXHAUST_DIVISOR); total_leader_fee += leader_fee; - Ok(Command::Accept(t.get_final_transaction_atom(leader_fee))) + let tx_atom = t.get_final_transaction_atom(leader_fee); + if tx_atom.decision.is_commit() { + let transaction = t.get_transaction(tx)?; + let result = transaction.result().ok_or_else(|| { + HotStuffError::InvariantError(format!( + "Transaction {} is committed but has no result when proposing", + t.transaction_id(), + )) + })?; + + let diff = result.finalize.result.accept().ok_or_else(|| { + HotStuffError::InvariantError(format!( + "Transaction {} has COMMIT decision but execution failed when proposing", + t.transaction_id(), + )) + })?; + substate_changes.extend(diff_to_substate_changes(diff)); + } + Ok(Command::Accept(tx_atom)) }, // Not reachable as there is nothing to propose for these stages. To confirm that all local nodes // agreed with the Accept, more (possibly empty) blocks with QCs will be @@ -244,6 +277,16 @@ where TConsensusSpec: ConsensusSpec commands.iter().map(|c| c.to_string()).collect::>().join(",") ); + let pending = PendingStateTreeDiff::get_all_up_to_commit_block(tx, high_qc.block_id())?; + + let (state_root, _) = calculate_state_merkle_diff( + tx.deref(), + current_version, + next_height.as_u64(), + pending, + substate_changes, + )?; + let non_local_buckets = proposer::get_non_local_shards_from_commands( tx, &commands, @@ -264,10 +307,11 @@ where TConsensusSpec: ConsensusSpec self.network, *parent_block.block_id(), high_qc, - parent_block.height() + NodeHeight(1), + next_height, epoch, proposed_by, commands, + state_root, total_leader_fee, foreign_indexes, None, diff --git a/dan_layer/consensus/src/hotstuff/on_ready_to_vote_on_local_block.rs b/dan_layer/consensus/src/hotstuff/on_ready_to_vote_on_local_block.rs index d7da48c1fe..3ef69d256b 100644 --- a/dan_layer/consensus/src/hotstuff/on_ready_to_vote_on_local_block.rs +++ b/dan_layer/consensus/src/hotstuff/on_ready_to_vote_on_local_block.rs @@ -24,6 +24,7 @@ use tari_dan_storage::{ LastVoted, LockedBlock, LockedOutput, + PendingStateTreeDiff, QuorumDecision, SubstateLockFlag, SubstateRecord, @@ -55,7 +56,7 @@ use crate::{ const LOG_TARGET: &str = "tari::dan::consensus::hotstuff::on_lock_block_ready"; pub struct OnReadyToVoteOnLocalBlock { - validator_addr: TConsensusSpec::Addr, + local_validator_addr: TConsensusSpec::Addr, store: TConsensusSpec::StateStore, epoch_manager: TConsensusSpec::EpochManager, vote_signing_service: TConsensusSpec::SignatureService, @@ -87,7 +88,7 @@ where TConsensusSpec: ConsensusSpec hooks: TConsensusSpec::Hooks, ) -> Self { Self { - validator_addr, + local_validator_addr: validator_addr, store, epoch_manager, vote_signing_service, @@ -119,8 +120,9 @@ where TConsensusSpec: ConsensusSpec let maybe_decision = self.store.with_write_tx(|tx| { let maybe_decision = self.decide_on_block(tx, &local_committee_shard, valid_block.block())?; + let is_accept_decision = maybe_decision.map(|d| d.is_accept()).unwrap_or(false); // Update nodes - if maybe_decision.map(|d| d.is_accept()).unwrap_or(false) { + if is_accept_decision { let high_qc = valid_block.block().update_nodes( tx, |tx, locked, block| { @@ -304,6 +306,7 @@ where TConsensusSpec: ConsensusSpec Command::ForeignProposal(_) => {}, } } + leaf.set_as_processed(tx)?; Ok(()) } @@ -506,7 +509,7 @@ where TConsensusSpec: ConsensusSpec warn!( target: LOG_TARGET, "{} ❌ Stage disagreement in block {} for transaction {}. Leader proposed LocalPrepared, but local stage is {}", - self.validator_addr, + self.local_validator_addr, block.id(), tx_rec.transaction_id(), tx_rec.current_stage() @@ -876,7 +879,7 @@ where TConsensusSpec: ConsensusSpec ) -> Result { let vn = self .epoch_manager - .get_validator_node(block.epoch(), &self.validator_addr) + .get_validator_node(block.epoch(), &self.local_validator_addr) .await?; let leaf_hash = vn.get_node_hash(self.network); @@ -1049,7 +1052,8 @@ where TConsensusSpec: ConsensusSpec // We are accepting the transaction so can remove the transaction from the pool debug!( target: LOG_TARGET, - "🗑️ Removing transaction {} from pool", tx_rec.transaction_id()); + "🗑️ Removing transaction {} from pool", tx_rec.transaction_id() + ); tx_rec.remove(tx)?; executed.set_final_decision(t.decision).update(tx)?; finalized_transactions.push(t.clone()); @@ -1060,6 +1064,13 @@ where TConsensusSpec: ConsensusSpec block.commit(tx)?; + // We don't store (empty) pending state diffs for dummy blocks + if !block.is_dummy() { + let pending = PendingStateTreeDiff::remove_by_block(tx, block.id())?; + let mut state_tree = tari_state_tree::SpreadPrefixStateTree::new(tx); + state_tree.commit_diff(pending.diff)?; + } + if total_transaction_fee > 0 { info!( target: LOG_TARGET, diff --git a/dan_layer/consensus/src/hotstuff/on_receive_local_proposal.rs b/dan_layer/consensus/src/hotstuff/on_receive_local_proposal.rs index a99d27f4b5..f3225254a6 100644 --- a/dan_layer/consensus/src/hotstuff/on_receive_local_proposal.rs +++ b/dan_layer/consensus/src/hotstuff/on_receive_local_proposal.rs @@ -5,7 +5,7 @@ // ----[foreign:LocalPrepared]--->(LocalPrepared, true) ----cmd:AllPrepare ---> (AllPrepared, true) ---cmd:Accept ---> // Complete -use std::ops::DerefMut; +use std::ops::{Deref, DerefMut}; use log::*; use tari_common::configuration::Network; @@ -23,6 +23,7 @@ use tari_dan_storage::{ ForeignProposal, ForeignSendCounters, HighQc, + PendingStateTreeDiff, TransactionPool, TransactionPoolStage, ValidBlock, @@ -31,11 +32,14 @@ use tari_dan_storage::{ StateStoreReadTransaction, }; use tari_epoch_manager::EpochManagerReader; +use tari_state_tree::StateHashTreeDiff; use tokio::sync::broadcast; use super::proposer::{self, Proposer}; use crate::{ hotstuff::{ + calculate_state_merkle_diff, + diff_to_substate_changes, error::HotStuffError, on_ready_to_vote_on_local_block::OnReadyToVoteOnLocalBlock, pacemaker_handle::PaceMakerHandle, @@ -138,12 +142,14 @@ impl OnReceiveLocalProposalHandler(Some((high_qc, valid_block))) })?; @@ -163,11 +169,16 @@ impl OnReceiveLocalProposalHandler::WriteTransaction<'_>, valid_block: &ValidBlock, + tree_diff: StateHashTreeDiff, ) -> Result { valid_block.block().save_foreign_send_counters(tx)?; valid_block.block().justify().save(tx)?; valid_block.save_all_dummy_blocks(tx)?; valid_block.block().save(tx)?; + + // Store the tree diff for the block + PendingStateTreeDiff::new(*valid_block.id(), valid_block.height(), tree_diff).save(tx)?; + let high_qc = valid_block.block().justify().update_high_qc(tx)?; Ok(high_qc) } @@ -178,9 +189,14 @@ impl OnReceiveLocalProposalHandler, local_committee_shard: &CommitteeShard, - ) -> Result, HotStuffError> { - match self.validate_local_proposed_block(tx, block, local_committee, local_committee_shard) { - Ok(validated) => Ok(Some(validated)), + ) -> Result, HotStuffError> { + match self + .validate_local_proposed_block(tx, block, local_committee, local_committee_shard) + .and_then(|valid_block| { + let diff = self.check_state_merkle_root(tx, valid_block.block())?; + Ok((valid_block, diff)) + }) { + Ok((validated, diff)) => Ok(Some((validated, diff))), // Propagate this error out as sync is needed in the case where we have a valid QC but do not know the // block Err(err @ HotStuffError::ProposalValidationError(ProposalValidationError::JustifyBlockNotFound { .. })) => { @@ -196,6 +212,37 @@ impl OnReceiveLocalProposalHandler::ReadTransaction<'_>, + block: &Block, + ) -> Result { + let current_version = block.justify().block_height().as_u64(); + let next_version = block.height().as_u64(); + let commit_substate_diffs = block.get_all_substate_diffs(tx)?; + + let pending = PendingStateTreeDiff::get_all_up_to_commit_block(tx, block.justify().block_id())?; + + let (state_root, state_tree_diff) = calculate_state_merkle_diff( + tx.deref(), + current_version, + next_version, + pending, + commit_substate_diffs.iter().flat_map(diff_to_substate_changes), + )?; + + if state_root != *block.merkle_root() { + return Err(ProposalValidationError::InvalidStateMerkleRoot { + block_id: *block.id(), + from_block: *block.merkle_root(), + calculated: state_root, + } + .into()); + } + + Ok(state_tree_diff) + } + fn check_foreign_indexes( &self, tx: &mut ::ReadTransaction<'_>, @@ -358,6 +405,7 @@ impl OnReceiveLocalProposalHandler HotstuffWorker { shutdown, } } -} -impl HotstuffWorker -where TConsensusSpec: ConsensusSpec -{ + pub async fn start(&mut self) -> Result<(), HotStuffError> { self.create_genesis_block_if_required()?; let (current_height, high_qc) = self.state_store.with_read_tx(|tx| { @@ -553,36 +549,36 @@ where TConsensusSpec: ConsensusSpec } fn create_genesis_block_if_required(&self) -> Result<(), HotStuffError> { - let mut tx = self.state_store.create_write_tx()?; - - // The parent for genesis blocks refer to this zero block - let zero_block = Block::zero_block(self.network); - if !zero_block.exists(tx.deref_mut())? { - debug!(target: LOG_TARGET, "Creating zero block"); - zero_block.justify().insert(&mut tx)?; - zero_block.insert(&mut tx)?; - zero_block.as_locked_block().set(&mut tx)?; - zero_block.as_leaf_block().set(&mut tx)?; - zero_block.as_last_executed().set(&mut tx)?; - zero_block.as_last_voted().set(&mut tx)?; - zero_block.justify().as_high_qc().set(&mut tx)?; - zero_block.commit(&mut tx)?; - } - - // let genesis = Block::genesis(); - // if !genesis.exists(tx.deref_mut())? { - // debug!(target: LOG_TARGET, "Creating genesis block"); - // genesis.justify().save(&mut tx)?; - // genesis.insert(&mut tx)?; - // genesis.as_locked().set(&mut tx)?; - // genesis.as_leaf_block().set(&mut tx)?; - // genesis.as_last_executed().set(&mut tx)?; - // genesis.justify().as_high_qc().set(&mut tx)?; - // } - - tx.commit()?; + self.state_store.with_write_tx(|tx| { + // The parent for genesis blocks refer to this zero block + let zero_block = Block::zero_block(self.network); + if !zero_block.exists(tx.deref_mut())? { + debug!(target: LOG_TARGET, "Creating zero block"); + zero_block.justify().insert(tx)?; + zero_block.insert(tx)?; + zero_block.as_locked_block().set(tx)?; + zero_block.as_leaf_block().set(tx)?; + zero_block.as_last_executed().set(tx)?; + zero_block.as_last_voted().set(tx)?; + zero_block.justify().as_high_qc().set(tx)?; + zero_block.commit(tx)?; + } - Ok(()) + // let mut state_tree = SpreadPrefixStateTree::new(tx); + // state_tree.put_substate_changes_at_next_version(None, vec![])?; + + // let genesis = Block::genesis(); + // if !genesis.exists(tx.deref_mut())? { + // debug!(target: LOG_TARGET, "Creating genesis block"); + // genesis.justify().save(&mut tx)?; + // genesis.insert(&mut tx)?; + // genesis.as_locked().set(&mut tx)?; + // genesis.as_leaf_block().set(&mut tx)?; + // genesis.as_last_executed().set(&mut tx)?; + // genesis.justify().as_high_qc().set(&mut tx)?; + // } + Ok(()) + }) } fn publish_event(&self, event: HotstuffEvent) { diff --git a/dan_layer/consensus/src/traits/sync.rs b/dan_layer/consensus/src/traits/sync.rs index 018c69676f..76aecc6238 100644 --- a/dan_layer/consensus/src/traits/sync.rs +++ b/dan_layer/consensus/src/traits/sync.rs @@ -9,7 +9,7 @@ pub trait SyncManager { async fn check_sync(&self) -> Result; - async fn sync(&self) -> Result<(), Self::Error>; + async fn sync(&mut self) -> Result<(), Self::Error>; } #[derive(Debug, Clone, Copy)] diff --git a/dan_layer/consensus_tests/Cargo.toml b/dan_layer/consensus_tests/Cargo.toml index 49fcaa53bc..a81601b43c 100644 --- a/dan_layer/consensus_tests/Cargo.toml +++ b/dan_layer/consensus_tests/Cargo.toml @@ -10,6 +10,7 @@ license.workspace = true [dependencies] [dev-dependencies] +tari_bor = { workspace = true } tari_dan_common_types = { workspace = true } tari_consensus = { workspace = true } tari_dan_storage = { workspace = true } diff --git a/dan_layer/consensus_tests/src/consensus.rs b/dan_layer/consensus_tests/src/consensus.rs index d6169f0bfb..80805f1b2f 100644 --- a/dan_layer/consensus_tests/src/consensus.rs +++ b/dan_layer/consensus_tests/src/consensus.rs @@ -298,8 +298,9 @@ async fn leader_failure_output_conflict() { setup_logger(); let mut test = Test::builder() .with_test_timeout(Duration::from_secs(60)) + .debug_sql("/tmp/test{}.db") .add_committee(0, vec!["1", "2"]) - .add_committee(1, vec!["3", "4"]) + // .add_committee(1, vec!["3", "4"]) .start() .await; diff --git a/dan_layer/consensus_tests/src/support/sync.rs b/dan_layer/consensus_tests/src/support/sync.rs index f51e2fc590..e7d207ff19 100644 --- a/dan_layer/consensus_tests/src/support/sync.rs +++ b/dan_layer/consensus_tests/src/support/sync.rs @@ -18,7 +18,7 @@ impl SyncManager for AlwaysSyncedSyncManager { Ok(SyncStatus::UpToDate) } - async fn sync(&self) -> Result<(), Self::Error> { + async fn sync(&mut self) -> Result<(), Self::Error> { Ok(()) } } diff --git a/dan_layer/consensus_tests/src/support/transaction.rs b/dan_layer/consensus_tests/src/support/transaction.rs index 6ce5b4b9c6..d47c35354f 100644 --- a/dan_layer/consensus_tests/src/support/transaction.rs +++ b/dan_layer/consensus_tests/src/support/transaction.rs @@ -3,15 +3,17 @@ use std::{iter, time::Duration}; +use rand::{distributions::Alphanumeric, rngs::OsRng, Rng}; use tari_common_types::types::PrivateKey; use tari_dan_common_types::SubstateAddress; use tari_dan_storage::consensus_models::{Decision, ExecutedTransaction}; use tari_engine_types::{ commit_result::{ExecuteResult, FinalizeResult, RejectReason, TransactionResult}, + component::{ComponentBody, ComponentHeader}, fees::{FeeCostBreakdown, FeeReceipt}, - substate::SubstateDiff, + substate::{Substate, SubstateDiff}, }; -use tari_transaction::Transaction; +use tari_transaction::{id_provider::IdProvider, Transaction}; use crate::support::helpers::random_shard_in_bucket; @@ -22,6 +24,7 @@ pub fn build_transaction_from( resulting_outputs: Vec, ) -> ExecutedTransaction { let tx_id = *tx.id(); + let id_provider = IdProvider::new(tx_id.into_array().into(), 1000); ExecutedTransaction::new( tx, ExecuteResult { @@ -30,7 +33,24 @@ pub fn build_transaction_from( vec![], vec![], if decision.is_commit() { - TransactionResult::Accept(SubstateDiff::new()) + let mut diff = SubstateDiff::new(); + for _ in &resulting_outputs { + let obj = id_provider.new_component_address(Default::default(), None).unwrap(); + let s = (0..100).map(|_| OsRng.sample(Alphanumeric) as char).collect::(); + let random_state = tari_bor::to_value(&s).unwrap(); + diff.up( + obj.into(), + Substate::new(0, ComponentHeader { + template_address: Default::default(), + module_name: "Test".to_string(), + owner_key: Default::default(), + owner_rule: Default::default(), + access_rules: Default::default(), + body: ComponentBody { state: random_state }, + }), + ) + } + TransactionResult::Accept(diff) } else { TransactionResult::Reject(RejectReason::ExecutionFailure("Test failure".to_string())) }, diff --git a/dan_layer/engine/src/runtime/tracker.rs b/dan_layer/engine/src/runtime/tracker.rs index 96f048f730..cf7f425422 100644 --- a/dan_layer/engine/src/runtime/tracker.rs +++ b/dan_layer/engine/src/runtime/tracker.rs @@ -119,10 +119,6 @@ impl StateTracker { self.read_with(|state| state.logs().len()) } - pub fn take_logs(&self) -> Vec { - self.write_with(|state| state.take_logs()) - } - pub fn get_template_address(&self) -> Result { self.read_with(|state| state.current_template().map(|(a, _)| *a)) } diff --git a/dan_layer/engine_types/src/commit_result.rs b/dan_layer/engine_types/src/commit_result.rs index 1307f7f4d5..144ff95975 100644 --- a/dan_layer/engine_types/src/commit_result.rs +++ b/dan_layer/engine_types/src/commit_result.rs @@ -137,6 +137,22 @@ impl FinalizeResult { } } + pub fn accept(&self) -> Option<&SubstateDiff> { + match self.result { + TransactionResult::Accept(ref diff) => Some(diff), + TransactionResult::AcceptFeeRejectRest(ref diff, _) => Some(diff), + TransactionResult::Reject(_) => None, + } + } + + pub fn into_accept(self) -> Option { + match self.result { + TransactionResult::Accept(diff) => Some(diff), + TransactionResult::AcceptFeeRejectRest(diff, _) => Some(diff), + TransactionResult::Reject(_) => None, + } + } + pub fn reject(&self) -> Option<&RejectReason> { match self.result { TransactionResult::Accept(_) => None, diff --git a/dan_layer/engine_types/src/hashing.rs b/dan_layer/engine_types/src/hashing.rs index 650f5b7344..f0317db570 100644 --- a/dan_layer/engine_types/src/hashing.rs +++ b/dan_layer/engine_types/src/hashing.rs @@ -41,6 +41,10 @@ pub fn template_hasher64() -> TariHasher64 { hasher64(EngineHashDomainLabel::Template) } +pub fn substate_value_hasher32() -> TariHasher32 { + hasher32(EngineHashDomainLabel::SubstateValue) +} + pub fn hasher32(label: EngineHashDomainLabel) -> TariHasher32 { TariHasher32::new_with_label::(label.as_label()) } @@ -175,6 +179,7 @@ pub enum EngineHashDomainLabel { TransactionReceipt, FeeClaimAddress, QuorumCertificate, + SubstateValue, } impl EngineHashDomainLabel { @@ -198,6 +203,7 @@ impl EngineHashDomainLabel { Self::TransactionReceipt => "TransactionReceipt", Self::FeeClaimAddress => "FeeClaimAddress", Self::QuorumCertificate => "QuorumCertificate", + Self::SubstateValue => "SubstateValue", } } } diff --git a/dan_layer/p2p/proto/rpc.proto b/dan_layer/p2p/proto/rpc.proto index 8d59877e32..7438ac3c40 100644 --- a/dan_layer/p2p/proto/rpc.proto +++ b/dan_layer/p2p/proto/rpc.proto @@ -178,9 +178,10 @@ message SubstateData { } message SubstateDestroyedProof { - bytes address = 1; - tari.dan.consensus.QuorumCertificate destroyed_justify = 2; - bytes destroyed_by_transaction = 3; + bytes substate_id = 1; + uint32 version = 2; + tari.dan.consensus.QuorumCertificate destroyed_justify = 3; + bytes destroyed_by_transaction = 4; } message SubstateUpdate { diff --git a/dan_layer/p2p/src/conversions/consensus.rs b/dan_layer/p2p/src/conversions/consensus.rs index 0f3d7b263a..dc97811b19 100644 --- a/dan_layer/p2p/src/conversions/consensus.rs +++ b/dan_layer/p2p/src/conversions/consensus.rs @@ -289,6 +289,7 @@ impl TryFrom for tari_dan_storage::consensus_models::Bl .into_iter() .map(TryInto::try_into) .collect::>()?, + value.merkle_root.try_into()?, value.total_leader_fee, decode_exact(&value.foreign_indexes)?, value.signature.map(TryInto::try_into).transpose()?, diff --git a/dan_layer/p2p/src/conversions/rpc.rs b/dan_layer/p2p/src/conversions/rpc.rs index 14e8e44cb0..b4babeee82 100644 --- a/dan_layer/p2p/src/conversions/rpc.rs +++ b/dan_layer/p2p/src/conversions/rpc.rs @@ -4,7 +4,7 @@ use std::convert::{TryFrom, TryInto}; use anyhow::anyhow; -use tari_dan_storage::consensus_models::{SubstateCreatedProof, SubstateData, SubstateUpdate}; +use tari_dan_storage::consensus_models::{SubstateCreatedProof, SubstateData, SubstateDestroyedProof, SubstateUpdate}; use tari_engine_types::substate::{SubstateId, SubstateValue}; use crate::proto; @@ -37,6 +37,34 @@ impl From for proto::rpc::SubstateCreatedProof { } } +impl TryFrom for SubstateDestroyedProof { + type Error = anyhow::Error; + + fn try_from(value: proto::rpc::SubstateDestroyedProof) -> Result { + Ok(Self { + substate_id: SubstateId::from_bytes(&value.substate_id)?, + version: value.version, + justify: value + .destroyed_justify + .map(TryInto::try_into) + .transpose()? + .ok_or_else(|| anyhow!("destroyed_justify not provided"))?, + destroyed_by_transaction: value.destroyed_by_transaction.try_into()?, + }) + } +} + +impl From for proto::rpc::SubstateDestroyedProof { + fn from(value: SubstateDestroyedProof) -> Self { + Self { + substate_id: value.substate_id.to_bytes(), + version: value.version, + destroyed_justify: Some((&value.justify).into()), + destroyed_by_transaction: value.destroyed_by_transaction.as_bytes().to_vec(), + } + } +} + impl TryFrom for SubstateUpdate { type Error = anyhow::Error; @@ -44,15 +72,7 @@ impl TryFrom for SubstateUpdate { let update = value.update.ok_or_else(|| anyhow!("update not provided"))?; match update { proto::rpc::substate_update::Update::Create(substate_proof) => Ok(Self::Create(substate_proof.try_into()?)), - proto::rpc::substate_update::Update::Destroy(proof) => Ok(Self::Destroy { - address: proof.address.try_into()?, - proof: proof - .destroyed_justify - .map(TryInto::try_into) - .transpose()? - .ok_or_else(|| anyhow!("destroyed_justify not provided"))?, - destroyed_by_transaction: proof.destroyed_by_transaction.try_into()?, - }), + proto::rpc::substate_update::Update::Destroy(proof) => Ok(Self::Destroy(proof.try_into()?)), } } } @@ -61,15 +81,7 @@ impl From for proto::rpc::SubstateUpdate { fn from(value: SubstateUpdate) -> Self { let update = match value { SubstateUpdate::Create(proof) => proto::rpc::substate_update::Update::Create(proof.into()), - SubstateUpdate::Destroy { - proof, - address, - destroyed_by_transaction, - } => proto::rpc::substate_update::Update::Destroy(proto::rpc::SubstateDestroyedProof { - address: address.as_bytes().to_vec(), - destroyed_justify: Some((&proof).into()), - destroyed_by_transaction: destroyed_by_transaction.as_bytes().to_vec(), - }), + SubstateUpdate::Destroy(proof) => proto::rpc::substate_update::Update::Destroy(proof.into()), }; Self { update: Some(update) } diff --git a/dan_layer/rpc_state_sync/Cargo.toml b/dan_layer/rpc_state_sync/Cargo.toml index aad981e7ce..e218a6b695 100644 --- a/dan_layer/rpc_state_sync/Cargo.toml +++ b/dan_layer/rpc_state_sync/Cargo.toml @@ -17,6 +17,7 @@ tari_dan_common_types = { workspace = true } tari_transaction = { workspace = true } tari_rpc_framework = { workspace = true } tari_dan_p2p = { workspace = true } +tari_state_tree = { workspace = true } anyhow = { workspace = true } async-trait = { workspace = true } diff --git a/dan_layer/rpc_state_sync/src/error.rs b/dan_layer/rpc_state_sync/src/error.rs index 25dc2feffb..1a2a5fdd92 100644 --- a/dan_layer/rpc_state_sync/src/error.rs +++ b/dan_layer/rpc_state_sync/src/error.rs @@ -30,6 +30,8 @@ pub enum CommsRpcConsensusSyncError { NoPeersAvailable { committee_size: usize }, #[error("Proposal validation error: {0}")] ProposalValidationError(#[from] ProposalValidationError), + #[error("State tree error: {0}")] + StateTreeError(#[from] tari_state_tree::StateTreeError), } impl From for HotStuffError { diff --git a/dan_layer/rpc_state_sync/src/manager.rs b/dan_layer/rpc_state_sync/src/manager.rs index 6abb9a5c14..af5d5c2a3e 100644 --- a/dan_layer/rpc_state_sync/src/manager.rs +++ b/dan_layer/rpc_state_sync/src/manager.rs @@ -1,14 +1,18 @@ // Copyright 2023 The Tari Project // SPDX-License-Identifier: BSD-3-Clause -use std::{fmt::Display, ops::DerefMut}; +use std::{ + collections::HashMap, + fmt::Display, + ops::{Deref, DerefMut}, +}; use async_trait::async_trait; use futures::StreamExt; use log::*; use tari_common::configuration::Network; use tari_consensus::{ - hotstuff::ProposalValidationError, + hotstuff::{calculate_state_merkle_diff, hash_substate, ProposalValidationError}, traits::{ConsensusSpec, LeaderStrategy, SyncManager, SyncStatus}, }; use tari_dan_common_types::{committee::Committee, optional::Optional, NodeHeight, PeerAddress}; @@ -19,6 +23,7 @@ use tari_dan_storage::{ BlockId, HighQc, LockedBlock, + PendingStateTreeDiff, QuorumCertificate, SubstateUpdate, TransactionPoolRecord, @@ -29,6 +34,7 @@ use tari_dan_storage::{ }; use tari_epoch_manager::EpochManagerReader; use tari_rpc_framework::RpcError; +use tari_state_tree::SubstateChange; use tari_transaction::Transaction; use tari_validator_node_rpc::{ client::{TariValidatorNodeRpcClientFactory, ValidatorNodeClientFactory}, @@ -78,7 +84,7 @@ where TConsensusSpec: ConsensusSpec } async fn sync_with_peer( - &self, + &mut self, addr: &TConsensusSpec::Addr, locked_block: &LockedBlock, ) -> Result<(), CommsRpcConsensusSyncError> { @@ -115,7 +121,7 @@ where TConsensusSpec: ConsensusSpec #[allow(clippy::too_many_lines)] async fn sync_blocks( - &self, + &mut self, client: &mut ValidatorNodeRpcClient, locked_block: &LockedBlock, ) -> Result<(), CommsRpcConsensusSyncError> { @@ -128,6 +134,9 @@ where TConsensusSpec: ConsensusSpec let mut counter = 0usize; let mut expected_height = locked_block.height + NodeHeight(1); + // Stores the uncommitted state updates for each block. When a block reaches a 3-chain, the updates are removed + // and applied. + let mut pending_state_updates = HashMap::new(); while let Some(resp) = stream.next().await { let msg = resp.map_err(RpcError::from)?; @@ -234,7 +243,8 @@ where TConsensusSpec: ConsensusSpec } else { expected_height = block.height() + NodeHeight(1); } - self.process_block(block, qcs, updates, transactions).await?; + self.process_block(block, qcs, updates, transactions, &mut pending_state_updates) + .await?; } info!(target: LOG_TARGET, "🌐 {counter} blocks synced to height {}", expected_height - NodeHeight(1)); @@ -243,11 +253,12 @@ where TConsensusSpec: ConsensusSpec } async fn process_block( - &self, + &mut self, block: Block, qcs: Vec, updates: Vec, transactions: Vec, + pending_state_updates: &mut HashMap>, ) -> Result<(), CommsRpcConsensusSyncError> { // Note: this is only used for dummy block calculation, so we avoid the epoch manager call unless it is needed. // Otherwise, the committee is empty. @@ -289,6 +300,7 @@ where TConsensusSpec: ConsensusSpec next_height, block.justify().clone(), block.epoch(), + *block.merkle_root(), ); dummy_block.save(tx)?; last_dummy_block = BlockIdAndHeight { id: *dummy_block.id(), height: next_height }; @@ -309,56 +321,119 @@ where TConsensusSpec: ConsensusSpec for qc in qcs { qc.save(tx)?; } - block.set_as_processed(tx)?; + + self.check_and_update_state_merkle_tree(tx, &block, &updates)?; + + if !updates.is_empty() { + pending_state_updates.insert(*block.id(), updates); + } block.update_nodes( tx, |_, _, _| Ok(()), - |tx, _, block| { - let last_exec = block.as_last_executed(); - block.commit(tx)?; - debug!( - target: LOG_TARGET, - "✅ COMMIT block {}, last executed height = {}", - block, - last_exec.height - ); - last_exec.set(tx)?; - - // Finalize any ACCEPTED transactions - for tx_atom in block.commands().iter().filter_map(|cmd| cmd.accept()) { - if let Some(mut transaction) = tx_atom.get_transaction(tx.deref_mut()).optional()? { - transaction.final_decision = Some(tx_atom.decision); - if tx_atom.decision.is_abort() { - transaction.abort_details = Some("Abort decision via sync".to_string()); - } - // TODO: execution result - we should execute or we should get the execution result via sync - transaction.update(tx)?; - } - } - - // Remove from pool including any pending updates - TransactionPoolRecord::remove_any( - tx, - block.commands().iter().filter_map(|cmd| cmd.accept()).map(|t| &t.id), - )?; - + |tx, _last_executed, block| { + debug!(target: LOG_TARGET, "Sync is committing block {}", block); + Self::commit_block(tx, block, pending_state_updates)?; + block.as_last_executed().set(tx)?; Ok::<_, CommsRpcConsensusSyncError>(()) }, )?; - // Ensure we dont vote on a synced block + + // Ensure we don't vote on or re-process a synced block block.as_last_voted().set(tx)?; + block.set_as_processed(tx)?; + + Ok(()) + }) + } + + fn check_and_update_state_merkle_tree( + &self, + tx: &mut ::WriteTransaction<'_>, + block: &Block, + updates: &[SubstateUpdate], + ) -> Result<(), CommsRpcConsensusSyncError> { + let pending_tree_updates = PendingStateTreeDiff::get_all_up_to_commit_block(tx.deref_mut(), block.id())?; + let current_version = block.justify().block_height().as_u64(); + let next_version = block.height().as_u64(); + + let changes = updates.iter().map(|update| match update { + SubstateUpdate::Create(create) => SubstateChange::Up { + id: create.substate.substate_id.clone(), + value_hash: hash_substate(&create.substate.clone().into_substate()), + }, + SubstateUpdate::Destroy(destroy) => SubstateChange::Down { + id: destroy.substate_id.clone(), + }, + }); + + let (root_hash, tree_diff) = + calculate_state_merkle_diff(tx.deref(), current_version, next_version, pending_tree_updates, changes)?; + + if root_hash != *block.merkle_root() { + return Err(CommsRpcConsensusSyncError::InvalidResponse(anyhow::anyhow!( + "Merkle root in block {} does not match the merkle root of the state tree. Block MR: {}, Calculated \ + MR: {}", + block, + block.merkle_root(), + root_hash + ))); + } + + // Persist pending state tree diff + PendingStateTreeDiff::new(*block.id(), block.height(), tree_diff).save(tx)?; + + Ok(()) + } + + fn commit_block( + tx: &mut ::WriteTransaction<'_>, + block: &Block, + pending_state_updates: &mut HashMap>, + ) -> Result<(), CommsRpcConsensusSyncError> { + block.commit(tx)?; + + if block.is_dummy() { + return Ok(()); + } + + // Finalize any ACCEPTED transactions + for tx_atom in block.commands().iter().filter_map(|cmd| cmd.accept()) { + if let Some(mut transaction) = tx_atom.get_transaction(tx.deref_mut()).optional()? { + transaction.final_decision = Some(tx_atom.decision); + if tx_atom.decision.is_abort() { + transaction.abort_details = Some("Abort decision via sync".to_string()); + } + // TODO: execution result - we should execute or we should get the execution result via sync + transaction.update(tx)?; + } + } + + // Remove from pool including any pending updates + TransactionPoolRecord::remove_any( + tx, + block.commands().iter().filter_map(|cmd| cmd.accept()).map(|t| &t.id), + )?; + + let diff = PendingStateTreeDiff::remove_by_block(tx, block.id())?; + let mut tree = tari_state_tree::SpreadPrefixStateTree::new(tx); + tree.commit_diff(diff.diff)?; + + // Commit the state if any + if let Some(updates) = pending_state_updates.remove(block.id()) { + debug!(target: LOG_TARGET, "Committed block {} has {} state update(s)", block, updates.len()); let (ups, downs) = updates.into_iter().partition::, _>(|u| u.is_create()); // First do UPs then do DOWNs - // TODO: stage the updates, then check against the state hash in the block, then persist for update in ups { - update.apply(tx, &block)?; + update.apply(tx, block)?; } for update in downs { - update.apply(tx, &block)?; + update.apply(tx, block)?; } - Ok(()) - }) + } + + debug!(target: LOG_TARGET, "✅ COMMIT block {}", block); + Ok(()) } } @@ -448,7 +523,7 @@ where TConsensusSpec: ConsensusSpec + Send + Sync + 'static Ok(SyncStatus::UpToDate) } - async fn sync(&self) -> Result<(), Self::Error> { + async fn sync(&mut self) -> Result<(), Self::Error> { let committee = self.get_sync_peers().await?; if committee.is_empty() { warn!(target: LOG_TARGET, "No peers available for sync"); diff --git a/dan_layer/state_store_sqlite/Cargo.toml b/dan_layer/state_store_sqlite/Cargo.toml index 7793d312a9..39a377efbd 100644 --- a/dan_layer/state_store_sqlite/Cargo.toml +++ b/dan_layer/state_store_sqlite/Cargo.toml @@ -8,12 +8,13 @@ repository.workspace = true license.workspace = true [dependencies] +# TODO: needed for FixedHash +tari_common_types = { workspace = true } tari_dan_storage = { workspace = true } tari_dan_common_types = { workspace = true } tari_transaction = { workspace = true } tari_engine_types = { workspace = true } -# TODO: needed for FixedHash -tari_common_types = { workspace = true } +tari_state_tree = { workspace = true } tari_utilities = { workspace = true } anyhow = { workspace = true } diff --git a/dan_layer/state_store_sqlite/migrations/2023-06-08-091819_create_state_store/up.sql b/dan_layer/state_store_sqlite/migrations/2023-06-08-091819_create_state_store/up.sql index 7a656d22cc..7d0272e7d5 100644 --- a/dan_layer/state_store_sqlite/migrations/2023-06-08-091819_create_state_store/up.sql +++ b/dan_layer/state_store_sqlite/migrations/2023-06-08-091819_create_state_store/up.sql @@ -15,6 +15,7 @@ create table blocks id integer not null primary key AUTOINCREMENT, block_id text not NULL, parent_block_id text not NULL, + merkle_root text not NULL, network text not NULL, height bigint not NULL, epoch bigint not NULL, @@ -40,6 +41,7 @@ create table parked_blocks id integer not null primary key AUTOINCREMENT, block_id text not NULL, parent_block_id text not NULL, + merkle_root text not NULL, network text not NULL, height bigint not NULL, epoch bigint not NULL, @@ -277,6 +279,32 @@ CREATE TABLE foreign_receive_counters created_at timestamp not NULL DEFAULT CURRENT_TIMESTAMP ); +CREATE TABLE state_tree +( + id integer not NULL primary key AUTOINCREMENT, + key text not NULL, + node text not NULL, + is_stale boolean not null default '0' +); + +-- Duplicate keys are not allowed +CREATE UNIQUE INDEX state_tree_uniq_idx_key on state_tree (key); +-- filtering out or by is_stale is used in every query +CREATE INDEX state_tree_idx_is_stale on state_tree (is_stale); + +CREATE TABLE pending_state_tree_diffs +( + id integer not NULL primary key AUTOINCREMENT, + block_id text not NULL, + block_height bigint not NULL, + diff_json text not NULL, + created_at timestamp not NULL DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (block_id) REFERENCES blocks (block_id) +); + +CREATE UNIQUE INDEX pending_state_tree_diffs_uniq_idx_block_id on pending_state_tree_diffs (block_id); + + -- Debug Triggers CREATE TABLE transaction_pool_history ( diff --git a/dan_layer/state_store_sqlite/src/lib.rs b/dan_layer/state_store_sqlite/src/lib.rs index 6a6f8f1536..58e1009f0c 100644 --- a/dan_layer/state_store_sqlite/src/lib.rs +++ b/dan_layer/state_store_sqlite/src/lib.rs @@ -7,6 +7,7 @@ mod serialization; mod sql_models; mod sqlite_transaction; mod store; +mod tree_store; mod writer; pub use store::SqliteStateStore; diff --git a/dan_layer/state_store_sqlite/src/reader.rs b/dan_layer/state_store_sqlite/src/reader.rs index 2a255cf6b2..34cd9c2bf7 100644 --- a/dan_layer/state_store_sqlite/src/reader.rs +++ b/dan_layer/state_store_sqlite/src/reader.rs @@ -44,6 +44,7 @@ use tari_dan_storage::{ LastVoted, LeafBlock, LockedBlock, + PendingStateTreeDiff, QcId, QuorumCertificate, SubstateLockFlag, @@ -84,7 +85,7 @@ impl<'a, TAddr> SqliteStateStoreReadTransaction<'a, TAddr> { } } - pub(crate) fn connection(&mut self) -> &mut SqliteConnection { + pub(crate) fn connection(&self) -> &mut SqliteConnection { self.transaction.connection() } @@ -202,6 +203,7 @@ impl<'a, TAddr: NodeAddressable + Serialize + DeserializeOwned> SqliteStateStore sql_frag } + /// Returns the blocks from the start_block (inclusive) to the end_block (inclusive). fn get_block_ids_between( &mut self, start_block: &BlockId, @@ -275,33 +277,33 @@ impl<'a, TAddr: NodeAddressable + Serialize + DeserializeOwned> SqliteStateStore } } -impl StateStoreReadTransaction - for SqliteStateStoreReadTransaction<'_, TAddr> +impl<'tx, TAddr: NodeAddressable + Serialize + DeserializeOwned + 'tx> StateStoreReadTransaction + for SqliteStateStoreReadTransaction<'tx, TAddr> { type Addr = TAddr; - fn last_voted_get(&mut self) -> Result { - use crate::schema::last_voted; + fn last_sent_vote_get(&mut self) -> Result { + use crate::schema::last_sent_vote; - let last_voted = last_voted::table - .order_by(last_voted::id.desc()) - .first::(self.connection()) + let last_voted = last_sent_vote::table + .order_by(last_sent_vote::id.desc()) + .first::(self.connection()) .map_err(|e| SqliteStorageError::DieselError { - operation: "last_voted_get", + operation: "last_sent_vote_get", source: e, })?; last_voted.try_into() } - fn last_sent_vote_get(&mut self) -> Result { - use crate::schema::last_sent_vote; + fn last_voted_get(&mut self) -> Result { + use crate::schema::last_voted; - let last_voted = last_sent_vote::table - .order_by(last_sent_vote::id.desc()) - .first::(self.connection()) + let last_voted = last_voted::table + .order_by(last_voted::id.desc()) + .first::(self.connection()) .map_err(|e| SqliteStorageError::DieselError { - operation: "last_sent_vote_get", + operation: "last_voted_get", source: e, })?; @@ -667,6 +669,61 @@ impl StateStoreReadTransa .collect() } + fn blocks_exists(&mut self, block_id: &BlockId) -> Result { + use crate::schema::blocks; + + let count = blocks::table + .filter(blocks::block_id.eq(serialize_hex(block_id))) + .count() + .limit(1) + .get_result::(self.connection()) + .map_err(|e| SqliteStorageError::DieselError { + operation: "blocks_exists", + source: e, + })?; + + Ok(count > 0) + } + + fn blocks_is_ancestor(&mut self, descendant: &BlockId, ancestor: &BlockId) -> Result { + if !self.blocks_exists(descendant)? { + return Err(StorageError::QueryError { + reason: format!("blocks_is_ancestor: descendant block {} does not exist", descendant), + }); + } + + if !self.blocks_exists(ancestor)? { + return Err(StorageError::QueryError { + reason: format!("blocks_is_ancestor: ancestor block {} does not exist", ancestor), + }); + } + + // TODO: this scans all the way to genesis for every query - can optimise though it's low priority for now + let is_ancestor = sql_query( + r#" + WITH RECURSIVE tree(bid, parent) AS ( + SELECT block_id, parent_block_id FROM blocks where block_id = ? + UNION ALL + SELECT block_id, parent_block_id + FROM blocks JOIN tree ON block_id = tree.parent AND tree.bid != tree.parent -- stop recursing at zero block (or any self referencing block) + ) + SELECT count(1) as "count" FROM tree WHERE bid = ? LIMIT 1 + "#, + ) + .bind::(serialize_hex(descendant)) + // .bind::(serialize_hex(BlockId::genesis())) // stop recursing at zero block + .bind::(serialize_hex(ancestor)) + .get_result::(self.connection()) + .map_err(|e| SqliteStorageError::DieselError { + operation: "blocks_is_ancestor", + source: e, + })?; + + debug!(target: LOG_TARGET, "blocks_is_ancestor: is_ancestor: {}", is_ancestor.count); + + Ok(is_ancestor.count > 0) + } + fn blocks_get_all_by_parent(&mut self, parent_id: &BlockId) -> Result, StorageError> { use crate::schema::{blocks, quorum_certificates}; @@ -742,61 +799,6 @@ impl StateStoreReadTransa .collect() } - fn blocks_exists(&mut self, block_id: &BlockId) -> Result { - use crate::schema::blocks; - - let count = blocks::table - .filter(blocks::block_id.eq(serialize_hex(block_id))) - .count() - .limit(1) - .get_result::(self.connection()) - .map_err(|e| SqliteStorageError::DieselError { - operation: "blocks_exists", - source: e, - })?; - - Ok(count > 0) - } - - fn blocks_is_ancestor(&mut self, descendant: &BlockId, ancestor: &BlockId) -> Result { - if !self.blocks_exists(descendant)? { - return Err(StorageError::QueryError { - reason: format!("blocks_is_ancestor: descendant block {} does not exist", descendant), - }); - } - - if !self.blocks_exists(ancestor)? { - return Err(StorageError::QueryError { - reason: format!("blocks_is_ancestor: ancestor block {} does not exist", ancestor), - }); - } - - // TODO: this scans all the way to genesis for every query - can optimise though it's low priority for now - let is_ancestor = sql_query( - r#" - WITH RECURSIVE tree(bid, parent) AS ( - SELECT block_id, parent_block_id FROM blocks where block_id = ? - UNION ALL - SELECT block_id, parent_block_id - FROM blocks JOIN tree ON block_id = tree.parent AND tree.bid != tree.parent -- stop recursing at zero block (or any self referencing block) - ) - SELECT count(1) as "count" FROM tree WHERE bid = ? LIMIT 1 - "#, - ) - .bind::(serialize_hex(descendant)) - // .bind::(serialize_hex(BlockId::genesis())) // stop recursing at zero block - .bind::(serialize_hex(ancestor)) - .get_result::(self.connection()) - .map_err(|e| SqliteStorageError::DieselError { - operation: "blocks_is_ancestor", - source: e, - })?; - - debug!(target: LOG_TARGET, "blocks_is_ancestor: is_ancestor: {}", is_ancestor.count); - - Ok(is_ancestor.count > 0) - } - fn blocks_get_pending_transactions(&mut self, block_id: &BlockId) -> Result, StorageError> { use crate::schema::missing_transactions; @@ -1026,86 +1028,6 @@ impl StateStoreReadTransa deserialize_json(&qc_json) } - fn transactions_fetch_involved_shards( - &mut self, - transaction_ids: HashSet, - ) -> Result, StorageError> { - use crate::schema::transactions; - - let tx_ids = transaction_ids.into_iter().map(serialize_hex).collect::>(); - - let involved_shards = transactions::table - .select((transactions::inputs, transactions::input_refs)) - .filter(transactions::transaction_id.eq_any(tx_ids)) - .load::<(String, String)>(self.connection()) - .map_err(|e| SqliteStorageError::DieselError { - operation: "transaction_pools_fetch_involved_shards", - source: e, - })?; - - let shards = involved_shards - .into_iter() - .map(|(inputs, input_refs)| { - ( - // a Result is very inconvenient with flat_map - deserialize_json::>(&inputs).unwrap(), - deserialize_json::>(&input_refs).unwrap(), - ) - }) - .flat_map(|(inputs, input_refs)| inputs.into_iter().chain(input_refs).collect::>()) - .collect(); - - Ok(shards) - } - - fn votes_count_for_block(&mut self, block_id: &BlockId) -> Result { - use crate::schema::votes; - - let count = votes::table - .filter(votes::block_id.eq(serialize_hex(block_id))) - .count() - .first::(self.connection()) - .map_err(|e| SqliteStorageError::DieselError { - operation: "votes_count_for_block", - source: e, - })?; - - Ok(count as u64) - } - - fn votes_get_for_block(&mut self, block_id: &BlockId) -> Result, StorageError> { - use crate::schema::votes; - - let votes = votes::table - .filter(votes::block_id.eq(serialize_hex(block_id))) - .get_results::(self.connection()) - .map_err(|e| SqliteStorageError::DieselError { - operation: "votes_get_for_block", - source: e, - })?; - - votes.into_iter().map(Vote::try_from).collect() - } - - fn votes_get_by_block_and_sender( - &mut self, - block_id: &BlockId, - sender_leaf_hash: &FixedHash, - ) -> Result { - use crate::schema::votes; - - let vote = votes::table - .filter(votes::block_id.eq(serialize_hex(block_id))) - .filter(votes::sender_leaf_hash.eq(serialize_hex(sender_leaf_hash))) - .first::(self.connection()) - .map_err(|e| SqliteStorageError::DieselError { - operation: "votes_get", - source: e, - })?; - - Vote::try_from(vote) - } - fn transaction_pool_get( &mut self, from_block_id: &BlockId, @@ -1327,6 +1249,86 @@ impl StateStoreReadTransa Ok(count as usize) } + fn transactions_fetch_involved_shards( + &mut self, + transaction_ids: HashSet, + ) -> Result, StorageError> { + use crate::schema::transactions; + + let tx_ids = transaction_ids.into_iter().map(serialize_hex).collect::>(); + + let involved_shards = transactions::table + .select((transactions::inputs, transactions::input_refs)) + .filter(transactions::transaction_id.eq_any(tx_ids)) + .load::<(String, String)>(self.connection()) + .map_err(|e| SqliteStorageError::DieselError { + operation: "transaction_pools_fetch_involved_shards", + source: e, + })?; + + let shards = involved_shards + .into_iter() + .map(|(inputs, input_refs)| { + ( + // a Result is very inconvenient with flat_map + deserialize_json::>(&inputs).unwrap(), + deserialize_json::>(&input_refs).unwrap(), + ) + }) + .flat_map(|(inputs, input_refs)| inputs.into_iter().chain(input_refs).collect::>()) + .collect(); + + Ok(shards) + } + + fn votes_get_by_block_and_sender( + &mut self, + block_id: &BlockId, + sender_leaf_hash: &FixedHash, + ) -> Result { + use crate::schema::votes; + + let vote = votes::table + .filter(votes::block_id.eq(serialize_hex(block_id))) + .filter(votes::sender_leaf_hash.eq(serialize_hex(sender_leaf_hash))) + .first::(self.connection()) + .map_err(|e| SqliteStorageError::DieselError { + operation: "votes_get", + source: e, + })?; + + Vote::try_from(vote) + } + + fn votes_count_for_block(&mut self, block_id: &BlockId) -> Result { + use crate::schema::votes; + + let count = votes::table + .filter(votes::block_id.eq(serialize_hex(block_id))) + .count() + .first::(self.connection()) + .map_err(|e| SqliteStorageError::DieselError { + operation: "votes_count_for_block", + source: e, + })?; + + Ok(count as u64) + } + + fn votes_get_for_block(&mut self, block_id: &BlockId) -> Result, StorageError> { + use crate::schema::votes; + + let votes = votes::table + .filter(votes::block_id.eq(serialize_hex(block_id))) + .get_results::(self.connection()) + .map_err(|e| SqliteStorageError::DieselError { + operation: "votes_get_for_block", + source: e, + })?; + + votes.into_iter().map(Vote::try_from).collect() + } + fn substates_get(&mut self, address: &SubstateAddress) -> Result { use crate::schema::substates; @@ -1486,6 +1488,7 @@ impl StateStoreReadTransa .eq(&transaction_id_hex) .or(substates::destroyed_by_transaction.eq(Some(&transaction_id_hex))), ) + .order_by(substates::id.asc()) .get_results::(self.connection()) .map_err(|e| SqliteStorageError::DieselError { operation: "substates_get_all_for_transaction", @@ -1570,6 +1573,59 @@ impl StateStoreReadTransa Ok(SubstateLockState::LockAcquired) } } + + fn pending_state_tree_diffs_exists_for_block(&mut self, block_id: &BlockId) -> Result { + use crate::schema::pending_state_tree_diffs; + + let count = pending_state_tree_diffs::table + .count() + .filter(pending_state_tree_diffs::block_id.eq(serialize_hex(block_id))) + .first::(self.connection()) + .map_err(|e| SqliteStorageError::DieselError { + operation: "pending_state_tree_diffs_exists_for_block", + source: e, + })?; + + Ok(count > 0) + } + + fn pending_state_tree_diffs_get_all_up_to_commit_block( + &mut self, + block_id: &BlockId, + ) -> Result, StorageError> { + use crate::schema::{blocks, pending_state_tree_diffs}; + + // Get the last committed block + let committed_block_id = blocks::table + .select(blocks::block_id) + .filter(blocks::is_committed.eq(true)) + .order_by(blocks::height.desc()) + .first::(self.connection()) + .map(|s| deserialize_hex_try_from::(&s)) + .map_err(|e| SqliteStorageError::DieselError { + operation: "pending_state_tree_diffs_get_all_pending", + source: e, + })??; + + let mut block_ids = self.get_block_ids_between(&committed_block_id, block_id)?; + + // Exclude commit block + block_ids.pop(); + if block_ids.is_empty() { + return Ok(Vec::new()); + } + + let diffs = pending_state_tree_diffs::table + .filter(pending_state_tree_diffs::block_id.eq_any(block_ids)) + .order_by(pending_state_tree_diffs::block_height.asc()) + .get_results::(self.connection()) + .map_err(|e| SqliteStorageError::DieselError { + operation: "pending_state_tree_diffs_get_all_pending", + source: e, + })?; + + diffs.into_iter().map(TryInto::try_into).collect() + } } #[derive(QueryableByName)] diff --git a/dan_layer/state_store_sqlite/src/schema.rs b/dan_layer/state_store_sqlite/src/schema.rs index 17f0b487eb..79453b285d 100644 --- a/dan_layer/state_store_sqlite/src/schema.rs +++ b/dan_layer/state_store_sqlite/src/schema.rs @@ -5,6 +5,7 @@ diesel::table! { id -> Integer, block_id -> Text, parent_block_id -> Text, + merkle_root -> Text, network -> Text, height -> BigInt, epoch -> BigInt, @@ -144,6 +145,7 @@ diesel::table! { id -> Integer, block_id -> Text, parent_block_id -> Text, + merkle_root -> Text, network -> Text, height -> BigInt, epoch -> BigInt, @@ -158,6 +160,16 @@ diesel::table! { } } +diesel::table! { + pending_state_tree_diffs (id) { + id -> Integer, + block_id -> Text, + block_height -> BigInt, + diff_json -> Text, + created_at -> Timestamp, + } +} + diesel::table! { quorum_certificates (id) { id -> Integer, @@ -168,6 +180,15 @@ diesel::table! { } } +diesel::table! { + state_tree (id) { + id -> Integer, + key -> Text, + node -> Text, + is_stale -> Bool, + } +} + diesel::table! { substates (id) { id -> Integer, @@ -297,7 +318,9 @@ diesel::allow_tables_to_appear_in_same_query!( locked_outputs, missing_transactions, parked_blocks, + pending_state_tree_diffs, quorum_certificates, + state_tree, substates, transaction_pool, transaction_pool_history, diff --git a/dan_layer/state_store_sqlite/src/sql_models/block.rs b/dan_layer/state_store_sqlite/src/sql_models/block.rs index 87d0e1bbcb..626b8c3e06 100644 --- a/dan_layer/state_store_sqlite/src/sql_models/block.rs +++ b/dan_layer/state_store_sqlite/src/sql_models/block.rs @@ -19,6 +19,7 @@ pub struct Block { pub id: i32, pub block_id: String, pub parent_block_id: String, + pub merkle_root: String, pub network: String, pub height: i64, pub epoch: i64, @@ -57,6 +58,7 @@ impl Block { } })?, deserialize_json(&self.commands)?, + deserialize_hex_try_from(&self.merkle_root)?, self.total_leader_fee as u64, self.is_dummy, self.is_processed, @@ -73,6 +75,7 @@ pub struct ParkedBlock { pub id: i32, pub block_id: String, pub parent_block_id: String, + pub merkle_root: String, pub network: String, pub height: i64, pub epoch: i64, @@ -110,6 +113,7 @@ impl TryFrom for consensus_models::Block { } })?, deserialize_json(&value.commands)?, + deserialize_hex_try_from(&value.merkle_root)?, value.total_leader_fee as u64, false, false, diff --git a/dan_layer/state_store_sqlite/src/sql_models/mod.rs b/dan_layer/state_store_sqlite/src/sql_models/mod.rs index d0cb91933e..2fe49effa8 100644 --- a/dan_layer/state_store_sqlite/src/sql_models/mod.rs +++ b/dan_layer/state_store_sqlite/src/sql_models/mod.rs @@ -4,6 +4,7 @@ mod block; mod bookkeeping; mod leaf_block; +mod pending_state_tree_diff; mod quorum_certificate; mod substate; mod transaction; @@ -13,6 +14,7 @@ mod vote; pub use block::*; pub use bookkeeping::*; pub use leaf_block::*; +pub use pending_state_tree_diff::*; pub use quorum_certificate::*; pub use substate::*; pub use transaction::*; diff --git a/dan_layer/state_store_sqlite/src/sql_models/pending_state_tree_diff.rs b/dan_layer/state_store_sqlite/src/sql_models/pending_state_tree_diff.rs new file mode 100644 index 0000000000..de87dcc1e1 --- /dev/null +++ b/dan_layer/state_store_sqlite/src/sql_models/pending_state_tree_diff.rs @@ -0,0 +1,28 @@ +// Copyright 2023 The Tari Project +// SPDX-License-Identifier: BSD-3-Clause + +use diesel::Queryable; +use tari_dan_storage::{consensus_models, StorageError}; +use time::PrimitiveDateTime; + +use crate::serialization::{deserialize_hex_try_from, deserialize_json}; + +#[derive(Debug, Clone, Queryable)] +pub struct PendingStateTreeDiff { + pub id: i32, + pub block_id: String, + pub block_height: i64, + pub diff: String, + pub created_at: PrimitiveDateTime, +} + +impl TryFrom for consensus_models::PendingStateTreeDiff { + type Error = StorageError; + + fn try_from(value: PendingStateTreeDiff) -> Result { + let block_id = deserialize_hex_try_from(&value.block_id)?; + let block_height = value.block_height as u64; + let diff = deserialize_json(&value.diff)?; + Ok(Self::new(block_id, block_height.into(), diff)) + } +} diff --git a/dan_layer/state_store_sqlite/src/sqlite_transaction.rs b/dan_layer/state_store_sqlite/src/sqlite_transaction.rs index 218f54af66..f1ab11894a 100644 --- a/dan_layer/state_store_sqlite/src/sqlite_transaction.rs +++ b/dan_layer/state_store_sqlite/src/sqlite_transaction.rs @@ -20,7 +20,7 @@ // WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE // USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -use std::sync::MutexGuard; +use std::{cell::UnsafeCell, sync::MutexGuard}; use diesel::{sql_query, RunQueryDsl, SqliteConnection}; @@ -29,22 +29,25 @@ use crate::error::SqliteStorageError; const _LOG_TARGET: &str = "tari::dan::storage::sqlite::transaction"; pub struct SqliteTransaction<'a> { - connection: MutexGuard<'a, SqliteConnection>, + connection: UnsafeCell>, is_done: bool, } impl<'a> SqliteTransaction<'a> { pub fn begin(connection: MutexGuard<'a, SqliteConnection>) -> Result { let mut this = Self { - connection, + connection: UnsafeCell::new(connection), is_done: false, }; this.execute_sql("BEGIN TRANSACTION")?; Ok(this) } - pub fn connection(&mut self) -> &mut SqliteConnection { - &mut self.connection + #[allow(clippy::mut_from_ref)] + pub fn connection(&self) -> &mut SqliteConnection { + // SAFETY: This is safe because only a single thread has access to this connection because SqliteStateStore + // transactions are obtained through a Mutex and UnsafeCell is not Sync + unsafe { &mut *self.connection.get() } } pub fn commit(mut self) -> Result<(), SqliteStorageError> { diff --git a/dan_layer/state_store_sqlite/src/tree_store.rs b/dan_layer/state_store_sqlite/src/tree_store.rs new file mode 100644 index 0000000000..d728d7f854 --- /dev/null +++ b/dan_layer/state_store_sqlite/src/tree_store.rs @@ -0,0 +1,70 @@ +// Copyright 2024 The Tari Project +// SPDX-License-Identifier: BSD-3-Clause + +use std::ops::Deref; + +use diesel::{ExpressionMethods, OptionalExtension, QueryDsl, RunQueryDsl}; +use tari_state_tree::{Node, NodeKey, StaleTreeNode, TreeNode, TreeStoreReader, TreeStoreWriter, Version}; + +use crate::{reader::SqliteStateStoreReadTransaction, writer::SqliteStateStoreWriteTransaction}; + +impl<'a, TAddr> TreeStoreReader for SqliteStateStoreReadTransaction<'a, TAddr> { + fn get_node(&self, key: &NodeKey) -> Result, tari_state_tree::JmtStorageError> { + use crate::schema::state_tree; + + let node = state_tree::table + .select(state_tree::node) + .filter(state_tree::key.eq(key.to_string())) + .filter(state_tree::is_stale.eq(false)) + .first::(self.connection()) + .optional() + .map_err(|e| tari_state_tree::JmtStorageError::UnexpectedError(e.to_string()))? + .ok_or_else(|| tari_state_tree::JmtStorageError::NotFound(key.clone()))?; + + let node = serde_json::from_str::(&node) + .map_err(|e| tari_state_tree::JmtStorageError::UnexpectedError(e.to_string()))?; + + Ok(node.into_node()) + } +} + +impl<'a, TAddr> TreeStoreReader for SqliteStateStoreWriteTransaction<'a, TAddr> { + fn get_node(&self, key: &NodeKey) -> Result, tari_state_tree::JmtStorageError> { + self.deref().get_node(key) + } +} + +impl<'a, TAddr> TreeStoreWriter for SqliteStateStoreWriteTransaction<'a, TAddr> { + fn insert_node(&mut self, key: NodeKey, node: Node) -> Result<(), tari_state_tree::JmtStorageError> { + use crate::schema::state_tree; + + let node = TreeNode::new_latest(node); + let node = serde_json::to_string(&node) + .map_err(|e| tari_state_tree::JmtStorageError::UnexpectedError(e.to_string()))?; + + let values = (state_tree::key.eq(key.to_string()), state_tree::node.eq(node)); + diesel::insert_into(state_tree::table) + .values(&values) + .on_conflict(state_tree::key) + .do_update() + .set(values.clone()) + .execute(self.connection()) + .map_err(|e| tari_state_tree::JmtStorageError::UnexpectedError(e.to_string()))?; + + Ok(()) + } + + fn record_stale_tree_node(&mut self, node: StaleTreeNode) -> Result<(), tari_state_tree::JmtStorageError> { + use crate::schema::state_tree; + let key = node.as_node_key(); + diesel::update(state_tree::table) + .filter(state_tree::key.eq(key.to_string())) + .set(state_tree::is_stale.eq(true)) + .execute(self.connection()) + .optional() + .map_err(|e| tari_state_tree::JmtStorageError::UnexpectedError(e.to_string()))? + .ok_or_else(|| tari_state_tree::JmtStorageError::NotFound(key.clone()))?; + + Ok(()) + } +} diff --git a/dan_layer/state_store_sqlite/src/writer.rs b/dan_layer/state_store_sqlite/src/writer.rs index 2c6f970b83..c5e0bc9406 100644 --- a/dan_layer/state_store_sqlite/src/writer.rs +++ b/dan_layer/state_store_sqlite/src/writer.rs @@ -27,6 +27,7 @@ use tari_dan_storage::{ LeafBlock, LockedBlock, LockedOutput, + PendingStateTreeDiff, QcId, QuorumCertificate, SubstateLockFlag, @@ -140,6 +141,7 @@ impl<'a, TAddr: NodeAddressable> SqliteStateStoreWriteTransaction<'a, TAddr> { parked_blocks::block_id.eq(&block_id), parked_blocks::parent_block_id.eq(serialize_hex(block.parent())), parked_blocks::network.eq(block.network().to_string()), + parked_blocks::merkle_root.eq(block.merkle_root().to_string()), parked_blocks::height.eq(block.height().as_u64() as i64), parked_blocks::epoch.eq(block.epoch().as_u64() as i64), parked_blocks::proposed_by.eq(serialize_hex(block.proposed_by().as_bytes())), @@ -162,7 +164,7 @@ impl<'a, TAddr: NodeAddressable> SqliteStateStoreWriteTransaction<'a, TAddr> { } } -impl StateStoreWriteTransaction for SqliteStateStoreWriteTransaction<'_, TAddr> { +impl<'tx, TAddr: NodeAddressable + 'tx> StateStoreWriteTransaction for SqliteStateStoreWriteTransaction<'tx, TAddr> { type Addr = TAddr; fn commit(mut self) -> Result<(), StorageError> { @@ -183,6 +185,7 @@ impl StateStoreWriteTransaction for SqliteStateStoreWrit let insert = ( blocks::block_id.eq(serialize_hex(block.id())), blocks::parent_block_id.eq(serialize_hex(block.parent())), + blocks::merkle_root.eq(block.merkle_root().to_string()), blocks::network.eq(block.network().to_string()), blocks::height.eq(block.height().as_u64() as i64), blocks::epoch.eq(block.epoch().as_u64() as i64), @@ -239,141 +242,6 @@ impl StateStoreWriteTransaction for SqliteStateStoreWrit Ok(()) } - fn missing_transactions_insert< - 'a, - IMissing: IntoIterator, - IAwaiting: IntoIterator, - >( - &mut self, - block: &Block, - missing_transaction_ids: IMissing, - awaiting_transaction_ids: IAwaiting, - ) -> Result<(), StorageError> { - use crate::schema::missing_transactions; - - let missing_transaction_ids = missing_transaction_ids - .into_iter() - .map(serialize_hex) - .collect::>(); - let awaiting_transaction_ids = awaiting_transaction_ids - .into_iter() - .map(serialize_hex) - .collect::>(); - let block_id_hex = serialize_hex(block.id()); - - self.parked_blocks_insert(block)?; - - let values = missing_transaction_ids - .iter() - .map(|tx_id| { - ( - missing_transactions::block_id.eq(&block_id_hex), - missing_transactions::block_height.eq(block.height().as_u64() as i64), - missing_transactions::transaction_id.eq(tx_id), - missing_transactions::is_awaiting_execution.eq(false), - ) - }) - .chain(awaiting_transaction_ids.iter().map(|tx_id| { - ( - missing_transactions::block_id.eq(&block_id_hex), - missing_transactions::block_height.eq(block.height().as_u64() as i64), - missing_transactions::transaction_id.eq(tx_id), - missing_transactions::is_awaiting_execution.eq(true), - ) - })) - .collect::>(); - - diesel::insert_into(missing_transactions::table) - .values(values) - .execute(self.connection()) - .map_err(|e| SqliteStorageError::DieselError { - operation: "missing_transactions_insert", - source: e, - })?; - - Ok(()) - } - - fn missing_transactions_remove( - &mut self, - current_height: NodeHeight, - transaction_id: &TransactionId, - ) -> Result, StorageError> { - use crate::schema::{missing_transactions, transactions}; - - // delete all entries that are for previous heights - diesel::delete(missing_transactions::table) - .filter(missing_transactions::block_height.lt(current_height.as_u64().saturating_sub(1) as i64)) - .execute(self.connection()) - .map_err(|e| SqliteStorageError::DieselError { - operation: "missing_transactions_remove", - source: e, - })?; - - let transaction_id = serialize_hex(transaction_id); - let block_id = missing_transactions::table - .select(missing_transactions::block_id) - .filter(missing_transactions::transaction_id.eq(&transaction_id)) - .filter(missing_transactions::block_height.eq(current_height.as_u64() as i64)) - .first::(self.connection()) - .optional() - .map_err(|e| SqliteStorageError::DieselError { - operation: "missing_transactions_remove", - source: e, - })?; - let Some(block_id) = block_id else { - return Ok(None); - }; - - diesel::delete(missing_transactions::table) - .filter(missing_transactions::transaction_id.eq(&transaction_id)) - .execute(self.connection()) - .map_err(|e| SqliteStorageError::DieselError { - operation: "missing_transactions_remove", - source: e, - })?; - let mut missing_transactions = missing_transactions::table - .select(missing_transactions::transaction_id) - .filter(missing_transactions::block_id.eq(&block_id)) - .get_results::(self.connection()) - .map_err(|e| SqliteStorageError::DieselError { - operation: "missing_transactions_remove", - source: e, - })?; - - if missing_transactions.is_empty() { - return self.parked_blocks_remove(&block_id).map(Some); - } - - // Make double sure that we dont have these transactions due to a race condition between inserting - // missing transactions and them completing execution. - let found_transaction_ids = transactions::table - .select(transactions::transaction_id) - .filter(transactions::transaction_id.eq_any(&missing_transactions)) - .filter(transactions::result.is_not_null()) - .get_results::(self.connection()) - .map_err(|e| SqliteStorageError::DieselError { - operation: "transactions_get_many", - source: e, - })?; - - diesel::delete(missing_transactions::table) - .filter(missing_transactions::transaction_id.eq_any(&found_transaction_ids)) - .execute(self.connection()) - .map_err(|e| SqliteStorageError::DieselError { - operation: "missing_transactions_remove", - source: e, - })?; - - missing_transactions.retain(|id| found_transaction_ids.iter().all(|found| found != id)); - - if missing_transactions.is_empty() { - return self.parked_blocks_remove(&block_id).map(Some); - } - - Ok(None) - } - fn quorum_certificates_insert(&mut self, qc: &QuorumCertificate) -> Result<(), StorageError> { use crate::schema::quorum_certificates; @@ -1001,6 +869,141 @@ impl StateStoreWriteTransaction for SqliteStateStoreWrit Ok(()) } + fn missing_transactions_insert< + 'a, + IMissing: IntoIterator, + IAwaiting: IntoIterator, + >( + &mut self, + block: &Block, + missing_transaction_ids: IMissing, + awaiting_transaction_ids: IAwaiting, + ) -> Result<(), StorageError> { + use crate::schema::missing_transactions; + + let missing_transaction_ids = missing_transaction_ids + .into_iter() + .map(serialize_hex) + .collect::>(); + let awaiting_transaction_ids = awaiting_transaction_ids + .into_iter() + .map(serialize_hex) + .collect::>(); + let block_id_hex = serialize_hex(block.id()); + + self.parked_blocks_insert(block)?; + + let values = missing_transaction_ids + .iter() + .map(|tx_id| { + ( + missing_transactions::block_id.eq(&block_id_hex), + missing_transactions::block_height.eq(block.height().as_u64() as i64), + missing_transactions::transaction_id.eq(tx_id), + missing_transactions::is_awaiting_execution.eq(false), + ) + }) + .chain(awaiting_transaction_ids.iter().map(|tx_id| { + ( + missing_transactions::block_id.eq(&block_id_hex), + missing_transactions::block_height.eq(block.height().as_u64() as i64), + missing_transactions::transaction_id.eq(tx_id), + missing_transactions::is_awaiting_execution.eq(true), + ) + })) + .collect::>(); + + diesel::insert_into(missing_transactions::table) + .values(values) + .execute(self.connection()) + .map_err(|e| SqliteStorageError::DieselError { + operation: "missing_transactions_insert", + source: e, + })?; + + Ok(()) + } + + fn missing_transactions_remove( + &mut self, + current_height: NodeHeight, + transaction_id: &TransactionId, + ) -> Result, StorageError> { + use crate::schema::{missing_transactions, transactions}; + + // delete all entries that are for previous heights + diesel::delete(missing_transactions::table) + .filter(missing_transactions::block_height.lt(current_height.as_u64().saturating_sub(1) as i64)) + .execute(self.connection()) + .map_err(|e| SqliteStorageError::DieselError { + operation: "missing_transactions_remove", + source: e, + })?; + + let transaction_id = serialize_hex(transaction_id); + let block_id = missing_transactions::table + .select(missing_transactions::block_id) + .filter(missing_transactions::transaction_id.eq(&transaction_id)) + .filter(missing_transactions::block_height.eq(current_height.as_u64() as i64)) + .first::(self.connection()) + .optional() + .map_err(|e| SqliteStorageError::DieselError { + operation: "missing_transactions_remove", + source: e, + })?; + let Some(block_id) = block_id else { + return Ok(None); + }; + + diesel::delete(missing_transactions::table) + .filter(missing_transactions::transaction_id.eq(&transaction_id)) + .execute(self.connection()) + .map_err(|e| SqliteStorageError::DieselError { + operation: "missing_transactions_remove", + source: e, + })?; + let mut missing_transactions = missing_transactions::table + .select(missing_transactions::transaction_id) + .filter(missing_transactions::block_id.eq(&block_id)) + .get_results::(self.connection()) + .map_err(|e| SqliteStorageError::DieselError { + operation: "missing_transactions_remove", + source: e, + })?; + + if missing_transactions.is_empty() { + return self.parked_blocks_remove(&block_id).map(Some); + } + + // Make double sure that we dont have these transactions due to a race condition between inserting + // missing transactions and them completing execution. + let found_transaction_ids = transactions::table + .select(transactions::transaction_id) + .filter(transactions::transaction_id.eq_any(&missing_transactions)) + .filter(transactions::result.is_not_null()) + .get_results::(self.connection()) + .map_err(|e| SqliteStorageError::DieselError { + operation: "transactions_get_many", + source: e, + })?; + + diesel::delete(missing_transactions::table) + .filter(missing_transactions::transaction_id.eq_any(&found_transaction_ids)) + .execute(self.connection()) + .map_err(|e| SqliteStorageError::DieselError { + operation: "missing_transactions_remove", + source: e, + })?; + + missing_transactions.retain(|id| found_transaction_ids.iter().all(|found| found != id)); + + if missing_transactions.is_empty() { + return self.parked_blocks_remove(&block_id).map(Some); + } + + Ok(None) + } + fn votes_insert(&mut self, vote: &Vote) -> Result<(), StorageError> { use crate::schema::votes; @@ -1368,6 +1371,51 @@ impl StateStoreWriteTransaction for SqliteStateStoreWrit // locked.into_iter().map(TryInto::try_into).collect() Ok(vec![]) } + + fn pending_state_tree_diffs_remove_by_block( + &mut self, + block_id: &BlockId, + ) -> Result { + use crate::schema::pending_state_tree_diffs; + + let diff = pending_state_tree_diffs::table + .filter(pending_state_tree_diffs::block_id.eq(serialize_hex(block_id))) + .first::(self.connection()) + .map_err(|e| SqliteStorageError::DieselError { + operation: "pending_state_tree_diffs_remove_by_block", + source: e, + })?; + + diesel::delete(pending_state_tree_diffs::table) + .filter(pending_state_tree_diffs::id.eq(diff.id)) + .execute(self.connection()) + .map_err(|e| SqliteStorageError::DieselError { + operation: "pending_state_tree_diffs_remove_by_block", + source: e, + })?; + + diff.try_into() + } + + fn pending_state_tree_diffs_insert(&mut self, pending_diff: &PendingStateTreeDiff) -> Result<(), StorageError> { + use crate::schema::pending_state_tree_diffs; + + let insert = ( + pending_state_tree_diffs::block_id.eq(serialize_hex(pending_diff.block_id)), + pending_state_tree_diffs::block_height.eq(pending_diff.block_height.as_u64() as i64), + pending_state_tree_diffs::diff_json.eq(serialize_json(&pending_diff.diff)?), + ); + + diesel::insert_into(pending_state_tree_diffs::table) + .values(insert) + .execute(self.connection()) + .map_err(|e| SqliteStorageError::DieselError { + operation: "pending_state_tree_diffs_insert", + source: e, + })?; + + Ok(()) + } } impl<'a, TAddr> Deref for SqliteStateStoreWriteTransaction<'a, TAddr> { diff --git a/dan_layer/state_store_sqlite/tests/tests.rs b/dan_layer/state_store_sqlite/tests/tests.rs index b59db37175..a08c44af64 100644 --- a/dan_layer/state_store_sqlite/tests/tests.rs +++ b/dan_layer/state_store_sqlite/tests/tests.rs @@ -58,6 +58,7 @@ mod confirm_all_transitions { [Command::Prepare(atom1.clone())].into_iter().collect(), Default::default(), Default::default(), + Default::default(), None, ); block1.insert(&mut tx).unwrap(); diff --git a/dan_layer/state_tree/Cargo.toml b/dan_layer/state_tree/Cargo.toml new file mode 100644 index 0000000000..2be4945dfd --- /dev/null +++ b/dan_layer/state_tree/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "tari_state_tree" +version.workspace = true +edition.workspace = true +authors.workspace = true +repository.workspace = true +license.workspace = true + +[dependencies] +tari_dan_common_types = { workspace = true } +tari_engine_types = { workspace = true } +tari_template_lib = { workspace = true } +tari_common_types = { workspace = true } +tari_crypto = { workspace = true } + +hex = { workspace = true } +thiserror = { workspace = true } +serde = { workspace = true, features = ["derive"] } +log = { workspace = true } + +[dev-dependencies] +indexmap = { workspace = true } +itertools = { workspace = true } \ No newline at end of file diff --git a/dan_layer/state_tree/src/error.rs b/dan_layer/state_tree/src/error.rs new file mode 100644 index 0000000000..0744fc9c0d --- /dev/null +++ b/dan_layer/state_tree/src/error.rs @@ -0,0 +1,10 @@ +// Copyright 2024 The Tari Project +// SPDX-License-Identifier: BSD-3-Clause + +use crate::jellyfish::JmtStorageError; + +#[derive(Debug, thiserror::Error)] +pub enum StateTreeError { + #[error("Storage error: {0}")] + StorageError(#[from] JmtStorageError), +} diff --git a/dan_layer/state_tree/src/jellyfish/hash.rs b/dan_layer/state_tree/src/jellyfish/hash.rs new file mode 100644 index 0000000000..823dbb6e55 --- /dev/null +++ b/dan_layer/state_tree/src/jellyfish/hash.rs @@ -0,0 +1,134 @@ +// Copyright 2024 The Tari Project +// SPDX-License-Identifier: BSD-3-Clause + +// Copyright 2022. The Tari Project +// +// Redistribution and use in source and binary forms, with or without modification, are permitted provided that the +// following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following +// disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the +// following disclaimer in the documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote +// products derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, +// INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +// USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +use std::{ + convert::TryFrom, + fmt::{Display, Formatter}, + ops::{Deref, DerefMut}, +}; + +const ZERO_HASH: [u8; TreeHash::byte_size()] = [0u8; TreeHash::byte_size()]; + +#[derive(thiserror::Error, Debug)] +#[error("Invalid size")] +pub struct TreeHashSizeError; + +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug, Default, Hash)] +pub struct TreeHash([u8; TreeHash::byte_size()]); + +impl TreeHash { + pub const fn new(hash: [u8; TreeHash::byte_size()]) -> Self { + Self(hash) + } + + pub const fn byte_size() -> usize { + 32 + } + + pub const fn zero() -> Self { + Self(ZERO_HASH) + } + + pub fn as_slice(&self) -> &[u8] { + &self.0 + } +} + +impl From<[u8; TreeHash::byte_size()]> for TreeHash { + fn from(hash: [u8; TreeHash::byte_size()]) -> Self { + Self(hash) + } +} + +impl TryFrom> for TreeHash { + type Error = TreeHashSizeError; + + fn try_from(value: Vec) -> Result { + TryFrom::try_from(value.as_slice()) + } +} + +impl TryFrom<&[u8]> for TreeHash { + type Error = TreeHashSizeError; + + fn try_from(bytes: &[u8]) -> Result { + if bytes.len() != TreeHash::byte_size() { + return Err(TreeHashSizeError); + } + + let mut buf = [0u8; TreeHash::byte_size()]; + buf.copy_from_slice(bytes); + Ok(Self(buf)) + } +} + +impl PartialEq<[u8]> for TreeHash { + fn eq(&self, other: &[u8]) -> bool { + self.0[..].eq(other) + } +} + +impl PartialEq for [u8] { + fn eq(&self, other: &TreeHash) -> bool { + self[..].eq(&other.0) + } +} + +impl PartialEq> for TreeHash { + fn eq(&self, other: &Vec) -> bool { + self == other.as_slice() + } +} +impl PartialEq for Vec { + fn eq(&self, other: &TreeHash) -> bool { + self == other.as_slice() + } +} + +impl AsRef<[u8]> for TreeHash { + fn as_ref(&self) -> &[u8] { + self.as_slice() + } +} + +impl Deref for TreeHash { + type Target = [u8; TreeHash::byte_size()]; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for TreeHash { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +impl Display for TreeHash { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + hex::encode(&self.0).fmt(f) + } +} diff --git a/dan_layer/state_tree/src/jellyfish/mod.rs b/dan_layer/state_tree/src/jellyfish/mod.rs new file mode 100644 index 0000000000..e4b8bb4ca9 --- /dev/null +++ b/dan_layer/state_tree/src/jellyfish/mod.rs @@ -0,0 +1,14 @@ +// Copyright 2024 The Tari Project +// SPDX-License-Identifier: BSD-3-Clause + +// mod hash; +// pub use hash::*; + +mod tree; +pub use tree::*; + +mod types; +pub use types::*; + +mod store; +pub use store::*; diff --git a/dan_layer/state_tree/src/jellyfish/store.rs b/dan_layer/state_tree/src/jellyfish/store.rs new file mode 100644 index 0000000000..65d1c78f3c --- /dev/null +++ b/dan_layer/state_tree/src/jellyfish/store.rs @@ -0,0 +1,79 @@ +// Copyright 2024 The Tari Project +// SPDX-License-Identifier: BSD-3-Clause + +use serde::{Deserialize, Serialize}; + +use crate::{ + jellyfish::{JmtStorageError, Node, NodeKey}, + Version, +}; + +/// Implementers are able to read nodes from a tree store. +pub trait TreeStoreReader

{ + /// Gets node by key, if it exists. + fn get_node(&self, key: &NodeKey) -> Result, JmtStorageError>; +} + +/// Implementers are able to insert nodes to a tree store. +pub trait TreeStoreWriter

{ + /// Inserts the node under a new, unique key (i.e. never an update). + fn insert_node(&mut self, key: NodeKey, node: Node

) -> Result<(), JmtStorageError>; + + /// Marks the given tree part for a (potential) future removal by an arbitrary external pruning + /// process. + fn record_stale_tree_node(&mut self, part: StaleTreeNode) -> Result<(), JmtStorageError>; +} + +/// Implementers are able to read and write nodes to a tree store. +pub trait TreeStore

: TreeStoreReader

+ TreeStoreWriter

{} +impl + TreeStoreWriter

> TreeStore

for S {} + +/// A part of a tree that may become stale (i.e. need eventual pruning). +#[derive(Clone, PartialEq, Eq, Hash, Debug, Serialize, Deserialize)] +pub enum StaleTreeNode { + /// A single node to be removed. + Node(NodeKey), + /// An entire subtree of descendants of a specific node (including itself). + Subtree(NodeKey), +} + +impl StaleTreeNode { + pub fn into_node_key(self) -> NodeKey { + match self { + Self::Node(key) | Self::Subtree(key) => key, + } + } + + pub fn as_node_key(&self) -> &NodeKey { + match self { + Self::Node(key) | Self::Subtree(key) => key, + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum TreeNode { + V1(Node), +} + +impl TreeNode { + pub fn new_latest(node: Node) -> Self { + Self::new_v1(node) + } + + pub fn new_v1(node: Node) -> Self { + Self::V1(node) + } + + pub fn as_node(&self) -> &Node { + match self { + Self::V1(node) => node, + } + } + + pub fn into_node(self) -> Node { + match self { + Self::V1(node) => node, + } + } +} diff --git a/dan_layer/state_tree/src/jellyfish/tree.rs b/dan_layer/state_tree/src/jellyfish/tree.rs new file mode 100644 index 0000000000..7eeaee9767 --- /dev/null +++ b/dan_layer/state_tree/src/jellyfish/tree.rs @@ -0,0 +1,733 @@ +// Copyright 2024 The Tari Project +// SPDX-License-Identifier: BSD-3-Clause + +// Copyright 2021 Radix Publishing Ltd incorporated in Jersey (Channel Islands). +// +// Licensed under the Radix License, Version 1.0 (the "License"); you may not use this +// file except in compliance with the License. You may obtain a copy of the License at: +// +// radixfoundation.org/licenses/LICENSE-v1 +// +// The Licensor hereby grants permission for the Canonical version of the Work to be +// published, distributed and used under or by reference to the Licensor's trademark +// Radix ® and use of any unregistered trade names, logos or get-up. +// +// The Licensor provides the Work (and each Contributor provides its Contributions) on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, +// including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, +// MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. +// +// Whilst the Work is capable of being deployed, used and adopted (instantiated) to create +// a distributed ledger it is your responsibility to test and validate the code, together +// with all logic and performance of that code under all foreseeable scenarios. +// +// The Licensor does not make or purport to make and hereby excludes liability for all +// and any representation, warranty or undertaking in any form whatsoever, whether express +// or implied, to any entity or person, including any representation, warranty or +// undertaking, as to the functionality security use, value or other characteristics of +// any distributed ledger nor in respect the functioning or value of any tokens which may +// be created stored or transferred using the Work. The Licensor does not warrant that the +// Work or any use of the Work complies with any law or regulation in any territory where +// it may be implemented or used or that it will be appropriate for any specific purpose. +// +// Neither the licensor nor any current or former employees, officers, directors, partners, +// trustees, representatives, agents, advisors, contractors, or volunteers of the Licensor +// shall be liable for any direct or indirect, special, incidental, consequential or other +// losses of any kind, in tort, contract or otherwise (including but not limited to loss +// of revenue, income or profits, or loss of use or data, or loss of reputation, or loss +// of any economic or other opportunity of whatsoever nature or howsoever arising), arising +// out of or in connection with (without limitation of any use, misuse, of any ledger system +// or use made or its functionality or any performance or operation of any code or protocol +// caused by bugs or programming or logic errors or otherwise); +// +// A. any offer, purchase, holding, use, sale, exchange or transmission of any +// cryptographic keys, tokens or assets created, exchanged, stored or arising from any +// interaction with the Work; +// +// B. any failure in a transmission or loss of any token or assets keys or other digital +// artefacts due to errors in transmission; +// +// C. bugs, hacks, logic errors or faults in the Work or any communication; +// +// D. system software or apparatus including but not limited to losses caused by errors +// in holding or transmitting tokens by any third-party; +// +// E. breaches or failure of security including hacker attacks, loss or disclosure of +// password, loss of private key, unauthorised use or misuse of such passwords or keys; +// +// F. any losses including loss of anticipated savings or other benefits resulting from +// use of the Work or any changes to the Work (however implemented). +// +// You are solely responsible for; testing, validating and evaluation of all operation +// logic, functionality, security and appropriateness of using the Work for any commercial +// or non-commercial purpose and for any reproduction or redistribution by You of the +// Work. You assume all risks associated with Your use of the Work and the exercise of +// permissions under this License. + +// This file contains code sourced from https://github.com/aptos-labs/aptos-core/tree/1.0.4 +// This original source is licensed under https://github.com/aptos-labs/aptos-core/blob/1.0.4/LICENSE +// +// The code in this file has been implemented by Radix® pursuant to an Apache 2 licence and has +// been modified by Radix® and is now licensed pursuant to the Radix® Open-Source Licence. +// +// Each sourced code fragment includes an inline attribution to the original source file in a +// comment starting "SOURCE: ..." +// +// Modifications from the original source are captured in two places: +// * Initial changes to get the code functional/integrated are marked by inline "INITIAL-MODIFICATION: ..." comments +// * Subsequent changes to the code are captured in the git commit history +// +// The following notice is retained from the original source +// Copyright (c) Aptos +// SPDX-License-Identifier: Apache-2.0 + +use std::{ + collections::{BTreeMap, HashMap}, + marker::PhantomData, +}; + +use super::{ + store::TreeStoreReader, + types::{ + Child, + InternalNode, + IteratedLeafKey, + JmtStorageError, + LeafKey, + LeafNode, + Nibble, + NibblePath, + Node, + NodeKey, + SparseMerkleProof, + SparseMerkleProofExt, + SparseMerkleRangeProof, + Version, + SPARSE_MERKLE_PLACEHOLDER_HASH, + }, + Hash, + LeafKeyRef, +}; + +// INITIAL-MODIFICATION: the original used a known key size (32) as a limit +const SANITY_NIBBLE_LIMIT: usize = 1000; + +pub type ProofValue

= (Hash, P, Version); + +// SOURCE: https://github.com/radixdlt/radixdlt-scrypto/blob/ca8e553c31a956c0851c1855291efe4a47fb5c97/radix-engine-stores/src/hash_tree/jellyfish.rs +// SOURCE: https://github.com/aptos-labs/aptos-core/blob/1.0.4/storage/jellyfish-merkle/src/lib.rs#L329 +/// The Jellyfish Merkle tree data structure. See [`crate`] for description. +pub struct JellyfishMerkleTree<'a, R, P> { + reader: &'a R, + _payload: PhantomData

, +} + +impl<'a, R: 'a + TreeStoreReader

, P: Clone> JellyfishMerkleTree<'a, R, P> { + /// Creates a `JellyfishMerkleTree` backed by the given [`TreeReader`](trait.TreeReader.html). + pub fn new(reader: &'a R) -> Self { + Self { + reader, + _payload: PhantomData, + } + } + + /// Get the node hash from the cache if cache is provided, otherwise (for test only) compute it. + fn get_hash(node_key: &NodeKey, node: &Node

, hash_cache: Option<&HashMap>) -> Hash { + if let Some(cache) = hash_cache { + match cache.get(node_key.nibble_path()) { + Some(hash) => *hash, + None => unreachable!("{:?} can not be found in hash cache", node_key), + } + } else { + node.hash() + } + } + + /// For each value set: + /// Returns the new nodes and values in a batch after applying `value_set`. For + /// example, if after transaction `T_i` the committed state of tree in the persistent storage + /// looks like the following structure: + /// + /// ```text + /// S_i + /// / \ + /// . . + /// . . + /// / \ + /// o x + /// / \ + /// A B + /// storage (disk) + /// ``` + /// + /// where `A` and `B` denote the states of two adjacent accounts, and `x` is a sibling subtree + /// of the path from root to A and B in the tree. Then a `value_set` produced by the next + /// transaction `T_{i+1}` modifies other accounts `C` and `D` exist in the subtree under `x`, a + /// new partial tree will be constructed in memory and the structure will be: + /// + /// ```text + /// S_i | S_{i+1} + /// / \ | / \ + /// . . | . . + /// . . | . . + /// / \ | / \ + /// / x | / x' + /// o<-------------+- / \ + /// / \ | C D + /// A B | + /// storage (disk) | cache (memory) + /// ``` + /// + /// With this design, we are able to query the global state in persistent storage and + /// generate the proposed tree delta based on a specific root hash and `value_set`. For + /// example, if we want to execute another transaction `T_{i+1}'`, we can use the tree `S_i` in + /// storage and apply the `value_set` of transaction `T_{i+1}`. Then if the storage commits + /// the returned batch, the state `S_{i+1}` is ready to be read from the tree by calling + /// [`get_with_proof`](struct.JellyfishMerkleTree.html#method.get_with_proof). Anything inside + /// the batch is not reachable from public interfaces before being committed. + pub fn batch_put_value_set( + &self, + value_set: Vec<(&LeafKey, Option<&(Hash, P)>)>, + node_hashes: Option<&HashMap>, + persisted_version: Option, + version: Version, + ) -> Result<(Hash, TreeUpdateBatch

), JmtStorageError> { + let deduped_and_sorted_kvs = value_set + .into_iter() + .collect::>() + .into_iter() + .collect::>(); + + let mut batch = TreeUpdateBatch::new(); + let root_node_opt = if let Some(persisted_version) = persisted_version { + self.batch_insert_at( + &NodeKey::new_empty_path(persisted_version), + version, + deduped_and_sorted_kvs.as_slice(), + 0, + node_hashes, + &mut batch, + )? + } else { + Self::batch_update_subtree( + &NodeKey::new_empty_path(version), + version, + deduped_and_sorted_kvs.as_slice(), + 0, + node_hashes, + &mut batch, + )? + }; + + let node_key = NodeKey::new_empty_path(version); + let root_hash = if let Some(root_node) = root_node_opt { + let hash = root_node.hash(); + batch.put_node(node_key, root_node); + hash + } else { + batch.put_node(node_key, Node::Null); + SPARSE_MERKLE_PLACEHOLDER_HASH + }; + + Ok((root_hash, batch)) + } + + fn batch_insert_at( + &self, + node_key: &NodeKey, + version: Version, + kvs: &[(&LeafKey, Option<&(Hash, P)>)], + depth: usize, + hash_cache: Option<&HashMap>, + batch: &mut TreeUpdateBatch

, + ) -> Result>, JmtStorageError> { + let node = self.reader.get_node(node_key)?; + batch.put_stale_node(node_key.clone(), version, &node); + + match node { + Node::Internal(internal_node) => { + // There is a small possibility that the old internal node is intact. + // Traverse all the path touched by `kvs` from this internal node. + let range_iter = NibbleRangeIterator::new(kvs, depth); + // INITIAL-MODIFICATION: there was a par_iter (conditionally) used here + let new_children = range_iter + .map(|(left, right)| { + self.insert_at_child( + node_key, + &internal_node, + version, + kvs, + left, + right, + depth, + hash_cache, + batch, + ) + }) + .collect::, JmtStorageError>>()?; + + // Reuse the current `InternalNode` in memory to create a new internal node. + let mut old_children = internal_node.into_children(); + let mut new_created_children: HashMap> = HashMap::new(); + for (child_nibble, child_option) in new_children { + if let Some(child) = child_option { + new_created_children.insert(child_nibble, child); + } else { + old_children.remove(&child_nibble); + } + } + + if old_children.is_empty() && new_created_children.is_empty() { + return Ok(None); + } + if old_children.len() <= 1 && new_created_children.len() <= 1 { + if let Some((new_nibble, new_child)) = new_created_children.iter().next() { + if let Some((old_nibble, _old_child)) = old_children.iter().next() { + if old_nibble == new_nibble && new_child.is_leaf() { + return Ok(Some(new_child.clone())); + } + } else if new_child.is_leaf() { + return Ok(Some(new_child.clone())); + } else { + // Nothing to do + } + } else { + let (old_child_nibble, old_child) = old_children.iter().next().expect("must exist"); + if old_child.is_leaf() { + let old_child_node_key = node_key.gen_child_node_key(old_child.version, *old_child_nibble); + let old_child_node = self.reader.get_node(&old_child_node_key)?; + batch.put_stale_node(old_child_node_key, version, &old_child_node); + return Ok(Some(old_child_node)); + } + } + } + + let mut new_children = old_children; + for (child_index, new_child_node) in new_created_children { + let new_child_node_key = node_key.gen_child_node_key(version, child_index); + new_children.insert( + child_index, + Child::new( + Self::get_hash(&new_child_node_key, &new_child_node, hash_cache), + version, + new_child_node.node_type(), + ), + ); + batch.put_node(new_child_node_key, new_child_node); + } + let new_internal_node = InternalNode::new(new_children); + Ok(Some(new_internal_node.into())) + }, + Node::Leaf(leaf_node) => Self::batch_update_subtree_with_existing_leaf( + node_key, version, leaf_node, kvs, depth, hash_cache, batch, + ), + Node::Null => { + assert_eq!(depth, 0, "Null node can only exist at depth 0"); + Self::batch_update_subtree(node_key, version, kvs, 0, hash_cache, batch) + }, + } + } + + fn insert_at_child( + &self, + node_key: &NodeKey, + internal_node: &InternalNode, + version: Version, + kvs: &[(&LeafKey, Option<&(Hash, P)>)], + left: usize, + right: usize, + depth: usize, + hash_cache: Option<&HashMap>, + batch: &mut TreeUpdateBatch

, + ) -> Result<(Nibble, Option>), JmtStorageError> { + let child_index = kvs[left].0.get_nibble(depth); + let child = internal_node.child(child_index); + + let new_child_node_option = match child { + Some(child) => self.batch_insert_at( + &node_key.gen_child_node_key(child.version, child_index), + version, + &kvs[left..=right], + depth + 1, + hash_cache, + batch, + )?, + None => Self::batch_update_subtree( + &node_key.gen_child_node_key(version, child_index), + version, + &kvs[left..=right], + depth + 1, + hash_cache, + batch, + )?, + }; + + Ok((child_index, new_child_node_option)) + } + + fn batch_update_subtree_with_existing_leaf( + node_key: &NodeKey, + version: Version, + existing_leaf_node: LeafNode

, + kvs: &[(&LeafKey, Option<&(Hash, P)>)], + depth: usize, + hash_cache: Option<&HashMap>, + batch: &mut TreeUpdateBatch

, + ) -> Result>, JmtStorageError> { + let existing_leaf_key = existing_leaf_node.leaf_key(); + + if kvs.len() == 1 && kvs[0].0 == existing_leaf_key { + if let (key, Some((value_hash, payload))) = kvs[0] { + let new_leaf_node = Node::new_leaf(key.clone(), *value_hash, payload.clone(), version); + Ok(Some(new_leaf_node)) + } else { + Ok(None) + } + } else { + let existing_leaf_bucket = existing_leaf_key.get_nibble(depth); + let mut isolated_existing_leaf = true; + let mut children = vec![]; + for (left, right) in NibbleRangeIterator::new(kvs, depth) { + let child_index = kvs[left].0.get_nibble(depth); + let child_node_key = node_key.gen_child_node_key(version, child_index); + if let Some(new_child_node) = if existing_leaf_bucket == child_index { + isolated_existing_leaf = false; + Self::batch_update_subtree_with_existing_leaf( + &child_node_key, + version, + existing_leaf_node.clone(), + &kvs[left..=right], + depth + 1, + hash_cache, + batch, + )? + } else { + Self::batch_update_subtree( + &child_node_key, + version, + &kvs[left..=right], + depth + 1, + hash_cache, + batch, + )? + } { + children.push((child_index, new_child_node)); + } + } + if isolated_existing_leaf { + children.push((existing_leaf_bucket, existing_leaf_node.into())); + } + + if children.is_empty() { + Ok(None) + } else if children.len() == 1 && children[0].1.is_leaf() { + let (_, child) = children.pop().expect("Must exist"); + Ok(Some(child)) + } else { + let new_internal_node = InternalNode::new( + children + .into_iter() + .map(|(child_index, new_child_node)| { + let new_child_node_key = node_key.gen_child_node_key(version, child_index); + let result = ( + child_index, + Child::new( + Self::get_hash(&new_child_node_key, &new_child_node, hash_cache), + version, + new_child_node.node_type(), + ), + ); + batch.put_node(new_child_node_key, new_child_node); + result + }) + .collect(), + ); + Ok(Some(new_internal_node.into())) + } + } + } + + fn batch_update_subtree( + node_key: &NodeKey, + version: Version, + kvs: &[(&LeafKey, Option<&(Hash, P)>)], + depth: usize, + hash_cache: Option<&HashMap>, + batch: &mut TreeUpdateBatch

, + ) -> Result>, JmtStorageError> { + if kvs.len() == 1 { + if let (key, Some((value_hash, payload))) = kvs[0] { + let new_leaf_node = Node::new_leaf(key.clone(), *value_hash, payload.clone(), version); + Ok(Some(new_leaf_node)) + } else { + Ok(None) + } + } else { + let mut children = vec![]; + for (left, right) in NibbleRangeIterator::new(kvs, depth) { + let child_index = kvs[left].0.get_nibble(depth); + let child_node_key = node_key.gen_child_node_key(version, child_index); + if let Some(new_child_node) = Self::batch_update_subtree( + &child_node_key, + version, + &kvs[left..=right], + depth + 1, + hash_cache, + batch, + )? { + children.push((child_index, new_child_node)) + } + } + if children.is_empty() { + Ok(None) + } else if children.len() == 1 && children[0].1.is_leaf() { + let (_, child) = children.pop().expect("Must exist"); + Ok(Some(child)) + } else { + let new_internal_node = InternalNode::new( + children + .into_iter() + .map(|(child_index, new_child_node)| { + let new_child_node_key = node_key.gen_child_node_key(version, child_index); + let result = ( + child_index, + Child::new( + Self::get_hash(&new_child_node_key, &new_child_node, hash_cache), + version, + new_child_node.node_type(), + ), + ); + batch.put_node(new_child_node_key, new_child_node); + result + }) + .collect(), + ); + Ok(Some(new_internal_node.into())) + } + } + } + + /// Returns the value (if applicable) and the corresponding merkle proof. + pub fn get_with_proof( + &self, + key: LeafKeyRef<'_>, + version: Version, + ) -> Result<(Option>, SparseMerkleProof), JmtStorageError> { + self.get_with_proof_ext(key, version) + .map(|(value, proof_ext)| (value, proof_ext.into())) + } + + pub fn get_with_proof_ext( + &self, + key: LeafKeyRef<'_>, + version: Version, + ) -> Result<(Option>, SparseMerkleProofExt), JmtStorageError> { + // Empty tree just returns proof with no sibling hash. + let mut next_node_key = NodeKey::new_empty_path(version); + let mut siblings = vec![]; + let nibble_path = NibblePath::new_even(key.bytes.to_vec()); + let mut nibble_iter = nibble_path.nibbles(); + + for _nibble_depth in 0..SANITY_NIBBLE_LIMIT { + let next_node = self.reader.get_node(&next_node_key)?; + match next_node { + Node::Internal(internal_node) => { + let queried_child_index = nibble_iter.next().ok_or(JmtStorageError::InconsistentState)?; + let (child_node_key, mut siblings_in_internal) = internal_node.get_child_with_siblings( + &next_node_key, + queried_child_index, + Some(self.reader), + )?; + siblings.append(&mut siblings_in_internal); + next_node_key = match child_node_key { + Some(node_key) => node_key, + None => { + return Ok(( + None, + SparseMerkleProofExt::new(None, { + siblings.reverse(); + siblings + }), + )) + }, + }; + }, + Node::Leaf(leaf_node) => { + return Ok(( + if leaf_node.leaf_key().as_ref() == key { + Some((leaf_node.value_hash(), leaf_node.payload().clone(), leaf_node.version())) + } else { + None + }, + SparseMerkleProofExt::new(Some(leaf_node.into()), { + siblings.reverse(); + siblings + }), + )); + }, + Node::Null => { + return Ok((None, SparseMerkleProofExt::new(None, vec![]))); + }, + } + } + Err(JmtStorageError::InconsistentState) + } + + /// Gets the proof that shows a list of keys up to `rightmost_key_to_prove` exist at `version`. + pub fn get_range_proof( + &self, + rightmost_key_to_prove: LeafKeyRef<'_>, + version: Version, + ) -> Result { + let (leaf, proof) = self.get_with_proof(rightmost_key_to_prove, version)?; + assert!(leaf.is_some(), "rightmost_key_to_prove must exist."); + + let siblings = proof + .siblings() + .iter() + .rev() + .zip(rightmost_key_to_prove.iter_bits()) + .filter_map(|(sibling, bit)| { + // We only need to keep the siblings on the right. + if bit { + None + } else { + Some(*sibling) + } + }) + .rev() + .collect(); + Ok(SparseMerkleRangeProof::new(siblings)) + } + + fn get_root_node(&self, version: Version) -> Result, JmtStorageError> { + let root_node_key = NodeKey::new_empty_path(version); + self.reader.get_node(&root_node_key) + } + + pub fn get_root_hash(&self, version: Version) -> Result { + self.get_root_node(version).map(|n| n.hash()) + } + + pub fn get_leaf_count(&self, version: Version) -> Result { + self.get_root_node(version).map(|n| n.leaf_count()) + } + + pub fn get_all_nodes_referenced(&self, key: NodeKey) -> Result, JmtStorageError> { + let mut out_keys = vec![]; + self.get_all_nodes_referenced_impl(key, &mut out_keys)?; + Ok(out_keys) + } + + fn get_all_nodes_referenced_impl(&self, key: NodeKey, out_keys: &mut Vec) -> Result<(), JmtStorageError> { + match self.reader.get_node(&key)? { + Node::Internal(internal_node) => { + for (child_nibble, child) in internal_node.children_sorted() { + self.get_all_nodes_referenced_impl(key.gen_child_node_key(child.version, *child_nibble), out_keys)?; + } + }, + Node::Leaf(_) | Node::Null => {}, + }; + + out_keys.push(key); + Ok(()) + } +} + +/// An iterator that iterates the index range (inclusive) of each different nibble at given +/// `nibble_idx` of all the keys in a sorted key-value pairs which have the identical Hash +/// prefix (up to nibble_idx). +struct NibbleRangeIterator<'a, P> { + sorted_kvs: &'a [(&'a LeafKey, P)], + nibble_idx: usize, + pos: usize, +} + +impl<'a, P> NibbleRangeIterator<'a, P> { + fn new(sorted_kvs: &'a [(&'a LeafKey, P)], nibble_idx: usize) -> Self { + NibbleRangeIterator { + sorted_kvs, + nibble_idx, + pos: 0, + } + } +} + +impl<'a, P> Iterator for NibbleRangeIterator<'a, P> { + type Item = (usize, usize); + + fn next(&mut self) -> Option { + let left = self.pos; + if self.pos < self.sorted_kvs.len() { + let cur_nibble = self.sorted_kvs[left].0.get_nibble(self.nibble_idx); + let (mut i, mut j) = (left, self.sorted_kvs.len() - 1); + // Find the last index of the cur_nibble. + while i < j { + let mid = j - (j - i) / 2; + if self.sorted_kvs[mid].0.get_nibble(self.nibble_idx) > cur_nibble { + j = mid - 1; + } else { + i = mid; + } + } + self.pos = i + 1; + Some((left, i)) + } else { + None + } + } +} + +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub struct TreeUpdateBatch

{ + pub node_batch: Vec<(NodeKey, Node

)>, + pub stale_node_index_batch: Vec, + pub num_new_leaves: usize, + pub num_stale_leaves: usize, +} + +impl TreeUpdateBatch

{ + pub fn new() -> Self { + Self { + node_batch: vec![], + stale_node_index_batch: vec![], + num_new_leaves: 0, + num_stale_leaves: 0, + } + } + + fn inc_num_new_leaves(&mut self) { + self.num_new_leaves += 1; + } + + fn inc_num_stale_leaves(&mut self) { + self.num_stale_leaves += 1; + } + + pub fn put_node(&mut self, node_key: NodeKey, node: Node

) { + if node.is_leaf() { + self.inc_num_new_leaves(); + } + self.node_batch.push((node_key, node)) + } + + pub fn put_stale_node(&mut self, node_key: NodeKey, stale_since_version: Version, node: &Node

) { + if node.is_leaf() { + self.inc_num_stale_leaves(); + } + self.stale_node_index_batch.push(StaleNodeIndex { + node_key, + stale_since_version, + }); + } +} + +/// Indicates a node becomes stale since `stale_since_version`. +#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct StaleNodeIndex { + /// The version since when the node is overwritten and becomes stale. + pub stale_since_version: Version, + /// The [`NodeKey`](node_type/struct.NodeKey.html) identifying the node associated with this + /// record. + pub node_key: NodeKey, +} diff --git a/dan_layer/state_tree/src/jellyfish/types.rs b/dan_layer/state_tree/src/jellyfish/types.rs new file mode 100644 index 0000000000..7849748328 --- /dev/null +++ b/dan_layer/state_tree/src/jellyfish/types.rs @@ -0,0 +1,1249 @@ +// Copyright 2024 The Tari Project +// SPDX-License-Identifier: BSD-3-Clause + +// Copyright 2021 Radix Publishing Ltd incorporated in Jersey (Channel Islands). +// +// Licensed under the Radix License, Version 1.0 (the "License"); you may not use this +// file except in compliance with the License. You may obtain a copy of the License at: +// +// radixfoundation.org/licenses/LICENSE-v1 +// +// The Licensor hereby grants permission for the Canonical version of the Work to be +// published, distributed and used under or by reference to the Licensor's trademark +// Radix ® and use of any unregistered trade names, logos or get-up. +// +// The Licensor provides the Work (and each Contributor provides its Contributions) on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, +// including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, +// MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. +// +// Whilst the Work is capable of being deployed, used and adopted (instantiated) to create +// a distributed ledger it is your responsibility to test and validate the code, together +// with all logic and performance of that code under all foreseeable scenarios. +// +// The Licensor does not make or purport to make and hereby excludes liability for all +// and any representation, warranty or undertaking in any form whatsoever, whether express +// or implied, to any entity or person, including any representation, warranty or +// undertaking, as to the functionality security use, value or other characteristics of +// any distributed ledger nor in respect the functioning or value of any tokens which may +// be created stored or transferred using the Work. The Licensor does not warrant that the +// Work or any use of the Work complies with any law or regulation in any territory where +// it may be implemented or used or that it will be appropriate for any specific purpose. +// +// Neither the licensor nor any current or former employees, officers, directors, partners, +// trustees, representatives, agents, advisors, contractors, or volunteers of the Licensor +// shall be liable for any direct or indirect, special, incidental, consequential or other +// losses of any kind, in tort, contract or otherwise (including but not limited to loss +// of revenue, income or profits, or loss of use or data, or loss of reputation, or loss +// of any economic or other opportunity of whatsoever nature or howsoever arising), arising +// out of or in connection with (without limitation of any use, misuse, of any ledger system +// or use made or its functionality or any performance or operation of any code or protocol +// caused by bugs or programming or logic errors or otherwise); +// +// A. any offer, purchase, holding, use, sale, exchange or transmission of any +// cryptographic keys, tokens or assets created, exchanged, stored or arising from any +// interaction with the Work; +// +// B. any failure in a transmission or loss of any token or assets keys or other digital +// artefacts due to errors in transmission; +// +// C. bugs, hacks, logic errors or faults in the Work or any communication; +// +// D. system software or apparatus including but not limited to losses caused by errors +// in holding or transmitting tokens by any third-party; +// +// E. breaches or failure of security including hacker attacks, loss or disclosure of +// password, loss of private key, unauthorised use or misuse of such passwords or keys; +// +// F. any losses including loss of anticipated savings or other benefits resulting from +// use of the Work or any changes to the Work (however implemented). +// +// You are solely responsible for; testing, validating and evaluation of all operation +// logic, functionality, security and appropriateness of using the Work for any commercial +// or non-commercial purpose and for any reproduction or redistribution by You of the +// Work. You assume all risks associated with Your use of the Work and the exercise of +// permissions under this License. + +// This file contains code sourced from https://github.com/aptos-labs/aptos-core/tree/1.0.4 +// This original source is licensed under https://github.com/aptos-labs/aptos-core/blob/1.0.4/LICENSE +// +// The code in this file has been implemented by Radix® pursuant to an Apache 2 licence and has +// been modified by Radix® and is now licensed pursuant to the Radix® Open-Source Licence. +// +// Each sourced code fragment includes an inline attribution to the original source file in a +// comment starting "SOURCE: ..." +// +// Modifications from the original source are captured in two places: +// * Initial changes to get the code functional/integrated are marked by inline "INITIAL-MODIFICATION: ..." comments +// * Subsequent changes to the code are captured in the git commit history +// +// The following notice is retained from the original source +// Copyright (c) Aptos +// SPDX-License-Identifier: Apache-2.0 + +use std::{collections::HashMap, fmt, ops::Range}; + +use serde::{Deserialize, Serialize}; +use tari_crypto::{hash_domain, tari_utilities::ByteArray}; +use tari_dan_common_types::{ + hasher::{tari_hasher, TariHasher}, + optional::IsNotFoundError, +}; +use tari_engine_types::serde_with; + +use crate::jellyfish::store::TreeStoreReader; + +pub type Hash = tari_common_types::types::FixedHash; + +hash_domain!(SparseMerkleTree, "com.tari.dan.state_tree", 0); + +fn hasher() -> TariHasher { + tari_hasher::("hash") +} + +pub fn hash>(data: T) -> Hash { + hasher().chain(data.as_ref()).result() +} + +pub fn hash2(d1: &[u8], d2: &[u8]) -> Hash { + hasher().chain(d1).chain(d2).result() +} + +// SOURCE: https://github.com/aptos-labs/aptos-core/blob/1.0.4/types/src/proof/definition.rs#L182 +/// A more detailed version of `SparseMerkleProof` with the only difference that all the leaf +/// siblings are explicitly set as `SparseMerkleLeafNode` instead of its hash value. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct SparseMerkleProofExt { + leaf: Option, + /// All siblings in this proof, including the default ones. Siblings are ordered from the bottom + /// level to the root level. + siblings: Vec, +} + +impl SparseMerkleProofExt { + /// Constructs a new `SparseMerkleProofExt` using leaf and a list of sibling nodes. + pub fn new(leaf: Option, siblings: Vec) -> Self { + Self { leaf, siblings } + } + + /// Returns the leaf node in this proof. + pub fn leaf(&self) -> Option { + self.leaf.clone() + } + + /// Returns the list of siblings in this proof. + pub fn siblings(&self) -> &[NodeInProof] { + &self.siblings + } +} + +impl From for SparseMerkleProof { + fn from(proof_ext: SparseMerkleProofExt) -> Self { + Self::new( + proof_ext.leaf, + proof_ext.siblings.into_iter().map(|node| node.hash()).collect(), + ) + } +} + +// SOURCE: https://github.com/aptos-labs/aptos-core/blob/1.0.4/types/src/proof/definition.rs#L135 +impl SparseMerkleProof { + /// Constructs a new `SparseMerkleProof` using leaf and a list of siblings. + pub fn new(leaf: Option, siblings: Vec) -> Self { + SparseMerkleProof { leaf, siblings } + } + + /// Returns the leaf node in this proof. + pub fn leaf(&self) -> Option { + self.leaf.clone() + } + + /// Returns the list of siblings in this proof. + pub fn siblings(&self) -> &[Hash] { + &self.siblings + } +} + +/// A proof that can be used to authenticate an element in a Sparse Merkle Tree given trusted root +/// hash. For example, `TransactionInfoToAccountProof` can be constructed on top of this structure. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct SparseMerkleProof { + /// This proof can be used to authenticate whether a given leaf exists in the tree or not. + /// - If this is `Some(leaf_node)` + /// - If `leaf_node.key` equals requested key, this is an inclusion proof and `leaf_node.value_hash` equals + /// the hash of the corresponding account blob. + /// - Otherwise this is a non-inclusion proof. `leaf_node.key` is the only key that exists in the subtree + /// and `leaf_node.value_hash` equals the hash of the corresponding account blob. + /// - If this is `None`, this is also a non-inclusion proof which indicates the subtree is empty. + leaf: Option, + + /// All siblings in this proof, including the default ones. Siblings are ordered from the bottom + /// level to the root level. + siblings: Vec, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum NodeInProof { + Leaf(SparseMerkleLeafNode), + Other(Hash), +} + +impl From for NodeInProof { + fn from(hash: Hash) -> Self { + Self::Other(hash) + } +} + +impl From for NodeInProof { + fn from(leaf: SparseMerkleLeafNode) -> Self { + Self::Leaf(leaf) + } +} + +impl NodeInProof { + pub fn hash(&self) -> Hash { + match self { + Self::Leaf(leaf) => leaf.hash(), + Self::Other(hash) => *hash, + } + } +} + +// SOURCE: https://github.com/aptos-labs/aptos-core/blob/1.0.4/types/src/proof/definition.rs#L681 +/// Note: this is not a range proof in the sense that a range of nodes is verified! +/// Instead, it verifies the entire left part of the tree up to a known rightmost node. +/// See the description below. +/// +/// A proof that can be used to authenticate a range of consecutive leaves, from the leftmost leaf to +/// the rightmost known one, in a sparse Merkle tree. For example, given the following sparse Merkle tree: +/// +/// ```text +/// root +/// / \ +/// / \ +/// / \ +/// o o +/// / \ / \ +/// a o o h +/// / \ / \ +/// o d e X +/// / \ / \ +/// b c f g +/// ``` +/// +/// if the proof wants show that `[a, b, c, d, e]` exists in the tree, it would need the siblings +/// `X` and `h` on the right. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct SparseMerkleRangeProof { + /// The vector of siblings on the right of the path from root to last leaf. The ones near the + /// bottom are at the beginning of the vector. In the above example, it's `[X, h]`. + right_siblings: Vec, +} + +impl SparseMerkleRangeProof { + /// Constructs a new `SparseMerkleRangeProof`. + pub fn new(right_siblings: Vec) -> Self { + Self { right_siblings } + } + + /// Returns the right siblings. + pub fn right_siblings(&self) -> &[Hash] { + &self.right_siblings + } +} + +// SOURCE: https://github.com/aptos-labs/aptos-core/blob/1.0.4/types/src/proof/mod.rs#L97 +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct SparseMerkleLeafNode { + key: LeafKey, + value_hash: Hash, +} + +impl SparseMerkleLeafNode { + pub fn new(key: LeafKey, value_hash: Hash) -> Self { + SparseMerkleLeafNode { key, value_hash } + } + + pub fn key(&self) -> &LeafKey { + &self.key + } + + pub fn value_hash(&self) -> &Hash { + &self.value_hash + } + + pub fn hash(&self) -> Hash { + hash2(self.key.bytes.as_slice(), self.value_hash.as_slice()) + } +} + +pub struct SparseMerkleInternalNode { + left_child: Hash, + right_child: Hash, +} + +impl SparseMerkleInternalNode { + pub fn new(left_child: Hash, right_child: Hash) -> Self { + Self { + left_child, + right_child, + } + } + + fn hash(&self) -> Hash { + hash2(self.left_child.as_bytes(), self.right_child.as_bytes()) + } +} + +// INITIAL-MODIFICATION: we propagate usage of our own `Hash` (instead of Aptos' `HashValue`) to avoid +// sourcing the entire https://github.com/aptos-labs/aptos-core/blob/1.0.4/crates/aptos-crypto/src/hash.rs +pub const SPARSE_MERKLE_PLACEHOLDER_HASH: Hash = Hash::new([0u8; Hash::byte_size()]); + +// CSOURCE: https://github.com/aptos-labs/aptos-core/blob/1.0.4/crates/aptos-crypto/src/hash.rs#L422 +/// An iterator over `LeafKey` that generates one bit for each iteration. +pub struct LeafKeyBitIterator<'a> { + /// The reference to the bytes that represent the `LeafKey`. + leaf_key_bytes: &'a [u8], + pos: Range, + // invariant pos.end == leaf_key_bytes.len() * 8; +} + +impl<'a> DoubleEndedIterator for LeafKeyBitIterator<'a> { + fn next_back(&mut self) -> Option { + self.pos.next_back().map(|x| self.get_bit(x)) + } +} + +impl<'a> ExactSizeIterator for LeafKeyBitIterator<'a> {} + +impl<'a> LeafKeyBitIterator<'a> { + /// Constructs a new `LeafKeyBitIterator` using given `leaf_key_bytes`. + fn new(leaf_key: LeafKeyRef<'a>) -> Self { + LeafKeyBitIterator { + leaf_key_bytes: leaf_key.bytes, + pos: (0..leaf_key.bytes.len() * 8), + } + } + + /// Returns the `index`-th bit in the bytes. + fn get_bit(&self, index: usize) -> bool { + let pos = index / 8; + let bit = 7 - index % 8; + (self.leaf_key_bytes[pos] >> bit) & 1 != 0 + } +} + +impl<'a> Iterator for LeafKeyBitIterator<'a> { + type Item = bool; + + fn next(&mut self) -> Option { + self.pos.next().map(|x| self.get_bit(x)) + } + + fn size_hint(&self) -> (usize, Option) { + self.pos.size_hint() + } +} + +// INITIAL-MODIFICATION: since we use our own `LeafKey` here, we need it to implement these for it +pub trait IteratedLeafKey { + fn iter_bits(&self) -> LeafKeyBitIterator<'_>; + + fn get_nibble(&self, index: usize) -> Nibble; +} + +impl IteratedLeafKey for LeafKey { + fn iter_bits(&self) -> LeafKeyBitIterator<'_> { + LeafKeyBitIterator::new(self.as_ref()) + } + + fn get_nibble(&self, index: usize) -> Nibble { + Nibble::from(if index % 2 == 0 { + self.bytes[index / 2] >> 4 + } else { + self.bytes[index / 2] & 0x0F + }) + } +} + +impl IteratedLeafKey for LeafKeyRef<'_> { + fn iter_bits(&self) -> LeafKeyBitIterator<'_> { + LeafKeyBitIterator::new(*self) + } + + fn get_nibble(&self, index: usize) -> Nibble { + Nibble::from(if index % 2 == 0 { + self.bytes[index / 2] >> 4 + } else { + self.bytes[index / 2] & 0x0F + }) + } +} + +// SOURCE: https://github.com/aptos-labs/aptos-core/blob/1.0.4/types/src/transaction/mod.rs#L57 +pub type Version = u64; + +// SOURCE: https://github.com/aptos-labs/aptos-core/blob/1.0.4/types/src/nibble/mod.rs#L20 +#[derive(Clone, Copy, Debug, Hash, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)] +#[serde(transparent)] +pub struct Nibble(u8); + +impl From for Nibble { + fn from(nibble: u8) -> Self { + assert!(nibble < 16, "Nibble out of range: {}", nibble); + Self(nibble) + } +} + +impl From for u8 { + fn from(nibble: Nibble) -> Self { + nibble.0 + } +} + +impl fmt::LowerHex for Nibble { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:x}", self.0) + } +} + +// SOURCE: https://github.com/aptos-labs/aptos-core/blob/1.0.4/types/src/nibble/nibble_path/mod.rs#L22 +/// NibblePath defines a path in Merkle tree in the unit of nibble (4 bits). +#[derive(Clone, Hash, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)] +pub struct NibblePath { + /// Indicates the total number of nibbles in bytes. Either `bytes.len() * 2 - 1` or + /// `bytes.len() * 2`. + // Guarantees intended ordering based on the top-to-bottom declaration order of the struct's + // members. + num_nibbles: usize, + /// The underlying bytes that stores the path, 2 nibbles per byte. If the number of nibbles is + /// odd, the second half of the last byte must be 0. + #[serde(with = "serde_with::hex")] + bytes: Vec, +} + +/// Supports debug format by concatenating nibbles literally. For example, [0x12, 0xa0] with 3 +/// nibbles will be printed as "12a". +impl fmt::Debug for NibblePath { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + self.nibbles().try_for_each(|x| write!(f, "{:x}", x)) + } +} + +// INITIAL-MODIFICATION: just to show it in errors +impl fmt::Display for NibblePath { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let hex_chars = self + .bytes + .iter() + .flat_map(|b| [b >> 4, b & 15]) + .map(|b| char::from_digit(u32::from(b), 16).unwrap()) + .take(self.num_nibbles); + + for ch in hex_chars { + write!(f, "{}", ch)?; + } + Ok(()) + } +} + +/// Convert a vector of bytes into `NibblePath` using the lower 4 bits of each byte as nibble. +impl FromIterator for NibblePath { + fn from_iter>(iter: I) -> Self { + let mut nibble_path = NibblePath::new_even(vec![]); + for nibble in iter { + nibble_path.push(nibble); + } + nibble_path + } +} + +impl NibblePath { + /// Creates a new `NibblePath` from a vector of bytes assuming each byte has 2 nibbles. + pub fn new_even(bytes: Vec) -> Self { + let num_nibbles = bytes.len() * 2; + NibblePath { num_nibbles, bytes } + } + + /// Similar to `new()` but asserts that the bytes have one less nibble. + pub fn new_odd(bytes: Vec) -> Self { + assert_eq!( + bytes.last().expect("Should have odd number of nibbles.") & 0x0F, + 0, + "Last nibble must be 0." + ); + let num_nibbles = bytes.len() * 2 - 1; + NibblePath { num_nibbles, bytes } + } + + /// Adds a nibble to the end of the nibble path. + pub fn push(&mut self, nibble: Nibble) { + if self.num_nibbles % 2 == 0 { + self.bytes.push(u8::from(nibble) << 4); + } else { + self.bytes[self.num_nibbles / 2] |= u8::from(nibble); + } + self.num_nibbles += 1; + } + + /// Pops a nibble from the end of the nibble path. + pub fn pop(&mut self) -> Option { + let poped_nibble = if self.num_nibbles % 2 == 0 { + self.bytes.last_mut().map(|last_byte| { + let nibble = *last_byte & 0x0F; + *last_byte &= 0xF0; + Nibble::from(nibble) + }) + } else { + self.bytes.pop().map(|byte| Nibble::from(byte >> 4)) + }; + if poped_nibble.is_some() { + self.num_nibbles -= 1; + } + poped_nibble + } + + /// Returns the last nibble. + pub fn last(&self) -> Option { + let last_byte_option = self.bytes.last(); + if self.num_nibbles % 2 == 0 { + last_byte_option.map(|last_byte| Nibble::from(*last_byte & 0x0F)) + } else { + let last_byte = last_byte_option.expect("Last byte must exist if num_nibbles is odd."); + Some(Nibble::from(*last_byte >> 4)) + } + } + + /// Get the i-th bit. + fn get_bit(&self, i: usize) -> bool { + assert!(i < self.num_nibbles * 4); + let pos = i / 8; + let bit = 7 - i % 8; + ((self.bytes[pos] >> bit) & 1) != 0 + } + + /// Get the i-th nibble. + pub fn get_nibble(&self, i: usize) -> Nibble { + assert!(i < self.num_nibbles); + Nibble::from((self.bytes[i / 2] >> (if i % 2 == 1 { 0 } else { 4 })) & 0xF) + } + + /// Get a bit iterator iterates over the whole nibble path. + pub fn bits(&self) -> BitIterator { + BitIterator { + nibble_path: self, + pos: (0..self.num_nibbles * 4), + } + } + + /// Get a nibble iterator iterates over the whole nibble path. + pub fn nibbles(&self) -> NibbleIterator { + NibbleIterator::new(self, 0, self.num_nibbles) + } + + /// Get the total number of nibbles stored. + pub fn num_nibbles(&self) -> usize { + self.num_nibbles + } + + /// Returns `true` if the nibbles contains no elements. + pub fn is_empty(&self) -> bool { + self.num_nibbles() == 0 + } + + /// Get the underlying bytes storing nibbles. + pub fn bytes(&self) -> &[u8] { + &self.bytes + } + + pub fn into_bytes(self) -> Vec { + self.bytes + } + + pub fn truncate(&mut self, len: usize) { + assert!(len <= self.num_nibbles); + self.num_nibbles = len; + self.bytes.truncate((len + 1) / 2); + if len % 2 != 0 { + *self.bytes.last_mut().expect("must exist.") &= 0xF0; + } + } +} + +pub trait Peekable: Iterator { + /// Returns the `next()` value without advancing the iterator. + fn peek(&self) -> Option; +} + +/// BitIterator iterates a nibble path by bit. +pub struct BitIterator<'a> { + nibble_path: &'a NibblePath, + pos: Range, +} + +impl<'a> Peekable for BitIterator<'a> { + /// Returns the `next()` value without advancing the iterator. + fn peek(&self) -> Option { + if self.pos.start < self.pos.end { + Some(self.nibble_path.get_bit(self.pos.start)) + } else { + None + } + } +} + +/// BitIterator spits out a boolean each time. True/false denotes 1/0. +impl<'a> Iterator for BitIterator<'a> { + type Item = bool; + + fn next(&mut self) -> Option { + self.pos.next().map(|i| self.nibble_path.get_bit(i)) + } +} + +/// Support iterating bits in reversed order. +impl<'a> DoubleEndedIterator for BitIterator<'a> { + fn next_back(&mut self) -> Option { + self.pos.next_back().map(|i| self.nibble_path.get_bit(i)) + } +} + +/// NibbleIterator iterates a nibble path by nibble. +#[derive(Debug)] +pub struct NibbleIterator<'a> { + /// The underlying nibble path that stores the nibbles + nibble_path: &'a NibblePath, + + /// The current index, `pos.start`, will bump by 1 after calling `next()` until `pos.start == + /// pos.end`. + pos: Range, + + /// The start index of the iterator. At the beginning, `pos.start == start`. [start, pos.end) + /// defines the range of `nibble_path` this iterator iterates over. `nibble_path` refers to + /// the entire underlying buffer but the range may only be partial. + start: usize, + // invariant self.start <= self.pos.start; + // invariant self.pos.start <= self.pos.end; +} + +/// NibbleIterator spits out a byte each time. Each byte must be in range [0, 16). +impl<'a> Iterator for NibbleIterator<'a> { + type Item = Nibble; + + fn next(&mut self) -> Option { + self.pos.next().map(|i| self.nibble_path.get_nibble(i)) + } +} + +impl<'a> Peekable for NibbleIterator<'a> { + /// Returns the `next()` value without advancing the iterator. + fn peek(&self) -> Option { + if self.pos.start < self.pos.end { + Some(self.nibble_path.get_nibble(self.pos.start)) + } else { + None + } + } +} + +impl<'a> NibbleIterator<'a> { + fn new(nibble_path: &'a NibblePath, start: usize, end: usize) -> Self { + assert!(start <= end); + Self { + nibble_path, + pos: (start..end), + start, + } + } + + /// Returns a nibble iterator that iterates all visited nibbles. + pub fn visited_nibbles(&self) -> NibbleIterator<'a> { + Self::new(self.nibble_path, self.start, self.pos.start) + } + + /// Returns a nibble iterator that iterates all remaining nibbles. + pub fn remaining_nibbles(&self) -> NibbleIterator<'a> { + Self::new(self.nibble_path, self.pos.start, self.pos.end) + } + + /// Turn it into a `BitIterator`. + pub fn bits(&self) -> BitIterator<'a> { + BitIterator { + nibble_path: self.nibble_path, + pos: (self.pos.start * 4..self.pos.end * 4), + } + } + + /// Cut and return the range of the underlying `nibble_path` that this iterator is iterating + /// over as a new `NibblePath` + pub fn get_nibble_path(&self) -> NibblePath { + self.visited_nibbles().chain(self.remaining_nibbles()).collect() + } + + /// Get the number of nibbles that this iterator covers. + pub fn num_nibbles(&self) -> usize { + assert!(self.start <= self.pos.end); // invariant + self.pos.end - self.start + } + + /// Return `true` if the iteration is over. + pub fn is_finished(&self) -> bool { + self.peek().is_none() + } +} + +// INITIAL-MODIFICATION: We will use this type (instead of `Hash`) to allow for arbitrary key length +/// A leaf key (i.e. a complete nibble path). +#[derive(Clone, Debug, Hash, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)] +pub struct LeafKey { + /// The underlying bytes. + /// All leaf keys of the same tree must be of the same length - otherwise the tree's behavior + /// becomes unspecified. + /// All leaf keys must be evenly distributed across their space - otherwise the tree's + /// performance degrades. + #[serde(with = "serde_with::hex")] + pub bytes: Vec, +} + +impl LeafKey { + pub fn new(bytes: Vec) -> Self { + Self { bytes } + } + + pub fn as_ref(&self) -> LeafKeyRef<'_> { + LeafKeyRef::new(&self.bytes) + } +} + +// INITIAL-MODIFICATION: We will use this type (instead of `Hash`) to allow for arbitrary key length +/// A leaf key (i.e. a complete nibble path). +#[derive(Clone, Copy, Debug, Hash, Eq, PartialEq, Ord, PartialOrd)] +pub struct LeafKeyRef<'a> { + /// The underlying bytes. + /// All leaf keys of the same tree must be of the same length - otherwise the tree's behavior + /// becomes unspecified. + /// All leaf keys must be evenly distributed across their space - otherwise the tree's + /// performance degrades. + pub bytes: &'a [u8], +} + +impl<'a> LeafKeyRef<'a> { + pub fn new(bytes: &'a [u8]) -> Self { + Self { bytes } + } +} + +impl PartialEq for LeafKeyRef<'_> { + fn eq(&self, other: &LeafKey) -> bool { + self.bytes == other.bytes + } +} + +// SOURCE: https://github.com/aptos-labs/aptos-core/blob/1.0.4/storage/jellyfish-merkle/src/node_type/mod.rs#L48 +/// The unique key of each node. +#[derive(Clone, Debug, Hash, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)] +pub struct NodeKey { + /// The version at which the node is created. + version: Version, + /// The nibble path this node represents in the tree. + nibble_path: NibblePath, +} + +impl NodeKey { + /// Creates a new `NodeKey`. + pub fn new(version: Version, nibble_path: NibblePath) -> Self { + Self { version, nibble_path } + } + + /// A shortcut to generate a node key consisting of a version and an empty nibble path. + pub fn new_empty_path(version: Version) -> Self { + Self::new(version, NibblePath::new_even(vec![])) + } + + /// Gets the version. + pub fn version(&self) -> Version { + self.version + } + + /// Gets the nibble path. + pub fn nibble_path(&self) -> &NibblePath { + &self.nibble_path + } + + /// Generates a child node key based on this node key. + pub fn gen_child_node_key(&self, version: Version, n: Nibble) -> Self { + let mut node_nibble_path = self.nibble_path().clone(); + node_nibble_path.push(n); + Self::new(version, node_nibble_path) + } + + /// Generates parent node key at the same version based on this node key. + pub fn gen_parent_node_key(&self) -> Self { + let mut node_nibble_path = self.nibble_path().clone(); + assert!(node_nibble_path.pop().is_some(), "Current node key is root.",); + Self::new(self.version, node_nibble_path) + } +} + +// INITIAL-MODIFICATION: just to show it in errors +impl fmt::Display for NodeKey { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "v{}:{}", self.version, self.nibble_path) + } +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub enum NodeType { + Leaf, + Null, + /// A internal node that haven't been finished the leaf count migration, i.e. None or not all + /// of the children leaf counts are known. + Internal { + leaf_count: usize, + }, +} + +/// Each child of [`InternalNode`] encapsulates a nibble forking at this node. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub struct Child { + /// The hash value of this child node. + #[serde(with = "serde_with::hex")] + pub hash: Hash, + /// `version`, the `nibble_path` of the [`NodeKey`] of this [`InternalNode`] the child belongs + /// to and the child's index constitute the [`NodeKey`] to uniquely identify this child node + /// from the storage. Used by `[`NodeKey::gen_child_node_key`]. + pub version: Version, + /// Indicates if the child is a leaf, or if it's an internal node, the total number of leaves + /// under it (though it can be unknown during migration). + pub node_type: NodeType, +} + +impl Child { + pub fn new(hash: Hash, version: Version, node_type: NodeType) -> Self { + Self { + hash, + version, + node_type, + } + } + + pub fn is_leaf(&self) -> bool { + matches!(self.node_type, NodeType::Leaf) + } + + pub fn leaf_count(&self) -> usize { + match self.node_type { + NodeType::Leaf => 1, + NodeType::Internal { leaf_count } => leaf_count, + NodeType::Null => unreachable!("Child cannot be Null"), + } + } +} + +/// [`Children`] is just a collection of children belonging to a [`InternalNode`], indexed from 0 to +/// 15, inclusive. +pub(crate) type Children = HashMap; + +/// Represents a 4-level subtree with 16 children at the bottom level. Theoretically, this reduces +/// IOPS to query a tree by 4x since we compress 4 levels in a standard Merkle tree into 1 node. +/// Though we choose the same internal node structure as that of Patricia Merkle tree, the root hash +/// computation logic is similar to a 4-level sparse Merkle tree except for some customizations. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub struct InternalNode { + /// Up to 16 children. + children: Children, + /// Total number of leaves under this internal node + leaf_count: usize, +} + +impl InternalNode { + /// Creates a new Internal node. + pub fn new(children: Children) -> Self { + let leaf_count = children.values().map(Child::leaf_count).sum(); + Self { children, leaf_count } + } + + pub fn leaf_count(&self) -> usize { + self.leaf_count + } + + pub fn node_type(&self) -> NodeType { + NodeType::Internal { + leaf_count: self.leaf_count, + } + } + + pub fn hash(&self) -> Hash { + self.merkle_hash( + 0, // start index + 16, // the number of leaves in the subtree of which we want the hash of root + self.generate_bitmaps(), + ) + } + + pub fn children_sorted(&self) -> impl Iterator { + let mut tmp = self.children.iter().collect::>(); + tmp.sort_by_key(|(nibble, _)| **nibble); + tmp.into_iter() + } + + pub fn into_children(self) -> Children { + self.children + } + + /// Gets the `n`-th child. + pub fn child(&self, n: Nibble) -> Option<&Child> { + self.children.get(&n) + } + + /// Generates `existence_bitmap` and `leaf_bitmap` as a pair of `u16`s: child at index `i` + /// exists if `existence_bitmap[i]` is set; child at index `i` is leaf node if + /// `leaf_bitmap[i]` is set. + pub fn generate_bitmaps(&self) -> (u16, u16) { + let mut existence_bitmap = 0; + let mut leaf_bitmap = 0; + for (nibble, child) in &self.children { + let i = u8::from(*nibble); + existence_bitmap |= 1u16 << i; + if child.is_leaf() { + leaf_bitmap |= 1u16 << i; + } + } + // `leaf_bitmap` must be a subset of `existence_bitmap`. + assert_eq!(existence_bitmap | leaf_bitmap, existence_bitmap); + (existence_bitmap, leaf_bitmap) + } + + /// Given a range [start, start + width), returns the sub-bitmap of that range. + fn range_bitmaps(start: u8, width: u8, bitmaps: (u16, u16)) -> (u16, u16) { + assert!(start < 16 && width.count_ones() == 1 && start % width == 0); + assert!(width <= 16 && (start + width) <= 16); + // A range with `start == 8` and `width == 4` will generate a mask 0b0000111100000000. + // use as converting to smaller integer types when 'width == 16' + let mask = (((1u32 << width) - 1) << start) as u16; + (bitmaps.0 & mask, bitmaps.1 & mask) + } + + fn merkle_hash(&self, start: u8, width: u8, (existence_bitmap, leaf_bitmap): (u16, u16)) -> Hash { + // Given a bit [start, 1 << nibble_height], return the value of that range. + let (range_existence_bitmap, range_leaf_bitmap) = + Self::range_bitmaps(start, width, (existence_bitmap, leaf_bitmap)); + if range_existence_bitmap == 0 { + // No child under this subtree + SPARSE_MERKLE_PLACEHOLDER_HASH + } else if width == 1 || (range_existence_bitmap.count_ones() == 1 && range_leaf_bitmap != 0) { + // Only 1 leaf child under this subtree or reach the lowest level + let only_child_index = Nibble::from(range_existence_bitmap.trailing_zeros() as u8); + self.child(only_child_index) + .expect("Corrupted internal node: existence_bitmap inconsistent") + .hash + } else { + let left_child = self.merkle_hash(start, width / 2, (range_existence_bitmap, range_leaf_bitmap)); + let right_child = self.merkle_hash( + start + width / 2, + width / 2, + (range_existence_bitmap, range_leaf_bitmap), + ); + SparseMerkleInternalNode::new(left_child, right_child).hash() + } + } + + fn gen_node_in_proof>( + &self, + start: u8, + width: u8, + (existence_bitmap, leaf_bitmap): (u16, u16), + (tree_reader, node_key): (&R, &NodeKey), + ) -> Result { + // Given a bit [start, 1 << nibble_height], return the value of that range. + let (range_existence_bitmap, range_leaf_bitmap) = + Self::range_bitmaps(start, width, (existence_bitmap, leaf_bitmap)); + Ok(if range_existence_bitmap == 0 { + // No child under this subtree + NodeInProof::Other(SPARSE_MERKLE_PLACEHOLDER_HASH) + } else if width == 1 || (range_existence_bitmap.count_ones() == 1 && range_leaf_bitmap != 0) { + // Only 1 leaf child under this subtree or reach the lowest level + let only_child_index = Nibble::from(range_existence_bitmap.trailing_zeros() as u8); + let only_child = self + .child(only_child_index) + .expect("Corrupted internal node: existence_bitmap inconsistent"); + if matches!(only_child.node_type, NodeType::Leaf) { + let only_child_node_key = node_key.gen_child_node_key(only_child.version, only_child_index); + match tree_reader.get_node(&only_child_node_key)? { + Node::Internal(_) => { + unreachable!("Corrupted internal node: in-memory leaf child is internal node on disk") + }, + Node::Leaf(leaf_node) => NodeInProof::Leaf(SparseMerkleLeafNode::from(leaf_node)), + Node::Null => unreachable!("Child cannot be Null"), + } + } else { + NodeInProof::Other(only_child.hash) + } + } else { + let left_child = self.merkle_hash(start, width / 2, (range_existence_bitmap, range_leaf_bitmap)); + let right_child = self.merkle_hash( + start + width / 2, + width / 2, + (range_existence_bitmap, range_leaf_bitmap), + ); + NodeInProof::Other(SparseMerkleInternalNode::new(left_child, right_child).hash()) + }) + } + + /// Gets the child and its corresponding siblings that are necessary to generate the proof for + /// the `n`-th child. If it is an existence proof, the returned child must be the `n`-th + /// child; otherwise, the returned child may be another child. See inline explanation for + /// details. When calling this function with n = 11 (node `b` in the following graph), the + /// range at each level is illustrated as a pair of square brackets: + /// + /// ```text + /// 4 [f e d c b a 9 8 7 6 5 4 3 2 1 0] -> root level + /// --------------------------------------------------------------- + /// 3 [f e d c b a 9 8] [7 6 5 4 3 2 1 0] width = 8 + /// chs <--┘ shs <--┘ + /// 2 [f e d c] [b a 9 8] [7 6 5 4] [3 2 1 0] width = 4 + /// shs <--┘ └--> chs + /// 1 [f e] [d c] [b a] [9 8] [7 6] [5 4] [3 2] [1 0] width = 2 + /// chs <--┘ └--> shs + /// 0 [f] [e] [d] [c] [b] [a] [9] [8] [7] [6] [5] [4] [3] [2] [1] [0] width = 1 + /// ^ chs <--┘ └--> shs + /// | MSB|<---------------------- uint 16 ---------------------------->|LSB + /// height chs: `child_half_start` shs: `sibling_half_start` + /// ``` + pub fn get_child_with_siblings>( + &self, + node_key: &NodeKey, + n: Nibble, + reader: Option<&R>, + ) -> Result<(Option, Vec), JmtStorageError> { + let mut siblings = vec![]; + let (existence_bitmap, leaf_bitmap) = self.generate_bitmaps(); + + // Nibble height from 3 to 0. + for h in (0..4).rev() { + // Get the number of children of the internal node that each subtree at this height + // covers. + let width = 1 << h; + let (child_half_start, sibling_half_start) = get_child_and_sibling_half_start(n, h); + // Compute the root hash of the subtree rooted at the sibling of `r`. + if let Some(reader) = reader { + siblings.push(self.gen_node_in_proof( + sibling_half_start, + width, + (existence_bitmap, leaf_bitmap), + (reader, node_key), + )?); + } else { + siblings.push( + self.merkle_hash(sibling_half_start, width, (existence_bitmap, leaf_bitmap)) + .into(), + ); + } + + let (range_existence_bitmap, range_leaf_bitmap) = + Self::range_bitmaps(child_half_start, width, (existence_bitmap, leaf_bitmap)); + + if range_existence_bitmap == 0 { + // No child in this range. + return Ok((None, siblings)); + } + + if width == 1 || (range_existence_bitmap.count_ones() == 1 && range_leaf_bitmap != 0) { + // Return the only 1 leaf child under this subtree or reach the lowest level + // Even this leaf child is not the n-th child, it should be returned instead of + // `None` because it's existence indirectly proves the n-th child doesn't exist. + // Please read proof format for details. + let only_child_index = Nibble::from(range_existence_bitmap.trailing_zeros() as u8); + return Ok(( + { + let only_child_version = self + .child(only_child_index) + // Should be guaranteed by the self invariants, but these are not easy to express at the moment + .expect("Corrupted internal node: child_bitmap inconsistent") + .version; + Some(node_key.gen_child_node_key(only_child_version, only_child_index)) + }, + siblings, + )); + } + } + unreachable!("Impossible to get here without returning even at the lowest level.") + } +} + +/// Given a nibble, computes the start position of its `child_half_start` and `sibling_half_start` +/// at `height` level. +pub(crate) fn get_child_and_sibling_half_start(n: Nibble, height: u8) -> (u8, u8) { + // Get the index of the first child belonging to the same subtree whose root, let's say `r` is + // at `height` that the n-th child belongs to. + // Note: `child_half_start` will be always equal to `n` at height 0. + let child_half_start = (0xFF << height) & u8::from(n); + + // Get the index of the first child belonging to the subtree whose root is the sibling of `r` + // at `height`. + let sibling_half_start = child_half_start ^ (1 << height); + + (child_half_start, sibling_half_start) +} + +/// Leaf node, capturing the value hash and carrying an arbitrary payload. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub struct LeafNode

{ + // The key of this leaf node (i.e. its full nibble path). + leaf_key: LeafKey, + // The hash of an externally-stored value. + // Note: do not confuse that value with the `payload`. + #[serde(with = "serde_with::hex")] + value_hash: Hash, + // The client payload. + // This is not the "value" whose changes are tracked by the tree (in fact, these values are + // supposed to be stored externally, and the tree only cares about their hashes - see + // `value_hash`). + // Rather, the payload is an arbitrary piece of data that the client wishes to store within the + // tree, in order to facilitate some related processing: + // - Many clients do not need it and will simply use a no-cost `()`. + // - A use-case designed by the original authors was to store a non-hashed element key as a payload (while the + // `leaf_key` contains that key's hash, to ensure the nibble paths are distributed over their space, for + // performance). + // - Our current use-case (specific to a "two layers" tree) is to store the nested tree's root metadata. + payload: P, + // The version at which this leaf was created. + version: Version, +} + +impl LeafNode

{ + /// Creates a new leaf node. + pub fn new(leaf_key: LeafKey, value_hash: Hash, payload: P, version: Version) -> Self { + Self { + leaf_key, + value_hash, + payload, + version, + } + } + + /// Gets the key. + pub fn leaf_key(&self) -> &LeafKey { + &self.leaf_key + } + + /// Gets the associated value hash. + pub fn value_hash(&self) -> Hash { + self.value_hash + } + + /// Gets the payload. + pub fn payload(&self) -> &P { + &self.payload + } + + /// Gets the version. + pub fn version(&self) -> Version { + self.version + } + + /// Gets the leaf's hash (not to be confused with a `value_hash()`). + /// This hash incorporates the node's key and the value's hash, in order to capture certain + /// changes within a sparse merkle tree (consider 2 trees, both containing a single element with + /// the same value, but stored under different keys - we want their root hashes to differ). + pub fn leaf_hash(&self) -> Hash { + hash2(self.leaf_key.bytes.as_slice(), self.value_hash.as_slice()) + } +} + +impl From> for SparseMerkleLeafNode { + fn from(leaf_node: LeafNode) -> Self { + Self::new(leaf_node.leaf_key, leaf_node.value_hash) + } +} + +/// The concrete node type of [`JellyfishMerkleTree`]. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub enum Node { + /// A wrapper of [`InternalNode`]. + Internal(InternalNode), + /// A wrapper of [`LeafNode`]. + Leaf(LeafNode), + /// Represents empty tree only + Null, +} + +impl From for Node { + fn from(node: InternalNode) -> Self { + Node::Internal(node) + } +} + +impl From> for Node { + fn from(node: LeafNode) -> Self { + Node::Leaf(node) + } +} + +impl Node

{ + /// Creates the [`Internal`](Node::Internal) variant. + #[cfg(any(test, feature = "fuzzing"))] + pub fn new_internal(children: Children) -> Self { + Node::Internal(InternalNode::new(children)) + } + + /// Creates the [`Leaf`](Node::Leaf) variant. + pub fn new_leaf(leaf_key: LeafKey, value_hash: Hash, payload: P, version: Version) -> Self { + Node::Leaf(LeafNode::new(leaf_key, value_hash, payload, version)) + } + + /// Returns `true` if the node is a leaf node. + pub fn is_leaf(&self) -> bool { + matches!(self, Node::Leaf(_)) + } + + /// Returns `NodeType` + pub fn node_type(&self) -> NodeType { + match self { + // The returning value will be used to construct a `Child` of a internal node, while an + // internal node will never have a child of Node::Null. + Self::Leaf(_) => NodeType::Leaf, + Self::Internal(n) => n.node_type(), + Self::Null => NodeType::Null, + } + } + + /// Returns leaf count if known + pub fn leaf_count(&self) -> usize { + match self { + Node::Leaf(_) => 1, + Node::Internal(internal_node) => internal_node.leaf_count, + Node::Null => 0, + } + } + + /// Computes the hash of nodes. + pub fn hash(&self) -> Hash { + match self { + Node::Internal(internal_node) => internal_node.hash(), + Node::Leaf(leaf_node) => leaf_node.leaf_hash(), + Node::Null => SPARSE_MERKLE_PLACEHOLDER_HASH, + } + } +} + +// INITIAL-MODIFICATION: we propagate usage of our own error enum (instead of `std::io::ErrorKind` +// used by Aptos) to allow for no-std build. +/// Error originating from underlying storage failure / inconsistency. +#[derive(Debug, thiserror::Error)] +pub enum JmtStorageError { + #[error("A node {0} expected to exist (according to JMT logic) was not found in the storage")] + NotFound(NodeKey), + + #[error("Nodes read from the storage are violating some JMT property (e.g. form a cycle).")] + InconsistentState, + + #[error("Unexpected error: {0}")] + UnexpectedError(String), +} + +impl IsNotFoundError for JmtStorageError { + fn is_not_found_error(&self) -> bool { + matches!(self, JmtStorageError::NotFound(_)) + } +} diff --git a/dan_layer/state_tree/src/key_mapper.rs b/dan_layer/state_tree/src/key_mapper.rs new file mode 100644 index 0000000000..55f4a72cc7 --- /dev/null +++ b/dan_layer/state_tree/src/key_mapper.rs @@ -0,0 +1,22 @@ +// Copyright 2024 The Tari Project +// SPDX-License-Identifier: BSD-3-Clause + +use tari_engine_types::substate::SubstateId; + +use crate::jellyfish::LeafKey; + +pub trait DbKeyMapper { + fn map_to_leaf_key(id: &SubstateId) -> LeafKey; +} + +const HASH_PREFIX_LENGTH: usize = 20; + +pub struct SpreadPrefixKeyMapper; + +impl DbKeyMapper for SpreadPrefixKeyMapper { + fn map_to_leaf_key(id: &SubstateId) -> LeafKey { + let hash = crate::jellyfish::hash(id.to_canonical_hash()); + let prefixed_key = [&hash[..HASH_PREFIX_LENGTH], hash.as_slice()].concat(); + LeafKey::new(prefixed_key) + } +} diff --git a/dan_layer/state_tree/src/lib.rs b/dan_layer/state_tree/src/lib.rs new file mode 100644 index 0000000000..2e9d254321 --- /dev/null +++ b/dan_layer/state_tree/src/lib.rs @@ -0,0 +1,16 @@ +// Copyright 2024 The Tari Project +// SPDX-License-Identifier: BSD-3-Clause + +mod error; +pub use error::*; + +mod jellyfish; +pub use jellyfish::*; +pub mod key_mapper; +pub mod memory_store; + +mod staged_store; +pub use staged_store::*; + +mod tree; +pub use tree::*; diff --git a/dan_layer/state_tree/src/memory_store.rs b/dan_layer/state_tree/src/memory_store.rs new file mode 100644 index 0000000000..2948cbb5e8 --- /dev/null +++ b/dan_layer/state_tree/src/memory_store.rs @@ -0,0 +1,75 @@ +// Copyright 2024 The Tari Project +// SPDX-License-Identifier: BSD-3-Clause + +use std::{collections::HashMap, fmt}; + +use crate::jellyfish::{ + JmtStorageError, + Node, + NodeKey, + StaleTreeNode, + TreeNode, + TreeStoreReader, + TreeStoreWriter, + Version, +}; + +#[derive(Debug, Default)] +pub struct MemoryTreeStore { + pub nodes: HashMap, + pub stale_nodes: Vec, +} + +impl MemoryTreeStore { + pub fn new() -> Self { + Self { + nodes: HashMap::new(), + stale_nodes: Vec::new(), + } + } + + pub fn clear_stale_nodes(&mut self) { + for stale in self.stale_nodes.drain(..) { + self.nodes.remove(stale.as_node_key()); + } + } +} + +impl TreeStoreReader for MemoryTreeStore { + fn get_node(&self, key: &NodeKey) -> Result, JmtStorageError> { + self.nodes + .get(key) + .map(|node| node.clone().into_node()) + .ok_or_else(|| JmtStorageError::NotFound(key.clone())) + } +} + +impl TreeStoreWriter for MemoryTreeStore { + fn insert_node(&mut self, key: NodeKey, node: Node) -> Result<(), JmtStorageError> { + let node = TreeNode::new_latest(node); + self.nodes.insert(key, node); + Ok(()) + } + + fn record_stale_tree_node(&mut self, stale: StaleTreeNode) -> Result<(), JmtStorageError> { + self.stale_nodes.push(stale); + Ok(()) + } +} + +impl fmt::Display for MemoryTreeStore { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + writeln!(f, "MemoryTreeStore")?; + writeln!(f, " Nodes:")?; + let mut store = self.nodes.iter().collect::>(); + store.sort_by_key(|(key, _)| *key); + for (key, node) in store { + writeln!(f, " {}: {:?}", key, node)?; + } + writeln!(f, " Stale Nodes:")?; + for stale in &self.stale_nodes { + writeln!(f, " {}", stale.as_node_key())?; + } + Ok(()) + } +} diff --git a/dan_layer/state_tree/src/staged_store.rs b/dan_layer/state_tree/src/staged_store.rs new file mode 100644 index 0000000000..cff18d48bc --- /dev/null +++ b/dan_layer/state_tree/src/staged_store.rs @@ -0,0 +1,92 @@ +// Copyright 2024 The Tari Project +// SPDX-License-Identifier: BSD-3-Clause + +// Copyright 2024 The Tari Project +// SPDX-License-Identifier: BSD-3-Clause +// Copyright 2024 The Tari Project +// SPDX-License-Identifier: BSD-3-Clause + +use std::collections::{HashMap, VecDeque}; + +use crate::{ + JmtStorageError, + Node, + NodeKey, + StaleTreeNode, + StateHashTreeDiff, + TreeStoreReader, + TreeStoreWriter, + Version, +}; + +pub struct StagedTreeStore<'s, S> { + readable_store: &'s S, + preceding_pending_state: HashMap>, + new_tree_nodes: HashMap>, + new_stale_nodes: Vec, +} + +impl<'s, S: TreeStoreReader> StagedTreeStore<'s, S> { + pub fn new(readable_store: &'s S) -> Self { + Self { + readable_store, + preceding_pending_state: HashMap::new(), + new_tree_nodes: HashMap::new(), + new_stale_nodes: Vec::new(), + } + } + + pub fn apply_ordered_diffs>(&mut self, diffs: I) { + for (key, node) in diffs.into_iter().flat_map(|diff| diff.new_nodes) { + self.preceding_pending_state.insert(key, node); + } + } + + pub fn into_diff(self) -> StateHashTreeDiff { + StateHashTreeDiff { + new_nodes: self.new_tree_nodes.into_iter().collect(), + stale_tree_nodes: self.new_stale_nodes, + } + } +} + +impl<'s, S: TreeStoreReader> TreeStoreReader for StagedTreeStore<'s, S> { + fn get_node(&self, key: &NodeKey) -> Result, JmtStorageError> { + if let Some(node) = self.new_tree_nodes.get(key).cloned() { + return Ok(node); + } + if let Some(node) = self.preceding_pending_state.get(key).cloned() { + return Ok(node); + } + + self.readable_store.get_node(key) + } +} + +impl<'s, S> TreeStoreWriter for StagedTreeStore<'s, S> { + fn insert_node(&mut self, key: NodeKey, node: Node) -> Result<(), JmtStorageError> { + self.new_tree_nodes.insert(key, node); + Ok(()) + } + + fn record_stale_tree_node(&mut self, stale: StaleTreeNode) -> Result<(), JmtStorageError> { + // Prune staged tree nodes immediately from preceding_pending_state. + let mut remove_queue = VecDeque::new(); + remove_queue.push_front(stale.as_node_key().clone()); + while let Some(key) = remove_queue.pop_front() { + if let Some(node) = self.preceding_pending_state.remove(&key) { + match node { + Node::Internal(node) => { + for (nibble, child) in node.into_children() { + remove_queue.push_back(key.gen_child_node_key(child.version, nibble)); + } + }, + Node::Leaf(_) | Node::Null => {}, + } + } + } + + self.new_stale_nodes.push(stale); + Ok(()) + } +} diff --git a/dan_layer/state_tree/src/tree.rs b/dan_layer/state_tree/src/tree.rs new file mode 100644 index 0000000000..ef7fd11b7a --- /dev/null +++ b/dan_layer/state_tree/src/tree.rs @@ -0,0 +1,165 @@ +// Copyright 2024 The Tari Project +// SPDX-License-Identifier: BSD-3-Clause + +use std::marker::PhantomData; + +use serde::{Deserialize, Serialize}; +use tari_engine_types::substate::SubstateId; + +use crate::{ + error::StateTreeError, + jellyfish::{Hash, JellyfishMerkleTree, LeafKey, SparseMerkleProofExt, TreeStore, Version}, + key_mapper::{DbKeyMapper, SpreadPrefixKeyMapper}, + Node, + NodeKey, + ProofValue, + StaleTreeNode, + TreeStoreReader, + TreeUpdateBatch, +}; + +pub type SpreadPrefixStateTree<'a, S> = StateTree<'a, S, SpreadPrefixKeyMapper>; + +pub struct StateTree<'a, S, M> { + store: &'a mut S, + _mapper: PhantomData, +} + +struct LeafChange { + key: LeafKey, + new_payload: Option<(Hash, Version)>, +} + +impl<'a, S, M> StateTree<'a, S, M> { + pub fn new(store: &'a mut S) -> Self { + Self { + store, + _mapper: PhantomData, + } + } +} + +impl<'a, S: TreeStoreReader, M: DbKeyMapper> StateTree<'a, S, M> { + pub fn get_proof( + &self, + version: Version, + substate_id: &SubstateId, + ) -> Result<(Option>, SparseMerkleProofExt), StateTreeError> { + let smt = JellyfishMerkleTree::new(self.store); + let key = M::map_to_leaf_key(substate_id); + let (maybe_value, proof) = smt.get_with_proof_ext(key.as_ref(), version)?; + Ok((maybe_value, proof)) + } +} + +impl<'a, S: TreeStore, M: DbKeyMapper> StateTree<'a, S, M> { + /// Stores the substate changes in the state tree and returns the new root hash. + pub fn put_substate_changes>( + &mut self, + current_version: Version, + next_version: Version, + changes: I, + ) -> Result { + let (root_hash, update_batch) = calculate_substate_changes::<_, M, _>( + self.store, + Some(current_version).filter(|v| *v > 0), + next_version, + changes, + )?; + + self.commit_diff(update_batch.into())?; + Ok(root_hash) + } + + pub fn commit_diff(&mut self, diff: StateHashTreeDiff) -> Result<(), StateTreeError> { + for (key, node) in diff.new_nodes { + log::debug!("Inserting node: {}", key); + self.store.insert_node(key, node)?; + } + + for stale_tree_node in diff.stale_tree_nodes { + log::debug!("Recording stale tree node: {}", stale_tree_node.as_node_key()); + self.store.record_stale_tree_node(stale_tree_node)?; + } + + Ok(()) + } +} + +/// Calculates the new root hash and tree updates for the given substate changes. +fn calculate_substate_changes, M: DbKeyMapper, I: IntoIterator>( + store: &mut S, + current_version: Option, + next_version: Version, + changes: I, +) -> Result<(Hash, TreeUpdateBatch), StateTreeError> { + let smt = JellyfishMerkleTree::new(store); + + let changes = changes + .into_iter() + .map(|ch| match ch { + SubstateChange::Up { id, value_hash } => LeafChange { + key: M::map_to_leaf_key(&id), + new_payload: Some((value_hash, next_version)), + }, + SubstateChange::Down { id } => LeafChange { + key: M::map_to_leaf_key(&id), + new_payload: None, + }, + }) + .collect::>(); + + let (root_hash, update_result) = smt.batch_put_value_set( + changes + .iter() + .map(|change| (&change.key, change.new_payload.as_ref())) + .collect(), + None, + current_version, + next_version, + )?; + + Ok((root_hash, update_result)) +} + +pub enum SubstateChange { + Up { id: SubstateId, value_hash: Hash }, + Down { id: SubstateId }, +} + +impl SubstateChange { + pub fn id(&self) -> &SubstateId { + match self { + Self::Up { id, .. } => id, + Self::Down { id } => id, + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct StateHashTreeDiff { + pub new_nodes: Vec<(NodeKey, Node)>, + pub stale_tree_nodes: Vec, +} + +impl StateHashTreeDiff { + pub fn new() -> Self { + Self { + new_nodes: Vec::new(), + stale_tree_nodes: Vec::new(), + } + } +} + +impl From> for StateHashTreeDiff { + fn from(batch: TreeUpdateBatch) -> Self { + Self { + new_nodes: batch.node_batch, + stale_tree_nodes: batch + .stale_node_index_batch + .into_iter() + .map(|node| StaleTreeNode::Node(node.node_key)) + .collect(), + } + } +} diff --git a/dan_layer/state_tree/tests/support.rs b/dan_layer/state_tree/tests/support.rs new file mode 100644 index 0000000000..6e299894de --- /dev/null +++ b/dan_layer/state_tree/tests/support.rs @@ -0,0 +1,79 @@ +// Copyright 2024 The Tari Project +// SPDX-License-Identifier: BSD-3-Clause + +use tari_engine_types::{hashing::substate_value_hasher32, substate::SubstateId}; +use tari_state_tree::{ + key_mapper::DbKeyMapper, + memory_store::MemoryTreeStore, + Hash, + LeafKey, + StateTree, + SubstateChange, + TreeStore, + Version, +}; +use tari_template_lib::models::ComponentAddress; + +pub fn change(substate_id_seed: u8, value_seed: Option) -> SubstateChange { + change_exact( + SubstateId::Component(ComponentAddress::from_array([substate_id_seed; 32])), + value_seed.map(from_seed), + ) +} + +fn hash_value(value: &[u8]) -> Hash { + substate_value_hasher32().chain(value).result().into_array().into() +} + +pub fn change_exact(substate_id: SubstateId, value: Option>) -> SubstateChange { + value + .map(|value| SubstateChange::Up { + id: substate_id.clone(), + value_hash: hash_value(&value), + }) + .unwrap_or_else(|| SubstateChange::Down { id: substate_id }) +} + +fn from_seed(node_key_seed: u8) -> Vec { + vec![node_key_seed; node_key_seed as usize] +} + +pub struct HashTreeTester { + pub tree_store: S, + pub current_version: Option, +} + +impl> HashTreeTester { + pub fn new(tree_store: S, current_version: Option) -> Self { + Self { + tree_store, + current_version, + } + } + + pub fn put_substate_changes(&mut self, changes: impl IntoIterator) -> Hash { + self.apply_database_updates(changes) + } + + fn apply_database_updates(&mut self, changes: impl IntoIterator) -> Hash { + let next_version = self.current_version.unwrap_or(0) + 1; + let current_version = self.current_version.replace(next_version).unwrap_or(0); + StateTree::<_, IdentityMapper>::new(&mut self.tree_store) + .put_substate_changes(current_version, next_version, changes) + .unwrap() + } +} + +impl HashTreeTester { + pub fn new_empty() -> Self { + Self::new(MemoryTreeStore::new(), None) + } +} + +pub struct IdentityMapper; + +impl DbKeyMapper for IdentityMapper { + fn map_to_leaf_key(id: &SubstateId) -> LeafKey { + LeafKey::new(id.to_canonical_hash().to_vec()) + } +} diff --git a/dan_layer/state_tree/tests/test.rs b/dan_layer/state_tree/tests/test.rs new file mode 100644 index 0000000000..c038d736ff --- /dev/null +++ b/dan_layer/state_tree/tests/test.rs @@ -0,0 +1,161 @@ +// Copyright 2024 The Tari Project +// SPDX-License-Identifier: BSD-3-Clause +// Adapted from https://github.com/radixdlt/radixdlt-scrypto/blob/868ba44ec3b806992864af27c706968c797eb961/radix-engine-stores/src/hash_tree/test.rs + +use std::collections::HashSet; + +use itertools::Itertools; +use tari_state_tree::{memory_store::MemoryTreeStore, StaleTreeNode, Version, SPARSE_MERKLE_PLACEHOLDER_HASH}; + +use crate::support::{change, HashTreeTester}; +mod support; + +#[test] +fn hash_of_next_version_differs_when_value_changed() { + let mut tester = HashTreeTester::new_empty(); + let hash_v1 = tester.put_substate_changes(vec![change(1, Some(30))]); + let hash_v2 = tester.put_substate_changes(vec![change(1, Some(70))]); + assert_ne!(hash_v1, hash_v2); +} + +#[test] +fn hash_of_next_version_same_when_write_repeated() { + let mut tester = HashTreeTester::new_empty(); + let hash_v1 = tester.put_substate_changes(vec![change(4, Some(30)), change(3, Some(40))]); + let hash_v2 = tester.put_substate_changes(vec![change(4, Some(30))]); + assert_eq!(hash_v1, hash_v2); +} + +#[test] +fn hash_of_next_version_same_when_write_empty() { + let mut tester = HashTreeTester::new_empty(); + let hash_v1 = tester.put_substate_changes(vec![change(1, Some(30)), change(3, Some(40))]); + let hash_v2 = tester.put_substate_changes(vec![]); + assert_eq!(hash_v1, hash_v2); +} + +#[test] +fn hash_of_next_version_differs_when_entry_added() { + let mut tester = HashTreeTester::new_empty(); + let hash_v1 = tester.put_substate_changes(vec![change(1, Some(30))]); + let hash_v2 = tester.put_substate_changes(vec![change(2, Some(30))]); + assert_ne!(hash_v1, hash_v2); +} + +#[test] +fn hash_of_next_version_differs_when_entry_removed() { + let mut tester = HashTreeTester::new_empty(); + let hash_v1 = tester.put_substate_changes(vec![change(1, Some(30)), change(4, Some(20))]); + let hash_v2 = tester.put_substate_changes(vec![change(1, None)]); + assert_ne!(hash_v1, hash_v2); +} + +#[test] +fn hash_returns_to_same_when_previous_state_restored() { + let mut tester = HashTreeTester::new_empty(); + let hash_v1 = tester.put_substate_changes(vec![change(1, Some(30)), change(2, Some(40))]); + tester.put_substate_changes(vec![change(1, Some(90)), change(2, None), change(3, Some(10))]); + let hash_v3 = tester.put_substate_changes(vec![change(1, Some(30)), change(2, Some(40)), change(3, None)]); + assert_eq!(hash_v1, hash_v3); +} + +#[test] +fn hash_computed_consistently_after_higher_tier_leafs_deleted() { + // Compute a "reference" hash of state containing simply [2:3:4, 2:3:5]. + let mut reference_tester = HashTreeTester::new_empty(); + let reference_root = reference_tester.put_substate_changes(vec![change(1, Some(234)), change(2, Some(235))]); + + // Compute a hash of the same state, at which we arrive after deleting some unrelated NodeId. + let mut tester = HashTreeTester::new_empty(); + tester.put_substate_changes(vec![change(3, Some(162)), change(4, Some(163)), change(1, Some(234))]); + tester.put_substate_changes(vec![change(3, None), change(4, None)]); + let root_after_deletes = tester.put_substate_changes(vec![change(2, Some(235))]); + + // We did [3,4,1] - [3,4] + [2] = [1,2] (i.e. same state). + assert_eq!(root_after_deletes, reference_root); +} + +#[test] +fn hash_computed_consistently_after_adding_higher_tier_sibling() { + // Compute a "reference" hash of state containing simply [1,2,3]. + let mut reference_tester = HashTreeTester::new_empty(); + let reference_root = + reference_tester.put_substate_changes(vec![change(1, Some(196)), change(2, Some(234)), change(3, Some(235))]); + + // Compute a hash of the same state, at which we arrive after adding some sibling NodeId. + let mut tester = HashTreeTester::new_empty(); + tester.put_substate_changes(vec![change(2, Some(234))]); + tester.put_substate_changes(vec![change(1, Some(196))]); + let root_after_adding_sibling = tester.put_substate_changes(vec![change(3, Some(235))]); + + // We did [2] + [1] + [3] = [1,2,3] (i.e. same state). + assert_eq!(root_after_adding_sibling, reference_root); +} + +#[test] +fn hash_differs_when_states_only_differ_by_node_key() { + let mut tester_1 = HashTreeTester::new_empty(); + let hash_1 = tester_1.put_substate_changes(vec![change(1, Some(30))]); + let mut tester_2 = HashTreeTester::new_empty(); + let hash_2 = tester_2.put_substate_changes(vec![change(2, Some(30))]); + assert_ne!(hash_1, hash_2); +} + +#[test] +fn hash_differs_when_states_only_differ_by_value() { + let mut tester_1 = HashTreeTester::new_empty(); + let hash_1 = tester_1.put_substate_changes(vec![change(1, Some(1))]); + let mut tester_2 = HashTreeTester::new_empty(); + let hash_2 = tester_2.put_substate_changes(vec![change(1, Some(2))]); + assert_ne!(hash_1, hash_2); +} + +#[test] +fn supports_empty_state() { + let mut tester = HashTreeTester::new_empty(); + let hash_v1 = tester.put_substate_changes(vec![]); + assert_eq!(hash_v1, SPARSE_MERKLE_PLACEHOLDER_HASH); + let hash_v2 = tester.put_substate_changes(vec![change(1, Some(30))]); + assert_ne!(hash_v2, SPARSE_MERKLE_PLACEHOLDER_HASH); + let hash_v3 = tester.put_substate_changes(vec![change(1, None)]); + assert_eq!(hash_v3, SPARSE_MERKLE_PLACEHOLDER_HASH); +} + +#[test] +fn records_stale_tree_node_keys() { + let mut tester = HashTreeTester::new_empty(); + tester.put_substate_changes(vec![change(4, Some(30))]); + tester.put_substate_changes(vec![change(3, Some(70))]); + tester.put_substate_changes(vec![change(3, Some(80))]); + let stale_versions = tester + .tree_store + .stale_nodes + .iter() + .map(|stale_part| { + let StaleTreeNode::Node(key) = stale_part else { + panic!("expected only single node removals"); + }; + key.version() + }) + .unique() + .sorted() + .collect::>(); + assert_eq!(stale_versions, vec![1, 2]); +} + +#[test] +fn serialized_keys_are_strictly_increasing() { + let mut tester = HashTreeTester::new(MemoryTreeStore::new(), None); + tester.put_substate_changes(vec![change(3, Some(90))]); + let previous_keys = tester.tree_store.nodes.keys().cloned().collect::>(); + tester.put_substate_changes(vec![change(1, Some(80))]); + let min_next_key = tester + .tree_store + .nodes + .keys() + .filter(|key| !previous_keys.contains(*key)) + .max() + .unwrap(); + let max_previous_key = previous_keys.iter().max().unwrap(); + assert!(min_next_key > max_previous_key); +} diff --git a/dan_layer/storage/Cargo.toml b/dan_layer/storage/Cargo.toml index 5d698fcfdd..5b2fb31b41 100644 --- a/dan_layer/storage/Cargo.toml +++ b/dan_layer/storage/Cargo.toml @@ -19,6 +19,7 @@ tari_transaction = { workspace = true } tari_core = { workspace = true, default-features = true } tari_mmr = { workspace = true } tari_crypto = { workspace = true } +tari_state_tree = { workspace = true } anyhow = { workspace = true } chrono = { workspace = true } diff --git a/dan_layer/storage/src/consensus_models/block.rs b/dan_layer/storage/src/consensus_models/block.rs index e32862afb4..00dc1b851c 100644 --- a/dan_layer/storage/src/consensus_models/block.rs +++ b/dan_layer/storage/src/consensus_models/block.rs @@ -19,15 +19,23 @@ use tari_dan_common_types::{ serde_with, shard::Shard, Epoch, + NodeAddressable, NodeHeight, SubstateAddress, }; +use tari_engine_types::substate::SubstateDiff; use tari_transaction::TransactionId; use time::PrimitiveDateTime; #[cfg(feature = "ts")] use ts_rs::TS; -use super::{ForeignProposal, ForeignSendCounters, QuorumCertificate, ValidatorSchnorrSignature}; +use super::{ + ForeignProposal, + ForeignSendCounters, + QuorumCertificate, + SubstateDestroyedProof, + ValidatorSchnorrSignature, +}; use crate::{ consensus_models::{ Command, @@ -100,6 +108,7 @@ impl Block { epoch: Epoch, proposed_by: PublicKey, commands: BTreeSet, + merkle_root: FixedHash, total_leader_fee: u64, sorted_foreign_indexes: IndexMap, signature: Option, @@ -112,8 +121,7 @@ impl Block { height, epoch, proposed_by, - // TODO - merkle_root: FixedHash::zero(), + merkle_root, commands, total_leader_fee, is_dummy: false, @@ -136,6 +144,7 @@ impl Block { epoch: Epoch, proposed_by: PublicKey, commands: BTreeSet, + merkle_root: FixedHash, total_leader_fee: u64, is_dummy: bool, is_processed: bool, @@ -152,8 +161,7 @@ impl Block { height, epoch, proposed_by, - // TODO - merkle_root: FixedHash::zero(), + merkle_root, commands, total_leader_fee, is_dummy, @@ -174,6 +182,7 @@ impl Block { Epoch(0), PublicKey::default(), Default::default(), + FixedHash::zero(), 0, IndexMap::new(), None, @@ -209,6 +218,7 @@ impl Block { node_height: NodeHeight, high_qc: QuorumCertificate, epoch: Epoch, + parent_merkle_root: FixedHash, ) -> Self { let mut block = Self::new( network, @@ -218,6 +228,7 @@ impl Block { epoch, proposed_by, Default::default(), + parent_merkle_root, 0, IndexMap::new(), None, @@ -369,6 +380,10 @@ impl Block { pub fn set_signature(&mut self, signature: ValidatorSchnorrSignature) { self.signature = Some(signature); } + + pub fn is_proposed_by_addr>(&self, address: &A) -> Option { + Some(A::try_from_public_key(&self.proposed_by)? == *address) + } } impl Block { @@ -380,6 +395,7 @@ impl Block { tx.blocks_get_tip() } + /// Returns all blocks from and excluding the start block (lower height) to the end block (inclusive) pub fn get_all_blocks_between( tx: &mut TTx, start_block_id_exclusive: &BlockId, @@ -575,11 +591,12 @@ impl Block { substate: substate.into(), })); } else { - updates.push(SubstateUpdate::Destroy { - address: substate.to_substate_address(), - proof: QuorumCertificate::get(tx, &destroyed.justify)?, + updates.push(SubstateUpdate::Destroy(SubstateDestroyedProof { + substate_id: substate.substate_id.clone(), + version: substate.version, + justify: QuorumCertificate::get(tx, &destroyed.justify)?, destroyed_by_transaction: destroyed.by_transaction, - }); + })); } } else { updates.push(SubstateUpdate::Create(SubstateCreatedProof { @@ -618,10 +635,12 @@ impl Block { return Ok(high_qc); }; - let locked = LockedBlock::get(tx.deref_mut())?; - if precommit_node.height() > locked.height { - on_locked_block_recurse(tx, &locked, &precommit_node, &mut on_lock_block)?; - precommit_node.as_locked_block().set(tx)?; + if !precommit_node.is_genesis() { + let locked = LockedBlock::get(tx.deref_mut())?; + if precommit_node.height() > locked.height { + on_locked_block_recurse(tx, &locked, &precommit_node, &mut on_lock_block)?; + precommit_node.as_locked_block().set(tx)?; + } } // b <- b'.justify.node @@ -638,10 +657,12 @@ impl Block { ); // Commit prepare_node (b) - let prepare_node = Block::get(tx.deref_mut(), prepare_node)?; - let last_executed = LastExecuted::get(tx.deref_mut())?; - on_commit_block_recurse(tx, &last_executed, &prepare_node, &mut on_commit)?; - prepare_node.as_last_executed().set(tx)?; + if !prepare_node.is_genesis() { + let prepare_node = Block::get(tx.deref_mut(), prepare_node)?; + let last_executed = LastExecuted::get(tx.deref_mut())?; + on_commit_block_recurse(tx, &last_executed, &prepare_node, &mut on_commit)?; + prepare_node.as_last_executed().set(tx)?; + } } else { debug!( target: LOG_TARGET, @@ -703,6 +724,26 @@ impl Block { } Ok(()) } + + pub fn get_all_substate_diffs( + &self, + tx: &mut TTx, + ) -> Result, StorageError> { + let transactions = self + .commands() + .iter() + .filter_map(|c| c.accept()) + .filter(|t| t.decision.is_commit()) + .map(|t| tx.transactions_get(t.id())) + .collect::, _>>()?; + + Ok(transactions + .into_iter() + // TODO: following two should never be None + .filter_map(|t_rec| t_rec.result) + .filter_map(|t_res| t_res.finalize.into_accept()) + .collect()) + } } impl Display for Block { diff --git a/dan_layer/storage/src/consensus_models/mod.rs b/dan_layer/storage/src/consensus_models/mod.rs index fcf9af7dca..d6d6e7783e 100644 --- a/dan_layer/storage/src/consensus_models/mod.rs +++ b/dan_layer/storage/src/consensus_models/mod.rs @@ -17,6 +17,7 @@ mod locked_block; mod locked_output; mod quorum; mod quorum_certificate; +mod state_tree_diff; mod substate; mod transaction; mod transaction_decision; @@ -42,6 +43,7 @@ pub use locked_block::*; pub use locked_output::*; pub use quorum::*; pub use quorum_certificate::*; +pub use state_tree_diff::*; pub use substate::*; pub use transaction::*; pub use transaction_decision::*; diff --git a/dan_layer/storage/src/consensus_models/state_tree_diff.rs b/dan_layer/storage/src/consensus_models/state_tree_diff.rs new file mode 100644 index 0000000000..a6de5746ea --- /dev/null +++ b/dan_layer/storage/src/consensus_models/state_tree_diff.rs @@ -0,0 +1,60 @@ +// Copyright 2024 The Tari Project +// SPDX-License-Identifier: BSD-3-Clause + +// Copyright 2023 The Tari Project +// SPDX-License-Identifier: BSD-3-Clause + +use std::ops::DerefMut; + +use tari_dan_common_types::NodeHeight; + +use crate::{consensus_models::BlockId, StateStoreReadTransaction, StateStoreWriteTransaction, StorageError}; + +#[derive(Debug, Clone)] +pub struct PendingStateTreeDiff { + pub block_id: BlockId, + pub block_height: NodeHeight, + pub diff: tari_state_tree::StateHashTreeDiff, +} + +impl PendingStateTreeDiff { + pub fn new(block_id: BlockId, block_height: NodeHeight, diff: tari_state_tree::StateHashTreeDiff) -> Self { + Self { + block_id, + block_height, + diff, + } + } +} + +impl PendingStateTreeDiff { + /// Returns all pending state tree diffs from the last committed block (exclusive) to the given block (inclusive). + pub fn get_all_up_to_commit_block(tx: &mut TTx, block_id: &BlockId) -> Result, StorageError> + where TTx: StateStoreReadTransaction { + tx.pending_state_tree_diffs_get_all_up_to_commit_block(block_id) + } + + pub fn remove_by_block(tx: &mut TTx, block_id: &BlockId) -> Result + where + TTx: DerefMut + StateStoreWriteTransaction, + TTx::Target: StateStoreReadTransaction, + { + tx.pending_state_tree_diffs_remove_by_block(block_id) + } + + pub fn save(&self, tx: &mut TTx) -> Result + where + TTx: DerefMut + StateStoreWriteTransaction, + TTx::Target: StateStoreReadTransaction, + { + if tx + .deref_mut() + .pending_state_tree_diffs_exists_for_block(&self.block_id)? + { + Ok(false) + } else { + tx.pending_state_tree_diffs_insert(self)?; + Ok(true) + } + } +} diff --git a/dan_layer/storage/src/consensus_models/substate.rs b/dan_layer/storage/src/consensus_models/substate.rs index 10fb997822..b15705c33c 100644 --- a/dan_layer/storage/src/consensus_models/substate.rs +++ b/dan_layer/storage/src/consensus_models/substate.rs @@ -284,6 +284,14 @@ pub struct SubstateCreatedProof { pub created_qc: QuorumCertificate, } +#[derive(Debug, Clone)] +pub struct SubstateDestroyedProof { + pub substate_id: SubstateId, + pub version: u32, + pub justify: QuorumCertificate, + pub destroyed_by_transaction: TransactionId, +} + #[derive(Debug, Clone)] pub struct SubstateData { pub substate_id: SubstateId, @@ -292,6 +300,12 @@ pub struct SubstateData { pub created_by_transaction: TransactionId, } +impl SubstateData { + pub fn into_substate(self) -> Substate { + Substate::new(self.version, self.substate_value) + } +} + impl From for SubstateData { fn from(value: SubstateRecord) -> Self { Self { @@ -306,11 +320,7 @@ impl From for SubstateData { #[derive(Debug, Clone)] pub enum SubstateUpdate { Create(SubstateCreatedProof), - Destroy { - address: SubstateAddress, - proof: QuorumCertificate, - destroyed_by_transaction: TransactionId, - }, + Destroy(SubstateDestroyedProof), } impl SubstateUpdate { @@ -351,21 +361,23 @@ impl SubstateUpdate { } .create(tx)?; }, - Self::Destroy { - address, - proof, + Self::Destroy(SubstateDestroyedProof { + substate_id, + version, + justify: proof, destroyed_by_transaction, - } => { + }) => { debug!( target: LOG_TARGET, - "🔥 Applying substate DESTROY for shard {} (transaction {})", - address, + "🔥 Applying substate DESTROY for substate {}v{} (transaction {})", + substate_id, + version, destroyed_by_transaction ); proof.save(tx)?; SubstateRecord::destroy_many( tx, - iter::once(address), + iter::once(SubstateAddress::from_address(&substate_id, version)), block.epoch(), block.id(), proof.id(), diff --git a/dan_layer/storage/src/consensus_models/transaction_pool.rs b/dan_layer/storage/src/consensus_models/transaction_pool.rs index c65ac0bc25..5f65577cc4 100644 --- a/dan_layer/storage/src/consensus_models/transaction_pool.rs +++ b/dan_layer/storage/src/consensus_models/transaction_pool.rs @@ -17,7 +17,15 @@ use tari_dan_common_types::{ use tari_transaction::TransactionId; use crate::{ - consensus_models::{Decision, LeafBlock, LockedBlock, QcId, TransactionAtom, TransactionPoolStatusUpdate}, + consensus_models::{ + Decision, + LeafBlock, + LockedBlock, + QcId, + TransactionAtom, + TransactionPoolStatusUpdate, + TransactionRecord, + }, StateStore, StateStoreReadTransaction, StateStoreWriteTransaction, @@ -448,6 +456,14 @@ impl TransactionPoolRecord { } Ok(()) } + + pub fn get_transaction( + &self, + tx: &mut TTx, + ) -> Result { + let transaction = TransactionRecord::get(tx, self.transaction_id())?; + Ok(transaction) + } } #[derive(Debug, thiserror::Error)] diff --git a/dan_layer/storage/src/consensus_models/validated_block.rs b/dan_layer/storage/src/consensus_models/validated_block.rs index f471691e0d..d82dbee1ba 100644 --- a/dan_layer/storage/src/consensus_models/validated_block.rs +++ b/dan_layer/storage/src/consensus_models/validated_block.rs @@ -7,7 +7,7 @@ use tari_common_types::types::PublicKey; use tari_dan_common_types::{Epoch, NodeHeight}; use crate::{ - consensus_models::{Block, BlockId}, + consensus_models::{Block, BlockId, QuorumCertificate}, StateStoreReadTransaction, StateStoreWriteTransaction, StorageError, @@ -50,6 +50,10 @@ impl ValidBlock { self.block.proposed_by() } + pub fn justify(&self) -> &QuorumCertificate { + self.block.justify() + } + pub fn dummy_blocks(&self) -> &[Block] { &self.dummy_blocks } diff --git a/dan_layer/storage/src/shard_store.rs b/dan_layer/storage/src/shard_store.rs deleted file mode 100644 index 23089153ee..0000000000 --- a/dan_layer/storage/src/shard_store.rs +++ /dev/null @@ -1,256 +0,0 @@ -// Copyright 2022. The Tari Project -// -// Redistribution and use in source and binary forms, with or without modification, are permitted provided that the -// following conditions are met: -// -// 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following -// disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the -// following disclaimer in the documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote -// products derived from this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, -// INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, -// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE -// USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -use std::ops::{Deref, DerefMut}; - -use tari_dan_common_types::{ - NodeAddressable, - NodeHeight, - ObjectPledge, - ObjectPledgeInfo, - PayloadId, - QuorumCertificate, - SubstateAddress, - SubstateState, - TreeNodeHash, -}; -use tari_engine_types::substate::{Substate, SubstateId}; - -use crate::{ - models::{ - ClaimLeaderFees, - CurrentLeaderStates, - HotStuffTreeNode, - LeafNode, - Payload, - PayloadResult, - RecentTransaction, - SQLSubstate, - SQLTransaction, - SubstateShardData, - VoteMessage, - }, - StorageError, -}; - -const LOG_TARGET: &str = "tari::dan::storage"; - -pub trait ShardStore { - type Addr: NodeAddressable; - type Payload: Payload; - - type ReadTransaction<'a>: ShardStoreReadTransaction - where Self: 'a; - type WriteTransaction<'a>: ShardStoreWriteTransaction - + Deref> - + DerefMut - where Self: 'a; - - fn create_read_tx(&self) -> Result, StorageError>; - fn create_write_tx(&self) -> Result, StorageError>; - - fn with_write_tx) -> Result, R, E>(&self, f: F) -> Result - where E: From { - let mut tx = self.create_write_tx()?; - match f(&mut tx) { - Ok(r) => { - tx.commit()?; - Ok(r) - }, - Err(e) => { - if let Err(err) = tx.rollback() { - log::error!(target: LOG_TARGET, "Failed to rollback transaction: {}", err); - } - Err(e) - }, - } - } - - fn with_read_tx) -> Result, R, E>(&self, f: F) -> Result - where E: From { - let mut tx = self.create_read_tx()?; - let ret = f(&mut tx)?; - Ok(ret) - } -} - -pub trait ShardStoreReadTransaction { - fn get_high_qc_for( - &mut self, - payload_id: PayloadId, - address: SubstateAddress, - ) -> Result, StorageError>; - fn get_high_qcs(&mut self, payload_id: PayloadId) -> Result>, StorageError>; - /// Returns the current leaf node for the substate address - fn get_leaf_node(&mut self, payload_id: &PayloadId, address: &SubstateAddress) -> Result; - fn get_current_leaders_states(&mut self, payload: &PayloadId) -> Result, StorageError>; - fn get_payload(&mut self, payload_id: &PayloadId) -> Result; - fn get_node(&mut self, node_hash: &TreeNodeHash) -> Result, StorageError>; - fn get_locked_node_hash_and_height( - &mut self, - payload_id: PayloadId, - address: SubstateAddress, - ) -> Result<(TreeNodeHash, NodeHeight), StorageError>; - fn get_last_executed_height(&mut self, address: SubstateAddress, payload_id: PayloadId) -> Result; - fn get_state_inventory(&mut self) -> Result, StorageError>; - fn get_substate_states(&mut self, addresses: &[SubstateAddress]) -> Result, StorageError>; - fn get_substate_states_by_range( - &mut self, - start_address: SubstateAddress, - end_address: SubstateAddress, - excluded_addresses: &[SubstateAddress], - ) -> Result, StorageError>; - /// Returns the last voted height. A height of 0 means that no previous vote height has been recorded for the - /// pair. - fn get_last_voted_height( - &mut self, - address: SubstateAddress, - payload_id: PayloadId, - ) -> Result<(NodeHeight, u32), StorageError>; - fn get_leader_proposals( - &mut self, - payload: PayloadId, - payload_height: NodeHeight, - addresses: &[SubstateAddress], - ) -> Result>, StorageError>; - fn get_last_payload_height_for_leader_proposal( - &mut self, - payload: PayloadId, - address: SubstateAddress, - ) -> Result; - fn has_vote_for(&mut self, from: &TAddr, node_hash: TreeNodeHash) -> Result; - fn get_received_votes_for(&mut self, node_hash: TreeNodeHash) -> Result, StorageError>; - fn get_recent_transactions(&mut self) -> Result, StorageError>; - fn get_transaction(&mut self, payload_id: Vec) -> Result, StorageError>; - fn get_substates_for_payload( - &mut self, - payload_id: Vec, - address: Vec, - ) -> Result, StorageError>; - fn get_fees_by_epoch( - &mut self, - epoch: u64, - claim_leader_public_key: Vec, - ) -> Result, StorageError>; - fn get_payload_result(&mut self, payload_id: &PayloadId) -> Result; - fn get_resolved_pledges_for_payload(&mut self, payload: PayloadId) -> Result, StorageError>; -} - -pub trait ShardStoreWriteTransaction { - fn commit(self) -> Result<(), StorageError>; - fn rollback(self) -> Result<(), StorageError>; - fn insert_high_qc(&mut self, from: TAddr, address: SubstateAddress, qc: QuorumCertificate) - -> Result<(), StorageError>; - fn save_payload(&mut self, payload: TPayload) -> Result<(), StorageError>; - fn save_current_leader_state( - &mut self, - payload: PayloadId, - address: SubstateAddress, - leader_round: u32, - leader: TAddr, - ) -> Result<(), StorageError>; - /// Inserts or updates the leaf node for the substate address - fn set_leaf_node( - &mut self, - payload_id: PayloadId, - address: SubstateAddress, - node: TreeNodeHash, - payload_height: NodeHeight, - height: NodeHeight, - ) -> Result<(), StorageError>; - fn save_node(&mut self, node: HotStuffTreeNode) -> Result<(), StorageError>; - fn set_locked( - &mut self, - payload_id: PayloadId, - address: SubstateAddress, - node_hash: TreeNodeHash, - node_height: NodeHeight, - ) -> Result<(), StorageError>; - - fn set_last_executed_height( - &mut self, - address: SubstateAddress, - payload_id: PayloadId, - height: NodeHeight, - ) -> Result<(), StorageError>; - fn commit_substate_changes( - &mut self, - node: HotStuffTreeNode, - changes: &[SubstateState], - ) -> Result<(), StorageError>; - fn insert_substates(&mut self, substate_data: SubstateShardData) -> Result<(), StorageError>; - fn set_last_voted_height( - &mut self, - address: SubstateAddress, - payload_id: PayloadId, - height: NodeHeight, - leader_round: u32, - ) -> Result<(), StorageError>; - - fn save_leader_proposals( - &mut self, - address: SubstateAddress, - payload: PayloadId, - payload_height: NodeHeight, - leader_round: u32, - node: HotStuffTreeNode, - ) -> Result<(), StorageError>; - - fn save_received_vote_for( - &mut self, - from: TAddr, - node_hash: TreeNodeHash, - vote_message: VoteMessage, - ) -> Result<(), StorageError>; - - /// Updates the result for an existing payload - fn update_payload_result(&mut self, payload_id: &PayloadId, result: PayloadResult) -> Result<(), StorageError>; - - fn mark_payload_finalized(&mut self, payload_id: &PayloadId) -> Result<(), StorageError>; - - // -------------------------------- Pledges -------------------------------- // - fn pledge_object( - &mut self, - address: SubstateAddress, - payload: PayloadId, - current_height: NodeHeight, - ) -> Result; - fn complete_pledges( - &mut self, - address: SubstateAddress, - payload_id: PayloadId, - node_hash: &TreeNodeHash, - ) -> Result<(), StorageError>; - fn abandon_pledges( - &mut self, - address: SubstateAddress, - payload_id: PayloadId, - node_hash: &TreeNodeHash, - ) -> Result<(), StorageError>; - - fn save_burnt_utxo( - &mut self, - substate: &Substate, - commitment_address: SubstateId, - address: SubstateAddress, - ) -> Result<(), StorageError>; -} diff --git a/dan_layer/storage/src/state_store/mod.rs b/dan_layer/storage/src/state_store/mod.rs index 979b8a00e2..368ca6aa5a 100644 --- a/dan_layer/storage/src/state_store/mod.rs +++ b/dan_layer/storage/src/state_store/mod.rs @@ -4,12 +4,13 @@ use std::{ borrow::Borrow, collections::HashSet, - ops::{Deref, DerefMut, RangeInclusive}, + ops::{DerefMut, RangeInclusive}, }; use serde::{Deserialize, Serialize}; use tari_common_types::types::{FixedHash, PublicKey}; use tari_dan_common_types::{Epoch, NodeAddressable, NodeHeight, SubstateAddress}; +use tari_state_tree::{TreeStore, TreeStoreReader, Version}; use tari_transaction::{Transaction, TransactionId}; #[cfg(feature = "ts")] use ts_rs::TS; @@ -31,6 +32,7 @@ use crate::{ LeafBlock, LockedBlock, LockedOutput, + PendingStateTreeDiff, QcId, QuorumCertificate, SubstateLockFlag, @@ -50,11 +52,11 @@ const LOG_TARGET: &str = "tari::dan::storage"; pub trait StateStore { type Addr: NodeAddressable; - type ReadTransaction<'a>: StateStoreReadTransaction + type ReadTransaction<'a>: StateStoreReadTransaction + TreeStoreReader where Self: 'a; type WriteTransaction<'a>: StateStoreWriteTransaction - + Deref> - + DerefMut + + TreeStore + + DerefMut> where Self: 'a; fn create_read_tx(&self) -> Result, StorageError>; @@ -87,7 +89,6 @@ pub trait StateStore { pub trait StateStoreReadTransaction { type Addr: NodeAddressable; - fn last_sent_vote_get(&mut self) -> Result; fn last_voted_get(&mut self) -> Result; fn last_executed_get(&mut self) -> Result; @@ -123,6 +124,7 @@ pub trait StateStoreReadTransaction { ) -> Result, StorageError>; fn blocks_get(&mut self, block_id: &BlockId) -> Result; fn blocks_get_tip(&mut self) -> Result; + /// Returns all blocks from and excluding the start block (lower height) to the end block (inclusive) fn blocks_get_all_between( &mut self, start_block_id_exclusive: &BlockId, @@ -237,6 +239,12 @@ pub trait StateStoreReadTransaction { where I: IntoIterator, B: Borrow; + + fn pending_state_tree_diffs_exists_for_block(&mut self, block_id: &BlockId) -> Result; + fn pending_state_tree_diffs_get_all_up_to_commit_block( + &mut self, + block_id: &BlockId, + ) -> Result, StorageError>; } pub trait StateStoreWriteTransaction { @@ -373,6 +381,12 @@ pub trait StateStoreWriteTransaction { where I: IntoIterator, B: Borrow; + // -------------------------------- Pending State Tree Diffs -------------------------------- // + fn pending_state_tree_diffs_insert(&mut self, diff: &PendingStateTreeDiff) -> Result<(), StorageError>; + fn pending_state_tree_diffs_remove_by_block( + &mut self, + block_id: &BlockId, + ) -> Result; } #[derive(Debug, Clone, Copy, Serialize, Deserialize)]