From 8a0399997fb68708a8ba8899adcc73d5c1d3c386 Mon Sep 17 00:00:00 2001 From: Stan Bondi Date: Mon, 5 Feb 2024 11:28:28 +0400 Subject: [PATCH] feat(consensus)!: implements state merkle root --- .license.ignore | 2 +- Cargo.lock | 27 +- Cargo.toml | 8 +- bindings/src/types/Account.ts | 5 + bindings/src/types/Claims.ts | 5 + bindings/src/types/JrpcPermission.ts | 5 + bindings/src/types/JrpcPermissions.ts | 5 + bindings/src/types/TransactionStatus.ts | 5 + dan_layer/common_types/src/hasher.rs | 7 +- dan_layer/consensus/Cargo.toml | 2 + dan_layer/consensus/src/hotstuff/common.rs | 67 +- dan_layer/consensus/src/hotstuff/error.rs | 10 + .../consensus/src/hotstuff/on_propose.rs | 52 +- .../on_ready_to_vote_on_local_block.rs | 24 +- .../src/hotstuff/on_receive_local_proposal.rs | 60 +- .../src/hotstuff/on_receive_new_view.rs | 12 +- dan_layer/consensus/src/hotstuff/worker.rs | 64 +- dan_layer/consensus/src/traits/sync.rs | 2 +- dan_layer/consensus_tests/Cargo.toml | 1 + dan_layer/consensus_tests/src/consensus.rs | 3 +- dan_layer/consensus_tests/src/support/sync.rs | 2 +- .../src/support/transaction.rs | 26 +- dan_layer/engine/src/runtime/tracker.rs | 4 - dan_layer/engine_types/src/commit_result.rs | 16 + dan_layer/engine_types/src/hashing.rs | 6 + dan_layer/p2p/proto/rpc.proto | 7 +- dan_layer/p2p/src/conversions/consensus.rs | 1 + dan_layer/p2p/src/conversions/rpc.rs | 50 +- dan_layer/rpc_state_sync/Cargo.toml | 1 + dan_layer/rpc_state_sync/src/error.rs | 2 + dan_layer/rpc_state_sync/src/manager.rs | 129 +- dan_layer/state_store_sqlite/Cargo.toml | 5 +- .../up.sql | 28 + dan_layer/state_store_sqlite/src/lib.rs | 1 + dan_layer/state_store_sqlite/src/reader.rs | 356 +++-- dan_layer/state_store_sqlite/src/schema.rs | 23 + .../src/sql_models/block.rs | 4 + .../state_store_sqlite/src/sql_models/mod.rs | 2 + .../src/sql_models/pending_state_tree_diff.rs | 28 + .../src/sqlite_transaction.rs | 13 +- .../state_store_sqlite/src/tree_store.rs | 70 + dan_layer/state_store_sqlite/src/writer.rs | 320 +++-- dan_layer/state_store_sqlite/tests/tests.rs | 1 + dan_layer/state_tree/Cargo.toml | 23 + dan_layer/state_tree/src/error.rs | 10 + dan_layer/state_tree/src/jellyfish/hash.rs | 134 ++ dan_layer/state_tree/src/jellyfish/mod.rs | 14 + dan_layer/state_tree/src/jellyfish/store.rs | 79 ++ dan_layer/state_tree/src/jellyfish/tree.rs | 733 ++++++++++ dan_layer/state_tree/src/jellyfish/types.rs | 1249 +++++++++++++++++ dan_layer/state_tree/src/key_mapper.rs | 22 + dan_layer/state_tree/src/lib.rs | 16 + dan_layer/state_tree/src/memory_store.rs | 75 + dan_layer/state_tree/src/staged_store.rs | 92 ++ dan_layer/state_tree/src/tree.rs | 165 +++ dan_layer/state_tree/tests/support.rs | 79 ++ dan_layer/state_tree/tests/test.rs | 161 +++ dan_layer/storage/Cargo.toml | 1 + .../storage/src/consensus_models/block.rs | 67 +- dan_layer/storage/src/consensus_models/mod.rs | 2 + .../src/consensus_models/state_tree_diff.rs | 60 + .../storage/src/consensus_models/substate.rs | 36 +- .../src/consensus_models/transaction_pool.rs | 18 +- .../src/consensus_models/validated_block.rs | 6 +- dan_layer/storage/src/shard_store.rs | 256 ---- dan_layer/storage/src/state_store/mod.rs | 24 +- 66 files changed, 4071 insertions(+), 712 deletions(-) create mode 100644 dan_layer/state_store_sqlite/src/sql_models/pending_state_tree_diff.rs create mode 100644 dan_layer/state_store_sqlite/src/tree_store.rs create mode 100644 dan_layer/state_tree/Cargo.toml create mode 100644 dan_layer/state_tree/src/error.rs create mode 100644 dan_layer/state_tree/src/jellyfish/hash.rs create mode 100644 dan_layer/state_tree/src/jellyfish/mod.rs create mode 100644 dan_layer/state_tree/src/jellyfish/store.rs create mode 100644 dan_layer/state_tree/src/jellyfish/tree.rs create mode 100644 dan_layer/state_tree/src/jellyfish/types.rs create mode 100644 dan_layer/state_tree/src/key_mapper.rs create mode 100644 dan_layer/state_tree/src/lib.rs create mode 100644 dan_layer/state_tree/src/memory_store.rs create mode 100644 dan_layer/state_tree/src/staged_store.rs create mode 100644 dan_layer/state_tree/src/tree.rs create mode 100644 dan_layer/state_tree/tests/support.rs create mode 100644 dan_layer/state_tree/tests/test.rs create mode 100644 dan_layer/storage/src/consensus_models/state_tree_diff.rs delete mode 100644 dan_layer/storage/src/shard_store.rs diff --git a/.license.ignore b/.license.ignore index d4cc847e2..a2cc470eb 100644 --- a/.license.ignore +++ b/.license.ignore @@ -6,4 +6,4 @@ ./dan_layer/storage_sqlite/src/global/schema.rs ./dan_layer/state_store_sqlite/src/schema.rs ./dan_layer/wallet/storage_sqlite/src/schema.rs -./scripts/env_sample +./scripts/env_sample \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index 2b80124cd..cbbf4c63d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1686,6 +1686,7 @@ dependencies = [ "log", "rand", "serde", + "tari_bor", "tari_common_types", "tari_consensus", "tari_crypto", @@ -4565,7 +4566,7 @@ dependencies = [ [[package]] name = "libp2p-identity" version = "0.2.8" -source = "git+https://github.com/tari-project/rust-libp2p.git?rev=9b73988f611a877595de8a4a205aa86fb78cd626#9b73988f611a877595de8a4a205aa86fb78cd626" +source = "git+https://github.com/tari-project/rust-libp2p.git?rev=49ca2a88961f7131d3e496b579b522a823ae0418#49ca2a88961f7131d3e496b579b522a823ae0418" dependencies = [ "bs58 0.5.0", "ed25519-dalek", @@ -5663,7 +5664,7 @@ dependencies = [ [[package]] name = "multiaddr" version = "0.18.1" -source = "git+https://github.com/tari-project/rust-libp2p.git?rev=9b73988f611a877595de8a4a205aa86fb78cd626#9b73988f611a877595de8a4a205aa86fb78cd626" +source = "git+https://github.com/tari-project/rust-libp2p.git?rev=49ca2a88961f7131d3e496b579b522a823ae0418#49ca2a88961f7131d3e496b579b522a823ae0418" dependencies = [ "arrayref", "byteorder", @@ -8935,9 +8936,11 @@ dependencies = [ "tari_common_types", "tari_dan_common_types", "tari_dan_storage", + "tari_engine_types", "tari_epoch_manager", "tari_mmr", "tari_shutdown", + "tari_state_tree", "tari_transaction", "thiserror", "tokio", @@ -9203,6 +9206,7 @@ dependencies = [ "tari_dan_common_types", "tari_engine_types", "tari_mmr", + "tari_state_tree", "tari_transaction", "thiserror", "time", @@ -9692,6 +9696,7 @@ dependencies = [ "tari_dan_storage", "tari_epoch_manager", "tari_rpc_framework", + "tari_state_tree", "tari_transaction", "tari_validator_node_rpc", "thiserror", @@ -9787,12 +9792,30 @@ dependencies = [ "tari_dan_common_types", "tari_dan_storage", "tari_engine_types", + "tari_state_tree", "tari_transaction", "tari_utilities", "thiserror", "time", ] +[[package]] +name = "tari_state_tree" +version = "0.3.0" +dependencies = [ + "hex", + "indexmap 2.1.0", + "itertools 0.11.0", + "log", + "serde", + "tari_common_types", + "tari_crypto", + "tari_dan_common_types", + "tari_engine_types", + "tari_template_lib", + "thiserror", +] + [[package]] name = "tari_storage" version = "1.0.0-dan.5" diff --git a/Cargo.toml b/Cargo.toml index 495d61a89..ad166a2a4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,6 +26,7 @@ members = [ "dan_layer/p2p", "dan_layer/rpc_state_sync", "dan_layer/state_store_sqlite", + "dan_layer/state_tree", "dan_layer/storage_lmdb", "dan_layer/storage_sqlite", "dan_layer/storage", @@ -87,6 +88,7 @@ tari_networking = { path = "networking/core" } tari_rpc_framework = { path = "networking/rpc_framework" } tari_rpc_macros = { path = "networking/rpc_macros" } tari_state_store_sqlite = { path = "dan_layer/state_store_sqlite" } +tari_state_tree = { path = "dan_layer/state_tree" } tari_swarm = { path = "networking/swarm" } tari_template_abi = { version = "0.3.0", path = "dan_layer/template_abi" } tari_template_builtin = { path = "dan_layer/template_builtin" } @@ -237,9 +239,9 @@ overflow-checks = true # Temporarily lock pgp to commit (master branch at time of writing) because the currently released crate locks zeroize to =1.3 liblmdb-sys = { git = "https://github.com/tari-project/lmdb-rs", tag = "0.7.6-tari.1" } # Use Tari's libp2p fork that adds support for Schnorr-Ristretto -multiaddr = { git = "https://github.com/tari-project/rust-libp2p.git", rev = "9b73988f611a877595de8a4a205aa86fb78cd626" } -#libp2p = { git = "https://github.com/tari-project/rust-libp2p.git", rev = "ae1c06a9615a0ac020e459904ec984a6ab253678" } -libp2p-identity = { git = "https://github.com/tari-project/rust-libp2p.git", rev = "9b73988f611a877595de8a4a205aa86fb78cd626" } +multiaddr = { git = "https://github.com/tari-project/rust-libp2p.git", rev = "49ca2a88961f7131d3e496b579b522a823ae0418" } +#libp2p = { git = "https://github.com/tari-project/rust-libp2p.git", rev = "49ca2a88961f7131d3e496b579b522a823ae0418" } +libp2p-identity = { git = "https://github.com/tari-project/rust-libp2p.git", rev = "49ca2a88961f7131d3e496b579b522a823ae0418" } # Make a copy of this code, uncomment and replace account and my-branch with the name of your fork and the branch you want to temporarily use #[patch."https://github.com/tari-project/tari.git"] diff --git a/bindings/src/types/Account.ts b/bindings/src/types/Account.ts index ab5adb79b..4895dbf4a 100644 --- a/bindings/src/types/Account.ts +++ b/bindings/src/types/Account.ts @@ -1,3 +1,8 @@ +/* + * // Copyright 2024 The Tari Project + * // SPDX-License-Identifier: BSD-3-Clause + */ + // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. import type { SubstateId } from "./SubstateId"; diff --git a/bindings/src/types/Claims.ts b/bindings/src/types/Claims.ts index 03c6b98d7..59f53a7c4 100644 --- a/bindings/src/types/Claims.ts +++ b/bindings/src/types/Claims.ts @@ -1,3 +1,8 @@ +/* + * // Copyright 2024 The Tari Project + * // SPDX-License-Identifier: BSD-3-Clause + */ + // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. import type { JrpcPermissions } from "./JrpcPermissions"; diff --git a/bindings/src/types/JrpcPermission.ts b/bindings/src/types/JrpcPermission.ts index c8ca7afe0..735a39e0b 100644 --- a/bindings/src/types/JrpcPermission.ts +++ b/bindings/src/types/JrpcPermission.ts @@ -1,3 +1,8 @@ +/* + * // Copyright 2024 The Tari Project + * // SPDX-License-Identifier: BSD-3-Clause + */ + // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. import type { ComponentAddress } from "./ComponentAddress"; import type { ResourceAddress } from "./ResourceAddress"; diff --git a/bindings/src/types/JrpcPermissions.ts b/bindings/src/types/JrpcPermissions.ts index 9c212ab4e..90158ef86 100644 --- a/bindings/src/types/JrpcPermissions.ts +++ b/bindings/src/types/JrpcPermissions.ts @@ -1,3 +1,8 @@ +/* + * // Copyright 2024 The Tari Project + * // SPDX-License-Identifier: BSD-3-Clause + */ + // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. import type { JrpcPermission } from "./JrpcPermission"; diff --git a/bindings/src/types/TransactionStatus.ts b/bindings/src/types/TransactionStatus.ts index ab9356ff7..cb3b43f71 100644 --- a/bindings/src/types/TransactionStatus.ts +++ b/bindings/src/types/TransactionStatus.ts @@ -1,3 +1,8 @@ +/* + * // Copyright 2024 The Tari Project + * // SPDX-License-Identifier: BSD-3-Clause + */ + // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. export type TransactionStatus = diff --git a/dan_layer/common_types/src/hasher.rs b/dan_layer/common_types/src/hasher.rs index b8dc8aca4..2bca82359 100644 --- a/dan_layer/common_types/src/hasher.rs +++ b/dan_layer/common_types/src/hasher.rs @@ -45,8 +45,11 @@ impl TariHasher { } pub fn result(self) -> FixedHash { - let hash: [u8; 32] = self.hasher.finalize().into(); - hash.into() + self.finalize_into_array().into() + } + + pub fn finalize_into_array(self) -> [u8; 32] { + self.hasher.finalize().into() } fn hash_writer(&mut self) -> impl Write + '_ { diff --git a/dan_layer/consensus/Cargo.toml b/dan_layer/consensus/Cargo.toml index 7fdaef864..827db23bf 100644 --- a/dan_layer/consensus/Cargo.toml +++ b/dan_layer/consensus/Cargo.toml @@ -10,8 +10,10 @@ license.workspace = true [dependencies] tari_dan_common_types = { workspace = true } tari_dan_storage = { workspace = true } +tari_engine_types = { workspace = true } tari_transaction = { workspace = true } tari_epoch_manager = { workspace = true } +tari_state_tree = { workspace = true } # Used for PublicKey and Signature and Network enum tari_common = { workspace = true } diff --git a/dan_layer/consensus/src/hotstuff/common.rs b/dan_layer/consensus/src/hotstuff/common.rs index dde7131ce..fb0e88323 100644 --- a/dan_layer/consensus/src/hotstuff/common.rs +++ b/dan_layer/consensus/src/hotstuff/common.rs @@ -3,8 +3,22 @@ use log::*; use tari_common::configuration::Network; +use tari_common_types::types::FixedHash; use tari_dan_common_types::{committee::Committee, Epoch, NodeAddressable, NodeHeight}; -use tari_dan_storage::consensus_models::{Block, QuorumCertificate}; +use tari_dan_storage::consensus_models::{Block, LeafBlock, PendingStateTreeDiff, QuorumCertificate}; +use tari_engine_types::{ + hashing::substate_value_hasher32, + substate::{Substate, SubstateDiff}, +}; +use tari_state_tree::{ + Hash, + StagedTreeStore, + StateHashTreeDiff, + StateTreeError, + SubstateChange, + TreeStoreReader, + Version, +}; use crate::traits::LeaderStrategy; @@ -15,14 +29,17 @@ const LOG_TARGET: &str = "tari::dan::consensus::hotstuff::common"; /// TODO: exhaust > 0 pub const EXHAUST_DIVISOR: u64 = 0; -pub fn calculate_dummy_blocks>( +/// Calculates the dummy block required to reach the new height and returns the last dummy block (parent for next +/// proposal). +pub fn calculate_last_dummy_block>( network: Network, epoch: Epoch, high_qc: &QuorumCertificate, + parent_merkle_root: FixedHash, new_height: NodeHeight, leader_strategy: &TLeaderStrategy, local_committee: &Committee, -) -> Vec { +) -> Option { let mut parent_block = high_qc.as_leaf_block(); let mut current_height = high_qc.block_height() + NodeHeight(1); if current_height > new_height { @@ -32,7 +49,7 @@ pub fn calculate_dummy_blocks impl Iterator + '_ { + diff.down_iter() + .map(|(substate_id, _version)| SubstateChange::Down { + id: substate_id.clone(), + }) + .chain(diff.up_iter().map(move |(substate_id, value)| SubstateChange::Up { + id: substate_id.clone(), + value_hash: hash_substate(value), + })) +} + +pub fn hash_substate(substate: &Substate) -> FixedHash { + substate_value_hasher32().chain(substate).result().into_array().into() +} + +pub fn calculate_state_merkle_diff, I: IntoIterator>( + tx: &TTx, + current_version: Version, + next_version: Version, + pending_tree_updates: Vec, + substate_changes: I, +) -> Result<(Hash, StateHashTreeDiff), StateTreeError> { + debug!( + target: LOG_TARGET, + "Calculating state merkle diff from version {} to {} with {} update(s)", + current_version, + next_version, + pending_tree_updates.len(), + ); + let mut store = StagedTreeStore::new(tx); + store.apply_ordered_diffs(pending_tree_updates.into_iter().map(|diff| diff.diff)); + let mut state_tree = tari_state_tree::SpreadPrefixStateTree::new(&mut store); + let state_root = state_tree.put_substate_changes(current_version, next_version, substate_changes)?; + Ok((state_root, store.into_diff())) } diff --git a/dan_layer/consensus/src/hotstuff/error.rs b/dan_layer/consensus/src/hotstuff/error.rs index f50ca271e..bf44a1cd8 100644 --- a/dan_layer/consensus/src/hotstuff/error.rs +++ b/dan_layer/consensus/src/hotstuff/error.rs @@ -1,6 +1,7 @@ // Copyright 2023 The Tari Project // SPDX-License-Identifier: BSD-3-Clause +use tari_common_types::types::FixedHash; use tari_dan_common_types::{Epoch, NodeHeight}; use tari_dan_storage::{ consensus_models::{BlockId, LeafBlock, LockedBlock, QuorumCertificate, TransactionPoolError}, @@ -8,6 +9,7 @@ use tari_dan_storage::{ }; use tari_epoch_manager::EpochManagerError; use tari_mmr::BalancedBinaryMerkleProofError; +use tari_state_tree::StateTreeError; use tari_transaction::TransactionId; use crate::traits::{InboundMessagingError, OutboundMessagingError}; @@ -16,6 +18,8 @@ use crate::traits::{InboundMessagingError, OutboundMessagingError}; pub enum HotStuffError { #[error("Storage error: {0}")] StorageError(#[from] StorageError), + #[error("State tree error: {0}")] + StateTreeError(#[from] StateTreeError), #[error("Internal channel send error when {context}")] InternalChannelClosed { context: &'static str }, #[error("Inbound messaging error: {0}")] @@ -176,4 +180,10 @@ pub enum ProposalValidationError { block_network: String, block_id: BlockId, }, + #[error("Invalid state merkle root for block {block_id}: calculated {calculated} but block has {from_block}")] + InvalidStateMerkleRoot { + block_id: BlockId, + calculated: FixedHash, + from_block: FixedHash, + }, } diff --git a/dan_layer/consensus/src/hotstuff/on_propose.rs b/dan_layer/consensus/src/hotstuff/on_propose.rs index 650228030..bf48aaa69 100644 --- a/dan_layer/consensus/src/hotstuff/on_propose.rs +++ b/dan_layer/consensus/src/hotstuff/on_propose.rs @@ -1,7 +1,11 @@ // Copyright 2023 The Tari Project // SPDX-License-Identifier: BSD-3-Clause -use std::{collections::BTreeSet, num::NonZeroU64, ops::DerefMut}; +use std::{ + collections::BTreeSet, + num::NonZeroU64, + ops::{Deref, DerefMut}, +}; use indexmap::IndexMap; use log::*; @@ -23,6 +27,7 @@ use tari_dan_storage::{ LastProposed, LeafBlock, LockedBlock, + PendingStateTreeDiff, QuorumCertificate, TransactionPool, TransactionPoolStage, @@ -32,7 +37,13 @@ use tari_dan_storage::{ use tari_epoch_manager::EpochManagerReader; use crate::{ - hotstuff::{common::EXHAUST_DIVISOR, error::HotStuffError, proposer}, + hotstuff::{ + calculate_state_merkle_diff, + common::EXHAUST_DIVISOR, + diff_to_substate_changes, + error::HotStuffError, + proposer, + }, messages::{HotstuffMessage, ProposalMessage}, traits::{ConsensusSpec, OutboundMessaging, ValidatorSignatureService}, }; @@ -171,6 +182,7 @@ where TConsensusSpec: ConsensusSpec Ok(()) } + #[allow(clippy::too_many_lines)] fn build_next_block( &self, tx: &mut ::ReadTransaction<'_>, @@ -188,8 +200,11 @@ where TConsensusSpec: ConsensusSpec } else { self.transaction_pool.get_batch_for_next_block(tx, TARGET_BLOCK_SIZE)? }; + let current_version = high_qc.block_height().as_u64(); // parent_block.height.as_u64(); + let next_height = parent_block.height() + NodeHeight(1); let mut total_leader_fee = 0; + let mut substate_changes = vec![]; let locked_block = LockedBlock::get(tx)?; let pending_proposals = ForeignProposal::get_all_pending(tx, locked_block.block_id(), parent_block.block_id())?; let commands = ForeignProposal::get_all_new(tx)? @@ -224,7 +239,25 @@ where TConsensusSpec: ConsensusSpec })?; let leader_fee = t.calculate_leader_fee(involved, EXHAUST_DIVISOR); total_leader_fee += leader_fee; - Ok(Command::Accept(t.get_final_transaction_atom(leader_fee))) + let tx_atom = t.get_final_transaction_atom(leader_fee); + if tx_atom.decision.is_commit() { + let transaction = t.get_transaction(tx)?; + let result = transaction.result().ok_or_else(|| { + HotStuffError::InvariantError(format!( + "Transaction {} is committed but has no result when proposing", + t.transaction_id(), + )) + })?; + + let diff = result.finalize.result.accept().ok_or_else(|| { + HotStuffError::InvariantError(format!( + "Transaction {} has COMMIT decision but execution failed when proposing", + t.transaction_id(), + )) + })?; + substate_changes.extend(diff_to_substate_changes(diff)); + } + Ok(Command::Accept(tx_atom)) }, // Not reachable as there is nothing to propose for these stages. To confirm that all local nodes // agreed with the Accept, more (possibly empty) blocks with QCs will be @@ -244,6 +277,16 @@ where TConsensusSpec: ConsensusSpec commands.iter().map(|c| c.to_string()).collect::>().join(",") ); + let pending = PendingStateTreeDiff::get_all_up_to_commit_block(tx, high_qc.block_id())?; + + let (state_root, _) = calculate_state_merkle_diff( + tx.deref(), + current_version, + next_height.as_u64(), + pending, + substate_changes, + )?; + let non_local_buckets = proposer::get_non_local_shards_from_commands( tx, &commands, @@ -264,10 +307,11 @@ where TConsensusSpec: ConsensusSpec self.network, *parent_block.block_id(), high_qc, - parent_block.height() + NodeHeight(1), + next_height, epoch, proposed_by, commands, + state_root, total_leader_fee, foreign_indexes, None, diff --git a/dan_layer/consensus/src/hotstuff/on_ready_to_vote_on_local_block.rs b/dan_layer/consensus/src/hotstuff/on_ready_to_vote_on_local_block.rs index 13349d5de..c802de5c8 100644 --- a/dan_layer/consensus/src/hotstuff/on_ready_to_vote_on_local_block.rs +++ b/dan_layer/consensus/src/hotstuff/on_ready_to_vote_on_local_block.rs @@ -24,6 +24,7 @@ use tari_dan_storage::{ LastVoted, LockedBlock, LockedOutput, + PendingStateTreeDiff, QuorumDecision, SubstateLockFlag, SubstateRecord, @@ -47,7 +48,7 @@ use crate::{ const LOG_TARGET: &str = "tari::dan::consensus::hotstuff::on_lock_block_ready"; pub struct OnReadyToVoteOnLocalBlock { - validator_addr: TConsensusSpec::Addr, + local_validator_addr: TConsensusSpec::Addr, store: TConsensusSpec::StateStore, epoch_manager: TConsensusSpec::EpochManager, vote_signing_service: TConsensusSpec::SignatureService, @@ -77,7 +78,7 @@ where TConsensusSpec: ConsensusSpec network: Network, ) -> Self { Self { - validator_addr, + local_validator_addr: validator_addr, store, epoch_manager, vote_signing_service, @@ -107,8 +108,9 @@ where TConsensusSpec: ConsensusSpec let maybe_decision = self.store.with_write_tx(|tx| { let maybe_decision = self.decide_on_block(tx, &local_committee_shard, valid_block.block())?; + let is_accept_decision = maybe_decision.map(|d| d.is_accept()).unwrap_or(false); // Update nodes - if maybe_decision.map(|d| d.is_accept()).unwrap_or(false) { + if is_accept_decision { let high_qc = valid_block.block().update_nodes( tx, |tx, locked, block, locked_blocks| self.on_lock_block(tx, locked, block, locked_blocks), @@ -125,6 +127,7 @@ where TConsensusSpec: ConsensusSpec } Ok::<_, HotStuffError>(maybe_decision) })?; + self.propose_newly_locked_blocks(locked_blocks).await?; if let Some(decision) = maybe_decision { @@ -279,6 +282,7 @@ where TConsensusSpec: ConsensusSpec Command::ForeignProposal(_) => {}, } } + leaf.set_as_processed(tx)?; Ok(()) } @@ -481,7 +485,7 @@ where TConsensusSpec: ConsensusSpec warn!( target: LOG_TARGET, "{} ❌ Stage disagreement in block {} for transaction {}. Leader proposed LocalPrepared, but local stage is {}", - self.validator_addr, + self.local_validator_addr, block.id(), tx_rec.transaction_id(), tx_rec.current_stage() @@ -851,7 +855,7 @@ where TConsensusSpec: ConsensusSpec ) -> Result { let vn = self .epoch_manager - .get_validator_node(block.epoch(), &self.validator_addr) + .get_validator_node(block.epoch(), &self.local_validator_addr) .await?; let leaf_hash = vn.get_node_hash(self.network); @@ -1013,7 +1017,8 @@ where TConsensusSpec: ConsensusSpec // We are accepting the transaction so can remove the transaction from the pool debug!( target: LOG_TARGET, - "🗑️ Removing transaction {} from pool", tx_rec.transaction_id()); + "🗑️ Removing transaction {} from pool", tx_rec.transaction_id() + ); tx_rec.remove(tx)?; executed.set_final_decision(t.decision).update(tx)?; }, @@ -1023,6 +1028,13 @@ where TConsensusSpec: ConsensusSpec block.commit(tx)?; + // We don't store (empty) pending state diffs for dummy blocks + if !block.is_dummy() { + let pending = PendingStateTreeDiff::remove_by_block(tx, block.id())?; + let mut state_tree = tari_state_tree::SpreadPrefixStateTree::new(tx); + state_tree.commit_diff(pending.diff)?; + } + if total_transaction_fee > 0 { info!( target: LOG_TARGET, diff --git a/dan_layer/consensus/src/hotstuff/on_receive_local_proposal.rs b/dan_layer/consensus/src/hotstuff/on_receive_local_proposal.rs index 20007e355..329c48ee9 100644 --- a/dan_layer/consensus/src/hotstuff/on_receive_local_proposal.rs +++ b/dan_layer/consensus/src/hotstuff/on_receive_local_proposal.rs @@ -5,7 +5,7 @@ // ----[foreign:LocalPrepared]--->(LocalPrepared, true) ----cmd:AllPrepare ---> (AllPrepared, true) ---cmd:Accept ---> // Complete -use std::ops::DerefMut; +use std::ops::{Deref, DerefMut}; use log::*; use tari_common::configuration::Network; @@ -23,6 +23,7 @@ use tari_dan_storage::{ ForeignProposal, ForeignSendCounters, HighQc, + PendingStateTreeDiff, TransactionPool, TransactionPoolStage, ValidBlock, @@ -31,11 +32,14 @@ use tari_dan_storage::{ StateStoreReadTransaction, }; use tari_epoch_manager::EpochManagerReader; +use tari_state_tree::StateHashTreeDiff; use tokio::sync::broadcast; use super::proposer::{self, Proposer}; use crate::{ hotstuff::{ + calculate_state_merkle_diff, + diff_to_substate_changes, error::HotStuffError, on_ready_to_vote_on_local_block::OnReadyToVoteOnLocalBlock, pacemaker_handle::PaceMakerHandle, @@ -126,12 +130,14 @@ impl OnReceiveLocalProposalHandler(Some((high_qc, valid_block))) })?; @@ -151,11 +157,16 @@ impl OnReceiveLocalProposalHandler::WriteTransaction<'_>, valid_block: &ValidBlock, + tree_diff: StateHashTreeDiff, ) -> Result { valid_block.block().save_foreign_send_counters(tx)?; valid_block.block().justify().save(tx)?; valid_block.save_all_dummy_blocks(tx)?; valid_block.block().save(tx)?; + + // Store the tree diff for the block + PendingStateTreeDiff::new(*valid_block.id(), valid_block.height(), tree_diff).save(tx)?; + let high_qc = valid_block.block().justify().update_high_qc(tx)?; Ok(high_qc) } @@ -166,9 +177,14 @@ impl OnReceiveLocalProposalHandler, local_committee_shard: &CommitteeShard, - ) -> Result, HotStuffError> { - match self.validate_local_proposed_block(tx, block, local_committee, local_committee_shard) { - Ok(validated) => Ok(Some(validated)), + ) -> Result, HotStuffError> { + match self + .validate_local_proposed_block(tx, block, local_committee, local_committee_shard) + .and_then(|valid_block| { + let diff = self.check_state_merkle_root(tx, valid_block.block())?; + Ok((valid_block, diff)) + }) { + Ok((validated, diff)) => Ok(Some((validated, diff))), // Propagate this error out as sync is needed in the case where we have a valid QC but do not know the // block Err(err @ HotStuffError::ProposalValidationError(ProposalValidationError::JustifyBlockNotFound { .. })) => { @@ -184,6 +200,37 @@ impl OnReceiveLocalProposalHandler::ReadTransaction<'_>, + block: &Block, + ) -> Result { + let current_version = block.justify().block_height().as_u64(); + let next_version = block.height().as_u64(); + let commit_substate_diffs = block.get_all_substate_diffs(tx)?; + + let pending = PendingStateTreeDiff::get_all_up_to_commit_block(tx, block.justify().block_id())?; + + let (state_root, state_tree_diff) = calculate_state_merkle_diff( + tx.deref(), + current_version, + next_version, + pending, + commit_substate_diffs.iter().flat_map(diff_to_substate_changes), + )?; + + if state_root != *block.merkle_root() { + return Err(ProposalValidationError::InvalidStateMerkleRoot { + block_id: *block.id(), + from_block: *block.merkle_root(), + calculated: state_root, + } + .into()); + } + + Ok(state_tree_diff) + } + fn check_foreign_indexes( &self, tx: &mut ::ReadTransaction<'_>, @@ -346,6 +393,7 @@ impl OnReceiveLocalProposalHandler HotstuffWorker { shutdown, } } -} -impl HotstuffWorker -where TConsensusSpec: ConsensusSpec -{ + pub async fn start(&mut self) -> Result<(), HotStuffError> { self.create_genesis_block_if_required()?; let (current_height, high_qc) = self.state_store.with_read_tx(|tx| { @@ -534,36 +530,36 @@ where TConsensusSpec: ConsensusSpec } fn create_genesis_block_if_required(&self) -> Result<(), HotStuffError> { - let mut tx = self.state_store.create_write_tx()?; - - // The parent for genesis blocks refer to this zero block - let zero_block = Block::zero_block(self.network); - if !zero_block.exists(tx.deref_mut())? { - debug!(target: LOG_TARGET, "Creating zero block"); - zero_block.justify().insert(&mut tx)?; - zero_block.insert(&mut tx)?; - zero_block.as_locked_block().set(&mut tx)?; - zero_block.as_leaf_block().set(&mut tx)?; - zero_block.as_last_executed().set(&mut tx)?; - zero_block.as_last_voted().set(&mut tx)?; - zero_block.justify().as_high_qc().set(&mut tx)?; - zero_block.commit(&mut tx)?; - } - - // let genesis = Block::genesis(); - // if !genesis.exists(tx.deref_mut())? { - // debug!(target: LOG_TARGET, "Creating genesis block"); - // genesis.justify().save(&mut tx)?; - // genesis.insert(&mut tx)?; - // genesis.as_locked().set(&mut tx)?; - // genesis.as_leaf_block().set(&mut tx)?; - // genesis.as_last_executed().set(&mut tx)?; - // genesis.justify().as_high_qc().set(&mut tx)?; - // } - - tx.commit()?; + self.state_store.with_write_tx(|tx| { + // The parent for genesis blocks refer to this zero block + let zero_block = Block::zero_block(self.network); + if !zero_block.exists(tx.deref_mut())? { + debug!(target: LOG_TARGET, "Creating zero block"); + zero_block.justify().insert(tx)?; + zero_block.insert(tx)?; + zero_block.as_locked_block().set(tx)?; + zero_block.as_leaf_block().set(tx)?; + zero_block.as_last_executed().set(tx)?; + zero_block.as_last_voted().set(tx)?; + zero_block.justify().as_high_qc().set(tx)?; + zero_block.commit(tx)?; + } - Ok(()) + // let mut state_tree = SpreadPrefixStateTree::new(tx); + // state_tree.put_substate_changes_at_next_version(None, vec![])?; + + // let genesis = Block::genesis(); + // if !genesis.exists(tx.deref_mut())? { + // debug!(target: LOG_TARGET, "Creating genesis block"); + // genesis.justify().save(&mut tx)?; + // genesis.insert(&mut tx)?; + // genesis.as_locked().set(&mut tx)?; + // genesis.as_leaf_block().set(&mut tx)?; + // genesis.as_last_executed().set(&mut tx)?; + // genesis.justify().as_high_qc().set(&mut tx)?; + // } + Ok(()) + }) } fn publish_event(&self, event: HotstuffEvent) { diff --git a/dan_layer/consensus/src/traits/sync.rs b/dan_layer/consensus/src/traits/sync.rs index 018c69676..76aecc623 100644 --- a/dan_layer/consensus/src/traits/sync.rs +++ b/dan_layer/consensus/src/traits/sync.rs @@ -9,7 +9,7 @@ pub trait SyncManager { async fn check_sync(&self) -> Result; - async fn sync(&self) -> Result<(), Self::Error>; + async fn sync(&mut self) -> Result<(), Self::Error>; } #[derive(Debug, Clone, Copy)] diff --git a/dan_layer/consensus_tests/Cargo.toml b/dan_layer/consensus_tests/Cargo.toml index 49fcaa53b..a81601b43 100644 --- a/dan_layer/consensus_tests/Cargo.toml +++ b/dan_layer/consensus_tests/Cargo.toml @@ -10,6 +10,7 @@ license.workspace = true [dependencies] [dev-dependencies] +tari_bor = { workspace = true } tari_dan_common_types = { workspace = true } tari_consensus = { workspace = true } tari_dan_storage = { workspace = true } diff --git a/dan_layer/consensus_tests/src/consensus.rs b/dan_layer/consensus_tests/src/consensus.rs index a9f9560cf..60e55e701 100644 --- a/dan_layer/consensus_tests/src/consensus.rs +++ b/dan_layer/consensus_tests/src/consensus.rs @@ -298,8 +298,9 @@ async fn leader_failure_output_conflict() { setup_logger(); let mut test = Test::builder() .with_test_timeout(Duration::from_secs(60)) + .debug_sql("/tmp/test{}.db") .add_committee(0, vec!["1", "2"]) - .add_committee(1, vec!["3", "4"]) + // .add_committee(1, vec!["3", "4"]) .start() .await; diff --git a/dan_layer/consensus_tests/src/support/sync.rs b/dan_layer/consensus_tests/src/support/sync.rs index f51e2fc59..e7d207ff1 100644 --- a/dan_layer/consensus_tests/src/support/sync.rs +++ b/dan_layer/consensus_tests/src/support/sync.rs @@ -18,7 +18,7 @@ impl SyncManager for AlwaysSyncedSyncManager { Ok(SyncStatus::UpToDate) } - async fn sync(&self) -> Result<(), Self::Error> { + async fn sync(&mut self) -> Result<(), Self::Error> { Ok(()) } } diff --git a/dan_layer/consensus_tests/src/support/transaction.rs b/dan_layer/consensus_tests/src/support/transaction.rs index 6ce5b4b9c..d47c35354 100644 --- a/dan_layer/consensus_tests/src/support/transaction.rs +++ b/dan_layer/consensus_tests/src/support/transaction.rs @@ -3,15 +3,17 @@ use std::{iter, time::Duration}; +use rand::{distributions::Alphanumeric, rngs::OsRng, Rng}; use tari_common_types::types::PrivateKey; use tari_dan_common_types::SubstateAddress; use tari_dan_storage::consensus_models::{Decision, ExecutedTransaction}; use tari_engine_types::{ commit_result::{ExecuteResult, FinalizeResult, RejectReason, TransactionResult}, + component::{ComponentBody, ComponentHeader}, fees::{FeeCostBreakdown, FeeReceipt}, - substate::SubstateDiff, + substate::{Substate, SubstateDiff}, }; -use tari_transaction::Transaction; +use tari_transaction::{id_provider::IdProvider, Transaction}; use crate::support::helpers::random_shard_in_bucket; @@ -22,6 +24,7 @@ pub fn build_transaction_from( resulting_outputs: Vec, ) -> ExecutedTransaction { let tx_id = *tx.id(); + let id_provider = IdProvider::new(tx_id.into_array().into(), 1000); ExecutedTransaction::new( tx, ExecuteResult { @@ -30,7 +33,24 @@ pub fn build_transaction_from( vec![], vec![], if decision.is_commit() { - TransactionResult::Accept(SubstateDiff::new()) + let mut diff = SubstateDiff::new(); + for _ in &resulting_outputs { + let obj = id_provider.new_component_address(Default::default(), None).unwrap(); + let s = (0..100).map(|_| OsRng.sample(Alphanumeric) as char).collect::(); + let random_state = tari_bor::to_value(&s).unwrap(); + diff.up( + obj.into(), + Substate::new(0, ComponentHeader { + template_address: Default::default(), + module_name: "Test".to_string(), + owner_key: Default::default(), + owner_rule: Default::default(), + access_rules: Default::default(), + body: ComponentBody { state: random_state }, + }), + ) + } + TransactionResult::Accept(diff) } else { TransactionResult::Reject(RejectReason::ExecutionFailure("Test failure".to_string())) }, diff --git a/dan_layer/engine/src/runtime/tracker.rs b/dan_layer/engine/src/runtime/tracker.rs index 96f048f73..cf7f42542 100644 --- a/dan_layer/engine/src/runtime/tracker.rs +++ b/dan_layer/engine/src/runtime/tracker.rs @@ -119,10 +119,6 @@ impl StateTracker { self.read_with(|state| state.logs().len()) } - pub fn take_logs(&self) -> Vec { - self.write_with(|state| state.take_logs()) - } - pub fn get_template_address(&self) -> Result { self.read_with(|state| state.current_template().map(|(a, _)| *a)) } diff --git a/dan_layer/engine_types/src/commit_result.rs b/dan_layer/engine_types/src/commit_result.rs index b9b46686a..c02517983 100644 --- a/dan_layer/engine_types/src/commit_result.rs +++ b/dan_layer/engine_types/src/commit_result.rs @@ -136,6 +136,22 @@ impl FinalizeResult { } } + pub fn accept(&self) -> Option<&SubstateDiff> { + match self.result { + TransactionResult::Accept(ref diff) => Some(diff), + TransactionResult::AcceptFeeRejectRest(ref diff, _) => Some(diff), + TransactionResult::Reject(_) => None, + } + } + + pub fn into_accept(self) -> Option { + match self.result { + TransactionResult::Accept(diff) => Some(diff), + TransactionResult::AcceptFeeRejectRest(diff, _) => Some(diff), + TransactionResult::Reject(_) => None, + } + } + pub fn reject(&self) -> Option<&RejectReason> { match self.result { TransactionResult::Accept(_) => None, diff --git a/dan_layer/engine_types/src/hashing.rs b/dan_layer/engine_types/src/hashing.rs index 650f5b734..f0317db57 100644 --- a/dan_layer/engine_types/src/hashing.rs +++ b/dan_layer/engine_types/src/hashing.rs @@ -41,6 +41,10 @@ pub fn template_hasher64() -> TariHasher64 { hasher64(EngineHashDomainLabel::Template) } +pub fn substate_value_hasher32() -> TariHasher32 { + hasher32(EngineHashDomainLabel::SubstateValue) +} + pub fn hasher32(label: EngineHashDomainLabel) -> TariHasher32 { TariHasher32::new_with_label::(label.as_label()) } @@ -175,6 +179,7 @@ pub enum EngineHashDomainLabel { TransactionReceipt, FeeClaimAddress, QuorumCertificate, + SubstateValue, } impl EngineHashDomainLabel { @@ -198,6 +203,7 @@ impl EngineHashDomainLabel { Self::TransactionReceipt => "TransactionReceipt", Self::FeeClaimAddress => "FeeClaimAddress", Self::QuorumCertificate => "QuorumCertificate", + Self::SubstateValue => "SubstateValue", } } } diff --git a/dan_layer/p2p/proto/rpc.proto b/dan_layer/p2p/proto/rpc.proto index 8d59877e3..7438ac3c4 100644 --- a/dan_layer/p2p/proto/rpc.proto +++ b/dan_layer/p2p/proto/rpc.proto @@ -178,9 +178,10 @@ message SubstateData { } message SubstateDestroyedProof { - bytes address = 1; - tari.dan.consensus.QuorumCertificate destroyed_justify = 2; - bytes destroyed_by_transaction = 3; + bytes substate_id = 1; + uint32 version = 2; + tari.dan.consensus.QuorumCertificate destroyed_justify = 3; + bytes destroyed_by_transaction = 4; } message SubstateUpdate { diff --git a/dan_layer/p2p/src/conversions/consensus.rs b/dan_layer/p2p/src/conversions/consensus.rs index 0f3d7b263..dc97811b1 100644 --- a/dan_layer/p2p/src/conversions/consensus.rs +++ b/dan_layer/p2p/src/conversions/consensus.rs @@ -289,6 +289,7 @@ impl TryFrom for tari_dan_storage::consensus_models::Bl .into_iter() .map(TryInto::try_into) .collect::>()?, + value.merkle_root.try_into()?, value.total_leader_fee, decode_exact(&value.foreign_indexes)?, value.signature.map(TryInto::try_into).transpose()?, diff --git a/dan_layer/p2p/src/conversions/rpc.rs b/dan_layer/p2p/src/conversions/rpc.rs index 14e8e44cb..b4babeee8 100644 --- a/dan_layer/p2p/src/conversions/rpc.rs +++ b/dan_layer/p2p/src/conversions/rpc.rs @@ -4,7 +4,7 @@ use std::convert::{TryFrom, TryInto}; use anyhow::anyhow; -use tari_dan_storage::consensus_models::{SubstateCreatedProof, SubstateData, SubstateUpdate}; +use tari_dan_storage::consensus_models::{SubstateCreatedProof, SubstateData, SubstateDestroyedProof, SubstateUpdate}; use tari_engine_types::substate::{SubstateId, SubstateValue}; use crate::proto; @@ -37,6 +37,34 @@ impl From for proto::rpc::SubstateCreatedProof { } } +impl TryFrom for SubstateDestroyedProof { + type Error = anyhow::Error; + + fn try_from(value: proto::rpc::SubstateDestroyedProof) -> Result { + Ok(Self { + substate_id: SubstateId::from_bytes(&value.substate_id)?, + version: value.version, + justify: value + .destroyed_justify + .map(TryInto::try_into) + .transpose()? + .ok_or_else(|| anyhow!("destroyed_justify not provided"))?, + destroyed_by_transaction: value.destroyed_by_transaction.try_into()?, + }) + } +} + +impl From for proto::rpc::SubstateDestroyedProof { + fn from(value: SubstateDestroyedProof) -> Self { + Self { + substate_id: value.substate_id.to_bytes(), + version: value.version, + destroyed_justify: Some((&value.justify).into()), + destroyed_by_transaction: value.destroyed_by_transaction.as_bytes().to_vec(), + } + } +} + impl TryFrom for SubstateUpdate { type Error = anyhow::Error; @@ -44,15 +72,7 @@ impl TryFrom for SubstateUpdate { let update = value.update.ok_or_else(|| anyhow!("update not provided"))?; match update { proto::rpc::substate_update::Update::Create(substate_proof) => Ok(Self::Create(substate_proof.try_into()?)), - proto::rpc::substate_update::Update::Destroy(proof) => Ok(Self::Destroy { - address: proof.address.try_into()?, - proof: proof - .destroyed_justify - .map(TryInto::try_into) - .transpose()? - .ok_or_else(|| anyhow!("destroyed_justify not provided"))?, - destroyed_by_transaction: proof.destroyed_by_transaction.try_into()?, - }), + proto::rpc::substate_update::Update::Destroy(proof) => Ok(Self::Destroy(proof.try_into()?)), } } } @@ -61,15 +81,7 @@ impl From for proto::rpc::SubstateUpdate { fn from(value: SubstateUpdate) -> Self { let update = match value { SubstateUpdate::Create(proof) => proto::rpc::substate_update::Update::Create(proof.into()), - SubstateUpdate::Destroy { - proof, - address, - destroyed_by_transaction, - } => proto::rpc::substate_update::Update::Destroy(proto::rpc::SubstateDestroyedProof { - address: address.as_bytes().to_vec(), - destroyed_justify: Some((&proof).into()), - destroyed_by_transaction: destroyed_by_transaction.as_bytes().to_vec(), - }), + SubstateUpdate::Destroy(proof) => proto::rpc::substate_update::Update::Destroy(proof.into()), }; Self { update: Some(update) } diff --git a/dan_layer/rpc_state_sync/Cargo.toml b/dan_layer/rpc_state_sync/Cargo.toml index aad981e7c..e218a6b69 100644 --- a/dan_layer/rpc_state_sync/Cargo.toml +++ b/dan_layer/rpc_state_sync/Cargo.toml @@ -17,6 +17,7 @@ tari_dan_common_types = { workspace = true } tari_transaction = { workspace = true } tari_rpc_framework = { workspace = true } tari_dan_p2p = { workspace = true } +tari_state_tree = { workspace = true } anyhow = { workspace = true } async-trait = { workspace = true } diff --git a/dan_layer/rpc_state_sync/src/error.rs b/dan_layer/rpc_state_sync/src/error.rs index 25dc2feff..1a2a5fdd9 100644 --- a/dan_layer/rpc_state_sync/src/error.rs +++ b/dan_layer/rpc_state_sync/src/error.rs @@ -30,6 +30,8 @@ pub enum CommsRpcConsensusSyncError { NoPeersAvailable { committee_size: usize }, #[error("Proposal validation error: {0}")] ProposalValidationError(#[from] ProposalValidationError), + #[error("State tree error: {0}")] + StateTreeError(#[from] tari_state_tree::StateTreeError), } impl From for HotStuffError { diff --git a/dan_layer/rpc_state_sync/src/manager.rs b/dan_layer/rpc_state_sync/src/manager.rs index c8d2fa96b..fb09c8f9b 100644 --- a/dan_layer/rpc_state_sync/src/manager.rs +++ b/dan_layer/rpc_state_sync/src/manager.rs @@ -1,14 +1,18 @@ // Copyright 2023 The Tari Project // SPDX-License-Identifier: BSD-3-Clause -use std::{fmt::Display, ops::DerefMut}; +use std::{ + collections::HashMap, + fmt::Display, + ops::{Deref, DerefMut}, +}; use async_trait::async_trait; use futures::StreamExt; use log::*; use tari_common::configuration::Network; use tari_consensus::{ - hotstuff::ProposalValidationError, + hotstuff::{calculate_state_merkle_diff, hash_substate, ProposalValidationError}, traits::{ConsensusSpec, LeaderStrategy, SyncManager, SyncStatus}, }; use tari_dan_common_types::{committee::Committee, optional::Optional, NodeHeight, PeerAddress}; @@ -20,6 +24,7 @@ use tari_dan_storage::{ HighQc, LastExecuted, LockedBlock, + PendingStateTreeDiff, QuorumCertificate, SubstateUpdate, TransactionPoolRecord, @@ -30,6 +35,7 @@ use tari_dan_storage::{ }; use tari_epoch_manager::EpochManagerReader; use tari_rpc_framework::RpcError; +use tari_state_tree::SubstateChange; use tari_transaction::Transaction; use tari_validator_node_rpc::{ client::{TariValidatorNodeRpcClientFactory, ValidatorNodeClientFactory}, @@ -79,7 +85,7 @@ where TConsensusSpec: ConsensusSpec } async fn sync_with_peer( - &self, + &mut self, addr: &TConsensusSpec::Addr, locked_block: &LockedBlock, ) -> Result<(), CommsRpcConsensusSyncError> { @@ -116,7 +122,7 @@ where TConsensusSpec: ConsensusSpec #[allow(clippy::too_many_lines)] async fn sync_blocks( - &self, + &mut self, client: &mut ValidatorNodeRpcClient, locked_block: &LockedBlock, ) -> Result<(), CommsRpcConsensusSyncError> { @@ -129,6 +135,9 @@ where TConsensusSpec: ConsensusSpec let mut counter = 0usize; let mut expected_height = locked_block.height + NodeHeight(1); + // Stores the uncommitted state updates for each block. When a block reaches a 3-chain, the updates are removed + // and applied. + let mut pending_state_updates = HashMap::new(); while let Some(resp) = stream.next().await { let msg = resp.map_err(RpcError::from)?; @@ -235,7 +244,8 @@ where TConsensusSpec: ConsensusSpec } else { expected_height = block.height() + NodeHeight(1); } - self.process_block(block, qcs, updates, transactions).await?; + self.process_block(block, qcs, updates, transactions, &mut pending_state_updates) + .await?; } info!(target: LOG_TARGET, "🌐 {counter} blocks synced to height {}", expected_height - NodeHeight(1)); @@ -244,11 +254,12 @@ where TConsensusSpec: ConsensusSpec } async fn process_block( - &self, + &mut self, block: Block, qcs: Vec, updates: Vec, transactions: Vec, + pending_state_updates: &mut HashMap>, ) -> Result<(), CommsRpcConsensusSyncError> { // Note: this is only used for dummy block calculation, so we avoid the epoch manager call unless it is needed. // Otherwise, the committee is empty. @@ -290,6 +301,7 @@ where TConsensusSpec: ConsensusSpec next_height, block.justify().clone(), block.epoch(), + *block.merkle_root(), ); dummy_block.save(tx)?; last_dummy_block = BlockIdAndHeight { id: *dummy_block.id(), height: next_height }; @@ -309,54 +321,117 @@ where TConsensusSpec: ConsensusSpec for qc in qcs { qc.save(tx)?; } + // Ensure we dont vote on a synced block + block.as_last_voted().set(tx)?; + + self.check_and_update_state_merkle_tree(tx, &block, &updates)?; + + if !updates.is_empty() { + pending_state_updates.insert(*block.id(), updates); + } + block.update_nodes( tx, |_, _, _, _| Ok(()), |tx, last_executed, block| { + debug!(target: LOG_TARGET, "Sync is committing block {}", block); let new_last_exec = block.as_last_executed(); - Self::mark_block_committed(tx, last_executed, block)?; + Self::commit_block(tx, last_executed, block, pending_state_updates)?; new_last_exec.set(tx)?; - TransactionPoolRecord::remove_any( - tx, - block.commands().iter().filter_map(|cmd| cmd.accept()).map(|t| &t.id), - )?; - Ok::<_, CommsRpcConsensusSyncError>(()) }, &mut Vec::new(), )?; - // Ensure we dont vote on a synced block - block.as_last_voted().set(tx)?; - let (ups, downs) = updates.into_iter().partition::, _>(|u| u.is_create()); - // First do UPs then do DOWNs - // TODO: stage the updates, then check against the state hash in the block, then persist - for update in ups { - update.apply(tx, &block)?; - } - for update in downs { - update.apply(tx, &block)?; - } + + block.set_as_processed(tx)?; + Ok(()) }) } - fn mark_block_committed( + fn check_and_update_state_merkle_tree( + &self, + tx: &mut ::WriteTransaction<'_>, + block: &Block, + updates: &[SubstateUpdate], + ) -> Result<(), CommsRpcConsensusSyncError> { + let pending_tree_updates = PendingStateTreeDiff::get_all_up_to_commit_block(tx.deref_mut(), block.id())?; + let current_version = block.justify().block_height().as_u64(); + let next_version = block.height().as_u64(); + + let changes = updates.iter().map(|update| match update { + SubstateUpdate::Create(create) => SubstateChange::Up { + id: create.substate.substate_id.clone(), + value_hash: hash_substate(&create.substate.clone().into_substate()), + }, + SubstateUpdate::Destroy(destroy) => SubstateChange::Down { + id: destroy.substate_id.clone(), + }, + }); + + let (root_hash, tree_diff) = + calculate_state_merkle_diff(tx.deref(), current_version, next_version, pending_tree_updates, changes)?; + + if root_hash != *block.merkle_root() { + return Err(CommsRpcConsensusSyncError::InvalidResponse(anyhow::anyhow!( + "Merkle root in block {} does not match the merkle root of the state tree. Block MR: {}, Calculated \ + MR: {}", + block, + block.merkle_root(), + root_hash + ))); + } + + // Persist pending state tree diff + PendingStateTreeDiff::new(*block.id(), block.height(), tree_diff).save(tx)?; + + Ok(()) + } + + fn commit_block( tx: &mut ::WriteTransaction<'_>, last_executed: &LastExecuted, block: &Block, + pending_state_updates: &mut HashMap>, ) -> Result<(), CommsRpcConsensusSyncError> { if last_executed.height < block.height() { let parent = block.get_parent(tx.deref_mut())?; - // Recurse to "catch up" any parent parent blocks we may not have executed + // Recurse to "catch up" any parent blocks we may not have executed block.commit(tx)?; - Self::mark_block_committed(tx, last_executed, &parent)?; + Self::commit_block(tx, last_executed, &parent, pending_state_updates)?; debug!( target: LOG_TARGET, "✅ COMMIT block {}, last executed height = {}", block, last_executed.height ); + + if block.is_dummy() { + return Ok(()); + } + + TransactionPoolRecord::remove_any( + tx, + block.commands().iter().filter_map(|cmd| cmd.accept()).map(|t| &t.id), + )?; + + let diff = PendingStateTreeDiff::remove_by_block(tx, block.id())?; + let mut tree = tari_state_tree::SpreadPrefixStateTree::new(tx); + tree.commit_diff(diff.diff)?; + + // Commit the state if any + if let Some(updates) = pending_state_updates.remove(block.id()) { + debug!(target: LOG_TARGET, "Committed block {} has {} state update(s)", block, updates.len()); + let (ups, downs) = updates.into_iter().partition::, _>(|u| u.is_create()); + // First do UPs then do DOWNs + for update in ups { + update.apply(tx, block)?; + } + for update in downs { + update.apply(tx, block)?; + } + } } Ok(()) } @@ -448,7 +523,7 @@ where TConsensusSpec: ConsensusSpec + Send + Sync + 'static Ok(SyncStatus::UpToDate) } - async fn sync(&self) -> Result<(), Self::Error> { + async fn sync(&mut self) -> Result<(), Self::Error> { let committee = self.get_sync_peers().await?; if committee.is_empty() { warn!(target: LOG_TARGET, "No peers available for sync"); diff --git a/dan_layer/state_store_sqlite/Cargo.toml b/dan_layer/state_store_sqlite/Cargo.toml index 7793d312a..39a377efb 100644 --- a/dan_layer/state_store_sqlite/Cargo.toml +++ b/dan_layer/state_store_sqlite/Cargo.toml @@ -8,12 +8,13 @@ repository.workspace = true license.workspace = true [dependencies] +# TODO: needed for FixedHash +tari_common_types = { workspace = true } tari_dan_storage = { workspace = true } tari_dan_common_types = { workspace = true } tari_transaction = { workspace = true } tari_engine_types = { workspace = true } -# TODO: needed for FixedHash -tari_common_types = { workspace = true } +tari_state_tree = { workspace = true } tari_utilities = { workspace = true } anyhow = { workspace = true } diff --git a/dan_layer/state_store_sqlite/migrations/2023-06-08-091819_create_state_store/up.sql b/dan_layer/state_store_sqlite/migrations/2023-06-08-091819_create_state_store/up.sql index 7a656d22c..7d0272e7d 100644 --- a/dan_layer/state_store_sqlite/migrations/2023-06-08-091819_create_state_store/up.sql +++ b/dan_layer/state_store_sqlite/migrations/2023-06-08-091819_create_state_store/up.sql @@ -15,6 +15,7 @@ create table blocks id integer not null primary key AUTOINCREMENT, block_id text not NULL, parent_block_id text not NULL, + merkle_root text not NULL, network text not NULL, height bigint not NULL, epoch bigint not NULL, @@ -40,6 +41,7 @@ create table parked_blocks id integer not null primary key AUTOINCREMENT, block_id text not NULL, parent_block_id text not NULL, + merkle_root text not NULL, network text not NULL, height bigint not NULL, epoch bigint not NULL, @@ -277,6 +279,32 @@ CREATE TABLE foreign_receive_counters created_at timestamp not NULL DEFAULT CURRENT_TIMESTAMP ); +CREATE TABLE state_tree +( + id integer not NULL primary key AUTOINCREMENT, + key text not NULL, + node text not NULL, + is_stale boolean not null default '0' +); + +-- Duplicate keys are not allowed +CREATE UNIQUE INDEX state_tree_uniq_idx_key on state_tree (key); +-- filtering out or by is_stale is used in every query +CREATE INDEX state_tree_idx_is_stale on state_tree (is_stale); + +CREATE TABLE pending_state_tree_diffs +( + id integer not NULL primary key AUTOINCREMENT, + block_id text not NULL, + block_height bigint not NULL, + diff_json text not NULL, + created_at timestamp not NULL DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (block_id) REFERENCES blocks (block_id) +); + +CREATE UNIQUE INDEX pending_state_tree_diffs_uniq_idx_block_id on pending_state_tree_diffs (block_id); + + -- Debug Triggers CREATE TABLE transaction_pool_history ( diff --git a/dan_layer/state_store_sqlite/src/lib.rs b/dan_layer/state_store_sqlite/src/lib.rs index 6a6f8f153..58e1009f0 100644 --- a/dan_layer/state_store_sqlite/src/lib.rs +++ b/dan_layer/state_store_sqlite/src/lib.rs @@ -7,6 +7,7 @@ mod serialization; mod sql_models; mod sqlite_transaction; mod store; +mod tree_store; mod writer; pub use store::SqliteStateStore; diff --git a/dan_layer/state_store_sqlite/src/reader.rs b/dan_layer/state_store_sqlite/src/reader.rs index 2a255cf6b..34cd9c2bf 100644 --- a/dan_layer/state_store_sqlite/src/reader.rs +++ b/dan_layer/state_store_sqlite/src/reader.rs @@ -44,6 +44,7 @@ use tari_dan_storage::{ LastVoted, LeafBlock, LockedBlock, + PendingStateTreeDiff, QcId, QuorumCertificate, SubstateLockFlag, @@ -84,7 +85,7 @@ impl<'a, TAddr> SqliteStateStoreReadTransaction<'a, TAddr> { } } - pub(crate) fn connection(&mut self) -> &mut SqliteConnection { + pub(crate) fn connection(&self) -> &mut SqliteConnection { self.transaction.connection() } @@ -202,6 +203,7 @@ impl<'a, TAddr: NodeAddressable + Serialize + DeserializeOwned> SqliteStateStore sql_frag } + /// Returns the blocks from the start_block (inclusive) to the end_block (inclusive). fn get_block_ids_between( &mut self, start_block: &BlockId, @@ -275,33 +277,33 @@ impl<'a, TAddr: NodeAddressable + Serialize + DeserializeOwned> SqliteStateStore } } -impl StateStoreReadTransaction - for SqliteStateStoreReadTransaction<'_, TAddr> +impl<'tx, TAddr: NodeAddressable + Serialize + DeserializeOwned + 'tx> StateStoreReadTransaction + for SqliteStateStoreReadTransaction<'tx, TAddr> { type Addr = TAddr; - fn last_voted_get(&mut self) -> Result { - use crate::schema::last_voted; + fn last_sent_vote_get(&mut self) -> Result { + use crate::schema::last_sent_vote; - let last_voted = last_voted::table - .order_by(last_voted::id.desc()) - .first::(self.connection()) + let last_voted = last_sent_vote::table + .order_by(last_sent_vote::id.desc()) + .first::(self.connection()) .map_err(|e| SqliteStorageError::DieselError { - operation: "last_voted_get", + operation: "last_sent_vote_get", source: e, })?; last_voted.try_into() } - fn last_sent_vote_get(&mut self) -> Result { - use crate::schema::last_sent_vote; + fn last_voted_get(&mut self) -> Result { + use crate::schema::last_voted; - let last_voted = last_sent_vote::table - .order_by(last_sent_vote::id.desc()) - .first::(self.connection()) + let last_voted = last_voted::table + .order_by(last_voted::id.desc()) + .first::(self.connection()) .map_err(|e| SqliteStorageError::DieselError { - operation: "last_sent_vote_get", + operation: "last_voted_get", source: e, })?; @@ -667,6 +669,61 @@ impl StateStoreReadTransa .collect() } + fn blocks_exists(&mut self, block_id: &BlockId) -> Result { + use crate::schema::blocks; + + let count = blocks::table + .filter(blocks::block_id.eq(serialize_hex(block_id))) + .count() + .limit(1) + .get_result::(self.connection()) + .map_err(|e| SqliteStorageError::DieselError { + operation: "blocks_exists", + source: e, + })?; + + Ok(count > 0) + } + + fn blocks_is_ancestor(&mut self, descendant: &BlockId, ancestor: &BlockId) -> Result { + if !self.blocks_exists(descendant)? { + return Err(StorageError::QueryError { + reason: format!("blocks_is_ancestor: descendant block {} does not exist", descendant), + }); + } + + if !self.blocks_exists(ancestor)? { + return Err(StorageError::QueryError { + reason: format!("blocks_is_ancestor: ancestor block {} does not exist", ancestor), + }); + } + + // TODO: this scans all the way to genesis for every query - can optimise though it's low priority for now + let is_ancestor = sql_query( + r#" + WITH RECURSIVE tree(bid, parent) AS ( + SELECT block_id, parent_block_id FROM blocks where block_id = ? + UNION ALL + SELECT block_id, parent_block_id + FROM blocks JOIN tree ON block_id = tree.parent AND tree.bid != tree.parent -- stop recursing at zero block (or any self referencing block) + ) + SELECT count(1) as "count" FROM tree WHERE bid = ? LIMIT 1 + "#, + ) + .bind::(serialize_hex(descendant)) + // .bind::(serialize_hex(BlockId::genesis())) // stop recursing at zero block + .bind::(serialize_hex(ancestor)) + .get_result::(self.connection()) + .map_err(|e| SqliteStorageError::DieselError { + operation: "blocks_is_ancestor", + source: e, + })?; + + debug!(target: LOG_TARGET, "blocks_is_ancestor: is_ancestor: {}", is_ancestor.count); + + Ok(is_ancestor.count > 0) + } + fn blocks_get_all_by_parent(&mut self, parent_id: &BlockId) -> Result, StorageError> { use crate::schema::{blocks, quorum_certificates}; @@ -742,61 +799,6 @@ impl StateStoreReadTransa .collect() } - fn blocks_exists(&mut self, block_id: &BlockId) -> Result { - use crate::schema::blocks; - - let count = blocks::table - .filter(blocks::block_id.eq(serialize_hex(block_id))) - .count() - .limit(1) - .get_result::(self.connection()) - .map_err(|e| SqliteStorageError::DieselError { - operation: "blocks_exists", - source: e, - })?; - - Ok(count > 0) - } - - fn blocks_is_ancestor(&mut self, descendant: &BlockId, ancestor: &BlockId) -> Result { - if !self.blocks_exists(descendant)? { - return Err(StorageError::QueryError { - reason: format!("blocks_is_ancestor: descendant block {} does not exist", descendant), - }); - } - - if !self.blocks_exists(ancestor)? { - return Err(StorageError::QueryError { - reason: format!("blocks_is_ancestor: ancestor block {} does not exist", ancestor), - }); - } - - // TODO: this scans all the way to genesis for every query - can optimise though it's low priority for now - let is_ancestor = sql_query( - r#" - WITH RECURSIVE tree(bid, parent) AS ( - SELECT block_id, parent_block_id FROM blocks where block_id = ? - UNION ALL - SELECT block_id, parent_block_id - FROM blocks JOIN tree ON block_id = tree.parent AND tree.bid != tree.parent -- stop recursing at zero block (or any self referencing block) - ) - SELECT count(1) as "count" FROM tree WHERE bid = ? LIMIT 1 - "#, - ) - .bind::(serialize_hex(descendant)) - // .bind::(serialize_hex(BlockId::genesis())) // stop recursing at zero block - .bind::(serialize_hex(ancestor)) - .get_result::(self.connection()) - .map_err(|e| SqliteStorageError::DieselError { - operation: "blocks_is_ancestor", - source: e, - })?; - - debug!(target: LOG_TARGET, "blocks_is_ancestor: is_ancestor: {}", is_ancestor.count); - - Ok(is_ancestor.count > 0) - } - fn blocks_get_pending_transactions(&mut self, block_id: &BlockId) -> Result, StorageError> { use crate::schema::missing_transactions; @@ -1026,86 +1028,6 @@ impl StateStoreReadTransa deserialize_json(&qc_json) } - fn transactions_fetch_involved_shards( - &mut self, - transaction_ids: HashSet, - ) -> Result, StorageError> { - use crate::schema::transactions; - - let tx_ids = transaction_ids.into_iter().map(serialize_hex).collect::>(); - - let involved_shards = transactions::table - .select((transactions::inputs, transactions::input_refs)) - .filter(transactions::transaction_id.eq_any(tx_ids)) - .load::<(String, String)>(self.connection()) - .map_err(|e| SqliteStorageError::DieselError { - operation: "transaction_pools_fetch_involved_shards", - source: e, - })?; - - let shards = involved_shards - .into_iter() - .map(|(inputs, input_refs)| { - ( - // a Result is very inconvenient with flat_map - deserialize_json::>(&inputs).unwrap(), - deserialize_json::>(&input_refs).unwrap(), - ) - }) - .flat_map(|(inputs, input_refs)| inputs.into_iter().chain(input_refs).collect::>()) - .collect(); - - Ok(shards) - } - - fn votes_count_for_block(&mut self, block_id: &BlockId) -> Result { - use crate::schema::votes; - - let count = votes::table - .filter(votes::block_id.eq(serialize_hex(block_id))) - .count() - .first::(self.connection()) - .map_err(|e| SqliteStorageError::DieselError { - operation: "votes_count_for_block", - source: e, - })?; - - Ok(count as u64) - } - - fn votes_get_for_block(&mut self, block_id: &BlockId) -> Result, StorageError> { - use crate::schema::votes; - - let votes = votes::table - .filter(votes::block_id.eq(serialize_hex(block_id))) - .get_results::(self.connection()) - .map_err(|e| SqliteStorageError::DieselError { - operation: "votes_get_for_block", - source: e, - })?; - - votes.into_iter().map(Vote::try_from).collect() - } - - fn votes_get_by_block_and_sender( - &mut self, - block_id: &BlockId, - sender_leaf_hash: &FixedHash, - ) -> Result { - use crate::schema::votes; - - let vote = votes::table - .filter(votes::block_id.eq(serialize_hex(block_id))) - .filter(votes::sender_leaf_hash.eq(serialize_hex(sender_leaf_hash))) - .first::(self.connection()) - .map_err(|e| SqliteStorageError::DieselError { - operation: "votes_get", - source: e, - })?; - - Vote::try_from(vote) - } - fn transaction_pool_get( &mut self, from_block_id: &BlockId, @@ -1327,6 +1249,86 @@ impl StateStoreReadTransa Ok(count as usize) } + fn transactions_fetch_involved_shards( + &mut self, + transaction_ids: HashSet, + ) -> Result, StorageError> { + use crate::schema::transactions; + + let tx_ids = transaction_ids.into_iter().map(serialize_hex).collect::>(); + + let involved_shards = transactions::table + .select((transactions::inputs, transactions::input_refs)) + .filter(transactions::transaction_id.eq_any(tx_ids)) + .load::<(String, String)>(self.connection()) + .map_err(|e| SqliteStorageError::DieselError { + operation: "transaction_pools_fetch_involved_shards", + source: e, + })?; + + let shards = involved_shards + .into_iter() + .map(|(inputs, input_refs)| { + ( + // a Result is very inconvenient with flat_map + deserialize_json::>(&inputs).unwrap(), + deserialize_json::>(&input_refs).unwrap(), + ) + }) + .flat_map(|(inputs, input_refs)| inputs.into_iter().chain(input_refs).collect::>()) + .collect(); + + Ok(shards) + } + + fn votes_get_by_block_and_sender( + &mut self, + block_id: &BlockId, + sender_leaf_hash: &FixedHash, + ) -> Result { + use crate::schema::votes; + + let vote = votes::table + .filter(votes::block_id.eq(serialize_hex(block_id))) + .filter(votes::sender_leaf_hash.eq(serialize_hex(sender_leaf_hash))) + .first::(self.connection()) + .map_err(|e| SqliteStorageError::DieselError { + operation: "votes_get", + source: e, + })?; + + Vote::try_from(vote) + } + + fn votes_count_for_block(&mut self, block_id: &BlockId) -> Result { + use crate::schema::votes; + + let count = votes::table + .filter(votes::block_id.eq(serialize_hex(block_id))) + .count() + .first::(self.connection()) + .map_err(|e| SqliteStorageError::DieselError { + operation: "votes_count_for_block", + source: e, + })?; + + Ok(count as u64) + } + + fn votes_get_for_block(&mut self, block_id: &BlockId) -> Result, StorageError> { + use crate::schema::votes; + + let votes = votes::table + .filter(votes::block_id.eq(serialize_hex(block_id))) + .get_results::(self.connection()) + .map_err(|e| SqliteStorageError::DieselError { + operation: "votes_get_for_block", + source: e, + })?; + + votes.into_iter().map(Vote::try_from).collect() + } + fn substates_get(&mut self, address: &SubstateAddress) -> Result { use crate::schema::substates; @@ -1486,6 +1488,7 @@ impl StateStoreReadTransa .eq(&transaction_id_hex) .or(substates::destroyed_by_transaction.eq(Some(&transaction_id_hex))), ) + .order_by(substates::id.asc()) .get_results::(self.connection()) .map_err(|e| SqliteStorageError::DieselError { operation: "substates_get_all_for_transaction", @@ -1570,6 +1573,59 @@ impl StateStoreReadTransa Ok(SubstateLockState::LockAcquired) } } + + fn pending_state_tree_diffs_exists_for_block(&mut self, block_id: &BlockId) -> Result { + use crate::schema::pending_state_tree_diffs; + + let count = pending_state_tree_diffs::table + .count() + .filter(pending_state_tree_diffs::block_id.eq(serialize_hex(block_id))) + .first::(self.connection()) + .map_err(|e| SqliteStorageError::DieselError { + operation: "pending_state_tree_diffs_exists_for_block", + source: e, + })?; + + Ok(count > 0) + } + + fn pending_state_tree_diffs_get_all_up_to_commit_block( + &mut self, + block_id: &BlockId, + ) -> Result, StorageError> { + use crate::schema::{blocks, pending_state_tree_diffs}; + + // Get the last committed block + let committed_block_id = blocks::table + .select(blocks::block_id) + .filter(blocks::is_committed.eq(true)) + .order_by(blocks::height.desc()) + .first::(self.connection()) + .map(|s| deserialize_hex_try_from::(&s)) + .map_err(|e| SqliteStorageError::DieselError { + operation: "pending_state_tree_diffs_get_all_pending", + source: e, + })??; + + let mut block_ids = self.get_block_ids_between(&committed_block_id, block_id)?; + + // Exclude commit block + block_ids.pop(); + if block_ids.is_empty() { + return Ok(Vec::new()); + } + + let diffs = pending_state_tree_diffs::table + .filter(pending_state_tree_diffs::block_id.eq_any(block_ids)) + .order_by(pending_state_tree_diffs::block_height.asc()) + .get_results::(self.connection()) + .map_err(|e| SqliteStorageError::DieselError { + operation: "pending_state_tree_diffs_get_all_pending", + source: e, + })?; + + diffs.into_iter().map(TryInto::try_into).collect() + } } #[derive(QueryableByName)] diff --git a/dan_layer/state_store_sqlite/src/schema.rs b/dan_layer/state_store_sqlite/src/schema.rs index 17f0b487e..79453b285 100644 --- a/dan_layer/state_store_sqlite/src/schema.rs +++ b/dan_layer/state_store_sqlite/src/schema.rs @@ -5,6 +5,7 @@ diesel::table! { id -> Integer, block_id -> Text, parent_block_id -> Text, + merkle_root -> Text, network -> Text, height -> BigInt, epoch -> BigInt, @@ -144,6 +145,7 @@ diesel::table! { id -> Integer, block_id -> Text, parent_block_id -> Text, + merkle_root -> Text, network -> Text, height -> BigInt, epoch -> BigInt, @@ -158,6 +160,16 @@ diesel::table! { } } +diesel::table! { + pending_state_tree_diffs (id) { + id -> Integer, + block_id -> Text, + block_height -> BigInt, + diff_json -> Text, + created_at -> Timestamp, + } +} + diesel::table! { quorum_certificates (id) { id -> Integer, @@ -168,6 +180,15 @@ diesel::table! { } } +diesel::table! { + state_tree (id) { + id -> Integer, + key -> Text, + node -> Text, + is_stale -> Bool, + } +} + diesel::table! { substates (id) { id -> Integer, @@ -297,7 +318,9 @@ diesel::allow_tables_to_appear_in_same_query!( locked_outputs, missing_transactions, parked_blocks, + pending_state_tree_diffs, quorum_certificates, + state_tree, substates, transaction_pool, transaction_pool_history, diff --git a/dan_layer/state_store_sqlite/src/sql_models/block.rs b/dan_layer/state_store_sqlite/src/sql_models/block.rs index 87d0e1bbc..626b8c3e0 100644 --- a/dan_layer/state_store_sqlite/src/sql_models/block.rs +++ b/dan_layer/state_store_sqlite/src/sql_models/block.rs @@ -19,6 +19,7 @@ pub struct Block { pub id: i32, pub block_id: String, pub parent_block_id: String, + pub merkle_root: String, pub network: String, pub height: i64, pub epoch: i64, @@ -57,6 +58,7 @@ impl Block { } })?, deserialize_json(&self.commands)?, + deserialize_hex_try_from(&self.merkle_root)?, self.total_leader_fee as u64, self.is_dummy, self.is_processed, @@ -73,6 +75,7 @@ pub struct ParkedBlock { pub id: i32, pub block_id: String, pub parent_block_id: String, + pub merkle_root: String, pub network: String, pub height: i64, pub epoch: i64, @@ -110,6 +113,7 @@ impl TryFrom for consensus_models::Block { } })?, deserialize_json(&value.commands)?, + deserialize_hex_try_from(&value.merkle_root)?, value.total_leader_fee as u64, false, false, diff --git a/dan_layer/state_store_sqlite/src/sql_models/mod.rs b/dan_layer/state_store_sqlite/src/sql_models/mod.rs index d0cb91933..2fe49effa 100644 --- a/dan_layer/state_store_sqlite/src/sql_models/mod.rs +++ b/dan_layer/state_store_sqlite/src/sql_models/mod.rs @@ -4,6 +4,7 @@ mod block; mod bookkeeping; mod leaf_block; +mod pending_state_tree_diff; mod quorum_certificate; mod substate; mod transaction; @@ -13,6 +14,7 @@ mod vote; pub use block::*; pub use bookkeeping::*; pub use leaf_block::*; +pub use pending_state_tree_diff::*; pub use quorum_certificate::*; pub use substate::*; pub use transaction::*; diff --git a/dan_layer/state_store_sqlite/src/sql_models/pending_state_tree_diff.rs b/dan_layer/state_store_sqlite/src/sql_models/pending_state_tree_diff.rs new file mode 100644 index 000000000..de87dcc1e --- /dev/null +++ b/dan_layer/state_store_sqlite/src/sql_models/pending_state_tree_diff.rs @@ -0,0 +1,28 @@ +// Copyright 2023 The Tari Project +// SPDX-License-Identifier: BSD-3-Clause + +use diesel::Queryable; +use tari_dan_storage::{consensus_models, StorageError}; +use time::PrimitiveDateTime; + +use crate::serialization::{deserialize_hex_try_from, deserialize_json}; + +#[derive(Debug, Clone, Queryable)] +pub struct PendingStateTreeDiff { + pub id: i32, + pub block_id: String, + pub block_height: i64, + pub diff: String, + pub created_at: PrimitiveDateTime, +} + +impl TryFrom for consensus_models::PendingStateTreeDiff { + type Error = StorageError; + + fn try_from(value: PendingStateTreeDiff) -> Result { + let block_id = deserialize_hex_try_from(&value.block_id)?; + let block_height = value.block_height as u64; + let diff = deserialize_json(&value.diff)?; + Ok(Self::new(block_id, block_height.into(), diff)) + } +} diff --git a/dan_layer/state_store_sqlite/src/sqlite_transaction.rs b/dan_layer/state_store_sqlite/src/sqlite_transaction.rs index 218f54af6..f1ab11894 100644 --- a/dan_layer/state_store_sqlite/src/sqlite_transaction.rs +++ b/dan_layer/state_store_sqlite/src/sqlite_transaction.rs @@ -20,7 +20,7 @@ // WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE // USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -use std::sync::MutexGuard; +use std::{cell::UnsafeCell, sync::MutexGuard}; use diesel::{sql_query, RunQueryDsl, SqliteConnection}; @@ -29,22 +29,25 @@ use crate::error::SqliteStorageError; const _LOG_TARGET: &str = "tari::dan::storage::sqlite::transaction"; pub struct SqliteTransaction<'a> { - connection: MutexGuard<'a, SqliteConnection>, + connection: UnsafeCell>, is_done: bool, } impl<'a> SqliteTransaction<'a> { pub fn begin(connection: MutexGuard<'a, SqliteConnection>) -> Result { let mut this = Self { - connection, + connection: UnsafeCell::new(connection), is_done: false, }; this.execute_sql("BEGIN TRANSACTION")?; Ok(this) } - pub fn connection(&mut self) -> &mut SqliteConnection { - &mut self.connection + #[allow(clippy::mut_from_ref)] + pub fn connection(&self) -> &mut SqliteConnection { + // SAFETY: This is safe because only a single thread has access to this connection because SqliteStateStore + // transactions are obtained through a Mutex and UnsafeCell is not Sync + unsafe { &mut *self.connection.get() } } pub fn commit(mut self) -> Result<(), SqliteStorageError> { diff --git a/dan_layer/state_store_sqlite/src/tree_store.rs b/dan_layer/state_store_sqlite/src/tree_store.rs new file mode 100644 index 000000000..d728d7f85 --- /dev/null +++ b/dan_layer/state_store_sqlite/src/tree_store.rs @@ -0,0 +1,70 @@ +// Copyright 2024 The Tari Project +// SPDX-License-Identifier: BSD-3-Clause + +use std::ops::Deref; + +use diesel::{ExpressionMethods, OptionalExtension, QueryDsl, RunQueryDsl}; +use tari_state_tree::{Node, NodeKey, StaleTreeNode, TreeNode, TreeStoreReader, TreeStoreWriter, Version}; + +use crate::{reader::SqliteStateStoreReadTransaction, writer::SqliteStateStoreWriteTransaction}; + +impl<'a, TAddr> TreeStoreReader for SqliteStateStoreReadTransaction<'a, TAddr> { + fn get_node(&self, key: &NodeKey) -> Result, tari_state_tree::JmtStorageError> { + use crate::schema::state_tree; + + let node = state_tree::table + .select(state_tree::node) + .filter(state_tree::key.eq(key.to_string())) + .filter(state_tree::is_stale.eq(false)) + .first::(self.connection()) + .optional() + .map_err(|e| tari_state_tree::JmtStorageError::UnexpectedError(e.to_string()))? + .ok_or_else(|| tari_state_tree::JmtStorageError::NotFound(key.clone()))?; + + let node = serde_json::from_str::(&node) + .map_err(|e| tari_state_tree::JmtStorageError::UnexpectedError(e.to_string()))?; + + Ok(node.into_node()) + } +} + +impl<'a, TAddr> TreeStoreReader for SqliteStateStoreWriteTransaction<'a, TAddr> { + fn get_node(&self, key: &NodeKey) -> Result, tari_state_tree::JmtStorageError> { + self.deref().get_node(key) + } +} + +impl<'a, TAddr> TreeStoreWriter for SqliteStateStoreWriteTransaction<'a, TAddr> { + fn insert_node(&mut self, key: NodeKey, node: Node) -> Result<(), tari_state_tree::JmtStorageError> { + use crate::schema::state_tree; + + let node = TreeNode::new_latest(node); + let node = serde_json::to_string(&node) + .map_err(|e| tari_state_tree::JmtStorageError::UnexpectedError(e.to_string()))?; + + let values = (state_tree::key.eq(key.to_string()), state_tree::node.eq(node)); + diesel::insert_into(state_tree::table) + .values(&values) + .on_conflict(state_tree::key) + .do_update() + .set(values.clone()) + .execute(self.connection()) + .map_err(|e| tari_state_tree::JmtStorageError::UnexpectedError(e.to_string()))?; + + Ok(()) + } + + fn record_stale_tree_node(&mut self, node: StaleTreeNode) -> Result<(), tari_state_tree::JmtStorageError> { + use crate::schema::state_tree; + let key = node.as_node_key(); + diesel::update(state_tree::table) + .filter(state_tree::key.eq(key.to_string())) + .set(state_tree::is_stale.eq(true)) + .execute(self.connection()) + .optional() + .map_err(|e| tari_state_tree::JmtStorageError::UnexpectedError(e.to_string()))? + .ok_or_else(|| tari_state_tree::JmtStorageError::NotFound(key.clone()))?; + + Ok(()) + } +} diff --git a/dan_layer/state_store_sqlite/src/writer.rs b/dan_layer/state_store_sqlite/src/writer.rs index 2c6f970b8..c5e0bc940 100644 --- a/dan_layer/state_store_sqlite/src/writer.rs +++ b/dan_layer/state_store_sqlite/src/writer.rs @@ -27,6 +27,7 @@ use tari_dan_storage::{ LeafBlock, LockedBlock, LockedOutput, + PendingStateTreeDiff, QcId, QuorumCertificate, SubstateLockFlag, @@ -140,6 +141,7 @@ impl<'a, TAddr: NodeAddressable> SqliteStateStoreWriteTransaction<'a, TAddr> { parked_blocks::block_id.eq(&block_id), parked_blocks::parent_block_id.eq(serialize_hex(block.parent())), parked_blocks::network.eq(block.network().to_string()), + parked_blocks::merkle_root.eq(block.merkle_root().to_string()), parked_blocks::height.eq(block.height().as_u64() as i64), parked_blocks::epoch.eq(block.epoch().as_u64() as i64), parked_blocks::proposed_by.eq(serialize_hex(block.proposed_by().as_bytes())), @@ -162,7 +164,7 @@ impl<'a, TAddr: NodeAddressable> SqliteStateStoreWriteTransaction<'a, TAddr> { } } -impl StateStoreWriteTransaction for SqliteStateStoreWriteTransaction<'_, TAddr> { +impl<'tx, TAddr: NodeAddressable + 'tx> StateStoreWriteTransaction for SqliteStateStoreWriteTransaction<'tx, TAddr> { type Addr = TAddr; fn commit(mut self) -> Result<(), StorageError> { @@ -183,6 +185,7 @@ impl StateStoreWriteTransaction for SqliteStateStoreWrit let insert = ( blocks::block_id.eq(serialize_hex(block.id())), blocks::parent_block_id.eq(serialize_hex(block.parent())), + blocks::merkle_root.eq(block.merkle_root().to_string()), blocks::network.eq(block.network().to_string()), blocks::height.eq(block.height().as_u64() as i64), blocks::epoch.eq(block.epoch().as_u64() as i64), @@ -239,141 +242,6 @@ impl StateStoreWriteTransaction for SqliteStateStoreWrit Ok(()) } - fn missing_transactions_insert< - 'a, - IMissing: IntoIterator, - IAwaiting: IntoIterator, - >( - &mut self, - block: &Block, - missing_transaction_ids: IMissing, - awaiting_transaction_ids: IAwaiting, - ) -> Result<(), StorageError> { - use crate::schema::missing_transactions; - - let missing_transaction_ids = missing_transaction_ids - .into_iter() - .map(serialize_hex) - .collect::>(); - let awaiting_transaction_ids = awaiting_transaction_ids - .into_iter() - .map(serialize_hex) - .collect::>(); - let block_id_hex = serialize_hex(block.id()); - - self.parked_blocks_insert(block)?; - - let values = missing_transaction_ids - .iter() - .map(|tx_id| { - ( - missing_transactions::block_id.eq(&block_id_hex), - missing_transactions::block_height.eq(block.height().as_u64() as i64), - missing_transactions::transaction_id.eq(tx_id), - missing_transactions::is_awaiting_execution.eq(false), - ) - }) - .chain(awaiting_transaction_ids.iter().map(|tx_id| { - ( - missing_transactions::block_id.eq(&block_id_hex), - missing_transactions::block_height.eq(block.height().as_u64() as i64), - missing_transactions::transaction_id.eq(tx_id), - missing_transactions::is_awaiting_execution.eq(true), - ) - })) - .collect::>(); - - diesel::insert_into(missing_transactions::table) - .values(values) - .execute(self.connection()) - .map_err(|e| SqliteStorageError::DieselError { - operation: "missing_transactions_insert", - source: e, - })?; - - Ok(()) - } - - fn missing_transactions_remove( - &mut self, - current_height: NodeHeight, - transaction_id: &TransactionId, - ) -> Result, StorageError> { - use crate::schema::{missing_transactions, transactions}; - - // delete all entries that are for previous heights - diesel::delete(missing_transactions::table) - .filter(missing_transactions::block_height.lt(current_height.as_u64().saturating_sub(1) as i64)) - .execute(self.connection()) - .map_err(|e| SqliteStorageError::DieselError { - operation: "missing_transactions_remove", - source: e, - })?; - - let transaction_id = serialize_hex(transaction_id); - let block_id = missing_transactions::table - .select(missing_transactions::block_id) - .filter(missing_transactions::transaction_id.eq(&transaction_id)) - .filter(missing_transactions::block_height.eq(current_height.as_u64() as i64)) - .first::(self.connection()) - .optional() - .map_err(|e| SqliteStorageError::DieselError { - operation: "missing_transactions_remove", - source: e, - })?; - let Some(block_id) = block_id else { - return Ok(None); - }; - - diesel::delete(missing_transactions::table) - .filter(missing_transactions::transaction_id.eq(&transaction_id)) - .execute(self.connection()) - .map_err(|e| SqliteStorageError::DieselError { - operation: "missing_transactions_remove", - source: e, - })?; - let mut missing_transactions = missing_transactions::table - .select(missing_transactions::transaction_id) - .filter(missing_transactions::block_id.eq(&block_id)) - .get_results::(self.connection()) - .map_err(|e| SqliteStorageError::DieselError { - operation: "missing_transactions_remove", - source: e, - })?; - - if missing_transactions.is_empty() { - return self.parked_blocks_remove(&block_id).map(Some); - } - - // Make double sure that we dont have these transactions due to a race condition between inserting - // missing transactions and them completing execution. - let found_transaction_ids = transactions::table - .select(transactions::transaction_id) - .filter(transactions::transaction_id.eq_any(&missing_transactions)) - .filter(transactions::result.is_not_null()) - .get_results::(self.connection()) - .map_err(|e| SqliteStorageError::DieselError { - operation: "transactions_get_many", - source: e, - })?; - - diesel::delete(missing_transactions::table) - .filter(missing_transactions::transaction_id.eq_any(&found_transaction_ids)) - .execute(self.connection()) - .map_err(|e| SqliteStorageError::DieselError { - operation: "missing_transactions_remove", - source: e, - })?; - - missing_transactions.retain(|id| found_transaction_ids.iter().all(|found| found != id)); - - if missing_transactions.is_empty() { - return self.parked_blocks_remove(&block_id).map(Some); - } - - Ok(None) - } - fn quorum_certificates_insert(&mut self, qc: &QuorumCertificate) -> Result<(), StorageError> { use crate::schema::quorum_certificates; @@ -1001,6 +869,141 @@ impl StateStoreWriteTransaction for SqliteStateStoreWrit Ok(()) } + fn missing_transactions_insert< + 'a, + IMissing: IntoIterator, + IAwaiting: IntoIterator, + >( + &mut self, + block: &Block, + missing_transaction_ids: IMissing, + awaiting_transaction_ids: IAwaiting, + ) -> Result<(), StorageError> { + use crate::schema::missing_transactions; + + let missing_transaction_ids = missing_transaction_ids + .into_iter() + .map(serialize_hex) + .collect::>(); + let awaiting_transaction_ids = awaiting_transaction_ids + .into_iter() + .map(serialize_hex) + .collect::>(); + let block_id_hex = serialize_hex(block.id()); + + self.parked_blocks_insert(block)?; + + let values = missing_transaction_ids + .iter() + .map(|tx_id| { + ( + missing_transactions::block_id.eq(&block_id_hex), + missing_transactions::block_height.eq(block.height().as_u64() as i64), + missing_transactions::transaction_id.eq(tx_id), + missing_transactions::is_awaiting_execution.eq(false), + ) + }) + .chain(awaiting_transaction_ids.iter().map(|tx_id| { + ( + missing_transactions::block_id.eq(&block_id_hex), + missing_transactions::block_height.eq(block.height().as_u64() as i64), + missing_transactions::transaction_id.eq(tx_id), + missing_transactions::is_awaiting_execution.eq(true), + ) + })) + .collect::>(); + + diesel::insert_into(missing_transactions::table) + .values(values) + .execute(self.connection()) + .map_err(|e| SqliteStorageError::DieselError { + operation: "missing_transactions_insert", + source: e, + })?; + + Ok(()) + } + + fn missing_transactions_remove( + &mut self, + current_height: NodeHeight, + transaction_id: &TransactionId, + ) -> Result, StorageError> { + use crate::schema::{missing_transactions, transactions}; + + // delete all entries that are for previous heights + diesel::delete(missing_transactions::table) + .filter(missing_transactions::block_height.lt(current_height.as_u64().saturating_sub(1) as i64)) + .execute(self.connection()) + .map_err(|e| SqliteStorageError::DieselError { + operation: "missing_transactions_remove", + source: e, + })?; + + let transaction_id = serialize_hex(transaction_id); + let block_id = missing_transactions::table + .select(missing_transactions::block_id) + .filter(missing_transactions::transaction_id.eq(&transaction_id)) + .filter(missing_transactions::block_height.eq(current_height.as_u64() as i64)) + .first::(self.connection()) + .optional() + .map_err(|e| SqliteStorageError::DieselError { + operation: "missing_transactions_remove", + source: e, + })?; + let Some(block_id) = block_id else { + return Ok(None); + }; + + diesel::delete(missing_transactions::table) + .filter(missing_transactions::transaction_id.eq(&transaction_id)) + .execute(self.connection()) + .map_err(|e| SqliteStorageError::DieselError { + operation: "missing_transactions_remove", + source: e, + })?; + let mut missing_transactions = missing_transactions::table + .select(missing_transactions::transaction_id) + .filter(missing_transactions::block_id.eq(&block_id)) + .get_results::(self.connection()) + .map_err(|e| SqliteStorageError::DieselError { + operation: "missing_transactions_remove", + source: e, + })?; + + if missing_transactions.is_empty() { + return self.parked_blocks_remove(&block_id).map(Some); + } + + // Make double sure that we dont have these transactions due to a race condition between inserting + // missing transactions and them completing execution. + let found_transaction_ids = transactions::table + .select(transactions::transaction_id) + .filter(transactions::transaction_id.eq_any(&missing_transactions)) + .filter(transactions::result.is_not_null()) + .get_results::(self.connection()) + .map_err(|e| SqliteStorageError::DieselError { + operation: "transactions_get_many", + source: e, + })?; + + diesel::delete(missing_transactions::table) + .filter(missing_transactions::transaction_id.eq_any(&found_transaction_ids)) + .execute(self.connection()) + .map_err(|e| SqliteStorageError::DieselError { + operation: "missing_transactions_remove", + source: e, + })?; + + missing_transactions.retain(|id| found_transaction_ids.iter().all(|found| found != id)); + + if missing_transactions.is_empty() { + return self.parked_blocks_remove(&block_id).map(Some); + } + + Ok(None) + } + fn votes_insert(&mut self, vote: &Vote) -> Result<(), StorageError> { use crate::schema::votes; @@ -1368,6 +1371,51 @@ impl StateStoreWriteTransaction for SqliteStateStoreWrit // locked.into_iter().map(TryInto::try_into).collect() Ok(vec![]) } + + fn pending_state_tree_diffs_remove_by_block( + &mut self, + block_id: &BlockId, + ) -> Result { + use crate::schema::pending_state_tree_diffs; + + let diff = pending_state_tree_diffs::table + .filter(pending_state_tree_diffs::block_id.eq(serialize_hex(block_id))) + .first::(self.connection()) + .map_err(|e| SqliteStorageError::DieselError { + operation: "pending_state_tree_diffs_remove_by_block", + source: e, + })?; + + diesel::delete(pending_state_tree_diffs::table) + .filter(pending_state_tree_diffs::id.eq(diff.id)) + .execute(self.connection()) + .map_err(|e| SqliteStorageError::DieselError { + operation: "pending_state_tree_diffs_remove_by_block", + source: e, + })?; + + diff.try_into() + } + + fn pending_state_tree_diffs_insert(&mut self, pending_diff: &PendingStateTreeDiff) -> Result<(), StorageError> { + use crate::schema::pending_state_tree_diffs; + + let insert = ( + pending_state_tree_diffs::block_id.eq(serialize_hex(pending_diff.block_id)), + pending_state_tree_diffs::block_height.eq(pending_diff.block_height.as_u64() as i64), + pending_state_tree_diffs::diff_json.eq(serialize_json(&pending_diff.diff)?), + ); + + diesel::insert_into(pending_state_tree_diffs::table) + .values(insert) + .execute(self.connection()) + .map_err(|e| SqliteStorageError::DieselError { + operation: "pending_state_tree_diffs_insert", + source: e, + })?; + + Ok(()) + } } impl<'a, TAddr> Deref for SqliteStateStoreWriteTransaction<'a, TAddr> { diff --git a/dan_layer/state_store_sqlite/tests/tests.rs b/dan_layer/state_store_sqlite/tests/tests.rs index b59db3717..a08c44af6 100644 --- a/dan_layer/state_store_sqlite/tests/tests.rs +++ b/dan_layer/state_store_sqlite/tests/tests.rs @@ -58,6 +58,7 @@ mod confirm_all_transitions { [Command::Prepare(atom1.clone())].into_iter().collect(), Default::default(), Default::default(), + Default::default(), None, ); block1.insert(&mut tx).unwrap(); diff --git a/dan_layer/state_tree/Cargo.toml b/dan_layer/state_tree/Cargo.toml new file mode 100644 index 000000000..2be4945df --- /dev/null +++ b/dan_layer/state_tree/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "tari_state_tree" +version.workspace = true +edition.workspace = true +authors.workspace = true +repository.workspace = true +license.workspace = true + +[dependencies] +tari_dan_common_types = { workspace = true } +tari_engine_types = { workspace = true } +tari_template_lib = { workspace = true } +tari_common_types = { workspace = true } +tari_crypto = { workspace = true } + +hex = { workspace = true } +thiserror = { workspace = true } +serde = { workspace = true, features = ["derive"] } +log = { workspace = true } + +[dev-dependencies] +indexmap = { workspace = true } +itertools = { workspace = true } \ No newline at end of file diff --git a/dan_layer/state_tree/src/error.rs b/dan_layer/state_tree/src/error.rs new file mode 100644 index 000000000..0744fc9c0 --- /dev/null +++ b/dan_layer/state_tree/src/error.rs @@ -0,0 +1,10 @@ +// Copyright 2024 The Tari Project +// SPDX-License-Identifier: BSD-3-Clause + +use crate::jellyfish::JmtStorageError; + +#[derive(Debug, thiserror::Error)] +pub enum StateTreeError { + #[error("Storage error: {0}")] + StorageError(#[from] JmtStorageError), +} diff --git a/dan_layer/state_tree/src/jellyfish/hash.rs b/dan_layer/state_tree/src/jellyfish/hash.rs new file mode 100644 index 000000000..823dbb6e5 --- /dev/null +++ b/dan_layer/state_tree/src/jellyfish/hash.rs @@ -0,0 +1,134 @@ +// Copyright 2024 The Tari Project +// SPDX-License-Identifier: BSD-3-Clause + +// Copyright 2022. The Tari Project +// +// Redistribution and use in source and binary forms, with or without modification, are permitted provided that the +// following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following +// disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the +// following disclaimer in the documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote +// products derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, +// INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +// USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +use std::{ + convert::TryFrom, + fmt::{Display, Formatter}, + ops::{Deref, DerefMut}, +}; + +const ZERO_HASH: [u8; TreeHash::byte_size()] = [0u8; TreeHash::byte_size()]; + +#[derive(thiserror::Error, Debug)] +#[error("Invalid size")] +pub struct TreeHashSizeError; + +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug, Default, Hash)] +pub struct TreeHash([u8; TreeHash::byte_size()]); + +impl TreeHash { + pub const fn new(hash: [u8; TreeHash::byte_size()]) -> Self { + Self(hash) + } + + pub const fn byte_size() -> usize { + 32 + } + + pub const fn zero() -> Self { + Self(ZERO_HASH) + } + + pub fn as_slice(&self) -> &[u8] { + &self.0 + } +} + +impl From<[u8; TreeHash::byte_size()]> for TreeHash { + fn from(hash: [u8; TreeHash::byte_size()]) -> Self { + Self(hash) + } +} + +impl TryFrom> for TreeHash { + type Error = TreeHashSizeError; + + fn try_from(value: Vec) -> Result { + TryFrom::try_from(value.as_slice()) + } +} + +impl TryFrom<&[u8]> for TreeHash { + type Error = TreeHashSizeError; + + fn try_from(bytes: &[u8]) -> Result { + if bytes.len() != TreeHash::byte_size() { + return Err(TreeHashSizeError); + } + + let mut buf = [0u8; TreeHash::byte_size()]; + buf.copy_from_slice(bytes); + Ok(Self(buf)) + } +} + +impl PartialEq<[u8]> for TreeHash { + fn eq(&self, other: &[u8]) -> bool { + self.0[..].eq(other) + } +} + +impl PartialEq for [u8] { + fn eq(&self, other: &TreeHash) -> bool { + self[..].eq(&other.0) + } +} + +impl PartialEq> for TreeHash { + fn eq(&self, other: &Vec) -> bool { + self == other.as_slice() + } +} +impl PartialEq for Vec { + fn eq(&self, other: &TreeHash) -> bool { + self == other.as_slice() + } +} + +impl AsRef<[u8]> for TreeHash { + fn as_ref(&self) -> &[u8] { + self.as_slice() + } +} + +impl Deref for TreeHash { + type Target = [u8; TreeHash::byte_size()]; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for TreeHash { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +impl Display for TreeHash { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + hex::encode(&self.0).fmt(f) + } +} diff --git a/dan_layer/state_tree/src/jellyfish/mod.rs b/dan_layer/state_tree/src/jellyfish/mod.rs new file mode 100644 index 000000000..e4b8bb4ca --- /dev/null +++ b/dan_layer/state_tree/src/jellyfish/mod.rs @@ -0,0 +1,14 @@ +// Copyright 2024 The Tari Project +// SPDX-License-Identifier: BSD-3-Clause + +// mod hash; +// pub use hash::*; + +mod tree; +pub use tree::*; + +mod types; +pub use types::*; + +mod store; +pub use store::*; diff --git a/dan_layer/state_tree/src/jellyfish/store.rs b/dan_layer/state_tree/src/jellyfish/store.rs new file mode 100644 index 000000000..65d1c78f3 --- /dev/null +++ b/dan_layer/state_tree/src/jellyfish/store.rs @@ -0,0 +1,79 @@ +// Copyright 2024 The Tari Project +// SPDX-License-Identifier: BSD-3-Clause + +use serde::{Deserialize, Serialize}; + +use crate::{ + jellyfish::{JmtStorageError, Node, NodeKey}, + Version, +}; + +/// Implementers are able to read nodes from a tree store. +pub trait TreeStoreReader

{ + /// Gets node by key, if it exists. + fn get_node(&self, key: &NodeKey) -> Result, JmtStorageError>; +} + +/// Implementers are able to insert nodes to a tree store. +pub trait TreeStoreWriter

{ + /// Inserts the node under a new, unique key (i.e. never an update). + fn insert_node(&mut self, key: NodeKey, node: Node

) -> Result<(), JmtStorageError>; + + /// Marks the given tree part for a (potential) future removal by an arbitrary external pruning + /// process. + fn record_stale_tree_node(&mut self, part: StaleTreeNode) -> Result<(), JmtStorageError>; +} + +/// Implementers are able to read and write nodes to a tree store. +pub trait TreeStore

: TreeStoreReader

+ TreeStoreWriter

{} +impl + TreeStoreWriter

> TreeStore

for S {} + +/// A part of a tree that may become stale (i.e. need eventual pruning). +#[derive(Clone, PartialEq, Eq, Hash, Debug, Serialize, Deserialize)] +pub enum StaleTreeNode { + /// A single node to be removed. + Node(NodeKey), + /// An entire subtree of descendants of a specific node (including itself). + Subtree(NodeKey), +} + +impl StaleTreeNode { + pub fn into_node_key(self) -> NodeKey { + match self { + Self::Node(key) | Self::Subtree(key) => key, + } + } + + pub fn as_node_key(&self) -> &NodeKey { + match self { + Self::Node(key) | Self::Subtree(key) => key, + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum TreeNode { + V1(Node), +} + +impl TreeNode { + pub fn new_latest(node: Node) -> Self { + Self::new_v1(node) + } + + pub fn new_v1(node: Node) -> Self { + Self::V1(node) + } + + pub fn as_node(&self) -> &Node { + match self { + Self::V1(node) => node, + } + } + + pub fn into_node(self) -> Node { + match self { + Self::V1(node) => node, + } + } +} diff --git a/dan_layer/state_tree/src/jellyfish/tree.rs b/dan_layer/state_tree/src/jellyfish/tree.rs new file mode 100644 index 000000000..7eeaee976 --- /dev/null +++ b/dan_layer/state_tree/src/jellyfish/tree.rs @@ -0,0 +1,733 @@ +// Copyright 2024 The Tari Project +// SPDX-License-Identifier: BSD-3-Clause + +// Copyright 2021 Radix Publishing Ltd incorporated in Jersey (Channel Islands). +// +// Licensed under the Radix License, Version 1.0 (the "License"); you may not use this +// file except in compliance with the License. You may obtain a copy of the License at: +// +// radixfoundation.org/licenses/LICENSE-v1 +// +// The Licensor hereby grants permission for the Canonical version of the Work to be +// published, distributed and used under or by reference to the Licensor's trademark +// Radix ® and use of any unregistered trade names, logos or get-up. +// +// The Licensor provides the Work (and each Contributor provides its Contributions) on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, +// including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, +// MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. +// +// Whilst the Work is capable of being deployed, used and adopted (instantiated) to create +// a distributed ledger it is your responsibility to test and validate the code, together +// with all logic and performance of that code under all foreseeable scenarios. +// +// The Licensor does not make or purport to make and hereby excludes liability for all +// and any representation, warranty or undertaking in any form whatsoever, whether express +// or implied, to any entity or person, including any representation, warranty or +// undertaking, as to the functionality security use, value or other characteristics of +// any distributed ledger nor in respect the functioning or value of any tokens which may +// be created stored or transferred using the Work. The Licensor does not warrant that the +// Work or any use of the Work complies with any law or regulation in any territory where +// it may be implemented or used or that it will be appropriate for any specific purpose. +// +// Neither the licensor nor any current or former employees, officers, directors, partners, +// trustees, representatives, agents, advisors, contractors, or volunteers of the Licensor +// shall be liable for any direct or indirect, special, incidental, consequential or other +// losses of any kind, in tort, contract or otherwise (including but not limited to loss +// of revenue, income or profits, or loss of use or data, or loss of reputation, or loss +// of any economic or other opportunity of whatsoever nature or howsoever arising), arising +// out of or in connection with (without limitation of any use, misuse, of any ledger system +// or use made or its functionality or any performance or operation of any code or protocol +// caused by bugs or programming or logic errors or otherwise); +// +// A. any offer, purchase, holding, use, sale, exchange or transmission of any +// cryptographic keys, tokens or assets created, exchanged, stored or arising from any +// interaction with the Work; +// +// B. any failure in a transmission or loss of any token or assets keys or other digital +// artefacts due to errors in transmission; +// +// C. bugs, hacks, logic errors or faults in the Work or any communication; +// +// D. system software or apparatus including but not limited to losses caused by errors +// in holding or transmitting tokens by any third-party; +// +// E. breaches or failure of security including hacker attacks, loss or disclosure of +// password, loss of private key, unauthorised use or misuse of such passwords or keys; +// +// F. any losses including loss of anticipated savings or other benefits resulting from +// use of the Work or any changes to the Work (however implemented). +// +// You are solely responsible for; testing, validating and evaluation of all operation +// logic, functionality, security and appropriateness of using the Work for any commercial +// or non-commercial purpose and for any reproduction or redistribution by You of the +// Work. You assume all risks associated with Your use of the Work and the exercise of +// permissions under this License. + +// This file contains code sourced from https://github.com/aptos-labs/aptos-core/tree/1.0.4 +// This original source is licensed under https://github.com/aptos-labs/aptos-core/blob/1.0.4/LICENSE +// +// The code in this file has been implemented by Radix® pursuant to an Apache 2 licence and has +// been modified by Radix® and is now licensed pursuant to the Radix® Open-Source Licence. +// +// Each sourced code fragment includes an inline attribution to the original source file in a +// comment starting "SOURCE: ..." +// +// Modifications from the original source are captured in two places: +// * Initial changes to get the code functional/integrated are marked by inline "INITIAL-MODIFICATION: ..." comments +// * Subsequent changes to the code are captured in the git commit history +// +// The following notice is retained from the original source +// Copyright (c) Aptos +// SPDX-License-Identifier: Apache-2.0 + +use std::{ + collections::{BTreeMap, HashMap}, + marker::PhantomData, +}; + +use super::{ + store::TreeStoreReader, + types::{ + Child, + InternalNode, + IteratedLeafKey, + JmtStorageError, + LeafKey, + LeafNode, + Nibble, + NibblePath, + Node, + NodeKey, + SparseMerkleProof, + SparseMerkleProofExt, + SparseMerkleRangeProof, + Version, + SPARSE_MERKLE_PLACEHOLDER_HASH, + }, + Hash, + LeafKeyRef, +}; + +// INITIAL-MODIFICATION: the original used a known key size (32) as a limit +const SANITY_NIBBLE_LIMIT: usize = 1000; + +pub type ProofValue

= (Hash, P, Version); + +// SOURCE: https://github.com/radixdlt/radixdlt-scrypto/blob/ca8e553c31a956c0851c1855291efe4a47fb5c97/radix-engine-stores/src/hash_tree/jellyfish.rs +// SOURCE: https://github.com/aptos-labs/aptos-core/blob/1.0.4/storage/jellyfish-merkle/src/lib.rs#L329 +/// The Jellyfish Merkle tree data structure. See [`crate`] for description. +pub struct JellyfishMerkleTree<'a, R, P> { + reader: &'a R, + _payload: PhantomData

, +} + +impl<'a, R: 'a + TreeStoreReader

, P: Clone> JellyfishMerkleTree<'a, R, P> { + /// Creates a `JellyfishMerkleTree` backed by the given [`TreeReader`](trait.TreeReader.html). + pub fn new(reader: &'a R) -> Self { + Self { + reader, + _payload: PhantomData, + } + } + + /// Get the node hash from the cache if cache is provided, otherwise (for test only) compute it. + fn get_hash(node_key: &NodeKey, node: &Node

, hash_cache: Option<&HashMap>) -> Hash { + if let Some(cache) = hash_cache { + match cache.get(node_key.nibble_path()) { + Some(hash) => *hash, + None => unreachable!("{:?} can not be found in hash cache", node_key), + } + } else { + node.hash() + } + } + + /// For each value set: + /// Returns the new nodes and values in a batch after applying `value_set`. For + /// example, if after transaction `T_i` the committed state of tree in the persistent storage + /// looks like the following structure: + /// + /// ```text + /// S_i + /// / \ + /// . . + /// . . + /// / \ + /// o x + /// / \ + /// A B + /// storage (disk) + /// ``` + /// + /// where `A` and `B` denote the states of two adjacent accounts, and `x` is a sibling subtree + /// of the path from root to A and B in the tree. Then a `value_set` produced by the next + /// transaction `T_{i+1}` modifies other accounts `C` and `D` exist in the subtree under `x`, a + /// new partial tree will be constructed in memory and the structure will be: + /// + /// ```text + /// S_i | S_{i+1} + /// / \ | / \ + /// . . | . . + /// . . | . . + /// / \ | / \ + /// / x | / x' + /// o<-------------+- / \ + /// / \ | C D + /// A B | + /// storage (disk) | cache (memory) + /// ``` + /// + /// With this design, we are able to query the global state in persistent storage and + /// generate the proposed tree delta based on a specific root hash and `value_set`. For + /// example, if we want to execute another transaction `T_{i+1}'`, we can use the tree `S_i` in + /// storage and apply the `value_set` of transaction `T_{i+1}`. Then if the storage commits + /// the returned batch, the state `S_{i+1}` is ready to be read from the tree by calling + /// [`get_with_proof`](struct.JellyfishMerkleTree.html#method.get_with_proof). Anything inside + /// the batch is not reachable from public interfaces before being committed. + pub fn batch_put_value_set( + &self, + value_set: Vec<(&LeafKey, Option<&(Hash, P)>)>, + node_hashes: Option<&HashMap>, + persisted_version: Option, + version: Version, + ) -> Result<(Hash, TreeUpdateBatch

), JmtStorageError> { + let deduped_and_sorted_kvs = value_set + .into_iter() + .collect::>() + .into_iter() + .collect::>(); + + let mut batch = TreeUpdateBatch::new(); + let root_node_opt = if let Some(persisted_version) = persisted_version { + self.batch_insert_at( + &NodeKey::new_empty_path(persisted_version), + version, + deduped_and_sorted_kvs.as_slice(), + 0, + node_hashes, + &mut batch, + )? + } else { + Self::batch_update_subtree( + &NodeKey::new_empty_path(version), + version, + deduped_and_sorted_kvs.as_slice(), + 0, + node_hashes, + &mut batch, + )? + }; + + let node_key = NodeKey::new_empty_path(version); + let root_hash = if let Some(root_node) = root_node_opt { + let hash = root_node.hash(); + batch.put_node(node_key, root_node); + hash + } else { + batch.put_node(node_key, Node::Null); + SPARSE_MERKLE_PLACEHOLDER_HASH + }; + + Ok((root_hash, batch)) + } + + fn batch_insert_at( + &self, + node_key: &NodeKey, + version: Version, + kvs: &[(&LeafKey, Option<&(Hash, P)>)], + depth: usize, + hash_cache: Option<&HashMap>, + batch: &mut TreeUpdateBatch

, + ) -> Result>, JmtStorageError> { + let node = self.reader.get_node(node_key)?; + batch.put_stale_node(node_key.clone(), version, &node); + + match node { + Node::Internal(internal_node) => { + // There is a small possibility that the old internal node is intact. + // Traverse all the path touched by `kvs` from this internal node. + let range_iter = NibbleRangeIterator::new(kvs, depth); + // INITIAL-MODIFICATION: there was a par_iter (conditionally) used here + let new_children = range_iter + .map(|(left, right)| { + self.insert_at_child( + node_key, + &internal_node, + version, + kvs, + left, + right, + depth, + hash_cache, + batch, + ) + }) + .collect::, JmtStorageError>>()?; + + // Reuse the current `InternalNode` in memory to create a new internal node. + let mut old_children = internal_node.into_children(); + let mut new_created_children: HashMap> = HashMap::new(); + for (child_nibble, child_option) in new_children { + if let Some(child) = child_option { + new_created_children.insert(child_nibble, child); + } else { + old_children.remove(&child_nibble); + } + } + + if old_children.is_empty() && new_created_children.is_empty() { + return Ok(None); + } + if old_children.len() <= 1 && new_created_children.len() <= 1 { + if let Some((new_nibble, new_child)) = new_created_children.iter().next() { + if let Some((old_nibble, _old_child)) = old_children.iter().next() { + if old_nibble == new_nibble && new_child.is_leaf() { + return Ok(Some(new_child.clone())); + } + } else if new_child.is_leaf() { + return Ok(Some(new_child.clone())); + } else { + // Nothing to do + } + } else { + let (old_child_nibble, old_child) = old_children.iter().next().expect("must exist"); + if old_child.is_leaf() { + let old_child_node_key = node_key.gen_child_node_key(old_child.version, *old_child_nibble); + let old_child_node = self.reader.get_node(&old_child_node_key)?; + batch.put_stale_node(old_child_node_key, version, &old_child_node); + return Ok(Some(old_child_node)); + } + } + } + + let mut new_children = old_children; + for (child_index, new_child_node) in new_created_children { + let new_child_node_key = node_key.gen_child_node_key(version, child_index); + new_children.insert( + child_index, + Child::new( + Self::get_hash(&new_child_node_key, &new_child_node, hash_cache), + version, + new_child_node.node_type(), + ), + ); + batch.put_node(new_child_node_key, new_child_node); + } + let new_internal_node = InternalNode::new(new_children); + Ok(Some(new_internal_node.into())) + }, + Node::Leaf(leaf_node) => Self::batch_update_subtree_with_existing_leaf( + node_key, version, leaf_node, kvs, depth, hash_cache, batch, + ), + Node::Null => { + assert_eq!(depth, 0, "Null node can only exist at depth 0"); + Self::batch_update_subtree(node_key, version, kvs, 0, hash_cache, batch) + }, + } + } + + fn insert_at_child( + &self, + node_key: &NodeKey, + internal_node: &InternalNode, + version: Version, + kvs: &[(&LeafKey, Option<&(Hash, P)>)], + left: usize, + right: usize, + depth: usize, + hash_cache: Option<&HashMap>, + batch: &mut TreeUpdateBatch

, + ) -> Result<(Nibble, Option>), JmtStorageError> { + let child_index = kvs[left].0.get_nibble(depth); + let child = internal_node.child(child_index); + + let new_child_node_option = match child { + Some(child) => self.batch_insert_at( + &node_key.gen_child_node_key(child.version, child_index), + version, + &kvs[left..=right], + depth + 1, + hash_cache, + batch, + )?, + None => Self::batch_update_subtree( + &node_key.gen_child_node_key(version, child_index), + version, + &kvs[left..=right], + depth + 1, + hash_cache, + batch, + )?, + }; + + Ok((child_index, new_child_node_option)) + } + + fn batch_update_subtree_with_existing_leaf( + node_key: &NodeKey, + version: Version, + existing_leaf_node: LeafNode

, + kvs: &[(&LeafKey, Option<&(Hash, P)>)], + depth: usize, + hash_cache: Option<&HashMap>, + batch: &mut TreeUpdateBatch

, + ) -> Result>, JmtStorageError> { + let existing_leaf_key = existing_leaf_node.leaf_key(); + + if kvs.len() == 1 && kvs[0].0 == existing_leaf_key { + if let (key, Some((value_hash, payload))) = kvs[0] { + let new_leaf_node = Node::new_leaf(key.clone(), *value_hash, payload.clone(), version); + Ok(Some(new_leaf_node)) + } else { + Ok(None) + } + } else { + let existing_leaf_bucket = existing_leaf_key.get_nibble(depth); + let mut isolated_existing_leaf = true; + let mut children = vec![]; + for (left, right) in NibbleRangeIterator::new(kvs, depth) { + let child_index = kvs[left].0.get_nibble(depth); + let child_node_key = node_key.gen_child_node_key(version, child_index); + if let Some(new_child_node) = if existing_leaf_bucket == child_index { + isolated_existing_leaf = false; + Self::batch_update_subtree_with_existing_leaf( + &child_node_key, + version, + existing_leaf_node.clone(), + &kvs[left..=right], + depth + 1, + hash_cache, + batch, + )? + } else { + Self::batch_update_subtree( + &child_node_key, + version, + &kvs[left..=right], + depth + 1, + hash_cache, + batch, + )? + } { + children.push((child_index, new_child_node)); + } + } + if isolated_existing_leaf { + children.push((existing_leaf_bucket, existing_leaf_node.into())); + } + + if children.is_empty() { + Ok(None) + } else if children.len() == 1 && children[0].1.is_leaf() { + let (_, child) = children.pop().expect("Must exist"); + Ok(Some(child)) + } else { + let new_internal_node = InternalNode::new( + children + .into_iter() + .map(|(child_index, new_child_node)| { + let new_child_node_key = node_key.gen_child_node_key(version, child_index); + let result = ( + child_index, + Child::new( + Self::get_hash(&new_child_node_key, &new_child_node, hash_cache), + version, + new_child_node.node_type(), + ), + ); + batch.put_node(new_child_node_key, new_child_node); + result + }) + .collect(), + ); + Ok(Some(new_internal_node.into())) + } + } + } + + fn batch_update_subtree( + node_key: &NodeKey, + version: Version, + kvs: &[(&LeafKey, Option<&(Hash, P)>)], + depth: usize, + hash_cache: Option<&HashMap>, + batch: &mut TreeUpdateBatch

, + ) -> Result>, JmtStorageError> { + if kvs.len() == 1 { + if let (key, Some((value_hash, payload))) = kvs[0] { + let new_leaf_node = Node::new_leaf(key.clone(), *value_hash, payload.clone(), version); + Ok(Some(new_leaf_node)) + } else { + Ok(None) + } + } else { + let mut children = vec![]; + for (left, right) in NibbleRangeIterator::new(kvs, depth) { + let child_index = kvs[left].0.get_nibble(depth); + let child_node_key = node_key.gen_child_node_key(version, child_index); + if let Some(new_child_node) = Self::batch_update_subtree( + &child_node_key, + version, + &kvs[left..=right], + depth + 1, + hash_cache, + batch, + )? { + children.push((child_index, new_child_node)) + } + } + if children.is_empty() { + Ok(None) + } else if children.len() == 1 && children[0].1.is_leaf() { + let (_, child) = children.pop().expect("Must exist"); + Ok(Some(child)) + } else { + let new_internal_node = InternalNode::new( + children + .into_iter() + .map(|(child_index, new_child_node)| { + let new_child_node_key = node_key.gen_child_node_key(version, child_index); + let result = ( + child_index, + Child::new( + Self::get_hash(&new_child_node_key, &new_child_node, hash_cache), + version, + new_child_node.node_type(), + ), + ); + batch.put_node(new_child_node_key, new_child_node); + result + }) + .collect(), + ); + Ok(Some(new_internal_node.into())) + } + } + } + + /// Returns the value (if applicable) and the corresponding merkle proof. + pub fn get_with_proof( + &self, + key: LeafKeyRef<'_>, + version: Version, + ) -> Result<(Option>, SparseMerkleProof), JmtStorageError> { + self.get_with_proof_ext(key, version) + .map(|(value, proof_ext)| (value, proof_ext.into())) + } + + pub fn get_with_proof_ext( + &self, + key: LeafKeyRef<'_>, + version: Version, + ) -> Result<(Option>, SparseMerkleProofExt), JmtStorageError> { + // Empty tree just returns proof with no sibling hash. + let mut next_node_key = NodeKey::new_empty_path(version); + let mut siblings = vec![]; + let nibble_path = NibblePath::new_even(key.bytes.to_vec()); + let mut nibble_iter = nibble_path.nibbles(); + + for _nibble_depth in 0..SANITY_NIBBLE_LIMIT { + let next_node = self.reader.get_node(&next_node_key)?; + match next_node { + Node::Internal(internal_node) => { + let queried_child_index = nibble_iter.next().ok_or(JmtStorageError::InconsistentState)?; + let (child_node_key, mut siblings_in_internal) = internal_node.get_child_with_siblings( + &next_node_key, + queried_child_index, + Some(self.reader), + )?; + siblings.append(&mut siblings_in_internal); + next_node_key = match child_node_key { + Some(node_key) => node_key, + None => { + return Ok(( + None, + SparseMerkleProofExt::new(None, { + siblings.reverse(); + siblings + }), + )) + }, + }; + }, + Node::Leaf(leaf_node) => { + return Ok(( + if leaf_node.leaf_key().as_ref() == key { + Some((leaf_node.value_hash(), leaf_node.payload().clone(), leaf_node.version())) + } else { + None + }, + SparseMerkleProofExt::new(Some(leaf_node.into()), { + siblings.reverse(); + siblings + }), + )); + }, + Node::Null => { + return Ok((None, SparseMerkleProofExt::new(None, vec![]))); + }, + } + } + Err(JmtStorageError::InconsistentState) + } + + /// Gets the proof that shows a list of keys up to `rightmost_key_to_prove` exist at `version`. + pub fn get_range_proof( + &self, + rightmost_key_to_prove: LeafKeyRef<'_>, + version: Version, + ) -> Result { + let (leaf, proof) = self.get_with_proof(rightmost_key_to_prove, version)?; + assert!(leaf.is_some(), "rightmost_key_to_prove must exist."); + + let siblings = proof + .siblings() + .iter() + .rev() + .zip(rightmost_key_to_prove.iter_bits()) + .filter_map(|(sibling, bit)| { + // We only need to keep the siblings on the right. + if bit { + None + } else { + Some(*sibling) + } + }) + .rev() + .collect(); + Ok(SparseMerkleRangeProof::new(siblings)) + } + + fn get_root_node(&self, version: Version) -> Result, JmtStorageError> { + let root_node_key = NodeKey::new_empty_path(version); + self.reader.get_node(&root_node_key) + } + + pub fn get_root_hash(&self, version: Version) -> Result { + self.get_root_node(version).map(|n| n.hash()) + } + + pub fn get_leaf_count(&self, version: Version) -> Result { + self.get_root_node(version).map(|n| n.leaf_count()) + } + + pub fn get_all_nodes_referenced(&self, key: NodeKey) -> Result, JmtStorageError> { + let mut out_keys = vec![]; + self.get_all_nodes_referenced_impl(key, &mut out_keys)?; + Ok(out_keys) + } + + fn get_all_nodes_referenced_impl(&self, key: NodeKey, out_keys: &mut Vec) -> Result<(), JmtStorageError> { + match self.reader.get_node(&key)? { + Node::Internal(internal_node) => { + for (child_nibble, child) in internal_node.children_sorted() { + self.get_all_nodes_referenced_impl(key.gen_child_node_key(child.version, *child_nibble), out_keys)?; + } + }, + Node::Leaf(_) | Node::Null => {}, + }; + + out_keys.push(key); + Ok(()) + } +} + +/// An iterator that iterates the index range (inclusive) of each different nibble at given +/// `nibble_idx` of all the keys in a sorted key-value pairs which have the identical Hash +/// prefix (up to nibble_idx). +struct NibbleRangeIterator<'a, P> { + sorted_kvs: &'a [(&'a LeafKey, P)], + nibble_idx: usize, + pos: usize, +} + +impl<'a, P> NibbleRangeIterator<'a, P> { + fn new(sorted_kvs: &'a [(&'a LeafKey, P)], nibble_idx: usize) -> Self { + NibbleRangeIterator { + sorted_kvs, + nibble_idx, + pos: 0, + } + } +} + +impl<'a, P> Iterator for NibbleRangeIterator<'a, P> { + type Item = (usize, usize); + + fn next(&mut self) -> Option { + let left = self.pos; + if self.pos < self.sorted_kvs.len() { + let cur_nibble = self.sorted_kvs[left].0.get_nibble(self.nibble_idx); + let (mut i, mut j) = (left, self.sorted_kvs.len() - 1); + // Find the last index of the cur_nibble. + while i < j { + let mid = j - (j - i) / 2; + if self.sorted_kvs[mid].0.get_nibble(self.nibble_idx) > cur_nibble { + j = mid - 1; + } else { + i = mid; + } + } + self.pos = i + 1; + Some((left, i)) + } else { + None + } + } +} + +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub struct TreeUpdateBatch

{ + pub node_batch: Vec<(NodeKey, Node

)>, + pub stale_node_index_batch: Vec, + pub num_new_leaves: usize, + pub num_stale_leaves: usize, +} + +impl TreeUpdateBatch

{ + pub fn new() -> Self { + Self { + node_batch: vec![], + stale_node_index_batch: vec![], + num_new_leaves: 0, + num_stale_leaves: 0, + } + } + + fn inc_num_new_leaves(&mut self) { + self.num_new_leaves += 1; + } + + fn inc_num_stale_leaves(&mut self) { + self.num_stale_leaves += 1; + } + + pub fn put_node(&mut self, node_key: NodeKey, node: Node

) { + if node.is_leaf() { + self.inc_num_new_leaves(); + } + self.node_batch.push((node_key, node)) + } + + pub fn put_stale_node(&mut self, node_key: NodeKey, stale_since_version: Version, node: &Node

) { + if node.is_leaf() { + self.inc_num_stale_leaves(); + } + self.stale_node_index_batch.push(StaleNodeIndex { + node_key, + stale_since_version, + }); + } +} + +/// Indicates a node becomes stale since `stale_since_version`. +#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct StaleNodeIndex { + /// The version since when the node is overwritten and becomes stale. + pub stale_since_version: Version, + /// The [`NodeKey`](node_type/struct.NodeKey.html) identifying the node associated with this + /// record. + pub node_key: NodeKey, +} diff --git a/dan_layer/state_tree/src/jellyfish/types.rs b/dan_layer/state_tree/src/jellyfish/types.rs new file mode 100644 index 000000000..784974832 --- /dev/null +++ b/dan_layer/state_tree/src/jellyfish/types.rs @@ -0,0 +1,1249 @@ +// Copyright 2024 The Tari Project +// SPDX-License-Identifier: BSD-3-Clause + +// Copyright 2021 Radix Publishing Ltd incorporated in Jersey (Channel Islands). +// +// Licensed under the Radix License, Version 1.0 (the "License"); you may not use this +// file except in compliance with the License. You may obtain a copy of the License at: +// +// radixfoundation.org/licenses/LICENSE-v1 +// +// The Licensor hereby grants permission for the Canonical version of the Work to be +// published, distributed and used under or by reference to the Licensor's trademark +// Radix ® and use of any unregistered trade names, logos or get-up. +// +// The Licensor provides the Work (and each Contributor provides its Contributions) on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, +// including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, +// MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. +// +// Whilst the Work is capable of being deployed, used and adopted (instantiated) to create +// a distributed ledger it is your responsibility to test and validate the code, together +// with all logic and performance of that code under all foreseeable scenarios. +// +// The Licensor does not make or purport to make and hereby excludes liability for all +// and any representation, warranty or undertaking in any form whatsoever, whether express +// or implied, to any entity or person, including any representation, warranty or +// undertaking, as to the functionality security use, value or other characteristics of +// any distributed ledger nor in respect the functioning or value of any tokens which may +// be created stored or transferred using the Work. The Licensor does not warrant that the +// Work or any use of the Work complies with any law or regulation in any territory where +// it may be implemented or used or that it will be appropriate for any specific purpose. +// +// Neither the licensor nor any current or former employees, officers, directors, partners, +// trustees, representatives, agents, advisors, contractors, or volunteers of the Licensor +// shall be liable for any direct or indirect, special, incidental, consequential or other +// losses of any kind, in tort, contract or otherwise (including but not limited to loss +// of revenue, income or profits, or loss of use or data, or loss of reputation, or loss +// of any economic or other opportunity of whatsoever nature or howsoever arising), arising +// out of or in connection with (without limitation of any use, misuse, of any ledger system +// or use made or its functionality or any performance or operation of any code or protocol +// caused by bugs or programming or logic errors or otherwise); +// +// A. any offer, purchase, holding, use, sale, exchange or transmission of any +// cryptographic keys, tokens or assets created, exchanged, stored or arising from any +// interaction with the Work; +// +// B. any failure in a transmission or loss of any token or assets keys or other digital +// artefacts due to errors in transmission; +// +// C. bugs, hacks, logic errors or faults in the Work or any communication; +// +// D. system software or apparatus including but not limited to losses caused by errors +// in holding or transmitting tokens by any third-party; +// +// E. breaches or failure of security including hacker attacks, loss or disclosure of +// password, loss of private key, unauthorised use or misuse of such passwords or keys; +// +// F. any losses including loss of anticipated savings or other benefits resulting from +// use of the Work or any changes to the Work (however implemented). +// +// You are solely responsible for; testing, validating and evaluation of all operation +// logic, functionality, security and appropriateness of using the Work for any commercial +// or non-commercial purpose and for any reproduction or redistribution by You of the +// Work. You assume all risks associated with Your use of the Work and the exercise of +// permissions under this License. + +// This file contains code sourced from https://github.com/aptos-labs/aptos-core/tree/1.0.4 +// This original source is licensed under https://github.com/aptos-labs/aptos-core/blob/1.0.4/LICENSE +// +// The code in this file has been implemented by Radix® pursuant to an Apache 2 licence and has +// been modified by Radix® and is now licensed pursuant to the Radix® Open-Source Licence. +// +// Each sourced code fragment includes an inline attribution to the original source file in a +// comment starting "SOURCE: ..." +// +// Modifications from the original source are captured in two places: +// * Initial changes to get the code functional/integrated are marked by inline "INITIAL-MODIFICATION: ..." comments +// * Subsequent changes to the code are captured in the git commit history +// +// The following notice is retained from the original source +// Copyright (c) Aptos +// SPDX-License-Identifier: Apache-2.0 + +use std::{collections::HashMap, fmt, ops::Range}; + +use serde::{Deserialize, Serialize}; +use tari_crypto::{hash_domain, tari_utilities::ByteArray}; +use tari_dan_common_types::{ + hasher::{tari_hasher, TariHasher}, + optional::IsNotFoundError, +}; +use tari_engine_types::serde_with; + +use crate::jellyfish::store::TreeStoreReader; + +pub type Hash = tari_common_types::types::FixedHash; + +hash_domain!(SparseMerkleTree, "com.tari.dan.state_tree", 0); + +fn hasher() -> TariHasher { + tari_hasher::("hash") +} + +pub fn hash>(data: T) -> Hash { + hasher().chain(data.as_ref()).result() +} + +pub fn hash2(d1: &[u8], d2: &[u8]) -> Hash { + hasher().chain(d1).chain(d2).result() +} + +// SOURCE: https://github.com/aptos-labs/aptos-core/blob/1.0.4/types/src/proof/definition.rs#L182 +/// A more detailed version of `SparseMerkleProof` with the only difference that all the leaf +/// siblings are explicitly set as `SparseMerkleLeafNode` instead of its hash value. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct SparseMerkleProofExt { + leaf: Option, + /// All siblings in this proof, including the default ones. Siblings are ordered from the bottom + /// level to the root level. + siblings: Vec, +} + +impl SparseMerkleProofExt { + /// Constructs a new `SparseMerkleProofExt` using leaf and a list of sibling nodes. + pub fn new(leaf: Option, siblings: Vec) -> Self { + Self { leaf, siblings } + } + + /// Returns the leaf node in this proof. + pub fn leaf(&self) -> Option { + self.leaf.clone() + } + + /// Returns the list of siblings in this proof. + pub fn siblings(&self) -> &[NodeInProof] { + &self.siblings + } +} + +impl From for SparseMerkleProof { + fn from(proof_ext: SparseMerkleProofExt) -> Self { + Self::new( + proof_ext.leaf, + proof_ext.siblings.into_iter().map(|node| node.hash()).collect(), + ) + } +} + +// SOURCE: https://github.com/aptos-labs/aptos-core/blob/1.0.4/types/src/proof/definition.rs#L135 +impl SparseMerkleProof { + /// Constructs a new `SparseMerkleProof` using leaf and a list of siblings. + pub fn new(leaf: Option, siblings: Vec) -> Self { + SparseMerkleProof { leaf, siblings } + } + + /// Returns the leaf node in this proof. + pub fn leaf(&self) -> Option { + self.leaf.clone() + } + + /// Returns the list of siblings in this proof. + pub fn siblings(&self) -> &[Hash] { + &self.siblings + } +} + +/// A proof that can be used to authenticate an element in a Sparse Merkle Tree given trusted root +/// hash. For example, `TransactionInfoToAccountProof` can be constructed on top of this structure. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct SparseMerkleProof { + /// This proof can be used to authenticate whether a given leaf exists in the tree or not. + /// - If this is `Some(leaf_node)` + /// - If `leaf_node.key` equals requested key, this is an inclusion proof and `leaf_node.value_hash` equals + /// the hash of the corresponding account blob. + /// - Otherwise this is a non-inclusion proof. `leaf_node.key` is the only key that exists in the subtree + /// and `leaf_node.value_hash` equals the hash of the corresponding account blob. + /// - If this is `None`, this is also a non-inclusion proof which indicates the subtree is empty. + leaf: Option, + + /// All siblings in this proof, including the default ones. Siblings are ordered from the bottom + /// level to the root level. + siblings: Vec, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum NodeInProof { + Leaf(SparseMerkleLeafNode), + Other(Hash), +} + +impl From for NodeInProof { + fn from(hash: Hash) -> Self { + Self::Other(hash) + } +} + +impl From for NodeInProof { + fn from(leaf: SparseMerkleLeafNode) -> Self { + Self::Leaf(leaf) + } +} + +impl NodeInProof { + pub fn hash(&self) -> Hash { + match self { + Self::Leaf(leaf) => leaf.hash(), + Self::Other(hash) => *hash, + } + } +} + +// SOURCE: https://github.com/aptos-labs/aptos-core/blob/1.0.4/types/src/proof/definition.rs#L681 +/// Note: this is not a range proof in the sense that a range of nodes is verified! +/// Instead, it verifies the entire left part of the tree up to a known rightmost node. +/// See the description below. +/// +/// A proof that can be used to authenticate a range of consecutive leaves, from the leftmost leaf to +/// the rightmost known one, in a sparse Merkle tree. For example, given the following sparse Merkle tree: +/// +/// ```text +/// root +/// / \ +/// / \ +/// / \ +/// o o +/// / \ / \ +/// a o o h +/// / \ / \ +/// o d e X +/// / \ / \ +/// b c f g +/// ``` +/// +/// if the proof wants show that `[a, b, c, d, e]` exists in the tree, it would need the siblings +/// `X` and `h` on the right. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct SparseMerkleRangeProof { + /// The vector of siblings on the right of the path from root to last leaf. The ones near the + /// bottom are at the beginning of the vector. In the above example, it's `[X, h]`. + right_siblings: Vec, +} + +impl SparseMerkleRangeProof { + /// Constructs a new `SparseMerkleRangeProof`. + pub fn new(right_siblings: Vec) -> Self { + Self { right_siblings } + } + + /// Returns the right siblings. + pub fn right_siblings(&self) -> &[Hash] { + &self.right_siblings + } +} + +// SOURCE: https://github.com/aptos-labs/aptos-core/blob/1.0.4/types/src/proof/mod.rs#L97 +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct SparseMerkleLeafNode { + key: LeafKey, + value_hash: Hash, +} + +impl SparseMerkleLeafNode { + pub fn new(key: LeafKey, value_hash: Hash) -> Self { + SparseMerkleLeafNode { key, value_hash } + } + + pub fn key(&self) -> &LeafKey { + &self.key + } + + pub fn value_hash(&self) -> &Hash { + &self.value_hash + } + + pub fn hash(&self) -> Hash { + hash2(self.key.bytes.as_slice(), self.value_hash.as_slice()) + } +} + +pub struct SparseMerkleInternalNode { + left_child: Hash, + right_child: Hash, +} + +impl SparseMerkleInternalNode { + pub fn new(left_child: Hash, right_child: Hash) -> Self { + Self { + left_child, + right_child, + } + } + + fn hash(&self) -> Hash { + hash2(self.left_child.as_bytes(), self.right_child.as_bytes()) + } +} + +// INITIAL-MODIFICATION: we propagate usage of our own `Hash` (instead of Aptos' `HashValue`) to avoid +// sourcing the entire https://github.com/aptos-labs/aptos-core/blob/1.0.4/crates/aptos-crypto/src/hash.rs +pub const SPARSE_MERKLE_PLACEHOLDER_HASH: Hash = Hash::new([0u8; Hash::byte_size()]); + +// CSOURCE: https://github.com/aptos-labs/aptos-core/blob/1.0.4/crates/aptos-crypto/src/hash.rs#L422 +/// An iterator over `LeafKey` that generates one bit for each iteration. +pub struct LeafKeyBitIterator<'a> { + /// The reference to the bytes that represent the `LeafKey`. + leaf_key_bytes: &'a [u8], + pos: Range, + // invariant pos.end == leaf_key_bytes.len() * 8; +} + +impl<'a> DoubleEndedIterator for LeafKeyBitIterator<'a> { + fn next_back(&mut self) -> Option { + self.pos.next_back().map(|x| self.get_bit(x)) + } +} + +impl<'a> ExactSizeIterator for LeafKeyBitIterator<'a> {} + +impl<'a> LeafKeyBitIterator<'a> { + /// Constructs a new `LeafKeyBitIterator` using given `leaf_key_bytes`. + fn new(leaf_key: LeafKeyRef<'a>) -> Self { + LeafKeyBitIterator { + leaf_key_bytes: leaf_key.bytes, + pos: (0..leaf_key.bytes.len() * 8), + } + } + + /// Returns the `index`-th bit in the bytes. + fn get_bit(&self, index: usize) -> bool { + let pos = index / 8; + let bit = 7 - index % 8; + (self.leaf_key_bytes[pos] >> bit) & 1 != 0 + } +} + +impl<'a> Iterator for LeafKeyBitIterator<'a> { + type Item = bool; + + fn next(&mut self) -> Option { + self.pos.next().map(|x| self.get_bit(x)) + } + + fn size_hint(&self) -> (usize, Option) { + self.pos.size_hint() + } +} + +// INITIAL-MODIFICATION: since we use our own `LeafKey` here, we need it to implement these for it +pub trait IteratedLeafKey { + fn iter_bits(&self) -> LeafKeyBitIterator<'_>; + + fn get_nibble(&self, index: usize) -> Nibble; +} + +impl IteratedLeafKey for LeafKey { + fn iter_bits(&self) -> LeafKeyBitIterator<'_> { + LeafKeyBitIterator::new(self.as_ref()) + } + + fn get_nibble(&self, index: usize) -> Nibble { + Nibble::from(if index % 2 == 0 { + self.bytes[index / 2] >> 4 + } else { + self.bytes[index / 2] & 0x0F + }) + } +} + +impl IteratedLeafKey for LeafKeyRef<'_> { + fn iter_bits(&self) -> LeafKeyBitIterator<'_> { + LeafKeyBitIterator::new(*self) + } + + fn get_nibble(&self, index: usize) -> Nibble { + Nibble::from(if index % 2 == 0 { + self.bytes[index / 2] >> 4 + } else { + self.bytes[index / 2] & 0x0F + }) + } +} + +// SOURCE: https://github.com/aptos-labs/aptos-core/blob/1.0.4/types/src/transaction/mod.rs#L57 +pub type Version = u64; + +// SOURCE: https://github.com/aptos-labs/aptos-core/blob/1.0.4/types/src/nibble/mod.rs#L20 +#[derive(Clone, Copy, Debug, Hash, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)] +#[serde(transparent)] +pub struct Nibble(u8); + +impl From for Nibble { + fn from(nibble: u8) -> Self { + assert!(nibble < 16, "Nibble out of range: {}", nibble); + Self(nibble) + } +} + +impl From for u8 { + fn from(nibble: Nibble) -> Self { + nibble.0 + } +} + +impl fmt::LowerHex for Nibble { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:x}", self.0) + } +} + +// SOURCE: https://github.com/aptos-labs/aptos-core/blob/1.0.4/types/src/nibble/nibble_path/mod.rs#L22 +/// NibblePath defines a path in Merkle tree in the unit of nibble (4 bits). +#[derive(Clone, Hash, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)] +pub struct NibblePath { + /// Indicates the total number of nibbles in bytes. Either `bytes.len() * 2 - 1` or + /// `bytes.len() * 2`. + // Guarantees intended ordering based on the top-to-bottom declaration order of the struct's + // members. + num_nibbles: usize, + /// The underlying bytes that stores the path, 2 nibbles per byte. If the number of nibbles is + /// odd, the second half of the last byte must be 0. + #[serde(with = "serde_with::hex")] + bytes: Vec, +} + +/// Supports debug format by concatenating nibbles literally. For example, [0x12, 0xa0] with 3 +/// nibbles will be printed as "12a". +impl fmt::Debug for NibblePath { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + self.nibbles().try_for_each(|x| write!(f, "{:x}", x)) + } +} + +// INITIAL-MODIFICATION: just to show it in errors +impl fmt::Display for NibblePath { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let hex_chars = self + .bytes + .iter() + .flat_map(|b| [b >> 4, b & 15]) + .map(|b| char::from_digit(u32::from(b), 16).unwrap()) + .take(self.num_nibbles); + + for ch in hex_chars { + write!(f, "{}", ch)?; + } + Ok(()) + } +} + +/// Convert a vector of bytes into `NibblePath` using the lower 4 bits of each byte as nibble. +impl FromIterator for NibblePath { + fn from_iter>(iter: I) -> Self { + let mut nibble_path = NibblePath::new_even(vec![]); + for nibble in iter { + nibble_path.push(nibble); + } + nibble_path + } +} + +impl NibblePath { + /// Creates a new `NibblePath` from a vector of bytes assuming each byte has 2 nibbles. + pub fn new_even(bytes: Vec) -> Self { + let num_nibbles = bytes.len() * 2; + NibblePath { num_nibbles, bytes } + } + + /// Similar to `new()` but asserts that the bytes have one less nibble. + pub fn new_odd(bytes: Vec) -> Self { + assert_eq!( + bytes.last().expect("Should have odd number of nibbles.") & 0x0F, + 0, + "Last nibble must be 0." + ); + let num_nibbles = bytes.len() * 2 - 1; + NibblePath { num_nibbles, bytes } + } + + /// Adds a nibble to the end of the nibble path. + pub fn push(&mut self, nibble: Nibble) { + if self.num_nibbles % 2 == 0 { + self.bytes.push(u8::from(nibble) << 4); + } else { + self.bytes[self.num_nibbles / 2] |= u8::from(nibble); + } + self.num_nibbles += 1; + } + + /// Pops a nibble from the end of the nibble path. + pub fn pop(&mut self) -> Option { + let poped_nibble = if self.num_nibbles % 2 == 0 { + self.bytes.last_mut().map(|last_byte| { + let nibble = *last_byte & 0x0F; + *last_byte &= 0xF0; + Nibble::from(nibble) + }) + } else { + self.bytes.pop().map(|byte| Nibble::from(byte >> 4)) + }; + if poped_nibble.is_some() { + self.num_nibbles -= 1; + } + poped_nibble + } + + /// Returns the last nibble. + pub fn last(&self) -> Option { + let last_byte_option = self.bytes.last(); + if self.num_nibbles % 2 == 0 { + last_byte_option.map(|last_byte| Nibble::from(*last_byte & 0x0F)) + } else { + let last_byte = last_byte_option.expect("Last byte must exist if num_nibbles is odd."); + Some(Nibble::from(*last_byte >> 4)) + } + } + + /// Get the i-th bit. + fn get_bit(&self, i: usize) -> bool { + assert!(i < self.num_nibbles * 4); + let pos = i / 8; + let bit = 7 - i % 8; + ((self.bytes[pos] >> bit) & 1) != 0 + } + + /// Get the i-th nibble. + pub fn get_nibble(&self, i: usize) -> Nibble { + assert!(i < self.num_nibbles); + Nibble::from((self.bytes[i / 2] >> (if i % 2 == 1 { 0 } else { 4 })) & 0xF) + } + + /// Get a bit iterator iterates over the whole nibble path. + pub fn bits(&self) -> BitIterator { + BitIterator { + nibble_path: self, + pos: (0..self.num_nibbles * 4), + } + } + + /// Get a nibble iterator iterates over the whole nibble path. + pub fn nibbles(&self) -> NibbleIterator { + NibbleIterator::new(self, 0, self.num_nibbles) + } + + /// Get the total number of nibbles stored. + pub fn num_nibbles(&self) -> usize { + self.num_nibbles + } + + /// Returns `true` if the nibbles contains no elements. + pub fn is_empty(&self) -> bool { + self.num_nibbles() == 0 + } + + /// Get the underlying bytes storing nibbles. + pub fn bytes(&self) -> &[u8] { + &self.bytes + } + + pub fn into_bytes(self) -> Vec { + self.bytes + } + + pub fn truncate(&mut self, len: usize) { + assert!(len <= self.num_nibbles); + self.num_nibbles = len; + self.bytes.truncate((len + 1) / 2); + if len % 2 != 0 { + *self.bytes.last_mut().expect("must exist.") &= 0xF0; + } + } +} + +pub trait Peekable: Iterator { + /// Returns the `next()` value without advancing the iterator. + fn peek(&self) -> Option; +} + +/// BitIterator iterates a nibble path by bit. +pub struct BitIterator<'a> { + nibble_path: &'a NibblePath, + pos: Range, +} + +impl<'a> Peekable for BitIterator<'a> { + /// Returns the `next()` value without advancing the iterator. + fn peek(&self) -> Option { + if self.pos.start < self.pos.end { + Some(self.nibble_path.get_bit(self.pos.start)) + } else { + None + } + } +} + +/// BitIterator spits out a boolean each time. True/false denotes 1/0. +impl<'a> Iterator for BitIterator<'a> { + type Item = bool; + + fn next(&mut self) -> Option { + self.pos.next().map(|i| self.nibble_path.get_bit(i)) + } +} + +/// Support iterating bits in reversed order. +impl<'a> DoubleEndedIterator for BitIterator<'a> { + fn next_back(&mut self) -> Option { + self.pos.next_back().map(|i| self.nibble_path.get_bit(i)) + } +} + +/// NibbleIterator iterates a nibble path by nibble. +#[derive(Debug)] +pub struct NibbleIterator<'a> { + /// The underlying nibble path that stores the nibbles + nibble_path: &'a NibblePath, + + /// The current index, `pos.start`, will bump by 1 after calling `next()` until `pos.start == + /// pos.end`. + pos: Range, + + /// The start index of the iterator. At the beginning, `pos.start == start`. [start, pos.end) + /// defines the range of `nibble_path` this iterator iterates over. `nibble_path` refers to + /// the entire underlying buffer but the range may only be partial. + start: usize, + // invariant self.start <= self.pos.start; + // invariant self.pos.start <= self.pos.end; +} + +/// NibbleIterator spits out a byte each time. Each byte must be in range [0, 16). +impl<'a> Iterator for NibbleIterator<'a> { + type Item = Nibble; + + fn next(&mut self) -> Option { + self.pos.next().map(|i| self.nibble_path.get_nibble(i)) + } +} + +impl<'a> Peekable for NibbleIterator<'a> { + /// Returns the `next()` value without advancing the iterator. + fn peek(&self) -> Option { + if self.pos.start < self.pos.end { + Some(self.nibble_path.get_nibble(self.pos.start)) + } else { + None + } + } +} + +impl<'a> NibbleIterator<'a> { + fn new(nibble_path: &'a NibblePath, start: usize, end: usize) -> Self { + assert!(start <= end); + Self { + nibble_path, + pos: (start..end), + start, + } + } + + /// Returns a nibble iterator that iterates all visited nibbles. + pub fn visited_nibbles(&self) -> NibbleIterator<'a> { + Self::new(self.nibble_path, self.start, self.pos.start) + } + + /// Returns a nibble iterator that iterates all remaining nibbles. + pub fn remaining_nibbles(&self) -> NibbleIterator<'a> { + Self::new(self.nibble_path, self.pos.start, self.pos.end) + } + + /// Turn it into a `BitIterator`. + pub fn bits(&self) -> BitIterator<'a> { + BitIterator { + nibble_path: self.nibble_path, + pos: (self.pos.start * 4..self.pos.end * 4), + } + } + + /// Cut and return the range of the underlying `nibble_path` that this iterator is iterating + /// over as a new `NibblePath` + pub fn get_nibble_path(&self) -> NibblePath { + self.visited_nibbles().chain(self.remaining_nibbles()).collect() + } + + /// Get the number of nibbles that this iterator covers. + pub fn num_nibbles(&self) -> usize { + assert!(self.start <= self.pos.end); // invariant + self.pos.end - self.start + } + + /// Return `true` if the iteration is over. + pub fn is_finished(&self) -> bool { + self.peek().is_none() + } +} + +// INITIAL-MODIFICATION: We will use this type (instead of `Hash`) to allow for arbitrary key length +/// A leaf key (i.e. a complete nibble path). +#[derive(Clone, Debug, Hash, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)] +pub struct LeafKey { + /// The underlying bytes. + /// All leaf keys of the same tree must be of the same length - otherwise the tree's behavior + /// becomes unspecified. + /// All leaf keys must be evenly distributed across their space - otherwise the tree's + /// performance degrades. + #[serde(with = "serde_with::hex")] + pub bytes: Vec, +} + +impl LeafKey { + pub fn new(bytes: Vec) -> Self { + Self { bytes } + } + + pub fn as_ref(&self) -> LeafKeyRef<'_> { + LeafKeyRef::new(&self.bytes) + } +} + +// INITIAL-MODIFICATION: We will use this type (instead of `Hash`) to allow for arbitrary key length +/// A leaf key (i.e. a complete nibble path). +#[derive(Clone, Copy, Debug, Hash, Eq, PartialEq, Ord, PartialOrd)] +pub struct LeafKeyRef<'a> { + /// The underlying bytes. + /// All leaf keys of the same tree must be of the same length - otherwise the tree's behavior + /// becomes unspecified. + /// All leaf keys must be evenly distributed across their space - otherwise the tree's + /// performance degrades. + pub bytes: &'a [u8], +} + +impl<'a> LeafKeyRef<'a> { + pub fn new(bytes: &'a [u8]) -> Self { + Self { bytes } + } +} + +impl PartialEq for LeafKeyRef<'_> { + fn eq(&self, other: &LeafKey) -> bool { + self.bytes == other.bytes + } +} + +// SOURCE: https://github.com/aptos-labs/aptos-core/blob/1.0.4/storage/jellyfish-merkle/src/node_type/mod.rs#L48 +/// The unique key of each node. +#[derive(Clone, Debug, Hash, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)] +pub struct NodeKey { + /// The version at which the node is created. + version: Version, + /// The nibble path this node represents in the tree. + nibble_path: NibblePath, +} + +impl NodeKey { + /// Creates a new `NodeKey`. + pub fn new(version: Version, nibble_path: NibblePath) -> Self { + Self { version, nibble_path } + } + + /// A shortcut to generate a node key consisting of a version and an empty nibble path. + pub fn new_empty_path(version: Version) -> Self { + Self::new(version, NibblePath::new_even(vec![])) + } + + /// Gets the version. + pub fn version(&self) -> Version { + self.version + } + + /// Gets the nibble path. + pub fn nibble_path(&self) -> &NibblePath { + &self.nibble_path + } + + /// Generates a child node key based on this node key. + pub fn gen_child_node_key(&self, version: Version, n: Nibble) -> Self { + let mut node_nibble_path = self.nibble_path().clone(); + node_nibble_path.push(n); + Self::new(version, node_nibble_path) + } + + /// Generates parent node key at the same version based on this node key. + pub fn gen_parent_node_key(&self) -> Self { + let mut node_nibble_path = self.nibble_path().clone(); + assert!(node_nibble_path.pop().is_some(), "Current node key is root.",); + Self::new(self.version, node_nibble_path) + } +} + +// INITIAL-MODIFICATION: just to show it in errors +impl fmt::Display for NodeKey { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "v{}:{}", self.version, self.nibble_path) + } +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub enum NodeType { + Leaf, + Null, + /// A internal node that haven't been finished the leaf count migration, i.e. None or not all + /// of the children leaf counts are known. + Internal { + leaf_count: usize, + }, +} + +/// Each child of [`InternalNode`] encapsulates a nibble forking at this node. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub struct Child { + /// The hash value of this child node. + #[serde(with = "serde_with::hex")] + pub hash: Hash, + /// `version`, the `nibble_path` of the [`NodeKey`] of this [`InternalNode`] the child belongs + /// to and the child's index constitute the [`NodeKey`] to uniquely identify this child node + /// from the storage. Used by `[`NodeKey::gen_child_node_key`]. + pub version: Version, + /// Indicates if the child is a leaf, or if it's an internal node, the total number of leaves + /// under it (though it can be unknown during migration). + pub node_type: NodeType, +} + +impl Child { + pub fn new(hash: Hash, version: Version, node_type: NodeType) -> Self { + Self { + hash, + version, + node_type, + } + } + + pub fn is_leaf(&self) -> bool { + matches!(self.node_type, NodeType::Leaf) + } + + pub fn leaf_count(&self) -> usize { + match self.node_type { + NodeType::Leaf => 1, + NodeType::Internal { leaf_count } => leaf_count, + NodeType::Null => unreachable!("Child cannot be Null"), + } + } +} + +/// [`Children`] is just a collection of children belonging to a [`InternalNode`], indexed from 0 to +/// 15, inclusive. +pub(crate) type Children = HashMap; + +/// Represents a 4-level subtree with 16 children at the bottom level. Theoretically, this reduces +/// IOPS to query a tree by 4x since we compress 4 levels in a standard Merkle tree into 1 node. +/// Though we choose the same internal node structure as that of Patricia Merkle tree, the root hash +/// computation logic is similar to a 4-level sparse Merkle tree except for some customizations. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub struct InternalNode { + /// Up to 16 children. + children: Children, + /// Total number of leaves under this internal node + leaf_count: usize, +} + +impl InternalNode { + /// Creates a new Internal node. + pub fn new(children: Children) -> Self { + let leaf_count = children.values().map(Child::leaf_count).sum(); + Self { children, leaf_count } + } + + pub fn leaf_count(&self) -> usize { + self.leaf_count + } + + pub fn node_type(&self) -> NodeType { + NodeType::Internal { + leaf_count: self.leaf_count, + } + } + + pub fn hash(&self) -> Hash { + self.merkle_hash( + 0, // start index + 16, // the number of leaves in the subtree of which we want the hash of root + self.generate_bitmaps(), + ) + } + + pub fn children_sorted(&self) -> impl Iterator { + let mut tmp = self.children.iter().collect::>(); + tmp.sort_by_key(|(nibble, _)| **nibble); + tmp.into_iter() + } + + pub fn into_children(self) -> Children { + self.children + } + + /// Gets the `n`-th child. + pub fn child(&self, n: Nibble) -> Option<&Child> { + self.children.get(&n) + } + + /// Generates `existence_bitmap` and `leaf_bitmap` as a pair of `u16`s: child at index `i` + /// exists if `existence_bitmap[i]` is set; child at index `i` is leaf node if + /// `leaf_bitmap[i]` is set. + pub fn generate_bitmaps(&self) -> (u16, u16) { + let mut existence_bitmap = 0; + let mut leaf_bitmap = 0; + for (nibble, child) in &self.children { + let i = u8::from(*nibble); + existence_bitmap |= 1u16 << i; + if child.is_leaf() { + leaf_bitmap |= 1u16 << i; + } + } + // `leaf_bitmap` must be a subset of `existence_bitmap`. + assert_eq!(existence_bitmap | leaf_bitmap, existence_bitmap); + (existence_bitmap, leaf_bitmap) + } + + /// Given a range [start, start + width), returns the sub-bitmap of that range. + fn range_bitmaps(start: u8, width: u8, bitmaps: (u16, u16)) -> (u16, u16) { + assert!(start < 16 && width.count_ones() == 1 && start % width == 0); + assert!(width <= 16 && (start + width) <= 16); + // A range with `start == 8` and `width == 4` will generate a mask 0b0000111100000000. + // use as converting to smaller integer types when 'width == 16' + let mask = (((1u32 << width) - 1) << start) as u16; + (bitmaps.0 & mask, bitmaps.1 & mask) + } + + fn merkle_hash(&self, start: u8, width: u8, (existence_bitmap, leaf_bitmap): (u16, u16)) -> Hash { + // Given a bit [start, 1 << nibble_height], return the value of that range. + let (range_existence_bitmap, range_leaf_bitmap) = + Self::range_bitmaps(start, width, (existence_bitmap, leaf_bitmap)); + if range_existence_bitmap == 0 { + // No child under this subtree + SPARSE_MERKLE_PLACEHOLDER_HASH + } else if width == 1 || (range_existence_bitmap.count_ones() == 1 && range_leaf_bitmap != 0) { + // Only 1 leaf child under this subtree or reach the lowest level + let only_child_index = Nibble::from(range_existence_bitmap.trailing_zeros() as u8); + self.child(only_child_index) + .expect("Corrupted internal node: existence_bitmap inconsistent") + .hash + } else { + let left_child = self.merkle_hash(start, width / 2, (range_existence_bitmap, range_leaf_bitmap)); + let right_child = self.merkle_hash( + start + width / 2, + width / 2, + (range_existence_bitmap, range_leaf_bitmap), + ); + SparseMerkleInternalNode::new(left_child, right_child).hash() + } + } + + fn gen_node_in_proof>( + &self, + start: u8, + width: u8, + (existence_bitmap, leaf_bitmap): (u16, u16), + (tree_reader, node_key): (&R, &NodeKey), + ) -> Result { + // Given a bit [start, 1 << nibble_height], return the value of that range. + let (range_existence_bitmap, range_leaf_bitmap) = + Self::range_bitmaps(start, width, (existence_bitmap, leaf_bitmap)); + Ok(if range_existence_bitmap == 0 { + // No child under this subtree + NodeInProof::Other(SPARSE_MERKLE_PLACEHOLDER_HASH) + } else if width == 1 || (range_existence_bitmap.count_ones() == 1 && range_leaf_bitmap != 0) { + // Only 1 leaf child under this subtree or reach the lowest level + let only_child_index = Nibble::from(range_existence_bitmap.trailing_zeros() as u8); + let only_child = self + .child(only_child_index) + .expect("Corrupted internal node: existence_bitmap inconsistent"); + if matches!(only_child.node_type, NodeType::Leaf) { + let only_child_node_key = node_key.gen_child_node_key(only_child.version, only_child_index); + match tree_reader.get_node(&only_child_node_key)? { + Node::Internal(_) => { + unreachable!("Corrupted internal node: in-memory leaf child is internal node on disk") + }, + Node::Leaf(leaf_node) => NodeInProof::Leaf(SparseMerkleLeafNode::from(leaf_node)), + Node::Null => unreachable!("Child cannot be Null"), + } + } else { + NodeInProof::Other(only_child.hash) + } + } else { + let left_child = self.merkle_hash(start, width / 2, (range_existence_bitmap, range_leaf_bitmap)); + let right_child = self.merkle_hash( + start + width / 2, + width / 2, + (range_existence_bitmap, range_leaf_bitmap), + ); + NodeInProof::Other(SparseMerkleInternalNode::new(left_child, right_child).hash()) + }) + } + + /// Gets the child and its corresponding siblings that are necessary to generate the proof for + /// the `n`-th child. If it is an existence proof, the returned child must be the `n`-th + /// child; otherwise, the returned child may be another child. See inline explanation for + /// details. When calling this function with n = 11 (node `b` in the following graph), the + /// range at each level is illustrated as a pair of square brackets: + /// + /// ```text + /// 4 [f e d c b a 9 8 7 6 5 4 3 2 1 0] -> root level + /// --------------------------------------------------------------- + /// 3 [f e d c b a 9 8] [7 6 5 4 3 2 1 0] width = 8 + /// chs <--┘ shs <--┘ + /// 2 [f e d c] [b a 9 8] [7 6 5 4] [3 2 1 0] width = 4 + /// shs <--┘ └--> chs + /// 1 [f e] [d c] [b a] [9 8] [7 6] [5 4] [3 2] [1 0] width = 2 + /// chs <--┘ └--> shs + /// 0 [f] [e] [d] [c] [b] [a] [9] [8] [7] [6] [5] [4] [3] [2] [1] [0] width = 1 + /// ^ chs <--┘ └--> shs + /// | MSB|<---------------------- uint 16 ---------------------------->|LSB + /// height chs: `child_half_start` shs: `sibling_half_start` + /// ``` + pub fn get_child_with_siblings>( + &self, + node_key: &NodeKey, + n: Nibble, + reader: Option<&R>, + ) -> Result<(Option, Vec), JmtStorageError> { + let mut siblings = vec![]; + let (existence_bitmap, leaf_bitmap) = self.generate_bitmaps(); + + // Nibble height from 3 to 0. + for h in (0..4).rev() { + // Get the number of children of the internal node that each subtree at this height + // covers. + let width = 1 << h; + let (child_half_start, sibling_half_start) = get_child_and_sibling_half_start(n, h); + // Compute the root hash of the subtree rooted at the sibling of `r`. + if let Some(reader) = reader { + siblings.push(self.gen_node_in_proof( + sibling_half_start, + width, + (existence_bitmap, leaf_bitmap), + (reader, node_key), + )?); + } else { + siblings.push( + self.merkle_hash(sibling_half_start, width, (existence_bitmap, leaf_bitmap)) + .into(), + ); + } + + let (range_existence_bitmap, range_leaf_bitmap) = + Self::range_bitmaps(child_half_start, width, (existence_bitmap, leaf_bitmap)); + + if range_existence_bitmap == 0 { + // No child in this range. + return Ok((None, siblings)); + } + + if width == 1 || (range_existence_bitmap.count_ones() == 1 && range_leaf_bitmap != 0) { + // Return the only 1 leaf child under this subtree or reach the lowest level + // Even this leaf child is not the n-th child, it should be returned instead of + // `None` because it's existence indirectly proves the n-th child doesn't exist. + // Please read proof format for details. + let only_child_index = Nibble::from(range_existence_bitmap.trailing_zeros() as u8); + return Ok(( + { + let only_child_version = self + .child(only_child_index) + // Should be guaranteed by the self invariants, but these are not easy to express at the moment + .expect("Corrupted internal node: child_bitmap inconsistent") + .version; + Some(node_key.gen_child_node_key(only_child_version, only_child_index)) + }, + siblings, + )); + } + } + unreachable!("Impossible to get here without returning even at the lowest level.") + } +} + +/// Given a nibble, computes the start position of its `child_half_start` and `sibling_half_start` +/// at `height` level. +pub(crate) fn get_child_and_sibling_half_start(n: Nibble, height: u8) -> (u8, u8) { + // Get the index of the first child belonging to the same subtree whose root, let's say `r` is + // at `height` that the n-th child belongs to. + // Note: `child_half_start` will be always equal to `n` at height 0. + let child_half_start = (0xFF << height) & u8::from(n); + + // Get the index of the first child belonging to the subtree whose root is the sibling of `r` + // at `height`. + let sibling_half_start = child_half_start ^ (1 << height); + + (child_half_start, sibling_half_start) +} + +/// Leaf node, capturing the value hash and carrying an arbitrary payload. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub struct LeafNode

{ + // The key of this leaf node (i.e. its full nibble path). + leaf_key: LeafKey, + // The hash of an externally-stored value. + // Note: do not confuse that value with the `payload`. + #[serde(with = "serde_with::hex")] + value_hash: Hash, + // The client payload. + // This is not the "value" whose changes are tracked by the tree (in fact, these values are + // supposed to be stored externally, and the tree only cares about their hashes - see + // `value_hash`). + // Rather, the payload is an arbitrary piece of data that the client wishes to store within the + // tree, in order to facilitate some related processing: + // - Many clients do not need it and will simply use a no-cost `()`. + // - A use-case designed by the original authors was to store a non-hashed element key as a payload (while the + // `leaf_key` contains that key's hash, to ensure the nibble paths are distributed over their space, for + // performance). + // - Our current use-case (specific to a "two layers" tree) is to store the nested tree's root metadata. + payload: P, + // The version at which this leaf was created. + version: Version, +} + +impl LeafNode

{ + /// Creates a new leaf node. + pub fn new(leaf_key: LeafKey, value_hash: Hash, payload: P, version: Version) -> Self { + Self { + leaf_key, + value_hash, + payload, + version, + } + } + + /// Gets the key. + pub fn leaf_key(&self) -> &LeafKey { + &self.leaf_key + } + + /// Gets the associated value hash. + pub fn value_hash(&self) -> Hash { + self.value_hash + } + + /// Gets the payload. + pub fn payload(&self) -> &P { + &self.payload + } + + /// Gets the version. + pub fn version(&self) -> Version { + self.version + } + + /// Gets the leaf's hash (not to be confused with a `value_hash()`). + /// This hash incorporates the node's key and the value's hash, in order to capture certain + /// changes within a sparse merkle tree (consider 2 trees, both containing a single element with + /// the same value, but stored under different keys - we want their root hashes to differ). + pub fn leaf_hash(&self) -> Hash { + hash2(self.leaf_key.bytes.as_slice(), self.value_hash.as_slice()) + } +} + +impl From> for SparseMerkleLeafNode { + fn from(leaf_node: LeafNode) -> Self { + Self::new(leaf_node.leaf_key, leaf_node.value_hash) + } +} + +/// The concrete node type of [`JellyfishMerkleTree`]. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub enum Node { + /// A wrapper of [`InternalNode`]. + Internal(InternalNode), + /// A wrapper of [`LeafNode`]. + Leaf(LeafNode), + /// Represents empty tree only + Null, +} + +impl From for Node { + fn from(node: InternalNode) -> Self { + Node::Internal(node) + } +} + +impl From> for Node { + fn from(node: LeafNode) -> Self { + Node::Leaf(node) + } +} + +impl Node

{ + /// Creates the [`Internal`](Node::Internal) variant. + #[cfg(any(test, feature = "fuzzing"))] + pub fn new_internal(children: Children) -> Self { + Node::Internal(InternalNode::new(children)) + } + + /// Creates the [`Leaf`](Node::Leaf) variant. + pub fn new_leaf(leaf_key: LeafKey, value_hash: Hash, payload: P, version: Version) -> Self { + Node::Leaf(LeafNode::new(leaf_key, value_hash, payload, version)) + } + + /// Returns `true` if the node is a leaf node. + pub fn is_leaf(&self) -> bool { + matches!(self, Node::Leaf(_)) + } + + /// Returns `NodeType` + pub fn node_type(&self) -> NodeType { + match self { + // The returning value will be used to construct a `Child` of a internal node, while an + // internal node will never have a child of Node::Null. + Self::Leaf(_) => NodeType::Leaf, + Self::Internal(n) => n.node_type(), + Self::Null => NodeType::Null, + } + } + + /// Returns leaf count if known + pub fn leaf_count(&self) -> usize { + match self { + Node::Leaf(_) => 1, + Node::Internal(internal_node) => internal_node.leaf_count, + Node::Null => 0, + } + } + + /// Computes the hash of nodes. + pub fn hash(&self) -> Hash { + match self { + Node::Internal(internal_node) => internal_node.hash(), + Node::Leaf(leaf_node) => leaf_node.leaf_hash(), + Node::Null => SPARSE_MERKLE_PLACEHOLDER_HASH, + } + } +} + +// INITIAL-MODIFICATION: we propagate usage of our own error enum (instead of `std::io::ErrorKind` +// used by Aptos) to allow for no-std build. +/// Error originating from underlying storage failure / inconsistency. +#[derive(Debug, thiserror::Error)] +pub enum JmtStorageError { + #[error("A node {0} expected to exist (according to JMT logic) was not found in the storage")] + NotFound(NodeKey), + + #[error("Nodes read from the storage are violating some JMT property (e.g. form a cycle).")] + InconsistentState, + + #[error("Unexpected error: {0}")] + UnexpectedError(String), +} + +impl IsNotFoundError for JmtStorageError { + fn is_not_found_error(&self) -> bool { + matches!(self, JmtStorageError::NotFound(_)) + } +} diff --git a/dan_layer/state_tree/src/key_mapper.rs b/dan_layer/state_tree/src/key_mapper.rs new file mode 100644 index 000000000..55f4a72cc --- /dev/null +++ b/dan_layer/state_tree/src/key_mapper.rs @@ -0,0 +1,22 @@ +// Copyright 2024 The Tari Project +// SPDX-License-Identifier: BSD-3-Clause + +use tari_engine_types::substate::SubstateId; + +use crate::jellyfish::LeafKey; + +pub trait DbKeyMapper { + fn map_to_leaf_key(id: &SubstateId) -> LeafKey; +} + +const HASH_PREFIX_LENGTH: usize = 20; + +pub struct SpreadPrefixKeyMapper; + +impl DbKeyMapper for SpreadPrefixKeyMapper { + fn map_to_leaf_key(id: &SubstateId) -> LeafKey { + let hash = crate::jellyfish::hash(id.to_canonical_hash()); + let prefixed_key = [&hash[..HASH_PREFIX_LENGTH], hash.as_slice()].concat(); + LeafKey::new(prefixed_key) + } +} diff --git a/dan_layer/state_tree/src/lib.rs b/dan_layer/state_tree/src/lib.rs new file mode 100644 index 000000000..2e9d25432 --- /dev/null +++ b/dan_layer/state_tree/src/lib.rs @@ -0,0 +1,16 @@ +// Copyright 2024 The Tari Project +// SPDX-License-Identifier: BSD-3-Clause + +mod error; +pub use error::*; + +mod jellyfish; +pub use jellyfish::*; +pub mod key_mapper; +pub mod memory_store; + +mod staged_store; +pub use staged_store::*; + +mod tree; +pub use tree::*; diff --git a/dan_layer/state_tree/src/memory_store.rs b/dan_layer/state_tree/src/memory_store.rs new file mode 100644 index 000000000..2948cbb5e --- /dev/null +++ b/dan_layer/state_tree/src/memory_store.rs @@ -0,0 +1,75 @@ +// Copyright 2024 The Tari Project +// SPDX-License-Identifier: BSD-3-Clause + +use std::{collections::HashMap, fmt}; + +use crate::jellyfish::{ + JmtStorageError, + Node, + NodeKey, + StaleTreeNode, + TreeNode, + TreeStoreReader, + TreeStoreWriter, + Version, +}; + +#[derive(Debug, Default)] +pub struct MemoryTreeStore { + pub nodes: HashMap, + pub stale_nodes: Vec, +} + +impl MemoryTreeStore { + pub fn new() -> Self { + Self { + nodes: HashMap::new(), + stale_nodes: Vec::new(), + } + } + + pub fn clear_stale_nodes(&mut self) { + for stale in self.stale_nodes.drain(..) { + self.nodes.remove(stale.as_node_key()); + } + } +} + +impl TreeStoreReader for MemoryTreeStore { + fn get_node(&self, key: &NodeKey) -> Result, JmtStorageError> { + self.nodes + .get(key) + .map(|node| node.clone().into_node()) + .ok_or_else(|| JmtStorageError::NotFound(key.clone())) + } +} + +impl TreeStoreWriter for MemoryTreeStore { + fn insert_node(&mut self, key: NodeKey, node: Node) -> Result<(), JmtStorageError> { + let node = TreeNode::new_latest(node); + self.nodes.insert(key, node); + Ok(()) + } + + fn record_stale_tree_node(&mut self, stale: StaleTreeNode) -> Result<(), JmtStorageError> { + self.stale_nodes.push(stale); + Ok(()) + } +} + +impl fmt::Display for MemoryTreeStore { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + writeln!(f, "MemoryTreeStore")?; + writeln!(f, " Nodes:")?; + let mut store = self.nodes.iter().collect::>(); + store.sort_by_key(|(key, _)| *key); + for (key, node) in store { + writeln!(f, " {}: {:?}", key, node)?; + } + writeln!(f, " Stale Nodes:")?; + for stale in &self.stale_nodes { + writeln!(f, " {}", stale.as_node_key())?; + } + Ok(()) + } +} diff --git a/dan_layer/state_tree/src/staged_store.rs b/dan_layer/state_tree/src/staged_store.rs new file mode 100644 index 000000000..cff18d48b --- /dev/null +++ b/dan_layer/state_tree/src/staged_store.rs @@ -0,0 +1,92 @@ +// Copyright 2024 The Tari Project +// SPDX-License-Identifier: BSD-3-Clause + +// Copyright 2024 The Tari Project +// SPDX-License-Identifier: BSD-3-Clause +// Copyright 2024 The Tari Project +// SPDX-License-Identifier: BSD-3-Clause + +use std::collections::{HashMap, VecDeque}; + +use crate::{ + JmtStorageError, + Node, + NodeKey, + StaleTreeNode, + StateHashTreeDiff, + TreeStoreReader, + TreeStoreWriter, + Version, +}; + +pub struct StagedTreeStore<'s, S> { + readable_store: &'s S, + preceding_pending_state: HashMap>, + new_tree_nodes: HashMap>, + new_stale_nodes: Vec, +} + +impl<'s, S: TreeStoreReader> StagedTreeStore<'s, S> { + pub fn new(readable_store: &'s S) -> Self { + Self { + readable_store, + preceding_pending_state: HashMap::new(), + new_tree_nodes: HashMap::new(), + new_stale_nodes: Vec::new(), + } + } + + pub fn apply_ordered_diffs>(&mut self, diffs: I) { + for (key, node) in diffs.into_iter().flat_map(|diff| diff.new_nodes) { + self.preceding_pending_state.insert(key, node); + } + } + + pub fn into_diff(self) -> StateHashTreeDiff { + StateHashTreeDiff { + new_nodes: self.new_tree_nodes.into_iter().collect(), + stale_tree_nodes: self.new_stale_nodes, + } + } +} + +impl<'s, S: TreeStoreReader> TreeStoreReader for StagedTreeStore<'s, S> { + fn get_node(&self, key: &NodeKey) -> Result, JmtStorageError> { + if let Some(node) = self.new_tree_nodes.get(key).cloned() { + return Ok(node); + } + if let Some(node) = self.preceding_pending_state.get(key).cloned() { + return Ok(node); + } + + self.readable_store.get_node(key) + } +} + +impl<'s, S> TreeStoreWriter for StagedTreeStore<'s, S> { + fn insert_node(&mut self, key: NodeKey, node: Node) -> Result<(), JmtStorageError> { + self.new_tree_nodes.insert(key, node); + Ok(()) + } + + fn record_stale_tree_node(&mut self, stale: StaleTreeNode) -> Result<(), JmtStorageError> { + // Prune staged tree nodes immediately from preceding_pending_state. + let mut remove_queue = VecDeque::new(); + remove_queue.push_front(stale.as_node_key().clone()); + while let Some(key) = remove_queue.pop_front() { + if let Some(node) = self.preceding_pending_state.remove(&key) { + match node { + Node::Internal(node) => { + for (nibble, child) in node.into_children() { + remove_queue.push_back(key.gen_child_node_key(child.version, nibble)); + } + }, + Node::Leaf(_) | Node::Null => {}, + } + } + } + + self.new_stale_nodes.push(stale); + Ok(()) + } +} diff --git a/dan_layer/state_tree/src/tree.rs b/dan_layer/state_tree/src/tree.rs new file mode 100644 index 000000000..ef7fd11b7 --- /dev/null +++ b/dan_layer/state_tree/src/tree.rs @@ -0,0 +1,165 @@ +// Copyright 2024 The Tari Project +// SPDX-License-Identifier: BSD-3-Clause + +use std::marker::PhantomData; + +use serde::{Deserialize, Serialize}; +use tari_engine_types::substate::SubstateId; + +use crate::{ + error::StateTreeError, + jellyfish::{Hash, JellyfishMerkleTree, LeafKey, SparseMerkleProofExt, TreeStore, Version}, + key_mapper::{DbKeyMapper, SpreadPrefixKeyMapper}, + Node, + NodeKey, + ProofValue, + StaleTreeNode, + TreeStoreReader, + TreeUpdateBatch, +}; + +pub type SpreadPrefixStateTree<'a, S> = StateTree<'a, S, SpreadPrefixKeyMapper>; + +pub struct StateTree<'a, S, M> { + store: &'a mut S, + _mapper: PhantomData, +} + +struct LeafChange { + key: LeafKey, + new_payload: Option<(Hash, Version)>, +} + +impl<'a, S, M> StateTree<'a, S, M> { + pub fn new(store: &'a mut S) -> Self { + Self { + store, + _mapper: PhantomData, + } + } +} + +impl<'a, S: TreeStoreReader, M: DbKeyMapper> StateTree<'a, S, M> { + pub fn get_proof( + &self, + version: Version, + substate_id: &SubstateId, + ) -> Result<(Option>, SparseMerkleProofExt), StateTreeError> { + let smt = JellyfishMerkleTree::new(self.store); + let key = M::map_to_leaf_key(substate_id); + let (maybe_value, proof) = smt.get_with_proof_ext(key.as_ref(), version)?; + Ok((maybe_value, proof)) + } +} + +impl<'a, S: TreeStore, M: DbKeyMapper> StateTree<'a, S, M> { + /// Stores the substate changes in the state tree and returns the new root hash. + pub fn put_substate_changes>( + &mut self, + current_version: Version, + next_version: Version, + changes: I, + ) -> Result { + let (root_hash, update_batch) = calculate_substate_changes::<_, M, _>( + self.store, + Some(current_version).filter(|v| *v > 0), + next_version, + changes, + )?; + + self.commit_diff(update_batch.into())?; + Ok(root_hash) + } + + pub fn commit_diff(&mut self, diff: StateHashTreeDiff) -> Result<(), StateTreeError> { + for (key, node) in diff.new_nodes { + log::debug!("Inserting node: {}", key); + self.store.insert_node(key, node)?; + } + + for stale_tree_node in diff.stale_tree_nodes { + log::debug!("Recording stale tree node: {}", stale_tree_node.as_node_key()); + self.store.record_stale_tree_node(stale_tree_node)?; + } + + Ok(()) + } +} + +/// Calculates the new root hash and tree updates for the given substate changes. +fn calculate_substate_changes, M: DbKeyMapper, I: IntoIterator>( + store: &mut S, + current_version: Option, + next_version: Version, + changes: I, +) -> Result<(Hash, TreeUpdateBatch), StateTreeError> { + let smt = JellyfishMerkleTree::new(store); + + let changes = changes + .into_iter() + .map(|ch| match ch { + SubstateChange::Up { id, value_hash } => LeafChange { + key: M::map_to_leaf_key(&id), + new_payload: Some((value_hash, next_version)), + }, + SubstateChange::Down { id } => LeafChange { + key: M::map_to_leaf_key(&id), + new_payload: None, + }, + }) + .collect::>(); + + let (root_hash, update_result) = smt.batch_put_value_set( + changes + .iter() + .map(|change| (&change.key, change.new_payload.as_ref())) + .collect(), + None, + current_version, + next_version, + )?; + + Ok((root_hash, update_result)) +} + +pub enum SubstateChange { + Up { id: SubstateId, value_hash: Hash }, + Down { id: SubstateId }, +} + +impl SubstateChange { + pub fn id(&self) -> &SubstateId { + match self { + Self::Up { id, .. } => id, + Self::Down { id } => id, + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct StateHashTreeDiff { + pub new_nodes: Vec<(NodeKey, Node)>, + pub stale_tree_nodes: Vec, +} + +impl StateHashTreeDiff { + pub fn new() -> Self { + Self { + new_nodes: Vec::new(), + stale_tree_nodes: Vec::new(), + } + } +} + +impl From> for StateHashTreeDiff { + fn from(batch: TreeUpdateBatch) -> Self { + Self { + new_nodes: batch.node_batch, + stale_tree_nodes: batch + .stale_node_index_batch + .into_iter() + .map(|node| StaleTreeNode::Node(node.node_key)) + .collect(), + } + } +} diff --git a/dan_layer/state_tree/tests/support.rs b/dan_layer/state_tree/tests/support.rs new file mode 100644 index 000000000..6e299894d --- /dev/null +++ b/dan_layer/state_tree/tests/support.rs @@ -0,0 +1,79 @@ +// Copyright 2024 The Tari Project +// SPDX-License-Identifier: BSD-3-Clause + +use tari_engine_types::{hashing::substate_value_hasher32, substate::SubstateId}; +use tari_state_tree::{ + key_mapper::DbKeyMapper, + memory_store::MemoryTreeStore, + Hash, + LeafKey, + StateTree, + SubstateChange, + TreeStore, + Version, +}; +use tari_template_lib::models::ComponentAddress; + +pub fn change(substate_id_seed: u8, value_seed: Option) -> SubstateChange { + change_exact( + SubstateId::Component(ComponentAddress::from_array([substate_id_seed; 32])), + value_seed.map(from_seed), + ) +} + +fn hash_value(value: &[u8]) -> Hash { + substate_value_hasher32().chain(value).result().into_array().into() +} + +pub fn change_exact(substate_id: SubstateId, value: Option>) -> SubstateChange { + value + .map(|value| SubstateChange::Up { + id: substate_id.clone(), + value_hash: hash_value(&value), + }) + .unwrap_or_else(|| SubstateChange::Down { id: substate_id }) +} + +fn from_seed(node_key_seed: u8) -> Vec { + vec![node_key_seed; node_key_seed as usize] +} + +pub struct HashTreeTester { + pub tree_store: S, + pub current_version: Option, +} + +impl> HashTreeTester { + pub fn new(tree_store: S, current_version: Option) -> Self { + Self { + tree_store, + current_version, + } + } + + pub fn put_substate_changes(&mut self, changes: impl IntoIterator) -> Hash { + self.apply_database_updates(changes) + } + + fn apply_database_updates(&mut self, changes: impl IntoIterator) -> Hash { + let next_version = self.current_version.unwrap_or(0) + 1; + let current_version = self.current_version.replace(next_version).unwrap_or(0); + StateTree::<_, IdentityMapper>::new(&mut self.tree_store) + .put_substate_changes(current_version, next_version, changes) + .unwrap() + } +} + +impl HashTreeTester { + pub fn new_empty() -> Self { + Self::new(MemoryTreeStore::new(), None) + } +} + +pub struct IdentityMapper; + +impl DbKeyMapper for IdentityMapper { + fn map_to_leaf_key(id: &SubstateId) -> LeafKey { + LeafKey::new(id.to_canonical_hash().to_vec()) + } +} diff --git a/dan_layer/state_tree/tests/test.rs b/dan_layer/state_tree/tests/test.rs new file mode 100644 index 000000000..c038d736f --- /dev/null +++ b/dan_layer/state_tree/tests/test.rs @@ -0,0 +1,161 @@ +// Copyright 2024 The Tari Project +// SPDX-License-Identifier: BSD-3-Clause +// Adapted from https://github.com/radixdlt/radixdlt-scrypto/blob/868ba44ec3b806992864af27c706968c797eb961/radix-engine-stores/src/hash_tree/test.rs + +use std::collections::HashSet; + +use itertools::Itertools; +use tari_state_tree::{memory_store::MemoryTreeStore, StaleTreeNode, Version, SPARSE_MERKLE_PLACEHOLDER_HASH}; + +use crate::support::{change, HashTreeTester}; +mod support; + +#[test] +fn hash_of_next_version_differs_when_value_changed() { + let mut tester = HashTreeTester::new_empty(); + let hash_v1 = tester.put_substate_changes(vec![change(1, Some(30))]); + let hash_v2 = tester.put_substate_changes(vec![change(1, Some(70))]); + assert_ne!(hash_v1, hash_v2); +} + +#[test] +fn hash_of_next_version_same_when_write_repeated() { + let mut tester = HashTreeTester::new_empty(); + let hash_v1 = tester.put_substate_changes(vec![change(4, Some(30)), change(3, Some(40))]); + let hash_v2 = tester.put_substate_changes(vec![change(4, Some(30))]); + assert_eq!(hash_v1, hash_v2); +} + +#[test] +fn hash_of_next_version_same_when_write_empty() { + let mut tester = HashTreeTester::new_empty(); + let hash_v1 = tester.put_substate_changes(vec![change(1, Some(30)), change(3, Some(40))]); + let hash_v2 = tester.put_substate_changes(vec![]); + assert_eq!(hash_v1, hash_v2); +} + +#[test] +fn hash_of_next_version_differs_when_entry_added() { + let mut tester = HashTreeTester::new_empty(); + let hash_v1 = tester.put_substate_changes(vec![change(1, Some(30))]); + let hash_v2 = tester.put_substate_changes(vec![change(2, Some(30))]); + assert_ne!(hash_v1, hash_v2); +} + +#[test] +fn hash_of_next_version_differs_when_entry_removed() { + let mut tester = HashTreeTester::new_empty(); + let hash_v1 = tester.put_substate_changes(vec![change(1, Some(30)), change(4, Some(20))]); + let hash_v2 = tester.put_substate_changes(vec![change(1, None)]); + assert_ne!(hash_v1, hash_v2); +} + +#[test] +fn hash_returns_to_same_when_previous_state_restored() { + let mut tester = HashTreeTester::new_empty(); + let hash_v1 = tester.put_substate_changes(vec![change(1, Some(30)), change(2, Some(40))]); + tester.put_substate_changes(vec![change(1, Some(90)), change(2, None), change(3, Some(10))]); + let hash_v3 = tester.put_substate_changes(vec![change(1, Some(30)), change(2, Some(40)), change(3, None)]); + assert_eq!(hash_v1, hash_v3); +} + +#[test] +fn hash_computed_consistently_after_higher_tier_leafs_deleted() { + // Compute a "reference" hash of state containing simply [2:3:4, 2:3:5]. + let mut reference_tester = HashTreeTester::new_empty(); + let reference_root = reference_tester.put_substate_changes(vec![change(1, Some(234)), change(2, Some(235))]); + + // Compute a hash of the same state, at which we arrive after deleting some unrelated NodeId. + let mut tester = HashTreeTester::new_empty(); + tester.put_substate_changes(vec![change(3, Some(162)), change(4, Some(163)), change(1, Some(234))]); + tester.put_substate_changes(vec![change(3, None), change(4, None)]); + let root_after_deletes = tester.put_substate_changes(vec![change(2, Some(235))]); + + // We did [3,4,1] - [3,4] + [2] = [1,2] (i.e. same state). + assert_eq!(root_after_deletes, reference_root); +} + +#[test] +fn hash_computed_consistently_after_adding_higher_tier_sibling() { + // Compute a "reference" hash of state containing simply [1,2,3]. + let mut reference_tester = HashTreeTester::new_empty(); + let reference_root = + reference_tester.put_substate_changes(vec![change(1, Some(196)), change(2, Some(234)), change(3, Some(235))]); + + // Compute a hash of the same state, at which we arrive after adding some sibling NodeId. + let mut tester = HashTreeTester::new_empty(); + tester.put_substate_changes(vec![change(2, Some(234))]); + tester.put_substate_changes(vec![change(1, Some(196))]); + let root_after_adding_sibling = tester.put_substate_changes(vec![change(3, Some(235))]); + + // We did [2] + [1] + [3] = [1,2,3] (i.e. same state). + assert_eq!(root_after_adding_sibling, reference_root); +} + +#[test] +fn hash_differs_when_states_only_differ_by_node_key() { + let mut tester_1 = HashTreeTester::new_empty(); + let hash_1 = tester_1.put_substate_changes(vec![change(1, Some(30))]); + let mut tester_2 = HashTreeTester::new_empty(); + let hash_2 = tester_2.put_substate_changes(vec![change(2, Some(30))]); + assert_ne!(hash_1, hash_2); +} + +#[test] +fn hash_differs_when_states_only_differ_by_value() { + let mut tester_1 = HashTreeTester::new_empty(); + let hash_1 = tester_1.put_substate_changes(vec![change(1, Some(1))]); + let mut tester_2 = HashTreeTester::new_empty(); + let hash_2 = tester_2.put_substate_changes(vec![change(1, Some(2))]); + assert_ne!(hash_1, hash_2); +} + +#[test] +fn supports_empty_state() { + let mut tester = HashTreeTester::new_empty(); + let hash_v1 = tester.put_substate_changes(vec![]); + assert_eq!(hash_v1, SPARSE_MERKLE_PLACEHOLDER_HASH); + let hash_v2 = tester.put_substate_changes(vec![change(1, Some(30))]); + assert_ne!(hash_v2, SPARSE_MERKLE_PLACEHOLDER_HASH); + let hash_v3 = tester.put_substate_changes(vec![change(1, None)]); + assert_eq!(hash_v3, SPARSE_MERKLE_PLACEHOLDER_HASH); +} + +#[test] +fn records_stale_tree_node_keys() { + let mut tester = HashTreeTester::new_empty(); + tester.put_substate_changes(vec![change(4, Some(30))]); + tester.put_substate_changes(vec![change(3, Some(70))]); + tester.put_substate_changes(vec![change(3, Some(80))]); + let stale_versions = tester + .tree_store + .stale_nodes + .iter() + .map(|stale_part| { + let StaleTreeNode::Node(key) = stale_part else { + panic!("expected only single node removals"); + }; + key.version() + }) + .unique() + .sorted() + .collect::>(); + assert_eq!(stale_versions, vec![1, 2]); +} + +#[test] +fn serialized_keys_are_strictly_increasing() { + let mut tester = HashTreeTester::new(MemoryTreeStore::new(), None); + tester.put_substate_changes(vec![change(3, Some(90))]); + let previous_keys = tester.tree_store.nodes.keys().cloned().collect::>(); + tester.put_substate_changes(vec![change(1, Some(80))]); + let min_next_key = tester + .tree_store + .nodes + .keys() + .filter(|key| !previous_keys.contains(*key)) + .max() + .unwrap(); + let max_previous_key = previous_keys.iter().max().unwrap(); + assert!(min_next_key > max_previous_key); +} diff --git a/dan_layer/storage/Cargo.toml b/dan_layer/storage/Cargo.toml index 5d698fcfd..5b2fb31b4 100644 --- a/dan_layer/storage/Cargo.toml +++ b/dan_layer/storage/Cargo.toml @@ -19,6 +19,7 @@ tari_transaction = { workspace = true } tari_core = { workspace = true, default-features = true } tari_mmr = { workspace = true } tari_crypto = { workspace = true } +tari_state_tree = { workspace = true } anyhow = { workspace = true } chrono = { workspace = true } diff --git a/dan_layer/storage/src/consensus_models/block.rs b/dan_layer/storage/src/consensus_models/block.rs index 02e180856..fb956abf8 100644 --- a/dan_layer/storage/src/consensus_models/block.rs +++ b/dan_layer/storage/src/consensus_models/block.rs @@ -19,15 +19,23 @@ use tari_dan_common_types::{ serde_with, shard::Shard, Epoch, + NodeAddressable, NodeHeight, SubstateAddress, }; +use tari_engine_types::substate::SubstateDiff; use tari_transaction::TransactionId; use time::PrimitiveDateTime; #[cfg(feature = "ts")] use ts_rs::TS; -use super::{ForeignProposal, ForeignSendCounters, QuorumCertificate, ValidatorSchnorrSignature}; +use super::{ + ForeignProposal, + ForeignSendCounters, + QuorumCertificate, + SubstateDestroyedProof, + ValidatorSchnorrSignature, +}; use crate::{ consensus_models::{ Command, @@ -96,6 +104,7 @@ impl Block { epoch: Epoch, proposed_by: PublicKey, commands: BTreeSet, + merkle_root: FixedHash, total_leader_fee: u64, sorted_foreign_indexes: IndexMap, signature: Option, @@ -108,8 +117,7 @@ impl Block { height, epoch, proposed_by, - // TODO - merkle_root: FixedHash::zero(), + merkle_root, commands, total_leader_fee, is_dummy: false, @@ -132,6 +140,7 @@ impl Block { epoch: Epoch, proposed_by: PublicKey, commands: BTreeSet, + merkle_root: FixedHash, total_leader_fee: u64, is_dummy: bool, is_processed: bool, @@ -148,8 +157,7 @@ impl Block { height, epoch, proposed_by, - // TODO - merkle_root: FixedHash::zero(), + merkle_root, commands, total_leader_fee, is_dummy, @@ -170,6 +178,7 @@ impl Block { Epoch(0), PublicKey::default(), Default::default(), + FixedHash::zero(), 0, IndexMap::new(), None, @@ -205,6 +214,7 @@ impl Block { node_height: NodeHeight, high_qc: QuorumCertificate, epoch: Epoch, + parent_merkle_root: FixedHash, ) -> Self { let mut block = Self::new( network, @@ -214,6 +224,7 @@ impl Block { epoch, proposed_by, Default::default(), + parent_merkle_root, 0, IndexMap::new(), None, @@ -365,6 +376,10 @@ impl Block { pub fn set_signature(&mut self, signature: ValidatorSchnorrSignature) { self.signature = Some(signature); } + + pub fn is_proposed_by_addr>(&self, address: &A) -> Option { + Some(A::try_from_public_key(&self.proposed_by)? == *address) + } } impl Block { @@ -376,6 +391,7 @@ impl Block { tx.blocks_get_tip() } + /// Returns all blocks from and excluding the start block (lower height) to the end block (inclusive) pub fn get_all_blocks_between( tx: &mut TTx, start_block_id_exclusive: &BlockId, @@ -571,11 +587,12 @@ impl Block { substate: substate.into(), })); } else { - updates.push(SubstateUpdate::Destroy { - address: substate.to_substate_address(), - proof: QuorumCertificate::get(tx, &destroyed.justify)?, + updates.push(SubstateUpdate::Destroy(SubstateDestroyedProof { + substate_id: substate.substate_id.clone(), + version: substate.version, + justify: QuorumCertificate::get(tx, &destroyed.justify)?, destroyed_by_transaction: destroyed.by_transaction, - }); + })); } } else { updates.push(SubstateUpdate::Create(SubstateCreatedProof { @@ -616,7 +633,7 @@ impl Block { }; let locked_block = LockedBlock::get(tx.deref_mut())?; - if precommit_node.height() > locked_block.height { + if precommit_node.height() > locked_block.height && !precommit_node.is_genesis() { on_lock_block(tx, &locked_block, &precommit_node, locked_blocks)?; precommit_node.as_locked_block().set(tx)?; } @@ -635,10 +652,12 @@ impl Block { ); // Commit prepare_node (b) - let prepare_node = Block::get(tx.deref_mut(), prepare_node)?; - let last_executed = LastExecuted::get(tx.deref_mut())?; - on_commit(tx, &last_executed, &prepare_node)?; - prepare_node.as_last_executed().set(tx)?; + if !prepare_node.is_genesis() { + let prepare_node = Block::get(tx.deref_mut(), prepare_node)?; + let last_executed = LastExecuted::get(tx.deref_mut())?; + on_commit(tx, &last_executed, &prepare_node)?; + prepare_node.as_last_executed().set(tx)?; + } } else { debug!( target: LOG_TARGET, @@ -700,6 +719,26 @@ impl Block { } Ok(()) } + + pub fn get_all_substate_diffs( + &self, + tx: &mut TTx, + ) -> Result, StorageError> { + let transactions = self + .commands() + .iter() + .filter_map(|c| c.accept()) + .filter(|t| t.decision.is_commit()) + .map(|t| tx.transactions_get(t.id())) + .collect::, _>>()?; + + Ok(transactions + .into_iter() + // TODO: following two should never be None + .filter_map(|t_rec| t_rec.result) + .filter_map(|t_res| t_res.finalize.into_accept()) + .collect()) + } } impl Display for Block { diff --git a/dan_layer/storage/src/consensus_models/mod.rs b/dan_layer/storage/src/consensus_models/mod.rs index fcf9af7dc..d6d6e7783 100644 --- a/dan_layer/storage/src/consensus_models/mod.rs +++ b/dan_layer/storage/src/consensus_models/mod.rs @@ -17,6 +17,7 @@ mod locked_block; mod locked_output; mod quorum; mod quorum_certificate; +mod state_tree_diff; mod substate; mod transaction; mod transaction_decision; @@ -42,6 +43,7 @@ pub use locked_block::*; pub use locked_output::*; pub use quorum::*; pub use quorum_certificate::*; +pub use state_tree_diff::*; pub use substate::*; pub use transaction::*; pub use transaction_decision::*; diff --git a/dan_layer/storage/src/consensus_models/state_tree_diff.rs b/dan_layer/storage/src/consensus_models/state_tree_diff.rs new file mode 100644 index 000000000..a6de5746e --- /dev/null +++ b/dan_layer/storage/src/consensus_models/state_tree_diff.rs @@ -0,0 +1,60 @@ +// Copyright 2024 The Tari Project +// SPDX-License-Identifier: BSD-3-Clause + +// Copyright 2023 The Tari Project +// SPDX-License-Identifier: BSD-3-Clause + +use std::ops::DerefMut; + +use tari_dan_common_types::NodeHeight; + +use crate::{consensus_models::BlockId, StateStoreReadTransaction, StateStoreWriteTransaction, StorageError}; + +#[derive(Debug, Clone)] +pub struct PendingStateTreeDiff { + pub block_id: BlockId, + pub block_height: NodeHeight, + pub diff: tari_state_tree::StateHashTreeDiff, +} + +impl PendingStateTreeDiff { + pub fn new(block_id: BlockId, block_height: NodeHeight, diff: tari_state_tree::StateHashTreeDiff) -> Self { + Self { + block_id, + block_height, + diff, + } + } +} + +impl PendingStateTreeDiff { + /// Returns all pending state tree diffs from the last committed block (exclusive) to the given block (inclusive). + pub fn get_all_up_to_commit_block(tx: &mut TTx, block_id: &BlockId) -> Result, StorageError> + where TTx: StateStoreReadTransaction { + tx.pending_state_tree_diffs_get_all_up_to_commit_block(block_id) + } + + pub fn remove_by_block(tx: &mut TTx, block_id: &BlockId) -> Result + where + TTx: DerefMut + StateStoreWriteTransaction, + TTx::Target: StateStoreReadTransaction, + { + tx.pending_state_tree_diffs_remove_by_block(block_id) + } + + pub fn save(&self, tx: &mut TTx) -> Result + where + TTx: DerefMut + StateStoreWriteTransaction, + TTx::Target: StateStoreReadTransaction, + { + if tx + .deref_mut() + .pending_state_tree_diffs_exists_for_block(&self.block_id)? + { + Ok(false) + } else { + tx.pending_state_tree_diffs_insert(self)?; + Ok(true) + } + } +} diff --git a/dan_layer/storage/src/consensus_models/substate.rs b/dan_layer/storage/src/consensus_models/substate.rs index 91aafef12..16f55f2b4 100644 --- a/dan_layer/storage/src/consensus_models/substate.rs +++ b/dan_layer/storage/src/consensus_models/substate.rs @@ -278,6 +278,14 @@ pub struct SubstateCreatedProof { pub created_qc: QuorumCertificate, } +#[derive(Debug, Clone)] +pub struct SubstateDestroyedProof { + pub substate_id: SubstateId, + pub version: u32, + pub justify: QuorumCertificate, + pub destroyed_by_transaction: TransactionId, +} + #[derive(Debug, Clone)] pub struct SubstateData { pub substate_id: SubstateId, @@ -286,6 +294,12 @@ pub struct SubstateData { pub created_by_transaction: TransactionId, } +impl SubstateData { + pub fn into_substate(self) -> Substate { + Substate::new(self.version, self.substate_value) + } +} + impl From for SubstateData { fn from(value: SubstateRecord) -> Self { Self { @@ -300,11 +314,7 @@ impl From for SubstateData { #[derive(Debug, Clone)] pub enum SubstateUpdate { Create(SubstateCreatedProof), - Destroy { - address: SubstateAddress, - proof: QuorumCertificate, - destroyed_by_transaction: TransactionId, - }, + Destroy(SubstateDestroyedProof), } impl SubstateUpdate { @@ -345,21 +355,23 @@ impl SubstateUpdate { } .create(tx)?; }, - Self::Destroy { - address, - proof, + Self::Destroy(SubstateDestroyedProof { + substate_id, + version, + justify: proof, destroyed_by_transaction, - } => { + }) => { debug!( target: LOG_TARGET, - "🔥 Applying substate DESTROY for shard {} (transaction {})", - address, + "🔥 Applying substate DESTROY for substate {}v{} (transaction {})", + substate_id, + version, destroyed_by_transaction ); proof.save(tx)?; SubstateRecord::destroy_many( tx, - iter::once(address), + iter::once(SubstateAddress::from_address(&substate_id, version)), block.epoch(), block.id(), proof.id(), diff --git a/dan_layer/storage/src/consensus_models/transaction_pool.rs b/dan_layer/storage/src/consensus_models/transaction_pool.rs index c65ac0bc2..5f65577cc 100644 --- a/dan_layer/storage/src/consensus_models/transaction_pool.rs +++ b/dan_layer/storage/src/consensus_models/transaction_pool.rs @@ -17,7 +17,15 @@ use tari_dan_common_types::{ use tari_transaction::TransactionId; use crate::{ - consensus_models::{Decision, LeafBlock, LockedBlock, QcId, TransactionAtom, TransactionPoolStatusUpdate}, + consensus_models::{ + Decision, + LeafBlock, + LockedBlock, + QcId, + TransactionAtom, + TransactionPoolStatusUpdate, + TransactionRecord, + }, StateStore, StateStoreReadTransaction, StateStoreWriteTransaction, @@ -448,6 +456,14 @@ impl TransactionPoolRecord { } Ok(()) } + + pub fn get_transaction( + &self, + tx: &mut TTx, + ) -> Result { + let transaction = TransactionRecord::get(tx, self.transaction_id())?; + Ok(transaction) + } } #[derive(Debug, thiserror::Error)] diff --git a/dan_layer/storage/src/consensus_models/validated_block.rs b/dan_layer/storage/src/consensus_models/validated_block.rs index f471691e0..d82dbee1b 100644 --- a/dan_layer/storage/src/consensus_models/validated_block.rs +++ b/dan_layer/storage/src/consensus_models/validated_block.rs @@ -7,7 +7,7 @@ use tari_common_types::types::PublicKey; use tari_dan_common_types::{Epoch, NodeHeight}; use crate::{ - consensus_models::{Block, BlockId}, + consensus_models::{Block, BlockId, QuorumCertificate}, StateStoreReadTransaction, StateStoreWriteTransaction, StorageError, @@ -50,6 +50,10 @@ impl ValidBlock { self.block.proposed_by() } + pub fn justify(&self) -> &QuorumCertificate { + self.block.justify() + } + pub fn dummy_blocks(&self) -> &[Block] { &self.dummy_blocks } diff --git a/dan_layer/storage/src/shard_store.rs b/dan_layer/storage/src/shard_store.rs deleted file mode 100644 index 23089153e..000000000 --- a/dan_layer/storage/src/shard_store.rs +++ /dev/null @@ -1,256 +0,0 @@ -// Copyright 2022. The Tari Project -// -// Redistribution and use in source and binary forms, with or without modification, are permitted provided that the -// following conditions are met: -// -// 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following -// disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the -// following disclaimer in the documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote -// products derived from this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, -// INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, -// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE -// USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -use std::ops::{Deref, DerefMut}; - -use tari_dan_common_types::{ - NodeAddressable, - NodeHeight, - ObjectPledge, - ObjectPledgeInfo, - PayloadId, - QuorumCertificate, - SubstateAddress, - SubstateState, - TreeNodeHash, -}; -use tari_engine_types::substate::{Substate, SubstateId}; - -use crate::{ - models::{ - ClaimLeaderFees, - CurrentLeaderStates, - HotStuffTreeNode, - LeafNode, - Payload, - PayloadResult, - RecentTransaction, - SQLSubstate, - SQLTransaction, - SubstateShardData, - VoteMessage, - }, - StorageError, -}; - -const LOG_TARGET: &str = "tari::dan::storage"; - -pub trait ShardStore { - type Addr: NodeAddressable; - type Payload: Payload; - - type ReadTransaction<'a>: ShardStoreReadTransaction - where Self: 'a; - type WriteTransaction<'a>: ShardStoreWriteTransaction - + Deref> - + DerefMut - where Self: 'a; - - fn create_read_tx(&self) -> Result, StorageError>; - fn create_write_tx(&self) -> Result, StorageError>; - - fn with_write_tx) -> Result, R, E>(&self, f: F) -> Result - where E: From { - let mut tx = self.create_write_tx()?; - match f(&mut tx) { - Ok(r) => { - tx.commit()?; - Ok(r) - }, - Err(e) => { - if let Err(err) = tx.rollback() { - log::error!(target: LOG_TARGET, "Failed to rollback transaction: {}", err); - } - Err(e) - }, - } - } - - fn with_read_tx) -> Result, R, E>(&self, f: F) -> Result - where E: From { - let mut tx = self.create_read_tx()?; - let ret = f(&mut tx)?; - Ok(ret) - } -} - -pub trait ShardStoreReadTransaction { - fn get_high_qc_for( - &mut self, - payload_id: PayloadId, - address: SubstateAddress, - ) -> Result, StorageError>; - fn get_high_qcs(&mut self, payload_id: PayloadId) -> Result>, StorageError>; - /// Returns the current leaf node for the substate address - fn get_leaf_node(&mut self, payload_id: &PayloadId, address: &SubstateAddress) -> Result; - fn get_current_leaders_states(&mut self, payload: &PayloadId) -> Result, StorageError>; - fn get_payload(&mut self, payload_id: &PayloadId) -> Result; - fn get_node(&mut self, node_hash: &TreeNodeHash) -> Result, StorageError>; - fn get_locked_node_hash_and_height( - &mut self, - payload_id: PayloadId, - address: SubstateAddress, - ) -> Result<(TreeNodeHash, NodeHeight), StorageError>; - fn get_last_executed_height(&mut self, address: SubstateAddress, payload_id: PayloadId) -> Result; - fn get_state_inventory(&mut self) -> Result, StorageError>; - fn get_substate_states(&mut self, addresses: &[SubstateAddress]) -> Result, StorageError>; - fn get_substate_states_by_range( - &mut self, - start_address: SubstateAddress, - end_address: SubstateAddress, - excluded_addresses: &[SubstateAddress], - ) -> Result, StorageError>; - /// Returns the last voted height. A height of 0 means that no previous vote height has been recorded for the - /// pair. - fn get_last_voted_height( - &mut self, - address: SubstateAddress, - payload_id: PayloadId, - ) -> Result<(NodeHeight, u32), StorageError>; - fn get_leader_proposals( - &mut self, - payload: PayloadId, - payload_height: NodeHeight, - addresses: &[SubstateAddress], - ) -> Result>, StorageError>; - fn get_last_payload_height_for_leader_proposal( - &mut self, - payload: PayloadId, - address: SubstateAddress, - ) -> Result; - fn has_vote_for(&mut self, from: &TAddr, node_hash: TreeNodeHash) -> Result; - fn get_received_votes_for(&mut self, node_hash: TreeNodeHash) -> Result, StorageError>; - fn get_recent_transactions(&mut self) -> Result, StorageError>; - fn get_transaction(&mut self, payload_id: Vec) -> Result, StorageError>; - fn get_substates_for_payload( - &mut self, - payload_id: Vec, - address: Vec, - ) -> Result, StorageError>; - fn get_fees_by_epoch( - &mut self, - epoch: u64, - claim_leader_public_key: Vec, - ) -> Result, StorageError>; - fn get_payload_result(&mut self, payload_id: &PayloadId) -> Result; - fn get_resolved_pledges_for_payload(&mut self, payload: PayloadId) -> Result, StorageError>; -} - -pub trait ShardStoreWriteTransaction { - fn commit(self) -> Result<(), StorageError>; - fn rollback(self) -> Result<(), StorageError>; - fn insert_high_qc(&mut self, from: TAddr, address: SubstateAddress, qc: QuorumCertificate) - -> Result<(), StorageError>; - fn save_payload(&mut self, payload: TPayload) -> Result<(), StorageError>; - fn save_current_leader_state( - &mut self, - payload: PayloadId, - address: SubstateAddress, - leader_round: u32, - leader: TAddr, - ) -> Result<(), StorageError>; - /// Inserts or updates the leaf node for the substate address - fn set_leaf_node( - &mut self, - payload_id: PayloadId, - address: SubstateAddress, - node: TreeNodeHash, - payload_height: NodeHeight, - height: NodeHeight, - ) -> Result<(), StorageError>; - fn save_node(&mut self, node: HotStuffTreeNode) -> Result<(), StorageError>; - fn set_locked( - &mut self, - payload_id: PayloadId, - address: SubstateAddress, - node_hash: TreeNodeHash, - node_height: NodeHeight, - ) -> Result<(), StorageError>; - - fn set_last_executed_height( - &mut self, - address: SubstateAddress, - payload_id: PayloadId, - height: NodeHeight, - ) -> Result<(), StorageError>; - fn commit_substate_changes( - &mut self, - node: HotStuffTreeNode, - changes: &[SubstateState], - ) -> Result<(), StorageError>; - fn insert_substates(&mut self, substate_data: SubstateShardData) -> Result<(), StorageError>; - fn set_last_voted_height( - &mut self, - address: SubstateAddress, - payload_id: PayloadId, - height: NodeHeight, - leader_round: u32, - ) -> Result<(), StorageError>; - - fn save_leader_proposals( - &mut self, - address: SubstateAddress, - payload: PayloadId, - payload_height: NodeHeight, - leader_round: u32, - node: HotStuffTreeNode, - ) -> Result<(), StorageError>; - - fn save_received_vote_for( - &mut self, - from: TAddr, - node_hash: TreeNodeHash, - vote_message: VoteMessage, - ) -> Result<(), StorageError>; - - /// Updates the result for an existing payload - fn update_payload_result(&mut self, payload_id: &PayloadId, result: PayloadResult) -> Result<(), StorageError>; - - fn mark_payload_finalized(&mut self, payload_id: &PayloadId) -> Result<(), StorageError>; - - // -------------------------------- Pledges -------------------------------- // - fn pledge_object( - &mut self, - address: SubstateAddress, - payload: PayloadId, - current_height: NodeHeight, - ) -> Result; - fn complete_pledges( - &mut self, - address: SubstateAddress, - payload_id: PayloadId, - node_hash: &TreeNodeHash, - ) -> Result<(), StorageError>; - fn abandon_pledges( - &mut self, - address: SubstateAddress, - payload_id: PayloadId, - node_hash: &TreeNodeHash, - ) -> Result<(), StorageError>; - - fn save_burnt_utxo( - &mut self, - substate: &Substate, - commitment_address: SubstateId, - address: SubstateAddress, - ) -> Result<(), StorageError>; -} diff --git a/dan_layer/storage/src/state_store/mod.rs b/dan_layer/storage/src/state_store/mod.rs index 979b8a00e..368ca6aa5 100644 --- a/dan_layer/storage/src/state_store/mod.rs +++ b/dan_layer/storage/src/state_store/mod.rs @@ -4,12 +4,13 @@ use std::{ borrow::Borrow, collections::HashSet, - ops::{Deref, DerefMut, RangeInclusive}, + ops::{DerefMut, RangeInclusive}, }; use serde::{Deserialize, Serialize}; use tari_common_types::types::{FixedHash, PublicKey}; use tari_dan_common_types::{Epoch, NodeAddressable, NodeHeight, SubstateAddress}; +use tari_state_tree::{TreeStore, TreeStoreReader, Version}; use tari_transaction::{Transaction, TransactionId}; #[cfg(feature = "ts")] use ts_rs::TS; @@ -31,6 +32,7 @@ use crate::{ LeafBlock, LockedBlock, LockedOutput, + PendingStateTreeDiff, QcId, QuorumCertificate, SubstateLockFlag, @@ -50,11 +52,11 @@ const LOG_TARGET: &str = "tari::dan::storage"; pub trait StateStore { type Addr: NodeAddressable; - type ReadTransaction<'a>: StateStoreReadTransaction + type ReadTransaction<'a>: StateStoreReadTransaction + TreeStoreReader where Self: 'a; type WriteTransaction<'a>: StateStoreWriteTransaction - + Deref> - + DerefMut + + TreeStore + + DerefMut> where Self: 'a; fn create_read_tx(&self) -> Result, StorageError>; @@ -87,7 +89,6 @@ pub trait StateStore { pub trait StateStoreReadTransaction { type Addr: NodeAddressable; - fn last_sent_vote_get(&mut self) -> Result; fn last_voted_get(&mut self) -> Result; fn last_executed_get(&mut self) -> Result; @@ -123,6 +124,7 @@ pub trait StateStoreReadTransaction { ) -> Result, StorageError>; fn blocks_get(&mut self, block_id: &BlockId) -> Result; fn blocks_get_tip(&mut self) -> Result; + /// Returns all blocks from and excluding the start block (lower height) to the end block (inclusive) fn blocks_get_all_between( &mut self, start_block_id_exclusive: &BlockId, @@ -237,6 +239,12 @@ pub trait StateStoreReadTransaction { where I: IntoIterator, B: Borrow; + + fn pending_state_tree_diffs_exists_for_block(&mut self, block_id: &BlockId) -> Result; + fn pending_state_tree_diffs_get_all_up_to_commit_block( + &mut self, + block_id: &BlockId, + ) -> Result, StorageError>; } pub trait StateStoreWriteTransaction { @@ -373,6 +381,12 @@ pub trait StateStoreWriteTransaction { where I: IntoIterator, B: Borrow; + // -------------------------------- Pending State Tree Diffs -------------------------------- // + fn pending_state_tree_diffs_insert(&mut self, diff: &PendingStateTreeDiff) -> Result<(), StorageError>; + fn pending_state_tree_diffs_remove_by_block( + &mut self, + block_id: &BlockId, + ) -> Result; } #[derive(Debug, Clone, Copy, Serialize, Deserialize)]