From 33a84ab39bfad0f16d5d096c74c28f6694c14283 Mon Sep 17 00:00:00 2001 From: Alex Pyattaev Date: Thu, 6 Feb 2025 21:52:32 +0000 Subject: [PATCH 1/2] basic ideas --- Cargo.lock | 36 ++++++++++++++++++++++++++++++++++++ gossip/Cargo.toml | 8 ++++++-- gossip/src/crds_data.rs | 21 ++++++++++++++++----- gossip/src/lib.rs | 3 +++ 4 files changed, 61 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 014977803e0df6..4f15a7bea615a7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1292,6 +1292,15 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" +[[package]] +name = "byteorder_slice" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b294e30387378958e8bf8f4242131b930ea615ff81e8cac2440cea0a6013190" +dependencies = [ + "byteorder", +] + [[package]] name = "bytes" version = "1.10.0" @@ -2025,6 +2034,17 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "derive-into-owned" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d94d81e3819a7b06a8638f448bc6339371ca9b6076a99d4a43eece3c4c923" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "derive-where" version = "1.2.7" @@ -4479,6 +4499,17 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "pcap-file" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fc1f139757b058f9f37b76c48501799d12c9aa0aa4c0d4c980b062ee925d1b2" +dependencies = [ + "byteorder_slice", + "derive-into-owned", + "thiserror 1.0.69", +] + [[package]] name = "pem" version = "1.1.1" @@ -7824,10 +7855,12 @@ dependencies = [ name = "solana-gossip" version = "2.2.0" dependencies = [ + "anyhow", "assert_matches", "bincode", "bs58", "bv", + "cfg-if 1.0.0", "clap 2.33.3", "criterion", "crossbeam-channel", @@ -7838,6 +7871,7 @@ dependencies = [ "lru", "num-traits", "num_cpus", + "pcap-file", "rand 0.7.3", "rand 0.8.5", "rand_chacha 0.2.2", @@ -7847,6 +7881,7 @@ dependencies = [ "serde-big-array", "serde_bytes", "serde_derive", + "serde_json", "serial_test", "siphasher", "solana-bloom", @@ -7857,6 +7892,7 @@ dependencies = [ "solana-feature-set", "solana-frozen-abi", "solana-frozen-abi-macro", + "solana-gossip", "solana-ledger", "solana-logger", "solana-measure", diff --git a/gossip/Cargo.toml b/gossip/Cargo.toml index 3b10339246f698..8f8f5ba79c3b9e 100644 --- a/gossip/Cargo.toml +++ b/gossip/Cargo.toml @@ -10,9 +10,11 @@ license = { workspace = true } edition = { workspace = true } [dependencies] +anyhow = { workspace = true } assert_matches = { workspace = true } bincode = { workspace = true } bv = { workspace = true, features = ["serde"] } +cfg-if = { workspace = true } clap = { workspace = true } crossbeam-channel = { workspace = true } flate2 = { workspace = true } @@ -21,6 +23,7 @@ itertools = { workspace = true } log = { workspace = true } lru = { workspace = true } num-traits = { workspace = true } +pcap-file = "2.0.0" rand = { workspace = true } rand_chacha = { workspace = true } rayon = { workspace = true } @@ -28,6 +31,7 @@ serde = { workspace = true } serde-big-array = { workspace = true } serde_bytes = { workspace = true } serde_derive = { workspace = true } +serde_json = { workspace = true, optional = true } siphasher = { workspace = true } solana-bloom = { workspace = true } solana-clap-utils = { workspace = true } @@ -62,7 +66,6 @@ solana-vote = { workspace = true } solana-vote-program = { workspace = true } static_assertions = { workspace = true } thiserror = { workspace = true } - [dev-dependencies] bs58 = { workspace = true } criterion = { workspace = true } @@ -70,10 +73,10 @@ num_cpus = { workspace = true } rand0-7 = { workspace = true } rand_chacha0-2 = { workspace = true } serial_test = { workspace = true } +solana-gossip = { workspace = true, features = ["dev-context-only-utils"] } solana-perf = { workspace = true, features = ["dev-context-only-utils"] } solana-runtime = { workspace = true, features = ["dev-context-only-utils"] } test-case = { workspace = true } - [features] frozen-abi = [ "dep:solana-frozen-abi", @@ -88,6 +91,7 @@ frozen-abi = [ "solana-vote/frozen-abi", "solana-vote-program/frozen-abi", ] +dev-context-only-utils = [] [[bench]] name = "crds" diff --git a/gossip/src/crds_data.rs b/gossip/src/crds_data.rs index 67cd680647f1ce..421d6ca890a0fd 100644 --- a/gossip/src/crds_data.rs +++ b/gossip/src/crds_data.rs @@ -7,7 +7,6 @@ use { legacy_contact_info::LegacyContactInfo, restart_crds_values::{RestartHeaviestFork, RestartLastVotedForkSlots}, }, - rand::Rng, serde::de::{Deserialize, Deserializer}, solana_sanitize::{Sanitize, SanitizeError}, solana_sdk::{ @@ -21,6 +20,9 @@ use { std::{cmp::Ordering, collections::BTreeSet}, }; +#[cfg(feature = "dev-context-only-utils")] +use {crate::format_validation::FormatValidation, rand::Rng, std::hint::black_box}; + pub(crate) const MAX_WALLCLOCK: u64 = 1_000_000_000_000_000; pub(crate) const MAX_SLOT: u64 = 1_000_000_000_000_000; /// Maximum number of hashes in AccountsHashes a node publishes @@ -233,9 +235,9 @@ impl Sanitize for AccountsHashes { } } -impl AccountsHashes { - /// New random AccountsHashes for tests and benchmarks. - pub(crate) fn new_rand(rng: &mut R, pubkey: Option) -> Self { +#[cfg(feature = "dev-context-only-utils")] +impl FormatValidation for AccountsHashes { + fn new_rand(rng: &mut R, pubkey: Option) -> Self { let num_hashes = rng.gen_range(0..MAX_ACCOUNTS_HASHES) + 1; let hashes = std::iter::repeat_with(|| { let slot = 47825632 + rng.gen_range(0..512); @@ -250,6 +252,13 @@ impl AccountsHashes { wallclock: new_rand_timestamp(rng), } } + + fn exercise(&self) -> anyhow::Result<()> { + FormatValidation::exercise(&self)?; + let s: u64 = self.hashes.iter().map(|v| v.0).sum(); + black_box(s); + Ok(()) + } } type LegacySnapshotHashes = AccountsHashes; @@ -303,8 +312,10 @@ impl LowestSlot { wallclock, } } +} - /// New random LowestSlot for tests and benchmarks. +#[cfg(feature = "dev-context-only-utils")] +impl FormatValidation for LowestSlot { fn new_rand(rng: &mut R, pubkey: Option) -> Self { Self { from: pubkey.unwrap_or_else(pubkey::new_rand), diff --git a/gossip/src/lib.rs b/gossip/src/lib.rs index 06df390c59213f..f9f6937448cee7 100644 --- a/gossip/src/lib.rs +++ b/gossip/src/lib.rs @@ -46,3 +46,6 @@ extern crate solana_frozen_abi_macro; #[macro_use] extern crate solana_metrics; + +#[cfg(feature = "dev-context-only-utils")] +pub mod format_validation; From 761c43a82c2e9e43df6851b426e9b79c9f8b1794 Mon Sep 17 00:00:00 2001 From: Alex Pyattaev Date: Sat, 8 Feb 2025 20:16:14 +0000 Subject: [PATCH 2/2] first working version --- gossip/Cargo.toml | 3 +- gossip/benches/crds.rs | 1 + gossip/benches/crds_gossip_pull.rs | 1 + gossip/benches/crds_shards.rs | 1 + gossip/src/cluster_info.rs | 4 +- gossip/src/crds.rs | 17 +++-- gossip/src/crds_data.rs | 64 +++++++++--------- gossip/src/crds_entry.rs | 2 +- gossip/src/crds_gossip_pull.rs | 3 +- gossip/src/crds_shards.rs | 1 + gossip/src/crds_value.rs | 42 +++++++----- gossip/src/epoch_slots.rs | 19 ++++-- gossip/src/lib.rs | 2 +- gossip/src/ping_pong.rs | 31 ++++++++- gossip/src/protocol.rs | 102 +++++++++++++++++++++-------- gossip/src/restart_crds_values.rs | 45 +++++++------ gossip/src/testing_fixtures.rs | 80 ++++++++++++++++++++++ programs/sbf/Cargo.lock | 35 ++++++++++ svm/examples/Cargo.lock | 35 ++++++++++ 19 files changed, 377 insertions(+), 111 deletions(-) create mode 100644 gossip/src/testing_fixtures.rs diff --git a/gossip/Cargo.toml b/gossip/Cargo.toml index 8f8f5ba79c3b9e..1e3174974a78ce 100644 --- a/gossip/Cargo.toml +++ b/gossip/Cargo.toml @@ -25,6 +25,7 @@ lru = { workspace = true } num-traits = { workspace = true } pcap-file = "2.0.0" rand = { workspace = true } +rand0-7 = { workspace = true } rand_chacha = { workspace = true } rayon = { workspace = true } serde = { workspace = true } @@ -73,7 +74,7 @@ num_cpus = { workspace = true } rand0-7 = { workspace = true } rand_chacha0-2 = { workspace = true } serial_test = { workspace = true } -solana-gossip = { workspace = true, features = ["dev-context-only-utils"] } +solana-gossip = { path = ".", features = ["dev-context-only-utils"] } solana-perf = { workspace = true, features = ["dev-context-only-utils"] } solana-runtime = { workspace = true, features = ["dev-context-only-utils"] } test-case = { workspace = true } diff --git a/gossip/benches/crds.rs b/gossip/benches/crds.rs index 71b2f4c870fca1..bdff76511c7937 100644 --- a/gossip/benches/crds.rs +++ b/gossip/benches/crds.rs @@ -9,6 +9,7 @@ use { crds::{Crds, GossipRoute}, crds_gossip_pull::{CrdsTimeouts, CRDS_GOSSIP_PULL_CRDS_TIMEOUT_MS}, crds_value::CrdsValue, + testing_fixtures::FormatValidation, }, solana_pubkey::Pubkey, std::{collections::HashMap, time::Duration}, diff --git a/gossip/benches/crds_gossip_pull.rs b/gossip/benches/crds_gossip_pull.rs index 76961259b82061..f93d81a559a7d9 100644 --- a/gossip/benches/crds_gossip_pull.rs +++ b/gossip/benches/crds_gossip_pull.rs @@ -9,6 +9,7 @@ use { crds::{Crds, GossipRoute}, crds_gossip_pull::{CrdsFilter, CrdsGossipPull}, crds_value::CrdsValue, + testing_fixtures::FormatValidation, }, solana_sdk::hash::Hash, std::sync::RwLock, diff --git a/gossip/benches/crds_shards.rs b/gossip/benches/crds_shards.rs index 1ed5816f4f1514..6b1c993060346f 100644 --- a/gossip/benches/crds_shards.rs +++ b/gossip/benches/crds_shards.rs @@ -8,6 +8,7 @@ use { crds::{Crds, GossipRoute, VersionedCrdsValue}, crds_shards::CrdsShards, crds_value::CrdsValue, + testing_fixtures::FormatValidation, }, solana_sdk::timing::timestamp, std::iter::repeat_with, diff --git a/gossip/src/cluster_info.rs b/gossip/src/cluster_info.rs index 2497925f20448d..357ec931ae47ed 100644 --- a/gossip/src/cluster_info.rs +++ b/gossip/src/cluster_info.rs @@ -3126,6 +3126,7 @@ mod tests { duplicate_shred::tests::new_rand_shred, protocol::tests::new_rand_remote_node, socketaddr, + testing_fixtures::{new_insecure_keypair, FormatValidation}, }, bincode::serialize, itertools::izip, @@ -3796,7 +3797,8 @@ mod tests { #[test] fn test_append_entrypoint_to_pulls() { let thread_pool = ThreadPoolBuilder::new().build().unwrap(); - let node_keypair = Arc::new(Keypair::new()); + let mut rng = rand::thread_rng(); + let node_keypair = Arc::new(new_insecure_keypair(&mut rng)); let cluster_info = ClusterInfo::new( ContactInfo::new_localhost(&node_keypair.pubkey(), timestamp()), node_keypair, diff --git a/gossip/src/crds.rs b/gossip/src/crds.rs index b30b7df94d85e3..a5e43a1de77a56 100644 --- a/gossip/src/crds.rs +++ b/gossip/src/crds.rs @@ -781,7 +781,10 @@ fn should_report_message_signature(signature: &Signature) -> bool { mod tests { use { super::*, - crate::crds_data::{new_rand_timestamp, AccountsHashes, NodeInstance}, + crate::{ + crds_data::{AccountsHashes, NodeInstance}, + testing_fixtures::*, + }, rand::{thread_rng, Rng, SeedableRng}, rand_chacha::ChaChaRng, rayon::ThreadPoolBuilder, @@ -1250,7 +1253,9 @@ mod tests { #[test] fn test_crds_value_indices() { let mut rng = thread_rng(); - let keypairs: Vec<_> = repeat_with(Keypair::new).take(128).collect(); + let keypairs: Vec<_> = repeat_with(|| new_insecure_keypair(&mut rng)) + .take(128) + .collect(); let mut crds = Crds::default(); let mut num_inserts = 0; for k in 0..4096 { @@ -1301,7 +1306,9 @@ mod tests { } } let mut rng = thread_rng(); - let keypairs: Vec<_> = repeat_with(Keypair::new).take(128).collect(); + let keypairs: Vec<_> = repeat_with(|| new_insecure_keypair(&mut rng)) + .take(128) + .collect(); let mut crds = Crds::default(); for k in 0..4096 { let keypair = &keypairs[rng.gen_range(0..keypairs.len())]; @@ -1400,7 +1407,9 @@ mod tests { .len() } let mut rng = thread_rng(); - let keypairs: Vec<_> = repeat_with(Keypair::new).take(64).collect(); + let keypairs: Vec<_> = repeat_with(|| new_insecure_keypair(&mut rng)) + .take(64) + .collect(); let stakes = keypairs .iter() .map(|k| (k.pubkey(), rng.gen_range(0..1000))) diff --git a/gossip/src/crds_data.rs b/gossip/src/crds_data.rs index 421d6ca890a0fd..559142be5a0f33 100644 --- a/gossip/src/crds_data.rs +++ b/gossip/src/crds_data.rs @@ -1,3 +1,9 @@ +#[cfg(feature = "dev-context-only-utils")] +use { + crate::testing_fixtures::{new_rand_timestamp, new_random_pubkey, FormatValidation}, + rand::Rng, + std::hint::black_box, +}; use { crate::{ contact_info::ContactInfo, @@ -9,20 +15,11 @@ use { }, serde::de::{Deserialize, Deserializer}, solana_sanitize::{Sanitize, SanitizeError}, - solana_sdk::{ - clock::Slot, - hash::Hash, - pubkey::{self, Pubkey}, - timing::timestamp, - transaction::Transaction, - }, + solana_sdk::{clock::Slot, hash::Hash, pubkey::Pubkey, transaction::Transaction}, solana_vote::vote_parser, std::{cmp::Ordering, collections::BTreeSet}, }; -#[cfg(feature = "dev-context-only-utils")] -use {crate::format_validation::FormatValidation, rand::Rng, std::hint::black_box}; - pub(crate) const MAX_WALLCLOCK: u64 = 1_000_000_000_000_000; pub(crate) const MAX_SLOT: u64 = 1_000_000_000_000_000; /// Maximum number of hashes in AccountsHashes a node publishes @@ -109,21 +106,16 @@ impl Sanitize for CrdsData { } } -/// Random timestamp for tests and benchmarks. -pub(crate) fn new_rand_timestamp(rng: &mut R) -> u64 { - const DELAY: u64 = 10 * 60 * 1000; // 10 minutes - timestamp() - DELAY + rng.gen_range(0..2 * DELAY) -} - -impl CrdsData { +#[cfg(feature = "dev-context-only-utils")] +impl crate::testing_fixtures::FormatValidation for CrdsData { /// New random CrdsData for tests and benchmarks. - pub(crate) fn new_rand(rng: &mut R, pubkey: Option) -> CrdsData { + fn new_rand(rng: &mut R, pubkey: Option) -> CrdsData { let kind = rng.gen_range(0..8); // TODO: Implement other kinds of CrdsData here. // TODO: Assign ranges to each arm proportional to their frequency in // the mainnet crds table. match kind { - 0 => CrdsData::from(ContactInfo::new_rand(rng, pubkey)), + 0 => CrdsData::ContactInfo(ContactInfo::new_rand(rng, pubkey)), // Index for LowestSlot is deprecated and should be zero. 1 => CrdsData::LowestSlot(0, LowestSlot::new_rand(rng, pubkey)), 2 => CrdsData::LegacySnapshotHashes(LegacySnapshotHashes::new_rand(rng, pubkey)), @@ -139,7 +131,9 @@ impl CrdsData { ), } } +} +impl CrdsData { pub(crate) fn wallclock(&self) -> u64 { match self { CrdsData::LegacyContactInfo(contact_info) => contact_info.wallclock(), @@ -226,6 +220,9 @@ pub(crate) struct AccountsHashes { impl Sanitize for AccountsHashes { fn sanitize(&self) -> Result<(), SanitizeError> { sanitize_wallclock(self.wallclock)?; + if self.hashes.len() > MAX_ACCOUNTS_HASHES { + return Err(SanitizeError::IndexOutOfBounds); + } for (slot, _) in &self.hashes { if *slot >= MAX_SLOT { return Err(SanitizeError::ValueOutOfBounds); @@ -247,14 +244,14 @@ impl FormatValidation for AccountsHashes { .take(num_hashes) .collect(); Self { - from: pubkey.unwrap_or_else(pubkey::new_rand), + from: pubkey.unwrap_or_else(|| new_random_pubkey(rng)), hashes, wallclock: new_rand_timestamp(rng), } } fn exercise(&self) -> anyhow::Result<()> { - FormatValidation::exercise(&self)?; + self.sanitize()?; let s: u64 = self.hashes.iter().map(|v| v.0).sum(); black_box(s); Ok(()) @@ -318,7 +315,7 @@ impl LowestSlot { impl FormatValidation for LowestSlot { fn new_rand(rng: &mut R, pubkey: Option) -> Self { Self { - from: pubkey.unwrap_or_else(pubkey::new_rand), + from: pubkey.unwrap_or_else(|| new_random_pubkey(rng)), root: rng.gen(), lowest: rng.gen(), slots: BTreeSet::default(), @@ -365,6 +362,19 @@ impl Sanitize for Vote { } } +#[cfg(feature = "dev-context-only-utils")] +impl FormatValidation for Vote { + /// New random Vote for tests and benchmarks. + fn new_rand(rng: &mut R, pubkey: Option) -> Self { + Self { + from: pubkey.unwrap_or_else(|| new_random_pubkey(rng)), + transaction: Transaction::default(), + wallclock: new_rand_timestamp(rng), + slot: None, + } + } +} + impl Vote { // Returns None if cannot parse transaction into a vote. pub fn new(from: Pubkey, transaction: Transaction, wallclock: u64) -> Option { @@ -376,16 +386,6 @@ impl Vote { }) } - /// New random Vote for tests and benchmarks. - fn new_rand(rng: &mut R, pubkey: Option) -> Self { - Self { - from: pubkey.unwrap_or_else(pubkey::new_rand), - transaction: Transaction::default(), - wallclock: new_rand_timestamp(rng), - slot: None, - } - } - pub(crate) fn transaction(&self) -> &Transaction { &self.transaction } diff --git a/gossip/src/crds_entry.rs b/gossip/src/crds_entry.rs index 43aeda1a686d38..9c54c78c9f6ce0 100644 --- a/gossip/src/crds_entry.rs +++ b/gossip/src/crds_entry.rs @@ -65,7 +65,7 @@ mod tests { super::*, crate::{ crds::{Crds, GossipRoute}, - crds_data::new_rand_timestamp, + testing_fixtures::*, }, rand::seq::SliceRandom, solana_sdk::signature::Keypair, diff --git a/gossip/src/crds_gossip_pull.rs b/gossip/src/crds_gossip_pull.rs index cc522e906da2b3..cbf54dd3145a71 100644 --- a/gossip/src/crds_gossip_pull.rs +++ b/gossip/src/crds_gossip_pull.rs @@ -82,7 +82,7 @@ impl solana_sanitize::Sanitize for CrdsFilter { } impl CrdsFilter { - #[cfg(test)] + #[cfg(feature = "dev-context-only-utils")] pub(crate) fn new_rand(num_items: usize, max_bytes: usize) -> Self { let max_bits = (max_bytes * 8) as f64; let max_items = Self::max_items(max_bits, FALSE_RATE, KEYS); @@ -666,6 +666,7 @@ pub(crate) mod tests { crds_data::{CrdsData, Vote}, legacy_contact_info::LegacyContactInfo, protocol::Protocol, + testing_fixtures::*, }, itertools::Itertools, rand::{seq::SliceRandom, SeedableRng}, diff --git a/gossip/src/crds_shards.rs b/gossip/src/crds_shards.rs index 319a18126debfb..a8cdc48b337835 100644 --- a/gossip/src/crds_shards.rs +++ b/gossip/src/crds_shards.rs @@ -137,6 +137,7 @@ mod test { crate::{ crds::{Crds, GossipRoute}, crds_value::CrdsValue, + testing_fixtures::*, }, rand::{thread_rng, Rng}, solana_sdk::timing::timestamp, diff --git a/gossip/src/crds_value.rs b/gossip/src/crds_value.rs index b40d7e6f6221c0..e419ae129efce1 100644 --- a/gossip/src/crds_value.rs +++ b/gossip/src/crds_value.rs @@ -1,3 +1,8 @@ +#[cfg(feature = "dev-context-only-utils")] +use { + crate::testing_fixtures::{new_insecure_keypair, FormatValidation}, + rand::Rng, +}; use { crate::{ contact_info::ContactInfo, @@ -6,7 +11,6 @@ use { epoch_slots::EpochSlots, }, bincode::serialize, - rand::Rng, serde::de::{Deserialize, Deserializer}, solana_sanitize::{Sanitize, SanitizeError}, solana_sdk::{ @@ -17,7 +21,8 @@ use { std::borrow::{Borrow, Cow}, }; -/// CrdsValue that is replicated across the cluster +/// CrdsValue is a wrapper around CrdsData that is replicated across the cluster +/// CrdsValue's purpose is to add signature to verify the origin of the CrdsData #[cfg_attr(feature = "frozen-abi", derive(AbiExample))] #[derive(Serialize, Clone, Debug, PartialEq, Eq)] pub struct CrdsValue { @@ -98,6 +103,24 @@ impl CrdsValueLabel { } } +#[cfg(feature = "dev-context-only-utils")] +impl FormatValidation<&Keypair> for CrdsValue { + fn new_rand(rng: &mut R, keypair: Option<&Keypair>) -> Self { + let mut random_keypair: Option = None; + let keypair = keypair + .unwrap_or_else(|| random_keypair.get_or_insert_with(|| new_insecure_keypair(rng))); + let data = CrdsData::new_rand(rng, Some(keypair.pubkey())); + Self::new(data, keypair) + } + + fn exercise(&self) -> anyhow::Result<()> { + self.sanitize()?; + if !self.verify() { + anyhow::bail!("sigverify"); + }; + self.data().exercise() + } +} impl CrdsValue { pub fn new(data: CrdsData, keypair: &Keypair) -> Self { let bincode_serialized_data = bincode::serialize(&data).unwrap(); @@ -122,21 +145,6 @@ impl CrdsValue { } } - /// New random CrdsValue for tests and benchmarks. - pub fn new_rand(rng: &mut R, keypair: Option<&Keypair>) -> CrdsValue { - match keypair { - None => { - let keypair = Keypair::new(); - let data = CrdsData::new_rand(rng, Some(keypair.pubkey())); - Self::new(data, &keypair) - } - Some(keypair) => { - let data = CrdsData::new_rand(rng, Some(keypair.pubkey())); - Self::new(data, keypair) - } - } - } - #[inline] pub(crate) fn signature(&self) -> &Signature { &self.signature diff --git a/gossip/src/epoch_slots.rs b/gossip/src/epoch_slots.rs index 200f63bbad1089..7c34bd12d2311c 100644 --- a/gossip/src/epoch_slots.rs +++ b/gossip/src/epoch_slots.rs @@ -1,6 +1,8 @@ +#[cfg(feature = "dev-context-only-utils")] +use crate::testing_fixtures::*; use { crate::{ - crds_data::{self, MAX_SLOT, MAX_WALLCLOCK}, + crds_data::{MAX_SLOT, MAX_WALLCLOCK}, protocol::MAX_CRDS_OBJECT_SIZE, }, bincode::serialized_size, @@ -341,10 +343,12 @@ impl EpochSlots { .filter_map(move |s| s.to_slots(min_slot).ok()) .flatten() } +} - /// New random EpochSlots for tests and simulations. - pub(crate) fn new_rand(rng: &mut R, pubkey: Option) -> Self { - let now = crds_data::new_rand_timestamp(rng); +#[cfg(feature = "dev-context-only-utils")] +impl FormatValidation for EpochSlots { + fn new_rand(rng: &mut R, pubkey: Option) -> Self { + let now = new_rand_timestamp(rng); let pubkey = pubkey.unwrap_or_else(solana_pubkey::new_rand); let mut epoch_slots = Self::new(pubkey, now); let num_slots = rng.gen_range(0..20); @@ -354,6 +358,13 @@ impl EpochSlots { epoch_slots.add(&slots); epoch_slots } + + fn exercise(&self) -> anyhow::Result<()> { + self.sanitize()?; + let s: Slot = self.to_slots(0).sum(); + std::hint::black_box(s); + Ok(()) + } } #[cfg(test)] diff --git a/gossip/src/lib.rs b/gossip/src/lib.rs index f9f6937448cee7..63098429c65713 100644 --- a/gossip/src/lib.rs +++ b/gossip/src/lib.rs @@ -48,4 +48,4 @@ extern crate solana_frozen_abi_macro; extern crate solana_metrics; #[cfg(feature = "dev-context-only-utils")] -pub mod format_validation; +pub mod testing_fixtures; diff --git a/gossip/src/ping_pong.rs b/gossip/src/ping_pong.rs index 7fef7c93918d81..2bcde6a9fecd7c 100644 --- a/gossip/src/ping_pong.rs +++ b/gossip/src/ping_pong.rs @@ -1,3 +1,5 @@ +#[cfg(feature = "dev-context-only-utils")] +use crate::testing_fixtures::*; use { lru::LruCache, rand::{CryptoRng, Rng}, @@ -82,6 +84,17 @@ impl Sanitize for Ping { } } +#[cfg(feature = "dev-context-only-utils")] +impl FormatValidation<&Keypair> for Ping<32> { + fn new_rand(rng: &mut R, keypair: Option<&Keypair>) -> Self { + let mut random_keypair: Option = None; + let keypair = keypair + .unwrap_or_else(|| random_keypair.get_or_insert_with(|| new_insecure_keypair(rng))); + let token: [u8; 32] = rng.gen(); + Ping::<32>::new(token, keypair) + } +} + impl Signable for Ping { #[inline] fn pubkey(&self) -> Pubkey { @@ -126,6 +139,21 @@ impl Sanitize for Pong { } } +#[cfg(feature = "dev-context-only-utils")] +impl FormatValidation<&Keypair> for Pong { + fn new_rand(rng: &mut R, keypair: Option<&Keypair>) -> Self { + let mut random_keypair: Option = None; + let keypair = keypair + .unwrap_or_else(|| random_keypair.get_or_insert_with(|| new_insecure_keypair(rng))); + let hash = Hash::new_unique(); + Pong { + from: keypair.pubkey(), + hash, + signature: keypair.sign_message(hash.as_ref()), + } + } +} + impl Signable for Pong { fn pubkey(&self) -> Pubkey { self.from @@ -271,6 +299,7 @@ fn hash_ping_token(token: &[u8; N]) -> Hash { mod tests { use { super::*, + crate::testing_fixtures::FormatValidation, std::{ collections::HashSet, iter::repeat_with, @@ -282,7 +311,7 @@ mod tests { fn test_ping_pong() { let mut rng = rand::thread_rng(); let keypair = Keypair::new(); - let ping = Ping::<32>::new(rng.gen(), &keypair); + let ping = Ping::<32>::new_rand(&mut rng, Some(&keypair)); assert!(ping.verify()); assert!(ping.sanitize().is_ok()); diff --git a/gossip/src/protocol.rs b/gossip/src/protocol.rs index 2b46d45e4799df..f31fb7a0613be1 100644 --- a/gossip/src/protocol.rs +++ b/gossip/src/protocol.rs @@ -1,4 +1,10 @@ //! Definitions for the base of all Gossip protocol messages +#[cfg(feature = "dev-context-only-utils")] +use { + crate::contact_info::ContactInfo, + crate::testing_fixtures::{new_insecure_keypair, new_rand_timestamp, FormatValidation}, + solana_sdk::{signature::Keypair, signer::Signer}, +}; use { crate::{ crds_data::{CrdsData, MAX_WALLCLOCK}, @@ -80,6 +86,35 @@ pub(crate) struct PruneData { pub(crate) wallclock: u64, } +#[cfg(feature = "dev-context-only-utils")] +impl FormatValidation<(&Keypair, usize)> for PruneData { + fn new_rand(rng: &mut R, source_material: Option<(&Keypair, usize)>) -> Self { + let wallclock = new_rand_timestamp(rng); + let make = |keypair: &Keypair, num_nodes| { + let prunes = std::iter::repeat_with(Pubkey::new_unique) + .take(num_nodes) + .collect(); + let mut prune_data = PruneData { + pubkey: keypair.pubkey(), + prunes, + signature: Signature::default(), + destination: Pubkey::new_unique(), + wallclock, + }; + prune_data.sign(keypair); + prune_data + }; + match source_material { + Some((keypair, num_nodes)) => make(keypair, num_nodes), + None => { + let num_nodes = rng.gen_range(5..=MAX_PRUNE_DATA_NODES); + let keypair = new_insecure_keypair(rng); + make(&keypair, num_nodes) + } + } + } +} + impl Protocol { /// Returns the bincode serialized size (in bytes) of the Protocol. #[cfg(test)] @@ -187,6 +222,37 @@ impl Sanitize for Protocol { } } +#[cfg(feature = "dev-context-only-utils")] +impl FormatValidation<&Keypair> for Protocol { + fn new_rand(rng: &mut R, keypair: Option<&Keypair>) -> Self { + let mut random_keypair: Option = None; + let keypair = keypair + .unwrap_or_else(|| random_keypair.get_or_insert_with(|| new_insecure_keypair(rng))); + let random_crds_value = |rng| CrdsValue::new_rand(rng, Some(keypair)); + match rng.gen_range(0..=5) { + 0 => Protocol::PullResponse(keypair.pubkey(), vec![random_crds_value(rng)]), + 1 => Protocol::PushMessage(keypair.pubkey(), vec![random_crds_value(rng)]), + 2 => Protocol::PullRequest( + CrdsFilter::new_rand(15, 256), + CrdsValue::new( + CrdsData::ContactInfo(ContactInfo::new_rand(rng, Some(keypair.pubkey()))), + keypair, + ), + ), + 3 => { + let num_nodes = rng.gen_range(1..=MAX_PRUNE_DATA_NODES); + Protocol::PruneMessage( + keypair.pubkey(), + PruneData::new_rand(rng, Some((keypair, num_nodes))), + ) + } + 4 => Protocol::PingMessage(Ping::new_rand(rng, Some(keypair))), + 5 => Protocol::PongMessage(Pong::new_rand(rng, Some(keypair))), + _ => unreachable!("All patterns should be covered"), + } + } +} + impl Sanitize for PruneData { fn sanitize(&self) -> Result<(), SanitizeError> { if self.wallclock >= MAX_WALLCLOCK { @@ -262,10 +328,9 @@ pub(crate) mod tests { super::*, crate::{ contact_info::ContactInfo, - crds_data::{ - self, AccountsHashes, CrdsData, LowestSlot, SnapshotHashes, Vote as CrdsVote, - }, + crds_data::{AccountsHashes, CrdsData, LowestSlot, SnapshotHashes, Vote as CrdsVote}, duplicate_shred::{self, tests::new_rand_shred, MAX_DUPLICATE_SHREDS}, + testing_fixtures::FormatValidation, }, rand::Rng, solana_ledger::shred::Shredder, @@ -312,27 +377,6 @@ pub(crate) mod tests { (keypair, socket) } - fn new_rand_prune_data( - rng: &mut R, - self_keypair: &Keypair, - num_nodes: Option, - ) -> PruneData { - let wallclock = crds_data::new_rand_timestamp(rng); - let num_nodes = num_nodes.unwrap_or_else(|| rng.gen_range(0..MAX_PRUNE_DATA_NODES + 1)); - let prunes = std::iter::repeat_with(Pubkey::new_unique) - .take(num_nodes) - .collect(); - let mut prune_data = PruneData { - pubkey: self_keypair.pubkey(), - prunes, - signature: Signature::default(), - destination: Pubkey::new_unique(), - wallclock, - }; - prune_data.sign(self_keypair); - prune_data - } - #[test] fn test_max_accounts_hashes_with_push_messages() { let mut rng = rand::thread_rng(); @@ -395,7 +439,7 @@ pub(crate) mod tests { for _ in 0..64 { let self_keypair = Keypair::new(); let prune_data = - new_rand_prune_data(&mut rng, &self_keypair, Some(MAX_PRUNE_DATA_NODES)); + PruneData::new_rand(&mut rng, Some((&self_keypair, MAX_PRUNE_DATA_NODES))); let prune_message = Protocol::PruneMessage(self_keypair.pubkey(), prune_data); let socket = new_rand_socket_addr(&mut rng); assert!(Packet::from_data(Some(&socket), prune_message).is_ok()); @@ -403,7 +447,7 @@ pub(crate) mod tests { // Assert that MAX_PRUNE_DATA_NODES is highest possible. let self_keypair = Keypair::new(); let prune_data = - new_rand_prune_data(&mut rng, &self_keypair, Some(MAX_PRUNE_DATA_NODES + 1)); + PruneData::new_rand(&mut rng, Some((&self_keypair, MAX_PRUNE_DATA_NODES + 1))); let prune_message = Protocol::PruneMessage(self_keypair.pubkey(), prune_data); let socket = new_rand_socket_addr(&mut rng); assert!(Packet::from_data(Some(&socket), prune_message).is_err()); @@ -668,7 +712,7 @@ pub(crate) mod tests { fn test_prune_data_sign_and_verify_without_prefix() { let mut rng = rand::thread_rng(); let keypair = Keypair::new(); - let mut prune_data = new_rand_prune_data(&mut rng, &keypair, Some(3)); + let mut prune_data = PruneData::new_rand(&mut rng, Some((&keypair, 3))); prune_data.sign(&keypair); @@ -680,7 +724,7 @@ pub(crate) mod tests { fn test_prune_data_sign_and_verify_with_prefix() { let mut rng = rand::thread_rng(); let keypair = Keypair::new(); - let mut prune_data = new_rand_prune_data(&mut rng, &keypair, Some(3)); + let mut prune_data = PruneData::new_rand(&mut rng, Some((&keypair, 3))); // Manually set the signature with prefixed data let prefixed_data = prune_data.signable_data_with_prefix(); @@ -695,7 +739,7 @@ pub(crate) mod tests { fn test_prune_data_verify_with_and_without_prefix() { let mut rng = rand::thread_rng(); let keypair = Keypair::new(); - let mut prune_data = new_rand_prune_data(&mut rng, &keypair, Some(3)); + let mut prune_data = PruneData::new_rand(&mut rng, Some((&keypair, 3))); // Sign with non-prefixed data prune_data.sign(&keypair); diff --git a/gossip/src/restart_crds_values.rs b/gossip/src/restart_crds_values.rs index 2543e9a8c63b04..3599d12c309a85 100644 --- a/gossip/src/restart_crds_values.rs +++ b/gossip/src/restart_crds_values.rs @@ -1,13 +1,17 @@ use { - crate::crds_data::{new_rand_timestamp, sanitize_wallclock}, + crate::crds_data::sanitize_wallclock, bv::BitVec, itertools::Itertools, - rand::Rng, solana_sanitize::{Sanitize, SanitizeError}, solana_sdk::{clock::Slot, hash::Hash, pubkey::Pubkey}, solana_serde_varint as serde_varint, thiserror::Error, }; +#[cfg(feature = "dev-context-only-utils")] +use { + crate::testing_fixtures::{new_rand_timestamp, new_random_pubkey, FormatValidation}, + rand::Rng, +}; #[cfg_attr(feature = "frozen-abi", derive(AbiExample))] #[derive(Serialize, Deserialize, Clone, PartialEq, Eq, Debug)] @@ -106,9 +110,22 @@ impl RestartLastVotedForkSlots { }) } - /// New random Version for tests and benchmarks. - pub(crate) fn new_rand(rng: &mut R, pubkey: Option) -> Self { - let pubkey = pubkey.unwrap_or_else(solana_pubkey::new_rand); + pub fn to_slots(&self, min_slot: Slot) -> Vec { + match &self.offsets { + SlotsOffsets::RunLengthEncoding(run_length_encoding) => { + run_length_encoding.to_slots(self.last_voted_slot, min_slot) + } + SlotsOffsets::RawOffsets(raw_offsets) => { + raw_offsets.to_slots(self.last_voted_slot, min_slot) + } + } + } +} + +#[cfg(feature = "dev-context-only-utils")] +impl FormatValidation for RestartLastVotedForkSlots { + fn new_rand(rng: &mut R, pubkey: Option) -> Self { + let pubkey = pubkey.unwrap_or_else(|| new_random_pubkey(rng)); let num_slots = rng.gen_range(2..20); let slots = std::iter::repeat_with(|| 47825632 + rng.gen_range(0..512)) .take(num_slots) @@ -122,17 +139,6 @@ impl RestartLastVotedForkSlots { ) .unwrap() } - - pub fn to_slots(&self, min_slot: Slot) -> Vec { - match &self.offsets { - SlotsOffsets::RunLengthEncoding(run_length_encoding) => { - run_length_encoding.to_slots(self.last_voted_slot, min_slot) - } - SlotsOffsets::RawOffsets(raw_offsets) => { - raw_offsets.to_slots(self.last_voted_slot, min_slot) - } - } - } } impl Sanitize for RestartHeaviestFork { @@ -142,9 +148,10 @@ impl Sanitize for RestartHeaviestFork { } } -impl RestartHeaviestFork { - pub(crate) fn new_rand(rng: &mut R, from: Option) -> Self { - let from = from.unwrap_or_else(solana_pubkey::new_rand); +#[cfg(feature = "dev-context-only-utils")] +impl FormatValidation for RestartHeaviestFork { + fn new_rand(rng: &mut R, from: Option) -> Self { + let from = from.unwrap_or_else(|| new_random_pubkey(rng)); Self { from, wallclock: new_rand_timestamp(rng), diff --git a/gossip/src/testing_fixtures.rs b/gossip/src/testing_fixtures.rs new file mode 100644 index 00000000000000..ad4f55148c1c50 --- /dev/null +++ b/gossip/src/testing_fixtures.rs @@ -0,0 +1,80 @@ +//! Contains fixtures for tests of gossip code +//! Some of these are cryptographically unsound and +//! do not belong in any prod code. +#[cfg(not(feature = "dev-context-only-utils"))] +use non_existent_crate_to_make_sure_this_is_never_imported_in_prod; +use { + anyhow::Context, + rand::Rng, + rand0_7::{CryptoRng as CryptoRng0_7, RngCore as RngCore0_7}, + solana_pubkey::{Pubkey, PUBKEY_BYTES}, + solana_sanitize::Sanitize, + solana_sdk::{signature::Keypair, timing::timestamp}, +}; + +pub trait FormatValidation: Sized + Sanitize { + /// New random instance for tests and benchmarks. + /// Implementations may customize which source material they need to build an instance. + fn new_rand(rng: &mut R, source_material: Option) -> Self; + + /// Run some code that uses/traverses the value in a meaningful way + /// This is primarily designed for use in fuzz tests. + fn exercise(&self) -> anyhow::Result<()> { + self.sanitize().context("Sanitize failed")?; + Ok(()) + } +} + +/// Random gossip timestamp for tests and benchmarks. +/// With seeded rng can be repeatable +pub fn new_rand_timestamp(rng: &mut R) -> u64 { + const DELAY: u64 = 10 * 60 * 1000; // 10 minutes + timestamp() - DELAY + rng.gen_range(0..2 * DELAY) +} + +/// *Insecurely* generated keypair. Designed for tests and benches. +/// Unlike a proper keypair generation, this can use any rng. +/// With seeded rng can be repeatable +pub fn new_insecure_keypair(rng: &mut R) -> Keypair { + let mut not_crypto_rng = FakeCryptoRng::new(rng); + Keypair::generate(&mut not_crypto_rng) +} + +/// Random public key bytes. Designed for tests and benches. +/// With seeded rng can be repeatable +pub fn new_random_pubkey(rng: &mut R) -> Pubkey { + let bytes = rng.gen::<[u8; PUBKEY_BYTES]>(); + Pubkey::from(bytes) +} + +/// Fake crypto RNG that can make signatures using any underlying RNG +/// Obviously, this is for tests and benchmakrs only +struct FakeCryptoRng<'r, R: Rng>(&'r mut R); +/// Marking the `FakeCryptoRng` as crypto RNG for tests only +impl CryptoRng0_7 for FakeCryptoRng<'_, R> {} + +impl<'r, R: Rng> FakeCryptoRng<'r, R> { + //just in case, prevent construction of instances outside of tests + #[cfg(feature = "dev-context-only-utils")] + pub fn new(rng: &'r mut R) -> Self { + Self(rng) + } +} + +impl RngCore0_7 for FakeCryptoRng<'_, R> { + fn next_u32(&mut self) -> u32 { + self.0.next_u32() + } + + fn next_u64(&mut self) -> u64 { + self.0.next_u64() + } + + fn fill_bytes(&mut self, dest: &mut [u8]) { + self.0.fill_bytes(dest); + } + + fn try_fill_bytes(&mut self, dest: &mut [u8]) -> Result<(), rand0_7::Error> { + self.0.try_fill_bytes(dest).map_err(rand0_7::Error::new) + } +} diff --git a/programs/sbf/Cargo.lock b/programs/sbf/Cargo.lock index dc5ec36eefecea..0e21b8031df7b2 100644 --- a/programs/sbf/Cargo.lock +++ b/programs/sbf/Cargo.lock @@ -939,6 +939,15 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" +[[package]] +name = "byteorder_slice" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b294e30387378958e8bf8f4242131b930ea615ff81e8cac2440cea0a6013190" +dependencies = [ + "byteorder 1.5.0", +] + [[package]] name = "bytes" version = "1.10.0" @@ -1438,6 +1447,17 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "derive-into-owned" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d94d81e3819a7b06a8638f448bc6339371ca9b6076a99d4a43eece3c4c923" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "derive-where" version = "1.2.7" @@ -3718,6 +3738,17 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "pcap-file" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fc1f139757b058f9f37b76c48501799d12c9aa0aa4c0d4c980b062ee925d1b2" +dependencies = [ + "byteorder_slice", + "derive-into-owned", + "thiserror 1.0.69", +] + [[package]] name = "pem" version = "1.1.1" @@ -6187,9 +6218,11 @@ dependencies = [ name = "solana-gossip" version = "2.2.0" dependencies = [ + "anyhow", "assert_matches", "bincode", "bv", + "cfg-if 1.0.0", "clap", "crossbeam-channel", "flate2", @@ -6198,6 +6231,8 @@ dependencies = [ "log", "lru", "num-traits", + "pcap-file", + "rand 0.7.3", "rand 0.8.5", "rand_chacha 0.3.1", "rayon", diff --git a/svm/examples/Cargo.lock b/svm/examples/Cargo.lock index 04ee263869f516..594c983eb24ed7 100644 --- a/svm/examples/Cargo.lock +++ b/svm/examples/Cargo.lock @@ -853,6 +853,15 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" +[[package]] +name = "byteorder_slice" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b294e30387378958e8bf8f4242131b930ea615ff81e8cac2440cea0a6013190" +dependencies = [ + "byteorder", +] + [[package]] name = "bytes" version = "1.10.0" @@ -1344,6 +1353,17 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "derive-into-owned" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d94d81e3819a7b06a8638f448bc6339371ca9b6076a99d4a43eece3c4c923" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "derive-where" version = "1.2.7" @@ -3599,6 +3619,17 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "pcap-file" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fc1f139757b058f9f37b76c48501799d12c9aa0aa4c0d4c980b062ee925d1b2" +dependencies = [ + "byteorder_slice", + "derive-into-owned", + "thiserror 1.0.69", +] + [[package]] name = "pem" version = "1.1.1" @@ -6007,9 +6038,11 @@ dependencies = [ name = "solana-gossip" version = "2.2.0" dependencies = [ + "anyhow", "assert_matches", "bincode", "bv", + "cfg-if 1.0.0", "clap", "crossbeam-channel", "flate2", @@ -6018,6 +6051,8 @@ dependencies = [ "log", "lru", "num-traits", + "pcap-file", + "rand 0.7.3", "rand 0.8.5", "rand_chacha 0.3.1", "rayon",