first working version

Author: alexpyattaev

gossip/Cargo.toml

@@ -25,6 +25,7 @@ lru = { workspace = true }
 num-traits = { workspace = true }
 pcap-file = "2.0.0"
 rand = { workspace = true }
+rand0-7 = { workspace = true }
 rand_chacha = { workspace = true }
 rayon = { workspace = true }
 serde = { workspace = true }
@@ -73,7 +74,7 @@ num_cpus = { workspace = true }
 rand0-7 = { workspace = true }
 rand_chacha0-2 = { workspace = true }
 serial_test = { workspace = true }
-solana-gossip = { workspace = true, features = ["dev-context-only-utils"] }
+solana-gossip = { path=".", features = ["dev-context-only-utils"] }
 solana-perf = { workspace = true, features = ["dev-context-only-utils"] }
 solana-runtime = { workspace = true, features = ["dev-context-only-utils"] }
 test-case = { workspace = true }

gossip/benches/crds.rs

@@ -9,6 +9,7 @@ use {
         crds::{Crds, GossipRoute},
         crds_gossip_pull::{CrdsTimeouts, CRDS_GOSSIP_PULL_CRDS_TIMEOUT_MS},
         crds_value::CrdsValue,
+        testing_fixtures::FormatValidation,
     },
     solana_pubkey::Pubkey,
     std::{collections::HashMap, time::Duration},

gossip/benches/crds_gossip_pull.rs

@@ -9,6 +9,7 @@ use {
         crds::{Crds, GossipRoute},
         crds_gossip_pull::{CrdsFilter, CrdsGossipPull},
         crds_value::CrdsValue,
+        testing_fixtures::FormatValidation,
     },
     solana_sdk::hash::Hash,
     std::sync::RwLock,

gossip/benches/crds_shards.rs

@@ -8,6 +8,7 @@ use {
         crds::{Crds, GossipRoute, VersionedCrdsValue},
         crds_shards::CrdsShards,
         crds_value::CrdsValue,
+        testing_fixtures::FormatValidation,
     },
     solana_sdk::timing::timestamp,
     std::iter::repeat_with,

gossip/src/cluster_info.rs

@@ -3126,6 +3126,7 @@ mod tests {
             duplicate_shred::tests::new_rand_shred,
             protocol::tests::new_rand_remote_node,
             socketaddr,
+            testing_fixtures::{new_insecure_keypair, FormatValidation},
         },
         bincode::serialize,
         itertools::izip,
@@ -3796,7 +3797,8 @@ mod tests {
     #[test]
     fn test_append_entrypoint_to_pulls() {
         let thread_pool = ThreadPoolBuilder::new().build().unwrap();
-        let node_keypair = Arc::new(Keypair::new());
+        let mut rng = rand::thread_rng();
+        let node_keypair = Arc::new(new_insecure_keypair(&mut rng));
         let cluster_info = ClusterInfo::new(
             ContactInfo::new_localhost(&node_keypair.pubkey(), timestamp()),
             node_keypair,

gossip/src/crds.rs

@@ -781,7 +781,10 @@ fn should_report_message_signature(signature: &Signature) -> bool {
 mod tests {
     use {
         super::*,
-        crate::crds_data::{new_rand_timestamp, AccountsHashes, NodeInstance},
+        crate::{
+            crds_data::{AccountsHashes, NodeInstance},
+            testing_fixtures::*,
+        },
         rand::{thread_rng, Rng, SeedableRng},
         rand_chacha::ChaChaRng,
         rayon::ThreadPoolBuilder,
@@ -1250,7 +1253,9 @@ mod tests {
     #[test]
     fn test_crds_value_indices() {
         let mut rng = thread_rng();
-        let keypairs: Vec<_> = repeat_with(Keypair::new).take(128).collect();
+        let keypairs: Vec<_> = repeat_with(|| new_insecure_keypair(&mut rng))
+            .take(128)
+            .collect();
         let mut crds = Crds::default();
         let mut num_inserts = 0;
         for k in 0..4096 {
@@ -1301,7 +1306,9 @@ mod tests {
             }
         }
         let mut rng = thread_rng();
-        let keypairs: Vec<_> = repeat_with(Keypair::new).take(128).collect();
+        let keypairs: Vec<_> = repeat_with(|| new_insecure_keypair(&mut rng))
+            .take(128)
+            .collect();
         let mut crds = Crds::default();
         for k in 0..4096 {
             let keypair = &keypairs[rng.gen_range(0..keypairs.len())];
@@ -1400,7 +1407,9 @@ mod tests {
                 .len()
         }
         let mut rng = thread_rng();
-        let keypairs: Vec<_> = repeat_with(Keypair::new).take(64).collect();
+        let keypairs: Vec<_> = repeat_with(|| new_insecure_keypair(&mut rng))
+            .take(64)
+            .collect();
         let stakes = keypairs
             .iter()
             .map(|k| (k.pubkey(), rng.gen_range(0..1000)))

gossip/src/crds_data.rs

@@ -1,3 +1,9 @@
+#[cfg(feature = "dev-context-only-utils")]
+use {
+    crate::testing_fixtures::{new_rand_timestamp, new_random_pubkey, FormatValidation},
+    rand::Rng,
+    std::hint::black_box,
+};
 use {
     crate::{
         contact_info::ContactInfo,
@@ -9,20 +15,11 @@ use {
     },
     serde::de::{Deserialize, Deserializer},
     solana_sanitize::{Sanitize, SanitizeError},
-    solana_sdk::{
-        clock::Slot,
-        hash::Hash,
-        pubkey::{self, Pubkey},
-        timing::timestamp,
-        transaction::Transaction,
-    },
+    solana_sdk::{clock::Slot, hash::Hash, pubkey::Pubkey, transaction::Transaction},
     solana_vote::vote_parser,
     std::{cmp::Ordering, collections::BTreeSet},
 };
 
-#[cfg(feature = "dev-context-only-utils")]
-use {crate::format_validation::FormatValidation, rand::Rng, std::hint::black_box};
-
 pub(crate) const MAX_WALLCLOCK: u64 = 1_000_000_000_000_000;
 pub(crate) const MAX_SLOT: u64 = 1_000_000_000_000_000;
 /// Maximum number of hashes in AccountsHashes a node publishes
@@ -109,21 +106,16 @@ impl Sanitize for CrdsData {
     }
 }
 
-/// Random timestamp for tests and benchmarks.
-pub(crate) fn new_rand_timestamp<R: Rng>(rng: &mut R) -> u64 {
-    const DELAY: u64 = 10 * 60 * 1000; // 10 minutes
-    timestamp() - DELAY + rng.gen_range(0..2 * DELAY)
-}
-
-impl CrdsData {
+#[cfg(feature = "dev-context-only-utils")]
+impl crate::testing_fixtures::FormatValidation<Pubkey> for CrdsData {
     /// New random CrdsData for tests and benchmarks.
-    pub(crate) fn new_rand<R: Rng>(rng: &mut R, pubkey: Option<Pubkey>) -> CrdsData {
+    fn new_rand<R: Rng>(rng: &mut R, pubkey: Option<Pubkey>) -> CrdsData {
         let kind = rng.gen_range(0..8);
         // TODO: Implement other kinds of CrdsData here.
         // TODO: Assign ranges to each arm proportional to their frequency in
         // the mainnet crds table.
         match kind {
-            0 => CrdsData::from(ContactInfo::new_rand(rng, pubkey)),
+            0 => CrdsData::ContactInfo(ContactInfo::new_rand(rng, pubkey)),
             // Index for LowestSlot is deprecated and should be zero.
             1 => CrdsData::LowestSlot(0, LowestSlot::new_rand(rng, pubkey)),
             2 => CrdsData::LegacySnapshotHashes(LegacySnapshotHashes::new_rand(rng, pubkey)),
@@ -139,7 +131,9 @@ impl CrdsData {
             ),
         }
     }
+}
 
+impl CrdsData {
     pub(crate) fn wallclock(&self) -> u64 {
         match self {
             CrdsData::LegacyContactInfo(contact_info) => contact_info.wallclock(),
@@ -226,6 +220,9 @@ pub(crate) struct AccountsHashes {
 impl Sanitize for AccountsHashes {
     fn sanitize(&self) -> Result<(), SanitizeError> {
         sanitize_wallclock(self.wallclock)?;
+        if self.hashes.len() > MAX_ACCOUNTS_HASHES {
+            return Err(SanitizeError::IndexOutOfBounds);
+        }
         for (slot, _) in &self.hashes {
             if *slot >= MAX_SLOT {
                 return Err(SanitizeError::ValueOutOfBounds);
@@ -247,14 +244,14 @@ impl FormatValidation for AccountsHashes {
         .take(num_hashes)
         .collect();
         Self {
-            from: pubkey.unwrap_or_else(pubkey::new_rand),
+            from: pubkey.unwrap_or_else(|| new_random_pubkey(rng)),
             hashes,
             wallclock: new_rand_timestamp(rng),
         }
     }
 
     fn exercise(&self) -> anyhow::Result<()> {
-        FormatValidation::exercise(&self)?;
+        self.sanitize()?;
         let s: u64 = self.hashes.iter().map(|v| v.0).sum();
         black_box(s);
         Ok(())
@@ -318,7 +315,7 @@ impl LowestSlot {
 impl FormatValidation for LowestSlot {
     fn new_rand<R: Rng>(rng: &mut R, pubkey: Option<Pubkey>) -> Self {
         Self {
-            from: pubkey.unwrap_or_else(pubkey::new_rand),
+            from: pubkey.unwrap_or_else(|| new_random_pubkey(rng)),
             root: rng.gen(),
             lowest: rng.gen(),
             slots: BTreeSet::default(),
@@ -365,6 +362,19 @@ impl Sanitize for Vote {
     }
 }
 
+#[cfg(feature = "dev-context-only-utils")]
+impl FormatValidation for Vote {
+    /// New random Vote for tests and benchmarks.
+    fn new_rand<R: Rng>(rng: &mut R, pubkey: Option<Pubkey>) -> Self {
+        Self {
+            from: pubkey.unwrap_or_else(|| new_random_pubkey(rng)),
+            transaction: Transaction::default(),
+            wallclock: new_rand_timestamp(rng),
+            slot: None,
+        }
+    }
+}
+
 impl Vote {
     // Returns None if cannot parse transaction into a vote.
     pub fn new(from: Pubkey, transaction: Transaction, wallclock: u64) -> Option<Self> {
@@ -376,16 +386,6 @@ impl Vote {
         })
     }
 
-    /// New random Vote for tests and benchmarks.
-    fn new_rand<R: Rng>(rng: &mut R, pubkey: Option<Pubkey>) -> Self {
-        Self {
-            from: pubkey.unwrap_or_else(pubkey::new_rand),
-            transaction: Transaction::default(),
-            wallclock: new_rand_timestamp(rng),
-            slot: None,
-        }
-    }
-
     pub(crate) fn transaction(&self) -> &Transaction {
         &self.transaction
     }

gossip/src/crds_entry.rs

@@ -65,7 +65,7 @@ mod tests {
         super::*,
         crate::{
             crds::{Crds, GossipRoute},
-            crds_data::new_rand_timestamp,
+            testing_fixtures::*,
         },
         rand::seq::SliceRandom,
         solana_sdk::signature::Keypair,

gossip/src/crds_gossip_pull.rs

@@ -82,7 +82,7 @@ impl solana_sanitize::Sanitize for CrdsFilter {
 }
 
 impl CrdsFilter {
-    #[cfg(test)]
+    #[cfg(feature = "dev-context-only-utils")]
     pub(crate) fn new_rand(num_items: usize, max_bytes: usize) -> Self {
         let max_bits = (max_bytes * 8) as f64;
         let max_items = Self::max_items(max_bits, FALSE_RATE, KEYS);
@@ -666,6 +666,7 @@ pub(crate) mod tests {
             crds_data::{CrdsData, Vote},
             legacy_contact_info::LegacyContactInfo,
             protocol::Protocol,
+            testing_fixtures::*,
         },
         itertools::Itertools,
         rand::{seq::SliceRandom, SeedableRng},

gossip/src/crds_shards.rs

@@ -137,6 +137,7 @@ mod test {
         crate::{
             crds::{Crds, GossipRoute},
             crds_value::CrdsValue,
+            testing_fixtures::*,
         },
         rand::{thread_rng, Rng},
         solana_sdk::timing::timestamp,

gossip/src/crds_value.rs

@@ -1,3 +1,8 @@
+#[cfg(feature = "dev-context-only-utils")]
+use {
+    crate::testing_fixtures::{new_insecure_keypair, FormatValidation},
+    rand::Rng,
+};
 use {
     crate::{
         contact_info::ContactInfo,
@@ -6,7 +11,6 @@ use {
         epoch_slots::EpochSlots,
     },
     bincode::serialize,
-    rand::Rng,
     serde::de::{Deserialize, Deserializer},
     solana_sanitize::{Sanitize, SanitizeError},
     solana_sdk::{
@@ -17,7 +21,8 @@ use {
     std::borrow::{Borrow, Cow},
 };
 
-/// CrdsValue that is replicated across the cluster
+/// CrdsValue is a wrapper around CrdsData that is replicated across the cluster
+/// CrdsValue's purpose is to add signature to verify the origin of the CrdsData
 #[cfg_attr(feature = "frozen-abi", derive(AbiExample))]
 #[derive(Serialize, Clone, Debug, PartialEq, Eq)]
 pub struct CrdsValue {
@@ -98,6 +103,24 @@ impl CrdsValueLabel {
     }
 }
 
+#[cfg(feature = "dev-context-only-utils")]
+impl FormatValidation<&Keypair> for CrdsValue {
+    fn new_rand<R: Rng>(rng: &mut R, keypair: Option<&Keypair>) -> Self {
+        let mut random_keypair: Option<Keypair> = None;
+        let keypair = keypair
+            .unwrap_or_else(|| random_keypair.get_or_insert_with(|| new_insecure_keypair(rng)));
+        let data = CrdsData::new_rand(rng, Some(keypair.pubkey()));
+        Self::new(data, keypair)
+    }
+
+    fn exercise(&self) -> anyhow::Result<()> {
+        self.sanitize()?;
+        if !self.verify() {
+            anyhow::bail!("sigverify");
+        };
+        self.data().exercise()
+    }
+}
 impl CrdsValue {
     pub fn new(data: CrdsData, keypair: &Keypair) -> Self {
         let bincode_serialized_data = bincode::serialize(&data).unwrap();
@@ -122,21 +145,6 @@ impl CrdsValue {
         }
     }
 
-    /// New random CrdsValue for tests and benchmarks.
-    pub fn new_rand<R: Rng>(rng: &mut R, keypair: Option<&Keypair>) -> CrdsValue {
-        match keypair {
-            None => {
-                let keypair = Keypair::new();
-                let data = CrdsData::new_rand(rng, Some(keypair.pubkey()));
-                Self::new(data, &keypair)
-            }
-            Some(keypair) => {
-                let data = CrdsData::new_rand(rng, Some(keypair.pubkey()));
-                Self::new(data, keypair)
-            }
-        }
-    }
-
     #[inline]
     pub(crate) fn signature(&self) -> &Signature {
         &self.signature

gossip/src/epoch_slots.rs

@@ -1,6 +1,8 @@
+#[cfg(feature = "dev-context-only-utils")]
+use crate::testing_fixtures::*;
 use {
     crate::{
-        crds_data::{self, MAX_SLOT, MAX_WALLCLOCK},
+        crds_data::{MAX_SLOT, MAX_WALLCLOCK},
         protocol::MAX_CRDS_OBJECT_SIZE,
     },
     bincode::serialized_size,
@@ -341,10 +343,12 @@ impl EpochSlots {
             .filter_map(move |s| s.to_slots(min_slot).ok())
             .flatten()
     }
+}
 
-    /// New random EpochSlots for tests and simulations.
-    pub(crate) fn new_rand<R: rand::Rng>(rng: &mut R, pubkey: Option<Pubkey>) -> Self {
-        let now = crds_data::new_rand_timestamp(rng);
+#[cfg(feature = "dev-context-only-utils")]
+impl FormatValidation<Pubkey> for EpochSlots {
+    fn new_rand<R: rand::Rng>(rng: &mut R, pubkey: Option<Pubkey>) -> Self {
+        let now = new_rand_timestamp(rng);
         let pubkey = pubkey.unwrap_or_else(solana_pubkey::new_rand);
         let mut epoch_slots = Self::new(pubkey, now);
         let num_slots = rng.gen_range(0..20);
@@ -354,6 +358,13 @@ impl EpochSlots {
         epoch_slots.add(&slots);
         epoch_slots
     }
+
+    fn exercise(&self) -> anyhow::Result<()> {
+        self.sanitize()?;
+        let s: Slot = self.to_slots(0).sum();
+        std::hint::black_box(s);
+        Ok(())
+    }
 }
 
 #[cfg(test)]

gossip/src/lib.rs

@@ -48,4 +48,4 @@ extern crate solana_frozen_abi_macro;
 extern crate solana_metrics;
 
 #[cfg(feature = "dev-context-only-utils")]
-pub mod format_validation;
+pub mod testing_fixtures;