diff --git a/chain/client/src/metrics.rs b/chain/client/src/metrics.rs index 400d76a3365..552693c3ca6 100644 --- a/chain/client/src/metrics.rs +++ b/chain/client/src/metrics.rs @@ -361,10 +361,11 @@ pub(crate) static CURRENT_PROTOCOL_VERSION: Lazy = Lazy::new(|| { .unwrap() }); -pub(crate) static NODE_PROTOCOL_UPGRADE_VOTING_START: Lazy = Lazy::new(|| { - try_create_int_gauge( +pub(crate) static NODE_PROTOCOL_UPGRADE_VOTING_START: Lazy = Lazy::new(|| { + try_create_int_gauge_vec( "near_node_protocol_upgrade_voting_start", - "Time in seconds since Unix epoch determining when node will start voting for the protocol upgrade; zero if there is no schedule for the voting") + "Time in seconds since Unix epoch determining when node will start voting for the protocol upgrade; zero if there is no schedule for the voting", + &["protocol_version"]) .unwrap() }); @@ -404,8 +405,12 @@ pub(crate) static PRODUCE_AND_DISTRIBUTE_CHUNK_TIME: Lazy = Lazy:: /// `neard_version` argument. pub(crate) fn export_version(neard_version: &near_primitives::version::Version) { NODE_PROTOCOL_VERSION.set(near_primitives::version::PROTOCOL_VERSION.into()); - NODE_PROTOCOL_UPGRADE_VOTING_START - .set(near_primitives::version::PROTOCOL_UPGRADE_SCHEDULE.timestamp()); + let schedule = near_primitives::version::PROTOCOL_UPGRADE_SCHEDULE; + for (datetime, protocol_version) in schedule.schedule().iter() { + NODE_PROTOCOL_UPGRADE_VOTING_START + .with_label_values(&[&protocol_version.to_string()]) + .set(datetime.timestamp()); + } NODE_DB_VERSION.set(near_store::metadata::DB_VERSION.into()); NODE_BUILD_INFO.reset(); NODE_BUILD_INFO diff --git a/core/primitives/src/upgrade_schedule.rs b/core/primitives/src/upgrade_schedule.rs index 97114768d94..a1f0560266c 100644 --- a/core/primitives/src/upgrade_schedule.rs +++ b/core/primitives/src/upgrade_schedule.rs @@ -1,245 +1,458 @@ -use chrono::{DateTime, NaiveDateTime, ParseError, Utc}; +use chrono::{DateTime, NaiveDateTime, Utc}; use near_primitives_core::types::ProtocolVersion; use std::env; -/// Defines the point in time after which validators are expected to vote on the -/// new protocol version. -#[derive(Clone, Copy, Debug, PartialEq)] -pub struct ProtocolUpgradeVotingSchedule { - timestamp: chrono::DateTime, +const NEAR_TESTS_PROTOCOL_UPGRADE_OVERRIDE: &str = "NEAR_TESTS_PROTOCOL_UPGRADE_OVERRIDE"; + +#[derive(thiserror::Error, Clone, Debug)] +pub enum ProtocolUpgradeVotingScheduleError { + #[error("The final upgrade must be the client protocol version! final version: {0}, client version: {1}")] + InvalidFinalUpgrade(ProtocolVersion, ProtocolVersion), + #[error("The upgrades must be sorted by datetime!")] + InvalidDateTimeOrder, + #[error("The upgrades must be sorted and increasing by one!")] + InvalidProtocolVersionOrder, + + #[error("The environment override has an invalid format! Input: {0} Error: {1}")] + InvalidOverrideFormat(String, String), } -impl Default for ProtocolUpgradeVotingSchedule { - fn default() -> Self { - Self { timestamp: DateTime::::from_naive_utc_and_offset(Default::default(), Utc) } - } +type ProtocolUpgradeVotingScheduleRaw = Vec<(chrono::DateTime, ProtocolVersion)>; + +/// Defines a schedule for validators to vote for the protocol version upgrades. +/// Multiple protocol version upgrades can be scheduled. The default schedule is +/// empty and in that case the node will always vote for the client protocol +/// version. +#[derive(Clone, Debug, PartialEq)] +pub struct ProtocolUpgradeVotingSchedule { + // The highest protocol version supported by the client. This class will + // check that the schedule ends with this version. + client_protocol_version: ProtocolVersion, + + /// The schedule is a sorted list of (datetime, version) tuples. The node + /// should vote for the highest version that is less than or equal to the + /// current time. + schedule: ProtocolUpgradeVotingScheduleRaw, } impl ProtocolUpgradeVotingSchedule { - pub fn is_in_future(&self) -> bool { - chrono::Utc::now() < self.timestamp - } - - pub fn timestamp(&self) -> i64 { - self.timestamp.timestamp() + /// This method creates an instance of the ProtocolUpgradeVotingSchedule + /// that will immediately vote for the client protocol version. + pub fn new_immediate(client_protocol_version: ProtocolVersion) -> Self { + Self { client_protocol_version, schedule: vec![] } } /// This method creates an instance of the ProtocolUpgradeVotingSchedule. /// - /// It will first check if the NEAR_TESTS_IMMEDIATE_PROTOCOL_UPGRADE is - /// set in the environment and if so return the immediate upgrade schedule. - /// This should only be used in tests, in particular in tests the in some + /// It will first check if the NEAR_TESTS_PROTOCOL_UPGRADE_OVERRIDE is set + /// in the environment and if so this override will be used as schedule. + /// This should only be used in tests, in particular in tests that in some /// way test neard upgrades. /// - /// Otherwise it will parse the given string and return the corresponding - /// upgrade schedule. - pub fn from_env_or_str(s: &str) -> Result { - let immediate_upgrade = env::var("NEAR_TESTS_IMMEDIATE_PROTOCOL_UPGRADE"); - if let Ok(_) = immediate_upgrade { - tracing::warn!("Setting immediate protocol upgrade. This is fine in tests but should be avoided otherwise"); - return Ok(Self::default()); + /// Otherwise it will use the provided schedule. + pub fn new_from_env_or_schedule( + client_protocol_version: ProtocolVersion, + mut schedule: ProtocolUpgradeVotingScheduleRaw, + ) -> Result { + let env_override = env::var(NEAR_TESTS_PROTOCOL_UPGRADE_OVERRIDE); + if let Ok(env_override) = env_override { + schedule = Self::parse_override(&env_override)?; + tracing::warn!( + target: "protocol_upgrade", + ?schedule, + "Setting protocol upgrade override. This is fine in tests but should be avoided otherwise" + ); } - Ok(Self { - timestamp: DateTime::::from_naive_utc_and_offset( - NaiveDateTime::parse_from_str(s, "%Y-%m-%d %H:%M:%S")?, - Utc, - ), - }) + // Sanity and invariant checks. + + // The final upgrade must be the client protocol version. + if let Some((_, version)) = schedule.last() { + if *version != client_protocol_version { + return Err(ProtocolUpgradeVotingScheduleError::InvalidFinalUpgrade( + *version, + client_protocol_version, + )); + } + } + + // The upgrades must be sorted by datetime. + for i in 1..schedule.len() { + let prev_time = schedule[i - 1].0; + let this_time = schedule[i].0; + if !(prev_time < this_time) { + return Err(ProtocolUpgradeVotingScheduleError::InvalidDateTimeOrder); + } + } + + // The upgrades must be increasing by 1. + for i in 1..schedule.len() { + let prev_protocol_version = schedule[i - 1].1; + let this_protocol_version = schedule[i].1; + if prev_protocol_version + 1 != this_protocol_version { + return Err(ProtocolUpgradeVotingScheduleError::InvalidProtocolVersionOrder); + } + } + + tracing::debug!(target: "protocol_upgrade", ?schedule, "created protocol upgrade schedule"); + + Ok(Self { client_protocol_version, schedule }) } -} -pub(crate) fn get_protocol_version_internal( - // Protocol version that will be used in the next epoch. - next_epoch_protocol_version: ProtocolVersion, - // Latest protocol version supported by this client. - client_protocol_version: ProtocolVersion, - // Point in time when voting for client_protocol_version version is expected - // to start. Use `Default::default()` to start voting immediately. - voting_start: ProtocolUpgradeVotingSchedule, -) -> ProtocolVersion { - if next_epoch_protocol_version >= client_protocol_version { - client_protocol_version - } else if voting_start.is_in_future() { - // Don't announce support for the latest protocol version yet. - next_epoch_protocol_version - } else { - // The time has passed, announce the latest supported protocol version. - client_protocol_version + /// This method returns the protocol version that the node should vote for. + pub(crate) fn get_protocol_version( + &self, + now: DateTime, + // Protocol version that will be used in the next epoch. + next_epoch_protocol_version: ProtocolVersion, + ) -> ProtocolVersion { + if next_epoch_protocol_version >= self.client_protocol_version { + return self.client_protocol_version; + } + + if self.schedule.is_empty() { + return self.client_protocol_version; + } + + // The datetime values in the schedule are sorted in ascending order. + // Find the last datetime value that is less than the current time. The + // schedule is sorted and the last value is the client_protocol_version + // so we are guaranteed to find a correct protocol version. + let mut result = next_epoch_protocol_version; + for (time, version) in &self.schedule { + if now < *time { + break; + } + result = *version; + } + + result + } + + /// Returns the schedule. Should only be used for exporting metrics. + pub fn schedule(&self) -> &Vec<(chrono::DateTime, ProtocolVersion)> { + &self.schedule + } + + /// A helper method to parse the datetime string. + pub fn parse_datetime(s: &str) -> Result, chrono::ParseError> { + let datetime = NaiveDateTime::parse_from_str(s, "%Y-%m-%d %H:%M:%S")?; + let datetime = DateTime::::from_naive_utc_and_offset(datetime, Utc); + Ok(datetime) + } + + // Parse the protocol version override from the environment. + // The format is comma separate datetime:=version pairs. + fn parse_override( + override_str: &str, + ) -> Result { + // The special value "now" means that the upgrade should happen immediately. + if override_str.to_lowercase() == "now" { + return Ok(vec![]); + } + + let mut result = vec![]; + let datetime_and_version_vec = override_str.split(',').collect::>(); + for datetime_and_version in datetime_and_version_vec { + let datetime_and_version = datetime_and_version.split('=').collect::>(); + let [datetime, version] = datetime_and_version[..] else { + let input = format!("{:?}", datetime_and_version); + let error = "The override must be in the format datetime=version!".to_string(); + return Err(ProtocolUpgradeVotingScheduleError::InvalidOverrideFormat( + input, error, + )); + }; + + let datetime = Self::parse_datetime(datetime).map_err(|err| { + ProtocolUpgradeVotingScheduleError::InvalidOverrideFormat( + datetime.to_string(), + err.to_string(), + ) + })?; + let version = version.parse::().map_err(|err| { + ProtocolUpgradeVotingScheduleError::InvalidOverrideFormat( + version.to_string(), + err.to_string(), + ) + })?; + result.push((datetime, version)); + } + Ok(result) } } #[cfg(test)] mod tests { + use std::vec; + use super::*; - // The tests call `get_protocol_version_internal()` with the following parameters: - // No schedule: (X-2,X), (X,X), (X+2,X) - // Before the scheduled upgrade: (X-2,X), (X,X), (X+2,X) - // After the scheduled upgrade: (X-2,X), (X,X), (X+2,X) + /// Make a simple schedule with a single protocol version upgrade. + fn make_simple_voting_schedule( + client_protocol_version: u32, + datetime_str: &str, + ) -> ProtocolUpgradeVotingSchedule { + make_voting_schedule(client_protocol_version, vec![(datetime_str, client_protocol_version)]) + } - #[test] - fn test_no_upgrade_schedule() { - // As no protocol upgrade voting schedule is set, always return the version supported by the client. + fn make_voting_schedule( + client_protocol_version: ProtocolVersion, + schedule: Vec<(&str, u32)>, + ) -> ProtocolUpgradeVotingSchedule { + make_voting_schedule_impl(schedule, client_protocol_version).unwrap() + } - let client_protocol_version = 100; - assert_eq!( + /// Make a schedule with a given list of protocol version upgrades. + fn make_voting_schedule_impl( + schedule: Vec<(&str, u32)>, + client_protocol_version: ProtocolVersion, + ) -> Result { + let parse = |(datetime_str, version)| { + let datetime = ProtocolUpgradeVotingSchedule::parse_datetime(datetime_str); + let datetime = datetime.unwrap(); + (datetime, version) + }; + let schedule = schedule.into_iter().map(parse).collect::>(); + let schedule = ProtocolUpgradeVotingSchedule::new_from_env_or_schedule( client_protocol_version, - get_protocol_version_internal( - client_protocol_version - 2, - client_protocol_version, - Default::default(), - ) - ); - assert_eq!( - client_protocol_version, - get_protocol_version_internal( - client_protocol_version, - client_protocol_version, - Default::default() - ) - ); - assert_eq!( - client_protocol_version, - get_protocol_version_internal( - client_protocol_version + 2, - client_protocol_version, - Default::default(), - ) + schedule, ); + schedule } + // The tests call `get_protocol_version()` with the following parameters: + // No schedule: (X-2,X), (X,X), (X+2,X) + // Before the scheduled upgrade: (X-2,X), (X,X), (X+2,X) + // After the scheduled upgrade: (X-2,X), (X,X), (X+2,X) + #[test] - fn test_none_upgrade_schedule() { + fn test_default_upgrade_schedule() { // As no protocol upgrade voting schedule is set, always return the version supported by the client. - + let now = chrono::Utc::now(); let client_protocol_version = 100; + let schedule = ProtocolUpgradeVotingSchedule::new_immediate(client_protocol_version); assert_eq!( client_protocol_version, - get_protocol_version_internal( - client_protocol_version - 2, - client_protocol_version, - Default::default(), - ) + schedule.get_protocol_version(now, client_protocol_version - 2) ); assert_eq!( client_protocol_version, - get_protocol_version_internal( - client_protocol_version, - client_protocol_version, - Default::default(), - ) + schedule.get_protocol_version(now, client_protocol_version) ); assert_eq!( client_protocol_version, - get_protocol_version_internal( - client_protocol_version + 2, - client_protocol_version, - Default::default(), - ) + schedule.get_protocol_version(now, client_protocol_version + 2) ); } #[test] fn test_before_scheduled_time() { + let now = chrono::Utc::now(); + let client_protocol_version = 100; - let schedule = - ProtocolUpgradeVotingSchedule::from_env_or_str("2050-01-01 00:00:00").unwrap(); + let schedule = make_simple_voting_schedule(client_protocol_version, "3000-01-01 00:00:00"); // The client supports a newer version than the version of the next epoch. // Upgrade voting will start in the far future, therefore don't announce the newest supported version. let next_epoch_protocol_version = client_protocol_version - 2; assert_eq!( next_epoch_protocol_version, - get_protocol_version_internal( - next_epoch_protocol_version, - client_protocol_version, - schedule, - ) + schedule.get_protocol_version(now, next_epoch_protocol_version) ); // An upgrade happened before the scheduled time. let next_epoch_protocol_version = client_protocol_version; assert_eq!( next_epoch_protocol_version, - get_protocol_version_internal( - next_epoch_protocol_version, - client_protocol_version, - schedule, - ) + schedule.get_protocol_version(now, next_epoch_protocol_version) ); // Several upgrades happened before the scheduled time. Announce only the currently supported protocol version. let next_epoch_protocol_version = client_protocol_version + 2; assert_eq!( client_protocol_version, - get_protocol_version_internal( - next_epoch_protocol_version, - client_protocol_version, - schedule, - ) + schedule.get_protocol_version(now, next_epoch_protocol_version) ); } #[test] fn test_after_scheduled_time() { + let now = chrono::Utc::now(); + let client_protocol_version = 100; - let schedule = - ProtocolUpgradeVotingSchedule::from_env_or_str("1900-01-01 00:00:00").unwrap(); + let schedule = make_simple_voting_schedule(client_protocol_version, "1900-01-01 00:00:00"); // Regardless of the protocol version of the next epoch, return the version supported by the client. assert_eq!( client_protocol_version, - get_protocol_version_internal( - client_protocol_version - 2, - client_protocol_version, - schedule, - ) + schedule.get_protocol_version(now, client_protocol_version - 2) ); assert_eq!( client_protocol_version, - get_protocol_version_internal( - client_protocol_version, - client_protocol_version, - schedule, - ) + schedule.get_protocol_version(now, client_protocol_version) ); assert_eq!( client_protocol_version, - get_protocol_version_internal( - client_protocol_version + 2, - client_protocol_version, - schedule, - ) + schedule.get_protocol_version(now, client_protocol_version + 2) + ); + } + + #[test] + fn test_double_upgrade_schedule() { + let current_protocol_version = 100; + let client_protocol_version = 102; + let schedule = vec![ + ("2000-01-10 00:00:00", current_protocol_version + 1), + ("2000-01-15 00:00:00", current_protocol_version + 2), + ]; + + let schedule = make_voting_schedule(client_protocol_version, schedule); + + // Test that the current version is returned before the first upgrade. + let now = ProtocolUpgradeVotingSchedule::parse_datetime("2000-01-05 00:00:00").unwrap(); + assert_eq!( + current_protocol_version, + schedule.get_protocol_version(now, current_protocol_version) ); + + // Test the first upgrade. + let now = ProtocolUpgradeVotingSchedule::parse_datetime("2000-01-10 00:00:00").unwrap(); + assert_eq!( + current_protocol_version + 1, + schedule.get_protocol_version(now, current_protocol_version) + ); + let now = ProtocolUpgradeVotingSchedule::parse_datetime("2000-01-12 00:00:00").unwrap(); + assert_eq!( + current_protocol_version + 1, + schedule.get_protocol_version(now, current_protocol_version) + ); + + // Test the final upgrade. + let now = ProtocolUpgradeVotingSchedule::parse_datetime("2000-01-15 00:00:00").unwrap(); + assert_eq!( + current_protocol_version + 2, + schedule.get_protocol_version(now, current_protocol_version) + ); + let now = ProtocolUpgradeVotingSchedule::parse_datetime("2000-01-20 00:00:00").unwrap(); + assert_eq!( + current_protocol_version + 2, + schedule.get_protocol_version(now, current_protocol_version) + ); + } + + #[test] + fn test_errors() { + let client_protocol_version = 110; + + // invalid last upgrade + let schedule = vec![("2000-01-10 00:00:00", 108), ("2000-01-15 00:00:00", 109)]; + let schedule = make_voting_schedule_impl(schedule, client_protocol_version); + assert!(schedule.is_err()); + + // invalid protocol version order - decreasing versions + let schedule = vec![("2000-01-10 00:00:00", 111), ("2000-01-15 00:00:00", 110)]; + let schedule = make_voting_schedule_impl(schedule, client_protocol_version); + assert!(schedule.is_err()); + + // invalid protocol version order - skip version + let schedule = vec![("2000-01-10 00:00:00", 108), ("2000-01-15 00:00:00", 110)]; + let schedule = make_voting_schedule_impl(schedule, client_protocol_version); + assert!(schedule.is_err()); + + // invalid datetime order + let schedule = vec![("2000-01-15 00:00:00", 109), ("2000-01-10 00:00:00", 110)]; + let schedule = make_voting_schedule_impl(schedule, client_protocol_version); + assert!(schedule.is_err()); } #[test] fn test_parse() { - assert!(ProtocolUpgradeVotingSchedule::from_env_or_str("2001-02-03 23:59:59").is_ok()); - assert!(ProtocolUpgradeVotingSchedule::from_env_or_str("123").is_err()); + assert!(ProtocolUpgradeVotingSchedule::parse_datetime("2001-02-03 23:59:59").is_ok()); + assert!(ProtocolUpgradeVotingSchedule::parse_datetime("123").is_err()); } #[test] - fn test_is_in_future() { - assert!(ProtocolUpgradeVotingSchedule::from_env_or_str("2999-02-03 23:59:59") - .unwrap() - .is_in_future()); - assert!(!ProtocolUpgradeVotingSchedule::from_env_or_str("1999-02-03 23:59:59") - .unwrap() - .is_in_future()); + fn test_parse_override() { + // prepare some datetime strings + + let datetime_str_1 = "2001-01-01 23:59:59"; + let datetime_str_2 = "2001-01-02 23:59:59"; + let datetime_str_3 = "2001-01-03 23:59:59"; + + let datetime_1 = ProtocolUpgradeVotingSchedule::parse_datetime(datetime_str_1).unwrap(); + let datetime_2 = ProtocolUpgradeVotingSchedule::parse_datetime(datetime_str_2).unwrap(); + let datetime_3 = ProtocolUpgradeVotingSchedule::parse_datetime(datetime_str_3).unwrap(); + + let datetime_version_str_1 = format!("{}={}", datetime_str_1, 101); + let datetime_version_str_2 = format!("{}={}", datetime_str_2, 102); + let datetime_version_str_3 = format!("{}={}", datetime_str_3, 103); + + // test immediate upgrade + + let override_str = "now"; + let raw_schedule = ProtocolUpgradeVotingSchedule::parse_override(override_str).unwrap(); + assert_eq!(raw_schedule.len(), 0); + + // test single upgrade + + let override_str = datetime_version_str_1.clone(); + let raw_schedule = ProtocolUpgradeVotingSchedule::parse_override(&override_str).unwrap(); + assert_eq!(raw_schedule.len(), 1); + + assert_eq!(raw_schedule[0].0, datetime_1); + assert_eq!(raw_schedule[0].1, 101); + + // test double upgrade + + let override_str = + [datetime_version_str_1.clone(), datetime_version_str_2.clone()].join(","); + let raw_schedule = ProtocolUpgradeVotingSchedule::parse_override(&override_str).unwrap(); + assert_eq!(raw_schedule.len(), 2); + + assert_eq!(raw_schedule[0].0, datetime_1); + assert_eq!(raw_schedule[0].1, 101); + + assert_eq!(raw_schedule[1].0, datetime_2); + assert_eq!(raw_schedule[1].1, 102); + + // test triple upgrade + + let override_str = + [datetime_version_str_1, datetime_version_str_2, datetime_version_str_3].join(","); + let raw_schedule = ProtocolUpgradeVotingSchedule::parse_override(&override_str).unwrap(); + assert_eq!(raw_schedule.len(), 3); + + assert_eq!(raw_schedule[0].0, datetime_1); + assert_eq!(raw_schedule[0].1, 101); + + assert_eq!(raw_schedule[1].0, datetime_2); + assert_eq!(raw_schedule[1].1, 102); + + assert_eq!(raw_schedule[2].0, datetime_3); + assert_eq!(raw_schedule[2].1, 103); } #[test] - fn test_env_overwrite() { - // The immediate protocol upgrade needs to be set for this test to pass in - // the release branch where the protocol upgrade date is set. - std::env::set_var("NEAR_TESTS_IMMEDIATE_PROTOCOL_UPGRADE", "1"); + fn test_env_override() { + let client_protocol_version = 100; + + std::env::set_var(NEAR_TESTS_PROTOCOL_UPGRADE_OVERRIDE, "now"); + let schedule = make_simple_voting_schedule(client_protocol_version, "2999-02-03 23:59:59"); + + assert_eq!(schedule, ProtocolUpgradeVotingSchedule::new_immediate(client_protocol_version)); + + let datetime_override = "2000-01-01 23:59:59"; + std::env::set_var( + NEAR_TESTS_PROTOCOL_UPGRADE_OVERRIDE, + format!("{}={}", datetime_override, client_protocol_version), + ); + let schedule = make_simple_voting_schedule(client_protocol_version, "2999-02-03 23:59:59"); assert_eq!( - ProtocolUpgradeVotingSchedule::from_env_or_str("2999-02-03 23:59:59").unwrap(), - ProtocolUpgradeVotingSchedule::default() + schedule.schedule()[0].0, + ProtocolUpgradeVotingSchedule::parse_datetime(datetime_override).unwrap() ); + assert_eq!(schedule.schedule()[0].1, client_protocol_version); } } diff --git a/core/primitives/src/version.rs b/core/primitives/src/version.rs index 58593f0feac..109e30e40a3 100644 --- a/core/primitives/src/version.rs +++ b/core/primitives/src/version.rs @@ -10,7 +10,7 @@ pub struct Version { pub rustc_version: String, } -use crate::upgrade_schedule::{get_protocol_version_internal, ProtocolUpgradeVotingSchedule}; +use crate::upgrade_schedule::ProtocolUpgradeVotingSchedule; /// near_primitives_core re-exports pub use near_primitives_core::checked_feature; @@ -59,21 +59,31 @@ pub const FIXED_MINIMUM_NEW_RECEIPT_GAS_VERSION: ProtocolVersion = 66; /// it’s set according to the schedule for that protocol upgrade. Release /// candidates usually have separate schedule to final releases. pub const PROTOCOL_UPGRADE_SCHEDULE: Lazy = Lazy::new(|| { - // Update to according to schedule when making a release. - // Keep in mind that the protocol upgrade will happen 1-2 epochs (15h-30h) - // after the set date. Ideally that should be during working hours. - // e.g. ProtocolUpgradeVotingSchedule::from_env_or_str("2000-01-01 15:00:00").unwrap()); - - ProtocolUpgradeVotingSchedule::default() + // Update according to the schedule when making a release. Keep in mind that + // the protocol upgrade will happen 1-2 epochs (15h-30h) after the set date. + // Ideally that should be during working hours. + // + // Multiple protocol version upgrades can be scheduled. Please make sure + // that they are ordered by datetime and version, the last one is the + // PROTOCOL_VERSION and that there is enough time between subsequent + // upgrades. + + // e.g. + // let v1_protocol_version = 50; + // let v2_protocol_version = 51; + // let v1_datetime = ProtocolUpgradeVotingSchedule::parse_datetime("2000-01-01 15:00:00").unwrap(); + // let v2_datetime = ProtocolUpgradeVotingSchedule::parse_datetime("2000-01-10 15:00:00").unwrap(); + // + // let schedule = vec![(v1_datetime, v1_protocol_version), (v2_datetime, v2_protocol_version)]; + // ProtocolUpgradeVotingSchedule::new_from_env_or_schedule(PROTOCOL_VERSION, schedule).unwrap() + + ProtocolUpgradeVotingSchedule::new_from_env_or_schedule(PROTOCOL_VERSION, vec![]).unwrap() }); /// Gives new clients an option to upgrade without announcing that they support /// the new version. This gives non-validator nodes time to upgrade. See /// pub fn get_protocol_version(next_epoch_protocol_version: ProtocolVersion) -> ProtocolVersion { - get_protocol_version_internal( - next_epoch_protocol_version, - PROTOCOL_VERSION, - *PROTOCOL_UPGRADE_SCHEDULE, - ) + let now = chrono::Utc::now(); + PROTOCOL_UPGRADE_SCHEDULE.get_protocol_version(now, next_epoch_protocol_version) } diff --git a/integration-tests/src/tests/client/features/account_id_in_function_call_permission.rs b/integration-tests/src/tests/client/features/account_id_in_function_call_permission.rs index b6f796abf17..205c841286f 100644 --- a/integration-tests/src/tests/client/features/account_id_in_function_call_permission.rs +++ b/integration-tests/src/tests/client/features/account_id_in_function_call_permission.rs @@ -13,7 +13,7 @@ use nearcore::test_utils::TestEnvNightshadeSetupExt; fn test_account_id_in_function_call_permission_upgrade() { // The immediate protocol upgrade needs to be set for this test to pass in // the release branch where the protocol upgrade date is set. - std::env::set_var("NEAR_TESTS_IMMEDIATE_PROTOCOL_UPGRADE", "1"); + std::env::set_var("NEAR_TESTS_PROTOCOL_UPGRADE_OVERRIDE", "now"); let old_protocol_version = near_primitives::version::ProtocolFeature::AccountIdInFunctionCallPermission diff --git a/integration-tests/src/tests/client/features/flat_storage.rs b/integration-tests/src/tests/client/features/flat_storage.rs index e27c41c043c..27cdd55c029 100644 --- a/integration-tests/src/tests/client/features/flat_storage.rs +++ b/integration-tests/src/tests/client/features/flat_storage.rs @@ -20,7 +20,7 @@ use nearcore::test_utils::TestEnvNightshadeSetupExt; fn test_flat_storage_upgrade() { // The immediate protocol upgrade needs to be set for this test to pass in // the release branch where the protocol upgrade date is set. - std::env::set_var("NEAR_TESTS_IMMEDIATE_PROTOCOL_UPGRADE", "1"); + std::env::set_var("NEAR_TESTS_PROTOCOL_UPGRADE_OVERRIDE", "now"); let mut genesis = Genesis::test(vec!["test0".parse().unwrap(), "test1".parse().unwrap()], 1); let epoch_length = 12; diff --git a/integration-tests/src/tests/client/features/increase_deployment_cost.rs b/integration-tests/src/tests/client/features/increase_deployment_cost.rs index 42ee74652db..d8ba1dc6aeb 100644 --- a/integration-tests/src/tests/client/features/increase_deployment_cost.rs +++ b/integration-tests/src/tests/client/features/increase_deployment_cost.rs @@ -15,7 +15,7 @@ use nearcore::test_utils::TestEnvNightshadeSetupExt; fn test_deploy_cost_increased() { // The immediate protocol upgrade needs to be set for this test to pass in // the release branch where the protocol upgrade date is set. - std::env::set_var("NEAR_TESTS_IMMEDIATE_PROTOCOL_UPGRADE", "1"); + std::env::set_var("NEAR_TESTS_PROTOCOL_UPGRADE_OVERRIDE", "now"); let new_protocol_version = ProtocolFeature::IncreaseDeploymentCost.protocol_version(); let old_protocol_version = new_protocol_version - 1; diff --git a/integration-tests/src/tests/client/features/increase_storage_compute_cost.rs b/integration-tests/src/tests/client/features/increase_storage_compute_cost.rs index 7b4c18555ab..b319e4ab3fe 100644 --- a/integration-tests/src/tests/client/features/increase_storage_compute_cost.rs +++ b/integration-tests/src/tests/client/features/increase_storage_compute_cost.rs @@ -177,7 +177,7 @@ fn assert_compute_limit_reached( ) { // The immediate protocol upgrade needs to be set for this test to pass in // the release branch where the protocol upgrade date is set. - std::env::set_var("NEAR_TESTS_IMMEDIATE_PROTOCOL_UPGRADE", "1"); + std::env::set_var("NEAR_TESTS_PROTOCOL_UPGRADE_OVERRIDE", "now"); near_o11y::testonly::init_test_logger(); let old_protocol_version = INCREASED_STORAGE_COSTS_PROTOCOL_VERSION - 1; diff --git a/integration-tests/src/tests/client/features/lower_storage_key_limit.rs b/integration-tests/src/tests/client/features/lower_storage_key_limit.rs index f56932cf0d3..9e065624cb5 100644 --- a/integration-tests/src/tests/client/features/lower_storage_key_limit.rs +++ b/integration-tests/src/tests/client/features/lower_storage_key_limit.rs @@ -33,7 +33,7 @@ fn protocol_upgrade() { // The immediate protocol upgrade needs to be set for this test to pass in // the release branch where the protocol upgrade date is set. - std::env::set_var("NEAR_TESTS_IMMEDIATE_PROTOCOL_UPGRADE", "1"); + std::env::set_var("NEAR_TESTS_PROTOCOL_UPGRADE_OVERRIDE", "now"); // Prepare TestEnv with a contract at the old protocol version. let mut env = { diff --git a/integration-tests/src/tests/client/features/zero_balance_account.rs b/integration-tests/src/tests/client/features/zero_balance_account.rs index 432dab19d4d..102fd353baa 100644 --- a/integration-tests/src/tests/client/features/zero_balance_account.rs +++ b/integration-tests/src/tests/client/features/zero_balance_account.rs @@ -242,7 +242,7 @@ fn test_zero_balance_account_add_key() { fn test_zero_balance_account_upgrade() { // The immediate protocol upgrade needs to be set for this test to pass in // the release branch where the protocol upgrade date is set. - std::env::set_var("NEAR_TESTS_IMMEDIATE_PROTOCOL_UPGRADE", "1"); + std::env::set_var("NEAR_TESTS_PROTOCOL_UPGRADE_OVERRIDE", "now"); let epoch_length = 5; let mut genesis = Genesis::test(vec!["test0".parse().unwrap(), "test1".parse().unwrap()], 1); diff --git a/integration-tests/src/tests/client/process_blocks.rs b/integration-tests/src/tests/client/process_blocks.rs index 9ca78818455..4d2223b6242 100644 --- a/integration-tests/src/tests/client/process_blocks.rs +++ b/integration-tests/src/tests/client/process_blocks.rs @@ -2758,28 +2758,8 @@ fn test_epoch_protocol_version_change() { .epoch_manager .get_epoch_id_from_prev_block(&head.last_block_hash) .unwrap(); - let chunk_producer = - env.clients[0].epoch_manager.get_chunk_producer(&epoch_id, i, 0).unwrap(); - let index = if chunk_producer == "test0" { 0 } else { 1 }; - let ProduceChunkResult { - chunk: encoded_chunk, - encoded_chunk_parts_paths: merkle_paths, - receipts, - .. - } = create_chunk_on_height(&mut env.clients[index], i); - for j in 0..2 { - let validator_id = - env.clients[j].validator_signer.as_ref().unwrap().validator_id().clone(); - env.clients[j] - .persist_and_distribute_encoded_chunk( - encoded_chunk.clone(), - merkle_paths.clone(), - receipts.clone(), - validator_id, - ) - .unwrap(); - } + produce_chunks(&mut env, &epoch_id, i); let epoch_id = env.clients[0] .epoch_manager @@ -2804,6 +2784,120 @@ fn test_epoch_protocol_version_change() { assert_eq!(protocol_version, PROTOCOL_VERSION); } +#[test] +fn test_epoch_multi_protocol_version_change() { + init_test_logger(); + + let v0 = PROTOCOL_VERSION - 2; + let v1 = PROTOCOL_VERSION - 1; + let v2 = PROTOCOL_VERSION; + + // produce blocks roughly every 100ms + // one epoch is roughly 500ms + // at least two epochs are needed for one protocol upgrade + // schedule the first protocol upgrade voting at now +1s + // schedule the second protocol upgrade voting at now +3s + + let start_time = chrono::Utc::now(); + let end_time = start_time + chrono::Duration::seconds(6); + + let v1_upgrade_time = start_time + chrono::Duration::seconds(1); + let v2_upgrade_time = start_time + chrono::Duration::seconds(3); + + let v1_upgrade_time = v1_upgrade_time.format("%Y-%m-%d %H:%M:%S").to_string(); + let v2_upgrade_time = v2_upgrade_time.format("%Y-%m-%d %H:%M:%S").to_string(); + + let protocol_version_override = + format!("{}={},{}={}", v1_upgrade_time, v1, v2_upgrade_time, v2); + + tracing::debug!(target: "test", ?protocol_version_override, "setting the protocol_version_override"); + std::env::set_var("NEAR_TESTS_PROTOCOL_UPGRADE_OVERRIDE", protocol_version_override); + + let epoch_length = 5; + let mut genesis = Genesis::test(vec!["test0".parse().unwrap(), "test1".parse().unwrap()], 2); + genesis.config.epoch_length = epoch_length; + genesis.config.protocol_version = v0; + let mut env = TestEnv::builder(&genesis.config) + .clients_count(2) + .validator_seats(2) + .nightshade_runtimes(&genesis) + .build(); + + let mut seen_v0 = false; + let mut seen_v1 = false; + let mut seen_v2 = false; + + let mut height = 1; + while chrono::Utc::now() < end_time { + let head = env.clients[0].chain.head().unwrap(); + let epoch_id = env.clients[0] + .epoch_manager + .get_epoch_id_from_prev_block(&head.last_block_hash) + .unwrap(); + let protocol_version = + env.clients[0].epoch_manager.get_epoch_protocol_version(&epoch_id).unwrap(); + + produce_chunks(&mut env, &epoch_id, height); + + produce_block(&mut env, &epoch_id, height); + + if protocol_version == v0 { + seen_v0 = true; + } + if protocol_version == v1 { + seen_v1 = true; + } + if protocol_version == v2 { + seen_v2 = true; + } + + tracing::debug!(target: "test", ?height, ?protocol_version, "loop iter finished"); + + height += 1; + std::thread::sleep(std::time::Duration::from_millis(100)); + } + + assert!(seen_v0); + assert!(seen_v1); + assert!(seen_v2); +} + +// helper for test_epoch_multi_protocol_version_change +fn produce_block(env: &mut TestEnv, epoch_id: &EpochId, height: u64) { + let block_producer = env.clients[0].epoch_manager.get_block_producer(epoch_id, height).unwrap(); + let index = if block_producer == "test0" { 0 } else { 1 }; + let block = env.clients[index].produce_block(height).unwrap().unwrap(); + for client in &mut env.clients { + client.process_block_test(block.clone().into(), Provenance::NONE).unwrap(); + } +} + +// helper for test_epoch_multi_protocol_version_change +fn produce_chunks(env: &mut TestEnv, epoch_id: &EpochId, height: u64) { + let shard_layout = env.clients[0].epoch_manager.get_shard_layout(epoch_id).unwrap(); + + for shard_id in shard_layout.shard_ids() { + let chunk_producer = + env.clients[0].epoch_manager.get_chunk_producer(epoch_id, height, shard_id).unwrap(); + + let produce_chunk_result = create_chunk_on_height(env.client(&chunk_producer), height); + let ProduceChunkResult { chunk, encoded_chunk_parts_paths, receipts, .. } = + produce_chunk_result; + + for client in &mut env.clients { + let validator_id = client.validator_signer.as_ref().unwrap().validator_id().clone(); + client + .persist_and_distribute_encoded_chunk( + chunk.clone(), + encoded_chunk_parts_paths.clone(), + receipts.clone(), + validator_id, + ) + .unwrap(); + } + } +} + #[test] fn test_discard_non_finalizable_block() { let genesis = Genesis::test(vec!["test0".parse().unwrap(), "test1".parse().unwrap()], 1); diff --git a/integration-tests/src/tests/client/resharding.rs b/integration-tests/src/tests/client/resharding.rs index 9c71e30be9a..9bdf874c8ca 100644 --- a/integration-tests/src/tests/client/resharding.rs +++ b/integration-tests/src/tests/client/resharding.rs @@ -2021,19 +2021,16 @@ fn test_shard_layout_upgrade_missing_chunks_high_missing_prob_v3() { // latest protocol -#[ignore] #[test] fn test_latest_protocol_missing_chunks_low_missing_prob() { test_latest_protocol_missing_chunks(0.1, 25); } -#[ignore] #[test] fn test_latest_protocol_missing_chunks_mid_missing_prob() { test_latest_protocol_missing_chunks(0.5, 26); } -#[ignore] #[test] fn test_latest_protocol_missing_chunks_high_missing_prob() { test_latest_protocol_missing_chunks(0.9, 27); diff --git a/pytest/tests/sanity/upgradable.py b/pytest/tests/sanity/upgradable.py index cc71ba1a5a8..69d490cb033 100755 --- a/pytest/tests/sanity/upgradable.py +++ b/pytest/tests/sanity/upgradable.py @@ -161,7 +161,7 @@ def test_upgrade() -> None: nodes[i].binary_name = executables.current.neard nodes[i].start( boot_node=nodes[0], - extra_env={"NEAR_TESTS_IMMEDIATE_PROTOCOL_UPGRADE": "1"}, + extra_env={"NEAR_TESTS_PROTOCOL_UPGRADE_OVERRIDE": "now"}, ) count = 0