From d9c48f8f8a69df28c38774afaad53b926a53a564 Mon Sep 17 00:00:00 2001 From: sfauvel Date: Thu, 13 Feb 2025 17:41:27 +0100 Subject: [PATCH 1/3] Use a unique name for the digest --- .../src/artifact_builder/cardano_database.rs | 6 +- .../cardano_database_artifacts/digest.rs | 69 +++++++++++++++---- .../builder/protocol/artifacts.rs | 1 + 3 files changed, 62 insertions(+), 14 deletions(-) diff --git a/mithril-aggregator/src/artifact_builder/cardano_database.rs b/mithril-aggregator/src/artifact_builder/cardano_database.rs index 52aa1da19f..82f31495ee 100644 --- a/mithril-aggregator/src/artifact_builder/cardano_database.rs +++ b/mithril-aggregator/src/artifact_builder/cardano_database.rs @@ -74,7 +74,7 @@ impl ArtifactBuilder for CardanoDataba .immutable_builder .upload(beacon.immutable_file_number) .await?; - let digest_locations = self.digest_builder.upload().await?; + let digest_locations = self.digest_builder.upload(&beacon).await?; let locations = ArtifactsLocations { ancillary: ancillary_locations, @@ -176,6 +176,7 @@ mod tests { let test_dir = get_test_directory("should_compute_valid_artifact"); let beacon = CardanoDbBeacon::new(123, 3); + let network = CardanoNetwork::DevNet(123); let immutable_trio_file_size = 777; let ledger_file_size = 6666; let volatile_file_size = 99; @@ -204,7 +205,7 @@ mod tests { AncillaryArtifactBuilder::new( vec![Arc::new(ancillary_uploader)], snapshotter.clone(), - CardanoNetwork::DevNet(123), + network, CompressionAlgorithm::Gzip, TestLogger::stdout(), ) @@ -245,6 +246,7 @@ mod tests { DigestArtifactBuilder::new( SanitizedUrlWithTrailingSlash::parse("http://aggregator_uri").unwrap(), vec![], + network, test_dir.join("digests"), Arc::new(immutable_file_digest_mapper), TestLogger::stdout(), diff --git a/mithril-aggregator/src/artifact_builder/cardano_database_artifacts/digest.rs b/mithril-aggregator/src/artifact_builder/cardano_database_artifacts/digest.rs index 3237962cbf..ce5dc0be04 100644 --- a/mithril-aggregator/src/artifact_builder/cardano_database_artifacts/digest.rs +++ b/mithril-aggregator/src/artifact_builder/cardano_database_artifacts/digest.rs @@ -7,8 +7,10 @@ use std::{ use anyhow::Context; use async_trait::async_trait; use mithril_common::{ - entities::DigestLocation, logging::LoggerExtensions, - messages::CardanoDatabaseDigestListItemMessage, StdResult, + entities::{CardanoDbBeacon, DigestLocation}, + logging::LoggerExtensions, + messages::CardanoDatabaseDigestListItemMessage, + CardanoNetwork, StdResult, }; use slog::{error, Logger}; @@ -60,6 +62,8 @@ pub struct DigestArtifactBuilder { /// Uploaders uploaders: Vec>, + network: CardanoNetwork, + digests_dir: PathBuf, immutable_file_digest_mapper: Arc, @@ -72,6 +76,7 @@ impl DigestArtifactBuilder { pub fn new( aggregator_url_prefix: SanitizedUrlWithTrailingSlash, uploaders: Vec>, + network: CardanoNetwork, digests_dir: PathBuf, immutable_file_digest_mapper: Arc, logger: Logger, @@ -79,14 +84,15 @@ impl DigestArtifactBuilder { Ok(Self { aggregator_url_prefix, uploaders, + network, digests_dir, immutable_file_digest_mapper, logger: logger.new_with_component_name::(), }) } - pub async fn upload(&self) -> StdResult> { - let digest_path = self.create_digest_file().await?; + pub async fn upload(&self, beacon: &CardanoDbBeacon) -> StdResult> { + let digest_path = self.create_digest_file(beacon).await?; let locations = self.upload_digest_file(&digest_path).await; fs::remove_file(&digest_path).with_context(|| { @@ -96,7 +102,7 @@ impl DigestArtifactBuilder { locations } - async fn create_digest_file(&self) -> StdResult { + async fn create_digest_file(&self, beacon: &CardanoDbBeacon) -> StdResult { let immutable_file_digest_map = self .immutable_file_digest_mapper .get_immutable_file_digest_map() @@ -110,7 +116,8 @@ impl DigestArtifactBuilder { ) .collect::>(); - let digests_file_path = DigestArtifactBuilder::get_digests_file_path(&self.digests_dir); + let digests_file_path = + DigestArtifactBuilder::get_digests_file_path(&self.digests_dir, &self.network, beacon); if let Some(digests_dir) = digests_file_path.parent() { fs::create_dir_all(digests_dir).with_context(|| { @@ -160,8 +167,16 @@ impl DigestArtifactBuilder { }) } - fn get_digests_file_path>(digests_dir: P) -> PathBuf { - digests_dir.as_ref().join("digests.json") + fn get_digests_file_path>( + digests_dir: P, + network: &CardanoNetwork, + beacon: &CardanoDbBeacon, + ) -> PathBuf { + let filename = format!( + "{}-e{}-i{}.digests.json", + network, *beacon.epoch, beacon.immutable_file_number + ); + digests_dir.as_ref().join(filename) } } @@ -174,6 +189,7 @@ mod tests { }; use anyhow::anyhow; use mithril_common::{ + entities::CardanoDbBeacon, messages::{CardanoDatabaseDigestListItemMessage, CardanoDatabaseDigestListMessage}, test_utils::{assert_equivalent, TempDir}, }; @@ -214,13 +230,14 @@ mod tests { let builder = DigestArtifactBuilder::new( SanitizedUrlWithTrailingSlash::parse("https://aggregator/").unwrap(), vec![], + CardanoNetwork::DevNet(123), temp_dir, Arc::new(immutable_file_digest_mapper), TestLogger::stdout(), ) .unwrap(); - let locations = builder.upload().await.unwrap(); + let locations = builder.upload(&CardanoDbBeacon::new(4, 123)).await.unwrap(); assert_eq!( vec!(DigestLocation::Aggregator { uri: "https://aggregator/artifact/cardano-database/digests".to_string() @@ -243,6 +260,7 @@ mod tests { let builder = DigestArtifactBuilder::new( SanitizedUrlWithTrailingSlash::parse("https://aggregator/").unwrap(), vec![Arc::new(uploader)], + CardanoNetwork::DevNet(123), PathBuf::from("/tmp/whatever"), Arc::new(MockImmutableFileDigestMapper::new()), TestLogger::file(&log_path), @@ -263,6 +281,7 @@ mod tests { let builder = DigestArtifactBuilder::new( SanitizedUrlWithTrailingSlash::parse("https://aggregator/").unwrap(), vec![Arc::new(uploader)], + CardanoNetwork::DevNet(123), PathBuf::from("/tmp/whatever"), Arc::new(MockImmutableFileDigestMapper::new()), TestLogger::stdout(), @@ -292,6 +311,7 @@ mod tests { let builder = DigestArtifactBuilder::new( SanitizedUrlWithTrailingSlash::parse("https://aggregator/").unwrap(), uploaders, + CardanoNetwork::DevNet(123), PathBuf::from("/tmp/whatever"), Arc::new(MockImmutableFileDigestMapper::new()), TestLogger::stdout(), @@ -327,6 +347,7 @@ mod tests { let builder = DigestArtifactBuilder::new( SanitizedUrlWithTrailingSlash::parse("https://aggregator/").unwrap(), uploaders, + CardanoNetwork::DevNet(123), PathBuf::from("/tmp/whatever"), Arc::new(MockImmutableFileDigestMapper::new()), TestLogger::stdout(), @@ -373,13 +394,17 @@ mod tests { let builder = DigestArtifactBuilder::new( SanitizedUrlWithTrailingSlash::parse("https://aggregator/").unwrap(), vec![], + CardanoNetwork::DevNet(123), temp_dir, Arc::new(immutable_file_digest_mapper), TestLogger::stdout(), ) .unwrap(); - let digest_file = builder.create_digest_file().await.unwrap(); + let digest_file = builder + .create_digest_file(&CardanoDbBeacon::new(4, 123)) + .await + .unwrap(); let file_content = read_to_string(digest_file).unwrap(); let digest_content: CardanoDatabaseDigestListMessage = @@ -408,7 +433,10 @@ mod tests { let mut digest_file_uploader = MockDigestFileUploader::new(); - let digest_file = DigestArtifactBuilder::get_digests_file_path(&digests_dir); + let beacon = CardanoDbBeacon::new(3, 456); + let network = CardanoNetwork::DevNet(24); + let digest_file = + DigestArtifactBuilder::get_digests_file_path(&digests_dir, &network, &beacon); let digest_file_clone = digest_file.clone(); digest_file_uploader @@ -424,14 +452,31 @@ mod tests { let builder = DigestArtifactBuilder::new( SanitizedUrlWithTrailingSlash::parse("https://aggregator/").unwrap(), vec![Arc::new(digest_file_uploader)], + network, digests_dir, Arc::new(immutable_file_digest_mapper), TestLogger::stdout(), ) .unwrap(); - let _locations = builder.upload().await.unwrap(); + let _locations = builder.upload(&beacon).await.unwrap(); assert!(!digest_file.exists()); } + + #[tokio::test] + async fn get_digest_file_path_include_beacon_information() { + let digests_dir = + TempDir::create("digests", "get_digest_file_path_include_beacon_information"); + + let beacon = CardanoDbBeacon::new(5, 456); + let network = CardanoNetwork::MainNet; + let digest_file = + DigestArtifactBuilder::get_digests_file_path(&digests_dir, &network, &beacon); + + assert_eq!( + digest_file, + digests_dir.join("mainnet-e5-i456.digests.json") + ); + } } diff --git a/mithril-aggregator/src/dependency_injection/builder/protocol/artifacts.rs b/mithril-aggregator/src/dependency_injection/builder/protocol/artifacts.rs index 050eaa0d1f..7def09bb22 100644 --- a/mithril-aggregator/src/dependency_injection/builder/protocol/artifacts.rs +++ b/mithril-aggregator/src/dependency_injection/builder/protocol/artifacts.rs @@ -353,6 +353,7 @@ impl DependenciesBuilder { let digest_builder = Arc::new(DigestArtifactBuilder::new( self.configuration.get_server_url()?, self.build_cardano_database_digests_uploaders()?, + self.configuration.get_network()?, snapshot_dir.join("pending_cardano_database_digests"), self.get_immutable_file_digest_mapper().await?, self.root_logger(), From a604819fd2d5124b77605589315a0fb0fd19a4f6 Mon Sep 17 00:00:00 2001 From: sfauvel Date: Fri, 14 Feb 2025 12:02:00 +0100 Subject: [PATCH 2/3] Add `cardano-database/digests` folder to the cloud storage lifecycle rule in the infra --- mithril-infra/variables.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mithril-infra/variables.tf b/mithril-infra/variables.tf index 829072270f..beb4baa688 100644 --- a/mithril-infra/variables.tf +++ b/mithril-infra/variables.tf @@ -135,7 +135,7 @@ variable "google_storage_bucket_max_age" { variable "google_storage_bucket_prefix_with_lifecyle_rule" { type = list(any) description = "The prefix of the object in the storage bucket to apply the lifecycle rule" - default = ["cardano-immutable-files-full", "cardano-database/ancillary"] + default = ["cardano-immutable-files-full", "cardano-database/ancillary", "cardano-database/digests"] } locals { From 19fa6c90fc0ad14ae90994056ef5e2bb943ef5a5 Mon Sep 17 00:00:00 2001 From: sfauvel Date: Mon, 17 Feb 2025 09:51:29 +0100 Subject: [PATCH 3/3] chore: upgrade crate versions * mithril-aggregator from `0.7.3` to `0.7.4` * mithril-infra from `0.3.12` to `0.3.13` --- Cargo.lock | 2 +- mithril-aggregator/Cargo.toml | 2 +- mithril-infra/assets/infra.version | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cd9aa70eac..6aa5aa2e5b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3642,7 +3642,7 @@ dependencies = [ [[package]] name = "mithril-aggregator" -version = "0.7.3" +version = "0.7.4" dependencies = [ "anyhow", "async-trait", diff --git a/mithril-aggregator/Cargo.toml b/mithril-aggregator/Cargo.toml index 694bfb01ab..c467a9fa56 100644 --- a/mithril-aggregator/Cargo.toml +++ b/mithril-aggregator/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "mithril-aggregator" -version = "0.7.3" +version = "0.7.4" description = "A Mithril Aggregator server" authors = { workspace = true } edition = { workspace = true } diff --git a/mithril-infra/assets/infra.version b/mithril-infra/assets/infra.version index 0b9c019963..e4737652ca 100644 --- a/mithril-infra/assets/infra.version +++ b/mithril-infra/assets/infra.version @@ -1 +1 @@ -0.3.12 +0.3.13