Skip to content

Commit e584c6f

Browse files
GrapeBaBaGrapeBaBa
committed
feat:cli arg to config and test
Signed-off-by: Chen Kai <[email protected]>
1 parent 2db5234 commit e584c6f

File tree

4 files changed

+284
-17
lines changed

4 files changed

+284
-17
lines changed

‎bin/archiver/src/api.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -166,6 +166,7 @@ mod tests {
166166
origin_block: *blob_test_helper::ORIGIN_BLOCK,
167167
beacon_config: Default::default(),
168168
storage_config: Default::default(),
169+
log_config: Default::default(),
169170
};
170171
let archiver = Archiver::new(beacon_client.clone(), storage, config, shutdown_rx);
171172
(archiver, beacon_client)

‎bin/archiver/src/archiver.rs‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ use tokio::sync::Mutex;
1313
use tokio::time::{interval, sleep};
1414
use tracing::log::{debug, error, info, trace};
1515

16+
use crate::LogConfig;
1617
use blob_archiver_beacon::beacon_client;
1718
use blob_archiver_beacon::beacon_client::BeaconClient;
1819
use blob_archiver_storage::storage;
@@ -56,6 +57,8 @@ pub struct Config {
5657
pub beacon_config: beacon_client::Config,
5758

5859
pub storage_config: storage::Config,
60+
61+
pub log_config: LogConfig,
5962
}
6063

6164
pub struct Archiver {
@@ -500,6 +503,7 @@ mod tests {
500503
origin_block: *blob_test_helper::ORIGIN_BLOCK,
501504
beacon_config: Default::default(),
502505
storage_config: Default::default(),
506+
log_config: Default::default(),
503507
};
504508
let archiver = Archiver::new(beacon_client.clone(), storage, config, shutdown_rx);
505509
(archiver, beacon_client)
@@ -1825,6 +1829,7 @@ mod tests {
18251829
origin_block: *blob_test_helper::ORIGIN_BLOCK,
18261830
beacon_config: Default::default(),
18271831
storage_config: Default::default(),
1832+
log_config: Default::default(),
18281833
};
18291834
let archiver = Archiver::new(
18301835
Arc::new(Mutex::new(beacon_client_eth2)),

‎bin/archiver/src/main.rs‎

Lines changed: 264 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,24 +1,30 @@
1-
use crate::archiver::{Archiver, Config, STARTUP_FETCH_BLOB_MAXIMUM_RETRIES};
2-
use again::RetryPolicy;
3-
use blob_archiver_beacon::beacon_client::BeaconClientEth2;
4-
use blob_archiver_beacon::blob_test_helper;
5-
use blob_archiver_storage::fs::FSStorage;
6-
use clap::Parser;
7-
use eth2::types::BlockId;
8-
use eth2::{BeaconNodeHttpClient, SensitiveUrl, Timeouts};
9-
use serde::Serialize;
101
use std::fs;
112
use std::path::PathBuf;
123
use std::str::FromStr;
134
use std::sync::Arc;
145
use std::time::Duration;
6+
7+
use again::RetryPolicy;
8+
use clap::Parser;
9+
use eth2::types::{BlockId, Hash256};
10+
use eth2::{BeaconNodeHttpClient, SensitiveUrl, Timeouts};
11+
use serde::{Deserialize, Serialize};
1512
use tokio::sync::Mutex;
1613
use tracing::log::error;
1714
use tracing_appender::rolling::{RollingFileAppender, Rotation};
1815
use tracing_subscriber::layer::SubscriberExt;
1916
use tracing_subscriber::util::SubscriberInitExt;
2017
use tracing_subscriber::{fmt, EnvFilter};
2118

19+
use blob_archiver_beacon::beacon_client::BeaconClientEth2;
20+
use blob_archiver_beacon::{beacon_client, blob_test_helper};
21+
use blob_archiver_storage::fs::FSStorage;
22+
use blob_archiver_storage::s3::S3Config;
23+
use blob_archiver_storage::storage;
24+
use blob_archiver_storage::storage::StorageType;
25+
26+
use crate::archiver::{Archiver, Config, STARTUP_FETCH_BLOB_MAXIMUM_RETRIES};
27+
2228
mod api;
2329
mod archiver;
2430

@@ -27,7 +33,9 @@ static INIT: std::sync::Once = std::sync::Once::new();
2733
#[tokio::main]
2834
async fn main() {
2935
let args = CliArgs::parse();
30-
println!("{}", args.verbose);
36+
37+
let config = args.to_config();
38+
println!("{:#?}", config);
3139
init_logging(0, None, None);
3240
let beacon_client = BeaconNodeHttpClient::new(
3341
SensitiveUrl::from_str("https://ethereum-beacon-api.publicnode.com").unwrap(),
@@ -42,6 +50,7 @@ async fn main() {
4250
origin_block: *blob_test_helper::ORIGIN_BLOCK,
4351
beacon_config: Default::default(),
4452
storage_config: Default::default(),
53+
log_config: Default::default(),
4554
};
4655
let archiver = Archiver::new(
4756
Arc::new(Mutex::new(beacon_client_eth2)),
@@ -128,14 +137,14 @@ pub fn setup_tracing(
128137

129138
#[derive(Parser, Serialize)]
130139
pub struct CliArgs {
131-
#[clap(short='v', long, action = clap::ArgAction::Count, default_value = "3")]
140+
#[clap(short = 'v', long, action = clap::ArgAction::Count, default_value = "3")]
132141
verbose: u8,
133142

134-
#[clap(long, default_value = "logs")]
135-
log_dir: String,
143+
#[clap(long)]
144+
log_dir: Option<String>,
136145

137-
#[clap(long, default_value = "DAILY")]
138-
log_rotation: String,
146+
#[clap(long, help = "Log rotation values: DAILY, HOURLY, MINUTELY, NEVER")]
147+
log_rotation: Option<String>,
139148

140149
#[clap(long, required = true)]
141150
beacon_endpoint: String,
@@ -169,3 +178,243 @@ pub struct CliArgs {
169178
#[clap(long)]
170179
fs_dir: Option<String>,
171180
}
181+
182+
impl CliArgs {
183+
fn to_config(&self) -> Config {
184+
let log_config = LogConfig {
185+
log_dir: self.log_dir.as_ref().map(PathBuf::from),
186+
log_rotation: self.log_rotation.clone(),
187+
verbose: self.verbose,
188+
};
189+
190+
let beacon_config = beacon_client::Config {
191+
beacon_endpoint: self.beacon_endpoint.clone(),
192+
beacon_client_timeout: Duration::from_secs(self.beacon_client_timeout),
193+
};
194+
195+
let storage_type = StorageType::from_str(self.storage_type.as_str()).unwrap();
196+
197+
let s3_config = if storage_type == StorageType::S3 {
198+
Some(S3Config {
199+
endpoint: self.s3_endpoint.clone().unwrap(),
200+
bucket: self.s3_bucket.clone().unwrap(),
201+
path: self.s3_path.clone().unwrap(),
202+
compression: self.s3_compress.unwrap(),
203+
})
204+
} else {
205+
None
206+
};
207+
208+
let fs_dir = self.fs_dir.as_ref().map(PathBuf::from);
209+
210+
let storage_config = storage::Config {
211+
storage_type,
212+
s3_config,
213+
fs_dir,
214+
};
215+
216+
let mut padded_hex = self
217+
.origin_block
218+
.strip_prefix("0x")
219+
.unwrap_or(&self.origin_block)
220+
.to_string();
221+
padded_hex = format!("{:0<64}", padded_hex);
222+
let origin_block = Hash256::from_slice(&hex::decode(padded_hex).expect("Invalid hex"));
223+
224+
Config {
225+
poll_interval: Duration::from_secs(self.poll_interval),
226+
listen_addr: self.listen_addr.clone(),
227+
origin_block,
228+
beacon_config,
229+
storage_config,
230+
log_config,
231+
}
232+
}
233+
}
234+
235+
#[allow(dead_code)]
236+
fn to_rotation(s: &str) -> Rotation {
237+
match s {
238+
"DAILY" => Rotation::DAILY,
239+
"HOURLY" => Rotation::HOURLY,
240+
"MINUTELY" => Rotation::MINUTELY,
241+
_ => Rotation::NEVER,
242+
}
243+
}
244+
245+
#[cfg(test)]
246+
mod tests {
247+
use std::path::PathBuf;
248+
use std::time::Duration;
249+
250+
use eth2::types::Hash256;
251+
252+
use blob_archiver_storage::s3::S3Config;
253+
254+
use crate::storage::StorageType;
255+
use crate::CliArgs;
256+
257+
#[test]
258+
fn test_cli_args_to_config_s3_storage() {
259+
let cli_args = CliArgs {
260+
verbose: 3,
261+
log_dir: Some("logs".to_string()),
262+
log_rotation: Some("DAILY".to_string()),
263+
beacon_endpoint: "http://localhost:5052".to_string(),
264+
beacon_client_timeout: 10,
265+
poll_interval: 6,
266+
listen_addr: "0.0.0.0:8000".to_string(),
267+
origin_block: "0x1234".to_string(),
268+
storage_type: "s3".to_string(),
269+
s3_endpoint: Some("https://s3.amazonaws.com".to_string()),
270+
s3_bucket: Some("my-bucket".to_string()),
271+
s3_path: Some("my-path".to_string()),
272+
s3_compress: Some(true),
273+
fs_dir: None,
274+
};
275+
276+
let config = cli_args.to_config();
277+
278+
assert_eq!(config.poll_interval, Duration::from_secs(6));
279+
assert_eq!(config.listen_addr, "0.0.0.0:8000");
280+
assert_eq!(
281+
config.origin_block,
282+
Hash256::from_slice(
283+
&hex::decode("1234000000000000000000000000000000000000000000000000000000000000")
284+
.unwrap()
285+
)
286+
);
287+
assert_eq!(
288+
config.beacon_config.beacon_endpoint,
289+
"http://localhost:5052"
290+
);
291+
assert_eq!(
292+
config.beacon_config.beacon_client_timeout,
293+
Duration::from_secs(10)
294+
);
295+
assert_eq!(config.storage_config.storage_type, StorageType::S3);
296+
assert_eq!(
297+
config.storage_config.s3_config,
298+
Some(S3Config {
299+
endpoint: "https://s3.amazonaws.com".to_string(),
300+
bucket: "my-bucket".to_string(),
301+
path: "my-path".to_string(),
302+
compression: true,
303+
})
304+
);
305+
assert_eq!(config.storage_config.fs_dir, None);
306+
}
307+
308+
#[test]
309+
fn test_cli_args_to_config_fs_storage() {
310+
let cli_args = CliArgs {
311+
verbose: 3,
312+
log_dir: Some("logs".to_string()),
313+
log_rotation: Some("DAILY".to_string()),
314+
beacon_endpoint: "http://localhost:5052".to_string(),
315+
beacon_client_timeout: 10,
316+
poll_interval: 6,
317+
listen_addr: "0.0.0.0:8000".to_string(),
318+
origin_block: "0xabcd".to_string(),
319+
storage_type: "fs".to_string(),
320+
s3_endpoint: None,
321+
s3_bucket: None,
322+
s3_path: None,
323+
s3_compress: None,
324+
fs_dir: Some("/path/to/storage".to_string()),
325+
};
326+
327+
let config = cli_args.to_config();
328+
329+
assert_eq!(config.poll_interval, Duration::from_secs(6));
330+
assert_eq!(config.listen_addr, "0.0.0.0:8000");
331+
assert_eq!(
332+
config.origin_block,
333+
Hash256::from_slice(
334+
&hex::decode("abcd000000000000000000000000000000000000000000000000000000000000")
335+
.unwrap()
336+
)
337+
);
338+
assert_eq!(
339+
config.beacon_config.beacon_endpoint,
340+
"http://localhost:5052"
341+
);
342+
assert_eq!(
343+
config.beacon_config.beacon_client_timeout,
344+
Duration::from_secs(10)
345+
);
346+
assert_eq!(config.storage_config.storage_type, StorageType::FS);
347+
assert_eq!(config.storage_config.s3_config, None);
348+
assert_eq!(
349+
config.storage_config.fs_dir,
350+
Some(PathBuf::from("/path/to/storage"))
351+
);
352+
}
353+
354+
#[test]
355+
fn test_cli_args_to_config_origin_block_padding() {
356+
let cli_args = CliArgs {
357+
verbose: 3,
358+
log_dir: Some("logs".to_string()),
359+
log_rotation: Some("DAILY".to_string()),
360+
beacon_endpoint: "http://localhost:5052".to_string(),
361+
beacon_client_timeout: 10,
362+
poll_interval: 6,
363+
listen_addr: "0.0.0.0:8000".to_string(),
364+
origin_block: "0x1".to_string(),
365+
storage_type: "fs".to_string(),
366+
s3_endpoint: None,
367+
s3_bucket: None,
368+
s3_path: None,
369+
s3_compress: None,
370+
fs_dir: Some("/path/to/storage".to_string()),
371+
};
372+
373+
let config = cli_args.to_config();
374+
375+
assert_eq!(
376+
config.origin_block,
377+
Hash256::from_slice(
378+
&hex::decode("1000000000000000000000000000000000000000000000000000000000000000")
379+
.unwrap()
380+
)
381+
);
382+
}
383+
384+
#[test]
385+
fn test_cli_args_to_config_origin_block_without_prefix() {
386+
let cli_args = CliArgs {
387+
verbose: 3,
388+
log_dir: Some("logs".to_string()),
389+
log_rotation: Some("DAILY".to_string()),
390+
beacon_endpoint: "http://localhost:5052".to_string(),
391+
beacon_client_timeout: 10,
392+
poll_interval: 6,
393+
listen_addr: "0.0.0.0:8000".to_string(),
394+
origin_block: "ff".to_string(),
395+
storage_type: "fs".to_string(),
396+
s3_endpoint: None,
397+
s3_bucket: None,
398+
s3_path: None,
399+
s3_compress: None,
400+
fs_dir: Some("/path/to/storage".to_string()),
401+
};
402+
403+
let config = cli_args.to_config();
404+
405+
assert_eq!(
406+
config.origin_block,
407+
Hash256::from_slice(
408+
&hex::decode("ff00000000000000000000000000000000000000000000000000000000000000")
409+
.unwrap()
410+
)
411+
);
412+
}
413+
}
414+
415+
#[derive(Debug, PartialEq, Eq, Clone, Default, Serialize, Deserialize)]
416+
pub struct LogConfig {
417+
log_dir: Option<PathBuf>,
418+
log_rotation: Option<String>,
419+
verbose: u8,
420+
}

‎crates/storage/src/storage.rs‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,11 +20,23 @@ pub enum StorageType {
2020
S3,
2121
}
2222

23+
impl std::str::FromStr for StorageType {
24+
type Err = String;
25+
26+
fn from_str(s: &str) -> Result<Self, Self::Err> {
27+
match s.to_lowercase().as_str() {
28+
"fs" => Ok(StorageType::FS),
29+
"s3" => Ok(StorageType::S3),
30+
_ => Err(format!("Invalid storage type: {}", s)),
31+
}
32+
}
33+
}
34+
2335
#[derive(Debug, PartialEq, Eq, Clone, Default, Serialize, Deserialize)]
2436
pub struct Config {
2537
pub storage_type: StorageType,
26-
pub s3_config: S3Config,
27-
pub fs_dir: PathBuf,
38+
pub s3_config: Option<S3Config>,
39+
pub fs_dir: Option<PathBuf>,
2840
}
2941

3042
#[derive(Debug, PartialEq, Clone, Serialize, Deserialize)]

0 commit comments

Comments
 (0)