ckb-export: Support export to stdout, then can pipe using zstd tar, etc.

Signed-off-by: Eval EXEC <[email protected]>

Author: eval-exec

Cargo.lock

@@ -388,7 +388,7 @@ fn export() -> Command {
                 .value_name("path")
                 .value_parser(clap::builder::PathBufValueParser::new())
                 .required(true)
-                .help("Specify the export target path"),
+                .help("Specify the export target path (use '-' for stdout)"),
         )
         .arg(
             Arg::new(ARG_FROM)

ckb-bin/src/cli.rs

@@ -1,9 +1,7 @@
 #[cfg(not(target_os = "windows"))]
 use ckb_app_config::DaemonArgs;
 use ckb_app_config::{
-    AppConfig, CustomizeSpec, ExitCode, ExportArgs, ImportArgs, InitArgs, MigrateArgs, MinerArgs,
-    PeerIDArgs, ReplayArgs, ResetDataArgs, RunArgs, StatsArgs, generate_random_key,
-    read_secret_key, write_secret_to_file,
+    generate_random_key, read_secret_key, write_secret_to_file, AppConfig, CustomizeSpec, ExitCode, ExportArgs, ExportTarget, ImportArgs, InitArgs, MigrateArgs, MinerArgs, PeerIDArgs, ReplayArgs, ResetDataArgs, RunArgs, StatsArgs
 };
 use ckb_chain_spec::{ChainSpec, consensus::Consensus};
 use ckb_jsonrpc_types::{Either, ScriptHashType};
@@ -245,13 +243,24 @@ H256::from_str(&target[2..]).expect("default assume_valid_target for testnet mus
     pub fn export(self, matches: &ArgMatches) -> Result<ExportArgs, ExitCode> {
         let consensus = self.consensus()?;
         let config = self.config.into_ckb()?;
-        let target = matches
-            .get_one::<PathBuf>(cli::ARG_TARGET)
-            .ok_or_else(|| {
-                eprintln!("Args Error: {:?} no found", cli::ARG_TARGET);
-                ExitCode::Cli
-            })?
-            .clone();
+        let target = {
+            let target = matches
+                .get_one::<String>(cli::ARG_TARGET)
+                .ok_or_else(|| {
+                    eprintln!("Args Error: {:?} no found", cli::ARG_TARGET);
+                    ExitCode::Cli
+                })?
+                .clone();
+            if target.eq("-") {
+                ExportTarget::Stdout
+            } else {
+                let target_path = PathBuf::from_str(&target).map_err(|err| {
+                    eprintln!("Args Error: failed to convert {} to PathBuf: {}", target, err);
+                    ExitCode::Cli
+                })?;
+                ExportTarget::Path(target_path)
+            }
+        };
 
         fn parse_either(arg: String) -> Either<u64, String> {
             match arg.parse::<u64>() {
@@ -269,6 +278,7 @@ H256::from_str(&target[2..]).expect("default assume_valid_target for testnet mus
             .get_one::<String>(cli::ARG_TO)
             .cloned()
             .map(parse_either);
+
         Ok(ExportArgs {
             from,
             to,

ckb-bin/src/setup.rs

@@ -31,7 +31,6 @@ pub fn export(args: ExportArgs, async_handle: Handle) -> Result<(), ExitCode> {
     let from = args.from.map(parse_from_to_arg).transpose()?;
 
     let to = args.to.map(parse_from_to_arg).transpose()?;
-
     Export::new(shared, args.target, from, to)
         .execute()
         .map_err(|err| {

ckb-bin/src/subcommand/export.rs

@@ -8,14 +8,19 @@ use ckb_verification_traits::Switch;
 use std::path::PathBuf;
 use std::sync::Arc;
 
+pub enum ExportTarget {
+    Path(PathBuf),
+    Stdout,
+}
+
 /// Parsed command line arguments for `ckb export`.
 pub struct ExportArgs {
     /// Parsed `ckb.toml`.
     pub config: Box<CKBAppConfig>,
     /// Loaded consensus.
     pub consensus: Consensus,
     /// The target directory to save the exported file.
-    pub target: PathBuf,
+    pub target: ExportTarget,
     /// The range start block number or block hash.
     pub from: Option<Either<u64, String>>,
     /// The range end block number or block hash.

util/app-config/src/args.rs

@@ -15,8 +15,8 @@ pub use app_config::{
     AppConfig, CKBAppConfig, ChainConfig, LogConfig, MetricsConfig, MinerAppConfig,
 };
 pub use args::{
-    CustomizeSpec, DaemonArgs, ExportArgs, ImportArgs, InitArgs, MigrateArgs, MinerArgs,
-    PeerIDArgs, ReplayArgs, ResetDataArgs, RunArgs, StatsArgs,
+    CustomizeSpec, DaemonArgs, ExportArgs, ExportTarget, ImportArgs, InitArgs, MigrateArgs,
+    MinerArgs, PeerIDArgs, ReplayArgs, ResetDataArgs, RunArgs, StatsArgs,
 };
 
 pub use configs::*;

util/app-config/src/lib.rs

@@ -22,6 +22,7 @@ indicatif = { workspace = true, optional = true }
 rayon.workspace = true
 rayon-progress-bar.workspace = true
 itertools.workspace = true
+ckb-app-config.workspace = true
 
 
 [features]

util/instrument/Cargo.toml

@@ -1,3 +1,4 @@
+use ckb_app_config::ExportTarget;
 use ckb_jsonrpc_types::BlockView as JsonBlock;
 use ckb_jsonrpc_types::Either;
 use ckb_shared::Snapshot;
@@ -13,13 +14,12 @@ use std::error::Error;
 use std::fs;
 use std::io;
 use std::io::Write;
-use std::path::PathBuf;
 use std::sync::Arc;
 
 /// Export block from database to specify file.
 pub struct Export {
     /// export target path
-    pub target: PathBuf,
+    pub target: ExportTarget,
     /// CKB shared data.
     pub shared: Shared,
     /// the snapshot of the shared data
@@ -35,7 +35,7 @@ impl Export {
     /// Creates the export job.
     pub fn new(
         shared: Shared,
-        target: PathBuf,
+        target: ExportTarget,
         from: Option<Either<BlockNumber, H256>>,
         to: Option<Either<BlockNumber, H256>>,
     ) -> Self {
@@ -52,17 +52,18 @@ impl Export {
     /// export file name
     fn file_name(&self, from_number: u64, to_number: u64) -> Result<String, Box<dyn Error>> {
         Ok(format!(
-            "{}-{}-{}.{}",
+            "{}-{}-{}.jsonl",
             self.shared.consensus().id,
             from_number,
-            to_number,
-            "jsonl"
+            to_number
         ))
     }
 
     /// Executes the export job.
     pub fn execute(self) -> Result<(), Box<dyn Error>> {
-        fs::create_dir_all(&self.target)?;
+        if let ExportTarget::Path(ref path) = self.target {
+            fs::create_dir_all(path)?;
+        }
         self.write_to_json()
     }
 
@@ -89,16 +90,24 @@ impl Export {
             return Err(format!("Invalid range: from {} to {}", from_number, to_number).into());
         }
 
-        let file_name = self.file_name(from_number, to_number)?;
-        let file_path = self.target.join(file_name);
-        println!("Writing to {}", file_path.display());
-        let f = fs::OpenOptions::new()
-            .create_new(true)
-            .read(true)
-            .write(true)
-            .open(file_path)?;
+        let mut writer: Box<dyn Write> = {
+            match self.target {
+                ExportTarget::Path(ref path_buf) => {
+                    let file_name = self.file_name(from_number, to_number)?;
+                    let file_path = path_buf.join(file_name);
+                    println!("Writing to {}", file_path.display());
+                    let f = fs::OpenOptions::new()
+                        .create_new(true)
+                        .read(true)
+                        .write(true)
+                        .open(file_path)?;
+
+                    Box::new(io::BufWriter::new(f))
+                }
+                ExportTarget::Stdout => Box::new(std::io::stdout()),
+            }
+        };
 
-        let mut writer = io::BufWriter::new(f);
         let snapshot = self.shared.snapshot();
 
         #[cfg(feature = "progress_bar")]