Skip to content

Enhance ckb export/import subcommand with range and verifier selection #4924

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 16 commits into
base: develop
Choose a base branch
from
Open

Enhance ckb export/import subcommand with range and verifier selection #4924

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
8ca2ed0
chain-iter: Support create ChainIterator with a range
eval-exec Jul 7, 2025
2018565
app-config: Add from and to for ExportArg
eval-exec Jul 7, 2025
8c47791
instrument: implement export with range feature
eval-exec Jul 7, 2025
55ee95d
add --skip-script-verify and --skip-all-verify for ckb import
eval-exec Jul 10, 2025
96b7aa8
Import: merge progress_bar related code
eval-exec Jul 14, 2025
eec4388
ckb-replay: print profile stastics even when profile period is very s…
eval-exec Jul 16, 2025
d23aa05
cli-test: exoport filename suffix is jsonl
eval-exec Jul 16, 2025
7f2d838
export: reword log, add from-to range check
eval-exec Jul 16, 2025
56f74d0
Import: check if first block parent in db
eval-exec Jul 22, 2025
477f931
bats-test: Add bats-test for ckb import and export
eval-exec Jul 22, 2025
f14886a
Remove ckb-chain-iter from export
eval-exec Jul 28, 2025
46eea89
Support concurrent export by rayon
eval-exec Jul 28, 2025
c68fb4d
ckb-export: Support export to stdout, then can pipe using zstd tar, etc.
eval-exec Jul 30, 2025
7420eb7
ckb-import: Support import from STDIN
eval-exec Aug 7, 2025
b4ec1ae
test: Add bats test for import and export
eval-exec Aug 6, 2025
832e33c
import: Add --num-threads to control parallel threads number
eval-exec Aug 14, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion Cargo.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

158 changes: 79 additions & 79 deletions Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,87 +30,87 @@ console-subscriber = { workspace = true, optional = true }

[target.'cfg(all(not(target_env = "msvc"), not(target_os="macos")))'.dependencies]
tikv-jemallocator = { version = "0.5.0", features = [
"unprefixed_malloc_on_supported_platforms",
"unprefixed_malloc_on_supported_platforms",
] }

[dev-dependencies]

[workspace]
# To get a list sorted by dependencies, run devtools/ci/check-cyclic-dependencies.py
members = [
"util",
"util/occupied-capacity/core",
"util/channel",
"util/systemtime",
"util/fixed-hash/core",
"util/build-info",
"util/logger",
"util/logger-config",
"util/metrics-config",
"util/hash",
"util/rational",
"benches",
"db-schema",
"util/spawn",
"util/occupied-capacity/macros",
"util/fixed-hash/macros",
"util/logger-service",
"util/runtime",
"util/stop-handler",
"util/metrics",
"util/metrics-service",
"util/fixed-hash",
"util/occupied-capacity",
"util/crypto",
"util/constant",
"error",
"util/multisig",
"util/gen-types",
"util/migrate/migration-template",
"util/types",
"util/jsonrpc-types",
"freezer",
"resource",
"pow",
"util/dao/utils",
"traits",
"spec",
"util/fee-estimator",
"util/proposal-table",
"script",
"util/app-config",
"db",
"network",
"miner",
"notify",
"util/memory-tracker",
"db-migration",
"util/network-alert",
"store",
"util/indexer-sync",
"util/indexer",
"util/rich-indexer",
"util/chain-iter",
"util/dao",
"util/test-chain-utils",
"util/reward-calculator",
"util/snapshot",
"util/migrate",
"verification/traits",
"verification",
"verification/contextual",
"tx-pool",
"shared",
"block-filter",
"chain",
"sync",
"util/instrument",
"rpc",
"util/light-client-protocol-server",
"util/launcher",
"devtools/doc/rpc-gen",
"ckb-bin",
"test",
"util",
"util/occupied-capacity/core",
"util/channel",
"util/systemtime",
"util/fixed-hash/core",
"util/build-info",
"util/logger",
"util/logger-config",
"util/metrics-config",
"util/hash",
"util/rational",
"benches",
"db-schema",
"util/spawn",
"util/occupied-capacity/macros",
"util/fixed-hash/macros",
"util/logger-service",
"util/runtime",
"util/stop-handler",
"util/metrics",
"util/metrics-service",
"util/fixed-hash",
"util/occupied-capacity",
"util/crypto",
"util/constant",
"error",
"util/multisig",
"util/gen-types",
"util/migrate/migration-template",
"util/types",
"util/jsonrpc-types",
"freezer",
"resource",
"pow",
"util/dao/utils",
"traits",
"spec",
"util/fee-estimator",
"util/proposal-table",
"script",
"util/app-config",
"db",
"network",
"miner",
"notify",
"util/memory-tracker",
"db-migration",
"util/network-alert",
"store",
"util/indexer-sync",
"util/indexer",
"util/rich-indexer",
"util/chain-iter",
"util/dao",
"util/test-chain-utils",
"util/reward-calculator",
"util/snapshot",
"util/migrate",
"verification/traits",
"verification",
"verification/contextual",
"tx-pool",
"shared",
"block-filter",
"chain",
"sync",
"util/instrument",
"rpc",
"util/light-client-protocol-server",
"util/launcher",
"devtools/doc/rpc-gen",
"ckb-bin",
"test",
]

[workspace.package]
Expand Down Expand Up @@ -231,9 +231,9 @@ futures = "0.3"
futures-util = "0.3.21"
golomb-coded-set = "0.2.0"
governor = { version = "0.10", default-features = false, features = [
"std",
"jitter",
"quanta",
"std",
"jitter",
"quanta",
] }
hex = "0.4"
hickory-resolver = "0.24.2"
Expand All @@ -244,7 +244,7 @@ hyper-util = "0.1"
include_dir = "0.7"
includedir = "0.6.0"
includedir_codegen = "0.6.0"
indicatif = "0.16"
indicatif = { version = "0.16", features = ["with_rayon"] }
ipnetwork = "0.20"
is-terminal = "0.4.7"
is_sorted = "0.1.1"
Expand Down Expand Up @@ -278,7 +278,7 @@ rand = "0.8.5"
rand_distr = "0.4"
rayon = "1.0"
regex = "1.1.6"
reqwest = { version = "0.12" ,features = ["blocking", "json"]}
reqwest = { version = "0.12", features = ["blocking", "json"] }
rhai = "1.16.0"
rocksdb = { version = "=0.21.1", package = "ckb-rocksdb", default-features = false }
rustc-hash = "1.1"
Expand Down
84 changes: 67 additions & 17 deletions ckb-bin/src/cli.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,12 @@ pub const ARG_FORMAT: &str = "format";
pub const ARG_TARGET: &str = "target";
/// Command line argument `--source`.
pub const ARG_SOURCE: &str = "source";
/// Command line argument `--num-threads`.
pub const ARG_NUM_THREADS: &str = "num-threads";
/// Command line flag: `--skip-script-verify`.
pub const ARG_SKIP_SCRIPT_VERIFY: &str = "skip-script-verify";
/// Command line flag: `--skip-all-verify`.
pub const ARG_SKIP_ALL_VERIFY: &str = "skip-all-verify";
/// Command line argument `--data`.
pub const ARG_DATA: &str = "data";
/// Command line argument `--list-chains`.
Expand Down Expand Up @@ -375,26 +381,70 @@ fn replay() -> Command {
}

fn export() -> Command {
Command::new(CMD_EXPORT).about("Export CKB data").arg(
Arg::new(ARG_TARGET)
.short('t')
.long(ARG_TARGET)
.value_name("path")
.value_parser(clap::builder::PathBufValueParser::new())
.required(true)
.help("Specify the export target path"),
)
Command::new(CMD_EXPORT)
.about("Export CKB data")
.arg(
Arg::new(ARG_TARGET)
.short('t')
.long(ARG_TARGET)
.value_name("path")
.value_parser(clap::value_parser!(String))
.allow_hyphen_values(true)
.required(true)
.help("Specify the export target path (use '-' for stdout)"),
)
.arg(
Arg::new(ARG_FROM)
.long(ARG_FROM)
.value_name("from")
.required(false)
.help("Specify the from block number/hash for export"),
)
.arg(
Arg::new(ARG_TO)
.long(ARG_TO)
.value_name("to")
.required(false)
.help("Specify the to block number/hash for export"),
)
}

fn import() -> Command {
Command::new(CMD_IMPORT).about("Import CKB data").arg(
Arg::new(ARG_SOURCE)
.index(1)
.value_name("path")
.value_parser(clap::builder::PathBufValueParser::new())
.required(true)
.help("Specify the exported data path"),
)
Command::new(CMD_IMPORT)
.about("Import CKB data")
.arg(
Arg::new(ARG_NUM_THREADS)
.long(ARG_NUM_THREADS)
.value_name(ARG_NUM_THREADS)
.value_parser(clap::value_parser!(usize))
.required(false)
.help("Specify the number of threads to use for parallel processing. If not specified, it will use the number of logical CPUs available on the system."),
)
.arg(
Arg::new(ARG_SOURCE)
.index(1)
.value_name("path")
.value_parser(clap::value_parser!(String))
.allow_hyphen_values(true)
.required(true)
.help("Specify the exported data path, import JSONL-formatted blocks. If path is '-', read JSONL blocks from STDIN."),
)
.arg(
//
Arg::new(ARG_SKIP_SCRIPT_VERIFY)
.long(ARG_SKIP_SCRIPT_VERIFY)
.action(clap::ArgAction::SetTrue)
.conflicts_with(ARG_SKIP_ALL_VERIFY)
.help("Skip script verification during import"),
)
.arg(
//
Arg::new(ARG_SKIP_ALL_VERIFY)
.long(ARG_SKIP_ALL_VERIFY)
.conflicts_with(ARG_SKIP_SCRIPT_VERIFY)
.action(clap::ArgAction::SetTrue)
.help("Skip all verifications during import"),
)
}

fn migrate() -> Command {
Expand Down
Loading
Loading