diff --git a/Cargo.lock b/Cargo.lock index d674b3af8..98cde08da 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1591,12 +1591,6 @@ dependencies = [ "hex-conservative 0.2.1", ] -[[package]] -name = "bitflags" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aad18937a628ec6abcd26d1489012cc0e18c21798210f491af69ded9b881106d" - [[package]] name = "bitflags" version = "1.3.2" @@ -2597,16 +2591,14 @@ dependencies = [ [[package]] name = "config" -version = "0.12.0" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54ad70579325f1a38ea4c13412b82241c5900700a69785d73e2736bd65a33f86" +checksum = "68578f196d2a33ff61b27fae256c3164f65e36382648e30666dde05b8cc9dfdf" dependencies = [ - "async-trait", - "lazy_static", "nom", "pathdiff", "serde", - "toml 0.5.11", + "toml 0.8.19", ] [[package]] @@ -5001,15 +4993,6 @@ dependencies = [ "syn 2.0.90", ] -[[package]] -name = "fdlimit" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c4c9e43643f5a3be4ca5b67d26b98031ff9db6806c3440ae32e02e3ceac3f1b" -dependencies = [ - "libc", -] - [[package]] name = "fdlimit" version = "0.3.0" @@ -5085,8 +5068,6 @@ dependencies = [ [[package]] name = "filecoin-hashers" version = "13.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85413176cea16bfe171caafab023044820c0033b243b535b19116776ffd3f285" dependencies = [ "anyhow", "bellperson", @@ -5105,8 +5086,6 @@ dependencies = [ [[package]] name = "filecoin-proofs" version = "18.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "096b8b483f6ed5823150daf6cd22ee8e32b3dabcb4fd70dab70044e73bcab107" dependencies = [ "anyhow", "bellperson", @@ -5277,15 +5256,13 @@ dependencies = [ [[package]] name = "fr32" version = "11.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "627a3f3108ee3287759a45f6d5aafe48b3017509df9b677115f88266d61e0815" dependencies = [ "anyhow", "blstrs", "byte-slice-cast", "byteorder", "ff", - "thiserror 1.0.69", + "thiserror 2.0.8", ] [[package]] @@ -6627,12 +6604,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] -name = "hwloc" -version = "0.5.0" +name = "hwloc2" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2934f84993b8b4bcae9b6a4e5f0aca638462dda9c7b4f26a570241494f21e0f4" +checksum = "0f2a4b6f52a58293f5a69375e8bedfc53b15e439f751d9d20a62689632fe348e" dependencies = [ - "bitflags 0.7.0", + "bitflags 1.3.2", "errno 0.2.8", "kernel32-sys", "libc", @@ -17809,7 +17786,7 @@ dependencies = [ "array-bytes", "chrono", "clap", - "fdlimit 0.3.0", + "fdlimit", "futures", "itertools 0.11.0", "libp2p-identity", @@ -19627,8 +19604,6 @@ dependencies = [ [[package]] name = "sha2raw" version = "13.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73744f6a373edfc5624f452ec705a762e1154bb88c6699242bf37c56d99a6ebb" dependencies = [ "byteorder", "cpufeatures", @@ -19636,7 +19611,6 @@ dependencies = [ "fake-simd", "lazy_static", "opaque-debug 0.3.1", - "sha2-asm", ] [[package]] @@ -22509,8 +22483,6 @@ dependencies = [ [[package]] name = "storage-proofs-core" version = "18.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "390385ae6787ad5d4312f3b6f0462c9f6615d9a7863376f8636604e6e43f3d28" dependencies = [ "aes", "anyhow", @@ -22525,7 +22497,7 @@ dependencies = [ "fr32", "fs2", "generic-array 0.14.7", - "itertools 0.10.5", + "itertools 0.13.0", "lazy_static", "log", "memmap2 0.5.10", @@ -22538,14 +22510,12 @@ dependencies = [ "serde", "serde_json", "sha2 0.10.8", - "thiserror 1.0.69", + "thiserror 2.0.8", ] [[package]] name = "storage-proofs-porep" version = "18.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dd8c6bbeb00933edb41152fdabf28d2519d6a1b6ea176a793e3198a07bb9acd" dependencies = [ "anyhow", "bellperson", @@ -22556,14 +22526,14 @@ dependencies = [ "byteorder", "chacha20", "crossbeam", - "fdlimit 0.2.1", + "fdlimit", "ff", "filecoin-hashers", "fr32", "generic-array 0.14.7", "glob", "hex", - "hwloc", + "hwloc2", "lazy_static", "libc", "log", @@ -22587,8 +22557,6 @@ dependencies = [ [[package]] name = "storage-proofs-post" version = "18.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "779fbfe1455a57d2a7fd655ce1b2e97bf9f238b65c71919e92aa9df8f2ced8b1" dependencies = [ "anyhow", "bellperson", @@ -22607,8 +22575,6 @@ dependencies = [ [[package]] name = "storage-proofs-update" version = "18.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee4b391917dbcffa2297295971a02cc54aa19aa8b5378d539a435e78f8050153" dependencies = [ "anyhow", "bellperson", diff --git a/Cargo.toml b/Cargo.toml index af1684c16..9ecefe9cb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -156,13 +156,13 @@ storagext = { path = "storagext/lib" } bellpepper-core = "0.2" bellperson = "0.26" blstrs = "0.7" -filecoin-hashers = { version = "13.1.0", default-features = false } -filecoin-proofs = { version = "18.1.0", default-features = false } -fr32 = "11.1.0" +filecoin-hashers = { path = "../rust-fil-proofs/filecoin-hashers", default-features = false } +filecoin-proofs = { path = "../rust-fil-proofs/filecoin-proofs", default-features = false } +fr32 = { path = "../rust-fil-proofs/fr32" } generic-array = "1.1.0" -storage-proofs-core = { version = "18.1.0", default-features = false } -storage-proofs-porep = { version = "18.1.0", default-features = false } -storage-proofs-post = { version = "18.1.0", default-features = false } +storage-proofs-core = { path = "../rust-fil-proofs/storage-proofs-core", default-features = false } +storage-proofs-porep = { path = "../rust-fil-proofs/storage-proofs-porep", default-features = false } +storage-proofs-post = { path = "../rust-fil-proofs/storage-proofs-post", default-features = false } # Substrate pallet-transaction-payment-rpc = { git = "https://github.com/paritytech/polkadot-sdk", tag = "polkadot-stable2412" } diff --git a/benches/generate_porep_proof.sh b/benches/generate_porep_proof.sh new file mode 100755 index 000000000..4f2043a87 --- /dev/null +++ b/benches/generate_porep_proof.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +# Default number of sectors is 1 if not specified +NUM_SECTORS=${1:-1} +START_SECTOR=1 +END_SECTOR=$NUM_SECTORS + +SEAL_RANDOMNESS_HEIGHT=20 +PRE_COMMIT_BLOCK_NUMBER=30 + +# Fixed parameters +PROVIDER="//Charlie" +CAR_FILE="../examples/big_file_184k.car" +COMMP="baga6ea4seaqhx2sxpfc2f3k2o75m3acskihug7me3g4coyw6adjqnd6ioszfqay" +PARAMS_PATH="../1GiB.porep.params" + +echo "Processing $NUM_SECTORS sector(s) starting from $START_SECTOR" + +# Loop through sector IDs +for ((SECTOR_ID=START_SECTOR; SECTOR_ID<=END_SECTOR; SECTOR_ID++)) +do + echo "Processing Sector ID: $SECTOR_ID" + + # Clear cache before run + CACHE_FOLDER="/eiger/tmp/sector-$SECTOR_ID-cache" + rm -r "$CACHE_FOLDER" 2>/dev/null # Suppress error if folder doesn't exist + mkdir "$CACHE_FOLDER" + + # Run the proof generation + polka-storage-provider-client proofs porep \ + --sr25519-key "$PROVIDER" \ + --proof-parameters-path "$PARAMS_PATH" \ + --cache-directory "$CACHE_FOLDER" \ + --sector-id "$SECTOR_ID" \ + --seal-randomness-height "$SEAL_RANDOMNESS_HEIGHT" \ + --pre-commit-block-number "$PRE_COMMIT_BLOCK_NUMBER" \ + "$CAR_FILE" \ + "$COMMP" + + echo "Finished processing Sector ID: $SECTOR_ID" + echo "----------------------------------------" +done + +echo "All $NUM_SECTORS sector(s) processed!" \ No newline at end of file diff --git a/benches/track_memory.sh b/benches/track_memory.sh new file mode 100755 index 000000000..bcd9daf82 --- /dev/null +++ b/benches/track_memory.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +# Check if a command was provided +if [ -z "$1" ]; then + echo "Usage: $0 " + exit 1 +fi + +# Log file +LOGFILE="memory_usage_$(date +%s).txt" +echo "Logging to $LOGFILE" + +# Start the process in the background +"$@" & # Runs the provided command +PID=$! +echo "Started process with PID: $PID" + +# Header for the log +echo "Timestamp %MEM RSS(kB) VSZ(kB)" > "$LOGFILE" + +# Monitor memory usage until the process ends +while ps -p $PID > /dev/null; do + TIMESTAMP=$(date '+%s') + MEMORY=$(ps -p $PID -o %mem,rss,vsz --no-headers) + if [ -n "$MEMORY" ]; then + echo "$TIMESTAMP $MEMORY" >> "$LOGFILE" + fi + sleep 1 # Adjust polling interval as needed (e.g., 0.5 for 500ms) +done + +echo "Process $PID has finished." + +# Generate a simple report +echo "Memory Usage Report" >> "$LOGFILE" +echo "-----------------" >> "$LOGFILE" +echo "Max %MEM: $(awk 'NR>1{print $2}' "$LOGFILE" | sort -n | tail -1)" >> "$LOGFILE" +echo "Max RSS (kB): $(awk 'NR>1{print $3}' "$LOGFILE" | sort -n | tail -1)" >> "$LOGFILE" +echo "Max VSZ (kB): $(awk 'NR>1{print $4}' "$LOGFILE" | sort -n | tail -1)" >> "$LOGFILE" + +echo "Report generated in $LOGFILE" \ No newline at end of file diff --git a/docs/src/getting-started/building/source.md b/docs/src/getting-started/building/source.md index 66d771b83..56b240438 100644 --- a/docs/src/getting-started/building/source.md +++ b/docs/src/getting-started/building/source.md @@ -40,6 +40,7 @@ $ sudo apt install -y libhwloc-dev \ clang \ build-essential \ git \ + libssl-dev \ curl ``` diff --git a/docs/src/getting-started/local-testnet/index.md b/docs/src/getting-started/local-testnet/index.md index 7ac2498eb..393e94865 100644 --- a/docs/src/getting-started/local-testnet/index.md +++ b/docs/src/getting-started/local-testnet/index.md @@ -25,9 +25,9 @@ We support `Linux x86_64 (Ubuntu 24.04.1 LTS/glibc 2.39+)` and `MacOS ARM x64`. 1. Download the binaries: ```bash -wget https://github.com/paritytech/polkadot-sdk/releases/download/polkadot-stable2407-1/polkadot -wget https://github.com/paritytech/polkadot-sdk/releases/download/polkadot-stable2407-1/polkadot-prepare-worker -wget https://github.com/paritytech/polkadot-sdk/releases/download/polkadot-stable2407-1/polkadot-execute-worker +wget https://github.com/paritytech/polkadot-sdk/releases/download/polkadot-stable2412/polkadot +wget https://github.com/paritytech/polkadot-sdk/releases/download/polkadot-stable2412/polkadot-prepare-worker +wget https://github.com/paritytech/polkadot-sdk/releases/download/polkadot-stable2412/polkadot-execute-worker wget https://github.com/eigerco/polka-storage/releases/download/polka-storage-node-v0.0.0/polka-storage-node-linux-x86 -O polka-storage-node wget https://github.com/eigerco/polka-storage/releases/download/polka-storage-provider-client-v0.1.0/polka-storage-provider-client-linux-x86 -O polka-storage-provider-client wget https://github.com/eigerco/polka-storage/releases/download/polka-storage-provider-server-v0.1.0/polka-storage-provider-server-linux-x86 -O polka-storage-provider-server diff --git a/lib/polka-storage-proofs/src/post/mod.rs b/lib/polka-storage-proofs/src/post/mod.rs index 3a1dfade3..5f5c9d562 100644 --- a/lib/polka-storage-proofs/src/post/mod.rs +++ b/lib/polka-storage-proofs/src/post/mod.rs @@ -188,18 +188,27 @@ pub fn generate_window_post( /// * fn seal_to_config(seal_proof: RegisteredPoStProof) -> filecoin_proofs::PoStConfig { match seal_proof { - // https://github.com/filecoin-project/rust-fil-proofs/blob/266acc39a3ebd6f3d28c6ee335d78e2b7cea06bc/filecoin-proofs/src/constants.rs#L104 RegisteredPoStProof::StackedDRGWindow2KiBV1P1 | RegisteredPoStProof::StackedDRGWindow8MiBV1 | RegisteredPoStProof::StackedDRGWindow512MiBV1 - | RegisteredPoStProof::StackedDRGWindow1GiBV1 => filecoin_proofs::PoStConfig { + => filecoin_proofs::PoStConfig { sector_size: filecoin_proofs::SectorSize(seal_proof.sector_size().bytes()), challenge_count: filecoin_proofs::WINDOW_POST_CHALLENGE_COUNT, + // https://github.com/filecoin-project/rust-fil-proofs/blob/266acc39a3ebd6f3d28c6ee335d78e2b7cea06bc/filecoin-proofs/src/constants.rs#L104 sector_count: 2, typ: PoStType::Window, priority: true, api_version: storage_proofs_core::api_version::ApiVersion::V1_2_0, }, + RegisteredPoStProof::StackedDRGWindow1GiBV1 => filecoin_proofs::PoStConfig { + sector_size: filecoin_proofs::SectorSize(seal_proof.sector_size().bytes()), + challenge_count: filecoin_proofs::WINDOW_POST_CHALLENGE_COUNT, + // https://github.com/filecoin-project/rust-fil-proofs/blob/266acc39a3ebd6f3d28c6ee335d78e2b7cea06bc/filecoin-proofs/src/constants.rs#L104 + sector_count: 2349, + typ: PoStType::Window, + priority: true, + api_version: storage_proofs_core::api_version::ApiVersion::V1_2_0, + }, } } diff --git a/maat/generate_porep_proof.sh b/maat/generate_porep_proof.sh deleted file mode 100755 index 8d20c50f8..000000000 --- a/maat/generate_porep_proof.sh +++ /dev/null @@ -1,22 +0,0 @@ -# Clear cache before run -CACHE_FOLDER="/tmp/psp-cache" -rm -r "$CACHE_FOLDER" -mkdir "$CACHE_FOLDER" - -PROVIDER="//Charlie" -CAR_FILE="../examples/test-data-big.car" -SECTOR_CID="baga6ea4seaqbfhdvmk5qygevit25ztjwl7voyikb5k2fqcl2lsuefhaqtukuiii" -PARAMS_PATH="../2KiB.porep.params" -SECTOR_ID=1 -SEAL_RANDOMNESS_HEIGHT=20 -PRE_COMMIT_BLOCK_NUMBER=30 - -polka-storage-provider-client proofs porep \ ---sr25519-key "$PROVIDER" \ ---proof-parameters-path "$PARAMS_PATH" \ ---cache-directory "$CACHE_FOLDER" \ ---sector-id "$SECTOR_ID" \ ---seal-randomness-height "$SEAL_RANDOMNESS_HEIGHT" \ ---pre-commit-block-number "$PRE_COMMIT_BLOCK_NUMBER" \ -"$CAR_FILE" \ -"$SECTOR_CID" diff --git a/maat/src/lib.rs b/maat/src/lib.rs index 208f6c158..d85bb5441 100644 --- a/maat/src/lib.rs +++ b/maat/src/lib.rs @@ -111,6 +111,7 @@ pub fn local_testnet_config(temp_dir_path: &std::path::Path) -> NetworkConfig { .with_node(|node| node.polkadot_node("relay-1")) .with_node(|node| node.polkadot_node("relay-2")) }) + .with_global_settings(|c| c.with_base_dir("/eiger/tmp/zombienet")) .with_parachain(|parachain| { parachain .with_id(1000) diff --git a/maat/tests/real_world.rs b/maat/tests/real_world.rs index 737420e97..f6b780d1c 100644 --- a/maat/tests/real_world.rs +++ b/maat/tests/real_world.rs @@ -271,8 +271,8 @@ async fn real_world_use_case() { let cache_dir_path = temp_dir.path().join("cache_dir"); let sealed_sector_path = temp_dir.path().join("sealed_sector"); - let seal_proof = primitives::proofs::RegisteredSealProof::StackedDRG8MiBV1; - let post_proof = primitives::proofs::RegisteredPoStProof::StackedDRGWindow8MiBV1; + let seal_proof = primitives::proofs::RegisteredSealProof::StackedDRG1GiBV1; + let post_proof = primitives::proofs::RegisteredPoStProof::StackedDRGWindow1GiBV1; let parameters_cache_path = Path::new(&workspace_root).join("../target/"); let porep_parameters_path = @@ -307,11 +307,12 @@ async fn real_world_use_case() { } else { tracing::info!( "using PoSt cached params at path: {}", - porep_parameters_path.display() + post_parameters_path.display() ); } // We need to read it again, as Proof Generating machine requires it in this form and that's the API of bellperson. + // TODO: It's not ideal, as we're loading 50GB two times into the memory. let porep_mapped_parameters = porep::load_groth16_parameters(porep_parameters_path).unwrap(); let post_mapped_parameters = Arc::new(post::load_groth16_parameters(post_parameters_path).unwrap()); @@ -378,7 +379,7 @@ async fn real_world_use_case() { data_file_path.display(), *piece_size ); - let sector_end_block = 165; + let sector_end_block = 480; // Publish a storage deal let deal = DealProposal { @@ -387,7 +388,7 @@ async fn real_world_use_case() { client: alice_kp.account_id().clone(), provider: charlie_kp.account_id().clone(), label: "My lovely big data".to_string(), - start_block: 85, + start_block: 400, end_block: sector_end_block, storage_price_per_block: 300_000_000, provider_collateral: 12_500_000_000, diff --git a/pallets/proofs/src/porep/config.rs b/pallets/proofs/src/porep/config.rs index a5ed72529..60af2129f 100644 --- a/pallets/proofs/src/porep/config.rs +++ b/pallets/proofs/src/porep/config.rs @@ -105,7 +105,7 @@ fn partitions(seal_proof: RegisteredSealProof) -> usize { RegisteredSealProof::StackedDRG2KiBV1P1 => 1, RegisteredSealProof::StackedDRG8MiBV1 => 1, RegisteredSealProof::StackedDRG512MiBV1 => 1, - RegisteredSealProof::StackedDRG1GiBV1 => 1, + RegisteredSealProof::StackedDRG1GiBV1 => 10, } } @@ -116,7 +116,7 @@ fn minimum_challenges(seal_proof: RegisteredSealProof) -> usize { RegisteredSealProof::StackedDRG2KiBV1P1 => 2, RegisteredSealProof::StackedDRG8MiBV1 => 2, RegisteredSealProof::StackedDRG512MiBV1 => 2, - RegisteredSealProof::StackedDRG1GiBV1 => 2, + RegisteredSealProof::StackedDRG1GiBV1 => 176, } } diff --git a/primitives/src/proofs.rs b/primitives/src/proofs.rs index ec0f84615..442a1d1bd 100644 --- a/primitives/src/proofs.rs +++ b/primitives/src/proofs.rs @@ -109,7 +109,7 @@ impl RegisteredSealProof { RegisteredSealProof::StackedDRG2KiBV1P1 => 192, RegisteredSealProof::StackedDRG8MiBV1 => 192, RegisteredSealProof::StackedDRG512MiBV1 => 192, - RegisteredSealProof::StackedDRG1GiBV1 => 192, + RegisteredSealProof::StackedDRG1GiBV1 => 1920, } } @@ -211,7 +211,7 @@ impl RegisteredPoStProof { RegisteredPoStProof::StackedDRGWindow2KiBV1P1 => 2, RegisteredPoStProof::StackedDRGWindow8MiBV1 => 2, RegisteredPoStProof::StackedDRGWindow512MiBV1 => 2, - RegisteredPoStProof::StackedDRGWindow1GiBV1 => 2, + RegisteredPoStProof::StackedDRGWindow1GiBV1 => 2349, } } @@ -224,7 +224,7 @@ impl RegisteredPoStProof { RegisteredPoStProof::StackedDRGWindow2KiBV1P1 => 2, RegisteredPoStProof::StackedDRGWindow8MiBV1 => 2, RegisteredPoStProof::StackedDRGWindow512MiBV1 => 2, - RegisteredPoStProof::StackedDRGWindow1GiBV1 => 2, + RegisteredPoStProof::StackedDRGWindow1GiBV1 => 2349, } } diff --git a/rust-fil-proofs.config.toml b/rust-fil-proofs.config.toml new file mode 100644 index 000000000..bccad738a --- /dev/null +++ b/rust-fil-proofs.config.toml @@ -0,0 +1,33 @@ +# The location to store downloaded parameter files required for proofs. +parameter_cache = "/eiger/tmp/filecoin-proofs-parameters/" + +# The location to store the on-disk parents cache. +parent_cache = "/eiger/tmp/filecoin-parents" +# The max number of parent cache elements to have mapped in RAM at a time. +sdr_parents_cache_size = 2_048 + +# This enables the use of the GPU for column tree building. +use_gpu_column_builder = true +# If the GPU is used for column building, this is the batch size to send to the GPU at a time. +max_gpu_column_batch_size = 400_000 +# This is the batch size for writing out the column tree elements to disk after it's generated. +column_write_batch_size = 262_144 + +# This enables the use of the GPU for tree r last building. +use_gpu_tree_builder = true +# If the GPU is used for tree r last building, this is the batch size to send to the GPU at a time. +max_gpu_tree_batch_size = 700_000 + +# This setting affects tree_r_last (MerkleTree) generation and access +# and determines the size of the on disk tree caches. This value MUST +# NOT be changed after tree_r_last caches have been generated on your +# system, as any remaining will no longer be accessible. A tool +# exists called 'update_tree_r_last' that can rebuild cache files if +# it's required, but updating this setting is NOT recommended. +rows_to_discard = 2 + +# This value is defaulted to the number of cores available on your system. +#window_post_synthesis_num_cpus = 8 + +# This enables multicore SDR replication +use_multicore_sdr = true