From d52eb6273969bdb04ac775a7d2689c29a45343d9 Mon Sep 17 00:00:00 2001 From: Markuu-s Date: Mon, 17 Jan 2022 13:12:25 +0300 Subject: [PATCH 01/10] Add CommitBatcher. --- core/miner/storage_fsm/CMakeLists.txt | 1 + core/miner/storage_fsm/commit_batcher.hpp | 36 ++++++ .../storage_fsm/impl/commit_batcher_impl.cpp | 107 ++++++++++++++++++ .../storage_fsm/impl/commit_batcher_impl.hpp | 94 +++++++++++++++ 4 files changed, 238 insertions(+) create mode 100644 core/miner/storage_fsm/commit_batcher.hpp create mode 100644 core/miner/storage_fsm/impl/commit_batcher_impl.cpp create mode 100644 core/miner/storage_fsm/impl/commit_batcher_impl.hpp diff --git a/core/miner/storage_fsm/CMakeLists.txt b/core/miner/storage_fsm/CMakeLists.txt index fd8042d0b8..ac7375eb66 100644 --- a/core/miner/storage_fsm/CMakeLists.txt +++ b/core/miner/storage_fsm/CMakeLists.txt @@ -38,6 +38,7 @@ target_link_libraries(events add_library(batcher impl/precommit_batcher_impl.cpp + impl/commit_batcher_impl.cpp ) target_link_libraries(batcher api diff --git a/core/miner/storage_fsm/commit_batcher.hpp b/core/miner/storage_fsm/commit_batcher.hpp new file mode 100644 index 0000000000..fcd287694f --- /dev/null +++ b/core/miner/storage_fsm/commit_batcher.hpp @@ -0,0 +1,36 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once +#include "api/full_node/node_api.hpp" +#include "miner/storage_fsm/types.hpp" +#include "primitives/sector/sector.hpp" + +namespace fc::mining { + using CommitCallback = std::function &)>; + using api::FullNodeApi; + using primitives::sector::RegisteredSealProof; + using sector_storage::Proof; + using types::SectorInfo; + + struct AggregateInput { + Proof proof; + // Info info; + RegisteredSealProof spt; + }; + + class CommitBatcher { + public: + virtual ~CommitBatcher() = default; + + virtual outcome::result addCommit( + const SectorInfo §or_info, + const AggregateInput &aggregate_input, + const CommitCallback &callBack) = 0; + + virtual void forceSend() = 0; + }; + +} // namespace fc::mining diff --git a/core/miner/storage_fsm/impl/commit_batcher_impl.cpp b/core/miner/storage_fsm/impl/commit_batcher_impl.cpp new file mode 100644 index 0000000000..630d7c4b4c --- /dev/null +++ b/core/miner/storage_fsm/impl/commit_batcher_impl.cpp @@ -0,0 +1,107 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once +#include "commit_batcher_impl.hpp" +#include + +namespace fc::mining { + CommitBatcherImpl::CommitBatcherImpl( + const std::chrono::milliseconds &max_time, + const size_t &max_size_callback, + const std::shared_ptr &scheduler) + : max_delay_(max_time), max_size_callback_(max_size_callback) { + cutoff_start_ = std::chrono::system_clock::now(); + handle_ = + scheduler_->scheduleWithHandle([&]() { sendCallbacks(); }, max_delay_); + } + + outcome::result CommitBatcherImpl::addCommit( + const SectorInfo §or_info, + const AggregateInput &aggregate_input, + const CommitCallback &callback) { + const SectorNumber §or_number = sector_info.sector_number; + + /* + * TODO batch_storage_ and callbacks_ may union Done + */ + union_storage_.push(sector_number, aggregate_input, callback); + + // setPreCommitCutoff(head->epoch(), sector_info); TODO same precommit + + if (union_storage_.size() >= max_size_callback_) { + sendCallbacks(); + } + return outcome::success(); + } + + void CommitBatcherImpl::forceSend() {} + + void CommitBatcherImpl::sendCallbacks() { + UnionStorage temp_union_storage(std::move(union_storage_)); + const auto maybe_result = sendBatch(); + for (const auto &[key, pair_storage] : temp_union_storage) { + pair_storage.commit_callback(maybe_result); + } + + cutoff_start_ = std::chrono::system_clock::now(); + closest_cutoff_ = max_delay_; + + handle_.reschedule(max_delay_).value(); + } + + outcome::result CommitBatcherImpl::sendBatch() {} + + void CommitBatcherImpl::UnionStorage::push( + const SectorNumber §or_number, + const AggregateInput &aggregate_input, + const CommitCallback &commit_callback) { + push(sector_number, PairStorage(aggregate_input, commit_callback)); + } + + void CommitBatcherImpl::UnionStorage::push( + const SectorNumber §or_number, + const CommitCallback &commit_callback, + const AggregateInput &aggregate_input) { + push(sector_number, PairStorage(aggregate_input, commit_callback)); + } + void CommitBatcherImpl::UnionStorage::push(const SectorNumber §or_number, + const PairStorage &pair_storage) { + std::unique_lock locker(mutex_); + storage_[sector_number] = pair_storage; + } + + + + CommitBatcherImpl::UnionStorage::UnionStorage( + CommitBatcherImpl::UnionStorage &&union_storage1) { + std::unique_lock locker(mutex_); + storage_.insert(std::make_move_iterator(union_storage1.storage_.begin()), + std::make_move_iterator(union_storage1.storage_.end())); + } + size_t CommitBatcherImpl::UnionStorage::size() const { + return storage_.size(); + } + + CommitBatcherImpl::UnionStorage::PairStorage::PairStorage( + const AggregateInput &aggregate_input, + const CommitCallback &commit_callback) + : aggregate_input(aggregate_input), commit_callback(commit_callback) {} + + CommitBatcherImpl::UnionStorage::PairStorage::PairStorage( + const CommitCallback &commit_callback, + const AggregateInput &aggregate_input) + : aggregate_input(aggregate_input), commit_callback(commit_callback) {} + + std::map::iterator + CommitBatcherImpl::UnionStorage::begin() { + return storage_.begin(); + } + + std::map::iterator + CommitBatcherImpl::UnionStorage::end() { + return storage_.end(); + } +} // namespace fc::mining diff --git a/core/miner/storage_fsm/impl/commit_batcher_impl.hpp b/core/miner/storage_fsm/impl/commit_batcher_impl.hpp new file mode 100644 index 0000000000..d420a52d22 --- /dev/null +++ b/core/miner/storage_fsm/impl/commit_batcher_impl.hpp @@ -0,0 +1,94 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once +#include +#include "miner/storage_fsm/commit_batcher.hpp" + +namespace fc::mining { + + using libp2p::basic::Scheduler; + using primitives::SectorNumber; + + class CommitBatcherImpl : public CommitBatcher { + public: + + class UnionStorage { + public: + UnionStorage() = default; + UnionStorage(UnionStorage &&); + + UnionStorage(const UnionStorage &) = delete; + + ~UnionStorage() = default; + + struct PairStorage { + AggregateInput aggregate_input; + CommitCallback commit_callback; + + PairStorage(const AggregateInput &aggregate_input, + const CommitCallback &commit_callback); + + PairStorage(const CommitCallback &commit_callback, + const AggregateInput &aggregate_input); + }; + + void push(const SectorNumber §or_number, + const AggregateInput &aggregate_input, + const CommitCallback &commit_callback); + void push(const SectorNumber §or_number, + const CommitCallback &commit_callback, + const AggregateInput &aggregate_input); + void push(const SectorNumber §or_number, + const PairStorage &pair_storage); + + size_t size() const; + + std::map::iterator begin(); + std::map::iterator end(); + + private: + std::mutex mutex_; + + std::map storage_; + }; + + CommitBatcherImpl(const std::chrono::milliseconds &max_time, + const size_t &max_size_callback, + const std::shared_ptr &scheduler); + + /* + * 1) Должен уметь отправлять информацию за время, которую ему задали + * NEW 2) Должен уметь отправлять информацию за размер, которую ему задали + * (например 10 коммитов макс вместимость мы должны его отправлять) + * 3) После любой отправки время отправки батчера заводится снова ( + * скедлинг, хэндлер) + */ + outcome::result addCommit(const SectorInfo §or_info, + const AggregateInput &aggregate_input, + const CommitCallback &callBack); + + void forceSend() override; + + void setCommitCutoff(const ChainEpoch ¤t_epoch, + const SectorInfo §or_info); + + private: + std::shared_ptr scheduler_; + Scheduler::Handle handle_; + std::chrono::milliseconds max_delay_; + std::chrono::milliseconds closest_cutoff_; + std::chrono::system_clock::time_point cutoff_start_; + size_t max_size_callback_; + UnionStorage union_storage_; + std::shared_ptr api_; + + void sendCallbacks(); + + outcome::result sendBatch(); + + }; + +} // namespace fc::mining \ No newline at end of file From cf975e705352389e2f33dc3585457f244173e6ed Mon Sep 17 00:00:00 2001 From: Markuu-s Date: Mon, 17 Jan 2022 16:38:13 +0300 Subject: [PATCH 02/10] Added setCommitCuttof. --- .../storage_fsm/impl/commit_batcher_impl.cpp | 39 +++++++++++++++++-- .../storage_fsm/impl/commit_batcher_impl.hpp | 2 +- 2 files changed, 37 insertions(+), 4 deletions(-) diff --git a/core/miner/storage_fsm/impl/commit_batcher_impl.cpp b/core/miner/storage_fsm/impl/commit_batcher_impl.cpp index 630d7c4b4c..000db725c9 100644 --- a/core/miner/storage_fsm/impl/commit_batcher_impl.cpp +++ b/core/miner/storage_fsm/impl/commit_batcher_impl.cpp @@ -8,6 +8,8 @@ #include namespace fc::mining { + using vm::actor::builtin::types::miner::kChainFinality; + CommitBatcherImpl::CommitBatcherImpl( const std::chrono::milliseconds &max_time, const size_t &max_size_callback, @@ -25,11 +27,11 @@ namespace fc::mining { const SectorNumber §or_number = sector_info.sector_number; /* - * TODO batch_storage_ and callbacks_ may union Done + * TODO batch_storage_ and callbacks_ may union DONE */ union_storage_.push(sector_number, aggregate_input, callback); - // setPreCommitCutoff(head->epoch(), sector_info); TODO same precommit + // setPreCommitCutoff(head->epoch(), sector_info); TODO same precommit DONE if (union_storage_.size() >= max_size_callback_) { sendCallbacks(); @@ -44,7 +46,7 @@ namespace fc::mining { const auto maybe_result = sendBatch(); for (const auto &[key, pair_storage] : temp_union_storage) { pair_storage.commit_callback(maybe_result); - } + } // TODO does it work? because I didn`t iterator++ cutoff_start_ = std::chrono::system_clock::now(); closest_cutoff_ = max_delay_; @@ -54,6 +56,37 @@ namespace fc::mining { outcome::result CommitBatcherImpl::sendBatch() {} + void CommitBatcherImpl::setCommitCutoff(const ChainEpoch ¤t_epoch, + const SectorInfo §or_info) { + ChainEpoch cutoff_epoch = + sector_info.ticket_epoch + + static_cast(kEpochsInDay + kChainFinality); + ChainEpoch start_epoch{}; + for (const auto &piece : sector_info.pieces) { + if (!piece.deal_info) { + continue; + } + start_epoch = piece.deal_info->deal_schedule.start_epoch; + if (start_epoch < cutoff_epoch) { + cutoff_epoch = start_epoch; + } + } + if (cutoff_epoch <= current_epoch) { + forceSend(); + } else { + const auto temp_cutoff = std::chrono::milliseconds( + (cutoff_epoch - current_epoch) * kEpochDurationSeconds); + if ((closest_cutoff_ + - std::chrono::duration_cast( + std::chrono::system_clock::now() - cutoff_start_) + > temp_cutoff)) { + cutoff_start_ = std::chrono::system_clock::now(); + handle_.reschedule(max_delay_).value(); + closest_cutoff_ = temp_cutoff; + } + } + } + void CommitBatcherImpl::UnionStorage::push( const SectorNumber §or_number, const AggregateInput &aggregate_input, diff --git a/core/miner/storage_fsm/impl/commit_batcher_impl.hpp b/core/miner/storage_fsm/impl/commit_batcher_impl.hpp index d420a52d22..93541adc60 100644 --- a/core/miner/storage_fsm/impl/commit_batcher_impl.hpp +++ b/core/miner/storage_fsm/impl/commit_batcher_impl.hpp @@ -42,7 +42,7 @@ namespace fc::mining { const CommitCallback &commit_callback, const AggregateInput &aggregate_input); void push(const SectorNumber §or_number, - const PairStorage &pair_storage); + const PairStorage &pair_storage); // TODO make one push size_t size() const; From e7ad72768186db9203012643707cffb751821313 Mon Sep 17 00:00:00 2001 From: Markuu-s Date: Thu, 20 Jan 2022 02:13:38 +0300 Subject: [PATCH 03/10] Added sendBatch. Signed-off-by: Markuu-s --- core/api/full_node/node_api.hpp | 8 ++ core/miner/storage_fsm/commit_batcher.hpp | 1 + .../storage_fsm/impl/commit_batcher_impl.cpp | 127 +++++++++++++----- .../storage_fsm/impl/commit_batcher_impl.hpp | 40 +++--- core/miner/storage_fsm/types.hpp | 1 + 5 files changed, 125 insertions(+), 52 deletions(-) diff --git a/core/api/full_node/node_api.hpp b/core/api/full_node/node_api.hpp index 9a399949ee..289b914f2f 100644 --- a/core/api/full_node/node_api.hpp +++ b/core/api/full_node/node_api.hpp @@ -463,6 +463,13 @@ namespace fc::api { const Address &, const TokenAmount &) + API_METHOD(ChainBaseFee, + jwt::kWritePermission, + TokenAmount, + Address, + SectorNumber, + TipsetKey) + API_METHOD(MinerCreateBlock, jwt::kWritePermission, BlockWithCids, @@ -787,6 +794,7 @@ namespace fc::api { visitNet(a, f); visitWallet(a, f); f(a.BeaconGetEntry); + f(a.ChainBaseFee); f(a.ChainGetBlock); f(a.ChainGetBlockMessages); f(a.ChainGetGenesis); diff --git a/core/miner/storage_fsm/commit_batcher.hpp b/core/miner/storage_fsm/commit_batcher.hpp index fcd287694f..5d6c03ff7b 100644 --- a/core/miner/storage_fsm/commit_batcher.hpp +++ b/core/miner/storage_fsm/commit_batcher.hpp @@ -21,6 +21,7 @@ namespace fc::mining { RegisteredSealProof spt; }; + class CommitBatcher { public: virtual ~CommitBatcher() = default; diff --git a/core/miner/storage_fsm/impl/commit_batcher_impl.cpp b/core/miner/storage_fsm/impl/commit_batcher_impl.cpp index 000db725c9..3793efe4f2 100644 --- a/core/miner/storage_fsm/impl/commit_batcher_impl.cpp +++ b/core/miner/storage_fsm/impl/commit_batcher_impl.cpp @@ -6,9 +6,17 @@ #pragma once #include "commit_batcher_impl.hpp" #include +#include "vm/actor/builtin/v5/miner/miner_actor.hpp" namespace fc::mining { using vm::actor::builtin::types::miner::kChainFinality; + using PairStorage = CommitBatcherImpl::UnionStorage::PairStorage; + using fc::primitives::ActorId; + using primitives::BigInt; + using primitives::sector::AggregateSealVerifyInfo; + using vm::actor::builtin::types::miner::SectorPreCommitOnChainInfo; + using vm::actor::builtin::v5::miner::ProveCommitAggregate; + namespace vm::actor::builtin::types::miner::SectorPreCommitOnChainInfo; CommitBatcherImpl::CommitBatcherImpl( const std::chrono::milliseconds &max_time, @@ -25,28 +33,27 @@ namespace fc::mining { const AggregateInput &aggregate_input, const CommitCallback &callback) { const SectorNumber §or_number = sector_info.sector_number; + OUTCOME_TRY(head, api_->ChainHead()); - /* - * TODO batch_storage_ and callbacks_ may union DONE - */ - union_storage_.push(sector_number, aggregate_input, callback); - - // setPreCommitCutoff(head->epoch(), sector_info); TODO same precommit DONE + union_storage_.push(sector_number, PairStorage(aggregate_input, callback)); if (union_storage_.size() >= max_size_callback_) { sendCallbacks(); } + // Вынести мьютексы + setCommitCutoff(head->epoch(), sector_info); + return outcome::success(); } void CommitBatcherImpl::forceSend() {} void CommitBatcherImpl::sendCallbacks() { - UnionStorage temp_union_storage(std::move(union_storage_)); - const auto maybe_result = sendBatch(); - for (const auto &[key, pair_storage] : temp_union_storage) { + UnionStorage union_storage_for_send_(std::move(union_storage_)); + const auto maybe_result = sendBatch(union_storage_for_send_); + for (const auto &[key, pair_storage] : union_storage_for_send_) { pair_storage.commit_callback(maybe_result); - } // TODO does it work? because I didn`t iterator++ + } cutoff_start_ = std::chrono::system_clock::now(); closest_cutoff_ = max_delay_; @@ -54,7 +61,63 @@ namespace fc::mining { handle_.reschedule(max_delay_).value(); } - outcome::result CommitBatcherImpl::sendBatch() {} + outcome::result CommitBatcherImpl::sendBatch( + UnionStorage &union_storage_for_send) { + if (not union_storage_for_send.size()) { + cutoff_start_ = std::chrono::system_clock::now(); + return ERROR_TEXT("Empty Batcher"); + } + OUTCOME_TRY(head, api_->ChainHead()); + + // TODO ? + const size_t total = union_storage_for_send.size(); + + ProveCommitAggregate::Params params; + + std::vector> proofs; + proofs.reserve(total); + + BigInt collateral = 0; + + for (const auto &[sector_number, pair_storage] : union_storage_for_send) { + + + TokenAmount sc = getSectorCollateral(head, sector_number, *head.get()); + collateral = collateral + sc; + + params.sectors.insert(sector_number); + } + + + + for (const auto &[sector_number, pair_storage] : union_storage_for_send) { + proofs.push_back( + pair_storage.aggregate_input.proof); + } + + const ActorId mid = miner_address_.getId(); + // TODO maybe long (AggregateSealProofs) + params.proof = proof_->AggregateSealProofs(); // OUTCOME_TRY + + // TODO CBOR::ENCODE params + + // BigDiv usage вместо /(обычное деление) + + const TokenAmount max_fee = + fee_config_->max_commit_batch_gas_fee.FeeForSector(proofs.size()); + + /* + * API_METHOD(StateMinerInfo, + * jwt::kReadPermission, + * MinerInfo, + * const Address &, + * const TipsetKey &) + */ + // OTCOME_TRY(mi, api_->StateMinerInfo()); + + OUTCOME_TRY(bf, api_->ChainBaseFee(head)); + OUTCOME_TRY(nv, api_->StateNetworkVersion(/*NetworkVersion*/, head)); + } void CommitBatcherImpl::setCommitCutoff(const ChainEpoch ¤t_epoch, const SectorInfo §or_info) { @@ -87,34 +150,38 @@ namespace fc::mining { } } - void CommitBatcherImpl::UnionStorage::push( + TokenAmount CommitBatcherImpl::getSectorCollateral( + std::shared_ptr &head, const SectorNumber §or_number, - const AggregateInput &aggregate_input, - const CommitCallback &commit_callback) { - push(sector_number, PairStorage(aggregate_input, commit_callback)); + const TipsetKey &tip_set_key) { + OUTCOME_TRY(pci, + api_->StateSectorPreCommitInfo( + miner_address_, sector_number, tip_set_key)); + OUTCOME_TRY(collateral, + api_->StateMinerInitialPledgeCollateral( + miner_address_, head->key, pci.info, tip_set_key)); + + collateral = collateral + pci.PreCommitDeposit; + collateral = max(0, collateral); + + return collateral; } - void CommitBatcherImpl::UnionStorage::push( - const SectorNumber §or_number, - const CommitCallback &commit_callback, - const AggregateInput &aggregate_input) { - push(sector_number, PairStorage(aggregate_input, commit_callback)); - } void CommitBatcherImpl::UnionStorage::push(const SectorNumber §or_number, const PairStorage &pair_storage) { std::unique_lock locker(mutex_); storage_[sector_number] = pair_storage; } - - CommitBatcherImpl::UnionStorage::UnionStorage( CommitBatcherImpl::UnionStorage &&union_storage1) { - std::unique_lock locker(mutex_); + std::unique_lock locker(union_storage1.mutex_); storage_.insert(std::make_move_iterator(union_storage1.storage_.begin()), std::make_move_iterator(union_storage1.storage_.end())); } - size_t CommitBatcherImpl::UnionStorage::size() const { + + size_t CommitBatcherImpl::UnionStorage::size() { + std::unique_lock locker(mutex_); return storage_.size(); } @@ -123,11 +190,6 @@ namespace fc::mining { const CommitCallback &commit_callback) : aggregate_input(aggregate_input), commit_callback(commit_callback) {} - CommitBatcherImpl::UnionStorage::PairStorage::PairStorage( - const CommitCallback &commit_callback, - const AggregateInput &aggregate_input) - : aggregate_input(aggregate_input), commit_callback(commit_callback) {} - std::map::iterator CommitBatcherImpl::UnionStorage::begin() { return storage_.begin(); @@ -137,4 +199,9 @@ namespace fc::mining { CommitBatcherImpl::UnionStorage::end() { return storage_.end(); } + + PairStorage CommitBatcherImpl::UnionStorage::get(const int index) { + std::unique_lock locker(mutex_); + return storage_[index]; + } } // namespace fc::mining diff --git a/core/miner/storage_fsm/impl/commit_batcher_impl.hpp b/core/miner/storage_fsm/impl/commit_batcher_impl.hpp index 93541adc60..01b8a90592 100644 --- a/core/miner/storage_fsm/impl/commit_batcher_impl.hpp +++ b/core/miner/storage_fsm/impl/commit_batcher_impl.hpp @@ -11,10 +11,14 @@ namespace fc::mining { using libp2p::basic::Scheduler; using primitives::SectorNumber; + using primitives::address::Address; + using primitives::tipset::Tipset; + using primitives::tipset::TipsetKey; + using types::FeeConfig; + using proofs::ProofEngine; class CommitBatcherImpl : public CommitBatcher { public: - class UnionStorage { public: UnionStorage() = default; @@ -30,21 +34,14 @@ namespace fc::mining { PairStorage(const AggregateInput &aggregate_input, const CommitCallback &commit_callback); - - PairStorage(const CommitCallback &commit_callback, - const AggregateInput &aggregate_input); }; void push(const SectorNumber §or_number, - const AggregateInput &aggregate_input, - const CommitCallback &commit_callback); - void push(const SectorNumber §or_number, - const CommitCallback &commit_callback, - const AggregateInput &aggregate_input); - void push(const SectorNumber §or_number, - const PairStorage &pair_storage); // TODO make one push + const PairStorage &pair_storage); + + size_t size(); - size_t size() const; + PairStorage get(const int index); std::map::iterator begin(); std::map::iterator end(); @@ -53,22 +50,15 @@ namespace fc::mining { std::mutex mutex_; std::map storage_; - }; + }; // TODO вынести PairStorage и удалить UnionStorage. storage_; CommitBatcherImpl(const std::chrono::milliseconds &max_time, const size_t &max_size_callback, const std::shared_ptr &scheduler); - /* - * 1) Должен уметь отправлять информацию за время, которую ему задали - * NEW 2) Должен уметь отправлять информацию за размер, которую ему задали - * (например 10 коммитов макс вместимость мы должны его отправлять) - * 3) После любой отправки время отправки батчера заводится снова ( - * скедлинг, хэндлер) - */ outcome::result addCommit(const SectorInfo §or_info, const AggregateInput &aggregate_input, - const CommitCallback &callBack); + const CommitCallback &callBack) override; void forceSend() override; @@ -84,11 +74,17 @@ namespace fc::mining { size_t max_size_callback_; UnionStorage union_storage_; std::shared_ptr api_; + Address miner_address_; + std::shared_ptr fee_config_; + std::shared_ptr proof_; void sendCallbacks(); - outcome::result sendBatch(); + outcome::result sendBatch(UnionStorage &union_storage_for_send); + TokenAmount getSectorCollateral(std::shared_ptr &head, + const SectorNumber §or_number, + const TipsetKey &tip_set_key); }; } // namespace fc::mining \ No newline at end of file diff --git a/core/miner/storage_fsm/types.hpp b/core/miner/storage_fsm/types.hpp index c93cb2aad0..0141050562 100644 --- a/core/miner/storage_fsm/types.hpp +++ b/core/miner/storage_fsm/types.hpp @@ -222,6 +222,7 @@ namespace fc::mining::types { // maxBatchFee = maxBase + maxPerSector * nSectors BatchConfing max_precommit_batch_gas_fee; + BatchConfing max_commit_batch_gas_fee; // TODO (Ruslan Gilvanov): TokenAmount max_terminate_gas_fee, // max_window_poSt_gas_fee, max_publish_deals_fee, From 5cb34220579cb922dd6571341eed4d3ffb2f3bf5 Mon Sep 17 00:00:00 2001 From: Markuu-s Date: Thu, 20 Jan 2022 16:54:24 +0300 Subject: [PATCH 04/10] Change unionStorage. Signed-off-by: Markuu-s --- core/miner/storage_fsm/commit_batcher.hpp | 1 - .../storage_fsm/impl/commit_batcher_impl.cpp | 91 ++++++------------- .../storage_fsm/impl/commit_batcher_impl.hpp | 49 +++------- 3 files changed, 39 insertions(+), 102 deletions(-) diff --git a/core/miner/storage_fsm/commit_batcher.hpp b/core/miner/storage_fsm/commit_batcher.hpp index 5d6c03ff7b..fcd287694f 100644 --- a/core/miner/storage_fsm/commit_batcher.hpp +++ b/core/miner/storage_fsm/commit_batcher.hpp @@ -21,7 +21,6 @@ namespace fc::mining { RegisteredSealProof spt; }; - class CommitBatcher { public: virtual ~CommitBatcher() = default; diff --git a/core/miner/storage_fsm/impl/commit_batcher_impl.cpp b/core/miner/storage_fsm/impl/commit_batcher_impl.cpp index 3793efe4f2..4899e50a21 100644 --- a/core/miner/storage_fsm/impl/commit_batcher_impl.cpp +++ b/core/miner/storage_fsm/impl/commit_batcher_impl.cpp @@ -9,14 +9,12 @@ #include "vm/actor/builtin/v5/miner/miner_actor.hpp" namespace fc::mining { - using vm::actor::builtin::types::miner::kChainFinality; - using PairStorage = CommitBatcherImpl::UnionStorage::PairStorage; using fc::primitives::ActorId; using primitives::BigInt; using primitives::sector::AggregateSealVerifyInfo; + using vm::actor::builtin::types::miner::kChainFinality; using vm::actor::builtin::types::miner::SectorPreCommitOnChainInfo; using vm::actor::builtin::v5::miner::ProveCommitAggregate; - namespace vm::actor::builtin::types::miner::SectorPreCommitOnChainInfo; CommitBatcherImpl::CommitBatcherImpl( const std::chrono::milliseconds &max_time, @@ -32,10 +30,12 @@ namespace fc::mining { const SectorInfo §or_info, const AggregateInput &aggregate_input, const CommitCallback &callback) { + std::unique_lock locker(mutex_storage_); + const SectorNumber §or_number = sector_info.sector_number; OUTCOME_TRY(head, api_->ChainHead()); - union_storage_.push(sector_number, PairStorage(aggregate_input, callback)); + union_storage_[sector_number] = PairStorage{aggregate_input, callback}; if (union_storage_.size() >= max_size_callback_) { sendCallbacks(); @@ -49,7 +49,12 @@ namespace fc::mining { void CommitBatcherImpl::forceSend() {} void CommitBatcherImpl::sendCallbacks() { - UnionStorage union_storage_for_send_(std::move(union_storage_)); + std::unique_lock locker(mutex_storage_); + MapPairStorage union_storage_for_send_( + std::make_move_iterator(union_storage_.begin()), + std::make_move_iterator(union_storage_.end())); + locker.unlock(); + const auto maybe_result = sendBatch(union_storage_for_send_); for (const auto &[key, pair_storage] : union_storage_for_send_) { pair_storage.commit_callback(maybe_result); @@ -62,8 +67,8 @@ namespace fc::mining { } outcome::result CommitBatcherImpl::sendBatch( - UnionStorage &union_storage_for_send) { - if (not union_storage_for_send.size()) { + MapPairStorage &union_storage_for_send) { + if (union_storage_for_send.empty()) { cutoff_start_ = std::chrono::system_clock::now(); return ERROR_TEXT("Empty Batcher"); } @@ -80,26 +85,24 @@ namespace fc::mining { BigInt collateral = 0; for (const auto &[sector_number, pair_storage] : union_storage_for_send) { - - - TokenAmount sc = getSectorCollateral(head, sector_number, *head.get()); + OUTCOME_TRY(sc, getSectorCollateral(sector_number, head->key)); collateral = collateral + sc; params.sectors.insert(sector_number); } - - for (const auto &[sector_number, pair_storage] : union_storage_for_send) { - proofs.push_back( - pair_storage.aggregate_input.proof); + proofs.push_back(pair_storage.aggregate_input.proof); } const ActorId mid = miner_address_.getId(); // TODO maybe long (AggregateSealProofs) - params.proof = proof_->AggregateSealProofs(); // OUTCOME_TRY - // TODO CBOR::ENCODE params + // TODO params.proof = proof_->AggregateSealProofs(); // OUTCOME_TRY + OUTCOME_TRY(a, proof_->AggregateSealProofs()); + + auto enc = codec::cbor::encode(params); + OUTCOME_TRY(mi, api_->StateMinerInfo(miner_address_, head->key)); // BigDiv usage вместо /(обычное деление) @@ -115,8 +118,8 @@ namespace fc::mining { */ // OTCOME_TRY(mi, api_->StateMinerInfo()); - OUTCOME_TRY(bf, api_->ChainBaseFee(head)); - OUTCOME_TRY(nv, api_->StateNetworkVersion(/*NetworkVersion*/, head)); + // TODO OUTCOME_TRY(bf, api_->ChainBaseFee(head)); + OUTCOME_TRY(nv, api_->StateNetworkVersion(head->key)); } void CommitBatcherImpl::setCommitCutoff(const ChainEpoch ¤t_epoch, @@ -124,7 +127,7 @@ namespace fc::mining { ChainEpoch cutoff_epoch = sector_info.ticket_epoch + static_cast(kEpochsInDay + kChainFinality); - ChainEpoch start_epoch{}; + ChainEpoch start_epoch; for (const auto &piece : sector_info.pieces) { if (!piece.deal_info) { continue; @@ -150,58 +153,20 @@ namespace fc::mining { } } - TokenAmount CommitBatcherImpl::getSectorCollateral( - std::shared_ptr &head, - const SectorNumber §or_number, - const TipsetKey &tip_set_key) { + outcome::result CommitBatcherImpl::getSectorCollateral( + const SectorNumber §or_number, const TipsetKey &tip_set_key) { OUTCOME_TRY(pci, api_->StateSectorPreCommitInfo( miner_address_, sector_number, tip_set_key)); + OUTCOME_TRY(collateral, api_->StateMinerInitialPledgeCollateral( - miner_address_, head->key, pci.info, tip_set_key)); + miner_address_, pci.info, tip_set_key)); - collateral = collateral + pci.PreCommitDeposit; - collateral = max(0, collateral); + collateral = collateral + pci.precommit_deposit; + collateral = std::max(BigInt(0), collateral); return collateral; } - void CommitBatcherImpl::UnionStorage::push(const SectorNumber §or_number, - const PairStorage &pair_storage) { - std::unique_lock locker(mutex_); - storage_[sector_number] = pair_storage; - } - - CommitBatcherImpl::UnionStorage::UnionStorage( - CommitBatcherImpl::UnionStorage &&union_storage1) { - std::unique_lock locker(union_storage1.mutex_); - storage_.insert(std::make_move_iterator(union_storage1.storage_.begin()), - std::make_move_iterator(union_storage1.storage_.end())); - } - - size_t CommitBatcherImpl::UnionStorage::size() { - std::unique_lock locker(mutex_); - return storage_.size(); - } - - CommitBatcherImpl::UnionStorage::PairStorage::PairStorage( - const AggregateInput &aggregate_input, - const CommitCallback &commit_callback) - : aggregate_input(aggregate_input), commit_callback(commit_callback) {} - - std::map::iterator - CommitBatcherImpl::UnionStorage::begin() { - return storage_.begin(); - } - - std::map::iterator - CommitBatcherImpl::UnionStorage::end() { - return storage_.end(); - } - - PairStorage CommitBatcherImpl::UnionStorage::get(const int index) { - std::unique_lock locker(mutex_); - return storage_[index]; - } } // namespace fc::mining diff --git a/core/miner/storage_fsm/impl/commit_batcher_impl.hpp b/core/miner/storage_fsm/impl/commit_batcher_impl.hpp index 01b8a90592..7c2ab4f0c3 100644 --- a/core/miner/storage_fsm/impl/commit_batcher_impl.hpp +++ b/core/miner/storage_fsm/impl/commit_batcher_impl.hpp @@ -14,43 +14,16 @@ namespace fc::mining { using primitives::address::Address; using primitives::tipset::Tipset; using primitives::tipset::TipsetKey; - using types::FeeConfig; using proofs::ProofEngine; + using types::FeeConfig; class CommitBatcherImpl : public CommitBatcher { public: - class UnionStorage { - public: - UnionStorage() = default; - UnionStorage(UnionStorage &&); - - UnionStorage(const UnionStorage &) = delete; - - ~UnionStorage() = default; - - struct PairStorage { - AggregateInput aggregate_input; - CommitCallback commit_callback; - - PairStorage(const AggregateInput &aggregate_input, - const CommitCallback &commit_callback); - }; - - void push(const SectorNumber §or_number, - const PairStorage &pair_storage); - - size_t size(); - - PairStorage get(const int index); - - std::map::iterator begin(); - std::map::iterator end(); - - private: - std::mutex mutex_; - - std::map storage_; - }; // TODO вынести PairStorage и удалить UnionStorage. storage_; + struct PairStorage { + AggregateInput aggregate_input; + CommitCallback commit_callback; + }; + typedef std::map MapPairStorage; CommitBatcherImpl(const std::chrono::milliseconds &max_time, const size_t &max_size_callback, @@ -72,19 +45,19 @@ namespace fc::mining { std::chrono::milliseconds closest_cutoff_; std::chrono::system_clock::time_point cutoff_start_; size_t max_size_callback_; - UnionStorage union_storage_; + MapPairStorage union_storage_; std::shared_ptr api_; Address miner_address_; std::shared_ptr fee_config_; std::shared_ptr proof_; + std::mutex mutex_storage_; void sendCallbacks(); - outcome::result sendBatch(UnionStorage &union_storage_for_send); + outcome::result sendBatch(MapPairStorage &union_storage_for_send); - TokenAmount getSectorCollateral(std::shared_ptr &head, - const SectorNumber §or_number, - const TipsetKey &tip_set_key); + outcome::result getSectorCollateral( + const SectorNumber §or_number, const TipsetKey &tip_set_key); }; } // namespace fc::mining \ No newline at end of file From 2abda47ee4b2aa30ee02f8f7e6072b9ef9e36bd5 Mon Sep 17 00:00:00 2001 From: Markuu-s Date: Mon, 24 Jan 2022 20:09:08 +0300 Subject: [PATCH 05/10] Need save change. Signed-off-by: Markuu-s --- core/miner/storage_fsm/commit_batcher.hpp | 3 +- .../storage_fsm/impl/commit_batcher_impl.cpp | 77 ++++++++++++++----- .../storage_fsm/impl/commit_batcher_impl.hpp | 15 +++- core/vm/actor/builtin/v6/monies.hpp | 26 +++++++ 4 files changed, 100 insertions(+), 21 deletions(-) create mode 100644 core/vm/actor/builtin/v6/monies.hpp diff --git a/core/miner/storage_fsm/commit_batcher.hpp b/core/miner/storage_fsm/commit_batcher.hpp index fcd287694f..f56018d7cc 100644 --- a/core/miner/storage_fsm/commit_batcher.hpp +++ b/core/miner/storage_fsm/commit_batcher.hpp @@ -11,13 +11,14 @@ namespace fc::mining { using CommitCallback = std::function &)>; using api::FullNodeApi; + using primitives::sector::AggregateSealVerifyInfo; using primitives::sector::RegisteredSealProof; using sector_storage::Proof; using types::SectorInfo; struct AggregateInput { Proof proof; - // Info info; + AggregateSealVerifyInfo info; RegisteredSealProof spt; }; diff --git a/core/miner/storage_fsm/impl/commit_batcher_impl.cpp b/core/miner/storage_fsm/impl/commit_batcher_impl.cpp index 4899e50a21..08d632f04d 100644 --- a/core/miner/storage_fsm/impl/commit_batcher_impl.cpp +++ b/core/miner/storage_fsm/impl/commit_batcher_impl.cpp @@ -7,14 +7,22 @@ #include "commit_batcher_impl.hpp" #include #include "vm/actor/builtin/v5/miner/miner_actor.hpp" +#include "vm/actor/builtin/v6/monies.hpp" namespace fc::mining { - using fc::primitives::ActorId; - using primitives::BigInt; + using api::kPushNoSpec; + using fc::BytesIn; + using fc::proofs::ProofEngine; + using primitives::ActorId; + using primitives::go::bigdiv; using primitives::sector::AggregateSealVerifyInfo; + using primitives::sector::AggregateSealVerifyProofAndInfos; + using primitives::tipset::TipsetCPtr; + using vm::actor::MethodParams; using vm::actor::builtin::types::miner::kChainFinality; using vm::actor::builtin::types::miner::SectorPreCommitOnChainInfo; using vm::actor::builtin::v5::miner::ProveCommitAggregate; + using vm::actor::builtin::v6::miner::AggregateProveCommitNetworkFee; CommitBatcherImpl::CommitBatcherImpl( const std::chrono::milliseconds &max_time, @@ -74,14 +82,16 @@ namespace fc::mining { } OUTCOME_TRY(head, api_->ChainHead()); - // TODO ? const size_t total = union_storage_for_send.size(); ProveCommitAggregate::Params params; - std::vector> proofs; + std::vector proofs; proofs.reserve(total); + std::vector infos; + infos.reserve(total); + BigInt collateral = 0; for (const auto &[sector_number, pair_storage] : union_storage_for_send) { @@ -89,6 +99,7 @@ namespace fc::mining { collateral = collateral + sc; params.sectors.insert(sector_number); + infos.push_back(pair_storage.aggregate_input.info); } for (const auto &[sector_number, pair_storage] : union_storage_for_send) { @@ -98,28 +109,56 @@ namespace fc::mining { const ActorId mid = miner_address_.getId(); // TODO maybe long (AggregateSealProofs) - // TODO params.proof = proof_->AggregateSealProofs(); // OUTCOME_TRY - OUTCOME_TRY(a, proof_->AggregateSealProofs()); - - auto enc = codec::cbor::encode(params); + AggregateSealVerifyProofAndInfos aggregate_seal = + AggregateSealVerifyProofAndInfos{ + .miner = mid, + .seal_proof = + union_storage_for_send[infos[0].number].aggregate_input.spt, + .aggregate_proof = arp_, + .proof = proofs[infos[0].number], // TODO is it correct? + .infos = infos}; + + OUTCOME_TRY(proof_->aggregateSealProofs(aggregate_seal, proofs)); + // need: std::vector> + // proofs: std::vector> + // proof: std::vector + // BytesIn: gsl::span; + + // proofs: std::vector> + auto b = gsl::make_span(proofs); + params.proof = aggregate_seal.proof; + OUTCOME_TRY(enc, codec::cbor::encode(params)); OUTCOME_TRY(mi, api_->StateMinerInfo(miner_address_, head->key)); - // BigDiv usage вместо /(обычное деление) - const TokenAmount max_fee = fee_config_->max_commit_batch_gas_fee.FeeForSector(proofs.size()); - /* - * API_METHOD(StateMinerInfo, - * jwt::kReadPermission, - * MinerInfo, - * const Address &, - * const TipsetKey &) - */ - // OTCOME_TRY(mi, api_->StateMinerInfo()); + OUTCOME_TRY(ts, api_->ChainGetTipSet(head->key)); + const BigInt bf = ts->blks[0].parent_base_fee; - // TODO OUTCOME_TRY(bf, api_->ChainBaseFee(head)); OUTCOME_TRY(nv, api_->StateNetworkVersion(head->key)); + + TokenAmount agg_fee_raw = AggregateProveCommitNetworkFee(infos.size(), bf); + + TokenAmount agg_fee = bigdiv(agg_fee_raw * agg_fee_num_, agg_fee_den_); + TokenAmount need_funds = collateral + agg_fee; + TokenAmount good_funds = max_fee + need_funds; + + OUTCOME_TRY(address, address_selector_(mi, good_funds, need_funds, api_)); + OUTCOME_TRY(mcid, + api_->MpoolPushMessage( + vm::message::UnsignedMessage(miner_address_, + address, + 0, + need_funds, + max_fee, + {}, + ProveCommitAggregate::Number, + MethodParams{enc}), + kPushNoSpec)); + + cutoff_start_ = std::chrono::system_clock::now(); + return mcid.getCid(); } void CommitBatcherImpl::setCommitCutoff(const ChainEpoch ¤t_epoch, diff --git a/core/miner/storage_fsm/impl/commit_batcher_impl.hpp b/core/miner/storage_fsm/impl/commit_batcher_impl.hpp index 7c2ab4f0c3..5201cfa57c 100644 --- a/core/miner/storage_fsm/impl/commit_batcher_impl.hpp +++ b/core/miner/storage_fsm/impl/commit_batcher_impl.hpp @@ -8,15 +8,23 @@ #include "miner/storage_fsm/commit_batcher.hpp" namespace fc::mining { - + using api::MinerInfo; using libp2p::basic::Scheduler; + using primitives::BigInt; using primitives::SectorNumber; using primitives::address::Address; + using primitives::sector::RegisteredAggregationProof; using primitives::tipset::Tipset; using primitives::tipset::TipsetKey; using proofs::ProofEngine; using types::FeeConfig; + using AddressSelector = std::function( + const MinerInfo &miner_info, + const TokenAmount &good_funds, + const TokenAmount &need_funds, + const std::shared_ptr &api)>; + class CommitBatcherImpl : public CommitBatcher { public: struct PairStorage { @@ -51,6 +59,11 @@ namespace fc::mining { std::shared_ptr fee_config_; std::shared_ptr proof_; std::mutex mutex_storage_; + AddressSelector address_selector_; + + const BigInt agg_fee_num_ = BigInt(110); + const BigInt agg_fee_den_ = BigInt(100); + const RegisteredAggregationProof arp_ = RegisteredAggregationProof(0); void sendCallbacks(); diff --git a/core/vm/actor/builtin/v6/monies.hpp b/core/vm/actor/builtin/v6/monies.hpp new file mode 100644 index 0000000000..9455eabb75 --- /dev/null +++ b/core/vm/actor/builtin/v6/monies.hpp @@ -0,0 +1,26 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +namespace fc::vm::actor::builtin::v6::miner { + using primitives::BigInt; + + const BigInt kEstimatedSingleProveCommitGasUsage = 49299973; + const BigInt kEstimatedSinglePreCommitGasUsage = 16433324; + const BigInt kBatchDiscountNumerator = 1; + const BigInt kBatchDiscountDenominator = 20; + const BigInt kBatchBalancer = 5 * kOneNanoFil; + + TokenAmount AggregateProveCommitNetworkFee(uint64_t aggregate_size, + const TokenAmount &base_fee) { + const TokenAmount effectiveGasFee = std::max(base_fee, kBatchBalancer); + const TokenAmount networkFeeNum = + effectiveGasFee * kEstimatedSinglePreCommitGasUsage * aggregate_size + * kBatchDiscountNumerator; + return bigdiv(networkFeeNum, kBatchDiscountDenominator); + } + +} // namespace fc::vm::actor::builtin::v6::miner \ No newline at end of file From cbf7438ff0d839767683c0a43a5f8024af8d0610 Mon Sep 17 00:00:00 2001 From: Markuu-s Date: Mon, 24 Jan 2022 20:39:10 +0300 Subject: [PATCH 06/10] Add completely sendBatch. Signed-off-by: Markuu-s --- .../storage_fsm/impl/commit_batcher_impl.cpp | 45 ++++++++++--------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/core/miner/storage_fsm/impl/commit_batcher_impl.cpp b/core/miner/storage_fsm/impl/commit_batcher_impl.cpp index 08d632f04d..b582fae7e9 100644 --- a/core/miner/storage_fsm/impl/commit_batcher_impl.cpp +++ b/core/miner/storage_fsm/impl/commit_batcher_impl.cpp @@ -48,7 +48,7 @@ namespace fc::mining { if (union_storage_.size() >= max_size_callback_) { sendCallbacks(); } - // Вынести мьютексы + setCommitCutoff(head->epoch(), sector_info); return outcome::success(); @@ -86,7 +86,7 @@ namespace fc::mining { ProveCommitAggregate::Params params; - std::vector proofs; + std::vector proofs; proofs.reserve(total); std::vector infos; @@ -95,8 +95,9 @@ namespace fc::mining { BigInt collateral = 0; for (const auto &[sector_number, pair_storage] : union_storage_for_send) { - OUTCOME_TRY(sc, getSectorCollateral(sector_number, head->key)); - collateral = collateral + sc; + OUTCOME_TRY(sector_collateral, + getSectorCollateral(sector_number, head->key)); + collateral = collateral + sector_collateral; params.sectors.insert(sector_number); infos.push_back(pair_storage.aggregate_input.info); @@ -118,34 +119,34 @@ namespace fc::mining { .proof = proofs[infos[0].number], // TODO is it correct? .infos = infos}; - OUTCOME_TRY(proof_->aggregateSealProofs(aggregate_seal, proofs)); - // need: std::vector> - // proofs: std::vector> - // proof: std::vector - // BytesIn: gsl::span; + std::vector proofsSpan; + for (const Proof &proof : proofs) { + proofsSpan.push_back(gsl::make_span(proof)); + } + OUTCOME_TRY(proof_->aggregateSealProofs(aggregate_seal, proofsSpan)); - // proofs: std::vector> - auto b = gsl::make_span(proofs); params.proof = aggregate_seal.proof; - OUTCOME_TRY(enc, codec::cbor::encode(params)); - OUTCOME_TRY(mi, api_->StateMinerInfo(miner_address_, head->key)); + OUTCOME_TRY(encode, codec::cbor::encode(params)); + OUTCOME_TRY(miner_info, api_->StateMinerInfo(miner_address_, head->key)); const TokenAmount max_fee = fee_config_->max_commit_batch_gas_fee.FeeForSector(proofs.size()); - OUTCOME_TRY(ts, api_->ChainGetTipSet(head->key)); - const BigInt bf = ts->blks[0].parent_base_fee; + OUTCOME_TRY(tipset, api_->ChainGetTipSet(head->key)); + const BigInt base_fee = tipset->blks[0].parent_base_fee; - OUTCOME_TRY(nv, api_->StateNetworkVersion(head->key)); + // OUTCOME_TRY(nv, api_->StateNetworkVersion(head->key)); - TokenAmount agg_fee_raw = AggregateProveCommitNetworkFee(infos.size(), bf); + TokenAmount agg_fee_raw = + AggregateProveCommitNetworkFee(infos.size(), base_fee); TokenAmount agg_fee = bigdiv(agg_fee_raw * agg_fee_num_, agg_fee_den_); TokenAmount need_funds = collateral + agg_fee; TokenAmount good_funds = max_fee + need_funds; - OUTCOME_TRY(address, address_selector_(mi, good_funds, need_funds, api_)); - OUTCOME_TRY(mcid, + OUTCOME_TRY(address, + address_selector_(miner_info, good_funds, need_funds, api_)); + OUTCOME_TRY(signed_messege, api_->MpoolPushMessage( vm::message::UnsignedMessage(miner_address_, address, @@ -154,11 +155,11 @@ namespace fc::mining { max_fee, {}, ProveCommitAggregate::Number, - MethodParams{enc}), + MethodParams{encode}), kPushNoSpec)); cutoff_start_ = std::chrono::system_clock::now(); - return mcid.getCid(); + return signed_messege.getCid(); } void CommitBatcherImpl::setCommitCutoff(const ChainEpoch ¤t_epoch, @@ -166,7 +167,7 @@ namespace fc::mining { ChainEpoch cutoff_epoch = sector_info.ticket_epoch + static_cast(kEpochsInDay + kChainFinality); - ChainEpoch start_epoch; + ChainEpoch start_epoch{}; for (const auto &piece : sector_info.pieces) { if (!piece.deal_info) { continue; From ed1d43893fa3d1016438a0dc1d2db4a7d6a111b2 Mon Sep 17 00:00:00 2001 From: Markuu-s Date: Fri, 4 Feb 2022 18:49:38 +0300 Subject: [PATCH 07/10] Add tests Signed-off-by: Markuu-s --- test/core/miner/CMakeLists.txt | 1 + test/core/miner/commit_batcher_test.cpp | 276 ++++++++++++++++++++++++ 2 files changed, 277 insertions(+) create mode 100644 test/core/miner/commit_batcher_test.cpp diff --git a/test/core/miner/CMakeLists.txt b/test/core/miner/CMakeLists.txt index 71b98efc9b..7d7f5b2c6a 100644 --- a/test/core/miner/CMakeLists.txt +++ b/test/core/miner/CMakeLists.txt @@ -12,6 +12,7 @@ target_link_libraries(events_test addtest(batcher_test precommit_batcher_test.cpp + commit_batcher_test.cpp ) target_link_libraries(batcher_test batcher diff --git a/test/core/miner/commit_batcher_test.cpp b/test/core/miner/commit_batcher_test.cpp new file mode 100644 index 0000000000..ad2c8ab9ec --- /dev/null +++ b/test/core/miner/commit_batcher_test.cpp @@ -0,0 +1,276 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include +#include +#include "miner/address_selector.hpp" +#include "miner/storage_fsm/impl/commit_batcher_impl.hpp" +#include "testutil/default_print.hpp" +#include "testutil/literals.hpp" +#include "testutil/mocks/api.hpp" +#include "testutil/mocks/proofs/proof_engine_mock.hpp" +#include "testutil/outcome.hpp" +#include "vm/actor/actor_method.hpp" +#include "vm/actor/builtin/v5/miner/miner_actor.hpp" +#include "vm/actor/builtin/v6/monies.hpp" +#include "vm/message/message.hpp" +//#include "primitives/block/block.hpp" + +namespace fc::mining { + // using tipset::block::BlockHeader; + using primitives::ActorId; + using testing::_; + using vm::actor::builtin::v5::miner::ProveCommitAggregate; + using vm::message::SignedMessage; + using vm::message::UnsignedMessage; + using PairStorage = CommitBatcherImpl::PairStorage; + using MapPairStorage = std::map; + using fc::mining::types::FeeConfig; + using fc::mining::types::PaddedPieceSize; + using fc::mining::types::Piece; + using fc::mining::types::PieceInfo; + using libp2p::basic::ManualSchedulerBackend; + using libp2p::basic::Scheduler; + using libp2p::basic::SchedulerImpl; + using mining::SelectAddress; + using proofs::ProofEngineMock; + using vm::actor::builtin::types::miner::SectorPreCommitOnChainInfo; + using BatcherCallbackMock = + std::function &cid)>; + + MATCHER_P(methodMatcher, method, "compare msg method name") { + return (arg.method == method); + } + + class CommitBatcherTest : public testing::Test { + protected: + void SetUp() override { + // EXPECT_CALL(mock_ChainHead, Call()).WillOnce(); + + scheduler_backend_ = std::make_shared(); + scheduler_ = std::make_shared(scheduler_backend_, + Scheduler::Config{}); + + miner_id_ = 1; + miner_address_ = Address::makeFromId(miner_id_); + side_address_ = Address::makeFromId(++miner_id_); + + api::BlockHeader block; + block.height = 2; + + tipset_ = std::make_shared( + TipsetKey(), std::vector({block})); + + EXPECT_CALL(mock_ChainHead, Call()) + .WillRepeatedly(testing::Return(tipset_)); + + fee_config_ = std::make_shared(); + fee_config_->max_precommit_batch_gas_fee.base = + TokenAmount{"50000000000000000"}; + fee_config_->max_precommit_batch_gas_fee.per_sector = + TokenAmount{"250000000000000"}; + + EXPECT_CALL(mock_StateMinerInitialPledgeCollateral, + Call(miner_address_, _, _)) + .WillRepeatedly(testing::Return(TokenAmount(100))); + + EXPECT_CALL(mock_StateSectorPreCommitInfo, Call(miner_address_, _, _)) + .WillRepeatedly(testing::Return(SectorPreCommitOnChainInfo())); + proof_ = std::make_shared(); + + EXPECT_CALL(mock_StateMinerInfo, Call(_, _)) + .WillRepeatedly(testing::Return(MinerInfo())); + + EXPECT_CALL(mock_ChainGetTipSet, Call(_)) + .WillRepeatedly(testing::Return(tipset_)); + + callback_mock_ = [](const outcome::result &cid) -> void { + EXPECT_TRUE(cid.has_value()); + }; + } + + std::shared_ptr api_ = std::make_shared(); + std::shared_ptr scheduler_; + MapPairStorage pair_storage_; + std::shared_ptr tipset_; + Address miner_address_; + Address side_address_; + Address wrong_side_address_; + ActorId miner_id_; + std::shared_ptr proof_; + std::shared_ptr fee_config_; + BatcherCallbackMock callback_mock_; + std::shared_ptr scheduler_backend_; + + MOCK_API(api_, ChainHead); + MOCK_API(api_, MpoolPushMessage); + MOCK_API(api_, ChainGetTipSet); + MOCK_API(api_, StateMinerInfo); + MOCK_API(api_, StateMinerInitialPledgeCollateral); + MOCK_API(api_, StateSectorPreCommitInfo); + }; + + /** + * @given 2 commits and max_size_callback is 2 + * @when send the 2 commits + * @then the result should be 2 messages in message pool with pair of commits + */ + TEST_F(CommitBatcherTest, SendAfterMaxSize) { + EXPECT_CALL(mock_MpoolPushMessage, + Call(methodMatcher(ProveCommitAggregate::Number), _)) + .WillOnce( + testing::Invoke([&](const UnsignedMessage &msg, + auto &) -> outcome::result { + return SignedMessage{.message = msg, .signature = BlsSignature()}; + })); + + EXPECT_CALL(*proof_, aggregateSealProofs(_, _)) + .WillOnce(testing::Return(outcome::success())); + + std::shared_ptr batcher = + std::make_shared( + std::chrono::seconds(9999), + api_, + miner_address_, + scheduler_, + [=](const MinerInfo &miner_info, + const TokenAmount &good_funds, + const TokenAmount &need_funds, // TODO is it needed? + const std::shared_ptr &api) + -> outcome::result
{ + return SelectAddress( + miner_info, good_funds, api); // TODO need_funds? + }, + fee_config_, + 2, + proof_); + + SectorInfo sector_info0 = SectorInfo(); + // const TokenAmount deposit = 10; + sector_info0.ticket_epoch = 5; + Piece piece0{.piece = PieceInfo{.size = PaddedPieceSize(128), + .cid = "010001020008"_cid}, + .deal_info = boost::none}; + sector_info0.pieces.push_back(piece0); + sector_info0.sector_number = 777; + + SectorInfo sector_info1 = SectorInfo(); + sector_info0.ticket_epoch = 5; + Piece piece1{.piece = PieceInfo{.size = PaddedPieceSize(128), + .cid = "010001020009"_cid}, + .deal_info = boost::none}; + sector_info1.pieces.push_back(piece1); + sector_info1.sector_number = 888; + /* + * Proof proof; +AggregateSealVerifyInfo info; +RegisteredSealProof spt; + */ + AggregateInput aggregate_input = AggregateInput{ + // .info = MultiCodec::FILECOIN_COMMITMENT_SEALED + }; + + EXPECT_OUTCOME_TRUE_1(batcher->addCommit( + sector_info0, AggregateInput{.info = 777}, callback_mock_)); + + EXPECT_OUTCOME_TRUE_1(batcher->addCommit( + sector_info1, AggregateInput{.info = 888}, callback_mock_)); + } + + /** + * @given a 1 commit + * @when send a 1 commit + * @then result should be 0 messages in message pool + */ + TEST_F(CommitBatcherTest, BatcherWrite) { + std::shared_ptr batcher = + std::make_shared( + std::chrono::seconds(9999), + api_, + miner_address_, + scheduler_, + [=](const MinerInfo &miner_info, + const TokenAmount &good_funds, + const TokenAmount &need_funds, // TODO is it needed? + const std::shared_ptr &api) + -> outcome::result
{ + return SelectAddress( + miner_info, good_funds, api); // TODO need_funds? + }, + fee_config_, + 999, + proof_); + + EXPECT_OUTCOME_TRUE_1(batcher->addCommit( + SectorInfo(), + AggregateInput(), + [](const outcome::result &cid) -> outcome::result { + return outcome::success(); + })); + } + + /** + * @given 4 commits + * @when send the first pair and after the rescheduling + * next pair + * @then The result should be 2 messages in message pool with pair of + * commits in each + */ + TEST_F(CommitBatcherTest, CallbackSend) { + std::shared_ptr batcher = + std::make_shared( + std::chrono::seconds(60), + api_, + miner_address_, + scheduler_, + [=](const MinerInfo &miner_info, + const TokenAmount &good_funds, + const TokenAmount &need_funds, // TODO is it needed? + const std::shared_ptr &api) + -> outcome::result
{ + return SelectAddress( + miner_info, good_funds, api); // TODO need_funds? + }, + fee_config_, + 999, + proof_); + + EXPECT_CALL(mock_MpoolPushMessage, + Call(methodMatcher(ProveCommitAggregate::Number), _)) + .Times(2) + .WillRepeatedly( + testing::Invoke([&](const UnsignedMessage &msg, + auto &) -> outcome::result { + return SignedMessage{.message = msg, .signature = BlsSignature()}; + })); + + + SectorInfo sector_info = SectorInfo(); + + sector_info.sector_number = 2; + + EXPECT_OUTCOME_TRUE_1(batcher->addCommit( + sector_info, AggregateInput{.info = 2}, callback_mock_)); + + sector_info.sector_number = 3; + + EXPECT_OUTCOME_TRUE_1(batcher->addCommit( + sector_info, AggregateInput{.info = 3}, callback_mock_)); + + try { + scheduler_backend_->shiftToTimer(); + } catch(std::exception &e) { + std::cerr << e.what(); + } + sector_info.sector_number = 6; + + EXPECT_OUTCOME_TRUE_1(batcher->addCommit( + sector_info, AggregateInput{.info = 6}, callback_mock_)); + + scheduler_backend_->shiftToTimer(); + } +} // namespace fc::mining From fe60a37fcbe667c92d56aa9e8b1c5b2e1236b51e Mon Sep 17 00:00:00 2001 From: Markuu-s Date: Tue, 8 Feb 2022 13:27:45 +0300 Subject: [PATCH 08/10] Fix sendBatch. Signed-off-by: Markuu-s --- core/const.hpp | 1 + core/miner/address_selector.hpp | 9 +- .../storage_fsm/impl/commit_batcher_impl.cpp | 108 ++++++++++++------ .../storage_fsm/impl/commit_batcher_impl.hpp | 19 ++- core/miner/storage_fsm/types.hpp | 9 +- core/vm/actor/builtin/v6/monies.hpp | 5 +- test/core/miner/commit_batcher_test.cpp | 40 ++----- 7 files changed, 110 insertions(+), 81 deletions(-) diff --git a/core/const.hpp b/core/const.hpp index 618c04f757..5fb2b4feed 100644 --- a/core/const.hpp +++ b/core/const.hpp @@ -46,6 +46,7 @@ namespace fc { const TokenAmount kMinimumBaseFee{100}; constexpr auto kPackingEfficiencyDenom{5}; constexpr auto kPackingEfficiencyNum{4}; + const TokenAmount kOneNanoFil{1000000000}; // ****************** // Network versions diff --git a/core/miner/address_selector.hpp b/core/miner/address_selector.hpp index 1addfbc2f9..0d0348c9a9 100644 --- a/core/miner/address_selector.hpp +++ b/core/miner/address_selector.hpp @@ -18,14 +18,15 @@ namespace fc::mining { /** * SelectAddress takes the maximal possible transaction fee from configs and - * chooses one of control addresses with minimal balance that is more than good - * funds to make miner work as long as possible. If no suitble control address - * were found, function returns worker address. + * chooses one of control addresses with minimal balance that is more than + * good funds to make miner work as long as possible. If no suitble control + * address were found, function returns worker address. */ inline outcome::result
SelectAddress( const MinerInfo &miner_info, const TokenAmount &good_funds, - const std::shared_ptr &api) { + const std::shared_ptr + &api) { // TODO update from lotus (Markuuu-s) TokenAmount finder_balance; auto finder = miner_info.control.end(); for (auto address = miner_info.control.begin(); diff --git a/core/miner/storage_fsm/impl/commit_batcher_impl.cpp b/core/miner/storage_fsm/impl/commit_batcher_impl.cpp index b582fae7e9..25024fe89f 100644 --- a/core/miner/storage_fsm/impl/commit_batcher_impl.cpp +++ b/core/miner/storage_fsm/impl/commit_batcher_impl.cpp @@ -5,10 +5,13 @@ #pragma once #include "commit_batcher_impl.hpp" -#include +#include +#include #include "vm/actor/builtin/v5/miner/miner_actor.hpp" #include "vm/actor/builtin/v6/monies.hpp" +#include + namespace fc::mining { using api::kPushNoSpec; using fc::BytesIn; @@ -24,14 +27,25 @@ namespace fc::mining { using vm::actor::builtin::v5::miner::ProveCommitAggregate; using vm::actor::builtin::v6::miner::AggregateProveCommitNetworkFee; - CommitBatcherImpl::CommitBatcherImpl( - const std::chrono::milliseconds &max_time, - const size_t &max_size_callback, - const std::shared_ptr &scheduler) - : max_delay_(max_time), max_size_callback_(max_size_callback) { + CommitBatcherImpl::CommitBatcherImpl(const std::chrono::seconds &max_time, + std::shared_ptr api, + Address miner_address, + std::shared_ptr scheduler, + AddressSelector address_selector, + std::shared_ptr fee_config, + const size_t &max_size_callback, + std::shared_ptr proof) + : scheduler_(std::move(scheduler)), + max_delay_(max_time), + closest_cutoff_(max_time), + max_size_callback_(max_size_callback), + api_(std::move(api)), + miner_address_(std::move(miner_address)), + fee_config_(std::move(fee_config)), + proof_(std::move(proof)), + address_selector_(std::move(address_selector)) { cutoff_start_ = std::chrono::system_clock::now(); - handle_ = - scheduler_->scheduleWithHandle([&]() { sendCallbacks(); }, max_delay_); + sendCallbacks(max_delay_); } outcome::result CommitBatcherImpl::addCommit( @@ -43,46 +57,68 @@ namespace fc::mining { const SectorNumber §or_number = sector_info.sector_number; OUTCOME_TRY(head, api_->ChainHead()); - union_storage_[sector_number] = PairStorage{aggregate_input, callback}; + pair_storage_[sector_number] = PairStorage{aggregate_input, callback}; - if (union_storage_.size() >= max_size_callback_) { - sendCallbacks(); + if (pair_storage_.size() >= max_size_callback_) { + locker.unlock(); + forceSend(); + return outcome::success(); } setCommitCutoff(head->epoch(), sector_info); - return outcome::success(); } - void CommitBatcherImpl::forceSend() {} + void CommitBatcherImpl::forceSend() { + MapPairStorage pair_storage_for_send_; - void CommitBatcherImpl::sendCallbacks() { std::unique_lock locker(mutex_storage_); - MapPairStorage union_storage_for_send_( - std::make_move_iterator(union_storage_.begin()), - std::make_move_iterator(union_storage_.end())); + pair_storage_for_send_ = std::move(pair_storage_); locker.unlock(); - const auto maybe_result = sendBatch(union_storage_for_send_); - for (const auto &[key, pair_storage] : union_storage_for_send_) { + const auto maybe_result = sendBatch(pair_storage_for_send_); + + for (auto &[key, pair_storage] : pair_storage_for_send_) { pair_storage.commit_callback(maybe_result); } cutoff_start_ = std::chrono::system_clock::now(); closest_cutoff_ = max_delay_; + sendCallbacks(max_delay_); + } + + void CommitBatcherImpl::sendCallbacks(std::chrono::milliseconds time) { + handle_ = scheduler_->scheduleWithHandle( + [&]() { + MapPairStorage pair_storage_for_send_; + + std::unique_lock locker(mutex_storage_); + pair_storage_for_send_ = std::move(pair_storage_); + locker.unlock(); - handle_.reschedule(max_delay_).value(); + const auto maybe_result = sendBatch(pair_storage_for_send_); + + for (auto &[key, pair_storage] : pair_storage_for_send_) { + pair_storage.commit_callback(maybe_result); + } + + cutoff_start_ = std::chrono::system_clock::now(); + closest_cutoff_ = max_delay_; + + handle_.reschedule(max_delay_).value(); + }, + time); } outcome::result CommitBatcherImpl::sendBatch( - MapPairStorage &union_storage_for_send) { - if (union_storage_for_send.empty()) { + const MapPairStorage &pair_storage_for_send) { + if (pair_storage_for_send.empty()) { cutoff_start_ = std::chrono::system_clock::now(); return ERROR_TEXT("Empty Batcher"); } OUTCOME_TRY(head, api_->ChainHead()); - const size_t total = union_storage_for_send.size(); + const size_t total = pair_storage_for_send.size(); ProveCommitAggregate::Params params; @@ -94,7 +130,7 @@ namespace fc::mining { BigInt collateral = 0; - for (const auto &[sector_number, pair_storage] : union_storage_for_send) { + for (const auto &[sector_number, pair_storage] : pair_storage_for_send) { OUTCOME_TRY(sector_collateral, getSectorCollateral(sector_number, head->key)); collateral = collateral + sector_collateral; @@ -103,26 +139,26 @@ namespace fc::mining { infos.push_back(pair_storage.aggregate_input.info); } - for (const auto &[sector_number, pair_storage] : union_storage_for_send) { + for (const auto &[sector_number, pair_storage] : pair_storage_for_send) { proofs.push_back(pair_storage.aggregate_input.proof); } const ActorId mid = miner_address_.getId(); // TODO maybe long (AggregateSealProofs) - - AggregateSealVerifyProofAndInfos aggregate_seal = - AggregateSealVerifyProofAndInfos{ - .miner = mid, - .seal_proof = - union_storage_for_send[infos[0].number].aggregate_input.spt, - .aggregate_proof = arp_, - .proof = proofs[infos[0].number], // TODO is it correct? - .infos = infos}; + AggregateSealVerifyProofAndInfos aggregate_seal{ + .miner = mid, + .seal_proof = + pair_storage_for_send.at(infos[0].number).aggregate_input.spt, + .aggregate_proof = arp_, + .infos = infos}; std::vector proofsSpan; + proofsSpan.reserve(proofs.size()); + for (const Proof &proof : proofs) { proofsSpan.push_back(gsl::make_span(proof)); } + OUTCOME_TRY(proof_->aggregateSealProofs(aggregate_seal, proofsSpan)); params.proof = aggregate_seal.proof; @@ -135,8 +171,6 @@ namespace fc::mining { OUTCOME_TRY(tipset, api_->ChainGetTipSet(head->key)); const BigInt base_fee = tipset->blks[0].parent_base_fee; - // OUTCOME_TRY(nv, api_->StateNetworkVersion(head->key)); - TokenAmount agg_fee_raw = AggregateProveCommitNetworkFee(infos.size(), base_fee); @@ -187,7 +221,7 @@ namespace fc::mining { std::chrono::system_clock::now() - cutoff_start_) > temp_cutoff)) { cutoff_start_ = std::chrono::system_clock::now(); - handle_.reschedule(max_delay_).value(); + sendCallbacks(temp_cutoff); closest_cutoff_ = temp_cutoff; } } diff --git a/core/miner/storage_fsm/impl/commit_batcher_impl.hpp b/core/miner/storage_fsm/impl/commit_batcher_impl.hpp index 5201cfa57c..cf7e46ef7b 100644 --- a/core/miner/storage_fsm/impl/commit_batcher_impl.hpp +++ b/core/miner/storage_fsm/impl/commit_batcher_impl.hpp @@ -31,11 +31,16 @@ namespace fc::mining { AggregateInput aggregate_input; CommitCallback commit_callback; }; - typedef std::map MapPairStorage; + using MapPairStorage = std::map; - CommitBatcherImpl(const std::chrono::milliseconds &max_time, + CommitBatcherImpl(const std::chrono::seconds &max_time, + std::shared_ptr api, + Address miner_address, + std::shared_ptr scheduler, + AddressSelector address_selector, + std::shared_ptr fee_config, const size_t &max_size_callback, - const std::shared_ptr &scheduler); + std::shared_ptr proof); outcome::result addCommit(const SectorInfo §or_info, const AggregateInput &aggregate_input, @@ -53,21 +58,23 @@ namespace fc::mining { std::chrono::milliseconds closest_cutoff_; std::chrono::system_clock::time_point cutoff_start_; size_t max_size_callback_; - MapPairStorage union_storage_; + MapPairStorage pair_storage_; std::shared_ptr api_; Address miner_address_; std::shared_ptr fee_config_; std::shared_ptr proof_; std::mutex mutex_storage_; AddressSelector address_selector_; + common::Logger logger_; const BigInt agg_fee_num_ = BigInt(110); const BigInt agg_fee_den_ = BigInt(100); const RegisteredAggregationProof arp_ = RegisteredAggregationProof(0); - void sendCallbacks(); + // TODO (Markuu-s) Add processIndividually + void sendCallbacks(std::chrono::milliseconds time); - outcome::result sendBatch(MapPairStorage &union_storage_for_send); + outcome::result sendBatch(const MapPairStorage &pair_storage_for_send); outcome::result getSectorCollateral( const SectorNumber §or_number, const TipsetKey &tip_set_key); diff --git a/core/miner/storage_fsm/types.hpp b/core/miner/storage_fsm/types.hpp index 0141050562..f2b6ce5e96 100644 --- a/core/miner/storage_fsm/types.hpp +++ b/core/miner/storage_fsm/types.hpp @@ -64,9 +64,12 @@ namespace fc::mining::types { is_keep_unsealed) inline bool operator==(const DealInfo &lhs, const DealInfo &rhs) { - return lhs.publish_cid == rhs.publish_cid && lhs.deal_id == rhs.deal_id - && lhs.deal_schedule == rhs.deal_schedule - && lhs.is_keep_unsealed == rhs.is_keep_unsealed; + if (lhs.is_keep_unsealed == rhs.is_keep_unsealed) { + return lhs.publish_cid == rhs.publish_cid && lhs.deal_id == rhs.deal_id + && lhs.deal_schedule == rhs.deal_schedule; + } else { + return false; + } } struct Piece { diff --git a/core/vm/actor/builtin/v6/monies.hpp b/core/vm/actor/builtin/v6/monies.hpp index 9455eabb75..6d8a5b873e 100644 --- a/core/vm/actor/builtin/v6/monies.hpp +++ b/core/vm/actor/builtin/v6/monies.hpp @@ -4,6 +4,7 @@ */ #pragma once +#include "const.hpp" namespace fc::vm::actor::builtin::v6::miner { using primitives::BigInt; @@ -14,8 +15,8 @@ namespace fc::vm::actor::builtin::v6::miner { const BigInt kBatchDiscountDenominator = 20; const BigInt kBatchBalancer = 5 * kOneNanoFil; - TokenAmount AggregateProveCommitNetworkFee(uint64_t aggregate_size, - const TokenAmount &base_fee) { + inline TokenAmount AggregateProveCommitNetworkFee( + uint64_t aggregate_size, const TokenAmount &base_fee) { const TokenAmount effectiveGasFee = std::max(base_fee, kBatchBalancer); const TokenAmount networkFeeNum = effectiveGasFee * kEstimatedSinglePreCommitGasUsage * aggregate_size diff --git a/test/core/miner/commit_batcher_test.cpp b/test/core/miner/commit_batcher_test.cpp index ad2c8ab9ec..a1f089c32e 100644 --- a/test/core/miner/commit_batcher_test.cpp +++ b/test/core/miner/commit_batcher_test.cpp @@ -18,10 +18,8 @@ #include "vm/actor/builtin/v5/miner/miner_actor.hpp" #include "vm/actor/builtin/v6/monies.hpp" #include "vm/message/message.hpp" -//#include "primitives/block/block.hpp" namespace fc::mining { - // using tipset::block::BlockHeader; using primitives::ActorId; using testing::_; using vm::actor::builtin::v5::miner::ProveCommitAggregate; @@ -49,8 +47,6 @@ namespace fc::mining { class CommitBatcherTest : public testing::Test { protected: void SetUp() override { - // EXPECT_CALL(mock_ChainHead, Call()).WillOnce(); - scheduler_backend_ = std::make_shared(); scheduler_ = std::make_shared(scheduler_backend_, Scheduler::Config{}); @@ -139,18 +135,16 @@ namespace fc::mining { scheduler_, [=](const MinerInfo &miner_info, const TokenAmount &good_funds, - const TokenAmount &need_funds, // TODO is it needed? + const TokenAmount &need_funds, const std::shared_ptr &api) -> outcome::result
{ - return SelectAddress( - miner_info, good_funds, api); // TODO need_funds? + return SelectAddress(miner_info, good_funds, api); }, fee_config_, 2, proof_); SectorInfo sector_info0 = SectorInfo(); - // const TokenAmount deposit = 10; sector_info0.ticket_epoch = 5; Piece piece0{.piece = PieceInfo{.size = PaddedPieceSize(128), .cid = "010001020008"_cid}, @@ -165,14 +159,6 @@ namespace fc::mining { .deal_info = boost::none}; sector_info1.pieces.push_back(piece1); sector_info1.sector_number = 888; - /* - * Proof proof; -AggregateSealVerifyInfo info; -RegisteredSealProof spt; - */ - AggregateInput aggregate_input = AggregateInput{ - // .info = MultiCodec::FILECOIN_COMMITMENT_SEALED - }; EXPECT_OUTCOME_TRUE_1(batcher->addCommit( sector_info0, AggregateInput{.info = 777}, callback_mock_)); @@ -195,11 +181,10 @@ RegisteredSealProof spt; scheduler_, [=](const MinerInfo &miner_info, const TokenAmount &good_funds, - const TokenAmount &need_funds, // TODO is it needed? + const TokenAmount &need_funds, const std::shared_ptr &api) -> outcome::result
{ - return SelectAddress( - miner_info, good_funds, api); // TODO need_funds? + return SelectAddress(miner_info, good_funds, api); }, fee_config_, 999, @@ -223,22 +208,24 @@ RegisteredSealProof spt; TEST_F(CommitBatcherTest, CallbackSend) { std::shared_ptr batcher = std::make_shared( - std::chrono::seconds(60), + std::chrono::seconds(999), api_, miner_address_, scheduler_, [=](const MinerInfo &miner_info, const TokenAmount &good_funds, - const TokenAmount &need_funds, // TODO is it needed? + const TokenAmount &need_funds, const std::shared_ptr &api) -> outcome::result
{ - return SelectAddress( - miner_info, good_funds, api); // TODO need_funds? + return SelectAddress(miner_info, good_funds, api); }, fee_config_, 999, proof_); + EXPECT_CALL(*proof_, aggregateSealProofs(_, _)) + .WillRepeatedly(testing::Return(outcome::success())); + EXPECT_CALL(mock_MpoolPushMessage, Call(methodMatcher(ProveCommitAggregate::Number), _)) .Times(2) @@ -248,7 +235,6 @@ RegisteredSealProof spt; return SignedMessage{.message = msg, .signature = BlsSignature()}; })); - SectorInfo sector_info = SectorInfo(); sector_info.sector_number = 2; @@ -256,16 +242,12 @@ RegisteredSealProof spt; EXPECT_OUTCOME_TRUE_1(batcher->addCommit( sector_info, AggregateInput{.info = 2}, callback_mock_)); + scheduler_backend_->shiftToTimer(); sector_info.sector_number = 3; EXPECT_OUTCOME_TRUE_1(batcher->addCommit( sector_info, AggregateInput{.info = 3}, callback_mock_)); - try { - scheduler_backend_->shiftToTimer(); - } catch(std::exception &e) { - std::cerr << e.what(); - } sector_info.sector_number = 6; EXPECT_OUTCOME_TRUE_1(batcher->addCommit( From 6d0d80f8044b99d9c53b12233dd1c6bf7612e81f Mon Sep 17 00:00:00 2001 From: Markuu-s Date: Tue, 8 Feb 2022 14:36:18 +0300 Subject: [PATCH 09/10] Small fixes. Signed-off-by: Markuu-s --- core/api/full_node/node_api.hpp | 8 -------- core/miner/storage_fsm/impl/commit_batcher_impl.cpp | 10 +++++----- core/miner/storage_fsm/impl/commit_batcher_impl.hpp | 2 +- core/vm/actor/builtin/v6/monies.hpp | 9 +++++++++ 4 files changed, 15 insertions(+), 14 deletions(-) diff --git a/core/api/full_node/node_api.hpp b/core/api/full_node/node_api.hpp index 2fdfd24679..df535f88fa 100644 --- a/core/api/full_node/node_api.hpp +++ b/core/api/full_node/node_api.hpp @@ -463,13 +463,6 @@ namespace fc::api { const Address &, const TokenAmount &) - API_METHOD(ChainBaseFee, - jwt::kWritePermission, - TokenAmount, - Address, - SectorNumber, - TipsetKey) - API_METHOD(MinerCreateBlock, jwt::kWritePermission, BlockWithCids, @@ -794,7 +787,6 @@ namespace fc::api { visitNet(a, f); visitWallet(a, f); f(a.BeaconGetEntry); - f(a.ChainBaseFee); f(a.ChainGetBlock); f(a.ChainGetBlockMessages); f(a.ChainGetGenesis); diff --git a/core/miner/storage_fsm/impl/commit_batcher_impl.cpp b/core/miner/storage_fsm/impl/commit_batcher_impl.cpp index 25024fe89f..cb435b01df 100644 --- a/core/miner/storage_fsm/impl/commit_batcher_impl.cpp +++ b/core/miner/storage_fsm/impl/commit_batcher_impl.cpp @@ -45,7 +45,7 @@ namespace fc::mining { proof_(std::move(proof)), address_selector_(std::move(address_selector)) { cutoff_start_ = std::chrono::system_clock::now(); - sendCallbacks(max_delay_); + reschedule(max_delay_); } outcome::result CommitBatcherImpl::addCommit( @@ -84,10 +84,10 @@ namespace fc::mining { cutoff_start_ = std::chrono::system_clock::now(); closest_cutoff_ = max_delay_; - sendCallbacks(max_delay_); + reschedule(max_delay_); } - void CommitBatcherImpl::sendCallbacks(std::chrono::milliseconds time) { + void CommitBatcherImpl::reschedule(std::chrono::milliseconds time) { handle_ = scheduler_->scheduleWithHandle( [&]() { MapPairStorage pair_storage_for_send_; @@ -172,7 +172,7 @@ namespace fc::mining { const BigInt base_fee = tipset->blks[0].parent_base_fee; TokenAmount agg_fee_raw = - AggregateProveCommitNetworkFee(infos.size(), base_fee); + AggregateProveCommitNetworkFee(infos.size(), base_fee); // TODO change to aggregateNetworkFee TokenAmount agg_fee = bigdiv(agg_fee_raw * agg_fee_num_, agg_fee_den_); TokenAmount need_funds = collateral + agg_fee; @@ -221,7 +221,7 @@ namespace fc::mining { std::chrono::system_clock::now() - cutoff_start_) > temp_cutoff)) { cutoff_start_ = std::chrono::system_clock::now(); - sendCallbacks(temp_cutoff); + reschedule(temp_cutoff); closest_cutoff_ = temp_cutoff; } } diff --git a/core/miner/storage_fsm/impl/commit_batcher_impl.hpp b/core/miner/storage_fsm/impl/commit_batcher_impl.hpp index cf7e46ef7b..0593a1b4f8 100644 --- a/core/miner/storage_fsm/impl/commit_batcher_impl.hpp +++ b/core/miner/storage_fsm/impl/commit_batcher_impl.hpp @@ -72,7 +72,7 @@ namespace fc::mining { const RegisteredAggregationProof arp_ = RegisteredAggregationProof(0); // TODO (Markuu-s) Add processIndividually - void sendCallbacks(std::chrono::milliseconds time); + void reschedule(std::chrono::milliseconds time); outcome::result sendBatch(const MapPairStorage &pair_storage_for_send); diff --git a/core/vm/actor/builtin/v6/monies.hpp b/core/vm/actor/builtin/v6/monies.hpp index 6d8a5b873e..9b77006052 100644 --- a/core/vm/actor/builtin/v6/monies.hpp +++ b/core/vm/actor/builtin/v6/monies.hpp @@ -24,4 +24,13 @@ namespace fc::vm::actor::builtin::v6::miner { return bigdiv(networkFeeNum, kBatchDiscountDenominator); } + inline TokenAmount aggregateNetworkFee(uint64_t aggregate_size, + const TokenAmount &gasUsage, + const TokenAmount &baseFee) { + TokenAmount effectiveGasFee = std::max(baseFee, kBatchBalancer); + TokenAmount networkFeeNum = effectiveGasFee * gasUsage * BigInt(aggregate_size) * kBatchDiscountNumerator; + TokenAmount networkFee = bigdiv(networkFeeNum, kBatchDiscountDenominator); + return networkFee; + } + } // namespace fc::vm::actor::builtin::v6::miner \ No newline at end of file From d64c1d0b6a8eebb12dba70426b586a87bfb370ae Mon Sep 17 00:00:00 2001 From: Markuu-s Date: Tue, 8 Feb 2022 15:50:47 +0300 Subject: [PATCH 10/10] Fix aggregate. Signed-off-by: Markuu-s --- core/miner/storage_fsm/impl/commit_batcher_impl.cpp | 4 ++-- core/vm/actor/builtin/v6/monies.hpp | 8 +++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/core/miner/storage_fsm/impl/commit_batcher_impl.cpp b/core/miner/storage_fsm/impl/commit_batcher_impl.cpp index cb435b01df..e0ea68d35a 100644 --- a/core/miner/storage_fsm/impl/commit_batcher_impl.cpp +++ b/core/miner/storage_fsm/impl/commit_batcher_impl.cpp @@ -171,8 +171,8 @@ namespace fc::mining { OUTCOME_TRY(tipset, api_->ChainGetTipSet(head->key)); const BigInt base_fee = tipset->blks[0].parent_base_fee; - TokenAmount agg_fee_raw = - AggregateProveCommitNetworkFee(infos.size(), base_fee); // TODO change to aggregateNetworkFee + TokenAmount agg_fee_raw = AggregateProveCommitNetworkFee( + infos.size(), base_fee); // TODO change to aggregateNetworkFee TokenAmount agg_fee = bigdiv(agg_fee_raw * agg_fee_num_, agg_fee_den_); TokenAmount need_funds = collateral + agg_fee; diff --git a/core/vm/actor/builtin/v6/monies.hpp b/core/vm/actor/builtin/v6/monies.hpp index 9b77006052..513f04ab5b 100644 --- a/core/vm/actor/builtin/v6/monies.hpp +++ b/core/vm/actor/builtin/v6/monies.hpp @@ -25,10 +25,12 @@ namespace fc::vm::actor::builtin::v6::miner { } inline TokenAmount aggregateNetworkFee(uint64_t aggregate_size, - const TokenAmount &gasUsage, - const TokenAmount &baseFee) { + const TokenAmount &gasUsage, + const TokenAmount &baseFee) { TokenAmount effectiveGasFee = std::max(baseFee, kBatchBalancer); - TokenAmount networkFeeNum = effectiveGasFee * gasUsage * BigInt(aggregate_size) * kBatchDiscountNumerator; + TokenAmount networkFeeNum = effectiveGasFee * gasUsage + * BigInt(aggregate_size) + * kBatchDiscountNumerator; TokenAmount networkFee = bigdiv(networkFeeNum, kBatchDiscountDenominator); return networkFee; }