Skip to content

Commit 0f8c95d

Browse files
committed
Merge bitcoin/bitcoin#27021: Implement Mini version of BlockAssembler to calculate mining scores
6b605b9 [fuzz] Add MiniMiner target + diff fuzz against BlockAssembler (glozow) 3f3f2d5 [unit test] GatherClusters and MiniMiner unit tests (glozow) 59afcc8 Implement Mini version of BlockAssembler to calculate mining scores (glozow) 56484f0 [mempool] find connected mempool entries with GatherClusters(…) (glozow) Pull request description: Implement Mini version of BlockAssembler to calculate mining scores Run the mining algorithm on a subset of the mempool, only disturbing the mempool to copy out fee information for relevant entries. Intended to be used by wallet to calculate amounts needed for fee-bumping unconfirmed transactions. From comments of sipa and glozow below: > > In what way does the code added here differ from the real block assembly code? > > * Only operates on the relevant transactions rather than full mempool > * Has the ability to remove transactions that will be replaced so they don't impact their ancestors > * Does not hold mempool lock outside of the constructor, makes copies of the entries it needs instead (though I'm not sure if this has an effect in practice) > * Doesn't do the sanity checks like keeping weight within max block weight and `IsFinalTx()` > * After the block template is built, additionally calculates fees to bump remaining ancestor packages to target feerate ACKs for top commit: achow101: ACK 6b605b9 Xekyo: > ACK [6b605b9](bitcoin/bitcoin@6b605b9) modulo `miniminer_overlap` test. furszy: ACK 6b605b9 modulo `miniminer_overlap` test. theStack: Code-review ACK 6b605b9 Tree-SHA512: f86a8b4ae0506858a7b15d90f417ebceea5038b395c05c825e3796123ad3b6cb8a98ebb948521316802a4c6d60ebd7041093356b1e2c2922a06b3b96b3b8acb6
2 parents fc4bee3 + 6b605b9 commit 0f8c95d

File tree

8 files changed

+1215
-2
lines changed

8 files changed

+1215
-2
lines changed

src/Makefile.am

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -217,6 +217,7 @@ BITCOIN_CORE_H = \
217217
node/mempool_args.h \
218218
node/mempool_persist_args.h \
219219
node/miner.h \
220+
node/mini_miner.h \
220221
node/minisketchwrapper.h \
221222
node/psbt.h \
222223
node/transaction.h \
@@ -410,6 +411,7 @@ libbitcoin_node_a_SOURCES = \
410411
node/mempool_args.cpp \
411412
node/mempool_persist_args.cpp \
412413
node/miner.cpp \
414+
node/mini_miner.cpp \
413415
node/minisketchwrapper.cpp \
414416
node/psbt.cpp \
415417
node/transaction.cpp \

src/Makefile.test.include

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,7 @@ BITCOIN_TESTS =\
106106
test/merkle_tests.cpp \
107107
test/merkleblock_tests.cpp \
108108
test/miner_tests.cpp \
109+
test/miniminer_tests.cpp \
109110
test/miniscript_tests.cpp \
110111
test/minisketch_tests.cpp \
111112
test/multisig_tests.cpp \
@@ -287,6 +288,7 @@ test_fuzz_fuzz_SOURCES = \
287288
test/fuzz/message.cpp \
288289
test/fuzz/miniscript.cpp \
289290
test/fuzz/minisketch.cpp \
291+
test/fuzz/mini_miner.cpp \
290292
test/fuzz/muhash.cpp \
291293
test/fuzz/multiplication_overflow.cpp \
292294
test/fuzz/net.cpp \

src/node/mini_miner.cpp

Lines changed: 366 additions & 0 deletions
Large diffs are not rendered by default.

src/node/mini_miner.h

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
// Copyright (c) 2022 The Bitcoin Core developers
2+
// Distributed under the MIT software license, see the accompanying
3+
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
4+
5+
#ifndef BITCOIN_NODE_MINI_MINER_H
6+
#define BITCOIN_NODE_MINI_MINER_H
7+
8+
#include <txmempool.h>
9+
10+
#include <memory>
11+
#include <optional>
12+
#include <stdint.h>
13+
14+
namespace node {
15+
16+
// Container for tracking updates to ancestor feerate as we include ancestors in the "block"
17+
class MiniMinerMempoolEntry
18+
{
19+
const CAmount fee_individual;
20+
const CTransactionRef tx;
21+
const int64_t vsize_individual;
22+
23+
// This class must be constructed while holding mempool.cs. After construction, the object's
24+
// methods can be called without holding that lock.
25+
public:
26+
CAmount fee_with_ancestors;
27+
int64_t vsize_with_ancestors;
28+
explicit MiniMinerMempoolEntry(CTxMemPool::txiter entry) :
29+
fee_individual{entry->GetModifiedFee()},
30+
tx{entry->GetSharedTx()},
31+
vsize_individual(entry->GetTxSize()),
32+
fee_with_ancestors{entry->GetModFeesWithAncestors()},
33+
vsize_with_ancestors(entry->GetSizeWithAncestors())
34+
{ }
35+
36+
CAmount GetModifiedFee() const { return fee_individual; }
37+
CAmount GetModFeesWithAncestors() const { return fee_with_ancestors; }
38+
int64_t GetTxSize() const { return vsize_individual; }
39+
int64_t GetSizeWithAncestors() const { return vsize_with_ancestors; }
40+
const CTransaction& GetTx() const LIFETIMEBOUND { return *tx; }
41+
};
42+
43+
// Comparator needed for std::set<MockEntryMap::iterator>
44+
struct IteratorComparator
45+
{
46+
template<typename I>
47+
bool operator()(const I& a, const I& b) const
48+
{
49+
return &(*a) < &(*b);
50+
}
51+
};
52+
53+
/** A minimal version of BlockAssembler. Allows us to run the mining algorithm on a subset of
54+
* mempool transactions, ignoring consensus rules, to calculate mining scores. */
55+
class MiniMiner
56+
{
57+
// When true, a caller may use CalculateBumpFees(). Becomes false if we failed to retrieve
58+
// mempool entries (i.e. cluster size too large) or bump fees have already been calculated.
59+
bool m_ready_to_calculate{true};
60+
61+
// Set once per lifetime, fill in during initialization.
62+
// txids of to-be-replaced transactions
63+
std::set<uint256> m_to_be_replaced;
64+
65+
// If multiple argument outpoints correspond to the same transaction, cache them together in
66+
// a single entry indexed by txid. Then we can just work with txids since all outpoints from
67+
// the same tx will have the same bumpfee. Excludes non-mempool transactions.
68+
std::map<uint256, std::vector<COutPoint>> m_requested_outpoints_by_txid;
69+
70+
// What we're trying to calculate.
71+
std::map<COutPoint, CAmount> m_bump_fees;
72+
73+
// The constructed block template
74+
std::set<uint256> m_in_block;
75+
76+
// Information on the current status of the block
77+
CAmount m_total_fees{0};
78+
int32_t m_total_vsize{0};
79+
80+
/** Main data structure holding the entries, can be indexed by txid */
81+
std::map<uint256, MiniMinerMempoolEntry> m_entries_by_txid;
82+
using MockEntryMap = decltype(m_entries_by_txid);
83+
84+
/** Vector of entries, can be sorted by ancestor feerate. */
85+
std::vector<MockEntryMap::iterator> m_entries;
86+
87+
/** Map of txid to its descendants. Should be inclusive. */
88+
std::map<uint256, std::vector<MockEntryMap::iterator>> m_descendant_set_by_txid;
89+
90+
/** Consider this ancestor package "mined" so remove all these entries from our data structures. */
91+
void DeleteAncestorPackage(const std::set<MockEntryMap::iterator, IteratorComparator>& ancestors);
92+
93+
/** Perform some checks. */
94+
void SanityCheck() const;
95+
96+
public:
97+
/** Returns true if CalculateBumpFees may be called, false if not. */
98+
bool IsReadyToCalculate() const { return m_ready_to_calculate; }
99+
100+
/** Build a block template until the target feerate is hit. */
101+
void BuildMockTemplate(const CFeeRate& target_feerate);
102+
103+
/** Returns set of txids in the block template if one has been constructed. */
104+
std::set<uint256> GetMockTemplateTxids() const { return m_in_block; }
105+
106+
MiniMiner(const CTxMemPool& mempool, const std::vector<COutPoint>& outpoints);
107+
108+
/** Construct a new block template and, for each outpoint corresponding to a transaction that
109+
* did not make it into the block, calculate the cost of bumping those transactions (and their
110+
* ancestors) to the minimum feerate. Returns a map from outpoint to bump fee, or an empty map
111+
* if they cannot be calculated. */
112+
std::map<COutPoint, CAmount> CalculateBumpFees(const CFeeRate& target_feerate);
113+
114+
/** Construct a new block template and, calculate the cost of bumping all transactions that did
115+
* not make it into the block to the target feerate. Returns the total bump fee, or std::nullopt
116+
* if it cannot be calculated. */
117+
std::optional<CAmount> CalculateTotalBumpFees(const CFeeRate& target_feerate);
118+
};
119+
} // namespace node
120+
121+
#endif // BITCOIN_NODE_MINI_MINER_H

src/test/fuzz/mini_miner.cpp

Lines changed: 192 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,192 @@
1+
#include <test/fuzz/FuzzedDataProvider.h>
2+
#include <test/fuzz/fuzz.h>
3+
#include <test/fuzz/util.h>
4+
#include <test/fuzz/util/mempool.h>
5+
#include <test/util/script.h>
6+
#include <test/util/setup_common.h>
7+
#include <test/util/txmempool.h>
8+
#include <test/util/mining.h>
9+
10+
#include <node/mini_miner.h>
11+
#include <node/miner.h>
12+
#include <primitives/transaction.h>
13+
#include <random.h>
14+
#include <txmempool.h>
15+
16+
#include <deque>
17+
#include <vector>
18+
19+
namespace {
20+
21+
const TestingSetup* g_setup;
22+
std::deque<COutPoint> g_available_coins;
23+
void initialize_miner()
24+
{
25+
static const auto testing_setup = MakeNoLogFileContext<const TestingSetup>();
26+
g_setup = testing_setup.get();
27+
for (uint32_t i = 0; i < uint32_t{100}; ++i) {
28+
g_available_coins.push_back(COutPoint{uint256::ZERO, i});
29+
}
30+
}
31+
32+
// Test that the MiniMiner can run with various outpoints and feerates.
33+
FUZZ_TARGET_INIT(mini_miner, initialize_miner)
34+
{
35+
FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()};
36+
CTxMemPool pool{CTxMemPool::Options{}};
37+
std::vector<COutPoint> outpoints;
38+
std::deque<COutPoint> available_coins = g_available_coins;
39+
LOCK2(::cs_main, pool.cs);
40+
// Cluster size cannot exceed 500
41+
LIMITED_WHILE(!available_coins.empty(), 500)
42+
{
43+
CMutableTransaction mtx = CMutableTransaction();
44+
const size_t num_inputs = fuzzed_data_provider.ConsumeIntegralInRange<size_t>(1, available_coins.size());
45+
const size_t num_outputs = fuzzed_data_provider.ConsumeIntegralInRange<size_t>(1, 50);
46+
for (size_t n{0}; n < num_inputs; ++n) {
47+
auto prevout = available_coins.front();
48+
mtx.vin.push_back(CTxIn(prevout, CScript()));
49+
available_coins.pop_front();
50+
}
51+
for (uint32_t n{0}; n < num_outputs; ++n) {
52+
mtx.vout.push_back(CTxOut(100, P2WSH_OP_TRUE));
53+
}
54+
CTransactionRef tx = MakeTransactionRef(mtx);
55+
TestMemPoolEntryHelper entry;
56+
const CAmount fee{ConsumeMoney(fuzzed_data_provider, /*max=*/MAX_MONEY/100000)};
57+
assert(MoneyRange(fee));
58+
pool.addUnchecked(entry.Fee(fee).FromTx(tx));
59+
60+
// All outputs are available to spend
61+
for (uint32_t n{0}; n < num_outputs; ++n) {
62+
if (fuzzed_data_provider.ConsumeBool()) {
63+
available_coins.push_back(COutPoint{tx->GetHash(), n});
64+
}
65+
}
66+
67+
if (fuzzed_data_provider.ConsumeBool() && !tx->vout.empty()) {
68+
// Add outpoint from this tx (may or not be spent by a later tx)
69+
outpoints.push_back(COutPoint{tx->GetHash(),
70+
(uint32_t)fuzzed_data_provider.ConsumeIntegralInRange<size_t>(0, tx->vout.size())});
71+
} else {
72+
// Add some random outpoint (will be interpreted as confirmed or not yet submitted
73+
// to mempool).
74+
auto outpoint = ConsumeDeserializable<COutPoint>(fuzzed_data_provider);
75+
if (outpoint.has_value() && std::find(outpoints.begin(), outpoints.end(), *outpoint) == outpoints.end()) {
76+
outpoints.push_back(*outpoint);
77+
}
78+
}
79+
80+
}
81+
82+
const CFeeRate target_feerate{CFeeRate{ConsumeMoney(fuzzed_data_provider, /*max=*/MAX_MONEY/1000)}};
83+
std::optional<CAmount> total_bumpfee;
84+
CAmount sum_fees = 0;
85+
{
86+
node::MiniMiner mini_miner{pool, outpoints};
87+
assert(mini_miner.IsReadyToCalculate());
88+
const auto bump_fees = mini_miner.CalculateBumpFees(target_feerate);
89+
for (const auto& outpoint : outpoints) {
90+
auto it = bump_fees.find(outpoint);
91+
assert(it != bump_fees.end());
92+
assert(it->second >= 0);
93+
sum_fees += it->second;
94+
}
95+
assert(!mini_miner.IsReadyToCalculate());
96+
}
97+
{
98+
node::MiniMiner mini_miner{pool, outpoints};
99+
assert(mini_miner.IsReadyToCalculate());
100+
total_bumpfee = mini_miner.CalculateTotalBumpFees(target_feerate);
101+
assert(total_bumpfee.has_value());
102+
assert(!mini_miner.IsReadyToCalculate());
103+
}
104+
// Overlapping ancestry across multiple outpoints can only reduce the total bump fee.
105+
assert (sum_fees >= *total_bumpfee);
106+
}
107+
108+
// Test that MiniMiner and BlockAssembler build the same block given the same transactions and constraints.
109+
FUZZ_TARGET_INIT(mini_miner_selection, initialize_miner)
110+
{
111+
FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()};
112+
CTxMemPool pool{CTxMemPool::Options{}};
113+
// Make a copy to preserve determinism.
114+
std::deque<COutPoint> available_coins = g_available_coins;
115+
std::vector<CTransactionRef> transactions;
116+
117+
LOCK2(::cs_main, pool.cs);
118+
LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 100)
119+
{
120+
CMutableTransaction mtx = CMutableTransaction();
121+
const size_t num_inputs = 2;
122+
const size_t num_outputs = fuzzed_data_provider.ConsumeIntegralInRange<size_t>(2, 5);
123+
for (size_t n{0}; n < num_inputs; ++n) {
124+
auto prevout = available_coins.front();
125+
mtx.vin.push_back(CTxIn(prevout, CScript()));
126+
available_coins.pop_front();
127+
}
128+
for (uint32_t n{0}; n < num_outputs; ++n) {
129+
mtx.vout.push_back(CTxOut(100, P2WSH_OP_TRUE));
130+
}
131+
CTransactionRef tx = MakeTransactionRef(mtx);
132+
133+
// First 2 outputs are available to spend. The rest are added to outpoints to calculate bumpfees.
134+
// There is no overlap between spendable coins and outpoints passed to MiniMiner because the
135+
// MiniMiner interprets spent coins as to-be-replaced and excludes them.
136+
for (uint32_t n{0}; n < num_outputs - 1; ++n) {
137+
if (fuzzed_data_provider.ConsumeBool()) {
138+
available_coins.push_front(COutPoint{tx->GetHash(), n});
139+
} else {
140+
available_coins.push_back(COutPoint{tx->GetHash(), n});
141+
}
142+
}
143+
144+
// Stop if pool reaches DEFAULT_BLOCK_MAX_WEIGHT because BlockAssembler will stop when the
145+
// block template reaches that, but the MiniMiner will keep going.
146+
if (pool.GetTotalTxSize() + GetVirtualTransactionSize(*tx) >= DEFAULT_BLOCK_MAX_WEIGHT) break;
147+
TestMemPoolEntryHelper entry;
148+
const CAmount fee{ConsumeMoney(fuzzed_data_provider, /*max=*/MAX_MONEY/100000)};
149+
assert(MoneyRange(fee));
150+
pool.addUnchecked(entry.Fee(fee).FromTx(tx));
151+
transactions.push_back(tx);
152+
}
153+
std::vector<COutPoint> outpoints;
154+
for (const auto& coin : g_available_coins) {
155+
if (!pool.GetConflictTx(coin)) outpoints.push_back(coin);
156+
}
157+
for (const auto& tx : transactions) {
158+
assert(pool.exists(GenTxid::Txid(tx->GetHash())));
159+
for (uint32_t n{0}; n < tx->vout.size(); ++n) {
160+
COutPoint coin{tx->GetHash(), n};
161+
if (!pool.GetConflictTx(coin)) outpoints.push_back(coin);
162+
}
163+
}
164+
const CFeeRate target_feerate{ConsumeMoney(fuzzed_data_provider, /*max=*/MAX_MONEY/100000)};
165+
166+
node::BlockAssembler::Options miner_options;
167+
miner_options.blockMinFeeRate = target_feerate;
168+
miner_options.nBlockMaxWeight = DEFAULT_BLOCK_MAX_WEIGHT;
169+
miner_options.test_block_validity = false;
170+
171+
node::BlockAssembler miner{g_setup->m_node.chainman->ActiveChainstate(), &pool, miner_options};
172+
node::MiniMiner mini_miner{pool, outpoints};
173+
assert(mini_miner.IsReadyToCalculate());
174+
175+
CScript spk_placeholder = CScript() << OP_0;
176+
// Use BlockAssembler as oracle. BlockAssembler and MiniMiner should select the same
177+
// transactions, stopping once packages do not meet target_feerate.
178+
const auto blocktemplate{miner.CreateNewBlock(spk_placeholder)};
179+
mini_miner.BuildMockTemplate(target_feerate);
180+
assert(!mini_miner.IsReadyToCalculate());
181+
auto mock_template_txids = mini_miner.GetMockTemplateTxids();
182+
// MiniMiner doesn't add a coinbase tx.
183+
assert(mock_template_txids.count(blocktemplate->block.vtx[0]->GetHash()) == 0);
184+
mock_template_txids.emplace(blocktemplate->block.vtx[0]->GetHash());
185+
assert(mock_template_txids.size() <= blocktemplate->block.vtx.size());
186+
assert(mock_template_txids.size() >= blocktemplate->block.vtx.size());
187+
assert(mock_template_txids.size() == blocktemplate->block.vtx.size());
188+
for (const auto& tx : blocktemplate->block.vtx) {
189+
assert(mock_template_txids.count(tx->GetHash()));
190+
}
191+
}
192+
} // namespace

0 commit comments

Comments
 (0)