From 218a109677914c69797d5ed55167d973b8837fde Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Tue, 29 Dec 2020 04:31:58 +0000 Subject: [PATCH] rpc: limit the number of txes for get_blocks.bin --- src/blockchain_db/blockchain_db.h | 9 ++++++--- src/blockchain_db/lmdb/db_lmdb.cpp | 11 ++++++++--- src/blockchain_db/lmdb/db_lmdb.h | 2 +- src/blockchain_db/testdb.h | 2 +- src/cryptonote_config.h | 3 ++- src/cryptonote_core/blockchain.cpp | 6 +++--- src/cryptonote_core/blockchain.h | 5 +++-- src/cryptonote_core/cryptonote_core.cpp | 4 ++-- src/cryptonote_core/cryptonote_core.h | 2 +- src/rpc/core_rpc_server.cpp | 8 ++++---- src/rpc/daemon_handler.cpp | 2 +- 11 files changed, 32 insertions(+), 22 deletions(-) diff --git a/src/blockchain_db/blockchain_db.h b/src/blockchain_db/blockchain_db.h index abebb52b4..74bd72332 100644 --- a/src/blockchain_db/blockchain_db.h +++ b/src/blockchain_db/blockchain_db.h @@ -1312,17 +1312,20 @@ public: * height. The number of blocks returned is variable, based on the max_size passed. * * @param start_height the height of the first block - * @param min_count the minimum number of blocks to return, if they exist - * @param max_count the maximum number of blocks to return + * @param min_block_count the minimum number of blocks to return, if they exist + * @param max_block_count the maximum number of blocks to return + * @param max_tx_count the maximum number of txes to return * @param max_size the maximum size of block/transaction data to return (will be exceeded by one blocks's worth at most, if min_count is met) * @param blocks the returned block/transaction data * @param pruned whether to return full or pruned tx data * @param skip_coinbase whether to return or skip coinbase transactions (they're in blocks regardless) * @param get_miner_tx_hash whether to calculate and return the miner (coinbase) tx hash * + * The call will return at least min_block_count if possible, even if this contravenes max_tx_count + * * @return true iff the blocks and transactions were found */ - virtual bool get_blocks_from(uint64_t start_height, size_t min_count, size_t max_count, size_t max_size, std::vector, std::vector>>>& blocks, bool pruned, bool skip_coinbase, bool get_miner_tx_hash) const = 0; + virtual bool get_blocks_from(uint64_t start_height, size_t min_block_count, size_t max_block_count, size_t max_tx_count, size_t max_size, std::vector, std::vector>>>& blocks, bool pruned, bool skip_coinbase, bool get_miner_tx_hash) const = 0; /** * @brief fetches the prunable transaction blob with the given hash diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index 8aa958825..4865ec952 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -3171,7 +3171,7 @@ bool BlockchainLMDB::get_pruned_tx_blobs_from(const crypto::hash& h, size_t coun return true; } -bool BlockchainLMDB::get_blocks_from(uint64_t start_height, size_t min_count, size_t max_count, size_t max_size, std::vector, std::vector>>>& blocks, bool pruned, bool skip_coinbase, bool get_miner_tx_hash) const +bool BlockchainLMDB::get_blocks_from(uint64_t start_height, size_t min_block_count, size_t max_block_count, size_t max_tx_count, size_t max_size, std::vector, std::vector>>>& blocks, bool pruned, bool skip_coinbase, bool get_miner_tx_hash) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); @@ -3185,14 +3185,15 @@ bool BlockchainLMDB::get_blocks_from(uint64_t start_height, size_t min_count, si RCURSOR(txs_prunable); } - blocks.reserve(std::min(max_count, 10000)); // guard against very large max count if only checking bytes + blocks.reserve(std::min(max_block_count, 10000)); // guard against very large max count if only checking bytes const uint64_t blockchain_height = height(); uint64_t size = 0; + size_t num_txes = 0; MDB_val_copy key(start_height); MDB_val k, v, val_tx_id; uint64_t tx_id = ~0; MDB_cursor_op op = MDB_SET; - for (uint64_t h = start_height; h < blockchain_height && blocks.size() < max_count && (size < max_size || blocks.size() < min_count); ++h) + for (uint64_t h = start_height; h < blockchain_height && blocks.size() < max_block_count && (size < max_size || blocks.size() < min_block_count); ++h) { MDB_cursor_op op = h == start_height ? MDB_SET : MDB_NEXT; int result = mdb_cursor_get(m_cur_blocks, &key, &v, op); @@ -3243,6 +3244,7 @@ bool BlockchainLMDB::get_blocks_from(uint64_t start_height, size_t min_count, si op = MDB_NEXT; current_block.second.reserve(b.tx_hashes.size()); + num_txes += b.tx_hashes.size() + (skip_coinbase ? 0 : 1); for (const auto &tx_hash: b.tx_hashes) { // get pruned data @@ -3262,6 +3264,9 @@ bool BlockchainLMDB::get_blocks_from(uint64_t start_height, size_t min_count, si current_block.second.push_back(std::make_pair(tx_hash, std::move(tx_blob))); size += current_block.second.back().second.size(); } + + if (blocks.size() >= min_block_count && num_txes >= max_tx_count) + break; } TXN_POSTFIX_RDONLY(); diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h index 568882ae5..0e6d70039 100644 --- a/src/blockchain_db/lmdb/db_lmdb.h +++ b/src/blockchain_db/lmdb/db_lmdb.h @@ -257,7 +257,7 @@ public: virtual bool get_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const; virtual bool get_pruned_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const; virtual bool get_pruned_tx_blobs_from(const crypto::hash& h, size_t count, std::vector &bd) const; - virtual bool get_blocks_from(uint64_t start_height, size_t min_count, size_t max_count, size_t max_size, std::vector, std::vector>>>& blocks, bool pruned, bool skip_coinbase, bool get_miner_tx_hash) const; + virtual bool get_blocks_from(uint64_t start_height, size_t min_block_count, size_t max_block_count, size_t max_tx_count, size_t max_size, std::vector, std::vector>>>& blocks, bool pruned, bool skip_coinbase, bool get_miner_tx_hash) const; virtual bool get_prunable_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const; virtual bool get_prunable_tx_hash(const crypto::hash& tx_hash, crypto::hash &prunable_hash) const; diff --git a/src/blockchain_db/testdb.h b/src/blockchain_db/testdb.h index 9e74b33f1..e36026e07 100644 --- a/src/blockchain_db/testdb.h +++ b/src/blockchain_db/testdb.h @@ -70,7 +70,7 @@ public: virtual bool get_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const override { return false; } virtual bool get_pruned_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const override { return false; } virtual bool get_pruned_tx_blobs_from(const crypto::hash& h, size_t count, std::vector &bd) const override { return false; } - virtual bool get_blocks_from(uint64_t start_height, size_t min_count, size_t max_count, size_t max_size, std::vector, std::vector>>>& blocks, bool pruned, bool skip_coinbase, bool get_miner_tx_hash) const override { return false; } + virtual bool get_blocks_from(uint64_t start_height, size_t min_block_count, size_t max_block_count, size_t max_tx_count, size_t max_size, std::vector, std::vector>>>& blocks, bool pruned, bool skip_coinbase, bool get_miner_tx_hash) const override { return false; } virtual bool get_prunable_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const override { return false; } virtual bool get_prunable_tx_hash(const crypto::hash& tx_hash, crypto::hash &prunable_hash) const override { return false; } virtual uint64_t get_block_height(const crypto::hash& h) const override { return 0; } diff --git a/src/cryptonote_config.h b/src/cryptonote_config.h index 97706f745..2cb28b2b1 100644 --- a/src/cryptonote_config.h +++ b/src/cryptonote_config.h @@ -124,7 +124,8 @@ #define CRYPTONOTE_MAX_FRAGMENTS 20 // ~20 * NOISE_BYTES max payload size for covert/noise send -#define COMMAND_RPC_GET_BLOCKS_FAST_MAX_COUNT 1000 +#define COMMAND_RPC_GET_BLOCKS_FAST_MAX_BLOCK_COUNT 1000 +#define COMMAND_RPC_GET_BLOCKS_FAST_MAX_TX_COUNT 20000 #define P2P_LOCAL_WHITE_PEERLIST_LIMIT 1000 #define P2P_LOCAL_GRAY_PEERLIST_LIMIT 5000 diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index 1e6df49da..594aa1e9c 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -2701,7 +2701,7 @@ bool Blockchain::find_blockchain_supplement(const std::list& qbloc // find split point between ours and foreign blockchain (or start at // blockchain height ), and return up to max_count FULL // blocks by reference. -bool Blockchain::find_blockchain_supplement(const uint64_t req_start_block, const std::list& qblock_ids, std::vector, std::vector > > >& blocks, uint64_t& total_height, uint64_t& start_height, bool pruned, bool get_miner_tx_hash, size_t max_count) const +bool Blockchain::find_blockchain_supplement(const uint64_t req_start_block, const std::list& qblock_ids, std::vector, std::vector > > >& blocks, uint64_t& total_height, uint64_t& start_height, bool pruned, bool get_miner_tx_hash, size_t max_block_count, size_t max_tx_count) const { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -2726,8 +2726,8 @@ bool Blockchain::find_blockchain_supplement(const uint64_t req_start_block, cons db_rtxn_guard rtxn_guard(m_db); total_height = get_current_blockchain_height(); - blocks.reserve(std::min(std::min(max_count, (size_t)10000), (size_t)(total_height - start_height))); - CHECK_AND_ASSERT_MES(m_db->get_blocks_from(start_height, 3, max_count, FIND_BLOCKCHAIN_SUPPLEMENT_MAX_SIZE, blocks, pruned, true, get_miner_tx_hash), + blocks.reserve(std::min(std::min(max_block_count, (size_t)10000), (size_t)(total_height - start_height))); + CHECK_AND_ASSERT_MES(m_db->get_blocks_from(start_height, 3, max_block_count, max_tx_count, FIND_BLOCKCHAIN_SUPPLEMENT_MAX_SIZE, blocks, pruned, true, get_miner_tx_hash), false, "Error getting blocks"); return true; diff --git a/src/cryptonote_core/blockchain.h b/src/cryptonote_core/blockchain.h index a9b7ca1da..66c3bc395 100644 --- a/src/cryptonote_core/blockchain.h +++ b/src/cryptonote_core/blockchain.h @@ -464,11 +464,12 @@ namespace cryptonote * @param total_height return-by-reference our current blockchain height * @param start_height return-by-reference the height of the first block returned * @param pruned whether to return full or pruned tx blobs - * @param max_count the max number of blocks to get + * @param max_block_count the max number of blocks to get + * @param max_tx_count the max number of txes to get (it can get overshot by the last block's number of txes minus 1) * * @return true if a block found in common or req_start_block specified, else false */ - bool find_blockchain_supplement(const uint64_t req_start_block, const std::list& qblock_ids, std::vector, std::vector > > >& blocks, uint64_t& total_height, uint64_t& start_height, bool pruned, bool get_miner_tx_hash, size_t max_count) const; + bool find_blockchain_supplement(const uint64_t req_start_block, const std::list& qblock_ids, std::vector, std::vector > > >& blocks, uint64_t& total_height, uint64_t& start_height, bool pruned, bool get_miner_tx_hash, size_t max_block_count, size_t max_tx_count) const; /** * @brief retrieves a set of blocks and their transactions, and possibly other transactions diff --git a/src/cryptonote_core/cryptonote_core.cpp b/src/cryptonote_core/cryptonote_core.cpp index 42b1c6c84..507c47a51 100644 --- a/src/cryptonote_core/cryptonote_core.cpp +++ b/src/cryptonote_core/cryptonote_core.cpp @@ -1406,9 +1406,9 @@ namespace cryptonote return m_blockchain_storage.find_blockchain_supplement(qblock_ids, clip_pruned, resp); } //----------------------------------------------------------------------------------------------- - bool core::find_blockchain_supplement(const uint64_t req_start_block, const std::list& qblock_ids, std::vector, std::vector > > >& blocks, uint64_t& total_height, uint64_t& start_height, bool pruned, bool get_miner_tx_hash, size_t max_count) const + bool core::find_blockchain_supplement(const uint64_t req_start_block, const std::list& qblock_ids, std::vector, std::vector > > >& blocks, uint64_t& total_height, uint64_t& start_height, bool pruned, bool get_miner_tx_hash, size_t max_block_count, size_t max_tx_count) const { - return m_blockchain_storage.find_blockchain_supplement(req_start_block, qblock_ids, blocks, total_height, start_height, pruned, get_miner_tx_hash, max_count); + return m_blockchain_storage.find_blockchain_supplement(req_start_block, qblock_ids, blocks, total_height, start_height, pruned, get_miner_tx_hash, max_block_count, max_tx_count); } //----------------------------------------------------------------------------------------------- bool core::get_outs(const COMMAND_RPC_GET_OUTPUTS_BIN::request& req, COMMAND_RPC_GET_OUTPUTS_BIN::response& res) const diff --git a/src/cryptonote_core/cryptonote_core.h b/src/cryptonote_core/cryptonote_core.h index da7a21b27..fe55d76bf 100644 --- a/src/cryptonote_core/cryptonote_core.h +++ b/src/cryptonote_core/cryptonote_core.h @@ -564,7 +564,7 @@ namespace cryptonote * * @note see Blockchain::find_blockchain_supplement(const uint64_t, const std::list&, std::vector > >&, uint64_t&, uint64_t&, size_t) const */ - bool find_blockchain_supplement(const uint64_t req_start_block, const std::list& qblock_ids, std::vector, std::vector > > >& blocks, uint64_t& total_height, uint64_t& start_height, bool pruned, bool get_miner_tx_hash, size_t max_count) const; + bool find_blockchain_supplement(const uint64_t req_start_block, const std::list& qblock_ids, std::vector, std::vector > > >& blocks, uint64_t& total_height, uint64_t& start_height, bool pruned, bool get_miner_tx_hash, size_t max_block_count, size_t max_tx_count) const; /** * @copydoc Blockchain::get_tx_outputs_gindexs diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index 1abeb3903..664f058e2 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -565,12 +565,12 @@ namespace cryptonote } } - size_t max_blocks = COMMAND_RPC_GET_BLOCKS_FAST_MAX_COUNT; + size_t max_blocks = COMMAND_RPC_GET_BLOCKS_FAST_MAX_BLOCK_COUNT; if (m_rpc_payment) { max_blocks = res.credits / COST_PER_BLOCK; - if (max_blocks > COMMAND_RPC_GET_BLOCKS_FAST_MAX_COUNT) - max_blocks = COMMAND_RPC_GET_BLOCKS_FAST_MAX_COUNT; + if (max_blocks > COMMAND_RPC_GET_BLOCKS_FAST_MAX_BLOCK_COUNT) + max_blocks = COMMAND_RPC_GET_BLOCKS_FAST_MAX_BLOCK_COUNT; if (max_blocks == 0) { res.status = CORE_RPC_STATUS_PAYMENT_REQUIRED; @@ -579,7 +579,7 @@ namespace cryptonote } std::vector, std::vector > > > bs; - if(!m_core.find_blockchain_supplement(req.start_height, req.block_ids, bs, res.current_height, res.start_height, req.prune, !req.no_miner_tx, max_blocks)) + if(!m_core.find_blockchain_supplement(req.start_height, req.block_ids, bs, res.current_height, res.start_height, req.prune, !req.no_miner_tx, max_blocks, COMMAND_RPC_GET_BLOCKS_FAST_MAX_TX_COUNT)) { res.status = "Failed"; add_host_fail(ctx); diff --git a/src/rpc/daemon_handler.cpp b/src/rpc/daemon_handler.cpp index e256322cb..cdfc363a9 100644 --- a/src/rpc/daemon_handler.cpp +++ b/src/rpc/daemon_handler.cpp @@ -128,7 +128,7 @@ namespace rpc { std::vector, std::vector > > > blocks; - if(!m_core.find_blockchain_supplement(req.start_height, req.block_ids, blocks, res.current_height, res.start_height, req.prune, true, COMMAND_RPC_GET_BLOCKS_FAST_MAX_COUNT)) + if(!m_core.find_blockchain_supplement(req.start_height, req.block_ids, blocks, res.current_height, res.start_height, req.prune, true, COMMAND_RPC_GET_BLOCKS_FAST_MAX_BLOCK_COUNT, COMMAND_RPC_GET_BLOCKS_FAST_MAX_TX_COUNT)) { res.status = Message::STATUS_FAILED; res.error_details = "core::find_blockchain_supplement() returned false";