From fc98f7a0a13c1246b67ed4d7fb40705bb68a8b93 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Wed, 7 Nov 2018 21:13:00 +0000 Subject: [PATCH] rpc: speedup get_outs.bin --- contrib/epee/include/span.h | 2 ++ src/blockchain_db/blockchain_db.h | 4 +-- src/blockchain_db/lmdb/db_lmdb.cpp | 14 ++++++---- src/blockchain_db/lmdb/db_lmdb.h | 2 +- src/cryptonote_core/blockchain.cpp | 34 +++++++++++++++++++------ src/rpc/core_rpc_server_commands_defs.h | 4 ++- src/wallet/wallet2.cpp | 1 + tests/unit_tests/testdb.h | 2 +- 8 files changed, 45 insertions(+), 18 deletions(-) diff --git a/contrib/epee/include/span.h b/contrib/epee/include/span.h index 174915ecf..b1296a0b7 100644 --- a/contrib/epee/include/span.h +++ b/contrib/epee/include/span.h @@ -109,6 +109,8 @@ namespace epee constexpr std::size_t size() const noexcept { return len; } constexpr std::size_t size_bytes() const noexcept { return size() * sizeof(value_type); } + const T &operator[](size_t idx) const { return ptr[idx]; } + private: T* ptr; std::size_t len; diff --git a/src/blockchain_db/blockchain_db.h b/src/blockchain_db/blockchain_db.h index 7118b0881..53e33898a 100644 --- a/src/blockchain_db/blockchain_db.h +++ b/src/blockchain_db/blockchain_db.h @@ -1307,11 +1307,11 @@ public: * get_output_data(const uint64_t& amount, const uint64_t& index) * but for a list of outputs rather than just one. * - * @param amount an output amount + * @param amounts an output amount, or as many as offsets * @param offsets a list of amount-specific output indices * @param outputs return-by-reference a list of outputs' metadata */ - virtual void get_output_key(const uint64_t &amount, const std::vector &offsets, std::vector &outputs, bool allow_partial = false) = 0; + virtual void get_output_key(const epee::span &amounts, const std::vector &offsets, std::vector &outputs, bool allow_partial = false) = 0; /* * FIXME: Need to check with git blame and ask what this does to diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index ea3638a85..19f2de4dc 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -3197,8 +3197,11 @@ void BlockchainLMDB::get_output_tx_and_index_from_global(const std::vector &offsets, std::vector &outputs, bool allow_partial) +void BlockchainLMDB::get_output_key(const epee::span &amounts, const std::vector &offsets, std::vector &outputs, bool allow_partial) { + if (amounts.size() != 1 && amounts.size() != offsets.size()) + throw0(DB_ERROR("Invalid sizes of amounts and offets")); + LOG_PRINT_L3("BlockchainLMDB::" << __func__); TIME_MEASURE_START(db3); check_open(); @@ -3209,10 +3212,11 @@ void BlockchainLMDB::get_output_key(const uint64_t &amount, const std::vector(amount) + ", index " + boost::lexical_cast(index) + ", count " + boost::lexical_cast(get_num_outputs(amount)) + "), but key does not exist (current height " + boost::lexical_cast(height()) + ")").c_str())); + throw1(OUTPUT_DNE((std::string("Attempting to get output pubkey by global index (amount ") + boost::lexical_cast(amount) + ", index " + boost::lexical_cast(offsets[i]) + ", count " + boost::lexical_cast(get_num_outputs(amount)) + "), but key does not exist (current height " + boost::lexical_cast(height()) + ")").c_str())); } else if (get_result) throw0(DB_ERROR(lmdb_error("Error attempting to retrieve an output pubkey from the db", get_result).c_str())); diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h index 7e76236a5..26159ab4d 100644 --- a/src/blockchain_db/lmdb/db_lmdb.h +++ b/src/blockchain_db/lmdb/db_lmdb.h @@ -243,7 +243,7 @@ public: virtual uint64_t get_num_outputs(const uint64_t& amount) const; virtual output_data_t get_output_key(const uint64_t& amount, const uint64_t& index); - virtual void get_output_key(const uint64_t &amount, const std::vector &offsets, std::vector &outputs, bool allow_partial = false); + virtual void get_output_key(const epee::span &amounts, const std::vector &offsets, std::vector &outputs, bool allow_partial = false); virtual tx_out_index get_output_tx_and_index_from_global(const uint64_t& index) const; virtual void get_output_tx_and_index_from_global(const std::vector &global_indices, diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index e80e3f66c..bfc5dbbe0 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -229,7 +229,7 @@ bool Blockchain::scan_outputkeys_for_indexes(size_t tx_version, const txin_to_ke { try { - m_db->get_output_key(tx_in_to_key.amount, absolute_offsets, outputs, true); + m_db->get_output_key(epee::span(&tx_in_to_key.amount, 1), absolute_offsets, outputs, true); if (absolute_offsets.size() != outputs.size()) { MERROR_VER("Output does not exist! amount = " << tx_in_to_key.amount); @@ -255,7 +255,7 @@ bool Blockchain::scan_outputkeys_for_indexes(size_t tx_version, const txin_to_ke add_offsets.push_back(absolute_offsets[i]); try { - m_db->get_output_key(tx_in_to_key.amount, add_offsets, add_outputs, true); + m_db->get_output_key(epee::span(&tx_in_to_key.amount, 1), add_offsets, add_outputs, true); if (add_offsets.size() != add_outputs.size()) { MERROR_VER("Output does not exist! amount = " << tx_in_to_key.amount); @@ -1767,16 +1767,34 @@ bool Blockchain::get_outs(const COMMAND_RPC_GET_OUTPUTS_BIN::request& req, COMMA res.outs.clear(); res.outs.reserve(req.outputs.size()); + + std::vector data; try { + std::vector amounts, offsets; + amounts.reserve(req.outputs.size()); + offsets.reserve(req.outputs.size()); for (const auto &i: req.outputs) { - // get tx_hash, tx_out_index from DB - const output_data_t od = m_db->get_output_key(i.amount, i.index); - tx_out_index toi = m_db->get_output_tx_and_index(i.amount, i.index); - bool unlocked = is_tx_spendtime_unlocked(m_db->get_tx_unlock_time(toi.first)); + amounts.push_back(i.amount); + offsets.push_back(i.index); + } + m_db->get_output_key(epee::span(amounts.data(), amounts.size()), offsets, data); + if (data.size() != req.outputs.size()) + { + MERROR("Unexpected output data size: expected " << req.outputs.size() << ", got " << data.size()); + return false; + } + for (const auto &t: data) + res.outs.push_back({t.pubkey, t.commitment, is_tx_spendtime_unlocked(t.unlock_time), t.height, crypto::null_hash}); - res.outs.push_back({od.pubkey, od.commitment, unlocked, od.height, toi.first}); + if (req.get_txid) + { + for (size_t i = 0; i < req.outputs.size(); ++i) + { + tx_out_index toi = m_db->get_output_tx_and_index(req.outputs[i].amount, req.outputs[i].index); + res.outs[i].txid = toi.first; + } } } catch (const std::exception &e) @@ -3795,7 +3813,7 @@ void Blockchain::output_scan_worker(const uint64_t amount, const std::vectorget_output_key(amount, offsets, outputs, true); + m_db->get_output_key(epee::span(&amount, 1), offsets, outputs, true); } catch (const std::exception& e) { diff --git a/src/rpc/core_rpc_server_commands_defs.h b/src/rpc/core_rpc_server_commands_defs.h index 8e8df7a52..94caab9f8 100644 --- a/src/rpc/core_rpc_server_commands_defs.h +++ b/src/rpc/core_rpc_server_commands_defs.h @@ -50,7 +50,7 @@ namespace cryptonote // advance which version they will stop working with // Don't go over 32767 for any of these #define CORE_RPC_VERSION_MAJOR 2 -#define CORE_RPC_VERSION_MINOR 1 +#define CORE_RPC_VERSION_MINOR 2 #define MAKE_CORE_RPC_VERSION(major,minor) (((major)<<16)|(minor)) #define CORE_RPC_VERSION MAKE_CORE_RPC_VERSION(CORE_RPC_VERSION_MAJOR, CORE_RPC_VERSION_MINOR) @@ -697,9 +697,11 @@ namespace cryptonote struct request { std::vector outputs; + bool get_txid; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(outputs) + KV_SERIALIZE_OPT(get_txid, true) END_KV_SERIALIZE_MAP() }; diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index 31be95a61..1638e920d 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -7185,6 +7185,7 @@ void wallet2::get_outs(std::vector> } // get the keys for those + req.get_txid = false; m_daemon_rpc_mutex.lock(); bool r = epee::net_utils::invoke_http_bin("/get_outs.bin", req, daemon_resp, m_http_client, rpc_timeout); m_daemon_rpc_mutex.unlock(); diff --git a/tests/unit_tests/testdb.h b/tests/unit_tests/testdb.h index b6962cc41..a9c772920 100644 --- a/tests/unit_tests/testdb.h +++ b/tests/unit_tests/testdb.h @@ -93,7 +93,7 @@ public: virtual cryptonote::tx_out_index get_output_tx_and_index_from_global(const uint64_t& index) const { return cryptonote::tx_out_index(); } virtual cryptonote::tx_out_index get_output_tx_and_index(const uint64_t& amount, const uint64_t& index) const { return cryptonote::tx_out_index(); } virtual void get_output_tx_and_index(const uint64_t& amount, const std::vector &offsets, std::vector &indices) const {} - virtual void get_output_key(const uint64_t &amount, const std::vector &offsets, std::vector &outputs, bool allow_partial = false) {} + virtual void get_output_key(const epee::span &amounts, const std::vector &offsets, std::vector &outputs, bool allow_partial = false) {} virtual bool can_thread_bulk_indices() const { return false; } virtual std::vector get_tx_output_indices(const crypto::hash& h) const { return std::vector(); } virtual std::vector get_tx_amount_output_indices(const uint64_t tx_index) const { return std::vector(); }