From 50cb370d5babcef5caa9f90981c475ca5fe3f887 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Sun, 12 Aug 2018 21:41:44 +0000 Subject: [PATCH] ringdb: allow blackballing many outputs at once It cuts down on txn commits, and speeds up blackballing substantially --- .../blockchain_blackball.cpp | 21 +++- src/wallet/ringdb.cpp | 95 +++++++++++-------- src/wallet/ringdb.h | 3 +- src/wallet/wallet2.cpp | 3 +- 4 files changed, 77 insertions(+), 45 deletions(-) diff --git a/src/blockchain_utilities/blockchain_blackball.cpp b/src/blockchain_utilities/blockchain_blackball.cpp index 1653910fc..04a894fe4 100644 --- a/src/blockchain_utilities/blockchain_blackball.cpp +++ b/src/blockchain_utilities/blockchain_blackball.cpp @@ -417,6 +417,7 @@ int main(int argc, char* argv[]) if (it != state.processed_heights.end()) start_idx = it->second; LOG_PRINT_L0("Reading blockchain from " << inputs[n] << " from " << start_idx); + std::vector blackballs; for_all_transactions(inputs[n], start_idx, [&](const cryptonote::transaction_prefix &tx)->bool { for (const auto &in: tx.vin) @@ -439,7 +440,7 @@ int main(int argc, char* argv[]) { const crypto::public_key pkey = core_storage[n]->get_output_key(txin.amount, absolute[0]); MINFO("Blackballing output " << pkey << ", due to being used in a 1-ring"); - ringdb.blackball(pkey); + blackballs.push_back(pkey); newly_spent.insert(output_data(txin.amount, absolute[0])); } else if (state.ring_instances[new_ring] == new_ring.size()) @@ -448,7 +449,7 @@ int main(int argc, char* argv[]) { const crypto::public_key pkey = core_storage[n]->get_output_key(txin.amount, absolute[o]); MINFO("Blackballing output " << pkey << ", due to being used in " << new_ring.size() << " identical " << new_ring.size() << "-rings"); - ringdb.blackball(pkey); + blackballs.push_back(pkey); newly_spent.insert(output_data(txin.amount, absolute[o])); } } @@ -476,7 +477,7 @@ int main(int argc, char* argv[]) { const crypto::public_key pkey = core_storage[n]->get_output_key(txin.amount, common[0]); MINFO("Blackballing output " << pkey << ", due to being used in rings with a single common element"); - ringdb.blackball(pkey); + blackballs.push_back(pkey); newly_spent.insert(output_data(txin.amount, common[0])); } else @@ -490,6 +491,11 @@ int main(int argc, char* argv[]) } } state.relative_rings[txin.k_image] = new_ring; + if (!blackballs.empty()) + { + ringdb.blackball(blackballs); + blackballs.clear(); + } } if (stop_requested) { @@ -513,6 +519,7 @@ int main(int argc, char* argv[]) for (const auto &e: work_spent) state.spent.insert(e); + std::vector blackballs; for (const output_data &od: work_spent) { for (const crypto::key_image &ki: state.outputs[od]) @@ -533,13 +540,19 @@ int main(int argc, char* argv[]) const crypto::public_key pkey = core_storage[0]->get_output_key(od.amount, last_unknown); MINFO("Blackballing output " << pkey << ", due to being used in a " << absolute.size() << "-ring where all other outputs are known to be spent"); - ringdb.blackball(pkey); + blackballs.push_back(pkey); newly_spent.insert(output_data(od.amount, last_unknown)); } } } } + if (!blackballs.empty()) + { + ringdb.blackball(blackballs); + blackballs.clear(); + } + LOG_PRINT_L0("Saving state data to " << state_file_path); std::ofstream state_data_out; state_data_out.open(state_file_path, std::ios_base::binary | std::ios_base::out | std::ios::trunc); diff --git a/src/wallet/ringdb.cpp b/src/wallet/ringdb.cpp index 3f2634c8b..bf5478f5e 100644 --- a/src/wallet/ringdb.cpp +++ b/src/wallet/ringdb.cpp @@ -146,7 +146,7 @@ static int resize_env(MDB_env *env, const char *db_path, size_t needed) MDB_stat mst; int ret; - needed = std::max(needed, (size_t)(2ul * 1024 * 1024)); // at least 2 MB + needed = std::max(needed, (size_t)(100ul * 1024 * 1024)); // at least 100 MB ret = mdb_env_info(env, &mei); if (ret) @@ -374,7 +374,7 @@ bool ringdb::set_ring(const crypto::chacha_key &chacha_key, const crypto::key_im return true; } -bool ringdb::blackball_worker(const crypto::public_key &output, int op) +bool ringdb::blackball_worker(const std::vector &outputs, int op) { MDB_txn *txn; MDB_cursor *cursor; @@ -382,7 +382,9 @@ bool ringdb::blackball_worker(const crypto::public_key &output, int op) bool tx_active = false; bool ret = true; - dbr = resize_env(env, filename.c_str(), 32 * 2); // a pubkey, and some slack + THROW_WALLET_EXCEPTION_IF(outputs.size() > 1 && op == BLACKBALL_QUERY, tools::error::wallet_internal_error, "Blackball query only makes sense for a single output"); + + dbr = resize_env(env, filename.c_str(), 32 * 2 * outputs.size()); // a pubkey, and some slack THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to set env map size: " + std::string(mdb_strerror(dbr))); dbr = mdb_txn_begin(env, NULL, 0, &txn); THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); @@ -391,40 +393,49 @@ bool ringdb::blackball_worker(const crypto::public_key &output, int op) MDB_val key = zerokeyval; MDB_val data; - data.mv_data = (void*)&output; - data.mv_size = sizeof(output); - switch (op) + for (const crypto::public_key &output: outputs) + { + data.mv_data = (void*)&output; + data.mv_size = sizeof(output); + + switch (op) + { + case BLACKBALL_BLACKBALL: + MDEBUG("Blackballing output " << output); + dbr = mdb_put(txn, dbi_blackballs, &key, &data, MDB_NODUPDATA); + if (dbr == MDB_KEYEXIST) + dbr = 0; + break; + case BLACKBALL_UNBLACKBALL: + MDEBUG("Unblackballing output " << output); + dbr = mdb_del(txn, dbi_blackballs, &key, &data); + if (dbr == MDB_NOTFOUND) + dbr = 0; + break; + case BLACKBALL_QUERY: + dbr = mdb_cursor_open(txn, dbi_blackballs, &cursor); + THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to create cursor for blackballs table: " + std::string(mdb_strerror(dbr))); + dbr = mdb_cursor_get(cursor, &key, &data, MDB_GET_BOTH); + THROW_WALLET_EXCEPTION_IF(dbr && dbr != MDB_NOTFOUND, tools::error::wallet_internal_error, "Failed to lookup in blackballs table: " + std::string(mdb_strerror(dbr))); + ret = dbr != MDB_NOTFOUND; + if (dbr == MDB_NOTFOUND) + dbr = 0; + mdb_cursor_close(cursor); + break; + case BLACKBALL_CLEAR: + break; + default: + THROW_WALLET_EXCEPTION(tools::error::wallet_internal_error, "Invalid blackball op"); + } + THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to query blackballs table: " + std::string(mdb_strerror(dbr))); + } + + if (op == BLACKBALL_CLEAR) { - case BLACKBALL_BLACKBALL: - MDEBUG("Blackballing output " << output); - dbr = mdb_put(txn, dbi_blackballs, &key, &data, MDB_NODUPDATA); - if (dbr == MDB_KEYEXIST) - dbr = 0; - break; - case BLACKBALL_UNBLACKBALL: - MDEBUG("Unblackballing output " << output); - dbr = mdb_del(txn, dbi_blackballs, &key, &data); - if (dbr == MDB_NOTFOUND) - dbr = 0; - break; - case BLACKBALL_QUERY: - dbr = mdb_cursor_open(txn, dbi_blackballs, &cursor); - THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to create cursor for blackballs table: " + std::string(mdb_strerror(dbr))); - dbr = mdb_cursor_get(cursor, &key, &data, MDB_GET_BOTH); - THROW_WALLET_EXCEPTION_IF(dbr && dbr != MDB_NOTFOUND, tools::error::wallet_internal_error, "Failed to lookup in blackballs table: " + std::string(mdb_strerror(dbr))); - ret = dbr != MDB_NOTFOUND; - if (dbr == MDB_NOTFOUND) - dbr = 0; - mdb_cursor_close(cursor); - break; - case BLACKBALL_CLEAR: - dbr = mdb_drop(txn, dbi_blackballs, 0); - break; - default: - THROW_WALLET_EXCEPTION(tools::error::wallet_internal_error, "Invalid blackball op"); + dbr = mdb_drop(txn, dbi_blackballs, 0); + THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to clear blackballs table: " + std::string(mdb_strerror(dbr))); } - THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to query blackballs table: " + std::string(mdb_strerror(dbr))); dbr = mdb_txn_commit(txn); THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to commit txn blackballing output to database: " + std::string(mdb_strerror(dbr))); @@ -432,24 +443,32 @@ bool ringdb::blackball_worker(const crypto::public_key &output, int op) return ret; } +bool ringdb::blackball(const std::vector &outputs) +{ + return blackball_worker(outputs, BLACKBALL_BLACKBALL); +} + bool ringdb::blackball(const crypto::public_key &output) { - return blackball_worker(output, BLACKBALL_BLACKBALL); + std::vector outputs(1, output); + return blackball_worker(outputs, BLACKBALL_BLACKBALL); } bool ringdb::unblackball(const crypto::public_key &output) { - return blackball_worker(output, BLACKBALL_UNBLACKBALL); + std::vector outputs(1, output); + return blackball_worker(outputs, BLACKBALL_UNBLACKBALL); } bool ringdb::blackballed(const crypto::public_key &output) { - return blackball_worker(output, BLACKBALL_QUERY); + std::vector outputs(1, output); + return blackball_worker(outputs, BLACKBALL_QUERY); } bool ringdb::clear_blackballs() { - return blackball_worker(crypto::public_key(), BLACKBALL_CLEAR); + return blackball_worker(std::vector(), BLACKBALL_CLEAR); } } diff --git a/src/wallet/ringdb.h b/src/wallet/ringdb.h index 6b4bce124..4a78f61f8 100644 --- a/src/wallet/ringdb.h +++ b/src/wallet/ringdb.h @@ -50,12 +50,13 @@ namespace tools bool set_ring(const crypto::chacha_key &chacha_key, const crypto::key_image &key_image, const std::vector &outs, bool relative); bool blackball(const crypto::public_key &output); + bool blackball(const std::vector &outputs); bool unblackball(const crypto::public_key &output); bool blackballed(const crypto::public_key &output); bool clear_blackballs(); private: - bool blackball_worker(const crypto::public_key &output, int op); + bool blackball_worker(const std::vector &outputs, int op); private: std::string filename; diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index a2e36706e..ffd7131cd 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -6238,8 +6238,7 @@ bool wallet2::set_blackballed_outputs(const std::vector &out bool ret = true; if (!add) ret &= m_ringdb->clear_blackballs(); - for (const auto &output: outputs) - ret &= m_ringdb->blackball(output); + ret &= m_ringdb->blackball(outputs); return ret; } catch (const std::exception &e) { return false; }