From 50cb370d5babcef5caa9f90981c475ca5fe3f887 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Sun, 12 Aug 2018 21:41:44 +0000 Subject: [PATCH 01/13] ringdb: allow blackballing many outputs at once It cuts down on txn commits, and speeds up blackballing substantially --- .../blockchain_blackball.cpp | 21 +++- src/wallet/ringdb.cpp | 95 +++++++++++-------- src/wallet/ringdb.h | 3 +- src/wallet/wallet2.cpp | 3 +- 4 files changed, 77 insertions(+), 45 deletions(-) diff --git a/src/blockchain_utilities/blockchain_blackball.cpp b/src/blockchain_utilities/blockchain_blackball.cpp index 1653910fc..04a894fe4 100644 --- a/src/blockchain_utilities/blockchain_blackball.cpp +++ b/src/blockchain_utilities/blockchain_blackball.cpp @@ -417,6 +417,7 @@ int main(int argc, char* argv[]) if (it != state.processed_heights.end()) start_idx = it->second; LOG_PRINT_L0("Reading blockchain from " << inputs[n] << " from " << start_idx); + std::vector blackballs; for_all_transactions(inputs[n], start_idx, [&](const cryptonote::transaction_prefix &tx)->bool { for (const auto &in: tx.vin) @@ -439,7 +440,7 @@ int main(int argc, char* argv[]) { const crypto::public_key pkey = core_storage[n]->get_output_key(txin.amount, absolute[0]); MINFO("Blackballing output " << pkey << ", due to being used in a 1-ring"); - ringdb.blackball(pkey); + blackballs.push_back(pkey); newly_spent.insert(output_data(txin.amount, absolute[0])); } else if (state.ring_instances[new_ring] == new_ring.size()) @@ -448,7 +449,7 @@ int main(int argc, char* argv[]) { const crypto::public_key pkey = core_storage[n]->get_output_key(txin.amount, absolute[o]); MINFO("Blackballing output " << pkey << ", due to being used in " << new_ring.size() << " identical " << new_ring.size() << "-rings"); - ringdb.blackball(pkey); + blackballs.push_back(pkey); newly_spent.insert(output_data(txin.amount, absolute[o])); } } @@ -476,7 +477,7 @@ int main(int argc, char* argv[]) { const crypto::public_key pkey = core_storage[n]->get_output_key(txin.amount, common[0]); MINFO("Blackballing output " << pkey << ", due to being used in rings with a single common element"); - ringdb.blackball(pkey); + blackballs.push_back(pkey); newly_spent.insert(output_data(txin.amount, common[0])); } else @@ -490,6 +491,11 @@ int main(int argc, char* argv[]) } } state.relative_rings[txin.k_image] = new_ring; + if (!blackballs.empty()) + { + ringdb.blackball(blackballs); + blackballs.clear(); + } } if (stop_requested) { @@ -513,6 +519,7 @@ int main(int argc, char* argv[]) for (const auto &e: work_spent) state.spent.insert(e); + std::vector blackballs; for (const output_data &od: work_spent) { for (const crypto::key_image &ki: state.outputs[od]) @@ -533,13 +540,19 @@ int main(int argc, char* argv[]) const crypto::public_key pkey = core_storage[0]->get_output_key(od.amount, last_unknown); MINFO("Blackballing output " << pkey << ", due to being used in a " << absolute.size() << "-ring where all other outputs are known to be spent"); - ringdb.blackball(pkey); + blackballs.push_back(pkey); newly_spent.insert(output_data(od.amount, last_unknown)); } } } } + if (!blackballs.empty()) + { + ringdb.blackball(blackballs); + blackballs.clear(); + } + LOG_PRINT_L0("Saving state data to " << state_file_path); std::ofstream state_data_out; state_data_out.open(state_file_path, std::ios_base::binary | std::ios_base::out | std::ios::trunc); diff --git a/src/wallet/ringdb.cpp b/src/wallet/ringdb.cpp index 3f2634c8b..bf5478f5e 100644 --- a/src/wallet/ringdb.cpp +++ b/src/wallet/ringdb.cpp @@ -146,7 +146,7 @@ static int resize_env(MDB_env *env, const char *db_path, size_t needed) MDB_stat mst; int ret; - needed = std::max(needed, (size_t)(2ul * 1024 * 1024)); // at least 2 MB + needed = std::max(needed, (size_t)(100ul * 1024 * 1024)); // at least 100 MB ret = mdb_env_info(env, &mei); if (ret) @@ -374,7 +374,7 @@ bool ringdb::set_ring(const crypto::chacha_key &chacha_key, const crypto::key_im return true; } -bool ringdb::blackball_worker(const crypto::public_key &output, int op) +bool ringdb::blackball_worker(const std::vector &outputs, int op) { MDB_txn *txn; MDB_cursor *cursor; @@ -382,7 +382,9 @@ bool ringdb::blackball_worker(const crypto::public_key &output, int op) bool tx_active = false; bool ret = true; - dbr = resize_env(env, filename.c_str(), 32 * 2); // a pubkey, and some slack + THROW_WALLET_EXCEPTION_IF(outputs.size() > 1 && op == BLACKBALL_QUERY, tools::error::wallet_internal_error, "Blackball query only makes sense for a single output"); + + dbr = resize_env(env, filename.c_str(), 32 * 2 * outputs.size()); // a pubkey, and some slack THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to set env map size: " + std::string(mdb_strerror(dbr))); dbr = mdb_txn_begin(env, NULL, 0, &txn); THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); @@ -391,40 +393,49 @@ bool ringdb::blackball_worker(const crypto::public_key &output, int op) MDB_val key = zerokeyval; MDB_val data; - data.mv_data = (void*)&output; - data.mv_size = sizeof(output); - switch (op) + for (const crypto::public_key &output: outputs) + { + data.mv_data = (void*)&output; + data.mv_size = sizeof(output); + + switch (op) + { + case BLACKBALL_BLACKBALL: + MDEBUG("Blackballing output " << output); + dbr = mdb_put(txn, dbi_blackballs, &key, &data, MDB_NODUPDATA); + if (dbr == MDB_KEYEXIST) + dbr = 0; + break; + case BLACKBALL_UNBLACKBALL: + MDEBUG("Unblackballing output " << output); + dbr = mdb_del(txn, dbi_blackballs, &key, &data); + if (dbr == MDB_NOTFOUND) + dbr = 0; + break; + case BLACKBALL_QUERY: + dbr = mdb_cursor_open(txn, dbi_blackballs, &cursor); + THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to create cursor for blackballs table: " + std::string(mdb_strerror(dbr))); + dbr = mdb_cursor_get(cursor, &key, &data, MDB_GET_BOTH); + THROW_WALLET_EXCEPTION_IF(dbr && dbr != MDB_NOTFOUND, tools::error::wallet_internal_error, "Failed to lookup in blackballs table: " + std::string(mdb_strerror(dbr))); + ret = dbr != MDB_NOTFOUND; + if (dbr == MDB_NOTFOUND) + dbr = 0; + mdb_cursor_close(cursor); + break; + case BLACKBALL_CLEAR: + break; + default: + THROW_WALLET_EXCEPTION(tools::error::wallet_internal_error, "Invalid blackball op"); + } + THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to query blackballs table: " + std::string(mdb_strerror(dbr))); + } + + if (op == BLACKBALL_CLEAR) { - case BLACKBALL_BLACKBALL: - MDEBUG("Blackballing output " << output); - dbr = mdb_put(txn, dbi_blackballs, &key, &data, MDB_NODUPDATA); - if (dbr == MDB_KEYEXIST) - dbr = 0; - break; - case BLACKBALL_UNBLACKBALL: - MDEBUG("Unblackballing output " << output); - dbr = mdb_del(txn, dbi_blackballs, &key, &data); - if (dbr == MDB_NOTFOUND) - dbr = 0; - break; - case BLACKBALL_QUERY: - dbr = mdb_cursor_open(txn, dbi_blackballs, &cursor); - THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to create cursor for blackballs table: " + std::string(mdb_strerror(dbr))); - dbr = mdb_cursor_get(cursor, &key, &data, MDB_GET_BOTH); - THROW_WALLET_EXCEPTION_IF(dbr && dbr != MDB_NOTFOUND, tools::error::wallet_internal_error, "Failed to lookup in blackballs table: " + std::string(mdb_strerror(dbr))); - ret = dbr != MDB_NOTFOUND; - if (dbr == MDB_NOTFOUND) - dbr = 0; - mdb_cursor_close(cursor); - break; - case BLACKBALL_CLEAR: - dbr = mdb_drop(txn, dbi_blackballs, 0); - break; - default: - THROW_WALLET_EXCEPTION(tools::error::wallet_internal_error, "Invalid blackball op"); + dbr = mdb_drop(txn, dbi_blackballs, 0); + THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to clear blackballs table: " + std::string(mdb_strerror(dbr))); } - THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to query blackballs table: " + std::string(mdb_strerror(dbr))); dbr = mdb_txn_commit(txn); THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to commit txn blackballing output to database: " + std::string(mdb_strerror(dbr))); @@ -432,24 +443,32 @@ bool ringdb::blackball_worker(const crypto::public_key &output, int op) return ret; } +bool ringdb::blackball(const std::vector &outputs) +{ + return blackball_worker(outputs, BLACKBALL_BLACKBALL); +} + bool ringdb::blackball(const crypto::public_key &output) { - return blackball_worker(output, BLACKBALL_BLACKBALL); + std::vector outputs(1, output); + return blackball_worker(outputs, BLACKBALL_BLACKBALL); } bool ringdb::unblackball(const crypto::public_key &output) { - return blackball_worker(output, BLACKBALL_UNBLACKBALL); + std::vector outputs(1, output); + return blackball_worker(outputs, BLACKBALL_UNBLACKBALL); } bool ringdb::blackballed(const crypto::public_key &output) { - return blackball_worker(output, BLACKBALL_QUERY); + std::vector outputs(1, output); + return blackball_worker(outputs, BLACKBALL_QUERY); } bool ringdb::clear_blackballs() { - return blackball_worker(crypto::public_key(), BLACKBALL_CLEAR); + return blackball_worker(std::vector(), BLACKBALL_CLEAR); } } diff --git a/src/wallet/ringdb.h b/src/wallet/ringdb.h index 6b4bce124..4a78f61f8 100644 --- a/src/wallet/ringdb.h +++ b/src/wallet/ringdb.h @@ -50,12 +50,13 @@ namespace tools bool set_ring(const crypto::chacha_key &chacha_key, const crypto::key_image &key_image, const std::vector &outs, bool relative); bool blackball(const crypto::public_key &output); + bool blackball(const std::vector &outputs); bool unblackball(const crypto::public_key &output); bool blackballed(const crypto::public_key &output); bool clear_blackballs(); private: - bool blackball_worker(const crypto::public_key &output, int op); + bool blackball_worker(const std::vector &outputs, int op); private: std::string filename; diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index a2e36706e..ffd7131cd 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -6238,8 +6238,7 @@ bool wallet2::set_blackballed_outputs(const std::vector &out bool ret = true; if (!add) ret &= m_ringdb->clear_blackballs(); - for (const auto &output: outputs) - ret &= m_ringdb->blackball(output); + ret &= m_ringdb->blackball(outputs); return ret; } catch (const std::exception &e) { return false; } From daa6cc7d738c80a558b3ef1beb0c51a06ac9def3 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Sun, 12 Aug 2018 21:43:33 +0000 Subject: [PATCH 02/13] blockchain_blackball: use LMDB for the cache This uses less memory and makes it faster to load/save, though makes it slower to run (which is actually faster since it would previously start swapping anyway). --- .../blockchain_blackball.cpp | 698 ++++++++++++++---- 1 file changed, 561 insertions(+), 137 deletions(-) diff --git a/src/blockchain_utilities/blockchain_blackball.cpp b/src/blockchain_utilities/blockchain_blackball.cpp index 04a894fe4..b1369f4c0 100644 --- a/src/blockchain_utilities/blockchain_blackball.cpp +++ b/src/blockchain_utilities/blockchain_blackball.cpp @@ -50,6 +50,19 @@ namespace po = boost::program_options; using namespace epee; using namespace cryptonote; +static const char zerokey[8] = {0}; +static const MDB_val zerokval = { sizeof(zerokey), (void *)zerokey }; + +static uint64_t records_per_sync = 200; +static uint64_t db_flags = 0; +static MDB_dbi dbi_relative_rings; +static MDB_dbi dbi_outputs; +static MDB_dbi dbi_processed_txidx; +static MDB_dbi dbi_spent; +static MDB_dbi dbi_ring_instances; +static MDB_dbi dbi_newly_spent; +static MDB_env *env = NULL; + struct output_data { uint64_t amount; @@ -57,57 +70,66 @@ struct output_data output_data(): amount(0), index(0) {} output_data(uint64_t a, uint64_t i): amount(a), index(i) {} bool operator==(const output_data &other) const { return other.amount == amount && other.index == index; } - template void serialize(t_archive &a, const unsigned int ver) - { - a & amount; - a & index; - } }; -BOOST_CLASS_VERSION(output_data, 0) -namespace std +// +// relative_rings: key_image -> vector +// outputs: 128 bits -> set of key images +// processed_txidx: string -> uint64_t +// spent: 128 bits, zerokval +// ring_instances: vector -> uint64_t +// newly_spent: 128 bits, zerokval +// + +static bool parse_db_sync_mode(std::string db_sync_mode) { - template<> struct hash + std::vector options; + boost::trim(db_sync_mode); + boost::split(options, db_sync_mode, boost::is_any_of(" :")); + + for(const auto &option : options) + MDEBUG("option: " << option); + + // default to fast:async:1 + uint64_t DEFAULT_FLAGS = DBF_FAST; + + if(options.size() == 0) { - size_t operator()(const output_data &od) const + // default to fast:async:1 + db_flags = DEFAULT_FLAGS; + } + + bool safemode = false; + if(options.size() >= 1) + { + if(options[0] == "safe") { - const uint64_t data[2] = {od.amount, od.index}; - crypto::hash h; - crypto::cn_fast_hash(data, 2 * sizeof(uint64_t), h); - return reinterpret_cast(h); + safemode = true; + db_flags = DBF_SAFE; } - }; - template<> struct hash> - { - size_t operator()(const std::vector &v) const + else if(options[0] == "fast") { - crypto::hash h; - crypto::cn_fast_hash(v.data(), v.size() * sizeof(uint64_t), h); - return reinterpret_cast(h); + db_flags = DBF_FAST; } - }; -} - -struct blackball_state_t -{ - std::unordered_map> relative_rings; - std::unordered_map> outputs; - std::unordered_map processed_heights; - std::unordered_set spent; - std::unordered_map, size_t> ring_instances; + else if(options[0] == "fastest") + { + db_flags = DBF_FASTEST; + records_per_sync = 1000; // default to fastest:async:1000 + } + else + db_flags = DEFAULT_FLAGS; + } - template void serialize(t_archive &a, const unsigned int ver) + if(options.size() >= 2 && !safemode) { - a & relative_rings; - a & outputs; - a & processed_heights; - a & spent; - if (ver < 1) - return; - a & ring_instances; + char *endptr; + uint64_t bps = strtoull(options[1].c_str(), &endptr, 0); + if (*endptr == '\0') + records_per_sync = bps; } -}; -BOOST_CLASS_VERSION(blackball_state_t, 1) + + return true; +} static std::string get_default_db_path() { @@ -118,6 +140,174 @@ static std::string get_default_db_path() return dir.string(); } +static std::string get_cache_filename(boost::filesystem::path filename) +{ + if (!boost::filesystem::is_directory(filename)) + filename.remove_filename(); + return filename.string(); +} + +static int compare_hash32(const MDB_val *a, const MDB_val *b) +{ + const uint32_t *va = (const uint32_t*) a->mv_data; + const uint32_t *vb = (const uint32_t*) b->mv_data; + for (int n = 7; n >= 0; n--) + { + if (va[n] == vb[n]) + continue; + return va[n] < vb[n] ? -1 : 1; + } + + return 0; +} + +static int compare_double64(const MDB_val *a, const MDB_val *b) +{ + const uint64_t va = *(const uint64_t*) a->mv_data; + const uint64_t vb = *(const uint64_t*) b->mv_data; + if (va == vb) + { + const uint64_t va = ((const uint64_t*) a->mv_data)[1]; + const uint64_t vb = ((const uint64_t*) b->mv_data)[1]; + return va < vb ? -1 : va > vb; + } + return va < vb ? -1 : va > vb; +} + +static int resize_env(const char *db_path) +{ + MDB_envinfo mei; + MDB_stat mst; + int ret; + + size_t needed = 1000ul * 1024 * 1024; // at least 1000 MB + + ret = mdb_env_info(env, &mei); + if (ret) + return ret; + ret = mdb_env_stat(env, &mst); + if (ret) + return ret; + uint64_t size_used = mst.ms_psize * mei.me_last_pgno; + uint64_t mapsize = mei.me_mapsize; + if (size_used + needed > mei.me_mapsize) + { + try + { + boost::filesystem::path path(db_path); + boost::filesystem::space_info si = boost::filesystem::space(path); + if(si.available < needed) + { + MERROR("!! WARNING: Insufficient free space to extend database !!: " << (si.available >> 20L) << " MB available"); + return ENOSPC; + } + } + catch(...) + { + // print something but proceed. + MWARNING("Unable to query free disk space."); + } + + mapsize += needed; + } + return mdb_env_set_mapsize(env, mapsize); +} + +static void init(std::string cache_filename) +{ + MDB_txn *txn; + bool tx_active = false; + int dbr; + + MINFO("Creating blackball cache in " << cache_filename); + + tools::create_directories_if_necessary(cache_filename); + + int flags = 0; + if (db_flags & DBF_FAST) + flags |= MDB_NOSYNC; + if (db_flags & DBF_FASTEST) + flags |= MDB_NOSYNC | MDB_WRITEMAP | MDB_MAPASYNC; + + dbr = mdb_env_create(&env); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LDMB environment: " + std::string(mdb_strerror(dbr))); + dbr = mdb_env_set_maxdbs(env, 6); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to set max env dbs: " + std::string(mdb_strerror(dbr))); + const std::string actual_filename = get_cache_filename(cache_filename); + dbr = mdb_env_open(env, actual_filename.c_str(), flags, 0664); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open rings database file '" + + actual_filename + "': " + std::string(mdb_strerror(dbr))); + + dbr = mdb_txn_begin(env, NULL, 0, &txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + epee::misc_utils::auto_scope_leave_caller txn_dtor = epee::misc_utils::create_scope_leave_handler([&](){if (tx_active) mdb_txn_abort(txn);}); + tx_active = true; + + dbr = mdb_dbi_open(txn, "relative_rings", MDB_CREATE | MDB_INTEGERKEY, &dbi_relative_rings); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + mdb_set_compare(txn, dbi_relative_rings, compare_hash32); + + dbr = mdb_dbi_open(txn, "outputs", MDB_CREATE | MDB_INTEGERKEY, &dbi_outputs); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + mdb_set_compare(txn, dbi_outputs, compare_double64); + + dbr = mdb_dbi_open(txn, "processed_txidx", MDB_CREATE, &dbi_processed_txidx); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + + dbr = mdb_dbi_open(txn, "spent", MDB_CREATE | MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, &dbi_spent); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + mdb_set_dupsort(txn, dbi_spent, compare_double64); + + dbr = mdb_dbi_open(txn, "ring_instances", MDB_CREATE, &dbi_ring_instances); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + + dbr = mdb_dbi_open(txn, "newly_spent", MDB_CREATE | MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, &dbi_newly_spent); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + mdb_set_dupsort(txn, dbi_newly_spent, compare_double64); + + dbr = mdb_txn_commit(txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to commit txn creating/opening database: " + std::string(mdb_strerror(dbr))); + tx_active = false; +} + +static void close() +{ + if (env) + { + mdb_dbi_close(env, dbi_relative_rings); + mdb_dbi_close(env, dbi_outputs); + mdb_dbi_close(env, dbi_processed_txidx); + mdb_dbi_close(env, dbi_spent); + mdb_dbi_close(env, dbi_ring_instances); + mdb_dbi_close(env, dbi_newly_spent); + mdb_env_close(env); + env = NULL; + } +} + +static std::string compress_ring(const std::vector &ring) +{ + std::string s; + for (uint64_t out: ring) + s += tools::get_varint_data(out); + return s; +} + +static std::vector decompress_ring(const std::string &s) +{ + std::vector ring; + int read = 0; + for (std::string::const_iterator i = s.begin(); i != s.cend(); std::advance(i, read)) + { + uint64_t out; + std::string tmp(i, s.cend()); + read = tools::read_varint(tmp.begin(), tmp.end(), out); + CHECK_AND_ASSERT_THROW_MES(read > 0 && read <= 256, "Internal error decompressing ring"); + ring.push_back(out); + } + return ring; +} + static bool for_all_transactions(const std::string &filename, uint64_t &start_idx, const std::function &f) { MDB_env *env; @@ -212,6 +402,234 @@ static std::vector canonicalize(const std::vector &v) return c; } +static uint64_t get_num_spent_outputs(bool newly) +{ + MDB_txn *txn; + bool tx_active = false; + + int dbr = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + epee::misc_utils::auto_scope_leave_caller txn_dtor = epee::misc_utils::create_scope_leave_handler([&](){if (tx_active) mdb_txn_abort(txn);}); + tx_active = true; + + MDB_cursor *cur; + dbr = mdb_cursor_open(txn, newly ? dbi_newly_spent : dbi_spent, &cur); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open cursor for spent outputs: " + std::string(mdb_strerror(dbr))); + MDB_val k, v; + mdb_size_t count = 0; + dbr = mdb_cursor_get(cur, &k, &v, MDB_FIRST); + if (dbr != MDB_NOTFOUND) + { + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to get first spent output: " + std::string(mdb_strerror(dbr))); + dbr = mdb_cursor_count(cur, &count); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to count entries: " + std::string(mdb_strerror(dbr))); + } + + mdb_cursor_close(cur); + dbr = mdb_txn_commit(txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to commit txn: " + std::string(mdb_strerror(dbr))); + tx_active = false; + + return count; +} + +static void add_spent_output(MDB_txn *txn, const output_data &od, bool newly) +{ + MDB_cursor *cur; + int dbr = mdb_cursor_open(txn, newly ? dbi_newly_spent : dbi_spent, &cur); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open cursor for spent outputs: " + std::string(mdb_strerror(dbr))); + MDB_val v = {sizeof(od), (void*)&od}; + dbr = mdb_cursor_put(cur, (MDB_val *)&zerokval, &v, MDB_NODUPDATA); + CHECK_AND_ASSERT_THROW_MES(!dbr || dbr == MDB_KEYEXIST, "Failed to add spent output: " + std::string(mdb_strerror(dbr))); + mdb_cursor_close(cur); +} + +static bool is_output_spent(MDB_txn *txn, const output_data &od, bool newly) +{ + MDB_cursor *cur; + int dbr = mdb_cursor_open(txn, newly ? dbi_newly_spent : dbi_spent, &cur); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open cursor for spent outputs: " + std::string(mdb_strerror(dbr))); + MDB_val v = {sizeof(od), (void*)&od}; + dbr = mdb_cursor_get(cur, (MDB_val *)&zerokval, &v, MDB_GET_BOTH); + CHECK_AND_ASSERT_THROW_MES(!dbr || dbr == MDB_NOTFOUND, "Failed to get spent output: " + std::string(mdb_strerror(dbr))); + bool spent = dbr == 0; + mdb_cursor_close(cur); + return spent; +} + +static std::vector get_spent_outputs(MDB_txn *txn, bool newly) +{ + MDB_cursor *cur; + int dbr = mdb_cursor_open(txn, newly ? dbi_newly_spent : dbi_spent, &cur); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open cursor for spent outputs: " + std::string(mdb_strerror(dbr))); + MDB_val k, v; + uint64_t count = 0; + dbr = mdb_cursor_get(cur, &k, &v, MDB_FIRST); + if (dbr != MDB_NOTFOUND) + { + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to get first spent output: " + std::string(mdb_strerror(dbr))); + dbr = mdb_cursor_count(cur, &count); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to count entries: " + std::string(mdb_strerror(dbr))); + } + std::vector outs; + outs.reserve(count); + while (1) + { + const output_data *od = (const output_data*)v.mv_data; + outs.push_back(*od); + dbr = mdb_cursor_get(cur, &k, &v, MDB_NEXT); + if (dbr == MDB_NOTFOUND) + break; + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to get next spent output: " + std::string(mdb_strerror(dbr))); + } + mdb_cursor_close(cur); + return outs; +} + +static void clear_spent_outputs(MDB_txn *txn, bool newly) +{ + int dbr = mdb_drop(txn, newly ? dbi_newly_spent : dbi_spent, 0); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to clear spent outputs: " + std::string(mdb_strerror(dbr))); +} + +static uint64_t get_processed_txidx(const std::string &name) +{ + MDB_txn *txn; + bool tx_active = false; + + int dbr = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + epee::misc_utils::auto_scope_leave_caller txn_dtor = epee::misc_utils::create_scope_leave_handler([&](){if (tx_active) mdb_txn_abort(txn);}); + tx_active = true; + + uint64_t height = 0; + MDB_val k, v; + k.mv_data = (void*)name.c_str(); + k.mv_size = name.size(); + dbr = mdb_get(txn, dbi_processed_txidx, &k, &v); + if (dbr != MDB_NOTFOUND) + { + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to get processed height: " + std::string(mdb_strerror(dbr))); + height = *(const uint64_t*)v.mv_data; + } + + dbr = mdb_txn_commit(txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to commit txn: " + std::string(mdb_strerror(dbr))); + tx_active = false; + + return height; +} + +static void set_processed_txidx(MDB_txn *txn, const std::string &name, uint64_t height) +{ + MDB_val k, v; + k.mv_data = (void*)name.c_str(); + k.mv_size = name.size(); + v.mv_data = (void*)&height; + v.mv_size = sizeof(height); + int dbr = mdb_put(txn, dbi_processed_txidx, &k, &v, 0); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to set processed height: " + std::string(mdb_strerror(dbr))); +} + +static bool get_relative_ring(MDB_txn *txn, const crypto::key_image &ki, std::vector &ring) +{ + const std::string sring = compress_ring(ring); + MDB_val k, v; + k.mv_data = (void*)&ki; + k.mv_size = sizeof(ki); + int dbr = mdb_get(txn, dbi_relative_rings, &k, &v); + if (dbr == MDB_NOTFOUND) + return false; + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to get relative ring: " + std::string(mdb_strerror(dbr))); + ring = decompress_ring(std::string((const char*)v.mv_data, v.mv_size)); + return true; +} + +static void set_relative_ring(MDB_txn *txn, const crypto::key_image &ki, const std::vector &ring) +{ + const std::string sring = compress_ring(ring); + MDB_val k, v; + k.mv_data = (void*)&ki; + k.mv_size = sizeof(ki); + v.mv_data = (void*)sring.c_str(); + v.mv_size = sring.size(); + int dbr = mdb_put(txn, dbi_relative_rings, &k, &v, 0); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to set relative ring: " + std::string(mdb_strerror(dbr))); +} + +static std::string keep_under_511(const std::string &s) +{ + if (s.size() <= 511) + return s; + crypto::hash hash; + crypto::cn_fast_hash(s.data(), s.size(), hash); + return std::string((const char*)&hash, 32); +} + +static uint64_t get_ring_instances(MDB_txn *txn, const std::vector &ring) +{ + const std::string sring = keep_under_511(compress_ring(ring)); + MDB_val k, v; + k.mv_data = (void*)sring.data(); + k.mv_size = sring.size(); + int dbr = mdb_get(txn, dbi_ring_instances, &k, &v); + if (dbr == MDB_NOTFOUND) + return 0; + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to get ring instances: " + std::string(mdb_strerror(dbr))); + return *(const uint64_t*)v.mv_data; +} + +static void set_ring_instances(MDB_txn *txn, const std::vector &ring, uint64_t count) +{ + const std::string sring = keep_under_511(compress_ring(ring)); + MDB_val k, v; + k.mv_data = (void*)sring.data(); + k.mv_size = sring.size(); + v.mv_data = &count; + v.mv_size = sizeof(count); + int dbr = mdb_put(txn, dbi_ring_instances, &k, &v, 0); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to get ring instances: " + std::string(mdb_strerror(dbr))); +} + +static std::vector get_key_images(MDB_txn *txn, const output_data &od) +{ + MDB_val k, v; + k.mv_data = (void*)&od; + k.mv_size = sizeof(od); + int dbr = mdb_get(txn, dbi_outputs, &k, &v); + CHECK_AND_ASSERT_THROW_MES(!dbr || dbr == MDB_NOTFOUND, "Failed to get output: " + std::string(mdb_strerror(dbr))); + if (dbr == MDB_NOTFOUND) + return {}; + CHECK_AND_ASSERT_THROW_MES(v.mv_size % 32 == 0, "Unexpected record size"); + std::vector key_images; + key_images.reserve(v.mv_size / 32); + const crypto::key_image *ki = (const crypto::key_image*)v.mv_data; + for (size_t n = 0; n < v.mv_size / 32; ++n) + key_images.push_back(*ki++); + return key_images; +} + +static void add_key_image(MDB_txn *txn, const output_data &od, const crypto::key_image &ki) +{ + MDB_val k, v; + k.mv_data = (void*)&od; + k.mv_size = sizeof(od); + int dbr = mdb_get(txn, dbi_outputs, &k, &v); + CHECK_AND_ASSERT_THROW_MES(!dbr || dbr == MDB_NOTFOUND, "Failed to get output"); + std::string data; + if (!dbr) + { + CHECK_AND_ASSERT_THROW_MES(v.mv_size % 32 == 0, "Unexpected record size"); + data = std::string((const char*)v.mv_data, v.mv_size); + } + data += std::string((const char*)&ki, sizeof(ki)); + + v.mv_data = (void*)data.data(); + v.mv_size = data.size(); + dbr = mdb_put(txn, dbi_outputs, &k, &v, 0); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to set outputs: " + std::string(mdb_strerror(dbr))); +} + int main(int argc, char* argv[]) { TRY_ENTRY(); @@ -249,6 +667,11 @@ int main(int argc, char* argv[]) }; const command_line::arg_descriptor arg_rct_only = {"rct-only", "Only work on ringCT outputs", false}; const command_line::arg_descriptor > arg_inputs = {"inputs", "Path to Monero DB, and path to any fork DBs"}; + const command_line::arg_descriptor arg_db_sync_mode = { + "db-sync-mode" + , "Specify sync option, using format [safe|fast|fastest]:[nrecords_per_sync]." + , "fast:1000" + }; command_line::add_arg(desc_cmd_sett, arg_blackball_db_dir); command_line::add_arg(desc_cmd_sett, cryptonote::arg_testnet_on); @@ -256,6 +679,7 @@ int main(int argc, char* argv[]) command_line::add_arg(desc_cmd_sett, arg_log_level); command_line::add_arg(desc_cmd_sett, arg_database); command_line::add_arg(desc_cmd_sett, arg_rct_only); + command_line::add_arg(desc_cmd_sett, arg_db_sync_mode); command_line::add_arg(desc_cmd_sett, arg_inputs); command_line::add_arg(desc_cmd_only, command_line::arg_help); @@ -304,6 +728,13 @@ int main(int argc, char* argv[]) return 1; } + std::string db_sync_mode = command_line::get_arg(vm, arg_db_sync_mode); + if (!parse_db_sync_mode(db_sync_mode)) + { + MERROR("Invalid db sync mode: " << db_sync_mode); + return 1; + } + // If we wanted to use the memory pool, we would set up a fake_core. // Use Blockchain instead of lower-level BlockchainDB for two reasons: @@ -337,7 +768,7 @@ int main(int argc, char* argv[]) } LOG_PRINT_L0("database: " << db_type); - std::string filename = inputs[n]; + std::string filename = (boost::filesystem::path(inputs[n]) / db->get_db_name()).string(); while (boost::ends_with(filename, "/") || boost::ends_with(filename, "\\")) filename.pop_back(); LOG_PRINT_L0("Loading blockchain from folder " << filename << " ..."); @@ -357,49 +788,14 @@ int main(int argc, char* argv[]) LOG_PRINT_L0("Source blockchain storage initialized OK"); } - boost::filesystem::path direc(output_file_path.string()); - if (boost::filesystem::exists(direc)) - { - if (!boost::filesystem::is_directory(direc)) - { - MERROR("LMDB needs a directory path, but a file was passed: " << output_file_path.string()); - return 1; - } - } - else - { - if (!boost::filesystem::create_directories(direc)) - { - MERROR("Failed to create directory: " << output_file_path.string()); - return 1; - } - } + const std::string cache_dir = (output_file_path / "blackball-cache").string(); + init(cache_dir); LOG_PRINT_L0("Scanning for blackballable outputs..."); size_t done = 0; - blackball_state_t state; - std::unordered_set newly_spent; - const std::string state_file_path = (boost::filesystem::path(output_file_path) / "blackball-state.bin").string(); - - LOG_PRINT_L0("Loading state data from " << state_file_path); - std::ifstream state_data_in; - state_data_in.open(state_file_path, std::ios_base::binary | std::ios_base::in); - if (!state_data_in.fail()) - { - try - { - boost::archive::portable_binary_iarchive a(state_data_in); - a >> state; - } - catch (const std::exception &e) - { - MERROR("Failed to load state data from " << state_file_path << ", restarting from scratch"); - state = blackball_state_t(); - } - state_data_in.close(); - } - uint64_t start_blackballed_outputs = state.spent.size(); + + const uint64_t start_blackballed_outputs = get_num_spent_outputs(false); cryptonote::block b = core_storage[0]->get_db().get_block_from_height(0); tools::ringdb ringdb(output_file_path.string(), epee::string_tools::pod_to_hex(get_block_hash(b))); @@ -409,17 +805,24 @@ int main(int argc, char* argv[]) stop_requested = true; }); + int dbr = resize_env(cache_dir.c_str()); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to resize LMDB database: " + std::string(mdb_strerror(dbr))); + for (size_t n = 0; n < inputs.size(); ++n) { const std::string canonical = boost::filesystem::canonical(inputs[n]).string(); - uint64_t start_idx = 0; - auto it = state.processed_heights.find(canonical); - if (it != state.processed_heights.end()) - start_idx = it->second; + uint64_t start_idx = get_processed_txidx(canonical); LOG_PRINT_L0("Reading blockchain from " << inputs[n] << " from " << start_idx); + MDB_txn *txn; + int dbr = mdb_txn_begin(env, NULL, 0, &txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + size_t records = 0; + const uint64_t n_txes = core_storage[n]->get_db().get_tx_count(); + const std::string filename = (boost::filesystem::path(inputs[n]) / core_storage[n]->get_db().get_db_name()).string(); std::vector blackballs; - for_all_transactions(inputs[n], start_idx, [&](const cryptonote::transaction_prefix &tx)->bool + for_all_transactions(filename, start_idx, [&](const cryptonote::transaction_prefix &tx)->bool { + std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; for (const auto &in: tx.vin) { if (in.type() != typeid(txin_to_key)) @@ -431,37 +834,44 @@ int main(int argc, char* argv[]) const std::vector absolute = cryptonote::relative_output_offsets_to_absolute(txin.key_offsets); if (n == 0) for (uint64_t out: absolute) - state.outputs[output_data(txin.amount, out)].insert(txin.k_image); + add_key_image(txn, output_data(txin.amount, out), txin.k_image); + std::vector relative_ring; std::vector new_ring = canonicalize(txin.key_offsets); const uint32_t ring_size = txin.key_offsets.size(); - state.ring_instances[new_ring] += 1; + uint64_t instances = get_ring_instances(txn, new_ring); + ++instances; + set_ring_instances(txn, new_ring, instances); if (ring_size == 1) { const crypto::public_key pkey = core_storage[n]->get_output_key(txin.amount, absolute[0]); MINFO("Blackballing output " << pkey << ", due to being used in a 1-ring"); + std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; blackballs.push_back(pkey); - newly_spent.insert(output_data(txin.amount, absolute[0])); + add_spent_output(txn, output_data(txin.amount, absolute[0]), true); } - else if (state.ring_instances[new_ring] == new_ring.size()) + else if (instances == new_ring.size()) { for (size_t o = 0; o < new_ring.size(); ++o) { const crypto::public_key pkey = core_storage[n]->get_output_key(txin.amount, absolute[o]); MINFO("Blackballing output " << pkey << ", due to being used in " << new_ring.size() << " identical " << new_ring.size() << "-rings"); + std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; blackballs.push_back(pkey); - newly_spent.insert(output_data(txin.amount, absolute[o])); + add_spent_output(txn, output_data(txin.amount, absolute[o]), true); } } - else if (state.relative_rings.find(txin.k_image) != state.relative_rings.end()) + else if (get_relative_ring(txn, txin.k_image, relative_ring)) { MINFO("Key image " << txin.k_image << " already seen: rings " << - boost::join(state.relative_rings[txin.k_image] | boost::adaptors::transformed([](uint64_t out){return std::to_string(out);}), " ") << + boost::join(relative_ring | boost::adaptors::transformed([](uint64_t out){return std::to_string(out);}), " ") << ", " << boost::join(txin.key_offsets | boost::adaptors::transformed([](uint64_t out){return std::to_string(out);}), " ")); - if (state.relative_rings[txin.k_image] != txin.key_offsets) + std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; + if (relative_ring != txin.key_offsets) { MINFO("Rings are different"); - const std::vector r0 = cryptonote::relative_output_offsets_to_absolute(state.relative_rings[txin.k_image]); + std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; + const std::vector r0 = cryptonote::relative_output_offsets_to_absolute(relative_ring); const std::vector r1 = cryptonote::relative_output_offsets_to_absolute(txin.key_offsets); std::vector common; for (uint64_t out: r0) @@ -472,17 +882,20 @@ int main(int argc, char* argv[]) if (common.empty()) { MERROR("Rings for the same key image are disjoint"); + std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; } else if (common.size() == 1) { const crypto::public_key pkey = core_storage[n]->get_output_key(txin.amount, common[0]); MINFO("Blackballing output " << pkey << ", due to being used in rings with a single common element"); + std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; blackballs.push_back(pkey); - newly_spent.insert(output_data(txin.amount, common[0])); + add_spent_output(txn, output_data(txin.amount, common[0]), true); } else { MINFO("The intersection has more than one element, it's still ok"); + std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; for (const auto &out: r0) if (std::find(common.begin(), common.end(), out) != common.end()) new_ring.push_back(out); @@ -490,13 +903,27 @@ int main(int argc, char* argv[]) } } } - state.relative_rings[txin.k_image] = new_ring; - if (!blackballs.empty()) - { - ringdb.blackball(blackballs); - blackballs.clear(); - } + set_relative_ring(txn, txin.k_image, new_ring); + } + if (!blackballs.empty()) + { + ringdb.blackball(blackballs); + blackballs.clear(); + } + set_processed_txidx(txn, canonical, start_idx); + + ++records; + if (records >= records_per_sync) + { + dbr = mdb_txn_commit(txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to commit txn creating/opening database: " + std::string(mdb_strerror(dbr))); + int dbr = resize_env(cache_dir.c_str()); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to resize LMDB database: " + std::string(mdb_strerror(dbr))); + dbr = mdb_txn_begin(env, NULL, 0, &txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + records = 0; } + if (stop_requested) { MINFO("Stopping scan, secondary passes will still happen..."); @@ -504,33 +931,45 @@ int main(int argc, char* argv[]) } return true; }); - LOG_PRINT_L0("blockchain from " << inputs[n] << " processed still height " << start_idx); - state.processed_heights[canonical] = start_idx; + dbr = mdb_txn_commit(txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to commit txn creating/opening database: " + std::string(mdb_strerror(dbr))); + LOG_PRINT_L0("blockchain from " << inputs[n] << " processed till tx idx " << start_idx); if (stop_requested) break; } - while (!newly_spent.empty()) + while (get_num_spent_outputs(true) != 0) { - LOG_PRINT_L0("Secondary pass due to " << newly_spent.size() << " newly found spent outputs"); - std::unordered_set work_spent = std::move(newly_spent); - newly_spent.clear(); + LOG_PRINT_L0("Secondary pass due to " << get_num_spent_outputs(true) << " newly found spent outputs"); + + int dbr = resize_env(cache_dir.c_str()); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to resize LMDB database: " + std::string(mdb_strerror(dbr))); + + MDB_txn *txn; + dbr = mdb_txn_begin(env, NULL, 0, &txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + + std::vector work_spent = get_spent_outputs(txn, true); + clear_spent_outputs(txn, true); - for (const auto &e: work_spent) - state.spent.insert(e); + for (const auto &od: work_spent) + add_spent_output(txn, od, false); std::vector blackballs; for (const output_data &od: work_spent) { - for (const crypto::key_image &ki: state.outputs[od]) + std::vector key_images = get_key_images(txn, od); + for (const crypto::key_image &ki: key_images) { - std::vector absolute = cryptonote::relative_output_offsets_to_absolute(state.relative_rings[ki]); + std::vector relative_ring; + CHECK_AND_ASSERT_THROW_MES(get_relative_ring(txn, ki, relative_ring), "Relative ring not found"); + std::vector absolute = cryptonote::relative_output_offsets_to_absolute(relative_ring); size_t known = 0; uint64_t last_unknown = 0; for (uint64_t out: absolute) { output_data new_od(od.amount, out); - if (state.spent.find(new_od) != state.spent.end()) + if (is_output_spent(txn, new_od, false)) ++known; else last_unknown = out; @@ -541,39 +980,24 @@ int main(int argc, char* argv[]) MINFO("Blackballing output " << pkey << ", due to being used in a " << absolute.size() << "-ring where all other outputs are known to be spent"); blackballs.push_back(pkey); - newly_spent.insert(output_data(od.amount, last_unknown)); + add_spent_output(txn, output_data(od.amount, last_unknown), true); } } } - } - - if (!blackballs.empty()) - { - ringdb.blackball(blackballs); - blackballs.clear(); - } - - LOG_PRINT_L0("Saving state data to " << state_file_path); - std::ofstream state_data_out; - state_data_out.open(state_file_path, std::ios_base::binary | std::ios_base::out | std::ios::trunc); - if (!state_data_out.fail()) - { - try - { - boost::archive::portable_binary_oarchive a(state_data_out); - a << state; - } - catch (const std::exception &e) + if (!blackballs.empty()) { - MERROR("Failed to save state data to " << state_file_path); + ringdb.blackball(blackballs); + blackballs.clear(); } - state_data_out.close(); + dbr = mdb_txn_commit(txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to commit txn creating/opening database: " + std::string(mdb_strerror(dbr))); } - uint64_t diff = state.spent.size() - start_blackballed_outputs; - LOG_PRINT_L0(std::to_string(diff) << " new outputs blackballed, " << state.spent.size() << " total outputs blackballed"); + uint64_t diff = get_num_spent_outputs(false) - start_blackballed_outputs; + LOG_PRINT_L0(std::to_string(diff) << " new outputs blackballed, " << get_num_spent_outputs(false) << " total outputs blackballed"); LOG_PRINT_L0("Blockchain blackball data exported OK"); + close(); return 0; - CATCH_ENTRY("Export error", 1); + CATCH_ENTRY("Error", 1); } From 846190fd18a6b16ba2406db56b533b5bb87b1f3e Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Sun, 12 Aug 2018 05:54:38 +0000 Subject: [PATCH 03/13] blockchain_blackball: support pre-v2 databases --- .../blockchain_blackball.cpp | 202 +++++++++++------- 1 file changed, 123 insertions(+), 79 deletions(-) diff --git a/src/blockchain_utilities/blockchain_blackball.cpp b/src/blockchain_utilities/blockchain_blackball.cpp index b1369f4c0..f034acb24 100644 --- a/src/blockchain_utilities/blockchain_blackball.cpp +++ b/src/blockchain_utilities/blockchain_blackball.cpp @@ -161,6 +161,13 @@ static int compare_hash32(const MDB_val *a, const MDB_val *b) return 0; } +int compare_uint64(const MDB_val *a, const MDB_val *b) +{ + const uint64_t va = *(const uint64_t *)a->mv_data; + const uint64_t vb = *(const uint64_t *)b->mv_data; + return (va < vb) ? -1 : va > vb; +} + static int compare_double64(const MDB_val *a, const MDB_val *b) { const uint64_t va = *(const uint64_t*) a->mv_data; @@ -308,7 +315,7 @@ static std::vector decompress_ring(const std::string &s) return ring; } -static bool for_all_transactions(const std::string &filename, uint64_t &start_idx, const std::function &f) +static bool for_all_transactions(const std::string &filename, uint64_t &start_idx, uint64_t &n_txes, const std::function &f) { MDB_env *env; MDB_dbi dbi; @@ -316,6 +323,8 @@ static bool for_all_transactions(const std::string &filename, uint64_t &start_id MDB_cursor *cur; int dbr; bool tx_active = false; + MDB_val k; + MDB_val v; dbr = mdb_env_create(&env); if (dbr) throw std::runtime_error("Failed to create LDMB environment: " + std::string(mdb_strerror(dbr))); @@ -337,9 +346,11 @@ static bool for_all_transactions(const std::string &filename, uint64_t &start_id if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); dbr = mdb_cursor_open(txn, dbi, &cur); if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr))); + MDB_stat stat; + dbr = mdb_stat(txn, dbi, &stat); + if (dbr) throw std::runtime_error("Failed to query m_block_info: " + std::string(mdb_strerror(dbr))); + n_txes = stat.ms_entries; - MDB_val k; - MDB_val v; bool fret = true; k.mv_size = sizeof(uint64_t); @@ -630,6 +641,90 @@ static void add_key_image(MDB_txn *txn, const output_data &od, const crypto::key CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to set outputs: " + std::string(mdb_strerror(dbr))); } +static void open_db(const std::string &filename, MDB_env **env, MDB_txn **txn, MDB_cursor **cur, MDB_dbi *dbi) +{ + tools::create_directories_if_necessary(filename); + + int flags = MDB_RDONLY; + if (db_flags & DBF_FAST) + flags |= MDB_NOSYNC; + if (db_flags & DBF_FASTEST) + flags |= MDB_NOSYNC | MDB_WRITEMAP | MDB_MAPASYNC; + + int dbr = mdb_env_create(env); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LDMB environment: " + std::string(mdb_strerror(dbr))); + dbr = mdb_env_set_maxdbs(*env, 1); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to set max env dbs: " + std::string(mdb_strerror(dbr))); + const std::string actual_filename = filename; + MINFO("Opening monero blockchain at " << actual_filename); + dbr = mdb_env_open(*env, actual_filename.c_str(), flags, 0664); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open rings database file '" + + actual_filename + "': " + std::string(mdb_strerror(dbr))); + + dbr = mdb_txn_begin(*env, NULL, MDB_RDONLY, txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + + dbr = mdb_dbi_open(*txn, "output_amounts", MDB_CREATE | MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, dbi); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + mdb_set_dupsort(*txn, *dbi, compare_uint64); + + dbr = mdb_cursor_open(*txn, *dbi, cur); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr))); +} + +static void close_db(MDB_env *env, MDB_txn *txn, MDB_cursor *cur, MDB_dbi dbi) +{ + mdb_txn_abort(txn); + mdb_cursor_close(cur); + mdb_dbi_close(env, dbi); + mdb_env_close(env); +} + +static crypto::public_key get_output_key(MDB_cursor *cur, uint64_t amount, uint64_t offset) +{ + MDB_val k = { sizeof(amount), (void*)&amount }, v = { sizeof(offset), (void*)&offset }; + int dbr = mdb_cursor_get(cur, &k, &v, MDB_GET_BOTH); + if (dbr) throw std::runtime_error("Output key not found: " + std::string(mdb_strerror(dbr))); + return *(const crypto::public_key*)(((const char*)v.mv_data) + sizeof(uint64_t) * 2); +} + +static crypto::hash get_genesis_block_hash(const std::string &filename) +{ + MDB_env *env; + MDB_dbi dbi; + MDB_txn *txn; + int dbr; + bool tx_active = false; + + dbr = mdb_env_create(&env); + if (dbr) throw std::runtime_error("Failed to create LDMB environment: " + std::string(mdb_strerror(dbr))); + dbr = mdb_env_set_maxdbs(env, 1); + if (dbr) throw std::runtime_error("Failed to set max env dbs: " + std::string(mdb_strerror(dbr))); + const std::string actual_filename = filename; + dbr = mdb_env_open(env, actual_filename.c_str(), 0, 0664); + if (dbr) throw std::runtime_error("Failed to open rings database file '" + + actual_filename + "': " + std::string(mdb_strerror(dbr))); + + dbr = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); + if (dbr) throw std::runtime_error("Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + epee::misc_utils::auto_scope_leave_caller txn_dtor = epee::misc_utils::create_scope_leave_handler([&](){if (tx_active) mdb_txn_abort(txn);}); + tx_active = true; + + dbr = mdb_dbi_open(txn, "block_info", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, &dbi); + mdb_set_dupsort(txn, dbi, compare_uint64); + if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + uint64_t zero = 0; + MDB_val k = { sizeof(uint64_t), (void*)&zero}, v; + dbr = mdb_get(txn, dbi, &k, &v); + if (dbr) throw std::runtime_error("Failed to retrieve genesis block: " + std::string(mdb_strerror(dbr))); + crypto::hash genesis_block_hash = *(const crypto::hash*)(((const uint64_t*)v.mv_data) + 5); + mdb_dbi_close(env, dbi); + mdb_txn_abort(txn); + mdb_env_close(env); + tx_active = false; + return genesis_block_hash; +} + int main(int argc, char* argv[]) { TRY_ENTRY(); @@ -649,17 +744,9 @@ int main(int argc, char* argv[]) po::options_description desc_cmd_only("Command line options"); po::options_description desc_cmd_sett("Command line options and settings options"); - const command_line::arg_descriptor arg_blackball_db_dir = { + const command_line::arg_descriptor arg_blackball_db_dir = { "blackball-db-dir", "Specify blackball database directory", get_default_db_path(), - {{ &arg_testnet_on, &arg_stagenet_on }}, - [](std::array testnet_stagenet, bool defaulted, std::string val)->std::string { - if (testnet_stagenet[0]) - return (boost::filesystem::path(val) / "testnet").string(); - else if (testnet_stagenet[1]) - return (boost::filesystem::path(val) / "stagenet").string(); - return val; - } }; const command_line::arg_descriptor arg_log_level = {"log-level", "0-4 or categories", ""}; const command_line::arg_descriptor arg_database = { @@ -674,8 +761,6 @@ int main(int argc, char* argv[]) }; command_line::add_arg(desc_cmd_sett, arg_blackball_db_dir); - command_line::add_arg(desc_cmd_sett, cryptonote::arg_testnet_on); - command_line::add_arg(desc_cmd_sett, cryptonote::arg_stagenet_on); command_line::add_arg(desc_cmd_sett, arg_log_level); command_line::add_arg(desc_cmd_sett, arg_database); command_line::add_arg(desc_cmd_sett, arg_rct_only); @@ -715,9 +800,6 @@ int main(int argc, char* argv[]) LOG_PRINT_L0("Starting..."); - bool opt_testnet = command_line::get_arg(vm, cryptonote::arg_testnet_on); - bool opt_stagenet = command_line::get_arg(vm, cryptonote::arg_stagenet_on); - network_type net_type = opt_testnet ? TESTNET : opt_stagenet ? STAGENET : MAINNET; output_file_path = command_line::get_arg(vm, arg_blackball_db_dir); bool opt_rct_only = command_line::get_arg(vm, arg_rct_only); @@ -735,58 +817,12 @@ int main(int argc, char* argv[]) return 1; } - // If we wanted to use the memory pool, we would set up a fake_core. - - // Use Blockchain instead of lower-level BlockchainDB for two reasons: - // 1. Blockchain has the init() method for easy setup - // 2. exporter needs to use get_current_blockchain_height(), get_block_id_by_height(), get_block_by_hash() - // - // cannot match blockchain_storage setup above with just one line, - // e.g. - // Blockchain* core_storage = new Blockchain(NULL); - // because unlike blockchain_storage constructor, which takes a pointer to - // tx_memory_pool, Blockchain's constructor takes tx_memory_pool object. - LOG_PRINT_L0("Initializing source blockchain (BlockchainDB)"); const std::vector inputs = command_line::get_arg(vm, arg_inputs); if (inputs.empty()) { LOG_PRINT_L0("No inputs given"); return 1; } - std::vector> core_storage(inputs.size()); - Blockchain *blockchain = NULL; - tx_memory_pool m_mempool(*blockchain); - for (size_t n = 0; n < inputs.size(); ++n) - { - core_storage[n].reset(new Blockchain(m_mempool)); - - BlockchainDB* db = new_db(db_type); - if (db == NULL) - { - LOG_ERROR("Attempted to use non-existent database type: " << db_type); - throw std::runtime_error("Attempting to use non-existent database type"); - } - LOG_PRINT_L0("database: " << db_type); - - std::string filename = (boost::filesystem::path(inputs[n]) / db->get_db_name()).string(); - while (boost::ends_with(filename, "/") || boost::ends_with(filename, "\\")) - filename.pop_back(); - LOG_PRINT_L0("Loading blockchain from folder " << filename << " ..."); - - try - { - db->open(filename, DBF_RDONLY); - } - catch (const std::exception& e) - { - LOG_PRINT_L0("Error opening database: " << e.what()); - return 1; - } - r = core_storage[n]->init(db, net_type); - - CHECK_AND_ASSERT_MES(r, 1, "Failed to initialize source blockchain storage"); - LOG_PRINT_L0("Source blockchain storage initialized OK"); - } const std::string cache_dir = (output_file_path / "blackball-cache").string(); init(cache_dir); @@ -797,8 +833,7 @@ int main(int argc, char* argv[]) const uint64_t start_blackballed_outputs = get_num_spent_outputs(false); - cryptonote::block b = core_storage[0]->get_db().get_block_from_height(0); - tools::ringdb ringdb(output_file_path.string(), epee::string_tools::pod_to_hex(get_block_hash(b))); + tools::ringdb ringdb(output_file_path.string(), epee::string_tools::pod_to_hex(get_genesis_block_hash(inputs[0]))); bool stop_requested = false; tools::signal_handler::install([&stop_requested](int type) { @@ -808,6 +843,13 @@ int main(int argc, char* argv[]) int dbr = resize_env(cache_dir.c_str()); CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to resize LMDB database: " + std::string(mdb_strerror(dbr))); + // open first db + MDB_env *env0; + MDB_txn *txn0; + MDB_dbi dbi0; + MDB_cursor *cur0; + open_db(inputs[0], &env0, &txn0, &cur0, &dbi0); + for (size_t n = 0; n < inputs.size(); ++n) { const std::string canonical = boost::filesystem::canonical(inputs[n]).string(); @@ -817,10 +859,10 @@ int main(int argc, char* argv[]) int dbr = mdb_txn_begin(env, NULL, 0, &txn); CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); size_t records = 0; - const uint64_t n_txes = core_storage[n]->get_db().get_tx_count(); - const std::string filename = (boost::filesystem::path(inputs[n]) / core_storage[n]->get_db().get_db_name()).string(); + const std::string filename = inputs[n]; std::vector blackballs; - for_all_transactions(filename, start_idx, [&](const cryptonote::transaction_prefix &tx)->bool + uint64_t n_txes; + for_all_transactions(filename, start_idx, n_txes, [&](const cryptonote::transaction_prefix &tx)->bool { std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; for (const auto &in: tx.vin) @@ -842,34 +884,34 @@ int main(int argc, char* argv[]) uint64_t instances = get_ring_instances(txn, new_ring); ++instances; set_ring_instances(txn, new_ring, instances); - if (ring_size == 1) + if (n == 0 && ring_size == 1) { - const crypto::public_key pkey = core_storage[n]->get_output_key(txin.amount, absolute[0]); + const crypto::public_key pkey = get_output_key(cur0, txin.amount, absolute[0]); MINFO("Blackballing output " << pkey << ", due to being used in a 1-ring"); std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; blackballs.push_back(pkey); add_spent_output(txn, output_data(txin.amount, absolute[0]), true); } - else if (instances == new_ring.size()) + else if (n == 0 && instances == new_ring.size()) { for (size_t o = 0; o < new_ring.size(); ++o) { - const crypto::public_key pkey = core_storage[n]->get_output_key(txin.amount, absolute[o]); + const crypto::public_key pkey = get_output_key(cur0, txin.amount, absolute[o]); MINFO("Blackballing output " << pkey << ", due to being used in " << new_ring.size() << " identical " << new_ring.size() << "-rings"); std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; blackballs.push_back(pkey); add_spent_output(txn, output_data(txin.amount, absolute[o]), true); } } - else if (get_relative_ring(txn, txin.k_image, relative_ring)) + else if (n > 0 && get_relative_ring(txn, txin.k_image, relative_ring)) { - MINFO("Key image " << txin.k_image << " already seen: rings " << + MDEBUG("Key image " << txin.k_image << " already seen: rings " << boost::join(relative_ring | boost::adaptors::transformed([](uint64_t out){return std::to_string(out);}), " ") << ", " << boost::join(txin.key_offsets | boost::adaptors::transformed([](uint64_t out){return std::to_string(out);}), " ")); std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; if (relative_ring != txin.key_offsets) { - MINFO("Rings are different"); + MDEBUG("Rings are different"); std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; const std::vector r0 = cryptonote::relative_output_offsets_to_absolute(relative_ring); const std::vector r1 = cryptonote::relative_output_offsets_to_absolute(txin.key_offsets); @@ -886,7 +928,7 @@ int main(int argc, char* argv[]) } else if (common.size() == 1) { - const crypto::public_key pkey = core_storage[n]->get_output_key(txin.amount, common[0]); + const crypto::public_key pkey = get_output_key(cur0, txin.amount, common[0]); MINFO("Blackballing output " << pkey << ", due to being used in rings with a single common element"); std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; blackballs.push_back(pkey); @@ -894,7 +936,7 @@ int main(int argc, char* argv[]) } else { - MINFO("The intersection has more than one element, it's still ok"); + MDEBUG("The intersection has more than one element, it's still ok"); std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; for (const auto &out: r0) if (std::find(common.begin(), common.end(), out) != common.end()) @@ -903,7 +945,8 @@ int main(int argc, char* argv[]) } } } - set_relative_ring(txn, txin.k_image, new_ring); + if (n == 0) + set_relative_ring(txn, txin.k_image, new_ring); } if (!blackballs.empty()) { @@ -976,7 +1019,7 @@ int main(int argc, char* argv[]) } if (known == absolute.size() - 1) { - const crypto::public_key pkey = core_storage[0]->get_output_key(od.amount, last_unknown); + const crypto::public_key pkey = get_output_key(cur0, od.amount, last_unknown); MINFO("Blackballing output " << pkey << ", due to being used in a " << absolute.size() << "-ring where all other outputs are known to be spent"); blackballs.push_back(pkey); @@ -996,6 +1039,7 @@ int main(int argc, char* argv[]) uint64_t diff = get_num_spent_outputs(false) - start_blackballed_outputs; LOG_PRINT_L0(std::to_string(diff) << " new outputs blackballed, " << get_num_spent_outputs(false) << " total outputs blackballed"); LOG_PRINT_L0("Blockchain blackball data exported OK"); + close_db(env0, txn0, cur0, dbi0); close(); return 0; From 4801d6b514777ba32ab012620a5786281cb57295 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Sun, 12 Aug 2018 09:47:03 +0000 Subject: [PATCH 04/13] blockchain_blackball: add stats --- .../blockchain_blackball.cpp | 96 ++++++++++++++++++- 1 file changed, 94 insertions(+), 2 deletions(-) diff --git a/src/blockchain_utilities/blockchain_blackball.cpp b/src/blockchain_utilities/blockchain_blackball.cpp index f034acb24..7f80eecef 100644 --- a/src/blockchain_utilities/blockchain_blackball.cpp +++ b/src/blockchain_utilities/blockchain_blackball.cpp @@ -61,6 +61,7 @@ static MDB_dbi dbi_processed_txidx; static MDB_dbi dbi_spent; static MDB_dbi dbi_ring_instances; static MDB_dbi dbi_newly_spent; +static MDB_dbi dbi_stats; static MDB_env *env = NULL; struct output_data @@ -79,6 +80,7 @@ struct output_data // spent: 128 bits, zerokval // ring_instances: vector -> uint64_t // newly_spent: 128 bits, zerokval +// stats: string -> arbitrary // static bool parse_db_sync_mode(std::string db_sync_mode) @@ -238,7 +240,7 @@ static void init(std::string cache_filename) dbr = mdb_env_create(&env); CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LDMB environment: " + std::string(mdb_strerror(dbr))); - dbr = mdb_env_set_maxdbs(env, 6); + dbr = mdb_env_set_maxdbs(env, 7); CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to set max env dbs: " + std::string(mdb_strerror(dbr))); const std::string actual_filename = get_cache_filename(cache_filename); dbr = mdb_env_open(env, actual_filename.c_str(), flags, 0664); @@ -272,6 +274,9 @@ static void init(std::string cache_filename) CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); mdb_set_dupsort(txn, dbi_newly_spent, compare_double64); + dbr = mdb_dbi_open(txn, "stats", MDB_CREATE, &dbi_stats); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + dbr = mdb_txn_commit(txn); CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to commit txn creating/opening database: " + std::string(mdb_strerror(dbr))); tx_active = false; @@ -287,6 +292,7 @@ static void close() mdb_dbi_close(env, dbi_spent); mdb_dbi_close(env, dbi_ring_instances); mdb_dbi_close(env, dbi_newly_spent); + mdb_dbi_close(env, dbi_stats); mdb_env_close(env); env = NULL; } @@ -641,6 +647,40 @@ static void add_key_image(MDB_txn *txn, const output_data &od, const crypto::key CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to set outputs: " + std::string(mdb_strerror(dbr))); } +static bool get_stat(MDB_txn *txn, const char *key, uint64_t &data) +{ + MDB_val k, v; + k.mv_data = (void*)key; + k.mv_size = strlen(key); + int dbr = mdb_get(txn, dbi_stats, &k, &v); + if (dbr == MDB_NOTFOUND) + return false; + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to get stat record"); + CHECK_AND_ASSERT_THROW_MES(v.mv_size == sizeof(uint64_t), "Unexpected record size"); + data = *(const uint64_t*)v.mv_data; + return true; +} + +static void set_stat(MDB_txn *txn, const char *key, uint64_t data) +{ + MDB_val k, v; + k.mv_data = (void*)key; + k.mv_size = strlen(key); + v.mv_data = (void*)&data; + v.mv_size = sizeof(uint64_t); + int dbr = mdb_put(txn, dbi_stats, &k, &v, 0); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to set stat record"); +} + +static void inc_stat(MDB_txn *txn, const char *key) +{ + uint64_t data; + if (!get_stat(txn, key, data)) + data = 0; + ++data; + set_stat(txn, key, data); +} + static void open_db(const std::string &filename, MDB_env **env, MDB_txn **txn, MDB_cursor **cur, MDB_dbi *dbi) { tools::create_directories_if_necessary(filename); @@ -688,6 +728,30 @@ static crypto::public_key get_output_key(MDB_cursor *cur, uint64_t amount, uint6 return *(const crypto::public_key*)(((const char*)v.mv_data) + sizeof(uint64_t) * 2); } +static void get_num_outputs(MDB_txn *txn, MDB_cursor *cur, MDB_dbi dbi, uint64_t &pre_rct, uint64_t &rct) +{ + uint64_t amount = 0; + MDB_val k = { sizeof(amount), (void*)&amount }, v; + int dbr = mdb_cursor_get(cur, &k, &v, MDB_SET); + if (dbr == MDB_NOTFOUND) + { + rct = 0; + } + else + { + if (dbr) throw std::runtime_error("Record 0 not found: " + std::string(mdb_strerror(dbr))); + mdb_size_t count = 0; + dbr = mdb_cursor_count(cur, &count); + if (dbr) throw std::runtime_error("Failed to count records: " + std::string(mdb_strerror(dbr))); + rct = count; + } + MDB_stat s; + dbr = mdb_stat(txn, dbi, &s); + if (dbr) throw std::runtime_error("Failed to count records: " + std::string(mdb_strerror(dbr))); + if (s.ms_entries < rct) throw std::runtime_error("Inconsistent records: " + std::string(mdb_strerror(dbr))); + pre_rct = s.ms_entries - rct; +} + static crypto::hash get_genesis_block_hash(const std::string &filename) { MDB_env *env; @@ -891,6 +955,7 @@ int main(int argc, char* argv[]) std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; blackballs.push_back(pkey); add_spent_output(txn, output_data(txin.amount, absolute[0]), true); + inc_stat(txn, txin.amount ? "pre-rct-ring-size-1" : "rct-ring-size-1"); } else if (n == 0 && instances == new_ring.size()) { @@ -901,6 +966,7 @@ int main(int argc, char* argv[]) std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; blackballs.push_back(pkey); add_spent_output(txn, output_data(txin.amount, absolute[o]), true); + inc_stat(txn, txin.amount ? "pre-rct-duplicate-rings" : "rct-duplicate-rings"); } } else if (n > 0 && get_relative_ring(txn, txin.k_image, relative_ring)) @@ -933,6 +999,7 @@ int main(int argc, char* argv[]) std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; blackballs.push_back(pkey); add_spent_output(txn, output_data(txin.amount, common[0]), true); + inc_stat(txn, txin.amount ? "pre-rct-key-image-attack" : "rct-key-image-attack"); } else { @@ -1017,13 +1084,14 @@ int main(int argc, char* argv[]) else last_unknown = out; } - if (known == absolute.size() - 1) + if (known == absolute.size() - 1 && !is_output_spent(txn, output_data(od.amount, last_unknown), false) && !is_output_spent(txn, output_data(od.amount, last_unknown), true)) { const crypto::public_key pkey = get_output_key(cur0, od.amount, last_unknown); MINFO("Blackballing output " << pkey << ", due to being used in a " << absolute.size() << "-ring where all other outputs are known to be spent"); blackballs.push_back(pkey); add_spent_output(txn, output_data(od.amount, last_unknown), true); + inc_stat(txn, od.amount ? "pre-rct-chain-reaction" : "rct-chain-reaction"); } } } @@ -1038,6 +1106,30 @@ int main(int argc, char* argv[]) uint64_t diff = get_num_spent_outputs(false) - start_blackballed_outputs; LOG_PRINT_L0(std::to_string(diff) << " new outputs blackballed, " << get_num_spent_outputs(false) << " total outputs blackballed"); + + MDB_txn *txn; + dbr = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + uint64_t pre_rct = 0, rct = 0; + get_num_outputs(txn0, cur0, dbi0, pre_rct, rct); + MINFO("Total pre-rct outputs: " << pre_rct); + MINFO("Total rct outputs: " << rct); + static const struct { const char *key; uint64_t base; } stat_keys[] = { + { "pre-rct-ring-size-1", pre_rct }, { "rct-ring-size-1", rct }, + { "pre-rct-duplicate-rings", pre_rct }, { "rct-duplicate-rings", rct }, + { "pre-rct-key-image-attack", pre_rct }, { "rct-key-image-attack", rct }, + { "pre-rct-chain-reaction", pre_rct }, { "rct-chain-reaction", rct }, + }; + for (const auto &key: stat_keys) + { + uint64_t data; + if (!get_stat(txn, key.key, data)) + data = 0; + float percent = key.base ? 100.0f * data / key.base : 0.0f; + MINFO(key.key << ": " << data << " (" << percent << "%)"); + } + mdb_txn_abort(txn); + LOG_PRINT_L0("Blockchain blackball data exported OK"); close_db(env0, txn0, cur0, dbi0); close(); From 80e4fef3c6bc423bec147d936685a827a4300c31 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Sun, 12 Aug 2018 09:50:48 +0000 Subject: [PATCH 05/13] blockchain_blackball: set transaction looping txn to read only --- src/blockchain_utilities/blockchain_blackball.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/blockchain_utilities/blockchain_blackball.cpp b/src/blockchain_utilities/blockchain_blackball.cpp index 7f80eecef..833a19a0b 100644 --- a/src/blockchain_utilities/blockchain_blackball.cpp +++ b/src/blockchain_utilities/blockchain_blackball.cpp @@ -341,7 +341,7 @@ static bool for_all_transactions(const std::string &filename, uint64_t &start_id if (dbr) throw std::runtime_error("Failed to open rings database file '" + actual_filename + "': " + std::string(mdb_strerror(dbr))); - dbr = mdb_txn_begin(env, NULL, 0, &txn); + dbr = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); if (dbr) throw std::runtime_error("Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); epee::misc_utils::auto_scope_leave_caller txn_dtor = epee::misc_utils::create_scope_leave_handler([&](){if (tx_active) mdb_txn_abort(txn);}); tx_active = true; From 2b2a681b017683153ed747f693b21f67049a2b27 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Sun, 12 Aug 2018 21:45:10 +0000 Subject: [PATCH 06/13] blockchain_blackball: avoid false positives for different amounts Identical offset based rings may not actually be identical rings since they represent different outputs --- .../blockchain_blackball.cpp | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/blockchain_utilities/blockchain_blackball.cpp b/src/blockchain_utilities/blockchain_blackball.cpp index 833a19a0b..42e04f195 100644 --- a/src/blockchain_utilities/blockchain_blackball.cpp +++ b/src/blockchain_utilities/blockchain_blackball.cpp @@ -298,14 +298,18 @@ static void close() } } -static std::string compress_ring(const std::vector &ring) +static std::string compress_ring(const std::vector &ring, std::string s = "") { - std::string s; for (uint64_t out: ring) s += tools::get_varint_data(out); return s; } +static std::string compress_ring(uint64_t amount, const std::vector &ring) +{ + return compress_ring(ring, tools::get_varint_data(amount)); +} + static std::vector decompress_ring(const std::string &s) { std::vector ring; @@ -583,9 +587,9 @@ static std::string keep_under_511(const std::string &s) return std::string((const char*)&hash, 32); } -static uint64_t get_ring_instances(MDB_txn *txn, const std::vector &ring) +static uint64_t get_ring_instances(MDB_txn *txn, uint64_t amount, const std::vector &ring) { - const std::string sring = keep_under_511(compress_ring(ring)); + const std::string sring = keep_under_511(compress_ring(amount, ring)); MDB_val k, v; k.mv_data = (void*)sring.data(); k.mv_size = sring.size(); @@ -596,9 +600,9 @@ static uint64_t get_ring_instances(MDB_txn *txn, const std::vector &ri return *(const uint64_t*)v.mv_data; } -static void set_ring_instances(MDB_txn *txn, const std::vector &ring, uint64_t count) +static void set_ring_instances(MDB_txn *txn, uint64_t amount, const std::vector &ring, uint64_t count) { - const std::string sring = keep_under_511(compress_ring(ring)); + const std::string sring = keep_under_511(compress_ring(amount, ring)); MDB_val k, v; k.mv_data = (void*)sring.data(); k.mv_size = sring.size(); @@ -945,9 +949,9 @@ int main(int argc, char* argv[]) std::vector relative_ring; std::vector new_ring = canonicalize(txin.key_offsets); const uint32_t ring_size = txin.key_offsets.size(); - uint64_t instances = get_ring_instances(txn, new_ring); + uint64_t instances = get_ring_instances(txn, txin.amount, new_ring); ++instances; - set_ring_instances(txn, new_ring, instances); + set_ring_instances(txn, txin.amount, new_ring, instances); if (n == 0 && ring_size == 1) { const crypto::public_key pkey = get_output_key(cur0, txin.amount, absolute[0]); From d6d276c6045a2ad8659e646eb03e06bffd62765c Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Sun, 12 Aug 2018 21:46:02 +0000 Subject: [PATCH 07/13] blockchain_blackball: fix chain reaction phase in incremental mode It makes it a lot slower, unfortunately --- .../blockchain_blackball.cpp | 76 +++++++++---------- 1 file changed, 35 insertions(+), 41 deletions(-) diff --git a/src/blockchain_utilities/blockchain_blackball.cpp b/src/blockchain_utilities/blockchain_blackball.cpp index 42e04f195..feafdc740 100644 --- a/src/blockchain_utilities/blockchain_blackball.cpp +++ b/src/blockchain_utilities/blockchain_blackball.cpp @@ -60,7 +60,6 @@ static MDB_dbi dbi_outputs; static MDB_dbi dbi_processed_txidx; static MDB_dbi dbi_spent; static MDB_dbi dbi_ring_instances; -static MDB_dbi dbi_newly_spent; static MDB_dbi dbi_stats; static MDB_env *env = NULL; @@ -79,7 +78,6 @@ struct output_data // processed_txidx: string -> uint64_t // spent: 128 bits, zerokval // ring_instances: vector -> uint64_t -// newly_spent: 128 bits, zerokval // stats: string -> arbitrary // @@ -240,7 +238,7 @@ static void init(std::string cache_filename) dbr = mdb_env_create(&env); CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LDMB environment: " + std::string(mdb_strerror(dbr))); - dbr = mdb_env_set_maxdbs(env, 7); + dbr = mdb_env_set_maxdbs(env, 6); CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to set max env dbs: " + std::string(mdb_strerror(dbr))); const std::string actual_filename = get_cache_filename(cache_filename); dbr = mdb_env_open(env, actual_filename.c_str(), flags, 0664); @@ -270,10 +268,6 @@ static void init(std::string cache_filename) dbr = mdb_dbi_open(txn, "ring_instances", MDB_CREATE, &dbi_ring_instances); CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); - dbr = mdb_dbi_open(txn, "newly_spent", MDB_CREATE | MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, &dbi_newly_spent); - CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); - mdb_set_dupsort(txn, dbi_newly_spent, compare_double64); - dbr = mdb_dbi_open(txn, "stats", MDB_CREATE, &dbi_stats); CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); @@ -291,7 +285,6 @@ static void close() mdb_dbi_close(env, dbi_processed_txidx); mdb_dbi_close(env, dbi_spent); mdb_dbi_close(env, dbi_ring_instances); - mdb_dbi_close(env, dbi_newly_spent); mdb_dbi_close(env, dbi_stats); mdb_env_close(env); env = NULL; @@ -423,7 +416,7 @@ static std::vector canonicalize(const std::vector &v) return c; } -static uint64_t get_num_spent_outputs(bool newly) +static uint64_t get_num_spent_outputs() { MDB_txn *txn; bool tx_active = false; @@ -434,7 +427,7 @@ static uint64_t get_num_spent_outputs(bool newly) tx_active = true; MDB_cursor *cur; - dbr = mdb_cursor_open(txn, newly ? dbi_newly_spent : dbi_spent, &cur); + dbr = mdb_cursor_open(txn, dbi_spent, &cur); CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open cursor for spent outputs: " + std::string(mdb_strerror(dbr))); MDB_val k, v; mdb_size_t count = 0; @@ -454,10 +447,10 @@ static uint64_t get_num_spent_outputs(bool newly) return count; } -static void add_spent_output(MDB_txn *txn, const output_data &od, bool newly) +static void add_spent_output(MDB_txn *txn, const output_data &od) { MDB_cursor *cur; - int dbr = mdb_cursor_open(txn, newly ? dbi_newly_spent : dbi_spent, &cur); + int dbr = mdb_cursor_open(txn, dbi_spent, &cur); CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open cursor for spent outputs: " + std::string(mdb_strerror(dbr))); MDB_val v = {sizeof(od), (void*)&od}; dbr = mdb_cursor_put(cur, (MDB_val *)&zerokval, &v, MDB_NODUPDATA); @@ -465,10 +458,10 @@ static void add_spent_output(MDB_txn *txn, const output_data &od, bool newly) mdb_cursor_close(cur); } -static bool is_output_spent(MDB_txn *txn, const output_data &od, bool newly) +static bool is_output_spent(MDB_txn *txn, const output_data &od) { MDB_cursor *cur; - int dbr = mdb_cursor_open(txn, newly ? dbi_newly_spent : dbi_spent, &cur); + int dbr = mdb_cursor_open(txn, dbi_spent, &cur); CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open cursor for spent outputs: " + std::string(mdb_strerror(dbr))); MDB_val v = {sizeof(od), (void*)&od}; dbr = mdb_cursor_get(cur, (MDB_val *)&zerokval, &v, MDB_GET_BOTH); @@ -478,10 +471,10 @@ static bool is_output_spent(MDB_txn *txn, const output_data &od, bool newly) return spent; } -static std::vector get_spent_outputs(MDB_txn *txn, bool newly) +static std::vector get_spent_outputs(MDB_txn *txn) { MDB_cursor *cur; - int dbr = mdb_cursor_open(txn, newly ? dbi_newly_spent : dbi_spent, &cur); + int dbr = mdb_cursor_open(txn, dbi_spent, &cur); CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open cursor for spent outputs: " + std::string(mdb_strerror(dbr))); MDB_val k, v; uint64_t count = 0; @@ -507,12 +500,6 @@ static std::vector get_spent_outputs(MDB_txn *txn, bool newly) return outs; } -static void clear_spent_outputs(MDB_txn *txn, bool newly) -{ - int dbr = mdb_drop(txn, newly ? dbi_newly_spent : dbi_spent, 0); - CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to clear spent outputs: " + std::string(mdb_strerror(dbr))); -} - static uint64_t get_processed_txidx(const std::string &name) { MDB_txn *txn; @@ -899,7 +886,7 @@ int main(int argc, char* argv[]) size_t done = 0; - const uint64_t start_blackballed_outputs = get_num_spent_outputs(false); + const uint64_t start_blackballed_outputs = get_num_spent_outputs(); tools::ringdb ringdb(output_file_path.string(), epee::string_tools::pod_to_hex(get_genesis_block_hash(inputs[0]))); @@ -958,7 +945,7 @@ int main(int argc, char* argv[]) MINFO("Blackballing output " << pkey << ", due to being used in a 1-ring"); std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; blackballs.push_back(pkey); - add_spent_output(txn, output_data(txin.amount, absolute[0]), true); + add_spent_output(txn, output_data(txin.amount, absolute[0])); inc_stat(txn, txin.amount ? "pre-rct-ring-size-1" : "rct-ring-size-1"); } else if (n == 0 && instances == new_ring.size()) @@ -969,7 +956,7 @@ int main(int argc, char* argv[]) MINFO("Blackballing output " << pkey << ", due to being used in " << new_ring.size() << " identical " << new_ring.size() << "-rings"); std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; blackballs.push_back(pkey); - add_spent_output(txn, output_data(txin.amount, absolute[o]), true); + add_spent_output(txn, output_data(txin.amount, absolute[o])); inc_stat(txn, txin.amount ? "pre-rct-duplicate-rings" : "rct-duplicate-rings"); } } @@ -1002,7 +989,7 @@ int main(int argc, char* argv[]) MINFO("Blackballing output " << pkey << ", due to being used in rings with a single common element"); std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; blackballs.push_back(pkey); - add_spent_output(txn, output_data(txin.amount, common[0]), true); + add_spent_output(txn, output_data(txin.amount, common[0])); inc_stat(txn, txin.amount ? "pre-rct-key-image-attack" : "rct-key-image-attack"); } else @@ -1024,7 +1011,7 @@ int main(int argc, char* argv[]) ringdb.blackball(blackballs); blackballs.clear(); } - set_processed_txidx(txn, canonical, start_idx); + set_processed_txidx(txn, canonical, start_idx+1); ++records; if (records >= records_per_sync) @@ -1052,9 +1039,19 @@ int main(int argc, char* argv[]) break; } - while (get_num_spent_outputs(true) != 0) + std::vector work_spent; + if (get_num_spent_outputs() > start_blackballed_outputs) + { + MDB_txn *txn; + dbr = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + work_spent = get_spent_outputs(txn); + mdb_txn_abort(txn); + } + + while (!work_spent.empty()) { - LOG_PRINT_L0("Secondary pass due to " << get_num_spent_outputs(true) << " newly found spent outputs"); + LOG_PRINT_L0("Secondary pass on " << work_spent.size() << " spent outputs"); int dbr = resize_env(cache_dir.c_str()); CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to resize LMDB database: " + std::string(mdb_strerror(dbr))); @@ -1063,14 +1060,10 @@ int main(int argc, char* argv[]) dbr = mdb_txn_begin(env, NULL, 0, &txn); CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); - std::vector work_spent = get_spent_outputs(txn, true); - clear_spent_outputs(txn, true); - - for (const auto &od: work_spent) - add_spent_output(txn, od, false); - std::vector blackballs; - for (const output_data &od: work_spent) + std::vector scan_spent = std::move(work_spent); + work_spent.clear(); + for (const output_data &od: scan_spent) { std::vector key_images = get_key_images(txn, od); for (const crypto::key_image &ki: key_images) @@ -1083,18 +1076,19 @@ int main(int argc, char* argv[]) for (uint64_t out: absolute) { output_data new_od(od.amount, out); - if (is_output_spent(txn, new_od, false)) + if (is_output_spent(txn, new_od)) ++known; else last_unknown = out; } - if (known == absolute.size() - 1 && !is_output_spent(txn, output_data(od.amount, last_unknown), false) && !is_output_spent(txn, output_data(od.amount, last_unknown), true)) + if (known == absolute.size() - 1 && !is_output_spent(txn, output_data(od.amount, last_unknown))) { const crypto::public_key pkey = get_output_key(cur0, od.amount, last_unknown); MINFO("Blackballing output " << pkey << ", due to being used in a " << absolute.size() << "-ring where all other outputs are known to be spent"); blackballs.push_back(pkey); - add_spent_output(txn, output_data(od.amount, last_unknown), true); + add_spent_output(txn, output_data(od.amount, last_unknown)); + work_spent.push_back(output_data(od.amount, last_unknown)); inc_stat(txn, od.amount ? "pre-rct-chain-reaction" : "rct-chain-reaction"); } } @@ -1108,8 +1102,8 @@ int main(int argc, char* argv[]) CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to commit txn creating/opening database: " + std::string(mdb_strerror(dbr))); } - uint64_t diff = get_num_spent_outputs(false) - start_blackballed_outputs; - LOG_PRINT_L0(std::to_string(diff) << " new outputs blackballed, " << get_num_spent_outputs(false) << " total outputs blackballed"); + uint64_t diff = get_num_spent_outputs() - start_blackballed_outputs; + LOG_PRINT_L0(std::to_string(diff) << " new outputs blackballed, " << get_num_spent_outputs() << " total outputs blackballed"); MDB_txn *txn; dbr = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); From eb8a51be68f9a32b85a5fcfccb5ab3baa461639e Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Tue, 14 Aug 2018 10:57:39 +0000 Subject: [PATCH 08/13] blockchain_blackball: detect spent outputs by partial ring reuse --- .../blockchain_blackball.cpp | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/src/blockchain_utilities/blockchain_blackball.cpp b/src/blockchain_utilities/blockchain_blackball.cpp index feafdc740..df1d14eaf 100644 --- a/src/blockchain_utilities/blockchain_blackball.cpp +++ b/src/blockchain_utilities/blockchain_blackball.cpp @@ -587,6 +587,25 @@ static uint64_t get_ring_instances(MDB_txn *txn, uint64_t amount, const std::vec return *(const uint64_t*)v.mv_data; } +static uint64_t get_ring_subset_instances(MDB_txn *txn, uint64_t amount, const std::vector &ring) +{ + uint64_t instances = get_ring_instances(txn, amount, ring); + if (ring.size() > 11) + return instances; + + uint64_t extra = 0; + for (uint64_t mask = 1; mask < (1u << ring.size()) - 1; ++mask) + { + std::vector subset; + subset.reserve(ring.size()); + for (size_t i = 0; i < ring.size(); ++i) + if ((mask >> i) & 1) + subset.push_back(ring[i]); + extra += get_ring_instances(txn, amount, subset); + } + return instances + extra; +} + static void set_ring_instances(MDB_txn *txn, uint64_t amount, const std::vector &ring, uint64_t count) { const std::string sring = keep_under_511(compress_ring(amount, ring)); @@ -808,6 +827,7 @@ int main(int argc, char* argv[]) "database", available_dbs.c_str(), default_db_type }; const command_line::arg_descriptor arg_rct_only = {"rct-only", "Only work on ringCT outputs", false}; + const command_line::arg_descriptor arg_check_subsets = {"check-subsets", "Check ring subsets (very expensive)", false}; const command_line::arg_descriptor > arg_inputs = {"inputs", "Path to Monero DB, and path to any fork DBs"}; const command_line::arg_descriptor arg_db_sync_mode = { "db-sync-mode" @@ -819,6 +839,7 @@ int main(int argc, char* argv[]) command_line::add_arg(desc_cmd_sett, arg_log_level); command_line::add_arg(desc_cmd_sett, arg_database); command_line::add_arg(desc_cmd_sett, arg_rct_only); + command_line::add_arg(desc_cmd_sett, arg_check_subsets); command_line::add_arg(desc_cmd_sett, arg_db_sync_mode); command_line::add_arg(desc_cmd_sett, arg_inputs); command_line::add_arg(desc_cmd_only, command_line::arg_help); @@ -857,6 +878,7 @@ int main(int argc, char* argv[]) output_file_path = command_line::get_arg(vm, arg_blackball_db_dir); bool opt_rct_only = command_line::get_arg(vm, arg_rct_only); + bool opt_check_subsets = command_line::get_arg(vm, arg_check_subsets); std::string db_type = command_line::get_arg(vm, arg_database); if (!cryptonote::blockchain_valid_db_type(db_type)) @@ -960,6 +982,18 @@ int main(int argc, char* argv[]) inc_stat(txn, txin.amount ? "pre-rct-duplicate-rings" : "rct-duplicate-rings"); } } + else if (n == 0 && opt_check_subsets && get_ring_subset_instances(txn, txin.amount, new_ring) >= new_ring.size()) + { + for (size_t o = 0; o < new_ring.size(); ++o) + { + const crypto::public_key pkey = get_output_key(cur0, txin.amount, absolute[o]); + MINFO("Blackballing output " << pkey << ", due to being used in " << new_ring.size() << " subsets of " << new_ring.size() << "-rings"); + std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; + blackballs.push_back(pkey); + add_spent_output(txn, output_data(txin.amount, absolute[o])); + inc_stat(txn, txin.amount ? "pre-rct-subset-rings" : "rct-subset-rings"); + } + } else if (n > 0 && get_relative_ring(txn, txin.k_image, relative_ring)) { MDEBUG("Key image " << txin.k_image << " already seen: rings " << @@ -1115,6 +1149,7 @@ int main(int argc, char* argv[]) static const struct { const char *key; uint64_t base; } stat_keys[] = { { "pre-rct-ring-size-1", pre_rct }, { "rct-ring-size-1", rct }, { "pre-rct-duplicate-rings", pre_rct }, { "rct-duplicate-rings", rct }, + { "pre-rct-subset-rings", pre_rct }, { "rct-subset-rings", rct }, { "pre-rct-key-image-attack", pre_rct }, { "rct-key-image-attack", rct }, { "pre-rct-chain-reaction", pre_rct }, { "rct-chain-reaction", rct }, }; From 639a3c019cf6ad45743a8432ca37bd5c162c4d35 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Tue, 14 Aug 2018 15:06:16 +0000 Subject: [PATCH 09/13] blockchain_blackball: make it clear secondary passes are not incremental yet --- src/blockchain_utilities/blockchain_blackball.cpp | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/blockchain_utilities/blockchain_blackball.cpp b/src/blockchain_utilities/blockchain_blackball.cpp index df1d14eaf..8a578a56d 100644 --- a/src/blockchain_utilities/blockchain_blackball.cpp +++ b/src/blockchain_utilities/blockchain_blackball.cpp @@ -1061,7 +1061,7 @@ int main(int argc, char* argv[]) if (stop_requested) { - MINFO("Stopping scan, secondary passes will still happen..."); + MINFO("Stopping scan..."); return false; } return true; @@ -1074,6 +1074,10 @@ int main(int argc, char* argv[]) } std::vector work_spent; + + if (stop_requested) + goto skip_secondary_passes; + if (get_num_spent_outputs() > start_blackballed_outputs) { MDB_txn *txn; @@ -1126,6 +1130,12 @@ int main(int argc, char* argv[]) inc_stat(txn, od.amount ? "pre-rct-chain-reaction" : "rct-chain-reaction"); } } + + if (stop_requested) + { + MINFO("Stopping secondary passes. Secondary passes are not incremental, they will re-run fully."); + return false; + } } if (!blackballs.empty()) { @@ -1136,6 +1146,7 @@ int main(int argc, char* argv[]) CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to commit txn creating/opening database: " + std::string(mdb_strerror(dbr))); } +skip_secondary_passes: uint64_t diff = get_num_spent_outputs() - start_blackballed_outputs; LOG_PRINT_L0(std::to_string(diff) << " new outputs blackballed, " << get_num_spent_outputs() << " total outputs blackballed"); From b66ba78306913f28fd73831e8667ca5e966a9f07 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Sat, 18 Aug 2018 12:40:26 +0000 Subject: [PATCH 10/13] blockchain_blackball: do not process duplicate blockchains parts --- .../blockchain_blackball.cpp | 76 +++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/src/blockchain_utilities/blockchain_blackball.cpp b/src/blockchain_utilities/blockchain_blackball.cpp index 8a578a56d..de7163208 100644 --- a/src/blockchain_utilities/blockchain_blackball.cpp +++ b/src/blockchain_utilities/blockchain_blackball.cpp @@ -398,6 +398,77 @@ static bool for_all_transactions(const std::string &filename, uint64_t &start_id return fret; } +static uint64_t find_first_diverging_transaction(const std::string &first_filename, const std::string &second_filename) +{ + MDB_env *env[2]; + MDB_dbi dbi[2]; + MDB_txn *txn[2]; + MDB_cursor *cur[2]; + int dbr; + bool tx_active[2] = { false, false }; + uint64_t n_txes[2]; + MDB_val k; + MDB_val v[2]; + + epee::misc_utils::auto_scope_leave_caller txn_dtor[2]; + for (int i = 0; i < 2; ++i) + { + dbr = mdb_env_create(&env[i]); + if (dbr) throw std::runtime_error("Failed to create LDMB environment: " + std::string(mdb_strerror(dbr))); + dbr = mdb_env_set_maxdbs(env[i], 2); + if (dbr) throw std::runtime_error("Failed to set max env dbs: " + std::string(mdb_strerror(dbr))); + const std::string actual_filename = i ? second_filename : first_filename; + dbr = mdb_env_open(env[i], actual_filename.c_str(), 0, 0664); + if (dbr) throw std::runtime_error("Failed to open rings database file '" + + actual_filename + "': " + std::string(mdb_strerror(dbr))); + + dbr = mdb_txn_begin(env[i], NULL, MDB_RDONLY, &txn[i]); + if (dbr) throw std::runtime_error("Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + txn_dtor[i] = epee::misc_utils::create_scope_leave_handler([&, i](){if (tx_active[i]) mdb_txn_abort(txn[i]);}); + tx_active[i] = true; + + dbr = mdb_dbi_open(txn[i], "txs_pruned", MDB_INTEGERKEY, &dbi[i]); + if (dbr) + dbr = mdb_dbi_open(txn[i], "txs", MDB_INTEGERKEY, &dbi[i]); + if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + dbr = mdb_cursor_open(txn[i], dbi[i], &cur[i]); + if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr))); + MDB_stat stat; + dbr = mdb_stat(txn[i], dbi[i], &stat); + if (dbr) throw std::runtime_error("Failed to query m_block_info: " + std::string(mdb_strerror(dbr))); + n_txes[i] = stat.ms_entries; + } + + if (n_txes[0] == 0 || n_txes[1] == 0) + throw std::runtime_error("No transaction in the database"); + uint64_t lo = 0, hi = std::min(n_txes[0], n_txes[1]) - 1; + while (lo <= hi) + { + uint64_t mid = (lo + hi) / 2; + + k.mv_size = sizeof(uint64_t); + k.mv_data = (void*)∣ + dbr = mdb_cursor_get(cur[0], &k, &v[0], MDB_SET); + if (dbr) throw std::runtime_error("Failed to query transaction: " + std::string(mdb_strerror(dbr))); + dbr = mdb_cursor_get(cur[1], &k, &v[1], MDB_SET); + if (dbr) throw std::runtime_error("Failed to query transaction: " + std::string(mdb_strerror(dbr))); + if (v[0].mv_size == v[1].mv_size && !memcmp(v[0].mv_data, v[1].mv_data, v[0].mv_size)) + lo = mid + 1; + else + hi = mid - 1; + } + + for (int i = 0; i < 2; ++i) + { + mdb_cursor_close(cur[i]); + mdb_txn_commit(txn[i]); + tx_active[i] = false; + mdb_dbi_close(env[i], dbi[i]); + mdb_env_close(env[i]); + } + return hi; +} + static std::vector canonicalize(const std::vector &v) { std::vector c; @@ -931,6 +1002,11 @@ int main(int argc, char* argv[]) { const std::string canonical = boost::filesystem::canonical(inputs[n]).string(); uint64_t start_idx = get_processed_txidx(canonical); + if (n > 0 && start_idx == 0) + { + start_idx = find_first_diverging_transaction(inputs[0], inputs[n]); + LOG_PRINT_L0("First diverging transaction at " << start_idx); + } LOG_PRINT_L0("Reading blockchain from " << inputs[n] << " from " << start_idx); MDB_txn *txn; int dbr = mdb_txn_begin(env, NULL, 0, &txn); From 4bce935b4038046ed47db63ddd63fc4feab566cd Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Sun, 19 Aug 2018 16:52:56 +0000 Subject: [PATCH 11/13] blockchain_blackball: more optimizations --- .../blockchain_blackball.cpp | 131 ++++++++++++------ 1 file changed, 86 insertions(+), 45 deletions(-) diff --git a/src/blockchain_utilities/blockchain_blackball.cpp b/src/blockchain_utilities/blockchain_blackball.cpp index de7163208..e68de9dc4 100644 --- a/src/blockchain_utilities/blockchain_blackball.cpp +++ b/src/blockchain_utilities/blockchain_blackball.cpp @@ -293,14 +293,24 @@ static void close() static std::string compress_ring(const std::vector &ring, std::string s = "") { + const size_t sz = s.size(); + s.resize(s.size() + 12 * ring.size()); + char *ptr = (char*)s.data() + sz; for (uint64_t out: ring) - s += tools::get_varint_data(out); + tools::write_varint(ptr, out); + if (ptr > s.data() + sz + 12 * ring.size()) + throw std::runtime_error("varint output overflow"); + s.resize(ptr - s.data()); return s; } static std::string compress_ring(uint64_t amount, const std::vector &ring) { - return compress_ring(ring, tools::get_varint_data(amount)); + char s[12], *ptr = s; + tools::write_varint(ptr, amount); + if (ptr > s + sizeof(s)) + throw std::runtime_error("varint output overflow"); + return compress_ring(ring, std::string(s, ptr-s)); } static std::vector decompress_ring(const std::string &s) @@ -518,27 +528,19 @@ static uint64_t get_num_spent_outputs() return count; } -static void add_spent_output(MDB_txn *txn, const output_data &od) +static void add_spent_output(MDB_cursor *cur, const output_data &od) { - MDB_cursor *cur; - int dbr = mdb_cursor_open(txn, dbi_spent, &cur); - CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open cursor for spent outputs: " + std::string(mdb_strerror(dbr))); MDB_val v = {sizeof(od), (void*)&od}; - dbr = mdb_cursor_put(cur, (MDB_val *)&zerokval, &v, MDB_NODUPDATA); + int dbr = mdb_cursor_put(cur, (MDB_val *)&zerokval, &v, MDB_NODUPDATA); CHECK_AND_ASSERT_THROW_MES(!dbr || dbr == MDB_KEYEXIST, "Failed to add spent output: " + std::string(mdb_strerror(dbr))); - mdb_cursor_close(cur); } -static bool is_output_spent(MDB_txn *txn, const output_data &od) +static bool is_output_spent(MDB_cursor *cur, const output_data &od) { - MDB_cursor *cur; - int dbr = mdb_cursor_open(txn, dbi_spent, &cur); - CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open cursor for spent outputs: " + std::string(mdb_strerror(dbr))); MDB_val v = {sizeof(od), (void*)&od}; - dbr = mdb_cursor_get(cur, (MDB_val *)&zerokval, &v, MDB_GET_BOTH); + int dbr = mdb_cursor_get(cur, (MDB_val *)&zerokval, &v, MDB_GET_BOTH); CHECK_AND_ASSERT_THROW_MES(!dbr || dbr == MDB_NOTFOUND, "Failed to get spent output: " + std::string(mdb_strerror(dbr))); bool spent = dbr == 0; - mdb_cursor_close(cur); return spent; } @@ -612,7 +614,6 @@ static void set_processed_txidx(MDB_txn *txn, const std::string &name, uint64_t static bool get_relative_ring(MDB_txn *txn, const crypto::key_image &ki, std::vector &ring) { - const std::string sring = compress_ring(ring); MDB_val k, v; k.mv_data = (void*)&ki; k.mv_size = sizeof(ki); @@ -665,10 +666,11 @@ static uint64_t get_ring_subset_instances(MDB_txn *txn, uint64_t amount, const s return instances; uint64_t extra = 0; + std::vector subset; + subset.reserve(ring.size()); for (uint64_t mask = 1; mask < (1u << ring.size()) - 1; ++mask) { - std::vector subset; - subset.reserve(ring.size()); + subset.resize(0); for (size_t i = 0; i < ring.size(); ++i) if ((mask >> i) & 1) subset.push_back(ring[i]); @@ -677,16 +679,28 @@ static uint64_t get_ring_subset_instances(MDB_txn *txn, uint64_t amount, const s return instances + extra; } -static void set_ring_instances(MDB_txn *txn, uint64_t amount, const std::vector &ring, uint64_t count) +static uint64_t inc_ring_instances(MDB_txn *txn, uint64_t amount, const std::vector &ring) { const std::string sring = keep_under_511(compress_ring(amount, ring)); MDB_val k, v; k.mv_data = (void*)sring.data(); k.mv_size = sring.size(); + + int dbr = mdb_get(txn, dbi_ring_instances, &k, &v); + CHECK_AND_ASSERT_THROW_MES(!dbr || dbr == MDB_NOTFOUND, "Failed to get ring instances: " + std::string(mdb_strerror(dbr))); + + uint64_t count; + if (dbr == MDB_NOTFOUND) + count = 1; + else + count = 1 + *(const uint64_t*)v.mv_data; + v.mv_data = &count; v.mv_size = sizeof(count); - int dbr = mdb_put(txn, dbi_ring_instances, &k, &v, 0); - CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to get ring instances: " + std::string(mdb_strerror(dbr))); + dbr = mdb_put(txn, dbi_ring_instances, &k, &v, 0); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to set ring instances: " + std::string(mdb_strerror(dbr))); + + return count; } static std::vector get_key_images(MDB_txn *txn, const output_data &od) @@ -899,6 +913,7 @@ int main(int argc, char* argv[]) }; const command_line::arg_descriptor arg_rct_only = {"rct-only", "Only work on ringCT outputs", false}; const command_line::arg_descriptor arg_check_subsets = {"check-subsets", "Check ring subsets (very expensive)", false}; + const command_line::arg_descriptor arg_verbose = {"verbose", "Verbose output)", false}; const command_line::arg_descriptor > arg_inputs = {"inputs", "Path to Monero DB, and path to any fork DBs"}; const command_line::arg_descriptor arg_db_sync_mode = { "db-sync-mode" @@ -911,6 +926,7 @@ int main(int argc, char* argv[]) command_line::add_arg(desc_cmd_sett, arg_database); command_line::add_arg(desc_cmd_sett, arg_rct_only); command_line::add_arg(desc_cmd_sett, arg_check_subsets); + command_line::add_arg(desc_cmd_sett, arg_verbose); command_line::add_arg(desc_cmd_sett, arg_db_sync_mode); command_line::add_arg(desc_cmd_sett, arg_inputs); command_line::add_arg(desc_cmd_only, command_line::arg_help); @@ -950,6 +966,7 @@ int main(int argc, char* argv[]) output_file_path = command_line::get_arg(vm, arg_blackball_db_dir); bool opt_rct_only = command_line::get_arg(vm, arg_rct_only); bool opt_check_subsets = command_line::get_arg(vm, arg_check_subsets); + bool opt_verbose = command_line::get_arg(vm, arg_verbose); std::string db_type = command_line::get_arg(vm, arg_database); if (!cryptonote::blockchain_valid_db_type(db_type)) @@ -1011,6 +1028,9 @@ int main(int argc, char* argv[]) MDB_txn *txn; int dbr = mdb_txn_begin(env, NULL, 0, &txn); CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + MDB_cursor *cur; + dbr = mdb_cursor_open(txn, dbi_spent, &cur); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB cursor: " + std::string(mdb_strerror(dbr))); size_t records = 0; const std::string filename = inputs[n]; std::vector blackballs; @@ -1034,16 +1054,17 @@ int main(int argc, char* argv[]) std::vector relative_ring; std::vector new_ring = canonicalize(txin.key_offsets); const uint32_t ring_size = txin.key_offsets.size(); - uint64_t instances = get_ring_instances(txn, txin.amount, new_ring); - ++instances; - set_ring_instances(txn, txin.amount, new_ring, instances); + const uint64_t instances = inc_ring_instances(txn, txin.amount, new_ring); if (n == 0 && ring_size == 1) { const crypto::public_key pkey = get_output_key(cur0, txin.amount, absolute[0]); - MINFO("Blackballing output " << pkey << ", due to being used in a 1-ring"); - std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; + if (opt_verbose) + { + MINFO("Blackballing output " << pkey << ", due to being used in a 1-ring"); + std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; + } blackballs.push_back(pkey); - add_spent_output(txn, output_data(txin.amount, absolute[0])); + add_spent_output(cur, output_data(txin.amount, absolute[0])); inc_stat(txn, txin.amount ? "pre-rct-ring-size-1" : "rct-ring-size-1"); } else if (n == 0 && instances == new_ring.size()) @@ -1051,10 +1072,13 @@ int main(int argc, char* argv[]) for (size_t o = 0; o < new_ring.size(); ++o) { const crypto::public_key pkey = get_output_key(cur0, txin.amount, absolute[o]); - MINFO("Blackballing output " << pkey << ", due to being used in " << new_ring.size() << " identical " << new_ring.size() << "-rings"); - std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; + if (opt_verbose) + { + MINFO("Blackballing output " << pkey << ", due to being used in " << new_ring.size() << " identical " << new_ring.size() << "-rings"); + std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; + } blackballs.push_back(pkey); - add_spent_output(txn, output_data(txin.amount, absolute[o])); + add_spent_output(cur, output_data(txin.amount, absolute[o])); inc_stat(txn, txin.amount ? "pre-rct-duplicate-rings" : "rct-duplicate-rings"); } } @@ -1063,10 +1087,13 @@ int main(int argc, char* argv[]) for (size_t o = 0; o < new_ring.size(); ++o) { const crypto::public_key pkey = get_output_key(cur0, txin.amount, absolute[o]); - MINFO("Blackballing output " << pkey << ", due to being used in " << new_ring.size() << " subsets of " << new_ring.size() << "-rings"); - std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; + if (opt_verbose) + { + MINFO("Blackballing output " << pkey << ", due to being used in " << new_ring.size() << " subsets of " << new_ring.size() << "-rings"); + std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; + } blackballs.push_back(pkey); - add_spent_output(txn, output_data(txin.amount, absolute[o])); + add_spent_output(cur, output_data(txin.amount, absolute[o])); inc_stat(txn, txin.amount ? "pre-rct-subset-rings" : "rct-subset-rings"); } } @@ -1096,10 +1123,13 @@ int main(int argc, char* argv[]) else if (common.size() == 1) { const crypto::public_key pkey = get_output_key(cur0, txin.amount, common[0]); - MINFO("Blackballing output " << pkey << ", due to being used in rings with a single common element"); - std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; + if (opt_verbose) + { + MINFO("Blackballing output " << pkey << ", due to being used in rings with a single common element"); + std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; + } blackballs.push_back(pkey); - add_spent_output(txn, output_data(txin.amount, common[0])); + add_spent_output(cur, output_data(txin.amount, common[0])); inc_stat(txn, txin.amount ? "pre-rct-key-image-attack" : "rct-key-image-attack"); } else @@ -1116,22 +1146,25 @@ int main(int argc, char* argv[]) if (n == 0) set_relative_ring(txn, txin.k_image, new_ring); } - if (!blackballs.empty()) - { - ringdb.blackball(blackballs); - blackballs.clear(); - } set_processed_txidx(txn, canonical, start_idx+1); ++records; if (records >= records_per_sync) { + if (!blackballs.empty()) + { + ringdb.blackball(blackballs); + blackballs.clear(); + } + mdb_cursor_close(cur); dbr = mdb_txn_commit(txn); CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to commit txn creating/opening database: " + std::string(mdb_strerror(dbr))); int dbr = resize_env(cache_dir.c_str()); CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to resize LMDB database: " + std::string(mdb_strerror(dbr))); dbr = mdb_txn_begin(env, NULL, 0, &txn); CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + dbr = mdb_cursor_open(txn, dbi_spent, &cur); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB cursor: " + std::string(mdb_strerror(dbr))); records = 0; } @@ -1142,6 +1175,7 @@ int main(int argc, char* argv[]) } return true; }); + mdb_cursor_close(cur); dbr = mdb_txn_commit(txn); CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to commit txn creating/opening database: " + std::string(mdb_strerror(dbr))); LOG_PRINT_L0("blockchain from " << inputs[n] << " processed till tx idx " << start_idx); @@ -1173,6 +1207,9 @@ int main(int argc, char* argv[]) MDB_txn *txn; dbr = mdb_txn_begin(env, NULL, 0, &txn); CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + MDB_cursor *cur; + dbr = mdb_cursor_open(txn, dbi_spent, &cur); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB cursor: " + std::string(mdb_strerror(dbr))); std::vector blackballs; std::vector scan_spent = std::move(work_spent); @@ -1190,18 +1227,21 @@ int main(int argc, char* argv[]) for (uint64_t out: absolute) { output_data new_od(od.amount, out); - if (is_output_spent(txn, new_od)) + if (is_output_spent(cur, new_od)) ++known; else last_unknown = out; } - if (known == absolute.size() - 1 && !is_output_spent(txn, output_data(od.amount, last_unknown))) + if (known == absolute.size() - 1) { const crypto::public_key pkey = get_output_key(cur0, od.amount, last_unknown); - MINFO("Blackballing output " << pkey << ", due to being used in a " << - absolute.size() << "-ring where all other outputs are known to be spent"); + if (opt_verbose) + { + MINFO("Blackballing output " << pkey << ", due to being used in a " << + absolute.size() << "-ring where all other outputs are known to be spent"); + } blackballs.push_back(pkey); - add_spent_output(txn, output_data(od.amount, last_unknown)); + add_spent_output(cur, output_data(od.amount, last_unknown)); work_spent.push_back(output_data(od.amount, last_unknown)); inc_stat(txn, od.amount ? "pre-rct-chain-reaction" : "rct-chain-reaction"); } @@ -1218,6 +1258,7 @@ int main(int argc, char* argv[]) ringdb.blackball(blackballs); blackballs.clear(); } + mdb_cursor_close(cur); dbr = mdb_txn_commit(txn); CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to commit txn creating/opening database: " + std::string(mdb_strerror(dbr))); } From 44439c32088dc684d686a392acf807f12ca0e0e9 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Wed, 29 Aug 2018 16:57:12 +0000 Subject: [PATCH 12/13] record blackballs as amount/offset, and add export ability --- .../blockchain_blackball.cpp | 241 +++++++++++++++--- src/simplewallet/simplewallet.cpp | 63 +++-- src/wallet/ringdb.cpp | 46 ++-- src/wallet/ringdb.h | 10 +- src/wallet/wallet2.cpp | 14 +- src/wallet/wallet2.h | 8 +- tests/unit_tests/ringdb.cpp | 8 +- 7 files changed, 287 insertions(+), 103 deletions(-) diff --git a/src/blockchain_utilities/blockchain_blackball.cpp b/src/blockchain_utilities/blockchain_blackball.cpp index e68de9dc4..9a31b95b3 100644 --- a/src/blockchain_utilities/blockchain_blackball.cpp +++ b/src/blockchain_utilities/blockchain_blackball.cpp @@ -66,17 +66,17 @@ static MDB_env *env = NULL; struct output_data { uint64_t amount; - uint64_t index; - output_data(): amount(0), index(0) {} - output_data(uint64_t a, uint64_t i): amount(a), index(i) {} - bool operator==(const output_data &other) const { return other.amount == amount && other.index == index; } + uint64_t offset; + output_data(): amount(0), offset(0) {} + output_data(uint64_t a, uint64_t i): amount(a), offset(i) {} + bool operator==(const output_data &other) const { return other.amount == amount && other.offset == offset; } }; // // relative_rings: key_image -> vector // outputs: 128 bits -> set of key images // processed_txidx: string -> uint64_t -// spent: 128 bits, zerokval +// spent: amount -> offset // ring_instances: vector -> uint64_t // stats: string -> arbitrary // @@ -263,7 +263,7 @@ static void init(std::string cache_filename) dbr = mdb_dbi_open(txn, "spent", MDB_CREATE | MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, &dbi_spent); CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); - mdb_set_dupsort(txn, dbi_spent, compare_double64); + mdb_set_dupsort(txn, dbi_spent, compare_uint64); dbr = mdb_dbi_open(txn, "ring_instances", MDB_CREATE, &dbi_ring_instances); CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); @@ -511,13 +511,19 @@ static uint64_t get_num_spent_outputs() dbr = mdb_cursor_open(txn, dbi_spent, &cur); CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open cursor for spent outputs: " + std::string(mdb_strerror(dbr))); MDB_val k, v; - mdb_size_t count = 0; - dbr = mdb_cursor_get(cur, &k, &v, MDB_FIRST); - if (dbr != MDB_NOTFOUND) + mdb_size_t count = 0, tmp; + + MDB_cursor_op op = MDB_FIRST; + while (1) { - CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to get first spent output: " + std::string(mdb_strerror(dbr))); - dbr = mdb_cursor_count(cur, &count); + dbr = mdb_cursor_get(cur, &k, &v, op); + op = MDB_NEXT_NODUP; + if (dbr == MDB_NOTFOUND) + break; + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to get first/next spent output: " + std::string(mdb_strerror(dbr))); + dbr = mdb_cursor_count(cur, &tmp); CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to count entries: " + std::string(mdb_strerror(dbr))); + count += tmp; } mdb_cursor_close(cur); @@ -530,15 +536,17 @@ static uint64_t get_num_spent_outputs() static void add_spent_output(MDB_cursor *cur, const output_data &od) { - MDB_val v = {sizeof(od), (void*)&od}; - int dbr = mdb_cursor_put(cur, (MDB_val *)&zerokval, &v, MDB_NODUPDATA); + MDB_val k = {sizeof(od.amount), (void*)&od.amount}; + MDB_val v = {sizeof(od.offset), (void*)&od.offset}; + int dbr = mdb_cursor_put(cur, &k, &v, 0); CHECK_AND_ASSERT_THROW_MES(!dbr || dbr == MDB_KEYEXIST, "Failed to add spent output: " + std::string(mdb_strerror(dbr))); } static bool is_output_spent(MDB_cursor *cur, const output_data &od) { - MDB_val v = {sizeof(od), (void*)&od}; - int dbr = mdb_cursor_get(cur, (MDB_val *)&zerokval, &v, MDB_GET_BOTH); + MDB_val k = {sizeof(od.amount), (void*)&od.amount}; + MDB_val v = {sizeof(od.offset), (void*)&od.offset}; + int dbr = mdb_cursor_get(cur, &k, &v, MDB_GET_BOTH); CHECK_AND_ASSERT_THROW_MES(!dbr || dbr == MDB_NOTFOUND, "Failed to get spent output: " + std::string(mdb_strerror(dbr))); bool spent = dbr == 0; return spent; @@ -562,8 +570,7 @@ static std::vector get_spent_outputs(MDB_txn *txn) outs.reserve(count); while (1) { - const output_data *od = (const output_data*)v.mv_data; - outs.push_back(*od); + outs.push_back({*(const uint64_t*)k.mv_data, *(const uint64_t*)v.mv_data}); dbr = mdb_cursor_get(cur, &k, &v, MDB_NEXT); if (dbr == MDB_NOTFOUND) break; @@ -815,14 +822,6 @@ static void close_db(MDB_env *env, MDB_txn *txn, MDB_cursor *cur, MDB_dbi dbi) mdb_env_close(env); } -static crypto::public_key get_output_key(MDB_cursor *cur, uint64_t amount, uint64_t offset) -{ - MDB_val k = { sizeof(amount), (void*)&amount }, v = { sizeof(offset), (void*)&offset }; - int dbr = mdb_cursor_get(cur, &k, &v, MDB_GET_BOTH); - if (dbr) throw std::runtime_error("Output key not found: " + std::string(mdb_strerror(dbr))); - return *(const crypto::public_key*)(((const char*)v.mv_data) + sizeof(uint64_t) * 2); -} - static void get_num_outputs(MDB_txn *txn, MDB_cursor *cur, MDB_dbi dbi, uint64_t &pre_rct, uint64_t &rct) { uint64_t amount = 0; @@ -884,6 +883,113 @@ static crypto::hash get_genesis_block_hash(const std::string &filename) return genesis_block_hash; } +static std::vector> load_outputs(const std::string &filename) +{ + std::vector> outputs; + uint64_t amount = std::numeric_limits::max(); + FILE *f; + + f = fopen(filename.c_str(), "r"); + if (!f) + { + MERROR("Failed to load outputs from " << filename << ": " << strerror(errno)); + return {}; + } + while (1) + { + char s[256]; + fgets(s, sizeof(s), f); + if (feof(f)) + break; + const size_t len = strlen(s); + if (len > 0 && s[len - 1] == '\n') + s[len - 1] = 0; + if (!s[0]) + continue; + std::pair output; + uint64_t offset, num_offsets; + if (sscanf(s, "@%" PRIu64, &amount) == 1) + { + continue; + } + if (amount == std::numeric_limits::max()) + { + MERROR("Bad format in " << filename); + continue; + } + if (sscanf(s, "%" PRIu64 "*%" PRIu64, &offset, &num_offsets) == 2 && num_offsets < std::numeric_limits::max() - offset) + { + while (num_offsets-- > 0) + outputs.push_back(std::make_pair(amount, offset++)); + } + else if (sscanf(s, "%" PRIu64, &offset) == 1) + { + outputs.push_back(std::make_pair(amount, offset)); + } + else + { + MERROR("Bad format in " << filename); + continue; + } + } + fclose(f); + return outputs; +} + +static bool export_spent_outputs(MDB_cursor *cur, const std::string &filename) +{ + FILE *f = fopen(filename.c_str(), "w"); + if (!f) + { + MERROR("Failed to open " << filename << ": " << strerror(errno)); + return false; + } + + uint64_t pending_amount = std::numeric_limits::max(); + std::vector pending_offsets; + MDB_val k, v; + MDB_cursor_op op = MDB_FIRST; + while (1) + { + int dbr = mdb_cursor_get(cur, &k, &v, op); + if (dbr == MDB_NOTFOUND) + break; + op = MDB_NEXT; + if (dbr) + { + fclose(f); + MERROR("Failed to enumerate spent outputs: " << mdb_strerror(dbr)); + return false; + } + const uint64_t amount = *(const uint64_t*)k.mv_data; + const uint64_t offset = *(const uint64_t*)v.mv_data; + if (!pending_offsets.empty() && (amount != pending_amount || pending_offsets.back()+1 != offset)) + { + if (pending_offsets.size() == 1) + fprintf(f, "%" PRIu64 "\n", pending_offsets.front()); + else + fprintf(f, "%" PRIu64 "*%" PRIu64 "\n", pending_offsets.front(), pending_offsets.size()); + pending_offsets.clear(); + } + if (pending_amount != amount) + { + fprintf(f, "@%" PRIu64 "\n", amount); + pending_amount = amount; + } + pending_offsets.push_back(offset); + } + if (!pending_offsets.empty()) + { + if (pending_offsets.size() == 1) + fprintf(f, "%" PRIu64 "\n", pending_offsets.front()); + else + fprintf(f, "%" PRIu64 "*%" PRIu64 "\n", pending_offsets.front(), pending_offsets.size()); + pending_offsets.clear(); + } + fclose(f); + return true; +} + int main(int argc, char* argv[]) { TRY_ENTRY(); @@ -920,6 +1026,8 @@ int main(int argc, char* argv[]) , "Specify sync option, using format [safe|fast|fastest]:[nrecords_per_sync]." , "fast:1000" }; + const command_line::arg_descriptor arg_extra_spent_list = {"extra-spent-list", "Optional list of known spent outputs",""}; + const command_line::arg_descriptor arg_export = {"export", "Filename to export the backball list to"}; command_line::add_arg(desc_cmd_sett, arg_blackball_db_dir); command_line::add_arg(desc_cmd_sett, arg_log_level); @@ -928,6 +1036,8 @@ int main(int argc, char* argv[]) command_line::add_arg(desc_cmd_sett, arg_check_subsets); command_line::add_arg(desc_cmd_sett, arg_verbose); command_line::add_arg(desc_cmd_sett, arg_db_sync_mode); + command_line::add_arg(desc_cmd_sett, arg_extra_spent_list); + command_line::add_arg(desc_cmd_sett, arg_export); command_line::add_arg(desc_cmd_sett, arg_inputs); command_line::add_arg(desc_cmd_only, command_line::arg_help); @@ -967,6 +1077,9 @@ int main(int argc, char* argv[]) bool opt_rct_only = command_line::get_arg(vm, arg_rct_only); bool opt_check_subsets = command_line::get_arg(vm, arg_check_subsets); bool opt_verbose = command_line::get_arg(vm, arg_verbose); + std::string opt_export = command_line::get_arg(vm, arg_export); + std::string extra_spent_list = command_line::get_arg(vm, arg_extra_spent_list); + std::vector> extra_spent_outputs = extra_spent_list.empty() ? std::vector>() : load_outputs(extra_spent_list); std::string db_type = command_line::get_arg(vm, arg_database); if (!cryptonote::blockchain_valid_db_type(db_type)) @@ -1015,6 +1128,36 @@ int main(int argc, char* argv[]) MDB_cursor *cur0; open_db(inputs[0], &env0, &txn0, &cur0, &dbi0); + if (!extra_spent_outputs.empty()) + { + MINFO("Adding " << extra_spent_outputs.size() << " extra spent outputs"); + MDB_txn *txn; + int dbr = mdb_txn_begin(env, NULL, 0, &txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + MDB_cursor *cur; + dbr = mdb_cursor_open(txn, dbi_spent, &cur); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB cursor: " + std::string(mdb_strerror(dbr))); + + std::vector> blackballs; + for (const std::pair &output: extra_spent_outputs) + { + if (!is_output_spent(cur, output_data(output.first, output.second))) + { + blackballs.push_back(output); + add_spent_output(cur, output_data(output.first, output.second)); + inc_stat(txn, output.first ? "pre-rct-extra" : "rct-ring-extra"); + } + } + if (!blackballs.empty()) + { + ringdb.blackball(blackballs); + blackballs.clear(); + } + mdb_cursor_close(cur); + dbr = mdb_txn_commit(txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to commit txn creating/opening database: " + std::string(mdb_strerror(dbr))); + } + for (size_t n = 0; n < inputs.size(); ++n) { const std::string canonical = boost::filesystem::canonical(inputs[n]).string(); @@ -1033,7 +1176,7 @@ int main(int argc, char* argv[]) CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB cursor: " + std::string(mdb_strerror(dbr))); size_t records = 0; const std::string filename = inputs[n]; - std::vector blackballs; + std::vector> blackballs; uint64_t n_txes; for_all_transactions(filename, start_idx, n_txes, [&](const cryptonote::transaction_prefix &tx)->bool { @@ -1057,13 +1200,13 @@ int main(int argc, char* argv[]) const uint64_t instances = inc_ring_instances(txn, txin.amount, new_ring); if (n == 0 && ring_size == 1) { - const crypto::public_key pkey = get_output_key(cur0, txin.amount, absolute[0]); + const std::pair output = std::make_pair(txin.amount, absolute[0]); if (opt_verbose) { - MINFO("Blackballing output " << pkey << ", due to being used in a 1-ring"); + MINFO("Blackballing output " << output.first << "/" << output.second << ", due to being used in a 1-ring"); std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; } - blackballs.push_back(pkey); + blackballs.push_back(output); add_spent_output(cur, output_data(txin.amount, absolute[0])); inc_stat(txn, txin.amount ? "pre-rct-ring-size-1" : "rct-ring-size-1"); } @@ -1071,13 +1214,13 @@ int main(int argc, char* argv[]) { for (size_t o = 0; o < new_ring.size(); ++o) { - const crypto::public_key pkey = get_output_key(cur0, txin.amount, absolute[o]); + const std::pair output = std::make_pair(txin.amount, absolute[o]); if (opt_verbose) { - MINFO("Blackballing output " << pkey << ", due to being used in " << new_ring.size() << " identical " << new_ring.size() << "-rings"); + MINFO("Blackballing output " << output.first << "/" << output.second << ", due to being used in " << new_ring.size() << " identical " << new_ring.size() << "-rings"); std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; } - blackballs.push_back(pkey); + blackballs.push_back(output); add_spent_output(cur, output_data(txin.amount, absolute[o])); inc_stat(txn, txin.amount ? "pre-rct-duplicate-rings" : "rct-duplicate-rings"); } @@ -1086,13 +1229,13 @@ int main(int argc, char* argv[]) { for (size_t o = 0; o < new_ring.size(); ++o) { - const crypto::public_key pkey = get_output_key(cur0, txin.amount, absolute[o]); + const std::pair output = std::make_pair(txin.amount, absolute[o]); if (opt_verbose) { - MINFO("Blackballing output " << pkey << ", due to being used in " << new_ring.size() << " subsets of " << new_ring.size() << "-rings"); + MINFO("Blackballing output " << output.first << "/" << output.second << ", due to being used in " << new_ring.size() << " subsets of " << new_ring.size() << "-rings"); std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; } - blackballs.push_back(pkey); + blackballs.push_back(output); add_spent_output(cur, output_data(txin.amount, absolute[o])); inc_stat(txn, txin.amount ? "pre-rct-subset-rings" : "rct-subset-rings"); } @@ -1122,13 +1265,13 @@ int main(int argc, char* argv[]) } else if (common.size() == 1) { - const crypto::public_key pkey = get_output_key(cur0, txin.amount, common[0]); + const std::pair output = std::make_pair(txin.amount, common[0]); if (opt_verbose) { - MINFO("Blackballing output " << pkey << ", due to being used in rings with a single common element"); + MINFO("Blackballing output " << output.first << "/" << output.second << ", due to being used in rings with a single common element"); std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; } - blackballs.push_back(pkey); + blackballs.push_back(output); add_spent_output(cur, output_data(txin.amount, common[0])); inc_stat(txn, txin.amount ? "pre-rct-key-image-attack" : "rct-key-image-attack"); } @@ -1211,7 +1354,7 @@ int main(int argc, char* argv[]) dbr = mdb_cursor_open(txn, dbi_spent, &cur); CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB cursor: " + std::string(mdb_strerror(dbr))); - std::vector blackballs; + std::vector> blackballs; std::vector scan_spent = std::move(work_spent); work_spent.clear(); for (const output_data &od: scan_spent) @@ -1234,13 +1377,13 @@ int main(int argc, char* argv[]) } if (known == absolute.size() - 1) { - const crypto::public_key pkey = get_output_key(cur0, od.amount, last_unknown); + const std::pair output = std::make_pair(od.amount, last_unknown); if (opt_verbose) { - MINFO("Blackballing output " << pkey << ", due to being used in a " << + MINFO("Blackballing output " << output.first << "/" << output.second << ", due to being used in a " << absolute.size() << "-ring where all other outputs are known to be spent"); } - blackballs.push_back(pkey); + blackballs.push_back(output); add_spent_output(cur, output_data(od.amount, last_unknown)); work_spent.push_back(output_data(od.amount, last_unknown)); inc_stat(txn, od.amount ? "pre-rct-chain-reaction" : "rct-chain-reaction"); @@ -1279,6 +1422,7 @@ skip_secondary_passes: { "pre-rct-duplicate-rings", pre_rct }, { "rct-duplicate-rings", rct }, { "pre-rct-subset-rings", pre_rct }, { "rct-subset-rings", rct }, { "pre-rct-key-image-attack", pre_rct }, { "rct-key-image-attack", rct }, + { "pre-rct-extra", pre_rct }, { "rct-ring-extra", rct }, { "pre-rct-chain-reaction", pre_rct }, { "rct-chain-reaction", rct }, }; for (const auto &key: stat_keys) @@ -1291,6 +1435,19 @@ skip_secondary_passes: } mdb_txn_abort(txn); + if (!opt_export.empty()) + { + MDB_txn *txn; + int dbr = mdb_txn_begin(env, NULL, 0, &txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + MDB_cursor *cur; + dbr = mdb_cursor_open(txn, dbi_spent, &cur); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB cursor: " + std::string(mdb_strerror(dbr))); + export_spent_outputs(cur, opt_export); + mdb_cursor_close(cur); + mdb_txn_abort(txn); + } + LOG_PRINT_L0("Blockchain blackball data exported OK"); close_db(env0, txn0, cur0, dbi0); close(); diff --git a/src/simplewallet/simplewallet.cpp b/src/simplewallet/simplewallet.cpp index f5aeabded..e0e0c441a 100644 --- a/src/simplewallet/simplewallet.cpp +++ b/src/simplewallet/simplewallet.cpp @@ -1624,23 +1624,23 @@ bool simple_wallet::set_ring(const std::vector &args) bool simple_wallet::blackball(const std::vector &args) { - crypto::public_key output; + uint64_t amount = std::numeric_limits::max(), offset, num_offsets; if (args.size() == 0) { - fail_msg_writer() << tr("usage: blackball | [add]"); + fail_msg_writer() << tr("usage: blackball / | [add]"); return true; } try { - if (epee::string_tools::hex_to_pod(args[0], output)) + if (sscanf(args[0].c_str(), "%" PRIu64 "/%" PRIu64, &amount, &offset) == 2) { - m_wallet->blackball_output(output); + m_wallet->blackball_output(std::make_pair(amount, offset)); } else if (epee::file_io_utils::is_file_exist(args[0])) { - std::vector outputs; - char str[65]; + std::vector> outputs; + char str[256]; std::unique_ptr f(fopen(args[0].c_str(), "r")); if (f) @@ -1654,10 +1654,27 @@ bool simple_wallet::blackball(const std::vector &args) str[len - 1] = 0; if (!str[0]) continue; - outputs.push_back(crypto::public_key()); - if (!epee::string_tools::hex_to_pod(str, outputs.back())) + if (sscanf(str, "@%" PRIu64, &amount) == 1) { - fail_msg_writer() << tr("Invalid public key: ") << str; + continue; + } + if (amount == std::numeric_limits::max()) + { + fail_msg_writer() << tr("First line is not an amount"); + return true; + } + if (sscanf(str, "%" PRIu64 "*%" PRIu64, &offset, &num_offsets) == 2 && num_offsets <= std::numeric_limits::max() - offset) + { + while (num_offsets--) + outputs.push_back(std::make_pair(amount, offset++)); + } + else if (sscanf(str, "%" PRIu64, &offset) == 1) + { + outputs.push_back(std::make_pair(amount, offset)); + } + else + { + fail_msg_writer() << tr("Invalid output: ") << str; return true; } } @@ -1682,7 +1699,7 @@ bool simple_wallet::blackball(const std::vector &args) } else { - fail_msg_writer() << tr("Invalid public key, and file doesn't exist"); + fail_msg_writer() << tr("Invalid output key, and file doesn't exist"); return true; } } @@ -1696,16 +1713,16 @@ bool simple_wallet::blackball(const std::vector &args) bool simple_wallet::unblackball(const std::vector &args) { - crypto::public_key output; + std::pair output; if (args.size() != 1) { - fail_msg_writer() << tr("usage: unblackball "); + fail_msg_writer() << tr("usage: unblackball /"); return true; } - if (!epee::string_tools::hex_to_pod(args[0], output)) + if (sscanf(args[0].c_str(), "%" PRIu64 "/%" PRIu64, &output.first, &output.second) != 2) { - fail_msg_writer() << tr("Invalid public key"); + fail_msg_writer() << tr("Invalid output"); return true; } @@ -1723,25 +1740,25 @@ bool simple_wallet::unblackball(const std::vector &args) bool simple_wallet::blackballed(const std::vector &args) { - crypto::public_key output; + std::pair output; if (args.size() != 1) { - fail_msg_writer() << tr("usage: blackballed "); + fail_msg_writer() << tr("usage: blackballed /"); return true; } - if (!epee::string_tools::hex_to_pod(args[0], output)) + if (sscanf(args[0].c_str(), "%" PRIu64 "/%" PRIu64, &output.first, &output.second) != 2) { - fail_msg_writer() << tr("Invalid public key"); + fail_msg_writer() << tr("Invalid output"); return true; } try { if (m_wallet->is_output_blackballed(output)) - message_writer() << tr("Blackballed: ") << output; + message_writer() << tr("Blackballed: ") << output.first << "/" << output.second; else - message_writer() << tr("not blackballed: ") << output; + message_writer() << tr("not blackballed: ") << output.first << "/" << output.second; } catch (const std::exception &e) { @@ -2554,15 +2571,15 @@ simple_wallet::simple_wallet() tr("Save known rings to the shared rings database")); m_cmd_binder.set_handler("blackball", boost::bind(&simple_wallet::blackball, this, _1), - tr("blackball | [add]"), + tr("blackball / | [add]"), tr("Blackball output(s) so they never get selected as fake outputs in a ring")); m_cmd_binder.set_handler("unblackball", boost::bind(&simple_wallet::unblackball, this, _1), - tr("unblackball "), + tr("unblackball /"), tr("Unblackballs an output so it may get selected as a fake output in a ring")); m_cmd_binder.set_handler("blackballed", boost::bind(&simple_wallet::blackballed, this, _1), - tr("blackballed "), + tr("blackballed /"), tr("Checks whether an output is blackballed")); m_cmd_binder.set_handler("version", boost::bind(&simple_wallet::version, this, _1), diff --git a/src/wallet/ringdb.cpp b/src/wallet/ringdb.cpp index bf5478f5e..e9fc6866d 100644 --- a/src/wallet/ringdb.cpp +++ b/src/wallet/ringdb.cpp @@ -55,6 +55,13 @@ static int compare_hash32(const MDB_val *a, const MDB_val *b) return 0; } +static int compare_uint64(const MDB_val *a, const MDB_val *b) +{ + const uint64_t va = *(const uint64_t*) a->mv_data; + const uint64_t vb = *(const uint64_t*) b->mv_data; + return va < vb ? -1 : va > vb; +} + static std::string compress_ring(const std::vector &ring) { std::string s; @@ -217,9 +224,9 @@ ringdb::ringdb(std::string filename, const std::string &genesis): THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); mdb_set_compare(txn, dbi_rings, compare_hash32); - dbr = mdb_dbi_open(txn, ("blackballs-" + genesis).c_str(), MDB_CREATE | MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, &dbi_blackballs); + dbr = mdb_dbi_open(txn, ("blackballs2-" + genesis).c_str(), MDB_CREATE | MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, &dbi_blackballs); THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); - mdb_set_dupsort(txn, dbi_blackballs, compare_hash32); + mdb_set_dupsort(txn, dbi_blackballs, compare_uint64); dbr = mdb_txn_commit(txn); THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to commit txn creating/opening database: " + std::string(mdb_strerror(dbr))); @@ -374,7 +381,7 @@ bool ringdb::set_ring(const crypto::chacha_key &chacha_key, const crypto::key_im return true; } -bool ringdb::blackball_worker(const std::vector &outputs, int op) +bool ringdb::blackball_worker(const std::vector> &outputs, int op) { MDB_txn *txn; MDB_cursor *cursor; @@ -391,24 +398,25 @@ bool ringdb::blackball_worker(const std::vector &outputs, in epee::misc_utils::auto_scope_leave_caller txn_dtor = epee::misc_utils::create_scope_leave_handler([&](){if (tx_active) mdb_txn_abort(txn);}); tx_active = true; - MDB_val key = zerokeyval; - MDB_val data; - for (const crypto::public_key &output: outputs) + MDB_val key, data; + for (const std::pair &output: outputs) { - data.mv_data = (void*)&output; - data.mv_size = sizeof(output); + key.mv_data = (void*)&output.first; + key.mv_size = sizeof(output.first); + data.mv_data = (void*)&output.second; + data.mv_size = sizeof(output.second); switch (op) { case BLACKBALL_BLACKBALL: - MDEBUG("Blackballing output " << output); - dbr = mdb_put(txn, dbi_blackballs, &key, &data, MDB_NODUPDATA); + MDEBUG("Blackballing output " << output.first << "/" << output.second); + dbr = mdb_put(txn, dbi_blackballs, &key, &data, MDB_APPENDDUP); if (dbr == MDB_KEYEXIST) dbr = 0; break; case BLACKBALL_UNBLACKBALL: - MDEBUG("Unblackballing output " << output); + MDEBUG("Unblackballing output " << output.first << "/" << output.second); dbr = mdb_del(txn, dbi_blackballs, &key, &data); if (dbr == MDB_NOTFOUND) dbr = 0; @@ -443,32 +451,32 @@ bool ringdb::blackball_worker(const std::vector &outputs, in return ret; } -bool ringdb::blackball(const std::vector &outputs) +bool ringdb::blackball(const std::vector> &outputs) { return blackball_worker(outputs, BLACKBALL_BLACKBALL); } -bool ringdb::blackball(const crypto::public_key &output) +bool ringdb::blackball(const std::pair &output) { - std::vector outputs(1, output); + std::vector> outputs(1, output); return blackball_worker(outputs, BLACKBALL_BLACKBALL); } -bool ringdb::unblackball(const crypto::public_key &output) +bool ringdb::unblackball(const std::pair &output) { - std::vector outputs(1, output); + std::vector> outputs(1, output); return blackball_worker(outputs, BLACKBALL_UNBLACKBALL); } -bool ringdb::blackballed(const crypto::public_key &output) +bool ringdb::blackballed(const std::pair &output) { - std::vector outputs(1, output); + std::vector> outputs(1, output); return blackball_worker(outputs, BLACKBALL_QUERY); } bool ringdb::clear_blackballs() { - return blackball_worker(std::vector(), BLACKBALL_CLEAR); + return blackball_worker(std::vector>(), BLACKBALL_CLEAR); } } diff --git a/src/wallet/ringdb.h b/src/wallet/ringdb.h index 4a78f61f8..7b448b0d7 100644 --- a/src/wallet/ringdb.h +++ b/src/wallet/ringdb.h @@ -49,14 +49,14 @@ namespace tools bool get_ring(const crypto::chacha_key &chacha_key, const crypto::key_image &key_image, std::vector &outs); bool set_ring(const crypto::chacha_key &chacha_key, const crypto::key_image &key_image, const std::vector &outs, bool relative); - bool blackball(const crypto::public_key &output); - bool blackball(const std::vector &outputs); - bool unblackball(const crypto::public_key &output); - bool blackballed(const crypto::public_key &output); + bool blackball(const std::pair &output); + bool blackball(const std::vector> &outputs); + bool unblackball(const std::pair &output); + bool blackballed(const std::pair &output); bool clear_blackballs(); private: - bool blackball_worker(const std::vector &outputs, int op); + bool blackball_worker(const std::vector> &outputs, int op); private: std::string filename; diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index ffd7131cd..9866aa215 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -6221,7 +6221,7 @@ bool wallet2::find_and_save_rings(bool force) return true; } -bool wallet2::blackball_output(const crypto::public_key &output) +bool wallet2::blackball_output(const std::pair &output) { if (!m_ringdb) return false; @@ -6229,7 +6229,7 @@ bool wallet2::blackball_output(const crypto::public_key &output) catch (const std::exception &e) { return false; } } -bool wallet2::set_blackballed_outputs(const std::vector &outputs, bool add) +bool wallet2::set_blackballed_outputs(const std::vector> &outputs, bool add) { if (!m_ringdb) return false; @@ -6244,7 +6244,7 @@ bool wallet2::set_blackballed_outputs(const std::vector &out catch (const std::exception &e) { return false; } } -bool wallet2::unblackball_output(const crypto::public_key &output) +bool wallet2::unblackball_output(const std::pair &output) { if (!m_ringdb) return false; @@ -6252,7 +6252,7 @@ bool wallet2::unblackball_output(const crypto::public_key &output) catch (const std::exception &e) { return false; } } -bool wallet2::is_output_blackballed(const crypto::public_key &output) const +bool wallet2::is_output_blackballed(const std::pair &output) const { if (!m_ringdb) return false; @@ -6297,8 +6297,8 @@ bool wallet2::tx_add_fake_output(std::vector> if (seen_indices.count(i)) continue; + if (is_output_blackballed(std::make_pair(amount, i))) // don't add blackballed outputs + continue; seen_indices.emplace(i); LOG_PRINT_L2("picking " << i << " as " << type); diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h index 556679f51..f0ed1c0e8 100644 --- a/src/wallet/wallet2.h +++ b/src/wallet/wallet2.h @@ -1161,10 +1161,10 @@ namespace tools bool set_ring(const crypto::key_image &key_image, const std::vector &outs, bool relative); bool find_and_save_rings(bool force = true); - bool blackball_output(const crypto::public_key &output); - bool set_blackballed_outputs(const std::vector &outputs, bool add = false); - bool unblackball_output(const crypto::public_key &output); - bool is_output_blackballed(const crypto::public_key &output) const; + bool blackball_output(const std::pair &output); + bool set_blackballed_outputs(const std::vector> &outputs, bool add = false); + bool unblackball_output(const std::pair &output); + bool is_output_blackballed(const std::pair &output) const; bool lock_keys_file(); bool unlock_keys_file(); diff --git a/tests/unit_tests/ringdb.cpp b/tests/unit_tests/ringdb.cpp index 9b842569a..416ae0890 100644 --- a/tests/unit_tests/ringdb.cpp +++ b/tests/unit_tests/ringdb.cpp @@ -59,17 +59,17 @@ static crypto::key_image generate_key_image() return key_image; } -static crypto::public_key generate_output() +static std::pair generate_output() { - return rct::rct2pk(rct::scalarmultBase(rct::skGen())); + return std::make_pair(rand(), rand()); } static const crypto::chacha_key KEY_1 = generate_chacha_key(); static const crypto::chacha_key KEY_2 = generate_chacha_key(); static const crypto::key_image KEY_IMAGE_1 = generate_key_image(); -static const crypto::public_key OUTPUT_1 = generate_output(); -static const crypto::public_key OUTPUT_2 = generate_output(); +static const std::pair OUTPUT_1 = generate_output(); +static const std::pair OUTPUT_2 = generate_output(); class RingDB: public tools::ringdb { From a54dbaee087a70df4be066d7d0f237fdfcba2300 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Thu, 30 Aug 2018 09:11:53 +0000 Subject: [PATCH 13/13] blockchain_blackball: add --force-chain-reaction-pass flag --- src/blockchain_utilities/blockchain_blackball.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/blockchain_utilities/blockchain_blackball.cpp b/src/blockchain_utilities/blockchain_blackball.cpp index 9a31b95b3..d9a179f64 100644 --- a/src/blockchain_utilities/blockchain_blackball.cpp +++ b/src/blockchain_utilities/blockchain_blackball.cpp @@ -1028,6 +1028,7 @@ int main(int argc, char* argv[]) }; const command_line::arg_descriptor arg_extra_spent_list = {"extra-spent-list", "Optional list of known spent outputs",""}; const command_line::arg_descriptor arg_export = {"export", "Filename to export the backball list to"}; + const command_line::arg_descriptor arg_force_chain_reaction_pass = {"force-chain-reaction-pass", "Run the chain reaction pass even if no new blockchain data was processed"}; command_line::add_arg(desc_cmd_sett, arg_blackball_db_dir); command_line::add_arg(desc_cmd_sett, arg_log_level); @@ -1038,6 +1039,7 @@ int main(int argc, char* argv[]) command_line::add_arg(desc_cmd_sett, arg_db_sync_mode); command_line::add_arg(desc_cmd_sett, arg_extra_spent_list); command_line::add_arg(desc_cmd_sett, arg_export); + command_line::add_arg(desc_cmd_sett, arg_force_chain_reaction_pass); command_line::add_arg(desc_cmd_sett, arg_inputs); command_line::add_arg(desc_cmd_only, command_line::arg_help); @@ -1077,6 +1079,7 @@ int main(int argc, char* argv[]) bool opt_rct_only = command_line::get_arg(vm, arg_rct_only); bool opt_check_subsets = command_line::get_arg(vm, arg_check_subsets); bool opt_verbose = command_line::get_arg(vm, arg_verbose); + bool opt_force_chain_reaction_pass = command_line::get_arg(vm, arg_force_chain_reaction_pass); std::string opt_export = command_line::get_arg(vm, arg_export); std::string extra_spent_list = command_line::get_arg(vm, arg_extra_spent_list); std::vector> extra_spent_outputs = extra_spent_list.empty() ? std::vector>() : load_outputs(extra_spent_list); @@ -1331,7 +1334,7 @@ int main(int argc, char* argv[]) if (stop_requested) goto skip_secondary_passes; - if (get_num_spent_outputs() > start_blackballed_outputs) + if (opt_force_chain_reaction_pass || get_num_spent_outputs() > start_blackballed_outputs) { MDB_txn *txn; dbr = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn);