From ae0854a43163d8480aeb030e2de4bde89b12b7fa Mon Sep 17 00:00:00 2001 From: warptangent Date: Fri, 4 Mar 2016 09:38:15 -0800 Subject: [PATCH] Schema update: tx_indices --- src/blockchain_db/lmdb/db_lmdb.cpp | 136 ++++++++++++++++++++++------- src/blockchain_db/lmdb/db_lmdb.h | 5 ++ 2 files changed, 109 insertions(+), 32 deletions(-) diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index 623b9478a..d26659ead 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -156,6 +156,7 @@ const char* const LMDB_BLOCK_INFO = "block_info"; const char* const LMDB_BLOCK_HEIGHTS = "block_heights"; const char* const LMDB_TXS = "txs"; +const char* const LMDB_TX_INDICES = "tx_indices"; const char* const LMDB_TX_UNLOCKS = "tx_unlocks"; const char* const LMDB_TX_HEIGHTS = "tx_heights"; const char* const LMDB_TX_OUTPUTS = "tx_outputs"; @@ -624,54 +625,76 @@ void BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, const tr int result = 0; CURSOR(txs) + CURSOR(tx_indices) CURSOR(tx_heights) CURSOR(tx_unlocks) + MDB_val_copy val_tx_index(m_num_txs); MDB_val_copy val_h(tx_hash); MDB_val unused; - if (mdb_cursor_get(m_cur_txs, &val_h, &unused, MDB_SET) == 0) - throw1(TX_EXISTS("Attempting to add transaction that's already in the db")); + result = mdb_cursor_get(m_cur_tx_indices, &val_h, &unused, MDB_SET); + if (result == 0) + throw1(TX_EXISTS(std::string("Attempting to add transaction that's already in the db (tx index ").append(boost::lexical_cast(*(const uint64_t*)unused.mv_data)).append(")").c_str())); + else if (result != MDB_NOTFOUND) + throw1(DB_ERROR(lmdb_error(std::string("Error checking if tx index exists for tx hash ") + epee::string_tools::pod_to_hex(tx_hash) + ": ", result).c_str())); + + result = mdb_cursor_put(m_cur_tx_indices, &val_h, &val_tx_index, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to add tx index to db transaction: ", result).c_str())); MDB_val_copy blob(tx_to_blob(tx)); - result = mdb_cursor_put(m_cur_txs, &val_h, &blob, 0); + result = mdb_cursor_put(m_cur_txs, &val_tx_index, &blob, 0); if (result) throw0(DB_ERROR(lmdb_error("Failed to add tx blob to db transaction: ", result).c_str())); MDB_val_copy height(m_height); - result = mdb_cursor_put(m_cur_tx_heights, &val_h, &height, 0); + result = mdb_cursor_put(m_cur_tx_heights, &val_tx_index, &height, 0); if (result) throw0(DB_ERROR(lmdb_error("Failed to add tx block height to db transaction: ", result).c_str())); MDB_val_copy unlock_time(tx.unlock_time); - result = mdb_cursor_put(m_cur_tx_unlocks, &val_h, &unlock_time, 0); + result = mdb_cursor_put(m_cur_tx_unlocks, &val_tx_index, &unlock_time, 0); if (result) throw0(DB_ERROR(lmdb_error("Failed to add tx unlock time to db transaction: ", result).c_str())); + + m_num_txs++; } +// TODO: compare pros and cons of looking up the tx hash's tx index once and +// passing it in to functions like this void BlockchainLMDB::remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); MDB_val_copy val_h(tx_hash); - MDB_val unused; - if (mdb_get(*m_write_txn, m_txs, &val_h, &unused)) + MDB_val val_tx_index; + + if (mdb_get(*m_write_txn, m_tx_indices, &val_h, &val_tx_index)) throw1(TX_DNE("Attempting to remove transaction that isn't in the db")); - if (mdb_del(*m_write_txn, m_txs, &val_h, NULL)) + if (mdb_del(*m_write_txn, m_txs, &val_tx_index, NULL)) throw1(DB_ERROR("Failed to add removal of tx to db transaction")); - if (mdb_del(*m_write_txn, m_tx_unlocks, &val_h, NULL)) + if (mdb_del(*m_write_txn, m_tx_unlocks, &val_tx_index, NULL)) throw1(DB_ERROR("Failed to add removal of tx unlock time to db transaction")); - if (mdb_del(*m_write_txn, m_tx_heights, &val_h, NULL)) + if (mdb_del(*m_write_txn, m_tx_heights, &val_tx_index, NULL)) throw1(DB_ERROR("Failed to add removal of tx block height to db transaction")); remove_tx_outputs(tx_hash, tx); - auto result = mdb_del(*m_write_txn, m_tx_outputs, &val_h, NULL); + int result = mdb_del(*m_write_txn, m_tx_outputs, &val_tx_index, NULL); if (result == MDB_NOTFOUND) LOG_PRINT_L1("tx has no outputs to remove: " << tx_hash); else if (result) throw1(DB_ERROR(lmdb_error("Failed to add removal of tx outputs to db transaction: ", result).c_str())); + + // Though other things could change, so long as earlier functions (like + // remove_tx_outputs) need to do the lookup of tx hash -> tx index, don't + // delete the tx_indices entry until the end. + if (mdb_del(*m_write_txn, m_tx_indices, &val_h, NULL)) + throw1(DB_ERROR("Failed to add removal of tx index to db transaction")); + + m_num_txs--; } void BlockchainLMDB::add_output(const crypto::hash& tx_hash, @@ -763,7 +786,12 @@ void BlockchainLMDB::add_amount_and_global_output_indices(const crypto::hash& tx v.mv_size = sizeof(uint64_t) * 2 * num_outputs; // LOG_PRINT_L1("tx_outputs[tx_hash] size: " << v.mv_size); - result = mdb_cursor_put(m_cur_tx_outputs, &k, &v, 0); + MDB_val val_tx_index; + result = mdb_cursor_get(m_cur_tx_indices, &k, &val_tx_index, MDB_SET); + if (result) + throw0(DB_ERROR(lmdb_error(std::string("Failed to get tx index for tx hash ") + epee::string_tools::pod_to_hex(tx_hash), result).c_str())); + + result = mdb_cursor_put(m_cur_tx_outputs, &val_tx_index, &v, 0); if (result) throw0(DB_ERROR(std::string("Failed to add to db transaction: ").append(mdb_strerror(result)).c_str())); } @@ -1072,10 +1100,11 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags) lmdb_db_open(txn, LMDB_BLOCK_INFO, MDB_INTEGERKEY | MDB_CREATE, m_block_info, "Failed to open db handle for m_block_info"); lmdb_db_open(txn, LMDB_BLOCK_HEIGHTS, MDB_CREATE, m_block_heights, "Failed to open db handle for m_block_heights"); - lmdb_db_open(txn, LMDB_TXS, MDB_CREATE, m_txs, "Failed to open db handle for m_txs"); - lmdb_db_open(txn, LMDB_TX_UNLOCKS, MDB_CREATE, m_tx_unlocks, "Failed to open db handle for m_tx_unlocks"); - lmdb_db_open(txn, LMDB_TX_HEIGHTS, MDB_CREATE, m_tx_heights, "Failed to open db handle for m_tx_heights"); - lmdb_db_open(txn, LMDB_TX_OUTPUTS, MDB_CREATE, m_tx_outputs, "Failed to open db handle for m_tx_outputs"); + lmdb_db_open(txn, LMDB_TXS, MDB_INTEGERKEY | MDB_CREATE, m_txs, "Failed to open db handle for m_txs"); + lmdb_db_open(txn, LMDB_TX_INDICES, MDB_CREATE, m_tx_indices, "Failed to open db handle for m_tx_indices"); + lmdb_db_open(txn, LMDB_TX_UNLOCKS, MDB_INTEGERKEY | MDB_CREATE, m_tx_unlocks, "Failed to open db handle for m_tx_unlocks"); + lmdb_db_open(txn, LMDB_TX_HEIGHTS, MDB_INTEGERKEY | MDB_CREATE, m_tx_heights, "Failed to open db handle for m_tx_heights"); + lmdb_db_open(txn, LMDB_TX_OUTPUTS, MDB_INTEGERKEY | MDB_CREATE, m_tx_outputs, "Failed to open db handle for m_tx_outputs"); lmdb_db_open(txn, LMDB_OUTPUT_TXS, MDB_INTEGERKEY | MDB_CREATE, m_output_txs, "Failed to open db handle for m_output_txs"); lmdb_db_open(txn, LMDB_OUTPUT_INDICES, MDB_INTEGERKEY | MDB_CREATE, m_output_indices, "Failed to open db handle for m_output_indices"); @@ -1092,9 +1121,7 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags) mdb_set_dupsort(txn, m_output_amounts, compare_uint64); mdb_set_compare(txn, m_spent_keys, compare_hash32); mdb_set_compare(txn, m_block_heights, compare_hash32); - mdb_set_compare(txn, m_txs, compare_hash32); - mdb_set_compare(txn, m_tx_unlocks, compare_hash32); - mdb_set_compare(txn, m_tx_heights, compare_hash32); + mdb_set_compare(txn, m_tx_indices, compare_hash32); mdb_set_compare(txn, m_hf_starting_heights, compare_uint8); mdb_set_compare(txn, m_hf_versions, compare_uint64); mdb_set_compare(txn, m_properties, compare_string); @@ -1106,6 +1133,11 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags) LOG_PRINT_L2("Setting m_height to: " << db_stats.ms_entries); m_height = db_stats.ms_entries; + // get and keep current number of txs + if (mdb_stat(txn, m_tx_indices, &db_stats)) + throw0(DB_ERROR("Failed to query m_tx_indices")); + m_num_txs = db_stats.ms_entries; + // get and keep current number of outputs if (mdb_stat(txn, m_output_indices, &db_stats)) throw0(DB_ERROR("Failed to query m_output_indices")); @@ -1659,25 +1691,37 @@ bool BlockchainLMDB::tx_exists(const crypto::hash& h) const TXN_PREFIX_RDONLY(); const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors; + RCURSOR(tx_indices); RCURSOR(txs); MDB_val_copy key(h); + MDB_val val_tx_index; MDB_val result; + bool tx_found = false; TIME_MEASURE_START(time1); - auto get_result = mdb_cursor_get(m_cur_txs, &key, &result, MDB_SET); + auto get_result = mdb_cursor_get(m_cur_tx_indices, &key, &val_tx_index, MDB_SET); + if (get_result == 0) + tx_found = true; + else if (get_result != MDB_NOTFOUND) + throw0(DB_ERROR(lmdb_error(std::string("DB error attempting to fetch transaction index from hash ") + epee::string_tools::pod_to_hex(h) + ": ", get_result).c_str())); + + get_result = mdb_cursor_get(m_cur_txs, &val_tx_index, &result, MDB_SET); TIME_MEASURE_FINISH(time1); time_tx_exists += time1; TXN_POSTFIX_RDONLY(); - if (get_result == MDB_NOTFOUND) + if (! tx_found) { LOG_PRINT_L1("transaction with hash " << epee::string_tools::pod_to_hex(h) << " not found in db"); return false; } + + if (get_result == MDB_NOTFOUND) + throw0(DB_ERROR(std::string("transaction with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found at index").c_str())); else if (get_result) - throw0(DB_ERROR("DB error attempting to fetch transaction from hash")); + throw0(DB_ERROR(lmdb_error(std::string("DB error attempting to fetch transaction ") + epee::string_tools::pod_to_hex(h) + " at index: ", get_result).c_str())); return true; } @@ -1689,15 +1733,23 @@ uint64_t BlockchainLMDB::get_tx_unlock_time(const crypto::hash& h) const TXN_PREFIX_RDONLY(); const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors; + RCURSOR(tx_indices); RCURSOR(tx_unlocks); MDB_val_copy key(h); + MDB_val val_tx_index; MDB_val result; - auto get_result = mdb_cursor_get(m_cur_tx_unlocks, &key, &result, MDB_SET); + auto get_result = mdb_cursor_get(m_cur_tx_indices, &key, &val_tx_index, MDB_SET); + if (get_result == MDB_NOTFOUND) + throw1(TX_DNE(lmdb_error(std::string("tx index with hash ") + epee::string_tools::pod_to_hex(h) + " not found in db: ", get_result).c_str())); + else if (get_result) + throw0(DB_ERROR(lmdb_error("DB error attempting to fetch tx index from hash: ", get_result).c_str())); + + get_result = mdb_cursor_get(m_cur_tx_unlocks, &val_tx_index, &result, MDB_SET); if (get_result == MDB_NOTFOUND) - throw1(TX_DNE(std::string("tx unlock time with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str())); + throw1(TX_DNE(lmdb_error(std::string("tx unlock time with hash ") + epee::string_tools::pod_to_hex(h) + " not found in db: ", get_result).c_str())); else if (get_result) - throw0(DB_ERROR("DB error attempting to fetch tx unlock time from hash")); + throw0(DB_ERROR(lmdb_error("DB error attempting to fetch tx unlock time from index: ", get_result).c_str())); uint64_t ret = *(const uint64_t*)result.mv_data; TXN_POSTFIX_RDONLY(); @@ -1711,11 +1763,15 @@ transaction BlockchainLMDB::get_tx(const crypto::hash& h) const TXN_PREFIX_RDONLY(); const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors; + RCURSOR(tx_indices); RCURSOR(txs); MDB_val_copy key(h); + MDB_val val_tx_index; MDB_val result; - auto get_result = mdb_cursor_get(m_cur_txs, &key, &result, MDB_SET); + auto get_result = mdb_cursor_get(m_cur_tx_indices, &key, &val_tx_index, MDB_SET); + if (get_result == 0) + get_result = mdb_cursor_get(m_cur_txs, &val_tx_index, &result, MDB_SET); if (get_result == MDB_NOTFOUND) throw1(TX_DNE(std::string("tx with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str())); else if (get_result) @@ -1741,8 +1797,8 @@ uint64_t BlockchainLMDB::get_tx_count() const TXN_PREFIX_RDONLY(); MDB_stat db_stats; - if (mdb_stat(m_txn, m_txs, &db_stats)) - throw0(DB_ERROR("Failed to query m_txs")); + if (mdb_stat(m_txn, m_tx_indices, &db_stats)) + throw0(DB_ERROR("Failed to query m_tx_indices")); TXN_POSTFIX_RDONLY(); @@ -1770,11 +1826,15 @@ uint64_t BlockchainLMDB::get_tx_block_height(const crypto::hash& h) const TXN_PREFIX_RDONLY(); const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors; + RCURSOR(tx_indices); RCURSOR(tx_heights); MDB_val_copy key(h); + MDB_val val_tx_index; MDB_val result; - auto get_result = mdb_cursor_get(m_cur_tx_heights, &key, &result, MDB_SET); + auto get_result = mdb_cursor_get(m_cur_tx_indices, &key, &val_tx_index, MDB_SET); + if (get_result == 0) + get_result = mdb_cursor_get(m_cur_tx_heights, &val_tx_index, &result, MDB_SET); if (get_result == MDB_NOTFOUND) { throw1(TX_DNE(std::string("tx height with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str())); @@ -1906,24 +1966,32 @@ void BlockchainLMDB::get_amount_and_global_output_indices(const crypto::hash& h, // create a new read-only txn here, which is incorrect. TXN_PREFIX_RDONLY(); const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors; + RCURSOR(tx_indices); RCURSOR(tx_outputs); int result = 0; MDB_val_copy k(h); + MDB_val val_tx_index; MDB_val v; - result = mdb_cursor_get(m_cur_tx_outputs, &k, &v, MDB_SET); + result = mdb_cursor_get(m_cur_tx_indices, &k, &val_tx_index, MDB_SET); + if (result == MDB_NOTFOUND) + throw1(OUTPUT_DNE(std::string("Failed to get tx index for tx hash ").append(epee::string_tools::pod_to_hex(h)).c_str())); + else if (result) + throw0(DB_ERROR(lmdb_error(std::string("DB error attempting to fetch transaction index from hash ") + epee::string_tools::pod_to_hex(h) + ": ", result).c_str())); + + result = mdb_cursor_get(m_cur_tx_outputs, &val_tx_index, &v, MDB_SET); if (result == MDB_NOTFOUND) LOG_PRINT_L0("WARNING: Unexpected: tx has no amount and global indices stored in " "tx_outputs, but it should have an empty entry even if it's a tx without " "outputs"); else if (result) - throw0(DB_ERROR("DB error attempting to get data for tx_outputs[tx_hash]")); + throw0(DB_ERROR("DB error attempting to get data for tx_outputs[tx_index]")); uint64_t* paired_indices = (uint64_t*)v.mv_data; int num_elems = v.mv_size / sizeof(uint64_t); if (num_elems % 2 != 0) - throw0(DB_ERROR("tx_outputs[tx_hash] does not have an even numer of indices")); + throw0(DB_ERROR("tx_outputs[tx_index] does not have an even numer of indices")); int num_outputs = num_elems / 2; for (int i = 0; i < num_outputs; ++i) @@ -2380,6 +2448,7 @@ uint64_t BlockchainLMDB::add_block(const block& blk, const size_t& block_size, c } } + uint64_t num_txs = m_num_txs; uint64_t num_outputs = m_num_outputs; try { @@ -2391,6 +2460,7 @@ uint64_t BlockchainLMDB::add_block(const block& blk, const size_t& block_size, c } catch (...) { + m_num_txs = num_txs; m_num_outputs = num_outputs; block_txn_abort(); throw; @@ -2406,6 +2476,7 @@ void BlockchainLMDB::pop_block(block& blk, std::vector& txs) block_txn_start(false); + uint64_t num_txs = m_num_txs; uint64_t num_outputs = m_num_outputs; try { @@ -2414,6 +2485,7 @@ void BlockchainLMDB::pop_block(block& blk, std::vector& txs) } catch (...) { + m_num_txs = num_txs; m_num_outputs = num_outputs; block_txn_abort(); throw; diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h index 1181b5b46..41dccfede 100644 --- a/src/blockchain_db/lmdb/db_lmdb.h +++ b/src/blockchain_db/lmdb/db_lmdb.h @@ -51,6 +51,7 @@ typedef struct mdb_txn_cursors MDB_cursor *m_txc_output_keys; MDB_cursor *m_txc_txs; + MDB_cursor *m_txc_tx_indices; MDB_cursor *m_txc_tx_heights; MDB_cursor *m_txc_tx_unlocks; MDB_cursor *m_txc_tx_outputs; @@ -68,6 +69,7 @@ typedef struct mdb_txn_cursors #define m_cur_output_amounts m_cursors->m_txc_output_amounts #define m_cur_output_keys m_cursors->m_txc_output_keys #define m_cur_txs m_cursors->m_txc_txs +#define m_cur_tx_indices m_cursors->m_txc_tx_indices #define m_cur_tx_heights m_cursors->m_txc_tx_heights #define m_cur_tx_unlocks m_cursors->m_txc_tx_unlocks #define m_cur_tx_outputs m_cursors->m_txc_tx_outputs @@ -85,6 +87,7 @@ typedef struct mdb_rflags bool m_rf_output_amounts; bool m_rf_output_keys; bool m_rf_txs; + bool m_rf_tx_indices; bool m_rf_tx_heights; bool m_rf_tx_unlocks; bool m_rf_tx_outputs; @@ -363,6 +366,7 @@ private: MDB_dbi m_block_info; MDB_dbi m_txs; + MDB_dbi m_tx_indices; MDB_dbi m_tx_unlocks; MDB_dbi m_tx_heights; MDB_dbi m_tx_outputs; @@ -380,6 +384,7 @@ private: MDB_dbi m_properties; uint64_t m_height; + uint64_t m_num_txs; uint64_t m_num_outputs; mutable uint64_t m_cum_size; // used in batch size estimation mutable int m_cum_count;