Merge pull request #4769

5808530f blockchain: remove unused output_scan_worker parameter (moneromooo-monero)
1426209a blockchain: don't run threads if we have just one function to run (moneromooo-monero)
6f7a5fd4 db_lmdb: slight speedup getting array data from the blockchain (moneromooo-monero)
99fbe100 db_lmdb: save some string copies for readonly db keys/values (moneromooo-monero)
bf31447e tx_pool: speed up take_tx for transactions from blocks (moneromooo-monero)
4f005a77 tx_pool: remove unnecessary get_transaction_hash (moneromooo-monero)
593ef598 perf_timer: call reserve on new timer array (moneromooo-monero)
6ecc99ad core: avoid unnecessary tx/blob conversions (moneromooo-monero)
00cc1a16 unit_tests: notify test special case for the usual weirdo (moneromooo-monero)
pull/127/head
Riccardo Spagni 6 years ago
commit 0cc3fc3756
No known key found for this signature in database
GPG Key ID: 55432DF31CCD4FCD

@ -1349,7 +1349,7 @@ public:
*
* @param details the details of the transaction to add
*/
virtual void add_txpool_tx(const transaction &tx, const txpool_tx_meta_t& details) = 0;
virtual void add_txpool_tx(const crypto::hash &txid, const cryptonote::blobdata &blob, const txpool_tx_meta_t& details) = 0;
/**
* @brief update a txpool transaction's metadata

@ -85,6 +85,10 @@ inline void throw1(const T &e)
#define MDB_val_set(var, val) MDB_val var = {sizeof(val), (void *)&val}
#define MDB_val_sized(var, val) MDB_val var = {val.size(), (void *)val.data()}
#define MDB_val_str(var, val) MDB_val var = {strlen(val) + 1, (void *)val}
template<typename T>
struct MDB_val_copy: public MDB_val
{
@ -714,7 +718,8 @@ void BlockchainLMDB::add_block(const block& blk, size_t block_weight, const diff
CURSOR(block_info)
// this call to mdb_cursor_put will change height()
MDB_val_copy<blobdata> blob(block_to_blob(blk));
cryptonote::blobdata block_blob(block_to_blob(blk));
MDB_val_sized(blob, block_blob);
result = mdb_cursor_put(m_cur_blocks, &key, &blob, MDB_APPEND);
if (result)
throw0(DB_ERROR(lmdb_error("Failed to add block blob to db transaction: ", result).c_str()));
@ -828,7 +833,7 @@ uint64_t BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, cons
throw0(DB_ERROR(lmdb_error("Failed to add tx data to db transaction: ", result).c_str()));
cryptonote::blobdata blob = tx_to_blob(tx);
MDB_val_copy<blobdata> blobval(blob);
MDB_val_sized(blobval, blob);
std::stringstream ss;
binary_archive<true> ba(ss);
@ -836,7 +841,7 @@ uint64_t BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, cons
if (!r)
throw0(DB_ERROR("Failed to serialize pruned tx"));
std::string pruned = ss.str();
MDB_val_copy<blobdata> pruned_blob(pruned);
MDB_val_sized(pruned_blob, pruned);
result = mdb_cursor_put(m_cur_txs_pruned, &val_tx_id, &pruned_blob, MDB_APPEND);
if (result)
throw0(DB_ERROR(lmdb_error("Failed to add pruned tx blob to db transaction: ", result).c_str()));
@ -844,7 +849,7 @@ uint64_t BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, cons
if (pruned.size() > blob.size())
throw0(DB_ERROR("pruned tx size is larger than tx size"));
cryptonote::blobdata prunable(blob.data() + pruned.size(), blob.size() - pruned.size());
MDB_val_copy<blobdata> prunable_blob(prunable);
MDB_val_sized(prunable_blob, prunable);
result = mdb_cursor_put(m_cur_txs_prunable, &val_tx_id, &prunable_blob, MDB_APPEND);
if (result)
throw0(DB_ERROR(lmdb_error("Failed to add prunable tx blob to db transaction: ", result).c_str()));
@ -1331,7 +1336,7 @@ void BlockchainLMDB::open(const std::string& filename, const int db_flags)
bool compatible = true;
MDB_val_copy<const char*> k("version");
MDB_val_str(k, "version");
MDB_val v;
auto get_result = mdb_get(txn, m_properties, &k, &v);
if(get_result == MDB_SUCCESS)
@ -1379,7 +1384,7 @@ void BlockchainLMDB::open(const std::string& filename, const int db_flags)
// only write version on an empty DB
if (m_height == 0)
{
MDB_val_copy<const char*> k("version");
MDB_val_str(k, "version");
MDB_val_copy<uint32_t> v(VERSION);
auto put_result = mdb_put(txn, m_properties, &k, &v, 0);
if (put_result != MDB_SUCCESS)
@ -1476,7 +1481,7 @@ void BlockchainLMDB::reset()
throw0(DB_ERROR(lmdb_error("Failed to drop m_properties: ", result).c_str()));
// init with current version
MDB_val_copy<const char*> k("version");
MDB_val_str(k, "version");
MDB_val_copy<uint32_t> v(VERSION);
if (auto result = mdb_put(txn, m_properties, &k, &v, 0))
throw0(DB_ERROR(lmdb_error("Failed to write version to database: ", result).c_str()));
@ -1591,7 +1596,7 @@ void BlockchainLMDB::unlock()
auto_txn.commit(); \
} while(0)
void BlockchainLMDB::add_txpool_tx(const transaction &tx, const txpool_tx_meta_t &meta)
void BlockchainLMDB::add_txpool_tx(const crypto::hash &txid, const cryptonote::blobdata &blob, const txpool_tx_meta_t &meta)
{
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
check_open();
@ -1600,8 +1605,6 @@ void BlockchainLMDB::add_txpool_tx(const transaction &tx, const txpool_tx_meta_t
CURSOR(txpool_meta)
CURSOR(txpool_blob)
const crypto::hash txid = get_transaction_hash(tx);
MDB_val k = {sizeof(txid), (void *)&txid};
MDB_val v = {sizeof(meta), (void *)&meta};
if (auto result = mdb_cursor_put(m_cur_txpool_meta, &k, &v, MDB_NODUPDATA)) {
@ -1610,7 +1613,7 @@ void BlockchainLMDB::add_txpool_tx(const transaction &tx, const txpool_tx_meta_t
else
throw1(DB_ERROR(lmdb_error("Error adding txpool tx metadata to db transaction: ", result).c_str()));
}
MDB_val_copy<cryptonote::blobdata> blob_val(tx_to_blob(tx));
MDB_val_sized(blob_val, blob);
if (auto result = mdb_cursor_put(m_cur_txpool_blob, &k, &blob_val, MDB_NODUPDATA)) {
if (result == MDB_KEYEXIST)
throw1(DB_ERROR("Attempting to add txpool tx blob that's already in the db"));
@ -3172,6 +3175,7 @@ void BlockchainLMDB::get_output_tx_and_index_from_global(const std::vector<uint6
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
check_open();
tx_out_indices.clear();
tx_out_indices.reserve(global_indices.size());
TXN_PREFIX_RDONLY();
RCURSOR(output_txs);
@ -3186,9 +3190,8 @@ void BlockchainLMDB::get_output_tx_and_index_from_global(const std::vector<uint6
else if (get_result)
throw0(DB_ERROR("DB error attempting to fetch output tx hash"));
outtx *ot = (outtx *)v.mv_data;
auto result = tx_out_index(ot->tx_hash, ot->local_index);
tx_out_indices.push_back(result);
const outtx *ot = (const outtx *)v.mv_data;
tx_out_indices.push_back(tx_out_index(ot->tx_hash, ot->local_index));
}
TXN_POSTFIX_RDONLY();
@ -3200,6 +3203,7 @@ void BlockchainLMDB::get_output_key(const uint64_t &amount, const std::vector<ui
TIME_MEASURE_START(db3);
check_open();
outputs.clear();
outputs.reserve(offsets.size());
TXN_PREFIX_RDONLY();
@ -3223,19 +3227,19 @@ void BlockchainLMDB::get_output_key(const uint64_t &amount, const std::vector<ui
else if (get_result)
throw0(DB_ERROR(lmdb_error("Error attempting to retrieve an output pubkey from the db", get_result).c_str()));
output_data_t data;
if (amount == 0)
{
const outkey *okp = (const outkey *)v.mv_data;
data = okp->data;
outputs.push_back(okp->data);
}
else
{
const pre_rct_outkey *okp = (const pre_rct_outkey *)v.mv_data;
outputs.resize(outputs.size() + 1);
output_data_t &data = outputs.back();
memcpy(&data, &okp->data, sizeof(pre_rct_output_data_t));
data.commitment = rct::zeroCommit(amount);
}
outputs.push_back(data);
}
TXN_POSTFIX_RDONLY();
@ -3251,6 +3255,7 @@ void BlockchainLMDB::get_output_tx_and_index(const uint64_t& amount, const std::
indices.clear();
std::vector <uint64_t> tx_indices;
tx_indices.reserve(offsets.size());
TXN_PREFIX_RDONLY();
RCURSOR(output_amounts);
@ -4064,7 +4069,7 @@ void BlockchainLMDB::migrate_0_1()
uint32_t version = 1;
v.mv_data = (void *)&version;
v.mv_size = sizeof(version);
MDB_val_copy<const char *> vk("version");
MDB_val_str(vk, "version");
result = mdb_txn_begin(m_env, NULL, 0, txn);
if (result)
throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str()));
@ -4206,7 +4211,7 @@ void BlockchainLMDB::migrate_1_2()
uint32_t version = 2;
v.mv_data = (void *)&version;
v.mv_size = sizeof(version);
MDB_val_copy<const char *> vk("version");
MDB_val_str(vk, "version");
result = mdb_txn_begin(m_env, NULL, 0, txn);
if (result)
throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str()));
@ -4341,7 +4346,7 @@ void BlockchainLMDB::migrate_2_3()
uint32_t version = 3;
v.mv_data = (void *)&version;
v.mv_size = sizeof(version);
MDB_val_copy<const char *> vk("version");
MDB_val_str(vk, "version");
result = mdb_txn_begin(m_env, NULL, 0, txn);
if (result)
throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str()));

@ -256,7 +256,7 @@ public:
virtual bool has_key_image(const crypto::key_image& img) const;
virtual void add_txpool_tx(const transaction &tx, const txpool_tx_meta_t& meta);
virtual void add_txpool_tx(const crypto::hash &txid, const cryptonote::blobdata &blob, const txpool_tx_meta_t& meta);
virtual void update_txpool_tx(const crypto::hash &txid, const txpool_tx_meta_t& meta);
virtual uint64_t get_txpool_tx_count(bool include_unrelayed_txes = true) const;
virtual bool txpool_has_tx(const crypto::hash &txid) const;

@ -110,6 +110,7 @@ LoggingPerformanceTimer::LoggingPerformanceTimer(const std::string &s, const std
{
MCLOG(level, cat.c_str(), "PERF ----------");
performance_timers = new std::vector<LoggingPerformanceTimer*>();
performance_timers->reserve(16); // how deep before realloc
}
else
{

@ -3786,8 +3786,7 @@ bool Blockchain::cleanup_handle_incoming_blocks(bool force_sync)
}
//------------------------------------------------------------------
//FIXME: unused parameter txs
void Blockchain::output_scan_worker(const uint64_t amount, const std::vector<uint64_t> &offsets, std::vector<output_data_t> &outputs, std::unordered_map<crypto::hash, cryptonote::transaction> &txs) const
void Blockchain::output_scan_worker(const uint64_t amount, const std::vector<uint64_t> &offsets, std::vector<output_data_t> &outputs) const
{
try
{
@ -4164,21 +4163,19 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::vector<block_complete
offsets.second.erase(last, offsets.second.end());
}
// [output] stores all transactions for each tx_out_index::hash found
std::vector<std::unordered_map<crypto::hash, cryptonote::transaction>> transactions(amounts.size());
// gather all the output keys
threads = tpool.get_max_concurrency();
if (!m_db->can_thread_bulk_indices())
threads = 1;
if (threads > 1)
if (threads > 1 && amounts.size() > 1)
{
tools::threadpool::waiter waiter;
for (size_t i = 0; i < amounts.size(); i++)
{
uint64_t amount = amounts[i];
tpool.submit(&waiter, boost::bind(&Blockchain::output_scan_worker, this, amount, std::cref(offset_map[amount]), std::ref(tx_map[amount]), std::ref(transactions[i])), true);
tpool.submit(&waiter, boost::bind(&Blockchain::output_scan_worker, this, amount, std::cref(offset_map[amount]), std::ref(tx_map[amount])), true);
}
waiter.wait(&tpool);
}
@ -4187,7 +4184,7 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::vector<block_complete
for (size_t i = 0; i < amounts.size(); i++)
{
uint64_t amount = amounts[i];
output_scan_worker(amount, offset_map[amount], tx_map[amount], transactions[i]);
output_scan_worker(amount, offset_map[amount], tx_map[amount]);
}
}
@ -4254,9 +4251,9 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::vector<block_complete
return true;
}
void Blockchain::add_txpool_tx(transaction &tx, const txpool_tx_meta_t &meta)
void Blockchain::add_txpool_tx(const crypto::hash &txid, const cryptonote::blobdata &blob, const txpool_tx_meta_t &meta)
{
m_db->add_txpool_tx(tx, meta);
m_db->add_txpool_tx(txid, blob, meta);
}
void Blockchain::update_txpool_tx(const crypto::hash &txid, const txpool_tx_meta_t &meta)

@ -916,11 +916,9 @@ namespace cryptonote
* @param amount the amount
* @param offsets the indices (indexed to the amount) of the outputs
* @param outputs return-by-reference the outputs collected
* @param txs unused, candidate for removal
*/
void output_scan_worker(const uint64_t amount,const std::vector<uint64_t> &offsets,
std::vector<output_data_t> &outputs, std::unordered_map<crypto::hash,
cryptonote::transaction> &txs) const;
std::vector<output_data_t> &outputs) const;
/**
* @brief computes the "short" and "long" hashes for a set of blocks
@ -939,7 +937,7 @@ namespace cryptonote
*/
std::list<std::pair<block_extended_info,std::vector<crypto::hash>>> get_alternative_chains() const;
void add_txpool_tx(transaction &tx, const txpool_tx_meta_t &meta);
void add_txpool_tx(const crypto::hash &txid, const cryptonote::blobdata &blob, const txpool_tx_meta_t &meta);
void update_txpool_tx(const crypto::hash &txid, const txpool_tx_meta_t &meta);
void remove_txpool_tx(const crypto::hash &txid);
uint64_t get_txpool_tx_count(bool include_unrelayed_txes = true) const;

@ -909,7 +909,7 @@ namespace cryptonote
}
const size_t weight = get_transaction_weight(results[i].tx, it->size());
ok &= add_new_tx(results[i].tx, results[i].hash, results[i].prefix_hash, weight, tvc[i], keeped_by_block, relayed, do_not_relay);
ok &= add_new_tx(results[i].tx, results[i].hash, tx_blobs[i], results[i].prefix_hash, weight, tvc[i], keeped_by_block, relayed, do_not_relay);
if(tvc[i].m_verifivation_failed)
{MERROR_VER("Transaction verification failed: " << results[i].hash);}
else if(tvc[i].m_verifivation_impossible)
@ -1127,7 +1127,7 @@ namespace cryptonote
blobdata bl;
t_serializable_object_to_blob(tx, bl);
size_t tx_weight = get_transaction_weight(tx, bl.size());
return add_new_tx(tx, tx_hash, tx_prefix_hash, tx_weight, tvc, keeped_by_block, relayed, do_not_relay);
return add_new_tx(tx, tx_hash, bl, tx_prefix_hash, tx_weight, tvc, keeped_by_block, relayed, do_not_relay);
}
//-----------------------------------------------------------------------------------------------
size_t core::get_blockchain_total_transactions() const
@ -1135,7 +1135,7 @@ namespace cryptonote
return m_blockchain_storage.get_total_transactions();
}
//-----------------------------------------------------------------------------------------------
bool core::add_new_tx(transaction& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prefix_hash, size_t tx_weight, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay)
bool core::add_new_tx(transaction& tx, const crypto::hash& tx_hash, const cryptonote::blobdata &blob, const crypto::hash& tx_prefix_hash, size_t tx_weight, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay)
{
if (keeped_by_block)
get_blockchain_storage().on_new_tx_from_block(tx);
@ -1153,7 +1153,7 @@ namespace cryptonote
}
uint8_t version = m_blockchain_storage.get_current_hard_fork_version();
return m_mempool.add_tx(tx, tx_hash, tx_weight, tvc, keeped_by_block, relayed, do_not_relay, version);
return m_mempool.add_tx(tx, tx_hash, blob, tx_weight, tvc, keeped_by_block, relayed, do_not_relay, version);
}
//-----------------------------------------------------------------------------------------------
bool core::relay_txpool_transactions()

@ -783,13 +783,14 @@ namespace cryptonote
* @copydoc add_new_tx(transaction&, tx_verification_context&, bool)
*
* @param tx_hash the transaction's hash
* @param blob the transaction as a blob
* @param tx_prefix_hash the transaction prefix' hash
* @param tx_weight the weight of the transaction
* @param relayed whether or not the transaction was relayed to us
* @param do_not_relay whether to prevent the transaction from being relayed
*
*/
bool add_new_tx(transaction& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prefix_hash, size_t tx_weight, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay);
bool add_new_tx(transaction& tx, const crypto::hash& tx_hash, const cryptonote::blobdata &blob, const crypto::hash& tx_prefix_hash, size_t tx_weight, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay);
/**
* @brief add a new transaction to the transaction pool

@ -111,7 +111,7 @@ namespace cryptonote
}
//---------------------------------------------------------------------------------
bool tx_memory_pool::add_tx(transaction &tx, /*const crypto::hash& tx_prefix_hash,*/ const crypto::hash &id, size_t tx_weight, tx_verification_context& tvc, bool kept_by_block, bool relayed, bool do_not_relay, uint8_t version)
bool tx_memory_pool::add_tx(transaction &tx, /*const crypto::hash& tx_prefix_hash,*/ const crypto::hash &id, const cryptonote::blobdata &blob, size_t tx_weight, tx_verification_context& tvc, bool kept_by_block, bool relayed, bool do_not_relay, uint8_t version)
{
// this should already be called with that lock, but let's make it explicit for clarity
CRITICAL_REGION_LOCAL(m_transactions_lock);
@ -247,9 +247,11 @@ namespace cryptonote
memset(meta.padding, 0, sizeof(meta.padding));
try
{
if (kept_by_block)
m_parsed_tx_cache.insert(std::make_pair(id, tx));
CRITICAL_REGION_LOCAL1(m_blockchain);
LockedTXN lock(m_blockchain);
m_blockchain.add_txpool_tx(tx, meta);
m_blockchain.add_txpool_tx(id, blob, meta);
if (!insert_key_images(tx, id, kept_by_block))
return false;
m_txs_by_fee_and_receive_time.emplace(std::pair<double, std::time_t>(fee / (double)tx_weight, receive_time), id);
@ -288,12 +290,13 @@ namespace cryptonote
try
{
if (kept_by_block)
m_parsed_tx_cache.insert(std::make_pair(id, tx));
CRITICAL_REGION_LOCAL1(m_blockchain);
LockedTXN lock(m_blockchain);
const crypto::hash txid = get_transaction_hash(tx);
m_blockchain.remove_txpool_tx(txid);
m_blockchain.add_txpool_tx(tx, meta);
if (!insert_key_images(tx, txid, kept_by_block))
m_blockchain.remove_txpool_tx(id);
m_blockchain.add_txpool_tx(id, blob, meta);
if (!insert_key_images(tx, id, kept_by_block))
return false;
m_txs_by_fee_and_receive_time.emplace(std::pair<double, std::time_t>(fee / (double)tx_weight, receive_time), id);
}
@ -324,9 +327,11 @@ namespace cryptonote
{
crypto::hash h = null_hash;
size_t blob_size = 0;
if (!get_transaction_hash(tx, h, blob_size) || blob_size == 0)
cryptonote::blobdata bl;
t_serializable_object_to_blob(tx, bl);
if (bl.size() == 0 || !get_transaction_hash(tx, h))
return false;
return add_tx(tx, h, get_transaction_weight(tx, blob_size), tvc, keeped_by_block, relayed, do_not_relay, version);
return add_tx(tx, h, bl, get_transaction_weight(tx, bl.size()), tvc, keeped_by_block, relayed, do_not_relay, version);
}
//---------------------------------------------------------------------------------
size_t tx_memory_pool::get_txpool_weight() const
@ -467,7 +472,12 @@ namespace cryptonote
return false;
}
cryptonote::blobdata txblob = m_blockchain.get_txpool_tx_blob(id);
if (!parse_and_validate_tx_from_blob(txblob, tx))
auto ci = m_parsed_tx_cache.find(id);
if (ci != m_parsed_tx_cache.end())
{
tx = ci->second;
}
else if (!parse_and_validate_tx_from_blob(txblob, tx))
{
MERROR("Failed to parse tx from txpool");
return false;
@ -910,6 +920,7 @@ namespace cryptonote
{
CRITICAL_REGION_LOCAL(m_transactions_lock);
m_input_cache.clear();
m_parsed_tx_cache.clear();
return true;
}
//---------------------------------------------------------------------------------
@ -917,6 +928,7 @@ namespace cryptonote
{
CRITICAL_REGION_LOCAL(m_transactions_lock);
m_input_cache.clear();
m_parsed_tx_cache.clear();
return true;
}
//---------------------------------------------------------------------------------

@ -107,7 +107,7 @@ namespace cryptonote
* @param id the transaction's hash
* @param tx_weight the transaction's weight
*/
bool add_tx(transaction &tx, const crypto::hash &id, size_t tx_weight, tx_verification_context& tvc, bool kept_by_block, bool relayed, bool do_not_relay, uint8_t version);
bool add_tx(transaction &tx, const crypto::hash &id, const cryptonote::blobdata &blob, size_t tx_weight, tx_verification_context& tvc, bool kept_by_block, bool relayed, bool do_not_relay, uint8_t version);
/**
* @brief add a transaction to the transaction pool
@ -584,6 +584,8 @@ private:
size_t m_txpool_weight;
mutable std::unordered_map<crypto::hash, std::tuple<bool, tx_verification_context, uint64_t, crypto::hash>> m_input_cache;
std::unordered_map<crypto::hash, transaction> m_parsed_tx_cache;
};
}

@ -118,7 +118,7 @@ public:
virtual std::map<uint64_t, std::tuple<uint64_t, uint64_t, uint64_t>> get_output_histogram(const std::vector<uint64_t> &amounts, bool unlocked, uint64_t recent_cutoff, uint64_t min_count) const { return std::map<uint64_t, std::tuple<uint64_t, uint64_t, uint64_t>>(); }
virtual bool get_output_distribution(uint64_t amount, uint64_t from_height, uint64_t to_height, std::vector<uint64_t> &distribution, uint64_t &base) const { return false; }
virtual void add_txpool_tx(const transaction &tx, const txpool_tx_meta_t& details) {}
virtual void add_txpool_tx(const crypto::hash &txid, const cryptonote::blobdata &blob, const txpool_tx_meta_t& details) {}
virtual void update_txpool_tx(const crypto::hash &txid, const txpool_tx_meta_t& details) {}
virtual uint64_t get_txpool_tx_count(bool include_unrelayed_txes = true) const { return 0; }
virtual bool txpool_has_tx(const crypto::hash &txid) const { return false; }

Loading…
Cancel
Save