performance optmizations and fix for duple pub key

https://github.com/moneroexamples/openmonero/issues/84
pull/93/merge
moneroexamples 6 years ago
parent 64c192790c
commit c73f25e6cc

@ -5,8 +5,7 @@ set(PROJECT_NAME
project(${PROJECT_NAME})
set(CMAKE_CXX_FLAGS
"${CMAKE_CXX_FLAGS} -std=c++11")
set(CMAKE_CXX_STANDARD 14)
if(CMAKE_SIZEOF_VOID_P EQUAL "4")
add_definitions(-DMDB_VL32)

@ -44,7 +44,8 @@
}
},
"refresh_block_status_every_seconds" : 10,
"search_thread_life_in_seconds" : 300,
"blocks_search_lookahead" : 200,
"search_thread_life_in_seconds" : 120,
"max_number_of_blocks_to_import" : 132000,
"ssl" :
{

@ -3,7 +3,7 @@ var config = {
mainnetExplorerUrl: "https://xmrchain.com/",
testnetExplorerUrl: "https://testnet.xmrchain.com/",
stagenetExplorerUrl: "http://162.210.173.150:8083/",
nettype: 0, /* 0 - MAINNET, 1 - TESTNET, 2 - STAGENET */
nettype: 2, /* 0 - MAINNET, 1 - TESTNET, 2 - STAGENET */
coinUnitPlaces: 12,
txMinConfirms: 10, // corresponds to CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE in Monero
txCoinbaseMinConfirms: 60, // corresponds to CRYPTONOTE_MINED_MONEY_UNLOCK_WINDOW in Monero

@ -21,6 +21,7 @@
<option value="1000">n = 1000 (since roughly 1 day ago)</option>
<option value="5500">n = 5500 (since roughly 7 days ago)</option>
<option value="22000">n = 22000 (since roughly 30 days ago)</option>
<option value="66000">n = 66000 (since roughly 90 days ago)</option>
<option value="132000">n = 132000 (since roughly 180 days ago)</option>
</select>
</div>

@ -83,6 +83,8 @@ xmreg::CurrentBlockchainStatus::do_not_relay
= do_not_relay;
xmreg::CurrentBlockchainStatus::refresh_block_status_every_seconds
= config_json["refresh_block_status_every_seconds"];
xmreg::CurrentBlockchainStatus::blocks_search_lookahead
= config_json["blocks_search_lookahead"];
xmreg::CurrentBlockchainStatus::max_number_of_blocks_to_import
= config_json["max_number_of_blocks_to_import"];
xmreg::CurrentBlockchainStatus::search_thread_life_in_seconds

@ -28,7 +28,8 @@ network_type CurrentBlockchainStatus::net_type {network_type::MAINNET
bool CurrentBlockchainStatus::do_not_relay{false};
bool CurrentBlockchainStatus::is_running{false};
std::thread CurrentBlockchainStatus::m_thread;
uint64_t CurrentBlockchainStatus::refresh_block_status_every_seconds{20};
uint64_t CurrentBlockchainStatus::refresh_block_status_every_seconds{10};
uint64_t CurrentBlockchainStatus::blocks_search_lookahead {100};
uint64_t CurrentBlockchainStatus::max_number_of_blocks_to_import{8000};
uint64_t CurrentBlockchainStatus::search_thread_life_in_seconds {600}; // 10 minutes
vector<pair<uint64_t, transaction>> CurrentBlockchainStatus::mempool_txs;
@ -214,16 +215,18 @@ CurrentBlockchainStatus::get_blocks_range(
}
bool
CurrentBlockchainStatus::get_block_txs(const block &blk, list <transaction> &blk_txs)
CurrentBlockchainStatus::get_block_txs(
const block &blk,
list<transaction>& blk_txs,
list<crypto::hash>& missed_txs)
{
// get all transactions in the block found
// initialize the first list with transaction for solving
// the block i.e. coinbase tx.
blk_txs.push_back(blk.miner_tx);
list <crypto::hash> missed_txs;
if (!core_storage->get_transactions(blk.tx_hashes, blk_txs, missed_txs)) {
if (!core_storage->get_transactions(blk.tx_hashes, blk_txs, missed_txs))
{
cerr << "Cant get transactions in block: " << get_block_hash(blk) << endl;
return false;
}
@ -231,6 +234,20 @@ CurrentBlockchainStatus::get_block_txs(const block &blk, list <transaction> &blk
return true;
}
bool
CurrentBlockchainStatus::get_txs(
vector<crypto::hash> const& txs_to_get,
list<transaction>& txs,
list<crypto::hash>& missed_txs)
{
if (!core_storage->get_transactions(txs_to_get, txs, missed_txs))
{
cerr << "CurrentBlockchainStatus::get_txs: cant get transactions!\n";
return false;
}
return true;
}
bool
CurrentBlockchainStatus::tx_exist(const crypto::hash& tx_hash)
@ -546,8 +563,9 @@ CurrentBlockchainStatus::search_if_payment_made(
}
list <cryptonote::transaction> blk_txs;
list<crypto::hash> missed_txs;
if (!get_block_txs(blk, blk_txs))
if (!get_block_txs(blk, blk_txs, missed_txs))
{
cerr << "Cant get transactions in block: " << to_string(blk_i) << endl;
return false;
@ -757,7 +775,7 @@ CurrentBlockchainStatus::start_tx_search_thread(XmrAccount acc)
if (search_thread_exist(acc.address))
{
// thread for this address exist, dont make new one
cout << "Thread exists, dont make new one" << endl;
//cout << "Thread exists, dont make new one\n";
return true; // this is still OK, so return true.
}
@ -778,6 +796,8 @@ CurrentBlockchainStatus::start_tx_search_thread(XmrAccount acc)
std::thread t1 {&TxSearch::search, searching_threads[acc.address].get()};
t1.detach();
cout << "Search thread created\n";
return true;
}
@ -821,7 +841,7 @@ CurrentBlockchainStatus::get_searched_blk_no(const string& address,
bool
CurrentBlockchainStatus::get_known_outputs_keys(
string const& address,
vector<pair<public_key, uint64_t>>& known_outputs_keys)
unordered_map<public_key, uint64_t>& known_outputs_keys)
{
std::lock_guard<std::mutex> lck (searching_threads_map_mtx);
@ -1089,4 +1109,58 @@ CurrentBlockchainStatus::construct_output_rct_field(
};
}
bool
CurrentBlockchainStatus::get_txs_in_blocks(
vector<block> const& blocks,
vector<txs_tuple_t>& txs_data)
{
// get height of the first block
uint64_t h1 = get_block_height(blocks[0]);
for(uint64_t blk_i = 0; blk_i < blocks.size(); blk_i++)
{
block const& blk = blocks[blk_i];
uint64_t blk_height = h1 + blk_i;
// first insert miner_tx
txs_data.emplace_back(get_transaction_hash(blk.miner_tx),
transaction{}, blk_height, blk.timestamp, true);
// now insert hashes of regular txs to be fatched later
// so for now, theys txs are null pointers
for (auto& tx_hash: blk.tx_hashes)
txs_data.emplace_back(tx_hash, transaction{}, blk_height, blk.timestamp, false);
}
// prepare vector<tx_hash> for CurrentBlockchainStatus::get_txs to fetch
// the correspoding transactions
std::vector<crypto::hash> txs_to_get;
for (auto const& tx_tuple: txs_data)
txs_to_get.push_back(std::get<0>(tx_tuple));
// fetch all txs from the blocks that we are analyzing in this iteration
std::list<cryptonote::transaction> txs;
std::list<crypto::hash> missed_txs;
if (!CurrentBlockchainStatus::get_txs(txs_to_get, txs, missed_txs) || !missed_txs.empty())
{
cerr << "Cant get transactions in blocks from : " << h1 << '\n';
return false;
}
size_t tx_idx {0};
for (auto& tx: txs)
{
auto& tx_tuple = txs_data[tx_idx++];
//assert(std::get<0>(tx_tuple) == get_transaction_hash(tx));
std::get<1>(tx_tuple) = std::move(tx);
}
return true;
}
}

@ -36,9 +36,13 @@ struct CurrentBlockchainStatus
// vector of mempool transactions that all threads
// can refer to
// <recieved_time, transaction>
// recieved_time, tx
using mempool_txs_t = vector<pair<uint64_t, transaction>>;
// tx_hash , tx, height , timestamp, is_coinbase
using txs_tuple_t = std::tuple<crypto::hash, transaction, uint64_t, uint64_t, bool>;
static string blockchain_path;
static atomic<uint64_t> current_height;
@ -55,6 +59,8 @@ struct CurrentBlockchainStatus
static uint64_t refresh_block_status_every_seconds;
static uint64_t blocks_search_lookahead;
static uint64_t max_number_of_blocks_to_import;
static uint64_t search_thread_life_in_seconds;
@ -109,7 +115,14 @@ struct CurrentBlockchainStatus
get_blocks_range(uint64_t const& h1, uint64_t const& h2);
static bool
get_block_txs(const block &blk, list <transaction> &blk_txs);
get_block_txs(const block &blk,
list<transaction> &blk_txs,
list<crypto::hash>& missed_txs);
static bool
get_txs(vector<crypto::hash> const& txs_to_get,
list<transaction>& txs,
list<crypto::hash>& missed_txs);
static bool
tx_exist(const crypto::hash& tx_hash);
@ -226,7 +239,7 @@ struct CurrentBlockchainStatus
static bool
get_known_outputs_keys(string const& address,
vector<pair<public_key, uint64_t>>& known_outputs_keys);
unordered_map<public_key, uint64_t>& known_outputs_keys);
static void
clean_search_thread_map();
@ -240,6 +253,10 @@ struct CurrentBlockchainStatus
const uint64_t global_amount_index,
const uint64_t out_amount);
static bool
get_txs_in_blocks(vector<block> const& blocks, vector<txs_tuple_t>& txs_data);
};

@ -111,12 +111,10 @@ MysqlInputs::insert(const XmrInput& in_data)
try
{
SimpleResult sr = query.execute(in_data.account_id,
in_data.tx_id,
in_data.output_id,
in_data.key_image,
in_data.amount,
in_data.timestamp);
query.insert(in_data);
SimpleResult sr = query.execute();
if (sr.rows() == 1)
return sr.insert_id();
@ -137,6 +135,34 @@ MysqlInputs::insert(const XmrInput& in_data)
return 0;
}
uint64_t
MysqlInputs::insert(vector<XmrInput> const& in_data)
{
Query query = conn->query(XmrInput::INSERT_STMT);
query.parse();
try
{
query.insert(in_data.begin(), in_data.end());
SimpleResult sr = query.execute();
return sr.rows();
}
catch (mysqlpp::Exception& e)
{
MYSQL_EXCEPTION_MSG(e);
//throw e;
//return 0;
}
catch (std::exception& e)
{
MYSQL_EXCEPTION_MSG(e);
//throw e;
}
return 0;
}
MysqlOutpus::MysqlOutpus(shared_ptr<MySqlConnector> _conn): conn {_conn}
@ -276,18 +302,9 @@ MysqlOutpus::insert(const XmrOutput& out_data)
try
{
SimpleResult sr = query.execute(out_data.account_id,
out_data.tx_id,
out_data.out_pub_key,
out_data.tx_pub_key,
out_data.rct_outpk,
out_data.rct_mask,
out_data.rct_amount,
out_data.amount,
out_data.global_index,
out_data.out_index,
out_data.mixin,
out_data.timestamp);
query.insert(out_data);
SimpleResult sr = query.execute();
if (sr.rows() == 1)
return sr.insert_id();
@ -308,9 +325,39 @@ MysqlOutpus::insert(const XmrOutput& out_data)
return 0;
}
uint64_t
MysqlOutpus::insert(vector<XmrOutput> const& out_data)
{
Query query = conn->query(XmrOutput::INSERT_STMT);
query.parse();
// cout << query << endl;
try
{
query.insert(out_data.begin(), out_data.end());
SimpleResult sr = query.execute();
MysqlTransactions::MysqlTransactions(shared_ptr<MySqlConnector> _conn): conn {_conn}
return sr.rows();
}
catch (mysqlpp::Exception& e)
{
MYSQL_EXCEPTION_MSG(e);
//throw e;
//return 0;
}
catch (std::exception& e)
{
MYSQL_EXCEPTION_MSG(e);
//throw e;
}
return 0;
}
MysqlTransactions::MysqlTransactions(shared_ptr<MySqlConnector> _conn): conn {_conn}
{}
bool
@ -772,11 +819,23 @@ MySqlAccounts::insert_output(const XmrOutput& tx_out)
}
uint64_t
MySqlAccounts::insert_output(vector<XmrOutput> const& out_data)
{
return mysql_out->insert(out_data);
}
uint64_t
MySqlAccounts::insert_input(const XmrInput& tx_in)
{
return mysql_in->insert(tx_in);
}
uint64_t
MySqlAccounts::insert_input(vector<XmrInput> const& in_data)
{
return mysql_in->insert(in_data);
}
bool
MySqlAccounts::select_txs(const string& xmr_address, vector<XmrTransaction>& txs)
{

@ -54,6 +54,9 @@ public:
uint64_t
insert(const XmrInput& in_data);
uint64_t
insert(vector<XmrInput> const& in_data);
};
@ -79,11 +82,12 @@ public:
bool
exist(const string& output_public_key_str, XmrOutput& out);
uint64_t
insert(const XmrOutput& out_data);
uint64_t
insert(vector<XmrOutput> const& out_data);
};
@ -186,9 +190,15 @@ public:
uint64_t
insert_output(const XmrOutput& tx_out);
uint64_t
insert_output(vector<XmrOutput> const& out_data);
uint64_t
insert_input(const XmrInput& tx_in);
uint64_t
insert_input(vector<XmrInput> const& in_data);
bool
select_txs(const string& xmr_address, vector<XmrTransaction>& txs);

@ -11,17 +11,19 @@ namespace xmreg
OutputInputIdentification::OutputInputIdentification(
const address_parse_info* _a,
const secret_key* _v,
const transaction* _tx)
const transaction* _tx,
crypto::hash const& _tx_hash,
bool is_coinbase)
: total_received {0}, mixin_no {0}
{
address_info = _a;
viewkey = _v;
tx = _tx;
tx_hash = get_transaction_hash(*tx);
tx_pub_key = xmreg::get_tx_pub_key_from_received_outs(*tx);
tx_is_coinbase = is_coinbase(*tx);
tx_is_coinbase = is_coinbase;
tx_hash = _tx_hash;
is_rct = (tx->version == 2);
@ -55,16 +57,13 @@ void
OutputInputIdentification::identify_outputs()
{
// <public_key , amount , out idx>
vector<tuple<txout_to_key, uint64_t, uint64_t>> outputs;
outputs = get_ouputs_tuple(*tx);
vector<tuple<txout_to_key, uint64_t, uint64_t>> outputs = get_ouputs_tuple(*tx);
for (auto& out: outputs)
{
txout_to_key txout_k = std::get<0>(out);
uint64_t amount = std::get<1>(out);
uint64_t output_idx_in_tx = std::get<2>(out);
txout_to_key const& txout_k = std::get<0>(out);
uint64_t amount = std::get<1>(out);
uint64_t output_idx_in_tx = std::get<2>(out);
// get the tx output public key
// that normally would be generated for us,
@ -79,11 +78,6 @@ OutputInputIdentification::identify_outputs()
// check if generated public key matches the current output's key
bool mine_output = (txout_k.key == generated_tx_pubkey);
//cout << "Chekcing output: " << pod_to_hex(txout_k.key) << " "
// << "mine_output: " << mine_output << endl;
// placeholder variable for ringct outputs info
// that we need to save in database
string rtc_outpk;
@ -130,22 +124,13 @@ OutputInputIdentification::identify_outputs()
}
amount = rct_amount_val;
}
} // if (!tx_is_coinbase)
} // if (mine_output && tx.version == 2)
if (mine_output)
{
string out_key_str = pod_to_hex(txout_k.key);
// found an output associated with the given address and viewkey
string msg = fmt::format("tx_hash: {:s}, output_pub_key: {:s}\n",
get_tx_hash_str(),
out_key_str);
cout << msg << endl;
total_received += amount;
identified_outputs.emplace_back(
@ -163,14 +148,14 @@ OutputInputIdentification::identify_outputs()
void
OutputInputIdentification::identify_inputs(
const vector<pair<public_key, uint64_t>>& known_outputs_keys)
unordered_map<public_key, uint64_t> const& known_outputs_keys)
{
vector<txin_to_key> input_key_imgs = xmreg::get_key_images(*tx);
size_t search_misses = {0};
size_t search_misses {0};
// make timescale maps for mixins in input
for (const txin_to_key& in_key: input_key_imgs)
for (txin_to_key const& in_key: input_key_imgs)
{
// get absolute offsets of mixins
std::vector<uint64_t> absolute_offsets
@ -189,7 +174,6 @@ OutputInputIdentification::identify_inputs(
continue;
}
// mixin counter
size_t count = 0;
@ -202,39 +186,27 @@ OutputInputIdentification::identify_inputs(
// get basic information about mixn's output
cryptonote::output_data_t output_data = mixin_outputs[count];
//string output_public_key_str = pod_to_hex(output_data.pubkey);
//cout << " - output_public_key_str: " << output_public_key_str << endl;
// before going to the mysql, check our known outputs cash
// if the key exists. Its much faster than going to mysql
// for this.
auto it = known_outputs_keys.find(output_data.pubkey);
auto it = std::find_if(
known_outputs_keys.begin(),
known_outputs_keys.end(),
[&output_data](pair<public_key, uint64_t> const& known_output)
{
return output_data.pubkey == known_output.first;
});
if (it == known_outputs_keys.end())
if (it != known_outputs_keys.end())
{
// this mixins's output is unknown.
++count;
continue;
}
// this seems to be our mixin.
// save it into identified_inputs vector
// this seems to be our mixin.
// save it into identified_inputs vector
identified_inputs.push_back(input_info {
pod_to_hex(in_key.k_image),
it->second, // amount
output_data.pubkey});
identified_inputs.push_back(input_info {
pod_to_hex(in_key.k_image),
(*it).second, // amount
output_data.pubkey});
found_a_match = true;
found_a_match = true;
}
++count;
@ -265,10 +237,7 @@ string const&
OutputInputIdentification::get_tx_hash_str()
{
if (tx_hash_str.empty())
{
tx_hash_str = pod_to_hex(tx_hash);
}
return tx_hash_str;
}

@ -8,6 +8,8 @@
#include "CurrentBlockchainStatus.h"
#include "tools.h"
#include <map>
namespace xmreg
{
@ -99,13 +101,16 @@ public:
OutputInputIdentification(const address_parse_info* _a,
const secret_key* _v,
const transaction* _tx);
const transaction* _tx,
crypto::hash const& _tx_hash,
bool is_coinbase);
/**
* FIRST step. search for the incoming xmr using address, viewkey and
* outputs public keys.
*/
void identify_outputs();
void
identify_outputs();
/**
@ -125,8 +130,8 @@ public:
* known_outputs_keys is pair of <output public key, output amount>
*
*/
void identify_inputs(
const vector<pair<public_key, uint64_t>>& known_outputs_keys);
void
identify_inputs(unordered_map<public_key, uint64_t> const& known_outputs_keys);
string const&
get_tx_hash_str();

@ -51,13 +51,11 @@ void
TxSearch::search()
{
uint64_t current_timestamp = chrono::duration_cast<chrono::seconds>(
chrono::system_clock::now().time_since_epoch()).count();
uint64_t current_timestamp = get_current_timestamp();
last_ping_timestamp = current_timestamp;
uint64_t loop_idx {0};
uint64_t blocks_lookahead = CurrentBlockchainStatus::blocks_search_lookahead;
// we put everything in massive catch, as there are plenty ways in which
// an exceptions can be thrown here. Mostly from mysql.
@ -68,52 +66,38 @@ TxSearch::search()
{
while(continue_search)
{
++loop_idx;
uint64_t loop_timestamp {current_timestamp};
if (loop_idx % 10 == 0)
{
// get loop time every fith iteration. no need to call it
// all the time.
loop_timestamp = chrono::duration_cast<chrono::seconds>(
chrono::system_clock::now().time_since_epoch()).count();
uint64_t last_block_height = CurrentBlockchainStatus::current_height;
if (loop_timestamp - last_ping_timestamp > thread_search_life)
{
uint64_t h1 = searched_blk_no;
uint64_t h2 = std::min(h1 + blocks_lookahead - 1, last_block_height);
// also check if we caught up with current blockchain height
if (searched_blk_no >= CurrentBlockchainStatus::current_height)
{
stop();
continue;
}
}
}
vector<block> blocks = CurrentBlockchainStatus::get_blocks_range(h1, h2);
if (searched_blk_no > CurrentBlockchainStatus::current_height)
if (blocks.empty())
{
fmt::print("searched_blk_no {:d} and current_height {:d}\n",
searched_blk_no, CurrentBlockchainStatus::current_height);
cout << "Cant get blocks from " << h1 << " to " << h2 << '\n';
std::this_thread::sleep_for(
std::chrono::seconds(
CurrentBlockchainStatus::refresh_block_status_every_seconds)
);
continue;
}
// get block cointaining this tx
block blk;
loop_timestamp = get_current_timestamp();
if (!CurrentBlockchainStatus::get_block(searched_blk_no, blk))
{
cerr << "Cant get block of height: " + to_string(searched_blk_no) << endl;
// if thread has lived longer than thread_search_li
// without last_ping_timestamp being updated,
// stop the thread
if (loop_timestamp - last_ping_timestamp > thread_search_life)
{
cout << "Search thread stopped for address "<< acc->address <<'\n';
stop();
}
// update current_height of blockchain, as maybe top block(s)
// were dropped due to reorganization.
CurrentBlockchainStatus::update_current_blockchain_height();
//CurrentBlockchainStatus::update_current_blockchain_height();
// if any txs that we already indexed got orphaned as a consequence of this
// MySqlAccounts::select_txs_for_account_spendability_check should
@ -122,22 +106,15 @@ TxSearch::search()
continue;
}
// get all txs in the block
list <cryptonote::transaction> blk_txs;
if (!CurrentBlockchainStatus::get_block_txs(blk, blk_txs))
{
throw TxSearchException("Cant get transactions in block: " + to_string(searched_blk_no));
}
cout << "Analyzing " << blocks.size() << " blocks from " << h1 << " to " << h2
<< " out of " << last_block_height << " blocks.\n";
vector<CurrentBlockchainStatus::txs_tuple_t> txs_data;
if (searched_blk_no % 100 == 0)
if (!CurrentBlockchainStatus::get_txs_in_blocks(blocks, txs_data))
{
// print status every 100th block
cout << " - searching block " << searched_blk_no
<< " of hash: "
<< searched_blk_no << get_block_hash(blk) << '\n';
cout << "Cant get tx in blocks from " << h1 << " to " << h2 << '\n';
return;
}
// we will only create mysql DateTime object once, anything is found
@ -159,18 +136,28 @@ TxSearch::search()
// mixin purposes. Thus, we sent to the front end the list of key images
// that we think are yours, and the frontend, because it has spend key,
// can filter out false positives.
for (transaction& tx: blk_txs)
//for (transaction& tx: txs)
for (auto const& tx_tuple: txs_data)
{
crypto::hash const& tx_hash = std::get<0>(tx_tuple);
transaction const& tx = std::get<1>(tx_tuple);
uint64_t blk_height = std::get<2>(tx_tuple);
uint64_t blk_timestamp = std::get<3>(tx_tuple);
bool is_coinbase = std::get<4>(tx_tuple);
//cout << "\n\n\n" << blk_height << '\n';
// Class that is responsible for identification of our outputs
// and inputs in a given tx.
OutputInputIdentification oi_identification {&address, &viewkey, &tx};
OutputInputIdentification oi_identification {&address, &viewkey, &tx,
tx_hash, is_coinbase};
// flag indicating whether the txs in the given block are spendable.
// this is true when block number is more than 10 blocks from current
// blockchain height.
bool is_spendable = CurrentBlockchainStatus::is_tx_unlocked(
tx.unlock_time, searched_blk_no);
tx.unlock_time, blk_height);
// this is id of txs in lmdb blockchain table.
@ -192,45 +179,12 @@ TxSearch::search()
// save them into mysql.
if (!oi_identification.identified_outputs.empty())
{
// before adding this tx and its outputs to mysql
// check if it already exists. So that we dont
// do it twice.
//
// 2018:02:01 Dont know why I added this before?
// this results in incorrect balances in some cases
// as it removes already existing tx data? Dont know
// why it was added.
// Maybe I added it to enable rescanning blockchain? Thus
// it would be deleting already exisitng tx when rescanning
// blockchain
//
// XmrTransaction tx_data_existing;
//
// if (xmr_accounts->tx_exists(acc->id,
// oi_identification.tx_hash_str,
// tx_data_existing))
// {
// cout << "\nTransaction " << oi_identification.tx_hash_str
// << " already present in mysql"
// << endl;
//
// // if tx is already present for that user,
// // we remove it, as we get it data from scrach
//
// if (xmr_accounts->delete_tx(tx_data_existing.id) == 0)
// {
// string msg = fmt::format("xmr_accounts->delete_tx(%d)",
// tx_data_existing.id);
// cerr << msg << endl;
// throw TxSearchException(msg);
// }
// }
if (!blk_timestamp_mysql_format)
{
blk_timestamp_mysql_format
= unique_ptr<DateTime>(
new DateTime(static_cast<time_t>(blk.timestamp)));
new DateTime(static_cast<time_t>(blk_timestamp)));
}
if (!mysql_transaction)
@ -241,16 +195,27 @@ TxSearch::search()
new mysqlpp::Transaction(
xmr_accounts->get_connection()
->get_connection()));
// when we rescan blockchain some txs can already
// be present in the mysql. So remove them, and their
// associated data in that case to repopulate fresh tx data
if (!delete_existing_tx_if_exists(oi_identification.get_tx_hash_str()))
throw TxSearchException("Cant delete tx " + oi_identification.tx_hash_str);
}
if (!CurrentBlockchainStatus::tx_exist(oi_identification.tx_hash, blockchain_tx_id))
if (!CurrentBlockchainStatus::tx_exist(tx_hash, blockchain_tx_id))
{
cerr << "Tx " << oi_identification.get_tx_hash_str()
<< "not found in blockchain !" << '\n';
continue;
<< " " << pod_to_hex(tx_hash)
<< " not found in blockchain !" << '\n';
throw TxSearchException("Cant get tx from blockchain: " + pod_to_hex(tx_hash));
}
cout << " - found some outputs in block " << blk_height
<< ", tx: " << oi_identification.get_tx_hash_str() << '\n';
XmrTransaction tx_data;
@ -269,7 +234,7 @@ TxSearch::search()
// for coinbase tx it is 60 blocks
tx_data.unlock_time = tx.unlock_time;
tx_data.height = searched_blk_no;
tx_data.height = blk_height;
tx_data.coinbase = oi_identification.tx_is_coinbase;
tx_data.is_rct = oi_identification.is_rct;
tx_data.rct_type = oi_identification.rct_type;
@ -284,7 +249,7 @@ TxSearch::search()
// get amount specific (i.e., global) indices of outputs
if (!CurrentBlockchainStatus::get_amount_specific_indices(
oi_identification.tx_hash, amount_specific_indices))
tx_hash, amount_specific_indices))
{
cerr << "cant get_amount_specific_indices!" << endl;
throw TxSearchException("cant get_amount_specific_indices!");
@ -293,10 +258,12 @@ TxSearch::search()
if (tx_mysql_id == 0)
{
//cerr << "tx_mysql_id is zero!" << endl;
//throw TxSearchException("tx_mysql_id is zero!");
throw TxSearchException("tx_mysql_id is zero!");
//todo what should be done when insert_tx fails?
}
vector<XmrOutput> outputs_found;
// now add the found outputs into Outputs tables
for (auto& out_info: oi_identification.identified_outputs)
{
@ -315,22 +282,27 @@ TxSearch::search()
out_data.mixin = tx_data.mixin;
out_data.timestamp = tx_data.timestamp;
// insert output into mysql's outputs table
uint64_t out_mysql_id = xmr_accounts->insert_output(out_data);
if (out_mysql_id == 0)
{
//cerr << "out_mysql_id is zero!" << endl;
//todo what should be done when insert_tx fails?
}
outputs_found.push_back(out_data);
{
// add the outputs found into known_outputs_keys map
std::lock_guard<std::mutex> lck (getting_known_outputs_keys);
known_outputs_keys.push_back(make_pair(out_info.pub_key, out_info.amount));
known_outputs_keys.insert({out_info.pub_key, out_info.amount});
}
} // for (auto& out_info: oi_identification.identified_outputs)
// insert all outputs found into mysql's outputs table
uint64_t no_rows_inserted = xmr_accounts->insert_output(outputs_found);
if (no_rows_inserted == 0)
{
//cerr << "out_mysql_id is zero!" << endl;
throw TxSearchException("no_rows_inserted is zero!");
//todo what should be done when insert_tx fails?
}
} // if (!found_mine_outputs.empty())
@ -347,12 +319,11 @@ TxSearch::search()
// so now, go over those inputs, and check
// get detail info for each found mixin output from database
if (!blk_timestamp_mysql_format)
{
blk_timestamp_mysql_format
= unique_ptr<DateTime>(
new DateTime(static_cast<time_t>(blk.timestamp)));
new DateTime(static_cast<time_t>(blk_timestamp)));
}
if (!mysql_transaction)
@ -364,6 +335,14 @@ TxSearch::search()
xmr_accounts->get_connection()
->get_connection()));
// when we rescan blockchain some txs can already
// be present in the mysql. So remove them, and their
// associated data in that case to repopulate fresh tx data
// this will only execture if no outputs were found
// above. So there is no risk of deleting same tx twice
if (!delete_existing_tx_if_exists(oi_identification.get_tx_hash_str()))
throw TxSearchException("Cant delete tx " + oi_identification.tx_hash_str);
}
if (blockchain_tx_id == 0)
@ -372,10 +351,13 @@ TxSearch::search()
{
cerr << "Tx " << oi_identification.get_tx_hash_str()
<< "not found in blockchain !" << '\n';
continue;
throw TxSearchException("Cant get tx from blockchain: " + pod_to_hex(tx_hash));
}
}
cout << " - found some possible inputs in block " << blk_height
<< ", tx: " << oi_identification.get_tx_hash_str() << '\n';
vector<XmrInput> inputs_found;
for (auto& in_info: oi_identification.identified_inputs)
@ -384,8 +366,7 @@ TxSearch::search()
if (xmr_accounts->output_exists(pod_to_hex(in_info.out_pub_key), out))
{
cout << "input uses some mixins which are our outputs"
<< out << '\n';
//cout << "input uses some mixins which are our outputs" << out << '\n';
// seems that this key image is ours.
// so get it information from database into XmrInput
@ -438,7 +419,7 @@ TxSearch::search()
tx_data.total_received = 0; // because this is spending, total_recieved is 0
tx_data.total_sent = total_sent;
tx_data.unlock_time = tx.unlock_time;
tx_data.height = searched_blk_no;
tx_data.height = blk_height;
tx_data.coinbase = oi_identification.tx_is_coinbase;
tx_data.is_rct = oi_identification.is_rct;
tx_data.rct_type = oi_identification.rct_type;
@ -453,7 +434,7 @@ TxSearch::search()
if (tx_mysql_id == 0)
{
//cerr << "tx_mysql_id is zero!" << endl;
//throw TxSearchException("tx_mysql_id is zero!");
throw TxSearchException("tx_mysql_id is zero!");
// it did not insert this tx, because maybe it already
// exisits in the MySQL. So maybe can now
// check if we have it and get tx_mysql_id this way.
@ -462,13 +443,16 @@ TxSearch::search()
} // if (tx_mysql_id == 0)
// save all input found into database
// save all input found into database at once
// but first update tx_mysql_id for these inputs
for (XmrInput& in_data: inputs_found)
{
in_data.tx_id = tx_mysql_id; // set tx id now. before we made it 0
uint64_t in_mysql_id = xmr_accounts->insert_input(in_data);
uint64_t no_rows_inserted = xmr_accounts->insert_input(inputs_found);
if (no_rows_inserted == 0)
{
throw TxSearchException("no_rows_inserted is zero!");
//todo what shoud we do when insert_input fails?
}
@ -483,38 +467,26 @@ TxSearch::search()
if (mysql_transaction)
mysql_transaction->commit();
} // for (const transaction& tx: blk_txs)
} // for (auto const& tx_pair: txs_map)
// update scanned_block_height every given interval
// or when we reached top of the blockchain
if ((loop_timestamp - current_timestamp > UPDATE_SCANNED_HEIGHT_INTERVAL)
|| searched_blk_no == CurrentBlockchainStatus::current_height)
{
// update scanned_block_height every given interval
// or when we reached top of the blockchain
XmrAccount updated_acc = *acc;
XmrAccount updated_acc = *acc;
updated_acc.scanned_block_height = h2;
updated_acc.scanned_block_timestamp = DateTime(static_cast<time_t>(blocks.back().timestamp));
if (!blk_timestamp_mysql_format)
{
blk_timestamp_mysql_format
= unique_ptr<DateTime>(
new DateTime(static_cast<time_t>(blk.timestamp)));
}
updated_acc.scanned_block_height = searched_blk_no;
updated_acc.scanned_block_timestamp = *blk_timestamp_mysql_format;
if (xmr_accounts->update(*acc, updated_acc))
{
// iff success, set acc to updated_acc;
cout << "scanned_block_height updated" << endl;
*acc = updated_acc;
}
current_timestamp = loop_timestamp;
if (xmr_accounts->update(*acc, updated_acc))
{
// iff success, set acc to updated_acc;
//cout << "scanned_block_height updated\n";
*acc = updated_acc;
}
++searched_blk_no;
//current_timestamp = loop_timestamp;
searched_blk_no = h2 + 1;
} // while(continue_search)
@ -535,6 +507,9 @@ TxSearch::search()
{
cerr << "Unknown exception in TxSearch for " << acc->address << '\n';
}
// it will stop anyway, but just call it so we get info message pritened out
stop();
}
void
@ -561,7 +536,14 @@ TxSearch::get_searched_blk_no() const
return searched_blk_no;
}
void
inline uint64_t
TxSearch::get_current_timestamp() const
{
return chrono::duration_cast<chrono::seconds>(
chrono::system_clock::now().time_since_epoch()).count();
}
void
TxSearch::ping()
{
cout << "new last_ping_timestamp: " << last_ping_timestamp << endl;
@ -589,7 +571,7 @@ TxSearch::populate_known_outputs()
hex_to_pod(out.out_pub_key, out_pub_key);
known_outputs_keys.push_back(make_pair(out_pub_key, out.amount));
known_outputs_keys[out_pub_key] = out.amount;
}
}
}
@ -627,9 +609,13 @@ TxSearch::find_txs_in_mempool(
const transaction& tx = mtx.second;
const crypto::hash tx_hash = get_transaction_hash(tx);
const bool coinbase = is_coinbase(tx);
// Class that is resposnible for idenficitaction of our outputs
// and inputs in a given tx.
OutputInputIdentification oi_identification {&address, &viewkey, &tx};
OutputInputIdentification oi_identification {&address, &viewkey, &tx,
tx_hash, coinbase};
// FIRSt step. to search for the incoming xmr, we use address, viewkey and
// outputs public key.
@ -781,6 +767,29 @@ TxSearch::set_search_thread_life(uint64_t life_seconds)
thread_search_life = life_seconds;
}
bool
TxSearch::delete_existing_tx_if_exists(string const& tx_hash)
{
XmrTransaction tx_data_existing;
if (xmr_accounts->tx_exists(acc->id, tx_hash, tx_data_existing))
{
cout << "\nTransaction " << tx_hash << " already present in mysql, so remove it\n";
// if tx is already present for that user,
// we remove it, as we get it data from scrach
if (xmr_accounts->delete_tx(tx_data_existing.id) == 0)
{
cerr << "cant remove tx " << tx_hash << '\n';
return false;
}
}
return true;
}
// default value of static veriables
uint64_t TxSearch::thread_search_life {600};

@ -5,13 +5,16 @@
#ifndef RESTBED_XMR_TXSEARCH_H
#define RESTBED_XMR_TXSEARCH_H
#include "MySqlAccounts.h"
#include "OutputInputIdentification.h"
#include <iostream>
#include <memory>
#include <mutex>
#include <thread>
#include <atomic>
#include "MySqlAccounts.h"
#include "OutputInputIdentification.h"
#include <algorithm>
#include <unordered_map>
namespace xmreg
{
@ -30,11 +33,12 @@ class TxSearchException: public std::runtime_error
class TxSearch
{
// out_pk, amount
using known_outputs_t = vector<pair<public_key, uint64_t>>;
// out_pk , amount
using known_outputs_t = unordered_map<public_key, uint64_t>;
// how frequently update scanned_block_height in Accounts table
static constexpr uint64_t UPDATE_SCANNED_HEIGHT_INTERVAL = 5; // seconds
//static constexpr uint64_t UPDATE_SCANNED_HEIGHT_INTERVAL = 5; // seconds
// how long should the search thread be live after no request
// are coming from the frontend. For example, when a user finishes
@ -88,6 +92,9 @@ public:
uint64_t
get_searched_blk_no() const;
inline uint64_t
get_current_timestamp() const;
void
ping();
@ -134,6 +141,9 @@ public:
static void
set_search_thread_life(uint64_t life_seconds);
bool
delete_existing_tx_if_exists(string const& tx_hash);
};

@ -1119,6 +1119,9 @@ YourMoneroRequests::import_recent_wallet_request(const shared_ptr< Session > ses
no_blocks_to_import = std::min(no_blocks_to_import,
CurrentBlockchainStatus::max_number_of_blocks_to_import);
no_blocks_to_import = std::min(no_blocks_to_import,
CurrentBlockchainStatus::get_current_blockchain_height());
XmrAccount acc;
if (xmr_accounts->select(xmr_address, acc))
@ -1128,7 +1131,7 @@ YourMoneroRequests::import_recent_wallet_request(const shared_ptr< Session > ses
// make sure scanned_block_height is larger than no_blocks_to_import so we dont
// end up with overflowing uint64_t.
if (updated_acc.scanned_block_height > no_blocks_to_import)
if (updated_acc.scanned_block_height >= no_blocks_to_import)
{
// repetead calls to import_recent_wallet_request will be moving the scanning backward.
// not sure yet if any protection is needed to make sure that a user does not
@ -1351,7 +1354,8 @@ YourMoneroRequests::get_tx(const shared_ptr< Session > session, const Bytes & bo
// implementation in the frontend.
if (CurrentBlockchainStatus::get_xmr_address_viewkey(xmr_address, address_info, viewkey))
{
OutputInputIdentification oi_identification {&address_info, &viewkey, &tx};
OutputInputIdentification oi_identification {&address_info, &viewkey, &tx,
tx_hash, coinbase};
oi_identification.identify_outputs();
@ -1444,7 +1448,7 @@ YourMoneroRequests::get_tx(const shared_ptr< Session > session, const Bytes & bo
// we have to redo this info from basically from scrach.
vector<pair<public_key, uint64_t>> known_outputs_keys;
unordered_map<public_key, uint64_t> known_outputs_keys;
if (CurrentBlockchainStatus::get_known_outputs_keys(
xmr_address, known_outputs_keys))
@ -1456,7 +1460,7 @@ YourMoneroRequests::get_tx(const shared_ptr< Session > session, const Bytes & bo
// Class that is resposnible for idenficitaction of our outputs
// and inputs in a given tx.
OutputInputIdentification oi_identification
{&address_info, &viewkey, &tx};
{&address_info, &viewkey, &tx, tx_hash, coinbase};
// no need mutex here, as this will be exectued only after
// the above. there is no threads here.
@ -1659,8 +1663,6 @@ YourMoneroRequests::login_and_start_search_thread(
if (CurrentBlockchainStatus::start_tx_search_thread(acc))
{
cout << "Search thread started" << endl;
j_response["status"] = "success";
j_response["new_address"] = false;

Loading…
Cancel
Save