From 3dba7f252e080ca9ce9c86dca033e6b0b00ee21d Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Fri, 2 Nov 2018 22:27:25 +0000 Subject: [PATCH 01/25] protocol: option to pad transaction relay to the next kB To help protect one's privacy from traffic volume analysis for people using Tor or I2P. This will really fly once we relay txes on a timer rather than on demand, though. Off by default for now since it's wasteful and doesn't bring anything until I2P's in. --- src/cryptonote_core/cryptonote_core.cpp | 10 +++++- src/cryptonote_core/cryptonote_core.h | 8 +++++ .../cryptonote_protocol_defs.h | 2 ++ .../cryptonote_protocol_handler.inl | 31 +++++++++++++++++++ tests/core_proxy/core_proxy.h | 1 + tests/unit_tests/ban.cpp | 1 + 6 files changed, 52 insertions(+), 1 deletion(-) diff --git a/src/cryptonote_core/cryptonote_core.cpp b/src/cryptonote_core/cryptonote_core.cpp index d8c38bf9e..c55499365 100644 --- a/src/cryptonote_core/cryptonote_core.cpp +++ b/src/cryptonote_core/cryptonote_core.cpp @@ -163,6 +163,11 @@ namespace cryptonote , "Relay blocks as normal blocks" , false }; + static const command_line::arg_descriptor arg_pad_transactions = { + "pad-transactions" + , "Pad relayed transactions to help defend against traffic volume analysis" + , false + }; static const command_line::arg_descriptor arg_max_txpool_weight = { "max-txpool-weight" , "Set maximum txpool weight in bytes." @@ -188,7 +193,8 @@ namespace cryptonote m_disable_dns_checkpoints(false), m_update_download(0), m_nettype(UNDEFINED), - m_update_available(false) + m_update_available(false), + m_pad_transactions(false) { m_checkpoints_updating.clear(); set_cryptonote_protocol(pprotocol); @@ -282,6 +288,7 @@ namespace cryptonote command_line::add_arg(desc, arg_offline); command_line::add_arg(desc, arg_disable_dns_checkpoints); command_line::add_arg(desc, arg_max_txpool_weight); + command_line::add_arg(desc, arg_pad_transactions); command_line::add_arg(desc, arg_block_notify); miner::init_options(desc); @@ -320,6 +327,7 @@ namespace cryptonote set_enforce_dns_checkpoints(command_line::get_arg(vm, arg_dns_checkpoints)); test_drop_download_height(command_line::get_arg(vm, arg_test_drop_download_height)); m_fluffy_blocks_enabled = !get_arg(vm, arg_no_fluffy_blocks); + m_pad_transactions = get_arg(vm, arg_pad_transactions); m_offline = get_arg(vm, arg_offline); m_disable_dns_checkpoints = get_arg(vm, arg_disable_dns_checkpoints); if (!command_line::is_arg_defaulted(vm, arg_fluffy_blocks)) diff --git a/src/cryptonote_core/cryptonote_core.h b/src/cryptonote_core/cryptonote_core.h index 80c452f53..cef42d207 100644 --- a/src/cryptonote_core/cryptonote_core.h +++ b/src/cryptonote_core/cryptonote_core.h @@ -756,6 +756,13 @@ namespace cryptonote */ bool fluffy_blocks_enabled() const { return m_fluffy_blocks_enabled; } + /** + * @brief get whether transaction relay should be padded + * + * @return whether transaction relay should be padded + */ + bool pad_transactions() const { return m_pad_transactions; } + /** * @brief check a set of hashes against the precompiled hash set * @@ -1014,6 +1021,7 @@ namespace cryptonote bool m_fluffy_blocks_enabled; bool m_offline; + bool m_pad_transactions; }; } diff --git a/src/cryptonote_protocol/cryptonote_protocol_defs.h b/src/cryptonote_protocol/cryptonote_protocol_defs.h index db159f0f4..d5bb50930 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_defs.h +++ b/src/cryptonote_protocol/cryptonote_protocol_defs.h @@ -146,9 +146,11 @@ namespace cryptonote struct request { std::vector txs; + std::string _; // padding BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(txs) + KV_SERIALIZE(_) END_KV_SERIALIZE_MAP() }; }; diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.inl b/src/cryptonote_protocol/cryptonote_protocol_handler.inl index c2c660e8c..6efdcc25e 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler.inl +++ b/src/cryptonote_protocol/cryptonote_protocol_handler.inl @@ -1724,8 +1724,39 @@ skip: bool t_cryptonote_protocol_handler::relay_transactions(NOTIFY_NEW_TRANSACTIONS::request& arg, cryptonote_connection_context& exclude_context) { // no check for success, so tell core they're relayed unconditionally + const bool pad_transactions = m_core.pad_transactions(); + size_t bytes = pad_transactions ? 9 /* header */ + 4 /* 1 + 'txs' */ + tools::get_varint_data(arg.txs.size()).size() : 0; for(auto tx_blob_it = arg.txs.begin(); tx_blob_it!=arg.txs.end(); ++tx_blob_it) + { m_core.on_transaction_relayed(*tx_blob_it); + if (pad_transactions) + bytes += tools::get_varint_data(tx_blob_it->size()).size() + tx_blob_it->size(); + } + + if (pad_transactions) + { + // stuff some dummy bytes in to stay safe from traffic volume analysis + static constexpr size_t granularity = 1024; + size_t padding = granularity - bytes % granularity; + const size_t overhead = 2 /* 1 + '_' */ + tools::get_varint_data(padding).size(); + if (overhead > padding) + padding = 0; + else + padding -= overhead; + arg._ = std::string(padding, ' '); + + std::string arg_buff; + epee::serialization::store_t_to_binary(arg, arg_buff); + + // we probably lowballed the payload size a bit, so added a but too much. Fix this now. + size_t remove = arg_buff.size() % granularity; + if (remove > arg._.size()) + arg._.clear(); + else + arg._.resize(arg._.size() - remove); + // if the size of _ moved enough, we might lose byte in size encoding, we don't care + } + return relay_post_notify(arg, exclude_context); } //------------------------------------------------------------------------------------------------------------------------ diff --git a/tests/core_proxy/core_proxy.h b/tests/core_proxy/core_proxy.h index 7d36a0f68..023c220ae 100644 --- a/tests/core_proxy/core_proxy.h +++ b/tests/core_proxy/core_proxy.h @@ -104,5 +104,6 @@ namespace tests cryptonote::difficulty_type get_block_cumulative_difficulty(uint64_t height) const { return 0; } bool fluffy_blocks_enabled() const { return false; } uint64_t prevalidate_block_hashes(uint64_t height, const std::vector &hashes) { return 0; } + bool pad_transactions() const { return false; } }; } diff --git a/tests/unit_tests/ban.cpp b/tests/unit_tests/ban.cpp index e3dbdaef1..12625a949 100644 --- a/tests/unit_tests/ban.cpp +++ b/tests/unit_tests/ban.cpp @@ -83,6 +83,7 @@ public: cryptonote::difficulty_type get_block_cumulative_difficulty(uint64_t height) const { return 0; } bool fluffy_blocks_enabled() const { return false; } uint64_t prevalidate_block_hashes(uint64_t height, const std::vector &hashes) { return 0; } + bool pad_transactions() { return false; } void stop() {} }; From 9b5efad29474416427cfb9b6a696b5dcb57cc919 Mon Sep 17 00:00:00 2001 From: stoffu Date: Sat, 24 Nov 2018 08:25:38 +0900 Subject: [PATCH 02/25] simplewallet: enable donation on testnet/stagenet for easier testing --- src/simplewallet/simplewallet.cpp | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/src/simplewallet/simplewallet.cpp b/src/simplewallet/simplewallet.cpp index d9fd0c13e..6b7681893 100644 --- a/src/simplewallet/simplewallet.cpp +++ b/src/simplewallet/simplewallet.cpp @@ -5942,12 +5942,6 @@ bool simple_wallet::sweep_below(const std::vector &args_) //---------------------------------------------------------------------------------------------------- bool simple_wallet::donate(const std::vector &args_) { - if(m_wallet->nettype() != cryptonote::MAINNET) - { - fail_msg_writer() << tr("donations are not enabled on the testnet or on the stagenet"); - return true; - } - std::vector local_args = args_; if(local_args.empty() || local_args.size() > 5) { @@ -5969,11 +5963,30 @@ bool simple_wallet::donate(const std::vector &args_) amount_str = local_args.back(); local_args.pop_back(); // push back address, amount, payment id - local_args.push_back(MONERO_DONATION_ADDR); + std::string address_str; + if (m_wallet->nettype() != cryptonote::MAINNET) + { + // if not mainnet, convert donation address string to the relevant network type + address_parse_info info; + if (!cryptonote::get_account_address_from_str(info, cryptonote::MAINNET, MONERO_DONATION_ADDR)) + { + fail_msg_writer() << tr("Failed to parse donation address: ") << MONERO_DONATION_ADDR; + return true; + } + address_str = cryptonote::get_account_address_as_str(m_wallet->nettype(), info.is_subaddress, info.address); + } + else + { + address_str = MONERO_DONATION_ADDR; + } + local_args.push_back(address_str); local_args.push_back(amount_str); if (!payment_id_str.empty()) local_args.push_back(payment_id_str); - message_writer() << (boost::format(tr("Donating %s %s to The Monero Project (donate.getmonero.org or %s).")) % amount_str % cryptonote::get_unit(cryptonote::get_default_decimal_point()) % MONERO_DONATION_ADDR).str(); + if (m_wallet->nettype() == cryptonote::MAINNET) + message_writer() << (boost::format(tr("Donating %s %s to The Monero Project (donate.getmonero.org or %s).")) % amount_str % cryptonote::get_unit(cryptonote::get_default_decimal_point()) % MONERO_DONATION_ADDR).str(); + else + message_writer() << (boost::format(tr("Donating %s %s to %s.")) % amount_str % cryptonote::get_unit(cryptonote::get_default_decimal_point()) % address_str).str(); transfer(local_args); return true; } From 2be31b4c9ceb4fde1a3968ab0ae2fa4d33facc5d Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Sat, 24 Nov 2018 10:52:20 +0000 Subject: [PATCH 03/25] blockchain_blackball: spot when all outputs of an amount are spent --- .../blockchain_blackball.cpp | 96 ++++++++++++++++++- 1 file changed, 95 insertions(+), 1 deletion(-) diff --git a/src/blockchain_utilities/blockchain_blackball.cpp b/src/blockchain_utilities/blockchain_blackball.cpp index d2ce5cf76..73819bd25 100644 --- a/src/blockchain_utilities/blockchain_blackball.cpp +++ b/src/blockchain_utilities/blockchain_blackball.cpp @@ -59,6 +59,7 @@ static MDB_dbi dbi_relative_rings; static MDB_dbi dbi_outputs; static MDB_dbi dbi_processed_txidx; static MDB_dbi dbi_spent; +static MDB_dbi dbi_per_amount; static MDB_dbi dbi_ring_instances; static MDB_dbi dbi_stats; static MDB_env *env = NULL; @@ -238,7 +239,7 @@ static void init(std::string cache_filename) dbr = mdb_env_create(&env); CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LDMB environment: " + std::string(mdb_strerror(dbr))); - dbr = mdb_env_set_maxdbs(env, 6); + dbr = mdb_env_set_maxdbs(env, 7); CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to set max env dbs: " + std::string(mdb_strerror(dbr))); const std::string actual_filename = get_cache_filename(cache_filename); dbr = mdb_env_open(env, actual_filename.c_str(), flags, 0664); @@ -265,6 +266,10 @@ static void init(std::string cache_filename) CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); mdb_set_dupsort(txn, dbi_spent, compare_uint64); + dbr = mdb_dbi_open(txn, "per_amount", MDB_CREATE | MDB_INTEGERKEY, &dbi_per_amount); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + mdb_set_compare(txn, dbi_per_amount, compare_uint64); + dbr = mdb_dbi_open(txn, "ring_instances", MDB_CREATE, &dbi_ring_instances); CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); @@ -283,6 +288,7 @@ static void close() mdb_dbi_close(env, dbi_relative_rings); mdb_dbi_close(env, dbi_outputs); mdb_dbi_close(env, dbi_processed_txidx); + mdb_dbi_close(env, dbi_per_amount); mdb_dbi_close(env, dbi_spent); mdb_dbi_close(env, dbi_ring_instances); mdb_dbi_close(env, dbi_stats); @@ -585,6 +591,55 @@ static std::vector get_spent_outputs(MDB_txn *txn) return outs; } +static void get_per_amount_outputs(MDB_txn *txn, uint64_t amount, uint64_t &total, uint64_t &spent) +{ + MDB_cursor *cur; + int dbr = mdb_cursor_open(txn, dbi_per_amount, &cur); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open cursor for per amount outputs: " + std::string(mdb_strerror(dbr))); + MDB_val k, v; + mdb_size_t count = 0; + k.mv_size = sizeof(uint64_t); + k.mv_data = (void*)&amount; + dbr = mdb_cursor_get(cur, &k, &v, MDB_SET); + if (dbr == MDB_NOTFOUND) + { + total = spent = 0; + } + else + { + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to get per amount outputs: " + std::string(mdb_strerror(dbr))); + total = ((const uint64_t*)v.mv_data)[0]; + spent = ((const uint64_t*)v.mv_data)[1]; + } + mdb_cursor_close(cur); +} + +static void inc_per_amount_outputs(MDB_txn *txn, uint64_t amount, uint64_t total, uint64_t spent) +{ + MDB_cursor *cur; + int dbr = mdb_cursor_open(txn, dbi_per_amount, &cur); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open cursor for per amount outputs: " + std::string(mdb_strerror(dbr))); + MDB_val k, v; + mdb_size_t count = 0; + k.mv_size = sizeof(uint64_t); + k.mv_data = (void*)&amount; + dbr = mdb_cursor_get(cur, &k, &v, MDB_SET); + if (dbr == 0) + { + total += ((const uint64_t*)v.mv_data)[0]; + spent += ((const uint64_t*)v.mv_data)[1]; + } + else + { + CHECK_AND_ASSERT_THROW_MES(dbr == MDB_NOTFOUND, "Failed to get per amount outputs: " + std::string(mdb_strerror(dbr))); + } + uint64_t data[2] = {total, spent}; + v.mv_size = 2 * sizeof(uint64_t); + v.mv_data = (void*)data; + dbr = mdb_cursor_put(cur, &k, &v, 0); + mdb_cursor_close(cur); +} + static uint64_t get_processed_txidx(const std::string &name) { MDB_txn *txn; @@ -1193,6 +1248,7 @@ int main(int argc, char* argv[]) for_all_transactions(filename, start_idx, n_txes, [&](const cryptonote::transaction_prefix &tx)->bool { std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; + const bool miner_tx = tx.vin.size() == 1 && tx.vin[0].type() == typeid(txin_gen); for (const auto &in: tx.vin) { if (in.type() != typeid(txin_to_key)) @@ -1210,6 +1266,9 @@ int main(int argc, char* argv[]) std::vector new_ring = canonicalize(txin.key_offsets); const uint32_t ring_size = txin.key_offsets.size(); const uint64_t instances = inc_ring_instances(txn, txin.amount, new_ring); + uint64_t pa_total = 0, pa_spent = 0; + if (!opt_rct_only) + get_per_amount_outputs(txn, txin.amount, pa_total, pa_spent); if (n == 0 && ring_size == 1) { const std::pair output = std::make_pair(txin.amount, absolute[0]); @@ -1237,6 +1296,21 @@ int main(int argc, char* argv[]) inc_stat(txn, txin.amount ? "pre-rct-duplicate-rings" : "rct-duplicate-rings"); } } + else if (n == 0 && !opt_rct_only && pa_spent + 1 == pa_total) + { + for (size_t o = 0; o < pa_total; ++o) + { + const std::pair output = std::make_pair(txin.amount, o); + if (opt_verbose) + { + MINFO("Marking output " << output.first << "/" << output.second << " as spent, due to as many outputs of that amount being spent as exist so far"); + std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; + } + blackballs.push_back(output); + if (add_spent_output(cur, output_data(txin.amount, o))) + inc_stat(txn, txin.amount ? "pre-rct-full-count" : "rct-full-count"); + } + } else if (n == 0 && opt_check_subsets && get_ring_subset_instances(txn, txin.amount, new_ring) >= new_ring.size()) { for (size_t o = 0; o < new_ring.size(); ++o) @@ -1299,9 +1373,28 @@ int main(int argc, char* argv[]) } } if (n == 0) + { set_relative_ring(txn, txin.k_image, new_ring); + if (!opt_rct_only) + inc_per_amount_outputs(txn, txin.amount, 0, 1); + } } set_processed_txidx(txn, canonical, start_idx+1); + if (!opt_rct_only) + { + for (const auto &out: tx.vout) + { + uint64_t amount = out.amount; + if (miner_tx && tx.version >= 2) + amount = 0; + + if (opt_rct_only && amount != 0) + continue; + if (out.target.type() != typeid(txout_to_key)) + continue; + inc_per_amount_outputs(txn, amount, 1, 0); + } + } ++records; if (records >= records_per_sync) @@ -1433,6 +1526,7 @@ skip_secondary_passes: { "pre-rct-ring-size-1", pre_rct }, { "rct-ring-size-1", rct }, { "pre-rct-duplicate-rings", pre_rct }, { "rct-duplicate-rings", rct }, { "pre-rct-subset-rings", pre_rct }, { "rct-subset-rings", rct }, + { "pre-rct-full-count", pre_rct }, { "rct-full-count", rct }, { "pre-rct-key-image-attack", pre_rct }, { "rct-key-image-attack", rct }, { "pre-rct-extra", pre_rct }, { "rct-ring-extra", rct }, { "pre-rct-chain-reaction", pre_rct }, { "rct-chain-reaction", rct }, From e98ae34e4b3487cfa2e7b2278349a49671c2e901 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Mon, 26 Nov 2018 23:42:14 +0000 Subject: [PATCH 04/25] core: fix adding new pre-hoh block when a tx is already in the pool --- src/cryptonote_core/cryptonote_core.cpp | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/cryptonote_core/cryptonote_core.cpp b/src/cryptonote_core/cryptonote_core.cpp index 10ab3fe65..348c788b9 100644 --- a/src/cryptonote_core/cryptonote_core.cpp +++ b/src/cryptonote_core/cryptonote_core.cpp @@ -900,13 +900,15 @@ namespace cryptonote bool ok = true; it = tx_blobs.begin(); for (size_t i = 0; i < tx_blobs.size(); i++, ++it) { - if (already_have[i]) - continue; if (!results[i].res) { ok = false; continue; } + if (keeped_by_block) + get_blockchain_storage().on_new_tx_from_block(results[i].tx); + if (already_have[i]) + continue; const size_t weight = get_transaction_weight(results[i].tx, it->size()); ok &= add_new_tx(results[i].tx, results[i].hash, tx_blobs[i], results[i].prefix_hash, weight, tvc[i], keeped_by_block, relayed, do_not_relay); @@ -1137,9 +1139,6 @@ namespace cryptonote //----------------------------------------------------------------------------------------------- bool core::add_new_tx(transaction& tx, const crypto::hash& tx_hash, const cryptonote::blobdata &blob, const crypto::hash& tx_prefix_hash, size_t tx_weight, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay) { - if (keeped_by_block) - get_blockchain_storage().on_new_tx_from_block(tx); - if(m_mempool.have_tx(tx_hash)) { LOG_PRINT_L2("tx " << tx_hash << "already have transaction in tx_pool"); From 7cc27b367ecdfe1dca80ef24c6dcc513cb046dba Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Sun, 11 Nov 2018 21:31:48 +0000 Subject: [PATCH 05/25] Revert "easylogging++: make the logger handle early/late logging" This reverts commit 7f8bdeb35c73c70b2b65e30aa2a1cb93696355b3. --- external/easylogging++/easylogging++.cc | 12 ------------ external/easylogging++/easylogging++.h | 9 ++++----- 2 files changed, 4 insertions(+), 17 deletions(-) diff --git a/external/easylogging++/easylogging++.cc b/external/easylogging++/easylogging++.cc index d57f3f3a0..81179b0a3 100644 --- a/external/easylogging++/easylogging++.cc +++ b/external/easylogging++/easylogging++.cc @@ -2191,17 +2191,6 @@ void VRegistry::setFromArgs(const base::utils::CommandLineArgs* commandLineArgs) # define ELPP_DEFAULT_LOGGING_FLAGS 0x0 #endif // !defined(ELPP_DEFAULT_LOGGING_FLAGS) // Storage -el::base::type::StoragePointer getresetELPP(bool reset) -{ - static el::base::type::StoragePointer p(new el::base::Storage(el::LogBuilderPtr(new el::base::DefaultLogBuilder()))); - if (reset) - p = NULL; - return p; -} -el::base::type::StoragePointer el::base::Storage::getELPP() -{ - return getresetELPP(false); -} #if ELPP_ASYNC_LOGGING Storage::Storage(const LogBuilderPtr& defaultLogBuilder, base::IWorker* asyncDispatchWorker) : #else @@ -2250,7 +2239,6 @@ Storage::Storage(const LogBuilderPtr& defaultLogBuilder) : Storage::~Storage(void) { ELPP_INTERNAL_INFO(4, "Destroying storage"); - getresetELPP(true); #if ELPP_ASYNC_LOGGING ELPP_INTERNAL_INFO(5, "Replacing log dispatch callback to synchronous"); uninstallLogDispatchCallback(std::string("AsyncLogDispatchCallback")); diff --git a/external/easylogging++/easylogging++.h b/external/easylogging++/easylogging++.h index 046252a5b..375010d46 100644 --- a/external/easylogging++/easylogging++.h +++ b/external/easylogging++/easylogging++.h @@ -2734,8 +2734,6 @@ class Storage : base::NoCopy, public base::threading::ThreadSafe { return it->second; } - static el::base::type::StoragePointer getELPP(); - private: base::RegisteredHitCounters* m_registeredHitCounters; base::RegisteredLoggers* m_registeredLoggers; @@ -2770,7 +2768,7 @@ class Storage : base::NoCopy, public base::threading::ThreadSafe { } }; extern ELPP_EXPORT base::type::StoragePointer elStorage; -#define ELPP el::base::Storage::getELPP() +#define ELPP el::base::elStorage class DefaultLogDispatchCallback : public LogDispatchCallback { protected: void handle(const LogDispatchData* data); @@ -4613,9 +4611,10 @@ el::base::debug::CrashHandler elCrashHandler(ELPP_USE_DEF_CRASH_HANDLER); \ } #if ELPP_ASYNC_LOGGING -# define INITIALIZE_EASYLOGGINGPP ELPP_INIT_EASYLOGGINGPP(NULL) +# define INITIALIZE_EASYLOGGINGPP ELPP_INIT_EASYLOGGINGPP(new el::base::Storage(el::LogBuilderPtr(new el::base::DefaultLogBuilder()),\ +new el::base::AsyncDispatchWorker())) #else -# define INITIALIZE_EASYLOGGINGPP ELPP_INIT_EASYLOGGINGPP(NULL) +# define INITIALIZE_EASYLOGGINGPP ELPP_INIT_EASYLOGGINGPP(new el::base::Storage(el::LogBuilderPtr(new el::base::DefaultLogBuilder()))) #endif // ELPP_ASYNC_LOGGING #define INITIALIZE_NULL_EASYLOGGINGPP \ namespace el {\ From 721aacd88e780b30416deb66a68b87da8fe244ae Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Sun, 11 Nov 2018 21:34:37 +0000 Subject: [PATCH 06/25] easylogging++: faster access to logging Turns out getting the global shared_ptr hits the profile, and passing it around still keeps it at close to ~1% CPU, which is too much for mostly silent logging. Leak the object instead, which is even safer for late logging. --- contrib/valgrind/monero.supp | 9 +++++++++ external/easylogging++/easylogging++.cc | 6 ++++++ external/easylogging++/easylogging++.h | 6 ++++-- 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/contrib/valgrind/monero.supp b/contrib/valgrind/monero.supp index 16e34e82f..015b05a1c 100644 --- a/contrib/valgrind/monero.supp +++ b/contrib/valgrind/monero.supp @@ -17,3 +17,12 @@ fun:maybe_unlock_and_signal_one > ... } + +{ + we leak the logger, for performance reasons in on-the-fly init + Memcheck:Leak + match-leak-kinds: definite + fun:_Znwm + fun:_ZN2el4base7Storage7getELPPEv + ... +} diff --git a/external/easylogging++/easylogging++.cc b/external/easylogging++/easylogging++.cc index 81179b0a3..c24fb80cf 100644 --- a/external/easylogging++/easylogging++.cc +++ b/external/easylogging++/easylogging++.cc @@ -2191,6 +2191,12 @@ void VRegistry::setFromArgs(const base::utils::CommandLineArgs* commandLineArgs) # define ELPP_DEFAULT_LOGGING_FLAGS 0x0 #endif // !defined(ELPP_DEFAULT_LOGGING_FLAGS) // Storage +el::base::type::StoragePointer &el::base::Storage::getELPP() +{ + if (!el::base::elStorage) + el::base::elStorage = new el::base::Storage(el::LogBuilderPtr(new el::base::DefaultLogBuilder())); + return el::base::elStorage; +} #if ELPP_ASYNC_LOGGING Storage::Storage(const LogBuilderPtr& defaultLogBuilder, base::IWorker* asyncDispatchWorker) : #else diff --git a/external/easylogging++/easylogging++.h b/external/easylogging++/easylogging++.h index 375010d46..acf2a7674 100644 --- a/external/easylogging++/easylogging++.h +++ b/external/easylogging++/easylogging++.h @@ -552,7 +552,7 @@ typedef std::ostream ostream_t; typedef unsigned int EnumType; typedef unsigned short VerboseLevel; typedef unsigned long int LineNumber; -typedef std::shared_ptr StoragePointer; +typedef base::Storage *StoragePointer; typedef std::shared_ptr LogDispatchCallbackPtr; typedef std::shared_ptr PerformanceTrackingCallbackPtr; typedef std::shared_ptr LoggerRegistrationCallbackPtr; @@ -2734,6 +2734,8 @@ class Storage : base::NoCopy, public base::threading::ThreadSafe { return it->second; } + static el::base::type::StoragePointer &getELPP(); + private: base::RegisteredHitCounters* m_registeredHitCounters; base::RegisteredLoggers* m_registeredLoggers; @@ -2768,7 +2770,7 @@ class Storage : base::NoCopy, public base::threading::ThreadSafe { } }; extern ELPP_EXPORT base::type::StoragePointer elStorage; -#define ELPP el::base::elStorage +#define ELPP el::base::Storage::getELPP() class DefaultLogDispatchCallback : public LogDispatchCallback { protected: void handle(const LogDispatchData* data); From 5ca4994c9c31228139317c4278e1868598a0b975 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Wed, 7 Nov 2018 14:57:41 +0000 Subject: [PATCH 07/25] rpc: speed up the common get_output_distribution case while syncing --- src/rpc/rpc_handler.cpp | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/src/rpc/rpc_handler.cpp b/src/rpc/rpc_handler.cpp index 63664bf8b..e0a81c70f 100644 --- a/src/rpc/rpc_handler.cpp +++ b/src/rpc/rpc_handler.cpp @@ -43,8 +43,25 @@ namespace rpc std::vector distribution; std::uint64_t start_height, base; - if (!f(amount, from_height, to_height, start_height, distribution, base)) - return boost::none; + + // see if we can extend the cache - a common case + if (d.cached && amount == 0 && d.cached_from == from_height && to_height > d.cached_to) + { + std::vector new_distribution; + if (!f(amount, d.cached_to + 1, to_height, start_height, new_distribution, base)) + return boost::none; + distribution = d.cached_distribution; + distribution.reserve(distribution.size() + new_distribution.size()); + for (const auto &e: new_distribution) + distribution.push_back(e); + start_height = d.cached_start_height; + base = d.cached_base; + } + else + { + if (!f(amount, from_height, to_height, start_height, distribution, base)) + return boost::none; + } if (to_height > 0 && to_height >= from_height) { From 756684bb28482d953f822bbb9940c9b482b40054 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Sat, 10 Nov 2018 13:46:37 +0000 Subject: [PATCH 08/25] blockchain: avoid unnecessary DB lookups when syncing Some of the inputs for block in a span will be from other earlier blocks in that span. Keep track of those outputs so we don't have to look them up again after those early blocks are added to the blockchain. --- src/cryptonote_core/blockchain.cpp | 117 +++++++++++++++++++++-------- src/cryptonote_core/blockchain.h | 4 +- 2 files changed, 86 insertions(+), 35 deletions(-) diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index bfc5dbbe0..a6e35acc8 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -3723,7 +3723,7 @@ void Blockchain::set_enforce_dns_checkpoints(bool enforce_checkpoints) } //------------------------------------------------------------------ -void Blockchain::block_longhash_worker(uint64_t height, const std::vector &blocks, std::unordered_map &map) const +void Blockchain::block_longhash_worker(uint64_t height, const epee::span &blocks, std::unordered_map &map) const { TIME_MEASURE_START(t); slow_hash_allocate_state(); @@ -3809,11 +3809,33 @@ bool Blockchain::cleanup_handle_incoming_blocks(bool force_sync) } //------------------------------------------------------------------ -void Blockchain::output_scan_worker(const uint64_t amount, const std::vector &offsets, std::vector &outputs) const +void Blockchain::output_scan_worker(const uint64_t amount, const std::vector &offsets, std::vector &outputs, const std::vector &extra_tx_map) const { try { m_db->get_output_key(epee::span(&amount, 1), offsets, outputs, true); + if (outputs.size() < offsets.size()) + { + const uint64_t n_outputs = m_db->get_num_outputs(amount); + for (size_t i = outputs.size(); i < offsets.size(); ++i) + { + uint64_t idx = offsets[i]; + if (idx < n_outputs) + { + MWARNING("Index " << idx << " not found in db for amount " << amount << ", but it is less than the number of entries"); + break; + } + else if (idx < n_outputs + extra_tx_map.size()) + { + outputs.push_back(extra_tx_map[idx - n_outputs]); + } + else + { + MWARNING("missed " << amount << "/" << idx << " in " << extra_tx_map.size() << " (chain " << n_outputs << ")"); + break; + } + } + } } catch (const std::exception& e) { @@ -3928,6 +3950,34 @@ uint64_t Blockchain::prevalidate_block_hashes(uint64_t height, const std::vector // vs [k_image, output_keys] (m_scan_table). This is faster because it takes advantage of bulk queries // and is threaded if possible. The table (m_scan_table) will be used later when querying output // keys. +static bool update_output_map(std::map> &extra_tx_map, const transaction &tx, uint64_t height, bool miner) +{ + MTRACE("Blockchain::" << __func__); + for (size_t i = 0; i < tx.vout.size(); ++i) + { + const auto &out = tx.vout[i]; + if (out.target.type() != typeid(txout_to_key)) + continue; + const txout_to_key &out_to_key = boost::get(out.target); + rct::key commitment; + uint64_t amount = out.amount; + if (miner && tx.version == 2) + { + commitment = rct::zeroCommit(amount); + amount = 0; + } + else if (tx.version > 1) + { + CHECK_AND_ASSERT_MES(i < tx.rct_signatures.outPk.size(), false, "Invalid outPk size"); + commitment = tx.rct_signatures.outPk[i].mask; + } + else + commitment = rct::zero(); + extra_tx_map[amount].push_back(output_data_t{out_to_key.key, tx.unlock_time, height, commitment}); + } + return true; +} + bool Blockchain::prepare_handle_incoming_blocks(const std::vector &blocks_entry) { MTRACE("Blockchain::" << __func__); @@ -3974,42 +4024,40 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::vectorheight() + blocks_entry.size()) < m_blocks_hash_check.size()) + const uint64_t height = m_db->height(); + if ((height + blocks_entry.size()) < m_blocks_hash_check.size()) return true; bool blocks_exist = false; tools::threadpool& tpool = tools::threadpool::getInstance(); - uint64_t threads = tpool.get_max_concurrency(); + unsigned threads = tpool.get_max_concurrency(); + std::vector blocks; + blocks.resize(blocks_entry.size()); - if (blocks_entry.size() > 1 && threads > 1 && m_max_prepare_blocks_threads > 1) + if (1) { // limit threads, default limit = 4 if(threads > m_max_prepare_blocks_threads) threads = m_max_prepare_blocks_threads; - uint64_t height = m_db->height(); - int batches = blocks_entry.size() / threads; - int extra = blocks_entry.size() % threads; + unsigned int batches = blocks_entry.size() / threads; + unsigned int extra = blocks_entry.size() % threads; MDEBUG("block_batches: " << batches); std::vector> maps(threads); - std::vector < std::vector < block >> blocks(threads); auto it = blocks_entry.begin(); + unsigned blockidx = 0; - for (uint64_t i = 0; i < threads; i++) + for (unsigned i = 0; i < threads; i++) { - blocks[i].reserve(batches + 1); - for (int j = 0; j < batches; j++) + for (unsigned int j = 0; j < batches; j++, ++blockidx) { - block block; + block &block = blocks[blockidx]; if (!parse_and_validate_block_from_blob(it->block, block)) - { - std::advance(it, 1); - continue; - } + return false; // check first block and skip all blocks if its not chained properly - if (i == 0 && j == 0) + if (blockidx == 0) { crypto::hash tophash = m_db->top_block_hash(); if (block.prev_id != tophash) @@ -4024,20 +4072,16 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::vectorblock, block)) - { - std::advance(it, 1); - continue; - } + return false; if (have_block(get_block_hash(block))) { @@ -4045,7 +4089,6 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::vector(&blocks[i], nblocks), std::ref(maps[i])), true); + thread_height += nblocks; } waiter.wait(&tpool); @@ -4100,7 +4146,7 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::vector> offset_map; // [output] stores all output_data_t for each absolute_offset - std::map> tx_map; + std::map> tx_map, extra_tx_map; std::vector> txes(total_txs); #define SCAN_TABLE_QUIT(m) \ @@ -4111,12 +4157,14 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::vector= txes.size()) @@ -4175,7 +4223,10 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::vector &offsets, - std::vector &outputs) const; + std::vector &outputs, const std::vector &extra_tx_map) const; /** * @brief computes the "short" and "long" hashes for a set of blocks @@ -927,7 +927,7 @@ namespace cryptonote * @param blocks the blocks to be hashed * @param map return-by-reference the hashes for each block */ - void block_longhash_worker(uint64_t height, const std::vector &blocks, + void block_longhash_worker(uint64_t height, const epee::span &blocks, std::unordered_map &map) const; /** From 17b45725af754a3887f0c59bd5c67c58278d71eb Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Sat, 24 Nov 2018 14:49:04 +0000 Subject: [PATCH 09/25] Outputs where all amounts are known spent can now be pruned Only for pre rct for obvious reasons. Note: DO NOT use a known spent list which includes outputs which are not known spent. If the list includes any output that's just strongly thought to be spent, but not provably so, you risk finding yourself unable to sync past the point where that output is spent. I estimate only 200 MB saved on current mainnet though, unless the new blackballing rule unearths a good amount of large-amount-set extra spent outs. --- src/blockchain_db/blockchain_db.h | 7 + src/blockchain_db/lmdb/db_lmdb.cpp | 70 ++++- src/blockchain_db/lmdb/db_lmdb.h | 2 + src/blockchain_utilities/CMakeLists.txt | 33 +++ .../blockchain_prune_known_spent_data.cpp | 268 ++++++++++++++++++ tests/unit_tests/testdb.h | 1 + 6 files changed, 377 insertions(+), 4 deletions(-) create mode 100644 src/blockchain_utilities/blockchain_prune_known_spent_data.cpp diff --git a/src/blockchain_db/blockchain_db.h b/src/blockchain_db/blockchain_db.h index 53e33898a..a8f4eafe0 100644 --- a/src/blockchain_db/blockchain_db.h +++ b/src/blockchain_db/blockchain_db.h @@ -1405,6 +1405,13 @@ public: */ virtual cryptonote::blobdata get_txpool_tx_blob(const crypto::hash& txid) const = 0; + /** + * @brief prune output data for the given amount + * + * @param amount the amount for which to prune data + */ + virtual void prune_outputs(uint64_t amount) = 0; + /** * @brief runs a function over all txpool transactions * diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index d260caa75..142e86b5d 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -1077,6 +1077,60 @@ void BlockchainLMDB::remove_output(const uint64_t amount, const uint64_t& out_in throw0(DB_ERROR(lmdb_error(std::string("Error deleting amount for output index ").append(boost::lexical_cast(out_index).append(": ")).c_str(), result).c_str())); } +void BlockchainLMDB::prune_outputs(uint64_t amount) +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + mdb_txn_cursors *m_cursors = &m_wcursors; + CURSOR(output_amounts); + CURSOR(output_txs); + + MINFO("Pruning outputs for amount " << amount); + + MDB_val v; + MDB_val_set(k, amount); + int result = mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_SET); + if (result == MDB_NOTFOUND) + return; + if (result) + throw0(DB_ERROR(lmdb_error("Error looking up outputs: ", result).c_str())); + + // gather output ids + mdb_size_t num_elems; + mdb_cursor_count(m_cur_output_amounts, &num_elems); + MINFO(num_elems << " outputs found"); + std::vector output_ids; + output_ids.reserve(num_elems); + while (1) + { + const pre_rct_outkey *okp = (const pre_rct_outkey *)v.mv_data; + output_ids.push_back(okp->output_id); + MDEBUG("output id " << okp->output_id); + result = mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_NEXT_DUP); + if (result == MDB_NOTFOUND) + break; + if (result) + throw0(DB_ERROR(lmdb_error("Error counting outputs: ", result).c_str())); + } + if (output_ids.size() != num_elems) + throw0(DB_ERROR("Unexpected number of outputs")); + + result = mdb_cursor_del(m_cur_output_amounts, MDB_NODUPDATA); + if (result) + throw0(DB_ERROR(lmdb_error("Error deleting outputs: ", result).c_str())); + + for (uint64_t output_id: output_ids) + { + MDB_val_set(v, output_id); + result = mdb_cursor_get(m_cur_output_txs, (MDB_val *)&zerokval, &v, MDB_GET_BOTH); + if (result) + throw0(DB_ERROR(lmdb_error("Error looking up output: ", result).c_str())); + result = mdb_cursor_del(m_cur_output_txs, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Error deleting output: ", result).c_str())); + } +} + void BlockchainLMDB::add_spent_key(const crypto::key_image& k_image) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); @@ -2231,11 +2285,19 @@ uint64_t BlockchainLMDB::num_outputs() const TXN_PREFIX_RDONLY(); int result; - // get current height - MDB_stat db_stats; - if ((result = mdb_stat(m_txn, m_output_txs, &db_stats))) + RCURSOR(output_txs) + + uint64_t num = 0; + MDB_val k, v; + result = mdb_cursor_get(m_cur_output_txs, &k, &v, MDB_LAST); + if (result == MDB_NOTFOUND) + num = 0; + else if (result == 0) + num = 1 + ((const outtx*)v.mv_data)->output_id; + else throw0(DB_ERROR(lmdb_error("Failed to query m_output_txs: ", result).c_str())); - return db_stats.ms_entries; + + return num; } bool BlockchainLMDB::tx_exists(const crypto::hash& h) const diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h index 26159ab4d..6db241240 100644 --- a/src/blockchain_db/lmdb/db_lmdb.h +++ b/src/blockchain_db/lmdb/db_lmdb.h @@ -345,6 +345,8 @@ private: void remove_output(const uint64_t amount, const uint64_t& out_index); + virtual void prune_outputs(uint64_t amount); + virtual void add_spent_key(const crypto::key_image& k_image); virtual void remove_spent_key(const crypto::key_image& k_image); diff --git a/src/blockchain_utilities/CMakeLists.txt b/src/blockchain_utilities/CMakeLists.txt index c9ad1cebe..ddf575c29 100644 --- a/src/blockchain_utilities/CMakeLists.txt +++ b/src/blockchain_utilities/CMakeLists.txt @@ -81,6 +81,17 @@ monero_private_headers(blockchain_usage +set(blockchain_prune_known_spent_data_sources + blockchain_prune_known_spent_data.cpp + ) + +set(blockchain_prune_known_spent_data_private_headers) + +monero_private_headers(blockchain_prune_known_spent_data + ${blockchain_prune_known_spent_data_private_headers}) + + + set(blockchain_ancestry_sources blockchain_ancestry.cpp ) @@ -265,3 +276,25 @@ set_property(TARGET blockchain_stats PROPERTY OUTPUT_NAME "monero-blockchain-stats") install(TARGETS blockchain_stats DESTINATION bin) + +monero_add_executable(blockchain_prune_known_spent_data + ${blockchain_prune_known_spent_data_sources} + ${blockchain_prune_known_spent_data_private_headers}) + +target_link_libraries(blockchain_prune_known_spent_data + PRIVATE + cryptonote_core + blockchain_db + p2p + version + epee + ${Boost_FILESYSTEM_LIBRARY} + ${Boost_SYSTEM_LIBRARY} + ${Boost_THREAD_LIBRARY} + ${CMAKE_THREAD_LIBS_INIT} + ${EXTRA_LIBRARIES}) + +set_property(TARGET blockchain_prune_known_spent_data + PROPERTY + OUTPUT_NAME "monero-blockchain-prune-known-spent-data") +install(TARGETS blockchain_prune_known_spent_data DESTINATION bin) diff --git a/src/blockchain_utilities/blockchain_prune_known_spent_data.cpp b/src/blockchain_utilities/blockchain_prune_known_spent_data.cpp new file mode 100644 index 000000000..1a5869e9d --- /dev/null +++ b/src/blockchain_utilities/blockchain_prune_known_spent_data.cpp @@ -0,0 +1,268 @@ +// Copyright (c) 2014-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include +#include "common/command_line.h" +#include "serialization/crypto.h" +#include "cryptonote_core/tx_pool.h" +#include "cryptonote_core/cryptonote_core.h" +#include "cryptonote_core/blockchain.h" +#include "blockchain_db/blockchain_db.h" +#include "blockchain_db/db_types.h" +#include "version.h" + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "bcutil" + +namespace po = boost::program_options; +using namespace epee; +using namespace cryptonote; + +static std::map load_outputs(const std::string &filename) +{ + std::map outputs; + uint64_t amount = std::numeric_limits::max(); + FILE *f; + + f = fopen(filename.c_str(), "r"); + if (!f) + { + MERROR("Failed to load outputs from " << filename << ": " << strerror(errno)); + return {}; + } + while (1) + { + char s[256]; + if (!fgets(s, sizeof(s), f)) + break; + if (feof(f)) + break; + const size_t len = strlen(s); + if (len > 0 && s[len - 1] == '\n') + s[len - 1] = 0; + if (!s[0]) + continue; + std::pair output; + uint64_t offset, num_offsets; + if (sscanf(s, "@%" PRIu64, &amount) == 1) + { + continue; + } + if (amount == std::numeric_limits::max()) + { + MERROR("Bad format in " << filename); + continue; + } + if (sscanf(s, "%" PRIu64 "*%" PRIu64, &offset, &num_offsets) == 2 && num_offsets < std::numeric_limits::max() - offset) + { + outputs[amount] += num_offsets; + } + else if (sscanf(s, "%" PRIu64, &offset) == 1) + { + outputs[amount] += 1; + } + else + { + MERROR("Bad format in " << filename); + continue; + } + } + fclose(f); + return outputs; +} + +int main(int argc, char* argv[]) +{ + TRY_ENTRY(); + + epee::string_tools::set_module_name_and_folder(argv[0]); + + std::string default_db_type = "lmdb"; + + std::string available_dbs = cryptonote::blockchain_db_types(", "); + available_dbs = "available: " + available_dbs; + + uint32_t log_level = 0; + + tools::on_startup(); + + po::options_description desc_cmd_only("Command line options"); + po::options_description desc_cmd_sett("Command line options and settings options"); + const command_line::arg_descriptor arg_log_level = {"log-level", "0-4 or categories", ""}; + const command_line::arg_descriptor arg_database = { + "database", available_dbs.c_str(), default_db_type + }; + const command_line::arg_descriptor arg_verbose = {"verbose", "Verbose output", false}; + const command_line::arg_descriptor arg_dry_run = {"dry-run", "Do not actually prune", false}; + const command_line::arg_descriptor arg_input = {"input", "Path to the known spent outputs file"}; + + command_line::add_arg(desc_cmd_sett, cryptonote::arg_data_dir); + command_line::add_arg(desc_cmd_sett, cryptonote::arg_testnet_on); + command_line::add_arg(desc_cmd_sett, cryptonote::arg_stagenet_on); + command_line::add_arg(desc_cmd_sett, arg_log_level); + command_line::add_arg(desc_cmd_sett, arg_database); + command_line::add_arg(desc_cmd_sett, arg_verbose); + command_line::add_arg(desc_cmd_sett, arg_dry_run); + command_line::add_arg(desc_cmd_sett, arg_input); + command_line::add_arg(desc_cmd_only, command_line::arg_help); + + po::options_description desc_options("Allowed options"); + desc_options.add(desc_cmd_only).add(desc_cmd_sett); + + po::variables_map vm; + bool r = command_line::handle_error_helper(desc_options, [&]() + { + auto parser = po::command_line_parser(argc, argv).options(desc_options); + po::store(parser.run(), vm); + po::notify(vm); + return true; + }); + if (! r) + return 1; + + if (command_line::get_arg(vm, command_line::arg_help)) + { + std::cout << "Monero '" << MONERO_RELEASE_NAME << "' (v" << MONERO_VERSION_FULL << ")" << ENDL << ENDL; + std::cout << desc_options << std::endl; + return 1; + } + + mlog_configure(mlog_get_default_log_path("monero-blockchain-prune-known-spent-data.log"), true); + if (!command_line::is_arg_defaulted(vm, arg_log_level)) + mlog_set_log(command_line::get_arg(vm, arg_log_level).c_str()); + else + mlog_set_log(std::string(std::to_string(log_level) + ",bcutil:INFO").c_str()); + + LOG_PRINT_L0("Starting..."); + + std::string opt_data_dir = command_line::get_arg(vm, cryptonote::arg_data_dir); + bool opt_testnet = command_line::get_arg(vm, cryptonote::arg_testnet_on); + bool opt_stagenet = command_line::get_arg(vm, cryptonote::arg_stagenet_on); + network_type net_type = opt_testnet ? TESTNET : opt_stagenet ? STAGENET : MAINNET; + bool opt_verbose = command_line::get_arg(vm, arg_verbose); + bool opt_dry_run = command_line::get_arg(vm, arg_dry_run); + + std::string db_type = command_line::get_arg(vm, arg_database); + if (!cryptonote::blockchain_valid_db_type(db_type)) + { + std::cerr << "Invalid database type: " << db_type << std::endl; + return 1; + } + + const std::string input = command_line::get_arg(vm, arg_input); + if (input.empty()) + { + LOG_PRINT_L0("No input given"); + return 1; + } + + LOG_PRINT_L0("Initializing source blockchain (BlockchainDB)"); + std::unique_ptr core_storage; + tx_memory_pool m_mempool(*core_storage); + core_storage.reset(new Blockchain(m_mempool)); + BlockchainDB *db = new_db(db_type); + if (db == NULL) + { + LOG_ERROR("Attempted to use non-existent database type: " << db_type); + throw std::runtime_error("Attempting to use non-existent database type"); + } + LOG_PRINT_L0("database: " << db_type); + + const std::string filename = (boost::filesystem::path(opt_data_dir) / db->get_db_name()).string(); + LOG_PRINT_L0("Loading blockchain from folder " << filename << " ..."); + + try + { + db->open(filename, 0); + } + catch (const std::exception& e) + { + LOG_PRINT_L0("Error opening database: " << e.what()); + return 1; + } + r = core_storage->init(db, net_type); + + CHECK_AND_ASSERT_MES(r, 1, "Failed to initialize source blockchain storage"); + LOG_PRINT_L0("Source blockchain storage initialized OK"); + + LOG_PRINT_L0("Loading known spent data..."); + const std::map known_spent_outputs = load_outputs(input); + + LOG_PRINT_L0("Pruning known spent data..."); + + bool stop_requested = false; + tools::signal_handler::install([&stop_requested](int type) { + stop_requested = true; + }); + + db->batch_start(); + + size_t num_total_outputs = 0, num_prunable_outputs = 0, num_known_spent_outputs = 0, num_eligible_outputs = 0, num_eligible_known_spent_outputs = 0; + for (auto i = known_spent_outputs.begin(); i != known_spent_outputs.end(); ++i) + { + uint64_t num_outputs = db->get_num_outputs(i->first); + num_total_outputs += num_outputs; + num_known_spent_outputs += i->second; + if (i->first == 0 || is_valid_decomposed_amount(i->first)) + { + if (opt_verbose) + MINFO("Ignoring output value " << i->first << ", with " << num_outputs << " outputs"); + continue; + } + num_eligible_outputs += num_outputs; + num_eligible_known_spent_outputs += i->second; + if (opt_verbose) + MINFO(i->first << ": " << i->second << "/" << num_outputs); + if (num_outputs > i->second) + continue; + if (num_outputs && num_outputs < i->second) + { + MERROR("More outputs are spent than known for amount " << i->first << ", not touching"); + continue; + } + if (opt_verbose) + MINFO("Pruning data for " << num_outputs << " outputs"); + if (!opt_dry_run) + db->prune_outputs(i->first); + num_prunable_outputs += i->second; + } + + db->batch_stop(); + + MINFO("Total outputs: " << num_total_outputs); + MINFO("Known spent outputs: " << num_known_spent_outputs); + MINFO("Eligible outputs: " << num_eligible_outputs); + MINFO("Eligible known spent outputs: " << num_eligible_known_spent_outputs); + MINFO("Prunable outputs: " << num_prunable_outputs); + + LOG_PRINT_L0("Blockchain known spent data pruned OK"); + core_storage->deinit(); + return 0; + + CATCH_ENTRY("Error", 1); +} diff --git a/tests/unit_tests/testdb.h b/tests/unit_tests/testdb.h index a9c772920..5d9ba5833 100644 --- a/tests/unit_tests/testdb.h +++ b/tests/unit_tests/testdb.h @@ -142,5 +142,6 @@ public: virtual bool prune_blockchain(uint32_t pruning_seed = 0) { return true; } virtual bool update_pruning() { return true; } virtual bool check_pruning() { return true; } + virtual void prune_outputs(uint64_t amount) {} }; From a48f2dab00a81ab716f38022dc3197d841b63b7e Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Tue, 27 Nov 2018 15:44:01 +0000 Subject: [PATCH 10/25] blockchain_prune_known_spent_data: blackball file is now optional If not present, the tool will scan the blockchain, since scanning for this is fairly fast. --- .../blockchain_prune_known_spent_data.cpp | 51 ++++++++++++++++--- 1 file changed, 44 insertions(+), 7 deletions(-) diff --git a/src/blockchain_utilities/blockchain_prune_known_spent_data.cpp b/src/blockchain_utilities/blockchain_prune_known_spent_data.cpp index 1a5869e9d..f6136c1ba 100644 --- a/src/blockchain_utilities/blockchain_prune_known_spent_data.cpp +++ b/src/blockchain_utilities/blockchain_prune_known_spent_data.cpp @@ -175,11 +175,6 @@ int main(int argc, char* argv[]) } const std::string input = command_line::get_arg(vm, arg_input); - if (input.empty()) - { - LOG_PRINT_L0("No input given"); - return 1; - } LOG_PRINT_L0("Initializing source blockchain (BlockchainDB)"); std::unique_ptr core_storage; @@ -210,8 +205,50 @@ int main(int argc, char* argv[]) CHECK_AND_ASSERT_MES(r, 1, "Failed to initialize source blockchain storage"); LOG_PRINT_L0("Source blockchain storage initialized OK"); - LOG_PRINT_L0("Loading known spent data..."); - const std::map known_spent_outputs = load_outputs(input); + std::map known_spent_outputs; + if (input.empty()) + { + std::map> outputs; + + LOG_PRINT_L0("Scanning for known spent data..."); + db->for_all_transactions([&](const crypto::hash &txid, const cryptonote::transaction &tx){ + const bool miner_tx = tx.vin.size() == 1 && tx.vin[0].type() == typeid(txin_gen); + for (const auto &in: tx.vin) + { + if (in.type() != typeid(txin_to_key)) + continue; + const auto &txin = boost::get(in); + if (txin.amount == 0) + continue; + + outputs[txin.amount].second++; + } + + for (const auto &out: tx.vout) + { + uint64_t amount = out.amount; + if (miner_tx && tx.version >= 2) + amount = 0; + if (amount == 0) + continue; + if (out.target.type() != typeid(txout_to_key)) + continue; + + outputs[amount].first++; + } + return true; + }, true); + + for (const auto &i: outputs) + { + known_spent_outputs[i.first] = i.second.second; + } + } + else + { + LOG_PRINT_L0("Loading known spent data..."); + known_spent_outputs = load_outputs(input); + } LOG_PRINT_L0("Pruning known spent data..."); From dc1c12528d5cf86759993614e1fdd7d7bf04ff15 Mon Sep 17 00:00:00 2001 From: Jason Wong Date: Sun, 25 Nov 2018 22:08:07 +0100 Subject: [PATCH 11/25] add command pop_blocks add new public method to Blockchain and update according to code review update after review: better lock/unlock, try catch and coding style --- src/cryptonote_core/blockchain.cpp | 32 +++++++++++++++++++++++++ src/cryptonote_core/blockchain.h | 7 ++++++ src/daemon/command_parser_executor.cpp | 25 +++++++++++++++++++ src/daemon/command_parser_executor.h | 2 ++ src/daemon/command_server.cpp | 6 +++++ src/daemon/rpc_command_executor.cpp | 27 +++++++++++++++++++++ src/daemon/rpc_command_executor.h | 2 ++ src/rpc/core_rpc_server.cpp | 12 ++++++++++ src/rpc/core_rpc_server.h | 2 ++ src/rpc/core_rpc_server_commands_defs.h | 23 ++++++++++++++++++ 10 files changed, 138 insertions(+) diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index bfc5dbbe0..009c5e2ed 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -576,6 +576,38 @@ bool Blockchain::deinit() return true; } //------------------------------------------------------------------ +// This function removes blocks from the top of blockchain. +// It starts a batch and calls private method pop_block_from_blockchain(). +void Blockchain::pop_blocks(uint64_t nblocks) +{ + uint64_t i; + CRITICAL_REGION_LOCAL(m_tx_pool); + CRITICAL_REGION_LOCAL1(m_blockchain_lock); + + while (!m_db->batch_start()) + { + m_blockchain_lock.unlock(); + m_tx_pool.unlock(); + epee::misc_utils::sleep_no_w(1000); + m_tx_pool.lock(); + m_blockchain_lock.lock(); + } + + try + { + for (i=0; i < nblocks; ++i) + { + pop_block_from_blockchain(); + } + } + catch (const std::exception& e) + { + LOG_ERROR("Error when popping blocks, only " << i << " blocks are popped: " << e.what()); + } + + m_db->batch_stop(); +} +//------------------------------------------------------------------ // This function tells BlockchainDB to remove the top block from the // blockchain and then returns all transactions (except the miner tx, of course) // from it to the tx_pool diff --git a/src/cryptonote_core/blockchain.h b/src/cryptonote_core/blockchain.h index dfe833fb4..f1e366c9e 100644 --- a/src/cryptonote_core/blockchain.h +++ b/src/cryptonote_core/blockchain.h @@ -967,6 +967,13 @@ namespace cryptonote */ std::vector get_last_block_timestamps(unsigned int blocks) const; + /** + * @brief removes blocks from the top of the blockchain + * + * @param nblocks number of blocks to be removed + */ + void pop_blocks(uint64_t nblocks); + private: // TODO: evaluate whether or not each of these typedefs are left over from blockchain_storage diff --git a/src/daemon/command_parser_executor.cpp b/src/daemon/command_parser_executor.cpp index 1638cf505..853cde9c3 100644 --- a/src/daemon/command_parser_executor.cpp +++ b/src/daemon/command_parser_executor.cpp @@ -674,6 +674,31 @@ bool t_command_parser_executor::sync_info(const std::vector& args) return m_executor.sync_info(); } +bool t_command_parser_executor::pop_blocks(const std::vector& args) +{ + if (args.size() != 1) + { + std::cout << "Exactly one parameter is needed" << std::endl; + return false; + } + + try + { + uint64_t nblocks = boost::lexical_cast(args[0]); + if (nblocks < 1) + { + std::cout << "number of blocks must be greater than 0" << std::endl; + return false; + } + return m_executor.pop_blocks(nblocks); + } + catch (const boost::bad_lexical_cast&) + { + std::cout << "number of blocks must be a number greater than 0" << std::endl; + } + return false; +} + bool t_command_parser_executor::version(const std::vector& args) { std::cout << "Monero '" << MONERO_RELEASE_NAME << "' (v" << MONERO_VERSION_FULL << ")" << std::endl; diff --git a/src/daemon/command_parser_executor.h b/src/daemon/command_parser_executor.h index a70070171..e2844e8b7 100644 --- a/src/daemon/command_parser_executor.h +++ b/src/daemon/command_parser_executor.h @@ -139,6 +139,8 @@ public: bool sync_info(const std::vector& args); + bool pop_blocks(const std::vector& args); + bool version(const std::vector& args); }; diff --git a/src/daemon/command_server.cpp b/src/daemon/command_server.cpp index 35504f2c9..527ed2cf1 100644 --- a/src/daemon/command_server.cpp +++ b/src/daemon/command_server.cpp @@ -281,6 +281,12 @@ t_command_server::t_command_server( , std::bind(&t_command_parser_executor::sync_info, &m_parser, p::_1) , "Print information about the blockchain sync state." ); + m_command_lookup.set_handler( + "pop_blocks" + , std::bind(&t_command_parser_executor::pop_blocks, &m_parser, p::_1) + , "pop_blocks " + , "Remove blocks from end of blockchain" + ); m_command_lookup.set_handler( "version" , std::bind(&t_command_parser_executor::version, &m_parser, p::_1) diff --git a/src/daemon/rpc_command_executor.cpp b/src/daemon/rpc_command_executor.cpp index 5ae9851a7..015e1e1f9 100644 --- a/src/daemon/rpc_command_executor.cpp +++ b/src/daemon/rpc_command_executor.cpp @@ -1967,4 +1967,31 @@ bool t_rpc_command_executor::sync_info() return true; } +bool t_rpc_command_executor::pop_blocks(uint64_t num_blocks) +{ + cryptonote::COMMAND_RPC_POP_BLOCKS::request req; + cryptonote::COMMAND_RPC_POP_BLOCKS::response res; + std::string fail_message = "pop_blocks failed"; + + req.nblocks = num_blocks; + if (m_is_rpc) + { + if (!m_rpc_client->rpc_request(req, res, "/pop_blocks", fail_message.c_str())) + { + return true; + } + } + else + { + if (!m_rpc_server->on_pop_blocks(req, res) || res.status != CORE_RPC_STATUS_OK) + { + tools::fail_msg_writer() << make_error(fail_message, res.status); + return true; + } + } + tools::success_msg_writer() << "new height: " << res.height; + + return true; +} + }// namespace daemonize diff --git a/src/daemon/rpc_command_executor.h b/src/daemon/rpc_command_executor.h index 9e6010c5b..592584a5f 100644 --- a/src/daemon/rpc_command_executor.h +++ b/src/daemon/rpc_command_executor.h @@ -152,6 +152,8 @@ public: bool relay_tx(const std::string &txid); bool sync_info(); + + bool pop_blocks(uint64_t num_blocks); }; } // namespace daemonize diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index f029d1d5a..2fb70c995 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -2021,6 +2021,18 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ + bool core_rpc_server::on_pop_blocks(const COMMAND_RPC_POP_BLOCKS::request& req, COMMAND_RPC_POP_BLOCKS::response& res) + { + PERF_TIMER(on_pop_blocks); + + m_core.get_blockchain_storage().pop_blocks(req.nblocks); + + res.height = m_core.get_current_blockchain_height(); + res.status = CORE_RPC_STATUS_OK; + + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ bool core_rpc_server::on_relay_tx(const COMMAND_RPC_RELAY_TX::request& req, COMMAND_RPC_RELAY_TX::response& res, epee::json_rpc::error& error_resp) { PERF_TIMER(on_relay_tx); diff --git a/src/rpc/core_rpc_server.h b/src/rpc/core_rpc_server.h index 8ada0af15..9edbc71d7 100644 --- a/src/rpc/core_rpc_server.h +++ b/src/rpc/core_rpc_server.h @@ -118,6 +118,7 @@ namespace cryptonote MAP_URI_AUTO_JON2("/get_outs", on_get_outs, COMMAND_RPC_GET_OUTPUTS) MAP_URI_AUTO_JON2_IF("/update", on_update, COMMAND_RPC_UPDATE, !m_restricted) MAP_URI_AUTO_BIN2("/get_output_distribution.bin", on_get_output_distribution_bin, COMMAND_RPC_GET_OUTPUT_DISTRIBUTION) + MAP_URI_AUTO_JON2_IF("/pop_blocks", on_pop_blocks, COMMAND_RPC_POP_BLOCKS, !m_restricted) BEGIN_JSON_RPC_MAP("/json_rpc") MAP_JON_RPC("get_block_count", on_getblockcount, COMMAND_RPC_GETBLOCKCOUNT) MAP_JON_RPC("getblockcount", on_getblockcount, COMMAND_RPC_GETBLOCKCOUNT) @@ -189,6 +190,7 @@ namespace cryptonote bool on_stop_save_graph(const COMMAND_RPC_STOP_SAVE_GRAPH::request& req, COMMAND_RPC_STOP_SAVE_GRAPH::response& res); bool on_update(const COMMAND_RPC_UPDATE::request& req, COMMAND_RPC_UPDATE::response& res); bool on_get_output_distribution_bin(const COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::request& req, COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::response& res); + bool on_pop_blocks(const COMMAND_RPC_POP_BLOCKS::request& req, COMMAND_RPC_POP_BLOCKS::response& res); //json_rpc bool on_getblockcount(const COMMAND_RPC_GETBLOCKCOUNT::request& req, COMMAND_RPC_GETBLOCKCOUNT::response& res); diff --git a/src/rpc/core_rpc_server_commands_defs.h b/src/rpc/core_rpc_server_commands_defs.h index ce0be9c41..c5dbdcbc4 100644 --- a/src/rpc/core_rpc_server_commands_defs.h +++ b/src/rpc/core_rpc_server_commands_defs.h @@ -2326,4 +2326,27 @@ namespace cryptonote }; }; + struct COMMAND_RPC_POP_BLOCKS + { + struct request + { + uint64_t nblocks; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(nblocks); + END_KV_SERIALIZE_MAP() + }; + + struct response + { + std::string status; + uint64_t height; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(status) + KV_SERIALIZE(height) + END_KV_SERIALIZE_MAP() + }; + }; + } From 318cc78457136ec778d6dff43b36f9447d95a43d Mon Sep 17 00:00:00 2001 From: Dusan Klinec Date: Sun, 11 Nov 2018 20:07:25 +0100 Subject: [PATCH 12/25] device/trezor: passphrase entry on host - simple device callback object added. Device can request passphrase/PIN entry via the callback or notify user some action is required - callback is routed to wallet2, which routes the callback to i_wallet_callback so CLI or GUI wallets can support passphrase entry for HW tokens - wallet: device open needs wallet callback first - passphrase protected device needs wallet callback so user can enter passphrase --- src/device/device.hpp | 9 ++++ src/device_trezor/device_trezor_base.cpp | 6 +-- src/device_trezor/device_trezor_base.hpp | 19 +++------ src/simplewallet/simplewallet.cpp | 37 ++++++++++++++++- src/simplewallet/simplewallet.h | 3 ++ src/wallet/wallet2.cpp | 52 ++++++++++++++++++++++-- src/wallet/wallet2.h | 22 ++++++++++ src/wallet/wallet_errors.h | 8 ++++ 8 files changed, 134 insertions(+), 22 deletions(-) diff --git a/src/device/device.hpp b/src/device/device.hpp index dd9ad4332..3e782d21a 100644 --- a/src/device/device.hpp +++ b/src/device/device.hpp @@ -80,6 +80,14 @@ namespace hw { return false; } + class i_device_callback { + public: + virtual void on_button_request() {} + virtual void on_pin_request(epee::wipeable_string & pin) {} + virtual void on_passphrase_request(bool on_device, epee::wipeable_string & passphrase) {} + virtual ~i_device_callback() = default; + }; + class device { protected: std::string name; @@ -129,6 +137,7 @@ namespace hw { virtual device_type get_type() const = 0; virtual device_protocol_t device_protocol() const { return PROTOCOL_DEFAULT; }; + virtual void set_callback(i_device_callback * callback) {}; /* ======================================================================= */ /* LOCKER */ diff --git a/src/device_trezor/device_trezor_base.cpp b/src/device_trezor/device_trezor_base.cpp index 38c20c30b..dcfc9cb67 100644 --- a/src/device_trezor/device_trezor_base.cpp +++ b/src/device_trezor/device_trezor_base.cpp @@ -39,7 +39,7 @@ namespace trezor { const uint32_t device_trezor_base::DEFAULT_BIP44_PATH[] = {0x8000002c, 0x80000080, 0x80000000}; - device_trezor_base::device_trezor_base() { + device_trezor_base::device_trezor_base(): m_callback(nullptr) { } @@ -332,10 +332,6 @@ namespace trezor { MDEBUG("on_passhprase_state_request"); CHECK_AND_ASSERT_THROW_MES(msg, "Empty message"); - if (m_callback){ - m_callback->on_passphrase_state_request(msg->state()); - } - messages::common::PassphraseStateAck m; resp = call_raw(&m); } diff --git a/src/device_trezor/device_trezor_base.hpp b/src/device_trezor/device_trezor_base.hpp index 88d419494..a4e92bf78 100644 --- a/src/device_trezor/device_trezor_base.hpp +++ b/src/device_trezor/device_trezor_base.hpp @@ -57,17 +57,6 @@ namespace trezor { #ifdef WITH_DEVICE_TREZOR class device_trezor_base; - /** - * Trezor device callbacks - */ - class trezor_callback { - public: - virtual void on_button_request() {}; - virtual void on_pin_request(epee::wipeable_string & pin) {}; - virtual void on_passphrase_request(bool on_device, epee::wipeable_string & passphrase) {}; - virtual void on_passphrase_state_request(const std::string & state) {}; - }; - /** * TREZOR device template with basic functions */ @@ -79,7 +68,7 @@ namespace trezor { mutable boost::mutex command_locker; std::shared_ptr m_transport; - std::shared_ptr m_callback; + i_device_callback * m_callback; std::string full_name; @@ -218,7 +207,11 @@ namespace trezor { return m_transport; } - std::shared_ptr getCallback(){ + void set_callback(i_device_callback * callback) override { + m_callback = callback; + } + + i_device_callback * get_callback(){ return m_callback; } diff --git a/src/simplewallet/simplewallet.cpp b/src/simplewallet/simplewallet.cpp index d9fd0c13e..6d19235c5 100644 --- a/src/simplewallet/simplewallet.cpp +++ b/src/simplewallet/simplewallet.cpp @@ -3786,6 +3786,7 @@ boost::optional simple_wallet::new_wallet(const boost::pr { auto rc = tools::wallet2::make_new(vm, false, password_prompter); m_wallet = std::move(rc.first); + m_wallet->callback(this); if (!m_wallet) { return {}; @@ -3893,7 +3894,7 @@ bool simple_wallet::open_wallet(const boost::program_options::variables_map& vm) epee::wipeable_string password; try { - auto rc = tools::wallet2::make_from_file(vm, false, m_wallet_file, password_prompter); + auto rc = tools::wallet2::make_from_file(vm, false, "", password_prompter); m_wallet = std::move(rc.first); password = std::move(std::move(rc.second).password()); if (!m_wallet) @@ -3901,6 +3902,8 @@ bool simple_wallet::open_wallet(const boost::program_options::variables_map& vm) return false; } + m_wallet->callback(this); + m_wallet->load(m_wallet_file, password); std::string prefix; bool ready; uint32_t threshold, total; @@ -4304,6 +4307,38 @@ boost::optional simple_wallet::on_get_password(const char return pwd_container->password(); } //---------------------------------------------------------------------------------------------------- +void simple_wallet::on_button_request() +{ + message_writer(console_color_white, false) << tr("Device requires attention"); +} +//---------------------------------------------------------------------------------------------------- +void simple_wallet::on_pin_request(epee::wipeable_string & pin) +{ +#ifdef HAVE_READLINE + rdln::suspend_readline pause_readline; +#endif + std::string msg = tr("Enter device PIN"); + auto pwd_container = tools::password_container::prompt(false, msg.c_str()); + THROW_WALLET_EXCEPTION_IF(!pwd_container, tools::error::password_entry_failed, tr("Failed to read device PIN")); + pin = pwd_container->password(); +} +//---------------------------------------------------------------------------------------------------- +void simple_wallet::on_passphrase_request(bool on_device, epee::wipeable_string & passphrase) +{ + if (on_device){ + message_writer(console_color_white, true) << tr("Please enter the device passphrase on the device"); + return; + } + +#ifdef HAVE_READLINE + rdln::suspend_readline pause_readline; +#endif + std::string msg = tr("Enter device passphrase"); + auto pwd_container = tools::password_container::prompt(false, msg.c_str()); + THROW_WALLET_EXCEPTION_IF(!pwd_container, tools::error::password_entry_failed, tr("Failed to read device passphrase")); + passphrase = pwd_container->password(); +} +//---------------------------------------------------------------------------------------------------- bool simple_wallet::refresh_main(uint64_t start_height, enum ResetType reset, bool is_init) { if (!try_connect_to_daemon(is_init)) diff --git a/src/simplewallet/simplewallet.h b/src/simplewallet/simplewallet.h index 421afbeda..7e2edba7f 100644 --- a/src/simplewallet/simplewallet.h +++ b/src/simplewallet/simplewallet.h @@ -287,6 +287,9 @@ namespace cryptonote virtual void on_money_spent(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& in_tx, uint64_t amount, const cryptonote::transaction& spend_tx, const cryptonote::subaddress_index& subaddr_index); virtual void on_skip_transaction(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& tx); virtual boost::optional on_get_password(const char *reason); + virtual void on_button_request(); + virtual void on_pin_request(epee::wipeable_string & pin); + virtual void on_passphrase_request(bool on_device, epee::wipeable_string & passphrase); //---------------------------------------------------------- friend class refresh_progress_reporter_t; diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index 49aef4350..ae6ed0937 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -818,6 +818,24 @@ wallet_keys_unlocker::~wallet_keys_unlocker() w.encrypt_keys(key); } +void wallet_device_callback::on_button_request() +{ + if (wallet) + wallet->on_button_request(); +} + +void wallet_device_callback::on_pin_request(epee::wipeable_string & pin) +{ + if (wallet) + wallet->on_pin_request(pin); +} + +void wallet_device_callback::on_passphrase_request(bool on_device, epee::wipeable_string & passphrase) +{ + if (wallet) + wallet->on_passphrase_request(on_device, passphrase); +} + wallet2::wallet2(network_type nettype, uint64_t kdf_rounds, bool unattended): m_multisig_rescan_info(NULL), m_multisig_rescan_k(NULL), @@ -929,7 +947,7 @@ std::pair, password_container> wallet2::make_from_file( return {nullptr, password_container{}}; } auto wallet = make_basic(vm, unattended, opts, password_prompter); - if (wallet) + if (wallet && !wallet_file.empty()) { wallet->load(wallet_file, pwd->password()); } @@ -1071,15 +1089,16 @@ bool wallet2::reconnect_device() hw::device &hwdev = lookup_device(m_device_name); hwdev.set_name(m_device_name); hwdev.set_network_type(m_nettype); + hwdev.set_callback(get_device_callback()); r = hwdev.init(); if (!r){ - LOG_PRINT_L2("Could not init device"); + MERROR("Could not init device"); return false; } r = hwdev.connect(); if (!r){ - LOG_PRINT_L2("Could not connect to the device"); + MERROR("Could not connect to the device"); return false; } @@ -3441,6 +3460,7 @@ bool wallet2::load_keys(const std::string& keys_file_name, const epee::wipeable_ hw::device &hwdev = lookup_device(m_device_name); THROW_WALLET_EXCEPTION_IF(!hwdev.set_name(m_device_name), error::wallet_internal_error, "Could not set device name " + m_device_name); hwdev.set_network_type(m_nettype); + hwdev.set_callback(get_device_callback()); THROW_WALLET_EXCEPTION_IF(!hwdev.init(), error::wallet_internal_error, "Could not initialize the device " + m_device_name); THROW_WALLET_EXCEPTION_IF(!hwdev.connect(), error::wallet_internal_error, "Could not connect to the device " + m_device_name); m_account.set_device(hwdev); @@ -3947,6 +3967,7 @@ void wallet2::restore(const std::string& wallet_, const epee::wipeable_string& p auto &hwdev = lookup_device(device_name); hwdev.set_name(device_name); hwdev.set_network_type(m_nettype); + hwdev.set_callback(get_device_callback()); m_account.create_from_device(hwdev); m_key_device_type = m_account.get_device().get_type(); @@ -11940,4 +11961,29 @@ uint64_t wallet2::get_segregation_fork_height() const void wallet2::generate_genesis(cryptonote::block& b) const { cryptonote::generate_genesis_block(b, get_config(m_nettype).GENESIS_TX, get_config(m_nettype).GENESIS_NONCE); } +//---------------------------------------------------------------------------------------------------- +wallet_device_callback * wallet2::get_device_callback() +{ + if (!m_device_callback){ + m_device_callback.reset(new wallet_device_callback(this)); + } + return m_device_callback.get(); +}//---------------------------------------------------------------------------------------------------- +void wallet2::on_button_request() +{ + if (0 != m_callback) + m_callback->on_button_request(); +} +//---------------------------------------------------------------------------------------------------- +void wallet2::on_pin_request(epee::wipeable_string & pin) +{ + if (0 != m_callback) + m_callback->on_pin_request(pin); +} +//---------------------------------------------------------------------------------------------------- +void wallet2::on_passphrase_request(bool on_device, epee::wipeable_string & passphrase) +{ + if (0 != m_callback) + m_callback->on_passphrase_request(on_device, passphrase); +} } diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h index eb0763131..c07054177 100644 --- a/src/wallet/wallet2.h +++ b/src/wallet/wallet2.h @@ -98,11 +98,26 @@ namespace tools virtual void on_lw_money_received(uint64_t height, const crypto::hash &txid, uint64_t amount) {} virtual void on_lw_unconfirmed_money_received(uint64_t height, const crypto::hash &txid, uint64_t amount) {} virtual void on_lw_money_spent(uint64_t height, const crypto::hash &txid, uint64_t amount) {} + // Device callbacks + virtual void on_button_request() {} + virtual void on_pin_request(epee::wipeable_string & pin) {} + virtual void on_passphrase_request(bool on_device, epee::wipeable_string & passphrase) {} // Common callbacks virtual void on_pool_tx_removed(const crypto::hash &txid) {} virtual ~i_wallet2_callback() {} }; + class wallet_device_callback : public hw::i_device_callback + { + public: + wallet_device_callback(wallet2 * wallet): wallet(wallet) {}; + void on_button_request() override; + void on_pin_request(epee::wipeable_string & pin) override; + void on_passphrase_request(bool on_device, epee::wipeable_string & passphrase) override; + private: + wallet2 * wallet; + }; + struct tx_dust_policy { uint64_t dust_threshold; @@ -154,6 +169,7 @@ namespace tools { friend class ::Serialization_portability_wallet_Test; friend class wallet_keys_unlocker; + friend class wallet_device_callback; public: static constexpr const std::chrono::seconds rpc_timeout = std::chrono::minutes(3) + std::chrono::seconds(30); @@ -1285,6 +1301,11 @@ namespace tools void setup_new_blockchain(); void create_keys_file(const std::string &wallet_, bool watch_only, const epee::wipeable_string &password, bool create_address_file); + wallet_device_callback * get_device_callback(); + void on_button_request(); + void on_pin_request(epee::wipeable_string & pin); + void on_passphrase_request(bool on_device, epee::wipeable_string & passphrase); + cryptonote::account_base m_account; boost::optional m_daemon_login; std::string m_daemon_address; @@ -1396,6 +1417,7 @@ namespace tools bool m_devices_registered; std::shared_ptr m_tx_notify; + std::unique_ptr m_device_callback; }; } BOOST_CLASS_VERSION(tools::wallet2, 26) diff --git a/src/wallet/wallet_errors.h b/src/wallet/wallet_errors.h index b3141985d..e2caee5d2 100644 --- a/src/wallet/wallet_errors.h +++ b/src/wallet/wallet_errors.h @@ -219,6 +219,14 @@ namespace tools } }; //---------------------------------------------------------------------------------------------------- + struct password_entry_failed : public wallet_runtime_error + { + explicit password_entry_failed(std::string&& loc, const std::string &msg = "Password entry failed") + : wallet_runtime_error(std::move(loc), msg) + { + } + }; + //---------------------------------------------------------------------------------------------------- const char* const file_error_messages[] = { "file already exists", "file not found", From ac665418f0a7ef92ce9b27ecdd88914482f19ee5 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Fri, 30 Nov 2018 13:33:29 +0000 Subject: [PATCH 13/25] ringct: fix dummy bulletproofs on ledger in fake mode Ledger does some basic checks on them --- src/ringct/rctSigs.cpp | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/src/ringct/rctSigs.cpp b/src/ringct/rctSigs.cpp index dccd18867..5497b927e 100644 --- a/src/ringct/rctSigs.cpp +++ b/src/ringct/rctSigs.cpp @@ -46,13 +46,34 @@ using namespace std; namespace { - rct::Bulletproof make_dummy_bulletproof(size_t n_outs) + rct::Bulletproof make_dummy_bulletproof(const std::vector &outamounts, rct::keyV &C, rct::keyV &masks) { + const size_t n_outs = outamounts.size(); const rct::key I = rct::identity(); size_t nrl = 0; while ((1u << nrl) < n_outs) ++nrl; nrl += 6; + + C.resize(n_outs); + masks.resize(n_outs); + for (size_t i = 0; i < n_outs; ++i) + { + masks[i] = I; + rct::key sv8, sv; + sv = rct::zero(); + sv.bytes[0] = outamounts[i] & 255; + sv.bytes[1] = (outamounts[i] >> 8) & 255; + sv.bytes[2] = (outamounts[i] >> 16) & 255; + sv.bytes[3] = (outamounts[i] >> 24) & 255; + sv.bytes[4] = (outamounts[i] >> 32) & 255; + sv.bytes[5] = (outamounts[i] >> 40) & 255; + sv.bytes[6] = (outamounts[i] >> 48) & 255; + sv.bytes[7] = (outamounts[i] >> 56) & 255; + sc_mul(sv8.bytes, sv.bytes, rct::INV_EIGHT.bytes); + rct::addKeys2(C[i], rct::INV_EIGHT, sv8, rct::H); + } + return rct::Bulletproof{rct::keyV(n_outs, I), I, I, I, I, I, I, rct::keyV(nrl, I), rct::keyV(nrl, I), I, I, I}; } } @@ -769,9 +790,7 @@ namespace rct { if (hwdev.get_mode() == hw::device::TRANSACTION_CREATE_FAKE) { // use a fake bulletproof for speed - rv.p.bulletproofs.push_back(make_dummy_bulletproof(outamounts.size())); - C = rct::keyV(outamounts.size(), I); - masks = rct::keyV(outamounts.size(), I); + rv.p.bulletproofs.push_back(make_dummy_bulletproof(outamounts, C, masks)); } else { @@ -799,9 +818,7 @@ namespace rct { if (hwdev.get_mode() == hw::device::TRANSACTION_CREATE_FAKE) { // use a fake bulletproof for speed - rv.p.bulletproofs.push_back(make_dummy_bulletproof(batch_amounts.size())); - C = rct::keyV(batch_amounts.size(), I); - masks = rct::keyV(batch_amounts.size(), I); + rv.p.bulletproofs.push_back(make_dummy_bulletproof(batch_amounts, C, masks)); } else { From aba9a9c27777cd0649e9946566d87030f78d8bec Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Fri, 30 Nov 2018 14:53:38 +0000 Subject: [PATCH 14/25] daemon: stop miner before we bring the whole thing down This avoids the miner erroring out trying to submit blocks to a core that's already shut down (and avoids pegging the CPU while we're busy shutting down). --- src/cryptonote_core/cryptonote_core.cpp | 1 + src/daemon/daemon.cpp | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/cryptonote_core/cryptonote_core.cpp b/src/cryptonote_core/cryptonote_core.cpp index 10ab3fe65..dd79c0e67 100644 --- a/src/cryptonote_core/cryptonote_core.cpp +++ b/src/cryptonote_core/cryptonote_core.cpp @@ -247,6 +247,7 @@ namespace cryptonote //----------------------------------------------------------------------------------- void core::stop() { + m_miner.stop(); m_blockchain_storage.cancel(); tools::download_async_handle handle; diff --git a/src/daemon/daemon.cpp b/src/daemon/daemon.cpp index 49d6d49cf..85b8780f0 100644 --- a/src/daemon/daemon.cpp +++ b/src/daemon/daemon.cpp @@ -198,7 +198,6 @@ bool t_daemon::run(bool interactive) for(auto& rpc : mp_internals->rpcs) rpc->stop(); - mp_internals->core.get().get_miner().stop(); MGINFO("Node stopped."); return true; } @@ -220,7 +219,6 @@ void t_daemon::stop() { throw std::runtime_error{"Can't stop stopped daemon"}; } - mp_internals->core.get().get_miner().stop(); mp_internals->p2p.stop(); for(auto& rpc : mp_internals->rpcs) rpc->stop(); From 0c5dd3161b0f8c962a77cd27cff58043d85e509c Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Fri, 30 Nov 2018 14:54:46 +0000 Subject: [PATCH 15/25] cryptonote: add a set_null for transaction_prefix Since it's all inline, I suspect the compiler will merge the duplicate stores anyway. --- src/cryptonote_basic/cryptonote_basic.h | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/cryptonote_basic/cryptonote_basic.h b/src/cryptonote_basic/cryptonote_basic.h index b0eabb0aa..645e99f91 100644 --- a/src/cryptonote_basic/cryptonote_basic.h +++ b/src/cryptonote_basic/cryptonote_basic.h @@ -175,7 +175,15 @@ namespace cryptonote END_SERIALIZE() public: - transaction_prefix(){} + transaction_prefix(){ set_null(); } + void set_null() + { + version = 1; + unlock_time = 0; + vin.clear(); + vout.clear(); + extra.clear(); + } }; class transaction: public transaction_prefix @@ -302,17 +310,12 @@ namespace cryptonote inline transaction::~transaction() { - //set_null(); } inline void transaction::set_null() { - version = 1; - unlock_time = 0; - vin.clear(); - vout.clear(); - extra.clear(); + transaction_prefix::set_null(); signatures.clear(); rct_signatures.type = rct::RCTTypeNull; set_hash_valid(false); From 4d71d463739fa20d21588266b70f58609d79cb2e Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Fri, 30 Nov 2018 16:52:50 +0000 Subject: [PATCH 16/25] mlocker: remove early page size log It comes before the logger is initialized, so gets displayed even though it should not be by default, and apparenly comes too early for (some versions of) Android, where it crashes. --- contrib/epee/src/mlocker.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/contrib/epee/src/mlocker.cpp b/contrib/epee/src/mlocker.cpp index c3262e8f4..4c48cbb58 100644 --- a/contrib/epee/src/mlocker.cpp +++ b/contrib/epee/src/mlocker.cpp @@ -47,7 +47,6 @@ static size_t query_page_size() MERROR("Failed to determine page size"); return 0; } - MINFO("Page size: " << ret); return ret; #else #warning Missing query_page_size implementation From 9b69a0ae019b366ad9511263680d6d08f50d6f25 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Fri, 30 Nov 2018 17:55:16 +0000 Subject: [PATCH 17/25] daemon: print monero version at startup when calling a detached daemon So people who want a timstamp get a timestamp --- src/daemon/main.cpp | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/daemon/main.cpp b/src/daemon/main.cpp index f483ba6c9..35017d9ef 100644 --- a/src/daemon/main.cpp +++ b/src/daemon/main.cpp @@ -216,6 +216,16 @@ int main(int argc, char const * argv[]) // after logs initialized tools::create_directories_if_necessary(data_dir.string()); +#ifdef STACK_TRACE + tools::set_stack_trace_log(log_file_path.filename().string()); +#endif // STACK_TRACE + + if (!command_line::is_arg_defaulted(vm, daemon_args::arg_max_concurrency)) + tools::set_max_concurrency(command_line::get_arg(vm, daemon_args::arg_max_concurrency)); + + // logging is now set up + MGINFO("Monero '" << MONERO_RELEASE_NAME << "' (v" << MONERO_VERSION_FULL << ")"); + // If there are positional options, we're running a daemon command { auto command = command_line::get_arg(vm, daemon_args::arg_command); @@ -276,16 +286,6 @@ int main(int argc, char const * argv[]) } } -#ifdef STACK_TRACE - tools::set_stack_trace_log(log_file_path.filename().string()); -#endif // STACK_TRACE - - if (!command_line::is_arg_defaulted(vm, daemon_args::arg_max_concurrency)) - tools::set_max_concurrency(command_line::get_arg(vm, daemon_args::arg_max_concurrency)); - - // logging is now set up - MGINFO("Monero '" << MONERO_RELEASE_NAME << "' (v" << MONERO_VERSION_FULL << ")"); - MINFO("Moving from main() into the daemonize now."); return daemonizer::daemonize(argc, argv, daemonize::t_executor{}, vm) ? 0 : 1; From ab783b1700fc14ed04bf14fc9df2c6c7b6e90a41 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Fri, 30 Nov 2018 17:58:58 +0000 Subject: [PATCH 18/25] easylogging++: ensure logger is initialized before main --- external/easylogging++/easylogging++.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/external/easylogging++/easylogging++.cc b/external/easylogging++/easylogging++.cc index d57f3f3a0..00b8b830b 100644 --- a/external/easylogging++/easylogging++.cc +++ b/external/easylogging++/easylogging++.cc @@ -2202,6 +2202,7 @@ el::base::type::StoragePointer el::base::Storage::getELPP() { return getresetELPP(false); } +static struct EnsureELPP { EnsureELPP() { el::base::Storage::getELPP(); } } ensureELPP; #if ELPP_ASYNC_LOGGING Storage::Storage(const LogBuilderPtr& defaultLogBuilder, base::IWorker* asyncDispatchWorker) : #else From 4f74a31ecdaac3d338985fecc2de0882675745d2 Mon Sep 17 00:00:00 2001 From: Dimitris Apostolou Date: Sat, 24 Nov 2018 15:31:49 +0200 Subject: [PATCH 19/25] http -> https --- contrib/depends/packages/bdb.mk | 2 +- contrib/depends/packages/ldns.mk | 2 +- contrib/depends/packages/libICE.mk | 2 +- contrib/depends/packages/libSM.mk | 2 +- contrib/depends/packages/libusb.mk | 2 +- contrib/depends/packages/native_cdrkit.mk | 2 +- contrib/depends/packages/qt.mk | 2 +- contrib/depends/packages/unbound.mk | 2 +- contrib/depends/packages/unwind.mk | 2 +- contrib/depends/packages/xproto.mk | 2 +- contrib/depends/packages/zlib.mk | 2 +- 11 files changed, 11 insertions(+), 11 deletions(-) diff --git a/contrib/depends/packages/bdb.mk b/contrib/depends/packages/bdb.mk index 6c9876c2c..050a60add 100644 --- a/contrib/depends/packages/bdb.mk +++ b/contrib/depends/packages/bdb.mk @@ -1,6 +1,6 @@ package=bdb $(package)_version=4.8.30 -$(package)_download_path=http://download.oracle.com/berkeley-db +$(package)_download_path=https://download.oracle.com/berkeley-db $(package)_file_name=db-$($(package)_version).NC.tar.gz $(package)_sha256_hash=12edc0df75bf9abd7f82f821795bcee50f42cb2e5f76a6a281b85732798364ef $(package)_build_subdir=build_unix diff --git a/contrib/depends/packages/ldns.mk b/contrib/depends/packages/ldns.mk index a9565a581..0b7c3806a 100644 --- a/contrib/depends/packages/ldns.mk +++ b/contrib/depends/packages/ldns.mk @@ -1,6 +1,6 @@ package=ldns $(package)_version=1.6.17 -$(package)_download_path=http://www.nlnetlabs.nl/downloads/ldns/ +$(package)_download_path=https://www.nlnetlabs.nl/downloads/ldns/ $(package)_file_name=$(package)-$($(package)_version).tar.gz $(package)_sha256_hash=8b88e059452118e8949a2752a55ce59bc71fa5bc414103e17f5b6b06f9bcc8cd $(package)_dependencies=openssl diff --git a/contrib/depends/packages/libICE.mk b/contrib/depends/packages/libICE.mk index fc60323b1..a897d9aed 100644 --- a/contrib/depends/packages/libICE.mk +++ b/contrib/depends/packages/libICE.mk @@ -1,6 +1,6 @@ package=libICE $(package)_version=1.0.9 -$(package)_download_path=http://xorg.freedesktop.org/releases/individual/lib/ +$(package)_download_path=https://xorg.freedesktop.org/releases/individual/lib/ $(package)_file_name=$(package)-$($(package)_version).tar.bz2 $(package)_sha256_hash=8f7032f2c1c64352b5423f6b48a8ebdc339cc63064af34d66a6c9aa79759e202 $(package)_dependencies=xtrans xproto diff --git a/contrib/depends/packages/libSM.mk b/contrib/depends/packages/libSM.mk index 0f9307ca7..83fcd4cdb 100644 --- a/contrib/depends/packages/libSM.mk +++ b/contrib/depends/packages/libSM.mk @@ -1,6 +1,6 @@ package=libSM $(package)_version=1.2.2 -$(package)_download_path=http://xorg.freedesktop.org/releases/individual/lib/ +$(package)_download_path=https://xorg.freedesktop.org/releases/individual/lib/ $(package)_file_name=$(package)-$($(package)_version).tar.bz2 $(package)_sha256_hash=0baca8c9f5d934450a70896c4ad38d06475521255ca63b717a6510fdb6e287bd $(package)_dependencies=xtrans xproto libICE diff --git a/contrib/depends/packages/libusb.mk b/contrib/depends/packages/libusb.mk index e9663ace0..6d60cce26 100644 --- a/contrib/depends/packages/libusb.mk +++ b/contrib/depends/packages/libusb.mk @@ -1,6 +1,6 @@ package=libusb $(package)_version=1.0.22 -$(package)_download_path=http://sourceforge.net/projects/libusb/files/libusb-1.0/libusb-$($(package)_version)/ +$(package)_download_path=https://sourceforge.net/projects/libusb/files/libusb-1.0/libusb-$($(package)_version)/ $(package)_file_name=$(package)-$($(package)_version).tar.bz2 $(package)_sha256_hash=75aeb9d59a4fdb800d329a545c2e6799f732362193b465ea198f2aa275518157 diff --git a/contrib/depends/packages/native_cdrkit.mk b/contrib/depends/packages/native_cdrkit.mk index cf694edb3..8243458ec 100644 --- a/contrib/depends/packages/native_cdrkit.mk +++ b/contrib/depends/packages/native_cdrkit.mk @@ -1,6 +1,6 @@ package=native_cdrkit $(package)_version=1.1.11 -$(package)_download_path=http://distro.ibiblio.org/fatdog/source/600/c +$(package)_download_path=https://distro.ibiblio.org/fatdog/source/600/c $(package)_file_name=cdrkit-$($(package)_version).tar.bz2 $(package)_sha256_hash=b50d64c214a65b1a79afe3a964c691931a4233e2ba605d793eb85d0ac3652564 $(package)_patches=cdrkit-deterministic.patch diff --git a/contrib/depends/packages/qt.mk b/contrib/depends/packages/qt.mk index 32ca4a84c..bca2926cb 100644 --- a/contrib/depends/packages/qt.mk +++ b/contrib/depends/packages/qt.mk @@ -1,6 +1,6 @@ PACKAGE=qt $(package)_version=5.7.1 -$(package)_download_path=http://download.qt.io/official_releases/qt/5.7/$($(package)_version)/submodules +$(package)_download_path=https://download.qt.io/archive/qt/5.7/5.7.1/submodules $(package)_suffix=opensource-src-$($(package)_version).tar.gz $(package)_file_name=qtbase-$($(package)_suffix) $(package)_sha256_hash=95f83e532d23b3ddbde7973f380ecae1bac13230340557276f75f2e37984e410 diff --git a/contrib/depends/packages/unbound.mk b/contrib/depends/packages/unbound.mk index beeeb54c1..733a7f232 100644 --- a/contrib/depends/packages/unbound.mk +++ b/contrib/depends/packages/unbound.mk @@ -1,6 +1,6 @@ package=unbound $(package)_version=1.6.8 -$(package)_download_path=http://www.unbound.net/downloads/ +$(package)_download_path=https://www.unbound.net/downloads/ $(package)_file_name=$(package)-$($(package)_version).tar.gz $(package)_sha256_hash=e3b428e33f56a45417107448418865fe08d58e0e7fea199b855515f60884dd49 $(package)_dependencies=openssl expat ldns diff --git a/contrib/depends/packages/unwind.mk b/contrib/depends/packages/unwind.mk index e1bcb35b2..543f868a5 100644 --- a/contrib/depends/packages/unwind.mk +++ b/contrib/depends/packages/unwind.mk @@ -1,6 +1,6 @@ package=unwind $(package)_version=1.2 -$(package)_download_path=http://download.savannah.nongnu.org/releases/libunwind +$(package)_download_path=https://download.savannah.nongnu.org/releases/libunwind $(package)_file_name=lib$(package)-$($(package)_version).tar.gz $(package)_sha256_hash=1de38ffbdc88bd694d10081865871cd2bfbb02ad8ef9e1606aee18d65532b992 diff --git a/contrib/depends/packages/xproto.mk b/contrib/depends/packages/xproto.mk index 50a90b268..52fe253c7 100644 --- a/contrib/depends/packages/xproto.mk +++ b/contrib/depends/packages/xproto.mk @@ -1,6 +1,6 @@ package=xproto $(package)_version=7.0.26 -$(package)_download_path=http://xorg.freedesktop.org/releases/individual/proto +$(package)_download_path=https://xorg.freedesktop.org/releases/individual/proto $(package)_file_name=$(package)-$($(package)_version).tar.bz2 $(package)_sha256_hash=636162c1759805a5a0114a369dffdeccb8af8c859ef6e1445f26a4e6e046514f diff --git a/contrib/depends/packages/zlib.mk b/contrib/depends/packages/zlib.mk index 589490800..1600b11a0 100644 --- a/contrib/depends/packages/zlib.mk +++ b/contrib/depends/packages/zlib.mk @@ -1,6 +1,6 @@ package=zlib $(package)_version=1.2.11 -$(package)_download_path=http://www.zlib.net +$(package)_download_path=https://www.zlib.net $(package)_file_name=$(package)-$($(package)_version).tar.gz $(package)_sha256_hash=c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1 From 6456cb415a16aab24694a246309f4c7f0ae5667c Mon Sep 17 00:00:00 2001 From: Tadeas Moravec Date: Sat, 1 Dec 2018 13:03:32 +0000 Subject: [PATCH 20/25] Bulletproof: Initialize members in default construtor. Fixing a build warning on g++ 7.3.0 --- src/ringct/rctTypes.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ringct/rctTypes.h b/src/ringct/rctTypes.h index 18290637b..487ea6f32 100644 --- a/src/ringct/rctTypes.h +++ b/src/ringct/rctTypes.h @@ -187,7 +187,8 @@ namespace rct { rct::keyV L, R; rct::key a, b, t; - Bulletproof() {} + Bulletproof(): + A({}), S({}), T1({}), T2({}), taux({}), mu({}), a({}), b({}), t({}) {} Bulletproof(const rct::key &V, const rct::key &A, const rct::key &S, const rct::key &T1, const rct::key &T2, const rct::key &taux, const rct::key &mu, const rct::keyV &L, const rct::keyV &R, const rct::key &a, const rct::key &b, const rct::key &t): V({V}), A(A), S(S), T1(T1), T2(T2), taux(taux), mu(mu), L(L), R(R), a(a), b(b), t(t) {} Bulletproof(const rct::keyV &V, const rct::key &A, const rct::key &S, const rct::key &T1, const rct::key &T2, const rct::key &taux, const rct::key &mu, const rct::keyV &L, const rct::keyV &R, const rct::key &a, const rct::key &b, const rct::key &t): From d21dad70ddda771bf33a19f9a53c48cc91d8464f Mon Sep 17 00:00:00 2001 From: Dusan Klinec Date: Mon, 12 Nov 2018 00:07:25 +0100 Subject: [PATCH 21/25] device: enable to use multiple independent device wallets - adds a new option `--hw-device-deriv-path` to the simple wallet. Enables to specify wallet derivation path / wallet code (path avoided so it can be misinterpreted as a file path). - devices can use different derivation mechanisms. Trezor uses standard SLIP-10 mechanism with fixed SLIP-44 prefix for Monero - Trezor: when empty, the default derivation mechanism is used with 44'/128'/0'. When entered the derivation path is 44'/128'/PATH. - Trezor: the path is always taken as elements are hardened (1<<31 bit turned on) --- src/device/device.hpp | 1 + src/device_trezor/device_trezor_base.cpp | 39 +++++++++++++++++++++++- src/device_trezor/device_trezor_base.hpp | 10 +++++- src/simplewallet/simplewallet.cpp | 2 ++ src/wallet/wallet2.cpp | 19 ++++++++++++ src/wallet/wallet2.h | 4 +++ 6 files changed, 73 insertions(+), 2 deletions(-) diff --git a/src/device/device.hpp b/src/device/device.hpp index 3e782d21a..399648f01 100644 --- a/src/device/device.hpp +++ b/src/device/device.hpp @@ -138,6 +138,7 @@ namespace hw { virtual device_protocol_t device_protocol() const { return PROTOCOL_DEFAULT; }; virtual void set_callback(i_device_callback * callback) {}; + virtual void set_derivation_path(const std::string &derivation_path) {}; /* ======================================================================= */ /* LOCKER */ diff --git a/src/device_trezor/device_trezor_base.cpp b/src/device_trezor/device_trezor_base.cpp index dcfc9cb67..fddc6082c 100644 --- a/src/device_trezor/device_trezor_base.cpp +++ b/src/device_trezor/device_trezor_base.cpp @@ -28,6 +28,9 @@ // #include "device_trezor_base.hpp" +#include +#include +#include namespace hw { namespace trezor { @@ -36,8 +39,9 @@ namespace trezor { #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "device.trezor" +#define TREZOR_BIP44_HARDENED_ZERO 0x80000000 - const uint32_t device_trezor_base::DEFAULT_BIP44_PATH[] = {0x8000002c, 0x80000080, 0x80000000}; + const uint32_t device_trezor_base::DEFAULT_BIP44_PATH[] = {0x8000002c, 0x80000080}; device_trezor_base::device_trezor_base(): m_callback(nullptr) { @@ -252,6 +256,39 @@ namespace trezor { } } + void device_trezor_base::ensure_derivation_path() noexcept { + if (m_wallet_deriv_path.empty()){ + m_wallet_deriv_path.push_back(TREZOR_BIP44_HARDENED_ZERO); // default 0' + } + } + + void device_trezor_base::set_derivation_path(const std::string &deriv_path){ + this->m_wallet_deriv_path.clear(); + + if (deriv_path.empty() || deriv_path == "-"){ + ensure_derivation_path(); + return; + } + + CHECK_AND_ASSERT_THROW_MES(deriv_path.size() <= 255, "Derivation path is too long"); + + std::vector fields; + boost::split(fields, deriv_path, boost::is_any_of("/")); + CHECK_AND_ASSERT_THROW_MES(fields.size() <= 10, "Derivation path is too long"); + + boost::regex rgx("^([0-9]+)'?$"); + boost::cmatch match; + + this->m_wallet_deriv_path.reserve(fields.size()); + for(const std::string & cur : fields){ + const bool ok = boost::regex_match(cur.c_str(), match, rgx); + CHECK_AND_ASSERT_THROW_MES(ok, "Invalid wallet code: " << deriv_path << ". Invalid path element: " << cur); + CHECK_AND_ASSERT_THROW_MES(match[0].length() > 0, "Invalid wallet code: " << deriv_path << ". Invalid path element: " << cur); + + const unsigned long cidx = std::stoul(match[0].str()) | TREZOR_BIP44_HARDENED_ZERO; + this->m_wallet_deriv_path.push_back((unsigned int)cidx); + } + } /* ======================================================================= */ /* TREZOR PROTOCOL */ diff --git a/src/device_trezor/device_trezor_base.hpp b/src/device_trezor/device_trezor_base.hpp index a4e92bf78..4d205ad7a 100644 --- a/src/device_trezor/device_trezor_base.hpp +++ b/src/device_trezor/device_trezor_base.hpp @@ -71,6 +71,7 @@ namespace trezor { i_device_callback * m_callback; std::string full_name; + std::vector m_wallet_deriv_path; cryptonote::network_type network_type; @@ -81,6 +82,7 @@ namespace trezor { void require_connected(); void call_ping_unsafe(); void test_ping(); + void ensure_derivation_path() noexcept; // Communication methods @@ -176,9 +178,13 @@ namespace trezor { msg->add_address_n(x); } } else { + ensure_derivation_path(); for (unsigned int i : DEFAULT_BIP44_PATH) { msg->add_address_n(i); } + for (unsigned int i : m_wallet_deriv_path) { + msg->add_address_n(i); + } } if (network_type){ @@ -201,7 +207,7 @@ namespace trezor { bool reset(); // Default derivation path for Monero - static const uint32_t DEFAULT_BIP44_PATH[3]; + static const uint32_t DEFAULT_BIP44_PATH[2]; std::shared_ptr getTransport(){ return m_transport; @@ -215,6 +221,8 @@ namespace trezor { return m_callback; } + void set_derivation_path(const std::string &deriv_path) override; + /* ======================================================================= */ /* SETUP/TEARDOWN */ /* ======================================================================= */ diff --git a/src/simplewallet/simplewallet.cpp b/src/simplewallet/simplewallet.cpp index 6d19235c5..07084d15b 100644 --- a/src/simplewallet/simplewallet.cpp +++ b/src/simplewallet/simplewallet.cpp @@ -3804,9 +3804,11 @@ boost::optional simple_wallet::new_wallet(const boost::pr m_wallet->set_refresh_from_block_height(m_restore_height); auto device_desc = tools::wallet2::device_name_option(vm); + auto device_derivation_path = tools::wallet2::device_derivation_path_option(vm); try { bool create_address_file = command_line::get_arg(vm, arg_create_address_file); + m_wallet->device_derivation_path(device_derivation_path); m_wallet->restore(m_wallet_file, std::move(rc.second).password(), device_desc.empty() ? "Ledger" : device_desc, create_address_file); message_writer(console_color_white, true) << tr("Generated new wallet on hw device: ") << m_wallet->get_account().get_public_address_str(m_wallet->nettype()); diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index ae6ed0937..83aae6de3 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -207,6 +207,7 @@ struct options { }; const command_line::arg_descriptor kdf_rounds = {"kdf-rounds", tools::wallet2::tr("Number of rounds for the key derivation function"), 1}; const command_line::arg_descriptor hw_device = {"hw-device", tools::wallet2::tr("HW device to use"), ""}; + const command_line::arg_descriptor hw_device_derivation_path = {"hw-device-deriv-path", tools::wallet2::tr("HW device wallet derivation path (e.g., SLIP-10)"), ""}; const command_line::arg_descriptor tx_notify = { "tx-notify" , "Run a program for each new incoming transaction, '%s' will be replaced by the transaction hash" , "" }; }; @@ -259,6 +260,7 @@ std::unique_ptr make_basic(const boost::program_options::variabl auto daemon_host = command_line::get_arg(vm, opts.daemon_host); auto daemon_port = command_line::get_arg(vm, opts.daemon_port); auto device_name = command_line::get_arg(vm, opts.hw_device); + auto device_derivation_path = command_line::get_arg(vm, opts.hw_device_derivation_path); THROW_WALLET_EXCEPTION_IF(!daemon_address.empty() && !daemon_host.empty() && 0 != daemon_port, tools::error::wallet_internal_error, tools::wallet2::tr("can't specify daemon host or port more than once")); @@ -314,6 +316,7 @@ std::unique_ptr make_basic(const boost::program_options::variabl boost::filesystem::path ringdb_path = command_line::get_arg(vm, opts.shared_ringdb_dir); wallet->set_ring_database(ringdb_path.string()); wallet->device_name(device_name); + wallet->device_derivation_path(device_derivation_path); try { @@ -912,6 +915,11 @@ std::string wallet2::device_name_option(const boost::program_options::variables_ return command_line::get_arg(vm, options().hw_device); } +std::string wallet2::device_derivation_path_option(const boost::program_options::variables_map &vm) +{ + return command_line::get_arg(vm, options().hw_device_derivation_path); +} + void wallet2::init_options(boost::program_options::options_description& desc_params) { const options opts{}; @@ -928,6 +936,7 @@ void wallet2::init_options(boost::program_options::options_description& desc_par command_line::add_arg(desc_params, opts.shared_ringdb_dir); command_line::add_arg(desc_params, opts.kdf_rounds); command_line::add_arg(desc_params, opts.hw_device); + command_line::add_arg(desc_params, opts.hw_device_derivation_path); command_line::add_arg(desc_params, opts.tx_notify); } @@ -1089,6 +1098,7 @@ bool wallet2::reconnect_device() hw::device &hwdev = lookup_device(m_device_name); hwdev.set_name(m_device_name); hwdev.set_network_type(m_nettype); + hwdev.set_derivation_path(m_device_derivation_path); hwdev.set_callback(get_device_callback()); r = hwdev.init(); if (!r){ @@ -3160,6 +3170,9 @@ bool wallet2::store_keys(const std::string& keys_file_name, const epee::wipeable value.SetString(m_device_name.c_str(), m_device_name.size()); json.AddMember("device_name", value, json.GetAllocator()); + value.SetString(m_device_derivation_path.c_str(), m_device_derivation_path.size()); + json.AddMember("device_derivation_path", value, json.GetAllocator()); + // Serialize the JSON object rapidjson::StringBuffer buffer; rapidjson::Writer writer(buffer); @@ -3279,6 +3292,7 @@ bool wallet2::load_keys(const std::string& keys_file_name, const epee::wipeable_ m_subaddress_lookahead_major = SUBADDRESS_LOOKAHEAD_MAJOR; m_subaddress_lookahead_minor = SUBADDRESS_LOOKAHEAD_MINOR; m_device_name = ""; + m_device_derivation_path = ""; m_key_device_type = hw::device::device_type::SOFTWARE; encrypted_secret_keys = false; } @@ -3446,6 +3460,9 @@ bool wallet2::load_keys(const std::string& keys_file_name, const epee::wipeable_ m_device_name = m_key_device_type == hw::device::device_type::LEDGER ? "Ledger" : "default"; } } + + GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, device_derivation_path, std::string, String, false, std::string()); + m_device_derivation_path = field_device_derivation_path; } else { @@ -3460,6 +3477,7 @@ bool wallet2::load_keys(const std::string& keys_file_name, const epee::wipeable_ hw::device &hwdev = lookup_device(m_device_name); THROW_WALLET_EXCEPTION_IF(!hwdev.set_name(m_device_name), error::wallet_internal_error, "Could not set device name " + m_device_name); hwdev.set_network_type(m_nettype); + hwdev.set_derivation_path(m_device_derivation_path); hwdev.set_callback(get_device_callback()); THROW_WALLET_EXCEPTION_IF(!hwdev.init(), error::wallet_internal_error, "Could not initialize the device " + m_device_name); THROW_WALLET_EXCEPTION_IF(!hwdev.connect(), error::wallet_internal_error, "Could not connect to the device " + m_device_name); @@ -3967,6 +3985,7 @@ void wallet2::restore(const std::string& wallet_, const epee::wipeable_string& p auto &hwdev = lookup_device(device_name); hwdev.set_name(device_name); hwdev.set_network_type(m_nettype); + hwdev.set_derivation_path(m_device_derivation_path); hwdev.set_callback(get_device_callback()); m_account.create_from_device(hwdev); diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h index c07054177..a588e8003 100644 --- a/src/wallet/wallet2.h +++ b/src/wallet/wallet2.h @@ -191,6 +191,7 @@ namespace tools static bool has_testnet_option(const boost::program_options::variables_map& vm); static bool has_stagenet_option(const boost::program_options::variables_map& vm); static std::string device_name_option(const boost::program_options::variables_map& vm); + static std::string device_derivation_path_option(const boost::program_options::variables_map &vm); static void init_options(boost::program_options::options_description& desc_params); //! Uses stdin and stdout. Returns a wallet2 if no errors. @@ -978,6 +979,8 @@ namespace tools void confirm_non_default_ring_size(bool always) { m_confirm_non_default_ring_size = always; } const std::string & device_name() const { return m_device_name; } void device_name(const std::string & device_name) { m_device_name = device_name; } + const std::string & device_derivation_path() const { return m_device_derivation_path; } + void device_derivation_path(const std::string &device_derivation_path) { m_device_derivation_path = device_derivation_path; } bool get_tx_key(const crypto::hash &txid, crypto::secret_key &tx_key, std::vector &additional_tx_keys) const; void set_tx_key(const crypto::hash &txid, const crypto::secret_key &tx_key, const std::vector &additional_tx_keys); @@ -1384,6 +1387,7 @@ namespace tools std::unordered_set m_scanned_pool_txs[2]; size_t m_subaddress_lookahead_major, m_subaddress_lookahead_minor; std::string m_device_name; + std::string m_device_derivation_path; // Aux transaction data from device std::unordered_map m_tx_device; From 9cf636af696edfa5ad5d0679c95a67f9a15e0635 Mon Sep 17 00:00:00 2001 From: Dusan Klinec Date: Mon, 12 Nov 2018 04:13:54 +0100 Subject: [PATCH 22/25] device/trezor: ask for KI sync on first refresh When doing a first refresh on HW-token based wallet KI sync is required if money were received. Received money may indicate wallet was already used before the restore I.e., some transaction could have been already sent from the wallet. The spent UTXO would not be detected as spent which could lead to double spending errors on submitting a new transaction. Thus if the wallet is HW-token based with the cold signing protocol and the first refresh detected received money the user is asked to perform the key image sync. --- src/simplewallet/simplewallet.cpp | 53 ++++++++++++++++++++++++------- src/simplewallet/simplewallet.h | 2 ++ src/wallet/wallet2.cpp | 14 +++++--- src/wallet/wallet2.h | 7 +++- 4 files changed, 58 insertions(+), 18 deletions(-) diff --git a/src/simplewallet/simplewallet.cpp b/src/simplewallet/simplewallet.cpp index 07084d15b..c85cbfb8a 100644 --- a/src/simplewallet/simplewallet.cpp +++ b/src/simplewallet/simplewallet.cpp @@ -4341,6 +4341,29 @@ void simple_wallet::on_passphrase_request(bool on_device, epee::wipeable_string passphrase = pwd_container->password(); } //---------------------------------------------------------------------------------------------------- +void simple_wallet::on_refresh_finished(uint64_t start_height, uint64_t fetched_blocks, bool is_init, bool received_money) +{ + // Key image sync after the first refresh + if (!m_wallet->get_account().get_device().has_tx_cold_sign()) { + return; + } + + if (!received_money || m_wallet->get_device_last_key_image_sync() != 0) { + return; + } + + // Finished first refresh for HW device and money received -> KI sync + message_writer() << "\n" << tr("The first refresh has finished for the HW-based wallet with received money. hw_key_images_sync is needed. "); + + std::string accepted = input_line(tr("Do you want to do it now? (Y/Yes/N/No): ")); + if (std::cin.eof() || !command_line::is_yes(accepted)) { + message_writer(console_color_red, false) << tr("hw_key_images_sync skipped. Run command manually before a transfer."); + return; + } + + key_images_sync_intern(); +} +//---------------------------------------------------------------------------------------------------- bool simple_wallet::refresh_main(uint64_t start_height, enum ResetType reset, bool is_init) { if (!try_connect_to_daemon(is_init)) @@ -4358,13 +4381,14 @@ bool simple_wallet::refresh_main(uint64_t start_height, enum ResetType reset, bo message_writer() << tr("Starting refresh..."); uint64_t fetched_blocks = 0; + bool received_money = false; bool ok = false; std::ostringstream ss; try { m_in_manual_refresh.store(true, std::memory_order_relaxed); epee::misc_utils::auto_scope_leave_caller scope_exit_handler = epee::misc_utils::create_scope_leave_handler([&](){m_in_manual_refresh.store(false, std::memory_order_relaxed);}); - m_wallet->refresh(m_wallet->is_trusted_daemon(), start_height, fetched_blocks); + m_wallet->refresh(m_wallet->is_trusted_daemon(), start_height, fetched_blocks, received_money); ok = true; // Clear line "Height xxx of xxx" std::cout << "\r \r"; @@ -4372,6 +4396,7 @@ bool simple_wallet::refresh_main(uint64_t start_height, enum ResetType reset, bo if (is_init) print_accounts(); show_balance_unlocked(); + on_refresh_finished(start_height, fetched_blocks, is_init, received_money); } catch (const tools::error::daemon_busy&) { @@ -8134,13 +8159,13 @@ bool simple_wallet::hw_key_images_sync(const std::vector &args) fail_msg_writer() << tr("hw wallet does not support cold KI sync"); return true; } - if (!m_wallet->is_trusted_daemon()) - { - fail_msg_writer() << tr("this command requires a trusted daemon. Enable with --trusted-daemon"); - return true; - } LOCK_IDLE_SCOPE(); + key_images_sync_intern(); + return true; +} +//---------------------------------------------------------------------------------------------------- +void simple_wallet::key_images_sync_intern(){ try { message_writer(console_color_white, false) << tr("Please confirm the key image sync on the device"); @@ -8149,19 +8174,23 @@ bool simple_wallet::hw_key_images_sync(const std::vector &args) uint64_t height = m_wallet->cold_key_image_sync(spent, unspent); if (height > 0) { - success_msg_writer() << tr("Signed key images imported to height ") << height << ", " - << print_money(spent) << tr(" spent, ") << print_money(unspent) << tr(" unspent"); - } else { + success_msg_writer() << tr("Key images synchronized to height ") << height; + if (!m_wallet->is_trusted_daemon()) + { + message_writer() << tr("Running untrusted daemon, cannot determine which transaction output is spent. Use a trusted daemon with --trusted-daemon and run rescan_spent"); + } else + { + success_msg_writer() << print_money(spent) << tr(" spent, ") << print_money(unspent) << tr(" unspent"); + } + } + else { fail_msg_writer() << tr("Failed to import key images"); } } catch (const std::exception &e) { fail_msg_writer() << tr("Failed to import key images: ") << e.what(); - return true; } - - return true; } //---------------------------------------------------------------------------------------------------- bool simple_wallet::hw_reconnect(const std::vector &args) diff --git a/src/simplewallet/simplewallet.h b/src/simplewallet/simplewallet.h index 7e2edba7f..665879cac 100644 --- a/src/simplewallet/simplewallet.h +++ b/src/simplewallet/simplewallet.h @@ -241,6 +241,8 @@ namespace cryptonote bool print_ring_members(const std::vector& ptx_vector, std::ostream& ostr); std::string get_prompt() const; bool print_seed(bool encrypted); + void key_images_sync_intern(); + void on_refresh_finished(uint64_t start_height, uint64_t fetched_blocks, bool is_init, bool received_money); struct transfer_view { diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index 83aae6de3..46bd47353 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -892,7 +892,8 @@ wallet2::wallet2(network_type nettype, uint64_t kdf_rounds, bool unattended): m_ringdb(), m_last_block_reward(0), m_encrypt_keys_after_refresh(boost::none), - m_unattended(unattended) + m_unattended(unattended), + m_device_last_key_image_sync(0) { } @@ -3009,6 +3010,7 @@ bool wallet2::clear() m_subaddresses.clear(); m_subaddress_labels.clear(); m_multisig_rounds_passed = 0; + m_device_last_key_image_sync = 0; return true; } @@ -9235,9 +9237,7 @@ void wallet2::cold_sign_tx(const std::vector& ptx_vector, signed_tx_ //---------------------------------------------------------------------------------------------------- uint64_t wallet2::cold_key_image_sync(uint64_t &spent, uint64_t &unspent) { auto & hwdev = get_account().get_device(); - if (!hwdev.has_ki_cold_sync()){ - throw std::invalid_argument("Device does not support cold ki sync protocol"); - } + CHECK_AND_ASSERT_THROW_MES(hwdev.has_ki_cold_sync(), "Device does not support cold ki sync protocol"); auto dev_cold = dynamic_cast<::hw::device_cold*>(&hwdev); CHECK_AND_ASSERT_THROW_MES(dev_cold, "Device does not implement cold signing interface"); @@ -9248,7 +9248,11 @@ uint64_t wallet2::cold_key_image_sync(uint64_t &spent, uint64_t &unspent) { dev_cold->ki_sync(&wallet_shim, m_transfers, ski); - return import_key_images(ski, 0, spent, unspent); + // Call COMMAND_RPC_IS_KEY_IMAGE_SPENT only if daemon is trusted. + uint64_t import_res = import_key_images(ski, 0, spent, unspent, is_trusted_daemon()); + m_device_last_key_image_sync = time(NULL); + + return import_res; } //---------------------------------------------------------------------------------------------------- void wallet2::get_hard_fork_info(uint8_t version, uint64_t &earliest_height) const diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h index a588e8003..4d121598b 100644 --- a/src/wallet/wallet2.h +++ b/src/wallet/wallet2.h @@ -810,6 +810,7 @@ namespace tools bool is_transfer_unlocked(uint64_t unlock_time, uint64_t block_height) const; uint64_t get_last_block_reward() const { return m_last_block_reward; } + uint64_t get_device_last_key_image_sync() const { return m_device_last_key_image_sync; } template inline void serialize(t_archive &a, const unsigned int ver) @@ -918,6 +919,9 @@ namespace tools if(ver < 26) return; a & m_tx_device; + if(ver < 27) + return; + a & m_device_last_key_image_sync; } /*! @@ -1388,6 +1392,7 @@ namespace tools size_t m_subaddress_lookahead_major, m_subaddress_lookahead_minor; std::string m_device_name; std::string m_device_derivation_path; + uint64_t m_device_last_key_image_sync; // Aux transaction data from device std::unordered_map m_tx_device; @@ -1424,7 +1429,7 @@ namespace tools std::unique_ptr m_device_callback; }; } -BOOST_CLASS_VERSION(tools::wallet2, 26) +BOOST_CLASS_VERSION(tools::wallet2, 27) BOOST_CLASS_VERSION(tools::wallet2::transfer_details, 10) BOOST_CLASS_VERSION(tools::wallet2::multisig_info, 1) BOOST_CLASS_VERSION(tools::wallet2::multisig_info::LR, 0) From 65b9bca70e3aa0aabc135f952da3695fdab00918 Mon Sep 17 00:00:00 2001 From: Dusan Klinec Date: Thu, 29 Nov 2018 04:22:52 +0100 Subject: [PATCH 23/25] device/trezor: python2 compatibility - bundle dependencies --- src/device_trezor/trezor/tools/README.md | 22 ++- src/device_trezor/trezor/tools/pb2cpp.py | 14 +- .../trezor/tools/py2backports/__init__.py | 0 .../trezor/tools/py2backports/tempfile.py | 72 +++++++++ .../trezor/tools/py2backports/weakref.py | 148 ++++++++++++++++++ 5 files changed, 248 insertions(+), 8 deletions(-) create mode 100644 src/device_trezor/trezor/tools/py2backports/__init__.py create mode 100644 src/device_trezor/trezor/tools/py2backports/tempfile.py create mode 100644 src/device_trezor/trezor/tools/py2backports/weakref.py diff --git a/src/device_trezor/trezor/tools/README.md b/src/device_trezor/trezor/tools/README.md index b176017ac..0802e734a 100644 --- a/src/device_trezor/trezor/tools/README.md +++ b/src/device_trezor/trezor/tools/README.md @@ -10,14 +10,28 @@ Install `protoc` for your distribution. Requirements: Soft requirement: Python 3, can be easily installed with [pyenv]. +If Python 3 is used there are no additional python dependencies. -### Python 2 +Since Cmake 3.12 the `FindPython` module is used to locate the Python +interpreter in your system. It preferably searches for Python 3, if none +is found, it searches for Python 2. -Workaround if there is no Python3 available: +Lower version of the cmake uses another module which does not guarantee +ordering. If you want to override the selected python you can do it in +the following way: ```bash -pip install backports.tempfile -``` +export TREZOR_PYTHON=`which python3` +``` + + +### Python 2.7+ + +Python 3 has `tempfile.TemporaryDirectory` available but Python 2 lacks +this class so the message generation code uses `backports.tempfile` package +bundled in the repository. + +The minimal Python versions are 2.7 and 3.4 ### Regenerate messages diff --git a/src/device_trezor/trezor/tools/pb2cpp.py b/src/device_trezor/trezor/tools/pb2cpp.py index 4d7cc775f..3e0318ea5 100644 --- a/src/device_trezor/trezor/tools/pb2cpp.py +++ b/src/device_trezor/trezor/tools/pb2cpp.py @@ -14,12 +14,18 @@ import hashlib try: from tempfile import TemporaryDirectory except: - # Py2 backward compatibility, optionally installed by user - # pip install backports.tempfile + # Py2 backward compatibility, using bundled sources. + # Original source: pip install backports.tempfile try: - from backports.tempfile import TemporaryDirectory + # Try bundled python version + import sys + sys.path.append(os.path.dirname(__file__)) + from py2backports.tempfile import TemporaryDirectory + except: - raise EnvironmentError('TemporaryDirectory could not be imported. Try: pip install backports.tempfile') + raise EnvironmentError('Python 2.7+ or 3.4+ is required. ' + 'TemporaryDirectory is not available in Python 2.' + 'Try to specify python to use, e.g.: "export TREZOR_PYTHON=`which python3`"') AUTO_HEADER = "# Automatically generated by pb2cpp\n" diff --git a/src/device_trezor/trezor/tools/py2backports/__init__.py b/src/device_trezor/trezor/tools/py2backports/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/device_trezor/trezor/tools/py2backports/tempfile.py b/src/device_trezor/trezor/tools/py2backports/tempfile.py new file mode 100644 index 000000000..e259dea3f --- /dev/null +++ b/src/device_trezor/trezor/tools/py2backports/tempfile.py @@ -0,0 +1,72 @@ +""" +https://github.com/pjdelport/backports.tempfile/blob/master/src/backports/tempfile.py +Partial backport of Python 3.5's tempfile module: + TemporaryDirectory +Backport modifications are marked with marked with "XXX backport". +""" +from __future__ import absolute_import + +import sys +import warnings as _warnings +from shutil import rmtree as _rmtree + +from py2backports.weakref import finalize + + +# XXX backport: Rather than backporting all of mkdtemp(), we just create a +# thin wrapper implementing its Python 3.5 signature. +if sys.version_info < (3, 5): + from tempfile import mkdtemp as old_mkdtemp + + def mkdtemp(suffix=None, prefix=None, dir=None): + """ + Wrap `tempfile.mkdtemp()` to make the suffix and prefix optional (like Python 3.5). + """ + kwargs = {k: v for (k, v) in + dict(suffix=suffix, prefix=prefix, dir=dir).items() + if v is not None} + return old_mkdtemp(**kwargs) + +else: + from tempfile import mkdtemp + + +# XXX backport: ResourceWarning was added in Python 3.2. +# For earlier versions, fall back to RuntimeWarning instead. +_ResourceWarning = RuntimeWarning if sys.version_info < (3, 2) else ResourceWarning + + +class TemporaryDirectory(object): + """Create and return a temporary directory. This has the same + behavior as mkdtemp but can be used as a context manager. For + example: + with TemporaryDirectory() as tmpdir: + ... + Upon exiting the context, the directory and everything contained + in it are removed. + """ + + def __init__(self, suffix=None, prefix=None, dir=None): + self.name = mkdtemp(suffix, prefix, dir) + self._finalizer = finalize( + self, self._cleanup, self.name, + warn_message="Implicitly cleaning up {!r}".format(self)) + + @classmethod + def _cleanup(cls, name, warn_message): + _rmtree(name) + _warnings.warn(warn_message, _ResourceWarning) + + + def __repr__(self): + return "<{} {!r}>".format(self.__class__.__name__, self.name) + + def __enter__(self): + return self.name + + def __exit__(self, exc, value, tb): + self.cleanup() + + def cleanup(self): + if self._finalizer.detach(): + _rmtree(self.name) diff --git a/src/device_trezor/trezor/tools/py2backports/weakref.py b/src/device_trezor/trezor/tools/py2backports/weakref.py new file mode 100644 index 000000000..eb646812b --- /dev/null +++ b/src/device_trezor/trezor/tools/py2backports/weakref.py @@ -0,0 +1,148 @@ +""" +https://github.com/pjdelport/backports.weakref/blob/master/src/backports/weakref.py +Partial backport of Python 3.6's weakref module: + finalize (new in Python 3.4) +Backport modifications are marked with "XXX backport". +""" +from __future__ import absolute_import + +import itertools +import sys +from weakref import ref + +__all__ = ['finalize'] + + +class finalize(object): + """Class for finalization of weakrefable objects + finalize(obj, func, *args, **kwargs) returns a callable finalizer + object which will be called when obj is garbage collected. The + first time the finalizer is called it evaluates func(*arg, **kwargs) + and returns the result. After this the finalizer is dead, and + calling it just returns None. + When the program exits any remaining finalizers for which the + atexit attribute is true will be run in reverse order of creation. + By default atexit is true. + """ + + # Finalizer objects don't have any state of their own. They are + # just used as keys to lookup _Info objects in the registry. This + # ensures that they cannot be part of a ref-cycle. + + __slots__ = () + _registry = {} + _shutdown = False + _index_iter = itertools.count() + _dirty = False + _registered_with_atexit = False + + class _Info(object): + __slots__ = ("weakref", "func", "args", "kwargs", "atexit", "index") + + def __init__(self, obj, func, *args, **kwargs): + if not self._registered_with_atexit: + # We may register the exit function more than once because + # of a thread race, but that is harmless + import atexit + atexit.register(self._exitfunc) + finalize._registered_with_atexit = True + info = self._Info() + info.weakref = ref(obj, self) + info.func = func + info.args = args + info.kwargs = kwargs or None + info.atexit = True + info.index = next(self._index_iter) + self._registry[self] = info + finalize._dirty = True + + def __call__(self, _=None): + """If alive then mark as dead and return func(*args, **kwargs); + otherwise return None""" + info = self._registry.pop(self, None) + if info and not self._shutdown: + return info.func(*info.args, **(info.kwargs or {})) + + def detach(self): + """If alive then mark as dead and return (obj, func, args, kwargs); + otherwise return None""" + info = self._registry.get(self) + obj = info and info.weakref() + if obj is not None and self._registry.pop(self, None): + return (obj, info.func, info.args, info.kwargs or {}) + + def peek(self): + """If alive then return (obj, func, args, kwargs); + otherwise return None""" + info = self._registry.get(self) + obj = info and info.weakref() + if obj is not None: + return (obj, info.func, info.args, info.kwargs or {}) + + @property + def alive(self): + """Whether finalizer is alive""" + return self in self._registry + + @property + def atexit(self): + """Whether finalizer should be called at exit""" + info = self._registry.get(self) + return bool(info) and info.atexit + + @atexit.setter + def atexit(self, value): + info = self._registry.get(self) + if info: + info.atexit = bool(value) + + def __repr__(self): + info = self._registry.get(self) + obj = info and info.weakref() + if obj is None: + return '<%s object at %#x; dead>' % (type(self).__name__, id(self)) + else: + return '<%s object at %#x; for %r at %#x>' % \ + (type(self).__name__, id(self), type(obj).__name__, id(obj)) + + @classmethod + def _select_for_exit(cls): + # Return live finalizers marked for exit, oldest first + L = [(f,i) for (f,i) in cls._registry.items() if i.atexit] + L.sort(key=lambda item:item[1].index) + return [f for (f,i) in L] + + @classmethod + def _exitfunc(cls): + # At shutdown invoke finalizers for which atexit is true. + # This is called once all other non-daemonic threads have been + # joined. + reenable_gc = False + try: + if cls._registry: + import gc + if gc.isenabled(): + reenable_gc = True + gc.disable() + pending = None + while True: + if pending is None or finalize._dirty: + pending = cls._select_for_exit() + finalize._dirty = False + if not pending: + break + f = pending.pop() + try: + # gc is disabled, so (assuming no daemonic + # threads) the following is the only line in + # this function which might trigger creation + # of a new finalizer + f() + except Exception: + sys.excepthook(*sys.exc_info()) + assert f not in cls._registry + finally: + # prevent any more finalizers from executing during shutdown + finalize._shutdown = True + if reenable_gc: + gc.enable() From d71f89e2a26720021f7509f8c1eee87645f48529 Mon Sep 17 00:00:00 2001 From: Dusan Klinec Date: Wed, 28 Nov 2018 22:22:11 +0100 Subject: [PATCH 24/25] device/trezor: device/trezor: correct device initialization, status check - checks if the device is in the correct usable state - implements check for the v2.0.9 firmware which does not support payment IDs - simple transacttion check, payment id fmt consistency - minor fixes, refactoring, webusb session counting fix --- cmake/CheckTrezor.cmake | 2 +- src/device_trezor/device_trezor.cpp | 77 +++++++++++++++++++++--- src/device_trezor/device_trezor.hpp | 5 +- src/device_trezor/device_trezor_base.cpp | 58 ++++++++++++++++-- src/device_trezor/device_trezor_base.hpp | 19 +++++- src/device_trezor/trezor/transport.cpp | 2 +- 6 files changed, 140 insertions(+), 23 deletions(-) diff --git a/cmake/CheckTrezor.cmake b/cmake/CheckTrezor.cmake index ea21237fd..bcd3cfc2c 100644 --- a/cmake/CheckTrezor.cmake +++ b/cmake/CheckTrezor.cmake @@ -56,7 +56,7 @@ if(Protobuf_FOUND AND USE_DEVICE_TREZOR AND TREZOR_PYTHON) endif() if(USE_DEVICE_TREZOR_UDP_RELEASE) - add_definitions(-DWITH_DEVICE_TREZOR_UDP_RELEASE=1) + add_definitions(-DUSE_DEVICE_TREZOR_UDP_RELEASE=1) endif() if (Protobuf_INCLUDE_DIR) diff --git a/src/device_trezor/device_trezor.cpp b/src/device_trezor/device_trezor.cpp index 5096fcea8..8868fb995 100644 --- a/src/device_trezor/device_trezor.cpp +++ b/src/device_trezor/device_trezor.cpp @@ -121,7 +121,8 @@ namespace trezor { const boost::optional & network_type){ AUTO_LOCK_CMD(); require_connected(); - test_ping(); + device_state_reset_unsafe(); + require_initialized(); auto req = std::make_shared(); this->set_msg_addr(req.get(), path, network_type); @@ -136,7 +137,8 @@ namespace trezor { const boost::optional & network_type){ AUTO_LOCK_CMD(); require_connected(); - test_ping(); + device_state_reset_unsafe(); + require_initialized(); auto req = std::make_shared(); this->set_msg_addr(req.get(), path, network_type); @@ -152,7 +154,8 @@ namespace trezor { { AUTO_LOCK_CMD(); require_connected(); - test_ping(); + device_state_reset_unsafe(); + require_initialized(); std::shared_ptr req; @@ -238,12 +241,11 @@ namespace trezor { cpend.construction_data = cdata.tx_data; // Transaction check - cryptonote::blobdata tx_blob; - cryptonote::transaction tx_deserialized; - bool r = cryptonote::t_serializable_object_to_blob(cpend.tx, tx_blob); - CHECK_AND_ASSERT_THROW_MES(r, "Transaction serialization failed"); - r = cryptonote::parse_and_validate_tx_from_blob(tx_blob, tx_deserialized); - CHECK_AND_ASSERT_THROW_MES(r, "Transaction deserialization failed"); + try { + transaction_check(cdata, aux_data); + } catch(const std::exception &e){ + throw exc::ProtocolException(std::string("Transaction verification failed: ") + e.what()); + } std::string key_images; bool all_are_txin_to_key = std::all_of(cdata.tx.vin.begin(), cdata.tx.vin.end(), [&](const cryptonote::txin_v& s_e) -> bool @@ -283,7 +285,8 @@ namespace trezor { { AUTO_LOCK_CMD(); require_connected(); - test_ping(); + device_state_reset_unsafe(); + require_initialized(); CHECK_AND_ASSERT_THROW_MES(idx < unsigned_tx.txes.size(), "Invalid transaction index"); signer = std::make_shared(wallet, &unsigned_tx, idx, &aux_data); @@ -294,6 +297,7 @@ namespace trezor { // Step: Init auto init_msg = signer->step_init(); this->set_msg_addr(init_msg.get()); + transaction_pre_check(init_msg); auto response = this->client_exchange(init_msg); signer->step_init_ack(response); @@ -351,6 +355,59 @@ namespace trezor { signer->step_final_ack(ack_final); } + void device_trezor::transaction_pre_check(std::shared_ptr init_msg) + { + CHECK_AND_ASSERT_THROW_MES(init_msg, "TransactionInitRequest is empty"); + CHECK_AND_ASSERT_THROW_MES(init_msg->has_tsx_data(), "TransactionInitRequest has no transaction data"); + CHECK_AND_ASSERT_THROW_MES(m_features, "Device state not initialized"); // make sure the caller did not reset features + const bool nonce_required = init_msg->tsx_data().has_payment_id() && init_msg->tsx_data().payment_id().size() > 0; + + if (nonce_required){ + // Versions 2.0.9 and lower do not support payment ID + CHECK_AND_ASSERT_THROW_MES(m_features->has_major_version() && m_features->has_minor_version() && m_features->has_patch_version(), "Invalid Trezor firmware version information"); + const uint32_t vma = m_features->major_version(); + const uint32_t vmi = m_features->minor_version(); + const uint32_t vpa = m_features->patch_version(); + if (vma < 2 || (vma == 2 && vmi == 0 && vpa <= 9)) { + throw exc::TrezorException("Trezor firmware 2.0.9 and lower does not support transactions with short payment IDs or integrated addresses. Please update."); + } + } + } + + void device_trezor::transaction_check(const protocol::tx::TData & tdata, const hw::tx_aux_data & aux_data) + { + // Simple serialization check + cryptonote::blobdata tx_blob; + cryptonote::transaction tx_deserialized; + bool r = cryptonote::t_serializable_object_to_blob(tdata.tx, tx_blob); + CHECK_AND_ASSERT_THROW_MES(r, "Transaction serialization failed"); + r = cryptonote::parse_and_validate_tx_from_blob(tx_blob, tx_deserialized); + CHECK_AND_ASSERT_THROW_MES(r, "Transaction deserialization failed"); + + // Extras check + std::vector tx_extra_fields; + cryptonote::tx_extra_nonce nonce; + + r = cryptonote::parse_tx_extra(tdata.tx.extra, tx_extra_fields); + CHECK_AND_ASSERT_THROW_MES(r, "tx.extra parsing failed"); + + const bool nonce_required = tdata.tsx_data.has_payment_id() && tdata.tsx_data.payment_id().size() > 0; + const bool has_nonce = cryptonote::find_tx_extra_field_by_type(tx_extra_fields, nonce); + CHECK_AND_ASSERT_THROW_MES(has_nonce == nonce_required, "Transaction nonce presence inconsistent"); + + if (nonce_required){ + const std::string & payment_id = tdata.tsx_data.payment_id(); + if (payment_id.size() == 32){ + crypto::hash payment_id_long{}; + CHECK_AND_ASSERT_THROW_MES(cryptonote::get_payment_id_from_tx_extra_nonce(nonce.nonce, payment_id_long), "Long payment ID not present"); + + } else if (payment_id.size() == 8){ + crypto::hash8 payment_id_short{}; + CHECK_AND_ASSERT_THROW_MES(cryptonote::get_encrypted_payment_id_from_tx_extra_nonce(nonce.nonce, payment_id_short), "Short payment ID not present"); + } + } + } + #else //WITH_DEVICE_TREZOR void register_all(std::map> ®istry) { diff --git a/src/device_trezor/device_trezor.hpp b/src/device_trezor/device_trezor.hpp index a2134574c..1f08be887 100644 --- a/src/device_trezor/device_trezor.hpp +++ b/src/device_trezor/device_trezor.hpp @@ -57,9 +57,8 @@ namespace trezor { */ class device_trezor : public hw::trezor::device_trezor_base, public hw::device_cold { protected: - // To speed up blockchain parsing the view key maybe handle here. - crypto::secret_key viewkey; - bool has_view_key; + void transaction_pre_check(std::shared_ptr init_msg); + void transaction_check(const protocol::tx::TData & tdata, const hw::tx_aux_data & aux_data); public: device_trezor(); diff --git a/src/device_trezor/device_trezor_base.cpp b/src/device_trezor/device_trezor_base.cpp index fddc6082c..5071932ee 100644 --- a/src/device_trezor/device_trezor_base.cpp +++ b/src/device_trezor/device_trezor_base.cpp @@ -65,7 +65,7 @@ namespace trezor { } bool device_trezor_base::set_name(const std::string & name) { - this->full_name = name; + this->m_full_name = name; this->name = ""; auto delim = name.find(':'); @@ -77,10 +77,10 @@ namespace trezor { } const std::string device_trezor_base::get_name() const { - if (this->full_name.empty()) { + if (this->m_full_name.empty()) { return std::string("name).append(">"); } - return this->full_name; + return this->m_full_name; } bool device_trezor_base::init() { @@ -139,6 +139,9 @@ namespace trezor { } bool device_trezor_base::disconnect() { + m_device_state.clear(); + m_features.reset(); + if (m_transport){ try { m_transport->close(); @@ -193,6 +196,25 @@ namespace trezor { } } + void device_trezor_base::require_initialized(){ + if (!m_features){ + throw exc::TrezorException("Device state not initialized"); + } + + if (m_features->has_bootloader_mode() && m_features->bootloader_mode()){ + throw exc::TrezorException("Device is in the bootloader mode"); + } + + if (m_features->has_firmware_present() && !m_features->firmware_present()){ + throw exc::TrezorException("Device has no firmware loaded"); + } + + // Hard requirement on initialized field, has to be there. + if (!m_features->has_initialized() || !m_features->initialized()){ + throw exc::TrezorException("Device is not initialized"); + } + } + void device_trezor_base::call_ping_unsafe(){ auto pingMsg = std::make_shared(); pingMsg->set_message("PING"); @@ -217,7 +239,7 @@ namespace trezor { void device_trezor_base::write_raw(const google::protobuf::Message * msg){ require_connected(); CHECK_AND_ASSERT_THROW_MES(msg, "Empty message"); - this->getTransport()->write(*msg); + this->get_transport()->write(*msg); } GenericMessage device_trezor_base::read_raw(){ @@ -225,7 +247,7 @@ namespace trezor { std::shared_ptr msg_resp; hw::trezor::messages::MessageType msg_resp_type; - this->getTransport()->read(msg_resp, &msg_resp_type); + this->get_transport()->read(msg_resp, &msg_resp_type); return GenericMessage(msg_resp_type, msg_resp); } @@ -314,6 +336,25 @@ namespace trezor { return false; } + void device_trezor_base::device_state_reset_unsafe() + { + require_connected(); + auto initMsg = std::make_shared(); + + if(!m_device_state.empty()) { + initMsg->set_allocated_state(&m_device_state); + } + + m_features = this->client_exchange(initMsg); + initMsg->release_state(); + } + + void device_trezor_base::device_state_reset() + { + AUTO_LOCK_CMD(); + device_state_reset_unsafe(); + } + void device_trezor_base::on_button_request(GenericMessage & resp, const messages::common::ButtonRequest * msg) { CHECK_AND_ASSERT_THROW_MES(msg, "Empty message"); @@ -361,7 +402,13 @@ namespace trezor { // TODO: remove passphrase from memory m.set_passphrase(passphrase.data(), passphrase.size()); } + + if (!m_device_state.empty()){ + m.set_allocated_state(&m_device_state); + } + resp = call_raw(&m); + m.release_state(); } void device_trezor_base::on_passphrase_state_request(GenericMessage & resp, const messages::common::PassphraseStateRequest * msg) @@ -369,6 +416,7 @@ namespace trezor { MDEBUG("on_passhprase_state_request"); CHECK_AND_ASSERT_THROW_MES(msg, "Empty message"); + m_device_state = msg->state(); messages::common::PassphraseStateAck m; resp = call_raw(&m); } diff --git a/src/device_trezor/device_trezor_base.hpp b/src/device_trezor/device_trezor_base.hpp index 4d205ad7a..3c35e8aca 100644 --- a/src/device_trezor/device_trezor_base.hpp +++ b/src/device_trezor/device_trezor_base.hpp @@ -70,8 +70,10 @@ namespace trezor { std::shared_ptr m_transport; i_device_callback * m_callback; - std::string full_name; + std::string m_full_name; std::vector m_wallet_deriv_path; + std::string m_device_state; // returned after passphrase entry, session + std::shared_ptr m_features; // features from the last device reset cryptonote::network_type network_type; @@ -80,8 +82,10 @@ namespace trezor { // void require_connected(); + void require_initialized(); void call_ping_unsafe(); void test_ping(); + void device_state_reset_unsafe(); void ensure_derivation_path() noexcept; // Communication methods @@ -130,7 +134,7 @@ namespace trezor { // Scoped session closer BOOST_SCOPE_EXIT_ALL(&, this) { if (open_session){ - this->getTransport()->close(); + this->get_transport()->close(); } }; @@ -209,7 +213,7 @@ namespace trezor { // Default derivation path for Monero static const uint32_t DEFAULT_BIP44_PATH[2]; - std::shared_ptr getTransport(){ + std::shared_ptr get_transport(){ return m_transport; } @@ -221,6 +225,10 @@ namespace trezor { return m_callback; } + std::shared_ptr & get_features() { + return m_features; + } + void set_derivation_path(const std::string &deriv_path) override; /* ======================================================================= */ @@ -250,6 +258,11 @@ namespace trezor { */ bool ping(); + /** + * Performs Initialize call to the Trezor, resets to known state. + */ + void device_state_reset(); + // Protocol callbacks void on_button_request(GenericMessage & resp, const messages::common::ButtonRequest * msg); void on_pin_request(GenericMessage & resp, const messages::common::PinMatrixRequest * msg); diff --git a/src/device_trezor/trezor/transport.cpp b/src/device_trezor/trezor/transport.cpp index 814537eb6..cd66e59e8 100644 --- a/src/device_trezor/trezor/transport.cpp +++ b/src/device_trezor/trezor/transport.cpp @@ -840,7 +840,7 @@ namespace trezor{ throw exc::DeviceAcquireException("Unable to claim libusb device"); } - m_conn_count += 1; + m_conn_count = 1; m_proto->session_begin(*this); #undef TREZOR_DESTROY_SESSION From b58f12e53e8bada75f407f9c0fa4565d2ec948d5 Mon Sep 17 00:00:00 2001 From: wowario Date: Wed, 12 Dec 2018 16:17:00 +0300 Subject: [PATCH 25/25] rename file name --- src/blockchain_utilities/CMakeLists.txt | 2 +- src/blockchain_utilities/blockchain_prune_known_spent_data.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/blockchain_utilities/CMakeLists.txt b/src/blockchain_utilities/CMakeLists.txt index 980f89d7c..002bf7986 100644 --- a/src/blockchain_utilities/CMakeLists.txt +++ b/src/blockchain_utilities/CMakeLists.txt @@ -296,5 +296,5 @@ target_link_libraries(blockchain_prune_known_spent_data set_property(TARGET blockchain_prune_known_spent_data PROPERTY - OUTPUT_NAME "monero-blockchain-prune-known-spent-data") + OUTPUT_NAME "wownero-blockchain-prune-known-spent-data") install(TARGETS blockchain_prune_known_spent_data DESTINATION bin) diff --git a/src/blockchain_utilities/blockchain_prune_known_spent_data.cpp b/src/blockchain_utilities/blockchain_prune_known_spent_data.cpp index f6136c1ba..68cff230e 100644 --- a/src/blockchain_utilities/blockchain_prune_known_spent_data.cpp +++ b/src/blockchain_utilities/blockchain_prune_known_spent_data.cpp @@ -152,7 +152,7 @@ int main(int argc, char* argv[]) return 1; } - mlog_configure(mlog_get_default_log_path("monero-blockchain-prune-known-spent-data.log"), true); + mlog_configure(mlog_get_default_log_path("wownero-blockchain-prune-known-spent-data.log"), true); if (!command_line::is_arg_defaulted(vm, arg_log_level)) mlog_set_log(command_line::get_arg(vm, arg_log_level).c_str()); else