diff --git a/src/device_trezor/device_trezor.cpp b/src/device_trezor/device_trezor.cpp index 07c03fc66..f55cbb15d 100644 --- a/src/device_trezor/device_trezor.cpp +++ b/src/device_trezor/device_trezor.cpp @@ -212,9 +212,10 @@ namespace trezor { tools::wallet2::signed_tx_set & signed_tx, hw::tx_aux_data & aux_data) { + CHECK_AND_ASSERT_THROW_MES(unsigned_tx.transfers.first == 0, "Unsuported non zero offset"); size_t num_tx = unsigned_tx.txes.size(); signed_tx.key_images.clear(); - signed_tx.key_images.resize(unsigned_tx.transfers.size()); + signed_tx.key_images.resize(unsigned_tx.transfers.second.size()); for(size_t tx_idx = 0; tx_idx < num_tx; ++tx_idx) { std::shared_ptr signer; @@ -360,4 +361,4 @@ namespace trezor { } #endif //WITH_DEVICE_TREZOR -}} \ No newline at end of file +}} diff --git a/src/simplewallet/simplewallet.cpp b/src/simplewallet/simplewallet.cpp index 58d4cdced..babc59ea3 100644 --- a/src/simplewallet/simplewallet.cpp +++ b/src/simplewallet/simplewallet.cpp @@ -6105,8 +6105,8 @@ bool simple_wallet::accept_loaded_tx(const std::function get_num_txes, bool simple_wallet::accept_loaded_tx(const tools::wallet2::unsigned_tx_set &txs) { std::string extra_message; - if (!txs.transfers.empty()) - extra_message = (boost::format("%u outputs to import. ") % (unsigned)txs.transfers.size()).str(); + if (!txs.transfers.second.empty()) + extra_message = (boost::format("%u outputs to import. ") % (unsigned)txs.transfers.second.size()).str(); return accept_loaded_tx([&txs](){return txs.txes.size();}, [&txs](size_t n)->const tools::wallet2::tx_construction_data&{return txs.txes[n];}, extra_message); } //---------------------------------------------------------------------------------------------------- diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index 31246d0b1..a90a93321 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -113,11 +113,11 @@ using namespace cryptonote; #define SUBADDRESS_LOOKAHEAD_MAJOR 50 #define SUBADDRESS_LOOKAHEAD_MINOR 200 -#define KEY_IMAGE_EXPORT_FILE_MAGIC "Monero key image export\002" +#define KEY_IMAGE_EXPORT_FILE_MAGIC "Monero key image export\003" #define MULTISIG_EXPORT_FILE_MAGIC "Monero multisig export\001" -#define OUTPUT_EXPORT_FILE_MAGIC "Monero output export\003" +#define OUTPUT_EXPORT_FILE_MAGIC "Monero output export\004" #define SEGREGATION_FORK_HEIGHT 99999999 #define TESTNET_SEGREGATION_FORK_HEIGHT 99999999 @@ -1606,6 +1606,7 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote td.m_txid = txid; td.m_key_image = tx_scan_info[o].ki; td.m_key_image_known = !m_watch_only && !m_multisig; + td.m_key_image_requested = false; td.m_key_image_partial = m_multisig; td.m_amount = amount; td.m_pk_index = pk_index - 1; @@ -5466,7 +5467,7 @@ std::string wallet2::dump_tx_to_str(const std::vector &ptx_vector) c txs.txes.push_back(get_construction_data_with_decrypted_short_payment_id(tx, m_account.get_device())); } - txs.transfers = m_transfers; + txs.transfers = export_outputs(); // save as binary std::ostringstream oss; boost::archive::portable_binary_oarchive ar(oss); @@ -7952,6 +7953,7 @@ void wallet2::light_wallet_get_unspent_outs() td.m_key_image = unspent_key_image; td.m_key_image_known = !m_watch_only && !m_multisig; + td.m_key_image_requested = false; td.m_key_image_partial = m_multisig; td.m_amount = o.amount; td.m_pk_index = 0; @@ -9125,7 +9127,7 @@ void wallet2::cold_sign_tx(const std::vector& ptx_vector, signed_tx_ { txs.txes.push_back(get_construction_data_with_decrypted_short_payment_id(tx, m_account.get_device())); } - txs.transfers = m_transfers; + txs.transfers = std::make_pair(0, m_transfers); auto dev_cold = dynamic_cast<::hw::device_cold*>(&hwdev); CHECK_AND_ASSERT_THROW_MES(dev_cold, "Device does not implement cold signing interface"); @@ -9156,7 +9158,7 @@ uint64_t wallet2::cold_key_image_sync(uint64_t &spent, uint64_t &unspent) { dev_cold->ki_sync(&wallet_shim, m_transfers, ski); - return import_key_images(ski, spent, unspent); + return import_key_images(ski, 0, spent, unspent); } //---------------------------------------------------------------------------------------------------- void wallet2::get_hard_fork_info(uint8_t version, uint64_t &earliest_height) const @@ -10519,15 +10521,21 @@ crypto::public_key wallet2::get_tx_pub_key_from_received_outs(const tools::walle bool wallet2::export_key_images(const std::string &filename) const { PERF_TIMER(export_key_images); - std::vector> ski = export_key_images(); + std::pair>> ski = export_key_images(); std::string magic(KEY_IMAGE_EXPORT_FILE_MAGIC, strlen(KEY_IMAGE_EXPORT_FILE_MAGIC)); const cryptonote::account_public_address &keys = get_account().get_keys().m_account_address; + const uint32_t offset = ski.first; std::string data; - data.reserve(ski.size() * (sizeof(crypto::key_image) + sizeof(crypto::signature)) + 2 * sizeof(crypto::public_key)); + data.reserve(4 + ski.second.size() * (sizeof(crypto::key_image) + sizeof(crypto::signature)) + 2 * sizeof(crypto::public_key)); + data.resize(4); + data[0] = offset & 0xff; + data[1] = (offset >> 8) & 0xff; + data[2] = (offset >> 16) & 0xff; + data[3] = (offset >> 24) & 0xff; data += std::string((const char *)&keys.m_spend_public_key, sizeof(crypto::public_key)); data += std::string((const char *)&keys.m_view_public_key, sizeof(crypto::public_key)); - for (const auto &i: ski) + for (const auto &i: ski.second) { data += std::string((const char *)&i.first, sizeof(crypto::key_image)); data += std::string((const char *)&i.second, sizeof(crypto::signature)); @@ -10540,13 +10548,17 @@ bool wallet2::export_key_images(const std::string &filename) const } //---------------------------------------------------------------------------------------------------- -std::vector> wallet2::export_key_images() const +std::pair>> wallet2::export_key_images() const { PERF_TIMER(export_key_images_raw); std::vector> ski; - ski.reserve(m_transfers.size()); - for (size_t n = 0; n < m_transfers.size(); ++n) + size_t offset = 0; + while (offset < m_transfers.size() && !m_transfers[offset].m_key_image_requested) + ++offset; + + ski.reserve(m_transfers.size() - offset); + for (size_t n = offset; n < m_transfers.size(); ++n) { const transfer_details &td = m_transfers[n]; @@ -10590,7 +10602,7 @@ std::vector> wallet2::export_key ski.push_back(std::make_pair(td.m_key_image, signature)); } - return ski; + return std::make_pair(offset, ski); } uint64_t wallet2::import_key_images(const std::string &filename, uint64_t &spent, uint64_t &unspent) @@ -10617,15 +10629,17 @@ uint64_t wallet2::import_key_images(const std::string &filename, uint64_t &spent THROW_WALLET_EXCEPTION(error::wallet_internal_error, std::string("Failed to decrypt ") + filename + ": " + e.what()); } - const size_t headerlen = 2 * sizeof(crypto::public_key); + const size_t headerlen = 4 + 2 * sizeof(crypto::public_key); THROW_WALLET_EXCEPTION_IF(data.size() < headerlen, error::wallet_internal_error, std::string("Bad data size from file ") + filename); - const crypto::public_key &public_spend_key = *(const crypto::public_key*)&data[0]; - const crypto::public_key &public_view_key = *(const crypto::public_key*)&data[sizeof(crypto::public_key)]; + const uint32_t offset = (uint8_t)data[0] | (((uint8_t)data[1]) << 8) | (((uint8_t)data[2]) << 16) | (((uint8_t)data[3]) << 24); + const crypto::public_key &public_spend_key = *(const crypto::public_key*)&data[4]; + const crypto::public_key &public_view_key = *(const crypto::public_key*)&data[4 + sizeof(crypto::public_key)]; const cryptonote::account_public_address &keys = get_account().get_keys().m_account_address; if (public_spend_key != keys.m_spend_public_key || public_view_key != keys.m_view_public_key) { THROW_WALLET_EXCEPTION(error::wallet_internal_error, std::string( "Key images from ") + filename + " are for a different account"); } + THROW_WALLET_EXCEPTION_IF(offset > m_transfers.size(), error::wallet_internal_error, "Offset larger than known outputs"); const size_t record_size = sizeof(crypto::key_image) + sizeof(crypto::signature); THROW_WALLET_EXCEPTION_IF((data.size() - headerlen) % record_size, @@ -10642,20 +10656,21 @@ uint64_t wallet2::import_key_images(const std::string &filename, uint64_t &spent ski.push_back(std::make_pair(key_image, signature)); } - return import_key_images(ski, spent, unspent); + return import_key_images(ski, offset, spent, unspent); } //---------------------------------------------------------------------------------------------------- -uint64_t wallet2::import_key_images(const std::vector> &signed_key_images, uint64_t &spent, uint64_t &unspent, bool check_spent) +uint64_t wallet2::import_key_images(const std::vector> &signed_key_images, size_t offset, uint64_t &spent, uint64_t &unspent, bool check_spent) { PERF_TIMER(import_key_images_lots); COMMAND_RPC_IS_KEY_IMAGE_SPENT::request req = AUTO_VAL_INIT(req); COMMAND_RPC_IS_KEY_IMAGE_SPENT::response daemon_resp = AUTO_VAL_INIT(daemon_resp); - THROW_WALLET_EXCEPTION_IF(signed_key_images.size() > m_transfers.size(), error::wallet_internal_error, + THROW_WALLET_EXCEPTION_IF(offset > m_transfers.size(), error::wallet_internal_error, "Offset larger than known outputs"); + THROW_WALLET_EXCEPTION_IF(signed_key_images.size() > m_transfers.size() - offset, error::wallet_internal_error, "The blockchain is out of date compared to the signed key images"); - if (signed_key_images.empty()) + if (signed_key_images.empty() && offset == 0) { spent = 0; unspent = 0; @@ -10667,7 +10682,7 @@ uint64_t wallet2::import_key_images(const std::vector pkeys; pkeys.push_back(&pkey); THROW_WALLET_EXCEPTION_IF(!(rct::scalarmultKey(rct::ki2rct(key_image), rct::curveOrder()) == rct::identity()), - error::wallet_internal_error, "Key image out of validity domain: input " + boost::lexical_cast(n) + "/" + error::wallet_internal_error, "Key image out of validity domain: input " + boost::lexical_cast(n + offset) + "/" + boost::lexical_cast(signed_key_images.size()) + ", key image " + epee::string_tools::pod_to_hex(key_image)); THROW_WALLET_EXCEPTION_IF(!crypto::check_ring_signature((const crypto::hash&)key_image, key_image, pkeys, &signature), - error::signature_check_failed, boost::lexical_cast(n) + "/" + error::signature_check_failed, boost::lexical_cast(n + offset) + "/" + boost::lexical_cast(signed_key_images.size()) + ", key image " + epee::string_tools::pod_to_hex(key_image) + ", signature " + epee::string_tools::pod_to_hex(signature) + ", pubkey " + epee::string_tools::pod_to_hex(*pkeys[0])); } @@ -10698,10 +10713,11 @@ uint64_t wallet2::import_key_images(const std::vector key_images) td.m_key_image = key_images[i]; m_key_images[m_transfers[i].m_key_image] = i; td.m_key_image_known = true; + td.m_key_image_requested = false; td.m_key_image_partial = false; m_pub_keys[m_transfers[i].get_public_key()] = i; } @@ -10984,20 +11001,24 @@ void wallet2::import_blockchain(const std::tuple wallet2::export_outputs() const +std::pair> wallet2::export_outputs() const { PERF_TIMER(export_outputs); std::vector outs; - outs.reserve(m_transfers.size()); - for (size_t n = 0; n < m_transfers.size(); ++n) + size_t offset = 0; + while (offset < m_transfers.size() && m_transfers[offset].m_key_image_known) + ++offset; + + outs.reserve(m_transfers.size() - offset); + for (size_t n = offset; n < m_transfers.size(); ++n) { const transfer_details &td = m_transfers[n]; outs.push_back(td); } - return outs; + return std::make_pair(offset, outs); } //---------------------------------------------------------------------------------------------------- std::string wallet2::export_outputs_to_str() const @@ -11006,7 +11027,7 @@ std::string wallet2::export_outputs_to_str() const std::stringstream oss; boost::archive::portable_binary_oarchive ar(oss); - ar << m_transfers; + ar << export_outputs(); std::string magic(OUTPUT_EXPORT_FILE_MAGIC, strlen(OUTPUT_EXPORT_FILE_MAGIC)); const cryptonote::account_public_address &keys = get_account().get_keys().m_account_address; @@ -11018,20 +11039,27 @@ std::string wallet2::export_outputs_to_str() const return magic + ciphertext; } //---------------------------------------------------------------------------------------------------- -size_t wallet2::import_outputs(const std::vector &outputs) +size_t wallet2::import_outputs(const std::pair> &outputs) { PERF_TIMER(import_outputs); + + THROW_WALLET_EXCEPTION_IF(outputs.first > m_transfers.size(), error::wallet_internal_error, + "Imported outputs omit more outputs that we know of"); + + const size_t offset = outputs.first; const size_t original_size = m_transfers.size(); - m_transfers.resize(outputs.size()); - for (size_t i = 0; i < outputs.size(); ++i) + m_transfers.resize(offset + outputs.second.size()); + for (size_t i = 0; i < offset; ++i) + m_transfers[i].m_key_image_requested = false; + for (size_t i = 0; i < outputs.second.size(); ++i) { - transfer_details td = outputs[i]; + transfer_details td = outputs.second[i]; // skip those we've already imported, or which have different data - if (i < original_size) + if (i + offset < original_size) { // compare the data used to create the key image below - const transfer_details &org_td = m_transfers[i]; + const transfer_details &org_td = m_transfers[i + offset]; if (!org_td.m_key_image_known) goto process; #define CMPF(f) if (!(td.f == org_td.f)) goto process @@ -11043,7 +11071,7 @@ size_t wallet2::import_outputs(const std::vector(i)); + THROW_WALLET_EXCEPTION_IF(td.m_tx.vout.empty(), error::wallet_internal_error, "tx with no outputs at index " + boost::lexical_cast(i + offset)); crypto::public_key tx_pub_key = get_tx_pub_key_from_received_outs(td); const std::vector additional_tx_pub_keys = get_additional_tx_pub_keys_from_extra(td.m_tx); @@ -11063,13 +11091,14 @@ process: THROW_WALLET_EXCEPTION_IF(!r, error::wallet_internal_error, "Failed to generate key image"); expand_subaddresses(td.m_subaddr_index); td.m_key_image_known = true; + td.m_key_image_requested = true; td.m_key_image_partial = false; THROW_WALLET_EXCEPTION_IF(in_ephemeral.pub != out_key, - error::wallet_internal_error, "key_image generated ephemeral public key not matched with output_key at index " + boost::lexical_cast(i)); + error::wallet_internal_error, "key_image generated ephemeral public key not matched with output_key at index " + boost::lexical_cast(i + offset)); - m_key_images[td.m_key_image] = i; - m_pub_keys[td.get_public_key()] = i; - m_transfers[i] = std::move(td); + m_key_images[td.m_key_image] = i + offset; + m_pub_keys[td.get_public_key()] = i + offset; + m_transfers[i + offset] = std::move(td); } return m_transfers.size(); @@ -11114,7 +11143,7 @@ size_t wallet2::import_outputs_from_str(const std::string &outputs_st) std::string body(data, headerlen); std::stringstream iss; iss << body; - std::vector outputs; + std::pair> outputs; try { boost::archive::portable_binary_iarchive ar(iss); @@ -11306,6 +11335,7 @@ void wallet2::update_multisig_rescan_info(const std::vector txes; - wallet2::transfer_container transfers; + std::pair transfers; }; struct signed_tx_set @@ -1071,9 +1073,9 @@ namespace tools bool verify_with_public_key(const std::string &data, const crypto::public_key &public_key, const std::string &signature) const; // Import/Export wallet data - std::vector export_outputs() const; + std::pair> export_outputs() const; std::string export_outputs_to_str() const; - size_t import_outputs(const std::vector &outputs); + size_t import_outputs(const std::pair> &outputs); size_t import_outputs_from_str(const std::string &outputs_st); payment_container export_payments() const; void import_payments(const payment_container &payments); @@ -1081,8 +1083,8 @@ namespace tools std::tuple> export_blockchain() const; void import_blockchain(const std::tuple> &bc); bool export_key_images(const std::string &filename) const; - std::vector> export_key_images() const; - uint64_t import_key_images(const std::vector> &signed_key_images, uint64_t &spent, uint64_t &unspent, bool check_spent = true); + std::pair>> export_key_images() const; + uint64_t import_key_images(const std::vector> &signed_key_images, size_t offset, uint64_t &spent, uint64_t &unspent, bool check_spent = true); uint64_t import_key_images(const std::string &filename, uint64_t &spent, uint64_t &unspent); bool import_key_images(std::vector key_images); crypto::public_key get_tx_pub_key_from_received_outs(const tools::wallet2::transfer_details &td) const; @@ -1396,7 +1398,7 @@ namespace tools }; } BOOST_CLASS_VERSION(tools::wallet2, 26) -BOOST_CLASS_VERSION(tools::wallet2::transfer_details, 9) +BOOST_CLASS_VERSION(tools::wallet2::transfer_details, 10) BOOST_CLASS_VERSION(tools::wallet2::multisig_info, 1) BOOST_CLASS_VERSION(tools::wallet2::multisig_info::LR, 0) BOOST_CLASS_VERSION(tools::wallet2::multisig_tx_set, 1) @@ -1454,6 +1456,10 @@ namespace boost x.m_multisig_k.clear(); x.m_multisig_info.clear(); } + if (ver < 10) + { + x.m_key_image_requested = false; + } } template @@ -1535,6 +1541,12 @@ namespace boost a & x.m_multisig_info; a & x.m_multisig_k; a & x.m_key_image_partial; + if (ver < 10) + { + initialize_transfer_details(a, x, ver); + return; + } + a & x.m_key_image_requested; } template diff --git a/src/wallet/wallet_rpc_server.cpp b/src/wallet/wallet_rpc_server.cpp index 5e6100dfd..eabdd9a6a 100644 --- a/src/wallet/wallet_rpc_server.cpp +++ b/src/wallet/wallet_rpc_server.cpp @@ -2460,12 +2460,13 @@ namespace tools if (!m_wallet) return not_open(er); try { - std::vector> ski = m_wallet->export_key_images(); - res.signed_key_images.resize(ski.size()); - for (size_t n = 0; n < ski.size(); ++n) + std::pair>> ski = m_wallet->export_key_images(); + res.offset = ski.first; + res.signed_key_images.resize(ski.second.size()); + for (size_t n = 0; n < ski.second.size(); ++n) { - res.signed_key_images[n].key_image = epee::string_tools::pod_to_hex(ski[n].first); - res.signed_key_images[n].signature = epee::string_tools::pod_to_hex(ski[n].second); + res.signed_key_images[n].key_image = epee::string_tools::pod_to_hex(ski.second[n].first); + res.signed_key_images[n].signature = epee::string_tools::pod_to_hex(ski.second[n].second); } } @@ -2518,7 +2519,7 @@ namespace tools ski[n].second = *reinterpret_cast(bd.data()); } uint64_t spent = 0, unspent = 0; - uint64_t height = m_wallet->import_key_images(ski, spent, unspent); + uint64_t height = m_wallet->import_key_images(ski, req.offset, spent, unspent); res.spent = spent; res.unspent = unspent; res.height = height; diff --git a/src/wallet/wallet_rpc_server_commands_defs.h b/src/wallet/wallet_rpc_server_commands_defs.h index 924f3a0f1..026b75a9e 100644 --- a/src/wallet/wallet_rpc_server_commands_defs.h +++ b/src/wallet/wallet_rpc_server_commands_defs.h @@ -47,7 +47,7 @@ // advance which version they will stop working with // Don't go over 32767 for any of these #define WALLET_RPC_VERSION_MAJOR 1 -#define WALLET_RPC_VERSION_MINOR 5 +#define WALLET_RPC_VERSION_MINOR 6 #define MAKE_WALLET_RPC_VERSION(major,minor) (((major)<<16)|(minor)) #define WALLET_RPC_VERSION MAKE_WALLET_RPC_VERSION(WALLET_RPC_VERSION_MAJOR, WALLET_RPC_VERSION_MINOR) namespace tools @@ -1579,9 +1579,11 @@ namespace wallet_rpc struct response { + uint32_t offset; std::vector signed_key_images; BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(offset); KV_SERIALIZE(signed_key_images); END_KV_SERIALIZE_MAP() }; @@ -1602,9 +1604,11 @@ namespace wallet_rpc struct request { + uint32_t offset; std::vector signed_key_images; BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE_OPT(offset, (uint32_t)0); KV_SERIALIZE(signed_key_images); END_KV_SERIALIZE_MAP() }; diff --git a/tests/data/fuzz/cold-outputs/OUTPUTS2 b/tests/data/fuzz/cold-outputs/OUTPUTS2 index 907bcdb91..33cf39024 100644 Binary files a/tests/data/fuzz/cold-outputs/OUTPUTS2 and b/tests/data/fuzz/cold-outputs/OUTPUTS2 differ diff --git a/tests/fuzz/cold-outputs.cpp b/tests/fuzz/cold-outputs.cpp index 488a3b931..29b3ed267 100644 --- a/tests/fuzz/cold-outputs.cpp +++ b/tests/fuzz/cold-outputs.cpp @@ -77,7 +77,7 @@ int ColdOutputsFuzzer::run(const std::string &filename) s = std::string("\x01\x16serialization::archive") + s; try { - std::vector outputs; + std::pair> outputs; std::stringstream iss; iss << s; boost::archive::portable_binary_iarchive ar(iss); diff --git a/tests/unit_tests/serialization.cpp b/tests/unit_tests/serialization.cpp index 2f7b5aac7..e1404d637 100644 --- a/tests/unit_tests/serialization.cpp +++ b/tests/unit_tests/serialization.cpp @@ -908,9 +908,21 @@ TEST(Serialization, portability_outputs) ASSERT_TRUE(td2.m_pk_index == 0); } +struct unsigned_tx_set +{ + std::vector txes; + tools::wallet2::transfer_container transfers; +}; +template +inline void serialize(Archive &a, unsigned_tx_set &x, const boost::serialization::version_type ver) +{ + a & x.txes; + a & x.transfers; +} #define UNSIGNED_TX_PREFIX "Monero unsigned tx set\003" TEST(Serialization, portability_unsigned_tx) { + const boost::filesystem::path filename = unit_test::data_dir / "unsigned_monero_tx"; std::string s; const cryptonote::network_type nettype = cryptonote::TESTNET; @@ -918,7 +930,7 @@ TEST(Serialization, portability_unsigned_tx) ASSERT_TRUE(r); const size_t magiclen = strlen(UNSIGNED_TX_PREFIX); ASSERT_FALSE(strncmp(s.c_str(), UNSIGNED_TX_PREFIX, magiclen)); - tools::wallet2::unsigned_tx_set exported_txs; + unsigned_tx_set exported_txs; s = s.substr(magiclen); r = false; try