wallet: add shared ring database

This maps key images to rings, so that different forks can reuse
the rings by key image. This avoids revealing the real inputs like
would happen if two forks spent the same outputs with different
rings. This database is meant to be shared with all Monero forks
which don't bother making a new chain, putting users' privacy at
risk in the process. It is placed in a shared data directory by
default ($HOME/.shared-ringdb on UNIX like systems). You may
use --shared-ringdb-dir to override this location, and should
then do so for all Monero forks for them to share the database.

src/simplewallet/simplewallet.cpp

@@ -1294,6 +1294,58 @@ bool simple_wallet::export_raw_multisig(const std::vector<std::string> &args)
   return true;
 }
 
+bool simple_wallet::print_ring(const std::vector<std::string> &args)
+{
+  crypto::key_image key_image;
+  if (args.size() != 1)
+  {
+    fail_msg_writer() << tr("usage: print_ring <key_image>");
+    return true;
+  }
+
+  if (!epee::string_tools::hex_to_pod(args[0], key_image))
+  {
+    fail_msg_writer() << tr("Invalid key image");
+    return true;
+  }
+
+  std::vector<uint64_t> ring;
+  try
+  {
+    if (m_wallet->get_ring(m_wallet->get_ring_database(), key_image, ring))
+    {
+      std::stringstream str;
+      for (const auto &x: ring)
+        str << x << " ";
+      success_msg_writer() << tr("Ring size ") << std::to_string(ring.size()) << ": " << str.str();
+    }
+    else
+    {
+      fail_msg_writer() << tr("Key image either not spent, or spent with mixin 0");
+    }
+  }
+  catch (const std::exception &e)
+  {
+    fail_msg_writer() << tr("Failed to get key image ring: ") << e.what();
+  }
+
+  return true;
+}
+
+bool simple_wallet::save_known_rings(const std::vector<std::string> &args)
+{
+  try
+  {
+    LOCK_IDLE_SCOPE();
+    m_wallet->find_and_save_rings(m_wallet->get_ring_database());
+  }
+  catch (const std::exception &e)
+  {
+    fail_msg_writer() << tr("Failed to save known rings: ") << e.what();
+  }
+  return true;
+}
+
 bool simple_wallet::set_always_confirm_transfers(const std::vector<std::string> &args/* = std::vector<std::string>()*/)
 {
   const auto pwd_container = get_and_verify_password();
@@ -1951,6 +2003,14 @@ simple_wallet::simple_wallet()
                            boost::bind(&simple_wallet::export_raw_multisig, this, _1),
                            tr("export_raw_multisig_tx <filename>"),
                            tr("Export a signed multisig transaction to a file"));
+  m_cmd_binder.set_handler("print_ring",
+                           boost::bind(&simple_wallet::print_ring, this, _1),
+                           tr("print_ring <key_image>"),
+                           tr("Print the ring used to spend a given key image (if the ring size is > 1)"));
+  m_cmd_binder.set_handler("save_known_rings",
+                           boost::bind(&simple_wallet::save_known_rings, this, _1),
+                           tr("save_known_rings"),
+                           tr("Save known rings to the shared rings database"));
   m_cmd_binder.set_handler("help",
                            boost::bind(&simple_wallet::help, this, _1),
                            tr("help [<command>]"),

src/simplewallet/simplewallet.h

@@ -208,6 +208,8 @@ namespace cryptonote
     bool sign_multisig(const std::vector<std::string>& args);
     bool submit_multisig(const std::vector<std::string>& args);
     bool export_raw_multisig(const std::vector<std::string>& args);
+    bool print_ring(const std::vector<std::string>& args);
+    bool save_known_rings(const std::vector<std::string>& args);
 
     uint64_t get_daemon_blockchain_height(std::string& err);
     bool try_connect_to_daemon(bool silent = false, uint32_t* version = nullptr);

src/wallet/CMakeLists.txt

@@ -33,6 +33,7 @@ set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib)
 set(wallet_sources
   wallet2.cpp
   wallet_args.cpp
+  ringdb.cpp
   node_rpc_proxy.cpp)
 
 set(wallet_private_headers
@@ -42,6 +43,7 @@ set(wallet_private_headers
   wallet_rpc_server.h
   wallet_rpc_server_commands_defs.h
   wallet_rpc_server_error_codes.h
+  ringdb.h
   node_rpc_proxy.h)
 
 monero_private_headers(wallet
@@ -55,6 +57,7 @@ target_link_libraries(wallet
     common
     cryptonote_core
     mnemonics
+    ${LMDB_LIBRARY}
     ${Boost_CHRONO_LIBRARY}
     ${Boost_SERIALIZATION_LIBRARY}
     ${Boost_FILESYSTEM_LIBRARY}

src/wallet/ringdb.cpp

@@ -0,0 +1,340 @@
+// Copyright (c) 2018, The Monero Project
+// 
+// All rights reserved.
+// 
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+// 
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+// 
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+// 
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+// 
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include <lmdb.h>
+#include <boost/algorithm/string.hpp>
+#include <boost/range/adaptor/transformed.hpp>
+#include <boost/filesystem.hpp>
+#include "misc_log_ex.h"
+#include "misc_language.h"
+#include "wallet_errors.h"
+#include "ringdb.h"
+
+#undef MONERO_DEFAULT_LOG_CATEGORY
+#define MONERO_DEFAULT_LOG_CATEGORY "wallet.ringdb"
+
+static int compare_hash32(const MDB_val *a, const MDB_val *b)
+{
+  uint32_t *va = (uint32_t*) a->mv_data;
+  uint32_t *vb = (uint32_t*) b->mv_data;
+  for (int n = 7; n >= 0; n--)
+  {
+    if (va[n] == vb[n])
+      continue;
+    return va[n] < vb[n] ? -1 : 1;
+  }
+
+  return 0;
+}
+
+static std::string compress_ring(const std::vector<uint64_t> &ring)
+{
+  std::string s;
+  for (uint64_t out: ring)
+    s += tools::get_varint_data(out);
+  return s;
+}
+
+static std::vector<uint64_t> decompress_ring(const std::string &s)
+{
+  std::vector<uint64_t> ring;
+  int read = 0;
+  for (std::string::const_iterator i = s.begin(); i != s.cend(); std::advance(i, read))
+  {
+    uint64_t out;
+    std::string tmp(i, s.cend());
+    read = tools::read_varint(tmp.begin(), tmp.end(), out);
+    THROW_WALLET_EXCEPTION_IF(read <= 0 || read > 256, tools::error::wallet_internal_error, "Internal error decompressing ring");
+    ring.push_back(out);
+  }
+  return ring;
+}
+
+std::string get_rings_filename(boost::filesystem::path filename)
+{
+  if (!boost::filesystem::is_directory(filename))
+    filename.remove_filename();
+  return filename.string();
+}
+
+static crypto::chacha_iv make_iv(const crypto::key_image &key_image, const crypto::chacha_key &key)
+{
+  static const char salt[] = "ringdsb";
+
+  uint8_t buffer[sizeof(key_image) + sizeof(key) + sizeof(salt)];
+  memcpy(buffer, &key_image, sizeof(key_image));
+  memcpy(buffer + sizeof(key_image), &key, sizeof(key));
+  memcpy(buffer + sizeof(key_image) + sizeof(key), salt, sizeof(salt));
+  crypto::hash hash;
+  crypto::cn_fast_hash(buffer, sizeof(buffer), hash.data);
+  static_assert(sizeof(hash) >= CHACHA_IV_SIZE, "Incompatible hash and chacha IV sizes");
+  crypto::chacha_iv iv;
+  memcpy(&iv, &hash, CHACHA_IV_SIZE);
+  return iv;
+}
+
+static std::string encrypt(const std::string &plaintext, const crypto::key_image &key_image, const crypto::chacha_key &key)
+{
+  const crypto::chacha_iv iv = make_iv(key_image, key);
+  std::string ciphertext;
+  ciphertext.resize(plaintext.size() + sizeof(iv));
+  crypto::chacha20(plaintext.data(), plaintext.size(), key, iv, &ciphertext[sizeof(iv)]);
+  memcpy(&ciphertext[0], &iv, sizeof(iv));
+  return ciphertext;
+}
+
+static std::string encrypt(const crypto::key_image &key_image, const crypto::chacha_key &key)
+{
+  return encrypt(std::string((const char*)&key_image, sizeof(key_image)), key_image, key);
+}
+
+static std::string decrypt(const std::string &ciphertext, const crypto::key_image &key_image, const crypto::chacha_key &key)
+{
+  const crypto::chacha_iv iv = make_iv(key_image, key);
+  std::string plaintext;
+  THROW_WALLET_EXCEPTION_IF(ciphertext.size() < sizeof(iv), tools::error::wallet_internal_error, "Bad ciphertext text");
+  plaintext.resize(ciphertext.size() - sizeof(iv));
+  crypto::chacha20(ciphertext.data() + sizeof(iv), ciphertext.size() - sizeof(iv), key, iv, &plaintext[0]);
+  return plaintext;
+}
+
+static int resize_env(MDB_env *env, const char *db_path, size_t n_entries)
+{
+  MDB_envinfo mei;
+  MDB_stat mst;
+  int ret;
+
+  ret = mdb_env_info(env, &mei);
+  if (ret)
+    return ret;
+  ret = mdb_env_stat(env, &mst);
+  if (ret)
+    return ret;
+  uint64_t size_used = mst.ms_psize * mei.me_last_pgno;
+  const size_t needed = n_entries * (32 + 1024); // highball 1kB for the ring data to make sure
+  uint64_t mapsize = mei.me_mapsize;
+  if (size_used + needed > mei.me_mapsize)
+  {
+    try
+    {
+      boost::filesystem::path path(db_path);
+      boost::filesystem::space_info si = boost::filesystem::space(path);
+      if(si.available < needed)
+      {
+        MERROR("!! WARNING: Insufficient free space to extend database !!: " << (si.available >> 20L) << " MB available");
+        return ENOSPC;
+      }
+    }
+    catch(...)
+    {
+      // print something but proceed.
+      MWARNING("Unable to query free disk space.");
+    }
+
+    mapsize += needed;
+  }
+  return mdb_env_set_mapsize(env, mapsize);
+}
+
+namespace tools { namespace ringdb
+{
+
+bool add_rings(const std::string &filename, const crypto::chacha_key &chacha_key, const cryptonote::transaction_prefix &tx)
+{
+  MDB_env *env;
+  MDB_dbi dbi;
+  MDB_txn *txn;
+  int dbr;
+  bool tx_active = false;
+
+  if (filename.empty())
+    return true;
+  tools::create_directories_if_necessary(filename);
+
+  dbr = mdb_env_create(&env);
+  THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to create LDMB environment: " + std::string(mdb_strerror(dbr)));
+  dbr = mdb_env_set_maxdbs(env, 1);
+  THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to set max env dbs: " + std::string(mdb_strerror(dbr)));
+  dbr = mdb_env_open(env, get_rings_filename(filename).c_str(), 0, 0664);
+  THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to open rings database file: " + std::string(mdb_strerror(dbr)));
+  epee::misc_utils::auto_scope_leave_caller env_dtor = epee::misc_utils::create_scope_leave_handler([&](){mdb_env_close(env);});
+  dbr = resize_env(env, filename.c_str(), tx.vin.size());
+  THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to set env map size: " + std::string(mdb_strerror(dbr)));
+  dbr = mdb_txn_begin(env, NULL, 0, &txn);
+  THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr)));
+  epee::misc_utils::auto_scope_leave_caller txn_dtor = epee::misc_utils::create_scope_leave_handler([&](){if (tx_active) mdb_txn_abort(txn);});
+  tx_active = true;
+  dbr = mdb_dbi_open(txn, "rings", MDB_CREATE, &dbi);
+  THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr)));
+  epee::misc_utils::auto_scope_leave_caller dbi_dtor = epee::misc_utils::create_scope_leave_handler([&](){mdb_dbi_close(env, dbi);});
+  mdb_set_compare(txn, dbi, compare_hash32);
+
+  for (const auto &in: tx.vin)
+  {
+    if (in.type() != typeid(cryptonote::txin_to_key))
+      continue;
+    const auto &txin = boost::get<cryptonote::txin_to_key>(in);
+    const uint32_t ring_size = txin.key_offsets.size();
+    if (ring_size == 1)
+      continue;
+
+    MDB_val key, data;
+    std::string key_ciphertext = encrypt(txin.k_image, chacha_key);
+    key.mv_data = (void*)key_ciphertext.data();
+    key.mv_size = key_ciphertext.size();
+    MDEBUG("Saving relative ring for key image " << txin.k_image << ": " <<
+        boost::join(txin.key_offsets | boost::adaptors::transformed([](uint64_t out){return std::to_string(out);}), " "));
+    std::string compressed_ring = compress_ring(txin.key_offsets);
+    std::string data_ciphertext = encrypt(compressed_ring, txin.k_image, chacha_key);
+    data.mv_size = data_ciphertext.size();
+    data.mv_data = (void*)data_ciphertext.c_str();
+    dbr = mdb_put(txn, dbi, &key, &data, 0);
+    THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to add ring to database: " + std::string(mdb_strerror(dbr)));
+  }
+
+  dbr = mdb_txn_commit(txn);
+  THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to commit txn adding ring to database: " + std::string(mdb_strerror(dbr)));
+  tx_active = false;
+  return true;
+}
+
+bool remove_rings(const std::string &filename, const crypto::chacha_key &chacha_key, const cryptonote::transaction_prefix &tx)
+{
+  MDB_env *env;
+  MDB_dbi dbi;
+  MDB_txn *txn;
+  int dbr;
+  bool tx_active = false;
+
+  if (filename.empty())
+    return true;
+  tools::create_directories_if_necessary(filename);
+
+  dbr = mdb_env_create(&env);
+  THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to create LDMB environment: " + std::string(mdb_strerror(dbr)));
+  dbr = mdb_env_set_maxdbs(env, 1);
+  THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to set max env dbs: " + std::string(mdb_strerror(dbr)));
+  dbr = mdb_env_open(env, get_rings_filename(filename).c_str(), 0, 0664);
+  THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to open rings database file: " + std::string(mdb_strerror(dbr)));
+  epee::misc_utils::auto_scope_leave_caller env_dtor = epee::misc_utils::create_scope_leave_handler([&](){mdb_env_close(env);});
+  dbr = resize_env(env, filename.c_str(), tx.vin.size());
+  THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to set env map size: " + std::string(mdb_strerror(dbr)));
+  dbr = mdb_txn_begin(env, NULL, 0, &txn);
+  THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr)));
+  epee::misc_utils::auto_scope_leave_caller txn_dtor = epee::misc_utils::create_scope_leave_handler([&](){if (tx_active) mdb_txn_abort(txn);});
+  tx_active = true;
+  dbr = mdb_dbi_open(txn, "rings", MDB_CREATE, &dbi);
+  THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr)));
+  epee::misc_utils::auto_scope_leave_caller dbi_dtor = epee::misc_utils::create_scope_leave_handler([&](){mdb_dbi_close(env, dbi);});
+  mdb_set_compare(txn, dbi, compare_hash32);
+
+  for (const auto &in: tx.vin)
+  {
+    if (in.type() != typeid(cryptonote::txin_to_key))
+      continue;
+    const auto &txin = boost::get<cryptonote::txin_to_key>(in);
+    const uint32_t ring_size = txin.key_offsets.size();
+    if (ring_size == 1)
+      continue;
+
+    MDB_val key, data;
+    std::string key_ciphertext = encrypt(txin.k_image, chacha_key);
+    key.mv_data = (void*)key_ciphertext.data();
+    key.mv_size = key_ciphertext.size();
+
+    dbr = mdb_get(txn, dbi, &key, &data);
+    THROW_WALLET_EXCEPTION_IF(dbr && dbr != MDB_NOTFOUND, tools::error::wallet_internal_error, "Failed to look for key image in LMDB table: " + std::string(mdb_strerror(dbr)));
+    if (dbr == MDB_NOTFOUND)
+      continue;
+    THROW_WALLET_EXCEPTION_IF(data.mv_size <= 0, tools::error::wallet_internal_error, "Invalid ring data size");
+
+    MDEBUG("Removing ring data for key image " << txin.k_image);
+    dbr = mdb_del(txn, dbi, &key, NULL);
+    THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to remove ring to database: " + std::string(mdb_strerror(dbr)));
+  }
+
+  dbr = mdb_txn_commit(txn);
+  THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to commit txn removing ring to database: " + std::string(mdb_strerror(dbr)));
+  tx_active = false;
+  return true;
+}
+
+bool get_ring(const std::string &filename, const crypto::chacha_key &chacha_key, const crypto::key_image &key_image, std::vector<uint64_t> &outs)
+{
+  MDB_env *env;
+  MDB_dbi dbi;
+  MDB_txn *txn;
+  int dbr;
+  bool tx_active = false;
+
+  if (filename.empty())
+    return false;
+  tools::create_directories_if_necessary(filename);
+
+  dbr = mdb_env_create(&env);
+  THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to create LDMB environment: " + std::string(mdb_strerror(dbr)));
+  dbr = mdb_env_set_maxdbs(env, 1);
+  THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to set max env dbs: " + std::string(mdb_strerror(dbr)));
+  dbr = mdb_env_open(env, get_rings_filename(filename).c_str(), 0, 0664);
+  THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to open rings database file: " + std::string(mdb_strerror(dbr)));
+  epee::misc_utils::auto_scope_leave_caller env_dtor = epee::misc_utils::create_scope_leave_handler([&](){mdb_env_close(env);});
+  dbr = resize_env(env, filename.c_str(), 0);
+  THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to set env map size: " + std::string(mdb_strerror(dbr)));
+  dbr = mdb_txn_begin(env, NULL, 0, &txn);
+  THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr)));
+  epee::misc_utils::auto_scope_leave_caller txn_dtor = epee::misc_utils::create_scope_leave_handler([&](){if (tx_active) mdb_txn_abort(txn);});
+  tx_active = true;
+  dbr = mdb_dbi_open(txn, "rings", MDB_CREATE, &dbi);
+  THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr)));
+  epee::misc_utils::auto_scope_leave_caller dbi_dtor = epee::misc_utils::create_scope_leave_handler([&](){mdb_dbi_close(env, dbi);});
+  mdb_set_compare(txn, dbi, compare_hash32);
+
+  MDB_val key, data;
+  std::string key_ciphertext = encrypt(key_image, chacha_key);
+  key.mv_data = (void*)key_ciphertext.data();
+  key.mv_size = key_ciphertext.size();
+  dbr = mdb_get(txn, dbi, &key, &data);
+  THROW_WALLET_EXCEPTION_IF(dbr && dbr != MDB_NOTFOUND, tools::error::wallet_internal_error, "Failed to look for key image in LMDB table: " + std::string(mdb_strerror(dbr)));
+  if (dbr == MDB_NOTFOUND)
+    return false;
+  THROW_WALLET_EXCEPTION_IF(data.mv_size <= 0, tools::error::wallet_internal_error, "Invalid ring data size");
+
+  std::string data_plaintext = decrypt(std::string((const char*)data.mv_data, data.mv_size), key_image, chacha_key);
+  outs = decompress_ring(data_plaintext);
+  MDEBUG("Found ring for key image " << key_image << ":");
+  MDEBUG("Relative: " << boost::join(outs | boost::adaptors::transformed([](uint64_t out){return std::to_string(out);}), " "));
+  outs = cryptonote::relative_output_offsets_to_absolute(outs);
+  MDEBUG("Absolute: " << boost::join(outs | boost::adaptors::transformed([](uint64_t out){return std::to_string(out);}), " "));
+
+  dbr = mdb_txn_commit(txn);
+  THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to commit txn getting ring from database: " + std::string(mdb_strerror(dbr)));
+  tx_active = false;
+  return true;
+}
+
+}}

src/wallet/ringdb.h

@@ -0,0 +1,45 @@
+// Copyright (c) 2018, The Monero Project
+// 
+// All rights reserved.
+// 
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+// 
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+// 
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+// 
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+// 
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#pragma once
+
+#include <string>
+#include <vector>
+#include "wipeable_string.h"
+#include "crypto/crypto.h"
+#include "cryptonote_basic/cryptonote_basic.h"
+
+namespace tools
+{
+  namespace ringdb
+  {
+    bool add_rings(const std::string &filename, const crypto::chacha_key &chacha_key, const cryptonote::transaction_prefix &tx);
+    bool remove_rings(const std::string &filename, const crypto::chacha_key &chacha_key, const cryptonote::transaction_prefix &tx);
+    bool get_ring(const std::string &filename, const crypto::chacha_key &chacha_key, const crypto::key_image &key_image, std::vector<uint64_t> &outs);
+  }
+}

src/wallet/wallet2.cpp

@@ -66,6 +66,7 @@ using namespace epee;
 #include "memwipe.h"
 #include "common/base58.h"
 #include "ringct/rctSigs.h"
+#include "ringdb.h"
 
 extern "C"
 {
@@ -106,6 +107,18 @@ using namespace cryptonote;
 
 #define MULTISIG_EXPORT_FILE_MAGIC "Monero multisig export\001"
 
+namespace
+{
+  std::string get_default_ringdb_path()
+  {
+    boost::filesystem::path dir = tools::get_default_data_dir();
+    // remove .bitmonero, replace with .shared-ringdb
+    dir = dir.remove_filename();
+    dir /= ".shared-ringdb";
+    return dir.string();
+  }
+}
+
 namespace
 {
 // Create on-demand to prevent static initialization order fiasco issues.
@@ -119,6 +132,16 @@ struct options {
   const command_line::arg_descriptor<bool> testnet = {"testnet", tools::wallet2::tr("For testnet. Daemon must also be launched with --testnet flag"), false};
   const command_line::arg_descriptor<bool> stagenet = {"stagenet", tools::wallet2::tr("For stagenet. Daemon must also be launched with --stagenet flag"), false};
   const command_line::arg_descriptor<bool> restricted = {"restricted-rpc", tools::wallet2::tr("Restricts to view-only commands"), false};
+  const command_line::arg_descriptor<std::string, false, true> shared_ringdb_dir = {
+    "shared-ringdb-dir", tools::wallet2::tr("Set shared ring database path"),
+    get_default_ringdb_path(),
+    testnet,
+    [](bool testnet, bool defaulted, std::string val) {
+      if (testnet)
+        return (boost::filesystem::path(val) / "testnet").string();
+      return val;
+    }
+  };
 };
 
 void do_prepare_file_names(const std::string& file_path, std::string& keys_file, std::string& wallet_file)
@@ -196,6 +219,8 @@ std::unique_ptr<tools::wallet2> make_basic(const boost::program_options::variabl
 
   std::unique_ptr<tools::wallet2> wallet(new tools::wallet2(testnet ? TESTNET : stagenet ? STAGENET : MAINNET, restricted));
   wallet->init(std::move(daemon_address), std::move(login));
+  boost::filesystem::path ringdb_path = command_line::get_arg(vm, opts.shared_ringdb_dir);
+  wallet->set_ring_database(ringdb_path.string());
   return wallet;
 }
 
@@ -639,7 +664,8 @@ wallet2::wallet2(network_type nettype, bool restricted):
   m_light_wallet_connected(false),
   m_light_wallet_balance(0),
   m_light_wallet_unlocked_balance(0),
-  m_key_on_device(false)
+  m_key_on_device(false),
+  m_ring_history_saved(false)
 {
 }
 
@@ -665,6 +691,7 @@ void wallet2::init_options(boost::program_options::options_description& desc_par
   command_line::add_arg(desc_params, opts.testnet);
   command_line::add_arg(desc_params, opts.stagenet);
   command_line::add_arg(desc_params, opts.restricted);
+  command_line::add_arg(desc_params, opts.shared_ringdb_dir);
 }
 
 std::unique_ptr<wallet2> wallet2::make_from_json(const boost::program_options::variables_map& vm, const std::string& json_file, const std::function<boost::optional<tools::password_container>(const char *, bool)> &password_prompter)
@@ -1482,6 +1509,8 @@ void wallet2::process_outgoing(const crypto::hash &txid, const cryptonote::trans
   entry.first->second.m_block_height = height;
   entry.first->second.m_timestamp = ts;
   entry.first->second.m_unlock_time = tx.unlock_time;
+
+  add_rings(get_ring_database(), tx);
 }
 //----------------------------------------------------------------------------------------------------
 void wallet2::process_new_blockchain_entry(const cryptonote::block& b, const cryptonote::block_complete_entry& bche, const crypto::hash& bl_id, uint64_t height, const cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices &o_indices)
@@ -1852,6 +1881,7 @@ void wallet2::update_pool_state(bool refreshed)
         pit->second.m_state = wallet2::unconfirmed_transfer_details::failed;
 
         // the inputs aren't spent anymore, since the tx failed
+        remove_rings(m_ring_database, pit->second.m_tx);
         for (size_t vini = 0; vini < pit->second.m_tx.vin.size(); ++vini)
         {
           if (pit->second.m_tx.vin[vini].type() == typeid(txin_to_key))
@@ -3742,6 +3772,15 @@ void wallet2::load(const std::string& wallet_, const epee::wipeable_string& pass
     add_subaddress_account(tr("Primary account"));
 
   m_local_bc_height = m_blockchain.size();
+
+  try
+  {
+    find_and_save_rings(get_ring_database(), false);
+  }
+  catch (const std::exception &e)
+  {
+    MERROR("Failed to save rings, will try again next time");
+  }
 }
 //----------------------------------------------------------------------------------------------------
 void wallet2::trim_hashchain()
@@ -4483,6 +4522,8 @@ void wallet2::commit_tx(pending_tx& ptx)
     m_additional_tx_keys.insert(std::make_pair(txid, ptx.additional_tx_keys));
   }
 
+  add_rings(m_ring_database, ptx.tx);
+
   LOG_PRINT_L2("transaction " << txid << " generated ok and sent to daemon, key_images: [" << ptx.key_images << "]");
 
   for(size_t idx: ptx.selected_transfers)
@@ -5385,6 +5426,102 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions(std::vector<crypto
   }
 }
 
+void wallet2::set_ring_database(const std::string &filename)
+{
+  m_ring_database = filename;
+  MINFO("ringdb path set to " << filename);
+}
+
+bool wallet2::add_rings(const std::string &filename, const crypto::chacha_key &key, const cryptonote::transaction_prefix &tx)
+{
+  return ringdb::add_rings(filename, key, tx);
+}
+
+bool wallet2::add_rings(const std::string &filename, const cryptonote::transaction_prefix &tx)
+{
+  crypto::chacha_key key;
+  generate_chacha_key_from_secret_keys(key);
+  return add_rings(filename, key, tx);
+}
+
+bool wallet2::remove_rings(const std::string &filename, const cryptonote::transaction_prefix &tx)
+{
+  crypto::chacha_key key;
+  generate_chacha_key_from_secret_keys(key);
+  return ringdb::remove_rings(filename, key, tx);
+}
+
+bool wallet2::get_ring(const std::string &filename, const crypto::chacha_key &key, const crypto::key_image &key_image, std::vector<uint64_t> &outs)
+{
+  return ringdb::get_ring(filename, key, key_image, outs);
+}
+
+bool wallet2::get_ring(const std::string &filename, const crypto::key_image &key_image, std::vector<uint64_t> &outs)
+{
+  crypto::chacha_key key;
+  generate_chacha_key_from_secret_keys(key);
+
+  return get_ring(filename, key, key_image, outs);
+}
+
+bool wallet2::find_and_save_rings(const std::string &filename, bool force)
+{
+  if (!force && m_ring_history_saved)
+    return true;
+
+  COMMAND_RPC_GET_TRANSACTIONS::request req = AUTO_VAL_INIT(req);
+  COMMAND_RPC_GET_TRANSACTIONS::response res = AUTO_VAL_INIT(res);
+
+  MDEBUG("Finding and saving rings...");
+
+  // get payments we made
+  std::list<std::pair<crypto::hash,wallet2::confirmed_transfer_details>> payments;
+  get_payments_out(payments, 0, std::numeric_limits<uint64_t>::max(), boost::none, std::set<uint32_t>());
+  for (const std::pair<crypto::hash,wallet2::confirmed_transfer_details> &entry: payments)
+  {
+    const crypto::hash &txid = entry.first;
+    req.txs_hashes.push_back(epee::string_tools::pod_to_hex(txid));
+  }
+
+  MDEBUG("Found " << std::to_string(req.txs_hashes.size()) << " transactions");
+
+  // get those transactions from the daemon
+  req.decode_as_json = false;
+  bool r;
+  {
+    const boost::lock_guard<boost::mutex> lock{m_daemon_rpc_mutex};
+    r = epee::net_utils::invoke_http_json("/gettransactions", req, res, m_http_client, rpc_timeout);
+  }
+  THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "gettransactions");
+  THROW_WALLET_EXCEPTION_IF(res.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "gettransactions");
+  THROW_WALLET_EXCEPTION_IF(res.status != CORE_RPC_STATUS_OK, error::wallet_internal_error, "gettransactions");
+  THROW_WALLET_EXCEPTION_IF(res.txs.size() != req.txs_hashes.size(), error::wallet_internal_error,
+    "daemon returned wrong response for gettransactions, wrong txs count = " +
+    std::to_string(res.txs.size()) + ", expected " + std::to_string(req.txs_hashes.size()));
+
+  MDEBUG("Scanning " << res.txs.size() << " transactions");
+
+  crypto::chacha_key key;
+  generate_chacha_key_from_secret_keys(key);
+
+  auto it = req.txs_hashes.begin();
+  for (size_t i = 0; i < res.txs.size(); ++i, ++it)
+  {
+    const auto &tx_info = res.txs[i];
+    THROW_WALLET_EXCEPTION_IF(tx_info.tx_hash != *it, error::wallet_internal_error, "Wrong txid received");
+    cryptonote::blobdata bd;
+    THROW_WALLET_EXCEPTION_IF(!epee::string_tools::parse_hexstr_to_binbuff(tx_info.as_hex, bd), error::wallet_internal_error, "failed to parse tx from hexstr");
+    cryptonote::transaction tx;
+    crypto::hash tx_hash, tx_prefix_hash;
+    THROW_WALLET_EXCEPTION_IF(!cryptonote::parse_and_validate_tx_from_blob(bd, tx, tx_hash, tx_prefix_hash), error::wallet_internal_error, "failed to parse tx from blob");
+    THROW_WALLET_EXCEPTION_IF(epee::string_tools::pod_to_hex(tx_hash) != tx_info.tx_hash, error::wallet_internal_error, "txid mismatch");
+    THROW_WALLET_EXCEPTION_IF(!add_rings(filename, key, tx), error::wallet_internal_error, "Failed to save ring");
+  }
+
+  MINFO("Found and saved rings for " << res.txs.size() << " transactions");
+  m_ring_history_saved = true;
+  return true;
+}
 
 bool wallet2::tx_add_fake_output(std::vector<std::vector<tools::wallet2::get_outs_entry>> &outs, uint64_t global_index, const crypto::public_key& tx_public_key, const rct::key& mask, uint64_t real_index, bool unlocked) const
 {
@@ -5514,6 +5651,9 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>>
     return;
   }
 
+  crypto::chacha_key key;
+  generate_chacha_key_from_secret_keys(key);
+
   if (fake_outputs_count > 0)
   {
     // get histogram for the amounts we need
@@ -5582,7 +5722,48 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>>
         --recent_outputs_count; // if the real out is recent, pick one less recent fake out
       LOG_PRINT_L1("Using " << recent_outputs_count << " recent outputs");
 
-      if (num_outs <= requested_outputs_count)
+      uint64_t num_found = 0;
+
+      // if we have a known ring, use it
+      bool existing_ring_found = false;
+      if (td.m_key_image_known && !td.m_key_image_partial)
+      {
+        std::vector<uint64_t> ring;
+        if (get_ring(get_ring_database(), key, td.m_key_image, ring))
+        {
+          MINFO("This output has a known ring, reusing (size " << ring.size() << ")");
+          THROW_WALLET_EXCEPTION_IF(ring.size() > fake_outputs_count + 1, error::wallet_internal_error,
+              "An output in this transaction was previously spent on another chain with ring size " +
+              std::to_string(ring.size()) + ", it cannot be spent now with ring size " +
+              std::to_string(fake_outputs_count + 1) + " as it is smaller: use a higher ring size");
+          bool own_found = false;
+          existing_ring_found = true;
+          for (const auto &out: ring)
+          {
+            MINFO("Ring has output " << out);
+            if (out < num_outs)
+            {
+              MINFO("Using it");
+              req.outputs.push_back({amount, out});
+              ++num_found;
+              seen_indices.emplace(out);
+              if (out == td.m_global_output_index)
+              {
+                MINFO("This is the real output");
+                own_found = true;
+              }
+            }
+            else
+            {
+              MINFO("Ignoring output " << out << ", too recent");
+            }
+          }
+          THROW_WALLET_EXCEPTION_IF(!own_found, error::wallet_internal_error,
+              "Known ring does not include the spent output: " + std::to_string(td.m_global_output_index));
+        }
+      }
+
+      if (num_outs <= requested_outputs_count && !existing_ring_found)
       {
         for (uint64_t i = 0; i < num_outs; i++)
           req.outputs.push_back({amount, i});
@@ -5595,10 +5776,13 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>>
       else
       {
         // start with real one
-        uint64_t num_found = 1;
-        seen_indices.emplace(td.m_global_output_index);
-        req.outputs.push_back({amount, td.m_global_output_index});
-        LOG_PRINT_L1("Selecting real output: " << td.m_global_output_index << " for " << print_money(amount));
+        if (num_found == 0)
+        {
+          num_found = 1;
+          seen_indices.emplace(td.m_global_output_index);
+          req.outputs.push_back({amount, td.m_global_output_index});
+          LOG_PRINT_L1("Selecting real output: " << td.m_global_output_index << " for " << print_money(amount));
+        }
 
         // while we still need more mixins
         while (num_found < requested_outputs_count)
@@ -5674,6 +5858,17 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>>
       outs.back().reserve(fake_outputs_count + 1);
       const rct::key mask = td.is_rct() ? rct::commit(td.amount(), td.m_mask) : rct::zeroCommit(td.amount());
 
+      uint64_t num_outs = 0;
+      const uint64_t amount = td.is_rct() ? 0 : td.amount();
+      for (const auto &he: resp_t.histogram)
+      {
+        if (he.amount == amount)
+        {
+          num_outs = he.unlocked_instances;
+          break;
+        }
+      }
+
       // make sure the real outputs we asked for are really included, along
       // with the correct key and mask: this guards against an active attack
       // where the node sends dummy data for all outputs, and we then send
@@ -5694,6 +5889,38 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>>
       // pick real out first (it will be sorted when done)
       outs.back().push_back(std::make_tuple(td.m_global_output_index, boost::get<txout_to_key>(td.m_tx.vout[td.m_internal_output_index].target).key, mask));
 
+      // then pick outs from an existing ring, if any
+      bool existing_ring_found = false;
+      if (td.m_key_image_known && !td.m_key_image_partial)
+      {
+        std::vector<uint64_t> ring;
+        if (get_ring(get_ring_database(), key, td.m_key_image, ring))
+        {
+          for (uint64_t out: ring)
+          {
+            if (out < num_outs)
+            {
+              if (out != td.m_global_output_index)
+              {
+                bool found = false;
+                for (size_t o = 0; o < requested_outputs_count; ++o)
+                {
+                  size_t i = base + o;
+                  if (req.outputs[i].index == out)
+                  {
+                    LOG_PRINT_L2("Index " << i << "/" << requested_outputs_count << ": idx " << req.outputs[i].index << " (real " << td.m_global_output_index << "), unlocked " << daemon_resp.outs[i].unlocked << ", key " << daemon_resp.outs[i].key << " (from existing ring)");
+                    tx_add_fake_output(outs, req.outputs[i].index, daemon_resp.outs[i].key, daemon_resp.outs[i].mask, td.m_global_output_index, daemon_resp.outs[i].unlocked);
+                    found = true;
+                    break;
+                  }
+                }
+                THROW_WALLET_EXCEPTION_IF(!found, error::wallet_internal_error, "Falied to find existing ring output in daemon out data");
+              }
+            }
+          }
+        }
+      }
+
       // then pick others in random order till we reach the required number
       // since we use an equiprobable pick here, we don't upset the triangular distribution
       std::vector<size_t> order;

src/wallet/wallet2.h

@@ -810,6 +810,9 @@ namespace tools
       if(ver < 23)
         return;
       a & m_account_tags;
+      if(ver < 24)
+        return;
+      a & m_ring_history_saved;
     }
 
     /*!
@@ -1046,6 +1049,11 @@ namespace tools
       return epee::net_utils::invoke_http_json_rpc(uri, method_name, req, res, m_http_client, timeout, http_method, req_id);
     }
 
+    void set_ring_database(const std::string &filename);
+    const std::string get_ring_database() const { return m_ring_database; }
+    bool get_ring(const std::string &filename, const crypto::key_image &key_image, std::vector<uint64_t> &outs);
+    bool find_and_save_rings(const std::string &filename, bool force = true);
+
   private:
     /*!
      * \brief  Stores wallet information to wallet file.
@@ -1102,6 +1110,10 @@ namespace tools
     rct::multisig_kLRki get_multisig_kLRki(size_t n, const rct::key &k) const;
     rct::key get_multisig_k(size_t idx, const std::unordered_set<rct::key> &used_L) const;
     void update_multisig_rescan_info(const std::vector<std::vector<rct::key>> &multisig_k, const std::vector<std::vector<tools::wallet2::multisig_info>> &info, size_t n);
+    bool add_rings(const std::string &filename, const crypto::chacha_key &key, const cryptonote::transaction_prefix &tx);
+    bool add_rings(const std::string &filename, const cryptonote::transaction_prefix &tx);
+    bool remove_rings(const std::string &filename, const cryptonote::transaction_prefix &tx);
+    bool get_ring(const std::string &filename, const crypto::chacha_key &key, const crypto::key_image &key_image, std::vector<uint64_t> &outs);
 
     bool get_output_distribution(uint64_t &start_height, std::vector<uint64_t> &distribution);
 
@@ -1187,9 +1199,12 @@ namespace tools
     std::unordered_map<crypto::hash, address_tx> m_light_wallet_address_txs;
     // store calculated key image for faster lookup
     std::unordered_map<crypto::public_key, std::map<uint64_t, crypto::key_image> > m_key_image_cache;
+
+    std::string m_ring_database;
+    bool m_ring_history_saved;
   };
 }
-BOOST_CLASS_VERSION(tools::wallet2, 23)
+BOOST_CLASS_VERSION(tools::wallet2, 24)
 BOOST_CLASS_VERSION(tools::wallet2::transfer_details, 9)
 BOOST_CLASS_VERSION(tools::wallet2::multisig_info, 1)
 BOOST_CLASS_VERSION(tools::wallet2::multisig_info::LR, 0)

src/wallet/wallet_errors.h

@@ -36,6 +36,7 @@
 #include <vector>
 
 #include "cryptonote_basic/cryptonote_format_utils.h"
+#include "cryptonote_core/cryptonote_tx_utils.h"
 #include "rpc/core_rpc_server_commands_defs.h"
 #include "include_base_utils.h"