Merge pull request #5211

c9b13fbb tests/trezor: HF9 and HF10 tests (Dusan Klinec)
a1fd1d49 device/trezor: HF10 support added, wallet::API (Dusan Klinec)
d74d26f2 crypto: hmac_keccak added (Dusan Klinec)

external/trezor-common

@@ -1 +1 @@
-Subproject commit 588f8e03f5ac111adf719f0a437de67481a26aed
+Subproject commit cb238cb1f134accc4200217d9511115a8f61c6cb

src/crypto/CMakeLists.txt

@@ -39,6 +39,7 @@ set(crypto_sources
   hash-extra-jh.c
   hash-extra-skein.c
   hash.c
+  hmac-keccak.c
   jh.c
   keccak.c
   oaes_lib.c
@@ -64,6 +65,7 @@ set(crypto_private_headers
   groestl_tables.h
   hash-ops.h
   hash.h
+  hmac-keccak.h
   initializer.h
   jh.h
   keccak.h

src/crypto/hmac-keccak.c

@@ -0,0 +1,81 @@
+// Copyright (c) 2014-2018, The Monero Project
+//
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "hmac-keccak.h"
+#include "memwipe.h"
+
+#define KECCAK_BLOCKLEN 136
+#define HASH_SIZE 32
+
+void hmac_keccak_init(hmac_keccak_state *S, const uint8_t *_key, size_t keylen) {
+  const uint8_t *key = _key;
+  uint8_t keyhash[HASH_SIZE];
+  uint8_t pad[KECCAK_BLOCKLEN];
+  uint64_t i;
+
+  if (keylen > KECCAK_BLOCKLEN) {
+    keccak(key, keylen, keyhash, HASH_SIZE);
+    key = keyhash;
+    keylen = HASH_SIZE;
+  }
+
+  keccak_init(&S->inner);
+  memset(pad, 0x36, KECCAK_BLOCKLEN);
+  for (i = 0; i < keylen; ++i) {
+    pad[i] ^= key[i];
+  }
+  keccak_update(&S->inner, pad, KECCAK_BLOCKLEN);
+
+  keccak_init(&S->outer);
+  memset(pad, 0x5c, KECCAK_BLOCKLEN);
+  for (i = 0; i < keylen; ++i) {
+    pad[i] ^= key[i];
+  }
+  keccak_update(&S->outer, pad, KECCAK_BLOCKLEN);
+
+  memwipe(keyhash, HASH_SIZE);
+}
+
+void hmac_keccak_update(hmac_keccak_state *S, const uint8_t *data, size_t datalen) {
+  keccak_update(&S->inner, data, datalen);
+}
+
+void hmac_keccak_finish(hmac_keccak_state *S, uint8_t *digest) {
+  uint8_t ihash[HASH_SIZE];
+  keccak_finish(&S->inner, ihash);
+  keccak_update(&S->outer, ihash, HASH_SIZE);
+  keccak_finish(&S->outer, digest);
+  memwipe(ihash, HASH_SIZE);
+}
+
+void hmac_keccak_hash(uint8_t *out, const uint8_t *key, size_t keylen, const uint8_t *in, size_t inlen) {
+  hmac_keccak_state S;
+  hmac_keccak_init(&S, key, keylen);
+  hmac_keccak_update(&S, in, inlen);
+  hmac_keccak_finish(&S, out);
+}

src/crypto/hmac-keccak.h

@@ -0,0 +1,59 @@
+// Copyright (c) 2014-2018, The Monero Project
+//
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef HMAC_KECCAK_H
+#define HMAC_KECCAK_H
+
+#include "keccak.h"
+
+// HMAC RFC 2104 with Keccak-256 base hash function
+//
+// B = KECCAK_BLOCKLEN = 136 B
+// L = HASH_SIZE = 32 B
+//
+// Note this is not HMAC-SHA3 as SHA3 and Keccak differs in
+// the padding constant.
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct {
+  KECCAK_CTX inner;
+  KECCAK_CTX outer;
+} hmac_keccak_state;
+
+void hmac_keccak_init(hmac_keccak_state *S, const uint8_t *_key, size_t keylen);
+void hmac_keccak_update(hmac_keccak_state *S, const uint8_t *data, size_t datalen);
+void hmac_keccak_finish(hmac_keccak_state *S, uint8_t *digest);
+void hmac_keccak_hash(uint8_t *out, const uint8_t *key, size_t keylen, const uint8_t *in, size_t inlen);
+
+#ifdef __cplusplus
+}
+#endif
+#endif //HMAC_KECCAK_H

src/cryptonote_basic/cryptonote_format_utils.cpp

@@ -282,6 +282,11 @@ namespace cryptonote
   //---------------------------------------------------------------
   bool generate_key_image_helper_precomp(const account_keys& ack, const crypto::public_key& out_key, const crypto::key_derivation& recv_derivation, size_t real_output_index, const subaddress_index& received_index, keypair& in_ephemeral, crypto::key_image& ki, hw::device &hwdev)
   {
+    if (hwdev.compute_key_image(ack, out_key, recv_derivation, real_output_index, received_index, in_ephemeral, ki))
+    {
+      return true;
+    }
+
     if (ack.m_spend_secret_key == crypto::null_skey)
     {
       // for watch-only wallet, simply copy the known output pubkey

src/device/device.hpp

@@ -70,6 +70,7 @@ namespace cryptonote
     struct account_keys;
     struct subaddress_index;
     struct tx_destination_entry;
+    struct keypair;
 }
 
 namespace hw {
@@ -81,11 +82,18 @@ namespace hw {
            return false;
     }
 
+    class device_progress {
+    public:
+      virtual double progress() const { return 0; }
+      virtual bool indeterminate() const { return false; }
+    };
+
     class i_device_callback {
     public:
-        virtual void on_button_request() {}
-        virtual void on_pin_request(epee::wipeable_string & pin) {}
-        virtual void on_passphrase_request(bool on_device, epee::wipeable_string & passphrase) {}
+        virtual void on_button_request(uint64_t code=0) {}
+        virtual boost::optional<epee::wipeable_string> on_pin_request() { return boost::none; }
+        virtual boost::optional<epee::wipeable_string> on_passphrase_request(bool on_device) { return boost::none; }
+        virtual void on_progress(const device_progress& event) {}
         virtual ~i_device_callback() = default;
     };
 
@@ -141,6 +149,9 @@ namespace hw {
         virtual void set_callback(i_device_callback * callback) {};
         virtual void set_derivation_path(const std::string &derivation_path) {};
 
+        virtual void set_pin(const epee::wipeable_string & pin) {}
+        virtual void set_passphrase(const epee::wipeable_string & passphrase) {}
+
         /* ======================================================================= */
         /*  LOCKER                                                                 */
         /* ======================================================================= */ 
@@ -229,7 +240,9 @@ namespace hw {
 
         virtual bool  has_ki_cold_sync(void) const { return false; }
         virtual bool  has_tx_cold_sign(void) const { return false; }
-
+        virtual bool  has_ki_live_refresh(void) const { return true; }
+        virtual bool  compute_key_image(const cryptonote::account_keys& ack, const crypto::public_key& out_key, const crypto::key_derivation& recv_derivation, size_t real_output_index, const cryptonote::subaddress_index& received_index, cryptonote::keypair& in_ephemeral, crypto::key_image& ki) { return false; }
+        virtual void  computing_key_images(bool started) {};
         virtual void  set_network_type(cryptonote::network_type network_type) { }
 
     protected:

src/device/device_cold.hpp

@@ -31,6 +31,7 @@
 #define MONERO_DEVICE_COLD_H
 
 #include "wallet/wallet2.h"
+#include <boost/optional/optional.hpp>
 #include <boost/function.hpp>
 
 
@@ -44,6 +45,8 @@ namespace hw {
   public:
     std::vector<std::string> tx_device_aux;  // device generated aux data
     std::vector<cryptonote::address_parse_info> tx_recipients;  // as entered by user
+    boost::optional<int> bp_version;  // BP version to use
+    boost::optional<unsigned> client_version;  // Signing client version to use (testing)
   };
 
   class device_cold {
@@ -51,6 +54,53 @@ namespace hw {
 
     using exported_key_image = std::vector<std::pair<crypto::key_image, crypto::signature>>;
 
+    class op_progress : public hw::device_progress {
+    public:
+      op_progress():m_progress(0), m_indeterminate(false) {};
+      explicit op_progress(double progress, bool indeterminate=false): m_progress(progress), m_indeterminate(indeterminate){}
+
+      double progress() const override { return m_progress; }
+      bool indeterminate() const override { return m_indeterminate; }
+    protected:
+      double m_progress;
+      bool m_indeterminate;
+    };
+
+    class tx_progress : public op_progress {
+    public:
+      tx_progress():
+        m_cur_tx(0), m_max_tx(1),
+        m_cur_step(0), m_max_step(1),
+        m_cur_substep(0), m_max_substep(1){};
+
+      tx_progress(size_t cur_tx, size_t max_tx, size_t cur_step, size_t max_step, size_t cur_substep, size_t max_substep):
+        m_cur_tx(cur_tx), m_max_tx(max_tx),
+        m_cur_step(cur_tx), m_max_step(max_tx),
+        m_cur_substep(cur_tx), m_max_substep(max_tx){}
+
+      double progress() const override {
+        return std::max(1.0, (double)m_cur_tx / m_max_tx
+          + (double)m_cur_step / (m_max_tx * m_max_step)
+          + (double)m_cur_substep / (m_max_tx * m_max_step * m_max_substep));
+      }
+      bool indeterminate() const override { return false; }
+
+    protected:
+      size_t m_cur_tx;
+      size_t m_max_tx;
+      size_t m_cur_step;
+      size_t m_max_step;
+      size_t m_cur_substep;
+      size_t m_max_substep;
+    };
+
+    typedef struct {
+      std::string salt1;
+      std::string salt2;
+      std::string tx_enc_keys;
+      std::string tx_prefix_hash;
+    } tx_key_data_t;
+
     /**
      * Key image sync with the cold protocol.
      */
@@ -65,6 +115,52 @@ namespace hw {
                  const ::tools::wallet2::unsigned_tx_set & unsigned_tx,
                  ::tools::wallet2::signed_tx_set & signed_tx,
                  tx_aux_data & aux_data) =0;
+
+    /**
+     * Get tx key support check.
+     */
+    virtual bool is_get_tx_key_supported() const { return false; }
+
+    /**
+     * Loads TX aux data required for tx key.
+     */
+    virtual void load_tx_key_data(tx_key_data_t & res, const std::string & tx_aux_data) =0;
+
+    /**
+     * Decrypts TX keys.
+     */
+    virtual void get_tx_key(
+        std::vector<::crypto::secret_key> & tx_keys,
+        const tx_key_data_t & tx_aux_data,
+        const ::crypto::secret_key & view_key_priv) =0;
+
+    /**
+     * Live refresh support check
+     */
+    virtual bool is_live_refresh_supported() const { return false; };
+
+    /**
+     * Starts live refresh process with the device
+     */
+    virtual void live_refresh_start() =0;
+
+    /**
+     * One live refresh step
+     */
+    virtual void live_refresh(
+        const ::crypto::secret_key & view_key_priv,
+        const crypto::public_key& out_key,
+        const crypto::key_derivation& recv_derivation,
+        size_t real_output_index,
+        const cryptonote::subaddress_index& received_index,
+        cryptonote::keypair& in_ephemeral,
+        crypto::key_image& ki
+    ) =0;
+
+    /**
+     * Live refresh process termination
+     */
+    virtual void live_refresh_finish() =0;
   };
 }
 

src/device_trezor/device_trezor.cpp

@@ -57,7 +57,9 @@ namespace trezor {
     }
 
     device_trezor::device_trezor() {
-
+      m_live_refresh_in_progress = false;
+      m_live_refresh_enabled = true;
+      m_live_refresh_thread_running = false;
     }
 
     device_trezor::~device_trezor() {
@@ -69,6 +71,89 @@ namespace trezor {
       }
     }
 
+    bool device_trezor::init()
+    {
+      m_live_refresh_in_progress = false;
+      bool r = device_trezor_base::init();
+      if (r && !m_live_refresh_thread)
+      {
+        m_live_refresh_thread_running = true;
+        m_live_refresh_thread.reset(new boost::thread(boost::bind(&device_trezor::live_refresh_thread_main, this)));
+      }
+      return r;
+    }
+
+    bool device_trezor::release()
+    {
+      m_live_refresh_in_progress = false;
+      m_live_refresh_thread_running = false;
+      if (m_live_refresh_thread)
+      {
+        m_live_refresh_thread->join();
+        m_live_refresh_thread = nullptr;
+      }
+      return device_trezor_base::release();
+    }
+
+    bool device_trezor::disconnect()
+    {
+      m_live_refresh_in_progress = false;
+      return device_trezor_base::disconnect();
+    }
+
+    void device_trezor::device_state_reset_unsafe()
+    {
+      require_connected();
+      if (m_live_refresh_in_progress)
+      {
+        try
+        {
+          live_refresh_finish_unsafe();
+        }
+        catch(const std::exception & e)
+        {
+          MERROR("Live refresh could not be terminated: " << e.what());
+        }
+      }
+
+      m_live_refresh_in_progress = false;
+      device_trezor_base::device_state_reset_unsafe();
+    }
+
+    void device_trezor::live_refresh_thread_main()
+    {
+      while(m_live_refresh_thread_running)
+      {
+        boost::this_thread::sleep_for(boost::chrono::milliseconds(100));
+        if (!m_live_refresh_in_progress)
+        {
+          continue;
+        }
+
+        TREZOR_AUTO_LOCK_DEVICE();
+        if (!m_transport || !m_live_refresh_in_progress)
+        {
+          continue;
+        }
+
+        auto current_time = std::chrono::steady_clock::now();
+        if (current_time - m_last_live_refresh_time <= std::chrono::seconds(20))
+        {
+          continue;
+        }
+
+        MTRACE("Closing live refresh process due to inactivity");
+        try
+        {
+          live_refresh_finish();
+        }
+        catch(const std::exception &e)
+        {
+          MWARNING("Live refresh auto-finish failed: " << e.what());
+        }
+      }
+    }
+
     /* ======================================================================= */
     /*                             WALLET & ADDRESS                            */
     /* ======================================================================= */
@@ -126,7 +211,7 @@ namespace trezor {
     std::shared_ptr<messages::monero::MoneroAddress> device_trezor::get_address(
         const boost::optional<std::vector<uint32_t>> & path,
         const boost::optional<cryptonote::network_type> & network_type){
-      AUTO_LOCK_CMD();
+      TREZOR_AUTO_LOCK_CMD();
       require_connected();
       device_state_reset_unsafe();
       require_initialized();
@@ -142,7 +227,7 @@ namespace trezor {
     std::shared_ptr<messages::monero::MoneroWatchKey> device_trezor::get_view_key(
         const boost::optional<std::vector<uint32_t>> & path,
         const boost::optional<cryptonote::network_type> & network_type){
-      AUTO_LOCK_CMD();
+      TREZOR_AUTO_LOCK_CMD();
       require_connected();
       device_state_reset_unsafe();
       require_initialized();
@@ -155,11 +240,43 @@ namespace trezor {
       return response;
     }
 
+    bool device_trezor::is_get_tx_key_supported() const
+    {
+      require_initialized();
+      return get_version() > pack_version(2, 0, 10);
+    }
+
+    void device_trezor::load_tx_key_data(::hw::device_cold::tx_key_data_t & res, const std::string & tx_aux_data)
+    {
+      protocol::tx::load_tx_key_data(res, tx_aux_data);
+    }
+
+    void device_trezor::get_tx_key(
+        std::vector<::crypto::secret_key> & tx_keys,
+        const ::hw::device_cold::tx_key_data_t & tx_aux_data,
+        const ::crypto::secret_key & view_key_priv)
+    {
+      TREZOR_AUTO_LOCK_CMD();
+      require_connected();
+      device_state_reset_unsafe();
+      require_initialized();
+
+      auto req = protocol::tx::get_tx_key(tx_aux_data);
+      this->set_msg_addr<messages::monero::MoneroGetTxKeyRequest>(req.get());
+
+      auto response = this->client_exchange<messages::monero::MoneroGetTxKeyAck>(req);
+      MTRACE("Get TX key response received");
+
+      protocol::tx::get_tx_key_ack(tx_keys, tx_aux_data.tx_prefix_hash, view_key_priv, response);
+    }
+
     void device_trezor::ki_sync(wallet_shim * wallet,
                                 const std::vector<tools::wallet2::transfer_details> & transfers,
                                 hw::device_cold::exported_key_image & ski)
     {
-      AUTO_LOCK_CMD();
+#define EVENT_PROGRESS(P) do { if (m_callback) {(m_callback)->on_progress(device_cold::op_progress(P)); } }while(0)
+
+      TREZOR_AUTO_LOCK_CMD();
       require_connected();
       device_state_reset_unsafe();
       require_initialized();
@@ -171,6 +288,7 @@ namespace trezor {
       protocol::ki::key_image_data(wallet, transfers, mtds);
       protocol::ki::generate_commitment(mtds, transfers, req);
 
+      EVENT_PROGRESS(0.);
       this->set_msg_addr<messages::monero::MoneroKeyImageExportInitRequest>(req.get());
       auto ack1 = this->client_exchange<messages::monero::MoneroKeyImageExportInitAck>(req);
 
@@ -194,27 +312,160 @@ namespace trezor {
         }
 
         MTRACE("Batch " << cur << " / " << num_batches << " batches processed");
+        EVENT_PROGRESS((double)cur * batch_size / mtds.size());
       }
+      EVENT_PROGRESS(1.);
 
       auto final_req = std::make_shared<messages::monero::MoneroKeyImageSyncFinalRequest>();
       auto final_ack = this->client_exchange<messages::monero::MoneroKeyImageSyncFinalAck>(final_req);
       ski.reserve(kis.size());
 
       for(auto & sub : kis){
-        char buff[32*3];
+        ::crypto::signature sig{};
+        ::crypto::key_image ki;
+        char buff[sizeof(ki.data)*3];
+
+        size_t buff_len = sizeof(buff);
+
         protocol::crypto::chacha::decrypt(sub.blob().data(), sub.blob().size(),
                                           reinterpret_cast<const uint8_t *>(final_ack->enc_key().data()),
-                                          reinterpret_cast<const uint8_t *>(sub.iv().data()), buff);
+                                          reinterpret_cast<const uint8_t *>(sub.iv().data()), buff, &buff_len);
+        CHECK_AND_ASSERT_THROW_MES(buff_len == sizeof(buff), "Plaintext size invalid");
 
-        ::crypto::signature sig{};
-        ::crypto::key_image ki;
-        memcpy(ki.data, buff, 32);
-        memcpy(sig.c.data, buff + 32, 32);
-        memcpy(sig.r.data, buff + 64, 32);
+        memcpy(ki.data, buff, sizeof(ki.data));
+        memcpy(sig.c.data, buff + sizeof(ki.data), sizeof(ki.data));
+        memcpy(sig.r.data, buff + 2*sizeof(ki.data), sizeof(ki.data));
         ski.push_back(std::make_pair(ki, sig));
       }
+#undef EVENT_PROGRESS
+    }
+
+    bool device_trezor::is_live_refresh_supported() const
+    {
+      require_initialized();
+      return get_version() > pack_version(2, 0, 10);
     }
 
+    bool device_trezor::is_live_refresh_enabled() const
+    {
+      return is_live_refresh_supported() && (mode == NONE || mode == TRANSACTION_PARSE) && m_live_refresh_enabled;
+    }
+
+    bool device_trezor::has_ki_live_refresh() const
+    {
+      try{
+        return is_live_refresh_enabled();
+      } catch(const std::exception & e){
+        MERROR("Could not detect if live refresh is enabled: " << e.what());
+      }
+      return false;
+    }
+
+    void device_trezor::live_refresh_start()
+    {
+      TREZOR_AUTO_LOCK_CMD();
+      require_connected();
+      live_refresh_start_unsafe();
+    }
+
+    void device_trezor::live_refresh_start_unsafe()
+    {
+      device_state_reset_unsafe();
+      require_initialized();
+
+      auto req = std::make_shared<messages::monero::MoneroLiveRefreshStartRequest>();
+      this->set_msg_addr<messages::monero::MoneroLiveRefreshStartRequest>(req.get());
+      this->client_exchange<messages::monero::MoneroLiveRefreshStartAck>(req);
+      m_live_refresh_in_progress = true;
+      m_last_live_refresh_time = std::chrono::steady_clock::now();
+    }
+
+    void device_trezor::live_refresh(
+        const ::crypto::secret_key & view_key_priv,
+        const crypto::public_key& out_key,
+        const crypto::key_derivation& recv_derivation,
+        size_t real_output_index,
+        const cryptonote::subaddress_index& received_index,
+        cryptonote::keypair& in_ephemeral,
+        crypto::key_image& ki
+    )
+    {
+      TREZOR_AUTO_LOCK_CMD();
+      require_connected();
+
+      if (!m_live_refresh_in_progress)
+      {
+        live_refresh_start_unsafe();
+      }
+
+      m_last_live_refresh_time = std::chrono::steady_clock::now();
+
+      auto req = std::make_shared<messages::monero::MoneroLiveRefreshStepRequest>();
+      req->set_out_key(out_key.data, 32);
+      req->set_recv_deriv(recv_derivation.data, 32);
+      req->set_real_out_idx(real_output_index);
+      req->set_sub_addr_major(received_index.major);
+      req->set_sub_addr_minor(received_index.minor);
+
+      auto ack = this->client_exchange<messages::monero::MoneroLiveRefreshStepAck>(req);
+      protocol::ki::live_refresh_ack(view_key_priv, out_key, ack, in_ephemeral, ki);
+    }
+
+    void device_trezor::live_refresh_finish_unsafe()
+    {
+      auto req = std::make_shared<messages::monero::MoneroLiveRefreshFinalRequest>();
+      this->client_exchange<messages::monero::MoneroLiveRefreshFinalAck>(req);
+      m_live_refresh_in_progress = false;
+    }
+
+    void device_trezor::live_refresh_finish()
+    {
+      TREZOR_AUTO_LOCK_CMD();
+      require_connected();
+      if (m_live_refresh_in_progress)
+      {
+        live_refresh_finish_unsafe();
+      }
+    }
+
+    void device_trezor::computing_key_images(bool started)
+    {
+      try
+      {
+        if (!is_live_refresh_enabled())
+        {
+          return;
+        }
+
+        // React only on termination as the process can auto-start itself.
+        if (!started && m_live_refresh_in_progress)
+        {
+          live_refresh_finish();
+        }
+      }
+      catch(const std::exception & e)
+      {
+        MWARNING("KI computation state change failed, started: " << started << ", e: " << e.what());
+      }
+    }
+
+    bool device_trezor::compute_key_image(
+        const ::cryptonote::account_keys& ack,
+        const ::crypto::public_key& out_key,
+        const ::crypto::key_derivation& recv_derivation,
+        size_t real_output_index,
+        const ::cryptonote::subaddress_index& received_index,
+        ::cryptonote::keypair& in_ephemeral,
+        ::crypto::key_image& ki)
+    {
+      if (!is_live_refresh_enabled())
+      {
+        return false;
+      }
+
+      live_refresh(ack.m_view_secret_key, out_key, recv_derivation, real_output_index, received_index, in_ephemeral, ki);
+      return true;
+    }
 
     void device_trezor::tx_sign(wallet_shim * wallet,
                                 const tools::wallet2::unsigned_tx_set & unsigned_tx,
@@ -222,7 +473,15 @@ namespace trezor {
                                 hw::tx_aux_data & aux_data)
     {
       CHECK_AND_ASSERT_THROW_MES(unsigned_tx.transfers.first == 0, "Unsuported non zero offset");
-      size_t num_tx = unsigned_tx.txes.size();
+
+      TREZOR_AUTO_LOCK_CMD();
+      require_connected();
+      device_state_reset_unsafe();
+      require_initialized();
+      transaction_versions_check(unsigned_tx, aux_data);
+
+      const size_t num_tx = unsigned_tx.txes.size();
+      m_num_transations_to_sign = num_tx;
       signed_tx.key_images.clear();
       signed_tx.key_images.resize(unsigned_tx.transfers.second.size());
 
@@ -267,6 +526,10 @@ namespace trezor {
         cpend.key_images = key_images;
 
         // KI sync
+        for(size_t cidx=0, trans_max=unsigned_tx.transfers.second.size(); cidx < trans_max; ++cidx){
+          signed_tx.key_images[cidx] = unsigned_tx.transfers.second[cidx].m_key_image;
+        }
+
         size_t num_sources = cdata.tx_data.sources.size();
         CHECK_AND_ASSERT_THROW_MES(num_sources == cdata.source_permutation.size(), "Invalid permutation size");
         CHECK_AND_ASSERT_THROW_MES(num_sources == cdata.tx.vin.size(), "Invalid tx.vin size");
@@ -276,12 +539,19 @@ namespace trezor {
           CHECK_AND_ASSERT_THROW_MES(src_idx < cdata.tx.vin.size(), "Invalid idx_mapped");
 
           size_t idx_map_src = cdata.tx_data.selected_transfers[idx_mapped];
-          auto vini = boost::get<cryptonote::txin_to_key>(cdata.tx.vin[src_idx]);
+          CHECK_AND_ASSERT_THROW_MES(idx_map_src >= unsigned_tx.transfers.first, "Invalid offset");
 
+          idx_map_src -= unsigned_tx.transfers.first;
           CHECK_AND_ASSERT_THROW_MES(idx_map_src < signed_tx.key_images.size(), "Invalid key image index");
+
+          const auto vini = boost::get<cryptonote::txin_to_key>(cdata.tx.vin[src_idx]);
           signed_tx.key_images[idx_map_src] = vini.k_image;
         }
       }
+
+      if (m_callback){
+        m_callback->on_progress(device_cold::tx_progress(m_num_transations_to_sign, m_num_transations_to_sign, 1, 1, 1, 1));
+      }
     }
 
     void device_trezor::tx_sign(wallet_shim * wallet,
@@ -290,10 +560,16 @@ namespace trezor {
                    hw::tx_aux_data & aux_data,
                    std::shared_ptr<protocol::tx::Signer> & signer)
     {
-      AUTO_LOCK_CMD();
+#define EVENT_PROGRESS(S, SUB, SUBMAX) do { if (m_callback) { \
+      (m_callback)->on_progress(device_cold::tx_progress(idx, m_num_transations_to_sign, S, 10, SUB, SUBMAX)); \
+} }while(0)
+
       require_connected();
-      device_state_reset_unsafe();
+      if (idx > 0)
+        device_state_reset_unsafe();
+
       require_initialized();
+      EVENT_PROGRESS(0, 1, 1);
 
       CHECK_AND_ASSERT_THROW_MES(idx < unsigned_tx.txes.size(), "Invalid transaction index");
       signer = std::make_shared<protocol::tx::Signer>(wallet, &unsigned_tx, idx, &aux_data);
@@ -305,6 +581,7 @@ namespace trezor {
       auto init_msg = signer->step_init();
       this->set_msg_addr(init_msg.get());
       transaction_pre_check(init_msg);
+      EVENT_PROGRESS(1, 1, 1);
 
       auto response = this->client_exchange<messages::monero::MoneroTransactionInitAck>(init_msg);
       signer->step_init_ack(response);
@@ -314,6 +591,7 @@ namespace trezor {
         auto src = signer->step_set_input(cur_src);
         auto ack = this->client_exchange<messages::monero::MoneroTransactionSetInputAck>(src);
         signer->step_set_input_ack(ack);
+        EVENT_PROGRESS(2, cur_src, num_sources);
       }
 
       // Step: sort
@@ -322,44 +600,82 @@ namespace trezor {
         auto perm_ack = this->client_exchange<messages::monero::MoneroTransactionInputsPermutationAck>(perm_req);
         signer->step_permutation_ack(perm_ack);
       }
+      EVENT_PROGRESS(3, 1, 1);
 
       // Step: input_vini
-      if (!signer->in_memory()){
-        for(size_t cur_src = 0; cur_src < num_sources; ++cur_src){
-          auto src = signer->step_set_vini_input(cur_src);
-          auto ack = this->client_exchange<messages::monero::MoneroTransactionInputViniAck>(src);
-          signer->step_set_vini_input_ack(ack);
-        }
+      for(size_t cur_src = 0; cur_src < num_sources; ++cur_src){
+        auto src = signer->step_set_vini_input(cur_src);
+        auto ack = this->client_exchange<messages::monero::MoneroTransactionInputViniAck>(src);
+        signer->step_set_vini_input_ack(ack);
+        EVENT_PROGRESS(4, cur_src, num_sources);
       }
 
       // Step: all inputs set
       auto all_inputs_set = signer->step_all_inputs_set();
       auto ack_all_inputs = this->client_exchange<messages::monero::MoneroTransactionAllInputsSetAck>(all_inputs_set);
       signer->step_all_inputs_set_ack(ack_all_inputs);
+      EVENT_PROGRESS(5, 1, 1);
 
       // Step: outputs
       for(size_t cur_dst = 0; cur_dst < num_outputs; ++cur_dst){
         auto src = signer->step_set_output(cur_dst);
         auto ack = this->client_exchange<messages::monero::MoneroTransactionSetOutputAck>(src);
         signer->step_set_output_ack(ack);
+
+        // If BP is offloaded to host, another step with computed BP may be needed.
+        auto offloaded_bp = signer->step_rsig(cur_dst);
+        if (offloaded_bp){
+          auto bp_ack = this->client_exchange<messages::monero::MoneroTransactionSetOutputAck>(offloaded_bp);
+          signer->step_set_rsig_ack(ack);
+        }
+
+        EVENT_PROGRESS(6, cur_dst, num_outputs);
       }
 
       // Step: all outs set
       auto all_out_set = signer->step_all_outs_set();
       auto ack_all_out_set = this->client_exchange<messages::monero::MoneroTransactionAllOutSetAck>(all_out_set);
       signer->step_all_outs_set_ack(ack_all_out_set, *this);
+      EVENT_PROGRESS(7, 1, 1);
 
       // Step: sign each input
       for(size_t cur_src = 0; cur_src < num_sources; ++cur_src){
         auto src = signer->step_sign_input(cur_src);
         auto ack_sign = this->client_exchange<messages::monero::MoneroTransactionSignInputAck>(src);
         signer->step_sign_input_ack(ack_sign);
+        EVENT_PROGRESS(8, cur_src, num_sources);
       }
 
       // Step: final
       auto final_msg = signer->step_final();
       auto ack_final = this->client_exchange<messages::monero::MoneroTransactionFinalAck>(final_msg);
       signer->step_final_ack(ack_final);
+      EVENT_PROGRESS(9, 1, 1);
+#undef EVENT_PROGRESS
+    }
+
+    void device_trezor::transaction_versions_check(const ::tools::wallet2::unsigned_tx_set & unsigned_tx, hw::tx_aux_data & aux_data)
+    {
+      auto trezor_version = get_version();
+      unsigned client_version = 1;  // default client version for tx
+
+      if (trezor_version <= pack_version(2, 0, 10)){
+        client_version = 0;
+      }
+
+      if (aux_data.client_version){
+        auto wanted_client_version = aux_data.client_version.get();
+        if (wanted_client_version > client_version){
+          throw exc::TrezorException("Trezor firmware 2.0.10 and lower does not support current transaction sign protocol. Please update.");
+        } else {
+          client_version = wanted_client_version;
+        }
+      }
+      aux_data.client_version = client_version;
+
+      if (client_version == 0 && aux_data.bp_version && aux_data.bp_version.get() != 1){
+        throw exc::TrezorException("Trezor firmware 2.0.10 and lower does not support current transaction sign protocol (BPv2+). Please update.");
+      }
     }
 
     void device_trezor::transaction_pre_check(std::shared_ptr<messages::monero::MoneroTransactionInitRequest> init_msg)

src/device_trezor/device_trezor.hpp

@@ -30,18 +30,21 @@
 #ifndef MONERO_DEVICE_TREZOR_H
 #define MONERO_DEVICE_TREZOR_H
 
+#include "trezor.hpp"
+#include "device/device.hpp"
 
+#ifdef WITH_DEVICE_TREZOR
 #include <cstddef>
 #include <string>
-#include "device/device.hpp"
-#include "device/device_default.hpp"
-#include "device/device_cold.hpp"
 #include <boost/scope_exit.hpp>
 #include <boost/thread/mutex.hpp>
 #include <boost/thread/recursive_mutex.hpp>
+
+#include "device/device_default.hpp"
+#include "device/device_cold.hpp"
 #include "cryptonote_config.h"
-#include "trezor.hpp"
 #include "device_trezor_base.hpp"
+#endif
 
 namespace hw {
 namespace trezor {
@@ -57,8 +60,29 @@ namespace trezor {
    */
   class device_trezor : public hw::trezor::device_trezor_base, public hw::device_cold {
     protected:
+      std::atomic<bool> m_live_refresh_in_progress;
+      std::chrono::steady_clock::time_point m_last_live_refresh_time;
+      std::unique_ptr<boost::thread> m_live_refresh_thread;
+      std::atomic<bool> m_live_refresh_thread_running;
+      bool m_live_refresh_enabled;
+      size_t m_num_transations_to_sign;
+
+      void transaction_versions_check(const ::tools::wallet2::unsigned_tx_set & unsigned_tx, hw::tx_aux_data & aux_data);
       void transaction_pre_check(std::shared_ptr<messages::monero::MoneroTransactionInitRequest> init_msg);
       void transaction_check(const protocol::tx::TData & tdata, const hw::tx_aux_data & aux_data);
+      void device_state_reset_unsafe() override;
+      void live_refresh_start_unsafe();
+      void live_refresh_finish_unsafe();
+      void live_refresh_thread_main();
+
+      /**
+       * Signs particular transaction idx in the unsigned set, keeps state in the signer
+       */
+      virtual void tx_sign(wallet_shim * wallet,
+                   const ::tools::wallet2::unsigned_tx_set & unsigned_tx,
+                   size_t idx,
+                   hw::tx_aux_data & aux_data,
+                   std::shared_ptr<protocol::tx::Signer> & signer);
 
     public:
       device_trezor();
@@ -69,11 +93,17 @@ namespace trezor {
 
       explicit operator bool() const override {return true;}
 
+      bool init() override;
+      bool release() override;
+      bool disconnect() override;
+
       device_protocol_t device_protocol() const override { return PROTOCOL_COLD; };
 
       bool  has_ki_cold_sync() const override { return true; }
       bool  has_tx_cold_sign() const override { return true; }
       void  set_network_type(cryptonote::network_type network_type) override { this->network_type = network_type; }
+      void  set_live_refresh_enabled(bool enabled) { m_live_refresh_enabled = enabled; }
+      bool  live_refresh_enabled() const { return m_live_refresh_enabled; }
 
       /* ======================================================================= */
       /*                             WALLET & ADDRESS                            */
@@ -99,6 +129,24 @@ namespace trezor {
           const boost::optional<std::vector<uint32_t>> & path = boost::none,
           const boost::optional<cryptonote::network_type> & network_type = boost::none);
 
+      /**
+       * Get_tx_key support check
+       */
+      bool is_get_tx_key_supported() const override;
+
+      /**
+       * Loads tx aux data
+       */
+      void load_tx_key_data(::hw::device_cold::tx_key_data_t & res, const std::string & tx_aux_data) override;
+
+      /**
+       * TX key load with the Trezor
+       */
+      void get_tx_key(
+        std::vector<::crypto::secret_key> & tx_keys,
+        const ::hw::device_cold::tx_key_data_t & tx_aux_data,
+        const ::crypto::secret_key & view_key_priv) override;
+
       /**
        * Key image sync with the Trezor.
        */
@@ -106,14 +154,44 @@ namespace trezor {
                    const std::vector<::tools::wallet2::transfer_details> & transfers,
                    hw::device_cold::exported_key_image & ski) override;
 
+      bool is_live_refresh_supported() const override;
+
+      bool is_live_refresh_enabled() const;
+
+      bool has_ki_live_refresh() const override;
+
+      void live_refresh_start() override;
+
+      void live_refresh(
+          const ::crypto::secret_key & view_key_priv,
+          const crypto::public_key& out_key,
+          const crypto::key_derivation& recv_derivation,
+          size_t real_output_index,
+          const cryptonote::subaddress_index& received_index,
+          cryptonote::keypair& in_ephemeral,
+          crypto::key_image& ki
+          ) override;
+
+      void live_refresh_finish() override;
+
       /**
-       * Signs particular transaction idx in the unsigned set, keeps state in the signer
+       * Letting device know the KI computation started / ended.
+       * During refresh
        */
-      void tx_sign(wallet_shim * wallet,
-                   const ::tools::wallet2::unsigned_tx_set & unsigned_tx,
-                   size_t idx,
-                   hw::tx_aux_data & aux_data,
-                   std::shared_ptr<protocol::tx::Signer> & signer);
+      void computing_key_images(bool started) override;
+
+      /**
+       * Implements hw::device interface
+       * called from generate_key_image_helper_precomp()
+       */
+      bool compute_key_image(
+          const ::cryptonote::account_keys& ack,
+          const ::crypto::public_key& out_key,
+          const ::crypto::key_derivation& recv_derivation,
+          size_t real_output_index,
+          const ::cryptonote::subaddress_index& received_index,
+          ::cryptonote::keypair& in_ephemeral,
+          ::crypto::key_image& ki) override;
 
       /**
        * Signs unsigned transaction with the Trezor.

src/device_trezor/device_trezor_base.cpp

@@ -109,6 +109,7 @@ namespace trezor {
       disconnect();
 
       // Enumerate all available devices
+      TREZOR_AUTO_LOCK_DEVICE();
       try {
         hw::trezor::t_transport_vect trans;
 
@@ -145,6 +146,7 @@ namespace trezor {
     }
 
     bool device_trezor_base::disconnect() {
+      TREZOR_AUTO_LOCK_DEVICE();
       m_device_state.clear();
       m_features.reset();
 
@@ -203,13 +205,13 @@ namespace trezor {
     /*  Helpers                                                                */
     /* ======================================================================= */
 
-    void device_trezor_base::require_connected(){
+    void device_trezor_base::require_connected() const {
       if (!m_transport){
         throw exc::NotConnectedException();
       }
     }
 
-    void device_trezor_base::require_initialized(){
+    void device_trezor_base::require_initialized() const {
       if (!m_features){
         throw exc::TrezorException("Device state not initialized");
       }
@@ -330,7 +332,7 @@ namespace trezor {
     /* ======================================================================= */
 
     bool device_trezor_base::ping() {
-      AUTO_LOCK_CMD();
+      TREZOR_AUTO_LOCK_CMD();
       if (!m_transport){
         MINFO("Ping failed, device not connected");
         return false;
@@ -364,7 +366,7 @@ namespace trezor {
 
     void device_trezor_base::device_state_reset()
     {
-      AUTO_LOCK_CMD();
+      TREZOR_AUTO_LOCK_CMD();
       device_state_reset_unsafe();
     }
 
@@ -373,6 +375,10 @@ namespace trezor {
   if (m_debug_callback) m_debug_callback->method(__VA_ARGS__); \
   if (m_callback) m_callback->method(__VA_ARGS__);             \
 }while(0)
+#define TREZOR_CALLBACK_GET(VAR, method, ...) do { \
+  if (m_debug_callback) VAR = m_debug_callback->method(__VA_ARGS__); \
+  if (m_callback) VAR = m_callback->method(__VA_ARGS__);             \
+}while(0)
 
     void device_trezor_base::setup_debug(){
       if (!m_debug){
@@ -392,6 +398,7 @@ namespace trezor {
 
 #else
 #define TREZOR_CALLBACK(method, ...) do { if (m_callback) m_callback->method(__VA_ARGS__); } while(0)
+#define TREZOR_CALLBACK_GET(VAR, method, ...) VAR = (m_callback ? m_callback->method(__VA_ARGS__) : boost::none)
 #endif
 
     void device_trezor_base::on_button_request(GenericMessage & resp, const messages::common::ButtonRequest * msg)
@@ -402,7 +409,7 @@ namespace trezor {
       messages::common::ButtonAck ack;
       write_raw(&ack);
 
-      TREZOR_CALLBACK(on_button_request);
+      TREZOR_CALLBACK(on_button_request, msg->code());
       resp = read_raw();
     }
 
@@ -411,13 +418,18 @@ namespace trezor {
       MDEBUG("on_pin_request");
       CHECK_AND_ASSERT_THROW_MES(msg, "Empty message");
 
-      epee::wipeable_string pin;
+      boost::optional<epee::wipeable_string> pin;
+      TREZOR_CALLBACK_GET(pin, on_pin_request);
 
-      TREZOR_CALLBACK(on_pin_request, pin);
+      if (!pin && m_pin){
+        pin = m_pin;
+      }
 
       // TODO: remove PIN from memory
       messages::common::PinMatrixAck m;
-      m.set_pin(pin.data(), pin.size());
+      if (pin) {
+        m.set_pin(pin.get().data(), pin.get().size());
+      }
       resp = call_raw(&m);
     }
 
@@ -425,14 +437,19 @@ namespace trezor {
     {
       CHECK_AND_ASSERT_THROW_MES(msg, "Empty message");
       MDEBUG("on_passhprase_request, on device: " << msg->on_device());
-      epee::wipeable_string passphrase;
+      boost::optional<epee::wipeable_string> passphrase;
+      TREZOR_CALLBACK_GET(passphrase, on_passphrase_request, msg->on_device());
 
-      TREZOR_CALLBACK(on_passphrase_request, msg->on_device(), passphrase);
+      if (!passphrase && m_passphrase){
+        passphrase = m_passphrase;
+      }
+
+      m_passphrase = boost::none;
 
       messages::common::PassphraseAck m;
-      if (!msg->on_device()){
+      if (!msg->on_device() && passphrase){
         // TODO: remove passphrase from memory
-        m.set_passphrase(passphrase.data(), passphrase.size());
+        m.set_passphrase(passphrase.get().data(), passphrase.get().size());
       }
 
       if (!m_device_state.empty()){
@@ -494,16 +511,16 @@ namespace trezor {
       m_debug_link->init(debug_transport);
     }
 
-    void trezor_debug_callback::on_button_request() {
+    void trezor_debug_callback::on_button_request(uint64_t code) {
       if (m_debug_link) m_debug_link->press_yes();
     }
 
-    void trezor_debug_callback::on_pin_request(epee::wipeable_string &pin) {
-
+    boost::optional<epee::wipeable_string> trezor_debug_callback::on_pin_request() {
+      return boost::none;
     }
 
-    void trezor_debug_callback::on_passphrase_request(bool on_device, epee::wipeable_string &passphrase) {
-
+    boost::optional<epee::wipeable_string> trezor_debug_callback::on_passphrase_request(bool on_device) {
+      return boost::none;
     }
 
     void trezor_debug_callback::on_passphrase_state_request(const std::string &state) {

src/device_trezor/device_trezor_base.hpp

@@ -47,14 +47,15 @@
 #endif
 
 //automatic lock one more level on device ensuring the current thread is allowed to use it
-#define AUTO_LOCK_CMD() \
+#define TREZOR_AUTO_LOCK_CMD() \
   /* lock both mutexes without deadlock*/ \
   boost::lock(device_locker, command_locker); \
   /* make sure both already-locked mutexes are unlocked at the end of scope */ \
   boost::lock_guard<boost::recursive_mutex> lock1(device_locker, boost::adopt_lock); \
   boost::lock_guard<boost::mutex> lock2(command_locker, boost::adopt_lock)
 
-  
+#define TREZOR_AUTO_LOCK_DEVICE() boost::lock_guard<boost::recursive_mutex> lock1_device(device_locker)
+
 namespace hw {
 namespace trezor {
 
@@ -62,14 +63,14 @@ namespace trezor {
   class device_trezor_base;
 
 #ifdef WITH_TREZOR_DEBUGGING
-    class trezor_debug_callback {
+    class trezor_debug_callback : public hw::i_device_callback {
     public:
       trezor_debug_callback()=default;
       explicit trezor_debug_callback(std::shared_ptr<Transport> & debug_transport);
 
-      void on_button_request();
-      void on_pin_request(epee::wipeable_string &pin);
-      void on_passphrase_request(bool on_device, epee::wipeable_string &passphrase);
+      void on_button_request(uint64_t code=0) override;
+      boost::optional<epee::wipeable_string> on_pin_request() override;
+      boost::optional<epee::wipeable_string> on_passphrase_request(bool on_device) override;
       void on_passphrase_state_request(const std::string &state);
       void on_disconnect();
     protected:
@@ -95,6 +96,8 @@ namespace trezor {
       std::vector<unsigned int> m_wallet_deriv_path;
       std::string m_device_state;  // returned after passphrase entry, session
       std::shared_ptr<messages::management::Features> m_features;  // features from the last device reset
+      boost::optional<epee::wipeable_string> m_pin;
+      boost::optional<epee::wipeable_string> m_passphrase;
 
       cryptonote::network_type network_type;
 
@@ -109,11 +112,11 @@ namespace trezor {
       // Internal methods
       //
 
-      void require_connected();
-      void require_initialized();
+      void require_connected() const;
+      void require_initialized() const;
       void call_ping_unsafe();
       void test_ping();
-      void device_state_reset_unsafe();
+      virtual void device_state_reset_unsafe();
       void ensure_derivation_path() noexcept;
 
       // Communication methods
@@ -265,6 +268,15 @@ namespace trezor {
 
     void set_derivation_path(const std::string &deriv_path) override;
 
+    virtual bool has_ki_live_refresh(void) const override { return false; }
+
+    virtual void set_pin(const epee::wipeable_string & pin) override {
+      m_pin = pin;
+    }
+    virtual void set_passphrase(const epee::wipeable_string & passphrase) override {
+      m_passphrase = passphrase;
+    }
+
     /* ======================================================================= */
     /*                              SETUP/TEARDOWN                             */
     /* ======================================================================= */

src/device_trezor/trezor/protocol.cpp

@@ -33,6 +33,8 @@
 #include <utility>
 #include <boost/endian/conversion.hpp>
 #include <common/apply_permutation.h>
+#include <common/json_util.h>
+#include <crypto/hmac-keccak.h>
 #include <ringct/rctSigs.h>
 #include <ringct/bulletproofs.h>
 #include "cryptonote_config.h"
@@ -40,6 +42,37 @@
 #include <sodium/crypto_verify_32.h>
 #include <sodium/crypto_aead_chacha20poly1305.h>
 
+#define GET_FIELD_STRING(name, type, jtype) field_##name = std::string(json[#name].GetString(), json[#name].GetStringLength())
+#define GET_FIELD_OTHER(name, type, jtype) field_##name = static_cast<type>(json[#name].Get##jtype())
+
+#define GET_STRING_FROM_JSON(json, name, type, mandatory, def) \
+  GET_FIELD_FROM_JSON_EX(json, name, type, String, mandatory, def, GET_FIELD_STRING)
+
+#define GET_FIELD_FROM_JSON(json, name, type, jtype, mandatory, def) \
+  GET_FIELD_FROM_JSON_EX(json, name, type, jtype, mandatory, def, GET_FIELD_OTHER)
+
+#define GET_FIELD_FROM_JSON_EX(json, name, type, jtype, mandatory, def, VAL) \
+  type field_##name = static_cast<type>(def);                        \
+  bool field_##name##_found = false;                                 \
+  (void)field_##name##_found;                                        \
+  do if (json.HasMember(#name))                                      \
+  {                                                                  \
+    if (json[#name].Is##jtype())                                     \
+    {                                                                \
+      VAL(name, type, jtype);                                        \
+      field_##name##_found = true;                                   \
+    }                                                                \
+    else                                                             \
+    {                                                                \
+      throw std::invalid_argument("Field " #name " found in JSON, but not " #jtype); \
+    }                                                                \
+  }                                                                  \
+  else if (mandatory)                                                \
+  {                                                                  \
+    throw std::invalid_argument("Field " #name " not found in JSON");\
+  } while(0)
+
+
 namespace hw{
 namespace trezor{
 namespace protocol{
@@ -84,19 +117,22 @@ namespace protocol{
 namespace crypto {
 namespace chacha {
 
-  void decrypt(const void* ciphertext, size_t length, const uint8_t* key, const uint8_t* iv, char* plaintext){
-    if (length < 16){
-      throw std::invalid_argument("Ciphertext length too small");
-    }
+  void decrypt(const void* ciphertext, size_t length, const uint8_t* key, const uint8_t* iv, char* plaintext, size_t *plaintext_len){
+    CHECK_AND_ASSERT_THROW_MES(length >= TAG_SIZE, "Ciphertext length too small");
+    CHECK_AND_ASSERT_THROW_MES(!plaintext_len || *plaintext_len >= (length - TAG_SIZE), "Plaintext length too small");
 
-    unsigned long long int cip_len = length;
+    unsigned long long int res_len = plaintext_len ? *plaintext_len : length;
     auto r = crypto_aead_chacha20poly1305_ietf_decrypt(
-        reinterpret_cast<unsigned char *>(plaintext), &cip_len, nullptr,
+        reinterpret_cast<unsigned char *>(plaintext), &res_len, nullptr,
         static_cast<const unsigned char *>(ciphertext), length, nullptr, 0, iv, key);
 
     if (r != 0){
       throw exc::Poly1305TagInvalid();
     }
+
+    if (plaintext_len){
+      *plaintext_len = (size_t) res_len;
+    }
   }
 
 }
@@ -185,6 +221,49 @@ namespace ki {
     }
   }
 
+  void live_refresh_ack(const ::crypto::secret_key & view_key_priv,
+                        const ::crypto::public_key& out_key,
+                        const std::shared_ptr<messages::monero::MoneroLiveRefreshStepAck> & ack,
+                        ::cryptonote::keypair& in_ephemeral,
+                        ::crypto::key_image& ki)
+  {
+    std::string str_out_key(out_key.data, sizeof(out_key.data));
+    auto enc_key = protocol::tx::compute_enc_key(view_key_priv, str_out_key, ack->salt());
+
+    const size_t len_ciphertext = ack->key_image().size();  // IV || keys
+    CHECK_AND_ASSERT_THROW_MES(len_ciphertext > crypto::chacha::IV_SIZE + crypto::chacha::TAG_SIZE, "Invalid size");
+
+    size_t ki_len = len_ciphertext - crypto::chacha::IV_SIZE - crypto::chacha::TAG_SIZE;
+    std::unique_ptr<uint8_t[]> plaintext(new uint8_t[ki_len]);
+    uint8_t * buff = plaintext.get();
+
+    protocol::crypto::chacha::decrypt(
+        ack->key_image().data() + crypto::chacha::IV_SIZE,
+        len_ciphertext - crypto::chacha::IV_SIZE,
+        reinterpret_cast<const uint8_t *>(enc_key.data),
+        reinterpret_cast<const uint8_t *>(ack->key_image().data()),
+        reinterpret_cast<char *>(buff), &ki_len);
+
+    CHECK_AND_ASSERT_THROW_MES(ki_len == 3*32, "Invalid size");
+    ::crypto::signature sig{};
+    memcpy(ki.data, buff, 32);
+    memcpy(sig.c.data, buff + 32, 32);
+    memcpy(sig.r.data, buff + 64, 32);
+    in_ephemeral.pub = out_key;
+    in_ephemeral.sec = ::crypto::null_skey;
+
+    // Verification
+    std::vector<const ::crypto::public_key*> pkeys;
+    pkeys.push_back(&out_key);
+
+    CHECK_AND_ASSERT_THROW_MES(rct::scalarmultKey(rct::ki2rct(ki), rct::curveOrder()) == rct::identity(),
+                               "Key image out of validity domain: key image " << epee::string_tools::pod_to_hex(ki));
+
+    CHECK_AND_ASSERT_THROW_MES(::crypto::check_ring_signature((const ::crypto::hash&)ki, ki, pkeys, &sig),
+                               "Signature failed for key image " << epee::string_tools::pod_to_hex(ki)
+                                                                 << ", signature " + epee::string_tools::pod_to_hex(sig)
+                                                                 << ", pubkey " + epee::string_tools::pod_to_hex(*pkeys[0]));
+  }
 }
 
 // Cold transaction signing
@@ -198,6 +277,8 @@ namespace tx {
   void translate_dst_entry(MoneroTransactionDestinationEntry * dst, const cryptonote::tx_destination_entry * src){
     dst->set_amount(src->amount);
     dst->set_is_subaddress(src->is_subaddress);
+    dst->set_is_integrated(src->is_integrated);
+    dst->set_original(src->original);
     translate_address(dst->mutable_addr(), &(src->addr));
   }
 
@@ -267,9 +348,29 @@ namespace tx {
     return std::string(buff, offset);
   }
 
+  ::crypto::secret_key compute_enc_key(const ::crypto::secret_key & private_view_key, const std::string & aux, const std::string & salt)
+  {
+    uint8_t hash[32];
+    KECCAK_CTX ctx;
+    ::crypto::secret_key res;
+
+    keccak_init(&ctx);
+    keccak_update(&ctx, (const uint8_t *) private_view_key.data, sizeof(private_view_key.data));
+    if (!aux.empty()){
+      keccak_update(&ctx, (const uint8_t *) aux.data(), aux.size());
+    }
+    keccak_finish(&ctx, hash);
+    keccak(hash, sizeof(hash), hash, sizeof(hash));
+
+    hmac_keccak_hash(hash, (const uint8_t *) salt.data(), salt.size(), hash, sizeof(hash));
+    memcpy(res.data, hash, sizeof(hash));
+    memwipe(hash, sizeof(hash));
+    return res;
+  }
+
   TData::TData() {
-    in_memory = false;
     rsig_type = 0;
+    bp_version = 0;
     cur_input_idx = 0;
     cur_output_idx = 0;
     cur_batch_idx = 0;
@@ -283,6 +384,7 @@ namespace tx {
     m_tx_idx = tx_idx;
     m_ct.tx_data = cur_tx();
     m_multisig = false;
+    m_client_version = 1;
   }
 
   void Signer::extract_payment_id(){
@@ -392,8 +494,10 @@ namespace tx {
 
     m_ct.tx.version = 2;
     m_ct.tx.unlock_time = tx.unlock_time;
+    m_client_version = (m_aux_data->client_version ? m_aux_data->client_version.get() : 1);
 
     tsx_data.set_version(1);
+    tsx_data.set_client_version(client_version());
     tsx_data.set_unlock_time(tx.unlock_time);
     tsx_data.set_num_inputs(static_cast<google::protobuf::uint32>(tx.sources.size()));
     tsx_data.set_mixin(static_cast<google::protobuf::uint32>(tx.sources[0].outputs.size() - 1));
@@ -404,6 +508,10 @@ namespace tx {
     auto rsig_data = tsx_data.mutable_rsig_data();
     m_ct.rsig_type = get_rsig_type(tx.use_bulletproofs, tx.splitted_dsts.size());
     rsig_data->set_rsig_type(m_ct.rsig_type);
+    if (tx.use_bulletproofs){
+      m_ct.bp_version = (m_aux_data->bp_version ? m_aux_data->bp_version.get() : 1);
+      rsig_data->set_bp_version((uint32_t) m_ct.bp_version);
+    }
 
     generate_rsig_batch_sizes(m_ct.grouping_vct, m_ct.rsig_type, tx.splitted_dsts.size());
     assign_to_repeatable(rsig_data->mutable_grouping(), m_ct.grouping_vct.begin(), m_ct.grouping_vct.end());
@@ -437,7 +545,6 @@ namespace tx {
   }
 
   void Signer::step_init_ack(std::shared_ptr<const messages::monero::MoneroTransactionInitAck> ack){
-    m_ct.in_memory = false;
     if (ack->has_rsig_data()){
       m_ct.rsig_param = std::make_shared<MoneroRsigData>(ack->rsig_data());
     }
@@ -505,10 +612,6 @@ namespace tx {
   std::shared_ptr<messages::monero::MoneroTransactionInputsPermutationRequest> Signer::step_permutation(){
     sort_ki();
 
-    if (in_memory()){
-      return nullptr;
-    }
-
     auto res = std::make_shared<messages::monero::MoneroTransactionInputsPermutationRequest>();
     assign_to_repeatable(res->mutable_perm(), m_ct.source_permutation.begin(), m_ct.source_permutation.end());
 
@@ -516,15 +619,10 @@ namespace tx {
   }
 
   void Signer::step_permutation_ack(std::shared_ptr<const messages::monero::MoneroTransactionInputsPermutationAck> ack){
-    if (in_memory()){
-      return;
-    }
+
   }
 
   std::shared_ptr<messages::monero::MoneroTransactionInputViniRequest> Signer::step_set_vini_input(size_t idx){
-    if (in_memory()){
-      return nullptr;
-    }
     CHECK_AND_ASSERT_THROW_MES(idx < m_ct.tx_data.sources.size(), "Invalid transaction index");
     CHECK_AND_ASSERT_THROW_MES(idx < m_ct.tx.vin.size(), "Invalid transaction index");
     CHECK_AND_ASSERT_THROW_MES(idx < m_ct.tx_in_hmacs.size(), "Invalid transaction index");
@@ -536,7 +634,8 @@ namespace tx {
     translate_src_entry(res->mutable_src_entr(), &(tx.sources[idx]));
     res->set_vini(cryptonote::t_serializable_object_to_blob(vini));
     res->set_vini_hmac(m_ct.tx_in_hmacs[idx]);
-    if (!in_memory()) {
+
+    if (client_version() == 0) {
       CHECK_AND_ASSERT_THROW_MES(idx < m_ct.pseudo_outs.size(), "Invalid transaction index");
       CHECK_AND_ASSERT_THROW_MES(idx < m_ct.pseudo_outs_hmac.size(), "Invalid transaction index");
       res->set_pseudo_out(m_ct.pseudo_outs[idx]);
@@ -547,9 +646,7 @@ namespace tx {
   }
 
   void Signer::step_set_vini_input_ack(std::shared_ptr<const messages::monero::MoneroTransactionInputViniAck> ack){
-    if (in_memory()){
-      return;
-    }
+
   }
 
   std::shared_ptr<messages::monero::MoneroTransactionAllInputsSetRequest> Signer::step_all_inputs_set(){
@@ -557,34 +654,37 @@ namespace tx {
   }
 
   void Signer::step_all_inputs_set_ack(std::shared_ptr<const messages::monero::MoneroTransactionAllInputsSetAck> ack){
-    if (is_offloading()){
-      // If offloading, expect rsig configuration.
-      if (!ack->has_rsig_data()){
-        throw exc::ProtocolException("Rsig offloading requires rsig param");
-      }
+    if (client_version() > 0 || !is_offloading()){
+      return;
+    }
 
-      auto & rsig_data = ack->rsig_data();
-      if (!rsig_data.has_mask()){
-        throw exc::ProtocolException("Gamma masks not present in offloaded version");
-      }
+    // If offloading, expect rsig configuration.
+    if (!ack->has_rsig_data()){
+      throw exc::ProtocolException("Rsig offloading requires rsig param");
+    }
 
-      auto & mask = rsig_data.mask();
-      if (mask.size() != 32 * num_outputs()){
-        throw exc::ProtocolException("Invalid number of gamma masks");
-      }
+    auto & rsig_data = ack->rsig_data();
+    if (!rsig_data.has_mask()){
+      throw exc::ProtocolException("Gamma masks not present in offloaded version");
+    }
 
-      m_ct.rsig_gamma.reserve(num_outputs());
-      for(size_t c=0; c < num_outputs(); ++c){
-        rct::key cmask{};
-        memcpy(cmask.bytes, mask.data() + c * 32, 32);
-        m_ct.rsig_gamma.emplace_back(cmask);
-      }
+    auto & mask = rsig_data.mask();
+    if (mask.size() != 32 * num_outputs()){
+      throw exc::ProtocolException("Invalid number of gamma masks");
+    }
+
+    m_ct.rsig_gamma.reserve(num_outputs());
+    for(size_t c=0; c < num_outputs(); ++c){
+      rct::key cmask{};
+      memcpy(cmask.bytes, mask.data() + c * 32, 32);
+      m_ct.rsig_gamma.emplace_back(cmask);
     }
   }
 
   std::shared_ptr<messages::monero::MoneroTransactionSetOutputRequest> Signer::step_set_output(size_t idx){
     CHECK_AND_ASSERT_THROW_MES(idx < m_ct.tx_data.splitted_dsts.size(), "Invalid transaction index");
     CHECK_AND_ASSERT_THROW_MES(idx < m_ct.tx_out_entr_hmacs.size(), "Invalid transaction index");
+    CHECK_AND_ASSERT_THROW_MES(is_req_bulletproof(), "Borromean rsig not supported");
 
     m_ct.cur_output_idx = idx;
     m_ct.cur_output_in_batch_idx += 1;   // assumes sequential call to step_set_output()
@@ -595,48 +695,11 @@ namespace tx {
     res->set_dst_entr_hmac(m_ct.tx_out_entr_hmacs[idx]);
 
     // Range sig offloading to the host
-    if (!is_offloading()) {
-      return res;
-    }
-
-    CHECK_AND_ASSERT_THROW_MES(m_ct.cur_batch_idx < m_ct.grouping_vct.size(), "Invalid batch index");
-    if (m_ct.grouping_vct[m_ct.cur_batch_idx] > m_ct.cur_output_in_batch_idx) {
-      return res;
-    }
-
-    auto rsig_data = res->mutable_rsig_data();
-    auto batch_size = m_ct.grouping_vct[m_ct.cur_batch_idx];
-
-    if (!is_req_bulletproof()){
-      if (batch_size > 1){
-        throw std::invalid_argument("Borromean cannot batch outputs");
-      }
-
-      CHECK_AND_ASSERT_THROW_MES(idx < m_ct.rsig_gamma.size(), "Invalid gamma index");
-      rct::key C{}, mask = m_ct.rsig_gamma[idx];
-      auto genRsig = rct::proveRange(C, mask, cur_dst.amount);  // TODO: rsig with given mask
-      auto serRsig = cn_serialize(genRsig);
-      m_ct.tx_out_rsigs.emplace_back(genRsig);
-      rsig_data->set_rsig(serRsig);
-
-    } else {
-      std::vector<uint64_t> amounts;
-      rct::keyV masks;
-      CHECK_AND_ASSERT_THROW_MES(idx + 1 >= batch_size, "Invalid index for batching");
-
-      for(size_t i = 0; i < batch_size; ++i){
-        const size_t bidx = 1 + idx - batch_size + i;
-        CHECK_AND_ASSERT_THROW_MES(bidx < m_ct.tx_data.splitted_dsts.size(), "Invalid gamma index");
-        CHECK_AND_ASSERT_THROW_MES(bidx < m_ct.rsig_gamma.size(), "Invalid gamma index");
-
-        amounts.push_back(m_ct.tx_data.splitted_dsts[bidx].amount);
-        masks.push_back(m_ct.rsig_gamma[bidx]);
-      }
-
-      auto bp = bulletproof_PROVE(amounts, masks);
-      auto serRsig = cn_serialize(bp);
-      m_ct.tx_out_rsigs.emplace_back(bp);
-      rsig_data->set_rsig(serRsig);
+    // ClientV0 sends offloaded BP with the last message in the batch.
+    // ClientV1 needs additional message after the last message in the batch as BP uses deterministic masks.
+    if (client_version() == 0 && is_offloading() && should_compute_bp_now()) {
+      auto rsig_data = res->mutable_rsig_data();
+      compute_bproof(*rsig_data);
     }
 
     return res;
@@ -644,7 +707,6 @@ namespace tx {
 
   void Signer::step_set_output_ack(std::shared_ptr<const messages::monero::MoneroTransactionSetOutputAck> ack){
     cryptonote::tx_out tx_out;
-    rct::rangeSig range_sig{};
     rct::Bulletproof bproof{};
     rct::ctkey out_pk{};
     rct::ecdhTuple ecdh{};
@@ -658,12 +720,12 @@ namespace tx {
       if (rsig_data.has_rsig() && !rsig_data.rsig().empty()){
         has_rsig = true;
         rsig_buff = rsig_data.rsig();
+      }
 
-      } else if (rsig_data.rsig_parts_size() > 0){
-        has_rsig = true;
-        for (const auto &it : rsig_data.rsig_parts()) {
-          rsig_buff += it;
-        }
+      if (client_version() >= 1 && rsig_data.has_mask()){
+        rct::key cmask{};
+        string_to_key(cmask, rsig_data.mask());
+        m_ct.rsig_gamma.emplace_back(cmask);
       }
     }
 
@@ -675,12 +737,13 @@ namespace tx {
       throw exc::ProtocolException("Cannot deserialize out_pk");
     }
 
-    if (!cn_deserialize(ack->ecdh_info(), ecdh)){
-      throw exc::ProtocolException("Cannot deserialize ecdhtuple");
-    }
-
-    if (has_rsig && !is_req_bulletproof() && !cn_deserialize(rsig_buff, range_sig)){
-      throw exc::ProtocolException("Cannot deserialize rangesig");
+    if (m_ct.bp_version <= 1) {
+      if (!cn_deserialize(ack->ecdh_info(), ecdh)){
+        throw exc::ProtocolException("Cannot deserialize ecdhtuple");
+      }
+    } else {
+      CHECK_AND_ASSERT_THROW_MES(8 == ack->ecdh_info().size(), "Invalid ECDH.amount size");
+      memcpy(ecdh.amount.bytes, ack->ecdh_info().data(), 8);
     }
 
     if (has_rsig && is_req_bulletproof() && !cn_deserialize(rsig_buff, bproof)){
@@ -692,35 +755,77 @@ namespace tx {
     m_ct.tx_out_pk.emplace_back(out_pk);
     m_ct.tx_out_ecdh.emplace_back(ecdh);
 
-    if (!has_rsig){
+    // ClientV0, if no rsig was generated on Trezor, do not continue.
+    // ClientV1+ generates BP after all masks in the current batch are generated
+    if (!has_rsig || (client_version() >= 1 && is_offloading())){
       return;
     }
 
-    if (is_req_bulletproof()){
-      CHECK_AND_ASSERT_THROW_MES(m_ct.cur_batch_idx < m_ct.grouping_vct.size(), "Invalid batch index");
-      auto batch_size = m_ct.grouping_vct[m_ct.cur_batch_idx];
-      for (size_t i = 0; i < batch_size; ++i){
-        const size_t bidx = 1 + m_ct.cur_output_idx - batch_size + i;
-        CHECK_AND_ASSERT_THROW_MES(bidx < m_ct.tx_out_pk.size(), "Invalid out index");
+    process_bproof(bproof);
+    m_ct.cur_batch_idx += 1;
+    m_ct.cur_output_in_batch_idx = 0;
+  }
 
-        rct::key commitment = m_ct.tx_out_pk[bidx].mask;
-        commitment = rct::scalarmultKey(commitment, rct::INV_EIGHT);
-        bproof.V.push_back(commitment);
-      }
+  bool Signer::should_compute_bp_now() const {
+    CHECK_AND_ASSERT_THROW_MES(m_ct.cur_batch_idx < m_ct.grouping_vct.size(), "Invalid batch index");
+    return m_ct.grouping_vct[m_ct.cur_batch_idx] <= m_ct.cur_output_in_batch_idx;
+  }
 
-      m_ct.tx_out_rsigs.emplace_back(bproof);
-      if (!rct::bulletproof_VERIFY(boost::get<rct::Bulletproof>(m_ct.tx_out_rsigs.back()))) {
-        throw exc::ProtocolException("Returned range signature is invalid");
-      }
+  void Signer::compute_bproof(messages::monero::MoneroTransactionRsigData & rsig_data){
+    auto batch_size = m_ct.grouping_vct[m_ct.cur_batch_idx];
+    std::vector<uint64_t> amounts;
+    rct::keyV masks;
+    CHECK_AND_ASSERT_THROW_MES(m_ct.cur_output_idx + 1 >= batch_size, "Invalid index for batching");
 
-    } else {
-      m_ct.tx_out_rsigs.emplace_back(range_sig);
+    for(size_t i = 0; i < batch_size; ++i){
+      const size_t bidx = 1 + m_ct.cur_output_idx - batch_size + i;
+      CHECK_AND_ASSERT_THROW_MES(bidx < m_ct.tx_data.splitted_dsts.size(), "Invalid gamma index");
+      CHECK_AND_ASSERT_THROW_MES(bidx < m_ct.rsig_gamma.size(), "Invalid gamma index");
 
-      if (!rct::verRange(out_pk.mask, boost::get<rct::rangeSig>(m_ct.tx_out_rsigs.back()))) {
-        throw exc::ProtocolException("Returned range signature is invalid");
-      }
+      amounts.push_back(m_ct.tx_data.splitted_dsts[bidx].amount);
+      masks.push_back(m_ct.rsig_gamma[bidx]);
     }
 
+    auto bp = bulletproof_PROVE(amounts, masks);
+    auto serRsig = cn_serialize(bp);
+    m_ct.tx_out_rsigs.emplace_back(bp);
+    rsig_data.set_rsig(serRsig);
+  }
+
+  void Signer::process_bproof(rct::Bulletproof & bproof){
+    CHECK_AND_ASSERT_THROW_MES(m_ct.cur_batch_idx < m_ct.grouping_vct.size(), "Invalid batch index");
+    auto batch_size = m_ct.grouping_vct[m_ct.cur_batch_idx];
+    for (size_t i = 0; i < batch_size; ++i){
+      const size_t bidx = 1 + m_ct.cur_output_idx - batch_size + i;
+      CHECK_AND_ASSERT_THROW_MES(bidx < m_ct.tx_out_pk.size(), "Invalid out index");
+
+      rct::key commitment = m_ct.tx_out_pk[bidx].mask;
+      commitment = rct::scalarmultKey(commitment, rct::INV_EIGHT);
+      bproof.V.push_back(commitment);
+    }
+
+    m_ct.tx_out_rsigs.emplace_back(bproof);
+    if (!rct::bulletproof_VERIFY(boost::get<rct::Bulletproof>(m_ct.tx_out_rsigs.back()))) {
+      throw exc::ProtocolException("Returned range signature is invalid");
+    }
+  }
+
+  std::shared_ptr<messages::monero::MoneroTransactionSetOutputRequest> Signer::step_rsig(size_t idx){
+    if (client_version() == 0 || !is_offloading() || !should_compute_bp_now()){
+      return nullptr;
+    }
+
+    auto res = std::make_shared<messages::monero::MoneroTransactionSetOutputRequest>();
+    auto & cur_dst = m_ct.tx_data.splitted_dsts[idx];
+    translate_dst_entry(res->mutable_dst_entr(), &cur_dst);
+    res->set_dst_entr_hmac(m_ct.tx_out_entr_hmacs[idx]);
+
+    compute_bproof(*(res->mutable_rsig_data()));
+    res->set_is_offloaded_bp(true);
+    return res;
+  }
+
+  void Signer::step_set_rsig_ack(std::shared_ptr<const messages::monero::MoneroTransactionSetOutputAck> ack){
     m_ct.cur_batch_idx += 1;
     m_ct.cur_output_in_batch_idx = 0;
   }
@@ -814,12 +919,11 @@ namespace tx {
     res->set_vini_hmac(m_ct.tx_in_hmacs[idx]);
     res->set_pseudo_out_alpha(m_ct.alphas[idx]);
     res->set_spend_key(m_ct.spend_encs[idx]);
-    if (!in_memory()){
-      CHECK_AND_ASSERT_THROW_MES(idx < m_ct.pseudo_outs.size(), "Invalid transaction index");
-      CHECK_AND_ASSERT_THROW_MES(idx < m_ct.pseudo_outs_hmac.size(), "Invalid transaction index");
-      res->set_pseudo_out(m_ct.pseudo_outs[idx]);
-      res->set_pseudo_out_hmac(m_ct.pseudo_outs_hmac[idx]);
-    }
+
+    CHECK_AND_ASSERT_THROW_MES(idx < m_ct.pseudo_outs.size(), "Invalid transaction index");
+    CHECK_AND_ASSERT_THROW_MES(idx < m_ct.pseudo_outs_hmac.size(), "Invalid transaction index");
+    res->set_pseudo_out(m_ct.pseudo_outs[idx]);
+    res->set_pseudo_out_hmac(m_ct.pseudo_outs_hmac[idx]);
     return res;
   }
 
@@ -829,6 +933,19 @@ namespace tx {
       throw exc::ProtocolException("Cannot deserialize mg[i]");
     }
 
+    // Sync updated pseudo_outputs, client_version>=1, HF10+
+    if (client_version() >= 1 && ack->has_pseudo_out()){
+      CHECK_AND_ASSERT_THROW_MES(m_ct.cur_input_idx < m_ct.pseudo_outs.size(), "Invalid pseudo-out index");
+      m_ct.pseudo_outs[m_ct.cur_input_idx] = ack->pseudo_out();
+      if (is_bulletproof()){
+        CHECK_AND_ASSERT_THROW_MES(m_ct.cur_input_idx < m_ct.rv->p.pseudoOuts.size(), "Invalid pseudo-out index");
+        string_to_key(m_ct.rv->p.pseudoOuts[m_ct.cur_input_idx], ack->pseudo_out());
+      } else {
+        CHECK_AND_ASSERT_THROW_MES(m_ct.cur_input_idx < m_ct.rv->pseudoOuts.size(), "Invalid pseudo-out index");
+        string_to_key(m_ct.rv->pseudoOuts[m_ct.cur_input_idx], ack->pseudo_out());
+      }
+    }
+
     m_ct.rv->p.MGs.push_back(mg);
   }
 
@@ -841,14 +958,14 @@ namespace tx {
     if (m_multisig){
       auto & cout_key = ack->cout_key();
       for(auto & cur : m_ct.couts){
-        if (cur.size() != 12 + 32){
+        if (cur.size() != crypto::chacha::IV_SIZE + 32){
           throw std::invalid_argument("Encrypted cout has invalid length");
         }
 
         char buff[32];
         auto data = cur.data();
 
-        crypto::chacha::decrypt(data + 12, 32, reinterpret_cast<const uint8_t *>(cout_key.data()), reinterpret_cast<const uint8_t *>(data), buff);
+        crypto::chacha::decrypt(data + crypto::chacha::IV_SIZE, 32, reinterpret_cast<const uint8_t *>(cout_key.data()), reinterpret_cast<const uint8_t *>(data), buff);
         m_ct.couts_dec.emplace_back(buff, 32);
       }
     }
@@ -887,6 +1004,82 @@ namespace tx {
     return sb.GetString();
   }
 
+  void load_tx_key_data(hw::device_cold::tx_key_data_t & res, const std::string & data)
+  {
+    rapidjson::Document json;
+
+    // The contents should be JSON if the wallet follows the new format.
+    if (json.Parse(data.c_str()).HasParseError())
+    {
+      throw std::invalid_argument("Data parsing error");
+    }
+    else if(!json.IsObject())
+    {
+      throw std::invalid_argument("Data parsing error - not an object");
+    }
+
+    GET_FIELD_FROM_JSON(json, version, int, Int, true, -1);
+    GET_STRING_FROM_JSON(json, salt1, std::string, true, std::string());
+    GET_STRING_FROM_JSON(json, salt2, std::string, true, std::string());
+    GET_STRING_FROM_JSON(json, enc_keys, std::string, true, std::string());
+    GET_STRING_FROM_JSON(json, tx_prefix_hash, std::string, false, std::string());
+
+    if (field_version != 1)
+    {
+      throw std::invalid_argument("Unknown version");
+    }
+
+    res.salt1 = field_salt1;
+    res.salt2 = field_salt2;
+    res.tx_enc_keys = field_enc_keys;
+    res.tx_prefix_hash = field_tx_prefix_hash;
+  }
+
+  std::shared_ptr<messages::monero::MoneroGetTxKeyRequest> get_tx_key(
+      const hw::device_cold::tx_key_data_t & tx_data)
+  {
+    auto req = std::make_shared<messages::monero::MoneroGetTxKeyRequest>();
+    req->set_salt1(tx_data.salt1);
+    req->set_salt2(tx_data.salt2);
+    req->set_tx_enc_keys(tx_data.tx_enc_keys);
+    req->set_tx_prefix_hash(tx_data.tx_prefix_hash);
+    req->set_reason(0);
+
+    return req;
+  }
+
+  void get_tx_key_ack(
+      std::vector<::crypto::secret_key> & tx_keys,
+      const std::string & tx_prefix_hash,
+      const ::crypto::secret_key & view_key_priv,
+      std::shared_ptr<const messages::monero::MoneroGetTxKeyAck> ack
+  )
+  {
+    auto enc_key = protocol::tx::compute_enc_key(view_key_priv, tx_prefix_hash, ack->salt());
+    auto & encrypted_keys = ack->has_tx_derivations() ? ack->tx_derivations() : ack->tx_keys();
+
+    const size_t len_ciphertext = encrypted_keys.size();  // IV || keys || TAG
+    CHECK_AND_ASSERT_THROW_MES(len_ciphertext > crypto::chacha::IV_SIZE + crypto::chacha::TAG_SIZE, "Invalid size");
+
+    size_t keys_len = len_ciphertext - crypto::chacha::IV_SIZE - crypto::chacha::TAG_SIZE;
+    std::unique_ptr<uint8_t[]> plaintext(new uint8_t[keys_len]);
+
+    protocol::crypto::chacha::decrypt(
+        encrypted_keys.data() + crypto::chacha::IV_SIZE,
+        len_ciphertext - crypto::chacha::IV_SIZE,
+        reinterpret_cast<const uint8_t *>(enc_key.data),
+        reinterpret_cast<const uint8_t *>(encrypted_keys.data()),
+        reinterpret_cast<char *>(plaintext.get()), &keys_len);
+
+    CHECK_AND_ASSERT_THROW_MES(keys_len % 32 == 0, "Invalid size");
+    tx_keys.resize(keys_len / 32);
+
+    for(unsigned i = 0; i < keys_len / 32; ++i)
+    {
+      memcpy(tx_keys[i].data, plaintext.get() + 32 * i, 32);
+    }
+    memwipe(plaintext.get(), keys_len);
+  }
 
 }
 }

src/device_trezor/trezor/protocol.hpp

@@ -92,11 +92,14 @@ namespace protocol{
 // Crypto / encryption
 namespace crypto {
 namespace chacha {
+  // Constants as defined in RFC 7539.
+  const unsigned IV_SIZE = 12;
+  const unsigned TAG_SIZE = 16;  // crypto_aead_chacha20poly1305_IETF_ABYTES;
 
   /**
    * Chacha20Poly1305 decryption with tag verification. RFC 7539.
    */
-  void decrypt(const void* ciphertext, size_t length, const uint8_t* key, const uint8_t* iv, char* plaintext);
+  void decrypt(const void* ciphertext, size_t length, const uint8_t* key, const uint8_t* iv, char* plaintext, size_t *plaintext_len=nullptr);
 
 }
 }
@@ -129,6 +132,14 @@ namespace ki {
                            const std::vector<tools::wallet2::transfer_details> & transfers,
                            std::shared_ptr<messages::monero::MoneroKeyImageExportInitRequest> & req);
 
+  /**
+   * Processes Live refresh step response, parses KI, checks the signature
+   */
+  void live_refresh_ack(const ::crypto::secret_key & view_key_priv,
+                        const ::crypto::public_key& out_key,
+                        const std::shared_ptr<messages::monero::MoneroLiveRefreshStepAck> & ack,
+                        ::cryptonote::keypair& in_ephemeral,
+                        ::crypto::key_image& ki);
 }
 
 // Cold transaction signing
@@ -153,6 +164,7 @@ namespace tx {
   std::string hash_addr(const MoneroAccountPublicAddress * addr, boost::optional<uint64_t> amount = boost::none, boost::optional<bool> is_subaddr = boost::none);
   std::string hash_addr(const std::string & spend_key, const std::string & view_key, boost::optional<uint64_t> amount = boost::none, boost::optional<bool> is_subaddr = boost::none);
   std::string hash_addr(const ::crypto::public_key * spend_key, const ::crypto::public_key * view_key, boost::optional<uint64_t> amount = boost::none, boost::optional<bool> is_subaddr = boost::none);
+  ::crypto::secret_key compute_enc_key(const ::crypto::secret_key & private_view_key, const std::string & aux, const std::string & salt);
 
   typedef boost::variant<rct::rangeSig, rct::Bulletproof> rsig_v;
 
@@ -164,8 +176,8 @@ namespace tx {
     TsxData tsx_data;
     tx_construction_data tx_data;
     cryptonote::transaction tx;
-    bool in_memory;
     unsigned rsig_type;
+    int bp_version;
     std::vector<uint64_t> grouping_vct;
     std::shared_ptr<MoneroRsigData> rsig_param;
     size_t cur_input_idx;
@@ -206,6 +218,7 @@ namespace tx {
     const unsigned_tx_set * m_unsigned_tx;
     hw::tx_aux_data * m_aux_data;
 
+    unsigned m_client_version;
     bool m_multisig;
 
     const tx_construction_data & cur_tx(){
@@ -215,6 +228,9 @@ namespace tx {
 
     void extract_payment_id();
     void compute_integrated_indices(TsxData * tsx_data);
+    bool should_compute_bp_now() const;
+    void compute_bproof(messages::monero::MoneroTransactionRsigData & rsig_data);
+    void process_bproof(rct::Bulletproof & bproof);
 
   public:
     Signer(wallet_shim * wallet2, const unsigned_tx_set * unsigned_tx, size_t tx_idx = 0, hw::tx_aux_data * aux_data = nullptr);
@@ -238,6 +254,9 @@ namespace tx {
     std::shared_ptr<messages::monero::MoneroTransactionSetOutputRequest> step_set_output(size_t idx);
     void step_set_output_ack(std::shared_ptr<const messages::monero::MoneroTransactionSetOutputAck> ack);
 
+    std::shared_ptr<messages::monero::MoneroTransactionSetOutputRequest> step_rsig(size_t idx);
+    void step_set_rsig_ack(std::shared_ptr<const messages::monero::MoneroTransactionSetOutputAck> ack);
+
     std::shared_ptr<messages::monero::MoneroTransactionAllOutSetRequest> step_all_outs_set();
     void step_all_outs_set_ack(std::shared_ptr<const messages::monero::MoneroTransactionAllOutSetAck> ack, hw::device &hwdev);
 
@@ -249,8 +268,8 @@ namespace tx {
 
     std::string store_tx_aux_info();
 
-    bool in_memory() const {
-      return m_ct.in_memory;
+    unsigned client_version() const {
+      return m_client_version;
     }
 
     bool is_simple() const {
@@ -290,6 +309,18 @@ namespace tx {
     }
   };
 
+  // TX Key decryption
+  void load_tx_key_data(hw::device_cold::tx_key_data_t & res, const std::string & data);
+
+  std::shared_ptr<messages::monero::MoneroGetTxKeyRequest> get_tx_key(
+      const hw::device_cold::tx_key_data_t & tx_data);
+
+  void get_tx_key_ack(
+      std::vector<::crypto::secret_key> & tx_keys,
+      const std::string & tx_prefix_hash,
+      const ::crypto::secret_key & view_key_priv,
+      std::shared_ptr<const messages::monero::MoneroGetTxKeyAck> ack
+  );
 }
 
 }

src/simplewallet/simplewallet.cpp

@@ -2049,7 +2049,7 @@ bool simple_wallet::cold_sign_tx(const std::vector<tools::wallet2::pending_tx>&
   m_wallet->cold_tx_aux_import(exported_txs.ptx, tx_aux);
 
   // import key images
-  return m_wallet->import_key_images(exported_txs.key_images);
+  return m_wallet->import_key_images(exported_txs, 0, true);
 }
 
 bool simple_wallet::set_always_confirm_transfers(const std::vector<std::string> &args/* = std::vector<std::string>()*/)
@@ -4686,12 +4686,12 @@ boost::optional<epee::wipeable_string> simple_wallet::on_get_password(const char
   return pwd_container->password();
 }
 //----------------------------------------------------------------------------------------------------
-void simple_wallet::on_button_request()
+void simple_wallet::on_device_button_request(uint64_t code)
 {
   message_writer(console_color_white, false) << tr("Device requires attention");
 }
 //----------------------------------------------------------------------------------------------------
-void simple_wallet::on_pin_request(epee::wipeable_string & pin)
+boost::optional<epee::wipeable_string> simple_wallet::on_device_pin_request()
 {
 #ifdef HAVE_READLINE
   rdln::suspend_readline pause_readline;
@@ -4699,14 +4699,14 @@ void simple_wallet::on_pin_request(epee::wipeable_string & pin)
   std::string msg = tr("Enter device PIN");
   auto pwd_container = tools::password_container::prompt(false, msg.c_str());
   THROW_WALLET_EXCEPTION_IF(!pwd_container, tools::error::password_entry_failed, tr("Failed to read device PIN"));
-  pin = pwd_container->password();
+  return pwd_container->password();
 }
 //----------------------------------------------------------------------------------------------------
-void simple_wallet::on_passphrase_request(bool on_device, epee::wipeable_string & passphrase)
+boost::optional<epee::wipeable_string> simple_wallet::on_device_passphrase_request(bool on_device)
 {
   if (on_device){
     message_writer(console_color_white, true) << tr("Please enter the device passphrase on the device");
-    return;
+    return boost::none;
   }
 
 #ifdef HAVE_READLINE
@@ -4715,13 +4715,13 @@ void simple_wallet::on_passphrase_request(bool on_device, epee::wipeable_string
   std::string msg = tr("Enter device passphrase");
   auto pwd_container = tools::password_container::prompt(false, msg.c_str());
   THROW_WALLET_EXCEPTION_IF(!pwd_container, tools::error::password_entry_failed, tr("Failed to read device passphrase"));
-  passphrase = pwd_container->password();
+  return pwd_container->password();
 }
 //----------------------------------------------------------------------------------------------------
 void simple_wallet::on_refresh_finished(uint64_t start_height, uint64_t fetched_blocks, bool is_init, bool received_money)
 {
   // Key image sync after the first refresh
-  if (!m_wallet->get_account().get_device().has_tx_cold_sign()) {
+  if (!m_wallet->get_account().get_device().has_tx_cold_sign() || m_wallet->get_account().get_device().has_ki_live_refresh()) {
     return;
   }
 
@@ -6772,7 +6772,7 @@ bool simple_wallet::get_tx_key(const std::vector<std::string> &args_)
 {
   std::vector<std::string> local_args = args_;
 
-  if (m_wallet->key_on_device())
+  if (m_wallet->key_on_device() && m_wallet->get_account().get_device().get_type() != hw::device::TREZOR)
   {
     fail_msg_writer() << tr("command not supported by HW wallet");
     return true;
@@ -6793,7 +6793,9 @@ bool simple_wallet::get_tx_key(const std::vector<std::string> &args_)
 
   crypto::secret_key tx_key;
   std::vector<crypto::secret_key> additional_tx_keys;
-  if (m_wallet->get_tx_key(txid, tx_key, additional_tx_keys))
+
+  bool found_tx_key = m_wallet->get_tx_key(txid, tx_key, additional_tx_keys);
+  if (found_tx_key)
   {
     ostringstream oss;
     oss << epee::string_tools::pod_to_hex(tx_key);
@@ -6869,7 +6871,7 @@ bool simple_wallet::set_tx_key(const std::vector<std::string> &args_)
 //----------------------------------------------------------------------------------------------------
 bool simple_wallet::get_tx_proof(const std::vector<std::string> &args)
 {
-  if (m_wallet->key_on_device())
+  if (m_wallet->key_on_device() && m_wallet->get_account().get_device().get_type() != hw::device::TREZOR)
   {
     fail_msg_writer() << tr("command not supported by HW wallet");
     return true;

src/simplewallet/simplewallet.h

@@ -300,9 +300,9 @@ namespace cryptonote
     virtual void on_money_spent(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& in_tx, uint64_t amount, const cryptonote::transaction& spend_tx, const cryptonote::subaddress_index& subaddr_index);
     virtual void on_skip_transaction(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& tx);
     virtual boost::optional<epee::wipeable_string> on_get_password(const char *reason);
-    virtual void on_button_request();
-    virtual void on_pin_request(epee::wipeable_string & pin);
-    virtual void on_passphrase_request(bool on_device, epee::wipeable_string & passphrase);
+    virtual void on_device_button_request(uint64_t code);
+    virtual boost::optional<epee::wipeable_string> on_device_pin_request();
+    virtual boost::optional<epee::wipeable_string> on_device_passphrase_request(bool on_device);
     //----------------------------------------------------------
 
     friend class refresh_progress_reporter_t;

src/wallet/api/pending_transaction.cpp

@@ -109,6 +109,23 @@ bool PendingTransactionImpl::commit(const std::string &filename, bool overwrite)
         }
 
         m_wallet.pauseRefresh();
+
+        const bool tx_cold_signed = m_wallet.m_wallet->get_account().get_device().has_tx_cold_sign();
+        if (tx_cold_signed){
+          std::unordered_set<size_t> selected_transfers;
+          for(const tools::wallet2::pending_tx & ptx : m_pending_tx){
+            for(size_t s : ptx.selected_transfers){
+              selected_transfers.insert(s);
+            }
+          }
+
+          m_wallet.m_wallet->cold_tx_aux_import(m_pending_tx, m_tx_device_aux);
+          bool r = m_wallet.m_wallet->import_key_images(m_key_images, 0, selected_transfers);
+          if (!r){
+            throw runtime_error("Cold sign transaction submit failed - key image sync fail");
+          }
+        }
+
         while (!m_pending_tx.empty()) {
             auto & ptx = m_pending_tx.back();
             m_wallet.m_wallet->commit_tx(ptx);

src/wallet/api/pending_transaction.h

@@ -67,6 +67,8 @@ private:
     std::string m_errorString;
     std::vector<tools::wallet2::pending_tx> m_pending_tx;
     std::unordered_set<crypto::public_key> m_signers;
+    std::vector<std::string> m_tx_device_aux;
+    std::vector<crypto::key_image> m_key_images;
 };
 
 

src/wallet/api/wallet.cpp

@@ -242,6 +242,42 @@ struct Wallet2CallbackImpl : public tools::i_wallet2_callback
       }
     }
 
+    virtual void on_device_button_request(uint64_t code)
+    {
+      if (m_listener) {
+        m_listener->onDeviceButtonRequest(code);
+      }
+    }
+
+    virtual boost::optional<epee::wipeable_string> on_device_pin_request()
+    {
+      if (m_listener) {
+        auto pin = m_listener->onDevicePinRequest();
+        if (pin){
+          return boost::make_optional(epee::wipeable_string((*pin).data(), (*pin).size()));
+        }
+      }
+      return boost::none;
+    }
+
+    virtual boost::optional<epee::wipeable_string> on_device_passphrase_request(bool on_device)
+    {
+      if (m_listener) {
+        auto passphrase = m_listener->onDevicePassphraseRequest(on_device);
+        if (!on_device && passphrase) {
+          return boost::make_optional(epee::wipeable_string((*passphrase).data(), (*passphrase).size()));
+        }
+      }
+      return boost::none;
+    }
+
+    virtual void on_device_progress(const hw::device_progress & event)
+    {
+      if (m_listener) {
+        m_listener->onDeviceProgress(DeviceProgress(event.progress(), event.indeterminate()));
+      }
+    }
+
     WalletListener * m_listener;
     WalletImpl     * m_wallet;
 };
@@ -785,6 +821,28 @@ bool WalletImpl::setPassword(const std::string &password)
     return status() == Status_Ok;
 }
 
+bool WalletImpl::setDevicePin(const std::string &pin)
+{
+    clearStatus();
+    try {
+        m_wallet->get_account().get_device().set_pin(epee::wipeable_string(pin.data(), pin.size()));
+    } catch (const std::exception &e) {
+        setStatusError(e.what());
+    }
+    return status() == Status_Ok;
+}
+
+bool WalletImpl::setDevicePassphrase(const std::string &passphrase)
+{
+    clearStatus();
+    try {
+        m_wallet->get_account().get_device().set_passphrase(epee::wipeable_string(passphrase.data(), passphrase.size()));
+    } catch (const std::exception &e) {
+        setStatusError(e.what());
+    }
+    return status() == Status_Ok;
+}
+
 std::string WalletImpl::address(uint32_t accountIndex, uint32_t addressIndex) const
 {
     return m_wallet->get_subaddress_as_str({accountIndex, addressIndex});
@@ -1428,6 +1486,8 @@ PendingTransaction *WalletImpl::createTransaction(const string &dst_addr, const
                                                                           extra, subaddr_account, subaddr_indices);
             }
 
+            pendingTxPostProcess(transaction);
+
             if (multisig().isMultisig) {
                 transaction->m_signers = m_wallet->make_multisig_tx_set(transaction->m_pending_tx).m_signers;
             }
@@ -1511,6 +1571,7 @@ PendingTransaction *WalletImpl::createSweepUnmixableTransaction()
     do {
         try {
             transaction->m_pending_tx = m_wallet->create_unmixable_sweep_transactions();
+            pendingTxPostProcess(transaction);
 
         } catch (const tools::error::daemon_busy&) {
             // TODO: make it translatable with "tr"?
@@ -2093,6 +2154,21 @@ bool WalletImpl::isNewWallet() const
     return !(blockChainHeight() > 1 || m_recoveringFromSeed || m_recoveringFromDevice || m_rebuildWalletCache) && !watchOnly();
 }
 
+void WalletImpl::pendingTxPostProcess(PendingTransactionImpl * pending)
+{
+  // If the device being used is HW device with cold signing protocol, cold sign then.
+  if (!m_wallet->get_account().get_device().has_tx_cold_sign()){
+    return;
+  }
+
+  tools::wallet2::signed_tx_set exported_txs;
+  std::vector<cryptonote::address_parse_info> dsts_info;
+
+  m_wallet->cold_sign_tx(pending->m_pending_tx, exported_txs, dsts_info, pending->m_tx_device_aux);
+  pending->m_key_images = exported_txs.key_images;
+  pending->m_pending_tx = exported_txs.ptx;
+}
+
 bool WalletImpl::doInit(const string &daemon_address, uint64_t upper_transaction_size_limit, bool ssl)
 {
     // claim RPC so there's no in-memory encryption for now
@@ -2325,6 +2401,11 @@ bool WalletImpl::isKeysFileLocked()
 {
     return m_wallet->is_keys_file_locked();
 }
+
+uint64_t WalletImpl::coldKeyImageSync(uint64_t &spent, uint64_t &unspent)
+{
+    return m_wallet->cold_key_image_sync(spent, unspent);
+}
 } // namespace
 
 namespace Bitmonero = Monero;

src/wallet/api/wallet.h

@@ -89,6 +89,8 @@ public:
     std::string errorString() const override;
     void statusWithErrorString(int& status, std::string& errorString) const override;
     bool setPassword(const std::string &password) override;
+    bool setDevicePin(const std::string &password) override;
+    bool setDevicePassphrase(const std::string &password) override;
     std::string address(uint32_t accountIndex = 0, uint32_t addressIndex = 0) const override;
     std::string integratedAddress(const std::string &payment_id) const override;
     std::string secretViewKey() const override;
@@ -198,6 +200,7 @@ public:
     virtual bool lockKeysFile() override;
     virtual bool unlockKeysFile() override;
     virtual bool isKeysFileLocked() override;
+    virtual uint64_t coldKeyImageSync(uint64_t &spent, uint64_t &unspent) override;
 
 private:
     void clearStatus() const;
@@ -209,6 +212,7 @@ private:
     bool daemonSynced() const;
     void stopRefresh();
     bool isNewWallet() const;
+    void pendingTxPostProcess(PendingTransactionImpl * pending);
     bool doInit(const std::string &daemon_address, uint64_t upper_transaction_size_limit = 0, bool ssl = false);
 
 private:

src/wallet/api/wallet2_api.h

@@ -324,6 +324,19 @@ struct MultisigState {
     uint32_t total;
 };
 
+
+struct DeviceProgress {
+    DeviceProgress(): m_progress(0), m_indeterminate(false) {}
+    DeviceProgress(double progress, bool indeterminate=false): m_progress(progress), m_indeterminate(indeterminate) {}
+
+    virtual double progress() const { return m_progress; }
+    virtual bool indeterminate() const { return m_indeterminate; }
+
+protected:
+    double m_progress;
+    bool m_indeterminate;
+};
+
 struct WalletListener
 {
     virtual ~WalletListener() = 0;
@@ -364,6 +377,31 @@ struct WalletListener
      * @brief refreshed - called when wallet refreshed by background thread or explicitly refreshed by calling "refresh" synchronously
      */
     virtual void refreshed() = 0;
+
+    /**
+     * @brief called by device if the action is required
+     */
+    virtual void onDeviceButtonRequest(uint64_t code) {}
+
+    /**
+     * @brief called by device when PIN is needed
+     */
+    virtual optional<std::string> onDevicePinRequest() {
+        throw std::runtime_error("Not supported");
+    }
+
+    /**
+     * @brief called by device when passphrase entry is needed
+     */
+    virtual optional<std::string> onDevicePassphraseRequest(bool on_device) {
+        if (!on_device) throw std::runtime_error("Not supported");
+        return optional<std::string>();
+    }
+
+    /**
+     * @brief Signalizes device operation progress
+     */
+    virtual void onDeviceProgress(const DeviceProgress & event) {};
 };
 
 
@@ -375,7 +413,8 @@ struct Wallet
 {
     enum Device {
         Device_Software = 0,
-        Device_Ledger = 1
+        Device_Ledger = 1,
+        Device_Trezor = 2
     };
 
     enum Status {
@@ -401,6 +440,8 @@ struct Wallet
     //! returns both error and error string atomically. suggested to use in instead of status() and errorString()
     virtual void statusWithErrorString(int& status, std::string& errorString) const = 0;
     virtual bool setPassword(const std::string &password) = 0;
+    virtual bool setDevicePin(const std::string &password) { return false; };
+    virtual bool setDevicePassphrase(const std::string &password) { return false; };
     virtual std::string address(uint32_t accountIndex = 0, uint32_t addressIndex = 0) const = 0;
     std::string mainAddress() const { return address(0, 0); }
     virtual std::string path() const = 0;
@@ -947,6 +988,9 @@ struct Wallet
      * \return Device they are on
      */
     virtual Device getDeviceType() const = 0;
+
+    //! cold-device protocol key image sync
+    virtual uint64_t coldKeyImageSync(uint64_t &spent, uint64_t &unspent) = 0;
 };
 
 /**

src/wallet/wallet2.cpp

@@ -928,22 +928,30 @@ wallet_keys_unlocker::~wallet_keys_unlocker()
   }
 }
 
-void wallet_device_callback::on_button_request()
+void wallet_device_callback::on_button_request(uint64_t code)
 {
   if (wallet)
-    wallet->on_button_request();
+    wallet->on_device_button_request(code);
 }
 
-void wallet_device_callback::on_pin_request(epee::wipeable_string & pin)
+boost::optional<epee::wipeable_string> wallet_device_callback::on_pin_request()
 {
   if (wallet)
-    wallet->on_pin_request(pin);
+    return wallet->on_device_pin_request();
+  return boost::none;
 }
 
-void wallet_device_callback::on_passphrase_request(bool on_device, epee::wipeable_string & passphrase)
+boost::optional<epee::wipeable_string> wallet_device_callback::on_passphrase_request(bool on_device)
 {
   if (wallet)
-    wallet->on_passphrase_request(on_device, passphrase);
+    return wallet->on_device_passphrase_request(on_device);
+  return boost::none;
+}
+
+void wallet_device_callback::on_progress(const hw::device_progress& event)
+{
+  if (wallet)
+    wallet->on_device_progress(event);
 }
 
 wallet2::wallet2(network_type nettype, uint64_t kdf_rounds, bool unattended):
@@ -2883,6 +2891,7 @@ void wallet2::refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blo
   std::vector<parsed_block> parsed_blocks;
   bool refreshed = false;
   std::shared_ptr<std::map<std::pair<uint64_t, uint64_t>, size_t>> output_tracker_cache;
+  hw::device &hwdev = m_account.get_device();
 
   // pull the first set of blocks
   get_short_chain_history(short_chain_history, (m_first_refresh_done || trusted_daemon) ? 1 : FIRST_REFRESH_GRANULARITY);
@@ -3039,6 +3048,7 @@ void wallet2::refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blo
     LOG_PRINT_L1("Failed to check pending transactions");
   }
 
+  hwdev.computing_key_images(false);
   m_first_refresh_done = true;
 
   LOG_PRINT_L1("Refresh done, blocks received: " << blocks_fetched << ", balance (all accounts): " << print_money(balance_all()) << ", unlocked: " << print_money(unlocked_balance_all()));
@@ -9670,6 +9680,7 @@ void wallet2::cold_sign_tx(const std::vector<pending_tx>& ptx_vector, signed_tx_
   hw::wallet_shim wallet_shim;
   setup_shim(&wallet_shim, this);
   aux_data.tx_recipients = dsts_info;
+  aux_data.bp_version = use_fork_rules(HF_VERSION_SMALLER_BP, -10) ? 2 : 1;
   dev_cold->tx_sign(&wallet_shim, txs, exported_txs, aux_data);
   tx_device_aux = aux_data.tx_device_aux;
 
@@ -9886,7 +9897,7 @@ void wallet2::discard_unmixable_outputs()
   }
 }
 
-bool wallet2::get_tx_key(const crypto::hash &txid, crypto::secret_key &tx_key, std::vector<crypto::secret_key> &additional_tx_keys) const
+bool wallet2::get_tx_key_cached(const crypto::hash &txid, crypto::secret_key &tx_key, std::vector<crypto::secret_key> &additional_tx_keys) const
 {
   additional_tx_keys.clear();
   const std::unordered_map<crypto::hash, crypto::secret_key>::const_iterator i = m_tx_keys.find(txid);
@@ -9899,6 +9910,82 @@ bool wallet2::get_tx_key(const crypto::hash &txid, crypto::secret_key &tx_key, s
   return true;
 }
 //----------------------------------------------------------------------------------------------------
+bool wallet2::get_tx_key(const crypto::hash &txid, crypto::secret_key &tx_key, std::vector<crypto::secret_key> &additional_tx_keys)
+{
+  bool r = get_tx_key_cached(txid, tx_key, additional_tx_keys);
+  if (r)
+  {
+    return true;
+  }
+
+  auto & hwdev = get_account().get_device();
+
+  // So far only Cold protocol devices are supported.
+  if (hwdev.device_protocol() != hw::device::PROTOCOL_COLD)
+  {
+    return false;
+  }
+
+  const auto tx_data_it = m_tx_device.find(txid);
+  if (tx_data_it == m_tx_device.end())
+  {
+    MDEBUG("Aux data not found for txid: " << txid);
+    return false;
+  }
+
+  auto dev_cold = dynamic_cast<::hw::device_cold*>(&hwdev);
+  CHECK_AND_ASSERT_THROW_MES(dev_cold, "Device does not implement cold signing interface");
+  if (!dev_cold->is_get_tx_key_supported())
+  {
+    MDEBUG("get_tx_key not supported by the device");
+    return false;
+  }
+
+  hw::device_cold::tx_key_data_t tx_key_data;
+  dev_cold->load_tx_key_data(tx_key_data, tx_data_it->second);
+
+  // Load missing tx prefix hash
+  if (tx_key_data.tx_prefix_hash.empty())
+  {
+    COMMAND_RPC_GET_TRANSACTIONS::request req;
+    COMMAND_RPC_GET_TRANSACTIONS::response res;
+    req.txs_hashes.push_back(epee::string_tools::pod_to_hex(txid));
+    req.decode_as_json = false;
+    req.prune = true;
+    m_daemon_rpc_mutex.lock();
+    bool ok = epee::net_utils::invoke_http_json("/gettransactions", req, res, m_http_client);
+    m_daemon_rpc_mutex.unlock();
+    THROW_WALLET_EXCEPTION_IF(!ok || (res.txs.size() != 1 && res.txs_as_hex.size() != 1),
+                              error::wallet_internal_error, "Failed to get transaction from daemon");
+
+    cryptonote::transaction tx;
+    crypto::hash tx_hash{};
+    cryptonote::blobdata tx_data;
+    crypto::hash tx_prefix_hash{};
+    ok = string_tools::parse_hexstr_to_binbuff(res.txs_as_hex.front(), tx_data);
+    THROW_WALLET_EXCEPTION_IF(!ok, error::wallet_internal_error, "Failed to parse transaction from daemon");
+    THROW_WALLET_EXCEPTION_IF(!cryptonote::parse_and_validate_tx_from_blob(tx_data, tx, tx_hash, tx_prefix_hash),
+                              error::wallet_internal_error, "Failed to validate transaction from daemon");
+    THROW_WALLET_EXCEPTION_IF(tx_hash != txid, error::wallet_internal_error,
+                              "Failed to get the right transaction from daemon");
+
+    tx_key_data.tx_prefix_hash = std::string(tx_prefix_hash.data, 32);
+  }
+
+  std::vector<crypto::secret_key> tx_keys;
+  dev_cold->get_tx_key(tx_keys, tx_key_data, m_account.get_keys().m_view_secret_key);
+  if (tx_keys.empty())
+  {
+    return false;
+  }
+
+  tx_key = tx_keys[0];
+  tx_keys.erase(tx_keys.begin());
+  additional_tx_keys = tx_keys;
+
+  return true;
+}
+//----------------------------------------------------------------------------------------------------
 void wallet2::set_tx_key(const crypto::hash &txid, const crypto::secret_key &tx_key, const std::vector<crypto::secret_key> &additional_tx_keys)
 {
   // fetch tx from daemon and check if secret keys agree with corresponding public keys
@@ -10301,7 +10388,8 @@ std::string wallet2::get_tx_proof(const crypto::hash &txid, const cryptonote::ac
   {
     crypto::secret_key tx_key;
     std::vector<crypto::secret_key> additional_tx_keys;
-    THROW_WALLET_EXCEPTION_IF(!get_tx_key(txid, tx_key, additional_tx_keys), error::wallet_internal_error, "Tx secret key wasn't found in the wallet file.");
+    bool found_tx_key = get_tx_key(txid, tx_key, additional_tx_keys);
+    THROW_WALLET_EXCEPTION_IF(!found_tx_key, error::wallet_internal_error, "Tx secret key wasn't found in the wallet file.");
 
     const size_t num_sigs = 1 + additional_tx_keys.size();
     shared_secret.resize(num_sigs);
@@ -11475,29 +11563,48 @@ uint64_t wallet2::import_key_images(const std::vector<std::pair<crypto::key_imag
   return m_transfers[signed_key_images.size() - 1].m_block_height;
 }
 
-bool wallet2::import_key_images(std::vector<crypto::key_image> key_images)
+bool wallet2::import_key_images(std::vector<crypto::key_image> key_images, size_t offset, boost::optional<std::unordered_set<size_t>> selected_transfers)
 {
-  if (key_images.size() > m_transfers.size())
+  if (key_images.size() + offset > m_transfers.size())
   {
     LOG_PRINT_L1("More key images returned that we know outputs for");
     return false;
   }
-  for (size_t i = 0; i < key_images.size(); ++i)
+  for (size_t ki_idx = 0; ki_idx < key_images.size(); ++ki_idx)
   {
-    transfer_details &td = m_transfers[i];
-    if (td.m_key_image_known && !td.m_key_image_partial && td.m_key_image != key_images[i])
-      LOG_PRINT_L0("WARNING: imported key image differs from previously known key image at index " << i << ": trusting imported one");
-    td.m_key_image = key_images[i];
-    m_key_images[m_transfers[i].m_key_image] = i;
+    const size_t transfer_idx = ki_idx + offset;
+    if (selected_transfers && selected_transfers.get().find(transfer_idx) == selected_transfers.get().end())
+      continue;
+
+    transfer_details &td = m_transfers[transfer_idx];
+    if (td.m_key_image_known && !td.m_key_image_partial && td.m_key_image != key_images[ki_idx])
+      LOG_PRINT_L0("WARNING: imported key image differs from previously known key image at index " << ki_idx << ": trusting imported one");
+    td.m_key_image = key_images[ki_idx];
+    m_key_images[td.m_key_image] = transfer_idx;
     td.m_key_image_known = true;
     td.m_key_image_request = false;
     td.m_key_image_partial = false;
-    m_pub_keys[m_transfers[i].get_public_key()] = i;
+    m_pub_keys[td.get_public_key()] = transfer_idx;
   }
 
   return true;
 }
 
+bool wallet2::import_key_images(signed_tx_set & signed_tx, size_t offset, bool only_selected_transfers)
+{
+  std::unordered_set<size_t> selected_transfers;
+  if (only_selected_transfers)
+  {
+    for (const pending_tx & ptx : signed_tx.ptx)
+    {
+      for (const size_t s: ptx.selected_transfers)
+        selected_transfers.insert(s);
+    }
+  }
+
+  return import_key_images(signed_tx.key_images, offset, only_selected_transfers ? boost::make_optional(selected_transfers) : boost::none);
+}
+
 wallet2::payment_container wallet2::export_payments() const
 {
   payment_container payments;
@@ -12478,22 +12585,30 @@ wallet_device_callback * wallet2::get_device_callback()
   }
   return m_device_callback.get();
 }//----------------------------------------------------------------------------------------------------
-void wallet2::on_button_request()
+void wallet2::on_device_button_request(uint64_t code)
 {
-  if (0 != m_callback)
-    m_callback->on_button_request();
+  if (nullptr != m_callback)
+    m_callback->on_device_button_request(code);
 }
 //----------------------------------------------------------------------------------------------------
-void wallet2::on_pin_request(epee::wipeable_string & pin)
+boost::optional<epee::wipeable_string> wallet2::on_device_pin_request()
 {
-  if (0 != m_callback)
-    m_callback->on_pin_request(pin);
+  if (nullptr != m_callback)
+    return m_callback->on_device_pin_request();
+  return boost::none;
 }
 //----------------------------------------------------------------------------------------------------
-void wallet2::on_passphrase_request(bool on_device, epee::wipeable_string & passphrase)
+boost::optional<epee::wipeable_string> wallet2::on_device_passphrase_request(bool on_device)
 {
-  if (0 != m_callback)
-    m_callback->on_passphrase_request(on_device, passphrase);
+  if (nullptr != m_callback)
+    return m_callback->on_device_passphrase_request(on_device);
+  return boost::none;
+}
+//----------------------------------------------------------------------------------------------------
+void wallet2::on_device_progress(const hw::device_progress& event)
+{
+  if (nullptr != m_callback)
+    m_callback->on_device_progress(event);
 }
 //----------------------------------------------------------------------------------------------------
 std::string wallet2::get_rpc_status(const std::string &s) const

src/wallet/wallet2.h

@@ -103,9 +103,10 @@ namespace tools
     virtual void on_lw_unconfirmed_money_received(uint64_t height, const crypto::hash &txid, uint64_t amount) {}
     virtual void on_lw_money_spent(uint64_t height, const crypto::hash &txid, uint64_t amount) {}
     // Device callbacks
-    virtual void on_button_request() {}
-    virtual void on_pin_request(epee::wipeable_string & pin) {}
-    virtual void on_passphrase_request(bool on_device, epee::wipeable_string & passphrase) {}
+    virtual void on_device_button_request(uint64_t code) {}
+    virtual boost::optional<epee::wipeable_string> on_device_pin_request() { return boost::none; }
+    virtual boost::optional<epee::wipeable_string> on_device_passphrase_request(bool on_device) { return boost::none; }
+    virtual void on_device_progress(const hw::device_progress& event) {};
     // Common callbacks
     virtual void on_pool_tx_removed(const crypto::hash &txid) {}
     virtual ~i_wallet2_callback() {}
@@ -115,9 +116,10 @@ namespace tools
   {
   public:
     wallet_device_callback(wallet2 * wallet): wallet(wallet) {};
-    void on_button_request() override;
-    void on_pin_request(epee::wipeable_string & pin) override;
-    void on_passphrase_request(bool on_device, epee::wipeable_string & passphrase) override;
+    void on_button_request(uint64_t code=0) override;
+    boost::optional<epee::wipeable_string> on_pin_request() override;
+    boost::optional<epee::wipeable_string> on_passphrase_request(bool on_device) override;
+    void on_progress(const hw::device_progress& event) override;
   private:
     wallet2 * wallet;
   };
@@ -1005,8 +1007,9 @@ namespace tools
     const std::string & device_derivation_path() const { return m_device_derivation_path; }
     void device_derivation_path(const std::string &device_derivation_path) { m_device_derivation_path = device_derivation_path; }
 
-    bool get_tx_key(const crypto::hash &txid, crypto::secret_key &tx_key, std::vector<crypto::secret_key> &additional_tx_keys) const;
+    bool get_tx_key_cached(const crypto::hash &txid, crypto::secret_key &tx_key, std::vector<crypto::secret_key> &additional_tx_keys) const;
     void set_tx_key(const crypto::hash &txid, const crypto::secret_key &tx_key, const std::vector<crypto::secret_key> &additional_tx_keys);
+    bool get_tx_key(const crypto::hash &txid, crypto::secret_key &tx_key, std::vector<crypto::secret_key> &additional_tx_keys);
     void check_tx_key(const crypto::hash &txid, const crypto::secret_key &tx_key, const std::vector<crypto::secret_key> &additional_tx_keys, const cryptonote::account_public_address &address, uint64_t &received, bool &in_pool, uint64_t &confirmations);
     void check_tx_key_helper(const crypto::hash &txid, const crypto::key_derivation &derivation, const std::vector<crypto::key_derivation> &additional_derivations, const cryptonote::account_public_address &address, uint64_t &received, bool &in_pool, uint64_t &confirmations);
     std::string get_tx_proof(const crypto::hash &txid, const cryptonote::account_public_address &address, bool is_subaddress, const std::string &message);
@@ -1128,7 +1131,8 @@ namespace tools
     std::pair<size_t, std::vector<std::pair<crypto::key_image, crypto::signature>>> export_key_images(bool all = false) const;
     uint64_t import_key_images(const std::vector<std::pair<crypto::key_image, crypto::signature>> &signed_key_images, size_t offset, uint64_t &spent, uint64_t &unspent, bool check_spent = true);
     uint64_t import_key_images(const std::string &filename, uint64_t &spent, uint64_t &unspent);
-    bool import_key_images(std::vector<crypto::key_image> key_images);
+    bool import_key_images(std::vector<crypto::key_image> key_images, size_t offset=0, boost::optional<std::unordered_set<size_t>> selected_transfers=boost::none);
+    bool import_key_images(signed_tx_set & signed_tx, size_t offset=0, bool only_selected_transfers=false);
     crypto::public_key get_tx_pub_key_from_received_outs(const tools::wallet2::transfer_details &td) const;
 
     void update_pool_state(bool refreshed = false);
@@ -1338,9 +1342,10 @@ namespace tools
     void create_keys_file(const std::string &wallet_, bool watch_only, const epee::wipeable_string &password, bool create_address_file);
 
     wallet_device_callback * get_device_callback();
-    void on_button_request();
-    void on_pin_request(epee::wipeable_string & pin);
-    void on_passphrase_request(bool on_device, epee::wipeable_string & passphrase);
+    void on_device_button_request(uint64_t code);
+    boost::optional<epee::wipeable_string> on_device_pin_request();
+    boost::optional<epee::wipeable_string> on_device_passphrase_request(bool on_device);
+    void on_device_progress(const hw::device_progress& event);
 
     std::string get_rpc_status(const std::string &s) const;
     void throw_on_rpc_response_error(const boost::optional<std::string> &status, const char *method) const;

tests/core_tests/chaingen.h

@@ -754,7 +754,7 @@ struct get_test_options {
 };
 //--------------------------------------------------------------------------
 template<class t_test_class>
-inline bool do_replay_events_get_core(std::vector<test_event_entry>& events, cryptonote::core **core)
+inline bool do_replay_events_get_core(std::vector<test_event_entry>& events, cryptonote::core *core)
 {
   boost::program_options::options_description desc("Allowed options");
   cryptonote::core::init_options(desc);
@@ -768,8 +768,7 @@ inline bool do_replay_events_get_core(std::vector<test_event_entry>& events, cry
   if (!r)
     return false;
 
-  *core = new cryptonote::core(nullptr);
-  auto & c = **core;
+  auto & c = *core;
 
   // FIXME: make sure that vm has arg_testnet_on set to true or false if
   // this test needs for it to be so.
@@ -825,9 +824,9 @@ inline bool replay_events_through_core_validate(std::vector<test_event_entry>& e
 template<class t_test_class>
 inline bool do_replay_events(std::vector<test_event_entry>& events)
 {
-  cryptonote::core * core;
+  cryptonote::core core(nullptr);
   bool ret = do_replay_events_get_core<t_test_class>(events, &core);
-  core->deinit();
+  core.deinit();
   return ret;
 }
 //--------------------------------------------------------------------------

tests/core_tests/wallet_tools.cpp

@@ -17,7 +17,6 @@ void wallet_accessor_test::set_account(tools::wallet2 * wallet, cryptonote::acco
 {
   wallet->clear();
   wallet->m_account = account;
-  wallet->m_nettype = MAINNET;
 
   wallet->m_key_device_type = account.get_device().get_type();
   wallet->m_account_public_address = account.get_keys().m_account_address;

tests/trezor/CMakeLists.txt

@@ -27,11 +27,15 @@
 # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 set(trezor_tests_sources
+        tools.cpp
+        daemon.cpp
         trezor_tests.cpp
         ../core_tests/chaingen.cpp
         ../core_tests/wallet_tools.cpp)
 
 set(trezor_tests_headers
+        tools.h
+        daemon.h
         trezor_tests.h
         ../core_tests/chaingen.h
         ../core_tests/wallet_tools.h)
@@ -50,6 +54,15 @@ target_link_libraries(trezor_tests
     device
     device_trezor
     wallet
+    wallet_api
+    rpc
+    cryptonote_protocol
+    daemon_rpc_server
+    ${Boost_CHRONO_LIBRARY}
+    ${Boost_FILESYSTEM_LIBRARY}
+    ${Boost_PROGRAM_OPTIONS_LIBRARY}
+    ${Boost_SYSTEM_LIBRARY}
+    ${ZMQ_LIB}
     ${CMAKE_THREAD_LIBS_INIT}
     ${EXTRA_LIBRARIES})
 

tests/trezor/daemon.cpp

@@ -0,0 +1,368 @@
+// Copyright (c) 2014-2018, The Monero Project
+//
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "daemon.h"
+#include <common/command_line.h>
+
+using namespace std;
+using namespace daemonize;
+namespace po = boost::program_options;
+
+bool mock_rpc_daemon::on_send_raw_tx_2(const cryptonote::COMMAND_RPC_SEND_RAW_TX::request& req, cryptonote::COMMAND_RPC_SEND_RAW_TX::response& res, const cryptonote::core_rpc_server::connection_context *ctx)
+{
+  cryptonote::COMMAND_RPC_SEND_RAW_TX::request req2(req);
+  req2.do_not_relay = true;  // Do not relay in test setup, only one daemon running.
+  return cryptonote::core_rpc_server::on_send_raw_tx(req2, res, ctx);
+}
+
+void mock_daemon::init_options(boost::program_options::options_description & option_spec)
+{
+  cryptonote::core::init_options(option_spec);
+  t_node_server::init_options(option_spec);
+  cryptonote::core_rpc_server::init_options(option_spec);
+
+  command_line::add_arg(option_spec, daemon_args::arg_zmq_rpc_bind_ip);
+  command_line::add_arg(option_spec, daemon_args::arg_zmq_rpc_bind_port);
+}
+
+void mock_daemon::default_options(boost::program_options::variables_map & vm)
+{
+  std::vector<std::string> exclusive_nodes{"127.0.0.1:65525"};
+  tools::options::set_option(vm, nodetool::arg_p2p_add_exclusive_node, po::variable_value(exclusive_nodes, false));
+
+  tools::options::set_option(vm, nodetool::arg_p2p_bind_ip, po::variable_value(std::string("127.0.0.1"), false));
+  tools::options::set_option(vm, nodetool::arg_no_igd, po::variable_value(true, false));
+  tools::options::set_option(vm, cryptonote::arg_offline, po::variable_value(true, false));
+  tools::options::set_option(vm, "disable-dns-checkpoints", po::variable_value(true, false));
+
+  const char *test_mainnet = getenv("TEST_MAINNET");
+  if (!test_mainnet || atoi(test_mainnet) == 0)
+  {
+    tools::options::set_option(vm, cryptonote::arg_testnet_on, po::variable_value(true, false));
+  }
+
+  // By default pick non-standard ports to avoid confusion with possibly locally running daemons (mainnet/testnet)
+  const char *test_p2p_port = getenv("TEST_P2P_PORT");
+  auto p2p_port = std::string(test_p2p_port && strlen(test_p2p_port) > 0 ? test_p2p_port : "61340");
+  tools::options::set_option(vm, nodetool::arg_p2p_bind_port, po::variable_value(p2p_port, false));
+
+  const char *test_rpc_port = getenv("TEST_RPC_PORT");
+  auto rpc_port = std::string(test_rpc_port && strlen(test_rpc_port) > 0 ? test_rpc_port : "61341");
+  tools::options::set_option(vm, cryptonote::core_rpc_server::arg_rpc_bind_port, po::variable_value(rpc_port, false));
+
+  const char *test_zmq_port = getenv("TEST_ZMQ_PORT");
+  auto zmq_port = std::string(test_zmq_port && strlen(test_zmq_port) > 0 ? test_zmq_port : "61342");
+  tools::options::set_option(vm, daemon_args::arg_zmq_rpc_bind_port, po::variable_value(zmq_port, false));
+
+  po::notify(vm);
+}
+
+void mock_daemon::set_ports(boost::program_options::variables_map & vm, unsigned initial_port)
+{
+  CHECK_AND_ASSERT_THROW_MES(initial_port < 65535-2, "Invalid port number");
+  tools::options::set_option(vm, nodetool::arg_p2p_bind_port, po::variable_value(std::to_string(initial_port), false));
+  tools::options::set_option(vm, cryptonote::core_rpc_server::arg_rpc_bind_port, po::variable_value(std::to_string(initial_port + 1), false));
+  tools::options::set_option(vm, daemon_args::arg_zmq_rpc_bind_port, po::variable_value(std::to_string(initial_port + 2), false));
+  po::notify(vm);
+}
+
+void mock_daemon::load_params(boost::program_options::variables_map const & vm)
+{
+  m_p2p_bind_port = command_line::get_arg(vm, nodetool::arg_p2p_bind_port);
+  m_rpc_bind_port = command_line::get_arg(vm, cryptonote::core_rpc_server::arg_rpc_bind_port);
+  m_zmq_bind_port = command_line::get_arg(vm, daemon_args::arg_zmq_rpc_bind_port);
+  m_network_type = command_line::get_arg(vm, cryptonote::arg_testnet_on) ? cryptonote::TESTNET : cryptonote::MAINNET;
+}
+
+mock_daemon::~mock_daemon()
+{
+  if (!m_terminated)
+  {
+    try
+    {
+      stop();
+    }
+    catch (...)
+    {
+      MERROR("Failed to stop");
+    }
+  }
+
+  if (!m_deinitalized)
+  {
+    deinit();
+  }
+}
+
+void mock_daemon::init()
+{
+  m_deinitalized = false;
+  const auto main_rpc_port = command_line::get_arg(m_vm, cryptonote::core_rpc_server::arg_rpc_bind_port);
+  m_rpc_server.nettype(m_network_type);
+
+  CHECK_AND_ASSERT_THROW_MES(m_protocol.init(m_vm), "Failed to initialize cryptonote protocol.");
+  CHECK_AND_ASSERT_THROW_MES(m_rpc_server.init(m_vm, false, main_rpc_port), "Failed to initialize RPC server.");
+
+  if (m_start_p2p)
+    CHECK_AND_ASSERT_THROW_MES(m_server.init(m_vm), "Failed to initialize p2p server.");
+
+  if(m_http_client.is_connected())
+    m_http_client.disconnect();
+
+  CHECK_AND_ASSERT_THROW_MES(m_http_client.set_server(rpc_addr(), boost::none), "RPC client init fail");
+}
+
+void mock_daemon::deinit()
+{
+  try
+  {
+    m_rpc_server.deinit();
+  }
+  catch (...)
+  {
+    MERROR("Failed to deinitialize RPC server...");
+  }
+
+  if (m_start_p2p)
+  {
+    try
+    {
+      m_server.deinit();
+    }
+    catch (...)
+    {
+      MERROR("Failed to deinitialize p2p...");
+    }
+  }
+
+  try
+  {
+    m_protocol.deinit();
+    m_protocol.set_p2p_endpoint(nullptr);
+  }
+  catch (...)
+  {
+    MERROR("Failed to stop cryptonote protocol!");
+  }
+
+  m_deinitalized = true;
+}
+
+void mock_daemon::init_and_run()
+{
+  init();
+  run();
+}
+
+void mock_daemon::stop_and_deinit()
+{
+  stop();
+  deinit();
+}
+
+void mock_daemon::try_init_and_run(boost::optional<unsigned> initial_port)
+{
+  const unsigned max_attempts = 3;
+  for(unsigned attempts=0; attempts < max_attempts; ++attempts)
+  {
+    if (initial_port)
+    {
+      set_ports(m_vm, initial_port.get());
+      load_params(m_vm);
+      MDEBUG("Ports changed, RPC: " << rpc_addr());
+    }
+
+    try
+    {
+      init_and_run();
+      return;
+    }
+    catch(const std::exception &e)
+    {
+      MWARNING("Could not init and start, attempt: " << attempts << ", reason: " << e.what());
+      if (attempts + 1 >= max_attempts)
+      {
+        throw;
+      }
+    }
+  }
+}
+
+void mock_daemon::run()
+{
+  m_run_thread = boost::thread(boost::bind(&mock_daemon::run_main, this));
+}
+
+bool mock_daemon::run_main()
+{
+  CHECK_AND_ASSERT_THROW_MES(!m_terminated, "Can't run stopped daemon");
+  CHECK_AND_ASSERT_THROW_MES(!m_start_zmq || m_start_p2p, "ZMQ requires P2P");
+  boost::thread stop_thread = boost::thread([this] {
+    while (!this->m_stopped)
+      epee::misc_utils::sleep_no_w(100);
+    this->stop_p2p();
+  });
+
+  epee::misc_utils::auto_scope_leave_caller scope_exit_handler = epee::misc_utils::create_scope_leave_handler([&](){
+    m_stopped = true;
+    stop_thread.join();
+  });
+
+  try
+  {
+    CHECK_AND_ASSERT_THROW_MES(m_rpc_server.run(2, false), "Failed to start RPC");
+    cryptonote::rpc::DaemonHandler rpc_daemon_handler(*m_core, m_server);
+    cryptonote::rpc::ZmqServer zmq_server(rpc_daemon_handler);
+
+    if (m_start_zmq)
+    {
+      if (!zmq_server.addTCPSocket("127.0.0.1", m_zmq_bind_port))
+      {
+        MERROR("Failed to add TCP Socket (127.0.0.1:" << m_zmq_bind_port << ") to ZMQ RPC Server");
+
+        stop_rpc();
+        return false;
+      }
+
+      MINFO("Starting ZMQ server...");
+      zmq_server.run();
+
+      MINFO("ZMQ server started at 127.0.0.1: " << m_zmq_bind_port);
+    }
+
+    if (m_start_p2p)
+    {
+      m_server.run();  // blocks until p2p goes down
+    }
+    else
+    {
+      while (!this->m_stopped)
+        epee::misc_utils::sleep_no_w(100);
+    }
+
+    if (m_start_zmq)
+      zmq_server.stop();
+
+    stop_rpc();
+    return true;
+  }
+  catch (std::exception const & ex)
+  {
+    MFATAL("Uncaught exception! " << ex.what());
+    return false;
+  }
+  catch (...)
+  {
+    MFATAL("Uncaught exception!");
+    return false;
+  }
+}
+
+void mock_daemon::stop()
+{
+  CHECK_AND_ASSERT_THROW_MES(!m_terminated, "Can't stop stopped daemon");
+  m_stopped = true;
+  m_terminated = true;
+  m_run_thread.join();
+}
+
+void mock_daemon::stop_rpc()
+{
+  m_rpc_server.send_stop_signal();
+  m_rpc_server.timed_wait_server_stop(5000);
+}
+
+void mock_daemon::stop_p2p()
+{
+  if (m_start_p2p)
+    m_server.send_stop_signal();
+}
+
+void mock_daemon::mine_blocks(size_t num_blocks, const std::string &miner_address)
+{
+  bool blocks_mined = false;
+  const uint64_t start_height = get_height();
+  const auto mining_timeout = std::chrono::seconds(30);
+  MDEBUG("Current height before mining: " << start_height);
+
+  start_mining(miner_address);
+  auto mining_started = std::chrono::system_clock::now();
+
+  while(true) {
+    epee::misc_utils::sleep_no_w(100);
+    const uint64_t cur_height = get_height();
+
+    if (cur_height - start_height >= num_blocks)
+    {
+      MDEBUG("Cur blocks: " << cur_height << " start: " << start_height);
+      blocks_mined = true;
+      break;
+    }
+
+    auto current_time = std::chrono::system_clock::now();
+    if (mining_timeout < current_time - mining_started)
+    {
+      break;
+    }
+  }
+
+  stop_mining();
+  CHECK_AND_ASSERT_THROW_MES(blocks_mined, "Mining failed in the time limit");
+}
+
+constexpr const std::chrono::seconds mock_daemon::rpc_timeout;
+
+void mock_daemon::start_mining(const std::string &miner_address, uint64_t threads_count, bool do_background_mining, bool ignore_battery)
+{
+  cryptonote::COMMAND_RPC_START_MINING::request req;
+  req.miner_address = miner_address;
+  req.threads_count = threads_count;
+  req.do_background_mining = do_background_mining;
+  req.ignore_battery = ignore_battery;
+
+  cryptonote::COMMAND_RPC_START_MINING::response resp;
+  bool r = epee::net_utils::invoke_http_json("/start_mining", req, resp, m_http_client, rpc_timeout);
+  CHECK_AND_ASSERT_THROW_MES(r, "RPC error - start mining");
+  CHECK_AND_ASSERT_THROW_MES(resp.status != CORE_RPC_STATUS_BUSY, "Daemon busy");
+  CHECK_AND_ASSERT_THROW_MES(resp.status == CORE_RPC_STATUS_OK, "Daemon response invalid: " << resp.status);
+}
+
+void mock_daemon::stop_mining()
+{
+  cryptonote::COMMAND_RPC_STOP_MINING::request req;
+  cryptonote::COMMAND_RPC_STOP_MINING::response resp;
+  bool r = epee::net_utils::invoke_http_json("/stop_mining", req, resp, m_http_client, rpc_timeout);
+  CHECK_AND_ASSERT_THROW_MES(r, "RPC error - stop mining");
+  CHECK_AND_ASSERT_THROW_MES(resp.status != CORE_RPC_STATUS_BUSY, "Daemon busy");
+  CHECK_AND_ASSERT_THROW_MES(resp.status == CORE_RPC_STATUS_OK, "Daemon response invalid: " << resp.status);
+}
+
+uint64_t mock_daemon::get_height()
+{
+  return m_core->get_blockchain_storage().get_current_blockchain_height();
+}

tests/trezor/daemon.h

@@ -0,0 +1,154 @@
+// Copyright (c) 2014-2018, The Monero Project
+//
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#pragma once
+
+#include "misc_log_ex.h"
+#include "daemon/daemon.h"
+#include "rpc/daemon_handler.h"
+#include "rpc/zmq_server.h"
+#include "common/password.h"
+#include "common/util.h"
+#include "daemon/core.h"
+#include "daemon/p2p.h"
+#include "daemon/protocol.h"
+#include "daemon/rpc.h"
+#include "daemon/command_server.h"
+#include "daemon/command_server.h"
+#include "daemon/command_line_args.h"
+#include "version.h"
+#include "tools.h"
+
+
+class mock_rpc_daemon : public cryptonote::core_rpc_server {
+public:
+  mock_rpc_daemon(
+    cryptonote::core& cr
+  , nodetool::node_server<cryptonote::t_cryptonote_protocol_handler<cryptonote::core> >& p2p
+  ): cryptonote::core_rpc_server(cr, p2p) {}
+
+  static void init_options(boost::program_options::options_description& desc){ cryptonote::core_rpc_server::init_options(desc); }
+  cryptonote::network_type nettype() const { return m_network_type; }
+  void nettype(cryptonote::network_type nettype) { m_network_type = nettype; }
+
+  CHAIN_HTTP_TO_MAP2(cryptonote::core_rpc_server::connection_context); //forward http requests to uri map
+  BEGIN_URI_MAP2()
+    MAP_URI_AUTO_JON2("/send_raw_transaction", on_send_raw_tx_2, cryptonote::COMMAND_RPC_SEND_RAW_TX)
+    MAP_URI_AUTO_JON2("/sendrawtransaction", on_send_raw_tx_2, cryptonote::COMMAND_RPC_SEND_RAW_TX)
+    else {  // Default to parent for non-overriden callbacks
+      return cryptonote::core_rpc_server::handle_http_request_map(query_info, response_info, m_conn_context);
+    }
+  END_URI_MAP2()
+
+  bool on_send_raw_tx_2(const cryptonote::COMMAND_RPC_SEND_RAW_TX::request& req, cryptonote::COMMAND_RPC_SEND_RAW_TX::response& res, const cryptonote::core_rpc_server::connection_context *ctx);
+
+protected:
+  cryptonote::network_type m_network_type;
+};
+
+class mock_daemon {
+public:
+  typedef cryptonote::t_cryptonote_protocol_handler<cryptonote::core> t_protocol_raw;
+  typedef nodetool::node_server<t_protocol_raw> t_node_server;
+
+  static constexpr const std::chrono::seconds rpc_timeout = std::chrono::seconds(60);
+
+  cryptonote::core * m_core;
+  t_protocol_raw m_protocol;
+  mock_rpc_daemon m_rpc_server;
+  t_node_server m_server;
+  cryptonote::network_type m_network_type;
+  epee::net_utils::http::http_simple_client m_http_client;
+
+  bool m_start_p2p;
+  bool m_start_zmq;
+  boost::program_options::variables_map m_vm;
+
+  std::string m_p2p_bind_port;
+  std::string m_rpc_bind_port;
+  std::string m_zmq_bind_port;
+
+  std::atomic<bool> m_stopped;
+  std::atomic<bool> m_terminated;
+  std::atomic<bool> m_deinitalized;
+  boost::thread m_run_thread;
+
+  mock_daemon(
+      cryptonote::core * core,
+      boost::program_options::variables_map const & vm
+  )
+      : m_core(core)
+      , m_vm(vm)
+      , m_start_p2p(false)
+      , m_start_zmq(false)
+      , m_terminated(false)
+      , m_deinitalized(false)
+      , m_stopped(false)
+      , m_protocol{*core, nullptr, command_line::get_arg(vm, cryptonote::arg_offline)}
+      , m_server{m_protocol}
+      , m_rpc_server{*core, m_server}
+  {
+    // Handle circular dependencies
+    m_protocol.set_p2p_endpoint(&m_server);
+    m_core->set_cryptonote_protocol(&m_protocol);
+    load_params(vm);
+  }
+
+  virtual ~mock_daemon();
+
+  static void init_options(boost::program_options::options_description & option_spec);
+  static void default_options(boost::program_options::variables_map & vm);
+  static void set_ports(boost::program_options::variables_map & vm, unsigned initial_port);
+
+  mock_daemon * set_start_p2p(bool fl) { m_start_p2p = fl; return this; }
+  mock_daemon * set_start_zmq(bool fl) { m_start_zmq = fl; return this; }
+
+  void init();
+  void deinit();
+  void run();
+  bool run_main();
+  void stop();
+  void stop_p2p();
+  void stop_rpc();
+  void init_and_run();
+  void stop_and_deinit();
+  void try_init_and_run(boost::optional<unsigned> initial_port=boost::none);
+
+  void mine_blocks(size_t num_blocks, const std::string &miner_address);
+  void start_mining(const std::string &miner_address, uint64_t threads_count=1, bool do_background_mining=false, bool ignore_battery=true);
+  void stop_mining();
+  uint64_t get_height();
+
+  void load_params(boost::program_options::variables_map const & vm);
+
+  std::string zmq_addr() const { return std::string("127.0.0.1:") + m_zmq_bind_port; }
+  std::string rpc_addr() const { return std::string("127.0.0.1:") + m_rpc_bind_port; }
+  std::string p2p_addr() const { return std::string("127.0.0.1:") + m_p2p_bind_port; }
+  cryptonote::network_type nettype() const { return m_network_type; }
+  cryptonote::core * core() const { return m_core; }
+};

tests/trezor/tools.cpp

@@ -0,0 +1,56 @@
+// Copyright (c) 2014-2018, The Monero Project
+//
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "tools.h"
+
+namespace tools {
+
+namespace po = boost::program_options;
+
+void options::set_option(boost::program_options::variables_map &vm, const std::string & key, const po::variable_value &pv)
+{
+  auto it = vm.find(key);
+  if (it == vm.end())
+  {
+    vm.insert(std::make_pair(key, pv));
+  }
+  else
+    {
+    it->second = pv;
+  }
+}
+
+void options::build_options(boost::program_options::variables_map & vm, const po::options_description & desc_params)
+{
+  const char *argv[2] = {nullptr};
+  po::store(po::parse_command_line(1, argv, desc_params), vm);
+  po::notify(vm);
+}
+
+}
+

tests/trezor/tools.h

@@ -0,0 +1,61 @@
+// Copyright (c) 2014-2018, The Monero Project
+//
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#pragma once
+
+#include "misc_log_ex.h"
+#include "daemon/daemon.h"
+#include "rpc/daemon_handler.h"
+#include "rpc/zmq_server.h"
+#include "common/password.h"
+#include "common/util.h"
+#include "daemon/core.h"
+#include "daemon/p2p.h"
+#include "daemon/protocol.h"
+#include "daemon/rpc.h"
+#include "daemon/command_server.h"
+#include "daemon/command_server.h"
+#include "daemon/command_line_args.h"
+#include "version.h"
+
+namespace tools {
+
+class options {
+public:
+  static void set_option(boost::program_options::variables_map &vm, const std::string &key, const boost::program_options::variable_value &pv);
+  static void build_options(boost::program_options::variables_map & vm, const boost::program_options::options_description & desc_params);
+
+  template<typename T, bool required, bool dependent, int NUM_DEPS>
+  static void set_option(boost::program_options::variables_map &vm, const command_line::arg_descriptor<T, required, dependent, NUM_DEPS> &arg, const boost::program_options::variable_value &pv)
+  {
+    set_option(vm, arg.name, pv);
+  }
+};
+
+};
+

tests/trezor/trezor_tests.cpp

@@ -41,6 +41,7 @@ using namespace cryptonote;
 #include "common/util.h"
 #include "common/command_line.h"
 #include "trezor_tests.h"
+#include "tools.h"
 #include "device/device_cold.hpp"
 #include "device_trezor/device_trezor.hpp"
 
@@ -57,7 +58,7 @@ namespace
   const command_line::arg_descriptor<bool>        arg_fix_chain                   = {"fix_chain", "If chain_patch is given and file cannot be used, it is ignored and overwriten", false};
 }
 
-
+#define HW_TREZOR_NAME "Trezor"
 #define TREZOR_ACCOUNT_ORDERING &m_miner_account, &m_alice_account, &m_bob_account, &m_eve_account
 #define TREZOR_COMMON_TEST_CASE(genclass, CORE, BASE)                                                           \
   rollback_chain(CORE, BASE.head_block());                                                                      \
@@ -70,14 +71,17 @@ namespace
 #define TREZOR_SETUP_CHAIN(NAME) do {                                                                           \
   ++tests_count;                                                                                                \
   try {                                                                                                         \
-    setup_chain(&core, trezor_base, chain_path, fix_chain);                                                     \
+    setup_chain(core, trezor_base, chain_path, fix_chain, vm_core);                                            \
   } catch (const std::exception& ex) {                                                                          \
     failed_tests.emplace_back("gen_trezor_base " #NAME);                                                        \
   }                                                                                                             \
 } while(0)
 
+
+static device_trezor_test *trezor_device = nullptr;
+static device_trezor_test *ensure_trezor_test_device();
 static void rollback_chain(cryptonote::core * core, const cryptonote::block & head);
-static void setup_chain(cryptonote::core ** core, gen_trezor_base & trezor_base, std::string chain_path, bool fix_chain);
+static void setup_chain(cryptonote::core * core, gen_trezor_base & trezor_base, std::string chain_path, bool fix_chain, const po::variables_map & vm_core);
 
 int main(int argc, char* argv[])
 {
@@ -123,38 +127,76 @@ int main(int argc, char* argv[])
     const bool heavy_tests = command_line::get_arg(vm, arg_heavy_tests);
     const bool fix_chain = command_line::get_arg(vm, arg_fix_chain);
 
-    hw::trezor::register_all();
+    hw::register_device(HW_TREZOR_NAME, ensure_trezor_test_device());
+    // hw::trezor::register_all();  // We use our shim instead.
 
     // Bootstrapping common chain & accounts
-    cryptonote::core * core = nullptr;
+    const uint8_t initial_hf = 9;
+    const uint8_t max_hf = 10;
+
+    cryptonote::core core_obj(nullptr);
+    cryptonote::core * const core = &core_obj;
+    std::shared_ptr<mock_daemon> daemon = nullptr;
 
     gen_trezor_base trezor_base;
     trezor_base.setup_args(trezor_path, heavy_tests);
-    trezor_base.rct_config({rct::RangeProofPaddedBulletproof, 1});  // HF9 tests
+    trezor_base.set_hard_fork(initial_hf);
 
-    TREZOR_SETUP_CHAIN("HF9");
-
-    // Individual test cases using shared pre-generated blockchain.
-    TREZOR_COMMON_TEST_CASE(gen_trezor_ki_sync, core, trezor_base);
+    // Arguments for core & daemon
+    po::variables_map vm_core;
+    po::options_description desc_params_core("Core");
+    mock_daemon::init_options(desc_params_core);
+    tools::options::build_options(vm_core, desc_params_core);
+    mock_daemon::default_options(vm_core);
 
     // Transaction tests
-    TREZOR_COMMON_TEST_CASE(gen_trezor_1utxo, core, trezor_base);
-    TREZOR_COMMON_TEST_CASE(gen_trezor_1utxo_paymentid_short, core, trezor_base);
-    TREZOR_COMMON_TEST_CASE(gen_trezor_1utxo_paymentid_short_integrated, core, trezor_base);
-    TREZOR_COMMON_TEST_CASE(gen_trezor_1utxo_paymentid_long, core, trezor_base);
-    TREZOR_COMMON_TEST_CASE(gen_trezor_4utxo, core, trezor_base);
-    TREZOR_COMMON_TEST_CASE(gen_trezor_4utxo_acc1, core, trezor_base);
-    TREZOR_COMMON_TEST_CASE(gen_trezor_4utxo_to_sub, core, trezor_base);
-    TREZOR_COMMON_TEST_CASE(gen_trezor_4utxo_to_2sub, core, trezor_base);
-    TREZOR_COMMON_TEST_CASE(gen_trezor_4utxo_to_1norm_2sub, core, trezor_base);
-    TREZOR_COMMON_TEST_CASE(gen_trezor_2utxo_sub_acc_to_1norm_2sub, core, trezor_base);
-    TREZOR_COMMON_TEST_CASE(gen_trezor_4utxo_to_7outs, core, trezor_base);
+    for(uint8_t hf=initial_hf; hf <= max_hf; ++hf)
+    {
+      MDEBUG("Transaction tests for HF " << (int)hf);
+      if (hf > initial_hf)
+      {
+        daemon->stop_and_deinit();
+        daemon = nullptr;
+        trezor_base.daemon(nullptr);
+      }
+
+      trezor_base.set_hard_fork(hf);
+      TREZOR_SETUP_CHAIN(std::string("HF") + std::to_string((int)hf));
+
+      daemon = std::make_shared<mock_daemon>(core, vm_core);
+      CHECK_AND_ASSERT_THROW_MES(daemon->nettype() == trezor_base.nettype(), "Serialized chain network type does not match");
+
+      daemon->try_init_and_run();
+      trezor_base.daemon(daemon);
+
+      // Hard-fork independent tests
+      if (hf == initial_hf)
+      {
+        TREZOR_COMMON_TEST_CASE(gen_trezor_ki_sync_without_refresh, core, trezor_base);
+        TREZOR_COMMON_TEST_CASE(gen_trezor_live_refresh, core, trezor_base);
+        TREZOR_COMMON_TEST_CASE(gen_trezor_ki_sync_with_refresh, core, trezor_base);
+      }
+
+      TREZOR_COMMON_TEST_CASE(gen_trezor_1utxo, core, trezor_base);
+      TREZOR_COMMON_TEST_CASE(gen_trezor_1utxo_paymentid_short, core, trezor_base);
+      TREZOR_COMMON_TEST_CASE(gen_trezor_1utxo_paymentid_short_integrated, core, trezor_base);
+      TREZOR_COMMON_TEST_CASE(gen_trezor_1utxo_paymentid_long, core, trezor_base);
+      TREZOR_COMMON_TEST_CASE(gen_trezor_4utxo, core, trezor_base);
+      TREZOR_COMMON_TEST_CASE(gen_trezor_4utxo_acc1, core, trezor_base);
+      TREZOR_COMMON_TEST_CASE(gen_trezor_4utxo_to_sub, core, trezor_base);
+      TREZOR_COMMON_TEST_CASE(gen_trezor_4utxo_to_2sub, core, trezor_base);
+      TREZOR_COMMON_TEST_CASE(gen_trezor_4utxo_to_1norm_2sub, core, trezor_base);
+      TREZOR_COMMON_TEST_CASE(gen_trezor_2utxo_sub_acc_to_1norm_2sub, core, trezor_base);
+      TREZOR_COMMON_TEST_CASE(gen_trezor_4utxo_to_7outs, core, trezor_base);
+      TREZOR_COMMON_TEST_CASE(wallet_api_tests, core, trezor_base);
+    }
 
     if (trezor_base.heavy_tests())
     {
       TREZOR_COMMON_TEST_CASE(gen_trezor_many_utxo, core, trezor_base);
     }
 
+    daemon->stop();
     core->deinit();
     el::Level level = (failed_tests.empty() ? el::Level::Info : el::Level::Error);
     MLOG(level, "\nREPORT:");
@@ -243,7 +285,36 @@ static bool serialize_chain_to_file(std::vector<test_event_entry>& events, gen_t
   CATCH_ENTRY_L0("serialize_chain_to_file", false);
 }
 
-static void setup_chain(cryptonote::core ** core, gen_trezor_base & trezor_base, std::string chain_path, bool fix_chain)
+template<class t_test_class>
+static bool init_core_replay_events(std::vector<test_event_entry>& events, cryptonote::core * core, const po::variables_map & vm_core)
+{
+  // this test needs for it to be so.
+  get_test_options<t_test_class> gto;
+
+  // Hardforks can be specified in events.
+  v_hardforks_t hardforks;
+  cryptonote::test_options test_options_tmp{};
+  const cryptonote::test_options * test_options_ = &gto.test_options;
+  if (extract_hard_forks(events, hardforks)){
+    hardforks.push_back(std::make_pair((uint8_t)0, (uint64_t)0));  // terminator
+    test_options_tmp.hard_forks = hardforks.data();
+    test_options_ = &test_options_tmp;
+  }
+
+  core->deinit();
+  CHECK_AND_ASSERT_THROW_MES(core->init(vm_core, test_options_), "Core init failed");
+  core->get_blockchain_storage().get_db().set_batch_transactions(true);
+
+  // start with a clean pool
+  std::vector<crypto::hash> pool_txs;
+  CHECK_AND_ASSERT_THROW_MES(core->get_pool_transaction_hashes(pool_txs), "Failed to flush txpool");
+  core->get_blockchain_storage().flush_txes_from_pool(pool_txs);
+
+  t_test_class validator;
+  return replay_events_through_core<t_test_class>(*core, events, validator);
+}
+
+static void setup_chain(cryptonote::core * core, gen_trezor_base & trezor_base, std::string chain_path, bool fix_chain, const po::variables_map & vm_core)
 {
   std::vector<test_event_entry> events;
   const bool do_serialize = !chain_path.empty();
@@ -272,6 +343,7 @@ static void setup_chain(cryptonote::core ** core, gen_trezor_base & trezor_base,
   {
     try
     {
+      trezor_base.clear();
       generated = trezor_base.generate(events);
 
       if (generated && !loaded && do_serialize)
@@ -287,7 +359,7 @@ static void setup_chain(cryptonote::core ** core, gen_trezor_base & trezor_base,
   }
 
   trezor_base.fix_hf(events);
-  if (generated && do_replay_events_get_core<gen_trezor_base>(events, core))
+  if (generated && init_core_replay_events<gen_trezor_base>(events, core, vm_core))
   {
     MGINFO_GREEN("#TEST-chain-init# Succeeded ");
   }
@@ -298,6 +370,14 @@ static void setup_chain(cryptonote::core ** core, gen_trezor_base & trezor_base,
   }
 }
 
+static device_trezor_test *ensure_trezor_test_device(){
+  if (!trezor_device) {
+    trezor_device = new device_trezor_test();
+    trezor_device->set_name(HW_TREZOR_NAME);
+  }
+  return trezor_device;
+}
+
 static void add_hforks(std::vector<test_event_entry>& events, const v_hardforks_t& hard_forks)
 {
   event_replay_settings repl_set;
@@ -483,8 +563,20 @@ static std::vector<tools::wallet2*> vct_wallets(tools::wallet2* w1=nullptr, tool
   return res;
 }
 
+static uint64_t get_available_funds(tools::wallet2* wallet, uint32_t account=0)
+{
+  tools::wallet2::transfer_container transfers;
+  wallet->get_transfers(transfers);
+  uint64_t sum = 0;
+  for(const auto & cur : transfers)
+  {
+    sum += !cur.m_spent && cur.m_subaddr_index.major == account ? cur.amount() : 0;
+  }
+  return sum;
+}
+
 // gen_trezor_base
-const uint64_t gen_trezor_base::m_ts_start = 1338224400;
+const uint64_t gen_trezor_base::m_ts_start = 1397862000;  // As default wallet timestamp is 1397516400
 const uint64_t gen_trezor_base::m_wallet_ts = m_ts_start - 60*60*24*4;
 const std::string gen_trezor_base::m_device_name = "Trezor:udp";
 const std::string gen_trezor_base::m_master_seed_str = "14821d0bc5659b24cafbc889dc4fc60785ee08b65d71c525f81eeaba4f3a570f";
@@ -494,12 +586,16 @@ const std::string gen_trezor_base::m_alice_view_private = "a6ccd4ac344a295d1387f
 
 gen_trezor_base::gen_trezor_base(){
   m_rct_config = {rct::RangeProofPaddedBulletproof, 1};
+  m_test_get_tx_key = true;
+  m_network_type = cryptonote::TESTNET;
 }
 
 gen_trezor_base::gen_trezor_base(const gen_trezor_base &other):
     m_generator(other.m_generator), m_bt(other.m_bt), m_miner_account(other.m_miner_account),
     m_bob_account(other.m_bob_account), m_alice_account(other.m_alice_account), m_eve_account(other.m_eve_account),
-    m_hard_forks(other.m_hard_forks), m_trezor(other.m_trezor), m_rct_config(other.m_rct_config)
+    m_hard_forks(other.m_hard_forks), m_trezor(other.m_trezor), m_rct_config(other.m_rct_config),
+    m_heavy_tests(other.m_heavy_tests), m_test_get_tx_key(other.m_test_get_tx_key), m_live_refresh_enabled(other.m_live_refresh_enabled),
+    m_network_type(other.m_network_type), m_daemon(other.m_daemon)
 {
 
 }
@@ -513,33 +609,27 @@ void gen_trezor_base::setup_args(const std::string & trezor_path, bool heavy_tes
 void gen_trezor_base::setup_trezor()
 {
   hw::device &hwdev = hw::get_device(m_trezor_path);
-  m_trezor = dynamic_cast<hw::trezor::device_trezor *>(&hwdev);
-  CHECK_AND_ASSERT_THROW_MES(m_trezor, "Dynamic cast failed");
-
-  m_trezor->set_debug(true);  // debugging commands on Trezor (auto-confirm transactions)
-
-  CHECK_AND_ASSERT_THROW_MES(m_trezor->set_name(m_trezor_path), "Could not set device name " << m_trezor_path);
-  m_trezor->set_network_type(MAINNET);
-  m_trezor->set_derivation_path("");  // empty derivation path
+  auto trezor = dynamic_cast<device_trezor_test *>(&hwdev);
+  CHECK_AND_ASSERT_THROW_MES(trezor, "Dynamic cast failed");
 
-  CHECK_AND_ASSERT_THROW_MES(m_trezor->init(), "Could not initialize the device " << m_trezor_path);
-  CHECK_AND_ASSERT_THROW_MES(m_trezor->connect(), "Could not connect to the device " << m_trezor_path);
-  m_trezor->wipe_device();
-  m_trezor->load_device(m_device_seed);
-  m_trezor->release();
-  m_trezor->disconnect();
+  trezor->setup_for_tests(m_trezor_path, m_device_seed, m_network_type);
+  m_trezor = trezor;
 }
 
 void gen_trezor_base::fork(gen_trezor_base & other)
 {
   other.m_generator = m_generator;
   other.m_bt = m_bt;
+  other.m_network_type = m_network_type;
+  other.m_daemon = m_daemon;
   other.m_events = m_events;
   other.m_head = m_head;
   other.m_hard_forks = m_hard_forks;
   other.m_trezor_path = m_trezor_path;
   other.m_heavy_tests = m_heavy_tests;
   other.m_rct_config = m_rct_config;
+  other.m_test_get_tx_key = m_test_get_tx_key;
+  other.m_live_refresh_enabled = m_live_refresh_enabled;
 
   other.m_miner_account = m_miner_account;
   other.m_bob_account = m_bob_account;
@@ -577,10 +667,22 @@ void gen_trezor_base::init_fields()
   m_alice_account.set_createtime(m_wallet_ts);
 }
 
+void gen_trezor_base::update_client_settings()
+{
+  auto dev_trezor = dynamic_cast<::hw::trezor::device_trezor*>(m_trezor);
+  CHECK_AND_ASSERT_THROW_MES(dev_trezor, "Could not cast to device_trezor");
+
+  dev_trezor->set_live_refresh_enabled(m_live_refresh_enabled);
+}
+
 bool gen_trezor_base::generate(std::vector<test_event_entry>& events)
 {
   init_fields();
   setup_trezor();
+
+  m_live_refresh_enabled = false;
+  update_client_settings();
+
   m_alice_account.create_from_device(*m_trezor);
   m_alice_account.set_createtime(m_wallet_ts);
 
@@ -589,7 +691,7 @@ bool gen_trezor_base::generate(std::vector<test_event_entry>& events)
 
   cryptonote::block blk_gen;
   std::vector<size_t> block_weights;
-  generate_genesis_block(blk_gen, get_config(MAINNET).GENESIS_TX, get_config(MAINNET).GENESIS_NONCE);
+  generate_genesis_block(blk_gen, get_config(m_network_type).GENESIS_TX, get_config(m_network_type).GENESIS_NONCE);
   events.push_back(blk_gen);
   generator.add_block(blk_gen, 0, block_weights, 0);
 
@@ -644,8 +746,8 @@ bool gen_trezor_base::generate(std::vector<test_event_entry>& events)
   MDEBUG("Hardfork height: " << hardfork_height << " at block: " << get_block_hash(blk_4r));
 
   // RCT transactions, wallets have to be used, wallet init
-  m_wl_alice.reset(new tools::wallet2(MAINNET, 1, true));
-  m_wl_bob.reset(new tools::wallet2(MAINNET, 1, true));
+  m_wl_alice.reset(new tools::wallet2(m_network_type, 1, true));
+  m_wl_bob.reset(new tools::wallet2(m_network_type, 1, true));
   wallet_accessor_test::set_account(m_wl_alice.get(), m_alice_account);
   wallet_accessor_test::set_account(m_wl_bob.get(), m_bob_account);
 
@@ -659,7 +761,7 @@ bool gen_trezor_base::generate(std::vector<test_event_entry>& events)
   auto addr_alice_sub_1_2 = m_wl_alice->get_subaddress({1, 2});
 
   // Miner -> Bob, RCT funds
-  MAKE_TX_LIST_START_RCT(events, txs_blk_5, m_miner_account, m_alice_account, MK_COINS(5), 10, blk_4);
+  MAKE_TX_LIST_START_RCT(events, txs_blk_5, m_miner_account, m_alice_account, MK_COINS(50), 10, blk_4);
 
   const size_t target_rct = m_heavy_tests ? 105 : 15;
   for(size_t i = 0; i < target_rct; ++i)
@@ -688,9 +790,9 @@ bool gen_trezor_base::generate(std::vector<test_event_entry>& events)
 
   // Simple RCT transactions
   MAKE_TX_MIX_LIST_RCT(events, txs_blk_5, m_miner_account, m_alice_account, MK_COINS(7), 10, blk_4);
-  MAKE_TX_MIX_LIST_RCT(events, txs_blk_5, m_miner_account, m_alice_account, MK_COINS(1), 10, blk_4);
-  MAKE_TX_MIX_LIST_RCT(events, txs_blk_5, m_miner_account, m_alice_account, MK_COINS(3), 10, blk_4);
-  MAKE_TX_MIX_LIST_RCT(events, txs_blk_5, m_miner_account, m_alice_account, MK_COINS(4), 10, blk_4);
+  MAKE_TX_MIX_LIST_RCT(events, txs_blk_5, m_miner_account, m_alice_account, MK_COINS(10), 10, blk_4);
+  MAKE_TX_MIX_LIST_RCT(events, txs_blk_5, m_miner_account, m_alice_account, MK_COINS(30), 10, blk_4);
+  MAKE_TX_MIX_LIST_RCT(events, txs_blk_5, m_miner_account, m_alice_account, MK_COINS(40), 10, blk_4);
   MAKE_NEXT_BLOCK_TX_LIST_HF(events, blk_5, blk_4r, m_miner_account, txs_blk_5, CUR_HF);
 
   // Simple transaction check
@@ -704,6 +806,8 @@ bool gen_trezor_base::generate(std::vector<test_event_entry>& events)
   // RCT transactions, wallets have to be used
   wallet_tools::process_transactions(m_wl_alice.get(), events, blk_5r, m_bt);
   wallet_tools::process_transactions(m_wl_bob.get(), events, blk_5r, m_bt);
+  MDEBUG("Available funds on Alice: " << get_available_funds(m_wl_alice.get()));
+  MDEBUG("Available funds on Bob: " << get_available_funds(m_wl_bob.get()));
 
   // Send Alice -> Bob, manually constructed. Simple TX test, precondition.
   cryptonote::transaction tx_1;
@@ -768,18 +872,21 @@ void gen_trezor_base::load(std::vector<test_event_entry>& events)
   m_eve_account.set_createtime(m_wallet_ts);
 
   setup_trezor();
+  update_client_settings();
   m_alice_account.create_from_device(*m_trezor);
   m_alice_account.set_createtime(m_wallet_ts);
 
-  m_wl_alice.reset(new tools::wallet2(MAINNET, 1, true));
-  m_wl_bob.reset(new tools::wallet2(MAINNET, 1, true));
-  m_wl_eve.reset(new tools::wallet2(MAINNET, 1, true));
+  m_wl_alice.reset(new tools::wallet2(m_network_type, 1, true));
+  m_wl_bob.reset(new tools::wallet2(m_network_type, 1, true));
+  m_wl_eve.reset(new tools::wallet2(m_network_type, 1, true));
   wallet_accessor_test::set_account(m_wl_alice.get(), m_alice_account);
   wallet_accessor_test::set_account(m_wl_bob.get(), m_bob_account);
   wallet_accessor_test::set_account(m_wl_eve.get(), m_eve_account);
 
   wallet_tools::process_transactions(m_wl_alice.get(), events, m_head, m_bt);
   wallet_tools::process_transactions(m_wl_bob.get(), events, m_head, m_bt);
+  MDEBUG("Available funds on Alice: " << get_available_funds(m_wl_alice.get()));
+  MDEBUG("Available funds on Bob: " << get_available_funds(m_wl_bob.get()));
 }
 
 void gen_trezor_base::fix_hf(std::vector<test_event_entry>& events)
@@ -804,12 +911,14 @@ void gen_trezor_base::test_setup(std::vector<test_event_entry>& events)
   add_shared_events(events);
 
   setup_trezor();
+  update_client_settings();
+
   m_alice_account.create_from_device(*m_trezor);
   m_alice_account.set_createtime(m_wallet_ts);
 
-  m_wl_alice.reset(new tools::wallet2(MAINNET, 1, true));
-  m_wl_bob.reset(new tools::wallet2(MAINNET, 1, true));
-  m_wl_eve.reset(new tools::wallet2(MAINNET, 1, true));
+  m_wl_alice.reset(new tools::wallet2(m_network_type, 1, true));
+  m_wl_bob.reset(new tools::wallet2(m_network_type, 1, true));
+  m_wl_eve.reset(new tools::wallet2(m_network_type, 1, true));
   wallet_accessor_test::set_account(m_wl_alice.get(), m_alice_account);
   wallet_accessor_test::set_account(m_wl_bob.get(), m_bob_account);
   wallet_accessor_test::set_account(m_wl_eve.get(), m_eve_account);
@@ -818,13 +927,11 @@ void gen_trezor_base::test_setup(std::vector<test_event_entry>& events)
   wallet_tools::process_transactions(m_wl_eve.get(), events, m_head, m_bt);
 }
 
-void gen_trezor_base::test_trezor_tx(std::vector<test_event_entry>& events, std::vector<tools::wallet2::pending_tx>& ptxs, std::vector<cryptonote::address_parse_info>& dsts_info, test_generator &generator, std::vector<tools::wallet2*> wallets, bool is_sweep)
+void gen_trezor_base::add_transactions_to_events(
+    std::vector<test_event_entry>& events,
+    test_generator &generator,
+    const std::vector<cryptonote::transaction> &txs)
 {
-  // Construct pending transaction for signature in the Trezor.
-  const uint64_t height_pre = num_blocks(events) - 1;
-  cryptonote::block head_block = get_head_block(events);
-  const crypto::hash head_hash = get_block_hash(head_block);
-
   // If current test requires higher hard-fork, move it up
   const auto current_hf = m_hard_forks.back().first;
   const uint8_t tx_hf = m_rct_config.bp_version == 2 ? 10 : 9;
@@ -832,8 +939,28 @@ void gen_trezor_base::test_trezor_tx(std::vector<test_event_entry>& events, std:
     throw std::runtime_error("Too late for HF change");
   }
 
-  tools::wallet2::unsigned_tx_set txs;
   std::list<cryptonote::transaction> tx_list;
+  for(const auto & tx : txs)
+  {
+    events.push_back(tx);
+    tx_list.push_back(tx);
+  }
+
+  MAKE_NEXT_BLOCK_TX_LIST_HF(events, blk_new, m_head, m_miner_account, tx_list, tx_hf);
+  MDEBUG("New tsx: " << (num_blocks(events) - 1) << " at block: " << get_block_hash(blk_new));
+
+  m_head = blk_new;
+}
+
+void gen_trezor_base::test_trezor_tx(std::vector<test_event_entry>& events, std::vector<tools::wallet2::pending_tx>& ptxs, std::vector<cryptonote::address_parse_info>& dsts_info, test_generator &generator, std::vector<tools::wallet2*> wallets, bool is_sweep)
+{
+  // Construct pending transaction for signature in the Trezor.
+  const uint64_t height_pre = num_blocks(events) - 1;
+  cryptonote::block head_block = get_head_block(events);
+  const crypto::hash head_hash = get_block_hash(head_block);
+
+  tools::wallet2::unsigned_tx_set txs;
+  std::vector<cryptonote::transaction> tx_list;
 
   for(auto &ptx : ptxs) {
     txs.txes.push_back(get_construction_data_with_decrypted_short_payment_id(ptx, *m_trezor));
@@ -848,6 +975,7 @@ void gen_trezor_base::test_trezor_tx(std::vector<test_event_entry>& events, std:
   hw::wallet_shim wallet_shim;
   setup_shim(&wallet_shim);
   aux_data.tx_recipients = dsts_info;
+  aux_data.bp_version = m_rct_config.bp_version;
   dev_cold->tx_sign(&wallet_shim, txs, exported_txs, aux_data);
 
   MDEBUG("Signed tx data from hw: " << exported_txs.ptx.size() << " transactions");
@@ -865,13 +993,11 @@ void gen_trezor_base::test_trezor_tx(std::vector<test_event_entry>& events, std:
     CHECK_AND_ASSERT_THROW_MES(resx, "Trezor tx_1 semantics failed");
     CHECK_AND_ASSERT_THROW_MES(resy, "Trezor tx_1 Nonsemantics failed");
 
-    events.push_back(c_ptx.tx);
     tx_list.push_back(c_ptx.tx);
     MDEBUG("Transaction: " << dump_data(c_ptx.tx));
   }
 
-  MAKE_NEXT_BLOCK_TX_LIST_HF(events, blk_7, m_head, m_miner_account, tx_list, tx_hf);
-  MDEBUG("Trezor tsx: " << (num_blocks(events) - 1) << " at block: " << get_block_hash(blk_7));
+  add_transactions_to_events(events, generator, tx_list);
 
   // TX receive test
   uint64_t sum_in = 0;
@@ -909,7 +1035,7 @@ void gen_trezor_base::test_trezor_tx(std::vector<test_event_entry>& events, std:
       const bool sender = widx == 0;
       tools::wallet2 *wl = wallets[widx];
 
-      wallet_tools::process_transactions(wl, events, blk_7, m_bt, boost::make_optional(head_hash));
+      wallet_tools::process_transactions(wl, events, m_head, m_bt, boost::make_optional(head_hash));
 
       tools::wallet2::transfer_container m_trans;
       tools::wallet2::transfer_container m_trans_txid;
@@ -972,6 +1098,120 @@ void gen_trezor_base::test_trezor_tx(std::vector<test_event_entry>& events, std:
   }
 
   CHECK_AND_ASSERT_THROW_MES(sum_in == sum_out, "Tx amount mismatch");
+
+  // Test get_tx_key feature for stored private tx keys
+  test_get_tx(events, wallets, exported_txs.ptx, aux_data.tx_device_aux);
+}
+
+bool gen_trezor_base::verify_tx_key(const ::crypto::secret_key & tx_priv, const ::crypto::public_key & tx_pub, const subaddresses_t & subs)
+{
+  ::crypto::public_key tx_pub_c;
+  ::crypto::secret_key_to_public_key(tx_priv, tx_pub_c);
+  if (tx_pub == tx_pub_c)
+    return true;
+
+  for(const auto & elem : subs)
+  {
+    tx_pub_c = rct::rct2pk(rct::scalarmultKey(rct::pk2rct(elem.first), rct::sk2rct(tx_priv)));
+    if (tx_pub == tx_pub_c)
+      return true;
+  }
+  return false;
+}
+
+void gen_trezor_base::test_get_tx(
+    std::vector<test_event_entry>& events,
+    std::vector<tools::wallet2*> wallets,
+    const std::vector<tools::wallet2::pending_tx> &ptxs,
+    const std::vector<std::string> &aux_tx_info)
+{
+  if (!m_test_get_tx_key)
+  {
+    return;
+  }
+
+  auto dev_cold = dynamic_cast<::hw::device_cold*>(m_trezor);
+  CHECK_AND_ASSERT_THROW_MES(dev_cold, "Device does not implement cold signing interface");
+
+  if (!dev_cold->is_get_tx_key_supported())
+  {
+    MERROR("Get TX key is not supported by the connected Trezor");
+    return;
+  }
+
+  subaddresses_t all_subs;
+  for(tools::wallet2 * wlt : wallets)
+  {
+    wlt->expand_subaddresses({10, 20});
+
+    const subaddresses_t & cur_sub = wallet_accessor_test::get_subaddresses(wlt);
+    all_subs.insert(cur_sub.begin(), cur_sub.end());
+  }
+
+  for(size_t txid = 0; txid < ptxs.size(); ++txid)
+  {
+    const auto &c_ptx = ptxs[txid];
+    const auto &c_tx = c_ptx.tx;
+    const ::crypto::hash tx_prefix_hash = cryptonote::get_transaction_prefix_hash(c_tx);
+
+    auto tx_pub = cryptonote::get_tx_pub_key_from_extra(c_tx.extra);
+    auto additional_pub_keys = cryptonote::get_additional_tx_pub_keys_from_extra(c_tx.extra);
+
+    hw::device_cold:: tx_key_data_t tx_key_data;
+    std::vector<::crypto::secret_key> tx_keys;
+
+    dev_cold->load_tx_key_data(tx_key_data, aux_tx_info[txid]);
+    CHECK_AND_ASSERT_THROW_MES(std::string(tx_prefix_hash.data, 32) == tx_key_data.tx_prefix_hash, "TX prefix mismatch");
+
+    dev_cold->get_tx_key(tx_keys, tx_key_data, m_alice_account.get_keys().m_view_secret_key);
+    CHECK_AND_ASSERT_THROW_MES(!tx_keys.empty(), "Empty TX keys");
+    CHECK_AND_ASSERT_THROW_MES(verify_tx_key(tx_keys[0], tx_pub, all_subs), "Tx pub mismatch");
+    CHECK_AND_ASSERT_THROW_MES(additional_pub_keys.size() == tx_keys.size() - 1, "Invalid additional keys count");
+
+    for(size_t i = 0; i < additional_pub_keys.size(); ++i)
+    {
+      CHECK_AND_ASSERT_THROW_MES(verify_tx_key(tx_keys[i + 1], additional_pub_keys[i], all_subs), "Tx pub mismatch");
+    }
+  }
+}
+
+void gen_trezor_base::mine_and_test(std::vector<test_event_entry>& events)
+{
+  cryptonote::core * core = daemon()->core();
+  const uint64_t height_before_mining = daemon()->get_height();
+
+  const auto miner_address = cryptonote::get_account_address_as_str(FAKECHAIN, false, get_address(m_miner_account));
+  daemon()->mine_blocks(1, miner_address);
+
+  const uint64_t cur_height = daemon()->get_height();
+  CHECK_AND_ASSERT_THROW_MES(height_before_mining < cur_height, "Mining fail");
+
+  const crypto::hash top_hash = core->get_blockchain_storage().get_block_id_by_height(height_before_mining);
+  cryptonote::block top_block{};
+  CHECK_AND_ASSERT_THROW_MES(core->get_blockchain_storage().get_block_by_hash(top_hash, top_block), "Block fetch fail");
+  CHECK_AND_ASSERT_THROW_MES(!top_block.tx_hashes.empty(), "Mined block is empty");
+
+  std::vector<cryptonote::transaction> txs_found;
+  std::vector<crypto::hash> txs_missed;
+  bool r = core->get_blockchain_storage().get_transactions(top_block.tx_hashes, txs_found, txs_missed);
+  CHECK_AND_ASSERT_THROW_MES(r, "Transaction lookup fail");
+  CHECK_AND_ASSERT_THROW_MES(!txs_found.empty(), "Transaction lookup fail");
+
+  // Transaction is not expanded, but mining verified it.
+  events.push_back(txs_found[0]);
+  events.push_back(top_block);
+}
+
+void gen_trezor_base::set_hard_fork(uint8_t hf)
+{
+  m_top_hard_fork = hf;
+  if (hf < 9){
+    throw std::runtime_error("Minimal supported Hardfork is 9");
+  } else if (hf == 9){
+    rct_config({rct::RangeProofPaddedBulletproof, 1});
+  } else {
+    rct_config({rct::RangeProofPaddedBulletproof, 2});
+  }
 }
 
 #define TREZOR_TEST_PREFIX()                              \
@@ -1182,6 +1422,48 @@ std::vector<tools::wallet2::pending_tx> tsx_builder::build()
   return m_ptxs;
 }
 
+device_trezor_test::device_trezor_test(): m_tx_sign_ctr(0), m_compute_key_image_ctr(0) {}
+
+void device_trezor_test::clear_test_counters(){
+  m_tx_sign_ctr = 0;
+  m_compute_key_image_ctr = 0;
+}
+
+void device_trezor_test::setup_for_tests(const std::string & trezor_path, const std::string & seed, cryptonote::network_type network_type){
+  this->clear_test_counters();
+  this->set_callback(nullptr);
+  this->set_debug(true);  // debugging commands on Trezor (auto-confirm transactions)
+
+  CHECK_AND_ASSERT_THROW_MES(this->set_name(trezor_path), "Could not set device name " << trezor_path);
+  this->set_network_type(network_type);
+  this->set_derivation_path("");  // empty derivation path
+
+  CHECK_AND_ASSERT_THROW_MES(this->init(), "Could not initialize the device " << trezor_path);
+  CHECK_AND_ASSERT_THROW_MES(this->connect(), "Could not connect to the device " << trezor_path);
+  this->wipe_device();
+  this->load_device(seed);
+  this->release();
+  this->disconnect();
+}
+
+bool device_trezor_test::compute_key_image(const ::cryptonote::account_keys &ack, const ::crypto::public_key &out_key,
+                                           const ::crypto::key_derivation &recv_derivation, size_t real_output_index,
+                                           const ::cryptonote::subaddress_index &received_index,
+                                           ::cryptonote::keypair &in_ephemeral, ::crypto::key_image &ki) {
+
+  bool res = device_trezor::compute_key_image(ack, out_key, recv_derivation, real_output_index, received_index,
+                                          in_ephemeral, ki);
+  m_compute_key_image_ctr += res;
+  return res;
+}
+
+void
+device_trezor_test::tx_sign(hw::wallet_shim *wallet, const ::tools::wallet2::unsigned_tx_set &unsigned_tx, size_t idx,
+                            hw::tx_aux_data &aux_data, std::shared_ptr<hw::trezor::protocol::tx::Signer> &signer) {
+  m_tx_sign_ctr += 1;
+  device_trezor::tx_sign(wallet, unsigned_tx, idx, aux_data, signer);
+}
+
 bool gen_trezor_ki_sync::generate(std::vector<test_event_entry>& events)
 {
   test_generator generator(m_generator);
@@ -1207,6 +1489,92 @@ bool gen_trezor_ki_sync::generate(std::vector<test_event_entry>& events)
 
   uint64_t spent = 0, unspent = 0;
   m_wl_alice->import_key_images(ski, 0, spent, unspent, false);
+
+  auto dev_trezor_test = dynamic_cast<device_trezor_test*>(m_trezor);
+  CHECK_AND_ASSERT_THROW_MES(dev_cold, "Device does not implement test interface");
+  if (!m_live_refresh_enabled)
+    CHECK_AND_ASSERT_THROW_MES(dev_trezor_test->m_compute_key_image_ctr == 0, "Live refresh should not happen: " << dev_trezor_test->m_compute_key_image_ctr);
+  else
+    CHECK_AND_ASSERT_THROW_MES(dev_trezor_test->m_compute_key_image_ctr == ski.size(), "Live refresh counts invalid");
+
+  return true;
+}
+
+bool gen_trezor_ki_sync_with_refresh::generate(std::vector<test_event_entry>& events)
+{
+  m_live_refresh_enabled = true;
+  return gen_trezor_ki_sync::generate(events);
+}
+
+bool gen_trezor_ki_sync_without_refresh::generate(std::vector<test_event_entry>& events)
+{
+  m_live_refresh_enabled = false;
+  return gen_trezor_ki_sync::generate(events);
+}
+
+bool gen_trezor_live_refresh::generate(std::vector<test_event_entry>& events)
+{
+  test_generator generator(m_generator);
+  test_setup(events);
+
+  auto dev_cold = dynamic_cast<::hw::device_cold*>(m_trezor);
+  CHECK_AND_ASSERT_THROW_MES(dev_cold, "Device does not implement cold signing interface");
+
+  if (!dev_cold->is_live_refresh_supported()){
+    MDEBUG("Trezor does not support live refresh");
+    return true;
+  }
+
+  hw::device & sw_device = hw::get_device("default");
+
+  dev_cold->live_refresh_start();
+  for(unsigned i=0; i<50; ++i)
+  {
+    cryptonote::subaddress_index subaddr = {0, i};
+
+    ::crypto::secret_key r;
+    ::crypto::public_key R;
+    ::crypto::key_derivation D;
+    ::crypto::public_key pub_ver;
+    ::crypto::key_image ki;
+
+    ::crypto::random32_unbiased((unsigned char*)r.data);
+    ::crypto::secret_key_to_public_key(r, R);
+    memcpy(D.data, rct::scalarmultKey(rct::pk2rct(R), rct::sk2rct(m_alice_account.get_keys().m_view_secret_key)).bytes, 32);
+
+    ::crypto::secret_key scalar_step1;
+    ::crypto::secret_key scalar_step2;
+    ::crypto::derive_secret_key(D, i, m_alice_account.get_keys().m_spend_secret_key, scalar_step1);
+    if (i == 0)
+    {
+      scalar_step2 = scalar_step1;
+    }
+    else
+    {
+      ::crypto::secret_key subaddr_sk = sw_device.get_subaddress_secret_key(m_alice_account.get_keys().m_view_secret_key, subaddr);
+      sw_device.sc_secret_add(scalar_step2, scalar_step1, subaddr_sk);
+    }
+
+    ::crypto::secret_key_to_public_key(scalar_step2, pub_ver);
+    ::crypto::generate_key_image(pub_ver, scalar_step2, ki);
+
+    cryptonote::keypair in_ephemeral;
+    ::crypto::key_image ki2;
+
+    dev_cold->live_refresh(
+        m_alice_account.get_keys().m_view_secret_key,
+        pub_ver,
+        D,
+        i,
+        subaddr,
+        in_ephemeral,
+        ki2
+    );
+
+    CHECK_AND_ASSERT_THROW_MES(ki == ki2, "Key image inconsistent");
+  }
+
+  dev_cold->live_refresh_finish();
   return true;
 }
 
@@ -1407,3 +1775,62 @@ bool gen_trezor_many_utxo::generate(std::vector<test_event_entry>& events)
   TREZOR_TEST_SUFFIX();
 }
 
+void wallet_api_tests::init()
+{
+  m_wallet_dir = boost::filesystem::unique_path();
+  boost::filesystem::create_directories(m_wallet_dir);
+}
+
+wallet_api_tests::~wallet_api_tests()
+{
+  try
+  {
+    if (!m_wallet_dir.empty() && boost::filesystem::exists(m_wallet_dir))
+    {
+      boost::filesystem::remove_all(m_wallet_dir);
+    }
+  }
+  catch(...)
+  {
+    MERROR("Could not remove wallet directory");
+  }
+}
+
+bool wallet_api_tests::generate(std::vector<test_event_entry>& events)
+{
+  init();
+  test_setup(events);
+  const std::string wallet_path = (m_wallet_dir / "wallet").string();
+  const auto api_net_type = m_network_type == TESTNET ? Monero::TESTNET : Monero::MAINNET;
+
+  Monero::WalletManager *wmgr = Monero::WalletManagerFactory::getWalletManager();
+  std::unique_ptr<Monero::Wallet> w{wmgr->createWalletFromDevice(wallet_path, "", api_net_type, m_trezor_path, 1)};
+  CHECK_AND_ASSERT_THROW_MES(w->init(daemon()->rpc_addr(), 0), "Wallet init fail");
+  CHECK_AND_ASSERT_THROW_MES(w->refresh(), "Refresh fail");
+  uint64_t balance = w->balance(0);
+  MINFO("Balance: " << balance);
+  CHECK_AND_ASSERT_THROW_MES(w->status() == Monero::PendingTransaction::Status_Ok, "Status nok");
+
+  auto addr = get_address(m_eve_account);
+  auto recepient_address = cryptonote::get_account_address_as_str(m_network_type, false, addr);
+  Monero::PendingTransaction * transaction = w->createTransaction(recepient_address,
+                                                                  "",
+                                                                  MK_COINS(10),
+                                                                  TREZOR_TEST_MIXIN,
+                                                                  Monero::PendingTransaction::Priority_Medium,
+                                                                  0,
+                                                                  std::set<uint32_t>{});
+  CHECK_AND_ASSERT_THROW_MES(transaction->status() == Monero::PendingTransaction::Status_Ok, "Status nok");
+  w->refresh();
+
+  CHECK_AND_ASSERT_THROW_MES(w->balance(0) == balance, "Err");
+  CHECK_AND_ASSERT_THROW_MES(transaction->amount() == MK_COINS(10), "Err");
+  CHECK_AND_ASSERT_THROW_MES(transaction->commit(), "Err");
+  CHECK_AND_ASSERT_THROW_MES(w->balance(0) != balance, "Err");
+  CHECK_AND_ASSERT_THROW_MES(wmgr->closeWallet(w.get()), "Err");
+  (void)w.release();
+
+  mine_and_test(events);
+  return true;
+}
+

tests/trezor/trezor_tests.h

@@ -31,6 +31,8 @@
 #pragma once
 
 #include <device_trezor/device_trezor.hpp>
+#include <wallet/api/wallet2_api.h>
+#include "daemon.h"
 #include "../core_tests/chaingen.h"
 #include "../core_tests/wallet_tools.h"
 
@@ -50,29 +52,48 @@ public:
   gen_trezor_base(const gen_trezor_base &other);
   virtual ~gen_trezor_base() {};
 
-  void setup_args(const std::string & trezor_path, bool heavy_tests=false);
+  virtual void setup_args(const std::string & trezor_path, bool heavy_tests=false);
   virtual bool generate(std::vector<test_event_entry>& events);
   virtual void load(std::vector<test_event_entry>& events);       // load events, init test obj
-  void fix_hf(std::vector<test_event_entry>& events);
-  void update_trackers(std::vector<test_event_entry>& events);
+  virtual void fix_hf(std::vector<test_event_entry>& events);
+  virtual void update_trackers(std::vector<test_event_entry>& events);
 
-  void fork(gen_trezor_base & other);                             // fork generated chain to another test
-  void clear();                                                   // clears m_events, bt, generator, hforks
-  void add_shared_events(std::vector<test_event_entry>& events);  // m_events -> events
-  void test_setup(std::vector<test_event_entry>& events);         // init setup env, wallets
+  virtual void fork(gen_trezor_base & other);                             // fork generated chain to another test
+  virtual void clear();                                                   // clears m_events, bt, generator, hforks
+  virtual void add_shared_events(std::vector<test_event_entry>& events);  // m_events -> events
+  virtual void test_setup(std::vector<test_event_entry>& events);         // init setup env, wallets
 
-  void test_trezor_tx(std::vector<test_event_entry>& events,
+  virtual void add_transactions_to_events(
+      std::vector<test_event_entry>& events,
+      test_generator &generator,
+      const std::vector<cryptonote::transaction> &txs);
+
+  virtual void test_trezor_tx(
+      std::vector<test_event_entry>& events,
       std::vector<tools::wallet2::pending_tx>& ptxs,
       std::vector<cryptonote::address_parse_info>& dsts_info,
       test_generator &generator,
       std::vector<tools::wallet2*> wallets,
       bool is_sweep=false);
 
-  crypto::hash head_hash() { return get_block_hash(m_head); }
-  cryptonote::block head_block() { return m_head; }
-  bool heavy_tests() { return m_heavy_tests; }
+  virtual void test_get_tx(
+      std::vector<test_event_entry>& events,
+      std::vector<tools::wallet2*> wallets,
+      const std::vector<tools::wallet2::pending_tx> &ptxs,
+      const std::vector<std::string> &aux_tx_info);
+
+  virtual void mine_and_test(std::vector<test_event_entry>& events);
+
+  virtual void set_hard_fork(uint8_t hf);
+
+  crypto::hash head_hash() const { return get_block_hash(m_head); }
+  cryptonote::block head_block() const { return m_head; }
+  bool heavy_tests() const { return m_heavy_tests; }
   void rct_config(rct::RCTConfig rct_config) { m_rct_config = rct_config; }
-  uint8_t cur_hf(){ return m_hard_forks.size() > 0 ? m_hard_forks.back().first : 0; }
+  uint8_t cur_hf() const { return m_hard_forks.size() > 0 ? m_hard_forks.back().first : 0; }
+  cryptonote::network_type nettype() const { return m_network_type; }
+  std::shared_ptr<mock_daemon> daemon() const { return m_daemon; }
+  void daemon(std::shared_ptr<mock_daemon> daemon){ m_daemon = std::move(daemon); }
 
   // Static configuration
   static const uint64_t m_ts_start;
@@ -84,19 +105,26 @@ public:
   static const std::string  m_alice_view_private;
 
 protected:
-  void setup_trezor();
-  void init_fields();
+  virtual void setup_trezor();
+  virtual void init_fields();
+  virtual void update_client_settings();
+  virtual bool verify_tx_key(const ::crypto::secret_key & tx_priv, const ::crypto::public_key & tx_pub, const subaddresses_t & subs);
 
   test_generator m_generator;
   block_tracker m_bt;
+  cryptonote::network_type m_network_type;
+  std::shared_ptr<mock_daemon> m_daemon;
 
+  uint8_t m_top_hard_fork;
   v_hardforks_t m_hard_forks;
   cryptonote::block m_head;
   std::vector<test_event_entry> m_events;
 
   std::string m_trezor_path;
   bool m_heavy_tests;
+  bool m_test_get_tx_key;
   rct::RCTConfig m_rct_config;
+  bool m_live_refresh_enabled;
 
   cryptonote::account_base m_miner_account;
   cryptonote::account_base m_bob_account;
@@ -113,6 +141,7 @@ protected:
   void serialize(Archive & ar, const unsigned int /*version*/)
   {
     ar & m_generator;
+    ar & m_network_type;
   }
 };
 
@@ -169,12 +198,52 @@ protected:
   rct::RCTConfig m_rct_config;
 };
 
+// Trezor device ship to track actual method calls.
+class device_trezor_test : public hw::trezor::device_trezor {
+public:
+  size_t m_tx_sign_ctr;
+  size_t m_compute_key_image_ctr;
+
+  device_trezor_test();
+
+  void clear_test_counters();
+  void setup_for_tests(const std::string & trezor_path, const std::string & seed, cryptonote::network_type network_type);
+
+  bool compute_key_image(const ::cryptonote::account_keys &ack, const ::crypto::public_key &out_key,
+                         const ::crypto::key_derivation &recv_derivation, size_t real_output_index,
+                         const ::cryptonote::subaddress_index &received_index, ::cryptonote::keypair &in_ephemeral,
+                         ::crypto::key_image &ki) override;
+
+protected:
+  void tx_sign(hw::wallet_shim *wallet, const ::tools::wallet2::unsigned_tx_set &unsigned_tx, size_t idx,
+               hw::tx_aux_data &aux_data, std::shared_ptr<hw::trezor::protocol::tx::Signer> &signer) override;
+};
+
+// Tests
 class gen_trezor_ki_sync : public gen_trezor_base
 {
 public:
   bool generate(std::vector<test_event_entry>& events) override;
 };
 
+class gen_trezor_ki_sync_with_refresh : public gen_trezor_ki_sync
+{
+public:
+  bool generate(std::vector<test_event_entry>& events) override;
+};
+
+class gen_trezor_ki_sync_without_refresh : public gen_trezor_ki_sync
+{
+public:
+  bool generate(std::vector<test_event_entry>& events) override;
+};
+
+class gen_trezor_live_refresh : public gen_trezor_base
+{
+public:
+  bool generate(std::vector<test_event_entry>& events) override;
+};
+
 class gen_trezor_1utxo : public gen_trezor_base
 {
 public:
@@ -246,3 +315,15 @@ class gen_trezor_many_utxo : public gen_trezor_base
 public:
   bool generate(std::vector<test_event_entry>& events) override;
 };
+
+// Wallet::API tests
+class wallet_api_tests : public gen_trezor_base
+{
+public:
+  virtual ~wallet_api_tests();
+  void init();
+  bool generate(std::vector<test_event_entry>& events) override;
+
+protected:
+  boost::filesystem::path m_wallet_dir;
+};

tests/unit_tests/CMakeLists.txt

@@ -52,6 +52,7 @@ set(unit_tests_sources
   json_serialization.cpp
   get_xtype_from_string.cpp
   hashchain.cpp
+  hmac_keccak.cpp
   http.cpp
   keccak.cpp
   logging.cpp

tests/unit_tests/hmac_keccak.cpp

@@ -0,0 +1,152 @@
+// Copyright (c) 2018, The Monero Project
+// 
+// All rights reserved.
+// 
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+// 
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+// 
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+// 
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+// 
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "gtest/gtest.h"
+#include "../io.h"
+
+extern "C" {
+#include "crypto/hmac-keccak.h"
+}
+
+#define KECCAK_BLOCKLEN 136
+
+static const struct {
+  const char *key;
+  const char *inp;
+  const char *res;
+} keccak_hmac_vectors[] = {
+    {"",
+     "",
+     "042186ec4e98680a0866091d6fb89b60871134b44327f8f467c14e9841d3e97b",
+    },
+    {"00",
+     "",
+     "042186ec4e98680a0866091d6fb89b60871134b44327f8f467c14e9841d3e97b",
+    },
+    {"00000000000000000000000000000000",
+     "",
+     "042186ec4e98680a0866091d6fb89b60871134b44327f8f467c14e9841d3e97b",
+    },
+    {"",
+     "00",
+     "402541d5d678ad30217023a12efbd2f367388dc0253ccddb8f915d187e03d414",
+    },
+    {"0000000000000000000000000000000000000000000000000000000000000000",
+     "0000000000000000000000000000000000000000000000000000000000000000",
+     "d164c38ca454176281b3c09dc83cf70bd00290835ff5490356993d368cde0577",
+    },
+    {"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff",
+     "0000000000000000000000000000000000000000000000000000000000000000",
+     "d52c9100301a9546d8c97d7f32e8178fd5f4b0a7646ab761db9e0f503f8b0542",
+    },
+    {"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff",
+     "d52c9100301a9546d8c97d7f32e8178fd5f4b0a7646ab761db9e0f503f8b0542",
+     "5b9d98c5d3249176230d70fdb215ee2f93e4783064c9564384ddc396e63c18cb",
+    },
+    {"00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",
+     "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",
+     "98d033ba0c95bb3aaf5709733443cbedba20f6451b710ccba4118fc5523013c8",
+    },
+    {"65768798a9bacbdcedfe0f2031425364758697a8b9cadbecfd0e1f30415263748596a7b8c9daebfc0d1e2f405162738495a6b7c8d9eafb0c1d2e3f5061728394a5b6c7d8e9fa0b1c2d3e4f60718293a4b5c6d7e8f90a1b2c3d4e5f708192a3b4c5d6e7f8091a2b3c4d5e6f8091a2b3c4d5e6f708192a3b4c5d6e7f90a1b2c3d4e5f60718293a4b5c6d7e8fa0b1c2d3e4f5061728394a5b6c7d8e9fb0c1d2e3f405162738495a6b7c8d9eafc0d1e2f30415263748596a7b8c9daebfd0e1f2031425364758697a8b9cadbecfe0f102132435465768798a9bacbdcedff00112233445566778899aabbccddeef00112233445566778899aabbccddeeff102132435465768798a9bacbdcedfe0f2031425364758697a8b9cadbecfd0e1f30415263748596a7b8c9daebfc0d1e2f405162738495a6b7c8d9eafb0c1d2e3f5061728394",
+     "7b8895a2afbcc9d6e3f0fd0a1724313e4b5865727f8c99a6b3c0cddae7f4010e1b2835424f5c697683909daab7c4d1deebf805121f2c394653606d7a8794a1aebbc8d5e2effc091623303d4a5764717e8b98a5b2bfccd9e6f3000d1a2734414e5b6875828f9ca9b6c3d0ddeaf704111e2b3845525f6c798693a0adbac7d4e1eefb0815222f3c495663707d8a97a4b1becbd8e5f2ff0c192633404d5a6774818e9ba8b5c2cfdce9f603101d2a3744515e6b7885929facb9c6d3e0edfa0714212e3b4855626f7c8996a3b0bdcad7e4f1fe0b1825323f4c596673808d9aa7b4c1cedbe8f5020f1c293643505d6a7784919eabb8c5d2dfecf90613202d3a4754616e7b8895a2afbcc9d6e3f0fd0a1724313e4b5865727f8c99a6b3c0cddae7f4010e1b2835424f5c697683909daab7c4d1deebf805121f2c394653606d7a8794a1ae",
+     "117b60a7f474fd2245adff82a69429367de8f5263fa96c411986009a743b0e70",
+    },
+    {"00112233445566778899aabbccddeeff102132435465768798a9bacbdcedfe0f",
+     "000d1a2734414e5b6875828f9ca9b6c3d0ddeaf704111e2b3845525f6c798693a0adbac7d4e1eefb0815222f3c495663707d8a97a4b1becbd8e5f2ff0c192633404d5a6774818e9ba8b5c2cfdce9f603101d2a3744515e6b7885929facb9c6d3e0edfa0714212e3b4855626f7c8996a3b0bdcad7e4f1fe0b1825323f4c596673808d9aa7b4c1cedbe8f5020f1c293643505d6a7784919eabb8c5d2dfecf90613202d3a4754616e7b8895a2afbcc9d6e3f0fd0a1724313e4b5865727f8c99a6b3c0cddae7f4010e1b",
+     "cbe3ca627c6432c9a66f07c6411fccca894c9b083b55d0773152460142a676ed",
+    },
+};
+
+static void test_keccak_hmac(const size_t * chunks)
+{
+  uint8_t inp_buff[1024];
+  uint8_t key_buff[1024];
+  uint8_t res_exp[32];
+  uint8_t res_comp[32];
+  const size_t len_chunks = chunks ? sizeof(chunks) / sizeof(*chunks) : 0;
+
+  for (size_t i = 0; i < (sizeof(keccak_hmac_vectors) / sizeof(*keccak_hmac_vectors)); i++)
+  {
+    const size_t inp_len = strlen(keccak_hmac_vectors[i].inp)/2;
+    const size_t key_len = strlen(keccak_hmac_vectors[i].key)/2;
+    ASSERT_TRUE(hexdecode(keccak_hmac_vectors[i].inp, inp_len, inp_buff));
+    ASSERT_TRUE(hexdecode(keccak_hmac_vectors[i].key, key_len, key_buff));
+    ASSERT_TRUE(hexdecode(keccak_hmac_vectors[i].res, 32, res_exp));
+    if (len_chunks == 0)
+    {
+      hmac_keccak_hash(res_comp, key_buff, key_len, inp_buff, inp_len);
+    }
+    else
+    {
+      hmac_keccak_state S;
+      hmac_keccak_init(&S, key_buff, key_len);
+      size_t inp_passed = 0;
+      size_t chunk_size = 0;
+      while(inp_passed < inp_len){
+        const size_t to_pass = std::min(inp_len - inp_passed, chunks[chunk_size]);
+        hmac_keccak_update(&S, inp_buff + inp_passed, to_pass);
+
+        inp_passed += to_pass;
+        chunk_size = (chunk_size + 1) % len_chunks;
+      }
+
+      hmac_keccak_finish(&S, res_comp);
+    }
+    ASSERT_EQ(memcmp(res_exp, res_comp, 32), 0);
+  }
+}
+
+
+TEST(keccak_hmac, )
+{
+  test_keccak_hmac({});
+}
+
+TEST(keccak_hmac, 1)
+{
+  static const size_t chunks[] = {1};
+  test_keccak_hmac(chunks);
+}
+
+TEST(keccak_hmac, 1_20)
+{
+  static const size_t chunks[] = {1, 20};
+  test_keccak_hmac(chunks);
+}
+
+TEST(keccak_hmac, 136_1)
+{
+  static const size_t chunks[] = {136, 1};
+  test_keccak_hmac(chunks);
+}
+
+TEST(keccak_hmac, 137_1)
+{
+  static const size_t chunks[] = {137, 1};
+  test_keccak_hmac(chunks);
+}