Merge pull request #250 from wowario/upstream

Upstream

CMakeLists.txt

@@ -48,6 +48,8 @@ message(STATUS "CMake version ${CMAKE_VERSION}")
 
 project(monero)
 
+include(FindCcache) # Has to be included after the project() macro, to be able to read the CXX variable.
+
 enable_language(C ASM)
 
 function (die msg)

Dockerfile

@@ -53,8 +53,8 @@ RUN set -ex \
 ENV BOOST_ROOT /usr/local/boost_${BOOST_VERSION}
 
 # OpenSSL
-ARG OPENSSL_VERSION=1.1.1b
+ARG OPENSSL_VERSION=1.1.1g
-ARG OPENSSL_HASH=5c557b023230413dfb0756f3137a13e6d726838ccd1430888ad15bfb2b43ea4b
+ARG OPENSSL_HASH=ddb04774f1e32f0c49751e21b67216ac87852ceb056b75209af2443400636d46
 RUN set -ex \
     && curl -s -O https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz \
     && echo "${OPENSSL_HASH}  openssl-${OPENSSL_VERSION}.tar.gz" | sha256sum -c \

cmake/FindCcache.cmake

@@ -0,0 +1,57 @@
+# Copyright (c) 2014-2020, The Monero Project
+#
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are
+# permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice, this list of
+#    conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice, this list
+#    of conditions and the following disclaimer in the documentation and/or other
+#    materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors may be
+#    used to endorse or promote products derived from this software without specific
+#    prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+# - Try to find readline include dirs and libraries 
+#
+# Automatically finds ccache build accelerator, if it's found in PATH.
+#
+# Usage of this module as follows:
+#
+#     project(monero)
+#     include(FindCcache) # Include AFTER the project() macro to be able to reach the CMAKE_CXX_COMPILER variable
+#
+# Properties modified by this module:
+#
+#  GLOBAL PROPERTY RULE_LAUNCH_COMPILE     set to ccache, when ccache found
+#  GLOBAL PROPERTY RULE_LAUNCH_LINK        set to ccache, when ccache found
+
+find_program(CCACHE_FOUND ccache)
+if (CCACHE_FOUND)
+	set(TEMP_CPP_FILE "${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeTmp/test-program.cpp")
+	file(WRITE "${TEMP_CPP_FILE}" "int main() { return 0; }")
+	execute_process(COMMAND "${CCACHE_FOUND}" "${CMAKE_CXX_COMPILER}" "${TEMP_CPP_FILE}"  RESULT_VARIABLE RET)
+	if (${RET} EQUAL 0)
+		message("found usable ccache: ${CCACHE_FOUND}")
+		set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE "${CCACHE_FOUND}")
+		set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK    "${CCACHE_FOUND}")
+	else()
+		message("found ccache ${CCACHE_FOUND}, but is UNUSABLE! Return code: ${RET}")
+	endif()	    	
+else()
+	message("ccache NOT found!")
+endif()
+

contrib/depends/packages/openssl.mk

@@ -1,6 +1,6 @@
 package=openssl
 $(package)_version=1.0.2r
-$(package)_download_path=https://www.openssl.org/source
+$(package)_download_path=https://ftp.openssl.org/source/old/1.0.2
 $(package)_file_name=$(package)-$($(package)_version).tar.gz
 $(package)_sha256_hash=ae51d08bba8a83958e894946f15303ff894d75c2b8bbd44a852b64e3fe11d0d6
 $(package)_patches=fix_arflags.patch

contrib/epee/include/byte_slice.h

@@ -42,7 +42,12 @@ namespace epee
 
   struct release_byte_slice
   {
-    void operator()(byte_slice_data*) const noexcept;
+    //! For use with `zmq_message_init_data`, use second arg for buffer pointer.
+    static void call(void*, void* ptr) noexcept;
+    void operator()(byte_slice_data* ptr) const noexcept
+    {
+      call(nullptr, ptr);
+    }
   };
 
   /*! Inspired by slices in golang. Storage is thread-safe reference counted,
@@ -140,6 +145,9 @@ namespace epee
         \throw std::out_of_range If `size() < end`.
         \return Slice starting at `data() + begin` of size `end - begin`. */
     byte_slice get_slice(std::size_t begin, std::size_t end) const;
+
+    //! \post `empty()` \return Ownership of ref-counted buffer.
+    std::unique_ptr<byte_slice_data, release_byte_slice> take_buffer() noexcept;
   };
 } // epee
 

contrib/epee/include/net/abstract_http_client.h

@@ -0,0 +1,87 @@
+// Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the name of the Andrey N. Sabelnikov nor the
+// names of its contributors may be used to endorse or promote products
+// derived from this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER  BE LIABLE FOR ANY
+// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#pragma once
+
+#include <string>
+#include <boost/optional/optional.hpp>
+#include "http_auth.h"
+#include "net/net_ssl.h"
+
+namespace epee
+{
+namespace net_utils
+{
+  inline const char* get_hex_vals()
+  {
+    static constexpr const char hexVals[16] = {'0','1','2','3','4','5','6','7','8','9','A','B','C','D','E','F'};
+    return hexVals;
+  }
+
+  inline const char* get_unsave_chars()
+  {
+    //static constexpr char unsave_chars[] = "\"<>%\\^[]`+$,@:;/!#?=&";
+    static constexpr const char unsave_chars[] = "\"<>%\\^[]`+$,@:;!#&";
+    return unsave_chars;
+  }
+
+  bool is_unsafe(unsigned char compare_char);
+  std::string dec_to_hex(char num, int radix);
+  int get_index(const char *s, char c);
+  std::string hex_to_dec_2bytes(const char *s);
+  std::string convert(char val);
+  std::string conver_to_url_format(const std::string& uri);
+  std::string convert_from_url_format(const std::string& uri);
+  std::string convert_to_url_format_force_all(const std::string& uri);
+
+namespace http
+{
+  class abstract_http_client
+  {
+  public:
+    abstract_http_client() {}
+    virtual ~abstract_http_client() {}
+    bool set_server(const std::string& address, boost::optional<login> user, ssl_options_t ssl_options = ssl_support_t::e_ssl_support_autodetect);
+    virtual void set_server(std::string host, std::string port, boost::optional<login> user, ssl_options_t ssl_options = ssl_support_t::e_ssl_support_autodetect) = 0;
+    virtual void set_auto_connect(bool auto_connect) = 0;
+    virtual bool connect(std::chrono::milliseconds timeout) = 0;
+    virtual bool disconnect() = 0;
+    virtual bool is_connected(bool *ssl = NULL) = 0;
+    virtual bool invoke(const boost::string_ref uri, const boost::string_ref method, const std::string& body, std::chrono::milliseconds timeout, const http_response_info** ppresponse_info = NULL, const fields_list& additional_params = fields_list()) = 0;
+    virtual bool invoke_get(const boost::string_ref uri, std::chrono::milliseconds timeout, const std::string& body = std::string(), const http_response_info** ppresponse_info = NULL, const fields_list& additional_params = fields_list()) = 0;
+    virtual bool invoke_post(const boost::string_ref uri, const std::string& body, std::chrono::milliseconds timeout, const http_response_info** ppresponse_info = NULL, const fields_list& additional_params = fields_list()) = 0;
+    virtual uint64_t get_bytes_sent() const = 0;
+    virtual uint64_t get_bytes_received() const = 0;
+  };
+
+  class http_client_factory
+  {
+  public:
+    virtual ~http_client_factory() {}
+    virtual std::unique_ptr<abstract_http_client> create() = 0;
+  };
+}
+}
+}

contrib/epee/include/net/http_base.h

@@ -33,6 +33,7 @@
 #include <string>
 #include <utility>
 
+#include "memwipe.h"
 #include "string_tools.h"
 
 #undef MONERO_DEFAULT_LOG_CATEGORY
@@ -200,6 +201,11 @@ namespace net_utils
 				this->~http_response_info();
 				new(this) http_response_info();
 			}
+
+			void wipe()
+			{
+				memwipe(&m_body[0], m_body.size());
+			}
 		};
 	}
 }

contrib/epee/include/net/http_client.h

@@ -47,6 +47,7 @@
 
 #include "string_tools.h"
 #include "reg_exp_definer.h"
+#include "abstract_http_client.h"
 #include "http_base.h" 
 #include "http_auth.h"
 #include "to_nonconst_iterator.h"
@@ -105,140 +106,11 @@ namespace net_utils
 
 
 	//---------------------------------------------------------------------------
-	static inline const char* get_hex_vals()
-	{
-		static const char hexVals[16] = {'0','1','2','3','4','5','6','7','8','9','A','B','C','D','E','F'};
-		return hexVals;
-	}
-
-	static inline const char* get_unsave_chars()
-	{
-		//static char unsave_chars[] = "\"<>%\\^[]`+$,@:;/!#?=&";
-		static const char unsave_chars[] = "\"<>%\\^[]`+$,@:;!#&";
-		return unsave_chars;
-	}
-
-	static inline bool is_unsafe(unsigned char compare_char)
-	{
-		if(compare_char <= 32 || compare_char >= 123)
-			return true;
-
-		const char* punsave = get_unsave_chars();
-
-		for(int ichar_pos = 0; 0!=punsave[ichar_pos] ;ichar_pos++)
-			if(compare_char == punsave[ichar_pos])
-				return true;
-
-		return false;
-	}
-	
-	static inline 
-		std::string dec_to_hex(char num, int radix)
-	{
-		int temp=0;
-		std::string csTmp;
-		int num_char;
-
-		num_char = (int) num;
-		if (num_char < 0)
-			num_char = 256 + num_char;
-
-		while (num_char >= radix)
-		{
-			temp = num_char % radix;
-			num_char = (int)floor((float)num_char / (float)radix);
-			csTmp = get_hex_vals()[temp];
-		}
-
-		csTmp += get_hex_vals()[num_char];
-
-		if(csTmp.size() < 2)
-		{
-			csTmp += '0';
-		}
-		
-		std::reverse(csTmp.begin(), csTmp.end());
-		//_mbsrev((unsigned char*)csTmp.data());
-
-		return csTmp;
-	}
-	static inline int get_index(const char *s, char c) { const char *ptr = (const char*)memchr(s, c, 16); return ptr ? ptr-s : -1; }
-	static inline 
-		std::string hex_to_dec_2bytes(const char *s)
-	{
-		const char *hex = get_hex_vals();
-		int i0 = get_index(hex, toupper(s[0]));
-		int i1 = get_index(hex, toupper(s[1]));
-		if (i0 < 0 || i1 < 0)
-			return std::string("%") + std::string(1, s[0]) + std::string(1, s[1]);
-		return std::string(1, i0 * 16 | i1);
-	}
-
-	static inline std::string convert(char val)
-	{
-		std::string csRet;
-		csRet += "%";
-		csRet += dec_to_hex(val, 16);
-		return  csRet;
-	}
-	static inline std::string conver_to_url_format(const std::string& uri)
-	{
-
-		std::string result;
-
-		for(size_t i = 0; i!= uri.size(); i++)
-		{
-			if(is_unsafe(uri[i]))
-				result += convert(uri[i]);
-			else
-				result += uri[i];
-
-		}
-
-		return result;
-	}
-	static inline std::string convert_from_url_format(const std::string& uri)
-	{
-
-		std::string result;
-
-		for(size_t i = 0; i!= uri.size(); i++)
-		{
-			if(uri[i] == '%' && i + 2 < uri.size())
-			{
-				result += hex_to_dec_2bytes(uri.c_str() + i + 1);
-				i += 2;
-			}
-			else
-				result += uri[i];
-
-		}
-
-		return result;
-	}
-
-  static inline std::string convert_to_url_format_force_all(const std::string& uri)
-  {
-
-    std::string result;
-
-    for(size_t i = 0; i!= uri.size(); i++)
-    {
-        result += convert(uri[i]);
-    }
-
-    return result;
-  }
-
-
-
-
-
 	namespace http
 	{
 
 		template<typename net_client_type>
-		class http_simple_client_template: public i_target_handler
+    class http_simple_client_template : public i_target_handler, public abstract_http_client
 		{
 		private:
 			enum reciev_machine_state
@@ -279,7 +151,7 @@ namespace net_utils
 
 		public:
 			explicit http_simple_client_template()
-				: i_target_handler()
+				: i_target_handler(), abstract_http_client()
 				, m_net_client()
 				, m_host_buff()
 				, m_port()
@@ -299,26 +171,19 @@ namespace net_utils
 			const std::string &get_host() const { return m_host_buff; };
 			const std::string &get_port() const { return m_port; };
 
-			bool set_server(const std::string& address, boost::optional<login> user, ssl_options_t ssl_options = ssl_support_t::e_ssl_support_autodetect)
+			using abstract_http_client::set_server;
-			{
-				http::url_content parsed{};
-				const bool r = parse_url(address, parsed);
-				CHECK_AND_ASSERT_MES(r, false, "failed to parse url: " << address);
-				set_server(std::move(parsed.host), std::to_string(parsed.port), std::move(user), std::move(ssl_options));
-				return true;
-			}
 
-			void set_server(std::string host, std::string port, boost::optional<login> user, ssl_options_t ssl_options = ssl_support_t::e_ssl_support_autodetect)
+			void set_server(std::string host, std::string port, boost::optional<login> user, ssl_options_t ssl_options = ssl_support_t::e_ssl_support_autodetect) override
 			{
 				CRITICAL_REGION_LOCAL(m_lock);
 				disconnect();
 				m_host_buff = std::move(host);
 				m_port = std::move(port);
-                                m_auth = user ? http_client_auth{std::move(*user)} : http_client_auth{};
+				m_auth = user ? http_client_auth{std::move(*user)} : http_client_auth{};
 				m_net_client.set_ssl(std::move(ssl_options));
 			}
 
-			void set_auto_connect(bool auto_connect)
+			void set_auto_connect(bool auto_connect) override
 			{
 				m_auto_connect = auto_connect;
 			}
@@ -330,25 +195,25 @@ namespace net_utils
 				m_net_client.set_connector(std::move(connector));
 			}
 
-      bool connect(std::chrono::milliseconds timeout)
+      bool connect(std::chrono::milliseconds timeout) override
       {
         CRITICAL_REGION_LOCAL(m_lock);
         return m_net_client.connect(m_host_buff, m_port, timeout);
       }
 			//---------------------------------------------------------------------------
-			bool disconnect()
+			bool disconnect() override
 			{
 				CRITICAL_REGION_LOCAL(m_lock);
 				return m_net_client.disconnect();
 			}
 			//---------------------------------------------------------------------------
-			bool is_connected(bool *ssl = NULL)
+			bool is_connected(bool *ssl = NULL) override
 			{
 				CRITICAL_REGION_LOCAL(m_lock);
 				return m_net_client.is_connected(ssl);
 			}
 			//---------------------------------------------------------------------------
-			virtual bool handle_target_data(std::string& piece_of_transfer)
+			virtual bool handle_target_data(std::string& piece_of_transfer) override
 			{
 				CRITICAL_REGION_LOCAL(m_lock);
 				m_response_info.m_body += piece_of_transfer;
@@ -361,15 +226,14 @@ namespace net_utils
         return true;
       }
 			//---------------------------------------------------------------------------
-			inline 
+			inline bool invoke_get(const boost::string_ref uri, std::chrono::milliseconds timeout, const std::string& body = std::string(), const http_response_info** ppresponse_info = NULL, const fields_list& additional_params = fields_list()) override
-				bool invoke_get(const boost::string_ref uri, std::chrono::milliseconds timeout, const std::string& body = std::string(), const http_response_info** ppresponse_info = NULL, const fields_list& additional_params = fields_list())
 			{
 					CRITICAL_REGION_LOCAL(m_lock);
 					return invoke(uri, "GET", body, timeout, ppresponse_info, additional_params);
 			}
 
 			//---------------------------------------------------------------------------
-			inline bool invoke(const boost::string_ref uri, const boost::string_ref method, const std::string& body, std::chrono::milliseconds timeout, const http_response_info** ppresponse_info = NULL, const fields_list& additional_params = fields_list())
+			inline bool invoke(const boost::string_ref uri, const boost::string_ref method, const std::string& body, std::chrono::milliseconds timeout, const http_response_info** ppresponse_info = NULL, const fields_list& additional_params = fields_list()) override
 			{
 				CRITICAL_REGION_LOCAL(m_lock);
 				if(!is_connected())
@@ -442,7 +306,7 @@ namespace net_utils
 				return false;
 			}
 			//---------------------------------------------------------------------------
-			inline bool invoke_post(const boost::string_ref uri, const std::string& body, std::chrono::milliseconds timeout, const http_response_info** ppresponse_info = NULL, const fields_list& additional_params = fields_list())
+			inline bool invoke_post(const boost::string_ref uri, const std::string& body, std::chrono::milliseconds timeout, const http_response_info** ppresponse_info = NULL, const fields_list& additional_params = fields_list()) override
 			{
 				CRITICAL_REGION_LOCAL(m_lock);
 				return invoke(uri, "POST", body, timeout, ppresponse_info, additional_params);
@@ -456,16 +320,21 @@ namespace net_utils
 				return handle_reciev(timeout);
 			}
 			//---------------------------------------------------------------------------
-			uint64_t get_bytes_sent() const
+			uint64_t get_bytes_sent() const override
 			{
 				return m_net_client.get_bytes_sent();
 			}
 			//---------------------------------------------------------------------------
-			uint64_t get_bytes_received() const
+			uint64_t get_bytes_received() const override
 			{
 				return m_net_client.get_bytes_received();
 			}
 			//---------------------------------------------------------------------------
+			void wipe_response()
+			{
+				m_response_info.wipe();
+			}
+			//---------------------------------------------------------------------------
 		private: 
 			//---------------------------------------------------------------------------
 			inline bool handle_reciev(std::chrono::milliseconds timeout)
@@ -1016,6 +885,14 @@ namespace net_utils
 			}
 		};
 		typedef http_simple_client_template<blocked_mode_client> http_simple_client;
+
+    class http_simple_client_factory : public http_client_factory
+    {
+    public:
+      std::unique_ptr<abstract_http_client> create() override {
+        return std::unique_ptr<epee::net_utils::http::abstract_http_client>(new epee::net_utils::http::http_simple_client());
+      }
+    };
 	}
 }
 }

contrib/epee/include/storages/http_abstract_invoke.h

@@ -38,7 +38,7 @@ namespace epee
   namespace net_utils
   {
     template<class t_request, class t_response, class t_transport>
-    bool invoke_http_json(const boost::string_ref uri, const t_request& out_struct, t_response& result_struct, t_transport& transport, std::chrono::milliseconds timeout = std::chrono::seconds(15), const boost::string_ref method = "GET")
+    bool invoke_http_json(const boost::string_ref uri, const t_request& out_struct, t_response& result_struct, t_transport& transport, std::chrono::milliseconds timeout = std::chrono::seconds(15), const boost::string_ref method = "POST")
     {
       std::string req_param;
       if(!serialization::store_t_to_json(out_struct, req_param))
@@ -72,7 +72,7 @@ namespace epee
 
 
     template<class t_request, class t_response, class t_transport>
-    bool invoke_http_bin(const boost::string_ref uri, const t_request& out_struct, t_response& result_struct, t_transport& transport, std::chrono::milliseconds timeout = std::chrono::seconds(15), const boost::string_ref method = "GET")
+    bool invoke_http_bin(const boost::string_ref uri, const t_request& out_struct, t_response& result_struct, t_transport& transport, std::chrono::milliseconds timeout = std::chrono::seconds(15), const boost::string_ref method = "POST")
     {
       std::string req_param;
       if(!serialization::store_t_to_binary(out_struct, req_param))
@@ -101,7 +101,7 @@ namespace epee
     }
 
     template<class t_request, class t_response, class t_transport>
-    bool invoke_http_json_rpc(const boost::string_ref uri, std::string method_name, const t_request& out_struct, t_response& result_struct, epee::json_rpc::error &error_struct, t_transport& transport, std::chrono::milliseconds timeout = std::chrono::seconds(15), const boost::string_ref http_method = "GET", const std::string& req_id = "0")
+    bool invoke_http_json_rpc(const boost::string_ref uri, std::string method_name, const t_request& out_struct, t_response& result_struct, epee::json_rpc::error &error_struct, t_transport& transport, std::chrono::milliseconds timeout = std::chrono::seconds(15), const boost::string_ref http_method = "POST", const std::string& req_id = "0")
     {
       epee::json_rpc::request<t_request> req_t = AUTO_VAL_INIT(req_t);
       req_t.jsonrpc = "2.0";
@@ -125,14 +125,14 @@ namespace epee
     }
 
     template<class t_request, class t_response, class t_transport>
-    bool invoke_http_json_rpc(const boost::string_ref uri, std::string method_name, const t_request& out_struct, t_response& result_struct, t_transport& transport, std::chrono::milliseconds timeout = std::chrono::seconds(15), const boost::string_ref http_method = "GET", const std::string& req_id = "0")
+    bool invoke_http_json_rpc(const boost::string_ref uri, std::string method_name, const t_request& out_struct, t_response& result_struct, t_transport& transport, std::chrono::milliseconds timeout = std::chrono::seconds(15), const boost::string_ref http_method = "POST", const std::string& req_id = "0")
     {
       epee::json_rpc::error error_struct;
       return invoke_http_json_rpc(uri, method_name, out_struct, result_struct, error_struct, transport, timeout, http_method, req_id);
     }
 
     template<class t_command, class t_transport>
-    bool invoke_http_json_rpc(const boost::string_ref uri, typename t_command::request& out_struct, typename t_command::response& result_struct, t_transport& transport, std::chrono::milliseconds timeout = std::chrono::seconds(15), const boost::string_ref http_method = "GET", const std::string& req_id = "0")
+    bool invoke_http_json_rpc(const boost::string_ref uri, typename t_command::request& out_struct, typename t_command::response& result_struct, t_transport& transport, std::chrono::milliseconds timeout = std::chrono::seconds(15), const boost::string_ref http_method = "POST", const std::string& req_id = "0")
     {
       return invoke_http_json_rpc(uri, t_command::methodname(), out_struct, result_struct, transport, timeout, http_method, req_id);
     }

contrib/epee/include/syncobj.h

@@ -150,7 +150,7 @@ namespace epee
   };
 
 
-#define  CRITICAL_REGION_LOCAL(x) {boost::this_thread::sleep_for(boost::chrono::milliseconds(epee::debug::g_test_dbg_lock_sleep()));}   epee::critical_region_t<decltype(x)>   critical_region_var(x)
+#define  CRITICAL_REGION_LOCAL(x) {} epee::critical_region_t<decltype(x)>   critical_region_var(x)
 #define  CRITICAL_REGION_BEGIN(x) { boost::this_thread::sleep_for(boost::chrono::milliseconds(epee::debug::g_test_dbg_lock_sleep())); epee::critical_region_t<decltype(x)>   critical_region_var(x)
 #define  CRITICAL_REGION_LOCAL1(x) {boost::this_thread::sleep_for(boost::chrono::milliseconds(epee::debug::g_test_dbg_lock_sleep()));} epee::critical_region_t<decltype(x)>   critical_region_var1(x)
 #define  CRITICAL_REGION_BEGIN1(x) {  boost::this_thread::sleep_for(boost::chrono::milliseconds(epee::debug::g_test_dbg_lock_sleep())); epee::critical_region_t<decltype(x)>   critical_region_var1(x)

contrib/epee/src/CMakeLists.txt

@@ -26,7 +26,7 @@
 # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
 # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-add_library(epee STATIC byte_slice.cpp hex.cpp http_auth.cpp mlog.cpp net_helper.cpp net_utils_base.cpp string_tools.cpp wipeable_string.cpp
+add_library(epee STATIC byte_slice.cpp hex.cpp abstract_http_client.cpp http_auth.cpp mlog.cpp net_helper.cpp net_utils_base.cpp string_tools.cpp wipeable_string.cpp
     levin_base.cpp memwipe.c connection_basic.cpp network_throttle.cpp network_throttle-detail.cpp mlocker.cpp buffer.cpp net_ssl.cpp
     int-util.cpp)
 

contrib/epee/src/abstract_http_client.cpp

@@ -0,0 +1,142 @@
+#include "net/abstract_http_client.h"
+#include "net/http_base.h"
+#include "net/net_parse_helpers.h"
+
+#undef MONERO_DEFAULT_LOG_CATEGORY
+#define MONERO_DEFAULT_LOG_CATEGORY "net.http"
+
+namespace epee
+{
+namespace net_utils
+{
+  //----------------------------------------------------------------------------------------------------
+  bool is_unsafe(unsigned char compare_char)
+  {
+    if(compare_char <= 32 || compare_char >= 123)
+      return true;
+
+    const char* punsave = get_unsave_chars();
+
+    for(int ichar_pos = 0; 0!=punsave[ichar_pos] ;ichar_pos++)
+      if(compare_char == punsave[ichar_pos])
+        return true;
+
+    return false;
+  }
+  //----------------------------------------------------------------------------------------------------
+  std::string dec_to_hex(char num, int radix)
+  {
+    int temp=0;
+    std::string csTmp;
+    int num_char;
+
+    num_char = (int) num;
+    if (num_char < 0)
+      num_char = 256 + num_char;
+
+    while (num_char >= radix)
+    {
+      temp = num_char % radix;
+      num_char = (int)floor((float)num_char / (float)radix);
+      csTmp = get_hex_vals()[temp];
+    }
+
+    csTmp += get_hex_vals()[num_char];
+
+    if(csTmp.size() < 2)
+    {
+      csTmp += '0';
+    }
+
+    std::reverse(csTmp.begin(), csTmp.end());
+    //_mbsrev((unsigned char*)csTmp.data());
+
+    return csTmp;
+  }
+  //----------------------------------------------------------------------------------------------------
+  int get_index(const char *s, char c)
+  {
+    const char *ptr = (const char*)memchr(s, c, 16);
+    return ptr ? ptr-s : -1;
+  }
+  //----------------------------------------------------------------------------------------------------
+  std::string hex_to_dec_2bytes(const char *s)
+  {
+    const char *hex = get_hex_vals();
+    int i0 = get_index(hex, toupper(s[0]));
+    int i1 = get_index(hex, toupper(s[1]));
+    if (i0 < 0 || i1 < 0)
+      return std::string("%") + std::string(1, s[0]) + std::string(1, s[1]);
+    return std::string(1, i0 * 16 | i1);
+  }
+  //----------------------------------------------------------------------------------------------------
+  std::string convert(char val)
+  {
+    std::string csRet;
+    csRet += "%";
+    csRet += dec_to_hex(val, 16);
+    return  csRet;
+  }
+  //----------------------------------------------------------------------------------------------------
+  std::string conver_to_url_format(const std::string& uri)
+  {
+
+    std::string result;
+
+    for(size_t i = 0; i!= uri.size(); i++)
+    {
+      if(is_unsafe(uri[i]))
+        result += convert(uri[i]);
+      else
+        result += uri[i];
+
+    }
+
+    return result;
+  }
+  //----------------------------------------------------------------------------------------------------
+  std::string convert_from_url_format(const std::string& uri)
+  {
+
+    std::string result;
+
+    for(size_t i = 0; i!= uri.size(); i++)
+    {
+      if(uri[i] == '%' && i + 2 < uri.size())
+      {
+        result += hex_to_dec_2bytes(uri.c_str() + i + 1);
+        i += 2;
+      }
+      else
+        result += uri[i];
+
+    }
+
+    return result;
+  }
+  //----------------------------------------------------------------------------------------------------
+  std::string convert_to_url_format_force_all(const std::string& uri)
+  {
+    std::string result;
+
+    for(size_t i = 0; i!= uri.size(); i++)
+    {
+        result += convert(uri[i]);
+    }
+    return result;
+  }
+
+namespace http
+{
+  //----------------------------------------------------------------------------------------------------
+  bool epee::net_utils::http::abstract_http_client::set_server(const std::string& address, boost::optional<login> user, ssl_options_t ssl_options)
+  {
+    http::url_content parsed{};
+    const bool r = parse_url(address, parsed);
+    CHECK_AND_ASSERT_MES(r, false, "failed to parse url: " << address);
+    set_server(std::move(parsed.host), std::to_string(parsed.port), std::move(user), std::move(ssl_options));
+    return true;
+  }
+}
+}
+}

contrib/epee/src/byte_slice.cpp

@@ -49,12 +49,16 @@ namespace epee
     std::atomic<std::size_t> ref_count;
   };
 
-  void release_byte_slice::operator()(byte_slice_data* ptr) const noexcept
+  void release_byte_slice::call(void*, void* ptr) noexcept
   {
-    if (ptr && --(ptr->ref_count) == 0)
+    if (ptr)
     {
-      ptr->~byte_slice_data();
+      byte_slice_data* self = static_cast<byte_slice_data*>(ptr);
-      free(ptr);
+      if (--(self->ref_count) == 0)
+      {
+        self->~byte_slice_data();
+        free(self);
+      }
     }
   }
 
@@ -206,4 +210,11 @@ namespace epee
       return {};
     return {storage_.get(), {portion_.begin() + begin, end - begin}};
   }
+
+  std::unique_ptr<byte_slice_data, release_byte_slice> byte_slice::take_buffer() noexcept
+  {
+    std::unique_ptr<byte_slice_data, release_byte_slice> out{std::move(storage_)};
+    portion_ = nullptr;
+    return out;
+  }
 } // epee

contrib/epee/src/hex.cpp

@@ -141,7 +141,7 @@ namespace epee
     result.reserve(count / 2);
 
     // the data to work with (std::string is always null-terminated)
-    auto data = src.data();
+    auto data = src.begin();
 
     // convert a single hex character to an unsigned integer
     auto char_to_int = [](const char *input) {
@@ -167,9 +167,9 @@ namespace epee
     };
 
     // keep going until we reach the end
-    while (data[0] != '\0') {
+    while (data != src.end()) {
       // skip unwanted characters
-      if (!include(data[0])) {
+      if (!include(*data)) {
         ++data;
         continue;
       }

external/trezor-common

@@ -1 +1 @@
-Subproject commit 31a0073c62738827b48d725facd3766879429124
+Subproject commit bff7fdfe436c727982cc553bdfb29a9021b423b0

src/common/util.cpp

@@ -66,7 +66,6 @@ using namespace epee;
 #include "util.h"
 #include "stack_trace.h"
 #include "memwipe.h"
-#include "cryptonote_config.h"
 #include "net/http_client.h"                        // epee::net_utils::...
 #include "readline_buffer.h"
 
@@ -1074,16 +1073,33 @@ std::string get_nix_version_display_string()
   {
     if (seconds < 60)
       return std::to_string(seconds) + " seconds";
+    std::stringstream ss;
+    ss << std::fixed << std::setprecision(1);
     if (seconds < 3600)
-      return std::to_string((uint64_t)(seconds / 60)) + " minutes";
+    {
+      ss << seconds / 60.f;
+      return ss.str() + " minutes";
+    }
     if (seconds < 3600 * 24)
-      return std::to_string((uint64_t)(seconds / 3600)) + " hours";
+    {
-    if (seconds < 3600 * 24 * 30.5)
+      ss << seconds / 3600.f;
-      return std::to_string((uint64_t)(seconds / (3600 * 24))) + " days";
+      return ss.str() + " hours";
-    if (seconds < 3600 * 24 * 365.25)
+    }
-      return std::to_string((uint64_t)(seconds / (3600 * 24 * 30.5))) + " months";
+    if (seconds < 3600 * 24 * 30.5f)
-    if (seconds < 3600 * 24 * 365.25 * 100)
+    {
-      return std::to_string((uint64_t)(seconds / (3600 * 24 * 30.5 * 365.25))) + " years";
+      ss << seconds / (3600 * 24.f);
+      return ss.str() + " days";
+    }
+    if (seconds < 3600 * 24 * 365.25f)
+    {
+      ss << seconds / (3600 * 24 * 30.5f);
+      return ss.str() + " months";
+    }
+    if (seconds < 3600 * 24 * 365.25f * 100)
+    {
+      ss << seconds / (3600 * 24 * 365.25f);
+      return ss.str() + " years";
+    }
     return "a long time";
   }
 
@@ -1279,6 +1295,86 @@ std::string get_nix_version_display_string()
     return lines;
   }
 
+  // Calculate a "sync weight" over ranges of blocks in the blockchain, suitable for
+  // calculating sync time estimates
+  uint64_t cumulative_block_sync_weight(cryptonote::network_type nettype, uint64_t start_block, uint64_t num_blocks)
+  {
+    if (nettype != cryptonote::MAINNET)
+    {
+      // No detailed data available except for Mainnet: Give back the number of blocks
+      // as a very simple and non-varying block sync weight for ranges of Testnet and
+      // Stagenet blocks
+      return num_blocks;
+    }
+
+    // The following is a table of average blocks sizes in bytes over the Monero mainnet
+    // blockchain, where the block size is averaged over ranges of 10,000 blocks
+    // (about 2 weeks worth of blocks each).
+    // The first array entry of 442 thus means "The average byte size of the blocks
+    // 0 .. 9,999 is 442". The info "block_size" from the "get_block_header_by_height"
+    // RPC call was used for calculating this. This table (and the whole mechanism
+    // of calculating a "sync weight") is most important when estimating times for
+    // syncing from scratch. Without it the fast progress through the (in comparison)
+    // rather small blocks in the early blockchain) would lead to vastly underestimated
+    // total sync times.
+    // It's no big problem for estimates that this table will, over time, and if not
+    // updated, miss larger and larger parts at the top of the blockchain, as long
+    // as block size averages there do not differ wildly.
+    // Without time-consuming tests it's hard to say how much the estimates would
+    // improve if one would not only take block sizes into account, but also varying
+    // verification times i.e. the different CPU effort needed for the different
+    // transaction types (pre / post RingCT, pre / post Bulletproofs).
+    // Testnet and Stagenet are neglected here because of their much smaller
+    // importance.
+    static const uint32_t average_block_sizes[] =
+    {
+      442, 1211, 1445, 1763, 2272, 8217, 5603, 9999, 16358, 10805, 5290, 4362,
+      4325, 5584, 4515, 5008, 4789, 5196, 7660, 3829, 6034, 2925, 3762, 2545,
+      2437, 2553, 2167, 2761, 2015, 1969, 2350, 1731, 2367, 2078, 2026, 3518,
+      2214, 1908, 1780, 1640, 1976, 1647, 1921, 1716, 1895, 2150, 2419, 2451,
+      2147, 2327, 2251, 1644, 1750, 1481, 1570, 1524, 1562, 1668, 1386, 1494,
+      1637, 1880, 1431, 1472, 1637, 1363, 1762, 1597, 1999, 1564, 1341, 1388,
+      1530, 1476, 1617, 1488, 1368, 1906, 1403, 1695, 1535, 1598, 1318, 1234,
+      1358, 1406, 1698, 1554, 1591, 1758, 1426, 2389, 1946, 1533, 1308, 2701,
+      1525, 1653, 3580, 1889, 2913, 8164, 5154, 3762, 3356, 4360, 3589, 4844,
+      4232, 3781, 3882, 5924, 10790, 7185, 7442, 8214, 8509, 7484, 6939, 7391,
+      8210, 15572, 39680, 44810, 53873, 54639, 68227, 63428, 62386, 68504,
+      83073, 103858, 117573, 98089, 96793, 102337, 94714, 129568, 251584,
+      132026, 94579, 94516, 95722, 106495, 121824, 153983, 162338, 136608,
+      137104, 109872, 91114, 84757, 96339, 74251, 94314, 143216, 155837,
+      129968, 120201, 109913, 101588, 97332, 104611, 95310, 93419, 113345,
+      100743, 92152, 57565, 22533, 37564, 21823, 19980, 18277, 18402, 14344,
+      12142, 15842, 13677, 17631, 18294, 22270, 41422, 39296, 36688, 33512,
+      33831, 27582, 22276, 27516, 27317, 25505, 24426, 20566, 23045, 26766,
+      28185, 26169, 27011,
+      28642    // Blocks 1,990,000 to 1,999,999 in December 2019
+    };
+    const uint64_t block_range_size = 10000;
+
+    uint64_t num_block_sizes = sizeof(average_block_sizes) / sizeof(average_block_sizes[0]);
+    uint64_t weight = 0;
+    uint64_t table_index = start_block / block_range_size;
+    for (;;) {
+      if (num_blocks == 0)
+      {
+        break;
+      }
+      if (table_index >= num_block_sizes)
+      {
+        // Take all blocks beyond our table as having the size of the blocks
+        // in the last table entry i.e. in the most recent known block range
+        weight += num_blocks * average_block_sizes[num_block_sizes - 1];
+        break;
+      }
+      uint64_t portion_size = std::min(num_blocks, block_range_size - start_block % block_range_size);
+      weight += portion_size * average_block_sizes[table_index];
+      table_index++;
+      num_blocks -= portion_size;
+      start_block += portion_size;
+    }
+    return weight;
+  }
+
   std::vector<std::pair<std::string, size_t>> split_string_by_width(const std::string &s, size_t columns)
   {
     std::vector<std::string> lines;

src/common/util.h

@@ -46,6 +46,7 @@
 #endif
 
 #include "crypto/hash.h"
+#include "cryptonote_config.h"
 
 /*! \brief Various Tools
  *
@@ -252,4 +253,6 @@ namespace tools
   void clear_screen();
 
   std::vector<std::pair<std::string, size_t>> split_string_by_width(const std::string &s, size_t columns);
+
+  uint64_t cumulative_block_sync_weight(cryptonote::network_type nettype, uint64_t start_block, uint64_t num_blocks);
 }

src/cryptonote_basic/cryptonote_format_utils.cpp

@@ -126,6 +126,20 @@ namespace cryptonote
 namespace cryptonote
 {
   //---------------------------------------------------------------
+  void get_transaction_prefix_hash(const transaction_prefix& tx, crypto::hash& h, hw::device &hwdev)
+  {
+    hwdev.get_transaction_prefix_hash(tx,h);    
+  }
+
+  //---------------------------------------------------------------  
+  crypto::hash get_transaction_prefix_hash(const transaction_prefix& tx, hw::device &hwdev)
+  {
+    crypto::hash h = null_hash;
+    get_transaction_prefix_hash(tx, h, hwdev);
+    return h;
+  }
+  
+  //---------------------------------------------------------------  
   void get_transaction_prefix_hash(const transaction_prefix& tx, crypto::hash& h)
   {
     std::ostringstream s;

src/cryptonote_basic/cryptonote_format_utils.h

@@ -48,6 +48,8 @@ namespace epee
 namespace cryptonote
 {
   //---------------------------------------------------------------
+  void get_transaction_prefix_hash(const transaction_prefix& tx, crypto::hash& h, hw::device &hwdev);
+  crypto::hash get_transaction_prefix_hash(const transaction_prefix& tx, hw::device &hwdev);
   void get_transaction_prefix_hash(const transaction_prefix& tx, crypto::hash& h);
   crypto::hash get_transaction_prefix_hash(const transaction_prefix& tx);
   bool parse_and_validate_tx_prefix_from_blob(const blobdata& tx_blob, transaction_prefix& tx);

src/cryptonote_core/cryptonote_core.cpp

@@ -428,9 +428,9 @@ namespace cryptonote
     return m_blockchain_storage.get_split_transactions_blobs(txs_ids, txs, missed_txs);
   }
   //-----------------------------------------------------------------------------------------------
-  bool core::get_txpool_backlog(std::vector<tx_backlog_entry>& backlog) const
+  bool core::get_txpool_backlog(std::vector<tx_backlog_entry>& backlog, bool include_sensitive_txes) const
   {
-    m_mempool.get_transaction_backlog(backlog);
+    m_mempool.get_transaction_backlog(backlog, include_sensitive_txes);
     return true;
   }
   //-----------------------------------------------------------------------------------------------
@@ -1584,9 +1584,9 @@ namespace cryptonote
     return m_blockchain_storage.get_db().get_block_cumulative_difficulty(height);
   }
   //-----------------------------------------------------------------------------------------------
-  size_t core::get_pool_transactions_count() const
+  size_t core::get_pool_transactions_count(bool include_sensitive_txes) const
   {
-    return m_mempool.get_transactions_count();
+    return m_mempool.get_transactions_count(include_sensitive_txes);
   }
   //-----------------------------------------------------------------------------------------------
   bool core::have_block(const crypto::hash& id) const

src/cryptonote_core/cryptonote_core.h

@@ -469,10 +469,11 @@ namespace cryptonote
 
      /**
       * @copydoc tx_memory_pool::get_txpool_backlog
+      * @param include_sensitive_txes include private transactions
       *
       * @note see tx_memory_pool::get_txpool_backlog
       */
-     bool get_txpool_backlog(std::vector<tx_backlog_entry>& backlog) const;
+     bool get_txpool_backlog(std::vector<tx_backlog_entry>& backlog, bool include_sensitive_txes = false) const;
      
      /**
       * @copydoc tx_memory_pool::get_transactions
@@ -514,10 +515,11 @@ namespace cryptonote
 
      /**
       * @copydoc tx_memory_pool::get_transactions_count
+      * @param include_sensitive_txes include private transactions
       *
       * @note see tx_memory_pool::get_transactions_count
       */
-     size_t get_pool_transactions_count() const;
+     size_t get_pool_transactions_count(bool include_sensitive_txes = false) const;
 
      /**
       * @copydoc Blockchain::get_total_transactions

src/cryptonote_core/cryptonote_tx_utils.cpp

@@ -590,7 +590,7 @@ namespace cryptonote
         tx.vout[i].amount = 0;
 
       crypto::hash tx_prefix_hash;
-      get_transaction_prefix_hash(tx, tx_prefix_hash);
+      get_transaction_prefix_hash(tx, tx_prefix_hash, hwdev);
       rct::ctkeyV outSk;
       if (rct_config.range_proof_type != rct::RangeProofPaddedBulletproof && use_simple_rct)
         tx.rct_signatures = rct::genRctSimple_old(rct::hash2rct(tx_prefix_hash), inSk, destinations, inamounts, outamounts, amount_in - amount_out, mixRing, amount_keys, msout ? &kLRki : NULL, msout, index, outSk, rct_config, hwdev);

src/cryptonote_core/tx_pool.cpp

@@ -210,7 +210,7 @@ namespace cryptonote
     // TODO: Investigate why not?
     if(!kept_by_block)
     {
-      if(have_tx_keyimges_as_spent(tx))
+      if(have_tx_keyimges_as_spent(tx, id))
       {
         mark_double_spend(tx);
         LOG_PRINT_L1("Transaction with id= "<< id << " used already spent key images");
@@ -253,7 +253,7 @@ namespace cryptonote
         meta.last_relayed_time = time(NULL);
         meta.relayed = relayed;
         meta.set_relay_method(tx_relay);
-        meta.double_spend_seen = have_tx_keyimges_as_spent(tx);
+        meta.double_spend_seen = have_tx_keyimges_as_spent(tx, id);
         meta.pruned = tx.pruned;
         meta.bf_padding = 0;
         memset(meta.padding, 0, sizeof(meta.padding));
@@ -615,8 +615,8 @@ namespace cryptonote
     CRITICAL_REGION_LOCAL1(m_blockchain);
 
     m_blockchain.for_all_txpool_txes([this, &hashes, &txes](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata*) {
-      const auto relay_method = meta.get_relay_method();
+      const auto tx_relay_method = meta.get_relay_method();
-      if (relay_method != relay_method::block && relay_method != relay_method::fluff)
+      if (tx_relay_method != relay_method::block && tx_relay_method != relay_method::fluff)
         return true;
       const auto i = std::find(hashes.begin(), hashes.end(), txid);
       if (i == hashes.end())
@@ -1098,30 +1098,32 @@ namespace cryptonote
     return m_blockchain.get_db().txpool_has_tx(id, tx_category);
   }
   //---------------------------------------------------------------------------------
-  bool tx_memory_pool::have_tx_keyimges_as_spent(const transaction& tx) const
+  bool tx_memory_pool::have_tx_keyimges_as_spent(const transaction& tx, const crypto::hash& txid) const
   {
     CRITICAL_REGION_LOCAL(m_transactions_lock);
     CRITICAL_REGION_LOCAL1(m_blockchain);
     for(const auto& in: tx.vin)
     {
       CHECKED_GET_SPECIFIC_VARIANT(in, const txin_to_key, tokey_in, true);//should never fail
-      if(have_tx_keyimg_as_spent(tokey_in.k_image))
+      if(have_tx_keyimg_as_spent(tokey_in.k_image, txid))
          return true;
     }
     return false;
   }
   //---------------------------------------------------------------------------------
-  bool tx_memory_pool::have_tx_keyimg_as_spent(const crypto::key_image& key_im) const
+  bool tx_memory_pool::have_tx_keyimg_as_spent(const crypto::key_image& key_im, const crypto::hash& txid) const
   {
     CRITICAL_REGION_LOCAL(m_transactions_lock);
-    bool spent = false;
     const auto found = m_spent_key_images.find(key_im);
-    if (found != m_spent_key_images.end())
+    if (found != m_spent_key_images.end() && !found->second.empty())
     {
-      for (const crypto::hash& tx_hash : found->second)
+      // If another tx is using the key image, always return as spent.
-        spent |= m_blockchain.txpool_tx_matches_category(tx_hash, relay_category::broadcasted);
+      // See `insert_key_images`.
+      if (1 < found->second.size() || *(found->second.cbegin()) != txid)
+        return true;
+      return m_blockchain.txpool_tx_matches_category(txid, relay_category::broadcasted);
     }
-    return spent;
+    return false;
   }
   //---------------------------------------------------------------------------------
   void tx_memory_pool::lock() const

src/cryptonote_core/tx_pool.h

@@ -470,10 +470,11 @@ namespace cryptonote
      * @brief check if a transaction in the pool has a given spent key image
      *
      * @param key_im the spent key image to look for
+     * @param txid hash of the new transaction where `key_im` was seen.
      *
      * @return true if the spent key image is present, otherwise false
      */
-    bool have_tx_keyimg_as_spent(const crypto::key_image& key_im) const;
+    bool have_tx_keyimg_as_spent(const crypto::key_image& key_im, const crypto::hash& txid) const;
 
     /**
      * @brief check if any spent key image in a transaction is in the pool
@@ -484,10 +485,11 @@ namespace cryptonote
      * @note see tx_pool::have_tx_keyimg_as_spent
      *
      * @param tx the transaction to check spent key images of
+     * @param txid hash of `tx`.
      *
      * @return true if any spent key images are present in the pool, otherwise false
      */
-    bool have_tx_keyimges_as_spent(const transaction& tx) const;
+    bool have_tx_keyimges_as_spent(const transaction& tx, const crypto::hash& txid) const;
 
     /**
      * @brief forget a transaction's spent key images

src/cryptonote_protocol/cryptonote_protocol_handler.h

@@ -170,6 +170,14 @@ namespace cryptonote
     size_t m_block_download_max_size;
     bool m_sync_pruned_blocks;
 
+    // Values for sync time estimates
+    boost::posix_time::ptime m_sync_start_time;
+    boost::posix_time::ptime m_period_start_time;
+    uint64_t m_sync_start_height;
+    uint64_t m_period_start_height;
+    uint64_t get_estimated_remaining_sync_seconds(uint64_t current_blockchain_height, uint64_t target_blockchain_height);
+    std::string get_periodic_sync_estimate(uint64_t current_blockchain_height, uint64_t target_blockchain_height);
+
     boost::mutex m_buffer_mutex;
     double get_avg_block_size();
     boost::circular_buffer<size_t> m_avg_buffer = boost::circular_buffer<size_t>(10);

src/cryptonote_protocol/cryptonote_protocol_handler.inl

@@ -43,6 +43,7 @@
 #include "profile_tools.h"
 #include "net/network_throttle-detail.hpp"
 #include "common/pruning.h"
+#include "common/util.h"
 
 #undef MONERO_DEFAULT_LOG_CATEGORY
 #define MONERO_DEFAULT_LOG_CATEGORY "net.cn"
@@ -362,7 +363,7 @@ namespace cryptonote
     uint64_t last_block_v1 = m_core.get_nettype() == TESTNET ? 0 : m_core.get_nettype() == MAINNET ? 0 : (uint64_t)-1;
     uint64_t diff_v2 = max_block_height > last_block_v1 ? std::min(abs_diff, max_block_height - last_block_v1) : 0;
     MCLOG(is_inital ? el::Level::Info : el::Level::Debug, "global", el::Color::Yellow, context <<  "Sync data returned a new top block candidate: " << m_core.get_current_blockchain_height() << " -> " << hshd.current_height
-      << " [Your node is " << abs_diff << " blocks (" << ((abs_diff - diff_v2) / (24 * 60 * 60 / DIFFICULTY_TARGET_V1)) + (diff_v2 / (24 * 60 * 60 / DIFFICULTY_TARGET_V2)) << " days) "
+      << " [Your node is " << abs_diff << " blocks (" << tools::get_human_readable_timespan((abs_diff - diff_v2) * DIFFICULTY_TARGET_V1 + diff_v2 * DIFFICULTY_TARGET_V2) << ") "
       << (0 <= diff ? std::string("behind") : std::string("ahead"))
       << "] " << ENDL << "SYNCHRONIZATION started");
       if (hshd.current_height >= m_core.get_current_blockchain_height() + 5) // don't switch to unsafe mode just for a few blocks
@@ -1212,6 +1213,55 @@ namespace cryptonote
     return 1;
   }
 
+  // Get an estimate for the remaining sync time from given current to target blockchain height, in seconds
+  template<class t_core>
+  uint64_t t_cryptonote_protocol_handler<t_core>::get_estimated_remaining_sync_seconds(uint64_t current_blockchain_height, uint64_t target_blockchain_height)
+  {
+    // The average sync speed varies so much, even averaged over quite long time periods like 10 minutes,
+    // that using some sliding window would be difficult to implement without often leading to bad estimates.
+    // The simplest strategy - always average sync speed over the maximum available interval i.e. since sync
+    // started at all (from "m_sync_start_time" and "m_sync_start_height") - gives already useful results
+    // and seems to be quite robust. Some quite special cases like "Internet connection suddenly becoming
+    // much faster after syncing already a long time, and staying fast" are not well supported however.
+
+    if (target_blockchain_height <= current_blockchain_height)
+    {
+      // Syncing stuck, or other special circumstance: Avoid errors, simply give back 0
+      return 0;
+    }
+
+    const boost::posix_time::ptime now = boost::posix_time::microsec_clock::universal_time();
+    const boost::posix_time::time_duration sync_time = now - m_sync_start_time;
+    cryptonote::network_type nettype = m_core.get_nettype();
+
+    // Don't simply use remaining number of blocks for the estimate but "sync weight" as provided by
+    // "cumulative_block_sync_weight" which knows about strongly varying Monero mainnet block sizes
+    uint64_t synced_weight = tools::cumulative_block_sync_weight(nettype, m_sync_start_height, current_blockchain_height - m_sync_start_height);
+    float us_per_weight = (float)sync_time.total_microseconds() / (float)synced_weight;
+    uint64_t remaining_weight = tools::cumulative_block_sync_weight(nettype, current_blockchain_height, target_blockchain_height - current_blockchain_height);
+    float remaining_us = us_per_weight * (float)remaining_weight;
+    return (uint64_t)(remaining_us / 1e6);
+  }
+
+  // Return a textual remaining sync time estimate, or the empty string if waiting period not yet over
+  template<class t_core>
+  std::string t_cryptonote_protocol_handler<t_core>::get_periodic_sync_estimate(uint64_t current_blockchain_height, uint64_t target_blockchain_height)
+  {
+    std::string text = "";
+    const boost::posix_time::ptime now = boost::posix_time::microsec_clock::universal_time();
+    boost::posix_time::time_duration period_sync_time = now - m_period_start_time;
+    if (period_sync_time > boost::posix_time::minutes(2))
+    {
+      // Period is over, time to report another estimate
+      uint64_t remaining_seconds = get_estimated_remaining_sync_seconds(current_blockchain_height, target_blockchain_height);
+      text = tools::get_human_readable_timespan(remaining_seconds);
+
+      // Start the new period
+      m_period_start_time = now;
+    }
+    return text;
+  }
+
   template<class t_core>
   int t_cryptonote_protocol_handler<t_core>::try_add_next_blocks(cryptonote_connection_context& context)
   {
@@ -1240,6 +1290,9 @@ namespace cryptonote
           if (!starting)
             m_last_add_end_time = tools::get_tick_count();
         });
+        m_sync_start_time = boost::posix_time::microsec_clock::universal_time();
+        m_sync_start_height = m_core.get_current_blockchain_height();
+        m_period_start_time = m_sync_start_time;
 
         while (1)
         {
@@ -1490,7 +1543,16 @@ namespace cryptonote
               if (completion_percent == 100) // never show 100% if not actually up to date
                 completion_percent = 99;
               progress_message = " (" + std::to_string(completion_percent) + "%, "
-                  + std::to_string(target_blockchain_height - current_blockchain_height) + " left)";
+                  + std::to_string(target_blockchain_height - current_blockchain_height) + " left";
+              std::string time_message = get_periodic_sync_estimate(current_blockchain_height, target_blockchain_height);
+              if (!time_message.empty())
+              {
+                uint64_t total_blocks_to_sync = target_blockchain_height - m_sync_start_height;
+                uint64_t total_blocks_synced = current_blockchain_height - m_sync_start_height;
+                progress_message += ", " + std::to_string(total_blocks_synced * 100 / total_blocks_to_sync) + "% of total synced";
+                progress_message += ", estimated " + time_message + " left";
+              }
+              progress_message += ")";
             }
             const uint32_t previous_stripe = tools::get_pruning_stripe(previous_height, target_blockchain_height, CRYPTONOTE_PRUNING_LOG_STRIPES);
             const uint32_t current_stripe = tools::get_pruning_stripe(current_blockchain_height, target_blockchain_height, CRYPTONOTE_PRUNING_LOG_STRIPES);
@@ -2228,8 +2290,26 @@ skip:
   bool t_cryptonote_protocol_handler<t_core>::on_connection_synchronized()
   {
     bool val_expected = false;
-    if(!m_core.is_within_compiled_block_hash_area(m_core.get_current_blockchain_height()) && m_synchronized.compare_exchange_strong(val_expected, true))
+    uint64_t current_blockchain_height = m_core.get_current_blockchain_height();
+    if(!m_core.is_within_compiled_block_hash_area(current_blockchain_height) && m_synchronized.compare_exchange_strong(val_expected, true))
     {
+      if ((current_blockchain_height > m_sync_start_height) && (m_sync_spans_downloaded > 0))
+      {
+        uint64_t synced_blocks = current_blockchain_height - m_sync_start_height;
+        // Report only after syncing an "interesting" number of blocks:
+        if (synced_blocks > 20)
+        {
+          const boost::posix_time::ptime now = boost::posix_time::microsec_clock::universal_time();
+          uint64_t synced_seconds = (now - m_sync_start_time).total_seconds();
+          if (synced_seconds == 0)
+          {
+            synced_seconds = 1;
+          }
+          float blocks_per_second = (1000 * synced_blocks / synced_seconds) / 1000.0f;
+          MGINFO_YELLOW("Synced " << synced_blocks << " blocks in "
+            << tools::get_human_readable_timespan(synced_seconds) << " (" << blocks_per_second << " blocks per second)");
+        }
+      }
       MGINFO_YELLOW(ENDL << "**********************************************************************" << ENDL
         << "You are now synchronized with the network. You may now start wownero-wallet-cli." << ENDL
         << ENDL

src/daemon/rpc_command_executor.cpp

@@ -695,11 +695,11 @@ bool t_rpc_command_executor::print_net_stats()
 
   if (m_is_rpc)
   {
-    if (!m_rpc_client->json_rpc_request(net_stats_req, net_stats_res, "get_net_stats", fail_message.c_str()))
+    if (!m_rpc_client->rpc_request(net_stats_req, net_stats_res, "/get_net_stats", fail_message.c_str()))
     {
       return true;
     }
-    if (!m_rpc_client->json_rpc_request(limit_req, limit_res, "get_limit", fail_message.c_str()))
+    if (!m_rpc_client->rpc_request(limit_req, limit_res, "/get_limit", fail_message.c_str()))
     {
       return true;
     }

src/device/device.hpp

@@ -56,6 +56,7 @@ namespace cryptonote
     struct subaddress_index;
     struct tx_destination_entry;
     struct keypair;
+    class transaction_prefix;
 }
 
 namespace hw {
@@ -78,7 +79,7 @@ namespace hw {
         virtual void on_button_request(uint64_t code=0) {}
         virtual void on_button_pressed() {}
         virtual boost::optional<epee::wipeable_string> on_pin_request() { return boost::none; }
-        virtual boost::optional<epee::wipeable_string> on_passphrase_request(bool on_device) { return boost::none; }
+        virtual boost::optional<epee::wipeable_string> on_passphrase_request(bool & on_device) { on_device = true; return boost::none; }
         virtual void on_progress(const device_progress& event) {}
         virtual ~i_device_callback() = default;
     };
@@ -203,6 +204,8 @@ namespace hw {
 
         virtual bool  open_tx(crypto::secret_key &tx_key) = 0;
 
+        virtual void get_transaction_prefix_hash(const cryptonote::transaction_prefix& tx, crypto::hash& h) = 0;
+        
         virtual bool  encrypt_payment_id(crypto::hash8 &payment_id, const crypto::public_key &public_key, const crypto::secret_key &secret_key) = 0;
         bool  decrypt_payment_id(crypto::hash8 &payment_id, const crypto::public_key &public_key, const crypto::secret_key &secret_key)
         {

src/device/device_default.cpp

@@ -281,6 +281,10 @@ namespace hw {
             return true;
         }
 
+        void device_default::get_transaction_prefix_hash(const cryptonote::transaction_prefix& tx, crypto::hash& h) {
+            cryptonote::get_transaction_prefix_hash(tx, h);
+        }
+
         bool device_default::generate_output_ephemeral_keys(const size_t tx_version,
                                                             const cryptonote::account_keys &sender_account_keys, const crypto::public_key &txkey_pub,  const crypto::secret_key &tx_key,
                                                             const cryptonote::tx_destination_entry &dst_entr, const boost::optional<cryptonote::account_public_address> &change_addr, const size_t output_index,

src/device/device_default.hpp

@@ -112,6 +112,7 @@ namespace hw {
                                    crypto::signature &sig) override;
 
             bool  open_tx(crypto::secret_key &tx_key) override;
+            void get_transaction_prefix_hash(const cryptonote::transaction_prefix& tx, crypto::hash& h) override;
 
             bool  encrypt_payment_id(crypto::hash8 &payment_id, const crypto::public_key &public_key, const crypto::secret_key &secret_key) override;
 

src/device/device_ledger.cpp

@@ -259,7 +259,7 @@ namespace hw {
 
     static int device_id = 0;
 
-    #define PROTOCOL_VERSION                    2
+    #define PROTOCOL_VERSION                    3
 
     #define INS_NONE                            0x00
     #define INS_RESET                           0x02
@@ -296,6 +296,7 @@ namespace hw {
     #define INS_BLIND                           0x78
     #define INS_UNBLIND                         0x7A
     #define INS_GEN_TXOUT_KEYS                  0x7B
+    #define INS_PREFIX_HASH                     0x7D
     #define INS_VALIDATE                        0x7C
     #define INS_MLSAG                           0x7E
     #define INS_CLOSE_TX                        0x80
@@ -1414,6 +1415,81 @@ namespace hw {
         return true;
     }
 
+    void device_ledger::get_transaction_prefix_hash(const cryptonote::transaction_prefix& tx, crypto::hash& h) {
+      AUTO_LOCK_CMD();
+      
+      int pref_length = 0, pref_offset = 0, offset = 0;
+
+      #ifdef DEBUG_HWDEVICE
+      crypto::hash h_x;
+      this->controle_device->get_transaction_prefix_hash(tx,h_x);
+      MDEBUG("get_transaction_prefix_hash [[IN]] h_x/1 "<<h_x);
+      #endif
+    
+      std::ostringstream s_x;
+      binary_archive<true> a_x(s_x);
+      CHECK_AND_ASSERT_THROW_MES(::serialization::serialize(a_x, const_cast<cryptonote::transaction_prefix&>(tx)),
+                                 "unable to serialize transaction prefix");
+      pref_length = s_x.str().size();
+      //auto pref = std::make_unique<unsigned char[]>(pref_length);
+      auto uprt_pref = std::unique_ptr<unsigned char[]>{ new unsigned char[pref_length] };
+      unsigned char* pref = uprt_pref.get();
+      memmove(pref, s_x.str().data(), pref_length);
+
+      offset = set_command_header_noopt(INS_PREFIX_HASH,1);
+      pref_offset = 0;
+      unsigned char v;
+
+      //version as varint     
+      do {
+        v = pref[pref_offset];
+        this->buffer_send[offset] = v;
+        offset += 1;
+        pref_offset += 1;
+      } while (v&0x80);
+
+      //locktime as var int
+      do {
+        v = pref[pref_offset];
+        this->buffer_send[offset] = v;
+        offset += 1;
+        pref_offset += 1;
+      } while (v&0x80);
+
+      this->buffer_send[4] = offset-5;
+      this->length_send = offset;
+      this->exchange_wait_on_input();
+
+      //hash remains
+      int cnt = 0;
+      while (pref_offset < pref_length) {
+        int len;
+        cnt++;
+        offset = set_command_header(INS_PREFIX_HASH,2,cnt);      
+        len = pref_length - pref_offset;
+        //options
+        if (len > (BUFFER_SEND_SIZE-7)) {
+          len = BUFFER_SEND_SIZE-7;
+          this->buffer_send[offset] = 0x80;
+        } else {
+          this->buffer_send[offset] = 0x00;
+        }
+        offset += 1;
+        //send chunk
+        memmove(&this->buffer_send[offset], pref+pref_offset, len);
+        offset += len;
+        pref_offset += len;
+        this->buffer_send[4] = offset-5;
+        this->length_send = offset;
+        this->exchange();
+      }
+      memmove(h.data, &this->buffer_recv[0], 32);
+      
+      #ifdef DEBUG_HWDEVICE
+      hw::ledger::check8("prefix_hash", "h", h_x.data, h.data);
+      #endif
+    }
+
     bool device_ledger::encrypt_payment_id(crypto::hash8 &payment_id, const crypto::public_key &public_key, const crypto::secret_key &secret_key) {
         AUTO_LOCK_CMD();
 

src/device/device_ledger.hpp

@@ -275,6 +275,8 @@ namespace hw {
         
         bool  open_tx(crypto::secret_key &tx_key) override;
 
+        void get_transaction_prefix_hash(const cryptonote::transaction_prefix& tx, crypto::hash& h) override;
+    
         bool  encrypt_payment_id(crypto::hash8 &payment_id, const crypto::public_key &public_key, const crypto::secret_key &secret_key) override;
 
         rct::key genCommitmentMask(const rct::key &amount_key) override;

src/device_trezor/device_trezor.cpp

@@ -101,7 +101,7 @@ namespace trezor {
       return device_trezor_base::disconnect();
     }
 
-    void device_trezor::device_state_reset_unsafe()
+    void device_trezor::device_state_initialize_unsafe()
     {
       require_connected();
       if (m_live_refresh_in_progress)
@@ -117,7 +117,7 @@ namespace trezor {
       }
 
       m_live_refresh_in_progress = false;
-      device_trezor_base::device_state_reset_unsafe();
+      device_trezor_base::device_state_initialize_unsafe();
     }
 
     void device_trezor::live_refresh_thread_main()
@@ -221,7 +221,7 @@ namespace trezor {
       CHECK_AND_ASSERT_THROW_MES(!payment_id || !subaddress || subaddress->is_zero(), "Subaddress cannot be integrated");
       TREZOR_AUTO_LOCK_CMD();
       require_connected();
-      device_state_reset_unsafe();
+      device_state_initialize_unsafe();
       require_initialized();
 
       auto req = std::make_shared<messages::monero::MoneroGetAddress>();
@@ -245,7 +245,7 @@ namespace trezor {
         const boost::optional<cryptonote::network_type> & network_type){
       TREZOR_AUTO_LOCK_CMD();
       require_connected();
-      device_state_reset_unsafe();
+      device_state_initialize_unsafe();
       require_initialized();
 
       auto req = std::make_shared<messages::monero::MoneroGetWatchKey>();
@@ -274,7 +274,7 @@ namespace trezor {
     {
       TREZOR_AUTO_LOCK_CMD();
       require_connected();
-      device_state_reset_unsafe();
+      device_state_initialize_unsafe();
       require_initialized();
 
       auto req = protocol::tx::get_tx_key(tx_aux_data);
@@ -294,15 +294,15 @@ namespace trezor {
 
       TREZOR_AUTO_LOCK_CMD();
       require_connected();
-      device_state_reset_unsafe();
+      device_state_initialize_unsafe();
       require_initialized();
 
       std::shared_ptr<messages::monero::MoneroKeyImageExportInitRequest> req;
 
       std::vector<protocol::ki::MoneroTransferDetails> mtds;
       std::vector<protocol::ki::MoneroExportedKeyImage> kis;
-      protocol::ki::key_image_data(wallet, transfers, mtds);
+      protocol::ki::key_image_data(wallet, transfers, mtds, client_version() <= 1);
-      protocol::ki::generate_commitment(mtds, transfers, req);
+      protocol::ki::generate_commitment(mtds, transfers, req, client_version() <= 1);
 
       EVENT_PROGRESS(0.);
       this->set_msg_addr<messages::monero::MoneroKeyImageExportInitRequest>(req.get());
@@ -386,7 +386,7 @@ namespace trezor {
 
     void device_trezor::live_refresh_start_unsafe()
     {
-      device_state_reset_unsafe();
+      device_state_initialize_unsafe();
       require_initialized();
 
       auto req = std::make_shared<messages::monero::MoneroLiveRefreshStartRequest>();
@@ -492,7 +492,7 @@ namespace trezor {
 
       TREZOR_AUTO_LOCK_CMD();
       require_connected();
-      device_state_reset_unsafe();
+      device_state_initialize_unsafe();
       require_initialized();
       transaction_versions_check(unsigned_tx, aux_data);
 
@@ -514,7 +514,7 @@ namespace trezor {
         auto & cpend = signed_tx.ptx.back();
         cpend.tx = cdata.tx;
         cpend.dust = 0;
-        cpend.fee = 0;
+        cpend.fee = cpend.tx.rct_signatures.txnFee;
         cpend.dust_added_to_fee = false;
         cpend.change_dts = cdata.tx_data.change_dts;
         cpend.selected_transfers = cdata.tx_data.selected_transfers;
@@ -524,6 +524,7 @@ namespace trezor {
 
         // Transaction check
         try {
+          MDEBUG("signed transaction: " << cryptonote::get_transaction_hash(cpend.tx) << ENDL << cryptonote::obj_to_json_str(cpend.tx) << ENDL);
           transaction_check(cdata, aux_data);
         } catch(const std::exception &e){
           throw exc::ProtocolException(std::string("Transaction verification failed: ") + e.what());
@@ -582,7 +583,7 @@ namespace trezor {
 
       require_connected();
       if (idx > 0)
-        device_state_reset_unsafe();
+        device_state_initialize_unsafe();
 
       require_initialized();
       EVENT_PROGRESS(0, 1, 1);
@@ -670,28 +671,42 @@ namespace trezor {
 #undef EVENT_PROGRESS
     }
 
-    void device_trezor::transaction_versions_check(const ::tools::wallet2::unsigned_tx_set & unsigned_tx, hw::tx_aux_data & aux_data)
+    unsigned device_trezor::client_version()
     {
       auto trezor_version = get_version();
-      unsigned client_version = 1;  // default client version for tx
-
       if (trezor_version <= pack_version(2, 0, 10)){
-        client_version = 0;
+        throw exc::TrezorException("Trezor firmware 2.0.10 and lower are not supported. Please update.");
       }
 
+      // default client version, higher versions check will be added
+      unsigned client_version = 1;
+
+#ifdef WITH_TREZOR_DEBUGGING
+      // Override client version for tests
+      const char *env_trezor_client_version = nullptr;
+      if ((env_trezor_client_version = getenv("TREZOR_CLIENT_VERSION")) != nullptr){
+        auto succ = epee::string_tools::get_xtype_from_string(client_version, env_trezor_client_version);
+        if (succ){
+          MINFO("Trezor client version overriden by TREZOR_CLIENT_VERSION to: " << client_version);
+        }
+      }
+#endif
+      return client_version;
+    }
+
+    void device_trezor::transaction_versions_check(const ::tools::wallet2::unsigned_tx_set & unsigned_tx, hw::tx_aux_data & aux_data)
+    {
+      unsigned cversion = client_version();
+
       if (aux_data.client_version){
         auto wanted_client_version = aux_data.client_version.get();
-        if (wanted_client_version > client_version){
+        if (wanted_client_version > cversion){
-          throw exc::TrezorException("Trezor firmware 2.0.10 and lower does not support current transaction sign protocol. Please update.");
+          throw exc::TrezorException("Trezor has too old firmware version. Please update.");
         } else {
-          client_version = wanted_client_version;
+          cversion = wanted_client_version;
         }
       }
-      aux_data.client_version = client_version;
+      aux_data.client_version = cversion;
-
-      if (client_version == 0 && aux_data.bp_version && aux_data.bp_version.get() != 1){
-        throw exc::TrezorException("Trezor firmware 2.0.10 and lower does not support current transaction sign protocol (BPv2+). Please update.");
-      }
     }
 
     void device_trezor::transaction_pre_check(std::shared_ptr<messages::monero::MoneroTransactionInitRequest> init_msg)

src/device_trezor/device_trezor.hpp

@@ -67,10 +67,11 @@ namespace trezor {
       bool m_live_refresh_enabled;
       size_t m_num_transations_to_sign;
 
+      unsigned client_version();
       void transaction_versions_check(const ::tools::wallet2::unsigned_tx_set & unsigned_tx, hw::tx_aux_data & aux_data);
       void transaction_pre_check(std::shared_ptr<messages::monero::MoneroTransactionInitRequest> init_msg);
       void transaction_check(const protocol::tx::TData & tdata, const hw::tx_aux_data & aux_data);
-      void device_state_reset_unsafe() override;
+      void device_state_initialize_unsafe() override;
       void live_refresh_start_unsafe();
       void live_refresh_finish_unsafe();
       void live_refresh_thread_main();

src/device_trezor/device_trezor_base.cpp

@@ -28,6 +28,7 @@
 //
 
 #include "device_trezor_base.hpp"
+#include "memwipe.h"
 #include <boost/algorithm/string/classification.hpp>
 #include <boost/algorithm/string/split.hpp>
 #include <boost/regex.hpp>
@@ -151,7 +152,7 @@ namespace trezor {
 
     bool device_trezor_base::disconnect() {
       TREZOR_AUTO_LOCK_DEVICE();
-      m_device_state.clear();
+      m_device_session_id.clear();
       m_features.reset();
 
       if (m_transport){
@@ -292,8 +293,8 @@ namespace trezor {
         case messages::MessageType_PassphraseRequest:
           on_passphrase_request(input, dynamic_cast<const messages::common::PassphraseRequest*>(input.m_msg.get()));
           return true;
-        case messages::MessageType_PassphraseStateRequest:
+        case messages::MessageType_Deprecated_PassphraseStateRequest:
-          on_passphrase_state_request(input, dynamic_cast<const messages::common::PassphraseStateRequest*>(input.m_msg.get()));
+          on_passphrase_state_request(input, dynamic_cast<const messages::common::Deprecated_PassphraseStateRequest*>(input.m_msg.get()));
           return true;
         case messages::MessageType_PinMatrixRequest:
           on_pin_request(input, dynamic_cast<const messages::common::PinMatrixRequest*>(input.m_msg.get()));
@@ -361,23 +362,34 @@ namespace trezor {
       return false;
     }
 
-    void device_trezor_base::device_state_reset_unsafe()
+    void device_trezor_base::device_state_initialize_unsafe()
     {
       require_connected();
+      std::string tmp_session_id;
       auto initMsg = std::make_shared<messages::management::Initialize>();
+      const auto data_cleaner = epee::misc_utils::create_scope_leave_handler([&]() {
+        memwipe(&tmp_session_id[0], tmp_session_id.size());
+      });
 
-      if(!m_device_state.empty()) {
+      if(!m_device_session_id.empty()) {
-        initMsg->set_allocated_state(&m_device_state);
+        tmp_session_id.assign(m_device_session_id.data(), m_device_session_id.size());
+        initMsg->set_allocated_session_id(&tmp_session_id);
       }
 
       m_features = this->client_exchange<messages::management::Features>(initMsg);
-      initMsg->release_state();
+      if (m_features->has_session_id()){
+        m_device_session_id = m_features->session_id();
+      } else {
+        m_device_session_id.clear();
+      }
+
+      initMsg->release_session_id();
     }
 
     void device_trezor_base::device_state_reset()
     {
       TREZOR_AUTO_LOCK_CMD();
-      device_state_reset_unsafe();
+      device_state_initialize_unsafe();
     }
 
 #ifdef WITH_TREZOR_DEBUGGING
@@ -441,48 +453,89 @@ namespace trezor {
         pin = m_pin;
       }
 
-      // TODO: remove PIN from memory
+      std::string pin_field;
       messages::common::PinMatrixAck m;
       if (pin) {
-        m.set_pin(pin.get().data(), pin.get().size());
+        pin_field.assign(pin->data(), pin->size());
+        m.set_allocated_pin(&pin_field);
       }
+
+      const auto data_cleaner = epee::misc_utils::create_scope_leave_handler([&]() {
+        m.release_pin();
+        if (!pin_field.empty()){
+          memwipe(&pin_field[0], pin_field.size());
+        }
+      });
+
       resp = call_raw(&m);
     }
 
     void device_trezor_base::on_passphrase_request(GenericMessage & resp, const messages::common::PassphraseRequest * msg)
     {
       CHECK_AND_ASSERT_THROW_MES(msg, "Empty message");
-      MDEBUG("on_passhprase_request, on device: " << msg->on_device());
+      MDEBUG("on_passhprase_request");
-      boost::optional<epee::wipeable_string> passphrase;
-      TREZOR_CALLBACK_GET(passphrase, on_passphrase_request, msg->on_device());
 
-      if (!passphrase && m_passphrase){
+      // Backward compatibility, migration clause.
-        passphrase = m_passphrase;
+      if (msg->has__on_device() && msg->_on_device()){
+        messages::common::PassphraseAck m;
+        resp = call_raw(&m);
+        return;
       }
 
-      m_passphrase = boost::none;
+      bool on_device = true;
+      if (msg->has__on_device() && !msg->_on_device()){
+        on_device = false;  // do not enter on device, old devices.
+      }
 
-      messages::common::PassphraseAck m;
+      if (on_device && m_features && m_features->capabilities_size() > 0){
-      if (!msg->on_device() && passphrase){
+        on_device = false;
-        // TODO: remove passphrase from memory
+        for (auto it = m_features->capabilities().begin(); it != m_features->capabilities().end(); it++) {
-        m.set_passphrase(passphrase.get().data(), passphrase.get().size());
+          if (*it == messages::management::Features::Capability_PassphraseEntry){
+            on_device = true;
+          }
+        }
       }
 
-      if (!m_device_state.empty()){
+      boost::optional<epee::wipeable_string> passphrase;
-        m.set_allocated_state(&m_device_state);
+      TREZOR_CALLBACK_GET(passphrase, on_passphrase_request, on_device);
+
+      std::string passphrase_field;
+      messages::common::PassphraseAck m;
+      m.set_on_device(on_device);
+      if (!on_device) {
+        if (!passphrase && m_passphrase) {
+          passphrase = m_passphrase;
+        }
+
+        if (m_passphrase) {
+          m_passphrase = boost::none;
+        }
+
+        if (passphrase) {
+          passphrase_field.assign(passphrase->data(), passphrase->size());
+          m.set_allocated_passphrase(&passphrase_field);
+        }
       }
 
+      const auto data_cleaner = epee::misc_utils::create_scope_leave_handler([&]() {
+        m.release_passphrase();
+        if (!passphrase_field.empty()){
+          memwipe(&passphrase_field[0], passphrase_field.size());
+        }
+      });
+
       resp = call_raw(&m);
-      m.release_state();
     }
 
-    void device_trezor_base::on_passphrase_state_request(GenericMessage & resp, const messages::common::PassphraseStateRequest * msg)
+    void device_trezor_base::on_passphrase_state_request(GenericMessage & resp, const messages::common::Deprecated_PassphraseStateRequest * msg)
     {
       MDEBUG("on_passhprase_state_request");
       CHECK_AND_ASSERT_THROW_MES(msg, "Empty message");
 
-      m_device_state = msg->state();
+      if (msg->has_state()) {
-      messages::common::PassphraseStateAck m;
+        m_device_session_id = msg->state();
+      }
+      messages::common::Deprecated_PassphraseStateAck m;
       resp = call_raw(&m);
     }
 
@@ -510,7 +563,7 @@ namespace trezor {
       }
 
       auto msg = std::make_shared<messages::management::LoadDevice>();
-      msg->set_mnemonic(mnemonic);
+      msg->add_mnemonics(mnemonic);
       msg->set_pin(pin);
       msg->set_passphrase_protection(passphrase_protection);
       msg->set_label(label);
@@ -535,7 +588,8 @@ namespace trezor {
       return boost::none;
     }
 
-    boost::optional<epee::wipeable_string> trezor_debug_callback::on_passphrase_request(bool on_device) {
+    boost::optional<epee::wipeable_string> trezor_debug_callback::on_passphrase_request(bool & on_device) {
+      on_device = true;
       return boost::none;
     }
 

src/device_trezor/device_trezor_base.hpp

@@ -70,7 +70,7 @@ namespace trezor {
 
       void on_button_request(uint64_t code=0) override;
       boost::optional<epee::wipeable_string> on_pin_request() override;
-      boost::optional<epee::wipeable_string> on_passphrase_request(bool on_device) override;
+      boost::optional<epee::wipeable_string> on_passphrase_request(bool & on_device) override;
       void on_passphrase_state_request(const std::string &state);
       void on_disconnect();
     protected:
@@ -94,7 +94,7 @@ namespace trezor {
 
       std::string m_full_name;
       std::vector<unsigned int> m_wallet_deriv_path;
-      std::string m_device_state;  // returned after passphrase entry, session
+      epee::wipeable_string m_device_session_id;  // returned after passphrase entry, session
       std::shared_ptr<messages::management::Features> m_features;  // features from the last device reset
       boost::optional<epee::wipeable_string> m_pin;
       boost::optional<epee::wipeable_string> m_passphrase;
@@ -117,7 +117,7 @@ namespace trezor {
       void require_initialized() const;
       void call_ping_unsafe();
       void test_ping();
-      virtual void device_state_reset_unsafe();
+      virtual void device_state_initialize_unsafe();
       void ensure_derivation_path() noexcept;
 
       // Communication methods
@@ -315,7 +315,7 @@ namespace trezor {
     void on_button_pressed();
     void on_pin_request(GenericMessage & resp, const messages::common::PinMatrixRequest * msg);
     void on_passphrase_request(GenericMessage & resp, const messages::common::PassphraseRequest * msg);
-    void on_passphrase_state_request(GenericMessage & resp, const messages::common::PassphraseStateRequest * msg);
+    void on_passphrase_state_request(GenericMessage & resp, const messages::common::Deprecated_PassphraseStateRequest * msg);
 
 #ifdef WITH_TREZOR_DEBUGGING
     void set_debug(bool debug){

src/device_trezor/trezor/debug_link.cpp

@@ -71,9 +71,9 @@ namespace trezor{
     call(decision, boost::none, true);
   }
 
-  void DebugLink::input_swipe(bool swipe){
+  void DebugLink::input_swipe(messages::debug::DebugLinkDecision_DebugSwipeDirection direction){
     messages::debug::DebugLinkDecision decision;
-    decision.set_up_down(swipe);
+    decision.set_swipe(direction);
     call(decision, boost::none, true);
   }
 

src/device_trezor/trezor/debug_link.hpp

@@ -49,7 +49,7 @@ namespace trezor {
     std::shared_ptr<messages::debug::DebugLinkState> state();
     void input_word(const std::string & word);
     void input_button(bool button);
-    void input_swipe(bool swipe);
+    void input_swipe(messages::debug::DebugLinkDecision_DebugSwipeDirection direction);
     void press_yes() { input_button(true); }
     void press_no() { input_button(false); }
     void stop();

src/device_trezor/trezor/protocol.cpp

@@ -145,7 +145,8 @@ namespace ki {
 
   bool key_image_data(wallet_shim * wallet,
                       const std::vector<tools::wallet2::transfer_details> & transfers,
-                      std::vector<MoneroTransferDetails> & res)
+                      std::vector<MoneroTransferDetails> & res,
+                      bool need_all_additionals)
   {
     for(auto & td : transfers){
       ::crypto::public_key tx_pub_key = wallet->get_tx_pub_key_from_received_outs(td);
@@ -157,8 +158,14 @@ namespace ki {
       cres.set_out_key(key_to_string(boost::get<cryptonote::txout_to_key>(td.m_tx.vout[td.m_internal_output_index].target).key));
       cres.set_tx_pub_key(key_to_string(tx_pub_key));
       cres.set_internal_output_index(td.m_internal_output_index);
-      for(auto & aux : additional_tx_pub_keys){
+      cres.set_sub_addr_major(td.m_subaddr_index.major);
-        cres.add_additional_tx_pub_keys(key_to_string(aux));
+      cres.set_sub_addr_minor(td.m_subaddr_index.minor);
+      if (need_all_additionals) {
+        for (auto &aux : additional_tx_pub_keys) {
+          cres.add_additional_tx_pub_keys(key_to_string(aux));
+        }
+      } else if (!additional_tx_pub_keys.empty() && additional_tx_pub_keys.size() > td.m_internal_output_index) {
+        cres.add_additional_tx_pub_keys(key_to_string(additional_tx_pub_keys[td.m_internal_output_index]));
       }
     }
 
@@ -188,7 +195,8 @@ namespace ki {
 
   void generate_commitment(std::vector<MoneroTransferDetails> & mtds,
                            const std::vector<tools::wallet2::transfer_details> & transfers,
-                           std::shared_ptr<messages::monero::MoneroKeyImageExportInitRequest> & req)
+                           std::shared_ptr<messages::monero::MoneroKeyImageExportInitRequest> & req,
+                           bool need_subaddr_indices)
   {
     req = std::make_shared<messages::monero::MoneroKeyImageExportInitRequest>();
 
@@ -213,11 +221,13 @@ namespace ki {
       st.insert(cur.m_subaddr_index.minor);
     }
 
-    for (auto& x: sub_indices){
+    if (need_subaddr_indices) {
-      auto subs = req->add_subs();
+      for (auto &x: sub_indices) {
-      subs->set_account(x.first);
+        auto subs = req->add_subs();
-      for(auto minor : x.second){
+        subs->set_account(x.first);
-        subs->add_minor_indices(minor);
+        for (auto minor : x.second) {
+          subs->add_minor_indices(minor);
+        }
       }
     }
   }
@@ -283,26 +293,6 @@ namespace tx {
     translate_address(dst->mutable_addr(), &(src->addr));
   }
 
-  void translate_src_entry(MoneroTransactionSourceEntry * dst, const cryptonote::tx_source_entry * src){
-    for(auto & cur : src->outputs){
-      auto out = dst->add_outputs();
-      out->set_idx(cur.first);
-      translate_rct_key(out->mutable_key(), &(cur.second));
-    }
-
-    dst->set_real_output(src->real_output);
-    dst->set_real_out_tx_key(key_to_string(src->real_out_tx_key));
-    for(auto & cur : src->real_out_additional_tx_keys){
-      dst->add_real_out_additional_tx_keys(key_to_string(cur));
-    }
-
-    dst->set_real_output_in_tx_index(src->real_output_in_tx_index);
-    dst->set_amount(src->amount);
-    dst->set_rct(src->rct);
-    dst->set_mask(key_to_string(src->mask));
-    translate_klrki(dst->mutable_multisig_klrki(), &(src->multisig_kLRki));
-  }
-
   void translate_klrki(MoneroMultisigKLRki * dst, const rct::multisig_kLRki * src){
     dst->set_k(key_to_string(src->k));
     dst->set_l(key_to_string(src->L));
@@ -369,6 +359,31 @@ namespace tx {
     return res;
   }
 
+  std::string compute_sealing_key(const std::string & master_key, size_t idx, bool is_iv)
+  {
+    // master-key-32B || domain-sep-12B || index-4B
+    uint8_t hash[32] = {0};
+    KECCAK_CTX ctx;
+    std::string sep = is_iv ? "sig-iv" : "sig-key";
+    std::string idx_data = tools::get_varint_data(idx);
+    if (idx_data.size() > 4){
+      throw std::invalid_argument("index is too big");
+    }
+
+    keccak_init(&ctx);
+    keccak_update(&ctx, (const uint8_t *) master_key.data(), master_key.size());
+    keccak_update(&ctx, (const uint8_t *) sep.data(), sep.size());
+    keccak_update(&ctx, hash, 12 - sep.size());
+    keccak_update(&ctx, (const uint8_t *) idx_data.data(), idx_data.size());
+    if (idx_data.size() < 4) {
+      keccak_update(&ctx, hash, 4 - idx_data.size());
+    }
+
+    keccak_finish(&ctx, hash);
+    keccak(hash, sizeof(hash), hash, sizeof(hash));
+    return std::string((const char*) hash, 32);
+  }
+
   TData::TData() {
     rsig_type = 0;
     bp_version = 0;
@@ -383,7 +398,7 @@ namespace tx {
     m_unsigned_tx = unsigned_tx;
     m_aux_data = aux_data;
     m_tx_idx = tx_idx;
-    m_ct.tx_data = cur_tx();
+    m_ct.tx_data = cur_src_tx();
     m_multisig = false;
     m_client_version = 1;
   }
@@ -451,6 +466,41 @@ namespace tx {
     }
   }
 
+  void Signer::set_tx_input(MoneroTransactionSourceEntry * dst, size_t idx, bool need_ring_keys, bool need_ring_indices){
+    const cryptonote::tx_source_entry & src = cur_tx().sources[idx];
+    const tools::wallet2::transfer_details & transfer = get_source_transfer(idx);
+
+    dst->set_real_output(src.real_output);
+    for(size_t i = 0; i < src.outputs.size(); ++i){
+      auto & cur = src.outputs[i];
+      auto out = dst->add_outputs();
+
+      if (i == src.real_output || need_ring_indices || client_version() <= 1) {
+        out->set_idx(cur.first);
+      }
+      if (i == src.real_output || need_ring_keys || client_version() <= 1) {
+        translate_rct_key(out->mutable_key(), &(cur.second));
+      }
+    }
+
+    dst->set_real_out_tx_key(key_to_string(src.real_out_tx_key));
+    dst->set_real_output_in_tx_index(src.real_output_in_tx_index);
+
+    if (client_version() <= 1) {
+      for (auto &cur : src.real_out_additional_tx_keys) {
+        dst->add_real_out_additional_tx_keys(key_to_string(cur));
+      }
+    } else if (!src.real_out_additional_tx_keys.empty()) {
+      dst->add_real_out_additional_tx_keys(key_to_string(src.real_out_additional_tx_keys.at(src.real_output_in_tx_index)));
+    }
+
+    dst->set_amount(src.amount);
+    dst->set_rct(src.rct);
+    dst->set_mask(key_to_string(src.mask));
+    translate_klrki(dst->mutable_multisig_klrki(), &(src.multisig_kLRki));
+    dst->set_subaddr_minor(transfer.m_subaddr_index.minor);
+  }
+
   void Signer::compute_integrated_indices(TsxData * tsx_data){
     if (m_aux_data == nullptr || m_aux_data->tx_recipients.empty()){
       return;
@@ -492,6 +542,7 @@ namespace tx {
     // extract payment ID from construction data
     auto & tsx_data = m_ct.tsx_data;
     auto & tx = cur_tx();
+    const size_t input_size = tx.sources.size();
 
     m_ct.tx.version = 2;
     m_ct.tx.unlock_time = tx.unlock_time;
@@ -500,12 +551,20 @@ namespace tx {
     tsx_data.set_version(1);
     tsx_data.set_client_version(client_version());
     tsx_data.set_unlock_time(tx.unlock_time);
-    tsx_data.set_num_inputs(static_cast<google::protobuf::uint32>(tx.sources.size()));
+    tsx_data.set_num_inputs(static_cast<google::protobuf::uint32>(input_size));
     tsx_data.set_mixin(static_cast<google::protobuf::uint32>(tx.sources[0].outputs.size() - 1));
     tsx_data.set_account(tx.subaddr_account);
     tsx_data.set_monero_version(std::string(MONERO_VERSION) + "|" + MONERO_VERSION_TAG);
     tsx_data.set_hard_fork(m_aux_data->hard_fork ? m_aux_data->hard_fork.get() : 0);
-    assign_to_repeatable(tsx_data.mutable_minor_indices(), tx.subaddr_indices.begin(), tx.subaddr_indices.end());
+
+    if (client_version() <= 1){
+      assign_to_repeatable(tsx_data.mutable_minor_indices(), tx.subaddr_indices.begin(), tx.subaddr_indices.end());
+    }
+
+    // TODO: use HF_VERSION_CLSAG after CLSAG is merged
+    if (tsx_data.hard_fork() >= 13){
+      throw exc::ProtocolException("CLSAG is not yet implemented");
+    }
 
     // Rsig decision
     auto rsig_data = tsx_data.mutable_rsig_data();
@@ -525,6 +584,11 @@ namespace tx {
       translate_dst_entry(dst, &cur);
     }
 
+    m_ct.source_permutation.clear();
+    for (size_t n = 0; n < input_size; ++n){
+      m_ct.source_permutation.push_back(n);
+    }
+
     compute_integrated_indices(&tsx_data);
 
     int64_t fee = 0;
@@ -559,7 +623,7 @@ namespace tx {
     CHECK_AND_ASSERT_THROW_MES(idx < cur_tx().sources.size(), "Invalid source index");
     m_ct.cur_input_idx = idx;
     auto res = std::make_shared<messages::monero::MoneroTransactionSetInputRequest>();
-    translate_src_entry(res->mutable_src_entr(), &(cur_tx().sources[idx]));
+    set_tx_input(res->mutable_src_entr(), idx, false, true);
     return res;
   }
 
@@ -582,11 +646,6 @@ namespace tx {
   void Signer::sort_ki(){
     const size_t input_size = cur_tx().sources.size();
 
-    m_ct.source_permutation.clear();
-    for (size_t n = 0; n < input_size; ++n){
-      m_ct.source_permutation.push_back(n);
-    }
-
     CHECK_AND_ASSERT_THROW_MES(m_ct.tx.vin.size() == input_size, "Invalid vector size");
     std::sort(m_ct.source_permutation.begin(), m_ct.source_permutation.end(), [&](const size_t i0, const size_t i1) {
       const cryptonote::txin_to_key &tk0 = boost::get<cryptonote::txin_to_key>(m_ct.tx.vin[i0]);
@@ -614,6 +673,9 @@ namespace tx {
 
   std::shared_ptr<messages::monero::MoneroTransactionInputsPermutationRequest> Signer::step_permutation(){
     sort_ki();
+    if (client_version() >= 2){
+      return nullptr;
+    }
 
     auto res = std::make_shared<messages::monero::MoneroTransactionInputsPermutationRequest>();
     assign_to_repeatable(res->mutable_perm(), m_ct.source_permutation.begin(), m_ct.source_permutation.end());
@@ -634,17 +696,10 @@ namespace tx {
     auto tx = m_ct.tx_data;
     auto res = std::make_shared<messages::monero::MoneroTransactionInputViniRequest>();
     auto & vini = m_ct.tx.vin[idx];
-    translate_src_entry(res->mutable_src_entr(), &(tx.sources[idx]));
+    set_tx_input(res->mutable_src_entr(), idx, false, false);
     res->set_vini(cryptonote::t_serializable_object_to_blob(vini));
     res->set_vini_hmac(m_ct.tx_in_hmacs[idx]);
-
+    res->set_orig_idx(m_ct.source_permutation[idx]);
-    if (client_version() == 0) {
-      CHECK_AND_ASSERT_THROW_MES(idx < m_ct.pseudo_outs.size(), "Invalid transaction index");
-      CHECK_AND_ASSERT_THROW_MES(idx < m_ct.pseudo_outs_hmac.size(), "Invalid transaction index");
-      res->set_pseudo_out(m_ct.pseudo_outs[idx]);
-      res->set_pseudo_out_hmac(m_ct.pseudo_outs_hmac[idx]);
-    }
-
     return res;
   }
 
@@ -657,31 +712,6 @@ namespace tx {
   }
 
   void Signer::step_all_inputs_set_ack(std::shared_ptr<const messages::monero::MoneroTransactionAllInputsSetAck> ack){
-    if (client_version() > 0 || !is_offloading()){
-      return;
-    }
-
-    // If offloading, expect rsig configuration.
-    if (!ack->has_rsig_data()){
-      throw exc::ProtocolException("Rsig offloading requires rsig param");
-    }
-
-    auto & rsig_data = ack->rsig_data();
-    if (!rsig_data.has_mask()){
-      throw exc::ProtocolException("Gamma masks not present in offloaded version");
-    }
-
-    auto & mask = rsig_data.mask();
-    if (mask.size() != 32 * num_outputs()){
-      throw exc::ProtocolException("Invalid number of gamma masks");
-    }
-
-    m_ct.rsig_gamma.reserve(num_outputs());
-    for(size_t c=0; c < num_outputs(); ++c){
-      rct::key cmask{};
-      memcpy(cmask.bytes, mask.data() + c * 32, 32);
-      m_ct.rsig_gamma.emplace_back(cmask);
-    }
   }
 
   std::shared_ptr<messages::monero::MoneroTransactionSetOutputRequest> Signer::step_set_output(size_t idx){
@@ -696,15 +726,6 @@ namespace tx {
     auto & cur_dst = m_ct.tx_data.splitted_dsts[idx];
     translate_dst_entry(res->mutable_dst_entr(), &cur_dst);
     res->set_dst_entr_hmac(m_ct.tx_out_entr_hmacs[idx]);
-
-    // Range sig offloading to the host
-    // ClientV0 sends offloaded BP with the last message in the batch.
-    // ClientV1 needs additional message after the last message in the batch as BP uses deterministic masks.
-    if (client_version() == 0 && is_offloading() && should_compute_bp_now()) {
-      auto rsig_data = res->mutable_rsig_data();
-      compute_bproof(*rsig_data);
-    }
-
     return res;
   }
 
@@ -814,7 +835,7 @@ namespace tx {
   }
 
   std::shared_ptr<messages::monero::MoneroTransactionSetOutputRequest> Signer::step_rsig(size_t idx){
-    if (client_version() == 0 || !is_offloading() || !should_compute_bp_now()){
+    if (!is_offloading() || !should_compute_bp_now()){
       return nullptr;
     }
 
@@ -917,11 +938,12 @@ namespace tx {
     CHECK_AND_ASSERT_THROW_MES(idx < m_ct.spend_encs.size(), "Invalid transaction index");
 
     auto res = std::make_shared<messages::monero::MoneroTransactionSignInputRequest>();
-    translate_src_entry(res->mutable_src_entr(), &(m_ct.tx_data.sources[idx]));
+    set_tx_input(res->mutable_src_entr(), idx, true, true);
     res->set_vini(cryptonote::t_serializable_object_to_blob(m_ct.tx.vin[idx]));
     res->set_vini_hmac(m_ct.tx_in_hmacs[idx]);
     res->set_pseudo_out_alpha(m_ct.alphas[idx]);
     res->set_spend_key(m_ct.spend_encs[idx]);
+    res->set_orig_idx(m_ct.source_permutation[idx]);
 
     CHECK_AND_ASSERT_THROW_MES(idx < m_ct.pseudo_outs.size(), "Invalid transaction index");
     CHECK_AND_ASSERT_THROW_MES(idx < m_ct.pseudo_outs_hmac.size(), "Invalid transaction index");
@@ -931,10 +953,7 @@ namespace tx {
   }
 
   void Signer::step_sign_input_ack(std::shared_ptr<const messages::monero::MoneroTransactionSignInputAck> ack){
-    rct::mgSig mg;
+    m_ct.signatures.push_back(ack->signature());
-    if (!cn_deserialize(ack->signature(), mg)){
-      throw exc::ProtocolException("Cannot deserialize mg[i]");
-    }
 
     // Sync updated pseudo_outputs, client_version>=1, HF10+
     if (client_version() >= 1 && ack->has_pseudo_out()){
@@ -948,12 +967,9 @@ namespace tx {
         string_to_key(m_ct.rv->pseudoOuts[m_ct.cur_input_idx], ack->pseudo_out());
       }
     }
-
-    m_ct.rv->p.MGs.push_back(mg);
   }
 
   std::shared_ptr<messages::monero::MoneroTransactionFinalRequest> Signer::step_final(){
-    m_ct.tx.rct_signatures = *(m_ct.rv);
     return std::make_shared<messages::monero::MoneroTransactionFinalRequest>();
   }
 
@@ -976,6 +992,42 @@ namespace tx {
     m_ct.enc_salt1 = ack->salt();
     m_ct.enc_salt2 = ack->rand_mult();
     m_ct.enc_keys = ack->tx_enc_keys();
+
+    // Opening the sealed signatures
+    if (client_version() >= 3){
+      if(!ack->has_opening_key()){
+        throw exc::ProtocolException("Client version 3+ requires sealed signatures");
+      }
+
+      for(size_t i = 0; i < m_ct.signatures.size(); ++i){
+        CHECK_AND_ASSERT_THROW_MES(m_ct.signatures[i].size() > crypto::chacha::TAG_SIZE, "Invalid signature size");
+        std::string nonce = compute_sealing_key(ack->opening_key(), i, true);
+        std::string key = compute_sealing_key(ack->opening_key(), i, false);
+        size_t plen = m_ct.signatures[i].size() - crypto::chacha::TAG_SIZE;
+        std::unique_ptr<uint8_t[]> plaintext(new uint8_t[plen]);
+        uint8_t * buff = plaintext.get();
+
+        protocol::crypto::chacha::decrypt(
+            m_ct.signatures[i].data(),
+            m_ct.signatures[i].size(),
+            reinterpret_cast<const uint8_t *>(key.data()),
+            reinterpret_cast<const uint8_t *>(nonce.data()),
+            reinterpret_cast<char *>(buff), &plen);
+        m_ct.signatures[i].assign(reinterpret_cast<const char *>(buff), plen);
+      }
+    }
+
+    // CLSAG support comes here once it is merged to the Monero
+    m_ct.rv->p.MGs.reserve(m_ct.signatures.size());
+    for(size_t i = 0; i < m_ct.signatures.size(); ++i) {
+      rct::mgSig mg;
+      if (!cn_deserialize(m_ct.signatures[i], mg)) {
+        throw exc::ProtocolException("Cannot deserialize mg[i]");
+      }
+      m_ct.rv->p.MGs.push_back(mg);
+    }
+
+    m_ct.tx.rct_signatures = *(m_ct.rv);
   }
 
   std::string Signer::store_tx_aux_info(){

src/device_trezor/trezor/protocol.hpp

@@ -118,7 +118,8 @@ namespace ki {
    */
   bool key_image_data(wallet_shim * wallet,
                       const std::vector<tools::wallet2::transfer_details> & transfers,
-                      std::vector<MoneroTransferDetails> & res);
+                      std::vector<MoneroTransferDetails> & res,
+                      bool need_all_additionals=false);
 
   /**
    * Computes a hash over MoneroTransferDetails. Commitment used in the KI sync.
@@ -130,7 +131,8 @@ namespace ki {
    */
   void generate_commitment(std::vector<MoneroTransferDetails> & mtds,
                            const std::vector<tools::wallet2::transfer_details> & transfers,
-                           std::shared_ptr<messages::monero::MoneroKeyImageExportInitRequest> & req);
+                           std::shared_ptr<messages::monero::MoneroKeyImageExportInitRequest> & req,
+                           bool need_subaddr_indices=false);
 
   /**
    * Processes Live refresh step response, parses KI, checks the signature
@@ -158,13 +160,13 @@ namespace tx {
 
   void translate_address(MoneroAccountPublicAddress * dst, const cryptonote::account_public_address * src);
   void translate_dst_entry(MoneroTransactionDestinationEntry * dst, const cryptonote::tx_destination_entry * src);
-  void translate_src_entry(MoneroTransactionSourceEntry * dst, const cryptonote::tx_source_entry * src);
   void translate_klrki(MoneroMultisigKLRki * dst, const rct::multisig_kLRki * src);
   void translate_rct_key(MoneroRctKey * dst, const rct::ctkey * src);
   std::string hash_addr(const MoneroAccountPublicAddress * addr, boost::optional<uint64_t> amount = boost::none, boost::optional<bool> is_subaddr = boost::none);
   std::string hash_addr(const std::string & spend_key, const std::string & view_key, boost::optional<uint64_t> amount = boost::none, boost::optional<bool> is_subaddr = boost::none);
   std::string hash_addr(const ::crypto::public_key * spend_key, const ::crypto::public_key * view_key, boost::optional<uint64_t> amount = boost::none, boost::optional<bool> is_subaddr = boost::none);
   ::crypto::secret_key compute_enc_key(const ::crypto::secret_key & private_view_key, const std::string & aux, const std::string & salt);
+  std::string compute_sealing_key(const std::string & master_key, size_t idx, bool is_iv=false);
 
   typedef boost::variant<rct::rangeSig, rct::Bulletproof> rsig_v;
 
@@ -198,6 +200,7 @@ namespace tx {
     std::vector<std::string> pseudo_outs_hmac;
     std::vector<std::string> couts;
     std::vector<std::string> couts_dec;
+    std::vector<std::string> signatures;
     std::vector<rct::key> rsig_gamma;
     std::string tx_prefix_hash;
     std::string enc_salt1;
@@ -221,16 +224,33 @@ namespace tx {
     unsigned m_client_version;
     bool m_multisig;
 
-    const tx_construction_data & cur_tx(){
+    const tx_construction_data & cur_src_tx() const {
       CHECK_AND_ASSERT_THROW_MES(m_tx_idx < m_unsigned_tx->txes.size(), "Invalid transaction index");
       return m_unsigned_tx->txes[m_tx_idx];
     }
 
+    const tx_construction_data & cur_tx() const {
+      return m_ct.tx_data;
+    }
+
+    const tools::wallet2::transfer_details & get_transfer(size_t idx) const {
+      CHECK_AND_ASSERT_THROW_MES(idx < m_unsigned_tx->transfers.second.size() + m_unsigned_tx->transfers.first && idx >= m_unsigned_tx->transfers.first, "Invalid transfer index");
+      return m_unsigned_tx->transfers.second[idx - m_unsigned_tx->transfers.first];
+    }
+
+    const tools::wallet2::transfer_details & get_source_transfer(size_t idx) const {
+      const auto & sel_transfers = cur_tx().selected_transfers;
+      CHECK_AND_ASSERT_THROW_MES(idx < m_ct.source_permutation.size(), "Invalid source index - permutation");
+      CHECK_AND_ASSERT_THROW_MES(m_ct.source_permutation[idx] < sel_transfers.size(), "Invalid source index");
+      return get_transfer(sel_transfers.at(m_ct.source_permutation[idx]));
+    }
+
     void extract_payment_id();
     void compute_integrated_indices(TsxData * tsx_data);
     bool should_compute_bp_now() const;
     void compute_bproof(messages::monero::MoneroTransactionRsigData & rsig_data);
     void process_bproof(rct::Bulletproof & bproof);
+    void set_tx_input(MoneroTransactionSourceEntry * dst, size_t idx, bool need_ring_keys=false, bool need_ring_indices=false);
 
   public:
     Signer(wallet_shim * wallet2, const unsigned_tx_set * unsigned_tx, size_t tx_idx = 0, hw::tx_aux_data * aux_data = nullptr);

src/device_trezor/trezor/transport.cpp

@@ -56,6 +56,11 @@ namespace trezor{
     return true;
   }
 
+  bool t_serialize(const epee::wipeable_string & in, std::string & out){
+    out.assign(in.data(), in.size());
+    return true;
+  }
+
   bool t_serialize(const json_val & in, std::string & out){
     rapidjson::StringBuffer sb;
     rapidjson::Writer<rapidjson::StringBuffer> writer(sb);
@@ -75,6 +80,11 @@ namespace trezor{
     return true;
   }
 
+  bool t_deserialize(std::string & in, epee::wipeable_string & out){
+    out = epee::wipeable_string(in);
+    return true;
+  }
+
   bool t_deserialize(const std::string & in, json & out){
     if (out.Parse(in.c_str()).HasParseError()) {
       throw exc::CommunicationException("JSON parse error");
@@ -192,61 +202,69 @@ namespace trezor{
     const auto msg_size = message_size(req);
     const auto buff_size = serialize_message_buffer_size(msg_size) + 2;
 
-    std::unique_ptr<uint8_t[]> req_buff(new uint8_t[buff_size]);
+    epee::wipeable_string req_buff;
-    uint8_t * req_buff_raw = req_buff.get();
+    epee::wipeable_string chunk_buff;
+
+    req_buff.resize(buff_size);
+    chunk_buff.resize(REPLEN);
+
+    uint8_t * req_buff_raw = reinterpret_cast<uint8_t *>(req_buff.data());
+    uint8_t * chunk_buff_raw = reinterpret_cast<uint8_t *>(chunk_buff.data());
+
     req_buff_raw[0] = '#';
     req_buff_raw[1] = '#';
 
     serialize_message(req, msg_size, req_buff_raw + 2, buff_size - 2);
 
     size_t offset = 0;
-    uint8_t chunk_buff[REPLEN];
 
     // Chunk by chunk upload
     while(offset < buff_size){
       auto to_copy = std::min((size_t)(buff_size - offset), (size_t)(REPLEN - 1));
 
-      chunk_buff[0] = '?';
+      chunk_buff_raw[0] = '?';
-      memcpy(chunk_buff + 1, req_buff_raw + offset, to_copy);
+      memcpy(chunk_buff_raw + 1, req_buff_raw + offset, to_copy);
 
       // Pad with zeros
       if (to_copy < REPLEN - 1){
-        memset(chunk_buff + 1 + to_copy, 0, REPLEN - 1 - to_copy);
+        memset(chunk_buff_raw + 1 + to_copy, 0, REPLEN - 1 - to_copy);
       }
 
-      transport.write_chunk(chunk_buff, REPLEN);
+      transport.write_chunk(chunk_buff_raw, REPLEN);
       offset += REPLEN - 1;
     }
   }
 
   void ProtocolV1::read(Transport & transport, std::shared_ptr<google::protobuf::Message> & msg, messages::MessageType * msg_type){
-    char chunk[REPLEN];
+    epee::wipeable_string chunk_buff;
+    chunk_buff.resize(REPLEN);
+    char * chunk_buff_raw = chunk_buff.data();
 
     // Initial chunk read
-    size_t nread = transport.read_chunk(chunk, REPLEN);
+    size_t nread = transport.read_chunk(chunk_buff_raw, REPLEN);
     if (nread != REPLEN){
       throw exc::CommunicationException("Read chunk has invalid size");
     }
 
-    if (strncmp(chunk, "?##", 3) != 0){
+    if (memcmp(chunk_buff_raw, "?##", 3) != 0){
       throw exc::CommunicationException("Malformed chunk");
     }
 
     uint16_t tag;
     uint32_t len;
     nread -= 3 + 6;
-    deserialize_message_header(chunk + 3, tag, len);
+    deserialize_message_header(chunk_buff_raw + 3, tag, len);
 
-    std::string data_acc(chunk + 3 + 6, nread);
+    epee::wipeable_string data_acc(chunk_buff_raw + 3 + 6, nread);
     data_acc.reserve(len);
 
     while(nread < len){
-      const size_t cur = transport.read_chunk(chunk, REPLEN);
+      const size_t cur = transport.read_chunk(chunk_buff_raw, REPLEN);
-      if (chunk[0] != '?'){
+      if (chunk_buff_raw[0] != '?'){
         throw exc::CommunicationException("Chunk malformed");
       }
 
-      data_acc.append(chunk + 1, cur - 1);
+      data_acc.append(chunk_buff_raw + 1, cur - 1);
       nread += cur - 1;
     }
 
@@ -259,7 +277,7 @@ namespace trezor{
     }
 
     std::shared_ptr<google::protobuf::Message> msg_wrap(MessageMapper::get_message(tag));
-    if (!msg_wrap->ParseFromArray(data_acc.c_str(), len)){
+    if (!msg_wrap->ParseFromArray(data_acc.data(), len)){
       throw exc::CommunicationException("Message could not be parsed");
     }
 
@@ -426,15 +444,16 @@ namespace trezor{
 
     const auto msg_size = message_size(req);
     const auto buff_size = serialize_message_buffer_size(msg_size);
+    epee::wipeable_string req_buff;
+    req_buff.resize(buff_size);
 
-    std::unique_ptr<uint8_t[]> req_buff(new uint8_t[buff_size]);
+    uint8_t * req_buff_raw = reinterpret_cast<uint8_t *>(req_buff.data());
-    uint8_t * req_buff_raw = req_buff.get();
 
     serialize_message(req, msg_size, req_buff_raw, buff_size);
 
     std::string uri = "/call/" + m_session.get();
-    std::string req_hex = epee::to_hex::string(epee::span<const std::uint8_t>(req_buff_raw, buff_size));
+    epee::wipeable_string res_hex;
-    std::string res_hex;
+    epee::wipeable_string req_hex = epee::to_hex::wipeable_string(epee::span<const std::uint8_t>(req_buff_raw, buff_size));
 
     bool req_status = invoke_bridge_http(uri, req_hex, res_hex, m_http_client);
     if (!req_status){
@@ -449,15 +468,15 @@ namespace trezor{
       throw exc::CommunicationException("Could not read, no response stored");
     }
 
-    std::string bin_data;
+    boost::optional<epee::wipeable_string> bin_data = m_response->parse_hexstr();
-    if (!epee::string_tools::parse_hexstr_to_binbuff(m_response.get(), bin_data)){
+    if (!bin_data){
       throw exc::CommunicationException("Response is not well hexcoded");
     }
 
     uint16_t msg_tag;
     uint32_t msg_len;
-    deserialize_message_header(bin_data.c_str(), msg_tag, msg_len);
+    deserialize_message_header(bin_data->data(), msg_tag, msg_len);
-    if (bin_data.size() != msg_len + 6){
+    if (bin_data->size() != msg_len + 6){
       throw exc::CommunicationException("Response is not well hexcoded");
     }
 
@@ -466,7 +485,7 @@ namespace trezor{
     }
 
     std::shared_ptr<google::protobuf::Message> msg_wrap(MessageMapper::get_message(msg_tag));
-    if (!msg_wrap->ParseFromArray(bin_data.c_str() + 6, msg_len)){
+    if (!msg_wrap->ParseFromArray(bin_data->data() + 6, msg_len)){
       throw exc::EncodingException("Response is not well hexcoded");
     }
     msg = msg_wrap;

src/device_trezor/trezor/transport.hpp

@@ -66,10 +66,12 @@ namespace trezor {
 
   // Base HTTP comm serialization.
   bool t_serialize(const std::string & in, std::string & out);
+  bool t_serialize(const epee::wipeable_string & in, std::string & out);
   bool t_serialize(const json_val & in, std::string & out);
   std::string t_serialize(const json_val & in);
 
   bool t_deserialize(const std::string & in, std::string & out);
+  bool t_deserialize(std::string & in, epee::wipeable_string & out);
   bool t_deserialize(const std::string & in, json & out);
 
   // Flexible json serialization. HTTP client tailored for bridge API
@@ -84,6 +86,13 @@ namespace trezor {
     additional_params.push_back(std::make_pair("Content-Type","application/json; charset=utf-8"));
 
     const http::http_response_info* pri = nullptr;
+    const auto data_cleaner = epee::misc_utils::create_scope_leave_handler([&]() {
+      if (!req_param.empty()) {
+        memwipe(&req_param[0], req_param.size());
+      }
+      transport.wipe_response();
+    });
+
     if(!transport.invoke(uri, method, req_param, timeout, &pri, std::move(additional_params)))
     {
       MERROR("Failed to invoke http request to  " << uri);
@@ -103,7 +112,7 @@ namespace trezor {
       return false;
     }
 
-    return t_deserialize(pri->m_body, result_struct);
+    return t_deserialize(const_cast<http::http_response_info*>(pri)->m_body, result_struct);
   }
 
   // Forward decl
@@ -186,7 +195,7 @@ namespace trezor {
     std::string m_bridge_host;
     boost::optional<std::string> m_device_path;
     boost::optional<std::string> m_session;
-    boost::optional<std::string> m_response;
+    boost::optional<epee::wipeable_string> m_response;
     boost::optional<json> m_device_info;
   };
 

src/net/zmq.cpp

@@ -33,6 +33,8 @@
 #include <limits>
 #include <utility>
 
+#include "byte_slice.h"
+
 namespace net
 {
 namespace zmq
@@ -183,6 +185,22 @@ namespace zmq
     {
         return retry_op(zmq_send, socket, payload.data(), payload.size(), flags);
     }
+
+    expect<void> send(epee::byte_slice&& payload, void* socket, int flags) noexcept
+    {
+        void* const data = const_cast<std::uint8_t*>(payload.data());
+        const std::size_t size = payload.size();
+        auto buffer = payload.take_buffer(); // clears `payload` from callee
+
+        zmq_msg_t msg{};
+        MONERO_ZMQ_CHECK(zmq_msg_init_data(std::addressof(msg), data, size, epee::release_byte_slice::call, buffer.get()));
+        buffer.release(); // zmq will now decrement byte_slice ref-count
+
+        expect<void> sent = retry_op(zmq_msg_send, std::addressof(msg), socket, flags);
+        if (!sent) // beware if removing `noexcept` from this function - possible leak here
+            zmq_msg_close(std::addressof(msg));
+        return sent;
+    }
 } // zmq
 } // net
 

src/net/zmq.h

@@ -53,6 +53,11 @@
 #define MONERO_ZMQ_THROW(msg)                         \
     MONERO_THROW( ::net::zmq::get_error_code(), msg )
 
+namespace epee
+{
+    class byte_slice;
+}
+
 namespace net
 {
 namespace zmq
@@ -132,5 +137,24 @@ namespace zmq
         \param flags See `zmq_send` for possible flags.
         \return `success()` if sent, otherwise ZMQ error. */
     expect<void> send(epee::span<const std::uint8_t> payload, void* socket, int flags = 0) noexcept;
+
+    /*! Sends `payload` on `socket`. Blocks until the entire message is queued
+        for sending, or until `zmq_term` is called on the `zmq_context`
+        associated with `socket`. If the context is terminated,
+        `make_error_code(ETERM)` is returned.
+
+        \note This will automatically retry on `EINTR`, so exiting on
+            interrupts requires context termination.
+        \note If non-blocking behavior is requested on `socket` or by `flags`,
+            then `net::zmq::make_error_code(EAGAIN)` will be returned if this
+            would block.
+
+        \param payload sent as one message on `socket`.
+        \param socket Handle created with `zmq_socket`.
+        \param flags See `zmq_msg_send` for possible flags.
+
+        \post `payload.emtpy()` - ownership is transferred to zmq.
+        \return `success()` if sent, otherwise ZMQ error. */
+    expect<void> send(epee::byte_slice&& payload, void* socket, int flags = 0) noexcept;
 } // zmq
 } // net

src/ringct/bulletproofs.cc

@@ -905,7 +905,7 @@ bool bulletproof_VERIFY(const std::vector<const Bulletproof*> &proofs)
   rct::key m_y0 = rct::zero(), y1 = rct::zero();
   int proof_data_index = 0;
   rct::keyV w_cache;
-  rct::keyV proof8_V, proof8_L, proof8_R;
+  std::vector<ge_p3> proof8_V, proof8_L, proof8_R;
   for (const Bulletproof *p: proofs)
   {
     const Bulletproof &proof = *p;
@@ -918,13 +918,17 @@ bool bulletproof_VERIFY(const std::vector<const Bulletproof*> &proofs)
     const rct::key weight_z = rct::skGen();
 
     // pre-multiply some points by 8
-    proof8_V.resize(proof.V.size()); for (size_t i = 0; i < proof.V.size(); ++i) proof8_V[i] = rct::scalarmult8(proof.V[i]);
+    proof8_V.resize(proof.V.size()); for (size_t i = 0; i < proof.V.size(); ++i) rct::scalarmult8(proof8_V[i], proof.V[i]);
-    proof8_L.resize(proof.L.size()); for (size_t i = 0; i < proof.L.size(); ++i) proof8_L[i] = rct::scalarmult8(proof.L[i]);
+    proof8_L.resize(proof.L.size()); for (size_t i = 0; i < proof.L.size(); ++i) rct::scalarmult8(proof8_L[i], proof.L[i]);
-    proof8_R.resize(proof.R.size()); for (size_t i = 0; i < proof.R.size(); ++i) proof8_R[i] = rct::scalarmult8(proof.R[i]);
+    proof8_R.resize(proof.R.size()); for (size_t i = 0; i < proof.R.size(); ++i) rct::scalarmult8(proof8_R[i], proof.R[i]);
-    rct::key proof8_T1 = rct::scalarmult8(proof.T1);
+    ge_p3 proof8_T1;
-    rct::key proof8_T2 = rct::scalarmult8(proof.T2);
+    ge_p3 proof8_T2;
-    rct::key proof8_S = rct::scalarmult8(proof.S);
+    ge_p3 proof8_S;
-    rct::key proof8_A = rct::scalarmult8(proof.A);
+    ge_p3 proof8_A;
+    rct::scalarmult8(proof8_T1, proof.T1);
+    rct::scalarmult8(proof8_T2, proof.T2);
+    rct::scalarmult8(proof8_S, proof.S);
+    rct::scalarmult8(proof8_A, proof.A);
 
     PERF_TIMER_START_BP(VERIFY_line_61);
     sc_mulsub(m_y0.bytes, proof.taux.bytes, weight_y.bytes, m_y0.bytes);

src/ringct/rctOps.cpp

@@ -408,6 +408,18 @@ namespace rct {
         return res;
     }
 
+    //Computes 8P without byte conversion
+    void scalarmult8(ge_p3 &res, const key &P)
+    {
+        ge_p3 p3;
+        CHECK_AND_ASSERT_THROW_MES_L1(ge_frombytes_vartime(&p3, P.bytes) == 0, "ge_frombytes_vartime failed at "+boost::lexical_cast<std::string>(__LINE__));
+        ge_p2 p2;
+        ge_p3_to_p2(&p2, &p3);
+        ge_p1p1 p1;
+        ge_mul8(&p1, &p2);
+        ge_p1p1_to_p3(&res, &p1);
+    }
+
     //Computes lA where l is the curve order
     bool isInMainSubgroup(const key & A) {
         ge_p3 p3;

src/ringct/rctOps.h

@@ -124,6 +124,7 @@ namespace rct {
     key scalarmultH(const key & a);
     // multiplies a point by 8
     key scalarmult8(const key & P);
+    void scalarmult8(ge_p3 &res, const key & P);
     // checks a is in the main subgroup (ie, not a small one)
     bool isInMainSubgroup(const key & a);
 

src/rpc/bootstrap_node_selector.h

@@ -54,6 +54,8 @@ namespace bootstrap_node
 
   struct selector
   {
+    virtual ~selector() = default;
+
     virtual void handle_result(const std::string &address, bool success) = 0;
     virtual boost::optional<node_info> next_node() = 0;
   };

src/rpc/core_rpc_server.cpp

@@ -224,7 +224,7 @@ namespace cryptonote
     }
     else if (address == "auto")
     {
-      auto get_nodes = [this, credits_per_hash_threshold]() {
+      auto get_nodes = [this]() {
         return get_public_nodes(credits_per_hash_threshold);
       };
       m_bootstrap_daemon.reset(new bootstrap_daemon(std::move(get_nodes), rpc_payment_enabled));
@@ -438,7 +438,7 @@ namespace cryptonote
     store_difficulty(m_core.get_blockchain_storage().get_difficulty_for_next_block(), res.difficulty, res.wide_difficulty, res.difficulty_top64);
     res.target = m_core.get_blockchain_storage().get_difficulty_target();
     res.tx_count = m_core.get_blockchain_storage().get_total_transactions() - res.height; //without coinbase
-    res.tx_pool_size = m_core.get_pool_transactions_count();
+    res.tx_pool_size = m_core.get_pool_transactions_count(!restricted);
     res.alt_blocks_count = restricted ? 0 : m_core.get_blockchain_storage().get_alternative_blocks_count();
     uint64_t total_conn = restricted ? 0 : m_p2p.get_public_connections_count();
     res.outgoing_connections_count = restricted ? 0 : m_p2p.get_public_outgoing_connections_count();
@@ -1119,6 +1119,8 @@ namespace cryptonote
     }
     res.sanity_check_failed = false;
 
+    const bool restricted = m_restricted && ctx;
+
     tx_verification_context tvc{};
     if(!m_core.handle_incoming_tx({tx_blob, crypto::null_hash}, tvc, (req.do_not_relay ? relay_method::none : relay_method::local), false) || tvc.m_verifivation_failed)
     {
@@ -1423,12 +1425,13 @@ namespace cryptonote
 
     const bool restricted = m_restricted && ctx;
     const bool request_has_rpc_origin = ctx != NULL;
+    const bool allow_sensitive = !request_has_rpc_origin || !restricted;
 
-    size_t n_txes = m_core.get_pool_transactions_count();
+    size_t n_txes = m_core.get_pool_transactions_count(allow_sensitive);
     if (n_txes > 0)
     {
       CHECK_PAYMENT_SAME_TS(req, res, n_txes * COST_PER_TX);
-      m_core.get_pool_transactions_and_spent_keys_info(res.transactions, res.spent_key_images, !request_has_rpc_origin || !restricted);
+      m_core.get_pool_transactions_and_spent_keys_info(res.transactions, res.spent_key_images, allow_sensitive);
       for (tx_info& txi : res.transactions)
         txi.tx_blob = epee::string_tools::buff_to_hex_nodelimer(txi.tx_blob);
     }
@@ -1448,12 +1451,13 @@ namespace cryptonote
 
     const bool restricted = m_restricted && ctx;
     const bool request_has_rpc_origin = ctx != NULL;
+    const bool allow_sensitive = !request_has_rpc_origin || !restricted;
 
-    size_t n_txes = m_core.get_pool_transactions_count();
+    size_t n_txes = m_core.get_pool_transactions_count(allow_sensitive);
     if (n_txes > 0)
     {
       CHECK_PAYMENT_SAME_TS(req, res, n_txes * COST_PER_POOL_HASH);
-      m_core.get_pool_transaction_hashes(res.tx_hashes, !request_has_rpc_origin || !restricted);
+      m_core.get_pool_transaction_hashes(res.tx_hashes, allow_sensitive);
     }
 
     res.status = CORE_RPC_STATUS_OK;
@@ -1471,13 +1475,14 @@ namespace cryptonote
 
     const bool restricted = m_restricted && ctx;
     const bool request_has_rpc_origin = ctx != NULL;
+    const bool allow_sensitive = !request_has_rpc_origin || !restricted;
 
-    size_t n_txes = m_core.get_pool_transactions_count();
+    size_t n_txes = m_core.get_pool_transactions_count(allow_sensitive);
     if (n_txes > 0)
     {
       CHECK_PAYMENT_SAME_TS(req, res, n_txes * COST_PER_POOL_HASH);
       std::vector<crypto::hash> tx_hashes;
-      m_core.get_pool_transaction_hashes(tx_hashes, !request_has_rpc_origin || !restricted);
+      m_core.get_pool_transaction_hashes(tx_hashes, allow_sensitive);
       res.tx_hashes.reserve(tx_hashes.size());
       for (const crypto::hash &tx_hash: tx_hashes)
         res.tx_hashes.push_back(epee::string_tools::pod_to_hex(tx_hash));

src/rpc/daemon_handler.cpp

@@ -33,6 +33,7 @@
 #include <stdexcept>
 
 #include <boost/uuid/nil_generator.hpp>
+#include <boost/utility/string_ref.hpp>
 // likely included by daemon_handler.h's includes,
 // but including here for clarity
 #include "cryptonote_core/cryptonote_core.h"
@@ -48,7 +49,7 @@ namespace rpc
 {
   namespace
   {
-    using handler_function = std::string(DaemonHandler& handler, const rapidjson::Value& id, const rapidjson::Value& msg);
+    using handler_function = epee::byte_slice(DaemonHandler& handler, const rapidjson::Value& id, const rapidjson::Value& msg);
     struct handler_map
     {
       const char* method_name;
@@ -66,7 +67,7 @@ namespace rpc
     }
 
     template<typename Message>
-    std::string handle_message(DaemonHandler& handler, const rapidjson::Value& id, const rapidjson::Value& parameters)
+    epee::byte_slice handle_message(DaemonHandler& handler, const rapidjson::Value& id, const rapidjson::Value& parameters)
     {
       typename Message::Request request{};
       request.fromJson(parameters);
@@ -903,7 +904,7 @@ namespace rpc
     return true;
   }
 
-  std::string DaemonHandler::handle(const std::string& request)
+  epee::byte_slice DaemonHandler::handle(const std::string& request)
   {
     MDEBUG("Handling RPC request: " << request);
 
@@ -916,8 +917,11 @@ namespace rpc
       if (matched_handler == std::end(handlers) || matched_handler->method_name != request_type)
         return BAD_REQUEST(request_type, req_full.getID());
 
-      std::string response = matched_handler->call(*this, req_full.getID(), req_full.getMessage());
+      epee::byte_slice response = matched_handler->call(*this, req_full.getID(), req_full.getMessage());
-      MDEBUG("Returning RPC response: " << response);
+
+      const boost::string_ref response_view{reinterpret_cast<const char*>(response.data()), response.size()};
+      MDEBUG("Returning RPC response: " << response_view);
+
       return response;
     }
     catch (const std::exception& e)

src/rpc/daemon_handler.h

@@ -28,6 +28,7 @@
 
 #pragma once
 
+#include "byte_slice.h"
 #include "daemon_messages.h"
 #include "daemon_rpc_version.h"
 #include "rpc_handler.h"
@@ -132,7 +133,7 @@ class DaemonHandler : public RpcHandler
 
     void handle(const GetOutputDistribution::Request& req, GetOutputDistribution::Response& res);
 
-    std::string handle(const std::string& request);
+    epee::byte_slice handle(const std::string& request) override final;
 
   private:
 

src/rpc/message.cpp

@@ -149,7 +149,7 @@ cryptonote::rpc::error FullMessage::getError()
   return err;
 }
 
-std::string FullMessage::getRequest(const std::string& request, const Message& message, const unsigned id)
+epee::byte_slice FullMessage::getRequest(const std::string& request, const Message& message, const unsigned id)
 {
   rapidjson::StringBuffer buffer;
   {
@@ -172,11 +172,11 @@ std::string FullMessage::getRequest(const std::string& request, const Message& m
     if (!dest.IsComplete())
       throw std::logic_error{"Invalid JSON tree generated"};
   }
-  return std::string{buffer.GetString(), buffer.GetSize()};
+  return epee::byte_slice{{buffer.GetString(), buffer.GetSize()}};
 }
 
 
-std::string FullMessage::getResponse(const Message& message, const rapidjson::Value& id)
+epee::byte_slice FullMessage::getResponse(const Message& message, const rapidjson::Value& id)
 {
   rapidjson::StringBuffer buffer;
   {
@@ -207,17 +207,17 @@ std::string FullMessage::getResponse(const Message& message, const rapidjson::Va
     if (!dest.IsComplete())
       throw std::logic_error{"Invalid JSON tree generated"};
   }
-  return std::string{buffer.GetString(), buffer.GetSize()};
+  return epee::byte_slice{{buffer.GetString(), buffer.GetSize()}};
 }
 
 // convenience functions for bad input
-std::string BAD_REQUEST(const std::string& request)
+epee::byte_slice BAD_REQUEST(const std::string& request)
 {
   rapidjson::Value invalid;
   return BAD_REQUEST(request, invalid);
 }
 
-std::string BAD_REQUEST(const std::string& request, const rapidjson::Value& id)
+epee::byte_slice BAD_REQUEST(const std::string& request, const rapidjson::Value& id)
 {
   Message fail;
   fail.status = Message::STATUS_BAD_REQUEST;
@@ -225,7 +225,7 @@ std::string BAD_REQUEST(const std::string& request, const rapidjson::Value& id)
   return FullMessage::getResponse(fail, id);
 }
 
-std::string BAD_JSON(const std::string& error_details)
+epee::byte_slice BAD_JSON(const std::string& error_details)
 {
   rapidjson::Value invalid;
   Message fail;

src/rpc/message.h

@@ -33,6 +33,7 @@
 #include <rapidjson/writer.h>
 #include <string>
 
+#include "byte_slice.h"
 #include "rpc/message_data_structs.h"
 
 namespace cryptonote
@@ -85,8 +86,8 @@ namespace rpc
 
       cryptonote::rpc::error getError();
 
-      static std::string getRequest(const std::string& request, const Message& message, unsigned id);
+      static epee::byte_slice getRequest(const std::string& request, const Message& message, unsigned id);
-      static std::string getResponse(const Message& message, const rapidjson::Value& id);
+      static epee::byte_slice getResponse(const Message& message, const rapidjson::Value& id);
     private:
 
       FullMessage() = default;
@@ -99,10 +100,10 @@ namespace rpc
 
 
   // convenience functions for bad input
-  std::string BAD_REQUEST(const std::string& request);
+  epee::byte_slice BAD_REQUEST(const std::string& request);
-  std::string BAD_REQUEST(const std::string& request, const rapidjson::Value& id);
+  epee::byte_slice BAD_REQUEST(const std::string& request, const rapidjson::Value& id);
 
-  std::string BAD_JSON(const std::string& error_details);
+  epee::byte_slice BAD_JSON(const std::string& error_details);
 
 
 }  // namespace rpc

src/rpc/rpc_handler.h

@@ -32,6 +32,7 @@
 #include <cstdint>
 #include <string>
 #include <vector>
+#include "byte_slice.h"
 #include "crypto/hash.h"
 
 namespace cryptonote
@@ -54,7 +55,7 @@ class RpcHandler
     RpcHandler() { }
     virtual ~RpcHandler() { }
 
-    virtual std::string handle(const std::string& request) = 0;
+    virtual epee::byte_slice handle(const std::string& request) = 0;
 
     static boost::optional<output_distribution_data>
       get_output_distribution(const std::function<bool(uint64_t, uint64_t, uint64_t, uint64_t&, std::vector<uint64_t>&, uint64_t&)> &f, uint64_t amount, uint64_t from_height, uint64_t to_height, const std::function<crypto::hash(uint64_t)> &get_hash, bool cumulative, uint64_t blockchain_height);

src/rpc/zmq_server.cpp

@@ -28,10 +28,13 @@
 
 #include "zmq_server.h"
 
+#include <boost/utility/string_ref.hpp>
 #include <chrono>
 #include <cstdint>
 #include <system_error>
 
+#include "byte_slice.h"
+
 namespace cryptonote
 {
 
@@ -73,10 +76,11 @@ void ZmqServer::serve()
     {
       const std::string message = MONERO_UNWRAP(net::zmq::receive(socket.get()));
       MDEBUG("Received RPC request: \"" << message << "\"");
-      const std::string& response = handler.handle(message);
+      epee::byte_slice response = handler.handle(message);
 
-      MONERO_UNWRAP(net::zmq::send(epee::strspan<std::uint8_t>(response), socket.get()));
+      const boost::string_ref response_view{reinterpret_cast<const char*>(response.data()), response.size()};
-      MDEBUG("Sent RPC reply: \"" << response << "\"");
+      MDEBUG("Sending RPC reply: \"" << response_view << "\"");
+      MONERO_UNWRAP(net::zmq::send(std::move(response), socket.get()));
     }
   }
   catch (const std::system_error& e)

src/simplewallet/simplewallet.cpp

@@ -182,6 +182,7 @@ namespace
   const char* USAGE_LOCKED_TRANSFER("locked_transfer [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] (<URI> | <addr> <amount>) <lockblocks> [<payment_id (obsolete)>]");
   const char* USAGE_LOCKED_SWEEP_ALL("locked_sweep_all [index=<N1>[,<N2>,...] | index=all] [<priority>] [<ring_size>] <address> <lockblocks> [<payment_id (obsolete)>]");
   const char* USAGE_SWEEP_ALL("sweep_all [index=<N1>[,<N2>,...] | index=all] [<priority>] [<ring_size>] [outputs=<N>] <address> [<payment_id (obsolete)>]");
+  const char* USAGE_SWEEP_ACCOUNT("sweep_account <account> [index=<N1>[,<N2>,...] | index=all] [<priority>] [<ring_size>] [outputs=<N>] <address> [<payment_id (obsolete)>]");
   const char* USAGE_SWEEP_BELOW("sweep_below <amount_threshold> [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] <address> [<payment_id (obsolete)>]");
   const char* USAGE_SWEEP_SINGLE("sweep_single [<priority>] [<ring_size>] [outputs=<N>] <key_image> <address> [<payment_id (obsolete)>]");
   const char* USAGE_DONATE("donate [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] <amount> [<payment_id (obsolete)>]");
@@ -207,7 +208,7 @@ namespace
   const char* USAGE_CHECK_SPEND_PROOF("check_spend_proof <txid> <signature_file> [<message>]");
   const char* USAGE_GET_RESERVE_PROOF("get_reserve_proof (all|<amount>) [<message>]");
   const char* USAGE_CHECK_RESERVE_PROOF("check_reserve_proof <address> <signature_file> [<message>]");
-  const char* USAGE_SHOW_TRANSFERS("show_transfers [in|out|pending|failed|pool|coinbase] [index=<N1>[,<N2>,...]] [<min_height> [<max_height>]]");
+  const char* USAGE_SHOW_TRANSFERS("show_transfers [in|out|all|pending|failed|pool|coinbase] [index=<N1>[,<N2>,...]] [<min_height> [<max_height>]]");
   const char* USAGE_UNSPENT_OUTPUTS("unspent_outputs [index=<N1>[,<N2>,...]] [<min_amount> [<max_amount>]]");
   const char* USAGE_RESCAN_BC("rescan_bc [hard|soft|keep_ki] [start_height=0]");
   const char* USAGE_SET_TX_NOTE("set_tx_note <txid> [free text note]");
@@ -3145,6 +3146,9 @@ simple_wallet::simple_wallet()
   m_cmd_binder.set_handler("sweep_all", boost::bind(&simple_wallet::sweep_all, this, _1),
                            tr(USAGE_SWEEP_ALL),
                            tr("Send all unlocked balance to an address. If the parameter \"index=<N1>[,<N2>,...]\" or \"index=all\" is specified, the wallet sweeps outputs received by those or all address indices, respectively. If omitted, the wallet randomly chooses an address index to be used. If the parameter \"outputs=<N>\" is specified and  N > 0, wallet splits the transaction into N even outputs."));
+  m_cmd_binder.set_handler("sweep_account", boost::bind(&simple_wallet::sweep_account, this, _1),
+                           tr(USAGE_SWEEP_ACCOUNT),
+                           tr("Send all unlocked balance from a given account to an address. If the parameter \"index=<N1>[,<N2>,...]\" or \"index=all\" is specified, the wallet sweeps outputs received by those or all address indices, respectively. If omitted, the wallet randomly chooses an address index to be used. If the parameter \"outputs=<N>\" is specified and  N > 0, wallet splits the transaction into N even outputs."));
   m_cmd_binder.set_handler("sweep_below",
                            boost::bind(&simple_wallet::on_command, this, &simple_wallet::sweep_below, _1),
                            tr(USAGE_SWEEP_BELOW),
@@ -3338,7 +3342,7 @@ simple_wallet::simple_wallet()
                               "** Set of address indices used as inputs in this transfer."));
   m_cmd_binder.set_handler("export_transfers",
                            boost::bind(&simple_wallet::on_command, this, &simple_wallet::export_transfers, _1),
-                           tr("export_transfers [in|out|all|pending|failed|coinbase] [index=<N1>[,<N2>,...]] [<min_height> [<max_height>]] [output=<filepath>]"),
+                           tr("export_transfers [in|out|all|pending|failed|pool|coinbase] [index=<N1>[,<N2>,...]] [<min_height> [<max_height>]] [output=<filepath>]"),
                            tr("Export to CSV the incoming/outgoing transfers within an optional height range."));
   m_cmd_binder.set_handler("unspent_outputs",
                            boost::bind(&simple_wallet::on_command, this, &simple_wallet::unspent_outputs, _1),
@@ -5576,14 +5580,19 @@ boost::optional<epee::wipeable_string> simple_wallet::on_device_pin_request()
   return pwd_container->password();
 }
 //----------------------------------------------------------------------------------------------------
-boost::optional<epee::wipeable_string> simple_wallet::on_device_passphrase_request(bool on_device)
+boost::optional<epee::wipeable_string> simple_wallet::on_device_passphrase_request(bool & on_device)
 {
-  if (on_device){
+  if (on_device) {
-    message_writer(console_color_white, true) << tr("Please enter the device passphrase on the device");
+    std::string accepted = input_line(tr(
-    return boost::none;
+        "Device asks for passphrase. Do you want to enter the passphrase on device (Y) (or on the host (N))?"));
+    if (std::cin.eof() || command_line::is_yes(accepted)) {
+      message_writer(console_color_white, true) << tr("Please enter the device passphrase on the device");
+      return boost::none;
+    }
   }
 
   PAUSE_READLINE();
+  on_device = false;
   std::string msg = tr("Enter device passphrase");
   auto pwd_container = tools::password_container::prompt(false, msg.c_str());
   THROW_WALLET_EXCEPTION_IF(!pwd_container, tools::error::password_entry_failed, tr("Failed to read device passphrase"));
@@ -6221,6 +6230,7 @@ bool simple_wallet::prompt_if_old(const std::vector<tools::wallet2::pending_tx>
   }
   return true;
 }
+//----------------------------------------------------------------------------------------------------
 void simple_wallet::check_for_inactivity_lock(bool user)
 {
   if (m_locked)
@@ -6758,7 +6768,7 @@ bool simple_wallet::locked_transfer(const std::vector<std::string> &args_)
 //----------------------------------------------------------------------------------------------------
 bool simple_wallet::locked_sweep_all(const std::vector<std::string> &args_)
 {
-  sweep_main(0, true, args_);
+  sweep_main(m_current_subaddress_account, 0, true, args_);
   return true;
 }
 //----------------------------------------------------------------------------------------------------
@@ -6869,18 +6879,22 @@ bool simple_wallet::sweep_unmixable(const std::vector<std::string> &args_)
   return true;
 }
 //----------------------------------------------------------------------------------------------------
-bool simple_wallet::sweep_main(uint64_t below, bool locked, const std::vector<std::string> &args_)
+bool simple_wallet::sweep_main(uint32_t account, uint64_t below, bool locked, const std::vector<std::string> &args_)
 {
-  auto print_usage = [below]()
+  auto print_usage = [this, account, below]()
   {
     if (below)
     {
       PRINT_USAGE(USAGE_SWEEP_BELOW);
     }
-    else
+    else if (account == m_current_subaddress_account)
     {
       PRINT_USAGE(USAGE_SWEEP_ALL);
     }
+    else
+    {
+      PRINT_USAGE(USAGE_SWEEP_ACCOUNT);
+    }
   };
   if (args_.size() == 0)
   {
@@ -7054,7 +7068,7 @@ bool simple_wallet::sweep_main(uint64_t below, bool locked, const std::vector<st
   try
   {
     // figure out what tx will be necessary
-    auto ptx_vector = m_wallet->create_transactions_all(below, info.address, info.is_subaddress, outputs, fake_outs_count, unlock_block /* unlock_time */, priority, extra, m_current_subaddress_account, subaddr_indices);
+    auto ptx_vector = m_wallet->create_transactions_all(below, info.address, info.is_subaddress, outputs, fake_outs_count, unlock_block /* unlock_time */, priority, extra, account, subaddr_indices);
 
     if (ptx_vector.empty())
     {
@@ -7394,7 +7408,27 @@ bool simple_wallet::sweep_single(const std::vector<std::string> &args_)
 //----------------------------------------------------------------------------------------------------
 bool simple_wallet::sweep_all(const std::vector<std::string> &args_)
 {
-  sweep_main(0, false, args_);
+  sweep_main(m_current_subaddress_account, 0, false, args_);
+  return true;
+}
+//----------------------------------------------------------------------------------------------------
+bool simple_wallet::sweep_account(const std::vector<std::string> &args_)
+{
+  auto local_args = args_;
+  if (local_args.empty())
+  {
+    PRINT_USAGE(USAGE_SWEEP_ACCOUNT);
+    return true;
+  }
+  uint32_t account = 0;
+  if (!epee::string_tools::get_xtype_from_string(account, local_args[0]))
+  {
+    fail_msg_writer() << tr("Invalid account");
+    return true;
+  }
+  local_args.erase(local_args.begin());
+
+  sweep_main(account, 0, false, local_args);
   return true;
 }
 //----------------------------------------------------------------------------------------------------
@@ -7411,7 +7445,7 @@ bool simple_wallet::sweep_below(const std::vector<std::string> &args_)
     fail_msg_writer() << tr("invalid amount threshold");
     return true;
   }
-  sweep_main(below, false, std::vector<std::string>(++args_.begin(), args_.end()));
+  sweep_main(m_current_subaddress_account, below, false, std::vector<std::string>(++args_.begin(), args_.end()));
   return true;
 }
 //----------------------------------------------------------------------------------------------------
@@ -8508,7 +8542,7 @@ bool simple_wallet::show_transfers(const std::vector<std::string> &args_)
   std::vector<std::string> local_args = args_;
 
   if(local_args.size() > 4) {
-    fail_msg_writer() << tr("usage: show_transfers [in|out|all|pending|failed|coinbase] [index=<N1>[,<N2>,...]] [<min_height> [<max_height>]]");
+    fail_msg_writer() << tr("usage: show_transfers [in|out|all|pending|failed|pool|coinbase] [index=<N1>[,<N2>,...]] [<min_height> [<max_height>]]");
     return true;
   }
 
@@ -8561,7 +8595,7 @@ bool simple_wallet::export_transfers(const std::vector<std::string>& args_)
   std::vector<std::string> local_args = args_;
 
   if(local_args.size() > 5) {
-    fail_msg_writer() << tr("usage: export_transfers [in|out|all|pending|failed|coinbase] [index=<N1>[,<N2>,...]] [<min_height> [<max_height>]] [output=<path>]");
+    fail_msg_writer() << tr("usage: export_transfers [in|out|all|pending|failed|pool|coinbase] [index=<N1>[,<N2>,...]] [<min_height> [<max_height>]] [output=<path>]");
     return true;
   }
 

src/simplewallet/simplewallet.h

@@ -170,8 +170,9 @@ namespace cryptonote
     bool transfer(const std::vector<std::string> &args);
     bool locked_transfer(const std::vector<std::string> &args);
     bool locked_sweep_all(const std::vector<std::string> &args);
-    bool sweep_main(uint64_t below, bool locked, const std::vector<std::string> &args);
+    bool sweep_main(uint32_t account, uint64_t below, bool locked, const std::vector<std::string> &args);
     bool sweep_all(const std::vector<std::string> &args);
+    bool sweep_account(const std::vector<std::string> &args);
     bool sweep_below(const std::vector<std::string> &args);
     bool sweep_single(const std::vector<std::string> &args);
     bool sweep_unmixable(const std::vector<std::string> &args);
@@ -348,7 +349,7 @@ namespace cryptonote
     virtual boost::optional<epee::wipeable_string> on_get_password(const char *reason);
     virtual void on_device_button_request(uint64_t code);
     virtual boost::optional<epee::wipeable_string> on_device_pin_request();
-    virtual boost::optional<epee::wipeable_string> on_device_passphrase_request(bool on_device);
+    virtual boost::optional<epee::wipeable_string> on_device_passphrase_request(bool & on_device);
     //----------------------------------------------------------
 
     friend class refresh_progress_reporter_t;

src/wallet/api/wallet.cpp

@@ -267,13 +267,15 @@ struct Wallet2CallbackImpl : public tools::i_wallet2_callback
       return boost::none;
     }
 
-    virtual boost::optional<epee::wipeable_string> on_device_passphrase_request(bool on_device)
+    virtual boost::optional<epee::wipeable_string> on_device_passphrase_request(bool & on_device)
     {
       if (m_listener) {
         auto passphrase = m_listener->onDevicePassphraseRequest(on_device);
-        if (!on_device && passphrase) {
+        if (passphrase) {
           return boost::make_optional(epee::wipeable_string((*passphrase).data(), (*passphrase).size()));
         }
+      } else {
+        on_device = true;
       }
       return boost::none;
     }

src/wallet/api/wallet2_api.h

@@ -400,8 +400,8 @@ struct WalletListener
     /**
      * @brief called by device when passphrase entry is needed
      */
-    virtual optional<std::string> onDevicePassphraseRequest(bool on_device) {
+    virtual optional<std::string> onDevicePassphraseRequest(bool & on_device) {
-        if (!on_device) throw std::runtime_error("Not supported");
+        on_device = true;
         return optional<std::string>();
     }
 
@@ -1293,7 +1293,11 @@ struct WalletManager
     virtual std::string resolveOpenAlias(const std::string &address, bool &dnssec_valid) const = 0;
 
     //! checks for an update and returns version, hash and url
-    static std::tuple<bool, std::string, std::string, std::string, std::string> checkUpdates(const std::string &software, std::string subdir);
+    static std::tuple<bool, std::string, std::string, std::string, std::string> checkUpdates(
+        const std::string &software,
+        std::string subdir,
+        const char *buildtag = nullptr,
+        const char *current_version = nullptr);
 };
 
 

src/wallet/api/wallet_manager.cpp

@@ -342,22 +342,30 @@ std::string WalletManagerImpl::resolveOpenAlias(const std::string &address, bool
     return addresses.front();
 }
 
-std::tuple<bool, std::string, std::string, std::string, std::string> WalletManager::checkUpdates(const std::string &software, std::string subdir)
+std::tuple<bool, std::string, std::string, std::string, std::string> WalletManager::checkUpdates(
+    const std::string &software,
+    std::string subdir,
+    const char *buildtag/* = nullptr*/,
+    const char *current_version/* = nullptr*/)
 {
+    if (buildtag == nullptr)
+    {
 #ifdef BUILD_TAG
-    static const char buildtag[] = BOOST_PP_STRINGIZE(BUILD_TAG);
+        static const char buildtag_default[] = BOOST_PP_STRINGIZE(BUILD_TAG);
 #else
-    static const char buildtag[] = "source";
+        static const char buildtag_default[] = "source";
-    // Override the subdir string when built from source
+        // Override the subdir string when built from source
-    subdir = "source";
+        subdir = "source";
 #endif
+        buildtag = buildtag_default;
+    }
 
     std::string version, hash;
     MDEBUG("Checking for a new " << software << " version for " << buildtag);
     if (!tools::check_updates(software, buildtag, version, hash))
       return std::make_tuple(false, "", "", "", "");
 
-    if (tools::vercmp(version.c_str(), MONERO_VERSION) > 0)
+    if (tools::vercmp(version.c_str(), current_version != nullptr ? current_version : MONERO_VERSION) > 0)
     {
       std::string user_url = tools::get_update_url(software, subdir, buildtag, version, true);
       std::string auto_url = tools::get_update_url(software, subdir, buildtag, version, false);

src/wallet/message_store.cpp

@@ -48,7 +48,7 @@
 namespace mms
 {
 
-message_store::message_store()
+message_store::message_store(std::unique_ptr<epee::net_utils::http::abstract_http_client> http_client) : m_transporter(std::move(http_client))
 {
   m_active = false;
   m_auto_send = false;

src/wallet/message_store.h

@@ -43,6 +43,7 @@
 #include "common/i18n.h"
 #include "common/command_line.h"
 #include "wipeable_string.h"
+#include "net/abstract_http_client.h"
 #include "message_transporter.h"
 
 #undef MONERO_DEFAULT_LOG_CATEGORY
@@ -202,7 +203,8 @@ namespace mms
   class message_store
   {
   public:
-    message_store();
+    message_store(std::unique_ptr<epee::net_utils::http::abstract_http_client> http_client);
+
     // Initialize and start to use the MMS, set the first signer, this wallet itself
     // Filename, if not null and not empty, is used to create the ".mms" file
     // reset it if already used, with deletion of all signers and messages

src/wallet/message_transporter.cpp

@@ -80,7 +80,7 @@ namespace bitmessage_rpc
 
 }
 
-message_transporter::message_transporter()
+message_transporter::message_transporter(std::unique_ptr<epee::net_utils::http::abstract_http_client> http_client) : m_http_client(std::move(http_client))
 {
   m_run = true;
 }
@@ -96,7 +96,7 @@ void message_transporter::set_options(const std::string &bitmessage_address, con
   }
   m_bitmessage_login = bitmessage_login;
 
-  m_http_client.set_server(address_parts.host, std::to_string(address_parts.port), boost::none);
+  m_http_client->set_server(address_parts.host, std::to_string(address_parts.port), boost::none);
 }
 
 bool message_transporter::receive_messages(const std::vector<std::string> &destination_transport_addresses,
@@ -256,7 +256,7 @@ bool message_transporter::post_request(const std::string &request, std::string &
   additional_params.push_back(std::make_pair("Content-Type", "application/xml; charset=utf-8"));
   const epee::net_utils::http::http_response_info* response = NULL;
   std::chrono::milliseconds timeout = std::chrono::seconds(15);
-  bool r = m_http_client.invoke("/", "POST", request, timeout, std::addressof(response), std::move(additional_params));
+  bool r = m_http_client->invoke("/", "POST", request, timeout, std::addressof(response), std::move(additional_params));
   if (r)
   {
     answer = response->m_body;
@@ -266,7 +266,7 @@ bool message_transporter::post_request(const std::string &request, std::string &
     LOG_ERROR("POST request to Bitmessage failed: " << request.substr(0, 300));
     THROW_WALLET_EXCEPTION(tools::error::no_connection_to_bitmessage, m_bitmessage_url);
   }
-  m_http_client.disconnect();  // see comment above
+  m_http_client->disconnect();  // see comment above
   std::string string_value = get_str_between_tags(answer, "<string>", "</string>");
   if ((string_value.find("API Error") == 0) || (string_value.find("RPC ") == 0))
   {

src/wallet/message_transporter.h

@@ -34,9 +34,9 @@
 #include "cryptonote_basic/cryptonote_basic.h"
 #include "net/http_server_impl_base.h"
 #include "net/http_client.h"
+#include "net/abstract_http_client.h"
 #include "common/util.h"
 #include "wipeable_string.h"
-#include "serialization/keyvalue_serialization.h"
 #include <vector>
 
 namespace mms
@@ -83,7 +83,7 @@ typedef epee::misc_utils::struct_init<transport_message_t> transport_message;
 class message_transporter
 {
 public:
-  message_transporter();
+  message_transporter(std::unique_ptr<epee::net_utils::http::abstract_http_client> http_client);
   void set_options(const std::string &bitmessage_address, const epee::wipeable_string &bitmessage_login);
   bool send_message(const transport_message &message);
   bool receive_messages(const std::vector<std::string> &destination_transport_addresses,
@@ -94,7 +94,7 @@ public:
   bool delete_transport_address(const std::string &transport_address);
 
 private:
-  epee::net_utils::http::http_simple_client m_http_client;
+  const std::unique_ptr<epee::net_utils::http::abstract_http_client> m_http_client;
   std::string m_bitmessage_url;
   epee::wipeable_string m_bitmessage_login;
   std::atomic<bool> m_run;

src/wallet/node_rpc_proxy.cpp

@@ -51,7 +51,7 @@ namespace tools
 
 static const std::chrono::seconds rpc_timeout = std::chrono::minutes(3) + std::chrono::seconds(30);
 
-NodeRPCProxy::NodeRPCProxy(epee::net_utils::http::http_simple_client &http_client, rpc_payment_state_t &rpc_payment_state, boost::recursive_mutex &mutex)
+NodeRPCProxy::NodeRPCProxy(epee::net_utils::http::abstract_http_client &http_client, rpc_payment_state_t &rpc_payment_state, boost::recursive_mutex &mutex)
   : m_http_client(http_client)
   , m_rpc_payment_state(rpc_payment_state)
   , m_daemon_rpc_mutex(mutex)

src/wallet/node_rpc_proxy.h

@@ -31,7 +31,7 @@
 #include <string>
 #include <boost/thread/mutex.hpp>
 #include "include_base_utils.h"
-#include "net/http_client.h"
+#include "net/abstract_http_client.h"
 #include "rpc/core_rpc_server_commands_defs.h"
 #include "wallet_rpc_helpers.h"
 
@@ -41,7 +41,7 @@ namespace tools
 class NodeRPCProxy
 {
 public:
-  NodeRPCProxy(epee::net_utils::http::http_simple_client &http_client, rpc_payment_state_t &rpc_payment_state, boost::recursive_mutex &mutex);
+  NodeRPCProxy(epee::net_utils::http::abstract_http_client &http_client, rpc_payment_state_t &rpc_payment_state, boost::recursive_mutex &mutex);
 
   void set_client_secret_key(const crypto::secret_key &skey) { m_client_id_secret_key = skey; }
   void invalidate();
@@ -72,7 +72,7 @@ private:
 private:
   boost::optional<std::string> get_info();
 
-  epee::net_utils::http::http_simple_client &m_http_client;
+  epee::net_utils::http::abstract_http_client &m_http_client;
   rpc_payment_state_t &m_rpc_payment_state;
   boost::recursive_mutex &m_daemon_rpc_mutex;
   crypto::secret_key m_client_id_secret_key;

src/wallet/wallet2.cpp

@@ -1109,10 +1109,12 @@ boost::optional<epee::wipeable_string> wallet_device_callback::on_pin_request()
   return boost::none;
 }
 
-boost::optional<epee::wipeable_string> wallet_device_callback::on_passphrase_request(bool on_device)
+boost::optional<epee::wipeable_string> wallet_device_callback::on_passphrase_request(bool & on_device)
 {
   if (wallet)
     return wallet->on_device_passphrase_request(on_device);
+  else
+    on_device = true;
   return boost::none;
 }
 
@@ -1122,7 +1124,8 @@ void wallet_device_callback::on_progress(const hw::device_progress& event)
     wallet->on_device_progress(event);
 }
 
-wallet2::wallet2(network_type nettype, uint64_t kdf_rounds, bool unattended):
+wallet2::wallet2(network_type nettype, uint64_t kdf_rounds, bool unattended, std::unique_ptr<epee::net_utils::http::http_client_factory> http_client_factory):
+  m_http_client(std::move(http_client_factory->create())),
   m_multisig_rescan_info(NULL),
   m_multisig_rescan_k(NULL),
   m_upper_transaction_weight_limit(0),
@@ -1167,7 +1170,7 @@ wallet2::wallet2(network_type nettype, uint64_t kdf_rounds, bool unattended):
   m_watch_only(false),
   m_multisig(false),
   m_multisig_threshold(0),
-  m_node_rpc_proxy(m_http_client, m_rpc_payment_state, m_daemon_rpc_mutex),
+  m_node_rpc_proxy(*m_http_client, m_rpc_payment_state, m_daemon_rpc_mutex),
   m_account_public_address{crypto::null_pkey, crypto::null_pkey},
   m_subaddress_lookahead_major(SUBADDRESS_LOOKAHEAD_MAJOR),
   m_subaddress_lookahead_minor(SUBADDRESS_LOOKAHEAD_MINOR),
@@ -1178,7 +1181,7 @@ wallet2::wallet2(network_type nettype, uint64_t kdf_rounds, bool unattended):
   m_light_wallet_balance(0),
   m_light_wallet_unlocked_balance(0),
   m_original_keys_available(false),
-  m_message_store(),
+  m_message_store(http_client_factory->create()),
   m_key_device_type(hw::device::device_type::SOFTWARE),
   m_ring_history_saved(false),
   m_ringdb(),
@@ -1298,8 +1301,8 @@ bool wallet2::set_daemon(std::string daemon_address, boost::optional<epee::net_u
 {
   boost::lock_guard<boost::recursive_mutex> lock(m_daemon_rpc_mutex);
 
-  if(m_http_client.is_connected())
+  if(m_http_client->is_connected())
-    m_http_client.disconnect();
+    m_http_client->disconnect();
   const bool changed = m_daemon_address != daemon_address;
   m_daemon_address = std::move(daemon_address);
   m_daemon_login = std::move(daemon_login);
@@ -1313,7 +1316,7 @@ bool wallet2::set_daemon(std::string daemon_address, boost::optional<epee::net_u
 
   const std::string address = get_daemon_address();
   MINFO("setting daemon to " << address);
-  bool ret =  m_http_client.set_server(address, get_daemon_login(), std::move(ssl_options));
+  bool ret =  m_http_client->set_server(address, get_daemon_login(), std::move(ssl_options));
   if (ret)
   {
     CRITICAL_REGION_LOCAL(default_daemon_address_lock);
@@ -1328,7 +1331,12 @@ bool wallet2::init(std::string daemon_address, boost::optional<epee::net_utils::
   m_is_initialized = true;
   m_upper_transaction_weight_limit = upper_transaction_weight_limit;
   if (proxy != boost::asio::ip::tcp::endpoint{})
-    m_http_client.set_connector(net::socks::connector{std::move(proxy)});
+  {
+    epee::net_utils::http::abstract_http_client* abstract_http_client = m_http_client.get();
+    epee::net_utils::http::http_simple_client* http_simple_client = dynamic_cast<epee::net_utils::http::http_simple_client*>(abstract_http_client);
+    CHECK_AND_ASSERT_MES(http_simple_client != nullptr, false, "http_simple_client must be used to set proxy");
+    http_simple_client->set_connector(net::socks::connector{std::move(proxy)});
+  }
   return set_daemon(daemon_address, daemon_login, trusted_daemon, std::move(ssl_options));
 }
 //----------------------------------------------------------------------------------------------------
@@ -2595,7 +2603,7 @@ void wallet2::pull_blocks(uint64_t start_height, uint64_t &blocks_start_height,
     const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex};
     uint64_t pre_call_credits = m_rpc_payment_state.credits;
     req.client = get_client_signature();
-    bool r = net_utils::invoke_http_bin("/getblocks.bin", req, res, m_http_client, rpc_timeout);
+    bool r = net_utils::invoke_http_bin("/getblocks.bin", req, res, *m_http_client, rpc_timeout);
     THROW_ON_RPC_RESPONSE_ERROR(r, {}, res, "getblocks.bin", error::get_blocks_error, get_rpc_status(res.status));
     THROW_WALLET_EXCEPTION_IF(res.blocks.size() != res.output_indices.size(), error::wallet_internal_error,
         "mismatched blocks (" + boost::lexical_cast<std::string>(res.blocks.size()) + ") and output_indices (" +
@@ -2624,7 +2632,7 @@ void wallet2::pull_hashes(uint64_t start_height, uint64_t &blocks_start_height,
     const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex};
     req.client = get_client_signature();
     uint64_t pre_call_credits = m_rpc_payment_state.credits;
-    bool r = net_utils::invoke_http_bin("/gethashes.bin", req, res, m_http_client, rpc_timeout);
+    bool r = net_utils::invoke_http_bin("/gethashes.bin", req, res, *m_http_client, rpc_timeout);
     THROW_ON_RPC_RESPONSE_ERROR(r, {}, res, "gethashes.bin", error::get_hashes_error, get_rpc_status(res.status));
     check_rpc_cost("/gethashes.bin", res.credits, pre_call_credits, 1 + res.m_block_ids.size() * COST_PER_BLOCK_HASH);
   }
@@ -2909,7 +2917,7 @@ void wallet2::update_pool_state(std::vector<std::tuple<cryptonote::transaction,
     const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex};
     uint64_t pre_call_credits = m_rpc_payment_state.credits;
     req.client = get_client_signature();
-    bool r = epee::net_utils::invoke_http_json("/get_transaction_pool_hashes.bin", req, res, m_http_client, rpc_timeout);
+    bool r = epee::net_utils::invoke_http_json("/get_transaction_pool_hashes.bin", req, res, *m_http_client, rpc_timeout);
     THROW_ON_RPC_RESPONSE_ERROR(r, {}, res, "get_transaction_pool_hashes.bin", error::get_tx_pool_error);
     check_rpc_cost("/get_transaction_pool_hashes.bin", res.credits, pre_call_credits, 1 + res.tx_hashes.size() * COST_PER_POOL_HASH);
   }
@@ -3054,7 +3062,7 @@ void wallet2::update_pool_state(std::vector<std::tuple<cryptonote::transaction,
       const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex};
       uint64_t pre_call_credits = m_rpc_payment_state.credits;
       req.client = get_client_signature();
-      r = epee::net_utils::invoke_http_json("/gettransactions", req, res, m_http_client, rpc_timeout);
+      r = epee::net_utils::invoke_http_json("/gettransactions", req, res, *m_http_client, rpc_timeout);
       if (r && res.status == CORE_RPC_STATUS_OK)
         check_rpc_cost("/gettransactions", res.credits, pre_call_credits, res.txs.size() * COST_PER_TX);
     }
@@ -3540,7 +3548,7 @@ bool wallet2::get_rct_distribution(uint64_t &start_height, std::vector<uint64_t>
     const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex};
     uint64_t pre_call_credits = m_rpc_payment_state.credits;
     req.client = get_client_signature();
-    r = net_utils::invoke_http_bin("/get_output_distribution.bin", req, res, m_http_client, rpc_timeout);
+    r = net_utils::invoke_http_bin("/get_output_distribution.bin", req, res, *m_http_client, rpc_timeout);
     THROW_ON_RPC_RESPONSE_ERROR_GENERIC(r, {}, res, "/get_output_distribution.bin");
     check_rpc_cost("/get_output_distribution.bin", res.credits, pre_call_credits, COST_PER_OUTPUT_DISTRIBUTION_0);
   }
@@ -3698,6 +3706,30 @@ void wallet2::clear_soft(bool keep_key_images)
  * \return                Whether it was successful.
  */
 bool wallet2::store_keys(const std::string& keys_file_name, const epee::wipeable_string& password, bool watch_only)
+{
+  boost::optional<wallet2::keys_file_data> keys_file_data = get_keys_file_data(password, watch_only);
+  CHECK_AND_ASSERT_MES(keys_file_data != boost::none, false, "failed to generate wallet keys data");
+
+  std::string tmp_file_name = keys_file_name + ".new";
+  std::string buf;
+  bool r = ::serialization::dump_binary(keys_file_data.get(), buf);
+  r = r && save_to_file(tmp_file_name, buf);
+  CHECK_AND_ASSERT_MES(r, false, "failed to generate wallet keys file " << tmp_file_name);
+
+  unlock_keys_file();
+  std::error_code e = tools::replace_file(tmp_file_name, keys_file_name);
+  lock_keys_file();
+
+  if (e) {
+    boost::filesystem::remove(tmp_file_name);
+    LOG_ERROR("failed to update wallet keys file " << keys_file_name);
+    return false;
+  }
+
+  return true;
+}
+//----------------------------------------------------------------------------------------------------
+boost::optional<wallet2::keys_file_data> wallet2::get_keys_file_data(const epee::wipeable_string& password, bool watch_only)
 {
   std::string account_data;
   std::string multisig_signers;
@@ -3719,8 +3751,8 @@ bool wallet2::store_keys(const std::string& keys_file_name, const epee::wipeable
   account.encrypt_keys(key);
 
   bool r = epee::serialization::store_t_to_binary(account, account_data);
-  CHECK_AND_ASSERT_MES(r, false, "failed to serialize wallet keys");
+  CHECK_AND_ASSERT_MES(r, boost::none, "failed to serialize wallet keys");
-  wallet2::keys_file_data keys_file_data = {};
+  boost::optional<wallet2::keys_file_data> keys_file_data = (wallet2::keys_file_data) {};
 
   // Create a JSON object with "key_data" and "seed_language" as keys.
   rapidjson::Document json;
@@ -3751,12 +3783,12 @@ bool wallet2::store_keys(const std::string& keys_file_name, const epee::wipeable
   if (m_multisig)
   {
     bool r = ::serialization::dump_binary(m_multisig_signers, multisig_signers);
-    CHECK_AND_ASSERT_MES(r, false, "failed to serialize wallet multisig signers");
+    CHECK_AND_ASSERT_MES(r, boost::none, "failed to serialize wallet multisig signers");
     value.SetString(multisig_signers.c_str(), multisig_signers.length());
     json.AddMember("multisig_signers", value, json.GetAllocator());
 
     r = ::serialization::dump_binary(m_multisig_derivations, multisig_derivations);
-    CHECK_AND_ASSERT_MES(r, false, "failed to serialize wallet multisig derivations");
+    CHECK_AND_ASSERT_MES(r, boost::none, "failed to serialize wallet multisig derivations");
     value.SetString(multisig_derivations.c_str(), multisig_derivations.length());
     json.AddMember("multisig_derivations", value, json.GetAllocator());
 
@@ -3899,27 +3931,10 @@ bool wallet2::store_keys(const std::string& keys_file_name, const epee::wipeable
   // Encrypt the entire JSON object.
   std::string cipher;
   cipher.resize(account_data.size());
-  keys_file_data.iv = crypto::rand<crypto::chacha_iv>();
+  keys_file_data.get().iv = crypto::rand<crypto::chacha_iv>();
-  crypto::chacha20(account_data.data(), account_data.size(), key, keys_file_data.iv, &cipher[0]);
+  crypto::chacha20(account_data.data(), account_data.size(), key, keys_file_data.get().iv, &cipher[0]);
-  keys_file_data.account_data = cipher;
+  keys_file_data.get().account_data = cipher;
-
+  return keys_file_data;
-  std::string tmp_file_name = keys_file_name + ".new";
-  std::string buf;
-  r = ::serialization::dump_binary(keys_file_data, buf);
-  r = r && save_to_file(tmp_file_name, buf);
-  CHECK_AND_ASSERT_MES(r, false, "failed to generate wallet keys file " << tmp_file_name);
-
-  unlock_keys_file();
-  std::error_code e = tools::replace_file(tmp_file_name, keys_file_name);
-  lock_keys_file();
-
-  if (e) {
-      boost::filesystem::remove(tmp_file_name);
-      LOG_ERROR("failed to update wallet keys file " << keys_file_name);
-      return false;
-  }
-
-  return true;
 }
 //----------------------------------------------------------------------------------------------------
 void wallet2::setup_keys(const epee::wipeable_string &password)
@@ -3959,16 +3974,51 @@ void wallet2::change_password(const std::string &filename, const epee::wipeable_
  */
 bool wallet2::load_keys(const std::string& keys_file_name, const epee::wipeable_string& password)
 {
-  rapidjson::Document json;
+  std::string keys_file_buf;
-  wallet2::keys_file_data keys_file_data;
+  bool r = load_from_file(keys_file_name, keys_file_buf);
-  std::string buf;
-  bool encrypted_secret_keys = false;
-  bool r = load_from_file(keys_file_name, buf);
   THROW_WALLET_EXCEPTION_IF(!r, error::file_read_error, keys_file_name);
 
+  // Load keys from buffer
+  boost::optional<crypto::chacha_key> keys_to_encrypt;
+  try {
+    r = wallet2::load_keys_buf(keys_file_buf, password, keys_to_encrypt);
+  } catch (const std::exception& e) {
+    std::size_t found = string(e.what()).find("failed to deserialize keys buffer");
+    THROW_WALLET_EXCEPTION_IF(found != std::string::npos, error::wallet_internal_error, "internal error: failed to deserialize \"" + keys_file_name + '\"');
+    throw e;
+  }
+
+  // Rewrite with encrypted keys if unencrypted, ignore errors
+  if (r && keys_to_encrypt != boost::none)
+  {
+    if (m_ask_password == AskPasswordToDecrypt && !m_unattended && !m_watch_only)
+      encrypt_keys(keys_to_encrypt.get());
+    bool saved_ret = store_keys(keys_file_name, password, m_watch_only);
+    if (!saved_ret)
+    {
+      // just moan a bit, but not fatal
+      MERROR("Error saving keys file with encrypted keys, not fatal");
+    }
+    if (m_ask_password == AskPasswordToDecrypt && !m_unattended && !m_watch_only)
+      decrypt_keys(keys_to_encrypt.get());
+    m_keys_file_locker.reset();
+  }
+  return r;
+}
+//----------------------------------------------------------------------------------------------------
+bool wallet2::load_keys_buf(const std::string& keys_buf, const epee::wipeable_string& password) {
+  boost::optional<crypto::chacha_key> keys_to_encrypt;
+  return wallet2::load_keys_buf(keys_buf, password, keys_to_encrypt);
+}
+//----------------------------------------------------------------------------------------------------
+bool wallet2::load_keys_buf(const std::string& keys_buf, const epee::wipeable_string& password, boost::optional<crypto::chacha_key>& keys_to_encrypt) {
+
   // Decrypt the contents
-  r = ::serialization::parse_binary(buf, keys_file_data);
+  rapidjson::Document json;
-  THROW_WALLET_EXCEPTION_IF(!r, error::wallet_internal_error, "internal error: failed to deserialize \"" + keys_file_name + '\"');
+  wallet2::keys_file_data keys_file_data;
+  bool encrypted_secret_keys = false;
+  bool r = ::serialization::parse_binary(keys_buf, keys_file_data);
+  THROW_WALLET_EXCEPTION_IF(!r, error::wallet_internal_error, "internal error: failed to deserialize keys buffer");
   crypto::chacha_key key;
   crypto::generate_chacha_key(password.data(), password.size(), key, m_kdf_rounds);
   std::string account_data;
@@ -4252,8 +4302,8 @@ bool wallet2::load_keys(const std::string& keys_file_name, const epee::wipeable_
   }
   else
   {
-      THROW_WALLET_EXCEPTION(error::wallet_internal_error, "invalid password");
+    THROW_WALLET_EXCEPTION(error::wallet_internal_error, "invalid password");
-      return false;
+    return false;
   }
 
   r = epee::serialization::load_t_from_binary(m_account, account_data);
@@ -4287,24 +4337,13 @@ bool wallet2::load_keys(const std::string& keys_file_name, const epee::wipeable_
     }
     else
     {
-      // rewrite with encrypted keys, ignore errors
+      keys_to_encrypt = key;
-      if (m_ask_password == AskPasswordToDecrypt && !m_unattended && !m_watch_only)
-        encrypt_keys(key);
-      bool saved_ret = store_keys(keys_file_name, password, m_watch_only);
-      if (!saved_ret)
-      {
-        // just moan a bit, but not fatal
-        MERROR("Error saving keys file with encrypted keys, not fatal");
-      }
-      if (m_ask_password == AskPasswordToDecrypt && !m_unattended && !m_watch_only)
-        decrypt_keys(key);
-      m_keys_file_locker.reset();
     }
   }
   const cryptonote::account_keys& keys = m_account.get_keys();
   hw::device &hwdev = m_account.get_device();
   r = r && hwdev.verify_keys(keys.m_view_secret_key,  keys.m_account_address.m_view_public_key);
-  if(!m_watch_only && !m_multisig && hwdev.device_protocol() != hw::device::PROTOCOL_COLD)
+  if (!m_watch_only && !m_multisig && hwdev.device_protocol() != hw::device::PROTOCOL_COLD)
     r = r && hwdev.verify_keys(keys.m_spend_secret_key, keys.m_account_address.m_spend_public_key);
   THROW_WALLET_EXCEPTION_IF(!r, error::wallet_files_doesnt_correspond, m_keys_file, m_wallet_file);
 
@@ -4923,7 +4962,8 @@ std::string wallet2::make_multisig(const epee::wipeable_string &password,
   // re-encrypt keys
   keys_reencryptor = epee::misc_utils::auto_scope_leave_caller();
 
-  create_keys_file(m_wallet_file, false, password, boost::filesystem::exists(m_wallet_file + ".address.txt"));
+  if (!m_wallet_file.empty())
+    create_keys_file(m_wallet_file, false, password, boost::filesystem::exists(m_wallet_file + ".address.txt"));
 
   setup_new_blockchain();
 
@@ -5063,7 +5103,9 @@ std::string wallet2::exchange_multisig_keys(const epee::wipeable_string &passwor
 
   ++m_multisig_rounds_passed;
 
-  create_keys_file(m_wallet_file, false, password, boost::filesystem::exists(m_wallet_file + ".address.txt"));
+  if (!m_wallet_file.empty())
+    create_keys_file(m_wallet_file, false, password, boost::filesystem::exists(m_wallet_file + ".address.txt"));
+
   return extra_multisig_info;
 }
 
@@ -5437,13 +5479,13 @@ bool wallet2::check_connection(uint32_t *version, bool *ssl, uint32_t timeout)
 
   {
     boost::lock_guard<boost::recursive_mutex> lock(m_daemon_rpc_mutex);
-    if(!m_http_client.is_connected(ssl))
+    if(!m_http_client->is_connected(ssl))
     {
       m_rpc_version = 0;
       m_node_rpc_proxy.invalidate();
-      if (!m_http_client.connect(std::chrono::milliseconds(timeout)))
+      if (!m_http_client->connect(std::chrono::milliseconds(timeout)))
         return false;
-      if(!m_http_client.is_connected(ssl))
+      if(!m_http_client->is_connected(ssl))
         return false;
     }
   }
@@ -5471,12 +5513,12 @@ void wallet2::set_offline(bool offline)
 {
   m_offline = offline;
   m_node_rpc_proxy.set_offline(offline);
-  m_http_client.set_auto_connect(!offline);
+  m_http_client->set_auto_connect(!offline);
   if (offline)
   {
     boost::lock_guard<boost::recursive_mutex> lock(m_daemon_rpc_mutex);
-    if(m_http_client.is_connected())
+    if(m_http_client->is_connected())
-      m_http_client.disconnect();
+      m_http_client->disconnect();
   }
 }
 //----------------------------------------------------------------------------------------------------
@@ -5491,48 +5533,63 @@ void wallet2::generate_chacha_key_from_password(const epee::wipeable_string &pas
   crypto::generate_chacha_key(pass.data(), pass.size(), key, m_kdf_rounds);
 }
 //----------------------------------------------------------------------------------------------------
-void wallet2::load(const std::string& wallet_, const epee::wipeable_string& password)
+void wallet2::load(const std::string& wallet_, const epee::wipeable_string& password, const std::string& keys_buf, const std::string& cache_buf)
 {
   clear();
   prepare_file_names(wallet_);
 
+  // determine if loading from file system or string buffer
+  bool use_fs = !wallet_.empty();
+  THROW_WALLET_EXCEPTION_IF((use_fs && !keys_buf.empty()) || (!use_fs && keys_buf.empty()), error::file_read_error, "must load keys either from file system or from buffer");\
+
   boost::system::error_code e;
-  bool exists = boost::filesystem::exists(m_keys_file, e);
+  if (use_fs)
-  THROW_WALLET_EXCEPTION_IF(e || !exists, error::file_not_found, m_keys_file);
+  {
-  lock_keys_file();
+    bool exists = boost::filesystem::exists(m_keys_file, e);
-  THROW_WALLET_EXCEPTION_IF(!is_keys_file_locked(), error::wallet_internal_error, "internal error: \"" + m_keys_file + "\" is opened by another wallet program");
+    THROW_WALLET_EXCEPTION_IF(e || !exists, error::file_not_found, m_keys_file);
+    lock_keys_file();
+    THROW_WALLET_EXCEPTION_IF(!is_keys_file_locked(), error::wallet_internal_error, "internal error: \"" + m_keys_file + "\" is opened by another wallet program");
 
-  // this temporary unlocking is necessary for Windows (otherwise the file couldn't be loaded).
+    // this temporary unlocking is necessary for Windows (otherwise the file couldn't be loaded).
-  unlock_keys_file();
+    unlock_keys_file();
-  if (!load_keys(m_keys_file, password))
+    if (!load_keys(m_keys_file, password))
+    {
+      THROW_WALLET_EXCEPTION_IF(true, error::file_read_error, m_keys_file);
+    }
+    LOG_PRINT_L0("Loaded wallet keys file, with public address: " << m_account.get_public_address_str(m_nettype));
+    lock_keys_file();
+  }
+  else if (!load_keys_buf(keys_buf, password))
   {
-    THROW_WALLET_EXCEPTION_IF(true, error::file_read_error, m_keys_file);
+    THROW_WALLET_EXCEPTION_IF(true, error::file_read_error, "failed to load keys from buffer");
   }
-  LOG_PRINT_L0("Loaded wallet keys file, with public address: " << m_account.get_public_address_str(m_nettype));
-  lock_keys_file();
 
   wallet_keys_unlocker unlocker(*this, m_ask_password == AskPasswordToDecrypt && !m_unattended && !m_watch_only, password);
 
   //keys loaded ok!
   //try to load wallet file. but even if we failed, it is not big problem
-  if(!boost::filesystem::exists(m_wallet_file, e) || e)
+  if (use_fs && (!boost::filesystem::exists(m_wallet_file, e) || e))
   {
     LOG_PRINT_L0("file not found: " << m_wallet_file << ", starting with empty blockchain");
     m_account_public_address = m_account.get_keys().m_account_address;
   }
-  else
+  else if (use_fs || !cache_buf.empty())
   {
     wallet2::cache_file_data cache_file_data;
-    std::string buf;
+    std::string cache_file_buf;
-    bool r = load_from_file(m_wallet_file, buf, std::numeric_limits<size_t>::max());
+    bool r = true;
-    THROW_WALLET_EXCEPTION_IF(!r, error::file_read_error, m_wallet_file);
+    if (use_fs)
+    {
+      load_from_file(m_wallet_file, cache_file_buf, std::numeric_limits<size_t>::max());
+      THROW_WALLET_EXCEPTION_IF(!r, error::file_read_error, m_wallet_file);
+    }
 
     // try to read it as an encrypted cache
     try
     {
       LOG_PRINT_L1("Trying to decrypt cache data");
 
-      r = ::serialization::parse_binary(buf, cache_file_data);
+      r = ::serialization::parse_binary(use_fs ? cache_file_buf : cache_buf, cache_file_data);
       THROW_WALLET_EXCEPTION_IF(!r, error::wallet_internal_error, "internal error: failed to deserialize \"" + m_wallet_file + '\"');
       std::string cache_data;
       cache_data.resize(cache_file_data.cache_data.size());
@@ -5569,7 +5626,7 @@ void wallet2::load(const std::string& wallet_, const epee::wipeable_string& pass
           catch (...)
           {
             LOG_PRINT_L0("Failed to open portable binary, trying unportable");
-            boost::filesystem::copy_file(m_wallet_file, m_wallet_file + ".unportable", boost::filesystem::copy_option::overwrite_if_exists);
+            if (use_fs) boost::filesystem::copy_file(m_wallet_file, m_wallet_file + ".unportable", boost::filesystem::copy_option::overwrite_if_exists);
             std::stringstream iss;
             iss.str("");
             iss << cache_data;
@@ -5584,17 +5641,17 @@ void wallet2::load(const std::string& wallet_, const epee::wipeable_string& pass
       LOG_PRINT_L1("Failed to load encrypted cache, trying unencrypted");
       try {
         std::stringstream iss;
-        iss << buf;
+        iss << cache_file_buf;
         boost::archive::portable_binary_iarchive ar(iss);
         ar >> *this;
       }
       catch (...)
       {
         LOG_PRINT_L0("Failed to open portable binary, trying unportable");
-        boost::filesystem::copy_file(m_wallet_file, m_wallet_file + ".unportable", boost::filesystem::copy_option::overwrite_if_exists);
+        if (use_fs) boost::filesystem::copy_file(m_wallet_file, m_wallet_file + ".unportable", boost::filesystem::copy_option::overwrite_if_exists);
         std::stringstream iss;
         iss.str("");
-        iss << buf;
+        iss << cache_file_buf;
         boost::archive::binary_iarchive ar(iss);
         ar >> *this;
       }
@@ -5638,7 +5695,8 @@ void wallet2::load(const std::string& wallet_, const epee::wipeable_string& pass
   
   try
   {
-    m_message_store.read_from_file(get_multisig_wallet_state(), m_mms_file);
+    if (use_fs)
+      m_message_store.read_from_file(get_multisig_wallet_state(), m_mms_file);
   }
   catch (const std::exception &e)
   {
@@ -5666,7 +5724,7 @@ void wallet2::trim_hashchain()
       req.height = m_blockchain.size() - 1;
       uint64_t pre_call_credits = m_rpc_payment_state.credits;
       req.client = get_client_signature();
-      r = net_utils::invoke_http_json_rpc("/json_rpc", "getblockheaderbyheight", req, res, m_http_client, rpc_timeout);
+      r = net_utils::invoke_http_json_rpc("/json_rpc", "getblockheaderbyheight", req, res, *m_http_client, rpc_timeout);
       if (r && res.status == CORE_RPC_STATUS_OK)
         check_rpc_cost("getblockheaderbyheight", res.credits, pre_call_credits, COST_PER_BLOCK_HEADER);
     }
@@ -5741,18 +5799,10 @@ void wallet2::store_to(const std::string &path, const epee::wipeable_string &pas
       }
     }
   }
-  // preparing wallet data
-  std::stringstream oss;
-  boost::archive::portable_binary_oarchive ar(oss);
-  ar << *this;
 
-  wallet2::cache_file_data cache_file_data = {};
+  // get wallet cache data
-  cache_file_data.cache_data = oss.str();
+  boost::optional<wallet2::cache_file_data> cache_file_data = get_cache_file_data(password);
-  std::string cipher;
+  THROW_WALLET_EXCEPTION_IF(cache_file_data == boost::none, error::wallet_internal_error, "failed to generate wallet cache data");
-  cipher.resize(cache_file_data.cache_data.size());
-  cache_file_data.iv = crypto::rand<crypto::chacha_iv>();
-  crypto::chacha20(cache_file_data.cache_data.data(), cache_file_data.cache_data.size(), m_cache_key, cache_file_data.iv, &cipher[0]);
-  cache_file_data.cache_data = cipher;
 
   const std::string new_file = same_file ? m_wallet_file + ".new" : path;
   const std::string old_file = m_wallet_file;
@@ -5803,7 +5853,7 @@ void wallet2::store_to(const std::string &path, const epee::wipeable_string &pas
     // The price to pay is temporary higher memory consumption for string stream + binary archive
     std::ostringstream oss;
     binary_archive<true> oar(oss);
-    bool success = ::serialization::serialize(oar, cache_file_data);
+    bool success = ::serialization::serialize(oar, cache_file_data.get());
     if (success) {
         success = save_to_file(new_file, oss.str());
     }
@@ -5812,7 +5862,7 @@ void wallet2::store_to(const std::string &path, const epee::wipeable_string &pas
     std::ofstream ostr;
     ostr.open(new_file, std::ios_base::binary | std::ios_base::out | std::ios_base::trunc);
     binary_archive<true> oar(ostr);
-    bool success = ::serialization::serialize(oar, cache_file_data);
+    bool success = ::serialization::serialize(oar, cache_file_data.get());
     ostr.close();
     THROW_WALLET_EXCEPTION_IF(!success || !ostr.good(), error::file_save_error, new_file);
 #endif
@@ -5828,7 +5878,30 @@ void wallet2::store_to(const std::string &path, const epee::wipeable_string &pas
     // store should only exist if the MMS is really active
     m_message_store.write_to_file(get_multisig_wallet_state(), m_mms_file);
   }
-  
+}
+//----------------------------------------------------------------------------------------------------
+boost::optional<wallet2::cache_file_data> wallet2::get_cache_file_data(const epee::wipeable_string &passwords)
+{
+  trim_hashchain();
+  try
+  {
+    std::stringstream oss;
+    boost::archive::portable_binary_oarchive ar(oss);
+    ar << *this;
+
+    boost::optional<wallet2::cache_file_data> cache_file_data = (wallet2::cache_file_data) {};
+    cache_file_data.get().cache_data = oss.str();
+    std::string cipher;
+    cipher.resize(cache_file_data.get().cache_data.size());
+    cache_file_data.get().iv = crypto::rand<crypto::chacha_iv>();
+    crypto::chacha20(cache_file_data.get().cache_data.data(), cache_file_data.get().cache_data.size(), m_cache_key, cache_file_data.get().iv, &cipher[0]);
+    cache_file_data.get().cache_data = cipher;
+    return cache_file_data;
+  }
+  catch(...)
+  {
+    return boost::none;
+  }
 }
 //----------------------------------------------------------------------------------------------------
 uint64_t wallet2::balance(uint32_t index_major, bool strict) const
@@ -6032,7 +6105,7 @@ void wallet2::rescan_spent()
       const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex};
       uint64_t pre_call_credits = m_rpc_payment_state.credits;
       req.client = get_client_signature();
-      bool r = epee::net_utils::invoke_http_json("/is_key_image_spent", req, daemon_resp, m_http_client, rpc_timeout);
+      bool r = epee::net_utils::invoke_http_json("/is_key_image_spent", req, daemon_resp, *m_http_client, rpc_timeout);
       THROW_ON_RPC_RESPONSE_ERROR(r, {}, daemon_resp, "is_key_image_spent", error::is_key_image_spent_error, get_rpc_status(daemon_resp.status));
       THROW_WALLET_EXCEPTION_IF(daemon_resp.spent_status.size() != n_outputs, error::wallet_internal_error,
         "daemon returned wrong response for is_key_image_spent, wrong amounts count = " +
@@ -6360,7 +6433,7 @@ void wallet2::commit_tx(pending_tx& ptx)
     oreq.tx = epee::string_tools::buff_to_hex_nodelimer(tx_to_blob(ptx.tx));
     {
       const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex};
-      bool r = epee::net_utils::invoke_http_json("/submit_raw_tx", oreq, ores, m_http_client, rpc_timeout, "POST");
+      bool r = epee::net_utils::invoke_http_json("/submit_raw_tx", oreq, ores, *m_http_client, rpc_timeout, "POST");
       THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "submit_raw_tx");
       // MyMonero and OpenMonero use different status strings
       THROW_WALLET_EXCEPTION_IF(ores.status != "OK" && ores.status != "success" , error::tx_rejected, ptx.tx, get_rpc_status(ores.status), ores.error);
@@ -6379,7 +6452,7 @@ void wallet2::commit_tx(pending_tx& ptx)
       const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex};
       uint64_t pre_call_credits = m_rpc_payment_state.credits;
       req.client = get_client_signature();
-      bool r = epee::net_utils::invoke_http_json("/sendrawtransaction", req, daemon_send_resp, m_http_client, rpc_timeout);
+      bool r = epee::net_utils::invoke_http_json("/sendrawtransaction", req, daemon_send_resp, *m_http_client, rpc_timeout);
       THROW_ON_RPC_RESPONSE_ERROR(r, {}, daemon_send_resp, "sendrawtransaction", error::tx_rejected, ptx.tx, get_rpc_status(daemon_send_resp.status), get_text_reason(daemon_send_resp));
       check_rpc_cost("/sendrawtransaction", daemon_send_resp.credits, pre_call_credits, COST_PER_TX_RELAY);
     }
@@ -7349,7 +7422,7 @@ uint32_t wallet2::adjust_priority(uint32_t priority)
         const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex};
         uint64_t pre_call_credits = m_rpc_payment_state.credits;
         getbh_req.client = get_client_signature();
-        bool r = net_utils::invoke_http_json_rpc("/json_rpc", "getblockheadersrange", getbh_req, getbh_res, m_http_client, rpc_timeout);
+        bool r = net_utils::invoke_http_json_rpc("/json_rpc", "getblockheadersrange", getbh_req, getbh_res, *m_http_client, rpc_timeout);
         THROW_ON_RPC_RESPONSE_ERROR(r, {}, getbh_res, "getblockheadersrange", error::get_blocks_error, get_rpc_status(getbh_res.status));
         check_rpc_cost("/sendrawtransaction", getbh_res.credits, pre_call_credits, N * COST_PER_BLOCK_HEADER);
       }
@@ -7575,7 +7648,7 @@ bool wallet2::find_and_save_rings(bool force)
       const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex};
       uint64_t pre_call_credits = m_rpc_payment_state.credits;
       req.client = get_client_signature();
-      bool r = epee::net_utils::invoke_http_json("/gettransactions", req, res, m_http_client, rpc_timeout);
+      bool r = epee::net_utils::invoke_http_json("/gettransactions", req, res, *m_http_client, rpc_timeout);
       THROW_ON_RPC_RESPONSE_ERROR_GENERIC(r, {}, res, "/gettransactions");
       THROW_WALLET_EXCEPTION_IF(res.txs.size() != req.txs_hashes.size(), error::wallet_internal_error,
         "daemon returned wrong response for gettransactions, wrong txs count = " +
@@ -7723,7 +7796,7 @@ void wallet2::light_wallet_get_outs(std::vector<std::vector<tools::wallet2::get_
 
   {
     const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex};
-    bool r = epee::net_utils::invoke_http_json("/get_random_outs", oreq, ores, m_http_client, rpc_timeout, "POST");
+    bool r = epee::net_utils::invoke_http_json("/get_random_outs", oreq, ores, *m_http_client, rpc_timeout, "POST");
     m_daemon_rpc_mutex.unlock();
     THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "get_random_outs");
     THROW_WALLET_EXCEPTION_IF(ores.amount_outs.empty() , error::wallet_internal_error, "No outputs received from light wallet node. Error: " + ores.Error);
@@ -7911,7 +7984,7 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>>
         const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex};
         uint64_t pre_call_credits = m_rpc_payment_state.credits;
         req_t.client = get_client_signature();
-        bool r = net_utils::invoke_http_json_rpc("/json_rpc", "get_output_histogram", req_t, resp_t, m_http_client, rpc_timeout);
+        bool r = net_utils::invoke_http_json_rpc("/json_rpc", "get_output_histogram", req_t, resp_t, *m_http_client, rpc_timeout);
         THROW_ON_RPC_RESPONSE_ERROR(r, {}, resp_t, "get_output_histogram", error::get_histogram_error, get_rpc_status(resp_t.status));
         check_rpc_cost("get_output_histogram", resp_t.credits, pre_call_credits, COST_PER_OUTPUT_HISTOGRAM * req_t.amounts.size());
       }
@@ -7937,7 +8010,7 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>>
         const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex};
         uint64_t pre_call_credits = m_rpc_payment_state.credits;
         req_t.client = get_client_signature();
-        bool r = net_utils::invoke_http_json_rpc("/json_rpc", "get_output_distribution", req_t, resp_t, m_http_client, rpc_timeout * 1000);
+        bool r = net_utils::invoke_http_json_rpc("/json_rpc", "get_output_distribution", req_t, resp_t, *m_http_client, rpc_timeout * 1000);
         THROW_ON_RPC_RESPONSE_ERROR(r, {}, resp_t, "get_output_distribution", error::get_output_distribution, get_rpc_status(resp_t.status));
         uint64_t expected_cost = 0;
         for (uint64_t amount: req_t.amounts) expected_cost += (amount ? COST_PER_OUTPUT_DISTRIBUTION : COST_PER_OUTPUT_DISTRIBUTION_0);
@@ -8291,7 +8364,7 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>>
       const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex};
       uint64_t pre_call_credits = m_rpc_payment_state.credits;
       req.client = get_client_signature();
-      bool r = epee::net_utils::invoke_http_bin("/get_outs.bin", req, daemon_resp, m_http_client, rpc_timeout);
+      bool r = epee::net_utils::invoke_http_bin("/get_outs.bin", req, daemon_resp, *m_http_client, rpc_timeout);
       THROW_ON_RPC_RESPONSE_ERROR(r, {}, daemon_resp, "get_outs.bin", error::get_outs_error, get_rpc_status(daemon_resp.status));
       THROW_WALLET_EXCEPTION_IF(daemon_resp.outs.size() != req.outputs.size(), error::wallet_internal_error,
         "daemon returned wrong response for get_outs.bin, wrong amounts count = " +
@@ -10500,7 +10573,7 @@ uint8_t wallet2::get_current_hard_fork()
 
   m_daemon_rpc_mutex.lock();
   req_t.version = 0;
-  bool r = net_utils::invoke_http_json_rpc("/json_rpc", "hard_fork_info", req_t, resp_t, m_http_client, rpc_timeout);
+  bool r = net_utils::invoke_http_json_rpc("/json_rpc", "hard_fork_info", req_t, resp_t, *m_http_client, rpc_timeout);
   m_daemon_rpc_mutex.unlock();
   THROW_WALLET_EXCEPTION_IF(!r, tools::error::no_connection_to_daemon, "hard_fork_info");
   THROW_WALLET_EXCEPTION_IF(resp_t.status == CORE_RPC_STATUS_BUSY, tools::error::daemon_busy, "hard_fork_info");
@@ -10595,7 +10668,7 @@ std::vector<size_t> wallet2::select_available_outputs_from_histogram(uint64_t co
     const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex};
     uint64_t pre_call_credits = m_rpc_payment_state.credits;
     req_t.client = get_client_signature();
-    bool r = net_utils::invoke_http_json_rpc("/json_rpc", "get_output_histogram", req_t, resp_t, m_http_client, rpc_timeout);
+    bool r = net_utils::invoke_http_json_rpc("/json_rpc", "get_output_histogram", req_t, resp_t, *m_http_client, rpc_timeout);
     THROW_ON_RPC_RESPONSE_ERROR(r, {}, resp_t, "get_output_histogram", error::get_histogram_error, resp_t.status);
     uint64_t cost = req_t.amounts.empty() ? COST_PER_FULL_OUTPUT_HISTOGRAM : (COST_PER_OUTPUT_HISTOGRAM * req_t.amounts.size());
     check_rpc_cost("get_output_histogram", resp_t.credits, pre_call_credits, cost);
@@ -10637,7 +10710,7 @@ uint64_t wallet2::get_num_rct_outputs()
     const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex};
     uint64_t pre_call_credits = m_rpc_payment_state.credits;
     req_t.client = get_client_signature();
-    bool r = net_utils::invoke_http_json_rpc("/json_rpc", "get_output_histogram", req_t, resp_t, m_http_client, rpc_timeout);
+    bool r = net_utils::invoke_http_json_rpc("/json_rpc", "get_output_histogram", req_t, resp_t, *m_http_client, rpc_timeout);
     THROW_ON_RPC_RESPONSE_ERROR(r, {}, resp_t, "get_output_histogram", error::get_histogram_error, resp_t.status);
     THROW_WALLET_EXCEPTION_IF(resp_t.histogram.size() != 1, error::get_histogram_error, "Expected exactly one response");
     THROW_WALLET_EXCEPTION_IF(resp_t.histogram[0].amount != 0, error::get_histogram_error, "Expected 0 amount");
@@ -10768,7 +10841,7 @@ bool wallet2::get_tx_key(const crypto::hash &txid, crypto::secret_key &tx_key, s
       const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex};
       req.client = get_client_signature();
       uint64_t pre_call_credits = m_rpc_payment_state.credits;
-      bool ok = epee::net_utils::invoke_http_json("/gettransactions", req, res, m_http_client);
+      bool ok = epee::net_utils::invoke_http_json("/gettransactions", req, res, *m_http_client);
       THROW_WALLET_EXCEPTION_IF(!ok || (res.txs.size() != 1 && res.txs_as_hex.size() != 1),
                                 error::wallet_internal_error, "Failed to get transaction from daemon");
       check_rpc_cost("/gettransactions", res.credits, pre_call_credits, res.txs.size() * COST_PER_TX);
@@ -10821,7 +10894,7 @@ void wallet2::set_tx_key(const crypto::hash &txid, const crypto::secret_key &tx_
     const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex};
     pre_call_credits = m_rpc_payment_state.credits;
     req.client = get_client_signature();
-    r = epee::net_utils::invoke_http_json("/gettransactions", req, res, m_http_client, rpc_timeout);
+    r = epee::net_utils::invoke_http_json("/gettransactions", req, res, *m_http_client, rpc_timeout);
     THROW_ON_RPC_RESPONSE_ERROR_GENERIC(r, {}, res, "/gettransactions");
     THROW_WALLET_EXCEPTION_IF(res.txs.size() != 1, error::wallet_internal_error,
       "daemon returned wrong response for gettransactions, wrong txs count = " +
@@ -10874,7 +10947,7 @@ std::string wallet2::get_spend_proof(const crypto::hash &txid, const std::string
     const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex};
     pre_call_credits = m_rpc_payment_state.credits;
     req.client = get_client_signature();
-    r = epee::net_utils::invoke_http_json("/gettransactions", req, res, m_http_client, rpc_timeout);
+    r = epee::net_utils::invoke_http_json("/gettransactions", req, res, *m_http_client, rpc_timeout);
     THROW_ON_RPC_RESPONSE_ERROR_GENERIC(r, {}, res, "gettransactions");
     THROW_WALLET_EXCEPTION_IF(res.txs.size() != 1, error::wallet_internal_error,
       "daemon returned wrong response for gettransactions, wrong txs count = " +
@@ -10938,7 +11011,7 @@ std::string wallet2::get_spend_proof(const crypto::hash &txid, const std::string
       const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex};
       pre_call_credits = m_rpc_payment_state.credits;
       req.client = get_client_signature();
-      r = epee::net_utils::invoke_http_bin("/get_outs.bin", req, res, m_http_client, rpc_timeout);
+      r = epee::net_utils::invoke_http_bin("/get_outs.bin", req, res, *m_http_client, rpc_timeout);
       THROW_ON_RPC_RESPONSE_ERROR(r, {}, res, "get_outs.bin", error::get_outs_error, res.status);
       THROW_WALLET_EXCEPTION_IF(res.outs.size() != ring_size, error::wallet_internal_error,
         "daemon returned wrong response for get_outs.bin, wrong amounts count = " +
@@ -10996,7 +11069,7 @@ bool wallet2::check_spend_proof(const crypto::hash &txid, const std::string &mes
     const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex};
     pre_call_credits = m_rpc_payment_state.credits;
     req.client = get_client_signature();
-    r = epee::net_utils::invoke_http_json("/gettransactions", req, res, m_http_client, rpc_timeout);
+    r = epee::net_utils::invoke_http_json("/gettransactions", req, res, *m_http_client, rpc_timeout);
     THROW_ON_RPC_RESPONSE_ERROR_GENERIC(r, {}, res, "gettransactions");
     THROW_WALLET_EXCEPTION_IF(res.txs.size() != 1, error::wallet_internal_error,
       "daemon returned wrong response for gettransactions, wrong txs count = " +
@@ -11071,7 +11144,7 @@ bool wallet2::check_spend_proof(const crypto::hash &txid, const std::string &mes
       const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex};
       pre_call_credits = m_rpc_payment_state.credits;
       req.client = get_client_signature();
-      r = epee::net_utils::invoke_http_bin("/get_outs.bin", req, res, m_http_client, rpc_timeout);
+      r = epee::net_utils::invoke_http_bin("/get_outs.bin", req, res, *m_http_client, rpc_timeout);
       THROW_ON_RPC_RESPONSE_ERROR(r, {}, res, "get_outs.bin", error::get_outs_error, res.status);
       THROW_WALLET_EXCEPTION_IF(res.outs.size() != req.outputs.size(), error::wallet_internal_error,
         "daemon returned wrong response for get_outs.bin, wrong amounts count = " +
@@ -11173,7 +11246,7 @@ void wallet2::check_tx_key_helper(const crypto::hash &txid, const crypto::key_de
     const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex};
     uint64_t pre_call_credits = m_rpc_payment_state.credits;
     req.client = get_client_signature();
-    ok = epee::net_utils::invoke_http_json("/gettransactions", req, res, m_http_client);
+    ok = epee::net_utils::invoke_http_json("/gettransactions", req, res, *m_http_client);
     THROW_WALLET_EXCEPTION_IF(!ok || (res.txs.size() != 1 && res.txs_as_hex.size() != 1),
       error::wallet_internal_error, "Failed to get transaction from daemon");
     check_rpc_cost("/gettransactions", res.credits, pre_call_credits, COST_PER_TX);
@@ -11228,7 +11301,7 @@ std::string wallet2::get_tx_proof(const crypto::hash &txid, const cryptonote::ac
       const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex};
       uint64_t pre_call_credits = m_rpc_payment_state.credits;
       req.client = get_client_signature();
-      ok = net_utils::invoke_http_json("/gettransactions", req, res, m_http_client);
+      ok = net_utils::invoke_http_json("/gettransactions", req, res, *m_http_client);
       THROW_WALLET_EXCEPTION_IF(!ok || (res.txs.size() != 1 && res.txs_as_hex.size() != 1),
         error::wallet_internal_error, "Failed to get transaction from daemon");
       check_rpc_cost("/gettransactions", res.credits, pre_call_credits, COST_PER_TX);
@@ -11389,7 +11462,7 @@ bool wallet2::check_tx_proof(const crypto::hash &txid, const cryptonote::account
     const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex};
     uint64_t pre_call_credits = m_rpc_payment_state.credits;
     req.client = get_client_signature();
-    ok = net_utils::invoke_http_json("/gettransactions", req, res, m_http_client);
+    ok = net_utils::invoke_http_json("/gettransactions", req, res, *m_http_client);
     THROW_WALLET_EXCEPTION_IF(!ok || (res.txs.size() != 1 && res.txs_as_hex.size() != 1),
       error::wallet_internal_error, "Failed to get transaction from daemon");
     check_rpc_cost("/gettransactions", res.credits, pre_call_credits, COST_PER_TX);
@@ -11686,7 +11759,7 @@ bool wallet2::check_reserve_proof(const cryptonote::account_public_address &addr
     const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex};
     uint64_t pre_call_credits = m_rpc_payment_state.credits;
     gettx_req.client = get_client_signature();
-    bool ok = net_utils::invoke_http_json("/gettransactions", gettx_req, gettx_res, m_http_client);
+    bool ok = net_utils::invoke_http_json("/gettransactions", gettx_req, gettx_res, *m_http_client);
     THROW_WALLET_EXCEPTION_IF(!ok || gettx_res.txs.size() != proofs.size(),
       error::wallet_internal_error, "Failed to get transaction from daemon");
     check_rpc_cost("/gettransactions", gettx_res.credits, pre_call_credits, gettx_res.txs.size() * COST_PER_TX);
@@ -11703,7 +11776,7 @@ bool wallet2::check_reserve_proof(const cryptonote::account_public_address &addr
     const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex};
     uint64_t pre_call_credits = m_rpc_payment_state.credits;
     kispent_req.client = get_client_signature();
-    ok = epee::net_utils::invoke_http_json("/is_key_image_spent", kispent_req, kispent_res, m_http_client, rpc_timeout);
+    ok = epee::net_utils::invoke_http_json("/is_key_image_spent", kispent_req, kispent_res, *m_http_client, rpc_timeout);
     THROW_WALLET_EXCEPTION_IF(!ok || kispent_res.spent_status.size() != proofs.size(),
       error::wallet_internal_error, "Failed to get key image spent status from daemon");
     check_rpc_cost("/is_key_image_spent", kispent_res.credits, pre_call_credits, kispent_res.spent_status.size() * COST_PER_KEY_IMAGE);
@@ -12266,7 +12339,7 @@ uint64_t wallet2::import_key_images(const std::vector<std::pair<crypto::key_imag
       const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex};
       uint64_t pre_call_credits = m_rpc_payment_state.credits;
       req.client = get_client_signature();
-      bool r = epee::net_utils::invoke_http_json("/is_key_image_spent", req, daemon_resp, m_http_client, rpc_timeout);
+      bool r = epee::net_utils::invoke_http_json("/is_key_image_spent", req, daemon_resp, *m_http_client, rpc_timeout);
       THROW_ON_RPC_RESPONSE_ERROR_GENERIC(r, {},  daemon_resp, "is_key_image_spent");
       THROW_WALLET_EXCEPTION_IF(daemon_resp.spent_status.size() != signed_key_images.size(), error::wallet_internal_error,
         "daemon returned wrong response for is_key_image_spent, wrong amounts count = " +
@@ -12355,7 +12428,7 @@ uint64_t wallet2::import_key_images(const std::vector<std::pair<crypto::key_imag
       const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex};
       gettxs_req.client = get_client_signature();
       uint64_t pre_call_credits = m_rpc_payment_state.credits;
-      bool r = epee::net_utils::invoke_http_json("/gettransactions", gettxs_req, gettxs_res, m_http_client, rpc_timeout);
+      bool r = epee::net_utils::invoke_http_json("/gettransactions", gettxs_req, gettxs_res, *m_http_client, rpc_timeout);
       THROW_ON_RPC_RESPONSE_ERROR_GENERIC(r, {}, gettxs_res, "gettransactions");
       THROW_WALLET_EXCEPTION_IF(gettxs_res.txs.size() != spent_txids.size(), error::wallet_internal_error,
         "daemon returned wrong response for gettransactions, wrong count = " + std::to_string(gettxs_res.txs.size()) + ", expected " + std::to_string(spent_txids.size()));
@@ -13296,7 +13369,7 @@ uint64_t wallet2::get_blockchain_height_by_date(uint16_t year, uint8_t month, ui
       const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex};
       uint64_t pre_call_credits = m_rpc_payment_state.credits;
       req.client = get_client_signature();
-      r = net_utils::invoke_http_bin("/getblocks_by_height.bin", req, res, m_http_client, rpc_timeout);
+      r = net_utils::invoke_http_bin("/getblocks_by_height.bin", req, res, *m_http_client, rpc_timeout);
       if (r && res.status == CORE_RPC_STATUS_OK)
         check_rpc_cost("/getblocks_by_height.bin", res.credits, pre_call_credits, 3 * COST_PER_BLOCK);
     }
@@ -13374,7 +13447,7 @@ std::vector<std::pair<uint64_t, uint64_t>> wallet2::estimate_backlog(const std::
     const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex};
     uint64_t pre_call_credits = m_rpc_payment_state.credits;
     req.client = get_client_signature();
-    bool r = net_utils::invoke_http_json_rpc("/json_rpc", "get_txpool_backlog", req, res, m_http_client, rpc_timeout);
+    bool r = net_utils::invoke_http_json_rpc("/json_rpc", "get_txpool_backlog", req, res, *m_http_client, rpc_timeout);
     THROW_ON_RPC_RESPONSE_ERROR(r, {}, res, "get_txpool_backlog", error::get_tx_pool_error);
     check_rpc_cost("get_txpool_backlog", res.credits, pre_call_credits, COST_PER_TX_POOL_STATS * res.backlog.size());
   }
@@ -13532,10 +13605,12 @@ boost::optional<epee::wipeable_string> wallet2::on_device_pin_request()
   return boost::none;
 }
 //----------------------------------------------------------------------------------------------------
-boost::optional<epee::wipeable_string> wallet2::on_device_passphrase_request(bool on_device)
+boost::optional<epee::wipeable_string> wallet2::on_device_passphrase_request(bool & on_device)
 {
   if (nullptr != m_callback)
     return m_callback->on_device_passphrase_request(on_device);
+  else
+    on_device = true;
   return boost::none;
 }
 //----------------------------------------------------------------------------------------------------
@@ -13709,12 +13784,12 @@ void wallet2::finish_rescan_bc_keep_key_images(uint64_t transfer_height, const c
 //----------------------------------------------------------------------------------------------------
 uint64_t wallet2::get_bytes_sent() const
 {
-  return m_http_client.get_bytes_sent();
+  return m_http_client->get_bytes_sent();
 }
 //----------------------------------------------------------------------------------------------------
 uint64_t wallet2::get_bytes_received() const
 {
-  return m_http_client.get_bytes_received();
+  return m_http_client->get_bytes_received();
 }
 //----------------------------------------------------------------------------------------------------
 std::vector<cryptonote::public_node> wallet2::get_public_nodes(bool white_only)
@@ -13727,7 +13802,7 @@ std::vector<cryptonote::public_node> wallet2::get_public_nodes(bool white_only)
 
   {
     const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex};
-    bool r = epee::net_utils::invoke_http_json("/get_public_nodes", req, res, m_http_client, rpc_timeout);
+    bool r = epee::net_utils::invoke_http_json("/get_public_nodes", req, res, *m_http_client, rpc_timeout);
     THROW_ON_RPC_RESPONSE_ERROR_GENERIC(r, {}, res, "/get_public_nodes");
   }
 

src/wallet/wallet2.h

@@ -145,7 +145,7 @@ private:
     virtual void on_device_button_request(uint64_t code) {}
     virtual void on_device_button_pressed() {}
     virtual boost::optional<epee::wipeable_string> on_device_pin_request() { return boost::none; }
-    virtual boost::optional<epee::wipeable_string> on_device_passphrase_request(bool on_device) { return boost::none; }
+    virtual boost::optional<epee::wipeable_string> on_device_passphrase_request(bool & on_device) { on_device = true; return boost::none; }
     virtual void on_device_progress(const hw::device_progress& event) {};
     // Common callbacks
     virtual void on_pool_tx_removed(const crypto::hash &txid) {}
@@ -159,7 +159,7 @@ private:
     void on_button_request(uint64_t code=0) override;
     void on_button_pressed() override;
     boost::optional<epee::wipeable_string> on_pin_request() override;
-    boost::optional<epee::wipeable_string> on_passphrase_request(bool on_device) override;
+    boost::optional<epee::wipeable_string> on_passphrase_request(bool & on_device) override;
     void on_progress(const hw::device_progress& event) override;
   private:
     wallet2 * wallet;
@@ -269,7 +269,7 @@ private:
     static bool verify_password(const std::string& keys_file_name, const epee::wipeable_string& password, bool no_spend_key, hw::device &hwdev, uint64_t kdf_rounds);
     static bool query_device(hw::device::device_type& device_type, const std::string& keys_file_name, const epee::wipeable_string& password, uint64_t kdf_rounds = 1);
 
-    wallet2(cryptonote::network_type nettype = cryptonote::MAINNET, uint64_t kdf_rounds = 1, bool unattended = false);
+    wallet2(cryptonote::network_type nettype = cryptonote::MAINNET, uint64_t kdf_rounds = 1, bool unattended = false, std::unique_ptr<epee::net_utils::http::http_client_factory> http_client_factory = std::unique_ptr<epee::net_utils::http::http_simple_client_factory>(new epee::net_utils::http::http_simple_client_factory()));
     ~wallet2();
 
     struct multisig_info
@@ -708,7 +708,7 @@ private:
      */
     void rewrite(const std::string& wallet_name, const epee::wipeable_string& password);
     void write_watch_only_wallet(const std::string& wallet_name, const epee::wipeable_string& password, std::string &new_keys_filename);
-    void load(const std::string& wallet, const epee::wipeable_string& password);
+    void load(const std::string& wallet, const epee::wipeable_string& password, const std::string& keys_buf = "", const std::string& cache_buf = "");
     void store();
     /*!
      * \brief store_to  Stores wallet to another file(s), deleting old ones
@@ -716,6 +716,19 @@ private:
      * \param password  Password to protect new wallet (TODO: probably better save the password in the wallet object?)
      */
     void store_to(const std::string &path, const epee::wipeable_string &password);
+    /*!
+     * \brief get_keys_file_data  Get wallet keys data which can be stored to a wallet file.
+     * \param password            Password of the encrypted wallet buffer (TODO: probably better save the password in the wallet object?)
+     * \param watch_only          true to include only view key, false to include both spend and view keys
+     * \return                    Encrypted wallet keys data which can be stored to a wallet file
+     */
+    boost::optional<wallet2::keys_file_data> get_keys_file_data(const epee::wipeable_string& password, bool watch_only);
+    /*!
+     * \brief get_cache_file_data   Get wallet cache data which can be stored to a wallet file.
+     * \param password              Password to protect the wallet cache data (TODO: probably better save the password in the wallet object?)
+     * \return                      Encrypted wallet cache data which can be stored to a wallet file
+     */
+    boost::optional<wallet2::cache_file_data> get_cache_file_data(const epee::wipeable_string& password);
 
     std::string path() const;
 
@@ -1319,25 +1332,25 @@ private:
     crypto::public_key get_multisig_signing_public_key(const crypto::secret_key &skey) const;
 
     template<class t_request, class t_response>
-    inline bool invoke_http_json(const boost::string_ref uri, const t_request& req, t_response& res, std::chrono::milliseconds timeout = std::chrono::seconds(15), const boost::string_ref http_method = "GET")
+    inline bool invoke_http_json(const boost::string_ref uri, const t_request& req, t_response& res, std::chrono::milliseconds timeout = std::chrono::seconds(15), const boost::string_ref http_method = "POST")
     {
       if (m_offline) return false;
       boost::lock_guard<boost::recursive_mutex> lock(m_daemon_rpc_mutex);
-      return epee::net_utils::invoke_http_json(uri, req, res, m_http_client, timeout, http_method);
+      return epee::net_utils::invoke_http_json(uri, req, res, *m_http_client, timeout, http_method);
     }
     template<class t_request, class t_response>
-    inline bool invoke_http_bin(const boost::string_ref uri, const t_request& req, t_response& res, std::chrono::milliseconds timeout = std::chrono::seconds(15), const boost::string_ref http_method = "GET")
+    inline bool invoke_http_bin(const boost::string_ref uri, const t_request& req, t_response& res, std::chrono::milliseconds timeout = std::chrono::seconds(15), const boost::string_ref http_method = "POST")
     {
       if (m_offline) return false;
       boost::lock_guard<boost::recursive_mutex> lock(m_daemon_rpc_mutex);
-      return epee::net_utils::invoke_http_bin(uri, req, res, m_http_client, timeout, http_method);
+      return epee::net_utils::invoke_http_bin(uri, req, res, *m_http_client, timeout, http_method);
     }
     template<class t_request, class t_response>
-    inline bool invoke_http_json_rpc(const boost::string_ref uri, const std::string& method_name, const t_request& req, t_response& res, std::chrono::milliseconds timeout = std::chrono::seconds(15), const boost::string_ref http_method = "GET", const std::string& req_id = "0")
+    inline bool invoke_http_json_rpc(const boost::string_ref uri, const std::string& method_name, const t_request& req, t_response& res, std::chrono::milliseconds timeout = std::chrono::seconds(15), const boost::string_ref http_method = "POST", const std::string& req_id = "0")
     {
       if (m_offline) return false;
       boost::lock_guard<boost::recursive_mutex> lock(m_daemon_rpc_mutex);
-      return epee::net_utils::invoke_http_json_rpc(uri, method_name, req, res, m_http_client, timeout, http_method, req_id);
+      return epee::net_utils::invoke_http_json_rpc(uri, method_name, req, res, *m_http_client, timeout, http_method, req_id);
     }
 
     bool set_ring_database(const std::string &filename);
@@ -1403,11 +1416,18 @@ private:
      */
     bool store_keys(const std::string& keys_file_name, const epee::wipeable_string& password, bool watch_only = false);
     /*!
-     * \brief Load wallet information from wallet file.
+     * \brief Load wallet keys information from wallet file.
      * \param keys_file_name Name of wallet file
      * \param password       Password of wallet file
      */
     bool load_keys(const std::string& keys_file_name, const epee::wipeable_string& password);
+    /*!
+     * \brief Load wallet keys information from a string buffer.
+     * \param keys_buf       Keys buffer to load
+     * \param password       Password of keys buffer
+     */
+    bool load_keys_buf(const std::string& keys_buf, const epee::wipeable_string& password);
+    bool load_keys_buf(const std::string& keys_buf, const epee::wipeable_string& password, boost::optional<crypto::chacha_key>& keys_to_encrypt);
     void process_new_transaction(const crypto::hash &txid, const cryptonote::transaction& tx, const std::vector<uint64_t> &o_indices, uint64_t height, uint8_t block_version, uint64_t ts, bool miner_tx, bool pool, bool double_spend_seen, const tx_cache_data &tx_cache_data, std::map<std::pair<uint64_t, uint64_t>, size_t> *output_tracker_cache = NULL);
     bool should_skip_block(const cryptonote::block &b, uint64_t height) const;
     void process_new_blockchain_entry(const cryptonote::block& b, const cryptonote::block_complete_entry& bche, const parsed_block &parsed_block, const crypto::hash& bl_id, uint64_t height, const std::vector<tx_cache_data> &tx_cache_data, size_t tx_cache_data_offset, std::map<std::pair<uint64_t, uint64_t>, size_t> *output_tracker_cache = NULL);
@@ -1487,7 +1507,7 @@ private:
     void on_device_button_request(uint64_t code);
     void on_device_button_pressed();
     boost::optional<epee::wipeable_string> on_device_pin_request();
-    boost::optional<epee::wipeable_string> on_device_passphrase_request(bool on_device);
+    boost::optional<epee::wipeable_string> on_device_passphrase_request(bool & on_device);
     void on_device_progress(const hw::device_progress& event);
 
     std::string get_rpc_status(const std::string &s) const;
@@ -1502,7 +1522,7 @@ private:
     std::string m_wallet_file;
     std::string m_keys_file;
     std::string m_mms_file;
-    epee::net_utils::http::http_simple_client m_http_client;
+    const std::unique_ptr<epee::net_utils::http::abstract_http_client> m_http_client;
     hashchain m_blockchain;
     std::unordered_map<crypto::hash, unconfirmed_transfer_details> m_unconfirmed_txs;
     std::unordered_map<crypto::hash, confirmed_transfer_details> m_confirmed_txs;

src/wallet/wallet_rpc_payments.cpp

@@ -85,7 +85,7 @@ bool wallet2::make_rpc_payment(uint32_t nonce, uint32_t cookie, uint64_t &credit
   uint64_t pre_call_credits = m_rpc_payment_state.credits;
   req.client = get_client_signature();
   epee::json_rpc::error error;
-  bool r = epee::net_utils::invoke_http_json_rpc("/json_rpc", "rpc_access_submit_nonce", req, res, error, m_http_client, rpc_timeout);
+  bool r = epee::net_utils::invoke_http_json_rpc("/json_rpc", "rpc_access_submit_nonce", req, res, error, *m_http_client, rpc_timeout);
   m_daemon_rpc_mutex.unlock();
   THROW_ON_RPC_RESPONSE_ERROR_GENERIC(r, error, res, "rpc_access_submit_nonce");
   THROW_WALLET_EXCEPTION_IF(res.credits < pre_call_credits, error::wallet_internal_error, "RPC payment did not increase balance");

tests/functional_tests/functional_tests_rpc.py

@@ -34,8 +34,8 @@ try:
 except:
   tests = DEFAULT_TESTS
 
-N_MONERODS = 2
+N_MONERODS = 3
-N_WALLETS = 4
+N_WALLETS = 7
 WALLET_DIRECTORY = builddir + "/functional-tests-directory"
 DIFFICULTY = 10
 
@@ -43,9 +43,17 @@ monerod_base = [builddir + "/bin/monerod", "--regtest", "--fixed-difficulty", st
 monerod_extra = [
   [],
   ["--rpc-payment-address", "44SKxxLQw929wRF6BA9paQ1EWFshNnKhXM3qz6Mo3JGDE2YG3xyzVutMStEicxbQGRfrYvAAYxH6Fe8rnD56EaNwUiqhcwR", "--rpc-payment-difficulty", str(DIFFICULTY), "--rpc-payment-credits", "5000", "--data-dir", builddir + "/functional-tests-directory/monerod1"],
+  ["--rpc-restricted-bind-port", "18482", "--data-dir", builddir + "/functional-tests-directory/monerod2"]
 ]
-wallet_base = [builddir + "/bin/monero-wallet-rpc", "--wallet-dir", WALLET_DIRECTORY, "--rpc-bind-port", "wallet_port", "--disable-rpc-login", "--rpc-ssl", "disabled", "--daemon-ssl", "disabled", "--daemon-port", "18180", "--log-level", "1"]
+wallet_base = [builddir + "/bin/monero-wallet-rpc", "--wallet-dir", WALLET_DIRECTORY, "--rpc-bind-port", "wallet_port", "--disable-rpc-login", "--rpc-ssl", "disabled", "--daemon-ssl", "disabled", "--log-level", "1"]
 wallet_extra = [
+  ["--daemon-port", "18180"],
+  ["--daemon-port", "18180"],
+  ["--daemon-port", "18180"],
+  ["--daemon-port", "18180"],
+  ["--daemon-port", "18182"],
+  ["--daemon-port", "18182"],
+  ["--daemon-port", "18182"]
 ]
 
 command_lines = []

tests/functional_tests/make_test_signature.cc

@@ -27,6 +27,7 @@
 // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include <stdio.h>
+#include "misc_language.h"
 #include "string_tools.h"
 #include "rpc/rpc_payment_signature.h"
 
@@ -69,6 +70,7 @@ int main(int argc, const char **argv)
   while (count--)
   {
     std::string signature = cryptonote::make_rpc_payment_signature(skey);
+    epee::misc_utils::sleep_no_w(1);
     printf("%s\n", signature.c_str());
   }
   return 0;

tests/functional_tests/rpc_payment.py

@@ -59,12 +59,14 @@ class RPCPaymentTest():
         return fields
 
     def refill_signatures(self):
+        self.signatures_time = time.time()
+        self.signatures = []
         signatures = subprocess.check_output([self.make_test_signature, self.secret_key, '256']).decode('utf-8')
         for line in signatures.split():
             self.signatures.append(line.rstrip())
 
     def get_signature(self):
-        if len(self.signatures) == 0:
+        if len(self.signatures) == 0 or self.signatures_time + 10 < time.time():
             self.refill_signatures()
         s = self.signatures[0]
         self.signatures = self.signatures[1:]

tests/functional_tests/transfer.py

@@ -55,7 +55,7 @@ class TransferTest():
 
     def reset(self):
         print('Resetting blockchain')
-        daemon = Daemon()
+        daemon = Daemon(idx = 2)
         res = daemon.get_height()
         daemon.pop_blocks(res.height - 1)
         daemon.flush_txpool()
@@ -69,7 +69,7 @@ class TransferTest():
         ]
         self.wallet = [None] * len(seeds)
         for i in range(len(seeds)):
-            self.wallet[i] = Wallet(idx = i)
+            self.wallet[i] = Wallet(idx = i + 4)
             # close the wallet if any, will throw if none is loaded
             try: self.wallet[i].close_wallet()
             except: pass
@@ -77,7 +77,7 @@ class TransferTest():
 
     def mine(self):
         print("Mining some blocks")
-        daemon = Daemon()
+        daemon = Daemon(idx = 2)
 
         res = daemon.get_info()
         height = res.height
@@ -89,7 +89,7 @@ class TransferTest():
             assert res.height == height + 80
 
     def transfer(self):
-        daemon = Daemon()
+        daemon = Daemon(idx = 2)
 
         print("Creating transfer to self")
 
@@ -508,7 +508,7 @@ class TransferTest():
     def check_get_bulk_payments(self):
         print('Checking get_bulk_payments')
 
-        daemon = Daemon()
+        daemon = Daemon(idx = 2)
         res = daemon.get_info()
         height = res.height
 
@@ -544,7 +544,7 @@ class TransferTest():
     def check_get_payments(self):
         print('Checking get_payments')
 
-        daemon = Daemon()
+        daemon = Daemon(idx = 2)
         res = daemon.get_info()
         height = res.height
 
@@ -587,7 +587,8 @@ class TransferTest():
             assert len(res.tx_blob_list) == 1
             txes[i][1] = res.tx_blob_list[0]
 
-        daemon = Daemon()
+        daemon = Daemon(idx = 2)
+        restricted_daemon = Daemon(idx = 2, restricted_rpc = True)
         res = daemon.send_raw_transaction(txes[0][1])
         assert res.not_relayed == False
         assert res.low_mixin == False
@@ -598,6 +599,18 @@ class TransferTest():
         assert res.overspend == False
         assert res.fee_too_low == False
 
+        res = restricted_daemon.send_raw_transaction(txes[0][1])
+        assert res.not_relayed == False
+        assert res.low_mixin == False
+        assert res.double_spend == False
+        assert res.invalid_input == False
+        assert res.invalid_output == False
+        assert res.too_big == False
+        assert res.overspend == False
+        assert res.fee_too_low == False
+
+        res = restricted_daemon.get_transactions([txes[0][0]])
+        assert not 'txs' in res or len(res.txs) == 0
         res = daemon.get_transactions([txes[0][0]])
         assert len(res.txs) >= 1
         tx = [tx for tx in res.txs if tx.tx_hash == txes[0][0]][0]
@@ -615,6 +628,19 @@ class TransferTest():
         assert res.fee_too_low == False
         assert res.too_few_outputs == False
 
+        res = restricted_daemon.send_raw_transaction(txes[1][1])
+        assert res.not_relayed == False
+        assert res.low_mixin == False
+        assert res.double_spend == True
+        assert res.invalid_input == False
+        assert res.invalid_output == False
+        assert res.too_big == False
+        assert res.overspend == False
+        assert res.fee_too_low == False
+        assert res.too_few_outputs == False
+
+        res = restricted_daemon.get_transactions([txes[0][0]])
+        assert not 'txs' in res or len(res.txs) == 0
         res = daemon.get_transactions([txes[0][0]])
         assert len(res.txs) >= 1
         tx = [tx for tx in res.txs if tx.tx_hash == txes[0][0]][0]
@@ -623,13 +649,13 @@ class TransferTest():
 
     def sweep_dust(self):
         print("Sweeping dust")
-        daemon = Daemon()
+        daemon = Daemon(idx = 2)
         self.wallet[0].refresh()
         res = self.wallet[0].sweep_dust()
         assert not 'tx_hash_list' in res or len(res.tx_hash_list) == 0 # there's just one, but it cannot meet the fee
 
     def sweep_single(self):
-        daemon = Daemon()
+        daemon = Daemon(idx = 2)
 
         print("Sending single output")
 
@@ -685,7 +711,7 @@ class TransferTest():
         assert len([t for t in res.transfers if t.key_image == ki]) == 1
 
     def check_destinations(self):
-        daemon = Daemon()
+        daemon = Daemon(idx = 2)
 
         print("Checking transaction destinations")
 
@@ -741,7 +767,7 @@ class TransferTest():
                 self.wallet[0].refresh()
 
     def check_tx_notes(self):
-        daemon = Daemon()
+        daemon = Daemon(idx = 2)
 
         print('Testing tx notes')
         res = self.wallet[0].get_transfers()
@@ -758,7 +784,7 @@ class TransferTest():
         assert res.notes == ['out txid', 'in txid']
 
     def check_rescan(self):
-        daemon = Daemon()
+        daemon = Daemon(idx = 2)
 
         print('Testing rescan_spent')
         res = self.wallet[0].incoming_transfers(transfer_type = 'all')
@@ -798,7 +824,7 @@ class TransferTest():
             assert sorted(old_t_out, key = lambda k: k['txid']) == sorted(new_t_out, key = lambda k: k['txid'])
 
     def check_is_key_image_spent(self):
-        daemon = Daemon()
+        daemon = Daemon(idx = 2)
 
         print('Testing is_key_image_spent')
         res = self.wallet[0].incoming_transfers(transfer_type = 'all')

tests/functional_tests/txpool.py

@@ -45,14 +45,14 @@ class TransferTest():
 
     def reset(self):
         print('Resetting blockchain')
-        daemon = Daemon()
+        daemon = Daemon(idx=2)
         res = daemon.get_height()
         daemon.pop_blocks(res.height - 1)
         daemon.flush_txpool()
 
     def create(self):
         print('Creating wallet')
-        wallet = Wallet()
+        wallet = Wallet(idx = 4)
         # close the wallet if any, will throw if none is loaded
         try: wallet.close_wallet()
         except: pass
@@ -61,8 +61,8 @@ class TransferTest():
 
     def mine(self):
         print("Mining some blocks")
-        daemon = Daemon()
+        daemon = Daemon(idx = 2)
-        wallet = Wallet()
+        wallet = Wallet(idx = 4)
 
         daemon.generateblocks('42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm', 80)
         wallet.refresh()
@@ -70,8 +70,8 @@ class TransferTest():
     def create_txes(self, address, ntxes):
         print('Creating ' + str(ntxes) + ' transactions')
 
-        daemon = Daemon()
+        daemon = Daemon(idx = 2)
-        wallet = Wallet()
+        wallet = Wallet(idx = 4)
 
         dst = {'address': address, 'amount': 1000000000000}
 
@@ -83,8 +83,10 @@ class TransferTest():
         return txes
 
     def check_empty_pool(self):
-        daemon = Daemon()
+        self.check_empty_rpc_pool(Daemon(idx = 2))
+        self.check_empty_rpc_pool(Daemon(idx = 2, restricted_rpc = True))
 
+    def check_empty_rpc_pool(self, daemon):
         res = daemon.get_transaction_pool_hashes()
         assert not 'tx_hashes' in res or len(res.tx_hashes) == 0
         res = daemon.get_transaction_pool_stats()
@@ -103,8 +105,9 @@ class TransferTest():
         assert res.pool_stats.num_double_spends == 0
 
     def check_txpool(self):
-        daemon = Daemon()
+        daemon = Daemon(idx = 2)
-        wallet = Wallet()
+        restricted_daemon = Daemon(idx = 2, restricted_rpc = True)
+        wallet = Wallet(idx = 4)
 
         res = daemon.get_info()
         height = res.height
@@ -117,6 +120,7 @@ class TransferTest():
         res = daemon.get_info()
         assert res.tx_pool_size == txpool_size + 5
         txpool_size = res.tx_pool_size
+        self.check_empty_rpc_pool(restricted_daemon)
 
         res = daemon.get_transaction_pool()
         assert len(res.transactions) == txpool_size
@@ -160,6 +164,7 @@ class TransferTest():
         print('Flushing 2 transactions')
         txes_keys = list(txes.keys())
         daemon.flush_txpool([txes_keys[1], txes_keys[3]])
+        self.check_empty_rpc_pool(restricted_daemon)
         res = daemon.get_transaction_pool()
         assert len(res.transactions) == txpool_size - 2
         assert len([x for x in res.transactions if x.id_hash == txes_keys[1]]) == 0
@@ -210,6 +215,7 @@ class TransferTest():
         print('Flushing unknown transactions')
         unknown_txids = ['1'*64, '2'*64, '3'*64]
         daemon.flush_txpool(unknown_txids)
+        self.check_empty_rpc_pool(restricted_daemon)
         res = daemon.get_transaction_pool()
         assert len(res.transactions) == txpool_size - 2
 

tests/fuzz/http-client.cpp

@@ -29,6 +29,7 @@
 #include "include_base_utils.h"
 #include "file_io_utils.h"
 #include "net/http_client.h"
+#include "net/net_ssl.h"
 #include "fuzzer.h"
 
 class dummy_client
@@ -46,6 +47,10 @@ public:
     data.clear();
     return true;
   }
+  void set_ssl(epee::net_utils::ssl_options_t ssl_options) { }
+  bool is_connected(bool *ssl = NULL) { return true; }
+  uint64_t get_bytes_sent() const  { return 1; }
+  uint64_t get_bytes_received() const { return 1; }
 
   void set_test_data(const std::string &s) { data = s; }
 

tests/performance_tests/crypto_ops.h

@@ -48,6 +48,7 @@ enum test_op
   op_scalarmultKey,
   op_scalarmultH,
   op_scalarmult8,
+  op_scalarmult8_p3,
   op_ge_dsm_precomp,
   op_ge_double_scalarmult_base_vartime,
   op_ge_double_scalarmult_precomp_vartime,
@@ -105,6 +106,7 @@ public:
       case op_scalarmultKey: rct::scalarmultKey(point0, scalar0); break;
       case op_scalarmultH: rct::scalarmultH(scalar0); break;
       case op_scalarmult8: rct::scalarmult8(point0); break;
+      case op_scalarmult8_p3: rct::scalarmult8(p3_0,point0); break;
       case op_ge_dsm_precomp: ge_dsm_precomp(dsmp, &p3_0); break;
       case op_ge_double_scalarmult_base_vartime: ge_double_scalarmult_base_vartime(&tmp_p2, scalar0.bytes, &p3_0, scalar1.bytes); break;
       case op_ge_double_scalarmult_precomp_vartime: ge_double_scalarmult_precomp_vartime(&tmp_p2, scalar0.bytes, &p3_0, scalar1.bytes, precomp0); break;

tests/performance_tests/main.cpp

@@ -254,6 +254,7 @@ int main(int argc, char** argv)
   TEST_PERFORMANCE1(filter, p, test_crypto_ops, op_scalarmultKey);
   TEST_PERFORMANCE1(filter, p, test_crypto_ops, op_scalarmultH);
   TEST_PERFORMANCE1(filter, p, test_crypto_ops, op_scalarmult8);
+  TEST_PERFORMANCE1(filter, p, test_crypto_ops, op_scalarmult8_p3);
   TEST_PERFORMANCE1(filter, p, test_crypto_ops, op_ge_dsm_precomp);
   TEST_PERFORMANCE1(filter, p, test_crypto_ops, op_ge_double_scalarmult_base_vartime);
   TEST_PERFORMANCE1(filter, p, test_crypto_ops, op_ge_double_scalarmult_precomp_vartime);

tests/trezor/daemon.cpp

@@ -129,7 +129,7 @@ void mock_daemon::init()
   m_rpc_server.nettype(m_network_type);
 
   CHECK_AND_ASSERT_THROW_MES(m_protocol.init(m_vm), "Failed to initialize cryptonote protocol.");
-  CHECK_AND_ASSERT_THROW_MES(m_rpc_server.init(m_vm, false, main_rpc_port), "Failed to initialize RPC server.");
+  CHECK_AND_ASSERT_THROW_MES(m_rpc_server.init(m_vm, false, main_rpc_port, false), "Failed to initialize RPC server.");
 
   if (m_start_p2p)
     CHECK_AND_ASSERT_THROW_MES(m_server.init(m_vm), "Failed to initialize p2p server.");
@@ -313,7 +313,7 @@ void mock_daemon::mine_blocks(size_t num_blocks, const std::string &miner_addres
 {
   bool blocks_mined = false;
   const uint64_t start_height = get_height();
-  const auto mining_timeout = std::chrono::seconds(30);
+  const auto mining_timeout = std::chrono::seconds(120);
   MDEBUG("Current height before mining: " << start_height);
 
   start_mining(miner_address);

tests/trezor/daemon.h

@@ -76,7 +76,7 @@ public:
   typedef cryptonote::t_cryptonote_protocol_handler<cryptonote::core> t_protocol_raw;
   typedef nodetool::node_server<t_protocol_raw> t_node_server;
 
-  static constexpr const std::chrono::seconds rpc_timeout = std::chrono::seconds(60);
+  static constexpr const std::chrono::seconds rpc_timeout = std::chrono::seconds(120);
 
   cryptonote::core * m_core;
   t_protocol_raw m_protocol;

tests/trezor/trezor_tests.cpp

@@ -38,6 +38,7 @@
 using namespace cryptonote;
 
 #include <boost/regex.hpp>
+#include <common/apply_permutation.h>
 #include "common/util.h"
 #include "common/command_line.h"
 #include "trezor_tests.h"
@@ -72,9 +73,10 @@ namespace
 #define TREZOR_SETUP_CHAIN(NAME) do {                                                                           \
   ++tests_count;                                                                                                \
   try {                                                                                                         \
-    setup_chain(core, trezor_base, chain_path, fix_chain, vm_core);                                            \
+    setup_chain(core, trezor_base, chain_path, fix_chain, vm_core);                                             \
   } catch (const std::exception& ex) {                                                                          \
-    failed_tests.emplace_back("gen_trezor_base " #NAME);                                                        \
+    MERROR("Chain setup failed for " << NAME);                                                                  \
+    throw;                                                                                                      \
   }                                                                                                             \
 } while(0)
 
@@ -136,10 +138,11 @@ int main(int argc, char* argv[])
     hw::register_device(HW_TREZOR_NAME, ensure_trezor_test_device());  // shim device for call tracking
 
     // Bootstrapping common chain & accounts
-    const uint8_t initial_hf =  (uint8_t)get_env_long("TEST_MIN_HF", 11);
+    const uint8_t initial_hf =  (uint8_t)get_env_long("TEST_MIN_HF", 12);
-    const uint8_t max_hf = (uint8_t)get_env_long("TEST_MAX_HF", 11);
+    const uint8_t max_hf = (uint8_t)get_env_long("TEST_MAX_HF", 12);
+    auto sync_test = get_env_long("TEST_KI_SYNC", 1);
     MINFO("Test versions " << MONERO_RELEASE_NAME << "' (v" << MONERO_VERSION_FULL << ")");
-    MINFO("Testing hardforks [" << (int)initial_hf << ", " << (int)max_hf << "]");
+    MINFO("Testing hardforks [" << (int)initial_hf << ", " << (int)max_hf << "], sync-test: " << sync_test);
 
     cryptonote::core core_obj(nullptr);
     cryptonote::core * const core = &core_obj;
@@ -181,7 +184,7 @@ int main(int argc, char* argv[])
       trezor_base.daemon(daemon);
 
       // Hard-fork independent tests
-      if (hf == initial_hf)
+      if (hf == initial_hf && sync_test > 0)
       {
         TREZOR_COMMON_TEST_CASE(gen_trezor_ki_sync_without_refresh, core, trezor_base);
         TREZOR_COMMON_TEST_CASE(gen_trezor_live_refresh, core, trezor_base);
@@ -191,7 +194,6 @@ int main(int argc, char* argv[])
       TREZOR_COMMON_TEST_CASE(gen_trezor_1utxo, core, trezor_base);
       TREZOR_COMMON_TEST_CASE(gen_trezor_1utxo_paymentid_short, core, trezor_base);
       TREZOR_COMMON_TEST_CASE(gen_trezor_1utxo_paymentid_short_integrated, core, trezor_base);
-      TREZOR_COMMON_TEST_CASE(gen_trezor_1utxo_paymentid_long, core, trezor_base);
       TREZOR_COMMON_TEST_CASE(gen_trezor_4utxo, core, trezor_base);
       TREZOR_COMMON_TEST_CASE(gen_trezor_4utxo_acc1, core, trezor_base);
       TREZOR_COMMON_TEST_CASE(gen_trezor_4utxo_to_sub, core, trezor_base);
@@ -338,10 +340,7 @@ static void setup_chain(cryptonote::core * core, gen_trezor_base & trezor_base,
     if (!unserialize_chain_from_file(events, trezor_base, chain_path))
     {
       MERROR("Failed to deserialize data from file: " << chain_path);
-      if (!fix_chain)
+      CHECK_AND_ASSERT_THROW_MES(fix_chain, "Chain load error");
-      {
-        throw std::runtime_error("Chain load error");
-      }
     } else
     {
       trezor_base.load(events);
@@ -648,6 +647,8 @@ void gen_trezor_base::fork(gen_trezor_base & other)
   other.m_alice_account = m_alice_account;
   other.m_eve_account = m_eve_account;
   other.m_trezor = m_trezor;
+  other.m_generator.set_events(&other.m_events);
+  other.m_generator.set_network_type(m_network_type);
 }
 
 void gen_trezor_base::clear()
@@ -700,6 +701,8 @@ bool gen_trezor_base::generate(std::vector<test_event_entry>& events)
 
   // Events, custom genesis so it matches wallet genesis
   auto & generator = m_generator;  // macro shortcut
+  generator.set_events(&events);
+  generator.set_network_type(m_network_type);
 
   cryptonote::block blk_gen;
   std::vector<size_t> block_weights;
@@ -852,6 +855,8 @@ void gen_trezor_base::load(std::vector<test_event_entry>& events)
 {
   init_fields();
   m_events = events;
+  m_generator.set_events(&m_events);
+  m_generator.set_network_type(m_network_type);
 
   unsigned acc_idx = 0;
   cryptonote::account_base * accounts[] = {TREZOR_ACCOUNT_ORDERING};
@@ -919,29 +924,19 @@ void gen_trezor_base::rewind_blocks(std::vector<test_event_entry>& events, size_
 void gen_trezor_base::fix_hf(std::vector<test_event_entry>& events)
 {
   // If current test requires higher hard-fork, move it up
-  const auto current_hf = m_hard_forks.back().first;
+  auto current_hf = m_hard_forks.back().first;
-
+  CHECK_AND_ASSERT_THROW_MES(current_hf <= m_top_hard_fork, "Generated chain hardfork is higher than desired maximum");
-  if (current_hf > m_top_hard_fork)
+  CHECK_AND_ASSERT_THROW_MES(m_rct_config.bp_version != 2 || m_top_hard_fork >= 10, "Desired maximum is too low for BPv2");
-  {
-    throw std::runtime_error("Generated chain hardfork is higher than desired maximum");
-  }
-
-  if (m_rct_config.bp_version == 2 && m_top_hard_fork < 10)
-  {
-    throw std::runtime_error("Desired maximum is too low for BPv2");
-  }
 
-  if (current_hf < m_top_hard_fork)
+  for(;current_hf < m_top_hard_fork; current_hf+=1)
   {
+    auto const hf_to_add = current_hf + 1;
     auto hardfork_height = num_blocks(events);
-    ADD_HARDFORK(m_hard_forks, m_top_hard_fork, hardfork_height);
-    add_top_hfork(events, m_hard_forks);
-    MDEBUG("Hardfork added at height: " << hardfork_height << ", from " << (int)current_hf << " to " << (int)m_top_hard_fork);
 
-    if (current_hf < 10)
+    ADD_HARDFORK(m_hard_forks, hf_to_add, hardfork_height);
-    { // buffer blocks, add 10 to apply v10 rules
+    add_top_hfork(events, m_hard_forks);
-      rewind_blocks(events, 10, m_top_hard_fork);
+    MDEBUG("Hardfork added at height: " << hardfork_height << ", from " << (int)current_hf << " to " << (int)hf_to_add);
-    }
+    rewind_blocks(events, 10, hf_to_add);
   }
 }
 
@@ -1271,7 +1266,6 @@ void gen_trezor_base::set_hard_fork(uint8_t hf)
 
 #define TREZOR_SKIP_IF_VERSION_LEQ(x) if (m_trezor->get_version() <= x) { MDEBUG("Test skipped"); return true; }
 #define TREZOR_TEST_PAYMENT_ID "\xde\xad\xc0\xde\xde\xad\xc0\xde"
-#define TREZOR_TEST_PAYMENT_ID_LONG "\xde\xad\xc0\xde\xde\xad\xc0\xde\xde\xad\xc0\xde\xde\xad\xc0\xde\xde\xad\xc0\xde\xde\xad\xc0\xde\xde\xad\xc0\xde\xde\xad\xc0\xde"
 
 tsx_builder * tsx_builder::sources(std::vector<cryptonote::tx_source_entry> & sources, std::vector<size_t> & selected_transfers)
 {
@@ -1424,13 +1418,26 @@ tsx_builder * tsx_builder::construct_pending_tx(tools::wallet2::pending_tx &ptx,
   std::vector<crypto::secret_key> additional_tx_keys;
   std::vector<tx_destination_entry> destinations_copy = m_destinations;
 
+  auto sources_copy = m_sources;
   auto change_addr = m_from->get_account().get_keys().m_account_address;
   bool r = construct_tx_and_get_tx_key(m_from->get_account().get_keys(), subaddresses, m_sources, destinations_copy,
                                        change_addr, extra ? extra.get() : std::vector<uint8_t>(), tx, 0, tx_key,
                                        additional_tx_keys, true, m_rct_config, nullptr);
-
   CHECK_AND_ASSERT_THROW_MES(r, "Transaction construction failed");
 
+  // Selected transfers permutation
+  std::vector<size_t> ins_order;
+  for (size_t n = 0; n < m_sources.size(); ++n)
+  {
+    for (size_t idx = 0; idx < sources_copy.size(); ++idx)
+    {
+      CHECK_AND_ASSERT_THROW_MES((size_t)sources_copy[idx].real_output < sources_copy[idx].outputs.size(), "Invalid real_output");
+      if (sources_copy[idx].outputs[sources_copy[idx].real_output].second.dest == m_sources[n].outputs[m_sources[n].real_output].second.dest)
+        ins_order.push_back(idx);
+    }
+  }
+  CHECK_AND_ASSERT_THROW_MES(ins_order.size() == m_sources.size(), "Failed to work out sources permutation");
+
   ptx.key_images = "";
   ptx.fee = TESTS_DEFAULT_FEE;
   ptx.dust = 0;
@@ -1438,6 +1445,7 @@ tsx_builder * tsx_builder::construct_pending_tx(tools::wallet2::pending_tx &ptx,
   ptx.tx = tx;
   ptx.change_dts = m_destinations.back();
   ptx.selected_transfers = m_selected_transfers;
+  tools::apply_permutation(ins_order, ptx.selected_transfers);
   ptx.tx_key = tx_key;
   ptx.additional_tx_keys = additional_tx_keys;
   ptx.dests = m_destinations;
@@ -1671,22 +1679,6 @@ bool gen_trezor_1utxo_paymentid_short_integrated::generate(std::vector<test_even
   TREZOR_TEST_SUFFIX();
 }
 
-bool gen_trezor_1utxo_paymentid_long::generate(std::vector<test_event_entry>& events)
-{
-  TREZOR_TEST_PREFIX();
-  t_builder->cur_height(num_blocks(events) - 1)
-      ->mixin(TREZOR_TEST_MIXIN)
-      ->fee(TREZOR_TEST_FEE)
-      ->from(m_wl_alice.get(), 0)
-      ->compute_sources(boost::none, MK_COINS(1), -1, -1)
-      ->add_destination(m_eve_account, false, 1000)
-      ->payment_id(TREZOR_TEST_PAYMENT_ID_LONG)
-      ->rct_config(m_rct_config)
-      ->build_tx();
-
-  TREZOR_TEST_SUFFIX();
-}
-
 bool gen_trezor_4utxo::generate(std::vector<test_event_entry>& events)
 {
   TREZOR_TEST_PREFIX();

tests/trezor/trezor_tests.h

@@ -264,12 +264,6 @@ public:
   bool generate(std::vector<test_event_entry>& events) override;
 };
 
-class gen_trezor_1utxo_paymentid_long : public gen_trezor_base
-{
-public:
-  bool generate(std::vector<test_event_entry>& events) override;
-};
-
 class gen_trezor_4utxo : public gen_trezor_base
 {
 public:

tests/unit_tests/epee_utils.cpp

@@ -841,6 +841,9 @@ TEST(HexLocale, String)
     // decoding it this way also, ignoring spaces and colons between the numbers
     hex.assign("00:ff 0f:f0");
     EXPECT_EQ(source, epee::from_hex_locale::to_vector(hex));
+
+    hex.append("f0");
+    EXPECT_EQ(source, epee::from_hex_locale::to_vector(boost::string_ref{hex.data(), hex.size() - 2}));
 }
 
 TEST(ToHex, Array)

tests/unit_tests/levin.cpp

@@ -583,7 +583,9 @@ TEST_F(levin_notify, stem_without_padding)
         {
             const std::size_t sent = context->process_send_queue();
             if (sent && is_stem)
+            {
                 EXPECT_EQ(1u, (context - contexts_.begin()) % 2);
+            }
             send_count += sent;
         }
 
@@ -653,7 +655,9 @@ TEST_F(levin_notify, local_without_padding)
         {
             const std::size_t sent = context->process_send_queue();
             if (sent && is_stem)
+            {
                 EXPECT_EQ(1u, (context - contexts_.begin()) % 2);
+            }
             send_count += sent;
         }
 

tests/unit_tests/net.cpp

@@ -1702,6 +1702,45 @@ TEST(zmq, read_write)
     EXPECT_EQ(message, *received);
 }
 
+TEST(zmq, read_write_slice)
+{
+    net::zmq::context context{zmq_init(1)};
+    ASSERT_NE(nullptr, context);
+
+    net::zmq::socket send_socket{zmq_socket(context.get(), ZMQ_REQ)};
+    net::zmq::socket recv_socket{zmq_socket(context.get(), ZMQ_REP)};
+    ASSERT_NE(nullptr, send_socket);
+    ASSERT_NE(nullptr, recv_socket);
+
+    ASSERT_EQ(0u, zmq_bind(recv_socket.get(), "inproc://testing"));
+    ASSERT_EQ(0u, zmq_connect(send_socket.get(), "inproc://testing"));
+
+    std::string message;
+    message.resize(1024);
+    crypto::rand(message.size(), reinterpret_cast<std::uint8_t*>(std::addressof(message[0])));
+
+    {
+        epee::byte_slice slice_message{{epee::strspan<std::uint8_t>(message)}};
+        ASSERT_TRUE(bool(net::zmq::send(std::move(slice_message), send_socket.get())));
+        EXPECT_TRUE(slice_message.empty());
+    }
+
+    const expect<std::string> received = net::zmq::receive(recv_socket.get());
+    ASSERT_TRUE(bool(received));
+    EXPECT_EQ(message, *received);
+}
+
+TEST(zmq, write_slice_fail)
+{
+    std::string message;
+    message.resize(1024);
+    crypto::rand(message.size(), reinterpret_cast<std::uint8_t*>(std::addressof(message[0])));
+
+    epee::byte_slice slice_message{std::move(message)};
+    EXPECT_FALSE(bool(net::zmq::send(std::move(slice_message), nullptr)));
+    EXPECT_TRUE(slice_message.empty());
+}
+
 TEST(zmq, read_write_multipart)
 {
     net::zmq::context context{zmq_init(1)};

tests/unit_tests/ringct.cpp

@@ -1077,8 +1077,16 @@ TEST(ringct, H)
 
 TEST(ringct, mul8)
 {
+  ge_p3 p3;
+  rct::key key;
   ASSERT_EQ(rct::scalarmult8(rct::identity()), rct::identity());
+  rct::scalarmult8(p3,rct::identity());
+  ge_p3_tobytes(key.bytes, &p3);
+  ASSERT_EQ(key, rct::identity());
   ASSERT_EQ(rct::scalarmult8(rct::H), rct::scalarmultKey(rct::H, rct::EIGHT));
+  rct::scalarmult8(p3,rct::H);
+  ge_p3_tobytes(key.bytes, &p3);
+  ASSERT_EQ(key, rct::scalarmultKey(rct::H, rct::EIGHT));
   ASSERT_EQ(rct::scalarmultKey(rct::scalarmultKey(rct::H, rct::INV_EIGHT), rct::EIGHT), rct::H);
 }
 

utils/python-rpc/framework/daemon.py

@@ -32,10 +32,11 @@ from .rpc import JSONRPC
 
 class Daemon(object):
 
-    def __init__(self, protocol='http', host='127.0.0.1', port=0, idx=0):
+    def __init__(self, protocol='http', host='127.0.0.1', port=0, idx=0, restricted_rpc = False):
+        base = 18480 if restricted_rpc else 18180
         self.host = host
         self.port = port
-        self.rpc = JSONRPC('{protocol}://{host}:{port}'.format(protocol=protocol, host=host, port=port if port else 18180+idx))
+        self.rpc = JSONRPC('{protocol}://{host}:{port}'.format(protocol=protocol, host=host, port=port if port else base+idx))
 
     def getblocktemplate(self, address, prev_block = "", client = ""):
         getblocktemplate = {