Merge pull request #6737

0325474 epee: further defending against exceptions in command handlers (moneromooo-monero)

README.md

@@ -132,7 +132,7 @@ Dates are provided in the format YYYY-MM-DD.
 | 1686275                        | 2018-10-19 | v9                | v0.13.0.0              | v0.13.0.4                  | bulletproofs required
 | 1788000                        | 2019-03-09 | v10               | v0.14.0.0              | v0.14.1.2                  | New PoW based on Cryptonight-R, new block weight algorithm, slightly more efficient RingCT format
 | 1788720                        | 2019-03-10 | v11               | v0.14.0.0              | v0.14.1.2                  | forbid old RingCT transaction format
-| 1978433                        | 2019-11-30* | v12                | v0.15.0.0              | v0.15.0.0                  | New PoW based on RandomX, only allow >= 2 outputs, change to the block median used to calculate penalty, v1 coinbases are forbidden, rct sigs in coinbase forbidden, 10 block lock time for incoming outputs
+| 1978433                        | 2019-11-30* | v12                | v0.15.0.0              | v0.16.0.3                  | New PoW based on RandomX, only allow >= 2 outputs, change to the block median used to calculate penalty, v1 coinbases are forbidden, rct sigs in coinbase forbidden, 10 block lock time for incoming outputs
 | XXXXXXX                        | XXX-XX-XX | XXX                | vX.XX.X.X              | vX.XX.X.X                  | XXX |
 
 X's indicate that these details have not been determined as of commit date.
@@ -219,7 +219,7 @@ invokes cmake commands as needed.
 
     ```bash
     cd monero
-    git checkout release-v0.15
+    git checkout release-v0.16
     make
     ```
 
@@ -292,7 +292,7 @@ Tested on a Raspberry Pi Zero with a clean install of minimal Raspbian Stretch (
     ```bash
     git clone https://github.com/monero-project/monero.git
     cd monero
-    git checkout tags/v0.15.0.0
+    git checkout tags/v0.16.0.3
     ```
 
 * Build:
@@ -409,10 +409,10 @@ application.
     cd monero
     ```
 
-* If you would like a specific [version/tag](https://github.com/monero-project/monero/tags), do a git checkout for that version. eg. 'v0.15.0.0'. If you don't care about the version and just want binaries from master, skip this step:
+* If you would like a specific [version/tag](https://github.com/monero-project/monero/tags), do a git checkout for that version. eg. 'v0.16.0.3'. If you don't care about the version and just want binaries from master, skip this step:
 	
     ```bash
-    git checkout v0.15.0.0
+    git checkout v0.16.0.3
     ```
 
 * If you are on a 64-bit system, run:

contrib/epee/include/console_handler.h

@@ -465,7 +465,7 @@ eof:
   bool run_default_console_handler_no_srv_param(t_server* ptsrv, t_handler handlr, std::function<std::string(void)> prompt, const std::string& usage = "")
   {
     async_console_handler console_handler;
-    return console_handler.run(ptsrv, boost::bind<bool>(no_srv_param_adapter<t_server, t_handler>, _1, _2, handlr), prompt, usage);
+    return console_handler.run(ptsrv, boost::bind<bool>(no_srv_param_adapter<t_server, t_handler>, boost::placeholders::_1, boost::placeholders::_2, handlr), prompt, usage);
   }
 
   template<class t_server, class t_handler>
@@ -634,7 +634,7 @@ eof:
 
     bool run_handling(std::function<std::string(void)> prompt, const std::string& usage_string, std::function<void(void)> exit_handler = NULL)
     {
-      return m_console_handler.run(boost::bind(&console_handlers_binder::process_command_str, this, _1), prompt, usage_string, exit_handler);
+      return m_console_handler.run(boost::bind(&console_handlers_binder::process_command_str, this, boost::placeholders::_1), prompt, usage_string, exit_handler);
     }
 
     void print_prompt()

contrib/epee/include/md5_l.inl

@@ -277,7 +277,7 @@ namespace md5
 		/* Zeroize sensitive information.
 
 		*/
-		MD5_memset ((POINTER)context, 0, sizeof (*context)); 
+		memwipe ((POINTER)context, sizeof (*context));
 	}
 
 	/* MD5 basic transformation. Transforms state based on block.
@@ -369,7 +369,7 @@ namespace md5
 
 		/* Zeroize sensitive information.
 		*/
-		MD5_memset ((POINTER)x, 0, sizeof (x)); 
+		memwipe ((POINTER)x, sizeof (x));
 	}
 
 	/* Note: Replace "for loop" with standard memcpy if possible.
@@ -431,9 +431,9 @@ namespace md5
 		MD5Update(&hmac->octx, k_opad, 64);     /* apply outer pad */
 
 		/* scrub the pads and key context (if used) */
-		MD5_memset( (POINTER)&k_ipad, 0, sizeof(k_ipad));
-		MD5_memset( (POINTER)&k_opad, 0, sizeof(k_opad));
-		MD5_memset( (POINTER)&tk, 0, sizeof(tk));
+		memwipe( (POINTER)&k_ipad, sizeof(k_ipad));
+		memwipe( (POINTER)&k_opad, sizeof(k_opad));
+		memwipe( (POINTER)&tk, sizeof(tk));
 
 		/* and we're done. */
 	}
@@ -459,7 +459,7 @@ namespace md5
 			state->istate[lupe] = htonl(hmac.ictx.state[lupe]);
 			state->ostate[lupe] = htonl(hmac.octx.state[lupe]);
 		}
-		MD5_memset( (POINTER)&hmac, 0, sizeof(hmac));
+		memwipe( (POINTER)&hmac, sizeof(hmac));
 	}
 
 

contrib/epee/include/net/http_server_handlers_map2.h

@@ -42,8 +42,17 @@
   MINFO("HTTP [" << m_conn_context.m_remote_address.host_str() << "] " << query_info.m_http_method_str << " " << query_info.m_URI); \
   response.m_response_code = 200; \
   response.m_response_comment = "Ok"; \
-  if(!handle_http_request_map(query_info, response, m_conn_context)) \
-  {response.m_response_code = 404;response.m_response_comment = "Not found";} \
+  try \
+  { \
+    if(!handle_http_request_map(query_info, response, m_conn_context)) \
+    {response.m_response_code = 404;response.m_response_comment = "Not found";} \
+  } \
+  catch (const std::exception &e) \
+  { \
+    MERROR(m_conn_context << "Exception in handle_http_request_map: " << e.what()); \
+    response.m_response_code = 500; \
+    response.m_response_comment = "Internal Server Error"; \
+  } \
   return true; \
 }
 
@@ -69,9 +78,11 @@
       uint64_t ticks1 = epee::misc_utils::get_tick_count(); \
       boost::value_initialized<command_type::response> resp;\
       MINFO(m_conn_context << "calling " << s_pattern); \
-      if(!callback_f(static_cast<command_type::request&>(req), static_cast<command_type::response&>(resp), &m_conn_context)) \
+      bool res = false; \
+      try { res = callback_f(static_cast<command_type::request&>(req), static_cast<command_type::response&>(resp), &m_conn_context); } \
+      catch (const std::exception &e) { MERROR(m_conn_context << "Failed to " << #callback_f << "(): " << e.what()); } \
+      if (!res) \
       { \
-        MERROR(m_conn_context << "Failed to " << #callback_f << "()"); \
         response_info.m_response_code = 500; \
         response_info.m_response_comment = "Internal Server Error"; \
         return true; \
@@ -97,9 +108,11 @@
       uint64_t ticks1 = misc_utils::get_tick_count(); \
       boost::value_initialized<command_type::response> resp;\
       MINFO(m_conn_context << "calling " << s_pattern); \
-      if(!callback_f(static_cast<command_type::request&>(req), static_cast<command_type::response&>(resp), &m_conn_context)) \
+      bool res = false; \
+      try { res = callback_f(static_cast<command_type::request&>(req), static_cast<command_type::response&>(resp), &m_conn_context); } \
+      catch (const std::exception &e) { MERROR(m_conn_context << "Failed to " << #callback_f << "()"); } \
+      if (!res) \
       { \
-        MERROR(m_conn_context << "Failed to " << #callback_f << "()"); \
         response_info.m_response_code = 500; \
         response_info.m_response_comment = "Internal Server Error"; \
         return true; \
@@ -184,7 +197,10 @@
   fail_resp.jsonrpc = "2.0"; \
   fail_resp.id = req.id; \
   MINFO(m_conn_context << "Calling RPC method " << method_name); \
-  if(!callback_f(req.params, resp.result, fail_resp.error, &m_conn_context)) \
+  bool res = false; \
+  try { res = callback_f(req.params, resp.result, fail_resp.error, &m_conn_context); } \
+  catch (const std::exception &e) { MERROR(m_conn_context << "Failed to " << #callback_f << "(): " << e.what()); } \
+  if (!res) \
   { \
     epee::serialization::store_t_to_json(static_cast<epee::json_rpc::error_response&>(fail_resp), response_info.m_body); \
     return true; \
@@ -203,7 +219,10 @@
   fail_resp.jsonrpc = "2.0"; \
   fail_resp.id = req.id; \
   MINFO(m_conn_context << "calling RPC method " << method_name); \
-  if(!callback_f(req.params, resp.result, fail_resp.error, response_info, &m_conn_context)) \
+  bool res = false; \
+  try { res = callback_f(req.params, resp.result, fail_resp.error, response_info, &m_conn_context); } \
+  catch (const std::exception &e) { MERROR(m_conn_context << "Failed to " << #callback_f << "(): " << e.what()); } \
+  if (!res) \
   { \
     epee::serialization::store_t_to_json(static_cast<epee::json_rpc::error_response&>(fail_resp), response_info.m_body); \
     return true; \
@@ -217,7 +236,10 @@
 { \
   PREPARE_OBJECTS_FROM_JSON(command_type) \
   MINFO(m_conn_context << "calling RPC method " << method_name); \
-  if(!callback_f(req.params, resp.result, &m_conn_context)) \
+  bool res = false; \
+  try { res = callback_f(req.params, resp.result, &m_conn_context); } \
+  catch (const std::exception &e) { MERROR(m_conn_context << "Failed to " << #callback_f << "(): " << e.what()); } \
+  if (!res) \
   { \
     epee::json_rpc::error_response fail_resp = AUTO_VAL_INIT(fail_resp); \
     fail_resp.jsonrpc = "2.0"; \

contrib/epee/include/storages/levin_abstract_invoke2.h

@@ -290,24 +290,25 @@ namespace epee
 #define BEGIN_INVOKE_MAP2(owner_type) \
   template <class t_context> int handle_invoke_map(bool is_notify, int command, const epee::span<const uint8_t> in_buff, std::string& buff_out, t_context& context, bool& handled) \
   { \
+  try { \
   typedef owner_type internal_owner_type_name;
 
 #define HANDLE_INVOKE2(command_id, func, type_name_in, typename_out) \
   if(!is_notify && command_id == command) \
-  {handled=true;return epee::net_utils::buff_to_t_adapter<internal_owner_type_name, type_name_in, typename_out>(this, command, in_buff, buff_out, boost::bind(func, this, _1, _2, _3, _4), context);}
+  {handled=true;return epee::net_utils::buff_to_t_adapter<internal_owner_type_name, type_name_in, typename_out>(this, command, in_buff, buff_out, boost::bind(func, this, boost::placeholders::_1, boost::placeholders::_2, boost::placeholders::_3, boost::placeholders::_4), context);}
 
 #define HANDLE_INVOKE_T2(COMMAND, func) \
   if(!is_notify && COMMAND::ID == command) \
-  {handled=true;return epee::net_utils::buff_to_t_adapter<internal_owner_type_name, typename COMMAND::request, typename COMMAND::response>(command, in_buff, buff_out, boost::bind(func, this, _1, _2, _3, _4), context);}
+  {handled=true;return epee::net_utils::buff_to_t_adapter<internal_owner_type_name, typename COMMAND::request, typename COMMAND::response>(command, in_buff, buff_out, boost::bind(func, this, boost::placeholders::_1, boost::placeholders::_2, boost::placeholders::_3, boost::placeholders::_4), context);}
 
 
 #define HANDLE_NOTIFY2(command_id, func, type_name_in) \
   if(is_notify && command_id == command) \
-  {handled=true;return epee::net_utils::buff_to_t_adapter<internal_owner_type_name, type_name_in>(this, command, in_buff, boost::bind(func, this, _1, _2, _3), context);}
+  {handled=true;return epee::net_utils::buff_to_t_adapter<internal_owner_type_name, type_name_in>(this, command, in_buff, boost::bind(func, this, boost::placeholders::_1, boost::placeholders::_2, boost::placeholders::_3), context);}
 
 #define HANDLE_NOTIFY_T2(NOTIFY, func) \
   if(is_notify && NOTIFY::ID == command) \
-  {handled=true;return epee::net_utils::buff_to_t_adapter<internal_owner_type_name, typename NOTIFY::request>(this, command, in_buff, boost::bind(func, this, _1, _2, _3), context);}
+  {handled=true;return epee::net_utils::buff_to_t_adapter<internal_owner_type_name, typename NOTIFY::request>(this, command, in_buff, boost::bind(func, this, boost::placeholders::_1, boost::placeholders::_2, boost::placeholders::_3), context);}
 
 
 #define CHAIN_INVOKE_MAP2(func) \
@@ -335,7 +336,13 @@ namespace epee
   LOG_ERROR("Unknown command:" << command); \
   on_levin_traffic(context, false, false, true, in_buff.size(), "invalid-command"); \
   return LEVIN_ERROR_CONNECTION_HANDLER_NOT_DEFINED; \
+  } \
+  catch (const std::exception &e) { \
+    MERROR("Error in handle_invoke_map: " << e.what()); \
+    return LEVIN_ERROR_CONNECTION_TIMEDOUT; /* seems kinda appropriate */ \
+  } \
   }
+
   }
 }
 

contrib/epee/include/storages/parserse_base_utils.h

@@ -196,7 +196,7 @@ namespace misc_utils
                 uint32_t dst = 0;
                 for (int i = 0; i < 4; ++i)
                 {
-                  const unsigned char tmp = isx[(int)*++it];
+                  const unsigned char tmp = isx[(unsigned char)*++it];
                   CHECK_AND_ASSERT_THROW_MES(tmp != 0xff, "Bad Unicode encoding");
                   dst = dst << 4 | tmp;
                 }

contrib/epee/src/byte_slice.cpp

@@ -133,10 +133,13 @@ namespace epee
 
   template<typename T>
   byte_slice::byte_slice(const adapt_buffer, T&& buffer)
-    : storage_(nullptr), portion_(to_byte_span(to_span(buffer)))
+    : storage_(nullptr), portion_(nullptr)
   {
     if (!buffer.empty())
+    {
       storage_ = allocate_slice<adapted_byte_slice<T>>(0, std::move(buffer));
+      portion_ = to_byte_span(to_span(static_cast<adapted_byte_slice<T> *>(storage_.get())->buffer));
+    }
   }
 
   byte_slice::byte_slice(std::initializer_list<span<const std::uint8_t>> sources)
@@ -173,9 +176,14 @@ namespace epee
   byte_slice::byte_slice(byte_stream&& stream) noexcept
     : storage_(nullptr), portion_(stream.data(), stream.size())
   {
-    std::uint8_t* const data = stream.take_buffer().release() - sizeof(raw_byte_slice);
-    new (data) raw_byte_slice{};
-    storage_.reset(reinterpret_cast<raw_byte_slice*>(data));
+    if (stream.size())
+    {
+      std::uint8_t* const data = stream.take_buffer().release() - sizeof(raw_byte_slice);
+      new (data) raw_byte_slice{};
+      storage_.reset(reinterpret_cast<raw_byte_slice*>(data));
+    }
+    else
+      portion_ = nullptr;
   }
 
   byte_slice::byte_slice(byte_slice&& source) noexcept
@@ -205,14 +213,17 @@ namespace epee
   byte_slice byte_slice::take_slice(const std::size_t max_bytes) noexcept
   {
     byte_slice out{};
-    std::uint8_t const* const ptr = data();
-    out.portion_ = {ptr, portion_.remove_prefix(max_bytes)};
 
-    if (portion_.empty())
-      out.storage_ = std::move(storage_); // no atomic inc/dec
-    else
-      out = {storage_.get(), out.portion_};
+    if (max_bytes)
+    {
+      std::uint8_t const* const ptr = data();
+      out.portion_ = {ptr, portion_.remove_prefix(max_bytes)};
 
+      if (portion_.empty())
+        out.storage_ = std::move(storage_); // no atomic inc/dec
+      else
+        out = {storage_.get(), out.portion_};
+    }
     return out;
   }
 

contrib/epee/src/mlog.cpp

@@ -100,7 +100,7 @@ static const char *get_default_categories(int level)
   switch (level)
   {
     case 0:
-      categories = "*:WARNING,net:FATAL,net.http:FATAL,net.ssl:FATAL,net.p2p:FATAL,net.cn:FATAL,global:INFO,verify:FATAL,serialization:FATAL,daemon.rpc.payment:ERROR,stacktrace:INFO,logging:INFO,msgwriter:INFO";
+      categories = "*:WARNING,net:FATAL,net.http:FATAL,net.ssl:FATAL,net.p2p:FATAL,net.cn:FATAL,daemon.rpc:FATAL,global:INFO,verify:FATAL,serialization:FATAL,daemon.rpc.payment:ERROR,stacktrace:INFO,logging:INFO,msgwriter:INFO";
       break;
     case 1:
       categories = "*:INFO,global:INFO,stacktrace:INFO,logging:INFO,msgwriter:INFO,perf.*:DEBUG";

contrib/gitian/README.md

@@ -126,7 +126,7 @@ Setup for LXC:
 
 ```bash
 GH_USER=fluffypony
-VERSION=v0.15.0.0
+VERSION=v0.16.0.3
 
 ./gitian-build.py --setup $GH_USER $VERSION
 ```
@@ -182,7 +182,7 @@ If you chose to do detached signing using `--detach-sign` above (recommended), y
 
 ```bash
 GH_USER=fluffypony
-VERSION=v0.15.0.0
+VERSION=v0.16.0.3
 
 gpg --detach-sign ${VERSION}-linux/${GH_USER}/monero-linux-*-build.assert
 gpg --detach-sign ${VERSION}-win/${GH_USER}/monero-win-*-build.assert

contrib/gitian/gitian-android.yml

@@ -1,5 +1,5 @@
 ---
-name: "monero-android-0.15"
+name: "monero-android-0.16"
 enable_cache: true
 suites:
 - "bionic"

contrib/gitian/gitian-freebsd.yml

@@ -1,5 +1,5 @@
 ---
-name: "monero-freebsd-0.15"
+name: "monero-freebsd-0.16"
 enable_cache: true
 suites:
 - "bionic"

contrib/gitian/gitian-linux.yml

@@ -1,5 +1,5 @@
 ---
-name: "monero-linux-0.15"
+name: "monero-linux-0.16"
 enable_cache: true
 suites:
 - "bionic"

contrib/gitian/gitian-osx.yml

@@ -1,5 +1,5 @@
 ---
-name: "monero-osx-0.15"
+name: "monero-osx-0.16"
 enable_cache: true
 suites:
 - "bionic"

contrib/gitian/gitian-win.yml

@@ -1,5 +1,5 @@
 ---
-name: "monero-win-0.15"
+name: "monero-win-0.16"
 enable_cache: true
 suites:
 - "bionic"

external/easylogging++/easylogging++.cc

@@ -2475,6 +2475,100 @@ void DefaultLogDispatchCallback::handle(const LogDispatchData* data) {
   }
 }
 
+
+template<typename Transform>
+static inline std::string utf8canonical(const std::string &s, Transform t = [](wint_t c)->wint_t { return c; })
+{
+    std::string sc = "";
+    size_t avail = s.size();
+    const char *ptr = s.data();
+    wint_t cp = 0;
+    int bytes = 1;
+    char wbuf[8], *wptr;
+    while (avail--)
+    {
+      if ((*ptr & 0x80) == 0)
+      {
+        cp = *ptr++;
+        bytes = 1;
+      }
+      else if ((*ptr & 0xe0) == 0xc0)
+      {
+        if (avail < 1)
+          throw std::runtime_error("Invalid UTF-8");
+        cp = (*ptr++ & 0x1f) << 6;
+        cp |= *ptr++ & 0x3f;
+        --avail;
+        bytes = 2;
+      }
+      else if ((*ptr & 0xf0) == 0xe0)
+      {
+        if (avail < 2)
+          throw std::runtime_error("Invalid UTF-8");
+        cp = (*ptr++ & 0xf) << 12;
+        cp |= (*ptr++ & 0x3f) << 6;
+        cp |= *ptr++ & 0x3f;
+        avail -= 2;
+        bytes = 3;
+      }
+      else if ((*ptr & 0xf8) == 0xf0)
+      {
+        if (avail < 3)
+          throw std::runtime_error("Invalid UTF-8");
+        cp = (*ptr++ & 0x7) << 18;
+        cp |= (*ptr++ & 0x3f) << 12;
+        cp |= (*ptr++ & 0x3f) << 6;
+        cp |= *ptr++ & 0x3f;
+        avail -= 3;
+        bytes = 4;
+      }
+      else
+        throw std::runtime_error("Invalid UTF-8");
+
+      cp = t(cp);
+      if (cp <= 0x7f)
+        bytes = 1;
+      else if (cp <= 0x7ff)
+        bytes = 2;
+      else if (cp <= 0xffff)
+        bytes = 3;
+      else if (cp <= 0x10ffff)
+        bytes = 4;
+      else
+        throw std::runtime_error("Invalid code point UTF-8 transformation");
+
+      wptr = wbuf;
+      switch (bytes)
+      {
+        case 1: *wptr++ = cp; break;
+        case 2: *wptr++ = 0xc0 | (cp >> 6); *wptr++ = 0x80 | (cp & 0x3f); break;
+        case 3: *wptr++ = 0xe0 | (cp >> 12); *wptr++ = 0x80 | ((cp >> 6) & 0x3f); *wptr++ = 0x80 | (cp & 0x3f); break;
+        case 4: *wptr++ = 0xf0 | (cp >> 18); *wptr++ = 0x80 | ((cp >> 12) & 0x3f); *wptr++ = 0x80 | ((cp >> 6) & 0x3f); *wptr++ = 0x80 | (cp & 0x3f); break;
+        default: throw std::runtime_error("Invalid UTF-8");
+      }
+      *wptr = 0;
+      sc.append(wbuf, bytes);
+      cp = 0;
+      bytes = 1;
+    }
+    return sc;
+}
+
+void sanitize(std::string &s)
+{
+  s = utf8canonical(s, [](wint_t c)->wint_t {
+    if (c == 9 || c == 10 || c == 13)
+      return c;
+    if (c < 0x20)
+      return '?';
+    if (c == 0x7f)
+      return '?';
+    if (c >= 0x80 && c <= 0x9f)
+      return '?';
+    return c;
+  });
+}
+
 void DefaultLogDispatchCallback::dispatch(base::type::string_t&& rawLinePrefix, base::type::string_t&& rawLinePayload, base::type::string_t&& logLine) {
   if (m_data->dispatchAction() == base::DispatchAction::NormalLog || m_data->dispatchAction() == base::DispatchAction::FileOnlyLog) {
     if (m_data->logMessage()->logger()->m_typedConfigurations->toFile(m_data->logMessage()->level())) {
@@ -2506,6 +2600,8 @@ void DefaultLogDispatchCallback::dispatch(base::type::string_t&& rawLinePrefix,
         m_data->logMessage()->logger()->logBuilder()->setColor(el::base::utils::colorFromLevel(level), false);
         ELPP_COUT << rawLinePrefix;
         m_data->logMessage()->logger()->logBuilder()->setColor(color == el::Color::Default ? el::base::utils::colorFromLevel(level): color, color != el::Color::Default);
+        try { sanitize(rawLinePayload); }
+        catch (const std::exception &e) { rawLinePayload = "<Invalid UTF-8 in log>"; }
         ELPP_COUT << rawLinePayload;
         m_data->logMessage()->logger()->logBuilder()->setColor(el::Color::Default, false);
         ELPP_COUT << std::flush;
@@ -2872,6 +2968,16 @@ void Writer::initializeLogger(Logger *logger, bool needLock) {
 }
 
 void Writer::processDispatch() {
+  static std::atomic_flag in_dispatch;
+  if (in_dispatch.test_and_set())
+  {
+    if (m_proceed && m_logger != NULL)
+    {
+      m_logger->stream().str(ELPP_LITERAL(""));
+      m_logger->releaseLock();
+    }
+    return;
+  }
 #if ELPP_LOGGING_ENABLED
   if (ELPP->hasFlag(LoggingFlag::MultiLoggerSupport)) {
     bool firstDispatched = false;
@@ -2910,6 +3016,7 @@ void Writer::processDispatch() {
     m_logger->releaseLock();
   }
 #endif // ELPP_LOGGING_ENABLED
+  in_dispatch.clear();
 }
 
 void Writer::triggerDispatch(void) {

src/blockchain_utilities/blockchain_stats.cpp

@@ -68,6 +68,9 @@ int main(int argc, char* argv[])
   const command_line::arg_descriptor<bool> arg_outputs  = {"with-outputs", "with output stats", false};
   const command_line::arg_descriptor<bool> arg_ringsize  = {"with-ringsize", "with ringsize stats", false};
   const command_line::arg_descriptor<bool> arg_hours  = {"with-hours", "with txns per hour", false};
+  const command_line::arg_descriptor<bool> arg_emission  = {"with-emission", "with coin emission", false};
+  const command_line::arg_descriptor<bool> arg_fees  = {"with-fees", "with txn fees", false};
+  const command_line::arg_descriptor<bool> arg_diff  = {"with-diff", "with difficulty", false};
 
   command_line::add_arg(desc_cmd_sett, cryptonote::arg_data_dir);
   command_line::add_arg(desc_cmd_sett, cryptonote::arg_testnet_on);
@@ -79,6 +82,9 @@ int main(int argc, char* argv[])
   command_line::add_arg(desc_cmd_sett, arg_outputs);
   command_line::add_arg(desc_cmd_sett, arg_ringsize);
   command_line::add_arg(desc_cmd_sett, arg_hours);
+  command_line::add_arg(desc_cmd_sett, arg_emission);
+  command_line::add_arg(desc_cmd_sett, arg_fees);
+  command_line::add_arg(desc_cmd_sett, arg_diff);
   command_line::add_arg(desc_cmd_only, command_line::arg_help);
 
   po::options_description desc_options("Allowed options");
@@ -120,6 +126,9 @@ int main(int argc, char* argv[])
   bool do_outputs = command_line::get_arg(vm, arg_outputs);
   bool do_ringsize = command_line::get_arg(vm, arg_ringsize);
   bool do_hours = command_line::get_arg(vm, arg_hours);
+  bool do_emission = command_line::get_arg(vm, arg_emission);
+  bool do_fees = command_line::get_arg(vm, arg_fees);
+  bool do_diff = command_line::get_arg(vm, arg_diff);
 
   LOG_PRINT_L0("Initializing source blockchain (BlockchainDB)");
   std::unique_ptr<Blockchain> core_storage;
@@ -177,12 +186,20 @@ plot 'stats.csv' index "DATA" using (timecolumn(1,"%Y-%m-%d")):4 with lines, ''
   // spit out a comment that GnuPlot can use as an index
   std::cout << ENDL << "# DATA" << ENDL;
   std::cout << "Date\tBlocks/day\tBlocks\tTxs/Day\tTxs\tBytes/Day\tBytes";
+  if (do_emission)
+    std::cout << "\tEmission/day\tEmission";
+  if (do_fees)
+    std::cout << "\tFees/day\tFees";
+  if (do_diff)
+    std::cout << "\tDiffMin\tDiffMax\tDiffAvg";
   if (do_inputs)
     std::cout << "\tInMin\tInMax\tInAvg";
   if (do_outputs)
     std::cout << "\tOutMin\tOutMax\tOutAvg";
   if (do_ringsize)
     std::cout << "\tRingMin\tRingMax\tRingAvg";
+  if (do_inputs || do_outputs || do_ringsize)
+    std::cout << std::setprecision(2) << std::fixed;
   if (do_hours) {
     char buf[8];
     unsigned int i;
@@ -193,14 +210,20 @@ plot 'stats.csv' index "DATA" using (timecolumn(1,"%Y-%m-%d")):4 with lines, ''
   }
   std::cout << ENDL;
 
+#define MAX_INOUT	0xffffffff
+#define MAX_RINGS	0xffffffff
+
   struct tm prevtm = {0}, currtm;
   uint64_t prevsz = 0, currsz = 0;
   uint64_t prevtxs = 0, currtxs = 0;
   uint64_t currblks = 0;
   uint64_t totins = 0, totouts = 0, totrings = 0;
-  uint32_t minins = 10, maxins = 0;
-  uint32_t minouts = 10, maxouts = 0;
-  uint32_t minrings = 50, maxrings = 0;
+  boost::multiprecision::uint128_t prevemission = 0, prevfees = 0;
+  boost::multiprecision::uint128_t emission = 0, fees = 0;
+  boost::multiprecision::uint128_t totdiff = 0, mindiff = 0, maxdiff = 0;
+  uint32_t minins = MAX_INOUT, maxins = 0;
+  uint32_t minouts = MAX_INOUT, maxouts = 0;
+  uint32_t minrings = MAX_RINGS, maxrings = 0;
   uint32_t io, tottxs = 0;
   uint32_t txhr[24] = {0};
   unsigned int i;
@@ -230,34 +253,50 @@ plot 'stats.csv' index "DATA" using (timecolumn(1,"%Y-%m-%d")):4 with lines, ''
       std::cout << timebuf << "\t" << currblks << "\t" << h << "\t" << currtxs << "\t" << prevtxs + currtxs << "\t" << currsz << "\t" << prevsz + currsz;
       prevsz += currsz;
       currsz = 0;
-      currblks = 0;
       prevtxs += currtxs;
       currtxs = 0;
       if (!tottxs)
         tottxs = 1;
+      if (do_emission) {
+        std::cout << "\t" << print_money(emission) << "\t" << print_money(prevemission + emission);
+        prevemission += emission;
+        emission = 0;
+      }
+      if (do_fees) {
+        std::cout << "\t" << print_money(fees) << "\t" << print_money(prevfees + fees);
+        prevfees += fees;
+        fees = 0;
+      }
+      if (do_diff) {
+        std::cout << "\t" << (maxdiff ? mindiff : 0) << "\t" << maxdiff << "\t" << totdiff / currblks;
+        mindiff = 0; maxdiff = 0; totdiff = 0;
+      }
       if (do_inputs) {
-        std::cout << "\t" << (maxins ? minins : 0) << "\t" << maxins << "\t" << totins / tottxs;
-        minins = 10; maxins = 0; totins = 0;
+        std::cout << "\t" << (maxins ? minins : 0) << "\t" << maxins << "\t" << totins * 1.0 / tottxs;
+        minins = MAX_INOUT; maxins = 0; totins = 0;
       }
       if (do_outputs) {
-        std::cout << "\t" << (maxouts ? minouts : 0) << "\t" << maxouts << "\t" << totouts / tottxs;
-        minouts = 10; maxouts = 0; totouts = 0;
+        std::cout << "\t" << (maxouts ? minouts : 0) << "\t" << maxouts << "\t" << totouts * 1.0 / tottxs;
+        minouts = MAX_INOUT; maxouts = 0; totouts = 0;
       }
       if (do_ringsize) {
-        std::cout << "\t" << (maxrings ? minrings : 0) << "\t" << maxrings << "\t" << totrings / tottxs;
-        minrings = 50; maxrings = 0; totrings = 0;
+        std::cout << "\t" << (maxrings ? minrings : 0) << "\t" << maxrings << "\t" << totrings * 1.0 / tottxs;
+        minrings = MAX_RINGS; maxrings = 0; totrings = 0;
       }
-      tottxs = 0;
       if (do_hours) {
         for (i=0; i<24; i++) {
           std::cout << "\t" << txhr[i];
           txhr[i] = 0;
         }
       }
+      currblks = 0;
+      tottxs = 0;
       std::cout << ENDL;
     }
 skip:
     currsz += bd.size();
+    uint64_t coinbase_amount;
+    uint64_t tx_fee_amount = 0;
     for (const auto& tx_id : blk.tx_hashes)
     {
       if (tx_id == crypto::null_hash)
@@ -275,7 +314,12 @@ skip:
         return 1;
       }
       currsz += bd.size();
+      if (db->get_prunable_tx_blob(tx_id, bd))
+        currsz += bd.size();
       currtxs++;
+      if (do_fees || do_emission) {
+        tx_fee_amount += get_tx_fee(tx);
+      }
       if (do_hours)
         txhr[currtm.tm_hour]++;
       if (do_inputs) {
@@ -306,6 +350,21 @@ skip:
       }
       tottxs++;
     }
+    if (do_diff) {
+      difficulty_type diff = db->get_block_difficulty(h);
+      if (!mindiff || diff < mindiff)
+        mindiff = diff;
+      if (diff > maxdiff)
+        maxdiff = diff;
+      totdiff += diff;
+    }
+    if (do_emission) {
+      coinbase_amount = get_outs_money_amount(blk.miner_tx);
+      emission += coinbase_amount - tx_fee_amount;
+    }
+    if (do_fees) {
+      fees += tx_fee_amount;
+    }
     currblks++;
 
     if (stop_requested)

src/checkpoints/checkpoints.cpp

@@ -135,8 +135,8 @@ namespace cryptonote
   {
     std::map< uint64_t, crypto::hash >::const_iterator highest = 
         std::max_element( m_points.begin(), m_points.end(),
-                         ( boost::bind(&std::map< uint64_t, crypto::hash >::value_type::first, _1) < 
-                           boost::bind(&std::map< uint64_t, crypto::hash >::value_type::first, _2 ) ) );
+                         ( boost::bind(&std::map< uint64_t, crypto::hash >::value_type::first, boost::placeholders::_1) <
+                           boost::bind(&std::map< uint64_t, crypto::hash >::value_type::first, boost::placeholders::_2 ) ) );
     return highest->first;
   }
   //---------------------------------------------------------------------------
@@ -211,6 +211,10 @@ namespace cryptonote
     ADD_CHECKPOINT(1775600, "1c6e01c661dc22cab939e79ec6a5272190624ce8356d2f7b958e4f9a57fdb05e");
     ADD_CHECKPOINT(1856000, "9b57f17f29c71a3acd8a7904b93c41fa6eb8d2b7c73936ce4f1702d14880ba29");
     ADD_CHECKPOINT(1958000, "98a5d6e51afdf3146e0eefb10a66e8648d8d4d5c2742be8835e976ba217c9bb2");
+    ADD_CHECKPOINT(2046000, "5e867f0b8baefed9244a681df97fc885d8ab36c3dfcd24c7a3abf3b8ac8b8314");
+    ADD_CHECKPOINT(2092500, "c4e00820c9c7989b49153d5e90ae095a18a11d990e82fcc3be54e6ed785472b5");
+    ADD_CHECKPOINT(2125000, "a8e49c62792a2aa56ba62603fe015303647e2c19203c56999c7f6f2498cd3e6d");
+    ADD_CHECKPOINT(2153000, "05952ed32f92647c44f91b222cc95cc8fb6f04cefd59b76ef30884fec957797e");
     return true;
   }
 

src/common/CMakeLists.txt

@@ -86,7 +86,8 @@ set(common_private_headers
   updates.h
   aligned.h
   timings.h
-  combinator.h)
+  combinator.h
+  utf8.h)
 
 monero_private_headers(common
   ${common_private_headers})

src/common/updates.cpp

@@ -102,6 +102,8 @@ namespace tools
     const char *base = user ? "https://downloads.getmonero.org/" : "https://updates.getmonero.org/";
 #ifdef _WIN32
     static const char *extension = strncmp(buildtag.c_str(), "source", 6) ? (strncmp(buildtag.c_str(), "install-", 8) ? ".zip" : ".exe") : ".tar.bz2";
+#elif defined(__APPLE__)
+    static const char *extension = strncmp(software.c_str(), "monero-gui", 10) ? ".tar.bz2" : ".dmg";
 #else
     static const char extension[] = ".tar.bz2";
 #endif

src/common/utf8.h

@@ -0,0 +1,114 @@
+// Copyright (c) 2019, The Monero Project
+// 
+// All rights reserved.
+// 
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+// 
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+// 
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+// 
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+// 
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#pragma once 
+
+#include <cctype>
+#include <cwchar>
+#include <stdexcept>
+
+namespace tools
+{
+  template<typename T, typename Transform>
+  inline T utf8canonical(const T &s, Transform t = [](wint_t c)->wint_t { return c; })
+  {
+    T sc = "";
+    size_t avail = s.size();
+    const char *ptr = s.data();
+    wint_t cp = 0;
+    int bytes = 1;
+    char wbuf[8], *wptr;
+    while (avail--)
+    {
+      if ((*ptr & 0x80) == 0)
+      {
+        cp = *ptr++;
+        bytes = 1;
+      }
+      else if ((*ptr & 0xe0) == 0xc0)
+      {
+        if (avail < 1)
+          throw std::runtime_error("Invalid UTF-8");
+        cp = (*ptr++ & 0x1f) << 6;
+        cp |= *ptr++ & 0x3f;
+        --avail;
+        bytes = 2;
+      }
+      else if ((*ptr & 0xf0) == 0xe0)
+      {
+        if (avail < 2)
+          throw std::runtime_error("Invalid UTF-8");
+        cp = (*ptr++ & 0xf) << 12;
+        cp |= (*ptr++ & 0x3f) << 6;
+        cp |= *ptr++ & 0x3f;
+        avail -= 2;
+        bytes = 3;
+      }
+      else if ((*ptr & 0xf8) == 0xf0)
+      {
+        if (avail < 3)
+          throw std::runtime_error("Invalid UTF-8");
+        cp = (*ptr++ & 0x7) << 18;
+        cp |= (*ptr++ & 0x3f) << 12;
+        cp |= (*ptr++ & 0x3f) << 6;
+        cp |= *ptr++ & 0x3f;
+        avail -= 3;
+        bytes = 4;
+      }
+      else
+        throw std::runtime_error("Invalid UTF-8");
+
+      cp = t(cp);
+      if (cp <= 0x7f)
+        bytes = 1;
+      else if (cp <= 0x7ff)
+        bytes = 2;
+      else if (cp <= 0xffff)
+        bytes = 3;
+      else if (cp <= 0x10ffff)
+        bytes = 4;
+      else
+        throw std::runtime_error("Invalid code point UTF-8 transformation");
+
+      wptr = wbuf;
+      switch (bytes)
+      {
+        case 1: *wptr++ = cp; break;
+        case 2: *wptr++ = 0xc0 | (cp >> 6); *wptr++ = 0x80 | (cp & 0x3f); break;
+        case 3: *wptr++ = 0xe0 | (cp >> 12); *wptr++ = 0x80 | ((cp >> 6) & 0x3f); *wptr++ = 0x80 | (cp & 0x3f); break;
+        case 4: *wptr++ = 0xf0 | (cp >> 18); *wptr++ = 0x80 | ((cp >> 12) & 0x3f); *wptr++ = 0x80 | ((cp >> 6) & 0x3f); *wptr++ = 0x80 | (cp & 0x3f); break;
+        default: throw std::runtime_error("Invalid UTF-8");
+      }
+      *wptr = 0;
+      sc.append(wbuf, bytes);
+      cp = 0;
+      bytes = 1;
+    }
+    return sc;
+  }
+}

src/cryptonote_core/blockchain.cpp

@@ -86,7 +86,7 @@ DISABLE_VS_WARNINGS(4267)
 
 //------------------------------------------------------------------
 Blockchain::Blockchain(tx_memory_pool& tx_pool) :
-  m_db(), m_tx_pool(tx_pool), m_hardfork(NULL), m_timestamps_and_difficulties_height(0), m_current_block_cumul_weight_limit(0), m_current_block_cumul_weight_median(0),
+  m_db(), m_tx_pool(tx_pool), m_hardfork(NULL), m_timestamps_and_difficulties_height(0), m_reset_timestamps_and_difficulties_height(true), m_current_block_cumul_weight_limit(0), m_current_block_cumul_weight_median(0),
   m_enforce_dns_checkpoints(false), m_max_prepare_blocks_threads(4), m_db_sync_on_blocks(true), m_db_sync_threshold(1), m_db_sync_mode(db_async), m_db_default_sync(false), m_fast_sync(true), m_show_time_stats(false), m_sync_counter(0), m_bytes_to_sync(0), m_cancel(false),
   m_long_term_block_weights_window(CRYPTONOTE_LONG_TERM_BLOCK_WEIGHT_WINDOW_SIZE),
   m_long_term_effective_median_block_weight(0),
@@ -427,6 +427,7 @@ bool Blockchain::init(BlockchainDB* db, const network_type nettype, bool offline
   if (num_popped_blocks > 0)
   {
     m_timestamps_and_difficulties_height = 0;
+    m_reset_timestamps_and_difficulties_height = true;
     m_hardfork->reorganize_from_chain_height(get_current_blockchain_height());
     uint64_t top_block_height;
     crypto::hash top_block_hash = get_tail_id(top_block_height);
@@ -567,6 +568,7 @@ block Blockchain::pop_block_from_blockchain()
   CRITICAL_REGION_LOCAL(m_blockchain_lock);
 
   m_timestamps_and_difficulties_height = 0;
+  m_reset_timestamps_and_difficulties_height = true;
 
   block popped_block;
   std::vector<transaction> popped_txs;
@@ -644,6 +646,7 @@ bool Blockchain::reset_and_set_genesis_block(const block& b)
   LOG_PRINT_L3("Blockchain::" << __func__);
   CRITICAL_REGION_LOCAL(m_blockchain_lock);
   m_timestamps_and_difficulties_height = 0;
+  m_reset_timestamps_and_difficulties_height = true;
   invalidate_block_template_cache();
   m_db->reset();
   m_db->drop_alt_blocks();
@@ -812,12 +815,20 @@ bool Blockchain::get_block_by_hash(const crypto::hash &h, block &blk, bool *orph
 // less blocks than desired if there aren't enough.
 difficulty_type Blockchain::get_difficulty_for_next_block()
 {
+  LOG_PRINT_L3("Blockchain::" << __func__);
+
+  std::stringstream ss;
+  bool print = false;
+
+  int done = 0;
+  ss << "get_difficulty_for_next_block: height " << m_db->height() << std::endl;
   if (m_fixed_difficulty)
   {
     return m_db->height() ? m_fixed_difficulty : 1;
   }
 
-  LOG_PRINT_L3("Blockchain::" << __func__);
+start:
+  difficulty_type D = 0;
 
   crypto::hash top_hash = get_tail_id();
   {
@@ -826,21 +837,32 @@ difficulty_type Blockchain::get_difficulty_for_next_block()
     // something a bit out of date, but that's fine since anything which
     // requires the blockchain lock will have acquired it in the first place,
     // and it will be unlocked only when called from the getinfo RPC
+    ss << "Locked, tail id " << top_hash << ", cached is " << m_difficulty_for_next_block_top_hash << std::endl;
     if (top_hash == m_difficulty_for_next_block_top_hash)
-      return m_difficulty_for_next_block;
+    {
+      ss << "Same, using cached diff " << m_difficulty_for_next_block << std::endl;
+      D = m_difficulty_for_next_block;
+    }
   }
 
   CRITICAL_REGION_LOCAL(m_blockchain_lock);
   std::vector<uint64_t> timestamps;
   std::vector<difficulty_type> difficulties;
   uint64_t height;
-  top_hash = get_tail_id(height); // get it again now that we have the lock
-  ++height; // top block height to blockchain height
+  auto new_top_hash = get_tail_id(height); // get it again now that we have the lock
+  ++height;
+  if (!(new_top_hash == top_hash)) D=0;
+  ss << "Re-locked, height " << height << ", tail id " << new_top_hash << (new_top_hash == top_hash ? "" : " (different)") << std::endl;
+  top_hash = new_top_hash;
+
   // ND: Speedup
   // 1. Keep a list of the last 735 (or less) blocks that is used to compute difficulty,
   //    then when the next block difficulty is queried, push the latest height data and
   //    pop the oldest one from the list. This only requires 1x read per height instead
   //    of doing 735 (DIFFICULTY_BLOCKS_COUNT).
+  bool check = false;
+  if (m_reset_timestamps_and_difficulties_height)
+    m_timestamps_and_difficulties_height = 0;
   if (m_timestamps_and_difficulties_height != 0 && ((height - m_timestamps_and_difficulties_height) == 1) && m_timestamps.size() >= DIFFICULTY_BLOCKS_COUNT)
   {
     uint64_t index = height - 1;
@@ -855,8 +877,12 @@ difficulty_type Blockchain::get_difficulty_for_next_block()
     m_timestamps_and_difficulties_height = height;
     timestamps = m_timestamps;
     difficulties = m_difficulties;
+    check = true;
   }
-  else
+  //else
+  std::vector<uint64_t> timestamps_from_cache = timestamps;
+  std::vector<difficulty_type> difficulties_from_cache = difficulties;
+
   {
     uint64_t offset = height - std::min <uint64_t> (height, static_cast<uint64_t>(DIFFICULTY_BLOCKS_COUNT));
     if (offset == 0)
@@ -869,22 +895,68 @@ difficulty_type Blockchain::get_difficulty_for_next_block()
       timestamps.reserve(height - offset);
       difficulties.reserve(height - offset);
     }
+    ss << "Looking up " << (height - offset) << " from " << offset << std::endl;
     for (; offset < height; offset++)
     {
       timestamps.push_back(m_db->get_block_timestamp(offset));
       difficulties.push_back(m_db->get_block_cumulative_difficulty(offset));
     }
 
+    if (check) if (timestamps != timestamps_from_cache || difficulties !=difficulties_from_cache)
+    {
+      ss << "Inconsistency XXX:" << std::endl;
+      ss << "top hash: "<<top_hash << std::endl;
+      ss << "timestamps: " << timestamps_from_cache.size() << " from cache, but " << timestamps.size() << " without" << std::endl;
+      ss << "difficulties: " << difficulties_from_cache.size() << " from cache, but " << difficulties.size() << " without" << std::endl;
+      ss << "timestamps_from_cache:" << std::endl; for (const auto &v :timestamps_from_cache) ss << "  " << v << std::endl;
+      ss << "timestamps:" << std::endl; for (const auto &v :timestamps) ss << "  " << v << std::endl;
+      ss << "difficulties_from_cache:" << std::endl; for (const auto &v :difficulties_from_cache) ss << "  " << v << std::endl;
+      ss << "difficulties:" << std::endl; for (const auto &v :difficulties) ss << "  " << v << std::endl;
+
+      uint64_t dbh = m_db->height();
+      uint64_t sh = dbh < 10000 ? 0 : dbh - 10000;
+      ss << "History from -10k at :" << dbh << ", from " << sh << std::endl;
+      for (uint64_t h = sh; h < dbh; ++h)
+      {
+        uint64_t ts = m_db->get_block_timestamp(h);
+        difficulty_type d = m_db->get_block_cumulative_difficulty(h);
+        ss << "  " << h << " " << ts << " " << d << std::endl;
+      }
+      print = true;
+    }
     m_timestamps_and_difficulties_height = height;
     m_timestamps = timestamps;
     m_difficulties = difficulties;
   }
+
   size_t target = get_difficulty_target();
   difficulty_type diff = next_difficulty(timestamps, difficulties, target);
 
   CRITICAL_REGION_LOCAL1(m_difficulty_lock);
   m_difficulty_for_next_block_top_hash = top_hash;
   m_difficulty_for_next_block = diff;
+  if (D && D != diff)
+  {
+    ss << "XXX Mismatch at " << height << "/" << top_hash << "/" << get_tail_id() << ": cached " << D << ", real " << diff << std::endl;
+    print = true;
+  }
+
+  ++done;
+  if (done == 1 && D && D != diff)
+  {
+    print = true;
+    ss << "Might be a race. Let's see what happens if we try again..." << std::endl;
+    epee::misc_utils::sleep_no_w(100);
+    goto start;
+  }
+  ss << "Diff for " << top_hash << ": " << diff << std::endl;
+  if (print)
+  {
+    MGINFO("START DUMP");
+    MGINFO(ss.str());
+    MGINFO("END DUMP");
+    MGINFO("Please send moneromooo on Freenode the contents of this log, from a couple dozen lines before START DUMP to END DUMP");
+  }
   return diff;
 }
 //------------------------------------------------------------------
@@ -914,6 +986,7 @@ bool Blockchain::rollback_blockchain_switching(std::list<block>& original_chain,
   }
 
   m_timestamps_and_difficulties_height = 0;
+  m_reset_timestamps_and_difficulties_height = true;
 
   // remove blocks from blockchain until we get back to where we should be.
   while (m_db->height() != rollback_height)
@@ -950,6 +1023,7 @@ bool Blockchain::switch_to_alternative_blockchain(std::list<block_extended_info>
   CRITICAL_REGION_LOCAL(m_blockchain_lock);
 
   m_timestamps_and_difficulties_height = 0;
+  m_reset_timestamps_and_difficulties_height = true;
 
   // if empty alt chain passed (not sure how that could happen), return false
   CHECK_AND_ASSERT_MES(alt_chain.size(), false, "switch_to_alternative_blockchain: empty chain passed");
@@ -1639,6 +1713,7 @@ bool Blockchain::handle_alternative_block(const block& b, const crypto::hash& id
   LOG_PRINT_L3("Blockchain::" << __func__);
   CRITICAL_REGION_LOCAL(m_blockchain_lock);
   m_timestamps_and_difficulties_height = 0;
+  m_reset_timestamps_and_difficulties_height = true;
   uint64_t block_height = get_block_height(b);
   if(0 == block_height)
   {
@@ -2493,6 +2568,7 @@ bool Blockchain::find_blockchain_supplement(const uint64_t req_start_block, cons
   }
 
   db_rtxn_guard rtxn_guard(m_db);
+  total_height = get_current_blockchain_height();
   blocks.reserve(std::min(std::min(max_count, (size_t)10000), (size_t)(total_height - start_height)));
   CHECK_AND_ASSERT_MES(m_db->get_blocks_from(start_height, 3, max_count, FIND_BLOCKCHAIN_SUPPLEMENT_MAX_SIZE, blocks, pruned, true, get_miner_tx_hash),
       false, "Error getting blocks");
@@ -4177,6 +4253,9 @@ bool Blockchain::update_next_cumulative_weight_limit(uint64_t *long_term_effecti
 //------------------------------------------------------------------
 bool Blockchain::add_new_block(const block& bl, block_verification_context& bvc)
 {
+  try
+  {
+
   LOG_PRINT_L3("Blockchain::" << __func__);
   crypto::hash id = get_block_hash(bl);
   CRITICAL_REGION_LOCAL(m_tx_pool);//to avoid deadlock lets lock tx_pool for whole add/reorganize process
@@ -4204,6 +4283,14 @@ bool Blockchain::add_new_block(const block& bl, block_verification_context& bvc)
 
   rtxn_guard.stop();
   return handle_block_to_main_chain(bl, id, bvc);
+
+  }
+  catch (const std::exception &e)
+  {
+    LOG_ERROR("Exception at [add_new_block], what=" << e.what());
+    bvc.m_verifivation_failed = true;
+    return false;
+  }
 }
 //------------------------------------------------------------------
 //TODO: Refactor, consider returning a failure height and letting
@@ -4317,7 +4404,14 @@ bool Blockchain::cleanup_handle_incoming_blocks(bool force_sync)
   try
   {
     if (m_batch_success)
+    {
       m_db->batch_stop();
+      if (m_reset_timestamps_and_difficulties_height)
+      {
+        m_timestamps_and_difficulties_height = 0;
+        m_reset_timestamps_and_difficulties_height = false;
+      }
+    }
     else
       m_db->batch_abort();
     success = true;
@@ -5028,7 +5122,7 @@ void Blockchain::cancel()
 }
 
 #if defined(PER_BLOCK_CHECKPOINT)
-static const char expected_block_hashes_hash[] = "fce1dc7c17f7679f5f447df206b8f5fe2ef6b1a2845e59f650850a0ef00d265f";
+static const char expected_block_hashes_hash[] = "37e15136d7527e47940ef85bff9d258b940c583bcc2e820aa9a98833a7344ece";
 void Blockchain::load_compiled_in_block_hashes(const GetCheckpointsCallback& get_checkpoints)
 {
   if (get_checkpoints == nullptr || !m_fast_sync)

src/cryptonote_core/blockchain.h

@@ -1067,6 +1067,7 @@ namespace cryptonote
     std::vector<uint64_t> m_timestamps;
     std::vector<difficulty_type> m_difficulties;
     uint64_t m_timestamps_and_difficulties_height;
+    bool m_reset_timestamps_and_difficulties_height;
     uint64_t m_long_term_block_weights_window;
     uint64_t m_long_term_effective_median_block_weight;
     mutable crypto::hash m_long_term_block_weights_cache_tip_hash;

src/cryptonote_core/cryptonote_core.cpp

@@ -650,7 +650,7 @@ namespace cryptonote
     r = m_blockchain_storage.init(db.release(), m_nettype, m_offline, regtest ? &regtest_test_options : test_options, fixed_difficulty, get_checkpoints);
     CHECK_AND_ASSERT_MES(r, false, "Failed to initialize blockchain storage");
 
-    r = m_mempool.init(max_txpool_weight);
+    r = m_mempool.init(max_txpool_weight, m_nettype == FAKECHAIN);
     CHECK_AND_ASSERT_MES(r, false, "Failed to initialize memory pool");
 
     // now that we have a valid m_blockchain_storage, we can clean out any
@@ -1649,14 +1649,12 @@ namespace cryptonote
         << "You can set the level of process detailization through \"set_log <level|categories>\" command," << ENDL
         << "where <level> is between 0 (no details) and 4 (very verbose), or custom category based levels (eg, *:WARNING)." << ENDL
         << ENDL
-        << "Use the \"help\" command to see a simplified list of available commands." << ENDL
-        << "Use the \"help_advanced\" command to see an advanced list of available commands." << ENDL
-        << "Use \"help_advanced <command>\" to see a command's documentation." << ENDL
+        << "Use the \"help\" command to see the list of available commands." << ENDL
+        << "Use \"help <command>\" to see a command's documentation." << ENDL
         << "**********************************************************************" << ENDL);
       m_starter_message_showed = true;
     }
 
-    m_fork_moaner.do_call(boost::bind(&core::check_fork_time, this));
     m_txpool_auto_relayer.do_call(boost::bind(&core::relay_txpool_transactions, this));
     m_check_updates_interval.do_call(boost::bind(&core::check_updates, this));
     m_check_disk_space_interval.do_call(boost::bind(&core::check_disk_space, this));
@@ -1667,29 +1665,6 @@ namespace cryptonote
     return true;
   }
   //-----------------------------------------------------------------------------------------------
-  bool core::check_fork_time()
-  {
-    if (m_nettype == FAKECHAIN)
-      return true;
-
-    HardFork::State state = m_blockchain_storage.get_hard_fork_state();
-    el::Level level;
-    switch (state) {
-      case HardFork::LikelyForked:
-        level = el::Level::Warning;
-        MCLOG_RED(level, "global", "**********************************************************************");
-        MCLOG_RED(level, "global", "Last scheduled hard fork is too far in the past.");
-        MCLOG_RED(level, "global", "We are most likely forked from the network. Daemon update needed now.");
-        MCLOG_RED(level, "global", "**********************************************************************");
-        break;
-      case HardFork::UpdateNeeded:
-        break;
-      default:
-        break;
-    }
-    return true;
-  }
-  //-----------------------------------------------------------------------------------------------
   uint8_t core::get_ideal_hard_fork_version() const
   {
     return get_blockchain_storage().get_ideal_hard_fork_version();

src/cryptonote_core/cryptonote_core.h

@@ -1000,18 +1000,6 @@ namespace cryptonote
       */
      bool check_tx_inputs_keyimages_domain(const transaction& tx) const;
 
-     /**
-      * @brief checks HardFork status and prints messages about it
-      *
-      * Checks the status of HardFork and logs/prints if an update to
-      * the daemon is necessary.
-      *
-      * @note see Blockchain::get_hard_fork_state and HardFork::State
-      *
-      * @return true
-      */
-     bool check_fork_time();
-
      /**
       * @brief attempts to relay any transactions in the mempool which need it
       *

src/cryptonote_core/tx_pool.cpp

@@ -116,7 +116,7 @@ namespace cryptonote
   }
   //---------------------------------------------------------------------------------
   //---------------------------------------------------------------------------------
-  tx_memory_pool::tx_memory_pool(Blockchain& bchs): m_blockchain(bchs), m_txpool_max_weight(DEFAULT_TXPOOL_MAX_WEIGHT), m_txpool_weight(0), m_cookie(0)
+  tx_memory_pool::tx_memory_pool(Blockchain& bchs): m_blockchain(bchs), m_cookie(0), m_txpool_max_weight(DEFAULT_TXPOOL_MAX_WEIGHT), m_txpool_weight(0), m_mine_stem_txes(false)
   {
 
   }
@@ -1351,13 +1351,18 @@ namespace cryptonote
     for (; sorted_it != m_txs_by_fee_and_receive_time.end(); ++sorted_it)
     {
       txpool_tx_meta_t meta;
-      if (!m_blockchain.get_txpool_tx_meta(sorted_it->second, meta) && !meta.matches(relay_category::legacy))
+      if (!m_blockchain.get_txpool_tx_meta(sorted_it->second, meta))
       {
         MERROR("  failed to find tx meta");
         continue;
       }
-      LOG_PRINT_L2("Considering " << sorted_it->second << ", weight " << meta.weight << ", current block weight " << total_weight << "/" << max_total_weight << ", current coinbase " << print_money(best_coinbase));
+      LOG_PRINT_L2("Considering " << sorted_it->second << ", weight " << meta.weight << ", current block weight " << total_weight << "/" << max_total_weight << ", current coinbase " << print_money(best_coinbase) << ", relay method " << (unsigned)meta.get_relay_method());
 
+      if (!meta.matches(relay_category::legacy) && !(m_mine_stem_txes && meta.get_relay_method() == relay_method::stem))
+      {
+        LOG_PRINT_L2("  tx relay method is " << (unsigned)meta.get_relay_method());
+        continue;
+      }
       if (meta.pruned)
       {
         LOG_PRINT_L2("  tx is pruned");
@@ -1522,7 +1527,7 @@ namespace cryptonote
     return n_removed;
   }
   //---------------------------------------------------------------------------------
-  bool tx_memory_pool::init(size_t max_txpool_weight)
+  bool tx_memory_pool::init(size_t max_txpool_weight, bool mine_stem_txes)
   {
     CRITICAL_REGION_LOCAL(m_transactions_lock);
     CRITICAL_REGION_LOCAL1(m_blockchain);
@@ -1578,6 +1583,7 @@ namespace cryptonote
       lock.commit();
     }
 
+    m_mine_stem_txes = mine_stem_txes;
     m_cookie = 0;
 
     // Ignore deserialization error

src/cryptonote_core/tx_pool.h

@@ -205,10 +205,11 @@ namespace cryptonote
      * @brief loads pool state (if any) from disk, and initializes pool
      *
      * @param max_txpool_weight the max weight in bytes
+     * @param mine_stem_txes whether to mine txes in stem relay mode
      *
      * @return true
      */
-    bool init(size_t max_txpool_weight = 0);
+    bool init(size_t max_txpool_weight = 0, bool mine_stem_txes = false);
 
     /**
      * @brief attempts to save the transaction pool state to disk
@@ -603,6 +604,7 @@ private:
 
     size_t m_txpool_max_weight;
     size_t m_txpool_weight;
+    bool m_mine_stem_txes;
 
     mutable std::unordered_map<crypto::hash, std::tuple<bool, tx_verification_context, uint64_t, crypto::hash>> m_input_cache;
 

src/cryptonote_protocol/cryptonote_protocol_handler.h

@@ -51,7 +51,8 @@ PUSH_WARNINGS
 DISABLE_VS_WARNINGS(4355)
 
 #define LOCALHOST_INT 2130706433
-#define CURRENCY_PROTOCOL_MAX_OBJECT_REQUEST_COUNT 500
+#define CURRENCY_PROTOCOL_MAX_OBJECT_REQUEST_COUNT 100
+static_assert(CURRENCY_PROTOCOL_MAX_OBJECT_REQUEST_COUNT >= BLOCKS_SYNCHRONIZING_DEFAULT_COUNT_PRE_V4, "Invalid CURRENCY_PROTOCOL_MAX_OBJECT_REQUEST_COUNT");
 
 namespace cryptonote
 {

src/cryptonote_protocol/cryptonote_protocol_handler.inl

@@ -308,9 +308,9 @@ namespace cryptonote
       if (version >= 6 && version != hshd.top_version)
       {
         if (version < hshd.top_version && version == m_core.get_ideal_hard_fork_version())
-          MCLOG_RED(el::Level::Warning, "global", context << " peer claims higher version than we think (" <<
+          MDEBUG(context << " peer claims higher version than we think (" <<
               (unsigned)hshd.top_version << " for " << (hshd.current_height - 1) << " instead of " << (unsigned)version <<
-              ") - we may be forked from the network and a software upgrade may be needed");
+              ") - we may be forked from the network and a software upgrade may be needed, or that peer is broken or malicious");
         return false;
       }
     }
@@ -793,6 +793,12 @@ namespace cryptonote
   int t_cryptonote_protocol_handler<t_core>::handle_request_fluffy_missing_tx(int command, NOTIFY_REQUEST_FLUFFY_MISSING_TX::request& arg, cryptonote_connection_context& context)
   {
     MLOG_P2P_MESSAGE("Received NOTIFY_REQUEST_FLUFFY_MISSING_TX (" << arg.missing_tx_indices.size() << " txes), block hash " << arg.block_hash);
+    if (context.m_state == cryptonote_connection_context::state_before_handshake)
+    {
+      LOG_ERROR_CCONTEXT("Requested fluffy tx before handshake, dropping connection");
+      drop_connection(context, false, false);
+      return 1;
+    }
     
     std::vector<std::pair<cryptonote::blobdata, block>> local_blocks;
     std::vector<cryptonote::blobdata> local_txs;
@@ -884,6 +890,8 @@ namespace cryptonote
   int t_cryptonote_protocol_handler<t_core>::handle_notify_get_txpool_complement(int command, NOTIFY_GET_TXPOOL_COMPLEMENT::request& arg, cryptonote_connection_context& context)
   {
     MLOG_P2P_MESSAGE("Received NOTIFY_GET_TXPOOL_COMPLEMENT (" << arg.hashes.size() << " txes)");
+    if(context.m_state != cryptonote_connection_context::state_normal)
+      return 1;
 
     std::vector<std::pair<cryptonote::blobdata, block>> local_blocks;
     std::vector<cryptonote::blobdata> local_txs;
@@ -987,6 +995,12 @@ namespace cryptonote
   template<class t_core>
   int t_cryptonote_protocol_handler<t_core>::handle_request_get_objects(int command, NOTIFY_REQUEST_GET_OBJECTS::request& arg, cryptonote_connection_context& context)
   {
+    if (context.m_state == cryptonote_connection_context::state_before_handshake)
+    {
+      LOG_ERROR_CCONTEXT("Requested objects before handshake, dropping connection");
+      drop_connection(context, false, false);
+      return 1;
+    }
     MLOG_P2P_MESSAGE("Received NOTIFY_REQUEST_GET_OBJECTS (" << arg.blocks.size() << " blocks)");
     if (arg.blocks.size() > CURRENCY_PROTOCOL_MAX_OBJECT_REQUEST_COUNT)
       {
@@ -1717,11 +1731,16 @@ skip:
   int t_cryptonote_protocol_handler<t_core>::handle_request_chain(int command, NOTIFY_REQUEST_CHAIN::request& arg, cryptonote_connection_context& context)
   {
     MLOG_P2P_MESSAGE("Received NOTIFY_REQUEST_CHAIN (" << arg.block_ids.size() << " blocks");
+    if (context.m_state == cryptonote_connection_context::state_before_handshake)
+    {
+      LOG_ERROR_CCONTEXT("Requested chain before handshake, dropping connection");
+      drop_connection(context, false, false);
+      return 1;
+    }
     NOTIFY_RESPONSE_CHAIN_ENTRY::request r;
     if(!m_core.find_blockchain_supplement(arg.block_ids, !arg.prune, r))
     {
       LOG_ERROR_CCONTEXT("Failed to handle NOTIFY_REQUEST_CHAIN.");
-      drop_connection(context, false, false);
       return 1;
     }
     MLOG_P2P_MESSAGE("-->>NOTIFY_RESPONSE_CHAIN_ENTRY: m_start_height=" << r.start_height << ", m_total_height=" << r.total_height << ", m_block_ids.size()=" << r.m_block_ids.size());
@@ -2318,8 +2337,7 @@ skip:
       MGINFO_YELLOW(ENDL << "**********************************************************************" << ENDL
         << "You are now synchronized with the network. You may now start monero-wallet-cli." << ENDL
         << ENDL
-        << "Use the \"help\" command to see a simplified list of available commands." << ENDL
-        << "Use the \"help_advanced\" command to see an advanced list of available commands." << ENDL
+        << "Use the \"help\" command to see the list of available commands." << ENDL
         << "**********************************************************************");
       m_sync_timer.pause();
       if (ELPP->vRegistry()->allowed(el::Level::Info, "sync-info"))

src/daemon/rpc_command_executor.cpp

@@ -511,7 +511,7 @@ bool t_rpc_command_executor::show_status() {
   }
 
   std::stringstream str;
-  str << boost::format("Height: %llu/%llu (%.1f%%) on %s%s, %s, net hash %s, v%u%s, %s, %u(out)+%u(in) connections")
+  str << boost::format("Height: %llu/%llu (%.1f%%) on %s%s, %s, net hash %s, v%u%s, %u(out)+%u(in) connections")
     % (unsigned long long)ires.height
     % (unsigned long long)net_height
     % get_sync_percentage(ires)
@@ -521,7 +521,6 @@ bool t_rpc_command_executor::show_status() {
     % get_mining_speed(cryptonote::difficulty_type(ires.wide_difficulty) / ires.target)
     % (unsigned)hfres.version
     % get_fork_extra_info(hfres.earliest_height, net_height, ires.target)
-    % (hfres.state == cryptonote::HardFork::Ready ? "up to date" : hfres.state == cryptonote::HardFork::UpdateNeeded ? "update needed" : "out of date, likely forked")
     % (unsigned)ires.outgoing_connections_count
     % (unsigned)ires.incoming_connections_count
   ;

src/device/device_ledger.cpp

@@ -1468,8 +1468,8 @@ namespace hw {
         offset = set_command_header(INS_PREFIX_HASH,2,cnt);      
         len = pref_length - pref_offset;
         //options
-        if (len > (BUFFER_SEND_SIZE-7)) {
-          len = BUFFER_SEND_SIZE-7;
+        if (len > (BUFFER_SEND_SIZE-offset-3)) {
+          len = BUFFER_SEND_SIZE-offset-3;
           this->buffer_send[offset] = 0x80;
         } else {
           this->buffer_send[offset] = 0x00;

src/device_trezor/device_trezor.cpp

@@ -678,8 +678,10 @@ namespace trezor {
         throw exc::TrezorException("Trezor firmware 2.0.10 and lower are not supported. Please update.");
       }
 
-      // default client version, higher versions check will be added
       unsigned client_version = 1;
+      if (trezor_version >= pack_version(2, 3, 1)){
+        client_version = 3;
+      }
 
 #ifdef WITH_TREZOR_DEBUGGING
       // Override client version for tests

src/device_trezor/trezor/transport.cpp

@@ -711,7 +711,7 @@ namespace trezor{
     // Start the asynchronous operation itself. The handle_receive function
     // used as a callback will update the ec and length variables.
     m_socket->async_receive_from(boost::asio::buffer(buffer), m_endpoint,
-                                 boost::bind(&UdpTransport::handle_receive, _1, _2, &ec, &length));
+                                 boost::bind(&UdpTransport::handle_receive, boost::placeholders::_1, boost::placeholders::_2, &ec, &length));
 
     // Block until the asynchronous operation has completed.
     do {

src/mnemonics/language_base.h

@@ -41,6 +41,7 @@
 #include <boost/algorithm/string.hpp>
 #include "misc_log_ex.h"
 #include "fnv1.h"
+#include "common/utf8.h"
 
 /*!
  * \namespace Language
@@ -73,78 +74,11 @@ namespace Language
     return prefix;
   }
 
-  template<typename T>
-  inline T utf8canonical(const T &s)
-  {
-    T sc = "";
-    size_t avail = s.size();
-    const char *ptr = s.data();
-    wint_t cp = 0;
-    int bytes = 1;
-    char wbuf[8], *wptr;
-    while (avail--)
-    {
-      if ((*ptr & 0x80) == 0)
-      {
-        cp = *ptr++;
-        bytes = 1;
-      }
-      else if ((*ptr & 0xe0) == 0xc0)
-      {
-        if (avail < 1)
-          throw std::runtime_error("Invalid UTF-8");
-        cp = (*ptr++ & 0x1f) << 6;
-        cp |= *ptr++ & 0x3f;
-        --avail;
-        bytes = 2;
-      }
-      else if ((*ptr & 0xf0) == 0xe0)
-      {
-        if (avail < 2)
-          throw std::runtime_error("Invalid UTF-8");
-        cp = (*ptr++ & 0xf) << 12;
-        cp |= (*ptr++ & 0x3f) << 6;
-        cp |= *ptr++ & 0x3f;
-        avail -= 2;
-        bytes = 3;
-      }
-      else if ((*ptr & 0xf8) == 0xf0)
-      {
-        if (avail < 3)
-          throw std::runtime_error("Invalid UTF-8");
-        cp = (*ptr++ & 0x7) << 18;
-        cp |= (*ptr++ & 0x3f) << 12;
-        cp |= (*ptr++ & 0x3f) << 6;
-        cp |= *ptr++ & 0x3f;
-        avail -= 3;
-        bytes = 4;
-      }
-      else
-        throw std::runtime_error("Invalid UTF-8");
-
-      cp = std::towlower(cp);
-      wptr = wbuf;
-      switch (bytes)
-      {
-        case 1: *wptr++ = cp; break;
-        case 2: *wptr++ = 0xc0 | (cp >> 6); *wptr++ = 0x80 | (cp & 0x3f); break;
-        case 3: *wptr++ = 0xe0 | (cp >> 12); *wptr++ = 0x80 | ((cp >> 6) & 0x3f); *wptr++ = 0x80 | (cp & 0x3f); break;
-        case 4: *wptr++ = 0xf0 | (cp >> 18); *wptr++ = 0x80 | ((cp >> 12) & 0x3f); *wptr++ = 0x80 | ((cp >> 6) & 0x3f); *wptr++ = 0x80 | (cp & 0x3f); break;
-        default: throw std::runtime_error("Invalid UTF-8");
-      }
-      *wptr = 0;
-      sc += T(wbuf, bytes);
-      cp = 0;
-      bytes = 1;
-    }
-    return sc;
-  }
-
   struct WordHash
   {
     std::size_t operator()(const epee::wipeable_string &s) const
     {
-      const epee::wipeable_string sc = utf8canonical(s);
+      const epee::wipeable_string sc = tools::utf8canonical(s, [](wint_t c) -> wint_t { return std::towlower(c); });
       return epee::fnv::FNV1a(sc.data(), sc.size());
     }
   };
@@ -153,8 +87,8 @@ namespace Language
   {
     bool operator()(const epee::wipeable_string &s0, const epee::wipeable_string &s1) const
     {
-      const epee::wipeable_string s0c = utf8canonical(s0);
-      const epee::wipeable_string s1c = utf8canonical(s1);
+      const epee::wipeable_string s0c = tools::utf8canonical(s0, [](wint_t c) -> wint_t { return std::towlower(c); });
+      const epee::wipeable_string s1c = tools::utf8canonical(s1, [](wint_t c) -> wint_t { return std::towlower(c); });
       return s0c == s1c;
     }
   };

src/rpc/core_rpc_server.cpp

@@ -68,6 +68,11 @@ using namespace epee;
 #define DEFAULT_PAYMENT_DIFFICULTY 1000
 #define DEFAULT_PAYMENT_CREDITS_PER_HASH 100
 
+#define RESTRICTED_BLOCK_HEADER_RANGE 1000
+#define RESTRICTED_TRANSACTIONS_COUNT 100
+#define RESTRICTED_SPENT_KEY_IMAGES_COUNT 5000
+#define RESTRICTED_BLOCK_COUNT 1000
+
 #define RPC_TRACKER(rpc) \
   PERF_TIMER(rpc); \
   RPCTracker tracker(#rpc, PERF_TIMER_NAME(rpc))
@@ -265,25 +270,25 @@ namespace cryptonote
     {
       if (!m_restricted && nettype() != FAKECHAIN)
       {
-        MERROR("RPC payment enabled, but server is not restricted, anyone can adjust their balance to bypass payment");
+        MFATAL("RPC payment enabled, but server is not restricted, anyone can adjust their balance to bypass payment");
         return false;
       }
       cryptonote::address_parse_info info;
       if (!get_account_address_from_str(info, nettype(), address))
       {
-        MERROR("Invalid payment address: " << address);
+        MFATAL("Invalid payment address: " << address);
         return false;
       }
       if (info.is_subaddress)
       {
-        MERROR("Payment address may not be a subaddress: " << address);
+        MFATAL("Payment address may not be a subaddress: " << address);
         return false;
       }
       uint64_t diff = command_line::get_arg(vm, arg_rpc_payment_difficulty);
       uint64_t credits = command_line::get_arg(vm, arg_rpc_payment_credits);
       if (diff == 0 || credits == 0)
       {
-        MERROR("Payments difficulty and/or payments credits are 0, but a payment address was given");
+        MFATAL("Payments difficulty and/or payments credits are 0, but a payment address was given");
         return false;
       }
       m_rpc_payment_allow_free_loopback = command_line::get_arg(vm, arg_rpc_payment_allow_free_loopback);
@@ -303,7 +308,7 @@ namespace cryptonote
     if (!set_bootstrap_daemon(command_line::get_arg(vm, arg_bootstrap_daemon_address),
       command_line::get_arg(vm, arg_bootstrap_daemon_login)))
     {
-      MERROR("Failed to parse bootstrap daemon address");
+      MFATAL("Failed to parse bootstrap daemon address");
       return false;
     }
 
@@ -639,6 +644,13 @@ namespace cryptonote
     if (use_bootstrap_daemon_if_necessary<COMMAND_RPC_GET_BLOCKS_BY_HEIGHT>(invoke_http_mode::BIN, "/getblocks_by_height.bin", req, res, r))
       return r;
 
+    const bool restricted = m_restricted && ctx;
+    if (restricted && req.heights.size() > RESTRICTED_BLOCK_COUNT)
+    {
+      res.status = "Too many blocks requested in restricted mode";
+      return true;
+    }
+
     res.status = "Failed";
     res.blocks.clear();
     res.blocks.reserve(req.heights.size());
@@ -793,11 +805,17 @@ namespace cryptonote
     if (use_bootstrap_daemon_if_necessary<COMMAND_RPC_GET_TRANSACTIONS>(invoke_http_mode::JON, "/gettransactions", req, res, ok))
       return ok;
 
-    CHECK_PAYMENT_MIN1(req, res, req.txs_hashes.size() * COST_PER_TX, false);
-
     const bool restricted = m_restricted && ctx;
     const bool request_has_rpc_origin = ctx != NULL;
 
+    if (restricted && req.txs_hashes.size() > RESTRICTED_TRANSACTIONS_COUNT)
+    {
+      res.status = "Too many transactions requested in restricted mode";
+      return true;
+    }
+
+    CHECK_PAYMENT_MIN1(req, res, req.txs_hashes.size() * COST_PER_TX, false);
+
     std::vector<crypto::hash> vh;
     for(const auto& tx_hex_str: req.txs_hashes)
     {
@@ -1027,11 +1045,17 @@ namespace cryptonote
     if (use_bootstrap_daemon_if_necessary<COMMAND_RPC_IS_KEY_IMAGE_SPENT>(invoke_http_mode::JON, "/is_key_image_spent", req, res, ok))
       return ok;
 
-    CHECK_PAYMENT_MIN1(req, res, req.key_images.size() * COST_PER_KEY_IMAGE, false);
-
     const bool restricted = m_restricted && ctx;
     const bool request_has_rpc_origin = ctx != NULL;
 
+    if (restricted && req.key_images.size() > RESTRICTED_SPENT_KEY_IMAGES_COUNT)
+    {
+      res.status = "Too many key images queried in restricted mode";
+      return true;
+    }
+
+    CHECK_PAYMENT_MIN1(req, res, req.key_images.size() * COST_PER_KEY_IMAGE, false);
+
     std::vector<crypto::key_image> key_images;
     for(const auto& ki_hex_str: req.key_images)
     {
@@ -2034,6 +2058,14 @@ namespace cryptonote
 
     CHECK_PAYMENT_MIN1(req, res, COST_PER_BLOCK_HEADER, false);
 
+    const bool restricted = m_restricted && ctx;
+    if (restricted && req.hashes.size() > RESTRICTED_BLOCK_COUNT)
+    {
+      error_resp.code = CORE_RPC_ERROR_CODE_RESTRICTED;
+      error_resp.message = "Too many block headers requested in restricted mode";
+      return false;
+    }
+
     auto get = [this](const std::string &hash, bool fill_pow_hash, block_header_response &block_header, bool restricted, epee::json_rpc::error& error_resp) -> bool {
       crypto::hash block_hash;
       bool hash_parsed = parse_hash256(hash, block_hash);
@@ -2069,7 +2101,6 @@ namespace cryptonote
       return true;
     };
 
-    const bool restricted = m_restricted && ctx;
     if (!req.hash.empty())
     {
       if (!get(req.hash, req.fill_pow_hash, res.block_header, restricted, error_resp))
@@ -2101,6 +2132,14 @@ namespace cryptonote
       error_resp.message = "Invalid start/end heights.";
       return false;
     }
+    const bool restricted = m_restricted && ctx;
+    if (restricted && req.end_height - req.start_height > RESTRICTED_BLOCK_HEADER_RANGE)
+    {
+      error_resp.code = CORE_RPC_ERROR_CODE_RESTRICTED;
+      error_resp.message = "Too many block headers requested.";
+      return false;
+    }
+
     CHECK_PAYMENT_MIN1(req, res, (req.end_height - req.start_height + 1) * COST_PER_BLOCK_HEADER, false);
     for (uint64_t h = req.start_height; h <= req.end_height; ++h)
     {
@@ -2127,7 +2166,6 @@ namespace cryptonote
         return false;
       }
       res.headers.push_back(block_header_response());
-      const bool restricted = m_restricted && ctx;
       bool response_filled = fill_block_header_response(blk, false, block_height, block_hash, res.headers.back(), req.fill_pow_hash && !restricted);
       if (!response_filled)
       {
@@ -2415,14 +2453,13 @@ namespace cryptonote
     {
       for (const auto &str: req.txids)
       {
-        cryptonote::blobdata txid_data;
-        if(!epee::string_tools::parse_hexstr_to_binbuff(str, txid_data))
+        crypto::hash txid;
+        if(!epee::string_tools::hex_to_pod(str, txid))
         {
           failed = true;
         }
         else
         {
-          crypto::hash txid = *reinterpret_cast<const crypto::hash*>(txid_data.data());
           txids.push_back(txid);
         }
       }
@@ -2767,18 +2804,17 @@ namespace cryptonote
     res.status = "";
     for (const auto &str: req.txids)
     {
-      cryptonote::blobdata txid_data;
-      if(!epee::string_tools::parse_hexstr_to_binbuff(str, txid_data))
+      crypto::hash txid;
+      if(!epee::string_tools::hex_to_pod(str, txid))
       {
         if (!res.status.empty()) res.status += ", ";
         res.status += std::string("invalid transaction id: ") + str;
         failed = true;
         continue;
       }
-      crypto::hash txid = *reinterpret_cast<const crypto::hash*>(txid_data.data());
 
       cryptonote::blobdata txblob;
-      if (!m_core.get_pool_transaction(txid, txblob, relay_category::legacy))
+      if (m_core.get_pool_transaction(txid, txblob, relay_category::legacy))
       {
         NOTIFY_NEW_TRANSACTIONS::request r;
         r.txs.push_back(std::move(txblob));

src/rpc/core_rpc_server_error_codes.h

@@ -48,6 +48,7 @@
 #define CORE_RPC_ERROR_CODE_PAYMENT_TOO_LOW       -16
 #define CORE_RPC_ERROR_CODE_DUPLICATE_PAYMENT     -17
 #define CORE_RPC_ERROR_CODE_STALE_PAYMENT         -18
+#define CORE_RPC_ERROR_CODE_RESTRICTED            -19
 
 static inline const char *get_rpc_server_error_message(int64_t code)
 {
@@ -70,6 +71,7 @@ static inline const char *get_rpc_server_error_message(int64_t code)
     case CORE_RPC_ERROR_CODE_PAYMENT_TOO_LOW: return "Payment too low";
     case CORE_RPC_ERROR_CODE_DUPLICATE_PAYMENT: return "Duplicate payment";
     case CORE_RPC_ERROR_CODE_STALE_PAYMENT: return "Stale payment";
+    case CORE_RPC_ERROR_CODE_RESTRICTED: return "Parameters beyond restricted allowance";
     default: MERROR("Unknown error: " << code); return "Unknown error";
   }
 }

src/rpc/rpc_payment.cpp

@@ -92,6 +92,7 @@ namespace cryptonote
 
   uint64_t rpc_payment::balance(const crypto::public_key &client, int64_t delta)
   {
+    boost::lock_guard<boost::mutex> lock(mutex);
     client_info &info = m_client_info[client]; // creates if not found
     uint64_t credits = info.credits;
     if (delta > 0 && credits > std::numeric_limits<uint64_t>::max() - delta)
@@ -107,6 +108,7 @@ namespace cryptonote
 
   bool rpc_payment::pay(const crypto::public_key &client, uint64_t ts, uint64_t payment, const std::string &rpc, bool same_ts, uint64_t &credits)
   {
+    boost::lock_guard<boost::mutex> lock(mutex);
     client_info &info = m_client_info[client]; // creates if not found
     if (ts < info.last_request_timestamp || (ts == info.last_request_timestamp && !same_ts))
     {
@@ -130,6 +132,7 @@ namespace cryptonote
 
   bool rpc_payment::get_info(const crypto::public_key &client, const std::function<bool(const cryptonote::blobdata&, cryptonote::block&, uint64_t &seed_height, crypto::hash &seed_hash)> &get_block_template, cryptonote::blobdata &hashing_blob, uint64_t &seed_height, crypto::hash &seed_hash, const crypto::hash &top, uint64_t &diff, uint64_t &credits_per_hash_found, uint64_t &credits, uint32_t &cookie)
   {
+    boost::lock_guard<boost::mutex> lock(mutex);
     client_info &info = m_client_info[client]; // creates if not found
     const uint64_t now = time(NULL);
     bool need_template = top != info.top || now >= info.block_template_update_time + STALE_THRESHOLD;
@@ -180,6 +183,7 @@ namespace cryptonote
 
   bool rpc_payment::submit_nonce(const crypto::public_key &client, uint32_t nonce, const crypto::hash &top, int64_t &error_code, std::string &error_message, uint64_t &credits, crypto::hash &hash, cryptonote::block &block, uint32_t cookie, bool &stale)
   {
+    boost::lock_guard<boost::mutex> lock(mutex);
     client_info &info = m_client_info[client]; // creates if not found
     if (cookie != info.cookie && cookie != info.cookie - 1)
     {
@@ -272,6 +276,7 @@ namespace cryptonote
 
   bool rpc_payment::foreach(const std::function<bool(const crypto::public_key &client, const client_info &info)> &f) const
   {
+    boost::lock_guard<boost::mutex> lock(mutex);
     for (std::unordered_map<crypto::public_key, client_info>::const_iterator i = m_client_info.begin(); i != m_client_info.end(); ++i)
     {
       if (!f(i->first, i->second))
@@ -283,8 +288,9 @@ namespace cryptonote
   bool rpc_payment::load(std::string directory)
   {
     TRY_ENTRY();
+    boost::lock_guard<boost::mutex> lock(mutex);
     m_directory = std::move(directory);
-    std::string state_file_path = directory + "/" + RPC_PAYMENTS_DATA_FILENAME;
+    std::string state_file_path = m_directory + "/" + RPC_PAYMENTS_DATA_FILENAME;
     MINFO("loading rpc payments data from " << state_file_path);
     std::ifstream data;
     data.open(state_file_path, std::ios_base::binary | std::ios_base::in);
@@ -313,6 +319,7 @@ namespace cryptonote
   bool rpc_payment::store(const std::string &directory_) const
   {
     TRY_ENTRY();
+    boost::lock_guard<boost::mutex> lock(mutex);
     const std::string &directory = directory_.empty() ? m_directory : directory_;
     MDEBUG("storing rpc payments data to " << directory);
     if (!tools::create_directories_if_necessary(directory))
@@ -345,6 +352,7 @@ namespace cryptonote
 
   unsigned int rpc_payment::flush_by_age(time_t seconds)
   {
+    boost::lock_guard<boost::mutex> lock(mutex);
     unsigned int count = 0;
     const time_t now = time(NULL);
     time_t seconds0 = seconds;
@@ -358,7 +366,7 @@ namespace cryptonote
     for (std::unordered_map<crypto::public_key, client_info>::iterator i = m_client_info.begin(); i != m_client_info.end(); )
     {
       std::unordered_map<crypto::public_key, client_info>::iterator j = i++;
-      const time_t t = std::max(j->second.last_request_timestamp, j->second.update_time);
+      const time_t t = std::max(j->second.last_request_timestamp / 1000000, j->second.update_time);
       const bool erase = t < ((j->second.credits == 0) ? threshold0 : threshold);
       if (erase)
       {
@@ -372,6 +380,7 @@ namespace cryptonote
 
   uint64_t rpc_payment::get_hashes(unsigned int seconds) const
   {
+    boost::lock_guard<boost::mutex> lock(mutex);
     const uint64_t now = time(NULL);
     uint64_t hashes = 0;
     for (std::map<uint64_t, uint64_t>::const_reverse_iterator i = m_hashrate.crbegin(); i != m_hashrate.crend(); ++i)
@@ -385,6 +394,7 @@ namespace cryptonote
 
   void rpc_payment::prune_hashrate(unsigned int seconds)
   {
+    boost::lock_guard<boost::mutex> lock(mutex);
     const uint64_t now = time(NULL);
     std::map<uint64_t, uint64_t>::iterator i;
     for (i = m_hashrate.begin(); i != m_hashrate.end(); ++i)

src/rpc/rpc_payment.h

@@ -31,6 +31,7 @@
 #include <string>
 #include <unordered_set>
 #include <unordered_map>
+#include <boost/thread/mutex.hpp>
 #include <boost/serialization/version.hpp>
 #include "cryptonote_basic/blobdatatype.h"
 #include "cryptonote_basic/cryptonote_basic.h"
@@ -139,6 +140,7 @@ namespace cryptonote
     uint64_t m_nonces_stale;
     uint64_t m_nonces_bad;
     uint64_t m_nonces_dupe;
+    mutable boost::mutex mutex;
   };
 }
 

src/serialization/json_object.cpp

@@ -120,7 +120,7 @@ void read_hex(const rapidjson::Value& val, epee::span<std::uint8_t> dest)
     throw WRONG_TYPE("string");
   }
 
-  if (!epee::from_hex::to_buffer(dest, {val.GetString(), val.Size()}))
+  if (!epee::from_hex::to_buffer(dest, {val.GetString(), val.GetStringLength()}))
   {
     throw BAD_INPUT();
   }

src/simplewallet/simplewallet.cpp

@@ -121,7 +121,7 @@ typedef cryptonote::simple_wallet sw;
 
 #define SCOPED_WALLET_UNLOCK() SCOPED_WALLET_UNLOCK_ON_BAD_PASSWORD(return true;)
 
-#define PRINT_USAGE(usage_help_advanced) fail_msg_writer() << boost::format(tr("usage: %s")) % usage_help_advanced;
+#define PRINT_USAGE(usage_help) fail_msg_writer() << boost::format(tr("usage: %s")) % usage_help;
 
 #define LONG_PAYMENT_ID_SUPPORT_CHECK() \
   do { \
@@ -250,6 +250,7 @@ namespace
   const char* USAGE_MMS_SET("mms set <option_name> [<option_value>]");
   const char* USAGE_MMS_SEND_SIGNER_CONFIG("mms send_signer_config");
   const char* USAGE_MMS_START_AUTO_CONFIG("mms start_auto_config [<label> <label> ...]");
+  const char* USAGE_MMS_CONFIG_CHECKSUM("mms config_checksum");
   const char* USAGE_MMS_STOP_AUTO_CONFIG("mms stop_auto_config");
   const char* USAGE_MMS_AUTO_CONFIG("mms auto_config <auto_config_token>");
   const char* USAGE_PRINT_RING("print_ring <key_image> | <txid>");
@@ -270,8 +271,7 @@ namespace
   const char* USAGE_START_MINING_FOR_RPC("start_mining_for_rpc");
   const char* USAGE_STOP_MINING_FOR_RPC("stop_mining_for_rpc");
   const char* USAGE_VERSION("version");
-  const char* USAGE_HELP_ADVANCED("help_advanced [<command>]");
-  const char* USAGE_HELP("help");
+  const char* USAGE_HELP("help [<command>]");
 
   std::string input_line(const std::string& prompt, bool yesno = false)
   {
@@ -2308,7 +2308,7 @@ bool simple_wallet::on_unknown_command(const std::vector<std::string> &args)
 {
   if (args[0] == "exit" || args[0] == "q") // backward compat
     return false;
-  fail_msg_writer() << boost::format(tr("Unknown command '%s', try 'help_advanced'")) % args.front();
+  fail_msg_writer() << boost::format(tr("Unknown command '%s', try 'help'")) % args.front();
   return true;
 }
 
@@ -3036,30 +3036,6 @@ bool simple_wallet::set_export_format(const std::vector<std::string> &args/* = s
 }
 
 bool simple_wallet::help(const std::vector<std::string> &args/* = std::vector<std::string>()*/)
-{
-  message_writer() << "";
-  message_writer() << tr("Commands:");
-  message_writer() << "";
-  message_writer() << tr("\"welcome\" - Read welcome message.");
-  message_writer() << tr("\"donate <amount>\" - Donate XMR to the development team.");
-  message_writer() << tr("\"balance\" - Show balance.");
-  message_writer() << tr("\"address new\" - Create new subaddress.");
-  message_writer() << tr("\"address all\" - Show all addresses.");
-  message_writer() << tr("\"transfer <address> <amount>\" - Send XMR to an address.");
-  message_writer() << tr("\"show_transfers [in|out|pending|failed|pool]\" - Show transactions.");
-  message_writer() << tr("\"sweep_all <address>\" - Send whole balance to another wallet.");
-  message_writer() << tr("\"seed\" - Show secret 25 words that can be used to recover this wallet.");
-  message_writer() << tr("\"refresh\" - Synchronize wallet with the Monero network.");
-  message_writer() << tr("\"status\" - Check current status of wallet.");
-  message_writer() << tr("\"version\" - Check software version.");
-  message_writer() << tr("\"help_advanced\" - Show list with more available commands.");
-  message_writer() << tr("\"save\" - Save wallet.");
-  message_writer() << tr("\"exit\" - Exit wallet.");
-  message_writer() << "";
-  return true;
-}
-
-bool simple_wallet::help_advanced(const std::vector<std::string> &args/* = std::vector<std::string>()*/)
 {
   if(args.empty())
   {
@@ -3067,7 +3043,7 @@ bool simple_wallet::help_advanced(const std::vector<std::string> &args/* = std::
   }
   else if ((args.size() == 2) && (args.front() == "mms"))
   {
-    // Little hack to be able to do "help_advanced mms <subcommand>"
+    // Little hack to be able to do "help mms <subcommand>"
     std::vector<std::string> mms_args(1, args.front() + " " + args.back());
     success_msg_writer() << get_command_usage(mms_args);
   }
@@ -3460,8 +3436,8 @@ simple_wallet::simple_wallet()
                            tr("Interface with the MMS (Multisig Messaging System)\n"
                               "<subcommand> is one of:\n"
                               "  init, info, signer, list, next, sync, transfer, delete, send, receive, export, note, show, set, help\n"
-                              "  send_signer_config, start_auto_config, stop_auto_config, auto_config\n"
-                              "Get help about a subcommand with: help_advanced mms <subcommand>"));
+                              "  send_signer_config, start_auto_config, stop_auto_config, auto_config, config_checksum\n"
+                              "Get help about a subcommand with: help mms <subcommand>, or mms help <subcommand>"));
   m_cmd_binder.set_handler("mms init",
                            boost::bind(&simple_wallet::on_command, this, &simple_wallet::mms, _1),
                            tr(USAGE_MMS_INIT),
@@ -3529,6 +3505,10 @@ simple_wallet::simple_wallet()
                            boost::bind(&simple_wallet::on_command, this, &simple_wallet::mms, _1),
                            tr(USAGE_MMS_START_AUTO_CONFIG),
                            tr("Start auto-config at the auto-config manager's wallet by issuing auto-config tokens and optionally set others' labels"));
+  m_cmd_binder.set_handler("mms config_checksum",
+                           boost::bind(&simple_wallet::on_command, this, &simple_wallet::mms, _1),
+                           tr(USAGE_MMS_CONFIG_CHECKSUM),
+                           tr("Get a checksum that allows signers to easily check for identical MMS configuration"));
   m_cmd_binder.set_handler("mms stop_auto_config",
                            boost::bind(&simple_wallet::on_command, this, &simple_wallet::mms, _1),
                            tr(USAGE_MMS_STOP_AUTO_CONFIG),
@@ -3611,14 +3591,10 @@ simple_wallet::simple_wallet()
                            boost::bind(&simple_wallet::stop_mining_for_rpc, this, _1),
                            tr(USAGE_STOP_MINING_FOR_RPC),
                            tr("Stop mining to pay for RPC access"));
-  m_cmd_binder.set_handler("help_advanced",
-                           boost::bind(&simple_wallet::on_command, this, &simple_wallet::help_advanced, _1),
-                           tr(USAGE_HELP_ADVANCED),
-                           tr("Show the help section or the documentation about a <command>."));
   m_cmd_binder.set_handler("help",
                            boost::bind(&simple_wallet::on_command, this, &simple_wallet::help, _1),
                            tr(USAGE_HELP),
-                           tr("Show simplified list of available commands."));
+                           tr("Show the help section or the documentation about a <command>."));
   m_cmd_binder.set_unknown_command_handler(boost::bind(&simple_wallet::on_command, this, &simple_wallet::on_unknown_command, _1));
   m_cmd_binder.set_empty_command_handler(boost::bind(&simple_wallet::on_empty_command, this));
   m_cmd_binder.set_cancel_handler(boost::bind(&simple_wallet::on_cancelled_command, this));
@@ -4770,9 +4746,8 @@ boost::optional<epee::wipeable_string> simple_wallet::new_wallet(const boost::pr
     "**********************************************************************\n" <<
     tr("Your wallet has been generated!\n"
     "To start synchronizing with the daemon, use the \"refresh\" command.\n"
-    "Use the \"help\" command to see a simplified list of available commands.\n"
-    "Use the \"help_advanced\" command to see an advanced list of available commands.\n"
-    "Use \"help_advanced <command>\" to see a command's documentation.\n"
+    "Use the \"help\" command to see the list of available commands.\n"
+    "Use \"help <command>\" to see a command's documentation.\n"
     "Always use the \"exit\" command when closing monero-wallet-cli to save \n"
     "your current session's state. Otherwise, you might need to synchronize \n"
     "your wallet again (your wallet keys are NOT at risk in any case).\n")
@@ -5031,9 +5006,8 @@ boost::optional<epee::wipeable_string> simple_wallet::open_wallet(const boost::p
   }
   success_msg_writer() <<
     "**********************************************************************\n" <<
-    tr("Use the \"help\" command to see a simplified list of available commands.\n") <<
-    tr("Use the \"help_advanced\" command to see an advanced list of available commands.\n") <<
-    tr("Use \"help_advanced <command>\" to see a command's documentation.\n") <<
+    tr("Use the \"help\" command to see the list of available commands.\n") <<
+    tr("Use \"help <command>\" to see a command's documentation.\n") <<
     "**********************************************************************";
   return password;
 }
@@ -5466,7 +5440,7 @@ void simple_wallet::on_new_block(uint64_t height, const cryptonote::block& block
     m_refresh_progress_reporter.update(height, false);
 }
 //----------------------------------------------------------------------------------------------------
-void simple_wallet::on_money_received(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& tx, uint64_t amount, const cryptonote::subaddress_index& subaddr_index, uint64_t unlock_time)
+void simple_wallet::on_money_received(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& tx, uint64_t amount, const cryptonote::subaddress_index& subaddr_index, bool is_change, uint64_t unlock_time)
 {
   if (m_locked)
     return;
@@ -5477,7 +5451,7 @@ void simple_wallet::on_money_received(uint64_t height, const crypto::hash &txid,
     tr("idx ") << subaddr_index;
 
   const uint64_t warn_height = m_wallet->nettype() == TESTNET ? 1000000 : m_wallet->nettype() == STAGENET ? 50000 : 1650000;
-  if (height >= warn_height)
+  if (height >= warn_height && !is_change)
   {
     std::vector<tx_extra_field> tx_extra_fields;
     parse_tx_extra(tx.extra, tx_extra_fields); // failure ok
@@ -6248,7 +6222,7 @@ void simple_wallet::check_for_inactivity_lock(bool user)
     m_in_command = true;
     if (!user)
     {
-      const std::string speech = tr("I locked your Monero wallet to protect you while you were away\nsee \"help_advanced set\" to configure/disable");
+      const std::string speech = tr("I locked your Monero wallet to protect you while you were away\nsee \"help set\" to configure/disable");
       std::vector<std::pair<std::string, size_t>> lines = tools::split_string_by_width(speech, 45);
 
       size_t max_len = 0;
@@ -7366,6 +7340,32 @@ bool simple_wallet::sweep_single(const std::vector<std::string> &args_)
         success_msg_writer(true) << tr("Unsigned transaction(s) successfully written to file: ") << "multisig_monero_tx";
       }
     }
+    else if (m_wallet->get_account().get_device().has_tx_cold_sign())
+    {
+      try
+      {
+        tools::wallet2::signed_tx_set signed_tx;
+        std::vector<cryptonote::address_parse_info> dsts_info;
+        dsts_info.push_back(info);
+
+        if (!cold_sign_tx(ptx_vector, signed_tx, dsts_info, [&](const tools::wallet2::signed_tx_set &tx){ return accept_loaded_tx(tx); })){
+          fail_msg_writer() << tr("Failed to cold sign transaction with HW wallet");
+          return true;
+        }
+
+        commit_or_save(signed_tx.ptx, m_do_not_relay);
+        success_msg_writer(true) << tr("Money successfully sent, transaction: ") << get_transaction_hash(ptx_vector[0].tx);
+      }
+      catch (const std::exception& e)
+      {
+        handle_transfer_exception(std::current_exception(), m_wallet->is_trusted_daemon());
+      }
+      catch (...)
+      {
+        LOG_ERROR("Unknown error");
+        fail_msg_writer() << tr("unknown error");
+      }
+    }
     else if (m_wallet->watch_only())
     {
       bool r = m_wallet->save_tx(ptx_vector, "unsigned_monero_tx");
@@ -10361,6 +10361,14 @@ bool simple_wallet::user_confirms(const std::string &question)
    return !std::cin.eof() && command_line::is_yes(answer);
 }
 
+bool simple_wallet::user_confirms_auto_config()
+{
+  message_writer(console_color_red, true) << tr("WARNING: Using MMS auto-config mechanisms is not trustless");
+  message_writer() << tr("A malicious auto-config manager could send you info about own wallets instead of other signers' info");
+  message_writer() << tr("If in doubt do not use auto-config or at least compare configs using the \"mms config_checksum\" command");
+  return user_confirms("Accept the risks and continue?");
+}
+
 bool simple_wallet::get_number_from_arg(const std::string &arg, uint32_t &number, const uint32_t lower_bound, const uint32_t upper_bound)
 {
   bool valid = false;
@@ -10513,7 +10521,7 @@ void simple_wallet::show_message(const mms::message &m)
   case mms::message_type::additional_key_set:
   case mms::message_type::note:
     display_content = true;
-    ms.get_sanitized_message_text(m, sanitized_text);
+    ms.get_sanitized_text(m.content, 1000, sanitized_text);
     break;
   default:
     display_content = false;
@@ -10862,6 +10870,11 @@ void simple_wallet::mms_next(const std::vector<std::string> &args)
         {
           break;
         }
+        if (!user_confirms_auto_config())
+        {
+          message_writer() << tr("You can use the \"mms delete\" command to delete any unwanted message");
+          break;
+        }
       }
       ms.process_signer_config(state, m.content);
       ms.stop_auto_config();
@@ -11120,7 +11133,7 @@ void simple_wallet::mms_help(const std::vector<std::string> &args)
 {
   if (args.size() > 1)
   {
-    fail_msg_writer() << tr("Usage: help_advanced mms [<subcommand>]");
+    fail_msg_writer() << tr("Usage: mms help [<subcommand>]");
     return;
   }
   std::vector<std::string> help_args;
@@ -11188,6 +11201,18 @@ void simple_wallet::mms_start_auto_config(const std::vector<std::string> &args)
   list_signers(ms.get_all_signers());
 }
 
+void simple_wallet::mms_config_checksum(const std::vector<std::string> &args)
+{
+  if (args.size() != 0)
+  {
+    fail_msg_writer() << tr("Usage: mms config_checksum");
+    return;
+  }
+  mms::message_store& ms = m_wallet->get_message_store();
+  LOCK_IDLE_SCOPE();
+  message_writer() << ms.get_config_checksum();
+}
+
 void simple_wallet::mms_stop_auto_config(const std::vector<std::string> &args)
 {
   if (args.size() != 0)
@@ -11218,6 +11243,10 @@ void simple_wallet::mms_auto_config(const std::vector<std::string> &args)
     fail_msg_writer() << tr("Invalid auto-config token");
     return;
   }
+  if (!user_confirms_auto_config())
+  {
+    return;
+  }
   mms::authorized_signer me = ms.get_signer(0);
   if (me.auto_config_running)
   {
@@ -11330,6 +11359,10 @@ bool simple_wallet::mms(const std::vector<std::string> &args)
     {
       mms_start_auto_config(mms_args);
     }
+    else if (sub_command == "config_checksum")
+    {
+      mms_config_checksum(mms_args);
+    }
     else if (sub_command == "stop_auto_config")
     {
       mms_stop_auto_config(mms_args);

src/simplewallet/simplewallet.h

@@ -154,7 +154,6 @@ namespace cryptonote
     bool set_persistent_rpc_client_id(const std::vector<std::string> &args = std::vector<std::string>());
     bool set_auto_mine_for_rpc_payment_threshold(const std::vector<std::string> &args = std::vector<std::string>());
     bool set_credits_target(const std::vector<std::string> &args = std::vector<std::string>());
-    bool help_advanced(const std::vector<std::string> &args = std::vector<std::string>());
     bool help(const std::vector<std::string> &args = std::vector<std::string>());
     bool start_mining(const std::vector<std::string> &args);
     bool stop_mining(const std::vector<std::string> &args);
@@ -342,7 +341,7 @@ namespace cryptonote
 
     //----------------- i_wallet2_callback ---------------------
     virtual void on_new_block(uint64_t height, const cryptonote::block& block);
-    virtual void on_money_received(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& tx, uint64_t amount, const cryptonote::subaddress_index& subaddr_index, uint64_t unlock_time);
+    virtual void on_money_received(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& tx, uint64_t amount, const cryptonote::subaddress_index& subaddr_index, bool is_change, uint64_t unlock_time);
     virtual void on_unconfirmed_money_received(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& tx, uint64_t amount, const cryptonote::subaddress_index& subaddr_index);
     virtual void on_money_spent(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& in_tx, uint64_t amount, const cryptonote::transaction& spend_tx, const cryptonote::subaddress_index& subaddr_index);
     virtual void on_skip_transaction(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& tx);
@@ -478,6 +477,7 @@ namespace cryptonote
     void ask_send_all_ready_messages();
     void check_for_messages();
     bool user_confirms(const std::string &question);
+    bool user_confirms_auto_config();
     bool get_message_from_arg(const std::string &arg, mms::message &m);
     bool get_number_from_arg(const std::string &arg, uint32_t &number, const uint32_t lower_bound, const uint32_t upper_bound); 
 
@@ -498,6 +498,7 @@ namespace cryptonote
     void mms_help(const std::vector<std::string> &args);
     void mms_send_signer_config(const std::vector<std::string> &args);
     void mms_start_auto_config(const std::vector<std::string> &args);
+    void mms_config_checksum(const std::vector<std::string> &args);
     void mms_stop_auto_config(const std::vector<std::string> &args);
     void mms_auto_config(const std::vector<std::string> &args);
   };

src/version.cpp.in

@@ -1,6 +1,6 @@
 #define DEF_MONERO_VERSION_TAG "@VERSIONTAG@"
-#define DEF_MONERO_VERSION "0.15.0.0"
-#define DEF_MONERO_RELEASE_NAME "Carbon Chamaeleon"
+#define DEF_MONERO_VERSION "0.16.0.3"
+#define DEF_MONERO_RELEASE_NAME "Nitrogen Nebula"
 #define DEF_MONERO_VERSION_FULL DEF_MONERO_VERSION "-" DEF_MONERO_VERSION_TAG
 #define DEF_MONERO_VERSION_IS_RELEASE @VERSION_IS_RELEASE@
 

src/wallet/api/wallet.cpp

@@ -157,7 +157,7 @@ struct Wallet2CallbackImpl : public tools::i_wallet2_callback
         }
     }
 
-    virtual void on_money_received(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& tx, uint64_t amount, const cryptonote::subaddress_index& subaddr_index, uint64_t unlock_time)
+    virtual void on_money_received(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& tx, uint64_t amount, const cryptonote::subaddress_index& subaddr_index, bool is_change, uint64_t unlock_time)
     {
 
         std::string tx_hash =  epee::string_tools::pod_to_hex(txid);

src/wallet/message_store.cpp

@@ -39,6 +39,7 @@
 #include "serialization/binary_utils.h"
 #include "common/base58.h"
 #include "common/util.h"
+#include "common/utf8.h"
 #include "string_tools.h"
 
 
@@ -129,18 +130,18 @@ void message_store::set_signer(const multisig_wallet_state &state,
   authorized_signer &m = m_signers[index];
   if (label)
   {
-    m.label = label.get();
+    get_sanitized_text(label.get(), 50, m.label);
   }
   if (transport_address)
   {
-    m.transport_address = transport_address.get();
+    get_sanitized_text(transport_address.get(), 200, m.transport_address);
   }
   if (monero_address)
   {
     m.monero_address_known = true;
     m.monero_address = monero_address.get();
   }
-  // Save to minimize the chance to loose that info (at least while in beta)
+  // Save to minimize the chance to loose that info
   save(state);
 }
 
@@ -202,6 +203,17 @@ void message_store::unpack_signer_config(const multisig_wallet_state &state, con
   }
   uint32_t num_signers = (uint32_t)signers.size();
   THROW_WALLET_EXCEPTION_IF(num_signers != m_num_authorized_signers, tools::error::wallet_internal_error, "Wrong number of signers in config: " + std::to_string(num_signers));
+  for (uint32_t i = 0; i < num_signers; ++i)
+  {
+    authorized_signer &m = signers[i];
+    std::string temp;
+    get_sanitized_text(m.label, 50, temp);
+    m.label = temp;
+    get_sanitized_text(m.transport_address, 200, temp);
+    m.transport_address = temp;
+    get_sanitized_text(m.auto_config_token, 20, temp);
+    m.auto_config_token = temp;
+  }
 }
 
 void message_store::process_signer_config(const multisig_wallet_state &state, const std::string &signer_config)
@@ -242,10 +254,10 @@ void message_store::process_signer_config(const multisig_wallet_state &state, co
       }
     }
     authorized_signer &modify = m_signers[take_index];
-    modify.label = m.label;  // ALWAYS set label, see comments above
+    get_sanitized_text(m.label, 50, modify.label);  // ALWAYS set label, see comments above
     if (!modify.me)
     {
-      modify.transport_address = m.transport_address;
+      get_sanitized_text(m.transport_address, 200, modify.transport_address);
       modify.monero_address_known = m.monero_address_known;
       if (m.monero_address_known)
       {
@@ -392,6 +404,45 @@ void message_store::process_auto_config_data_message(uint32_t id)
   signer.auto_config_running = false;
 }
 
+void add_hash(crypto::hash &sum, const crypto::hash &summand)
+{
+  for (uint32_t i = 0; i < crypto::HASH_SIZE; ++i)
+  {
+    uint32_t x = (uint32_t)sum.data[i];
+    uint32_t y = (uint32_t)summand.data[i];
+    sum.data[i] = (char)((x + y) % 256);
+  }
+}
+
+// Calculate a checksum that allows signers to make sure they work with an identical signer config
+// by exchanging and comparing checksums out-of-band i.e. not using the MMS;
+// Because different signers have a different order of signers in the config work with "adding"
+// individual hashes because that operation is commutative
+std::string message_store::get_config_checksum() const
+{
+  crypto::hash sum = crypto::null_hash;
+  uint32_t num = SWAP32LE(m_num_authorized_signers);
+  add_hash(sum, crypto::cn_fast_hash(&num, sizeof(num)));
+  num = SWAP32LE(m_num_required_signers);
+  add_hash(sum, crypto::cn_fast_hash(&num, sizeof(num)));
+  for (uint32_t i = 0; i < m_num_authorized_signers; ++i)
+  {
+    const authorized_signer &m = m_signers[i];
+    add_hash(sum, crypto::cn_fast_hash(m.transport_address.data(), m.transport_address.size()));
+    if (m.monero_address_known)
+    {
+      add_hash(sum, crypto::cn_fast_hash(&m.monero_address.m_spend_public_key, sizeof(m.monero_address.m_spend_public_key)));
+      add_hash(sum, crypto::cn_fast_hash(&m.monero_address.m_view_public_key, sizeof(m.monero_address.m_view_public_key)));
+    }
+  }
+  std::string checksum_bytes;
+  checksum_bytes += sum.data[0];
+  checksum_bytes += sum.data[1];
+  checksum_bytes += sum.data[2];
+  checksum_bytes += sum.data[3];
+  return epee::string_tools::buff_to_hex_nodelimer(checksum_bytes);
+}
+
 void message_store::stop_auto_config()
 {
   for (uint32_t i = 0; i < m_num_authorized_signers; ++i)
@@ -661,31 +712,36 @@ void message_store::delete_all_messages()
   m_messages.clear();
 }
 
-// Make a message text, which is "attacker controlled data", reasonably safe to display
+// Make a text, which is "attacker controlled data", reasonably safe to display
 // This is mostly geared towards the safe display of notes sent by "mms note" with a "mms show" command
-void message_store::get_sanitized_message_text(const message &m, std::string &sanitized_text) const
+void message_store::get_sanitized_text(const std::string &text, size_t max_length, std::string &sanitized_text) const
 {
-  sanitized_text.clear();
-
   // Restrict the size to fend of DOS-style attacks with heaps of data
-  size_t length = std::min(m.content.length(), (size_t)1000);
+  size_t length = std::min(text.length(), max_length);
+  sanitized_text = text.substr(0, length);
 
-  for (size_t i = 0; i < length; ++i)
+  try
   {
-    char c = m.content[i];
-    if ((int)c < 32)
+    sanitized_text = tools::utf8canonical(sanitized_text, [](wint_t c)
     {
-      // Strip out any controls, especially ESC for getting rid of potentially dangerous
-      // ANSI escape sequences that a console window might interpret
-      c = ' ';
-    }
-    else if ((c == '<') || (c == '>'))
-    {
-      // Make XML or HTML impossible that e.g. might contain scripts that Qt might execute
-      // when displayed in the GUI wallet
-      c = ' ';
-    }
-    sanitized_text += c;
+      if ((c < 0x20) || (c == 0x7f) || (c >= 0x80 && c <= 0x9f))
+      {
+        // Strip out any controls, especially ESC for getting rid of potentially dangerous
+        // ANSI escape sequences that a console window might interpret
+        c = '?';
+      }
+      else if ((c == '<') || (c == '>'))
+      {
+        // Make XML or HTML impossible that e.g. might contain scripts that Qt might execute
+        // when displayed in the GUI wallet
+        c = '?';
+      }
+      return c;
+    });
+  }
+  catch (const std::exception &e)
+  {
+    sanitized_text = "(Illegal UTF-8 string)";
   }
 }
 

src/wallet/message_store.h

@@ -242,6 +242,7 @@ namespace mms
     size_t add_auto_config_data_message(const multisig_wallet_state &state,
                                         const std::string &auto_config_token);
     void process_auto_config_data_message(uint32_t id);
+    std::string get_config_checksum() const;
     void stop_auto_config();
 
     // Process data just created by "me" i.e. the own local wallet, e.g. as the result of a "prepare_multisig" command
@@ -275,7 +276,7 @@ namespace mms
     void set_message_processed_or_sent(uint32_t id);
     void delete_message(uint32_t id);
     void delete_all_messages();
-    void get_sanitized_message_text(const message &m, std::string &sanitized_text) const;
+    void get_sanitized_text(const std::string &text, size_t max_length, std::string &sanitized_text) const;
 
     void send_message(const multisig_wallet_state &state, uint32_t id);
     bool check_for_messages(const multisig_wallet_state &state, std::vector<message> &messages);

src/wallet/wallet2.cpp

@@ -1867,6 +1867,20 @@ void wallet2::cache_tx_data(const cryptonote::transaction& tx, const crypto::has
   }
 }
 //----------------------------------------------------------------------------------------------------
+bool wallet2::spends_one_of_ours(const cryptonote::transaction &tx) const
+{
+  for (const auto &in: tx.vin)
+  {
+    if (in.type() != typeid(cryptonote::txin_to_key))
+      continue;
+    const cryptonote::txin_to_key &in_to_key = boost::get<cryptonote::txin_to_key>(in);
+    auto it = m_key_images.find(in_to_key.k_image);
+    if (it != m_key_images.end())
+      return true;
+  }
+  return false;
+}
+//----------------------------------------------------------------------------------------------------
 void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote::transaction& tx, const std::vector<uint64_t> &o_indices, uint64_t height, uint8_t block_version, uint64_t ts, bool miner_tx, bool pool, bool double_spend_seen, const tx_cache_data &tx_cache_data, std::map<std::pair<uint64_t, uint64_t>, size_t> *output_tracker_cache)
 {
   PERF_TIMER(process_new_transaction);
@@ -2153,7 +2167,7 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote
             }
 	    LOG_PRINT_L0("Received money: " << print_money(td.amount()) << ", with tx: " << txid);
 	    if (0 != m_callback)
-	      m_callback->on_money_received(height, txid, tx, td.m_amount, td.m_subaddr_index, td.m_tx.unlock_time);
+	      m_callback->on_money_received(height, txid, tx, td.m_amount, td.m_subaddr_index, spends_one_of_ours(tx), td.m_tx.unlock_time);
           }
           total_received_1 += amount;
           notify = true;
@@ -2230,7 +2244,7 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote
 
 	    LOG_PRINT_L0("Received money: " << print_money(td.amount()) << ", with tx: " << txid);
 	    if (0 != m_callback)
-	      m_callback->on_money_received(height, txid, tx, td.m_amount, td.m_subaddr_index, td.m_tx.unlock_time);
+	      m_callback->on_money_received(height, txid, tx, td.m_amount, td.m_subaddr_index, spends_one_of_ours(tx), td.m_tx.unlock_time);
           }
           total_received_1 += extra_amount;
           notify = true;
@@ -13080,7 +13094,7 @@ size_t wallet2::import_multisig(std::vector<cryptonote::blobdata> blobs)
   CHECK_AND_ASSERT_THROW_MES(info.size() + 1 <= m_multisig_signers.size() && info.size() + 1 >= m_multisig_threshold, "Wrong number of multisig sources");
 
   std::vector<std::vector<rct::key>> k;
-  auto wiper = epee::misc_utils::create_scope_leave_handler([&](){memwipe(k.data(), k.size() * sizeof(k[0]));});
+  auto wiper = epee::misc_utils::create_scope_leave_handler([&](){for (auto &v: k) memwipe(v.data(), v.size() * sizeof(v[0]));});
   k.reserve(m_transfers.size());
   for (const auto &td: m_transfers)
     k.push_back(td.m_multisig_k);

src/wallet/wallet2.h

@@ -131,7 +131,7 @@ private:
   public:
     // Full wallet callbacks
     virtual void on_new_block(uint64_t height, const cryptonote::block& block) {}
-    virtual void on_money_received(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& tx, uint64_t amount, const cryptonote::subaddress_index& subaddr_index, uint64_t unlock_time) {}
+    virtual void on_money_received(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& tx, uint64_t amount, const cryptonote::subaddress_index& subaddr_index, bool is_change, uint64_t unlock_time) {}
     virtual void on_unconfirmed_money_received(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& tx, uint64_t amount, const cryptonote::subaddress_index& subaddr_index) {}
     virtual void on_money_spent(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& in_tx, uint64_t amount, const cryptonote::transaction& spend_tx, const cryptonote::subaddress_index& subaddr_index) {}
     virtual void on_skip_transaction(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& tx) {}
@@ -1517,6 +1517,7 @@ private:
     void check_rpc_cost(const char *call, uint64_t post_call_credits, uint64_t pre_credits, double expected_cost);
 
     bool should_expand(const cryptonote::subaddress_index &index) const;
+    bool spends_one_of_ours(const cryptonote::transaction &tx) const;
 
     cryptonote::account_base m_account;
     boost::optional<epee::net_utils::http::login> m_daemon_login;

tests/core_tests/chaingen.h

@@ -856,10 +856,10 @@ inline bool do_replay_file(const std::string& filename)
 }
 
 #define REGISTER_CALLBACK(CB_NAME, CLBACK) \
-  register_callback(CB_NAME, boost::bind(&CLBACK, this, _1, _2, _3));
+  register_callback(CB_NAME, boost::bind(&CLBACK, this, boost::placeholders::_1, boost::placeholders::_2, boost::placeholders::_3));
 
 #define REGISTER_CALLBACK_METHOD(CLASS, METHOD) \
-  register_callback(#METHOD, boost::bind(&CLASS::METHOD, this, _1, _2, _3));
+  register_callback(#METHOD, boost::bind(&CLASS::METHOD, this, boost::placeholders::_1, boost::placeholders::_2, boost::placeholders::_3));
 
 #define MAKE_GENESIS_BLOCK(VEC_EVENTS, BLK_NAME, MINER_ACC, TS)                       \
   test_generator generator;                                                           \

tests/functional_tests/txpool.py

@@ -241,6 +241,17 @@ class TransferTest():
             assert x.fee == txes[txid].fee
             assert x.tx_blob == txes[txid].tx_blob
 
+        print('Checking relaying txes')
+        res = daemon.get_transaction_pool_hashes()
+        assert len(res.tx_hashes) > 0
+        txid = res.tx_hashes[0]
+        daemon.relay_tx([txid])
+        res = daemon.get_transactions([txid])
+        assert len(res.txs) == 1
+        assert res.txs[0].tx_hash == txid
+        assert res.txs[0].in_pool
+        assert res.txs[0].relayed
+
         daemon.flush_txpool()
         self.check_empty_pool()
 

tests/unit_tests/epee_utils.cpp

@@ -387,6 +387,29 @@ TEST(ByteSlice, Construction)
   EXPECT_FALSE(std::is_copy_assignable<epee::byte_slice>());
 }
 
+TEST(ByteSlice, DataReturnedMatches)
+{
+  for (int i = 64; i > 0; i--)
+  {
+    std::string sso_string(i, 'a');
+    std::string original = sso_string;
+    epee::byte_slice slice{std::move(sso_string)};
+
+    EXPECT_EQ(slice.size(), original.size());
+    EXPECT_EQ(memcmp(slice.data(), original.data(), original.size()), 0);
+  }
+
+  for (int i = 64; i > 0; i--)
+  {
+    std::vector<uint8_t> sso_vector(i, 'a');
+    std::vector<uint8_t> original = sso_vector;
+    epee::byte_slice slice{std::move(sso_vector)};
+
+    EXPECT_EQ(slice.size(), original.size());
+    EXPECT_EQ(memcmp(slice.data(), original.data(), original.size()), 0);
+  }
+}
+
 TEST(ByteSlice, NoExcept)
 {
   EXPECT_TRUE(std::is_nothrow_default_constructible<epee::byte_slice>());
@@ -667,6 +690,23 @@ TEST(ByteSlice, TakeSlice)
   EXPECT_TRUE(boost::range::equal(base_string, slice));
 
   const epee::span<const std::uint8_t> original = epee::to_span(slice);
+  const epee::byte_slice empty_slice = slice.take_slice(0);
+  EXPECT_EQ(original.begin(), slice.begin());
+  EXPECT_EQ(slice.begin(), slice.cbegin());
+  EXPECT_EQ(original.end(), slice.end());
+  EXPECT_EQ(slice.end(), slice.cend());
+
+  EXPECT_EQ(nullptr, empty_slice.begin());
+  EXPECT_EQ(nullptr, empty_slice.cbegin());
+  EXPECT_EQ(nullptr, empty_slice.end());
+  EXPECT_EQ(nullptr, empty_slice.cend());
+  EXPECT_EQ(nullptr, empty_slice.data());
+  EXPECT_TRUE(empty_slice.empty());
+  EXPECT_EQ(0u, empty_slice.size());
+
+  EXPECT_FALSE(slice.empty());
+  EXPECT_EQ(slice.cbegin(), slice.data());
+
   const epee::byte_slice slice2 = slice.take_slice(remove_size);
 
   EXPECT_EQ(original.begin() + remove_size, slice.begin());
@@ -1061,6 +1101,20 @@ TEST(ByteStream, ToByteSlice)
   EXPECT_EQ(nullptr, stream.data());
   EXPECT_EQ(nullptr, stream.tellp());
   EXPECT_TRUE(equal(source, slice));
+
+  stream = epee::byte_stream{};
+  stream.reserve(1);
+  EXPECT_NE(nullptr, stream.data());
+  EXPECT_NE(nullptr, stream.tellp());
+
+  const epee::byte_slice empty_slice{std::move(stream)};
+  EXPECT_TRUE(empty_slice.empty());
+  EXPECT_EQ(0u, empty_slice.size());
+  EXPECT_EQ(nullptr, empty_slice.begin());
+  EXPECT_EQ(nullptr, empty_slice.cbegin());
+  EXPECT_EQ(nullptr, empty_slice.end());
+  EXPECT_EQ(nullptr, empty_slice.cend());
+  EXPECT_EQ(nullptr, empty_slice.data());
 }
 
 TEST(ToHex, String)