Merge pull request #251 from wowario/upstream

Upstream

contrib/epee/include/byte_slice.h

@@ -1,4 +1,4 @@
-// Copyright (c) 2019, The Monero Project
+// Copyright (c) 2019-2020, The Monero Project
 //
 // All rights reserved.
 //
@@ -39,6 +39,7 @@
 namespace epee
 {
   struct byte_slice_data;
+  class byte_stream;
 
   struct release_byte_slice
   {
@@ -50,6 +51,12 @@ namespace epee
     }
   };
 
+  //! Frees ref count + buffer allocated internally by `byte_buffer`.
+  struct release_byte_buffer
+  {
+    void operator()(std::uint8_t* buf) const noexcept;
+  };
+
   /*! Inspired by slices in golang. Storage is thread-safe reference counted,
       allowing for cheap copies or range selection on the bytes. The bytes
       owned by this class are always immutable.
@@ -104,6 +111,9 @@ namespace epee
     //! Convert `buffer` into a slice using one allocation for shared count.
     explicit byte_slice(std::string&& buffer);
 
+    //! Convert `stream` into a slice with zero allocations.
+    explicit byte_slice(byte_stream&& stream) noexcept;
+
     byte_slice(byte_slice&& source) noexcept;
     ~byte_slice() noexcept = default;
 
@@ -149,5 +159,19 @@ namespace epee
     //! \post `empty()` \return Ownership of ref-counted buffer.
     std::unique_ptr<byte_slice_data, release_byte_slice> take_buffer() noexcept;
   };
+
+  //! Alias for a buffer that has space for a `byte_slice` ref count.
+  using byte_buffer = std::unique_ptr<std::uint8_t, release_byte_buffer>;
+
+  /*! \return `buf` with a new size of exactly `length`. New bytes not
+        initialized. A `nullptr` is returned on allocation failure. */
+  byte_buffer byte_buffer_resize(byte_buffer buf, std::size_t length) noexcept;
+
+  /*! Increase `buf` of size `current` by `more` bytes.
+
+    \throw std::range_error if `current + more` exceeds `size_t` bounds.
+    \return Buffer of `current + more` bytes. A `nullptr` is returned on
+      allocation failure. */
+  byte_buffer byte_buffer_increase(byte_buffer buf, std::size_t current, std::size_t more);
 } // epee
 

contrib/epee/include/byte_stream.h

@@ -0,0 +1,224 @@
+// Copyright (c) 2020, The Monero Project
+//
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#pragma once
+
+#include <cassert>
+#include <cstdint>
+#include <cstring>
+
+#include "byte_slice.h"
+#include "span.h"
+
+namespace epee
+{
+  /*! \brief A partial drop-in replacement for `std::ostream`.
+
+      Only a few base `std::ostream` functions are implemented - enough for
+      rapidjson output currently.
+
+      Advantages over `std::stringstream` or `rapidjson::StringBuffer`:
+        - The internal buffer can be taken without a copy.
+        - The internal buffer can be given to `byte_slice` with zero
+          allocations for reference count.
+        - The internal buffer can be given to `zmq_msg_data_init` without a
+          copy or extra allocation.
+      an additional advantage over `std::stringstream`:
+        - Construction is significantly faster - the global `std::locale`
+          does not have to be acquired (global thread synchronization), and
+          an extra allocation for `std::stringbuf` is not needed (which is an
+          addition to the buffer inside of that object). */
+  class byte_stream
+  {
+    byte_buffer buffer_;        //! Beginning of buffer
+    std::uint8_t* next_write_;  //! Current write position
+    const std::uint8_t* end_;   //! End of buffer
+    std::size_t increase_size_; //! Minimum buffer size increase
+
+    //! \post `requested <= available()`
+    void overflow(const std::size_t requested);
+
+    //! Ensures that at least `requested` bytes are available.
+    void check(const std::size_t requested)
+    {
+      const std::size_t remaining = available();
+      if (remaining < requested)
+        overflow(requested);
+    }
+
+  public:
+    using char_type = std::uint8_t;
+    using Ch = char_type;
+
+    //! \return Default minimum size increase on buffer overflow
+    static constexpr std::size_t default_increase() noexcept { return 4096; }
+
+    //! Increase internal buffer by at least `byte_stream_increase` bytes.
+    byte_stream() noexcept
+      : byte_stream(default_increase())
+    {}
+
+    //! Increase internal buffer by at least `increase` bytes.
+    explicit byte_stream(const std::size_t increase) noexcept
+      : buffer_(nullptr),
+        next_write_(nullptr),
+        end_(nullptr),
+        increase_size_(increase)
+    {}
+
+    byte_stream(byte_stream&& rhs) noexcept;
+    ~byte_stream() noexcept = default;
+    byte_stream& operator=(byte_stream&& rhs) noexcept;
+
+    //! \return The minimum increase size on buffer overflow
+    std::size_t increase_size() const noexcept { return increase_size_; }
+
+    const std::uint8_t* data() const noexcept { return buffer_.get(); }
+    std::uint8_t* tellp() const noexcept { return next_write_; }
+    std::size_t available() const noexcept { return end_ - next_write_; }
+    std::size_t size() const noexcept { return next_write_ - buffer_.get(); }
+    std::size_t capacity() const noexcept { return end_ - buffer_.get(); }
+
+    //! Compatibility with rapidjson.
+    void Flush() const noexcept
+    {}
+
+    /*! Reserve at least `more` bytes.
+        \post `size() + more <= available()`.
+        \throw std::range_error if exceeding max `size_t` value.
+        \throw std::bad_alloc if allocation fails. */
+    void reserve(const std::size_t more)
+    {
+      check(more);
+    }
+
+    /*! Copy `length` bytes starting at `ptr` to end of stream.
+        \throw std::range_error If exceeding max size_t value.
+        \throw std::bad_alloc If allocation fails. */
+    void write(const std::uint8_t* ptr, const std::size_t length)
+    {
+      check(length);
+      std::memcpy(tellp(), ptr, length);
+      next_write_ += length;
+    }
+
+    /*! Copy `length` bytes starting at `ptr` to end of stream.
+        \throw std::range_error if exceeding max `size_t` value.
+        \throw std::bad_alloc if allocation fails. */
+    void write(const char* ptr, const std::size_t length)
+    {
+      write(reinterpret_cast<const std::uint8_t*>(ptr), length);
+    }
+
+    /*! Copy `source` to end of stream.
+        \throw std::range_error if exceeding max `size_t` value.
+        \throw std::bad_alloc if allocation fails. */
+    void write(const epee::span<const std::uint8_t> source)
+    {
+      write(source.data(), source.size());
+    }
+
+    /*! Copy `source` to end of stream.
+        \throw std::range_error if exceeding max `size_t` value.
+        \throw std::bad_alloc if allocation fails. */
+    void write(const epee::span<const char> source)
+    {
+      write(source.data(), source.size());
+    }
+
+    /*! Copy `ch` to end of stream.
+        \throw std::range_error if exceeding max `size_t` value.
+        \throw std::bad_alloc if allocation fails. */
+    void put(const std::uint8_t ch)
+    {
+      check(1);
+      put_unsafe(ch);
+    }
+
+    /*! Copy `ch` to end of stream. Provides rapidjson compatability.
+        \throw std::range_error if exceeding max `size_t` value.
+        \throw std::bad_alloc if allocation fails. */
+    void Put(const std::uint8_t ch)
+    {
+      put(ch);
+    }
+
+    /*! Writes `ch` to end of stream without runtime capacity checks. Must use
+        `reserve` before calling this function. Primarily for use with
+        rapidjson, which writes characters at a time but reserves memory in
+        blocks. Most applications want to use `put` or `write`. */
+    void put_unsafe(const std::uint8_t ch) noexcept
+    {
+      assert(1 <= available());
+      *(tellp()) = ch;
+      ++next_write_;
+    }
+
+    /*! Write `ch` to end of stream `count` times.
+        \throw std::range_error if exceeding max `size_t` value.
+        \throw std::bad_alloc if allocation fails. */
+    void put_n(const std::uint8_t ch, const std::size_t count)
+    {
+      check(count);
+      std::memset(tellp(), count, ch);
+      next_write_ += count;
+    }
+
+    /*! Copy `ch` to end of stream.
+        \throw std::range_error if exceeding max `size_t` value.
+        \throw std::bad_alloc if allocation fails. */
+    void push_back(const std::uint8_t ch)
+    {
+      put(ch);
+    }
+
+    //! \return The internal buffer. \post `size() == capacity() == 0`.
+    byte_buffer take_buffer() noexcept;
+  };
+
+  //! Compatability/optimization for rapidjson.
+
+  inline void PutReserve(byte_stream& dest, const std::size_t length)
+  {
+    dest.reserve(length);
+  }
+
+  //! Compatability/optimization for rapidjson.
+
+  inline void PutUnsafe(byte_stream& dest, const std::uint8_t ch)
+  {
+    dest.put_unsafe(ch);
+  }
+
+  //! Compability/optimization for rapidjson.
+  inline void PutN(byte_stream& dest, const std::uint8_t ch, const std::size_t count)
+  {
+    dest.put_n(ch, count);
+  }
+} // epee
+

contrib/epee/src/CMakeLists.txt

@@ -26,8 +26,9 @@
 # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
 # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-add_library(epee STATIC byte_slice.cpp hex.cpp abstract_http_client.cpp http_auth.cpp mlog.cpp net_helper.cpp net_utils_base.cpp string_tools.cpp wipeable_string.cpp
+
-    levin_base.cpp memwipe.c connection_basic.cpp network_throttle.cpp network_throttle-detail.cpp mlocker.cpp buffer.cpp net_ssl.cpp
+add_library(epee STATIC byte_slice.cpp byte_stream.cpp hex.cpp abstract_http_client.cpp http_auth.cpp mlog.cpp net_helper.cpp net_utils_base.cpp string_tools.cpp
+    wipeable_string.cpp levin_base.cpp memwipe.c connection_basic.cpp network_throttle.cpp network_throttle-detail.cpp mlocker.cpp buffer.cpp net_ssl.cpp
     int-util.cpp)
 
 if (USE_READLINE AND (GNU_READLINE_FOUND OR (DEPENDS AND NOT MINGW)))

contrib/epee/src/byte_slice.cpp

@@ -1,4 +1,4 @@
-// Copyright (c) 2019, The Monero Project
+// Copyright (c) 2019-2020, The Monero Project
 //
 // All rights reserved.
 //
@@ -34,6 +34,7 @@
 #include <utility>
 
 #include "byte_slice.h"
+#include "byte_stream.h"
 
 namespace epee
 {
@@ -117,6 +118,12 @@ namespace epee
     }
   } // anonymous
 
+  void release_byte_buffer::operator()(std::uint8_t* buf) const noexcept
+  {
+    if (buf)
+      std::free(buf - sizeof(raw_byte_slice));
+  }
+
   byte_slice::byte_slice(byte_slice_data* storage, span<const std::uint8_t> portion) noexcept
     : storage_(storage), portion_(portion)
   {
@@ -163,6 +170,14 @@ namespace epee
     : byte_slice(adapt_buffer{}, std::move(buffer))
   {}
 
+  byte_slice::byte_slice(byte_stream&& stream) noexcept
+    : storage_(nullptr), portion_(stream.data(), stream.size())
+  {
+    std::uint8_t* const data = stream.take_buffer().release() - sizeof(raw_byte_slice);
+    new (data) raw_byte_slice{};
+    storage_.reset(reinterpret_cast<raw_byte_slice*>(data));
+  }
+
   byte_slice::byte_slice(byte_slice&& source) noexcept
     : storage_(std::move(source.storage_)), portion_(source.portion_)
   {
@@ -217,4 +232,29 @@ namespace epee
     portion_ = nullptr;
     return out;
   }
+
+  byte_buffer byte_buffer_resize(byte_buffer buf, const std::size_t length) noexcept
+  {
+    if (std::numeric_limits<std::size_t>::max() - sizeof(raw_byte_slice) < length)
+      return nullptr;
+
+    std::uint8_t* data = buf.get();
+    if (data != nullptr)
+      data -= sizeof(raw_byte_slice);
+
+    data = static_cast<std::uint8_t*>(std::realloc(data, sizeof(raw_byte_slice) + length));
+    if (data == nullptr)
+      return nullptr;
+
+    buf.release();
+    buf.reset(data + sizeof(raw_byte_slice));
+    return buf;
+  }
+
+  byte_buffer byte_buffer_increase(byte_buffer buf, const std::size_t current, const std::size_t more)
+  {
+    if (std::numeric_limits<std::size_t>::max() - current < more)
+      throw std::range_error{"byte_buffer_increase size_t overflow"};
+    return byte_buffer_resize(std::move(buf), current + more);
+  }
 } // epee

contrib/epee/src/byte_stream.cpp

@@ -0,0 +1,93 @@
+// Copyright (c) 2020, The Monero Project
+//
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "byte_stream.h"
+
+#include <algorithm>
+#include <limits>
+#include <utility>
+
+#include <iostream>
+
+namespace epee
+{
+  void byte_stream::overflow(const std::size_t requested)
+  {
+    // Recalculating `need` bytes removes at least one instruction from every
+    // inlined `put` call in header
+
+    assert(available() < requested);
+    const std::size_t need = requested - available();
+
+    const std::size_t len = size();
+    const std::size_t cap = capacity();
+    const std::size_t increase = std::max(need, increase_size());
+
+    next_write_ = nullptr;
+    end_ = nullptr;
+
+    buffer_ = byte_buffer_increase(std::move(buffer_), cap, increase);
+    if (!buffer_)
+      throw std::bad_alloc{};
+
+    next_write_ = buffer_.get() + len;
+    end_ = buffer_.get() + cap + increase;
+  }
+
+  byte_stream::byte_stream(byte_stream&& rhs) noexcept
+    : buffer_(std::move(rhs.buffer_)),
+      next_write_(rhs.next_write_),
+      end_(rhs.end_),
+      increase_size_(rhs.increase_size_)
+  {
+    rhs.next_write_ = nullptr;
+    rhs.end_ = nullptr;
+  }
+
+  byte_stream& byte_stream::operator=(byte_stream&& rhs) noexcept
+  {
+    if (this != std::addressof(rhs))
+    {
+      buffer_ = std::move(rhs.buffer_);
+      next_write_ = rhs.next_write_;
+      end_ = rhs.end_;
+      increase_size_ = rhs.increase_size_;
+      rhs.next_write_ = nullptr;
+      rhs.end_ = nullptr;
+    }
+    return *this;
+  }
+
+  byte_buffer byte_stream::take_buffer() noexcept
+  {
+    byte_buffer out{std::move(buffer_)};
+    next_write_ = nullptr;
+    end_ = nullptr;
+    return out;
+  }
+}

src/crypto/crypto.cpp

@@ -294,6 +294,7 @@ namespace crypto {
     sc_mulsub(&sig.r, &sig.c, &unwrap(sec), &k);
     if (!sc_isnonzero((const unsigned char*)sig.r.data))
       goto try_again;
+    memwipe(&k, sizeof(k));
   }
 
   bool crypto_ops::check_signature(const hash &prefix_hash, const public_key &pub, const signature &sig) {
@@ -390,6 +391,8 @@ namespace crypto {
 
     // sig.r = k - sig.c*r
     sc_mulsub(&sig.r, &sig.c, &unwrap(r), &k);
+
+    memwipe(&k, sizeof(k));
   }
 
   bool crypto_ops::check_tx_proof(const hash &prefix_hash, const public_key &R, const public_key &A, const boost::optional<public_key> &B, const public_key &D, const signature &sig) {
@@ -560,6 +563,7 @@ POP_WARNINGS
         random_scalar(sig[i].c);
         random_scalar(sig[i].r);
         if (ge_frombytes_vartime(&tmp3, &*pubs[i]) != 0) {
+          memwipe(&k, sizeof(k));
           local_abort("invalid pubkey");
         }
         ge_double_scalarmult_base_vartime(&tmp2, &sig[i].c, &tmp3, &sig[i].r);
@@ -573,6 +577,8 @@ POP_WARNINGS
     hash_to_scalar(buf.get(), rs_comm_size(pubs_count), h);
     sc_sub(&sig[sec_index].c, &h, &sum);
     sc_mulsub(&sig[sec_index].r, &sig[sec_index].c, &unwrap(sec), &k);
+
+    memwipe(&k, sizeof(k));
   }
 
   bool crypto_ops::check_ring_signature(const hash &prefix_hash, const key_image &image,

src/cryptonote_basic/cryptonote_format_utils.cpp

@@ -160,6 +160,8 @@ namespace cryptonote
     if (tx.version >= 2 && !is_coinbase(tx))
     {
       rct::rctSig &rv = tx.rct_signatures;
+      if (rv.type == rct::RCTTypeNull)
+        return true;
       if (rv.outPk.size() != tx.vout.size())
       {
         LOG_PRINT_L1("Failed to parse transaction from blob, bad outPk size in tx " << get_transaction_hash(tx));

src/cryptonote_core/cryptonote_core.cpp

@@ -263,14 +263,14 @@ namespace cryptonote
     m_blockchain_storage.set_enforce_dns_checkpoints(enforce_dns);
   }
   //-----------------------------------------------------------------------------------------------
-  bool core::update_checkpoints()
+  bool core::update_checkpoints(const bool skip_dns /* = false */)
   {
     if (m_nettype != MAINNET || m_disable_dns_checkpoints) return true;
 
     if (m_checkpoints_updating.test_and_set()) return true;
 
     bool res = true;
-    if (time(NULL) - m_last_dns_checkpoints_update >= 3600)
+    if (!skip_dns && time(NULL) - m_last_dns_checkpoints_update >= 3600)
     {
       res = m_blockchain_storage.update_checkpoints(m_checkpoints_path, true);
       m_last_dns_checkpoints_update = time(NULL);
@@ -669,7 +669,8 @@ namespace cryptonote
 
     // load json & DNS checkpoints, and verify them
     // with respect to what blocks we already have
-    CHECK_AND_ASSERT_MES(update_checkpoints(), false, "One or more checkpoints loaded from json or dns conflicted with existing checkpoints.");
+    const bool skip_dns_checkpoints = !command_line::get_arg(vm, arg_dns_checkpoints);
+    CHECK_AND_ASSERT_MES(update_checkpoints(skip_dns_checkpoints), false, "One or more checkpoints loaded from json or dns conflicted with existing checkpoints.");
 
    // DNS versions checking
     if (check_updates_string == "disabled")
@@ -1348,9 +1349,9 @@ namespace cryptonote
     std::vector<crypto::hash> tx_hashes{};
     tx_hashes.resize(tx_blobs.size());
 
-    cryptonote::transaction tx{};
     for (std::size_t i = 0; i < tx_blobs.size(); ++i)
     {
+      cryptonote::transaction tx{};
       if (!parse_and_validate_tx_from_blob(tx_blobs[i], tx, tx_hashes[i]))
       {
         LOG_ERROR("Failed to parse relayed transaction");
@@ -1747,8 +1748,6 @@ namespace cryptonote
         MCLOG_RED(level, "global", "**********************************************************************");
         break;
       case HardFork::UpdateNeeded:
-        level = el::Level::Info;
-        MCLOG(level, "global", el::Color::Default, "Last scheduled hard fork time suggests a daemon update will be released within the next couple months.");
         break;
       default:
         break;

src/cryptonote_core/cryptonote_core.h

@@ -699,7 +699,7 @@ namespace cryptonote
       *
       * @note see Blockchain::update_checkpoints()
       */
-     bool update_checkpoints();
+     bool update_checkpoints(const bool skip_dns = false);
 
      /**
       * @brief tells the daemon to wind down operations and stop running

src/cryptonote_core/tx_pool.cpp

@@ -1121,7 +1121,7 @@ namespace cryptonote
       // See `insert_key_images`.
       if (1 < found->second.size() || *(found->second.cbegin()) != txid)
         return true;
-      return m_blockchain.txpool_tx_matches_category(txid, relay_category::broadcasted);
+      return m_blockchain.txpool_tx_matches_category(txid, relay_category::legacy);
     }
     return false;
   }

src/cryptonote_protocol/cryptonote_protocol_handler.inl

@@ -1907,6 +1907,10 @@ skip:
     const uint32_t local_stripe = tools::get_pruning_stripe(m_core.get_blockchain_pruning_seed());
     if (local_stripe == 0)
       return false;
+    // don't request pre-bulletprooof pruned blocks, we can't reconstruct their weight (yet)
+    static const uint64_t bp_fork_height = m_core.get_earliest_ideal_height_for_version(8);
+    if (first_block_height + nblocks - 1 < bp_fork_height)
+      return false;
     // assumes the span size is less or equal to the stripe size
     bool full_data_needed = tools::get_pruning_stripe(first_block_height, context.m_remote_blockchain_height, CRYPTONOTE_PRUNING_LOG_STRIPES) == local_stripe
         || tools::get_pruning_stripe(first_block_height + nblocks - 1, context.m_remote_blockchain_height, CRYPTONOTE_PRUNING_LOG_STRIPES) == local_stripe;
@@ -2083,7 +2087,8 @@ skip:
         skip_unneeded_hashes(context, false);
 
         const uint64_t first_block_height = context.m_last_response_height - context.m_needed_objects.size() + 1;
-        bool sync_pruned_blocks = m_sync_pruned_blocks && m_core.get_blockchain_pruning_seed();
+        static const uint64_t bp_fork_height = m_core.get_earliest_ideal_height_for_version(8);
+        bool sync_pruned_blocks = m_sync_pruned_blocks && first_block_height >= bp_fork_height && m_core.get_blockchain_pruning_seed();
         span = m_block_queue.reserve_span(first_block_height, context.m_last_response_height, count_limit, context.m_connection_id, sync_pruned_blocks, m_core.get_blockchain_pruning_seed(), context.m_pruning_seed, context.m_remote_blockchain_height, context.m_needed_objects);
         MDEBUG(context << " span from " << first_block_height << ": " << span.first << "/" << span.second);
         if (span.second > 0)

src/multisig/multisig.cpp

@@ -82,6 +82,7 @@ namespace cryptonote
     {
       rct::key sk = rct::scalarmultKey(rct::pk2rct(k), rct::sk2rct(blinded_skey));
       crypto::secret_key msk = get_multisig_blinded_secret_key(rct::rct2sk(sk));
+      memwipe(&sk, sizeof(sk));
       multisig_keys.push_back(msk);
       sc_add(spend_skey.bytes, spend_skey.bytes, (const unsigned char*)msk.data);
     }
@@ -126,10 +127,10 @@ namespace cryptonote
   //-----------------------------------------------------------------
   crypto::secret_key generate_multisig_view_secret_key(const crypto::secret_key &skey, const std::vector<crypto::secret_key> &skeys)
   {
-    rct::key view_skey = rct::sk2rct(get_multisig_blinded_secret_key(skey));
+    crypto::secret_key view_skey = get_multisig_blinded_secret_key(skey);
     for (const auto &k: skeys)
-      sc_add(view_skey.bytes, view_skey.bytes, rct::sk2rct(k).bytes);
+      sc_add((unsigned char*)&view_skey, rct::sk2rct(view_skey).bytes, rct::sk2rct(k).bytes);
-    return rct::rct2sk(view_skey);
+    return view_skey;
   }
   //-----------------------------------------------------------------
   crypto::public_key generate_multisig_M_N_spend_public_key(const std::vector<crypto::public_key> &pkeys)

src/p2p/net_node.h

@@ -381,6 +381,7 @@ namespace nodetool
     bool is_addr_recently_failed(const epee::net_utils::network_address& addr);
     bool is_priority_node(const epee::net_utils::network_address& na);
     std::set<std::string> get_seed_nodes(cryptonote::network_type nettype) const;
+    std::set<std::string> get_seed_nodes();
     bool connect_to_seed();
 
     template <class Container>
@@ -464,7 +465,9 @@ namespace nodetool
     std::list<epee::net_utils::network_address>   m_priority_peers;
     std::vector<epee::net_utils::network_address> m_exclusive_peers;
     std::vector<epee::net_utils::network_address> m_seed_nodes;
-    bool m_fallback_seed_nodes_added;
+    bool m_seed_nodes_initialized = false;
+    boost::shared_mutex m_seed_nodes_lock;
+    std::atomic_flag m_fallback_seed_nodes_added;
     std::vector<nodetool::peerlist_entry> m_command_line_peers;
     uint64_t m_peer_livetime;
     //keep connections to initiate some interactions

src/p2p/net_node.inl

@@ -442,6 +442,8 @@ namespace nodetool
 
     if (command_line::has_arg(vm, arg_p2p_seed_node))
     {
+      boost::unique_lock<boost::shared_mutex> lock(m_seed_nodes_lock);
+
       if (!parse_peers_and_add_to_container(vm, arg_p2p_seed_node, m_seed_nodes))
         return false;
     }
@@ -627,6 +629,115 @@ namespace nodetool
   }
   //-----------------------------------------------------------------------------------
   template<class t_payload_net_handler>
+  std::set<std::string> node_server<t_payload_net_handler>::get_seed_nodes()
+  {
+    if (!m_exclusive_peers.empty() || m_offline)
+    {
+      return {};
+    }
+    if (m_nettype == cryptonote::TESTNET)
+    {
+      return get_seed_nodes(cryptonote::TESTNET);
+    }
+    if (m_nettype == cryptonote::STAGENET)
+    {
+      return get_seed_nodes(cryptonote::STAGENET);
+    }
+
+    std::set<std::string> full_addrs;
+
+    // for each hostname in the seed nodes list, attempt to DNS resolve and
+    // add the result addresses as seed nodes
+    // TODO: at some point add IPv6 support, but that won't be relevant
+    // for some time yet.
+
+    std::vector<std::vector<std::string>> dns_results;
+    dns_results.resize(m_seed_nodes_list.size());
+
+    // some libc implementation provide only a very small stack
+    // for threads, e.g. musl only gives +- 80kb, which is not
+    // enough to do a resolve with unbound. we request a stack
+    // of 1 mb, which should be plenty
+    boost::thread::attributes thread_attributes;
+    thread_attributes.set_stack_size(1024*1024);
+
+    std::list<boost::thread> dns_threads;
+    uint64_t result_index = 0;
+    for (const std::string& addr_str : m_seed_nodes_list)
+    {
+      boost::thread th = boost::thread(thread_attributes, [=, &dns_results, &addr_str]
+      {
+        MDEBUG("dns_threads[" << result_index << "] created for: " << addr_str);
+        // TODO: care about dnssec avail/valid
+        bool avail, valid;
+        std::vector<std::string> addr_list;
+
+        try
+        {
+          addr_list = tools::DNSResolver::instance().get_ipv4(addr_str, avail, valid);
+          MDEBUG("dns_threads[" << result_index << "] DNS resolve done");
+          boost::this_thread::interruption_point();
+        }
+        catch(const boost::thread_interrupted&)
+        {
+          // thread interruption request
+          // even if we now have results, finish thread without setting
+          // result variables, which are now out of scope in main thread
+          MWARNING("dns_threads[" << result_index << "] interrupted");
+          return;
+        }
+
+        MINFO("dns_threads[" << result_index << "] addr_str: " << addr_str << "  number of results: " << addr_list.size());
+        dns_results[result_index] = addr_list;
+      });
+
+      dns_threads.push_back(std::move(th));
+      ++result_index;
+    }
+
+    MDEBUG("dns_threads created, now waiting for completion or timeout of " << CRYPTONOTE_DNS_TIMEOUT_MS << "ms");
+    boost::chrono::system_clock::time_point deadline = boost::chrono::system_clock::now() + boost::chrono::milliseconds(CRYPTONOTE_DNS_TIMEOUT_MS);
+    uint64_t i = 0;
+    for (boost::thread& th : dns_threads)
+    {
+      if (! th.try_join_until(deadline))
+      {
+        MWARNING("dns_threads[" << i << "] timed out, sending interrupt");
+        th.interrupt();
+      }
+      ++i;
+    }
+
+    i = 0;
+    for (const auto& result : dns_results)
+    {
+      MDEBUG("DNS lookup for " << m_seed_nodes_list[i] << ": " << result.size() << " results");
+      // if no results for node, thread's lookup likely timed out
+      if (result.size())
+      {
+        for (const auto& addr_string : result)
+          full_addrs.insert(addr_string + ":" + std::to_string(cryptonote::get_config(m_nettype).P2P_DEFAULT_PORT));
+      }
+      ++i;
+    }
+
+    // append the fallback nodes if we have too few seed nodes to start with
+    if (full_addrs.size() < MIN_WANTED_SEED_NODES)
+    {
+      if (full_addrs.empty())
+        MINFO("DNS seed node lookup either timed out or failed, falling back to defaults");
+      else
+        MINFO("Not enough DNS seed nodes found, using fallback defaults too");
+
+      for (const auto &peer: get_seed_nodes(cryptonote::MAINNET))
+        full_addrs.insert(peer);
+      m_fallback_seed_nodes_added.test_and_set();
+    }
+
+    return full_addrs;
+  }
+  //-----------------------------------------------------------------------------------
+  template<class t_payload_net_handler>
   typename node_server<t_payload_net_handler>::network_zone& node_server<t_payload_net_handler>::add_zone(const epee::net_utils::zone zone)
   {
     const auto zone_ = m_network_zones.lower_bound(zone);
@@ -640,123 +751,21 @@ namespace nodetool
   template<class t_payload_net_handler>
   bool node_server<t_payload_net_handler>::init(const boost::program_options::variables_map& vm)
   {
-    std::set<std::string> full_addrs;
-
     bool res = handle_command_line(vm);
     CHECK_AND_ASSERT_MES(res, false, "Failed to handle command line");
 
-    m_fallback_seed_nodes_added = false;
     if (m_nettype == cryptonote::TESTNET)
     {
       memcpy(&m_network_id, &::config::testnet::NETWORK_ID, 16);
-      full_addrs = get_seed_nodes(cryptonote::TESTNET);
     }
     else if (m_nettype == cryptonote::STAGENET)
     {
       memcpy(&m_network_id, &::config::stagenet::NETWORK_ID, 16);
-      full_addrs = get_seed_nodes(cryptonote::STAGENET);
     }
     else
     {
       memcpy(&m_network_id, &::config::NETWORK_ID, 16);
-      if (m_exclusive_peers.empty() && !m_offline)
-      {
-      // for each hostname in the seed nodes list, attempt to DNS resolve and
-      // add the result addresses as seed nodes
-      // TODO: at some point add IPv6 support, but that won't be relevant
-      // for some time yet.
-
-      std::vector<std::vector<std::string>> dns_results;
-      dns_results.resize(m_seed_nodes_list.size());
-
-      // some libc implementation provide only a very small stack
-      // for threads, e.g. musl only gives +- 80kb, which is not
-      // enough to do a resolve with unbound. we request a stack
-      // of 1 mb, which should be plenty
-      boost::thread::attributes thread_attributes;
-      thread_attributes.set_stack_size(1024*1024);
-
-      std::list<boost::thread> dns_threads;
-      uint64_t result_index = 0;
-      for (const std::string& addr_str : m_seed_nodes_list)
-      {
-        boost::thread th = boost::thread(thread_attributes, [=, &dns_results, &addr_str]
-        {
-          MDEBUG("dns_threads[" << result_index << "] created for: " << addr_str);
-          // TODO: care about dnssec avail/valid
-          bool avail, valid;
-          std::vector<std::string> addr_list;
-
-          try
-          {
-            addr_list = tools::DNSResolver::instance().get_ipv4(addr_str, avail, valid);
-            MDEBUG("dns_threads[" << result_index << "] DNS resolve done");
-            boost::this_thread::interruption_point();
-          }
-          catch(const boost::thread_interrupted&)
-          {
-            // thread interruption request
-            // even if we now have results, finish thread without setting
-            // result variables, which are now out of scope in main thread
-            MWARNING("dns_threads[" << result_index << "] interrupted");
-            return;
-          }
-
-          MINFO("dns_threads[" << result_index << "] addr_str: " << addr_str << "  number of results: " << addr_list.size());
-          dns_results[result_index] = addr_list;
-        });
-
-        dns_threads.push_back(std::move(th));
-        ++result_index;
-      }
-
-      MDEBUG("dns_threads created, now waiting for completion or timeout of " << CRYPTONOTE_DNS_TIMEOUT_MS << "ms");
-      boost::chrono::system_clock::time_point deadline = boost::chrono::system_clock::now() + boost::chrono::milliseconds(CRYPTONOTE_DNS_TIMEOUT_MS);
-      uint64_t i = 0;
-      for (boost::thread& th : dns_threads)
-      {
-        if (! th.try_join_until(deadline))
-        {
-          MWARNING("dns_threads[" << i << "] timed out, sending interrupt");
-          th.interrupt();
-        }
-        ++i;
-      }
-
-      i = 0;
-      for (const auto& result : dns_results)
-      {
-        MDEBUG("DNS lookup for " << m_seed_nodes_list[i] << ": " << result.size() << " results");
-        // if no results for node, thread's lookup likely timed out
-        if (result.size())
-        {
-          for (const auto& addr_string : result)
-            full_addrs.insert(addr_string + ":" + std::to_string(cryptonote::get_config(m_nettype).P2P_DEFAULT_PORT));
-        }
-        ++i;
-      }
-
-      // append the fallback nodes if we have too few seed nodes to start with
-      if (full_addrs.size() < MIN_WANTED_SEED_NODES)
-      {
-        if (full_addrs.empty())
-          MINFO("DNS seed node lookup either timed out or failed, falling back to defaults");
-        else
-          MINFO("Not enough DNS seed nodes found, using fallback defaults too");
-
-        for (const auto &peer: get_seed_nodes(cryptonote::MAINNET))
-          full_addrs.insert(peer);
-        m_fallback_seed_nodes_added = true;
-      }
-    }
-    }
-
-    for (const auto& full_addr : full_addrs)
-    {
-      MDEBUG("Seed node: " << full_addr);
-      append_net_address(m_seed_nodes, full_addr, cryptonote::get_config(m_nettype).P2P_DEFAULT_PORT);
     }
-    MDEBUG("Number of seed nodes: " << m_seed_nodes.size());
 
     m_config_folder = command_line::get_arg(vm, cryptonote::arg_data_dir);
     network_zone& public_zone = m_network_zones.at(epee::net_utils::zone::public_);
@@ -1533,6 +1542,20 @@ namespace nodetool
   template<class t_payload_net_handler>
   bool node_server<t_payload_net_handler>::connect_to_seed()
   {
+      boost::upgrade_lock<boost::shared_mutex> seed_nodes_upgrade_lock(m_seed_nodes_lock);
+
+      if (!m_seed_nodes_initialized)
+      {
+        boost::upgrade_to_unique_lock<boost::shared_mutex> seed_nodes_lock(seed_nodes_upgrade_lock);
+        m_seed_nodes_initialized = true;
+        for (const auto& full_addr : get_seed_nodes())
+        {
+          MDEBUG("Seed node: " << full_addr);
+          append_net_address(m_seed_nodes, full_addr, cryptonote::get_config(m_nettype).P2P_DEFAULT_PORT);
+        }
+        MDEBUG("Number of seed nodes: " << m_seed_nodes.size());
+      }
+
       if (m_seed_nodes.empty() || m_offline || !m_exclusive_peers.empty())
         return true;
 
@@ -1553,16 +1576,19 @@ namespace nodetool
           break;
         if(++try_count > m_seed_nodes.size())
         {
-          if (!m_fallback_seed_nodes_added)
+          if (!m_fallback_seed_nodes_added.test_and_set())
           {
             MWARNING("Failed to connect to any of seed peers, trying fallback seeds");
             current_index = m_seed_nodes.size() - 1;
-            for (const auto &peer: get_seed_nodes(m_nettype))
             {
-              MDEBUG("Fallback seed node: " << peer);
+              boost::upgrade_to_unique_lock<boost::shared_mutex> seed_nodes_lock(seed_nodes_upgrade_lock);
-              append_net_address(m_seed_nodes, peer, cryptonote::get_config(m_nettype).P2P_DEFAULT_PORT);
+
+              for (const auto &peer: get_seed_nodes(m_nettype))
+              {
+                MDEBUG("Fallback seed node: " << peer);
+                append_net_address(m_seed_nodes, peer, cryptonote::get_config(m_nettype).P2P_DEFAULT_PORT);
+              }
             }
-            m_fallback_seed_nodes_added = true;
             if (current_index == m_seed_nodes.size() - 1)
             {
               MWARNING("No fallback seeds, continuing without seeds");
@@ -1596,10 +1622,9 @@ namespace nodetool
     // Only have seeds in the public zone right now.
 
     size_t start_conn_count = get_public_outgoing_connections_count();
-    if(!get_public_white_peers_count() && m_seed_nodes.size())
+    if(!get_public_white_peers_count() && !connect_to_seed())
     {
-      if (!connect_to_seed())
+      return false;
-        return false;
     }
 
     if (!connect_to_peerlist(m_priority_peers)) return false;

src/ringct/rctSigs.cpp

@@ -29,6 +29,7 @@
 // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include "misc_log_ex.h"
+#include "misc_language.h"
 #include "common/perf_timer.h"
 #include "common/threadpool.h"
 #include "common/util.h"
@@ -138,6 +139,7 @@ namespace rct {
     //Borromean (c.f. gmax/andytoshi's paper)
     boroSig genBorromean(const key64 x, const key64 P1, const key64 P2, const bits indices) {
         key64 L[2], alpha;
+        auto wiper = epee::misc_utils::create_scope_leave_handler([&](){memwipe(alpha, sizeof(alpha));});
         key c;
         int naught = 0, prime = 0, ii = 0, jj=0;
         boroSig bb;
@@ -220,6 +222,7 @@ namespace rct {
         vector<geDsmp> Ip(dsRows);
         rv.II = keyV(dsRows);
         keyV alpha(rows);
+        auto wiper = epee::misc_utils::create_scope_leave_handler([&](){memwipe(alpha.data(), alpha.size() * sizeof(alpha[0]));});
         keyV aG(rows);
         rv.ss = keyM(cols, aG);
         keyV aHP(dsRows);
@@ -578,7 +581,7 @@ namespace rct {
             subKeys(M[i][1], pubs[i].mask, Cout);
         }
         mgSig result = MLSAG_Gen(message, M, sk, kLRki, mscout, index, rows, hwdev);
-        memwipe(&sk[0], sizeof(key));
+        memwipe(sk.data(), sk.size() * sizeof(key));
         return result;
     }
 

src/ringct/rctTypes.h

@@ -48,6 +48,7 @@ extern "C" {
 
 #include "hex.h"
 #include "span.h"
+#include "memwipe.h"
 #include "serialization/vector.h"
 #include "serialization/debug_archive.h"
 #include "serialization/binary_archive.h"
@@ -106,6 +107,8 @@ namespace rct {
         key L;
         key R;
         key ki;
+
+        ~multisig_kLRki() { memwipe(&k, sizeof(k)); }
     };
 
     struct multisig_out {

src/rpc/core_rpc_server.cpp

@@ -1990,7 +1990,7 @@ namespace cryptonote
       r = false;
     }
     res.untrusted = true;
-    return true;
+    return r;
   }
   //------------------------------------------------------------------------------------------------------------------------------
   bool core_rpc_server::on_get_last_block_header(const COMMAND_RPC_GET_LAST_BLOCK_HEADER::request& req, COMMAND_RPC_GET_LAST_BLOCK_HEADER::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx)

src/rpc/daemon_handler.cpp

@@ -101,7 +101,8 @@ namespace rpc
       {u8"key_images_spent", handle_message<KeyImagesSpent>},
       {u8"mining_status", handle_message<MiningStatus>},
       {u8"save_bc", handle_message<SaveBC>},
-      {u8"send_raw_tx", handle_message<SendRawTxHex>},
+      {u8"send_raw_tx", handle_message<SendRawTx>},
+      {u8"send_raw_tx_hex", handle_message<SendRawTxHex>},
       {u8"set_log_level", handle_message<SetLogLevel>},
       {u8"start_mining", handle_message<StartMining>},
       {u8"stop_mining", handle_message<StopMining>}

src/rpc/daemon_messages.cpp

@@ -34,14 +34,14 @@ namespace cryptonote
 
 namespace rpc
 {
-void GetHeight::Request::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const
+void GetHeight::Request::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const
 {}
 
 void GetHeight::Request::fromJson(const rapidjson::Value& val)
 {
 }
 
-void GetHeight::Response::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const
+void GetHeight::Response::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const
 {
   INSERT_INTO_JSON_OBJECT(dest, height, height);
 }
@@ -52,7 +52,7 @@ void GetHeight::Response::fromJson(const rapidjson::Value& val)
 }
 
 
-void GetBlocksFast::Request::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const
+void GetBlocksFast::Request::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const
 {
   INSERT_INTO_JSON_OBJECT(dest, block_ids, block_ids);
   INSERT_INTO_JSON_OBJECT(dest, start_height, start_height);
@@ -66,7 +66,7 @@ void GetBlocksFast::Request::fromJson(const rapidjson::Value& val)
   GET_FROM_JSON_OBJECT(val, prune, prune);
 }
 
-void GetBlocksFast::Response::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const
+void GetBlocksFast::Response::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const
 {
   INSERT_INTO_JSON_OBJECT(dest, blocks, blocks);
   INSERT_INTO_JSON_OBJECT(dest, start_height, start_height);
@@ -83,7 +83,7 @@ void GetBlocksFast::Response::fromJson(const rapidjson::Value& val)
 }
 
 
-void GetHashesFast::Request::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const
+void GetHashesFast::Request::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const
 {
   INSERT_INTO_JSON_OBJECT(dest, known_hashes, known_hashes);
   INSERT_INTO_JSON_OBJECT(dest, start_height, start_height);
@@ -95,7 +95,7 @@ void GetHashesFast::Request::fromJson(const rapidjson::Value& val)
   GET_FROM_JSON_OBJECT(val, start_height, start_height);
 }
 
-void GetHashesFast::Response::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const
+void GetHashesFast::Response::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const
 {
   INSERT_INTO_JSON_OBJECT(dest, hashes, hashes);
   INSERT_INTO_JSON_OBJECT(dest, start_height, start_height);
@@ -110,7 +110,7 @@ void GetHashesFast::Response::fromJson(const rapidjson::Value& val)
 }
 
 
-void GetTransactions::Request::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const
+void GetTransactions::Request::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const
 {
   INSERT_INTO_JSON_OBJECT(dest, tx_hashes, tx_hashes);
 }
@@ -120,7 +120,7 @@ void GetTransactions::Request::fromJson(const rapidjson::Value& val)
   GET_FROM_JSON_OBJECT(val, tx_hashes, tx_hashes);
 }
 
-void GetTransactions::Response::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const
+void GetTransactions::Response::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const
 {
   INSERT_INTO_JSON_OBJECT(dest, txs, txs);
   INSERT_INTO_JSON_OBJECT(dest, missed_hashes, missed_hashes);
@@ -133,7 +133,7 @@ void GetTransactions::Response::fromJson(const rapidjson::Value& val)
 }
 
 
-void KeyImagesSpent::Request::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const
+void KeyImagesSpent::Request::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const
 {
   INSERT_INTO_JSON_OBJECT(dest, key_images, key_images);
 }
@@ -143,7 +143,7 @@ void KeyImagesSpent::Request::fromJson(const rapidjson::Value& val)
   GET_FROM_JSON_OBJECT(val, key_images, key_images);
 }
 
-void KeyImagesSpent::Response::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const
+void KeyImagesSpent::Response::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const
 {
   INSERT_INTO_JSON_OBJECT(dest, spent_status, spent_status);
 }
@@ -154,7 +154,7 @@ void KeyImagesSpent::Response::fromJson(const rapidjson::Value& val)
 }
 
 
-void GetTxGlobalOutputIndices::Request::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const
+void GetTxGlobalOutputIndices::Request::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const
 {
   INSERT_INTO_JSON_OBJECT(dest, tx_hash, tx_hash);
 }
@@ -164,7 +164,7 @@ void GetTxGlobalOutputIndices::Request::fromJson(const rapidjson::Value& val)
   GET_FROM_JSON_OBJECT(val, tx_hash, tx_hash);
 }
 
-void GetTxGlobalOutputIndices::Response::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const
+void GetTxGlobalOutputIndices::Response::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const
 {
   INSERT_INTO_JSON_OBJECT(dest, output_indices, output_indices);
 }
@@ -174,7 +174,7 @@ void GetTxGlobalOutputIndices::Response::fromJson(const rapidjson::Value& val)
   GET_FROM_JSON_OBJECT(val, output_indices, output_indices);
 }
 
-void SendRawTx::Request::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const
+void SendRawTx::Request::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const
 {
   INSERT_INTO_JSON_OBJECT(dest, tx, tx);
   INSERT_INTO_JSON_OBJECT(dest, relay, relay);
@@ -186,7 +186,7 @@ void SendRawTx::Request::fromJson(const rapidjson::Value& val)
   GET_FROM_JSON_OBJECT(val, relay, relay);
 }
 
-void SendRawTx::Response::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const
+void SendRawTx::Response::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const
 {
   INSERT_INTO_JSON_OBJECT(dest, relayed, relayed);
 }
@@ -197,7 +197,7 @@ void SendRawTx::Response::fromJson(const rapidjson::Value& val)
   GET_FROM_JSON_OBJECT(val, relayed, relayed);
 }
 
-void SendRawTxHex::Request::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const
+void SendRawTxHex::Request::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const
 {
   INSERT_INTO_JSON_OBJECT(dest, tx_as_hex, tx_as_hex);
   INSERT_INTO_JSON_OBJECT(dest, relay, relay);
@@ -209,7 +209,7 @@ void SendRawTxHex::Request::fromJson(const rapidjson::Value& val)
   GET_FROM_JSON_OBJECT(val, relay, relay);
 }
 
-void StartMining::Request::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const
+void StartMining::Request::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const
 {
   INSERT_INTO_JSON_OBJECT(dest, miner_address, miner_address);
   INSERT_INTO_JSON_OBJECT(dest, threads_count, threads_count);
@@ -225,7 +225,7 @@ void StartMining::Request::fromJson(const rapidjson::Value& val)
   GET_FROM_JSON_OBJECT(val, ignore_battery, ignore_battery);
 }
 
-void StartMining::Response::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const
+void StartMining::Response::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const
 {}
 
 void StartMining::Response::fromJson(const rapidjson::Value& val)
@@ -233,14 +233,14 @@ void StartMining::Response::fromJson(const rapidjson::Value& val)
 }
 
 
-void StopMining::Request::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const
+void StopMining::Request::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const
 {}
 
 void StopMining::Request::fromJson(const rapidjson::Value& val)
 {
 }
 
-void StopMining::Response::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const
+void StopMining::Response::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const
 {}
 
 void StopMining::Response::fromJson(const rapidjson::Value& val)
@@ -248,14 +248,14 @@ void StopMining::Response::fromJson(const rapidjson::Value& val)
 }
 
 
-void MiningStatus::Request::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const
+void MiningStatus::Request::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const
 {}
 
 void MiningStatus::Request::fromJson(const rapidjson::Value& val)
 {
 }
 
-void MiningStatus::Response::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const
+void MiningStatus::Response::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const
 {
   INSERT_INTO_JSON_OBJECT(dest, active, active);
   INSERT_INTO_JSON_OBJECT(dest, speed, speed);
@@ -274,14 +274,14 @@ void MiningStatus::Response::fromJson(const rapidjson::Value& val)
 }
 
 
-void GetInfo::Request::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const
+void GetInfo::Request::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const
 {}
 
 void GetInfo::Request::fromJson(const rapidjson::Value& val)
 {
 }
 
-void GetInfo::Response::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const
+void GetInfo::Response::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const
 {
   INSERT_INTO_JSON_OBJECT(dest, info, info);
 }
@@ -292,14 +292,14 @@ void GetInfo::Response::fromJson(const rapidjson::Value& val)
 }
 
 
-void SaveBC::Request::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const
+void SaveBC::Request::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const
 {}
 
 void SaveBC::Request::fromJson(const rapidjson::Value& val)
 {
 }
 
-void SaveBC::Response::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const
+void SaveBC::Response::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const
 {}
 
 void SaveBC::Response::fromJson(const rapidjson::Value& val)
@@ -307,7 +307,7 @@ void SaveBC::Response::fromJson(const rapidjson::Value& val)
 }
 
 
-void GetBlockHash::Request::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const
+void GetBlockHash::Request::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const
 {
   INSERT_INTO_JSON_OBJECT(dest, height, height);
 }
@@ -317,7 +317,7 @@ void GetBlockHash::Request::fromJson(const rapidjson::Value& val)
   GET_FROM_JSON_OBJECT(val, height, height);
 }
 
-void GetBlockHash::Response::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const
+void GetBlockHash::Response::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const
 {
   INSERT_INTO_JSON_OBJECT(dest, hash, hash);
 }
@@ -328,14 +328,14 @@ void GetBlockHash::Response::fromJson(const rapidjson::Value& val)
 }
 
 
-void GetLastBlockHeader::Request::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const
+void GetLastBlockHeader::Request::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const
 {}
 
 void GetLastBlockHeader::Request::fromJson(const rapidjson::Value& val)
 {
 }
 
-void GetLastBlockHeader::Response::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const
+void GetLastBlockHeader::Response::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const
 {
   INSERT_INTO_JSON_OBJECT(dest, header, header);
 }
@@ -346,7 +346,7 @@ void GetLastBlockHeader::Response::fromJson(const rapidjson::Value& val)
 }
 
 
-void GetBlockHeaderByHash::Request::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const
+void GetBlockHeaderByHash::Request::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const
 {
   INSERT_INTO_JSON_OBJECT(dest, hash, hash);
 }
@@ -356,7 +356,7 @@ void GetBlockHeaderByHash::Request::fromJson(const rapidjson::Value& val)
   GET_FROM_JSON_OBJECT(val, hash, hash);
 }
 
-void GetBlockHeaderByHash::Response::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const
+void GetBlockHeaderByHash::Response::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const
 {
   INSERT_INTO_JSON_OBJECT(dest, header, header);
 }
@@ -367,7 +367,7 @@ void GetBlockHeaderByHash::Response::fromJson(const rapidjson::Value& val)
 }
 
 
-void GetBlockHeaderByHeight::Request::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const
+void GetBlockHeaderByHeight::Request::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const
 {
   INSERT_INTO_JSON_OBJECT(dest, height, height);
 }
@@ -377,7 +377,7 @@ void GetBlockHeaderByHeight::Request::fromJson(const rapidjson::Value& val)
   GET_FROM_JSON_OBJECT(val, height, height);
 }
 
-void GetBlockHeaderByHeight::Response::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const
+void GetBlockHeaderByHeight::Response::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const
 {
   INSERT_INTO_JSON_OBJECT(dest, header, header);
 }
@@ -388,7 +388,7 @@ void GetBlockHeaderByHeight::Response::fromJson(const rapidjson::Value& val)
 }
 
 
-void GetBlockHeadersByHeight::Request::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const
+void GetBlockHeadersByHeight::Request::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const
 {
   INSERT_INTO_JSON_OBJECT(dest, heights, heights);
 }
@@ -398,7 +398,7 @@ void GetBlockHeadersByHeight::Request::fromJson(const rapidjson::Value& val)
   GET_FROM_JSON_OBJECT(val, heights, heights);
 }
 
-void GetBlockHeadersByHeight::Response::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const
+void GetBlockHeadersByHeight::Response::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const
 {
   INSERT_INTO_JSON_OBJECT(dest, headers, headers);
 }
@@ -409,14 +409,14 @@ void GetBlockHeadersByHeight::Response::fromJson(const rapidjson::Value& val)
 }
 
 
-void GetPeerList::Request::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const
+void GetPeerList::Request::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const
 {}
 
 void GetPeerList::Request::fromJson(const rapidjson::Value& val)
 {
 }
 
-void GetPeerList::Response::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const
+void GetPeerList::Response::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const
 {
   INSERT_INTO_JSON_OBJECT(dest, white_list, white_list);
   INSERT_INTO_JSON_OBJECT(dest, gray_list, gray_list);
@@ -429,7 +429,7 @@ void GetPeerList::Response::fromJson(const rapidjson::Value& val)
 }
 
 
-void SetLogLevel::Request::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const
+void SetLogLevel::Request::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const
 {
   INSERT_INTO_JSON_OBJECT(dest, level, level);
 }
@@ -439,7 +439,7 @@ void SetLogLevel::Request::fromJson(const rapidjson::Value& val)
   GET_FROM_JSON_OBJECT(val, level, level);
 }
 
-void SetLogLevel::Response::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const
+void SetLogLevel::Response::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const
 {}
 
 void SetLogLevel::Response::fromJson(const rapidjson::Value& val)
@@ -447,14 +447,14 @@ void SetLogLevel::Response::fromJson(const rapidjson::Value& val)
 }
 
 
-void GetTransactionPool::Request::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const
+void GetTransactionPool::Request::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const
 {}
 
 void GetTransactionPool::Request::fromJson(const rapidjson::Value& val)
 {
 }
 
-void GetTransactionPool::Response::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const
+void GetTransactionPool::Response::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const
 {
   INSERT_INTO_JSON_OBJECT(dest, transactions, transactions);
   INSERT_INTO_JSON_OBJECT(dest, key_images, key_images);
@@ -467,7 +467,7 @@ void GetTransactionPool::Response::fromJson(const rapidjson::Value& val)
 }
 
 
-void HardForkInfo::Request::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const
+void HardForkInfo::Request::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const
 {
   INSERT_INTO_JSON_OBJECT(dest, version, version);
 }
@@ -477,7 +477,7 @@ void HardForkInfo::Request::fromJson(const rapidjson::Value& val)
   GET_FROM_JSON_OBJECT(val, version, version);
 }
 
-void HardForkInfo::Response::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const
+void HardForkInfo::Response::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const
 {
   INSERT_INTO_JSON_OBJECT(dest, info, info);
 }
@@ -488,7 +488,7 @@ void HardForkInfo::Response::fromJson(const rapidjson::Value& val)
 }
 
 
-void GetOutputHistogram::Request::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const
+void GetOutputHistogram::Request::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const
 {
   INSERT_INTO_JSON_OBJECT(dest, amounts, amounts);
   INSERT_INTO_JSON_OBJECT(dest, min_count, min_count);
@@ -506,7 +506,7 @@ void GetOutputHistogram::Request::fromJson(const rapidjson::Value& val)
   GET_FROM_JSON_OBJECT(val, recent_cutoff, recent_cutoff);
 }
 
-void GetOutputHistogram::Response::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const
+void GetOutputHistogram::Response::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const
 {
   INSERT_INTO_JSON_OBJECT(dest, histogram, histogram);
 }
@@ -517,7 +517,7 @@ void GetOutputHistogram::Response::fromJson(const rapidjson::Value& val)
 }
 
 
-void GetOutputKeys::Request::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const
+void GetOutputKeys::Request::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const
 {
   INSERT_INTO_JSON_OBJECT(dest, outputs, outputs);
 }
@@ -527,7 +527,7 @@ void GetOutputKeys::Request::fromJson(const rapidjson::Value& val)
   GET_FROM_JSON_OBJECT(val, outputs, outputs);
 }
 
-void GetOutputKeys::Response::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const
+void GetOutputKeys::Response::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const
 {
   INSERT_INTO_JSON_OBJECT(dest, keys, keys);
 }
@@ -538,14 +538,14 @@ void GetOutputKeys::Response::fromJson(const rapidjson::Value& val)
 }
 
 
-void GetRPCVersion::Request::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const
+void GetRPCVersion::Request::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const
 {}
 
 void GetRPCVersion::Request::fromJson(const rapidjson::Value& val)
 {
 }
 
-void GetRPCVersion::Response::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const
+void GetRPCVersion::Response::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const
 {
   INSERT_INTO_JSON_OBJECT(dest, version, version);
 }
@@ -555,7 +555,7 @@ void GetRPCVersion::Response::fromJson(const rapidjson::Value& val)
   GET_FROM_JSON_OBJECT(val, version, version);
 }
 
-void GetFeeEstimate::Request::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const
+void GetFeeEstimate::Request::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const
 {
   INSERT_INTO_JSON_OBJECT(dest, num_grace_blocks, num_grace_blocks);
 }
@@ -565,7 +565,7 @@ void GetFeeEstimate::Request::fromJson(const rapidjson::Value& val)
   GET_FROM_JSON_OBJECT(val, num_grace_blocks, num_grace_blocks);
 }
 
-void GetFeeEstimate::Response::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const
+void GetFeeEstimate::Response::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const
 {
   INSERT_INTO_JSON_OBJECT(dest, estimated_base_fee, estimated_base_fee);
   INSERT_INTO_JSON_OBJECT(dest, fee_mask, fee_mask);
@@ -581,7 +581,7 @@ void GetFeeEstimate::Response::fromJson(const rapidjson::Value& val)
   GET_FROM_JSON_OBJECT(val, hard_fork_version, hard_fork_version);
 }
 
-void GetOutputDistribution::Request::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const
+void GetOutputDistribution::Request::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const
 {
   INSERT_INTO_JSON_OBJECT(dest, amounts, amounts);
   INSERT_INTO_JSON_OBJECT(dest, from_height, from_height);
@@ -597,7 +597,7 @@ void GetOutputDistribution::Request::fromJson(const rapidjson::Value& val)
   GET_FROM_JSON_OBJECT(val, cumulative, cumulative);
 }
 
-void GetOutputDistribution::Response::doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const
+void GetOutputDistribution::Response::doToJson(rapidjson::Writer<epee::byte_stream>& dest) const
 {
   INSERT_INTO_JSON_OBJECT(dest, status, status);
   INSERT_INTO_JSON_OBJECT(dest, distributions, distributions);

src/rpc/daemon_messages.h

@@ -28,11 +28,11 @@
 
 #pragma once
 
-#include <rapidjson/stringbuffer.h>
 #include <rapidjson/writer.h>
 #include <unordered_map>
 #include <vector>
 
+#include "byte_stream.h"
 #include "message.h"
 #include "cryptonote_protocol/cryptonote_protocol_defs.h"
 #include "rpc/message_data_structs.h"
@@ -50,7 +50,7 @@ class classname \
       public: \
         Request() { } \
         ~Request() { } \
-        void doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const override final; \
+        void doToJson(rapidjson::Writer<epee::byte_stream>& dest) const override final; \
         void fromJson(const rapidjson::Value& val) override final;
 
 #define BEGIN_RPC_MESSAGE_RESPONSE \
@@ -59,7 +59,7 @@ class classname \
       public: \
         Response() { } \
         ~Response() { } \
-        void doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const override final; \
+        void doToJson(rapidjson::Writer<epee::byte_stream>& dest) const override final; \
         void fromJson(const rapidjson::Value& val) override final;
 
 #define END_RPC_MESSAGE_REQUEST };

src/rpc/message.cpp

@@ -62,7 +62,7 @@ const rapidjson::Value& get_method_field(const rapidjson::Value& src)
 }
 }
 
-void Message::toJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const
+void Message::toJson(rapidjson::Writer<epee::byte_stream>& dest) const
 {
   dest.StartObject();
   INSERT_INTO_JSON_OBJECT(dest, status, status);
@@ -151,9 +151,9 @@ cryptonote::rpc::error FullMessage::getError()
 
 epee::byte_slice FullMessage::getRequest(const std::string& request, const Message& message, const unsigned id)
 {
-  rapidjson::StringBuffer buffer;
+  epee::byte_stream buffer;
   {
-    rapidjson::Writer<rapidjson::StringBuffer> dest{buffer};
+    rapidjson::Writer<epee::byte_stream> dest{buffer};
 
     dest.StartObject();
     INSERT_INTO_JSON_OBJECT(dest, jsonrpc, (boost::string_ref{"2.0", 3}));
@@ -172,15 +172,15 @@ epee::byte_slice FullMessage::getRequest(const std::string& request, const Messa
     if (!dest.IsComplete())
       throw std::logic_error{"Invalid JSON tree generated"};
   }
-  return epee::byte_slice{{buffer.GetString(), buffer.GetSize()}};
+  return epee::byte_slice{std::move(buffer)};
 }
 
 
 epee::byte_slice FullMessage::getResponse(const Message& message, const rapidjson::Value& id)
 {
-  rapidjson::StringBuffer buffer;
+  epee::byte_stream buffer;
   {
-    rapidjson::Writer<rapidjson::StringBuffer> dest{buffer};
+    rapidjson::Writer<epee::byte_stream> dest{buffer};
 
     dest.StartObject();
     INSERT_INTO_JSON_OBJECT(dest, jsonrpc, (boost::string_ref{"2.0", 3}));
@@ -207,7 +207,7 @@ epee::byte_slice FullMessage::getResponse(const Message& message, const rapidjso
     if (!dest.IsComplete())
       throw std::logic_error{"Invalid JSON tree generated"};
   }
-  return epee::byte_slice{{buffer.GetString(), buffer.GetSize()}};
+  return epee::byte_slice{std::move(buffer)};
 }
 
 // convenience functions for bad input

src/rpc/message.h

@@ -29,11 +29,11 @@
 #pragma once
 
 #include <rapidjson/document.h>
-#include <rapidjson/stringbuffer.h>
 #include <rapidjson/writer.h>
 #include <string>
 
 #include "byte_slice.h"
+#include "byte_stream.h"
 #include "rpc/message_data_structs.h"
 
 namespace cryptonote
@@ -44,7 +44,7 @@ namespace rpc
 
   class Message
   {
-      virtual void doToJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const
+      virtual void doToJson(rapidjson::Writer<epee::byte_stream>& dest) const
       {}
 
     public:
@@ -58,7 +58,7 @@ namespace rpc
 
       virtual ~Message() { }
 
-      void toJson(rapidjson::Writer<rapidjson::StringBuffer>& dest) const;
+      void toJson(rapidjson::Writer<epee::byte_stream>& dest) const;
 
       virtual void fromJson(const rapidjson::Value& val);
 

src/rpc/zmq_server.cpp

@@ -28,7 +28,6 @@
 
 #include "zmq_server.h"
 
-#include <boost/utility/string_ref.hpp>
 #include <chrono>
 #include <cstdint>
 #include <system_error>

src/serialization/json_object.cpp

@@ -126,12 +126,12 @@ void read_hex(const rapidjson::Value& val, epee::span<std::uint8_t> dest)
   }
 }
 
-void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const rapidjson::Value& src)
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const rapidjson::Value& src)
 {
   src.Accept(dest);
 }
 
-void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const boost::string_ref i)
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const boost::string_ref i)
 {
   dest.String(i.data(), i.size());
 }
@@ -146,7 +146,7 @@ void fromJsonValue(const rapidjson::Value& val, std::string& str)
   str = val.GetString();
 }
 
-void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, bool i)
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, bool i)
 {
   dest.Bool(i);
 }
@@ -185,7 +185,7 @@ void fromJsonValue(const rapidjson::Value& val, short& i)
   to_int(val, i);
 }
 
-void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const unsigned int i)
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const unsigned int i)
 {
   dest.Uint(i);
 }
@@ -195,7 +195,7 @@ void fromJsonValue(const rapidjson::Value& val, unsigned int& i)
   to_uint(val, i);
 }
 
-void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const int i)
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const int i)
 {
   dest.Int(i);
 }
@@ -205,7 +205,7 @@ void fromJsonValue(const rapidjson::Value& val, int& i)
   to_int(val, i);
 }
 
-void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const unsigned long long i)
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const unsigned long long i)
 {
   static_assert(!precision_loss<unsigned long long, std::uint64_t>(), "bad uint64 conversion");
   dest.Uint64(i);
@@ -216,7 +216,7 @@ void fromJsonValue(const rapidjson::Value& val, unsigned long long& i)
   to_uint64(val, i);
 }
 
-void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const long long i)
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const long long i)
 {
   static_assert(!precision_loss<long long, std::int64_t>(), "bad int64 conversion");
   dest.Int64(i);
@@ -237,7 +237,7 @@ void fromJsonValue(const rapidjson::Value& val, long& i)
   to_int64(val, i);
 }
 
-void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::transaction& tx)
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::transaction& tx)
 {
   dest.StartObject();
 
@@ -269,7 +269,7 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::transaction& tx)
   GET_FROM_JSON_OBJECT(val, tx.rct_signatures, ringct);
 }
 
-void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::block& b)
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::block& b)
 {
   dest.StartObject();
 
@@ -301,14 +301,14 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::block& b)
   GET_FROM_JSON_OBJECT(val, b.tx_hashes, tx_hashes);
 }
 
-void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::txin_v& txin)
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::txin_v& txin)
 {
   dest.StartObject();
   struct add_input
   {
     using result_type = void;
 
-    rapidjson::Writer<rapidjson::StringBuffer>& dest;
+    rapidjson::Writer<epee::byte_stream>& dest;
 
     void operator()(cryptonote::txin_to_key const& input) const
     {
@@ -373,7 +373,7 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::txin_v& txin)
   }
 }
 
-void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::txin_gen& txin)
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::txin_gen& txin)
 {
   dest.StartObject();
 
@@ -392,7 +392,7 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::txin_gen& txin)
   GET_FROM_JSON_OBJECT(val, txin.height, height);
 }
 
-void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::txin_to_script& txin)
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::txin_to_script& txin)
 {
   dest.StartObject();
 
@@ -417,7 +417,7 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::txin_to_script& txin
 }
 
 
-void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::txin_to_scripthash& txin)
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::txin_to_scripthash& txin)
 {
   dest.StartObject();
 
@@ -443,7 +443,7 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::txin_to_scripthash&
   GET_FROM_JSON_OBJECT(val, txin.sigset, sigset);
 }
 
-void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::txin_to_key& txin)
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::txin_to_key& txin)
 {
   dest.StartObject();
 
@@ -467,7 +467,7 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::txin_to_key& txin)
 }
 
 
-void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::txout_to_script& txout)
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::txout_to_script& txout)
 {
   dest.StartObject();
 
@@ -489,7 +489,7 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::txout_to_script& txo
 }
 
 
-void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::txout_to_scripthash& txout)
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::txout_to_scripthash& txout)
 {
   dest.StartObject();
 
@@ -509,7 +509,7 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::txout_to_scripthash&
 }
 
 
-void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::txout_to_key& txout)
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::txout_to_key& txout)
 {
   dest.StartObject();
 
@@ -528,7 +528,7 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::txout_to_key& txout)
   GET_FROM_JSON_OBJECT(val, txout.key, key);
 }
 
-void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::tx_out& txout)
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::tx_out& txout)
 {
   dest.StartObject();
   INSERT_INTO_JSON_OBJECT(dest, amount, txout.amount);
@@ -537,7 +537,7 @@ void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const crypton
   {
     using result_type = void;
 
-    rapidjson::Writer<rapidjson::StringBuffer>& dest;
+    rapidjson::Writer<epee::byte_stream>& dest;
 
     void operator()(cryptonote::txout_to_key const& output) const
     {
@@ -596,7 +596,7 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::tx_out& txout)
   }
 }
 
-void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::connection_info& info)
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::connection_info& info)
 {
   dest.StartObject();
 
@@ -668,7 +668,7 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::connection_info& inf
   GET_FROM_JSON_OBJECT(val, info.current_upload, current_upload);
 }
 
-void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::tx_blob_entry& tx)
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::tx_blob_entry& tx)
 {
   dest.StartObject();
 
@@ -689,7 +689,7 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::tx_blob_entry& tx)
   GET_FROM_JSON_OBJECT(val, tx.prunable_hash, prunable_hash);
 }
 
-void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::block_complete_entry& blk)
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::block_complete_entry& blk)
 {
   dest.StartObject();
 
@@ -711,7 +711,7 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::block_complete_entry
   GET_FROM_JSON_OBJECT(val, blk.txs, transactions);
 }
 
-void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::rpc::block_with_transactions& blk)
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::rpc::block_with_transactions& blk)
 {
   dest.StartObject();
 
@@ -733,7 +733,7 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::block_with_tran
   GET_FROM_JSON_OBJECT(val, blk.transactions, transactions);
 }
 
-void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::rpc::transaction_info& tx_info)
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::rpc::transaction_info& tx_info)
 {
   dest.StartObject();
 
@@ -757,7 +757,7 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::transaction_inf
   GET_FROM_JSON_OBJECT(val, tx_info.transaction, transaction);
 }
 
-void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::rpc::output_key_and_amount_index& out)
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::rpc::output_key_and_amount_index& out)
 {
   dest.StartObject();
 
@@ -779,7 +779,7 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::output_key_and_
   GET_FROM_JSON_OBJECT(val, out.key, key);
 }
 
-void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::rpc::amount_with_random_outputs& out)
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::rpc::amount_with_random_outputs& out)
 {
   dest.StartObject();
 
@@ -801,7 +801,7 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::amount_with_ran
   GET_FROM_JSON_OBJECT(val, out.outputs, outputs);
 }
 
-void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::rpc::peer& peer)
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::rpc::peer& peer)
 {
   dest.StartObject();
 
@@ -833,7 +833,7 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::peer& peer)
   GET_FROM_JSON_OBJECT(val, peer.pruning_seed, pruning_seed);
 }
 
-void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::rpc::tx_in_pool& tx)
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::rpc::tx_in_pool& tx)
 {
   dest.StartObject();
 
@@ -880,7 +880,7 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::tx_in_pool& tx)
   GET_FROM_JSON_OBJECT(val, tx.double_spend_seen, double_spend_seen);
 }
 
-void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::rpc::hard_fork_info& info)
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::rpc::hard_fork_info& info)
 {
   dest.StartObject();
 
@@ -914,7 +914,7 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::hard_fork_info&
   GET_FROM_JSON_OBJECT(val, info.earliest_height, earliest_height);
 }
 
-void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::rpc::output_amount_count& out)
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::rpc::output_amount_count& out)
 {
   dest.StartObject();
 
@@ -940,7 +940,7 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::output_amount_c
   GET_FROM_JSON_OBJECT(val, out.recent_count, recent_count);
 }
 
-void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::rpc::output_amount_and_index& out)
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::rpc::output_amount_and_index& out)
 {
   dest.StartObject();
 
@@ -962,7 +962,7 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::output_amount_a
   GET_FROM_JSON_OBJECT(val, out.index, index);
 }
 
-void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::rpc::output_key_mask_unlocked& out)
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::rpc::output_key_mask_unlocked& out)
 {
   dest.StartObject();
 
@@ -985,7 +985,7 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::output_key_mask
   GET_FROM_JSON_OBJECT(val, out.unlocked, unlocked);
 }
 
-void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::rpc::error& err)
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::rpc::error& err)
 {
   dest.StartObject();
 
@@ -1008,7 +1008,7 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::error& error)
   GET_FROM_JSON_OBJECT(val, error.message, message);
 }
 
-void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::rpc::BlockHeaderResponse& response)
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::rpc::BlockHeaderResponse& response)
 {
   dest.StartObject();
 
@@ -1045,7 +1045,7 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::BlockHeaderResp
   GET_FROM_JSON_OBJECT(val, response.reward, reward);
 }
 
-void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const rct::rctSig& sig)
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const rct::rctSig& sig)
 {
   using boost::adaptors::transform;
 
@@ -1115,7 +1115,7 @@ void fromJsonValue(const rapidjson::Value& val, rct::rctSig& sig)
   }
 }
 
-void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const rct::ecdhTuple& tuple)
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const rct::ecdhTuple& tuple)
 {
   dest.StartObject();
   INSERT_INTO_JSON_OBJECT(dest, mask, tuple.mask);
@@ -1134,7 +1134,7 @@ void fromJsonValue(const rapidjson::Value& val, rct::ecdhTuple& tuple)
   GET_FROM_JSON_OBJECT(val, tuple.amount, amount);
 }
 
-void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const rct::rangeSig& sig)
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const rct::rangeSig& sig)
 {
   dest.StartObject();
 
@@ -1171,7 +1171,7 @@ void fromJsonValue(const rapidjson::Value& val, rct::rangeSig& sig)
   }
 }
 
-void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const rct::Bulletproof& p)
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const rct::Bulletproof& p)
 {
   dest.StartObject();
 
@@ -1212,7 +1212,7 @@ void fromJsonValue(const rapidjson::Value& val, rct::Bulletproof& p)
   GET_FROM_JSON_OBJECT(val, p.t, t);
 }
 
-void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const rct::boroSig& sig)
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const rct::boroSig& sig)
 {
   dest.StartObject();
 
@@ -1257,7 +1257,7 @@ void fromJsonValue(const rapidjson::Value& val, rct::boroSig& sig)
   GET_FROM_JSON_OBJECT(val, sig.ee, ee);
 }
 
-void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const rct::mgSig& sig)
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const rct::mgSig& sig)
 {
   dest.StartObject();
 
@@ -1278,7 +1278,7 @@ void fromJsonValue(const rapidjson::Value& val, rct::mgSig& sig)
   GET_FROM_JSON_OBJECT(val, sig.cc, cc);
 }
 
-void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::rpc::DaemonInfo& info)
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::rpc::DaemonInfo& info)
 {
   dest.StartObject();
 
@@ -1339,7 +1339,7 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::DaemonInfo& inf
   GET_FROM_JSON_OBJECT(val, info.start_time, start_time);
 }
 
-void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::rpc::output_distribution& dist)
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::rpc::output_distribution& dist)
 {
   dest.StartObject();
 

src/serialization/json_object.h

@@ -31,9 +31,9 @@
 #include <boost/utility/string_ref.hpp>
 #include <cstring>
 #include <rapidjson/document.h>
-#include <rapidjson/stringbuffer.h>
 #include <rapidjson/writer.h>
 
+#include "byte_stream.h"
 #include "cryptonote_basic/cryptonote_basic.h"
 #include "rpc/message_data_structs.h"
 #include "cryptonote_protocol/cryptonote_protocol_defs.h"
@@ -123,7 +123,7 @@ void read_hex(const rapidjson::Value& val, epee::span<std::uint8_t> dest);
 
 // POD to json key
 template <class Type>
-inline typename std::enable_if<is_to_hex<Type>()>::type toJsonKey(rapidjson::Writer<rapidjson::StringBuffer>& dest, const Type& pod)
+inline typename std::enable_if<is_to_hex<Type>()>::type toJsonKey(rapidjson::Writer<epee::byte_stream>& dest, const Type& pod)
 {
   const auto hex = epee::to_hex::array(pod);
   dest.Key(hex.data(), hex.size());
@@ -131,7 +131,7 @@ inline typename std::enable_if<is_to_hex<Type>()>::type toJsonKey(rapidjson::Wri
 
 // POD to json value
 template <class Type>
-inline typename std::enable_if<is_to_hex<Type>()>::type toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const Type& pod)
+inline typename std::enable_if<is_to_hex<Type>()>::type toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const Type& pod)
 {
   const auto hex = epee::to_hex::array(pod);
   dest.String(hex.data(), hex.size());
@@ -144,16 +144,16 @@ inline typename std::enable_if<is_to_hex<Type>()>::type fromJsonValue(const rapi
   json::read_hex(val, epee::as_mut_byte_span(t));
 }
 
-void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const rapidjson::Value& src);
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const rapidjson::Value& src);
 
-void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, boost::string_ref i);
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, boost::string_ref i);
-inline void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const std::string& i)
+inline void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const std::string& i)
 {
   toJsonValue(dest, boost::string_ref{i});
 }
 void fromJsonValue(const rapidjson::Value& val, std::string& str);
 
-void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, bool i);
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, bool i);
 void fromJsonValue(const rapidjson::Value& val, bool& b);
 
 // integers overloads for toJsonValue are not needed for standard promotions
@@ -168,144 +168,144 @@ void fromJsonValue(const rapidjson::Value& val, unsigned short& i);
 
 void fromJsonValue(const rapidjson::Value& val, short& i);
 
-void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const unsigned i);
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const unsigned i);
 void fromJsonValue(const rapidjson::Value& val, unsigned& i);
 
-void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const int);
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const int);
 void fromJsonValue(const rapidjson::Value& val, int& i);
 
 
-void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const unsigned long long i);
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const unsigned long long i);
 void fromJsonValue(const rapidjson::Value& val, unsigned long long& i);
 
-void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const long long i);
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const long long i);
 void fromJsonValue(const rapidjson::Value& val, long long& i);
 
-inline void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const unsigned long i) {
+inline void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const unsigned long i) {
     toJsonValue(dest, static_cast<unsigned long long>(i));
 }
 void fromJsonValue(const rapidjson::Value& val, unsigned long& i);
 
-inline void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const long i) {
+inline void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const long i) {
     toJsonValue(dest, static_cast<long long>(i));
 }
 void fromJsonValue(const rapidjson::Value& val, long& i);
 
 // end integers
 
-void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::transaction& tx);
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::transaction& tx);
 void fromJsonValue(const rapidjson::Value& val, cryptonote::transaction& tx);
 
-void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::block& b);
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::block& b);
 void fromJsonValue(const rapidjson::Value& val, cryptonote::block& b);
 
-void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::txin_v& txin);
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::txin_v& txin);
 void fromJsonValue(const rapidjson::Value& val, cryptonote::txin_v& txin);
 
-void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::txin_gen& txin);
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::txin_gen& txin);
 void fromJsonValue(const rapidjson::Value& val, cryptonote::txin_gen& txin);
 
-void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::txin_to_script& txin);
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::txin_to_script& txin);
 void fromJsonValue(const rapidjson::Value& val, cryptonote::txin_to_script& txin);
 
-void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::txin_to_scripthash& txin);
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::txin_to_scripthash& txin);
 void fromJsonValue(const rapidjson::Value& val, cryptonote::txin_to_scripthash& txin);
 
-void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::txin_to_key& txin);
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::txin_to_key& txin);
 void fromJsonValue(const rapidjson::Value& val, cryptonote::txin_to_key& txin);
 
-void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::txout_target_v& txout);
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::txout_target_v& txout);
 void fromJsonValue(const rapidjson::Value& val, cryptonote::txout_target_v& txout);
 
-void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::txout_to_script& txout);
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::txout_to_script& txout);
 void fromJsonValue(const rapidjson::Value& val, cryptonote::txout_to_script& txout);
 
-void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::txout_to_scripthash& txout);
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::txout_to_scripthash& txout);
 void fromJsonValue(const rapidjson::Value& val, cryptonote::txout_to_scripthash& txout);
 
-void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::txout_to_key& txout);
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::txout_to_key& txout);
 void fromJsonValue(const rapidjson::Value& val, cryptonote::txout_to_key& txout);
 
-void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::tx_out& txout);
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::tx_out& txout);
 void fromJsonValue(const rapidjson::Value& val, cryptonote::tx_out& txout);
 
-void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::connection_info& info);
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::connection_info& info);
 void fromJsonValue(const rapidjson::Value& val, cryptonote::connection_info& info);
 
-void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::tx_blob_entry& tx);
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::tx_blob_entry& tx);
 void fromJsonValue(const rapidjson::Value& val, cryptonote::tx_blob_entry& tx);
 
-void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::block_complete_entry& blk);
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::block_complete_entry& blk);
 void fromJsonValue(const rapidjson::Value& val, cryptonote::block_complete_entry& blk);
 
-void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::rpc::block_with_transactions& blk);
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::rpc::block_with_transactions& blk);
 void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::block_with_transactions& blk);
 
-void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::rpc::transaction_info& tx_info);
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::rpc::transaction_info& tx_info);
 void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::transaction_info& tx_info);
 
-void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::rpc::output_key_and_amount_index& out);
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::rpc::output_key_and_amount_index& out);
 void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::output_key_and_amount_index& out);
 
-void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::rpc::amount_with_random_outputs& out);
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::rpc::amount_with_random_outputs& out);
 void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::amount_with_random_outputs& out);
 
-void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::rpc::peer& peer);
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::rpc::peer& peer);
 void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::peer& peer);
 
-void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::rpc::tx_in_pool& tx);
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::rpc::tx_in_pool& tx);
 void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::tx_in_pool& tx);
 
-void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::rpc::hard_fork_info& info);
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::rpc::hard_fork_info& info);
 void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::hard_fork_info& info);
 
-void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::rpc::output_amount_count& out);
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::rpc::output_amount_count& out);
 void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::output_amount_count& out);
 
-void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::rpc::output_amount_and_index& out);
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::rpc::output_amount_and_index& out);
 void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::output_amount_and_index& out);
 
-void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::rpc::output_key_mask_unlocked& out);
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::rpc::output_key_mask_unlocked& out);
 void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::output_key_mask_unlocked& out);
 
-void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::rpc::error& err);
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::rpc::error& err);
 void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::error& error);
 
-void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::rpc::BlockHeaderResponse& response);
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::rpc::BlockHeaderResponse& response);
 void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::BlockHeaderResponse& response);
 
-void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const rct::rctSig& i);
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const rct::rctSig& i);
 void fromJsonValue(const rapidjson::Value& val, rct::rctSig& sig);
 
-void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const rct::ecdhTuple& tuple);
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const rct::ecdhTuple& tuple);
 void fromJsonValue(const rapidjson::Value& val, rct::ecdhTuple& tuple);
 
-void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const rct::rangeSig& sig);
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const rct::rangeSig& sig);
 void fromJsonValue(const rapidjson::Value& val, rct::rangeSig& sig);
 
-void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const rct::Bulletproof& p);
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const rct::Bulletproof& p);
 void fromJsonValue(const rapidjson::Value& val, rct::Bulletproof& p);
 
-void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const rct::boroSig& sig);
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const rct::boroSig& sig);
 void fromJsonValue(const rapidjson::Value& val, rct::boroSig& sig);
 
-void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const rct::mgSig& sig);
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const rct::mgSig& sig);
 void fromJsonValue(const rapidjson::Value& val, rct::mgSig& sig);
 
-void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::rpc::DaemonInfo& info);
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::rpc::DaemonInfo& info);
 void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::DaemonInfo& info);
 
-void toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const cryptonote::rpc::output_distribution& dist);
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::rpc::output_distribution& dist);
 void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::output_distribution& dist);
 
 template <typename Map>
-typename std::enable_if<sfinae::is_map_like<Map>::value, void>::type toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const Map& map);
+typename std::enable_if<sfinae::is_map_like<Map>::value, void>::type toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const Map& map);
 
 template <typename Map>
 typename std::enable_if<sfinae::is_map_like<Map>::value, void>::type fromJsonValue(const rapidjson::Value& val, Map& map);
 
 template <typename Vec>
-typename std::enable_if<sfinae::is_vector_like<Vec>::value, void>::type toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const Vec &vec);
+typename std::enable_if<sfinae::is_vector_like<Vec>::value, void>::type toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const Vec &vec);
 
 template <typename Vec>
 typename std::enable_if<sfinae::is_vector_like<Vec>::value, void>::type fromJsonValue(const rapidjson::Value& val, Vec& vec);
@@ -315,7 +315,7 @@ typename std::enable_if<sfinae::is_vector_like<Vec>::value, void>::type fromJson
 // unfortunately because of how templates work they have to be here.
 
 template <typename Map>
-inline typename std::enable_if<sfinae::is_map_like<Map>::value, void>::type toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const Map& map)
+inline typename std::enable_if<sfinae::is_map_like<Map>::value, void>::type toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const Map& map)
 {
   using key_type = typename Map::key_type;
   static_assert(std::is_same<std::string, key_type>() || is_to_hex<key_type>(), "invalid map key type");
@@ -351,7 +351,7 @@ inline typename std::enable_if<sfinae::is_map_like<Map>::value, void>::type from
 }
 
 template <typename Vec>
-inline typename std::enable_if<sfinae::is_vector_like<Vec>::value, void>::type toJsonValue(rapidjson::Writer<rapidjson::StringBuffer>& dest, const Vec &vec)
+inline typename std::enable_if<sfinae::is_vector_like<Vec>::value, void>::type toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const Vec &vec)
 {
   dest.StartArray();
   for (const auto& t : vec)

src/simplewallet/simplewallet.cpp

@@ -143,6 +143,7 @@ enum TransferType {
 };
 
 static std::string get_human_readable_timespan(std::chrono::seconds seconds);
+static std::string get_human_readable_timespan(uint64_t seconds);
 
 namespace
 {
@@ -5757,15 +5758,19 @@ bool simple_wallet::show_balance_unlocked(bool detailed)
   success_msg_writer() << tr("Currently selected account: [") << m_current_subaddress_account << tr("] ") << m_wallet->get_subaddress_label({m_current_subaddress_account, 0});
   const std::string tag = m_wallet->get_account_tags().second[m_current_subaddress_account];
   success_msg_writer() << tr("Tag: ") << (tag.empty() ? std::string{tr("(No tag assigned)")} : tag);
-  uint64_t blocks_to_unlock;
+  uint64_t blocks_to_unlock, time_to_unlock;
-  uint64_t unlocked_balance = m_wallet->unlocked_balance(m_current_subaddress_account, false, &blocks_to_unlock);
+  uint64_t unlocked_balance = m_wallet->unlocked_balance(m_current_subaddress_account, false, &blocks_to_unlock, &time_to_unlock);
   std::string unlock_time_message;
-  if (blocks_to_unlock > 0)
+  if (blocks_to_unlock > 0 && time_to_unlock > 0)
+    unlock_time_message = (boost::format(" (%lu block(s) and %s to unlock)") % blocks_to_unlock % get_human_readable_timespan(time_to_unlock)).str();
+  else if (blocks_to_unlock > 0)
     unlock_time_message = (boost::format(" (%lu block(s) to unlock)") % blocks_to_unlock).str();
+  else if (time_to_unlock > 0)
+    unlock_time_message = (boost::format(" (%s to unlock)") % get_human_readable_timespan(time_to_unlock)).str();
   success_msg_writer() << tr("Balance: ") << print_money(m_wallet->balance(m_current_subaddress_account, false)) << ", "
     << tr("unlocked balance: ") << print_money(unlocked_balance) << unlock_time_message << extra;
   std::map<uint32_t, uint64_t> balance_per_subaddress = m_wallet->balance_per_subaddress(m_current_subaddress_account, false);
-  std::map<uint32_t, std::pair<uint64_t, uint64_t>> unlocked_balance_per_subaddress = m_wallet->unlocked_balance_per_subaddress(m_current_subaddress_account, false);
+  std::map<uint32_t, std::pair<uint64_t, std::pair<uint64_t, uint64_t>>> unlocked_balance_per_subaddress = m_wallet->unlocked_balance_per_subaddress(m_current_subaddress_account, false);
   if (!detailed || balance_per_subaddress.empty())
     return true;
   success_msg_writer() << tr("Balance per address:");
@@ -8280,6 +8285,11 @@ static std::string get_human_readable_timespan(std::chrono::seconds seconds)
   return sw::tr("a long time");
 }
 //----------------------------------------------------------------------------------------------------
+static std::string get_human_readable_timespan(uint64_t seconds)
+{
+  return get_human_readable_timespan(std::chrono::seconds(seconds));
+}
+//----------------------------------------------------------------------------------------------------
 // mutates local_args as it parses and consumes arguments
 bool simple_wallet::get_transfers(std::vector<std::string>& local_args, std::vector<transfer_view>& transfers)
 {

src/wallet/message_store.cpp

@@ -724,7 +724,7 @@ void message_store::read_from_file(const multisig_wallet_state &state, const std
   {
     // Simply do nothing if the file is not there; allows e.g. easy recovery
     // from problems with the MMS by deleting the file
-    MERROR("No message store file found: " << filename);
+    MINFO("No message store file found: " << filename);
     return;
   }
 

src/wallet/wallet2.cpp

@@ -1523,6 +1523,18 @@ void wallet2::add_subaddress(uint32_t index_major, const std::string& label)
   m_subaddress_labels[index_major][index_minor] = label;
 }
 //----------------------------------------------------------------------------------------------------
+bool wallet2::should_expand(const cryptonote::subaddress_index &index) const
+{
+  const uint32_t last_major = m_subaddress_labels.size() - 1 > (std::numeric_limits<uint32_t>::max() - m_subaddress_lookahead_major) ? std::numeric_limits<uint32_t>::max()  : (m_subaddress_labels.size() + m_subaddress_lookahead_major - 1);
+  if (index.major > last_major)
+    return false;
+  const size_t nsub = index.major < m_subaddress_labels.size() ? m_subaddress_labels[index.major].size() : 0;
+  const uint32_t last_minor = nsub - 1 > (std::numeric_limits<uint32_t>::max() - m_subaddress_lookahead_minor) ? std::numeric_limits<uint32_t>::max()  : (nsub + m_subaddress_lookahead_minor - 1);
+  if (index.minor > last_minor)
+    return false;
+  return true;
+}
+//----------------------------------------------------------------------------------------------------
 void wallet2::expand_subaddresses(const cryptonote::subaddress_index& index)
 {
   hw::device &hwdev = m_account.get_device();
@@ -2110,7 +2122,7 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote
             td.m_amount = amount;
             td.m_pk_index = pk_index - 1;
             td.m_subaddr_index = tx_scan_info[o].received->index;
-            if (tx_scan_info[o].received->index.major < m_subaddress_labels.size() && tx_scan_info[o].received->index.minor < m_subaddress_labels[tx_scan_info[o].received->index.major].size())
+            if (should_expand(tx_scan_info[o].received->index))
               expand_subaddresses(tx_scan_info[o].received->index);
             if (tx.vout[o].amount == 0)
             {
@@ -2189,7 +2201,7 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote
             td.m_amount = amount;
             td.m_pk_index = pk_index - 1;
             td.m_subaddr_index = tx_scan_info[o].received->index;
-            if (tx_scan_info[o].received->index.major < m_subaddress_labels.size() && tx_scan_info[o].received->index.minor < m_subaddress_labels[tx_scan_info[o].received->index.major].size())
+            if (should_expand(tx_scan_info[o].received->index))
               expand_subaddresses(tx_scan_info[o].received->index);
             if (tx.vout[o].amount == 0)
             {
@@ -3980,13 +3992,7 @@ bool wallet2::load_keys(const std::string& keys_file_name, const epee::wipeable_
 
   // Load keys from buffer
   boost::optional<crypto::chacha_key> keys_to_encrypt;
-  try {
+  r = wallet2::load_keys_buf(keys_file_buf, password, keys_to_encrypt);
-    r = wallet2::load_keys_buf(keys_file_buf, password, keys_to_encrypt);
-  } catch (const std::exception& e) {
-    std::size_t found = string(e.what()).find("failed to deserialize keys buffer");
-    THROW_WALLET_EXCEPTION_IF(found != std::string::npos, error::wallet_internal_error, "internal error: failed to deserialize \"" + keys_file_name + '\"');
-    throw e;
-  }
 
   // Rewrite with encrypted keys if unencrypted, ignore errors
   if (r && keys_to_encrypt != boost::none)
@@ -4850,6 +4856,7 @@ std::string wallet2::make_multisig(const epee::wipeable_string &password,
   std::vector<crypto::secret_key> multisig_keys;
   rct::key spend_pkey = rct::identity();
   rct::key spend_skey;
+  auto wiper = epee::misc_utils::create_scope_leave_handler([&](){memwipe(&spend_skey, sizeof(spend_skey));});
   std::vector<crypto::public_key> multisig_signers;
 
   // decrypt keys
@@ -5495,13 +5502,12 @@ bool wallet2::check_connection(uint32_t *version, bool *ssl, uint32_t timeout)
     cryptonote::COMMAND_RPC_GET_VERSION::request req_t = AUTO_VAL_INIT(req_t);
     cryptonote::COMMAND_RPC_GET_VERSION::response resp_t = AUTO_VAL_INIT(resp_t);
     bool r = invoke_http_json_rpc("/json_rpc", "get_version", req_t, resp_t);
-    if(!r) {
+    if(!r || resp_t.status != CORE_RPC_STATUS_OK) {
       if(version)
         *version = 0;
       return false;
     }
-    if (resp_t.status == CORE_RPC_STATUS_OK)
+    m_rpc_version = resp_t.version;
-      m_rpc_version = resp_t.version;
   }
   if (version)
     *version = m_rpc_version;
@@ -5914,18 +5920,22 @@ uint64_t wallet2::balance(uint32_t index_major, bool strict) const
   return amount;
 }
 //----------------------------------------------------------------------------------------------------
-uint64_t wallet2::unlocked_balance(uint32_t index_major, bool strict, uint64_t *blocks_to_unlock) const
+uint64_t wallet2::unlocked_balance(uint32_t index_major, bool strict, uint64_t *blocks_to_unlock, uint64_t *time_to_unlock) const
 {
   uint64_t amount = 0;
   if (blocks_to_unlock)
     *blocks_to_unlock = 0;
+  if (time_to_unlock)
+    *time_to_unlock = 0;
   if(m_light_wallet)
     return m_light_wallet_balance;
   for (const auto& i : unlocked_balance_per_subaddress(index_major, strict))
   {
     amount += i.second.first;
-    if (blocks_to_unlock && i.second.second > *blocks_to_unlock)
+    if (blocks_to_unlock && i.second.second.first > *blocks_to_unlock)
-      *blocks_to_unlock = i.second.second;
+      *blocks_to_unlock = i.second.second.first;
+    if (time_to_unlock && i.second.second.second > *time_to_unlock)
+      *time_to_unlock = i.second.second.second;
   }
   return amount;
 }
@@ -5962,35 +5972,40 @@ std::map<uint32_t, uint64_t> wallet2::balance_per_subaddress(uint32_t index_majo
   return amount_per_subaddr;
 }
 //----------------------------------------------------------------------------------------------------
-std::map<uint32_t, std::pair<uint64_t, uint64_t>> wallet2::unlocked_balance_per_subaddress(uint32_t index_major, bool strict) const
+std::map<uint32_t, std::pair<uint64_t, std::pair<uint64_t, uint64_t>>> wallet2::unlocked_balance_per_subaddress(uint32_t index_major, bool strict) const
 {
-  std::map<uint32_t, std::pair<uint64_t, uint64_t>> amount_per_subaddr;
+  std::map<uint32_t, std::pair<uint64_t, std::pair<uint64_t, uint64_t>>> amount_per_subaddr;
   const uint64_t blockchain_height = get_blockchain_current_height();
+  const uint64_t now = time(NULL);
   for(const transfer_details& td: m_transfers)
   {
     if(td.m_subaddr_index.major == index_major && !is_spent(td, strict) && !td.m_frozen)
     {
-      uint64_t amount = 0, blocks_to_unlock = 0;
+      uint64_t amount = 0, blocks_to_unlock = 0, time_to_unlock = 0;
       if (is_transfer_unlocked(td))
       {
         amount = td.amount();
         blocks_to_unlock = 0;
+        time_to_unlock = 0;
       }
       else
       {
         uint64_t unlock_height = td.m_block_height + std::max<uint64_t>(CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE, CRYPTONOTE_LOCKED_TX_ALLOWED_DELTA_BLOCKS);
         if (td.m_tx.unlock_time < CRYPTONOTE_MAX_BLOCK_NUMBER && td.m_tx.unlock_time > unlock_height)
           unlock_height = td.m_tx.unlock_time;
+        uint64_t unlock_time = td.m_tx.unlock_time >= CRYPTONOTE_MAX_BLOCK_NUMBER ? td.m_tx.unlock_time : 0;
         blocks_to_unlock = unlock_height > blockchain_height ? unlock_height - blockchain_height : 0;
+        time_to_unlock = unlock_time > now ? unlock_time - now : 0;
         amount = 0;
       }
       auto found = amount_per_subaddr.find(td.m_subaddr_index.minor);
       if (found == amount_per_subaddr.end())
-        amount_per_subaddr[td.m_subaddr_index.minor] = std::make_pair(amount, blocks_to_unlock);
+        amount_per_subaddr[td.m_subaddr_index.minor] = std::make_pair(amount, std::make_pair(blocks_to_unlock, time_to_unlock));
       else
       {
         found->second.first += amount;
-        found->second.second = std::max(found->second.second, blocks_to_unlock);
+        found->second.second.first = std::max(found->second.second.first, blocks_to_unlock);
+        found->second.second.second = std::max(found->second.second.second, time_to_unlock);
       }
     }
   }
@@ -6005,17 +6020,21 @@ uint64_t wallet2::balance_all(bool strict) const
   return r;
 }
 //----------------------------------------------------------------------------------------------------
-uint64_t wallet2::unlocked_balance_all(bool strict, uint64_t *blocks_to_unlock) const
+uint64_t wallet2::unlocked_balance_all(bool strict, uint64_t *blocks_to_unlock, uint64_t *time_to_unlock) const
 {
   uint64_t r = 0;
   if (blocks_to_unlock)
     *blocks_to_unlock = 0;
+  if (time_to_unlock)
+    *time_to_unlock = 0;
   for (uint32_t index_major = 0; index_major < get_num_subaddress_accounts(); ++index_major)
   {
-    uint64_t local_blocks_to_unlock;
+    uint64_t local_blocks_to_unlock, local_time_to_unlock;
-    r += unlocked_balance(index_major, strict, blocks_to_unlock ? &local_blocks_to_unlock : NULL);
+    r += unlocked_balance(index_major, strict, blocks_to_unlock ? &local_blocks_to_unlock : NULL, time_to_unlock ? &local_time_to_unlock : NULL);
     if (blocks_to_unlock)
       *blocks_to_unlock = std::max(*blocks_to_unlock, local_blocks_to_unlock);
+    if (time_to_unlock)
+      *time_to_unlock = std::max(*time_to_unlock, local_time_to_unlock);
   }
   return r;
 }
@@ -6493,7 +6512,7 @@ void wallet2::commit_tx(pending_tx& ptx)
 
   // tx generated, get rid of used k values
   for (size_t idx: ptx.selected_transfers)
-    m_transfers[idx].m_multisig_k.clear();
+    memwipe(m_transfers[idx].m_multisig_k.data(), m_transfers[idx].m_multisig_k.size() * sizeof(m_transfers[idx].m_multisig_k[0]));
 
   //fee includes dust if dust policy specified it.
   LOG_PRINT_L1("Transaction successfully sent. <" << txid << ">" << ENDL
@@ -6935,13 +6954,13 @@ std::string wallet2::save_multisig_tx(multisig_tx_set txs)
   // txes generated, get rid of used k values
   for (size_t n = 0; n < txs.m_ptx.size(); ++n)
     for (size_t idx: txs.m_ptx[n].construction_data.selected_transfers)
-      m_transfers[idx].m_multisig_k.clear();
+      memwipe(m_transfers[idx].m_multisig_k.data(), m_transfers[idx].m_multisig_k.size() * sizeof(m_transfers[idx].m_multisig_k[0]));
 
   // zero out some data we don't want to share
   for (auto &ptx: txs.m_ptx)
   {
     for (auto &e: ptx.construction_data.sources)
-      e.multisig_kLRki.k = rct::zero();
+      memwipe(&e.multisig_kLRki.k, sizeof(e.multisig_kLRki.k));
   }
 
   for (auto &ptx: txs.m_ptx)
@@ -7149,10 +7168,12 @@ bool wallet2::sign_multisig_tx(multisig_tx_set &exported_txs, std::vector<crypto
         ptx.tx.rct_signatures = sig.sigs;
 
         rct::keyV k;
+        rct::key skey = rct::zero();
+        auto wiper = epee::misc_utils::create_scope_leave_handler([&](){ memwipe(k.data(), k.size() * sizeof(k[0])); memwipe(&skey, sizeof(skey)); });
+
         for (size_t idx: sd.selected_transfers)
           k.push_back(get_multisig_k(idx, sig.used_L));
 
-        rct::key skey = rct::zero();
         for (const auto &msk: get_account().get_multisig_keys())
         {
           crypto::public_key pmsk = get_multisig_signing_public_key(msk);
@@ -7200,7 +7221,7 @@ bool wallet2::sign_multisig_tx(multisig_tx_set &exported_txs, std::vector<crypto
   // txes generated, get rid of used k values
   for (size_t n = 0; n < exported_txs.m_ptx.size(); ++n)
     for (size_t idx: exported_txs.m_ptx[n].construction_data.selected_transfers)
-      m_transfers[idx].m_multisig_k.clear();
+      memwipe(m_transfers[idx].m_multisig_k.data(), m_transfers[idx].m_multisig_k.size() * sizeof(m_transfers[idx].m_multisig_k[0]));
 
   exported_txs.m_signers.insert(get_multisig_signer_public_key());
 
@@ -9652,7 +9673,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp
   // throw if attempting a transaction with no money
   THROW_WALLET_EXCEPTION_IF(needed_money == 0, error::zero_destination);
 
-  std::map<uint32_t, std::pair<uint64_t, uint64_t>> unlocked_balance_per_subaddr = unlocked_balance_per_subaddress(subaddr_account, false);
+  std::map<uint32_t, std::pair<uint64_t, std::pair<uint64_t, uint64_t>>> unlocked_balance_per_subaddr = unlocked_balance_per_subaddress(subaddr_account, false);
   std::map<uint32_t, uint64_t> balance_per_subaddr = balance_per_subaddress(subaddr_account, false);
 
   if (subaddr_indices.empty()) // "index=<N1>[,<N2>,...]" wasn't specified -> use all the indices with non-zero unlocked balance
@@ -12746,7 +12767,7 @@ process:
     const crypto::public_key& out_key = boost::get<cryptonote::txout_to_key>(td.m_tx.vout[td.m_internal_output_index].target).key;
     bool r = cryptonote::generate_key_image_helper(m_account.get_keys(), m_subaddresses, out_key, tx_pub_key, additional_tx_pub_keys, td.m_internal_output_index, in_ephemeral, td.m_key_image, m_account.get_device());
     THROW_WALLET_EXCEPTION_IF(!r, error::wallet_internal_error, "Failed to generate key image");
-    if (td.m_subaddr_index.major < m_subaddress_labels.size() && td.m_subaddr_index.minor < m_subaddress_labels[td.m_subaddr_index.major].size())
+    if (should_expand(td.m_subaddr_index))
       expand_subaddresses(td.m_subaddr_index);
     td.m_key_image_known = true;
     td.m_key_image_request = true;
@@ -12939,7 +12960,7 @@ cryptonote::blobdata wallet2::export_multisig()
   {
     transfer_details &td = m_transfers[n];
     crypto::key_image ki;
-    td.m_multisig_k.clear();
+    memwipe(td.m_multisig_k.data(), td.m_multisig_k.size() * sizeof(td.m_multisig_k[0]));
     info[n].m_LR.clear();
     info[n].m_partial_key_images.clear();
 
@@ -13048,6 +13069,7 @@ size_t wallet2::import_multisig(std::vector<cryptonote::blobdata> blobs)
   CHECK_AND_ASSERT_THROW_MES(info.size() + 1 <= m_multisig_signers.size() && info.size() + 1 >= m_multisig_threshold, "Wrong number of multisig sources");
 
   std::vector<std::vector<rct::key>> k;
+  auto wiper = epee::misc_utils::create_scope_leave_handler([&](){memwipe(k.data(), k.size() * sizeof(k[0]));});
   k.reserve(m_transfers.size());
   for (const auto &td: m_transfers)
     k.push_back(td.m_multisig_k);

src/wallet/wallet2.h

@@ -835,13 +835,13 @@ private:
 
     // locked & unlocked balance of given or current subaddress account
     uint64_t balance(uint32_t subaddr_index_major, bool strict) const;
-    uint64_t unlocked_balance(uint32_t subaddr_index_major, bool strict, uint64_t *blocks_to_unlock = NULL) const;
+    uint64_t unlocked_balance(uint32_t subaddr_index_major, bool strict, uint64_t *blocks_to_unlock = NULL, uint64_t *time_to_unlock = NULL) const;
     // locked & unlocked balance per subaddress of given or current subaddress account
     std::map<uint32_t, uint64_t> balance_per_subaddress(uint32_t subaddr_index_major, bool strict) const;
-    std::map<uint32_t, std::pair<uint64_t, uint64_t>> unlocked_balance_per_subaddress(uint32_t subaddr_index_major, bool strict) const;
+    std::map<uint32_t, std::pair<uint64_t, std::pair<uint64_t, uint64_t>>> unlocked_balance_per_subaddress(uint32_t subaddr_index_major, bool strict) const;
     // all locked & unlocked balances of all subaddress accounts
     uint64_t balance_all(bool strict) const;
-    uint64_t unlocked_balance_all(bool strict, uint64_t *blocks_to_unlock = NULL) const;
+    uint64_t unlocked_balance_all(bool strict, uint64_t *blocks_to_unlock = NULL, uint64_t *time_to_unlock = NULL) const;
     template<typename T>
     void transfer_selected(const std::vector<cryptonote::tx_destination_entry>& dsts, const std::vector<size_t>& selected_transfers, size_t fake_outputs_count,
       std::vector<std::vector<tools::wallet2::get_outs_entry>> &outs,
@@ -1516,6 +1516,8 @@ private:
     std::string get_client_signature() const;
     void check_rpc_cost(const char *call, uint64_t post_call_credits, uint64_t pre_credits, double expected_cost);
 
+    bool should_expand(const cryptonote::subaddress_index &index) const;
+
     cryptonote::account_base m_account;
     boost::optional<epee::net_utils::http::login> m_daemon_login;
     std::string m_daemon_address;

src/wallet/wallet_rpc_server.cpp

@@ -428,10 +428,10 @@ namespace tools
     try
     {
       res.balance = req.all_accounts ? m_wallet->balance_all(req.strict) : m_wallet->balance(req.account_index, req.strict);
-      res.unlocked_balance = req.all_accounts ? m_wallet->unlocked_balance_all(req.strict, &res.blocks_to_unlock) : m_wallet->unlocked_balance(req.account_index, req.strict, &res.blocks_to_unlock);
+      res.unlocked_balance = req.all_accounts ? m_wallet->unlocked_balance_all(req.strict, &res.blocks_to_unlock, &res.time_to_unlock) : m_wallet->unlocked_balance(req.account_index, req.strict, &res.blocks_to_unlock, &res.time_to_unlock);
       res.multisig_import_needed = m_wallet->multisig() && m_wallet->has_multisig_partial_key_images();
       std::map<uint32_t, std::map<uint32_t, uint64_t>> balance_per_subaddress_per_account;
-      std::map<uint32_t, std::map<uint32_t, std::pair<uint64_t, uint64_t>>> unlocked_balance_per_subaddress_per_account;
+      std::map<uint32_t, std::map<uint32_t, std::pair<uint64_t, std::pair<uint64_t, uint64_t>>>> unlocked_balance_per_subaddress_per_account;
       if (req.all_accounts)
       {
         for (uint32_t account_index = 0; account_index < m_wallet->get_num_subaddress_accounts(); ++account_index)
@@ -451,7 +451,7 @@ namespace tools
       {
         uint32_t account_index = p.first;
         std::map<uint32_t, uint64_t> balance_per_subaddress = p.second;
-        std::map<uint32_t, std::pair<uint64_t, uint64_t>> unlocked_balance_per_subaddress = unlocked_balance_per_subaddress_per_account[account_index];
+        std::map<uint32_t, std::pair<uint64_t, std::pair<uint64_t, uint64_t>>> unlocked_balance_per_subaddress = unlocked_balance_per_subaddress_per_account[account_index];
         std::set<uint32_t> address_indices;
         if (!req.all_accounts && !req.address_indices.empty())
         {
@@ -471,7 +471,8 @@ namespace tools
           info.address = m_wallet->get_subaddress_as_str(index);
           info.balance = balance_per_subaddress[i];
           info.unlocked_balance = unlocked_balance_per_subaddress[i].first;
-          info.blocks_to_unlock = unlocked_balance_per_subaddress[i].second;
+          info.blocks_to_unlock = unlocked_balance_per_subaddress[i].second.first;
+          info.time_to_unlock = unlocked_balance_per_subaddress[i].second.second;
           info.label = m_wallet->get_subaddress_label(index);
           info.num_unspent_outputs = std::count_if(transfers.begin(), transfers.end(), [&](const tools::wallet2::transfer_details& td) { return !td.m_spent && td.m_subaddr_index == index; });
           res.per_subaddress.emplace_back(std::move(info));

src/wallet/wallet_rpc_server_commands_defs.h

@@ -47,7 +47,7 @@
 // advance which version they will stop working with
 // Don't go over 32767 for any of these
 #define WALLET_RPC_VERSION_MAJOR 1
-#define WALLET_RPC_VERSION_MINOR 17
+#define WALLET_RPC_VERSION_MINOR 18
 #define MAKE_WALLET_RPC_VERSION(major,minor) (((major)<<16)|(minor))
 #define WALLET_RPC_VERSION MAKE_WALLET_RPC_VERSION(WALLET_RPC_VERSION_MAJOR, WALLET_RPC_VERSION_MINOR)
 namespace tools
@@ -84,6 +84,7 @@ namespace wallet_rpc
       std::string label;
       uint64_t num_unspent_outputs;
       uint64_t blocks_to_unlock;
+      uint64_t time_to_unlock;
 
       BEGIN_KV_SERIALIZE_MAP()
         KV_SERIALIZE(account_index)
@@ -94,6 +95,7 @@ namespace wallet_rpc
         KV_SERIALIZE(label)
         KV_SERIALIZE(num_unspent_outputs)
         KV_SERIALIZE(blocks_to_unlock)
+        KV_SERIALIZE(time_to_unlock)
       END_KV_SERIALIZE_MAP()
     };
 
@@ -104,6 +106,7 @@ namespace wallet_rpc
       bool       multisig_import_needed;
       std::vector<per_subaddress_info> per_subaddress;
       uint64_t   blocks_to_unlock;
+      uint64_t   time_to_unlock;
 
       BEGIN_KV_SERIALIZE_MAP()
         KV_SERIALIZE(balance)
@@ -111,6 +114,7 @@ namespace wallet_rpc
         KV_SERIALIZE(multisig_import_needed)
         KV_SERIALIZE(per_subaddress)
         KV_SERIALIZE(blocks_to_unlock)
+        KV_SERIALIZE(time_to_unlock)
       END_KV_SERIALIZE_MAP()
     };
     typedef epee::misc_utils::struct_init<response_t> response;

tests/functional_tests/functional_tests_rpc.py

@@ -34,8 +34,8 @@ try:
 except:
   tests = DEFAULT_TESTS
 
-N_MONERODS = 3
+N_MONERODS = 2
-N_WALLETS = 7
+N_WALLETS = 4
 WALLET_DIRECTORY = builddir + "/functional-tests-directory"
 DIFFICULTY = 10
 
@@ -43,17 +43,9 @@ monerod_base = [builddir + "/bin/monerod", "--regtest", "--fixed-difficulty", st
 monerod_extra = [
   [],
   ["--rpc-payment-address", "44SKxxLQw929wRF6BA9paQ1EWFshNnKhXM3qz6Mo3JGDE2YG3xyzVutMStEicxbQGRfrYvAAYxH6Fe8rnD56EaNwUiqhcwR", "--rpc-payment-difficulty", str(DIFFICULTY), "--rpc-payment-credits", "5000", "--data-dir", builddir + "/functional-tests-directory/monerod1"],
-  ["--rpc-restricted-bind-port", "18482", "--data-dir", builddir + "/functional-tests-directory/monerod2"]
 ]
-wallet_base = [builddir + "/bin/monero-wallet-rpc", "--wallet-dir", WALLET_DIRECTORY, "--rpc-bind-port", "wallet_port", "--disable-rpc-login", "--rpc-ssl", "disabled", "--daemon-ssl", "disabled", "--log-level", "1"]
+wallet_base = [builddir + "/bin/monero-wallet-rpc", "--wallet-dir", WALLET_DIRECTORY, "--rpc-bind-port", "wallet_port", "--disable-rpc-login", "--rpc-ssl", "disabled", "--daemon-ssl", "disabled", "--daemon-port", "18180", "--log-level", "1"]
 wallet_extra = [
-  ["--daemon-port", "18180"],
-  ["--daemon-port", "18180"],
-  ["--daemon-port", "18180"],
-  ["--daemon-port", "18180"],
-  ["--daemon-port", "18182"],
-  ["--daemon-port", "18182"],
-  ["--daemon-port", "18182"]
 ]
 
 command_lines = []

tests/functional_tests/transfer.py

@@ -55,7 +55,7 @@ class TransferTest():
 
     def reset(self):
         print('Resetting blockchain')
-        daemon = Daemon(idx = 2)
+        daemon = Daemon()
         res = daemon.get_height()
         daemon.pop_blocks(res.height - 1)
         daemon.flush_txpool()
@@ -69,7 +69,7 @@ class TransferTest():
         ]
         self.wallet = [None] * len(seeds)
         for i in range(len(seeds)):
-            self.wallet[i] = Wallet(idx = i + 4)
+            self.wallet[i] = Wallet(idx = i)
             # close the wallet if any, will throw if none is loaded
             try: self.wallet[i].close_wallet()
             except: pass
@@ -77,7 +77,7 @@ class TransferTest():
 
     def mine(self):
         print("Mining some blocks")
-        daemon = Daemon(idx = 2)
+        daemon = Daemon()
 
         res = daemon.get_info()
         height = res.height
@@ -89,7 +89,7 @@ class TransferTest():
             assert res.height == height + 80
 
     def transfer(self):
-        daemon = Daemon(idx = 2)
+        daemon = Daemon()
 
         print("Creating transfer to self")
 
@@ -508,7 +508,7 @@ class TransferTest():
     def check_get_bulk_payments(self):
         print('Checking get_bulk_payments')
 
-        daemon = Daemon(idx = 2)
+        daemon = Daemon()
         res = daemon.get_info()
         height = res.height
 
@@ -544,7 +544,7 @@ class TransferTest():
     def check_get_payments(self):
         print('Checking get_payments')
 
-        daemon = Daemon(idx = 2)
+        daemon = Daemon()
         res = daemon.get_info()
         height = res.height
 
@@ -587,8 +587,7 @@ class TransferTest():
             assert len(res.tx_blob_list) == 1
             txes[i][1] = res.tx_blob_list[0]
 
-        daemon = Daemon(idx = 2)
+        daemon = Daemon()
-        restricted_daemon = Daemon(idx = 2, restricted_rpc = True)
         res = daemon.send_raw_transaction(txes[0][1])
         assert res.not_relayed == False
         assert res.low_mixin == False
@@ -599,18 +598,6 @@ class TransferTest():
         assert res.overspend == False
         assert res.fee_too_low == False
 
-        res = restricted_daemon.send_raw_transaction(txes[0][1])
-        assert res.not_relayed == False
-        assert res.low_mixin == False
-        assert res.double_spend == False
-        assert res.invalid_input == False
-        assert res.invalid_output == False
-        assert res.too_big == False
-        assert res.overspend == False
-        assert res.fee_too_low == False
-
-        res = restricted_daemon.get_transactions([txes[0][0]])
-        assert not 'txs' in res or len(res.txs) == 0
         res = daemon.get_transactions([txes[0][0]])
         assert len(res.txs) >= 1
         tx = [tx for tx in res.txs if tx.tx_hash == txes[0][0]][0]
@@ -628,19 +615,6 @@ class TransferTest():
         assert res.fee_too_low == False
         assert res.too_few_outputs == False
 
-        res = restricted_daemon.send_raw_transaction(txes[1][1])
-        assert res.not_relayed == False
-        assert res.low_mixin == False
-        assert res.double_spend == True
-        assert res.invalid_input == False
-        assert res.invalid_output == False
-        assert res.too_big == False
-        assert res.overspend == False
-        assert res.fee_too_low == False
-        assert res.too_few_outputs == False
-
-        res = restricted_daemon.get_transactions([txes[0][0]])
-        assert not 'txs' in res or len(res.txs) == 0
         res = daemon.get_transactions([txes[0][0]])
         assert len(res.txs) >= 1
         tx = [tx for tx in res.txs if tx.tx_hash == txes[0][0]][0]
@@ -649,13 +623,13 @@ class TransferTest():
 
     def sweep_dust(self):
         print("Sweeping dust")
-        daemon = Daemon(idx = 2)
+        daemon = Daemon()
         self.wallet[0].refresh()
         res = self.wallet[0].sweep_dust()
         assert not 'tx_hash_list' in res or len(res.tx_hash_list) == 0 # there's just one, but it cannot meet the fee
 
     def sweep_single(self):
-        daemon = Daemon(idx = 2)
+        daemon = Daemon()
 
         print("Sending single output")
 
@@ -711,7 +685,7 @@ class TransferTest():
         assert len([t for t in res.transfers if t.key_image == ki]) == 1
 
     def check_destinations(self):
-        daemon = Daemon(idx = 2)
+        daemon = Daemon()
 
         print("Checking transaction destinations")
 
@@ -767,7 +741,7 @@ class TransferTest():
                 self.wallet[0].refresh()
 
     def check_tx_notes(self):
-        daemon = Daemon(idx = 2)
+        daemon = Daemon()
 
         print('Testing tx notes')
         res = self.wallet[0].get_transfers()
@@ -784,7 +758,7 @@ class TransferTest():
         assert res.notes == ['out txid', 'in txid']
 
     def check_rescan(self):
-        daemon = Daemon(idx = 2)
+        daemon = Daemon()
 
         print('Testing rescan_spent')
         res = self.wallet[0].incoming_transfers(transfer_type = 'all')
@@ -824,7 +798,7 @@ class TransferTest():
             assert sorted(old_t_out, key = lambda k: k['txid']) == sorted(new_t_out, key = lambda k: k['txid'])
 
     def check_is_key_image_spent(self):
-        daemon = Daemon(idx = 2)
+        daemon = Daemon()
 
         print('Testing is_key_image_spent')
         res = self.wallet[0].incoming_transfers(transfer_type = 'all')

tests/functional_tests/txpool.py

@@ -45,14 +45,14 @@ class TransferTest():
 
     def reset(self):
         print('Resetting blockchain')
-        daemon = Daemon(idx=2)
+        daemon = Daemon()
         res = daemon.get_height()
         daemon.pop_blocks(res.height - 1)
         daemon.flush_txpool()
 
     def create(self):
         print('Creating wallet')
-        wallet = Wallet(idx = 4)
+        wallet = Wallet()
         # close the wallet if any, will throw if none is loaded
         try: wallet.close_wallet()
         except: pass
@@ -61,8 +61,8 @@ class TransferTest():
 
     def mine(self):
         print("Mining some blocks")
-        daemon = Daemon(idx = 2)
+        daemon = Daemon()
-        wallet = Wallet(idx = 4)
+        wallet = Wallet()
 
         daemon.generateblocks('42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm', 80)
         wallet.refresh()
@@ -70,8 +70,8 @@ class TransferTest():
     def create_txes(self, address, ntxes):
         print('Creating ' + str(ntxes) + ' transactions')
 
-        daemon = Daemon(idx = 2)
+        daemon = Daemon()
-        wallet = Wallet(idx = 4)
+        wallet = Wallet()
 
         dst = {'address': address, 'amount': 1000000000000}
 
@@ -83,10 +83,8 @@ class TransferTest():
         return txes
 
     def check_empty_pool(self):
-        self.check_empty_rpc_pool(Daemon(idx = 2))
+        daemon = Daemon()
-        self.check_empty_rpc_pool(Daemon(idx = 2, restricted_rpc = True))
 
-    def check_empty_rpc_pool(self, daemon):
         res = daemon.get_transaction_pool_hashes()
         assert not 'tx_hashes' in res or len(res.tx_hashes) == 0
         res = daemon.get_transaction_pool_stats()
@@ -105,9 +103,8 @@ class TransferTest():
         assert res.pool_stats.num_double_spends == 0
 
     def check_txpool(self):
-        daemon = Daemon(idx = 2)
+        daemon = Daemon()
-        restricted_daemon = Daemon(idx = 2, restricted_rpc = True)
+        wallet = Wallet()
-        wallet = Wallet(idx = 4)
 
         res = daemon.get_info()
         height = res.height
@@ -120,7 +117,6 @@ class TransferTest():
         res = daemon.get_info()
         assert res.tx_pool_size == txpool_size + 5
         txpool_size = res.tx_pool_size
-        self.check_empty_rpc_pool(restricted_daemon)
 
         res = daemon.get_transaction_pool()
         assert len(res.transactions) == txpool_size
@@ -164,7 +160,6 @@ class TransferTest():
         print('Flushing 2 transactions')
         txes_keys = list(txes.keys())
         daemon.flush_txpool([txes_keys[1], txes_keys[3]])
-        self.check_empty_rpc_pool(restricted_daemon)
         res = daemon.get_transaction_pool()
         assert len(res.transactions) == txpool_size - 2
         assert len([x for x in res.transactions if x.id_hash == txes_keys[1]]) == 0
@@ -215,7 +210,6 @@ class TransferTest():
         print('Flushing unknown transactions')
         unknown_txids = ['1'*64, '2'*64, '3'*64]
         daemon.flush_txpool(unknown_txids)
-        self.check_empty_rpc_pool(restricted_daemon)
         res = daemon.get_transaction_pool()
         assert len(res.transactions) == txpool_size - 2
 

tests/unit_tests/epee_utils.cpp

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2019, The Monero Project
+// Copyright (c) 2014-2020, The Monero Project
 //
 // All rights reserved.
 //
@@ -45,6 +45,7 @@
 #include "boost/archive/portable_binary_iarchive.hpp"
 #include "boost/archive/portable_binary_oarchive.hpp"
 #include "byte_slice.h"
+#include "byte_stream.h"
 #include "crypto/crypto.h"
 #include "hex.h"
 #include "net/net_utils_base.h"
@@ -809,6 +810,259 @@ TEST(ByteSlice, GetSlice)
   EXPECT_TRUE(boost::range::equal(base_string, original));
 }
 
+TEST(ByteStream, Construction)
+{
+  EXPECT_TRUE(std::is_default_constructible<epee::byte_stream>());
+  EXPECT_TRUE(std::is_move_constructible<epee::byte_stream>());
+  EXPECT_FALSE(std::is_copy_constructible<epee::byte_stream>());
+  EXPECT_TRUE(std::is_move_assignable<epee::byte_stream>());
+  EXPECT_FALSE(std::is_copy_assignable<epee::byte_stream>());
+}
+
+TEST(ByteStream, Noexcept)
+{
+  EXPECT_TRUE(std::is_nothrow_default_constructible<epee::byte_stream>());
+  EXPECT_TRUE(std::is_nothrow_move_constructible<epee::byte_stream>());
+  EXPECT_TRUE(std::is_nothrow_move_assignable<epee::byte_stream>());
+
+  epee::byte_stream lvalue;
+  const epee::byte_stream clvalue;
+
+  EXPECT_TRUE(noexcept(lvalue.data()));
+  EXPECT_TRUE(noexcept(clvalue.data()));
+  EXPECT_TRUE(noexcept(lvalue.tellp()));
+  EXPECT_TRUE(noexcept(clvalue.tellp()));
+  EXPECT_TRUE(noexcept(lvalue.available()));
+  EXPECT_TRUE(noexcept(clvalue.available()));
+  EXPECT_TRUE(noexcept(lvalue.size()));
+  EXPECT_TRUE(noexcept(clvalue.size()));
+  EXPECT_TRUE(noexcept(lvalue.capacity()));
+  EXPECT_TRUE(noexcept(clvalue.capacity()));
+  EXPECT_TRUE(noexcept(lvalue.put_unsafe(4)));
+  EXPECT_TRUE(noexcept(lvalue.take_buffer()));
+}
+
+TEST(ByteStream, Empty)
+{
+  epee::byte_stream stream;
+
+  EXPECT_EQ(epee::byte_stream::default_increase(), stream.increase_size());
+
+  EXPECT_EQ(nullptr, stream.data());
+  EXPECT_EQ(nullptr, stream.tellp());
+  EXPECT_EQ(0u, stream.available());
+  EXPECT_EQ(0u, stream.size());
+  EXPECT_EQ(0u, stream.capacity());
+
+  const auto buf = stream.take_buffer();
+  EXPECT_EQ(nullptr, buf.get());
+  EXPECT_EQ(nullptr, stream.data());
+  EXPECT_EQ(nullptr, stream.tellp());
+  EXPECT_EQ(0u, stream.available());
+  EXPECT_EQ(0u, stream.size());
+  EXPECT_EQ(0u, stream.capacity());
+}
+
+TEST(ByteStream, Write)
+{
+  using boost::range::equal;
+  using byte_span = epee::span<const std::uint8_t>;
+
+  static constexpr const std::uint8_t source[] =
+    {0xde, 0xad, 0xbe, 0xef, 0xef};
+
+  std::vector<std::uint8_t> bytes;
+  epee::byte_stream stream{4};
+
+  EXPECT_EQ(4u, stream.increase_size());
+
+  stream.write({source, 3});
+  bytes.insert(bytes.end(), source, source + 3);
+  EXPECT_EQ(3u, stream.size());
+  EXPECT_EQ(1u, stream.available());
+  EXPECT_EQ(4u, stream.capacity());
+  EXPECT_TRUE(equal(bytes, byte_span{stream.data(), stream.size()}));
+
+  stream.write({source, 2});
+  bytes.insert(bytes.end(), source, source + 2);
+  EXPECT_EQ(5u, stream.size());
+  EXPECT_EQ(3u, stream.available());
+  EXPECT_EQ(8u, stream.capacity());
+  EXPECT_TRUE(equal(bytes, byte_span{stream.data(), stream.size()}));
+
+  stream.write({source, 5});
+  bytes.insert(bytes.end(), source, source + 5);
+  EXPECT_EQ(10u, stream.size());
+  EXPECT_EQ(2u, stream.available());
+  EXPECT_EQ(12u, stream.capacity());
+  EXPECT_TRUE(equal(bytes, byte_span{stream.data(), stream.size()}));
+
+  stream.write({source, 2});
+  bytes.insert(bytes.end(), source, source + 2);
+  EXPECT_EQ(12u, stream.size());
+  EXPECT_EQ(0u, stream.available());
+  EXPECT_EQ(12u, stream.capacity());
+  EXPECT_TRUE(equal(bytes, byte_span{stream.data(), stream.size()}));
+
+  stream.write({source, 5});
+  bytes.insert(bytes.end(), source, source + 5);
+  EXPECT_EQ(17u, stream.size());
+  EXPECT_EQ(0u, stream.available());
+  EXPECT_EQ(17u, stream.capacity());
+  EXPECT_TRUE(equal(bytes, byte_span{stream.data(), stream.size()}));
+}
+
+TEST(ByteStream, Put)
+{
+  using boost::range::equal;
+  using byte_span = epee::span<const std::uint8_t>;
+
+  std::vector<std::uint8_t> bytes;
+  epee::byte_stream stream;
+
+  for (std::uint8_t i = 0; i < 200; ++i)
+  {
+    bytes.push_back(i);
+    stream.put(i);
+  }
+
+  EXPECT_EQ(200u, stream.size());
+  EXPECT_EQ(epee::byte_stream::default_increase() - 200, stream.available());
+  EXPECT_EQ(epee::byte_stream::default_increase(), stream.capacity());
+  EXPECT_TRUE(equal(bytes, byte_span{stream.data(), stream.size()}));
+}
+
+TEST(ByteStream, Reserve)
+{
+  using boost::range::equal;
+  using byte_span = epee::span<const std::uint8_t>;
+
+  static constexpr const std::uint8_t source[] =
+    {0xde, 0xad, 0xbe, 0xef, 0xef};
+
+  std::vector<std::uint8_t> bytes;
+  epee::byte_stream stream{4};
+
+  EXPECT_EQ(4u, stream.increase_size());
+
+  stream.reserve(100);
+  EXPECT_EQ(100u, stream.capacity());
+  EXPECT_EQ(0u, stream.size());
+  EXPECT_EQ(100u, stream.available());
+
+  for (std::size_t i = 0; i < 100 / sizeof(source); ++i)
+  {
+    stream.write(source);
+    bytes.insert(bytes.end(), source, source + sizeof(source));
+  }
+
+  EXPECT_EQ(100u, stream.size());
+  EXPECT_EQ(0u, stream.available());
+  EXPECT_EQ(100u, stream.capacity());
+  EXPECT_TRUE(equal(bytes, byte_span{stream.data(), stream.size()}));
+}
+
+TEST(ByteStream, TakeBuffer)
+{
+  using boost::range::equal;
+  using byte_span = epee::span<const std::uint8_t>;
+
+  static constexpr const std::uint8_t source[] =
+    {0xde, 0xad, 0xbe, 0xef, 0xef};
+
+  epee::byte_stream stream;
+
+  stream.write(source);
+  ASSERT_EQ(sizeof(source), stream.size());
+  EXPECT_TRUE(equal(source, byte_span{stream.data(), stream.size()}));
+
+  const auto buffer = stream.take_buffer();
+  EXPECT_EQ(0u, stream.size());
+  EXPECT_EQ(0u, stream.available());
+  EXPECT_EQ(0u, stream.capacity());
+  EXPECT_EQ(nullptr, stream.data());
+  EXPECT_EQ(nullptr, stream.tellp());
+  EXPECT_TRUE(equal(source, byte_span{buffer.get(), sizeof(source)}));
+}
+
+TEST(ByteStream, Move)
+{
+  using boost::range::equal;
+  using byte_span = epee::span<const std::uint8_t>;
+
+  static constexpr const std::uint8_t source[] =
+    {0xde, 0xad, 0xbe, 0xef, 0xef};
+
+  epee::byte_stream stream{10};
+  stream.write(source);
+
+  epee::byte_stream stream2{std::move(stream)};
+
+  EXPECT_EQ(10u, stream.increase_size());
+  EXPECT_EQ(0u, stream.size());
+  EXPECT_EQ(0u, stream.available());
+  EXPECT_EQ(0u, stream.capacity());
+  EXPECT_EQ(nullptr, stream.data());
+  EXPECT_EQ(nullptr, stream.tellp());
+
+  EXPECT_EQ(10u, stream2.increase_size());
+  EXPECT_EQ(5u, stream2.size());
+  EXPECT_EQ(5u, stream2.available());
+  EXPECT_EQ(10u, stream2.capacity());
+  EXPECT_NE(nullptr, stream2.data());
+  EXPECT_NE(nullptr, stream2.tellp());
+  EXPECT_TRUE(equal(source, byte_span{stream2.data(), stream2.size()}));
+
+  stream = epee::byte_stream{};
+
+  EXPECT_EQ(epee::byte_stream::default_increase(), stream.increase_size());
+  EXPECT_EQ(0u, stream.size());
+  EXPECT_EQ(0u, stream.available());
+  EXPECT_EQ(0u, stream.capacity());
+  EXPECT_EQ(nullptr, stream.data());
+  EXPECT_EQ(nullptr, stream.tellp());
+
+  stream = std::move(stream2);
+
+  EXPECT_EQ(10u, stream.increase_size());
+  EXPECT_EQ(5u, stream.size());
+  EXPECT_EQ(5u, stream.available());
+  EXPECT_EQ(10u, stream.capacity());
+  EXPECT_NE(nullptr, stream.data());
+  EXPECT_NE(nullptr, stream.tellp());
+  EXPECT_TRUE(equal(source, byte_span{stream.data(), stream.size()}));
+
+  EXPECT_EQ(10u, stream2.increase_size());
+  EXPECT_EQ(0u, stream2.size());
+  EXPECT_EQ(0u, stream2.available());
+  EXPECT_EQ(0u, stream2.capacity());
+  EXPECT_EQ(nullptr, stream2.data());
+  EXPECT_EQ(nullptr, stream2.tellp());
+}
+
+TEST(ByteStream, ToByteSlice)
+{
+  using boost::range::equal;
+  using byte_span = epee::span<const std::uint8_t>;
+
+  static constexpr const std::uint8_t source[] =
+    {0xde, 0xad, 0xbe, 0xef, 0xef};
+
+  epee::byte_stream stream;
+
+  stream.write(source);
+  EXPECT_EQ(sizeof(source), stream.size());
+  EXPECT_TRUE(equal(source, byte_span{stream.data(), stream.size()}));
+
+  const epee::byte_slice slice{std::move(stream)};
+  EXPECT_EQ(0u, stream.size());
+  EXPECT_EQ(0u, stream.available());
+  EXPECT_EQ(0u, stream.capacity());
+  EXPECT_EQ(nullptr, stream.data());
+  EXPECT_EQ(nullptr, stream.tellp());
+  EXPECT_TRUE(equal(source, slice));
+}
+
 TEST(ToHex, String)
 {
   EXPECT_TRUE(epee::to_hex::string(nullptr).empty());

tests/unit_tests/json_serialization.cpp

@@ -3,10 +3,10 @@
 #include <boost/range/adaptor/indexed.hpp>
 #include <gtest/gtest.h>
 #include <rapidjson/document.h>
-#include <rapidjson/stringbuffer.h>
 #include <rapidjson/writer.h>
 #include <vector>
 
+#include "byte_stream.h"
 #include "crypto/hash.h"
 #include "cryptonote_basic/account.h"
 #include "cryptonote_basic/cryptonote_basic.h"
@@ -86,14 +86,14 @@ namespace
     template<typename T>
     T test_json(const T& value)
     {
-      rapidjson::StringBuffer buffer;
+      epee::byte_stream buffer;
       {
-        rapidjson::Writer<rapidjson::StringBuffer> dest{buffer};
+        rapidjson::Writer<epee::byte_stream> dest{buffer};
         cryptonote::json::toJsonValue(dest, value);
       }
 
       rapidjson::Document doc;
-      doc.Parse(buffer.GetString());
+      doc.Parse(reinterpret_cast<const char*>(buffer.data()), buffer.size());
       if (doc.HasParseError() || !doc.IsObject())
       {
         throw cryptonote::json::PARSE_FAIL();