Merge pull request #6601

98c151ecb Optimize ZMQ-JSON vector reading; GetBlocksFast reads 24%+ faster (Lee Clagett)
60627c9f2 Switch to insitu parsing for ZMQ-JSON; GetBlocksFast reads 13%+ faster (Lee Clagett)
fe96e66eb Fix pruned tx for ZMQ's GetBlocksFast (Lee Clagett)

src/rpc/daemon_handler.cpp

@@ -182,6 +182,7 @@ namespace rpc
       for (const auto& blob : it->second)
       {
         bwt.transactions.emplace_back();
+        bwt.transactions.back().pruned = req.prune;
         if (!parse_and_validate_tx_from_blob(blob.second, bwt.transactions.back()))
         {
           res.blocks.clear();
@@ -905,13 +906,13 @@ namespace rpc
     return true;
   }
 
-  epee::byte_slice DaemonHandler::handle(const std::string& request)
+  epee::byte_slice DaemonHandler::handle(std::string&& request)
   {
     MDEBUG("Handling RPC request: " << request);
 
     try
     {
-      FullMessage req_full(request, true);
+      FullMessage req_full(std::move(request), true);
 
       const std::string request_type = req_full.getRequestType();
       const auto matched_handler = std::lower_bound(std::begin(handlers), std::end(handlers), request_type);

src/rpc/daemon_handler.h

@@ -133,7 +133,7 @@ class DaemonHandler : public RpcHandler
 
     void handle(const GetOutputDistribution::Request& req, GetOutputDistribution::Response& res);
 
-    epee::byte_slice handle(const std::string& request) override final;
+    epee::byte_slice handle(std::string&& request) override final;
 
   private:
 

src/rpc/message.cpp

@@ -79,9 +79,12 @@ void Message::fromJson(const rapidjson::Value& val)
   GET_FROM_JSON_OBJECT(val, rpc_version, rpc_version);
 }
 
-FullMessage::FullMessage(const std::string& json_string, bool request)
+FullMessage::FullMessage(std::string&& json_string, bool request)
+  : contents(std::move(json_string)), doc()
 {
-  doc.Parse(json_string.c_str());
+  /* Insitu parsing does not copy data from `contents` to DOM,
+     accelerating string heavy content. */
+  doc.ParseInsitu(std::addressof(contents[0]));
   if (doc.HasParseError() || !doc.IsObject())
   {
     throw cryptonote::json::PARSE_FAIL();

src/rpc/message.h

@@ -72,9 +72,7 @@ namespace rpc
     public:
       ~FullMessage() { }
 
-      FullMessage(FullMessage&& rhs) noexcept : doc(std::move(rhs.doc)) { }
+      FullMessage(std::string&& json_string, bool request=false);
-
-      FullMessage(const std::string& json_string, bool request=false);
 
       std::string getRequestType() const;
 
@@ -91,10 +89,13 @@ namespace rpc
     private:
 
       FullMessage() = default;
+      FullMessage(const FullMessage&) = delete;
+      FullMessage& operator=(const FullMessage&) = delete;
 
       FullMessage(const std::string& request, Message* message);
       FullMessage(Message* message);
 
+      std::string contents;
       rapidjson::Document doc;
   };
 

src/rpc/rpc_handler.h

@@ -55,7 +55,7 @@ class RpcHandler
     RpcHandler() { }
     virtual ~RpcHandler() { }
 
-    virtual epee::byte_slice handle(const std::string& request) = 0;
+    virtual epee::byte_slice handle(std::string&& request) = 0;
 
     static boost::optional<output_distribution_data>
       get_output_distribution(const std::function<bool(uint64_t, uint64_t, uint64_t, uint64_t&, std::vector<uint64_t>&, uint64_t&)> &f, uint64_t amount, uint64_t from_height, uint64_t to_height, const std::function<crypto::hash(uint64_t)> &get_hash, bool cumulative, uint64_t blockchain_height);

src/rpc/zmq_server.cpp

@@ -158,9 +158,9 @@ void ZmqServer::serve()
 
       if (!pub || sockets[2].revents)
       {
-        const std::string message = MONERO_UNWRAP(net::zmq::receive(rep.get(), read_flags));
+        std::string message = MONERO_UNWRAP(net::zmq::receive(rep.get(), read_flags));
         MDEBUG("Received RPC request: \"" << message << "\"");
-        epee::byte_slice response = handler.handle(message);
+        epee::byte_slice response = handler.handle(std::move(message));
 
         const boost::string_ref response_view{reinterpret_cast<const char*>(response.data()), response.size()};
         MDEBUG("Sending RPC reply: \"" << response_view << "\"");

src/serialization/CMakeLists.txt

@@ -42,6 +42,7 @@ monero_add_library(serialization
   ${serialization_private_headers})
 target_link_libraries(serialization
   LINK_PRIVATE
+    cryptonote_basic
     cryptonote_core
     cryptonote_protocol
     epee

src/serialization/json_object.cpp

@@ -33,6 +33,8 @@
 #include <limits>
 #include <type_traits>
 
+#include "cryptonote_basic/cryptonote_basic_impl.h"
+
 // drop macro from windows.h
 #ifdef GetObject
   #undef GetObject
@@ -246,7 +248,10 @@ void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::t
   INSERT_INTO_JSON_OBJECT(dest, inputs, tx.vin);
   INSERT_INTO_JSON_OBJECT(dest, outputs, tx.vout);
   INSERT_INTO_JSON_OBJECT(dest, extra, tx.extra);
-  INSERT_INTO_JSON_OBJECT(dest, signatures, tx.signatures);
+  if (!tx.pruned)
+  {
+    INSERT_INTO_JSON_OBJECT(dest, signatures, tx.signatures);
+  }
   INSERT_INTO_JSON_OBJECT(dest, ringct, tx.rct_signatures);
 
   dest.EndObject();
@@ -265,8 +270,17 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::transaction& tx)
   GET_FROM_JSON_OBJECT(val, tx.vin, inputs);
   GET_FROM_JSON_OBJECT(val, tx.vout, outputs);
   GET_FROM_JSON_OBJECT(val, tx.extra, extra);
-  GET_FROM_JSON_OBJECT(val, tx.signatures, signatures);
   GET_FROM_JSON_OBJECT(val, tx.rct_signatures, ringct);
+
+  const auto& sigs = val.FindMember("signatures");
+  if (sigs != val.MemberEnd())
+  {
+    fromJsonValue(sigs->value, tx.signatures);
+  }
+
+  const auto& rsig = tx.rct_signatures;
+  if (!cryptonote::is_coinbase(tx) && rsig.p.bulletproofs.empty() && rsig.p.rangeSigs.empty() && rsig.p.MGs.empty() && rsig.get_pseudo_outs().empty() && sigs == val.MemberEnd())
+    tx.pruned = true;
 }
 
 void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::block& b)
@@ -1062,6 +1076,7 @@ void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const rct::rctSig&
   INSERT_INTO_JSON_OBJECT(dest, fee, sig.txnFee);
 
   // prunable
+  if (!sig.p.bulletproofs.empty() || !sig.p.rangeSigs.empty() || !sig.p.MGs.empty() || !sig.get_pseudo_outs().empty())
   {
     dest.Key("prunable");
     dest.StartObject();
@@ -1086,35 +1101,39 @@ void fromJsonValue(const rapidjson::Value& val, rct::rctSig& sig)
     throw WRONG_TYPE("json object");
   }
 
-  std::vector<rct::key> commitments;
-
   GET_FROM_JSON_OBJECT(val, sig.type, type);
   GET_FROM_JSON_OBJECT(val, sig.ecdhInfo, encrypted);
-  GET_FROM_JSON_OBJECT(val, commitments, commitments);
+  GET_FROM_JSON_OBJECT(val, sig.outPk, commitments);
   GET_FROM_JSON_OBJECT(val, sig.txnFee, fee);
 
   // prunable
+  const auto prunable = val.FindMember("prunable");
+  if (prunable != val.MemberEnd())
   {
-    OBJECT_HAS_MEMBER_OR_THROW(val, "prunable");
+    rct::keyV pseudo_outs = std::move(sig.get_pseudo_outs());
-    const auto& prunable = val["prunable"];
-
-    rct::keyV pseudo_outs;
 
-    GET_FROM_JSON_OBJECT(prunable, sig.p.rangeSigs, range_proofs);
+    GET_FROM_JSON_OBJECT(prunable->value, sig.p.rangeSigs, range_proofs);
-    GET_FROM_JSON_OBJECT(prunable, sig.p.bulletproofs, bulletproofs);
+    GET_FROM_JSON_OBJECT(prunable->value, sig.p.bulletproofs, bulletproofs);
-    GET_FROM_JSON_OBJECT(prunable, sig.p.MGs, mlsags);
+    GET_FROM_JSON_OBJECT(prunable->value, sig.p.MGs, mlsags);
-    GET_FROM_JSON_OBJECT(prunable, pseudo_outs, pseudo_outs);
+    GET_FROM_JSON_OBJECT(prunable->value, pseudo_outs, pseudo_outs);
 
     sig.get_pseudo_outs() = std::move(pseudo_outs);
   }
-
+  else
-  sig.outPk.reserve(commitments.size());
-  for (rct::key const& commitment : commitments)
   {
-    sig.outPk.push_back({{}, commitment});
+    sig.p.rangeSigs.clear();
+    sig.p.bulletproofs.clear();
+    sig.p.MGs.clear();
+    sig.get_pseudo_outs().clear();
   }
 }
 
+void fromJsonValue(const rapidjson::Value& val, rct::ctkey& key)
+{
+  key.dest = {};
+  fromJsonValue(val, key.mask);
+}
+
 void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const rct::ecdhTuple& tuple)
 {
   dest.StartObject();

src/serialization/json_object.h

@@ -32,6 +32,7 @@
 #include <cstring>
 #include <rapidjson/document.h>
 #include <rapidjson/writer.h>
+#include <vector>
 
 #include "byte_stream.h"
 #include "cryptonote_basic/cryptonote_basic.h"
@@ -277,6 +278,8 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::BlockHeaderResp
 void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const rct::rctSig& i);
 void fromJsonValue(const rapidjson::Value& val, rct::rctSig& sig);
 
+void fromJsonValue(const rapidjson::Value& val, rct::ctkey& key);
+
 void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const rct::ecdhTuple& tuple);
 void fromJsonValue(const rapidjson::Value& val, rct::ecdhTuple& tuple);
 
@@ -339,6 +342,7 @@ inline typename std::enable_if<sfinae::is_map_like<Map>::value, void>::type from
 
   auto itr = val.MemberBegin();
 
+  map.clear();
   while (itr != val.MemberEnd())
   {
     typename Map::key_type k;
@@ -359,6 +363,19 @@ inline typename std::enable_if<sfinae::is_vector_like<Vec>::value, void>::type t
   dest.EndArray();
 }
 
+namespace traits
+{
+  template<typename T>
+  void reserve(const T&, std::size_t)
+  {}
+
+  template<typename T>
+  void reserve(std::vector<T>& vec, const std::size_t count)
+  {
+    vec.reserve(count);
+  }
+}
+
 template <typename Vec>
 inline typename std::enable_if<sfinae::is_vector_like<Vec>::value, void>::type fromJsonValue(const rapidjson::Value& val, Vec& vec)
 {
@@ -367,11 +384,12 @@ inline typename std::enable_if<sfinae::is_vector_like<Vec>::value, void>::type f
     throw WRONG_TYPE("json array");
   }
 
+  vec.clear();
+  traits::reserve(vec, val.Size());
   for (rapidjson::SizeType i=0; i < val.Size(); i++)
   {
-    typename Vec::value_type v;
+    vec.emplace_back();
-    fromJsonValue(val[i], v);
+    fromJsonValue(val[i], vec.back());
-    vec.push_back(v);
   }
 }
 

tests/unit_tests/zmq_rpc.cpp

@@ -304,7 +304,7 @@ namespace
       : cryptonote::rpc::RpcHandler()
     {}
 
-    virtual epee::byte_slice handle(const std::string& request) override final
+    virtual epee::byte_slice handle(std::string&& request) override final
     {
       throw std::logic_error{"not implemented"};
     }