From d41415ec484216185a6255c9b1b8bd15977f5372 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Thu, 6 Dec 2018 18:04:33 +0000 Subject: [PATCH] epee: better network buffer data structure avoids pointless allocs and memcpy --- contrib/epee/include/net/buffer.h | 62 ++++++++++++ .../include/net/http_server_handlers_map2.h | 2 +- contrib/epee/include/net/levin_base.h | 4 +- contrib/epee/include/net/levin_client.h | 4 +- contrib/epee/include/net/levin_client.inl | 4 +- .../net/levin_protocol_handler_async.h | 61 ++++++------ contrib/epee/include/span.h | 8 ++ .../include/storages/http_abstract_invoke.h | 2 +- .../include/storages/levin_abstract_invoke2.h | 21 ++-- .../epee/include/storages/portable_storage.h | 6 +- .../portable_storage_template_helper.h | 8 +- contrib/epee/src/CMakeLists.txt | 2 +- contrib/epee/src/buffer.cpp | 97 +++++++++++++++++++ .../cryptonote_protocol_handler.h | 4 +- .../cryptonote_protocol_handler.inl | 4 +- src/p2p/net_node.h | 8 +- src/p2p/net_node.inl | 8 +- src/p2p/net_node_common.h | 16 +-- tests/fuzz/levin.cpp | 8 +- tests/net_load_tests/net_load_tests.h | 4 +- .../epee_levin_protocol_handler_async.cpp | 8 +- tests/unit_tests/epee_utils.cpp | 69 +++++++++++++ 22 files changed, 325 insertions(+), 85 deletions(-) create mode 100644 contrib/epee/include/net/buffer.h create mode 100644 contrib/epee/src/buffer.cpp diff --git a/contrib/epee/include/net/buffer.h b/contrib/epee/include/net/buffer.h new file mode 100644 index 000000000..56c67f6bf --- /dev/null +++ b/contrib/epee/include/net/buffer.h @@ -0,0 +1,62 @@ +// Copyright (c) 2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include +#include "misc_log_ex.h" +#include "span.h" + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "net.buffer" + +//#define NET_BUFFER_LOG(x) MDEBUG(x) +#define NET_BUFFER_LOG(x) ((void)0) + +namespace epee +{ +namespace net_utils +{ +class buffer +{ +public: + buffer(size_t reserve = 0): offset(0) { storage.reserve(reserve); } + + void append(const void *data, size_t sz); + void erase(size_t sz) { NET_BUFFER_LOG("erasing " << sz << "/" << size()); CHECK_AND_ASSERT_THROW_MES(offset + sz <= storage.size(), "erase: sz too large"); offset += sz; if (offset == storage.size()) { storage.resize(0); offset = 0; } } + epee::span span(size_t sz) const { CHECK_AND_ASSERT_THROW_MES(sz <= size(), "span is too large"); return epee::span(storage.data() + offset, sz); } + // carve must keep the data in scope till next call, other API calls (such as append, erase) can invalidate the carved buffer + epee::span carve(size_t sz) { CHECK_AND_ASSERT_THROW_MES(sz <= size(), "span is too large"); offset += sz; return epee::span(storage.data() + offset - sz, sz); } + size_t size() const { return storage.size() - offset; } + +private: + std::vector storage; + size_t offset; +}; +} +} diff --git a/contrib/epee/include/net/http_server_handlers_map2.h b/contrib/epee/include/net/http_server_handlers_map2.h index 997c801d1..64d035df9 100644 --- a/contrib/epee/include/net/http_server_handlers_map2.h +++ b/contrib/epee/include/net/http_server_handlers_map2.h @@ -92,7 +92,7 @@ handled = true; \ uint64_t ticks = misc_utils::get_tick_count(); \ boost::value_initialized req; \ - bool parse_res = epee::serialization::load_t_from_binary(static_cast(req), query_info.m_body); \ + bool parse_res = epee::serialization::load_t_from_binary(static_cast(req), epee::strspan(query_info.m_body)); \ CHECK_AND_ASSERT_MES(parse_res, false, "Failed to parse bin body data, body size=" << query_info.m_body.size()); \ uint64_t ticks1 = misc_utils::get_tick_count(); \ boost::value_initialized resp;\ diff --git a/contrib/epee/include/net/levin_base.h b/contrib/epee/include/net/levin_base.h index 7d060f5ef..a88a1eb49 100644 --- a/contrib/epee/include/net/levin_base.h +++ b/contrib/epee/include/net/levin_base.h @@ -80,8 +80,8 @@ namespace levin template struct levin_commands_handler { - virtual int invoke(int command, const std::string& in_buff, std::string& buff_out, t_connection_context& context)=0; - virtual int notify(int command, const std::string& in_buff, t_connection_context& context)=0; + virtual int invoke(int command, const epee::span in_buff, std::string& buff_out, t_connection_context& context)=0; + virtual int notify(int command, const epee::span in_buff, t_connection_context& context)=0; virtual void callback(t_connection_context& context){}; virtual void on_connection_new(t_connection_context& context){}; diff --git a/contrib/epee/include/net/levin_client.h b/contrib/epee/include/net/levin_client.h index 335f6ba02..76d528234 100644 --- a/contrib/epee/include/net/levin_client.h +++ b/contrib/epee/include/net/levin_client.h @@ -57,7 +57,7 @@ namespace levin bool is_connected(); bool disconnect(); - virtual int invoke(int command, const std::string& in_buff, std::string& buff_out); + virtual int invoke(int command, const epee::span in_buff, std::string& buff_out); virtual int notify(int command, const std::string& in_buff); protected: @@ -72,7 +72,7 @@ namespace levin { public: - int invoke(int command, const std::string& in_buff, std::string& buff_out); + int invoke(int command, const epee::span in_buff, std::string& buff_out); int notify(int command, const std::string& in_buff); }; diff --git a/contrib/epee/include/net/levin_client.inl b/contrib/epee/include/net/levin_client.inl index a580e81fd..2f048b027 100644 --- a/contrib/epee/include/net/levin_client.inl +++ b/contrib/epee/include/net/levin_client.inl @@ -74,7 +74,7 @@ levin_client_impl::~levin_client_impl() } //------------------------------------------------------------------------------ inline -int levin_client_impl::invoke(int command, const std::string& in_buff, std::string& buff_out) +int levin_client_impl::invoke(int command, const epee::span in_buff, std::string& buff_out) { if(!is_connected()) return -1; @@ -133,7 +133,7 @@ int levin_client_impl::notify(int command, const std::string& in_buff) //------------------------------------------------------------------------------ //------------------------------------------------------------------------------ inline - int levin_client_impl2::invoke(int command, const std::string& in_buff, std::string& buff_out) + int levin_client_impl2::invoke(int command, epee::spanstring& in_buff, std::string& buff_out) { if(!is_connected()) return -1; diff --git a/contrib/epee/include/net/levin_protocol_handler_async.h b/contrib/epee/include/net/levin_protocol_handler_async.h index 11649febe..a1ea3e680 100644 --- a/contrib/epee/include/net/levin_protocol_handler_async.h +++ b/contrib/epee/include/net/levin_protocol_handler_async.h @@ -34,6 +34,7 @@ #include #include "levin_base.h" +#include "buffer.h" #include "misc_language.h" #include "syncobj.h" #include "misc_os_dependent.h" @@ -85,11 +86,11 @@ public: uint64_t m_max_packet_size; uint64_t m_invoke_timeout; - int invoke(int command, const std::string& in_buff, std::string& buff_out, boost::uuids::uuid connection_id); + int invoke(int command, const epee::span in_buff, std::string& buff_out, boost::uuids::uuid connection_id); template - int invoke_async(int command, const std::string& in_buff, boost::uuids::uuid connection_id, const callback_t &cb, size_t timeout = LEVIN_DEFAULT_TIMEOUT_PRECONFIGURED); + int invoke_async(int command, const epee::span in_buff, boost::uuids::uuid connection_id, const callback_t &cb, size_t timeout = LEVIN_DEFAULT_TIMEOUT_PRECONFIGURED); - int notify(int command, const std::string& in_buff, boost::uuids::uuid connection_id); + int notify(int command, const epee::span in_buff, boost::uuids::uuid connection_id); bool close(boost::uuids::uuid connection_id); bool update_connection_context(const t_connection_context& contxt); bool request_callback(boost::uuids::uuid connection_id); @@ -143,7 +144,7 @@ public: config_type& m_config; t_connection_context& m_connection_context; - std::string m_cache_in_buffer; + net_utils::buffer m_cache_in_buffer; stream_state m_state; int32_t m_oponent_protocol_ver; @@ -151,7 +152,7 @@ public: struct invoke_response_handler_base { - virtual bool handle(int res, const std::string& buff, connection_context& context)=0; + virtual bool handle(int res, const epee::span buff, connection_context& context)=0; virtual bool is_timer_started() const=0; virtual void cancel()=0; virtual bool cancel_timer()=0; @@ -173,7 +174,7 @@ public: if(ec == boost::asio::error::operation_aborted) return; MINFO(con.get_context_ref() << "Timeout on invoke operation happened, command: " << command << " timeout: " << timeout); - std::string fake; + epee::span fake; cb(LEVIN_ERROR_CONNECTION_TIMEDOUT, fake, con.get_context_ref()); con.close(); con.finish_outer_call(); @@ -191,7 +192,7 @@ public: bool m_timer_cancelled; uint64_t m_timeout; int m_command; - virtual bool handle(int res, const std::string& buff, typename async_protocol_handler::connection_context& context) + virtual bool handle(int res, const epee::span buff, typename async_protocol_handler::connection_context& context) { if(!cancel_timer()) return false; @@ -207,7 +208,7 @@ public: { if(cancel_timer()) { - std::string fake; + epee::span fake; m_cb(LEVIN_ERROR_CONNECTION_DESTROYED, fake, m_con.get_context_ref()); m_con.finish_outer_call(); } @@ -237,7 +238,7 @@ public: if(ec == boost::asio::error::operation_aborted) return; MINFO(con.get_context_ref() << "Timeout on invoke operation happened, command: " << command << " timeout: " << timeout); - std::string fake; + epee::span fake; cb(LEVIN_ERROR_CONNECTION_TIMEDOUT, fake, con.get_context_ref()); con.close(); con.finish_outer_call(); @@ -265,6 +266,7 @@ public: m_pservice_endpoint(psnd_hndlr), m_config(config), m_connection_context(conn_context), + m_cache_in_buffer(256 * 1024), m_state(stream_state_head) { m_close_called = 0; @@ -405,14 +407,7 @@ public: break; } { - std::string buff_to_invoke; - if(m_cache_in_buffer.size() == m_current_head.m_cb) - buff_to_invoke.swap(m_cache_in_buffer); - else - { - buff_to_invoke.assign(m_cache_in_buffer, 0, (std::string::size_type)m_current_head.m_cb); - m_cache_in_buffer.erase(0, (std::string::size_type)m_current_head.m_cb); - } + epee::span buff_to_invoke = m_cache_in_buffer.carve((std::string::size_type)m_current_head.m_cb); bool is_response = (m_oponent_protocol_ver == LEVIN_PROTOCOL_VER_1 && m_current_head.m_flags&LEVIN_PACKET_RESPONSE); @@ -449,8 +444,8 @@ public: }else { CRITICAL_REGION_BEGIN(m_local_inv_buff_lock); - buff_to_invoke.swap(m_local_inv_buff); - buff_to_invoke.clear(); + m_local_inv_buff = std::string((const char*)buff_to_invoke.data(), buff_to_invoke.size()); + buff_to_invoke = epee::span((const uint8_t*)NULL, 0); m_invoke_result_code = m_current_head.m_return_code; CRITICAL_REGION_END(); boost::interprocess::ipcdetail::atomic_write32(&m_invoke_buf_ready, 1); @@ -503,7 +498,7 @@ public: { if(m_cache_in_buffer.size() < sizeof(bucket_head2)) { - if(m_cache_in_buffer.size() >= sizeof(uint64_t) && *((uint64_t*)m_cache_in_buffer.data()) != SWAP64LE(LEVIN_SIGNATURE)) + if(m_cache_in_buffer.size() >= sizeof(uint64_t) && *((uint64_t*)m_cache_in_buffer.span(8).data()) != SWAP64LE(LEVIN_SIGNATURE)) { MWARNING(m_connection_context << "Signature mismatch, connection will be closed"); return false; @@ -513,9 +508,9 @@ public: } #if BYTE_ORDER == LITTLE_ENDIAN - bucket_head2& phead = *(bucket_head2*)m_cache_in_buffer.data(); + bucket_head2& phead = *(bucket_head2*)m_cache_in_buffer.span(sizeof(bucket_head2)).data(); #else - bucket_head2 phead = *(bucket_head2*)m_cache_in_buffer.data(); + bucket_head2 phead = *(bucket_head2*)m_cache_in_buffer.span(sizeof(bucket_head2)).data(); phead.m_signature = SWAP64LE(phead.m_signature); phead.m_cb = SWAP64LE(phead.m_cb); phead.m_command = SWAP32LE(phead.m_command); @@ -530,7 +525,7 @@ public: } m_current_head = phead; - m_cache_in_buffer.erase(0, sizeof(bucket_head2)); + m_cache_in_buffer.erase(sizeof(bucket_head2)); m_state = stream_state_body; m_oponent_protocol_ver = m_current_head.m_protocol_version; if(m_current_head.m_cb > m_config.m_max_packet_size) @@ -562,7 +557,7 @@ public: } template - bool async_invoke(int command, const std::string& in_buff, const callback_t &cb, size_t timeout = LEVIN_DEFAULT_TIMEOUT_PRECONFIGURED) + bool async_invoke(int command, const epee::span in_buff, const callback_t &cb, size_t timeout = LEVIN_DEFAULT_TIMEOUT_PRECONFIGURED) { misc_utils::auto_scope_leave_caller scope_exit_handler = misc_utils::create_scope_leave_handler( boost::bind(&async_protocol_handler::finish_outer_call, this)); @@ -606,7 +601,7 @@ public: break; } - if(!m_pservice_endpoint->do_send(in_buff.data(), (int)in_buff.size())) + if(!m_pservice_endpoint->do_send(in_buff.data(), in_buff.size())) { LOG_ERROR_CC(m_connection_context, "Failed to do_send"); err_code = LEVIN_ERROR_CONNECTION; @@ -623,7 +618,7 @@ public: if (LEVIN_OK != err_code) { - std::string stub_buff; + epee::span stub_buff{(const uint8_t*)"", 0}; // Never call callback inside critical section, that can cause deadlock cb(err_code, stub_buff, m_connection_context); return false; @@ -632,7 +627,7 @@ public: return true; } - int invoke(int command, const std::string& in_buff, std::string& buff_out) + int invoke(int command, const epee::span in_buff, std::string& buff_out) { misc_utils::auto_scope_leave_caller scope_exit_handler = misc_utils::create_scope_leave_handler( boost::bind(&async_protocol_handler::finish_outer_call, this)); @@ -662,7 +657,7 @@ public: return LEVIN_ERROR_CONNECTION; } - if(!m_pservice_endpoint->do_send(in_buff.data(), (int)in_buff.size())) + if(!m_pservice_endpoint->do_send(in_buff.data(), in_buff.size())) { LOG_ERROR_CC(m_connection_context, "Failed to do_send"); return LEVIN_ERROR_CONNECTION; @@ -706,7 +701,7 @@ public: return m_invoke_result_code; } - int notify(int command, const std::string& in_buff) + int notify(int command, const epee::span in_buff) { misc_utils::auto_scope_leave_caller scope_exit_handler = misc_utils::create_scope_leave_handler( boost::bind(&async_protocol_handler::finish_outer_call, this)); @@ -734,7 +729,7 @@ public: return -1; } - if(!m_pservice_endpoint->do_send(in_buff.data(), (int)in_buff.size())) + if(!m_pservice_endpoint->do_send(in_buff.data(), in_buff.size())) { LOG_ERROR_CC(m_connection_context, "Failed to do_send()"); return -1; @@ -839,7 +834,7 @@ int async_protocol_handler_config::find_and_lock_connectio } //------------------------------------------------------------------------------------------ template -int async_protocol_handler_config::invoke(int command, const std::string& in_buff, std::string& buff_out, boost::uuids::uuid connection_id) +int async_protocol_handler_config::invoke(int command, const epee::span in_buff, std::string& buff_out, boost::uuids::uuid connection_id) { async_protocol_handler* aph; int r = find_and_lock_connection(connection_id, aph); @@ -847,7 +842,7 @@ int async_protocol_handler_config::invoke(int command, con } //------------------------------------------------------------------------------------------ template template -int async_protocol_handler_config::invoke_async(int command, const std::string& in_buff, boost::uuids::uuid connection_id, const callback_t &cb, size_t timeout) +int async_protocol_handler_config::invoke_async(int command, const epee::span in_buff, boost::uuids::uuid connection_id, const callback_t &cb, size_t timeout) { async_protocol_handler* aph; int r = find_and_lock_connection(connection_id, aph); @@ -896,7 +891,7 @@ void async_protocol_handler_config::set_handler(levin_comm } //------------------------------------------------------------------------------------------ template -int async_protocol_handler_config::notify(int command, const std::string& in_buff, boost::uuids::uuid connection_id) +int async_protocol_handler_config::notify(int command, const epee::span in_buff, boost::uuids::uuid connection_id) { async_protocol_handler* aph; int r = find_and_lock_connection(connection_id, aph); diff --git a/contrib/epee/include/span.h b/contrib/epee/include/span.h index b1296a0b7..cfb5b1a17 100644 --- a/contrib/epee/include/span.h +++ b/contrib/epee/include/span.h @@ -163,4 +163,12 @@ namespace epee static_assert(!has_padding(), "source type may have padding"); return {reinterpret_cast(std::addressof(src)), sizeof(T)}; } + + //! make a span from a std::string + template + span strspan(const std::string &s) noexcept + { + static_assert(std::is_same() || std::is_same() || std::is_same() || std::is_same(), "Unexpected type"); + return {reinterpret_cast(s.data()), s.size()}; + } } diff --git a/contrib/epee/include/storages/http_abstract_invoke.h b/contrib/epee/include/storages/http_abstract_invoke.h index d93084ab0..18b7f10c1 100644 --- a/contrib/epee/include/storages/http_abstract_invoke.h +++ b/contrib/epee/include/storages/http_abstract_invoke.h @@ -97,7 +97,7 @@ namespace epee return false; } - return serialization::load_t_from_binary(result_struct, pri->m_body); + return serialization::load_t_from_binary(result_struct, epee::strspan(pri->m_body)); } template diff --git a/contrib/epee/include/storages/levin_abstract_invoke2.h b/contrib/epee/include/storages/levin_abstract_invoke2.h index d77e7a1f8..06eb9bdaf 100644 --- a/contrib/epee/include/storages/levin_abstract_invoke2.h +++ b/contrib/epee/include/storages/levin_abstract_invoke2.h @@ -28,6 +28,7 @@ #include "portable_storage_template_helper.h" #include +#include "span.h" #include "net/levin_base.h" #undef MONERO_DEFAULT_LOG_CATEGORY @@ -114,7 +115,7 @@ namespace epee const_cast(out_struct).store(stg);//TODO: add true const support to searilzation std::string buff_to_send; stg.store_to_binary(buff_to_send); - int res = transport.invoke_async(command, buff_to_send, conn_id, [cb, command](int code, const std::string& buff, typename t_transport::connection_context& context)->bool + int res = transport.invoke_async(command, epee::strspan(buff_to_send), conn_id, [cb, command](int code, const epee::span buff, typename t_transport::connection_context& context)->bool { t_result result_struct = AUTO_VAL_INIT(result_struct); if( code <=0 ) @@ -156,7 +157,7 @@ namespace epee std::string buff_to_send; stg.store_to_binary(buff_to_send); - int res = transport.notify(command, buff_to_send, conn_id); + int res = transport.notify(command, epee::strspan(buff_to_send), conn_id); if(res <=0 ) { MERROR("Failed to notify command " << command << " return code " << res); @@ -167,7 +168,7 @@ namespace epee //---------------------------------------------------------------------------------------------------- //---------------------------------------------------------------------------------------------------- template - int buff_to_t_adapter(int command, const std::string& in_buff, std::string& buff_out, callback_t cb, t_context& context ) + int buff_to_t_adapter(int command, const epee::span in_buff, std::string& buff_out, callback_t cb, t_context& context ) { serialization::portable_storage strg; if(!strg.load_from_binary(in_buff)) @@ -197,7 +198,7 @@ namespace epee } template - int buff_to_t_adapter(t_owner* powner, int command, const std::string& in_buff, callback_t cb, t_context& context) + int buff_to_t_adapter(t_owner* powner, int command, const epee::span in_buff, callback_t cb, t_context& context) { serialization::portable_storage strg; if(!strg.load_from_binary(in_buff)) @@ -215,14 +216,14 @@ namespace epee } #define CHAIN_LEVIN_INVOKE_MAP2(context_type) \ - int invoke(int command, const std::string& in_buff, std::string& buff_out, context_type& context) \ + int invoke(int command, const epee::span in_buff, std::string& buff_out, context_type& context) \ { \ bool handled = false; \ return handle_invoke_map(false, command, in_buff, buff_out, context, handled); \ } #define CHAIN_LEVIN_NOTIFY_MAP2(context_type) \ - int notify(int command, const std::string& in_buff, context_type& context) \ + int notify(int command, const epee::span in_buff, context_type& context) \ { \ bool handled = false; std::string fake_str;\ return handle_invoke_map(true, command, in_buff, fake_str, context, handled); \ @@ -230,27 +231,27 @@ namespace epee #define CHAIN_LEVIN_INVOKE_MAP() \ - int invoke(int command, const std::string& in_buff, std::string& buff_out, epee::net_utils::connection_context_base& context) \ + int invoke(int command, const epee::span in_buff, std::string& buff_out, epee::net_utils::connection_context_base& context) \ { \ bool handled = false; \ return handle_invoke_map(false, command, in_buff, buff_out, context, handled); \ } #define CHAIN_LEVIN_NOTIFY_MAP() \ - int notify(int command, const std::string& in_buff, epee::net_utils::connection_context_base& context) \ + int notify(int command, const epee::span in_buff, epee::net_utils::connection_context_base& context) \ { \ bool handled = false; std::string fake_str;\ return handle_invoke_map(true, command, in_buff, fake_str, context, handled); \ } #define CHAIN_LEVIN_NOTIFY_STUB() \ - int notify(int command, const std::string& in_buff, epee::net_utils::connection_context_base& context) \ + int notify(int command, const epee::span in_buff, epee::net_utils::connection_context_base& context) \ { \ return -1; \ } #define BEGIN_INVOKE_MAP2(owner_type) \ - template int handle_invoke_map(bool is_notify, int command, const std::string& in_buff, std::string& buff_out, t_context& context, bool& handled) \ + template int handle_invoke_map(bool is_notify, int command, const epee::span in_buff, std::string& buff_out, t_context& context, bool& handled) \ { \ typedef owner_type internal_owner_type_name; diff --git a/contrib/epee/include/storages/portable_storage.h b/contrib/epee/include/storages/portable_storage.h index 0f0c6210f..d0e40d606 100644 --- a/contrib/epee/include/storages/portable_storage.h +++ b/contrib/epee/include/storages/portable_storage.h @@ -35,6 +35,7 @@ #include "portable_storage_to_json.h" #include "portable_storage_from_json.h" #include "portable_storage_val_converters.h" +#include "span.h" #include "int-util.h" namespace epee @@ -81,7 +82,8 @@ namespace epee //------------------------------------------------------------------------------- bool store_to_binary(binarybuffer& target); - bool load_from_binary(const binarybuffer& target); + bool load_from_binary(const epee::span target); + bool load_from_binary(const std::string& target) { return load_from_binary(epee::strspan(target)); } template bool dump_as_xml(std::string& targetObj, const std::string& root_name = ""); bool dump_as_json(std::string& targetObj, size_t indent = 0, bool insert_newlines = true); @@ -146,7 +148,7 @@ namespace epee CATCH_ENTRY("portable_storage::store_to_binary", false) } inline - bool portable_storage::load_from_binary(const binarybuffer& source) + bool portable_storage::load_from_binary(const epee::span source) { m_root.m_entries.clear(); if(source.size() < sizeof(storage_block_header)) diff --git a/contrib/epee/include/storages/portable_storage_template_helper.h b/contrib/epee/include/storages/portable_storage_template_helper.h index bbd8413fc..13c870d44 100644 --- a/contrib/epee/include/storages/portable_storage_template_helper.h +++ b/contrib/epee/include/storages/portable_storage_template_helper.h @@ -84,7 +84,7 @@ namespace epee } //----------------------------------------------------------------------------------------------------------- template - bool load_t_from_binary(t_struct& out, const std::string& binary_buff) + bool load_t_from_binary(t_struct& out, const epee::span binary_buff) { portable_storage ps; bool rs = ps.load_from_binary(binary_buff); @@ -95,6 +95,12 @@ namespace epee } //----------------------------------------------------------------------------------------------------------- template + bool load_t_from_binary(t_struct& out, const std::string& binary_buff) + { + return load_t_from_binary(out, epee::strspan(binary_buff)); + } + //----------------------------------------------------------------------------------------------------------- + template bool load_t_from_binary_file(t_struct& out, const std::string& binary_file) { std::string f_buff; diff --git a/contrib/epee/src/CMakeLists.txt b/contrib/epee/src/CMakeLists.txt index f234904a0..16f2e43ab 100644 --- a/contrib/epee/src/CMakeLists.txt +++ b/contrib/epee/src/CMakeLists.txt @@ -27,7 +27,7 @@ # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. add_library(epee STATIC hex.cpp http_auth.cpp mlog.cpp net_utils_base.cpp string_tools.cpp wipeable_string.cpp memwipe.c - connection_basic.cpp network_throttle.cpp network_throttle-detail.cpp mlocker.cpp) + connection_basic.cpp network_throttle.cpp network_throttle-detail.cpp mlocker.cpp buffer.cpp) if (USE_READLINE AND GNU_READLINE_FOUND) add_library(epee_readline STATIC readline_buffer.cpp) endif() diff --git a/contrib/epee/src/buffer.cpp b/contrib/epee/src/buffer.cpp new file mode 100644 index 000000000..d637b905e --- /dev/null +++ b/contrib/epee/src/buffer.cpp @@ -0,0 +1,97 @@ +// Copyright (c) 2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include +#include "net/buffer.h" + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "net.buffer" + +namespace epee +{ +namespace net_utils +{ + +void buffer::append(const void *data, size_t sz) +{ + const size_t capacity = storage.capacity(); + const size_t avail = capacity - storage.size(); + + CHECK_AND_ASSERT_THROW_MES(storage.size() < std::numeric_limits::max() - sz, "Too much data to append"); + + // decide when to move + if (sz > avail) + { + // we have to reallocate or move + const bool move = size() + sz <= capacity; + if (move) + { + const size_t bytes = storage.size() - offset; + NET_BUFFER_LOG("appending " << sz << " from " << size() << " by moving " << bytes << " from offset " << offset << " first (forced)"); + memmove(storage.data(), storage.data() + offset, bytes); + storage.resize(bytes); + offset = 0; + } + else + { + NET_BUFFER_LOG("appending " << sz << " from " << size() << " by reallocating"); + std::vector new_storage; + size_t reserve = (((size() + sz) * 3 / 2) + 4095) & ~4095; + new_storage.reserve(reserve); + new_storage.resize(size()); + memcpy(new_storage.data(), storage.data() + offset, storage.size() - offset); + offset = 0; + std::swap(storage, new_storage); + } + } + else + { + // we have space already + if (size() <= 4096 && offset > 4096 * 16 && offset >= capacity / 2) + { + // we have little to move, and we're far enough into the buffer that it's probably a win to move anyway + const size_t pos = storage.size() - offset; + NET_BUFFER_LOG("appending " << sz << " from " << size() << " by moving " << pos << " from offset " << offset << " first (unforced)"); + memmove(storage.data(), storage.data() + offset, storage.size() - offset); + storage.resize(pos); + offset = 0; + } + else + { + NET_BUFFER_LOG("appending " << sz << " from " << size() << " by writing to existing capacity"); + } + } + + // add the new data + storage.insert(storage.end(), (const uint8_t*)data, (const uint8_t*)data + sz); + + NET_BUFFER_LOG("storage now " << offset << "/" << storage.size() << "/" << storage.capacity()); +} + +} +} diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.h b/src/cryptonote_protocol/cryptonote_protocol_handler.h index 77879ebac..9aa4a39a5 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler.h +++ b/src/cryptonote_protocol/cryptonote_protocol_handler.h @@ -157,7 +157,7 @@ namespace cryptonote std::string blob; epee::serialization::store_t_to_binary(arg, blob); //handler_response_blocks_now(blob.size()); // XXX - return m_p2p->invoke_notify_to_peer(t_parameter::ID, blob, context); + return m_p2p->invoke_notify_to_peer(t_parameter::ID, epee::strspan(blob), context); } template @@ -166,7 +166,7 @@ namespace cryptonote LOG_PRINT_L2("[" << epee::net_utils::print_connection_context_short(exclude_context) << "] post relay " << typeid(t_parameter).name() << " -->"); std::string arg_buff; epee::serialization::store_t_to_binary(arg, arg_buff); - return m_p2p->relay_notify_to_all(t_parameter::ID, arg_buff, exclude_context); + return m_p2p->relay_notify_to_all(t_parameter::ID, epee::strspan(arg_buff), exclude_context); } }; diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.inl b/src/cryptonote_protocol/cryptonote_protocol_handler.inl index 294617070..5935105f7 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler.inl +++ b/src/cryptonote_protocol/cryptonote_protocol_handler.inl @@ -1720,13 +1720,13 @@ skip: { std::string fluffyBlob; epee::serialization::store_t_to_binary(fluffy_arg, fluffyBlob); - m_p2p->relay_notify_to_list(NOTIFY_NEW_FLUFFY_BLOCK::ID, fluffyBlob, fluffyConnections); + m_p2p->relay_notify_to_list(NOTIFY_NEW_FLUFFY_BLOCK::ID, epee::strspan(fluffyBlob), fluffyConnections); } if (!fullConnections.empty()) { std::string fullBlob; epee::serialization::store_t_to_binary(arg, fullBlob); - m_p2p->relay_notify_to_list(NOTIFY_NEW_BLOCK::ID, fullBlob, fullConnections); + m_p2p->relay_notify_to_list(NOTIFY_NEW_BLOCK::ID, epee::strspan(fullBlob), fullConnections); } return true; diff --git a/src/p2p/net_node.h b/src/p2p/net_node.h index 6b3c2f1ca..80cf3b734 100644 --- a/src/p2p/net_node.h +++ b/src/p2p/net_node.h @@ -177,10 +177,10 @@ namespace nodetool virtual void on_connection_close(p2p_connection_context& context); virtual void callback(p2p_connection_context& context); //----------------- i_p2p_endpoint ------------------------------------------------------------- - virtual bool relay_notify_to_list(int command, const std::string& data_buff, const std::list &connections); - virtual bool relay_notify_to_all(int command, const std::string& data_buff, const epee::net_utils::connection_context_base& context); - virtual bool invoke_command_to_peer(int command, const std::string& req_buff, std::string& resp_buff, const epee::net_utils::connection_context_base& context); - virtual bool invoke_notify_to_peer(int command, const std::string& req_buff, const epee::net_utils::connection_context_base& context); + virtual bool relay_notify_to_list(int command, const epee::span data_buff, const std::list &connections); + virtual bool relay_notify_to_all(int command, const epee::span data_buff, const epee::net_utils::connection_context_base& context); + virtual bool invoke_command_to_peer(int command, const epee::span req_buff, std::string& resp_buff, const epee::net_utils::connection_context_base& context); + virtual bool invoke_notify_to_peer(int command, const epee::span req_buff, const epee::net_utils::connection_context_base& context); virtual bool drop_connection(const epee::net_utils::connection_context_base& context); virtual void request_callback(const epee::net_utils::connection_context_base& context); virtual void for_each_connection(std::function f); diff --git a/src/p2p/net_node.inl b/src/p2p/net_node.inl index 40e516a81..01839b0b4 100644 --- a/src/p2p/net_node.inl +++ b/src/p2p/net_node.inl @@ -1508,7 +1508,7 @@ namespace nodetool } //----------------------------------------------------------------------------------- template - bool node_server::relay_notify_to_list(int command, const std::string& data_buff, const std::list &connections) + bool node_server::relay_notify_to_list(int command, const epee::span data_buff, const std::list &connections) { for(const auto& c_id: connections) { @@ -1518,7 +1518,7 @@ namespace nodetool } //----------------------------------------------------------------------------------- template - bool node_server::relay_notify_to_all(int command, const std::string& data_buff, const epee::net_utils::connection_context_base& context) + bool node_server::relay_notify_to_all(int command, const epee::span data_buff, const epee::net_utils::connection_context_base& context) { std::list connections; m_net_server.get_config_object().foreach_connection([&](const p2p_connection_context& cntxt) @@ -1537,14 +1537,14 @@ namespace nodetool } //----------------------------------------------------------------------------------- template - bool node_server::invoke_notify_to_peer(int command, const std::string& req_buff, const epee::net_utils::connection_context_base& context) + bool node_server::invoke_notify_to_peer(int command, const epee::span req_buff, const epee::net_utils::connection_context_base& context) { int res = m_net_server.get_config_object().notify(command, req_buff, context.m_connection_id); return res > 0; } //----------------------------------------------------------------------------------- template - bool node_server::invoke_command_to_peer(int command, const std::string& req_buff, std::string& resp_buff, const epee::net_utils::connection_context_base& context) + bool node_server::invoke_command_to_peer(int command, const epee::span req_buff, std::string& resp_buff, const epee::net_utils::connection_context_base& context) { int res = m_net_server.get_config_object().invoke(command, req_buff, resp_buff, context.m_connection_id); return res > 0; diff --git a/src/p2p/net_node_common.h b/src/p2p/net_node_common.h index 34dcf6005..fe4581cc4 100644 --- a/src/p2p/net_node_common.h +++ b/src/p2p/net_node_common.h @@ -43,10 +43,10 @@ namespace nodetool template struct i_p2p_endpoint { - virtual bool relay_notify_to_list(int command, const std::string& data_buff, const std::list& connections)=0; - virtual bool relay_notify_to_all(int command, const std::string& data_buff, const epee::net_utils::connection_context_base& context)=0; - virtual bool invoke_command_to_peer(int command, const std::string& req_buff, std::string& resp_buff, const epee::net_utils::connection_context_base& context)=0; - virtual bool invoke_notify_to_peer(int command, const std::string& req_buff, const epee::net_utils::connection_context_base& context)=0; + virtual bool relay_notify_to_list(int command, const epee::span data_buff, const std::list& connections)=0; + virtual bool relay_notify_to_all(int command, const epee::span data_buff, const epee::net_utils::connection_context_base& context)=0; + virtual bool invoke_command_to_peer(int command, const epee::span req_buff, std::string& resp_buff, const epee::net_utils::connection_context_base& context)=0; + virtual bool invoke_notify_to_peer(int command, const epee::span req_buff, const epee::net_utils::connection_context_base& context)=0; virtual bool drop_connection(const epee::net_utils::connection_context_base& context)=0; virtual void request_callback(const epee::net_utils::connection_context_base& context)=0; virtual uint64_t get_connections_count()=0; @@ -61,19 +61,19 @@ namespace nodetool template struct p2p_endpoint_stub: public i_p2p_endpoint { - virtual bool relay_notify_to_list(int command, const std::string& data_buff, const std::list& connections) + virtual bool relay_notify_to_list(int command, const epee::span data_buff, const std::list& connections) { return false; } - virtual bool relay_notify_to_all(int command, const std::string& data_buff, const epee::net_utils::connection_context_base& context) + virtual bool relay_notify_to_all(int command, const epee::span data_buff, const epee::net_utils::connection_context_base& context) { return false; } - virtual bool invoke_command_to_peer(int command, const std::string& req_buff, std::string& resp_buff, const epee::net_utils::connection_context_base& context) + virtual bool invoke_command_to_peer(int command, const epee::span req_buff, std::string& resp_buff, const epee::net_utils::connection_context_base& context) { return false; } - virtual bool invoke_notify_to_peer(int command, const std::string& req_buff, const epee::net_utils::connection_context_base& context) + virtual bool invoke_notify_to_peer(int command, const epee::span req_buff, const epee::net_utils::connection_context_base& context) { return true; } diff --git a/tests/fuzz/levin.cpp b/tests/fuzz/levin.cpp index d0c5803f5..573abd1d3 100644 --- a/tests/fuzz/levin.cpp +++ b/tests/fuzz/levin.cpp @@ -65,22 +65,22 @@ namespace { } - virtual int invoke(int command, const std::string& in_buff, std::string& buff_out, test_levin_connection_context& context) + virtual int invoke(int command, const epee::span in_buff, std::string& buff_out, test_levin_connection_context& context) { m_invoke_counter.inc(); boost::unique_lock lock(m_mutex); m_last_command = command; - m_last_in_buf = in_buff; + m_last_in_buf = std::string((const char*)in_buff.data(), in_buff.size()); buff_out = m_invoke_out_buf; return m_return_code; } - virtual int notify(int command, const std::string& in_buff, test_levin_connection_context& context) + virtual int notify(int command, const epee::span in_buff, test_levin_connection_context& context) { m_notify_counter.inc(); boost::unique_lock lock(m_mutex); m_last_command = command; - m_last_in_buf = in_buff; + m_last_in_buf = std::string((const char*)in_buff.data(), in_buff.size()); return m_return_code; } diff --git a/tests/net_load_tests/net_load_tests.h b/tests/net_load_tests/net_load_tests.h index f210b0125..cdc2d267a 100644 --- a/tests/net_load_tests/net_load_tests.h +++ b/tests/net_load_tests/net_load_tests.h @@ -63,7 +63,7 @@ namespace net_load_tests { } - virtual int invoke(int command, const std::string& in_buff, std::string& buff_out, test_connection_context& context) + virtual int invoke(int command, const epee::span in_buff, std::string& buff_out, test_connection_context& context) { //m_invoke_counter.inc(); //std::unique_lock lock(m_mutex); @@ -74,7 +74,7 @@ namespace net_load_tests return LEVIN_OK; } - virtual int notify(int command, const std::string& in_buff, test_connection_context& context) + virtual int notify(int command, const epee::span in_buff, test_connection_context& context) { //m_notify_counter.inc(); //std::unique_lock lock(m_mutex); diff --git a/tests/unit_tests/epee_levin_protocol_handler_async.cpp b/tests/unit_tests/epee_levin_protocol_handler_async.cpp index a4cbd9845..697845f60 100644 --- a/tests/unit_tests/epee_levin_protocol_handler_async.cpp +++ b/tests/unit_tests/epee_levin_protocol_handler_async.cpp @@ -56,22 +56,22 @@ namespace { } - virtual int invoke(int command, const std::string& in_buff, std::string& buff_out, test_levin_connection_context& context) + virtual int invoke(int command, const epee::span in_buff, std::string& buff_out, test_levin_connection_context& context) { m_invoke_counter.inc(); boost::unique_lock lock(m_mutex); m_last_command = command; - m_last_in_buf = in_buff; + m_last_in_buf = std::string((const char*)in_buff.data(), in_buff.size()); buff_out = m_invoke_out_buf; return m_return_code; } - virtual int notify(int command, const std::string& in_buff, test_levin_connection_context& context) + virtual int notify(int command, const epee::span in_buff, test_levin_connection_context& context) { m_notify_counter.inc(); boost::unique_lock lock(m_mutex); m_last_command = command; - m_last_in_buf = in_buff; + m_last_in_buf = std::string((const char*)in_buff.data(), in_buff.size()); return m_return_code; } diff --git a/tests/unit_tests/epee_utils.cpp b/tests/unit_tests/epee_utils.cpp index 91c285295..3915c8903 100644 --- a/tests/unit_tests/epee_utils.cpp +++ b/tests/unit_tests/epee_utils.cpp @@ -46,6 +46,7 @@ #include "hex.h" #include "net/net_utils_base.h" #include "net/local_ip.h" +#include "net/buffer.h" #include "p2p/net_peerlist_boost_serialization.h" #include "span.h" #include "string_tools.h" @@ -764,3 +765,71 @@ TEST(NetUtils, PrivateRanges) ASSERT_EQ(is_local("0.0.30.172"), false); ASSERT_EQ(is_local("0.0.30.127"), false); } + +TEST(net_buffer, basic) +{ + epee::net_utils::buffer buf; + + ASSERT_EQ(buf.size(), 0); + EXPECT_THROW(buf.span(1), std::runtime_error); + buf.append("a", 1); + epee::span span = buf.span(1); + ASSERT_EQ(span.size(), 1); + ASSERT_EQ(span.data()[0], 'a'); + EXPECT_THROW(buf.span(2), std::runtime_error); + buf.append("bc", 2); + buf.erase(1); + EXPECT_THROW(buf.span(3), std::runtime_error); + span = buf.span(2); + ASSERT_EQ(span.size(), 2); + ASSERT_EQ(span.data()[0], 'b'); + ASSERT_EQ(span.data()[1], 'c'); + buf.erase(1); + EXPECT_THROW(buf.span(2), std::runtime_error); + span = buf.span(1); + ASSERT_EQ(span.size(), 1); + ASSERT_EQ(span.data()[0], 'c'); + EXPECT_THROW(buf.erase(2), std::runtime_error); + buf.erase(1); + EXPECT_EQ(buf.size(), 0); + EXPECT_THROW(buf.span(1), std::runtime_error); +} + +TEST(net_buffer, existing_capacity) +{ + epee::net_utils::buffer buf; + + buf.append("123456789", 9); + buf.erase(9); + buf.append("abc", 3); + buf.append("def", 3); + ASSERT_EQ(buf.size(), 6); + epee::span span = buf.span(6); + ASSERT_TRUE(!memcmp(span.data(), "abcdef", 6)); +} + +TEST(net_buffer, reallocate) +{ + epee::net_utils::buffer buf; + + buf.append(std::string(4000, ' ').c_str(), 4000); + buf.append(std::string(8000, '0').c_str(), 8000); + ASSERT_EQ(buf.size(), 12000); + epee::span span = buf.span(12000); + ASSERT_TRUE(!memcmp(span.data(), std::string(4000, ' ').c_str(), 4000)); + ASSERT_TRUE(!memcmp(span.data() + 4000, std::string(8000, '0').c_str(), 8000)); +} + +TEST(net_buffer, move) +{ + epee::net_utils::buffer buf; + + buf.append(std::string(400, ' ').c_str(), 400); + buf.erase(399); + buf.append(std::string(4000, '0').c_str(), 4000); + ASSERT_EQ(buf.size(), 4001); + epee::span span = buf.span(4001); + ASSERT_TRUE(!memcmp(span.data(), std::string(1, ' ').c_str(), 1)); + ASSERT_TRUE(!memcmp(span.data() + 1, std::string(4000, '0').c_str(), 4000)); +} +