wowario
/
wownero
forked from wownero/wownero
2
1
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Improved performance for epee serialization:

Browse Source 
- Removed copy of field names in binary deserialization
  - Removed copy of array values in binary deserialization
  - Removed copy of string values in json deserialization
  - Removed unhelpful allocation in json string value parsing
  - Removed copy of blob data on binary and json serialization
release-0.8.0.2
Lee Clagett 5 years ago
parent 411f1b0ee3
commit a9bdc6e4c4
7 changed files with 56 additions and 59 deletions
Show Stats Download Patch File Download Diff File

24
contrib/epee/include/serialization/keyvalue_serialization_overloads.h
Unescape View File

@ -46,24 +46,12 @@ namespace epee
namespace serialization namespace serialization
{ {
//-------------------------------------------------------------------------------------------------------------------
template<class t_type, class t_storage>
static bool serialize_t_val(const t_type& d, t_storage& stg, typename t_storage::hsection hparent_section, const char* pname)
{
return stg.set_value(pname, d, hparent_section);
}
//-------------------------------------------------------------------------------------------------------------------
template<class t_type, class t_storage>
static bool unserialize_t_val(t_type& d, t_storage& stg, typename t_storage::hsection hparent_section, const char* pname)
{
return stg.get_value(pname, d, hparent_section);
}
//------------------------------------------------------------------------------------------------------------------- //-------------------------------------------------------------------------------------------------------------------
template<class t_type, class t_storage> template<class t_type, class t_storage>
static bool serialize_t_val_as_blob(const t_type& d, t_storage& stg, typename t_storage::hsection hparent_section, const char* pname) static bool serialize_t_val_as_blob(const t_type& d, t_storage& stg, typename t_storage::hsection hparent_section, const char* pname)
{ {
std::string blob((const char *)&d, sizeof(d)); std::string blob((const char *)&d, sizeof(d));
return stg.set_value(pname, blob, hparent_section); return stg.set_value(pname, std::move(blob), hparent_section);
} }
//------------------------------------------------------------------------------------------------------------------- //-------------------------------------------------------------------------------------------------------------------
template<class t_type, class t_storage> template<class t_type, class t_storage>
@ -114,13 +102,15 @@ namespace epee
template<class stl_container, class t_storage> template<class stl_container, class t_storage>
static bool serialize_stl_container_t_val (const stl_container& container, t_storage& stg, typename t_storage::hsection hparent_section, const char* pname) static bool serialize_stl_container_t_val (const stl_container& container, t_storage& stg, typename t_storage::hsection hparent_section, const char* pname)
{ {
using value_type = typename stl_container::value_type;
if(!container.size()) return true; if(!container.size()) return true;
typename stl_container::const_iterator it = container.begin(); typename stl_container::const_iterator it = container.begin();
typename t_storage::harray hval_array = stg.insert_first_value(pname, *it, hparent_section); typename t_storage::harray hval_array = stg.insert_first_value(pname, value_type(*it), hparent_section);
CHECK_AND_ASSERT_MES(hval_array, false, "failed to insert first value to storage"); CHECK_AND_ASSERT_MES(hval_array, false, "failed to insert first value to storage");
it++; it++;
for(;it!= container.end();it++) for(;it!= container.end();it++)
stg.insert_next_value(hval_array, *it); stg.insert_next_value(hval_array, value_type(*it));
return true; return true;
} }
@ -149,7 +139,7 @@ namespace epee
*p_elem = v; *p_elem = v;
p_elem++; p_elem++;
} }
return stg.set_value(pname, mb, hparent_section); return stg.set_value(pname, std::move(mb), hparent_section);
} }
//-------------------------------------------------------------------------------------------------------------------- //--------------------------------------------------------------------------------------------------------------------
template<class stl_container, class t_storage> template<class stl_container, class t_storage>
@ -221,7 +211,7 @@ namespace epee
template<class t_type, class t_storage> template<class t_type, class t_storage>
static bool kv_serialize(const t_type& d, t_storage& stg, typename t_storage::hsection hparent_section, const char* pname) static bool kv_serialize(const t_type& d, t_storage& stg, typename t_storage::hsection hparent_section, const char* pname)
{ {
return stg.set_value(pname, d, hparent_section); return stg.set_value(pname, t_type(d), hparent_section);
} }
//------------------------------------------------------------------------------------------------------------------- //-------------------------------------------------------------------------------------------------------------------
template<class t_type, class t_storage> template<class t_type, class t_storage>

1
contrib/epee/include/storages/parserse_base_utils.h
Unescape View File

@ -157,7 +157,6 @@ namespace misc_utils
while (fi != buf_end && ((lut[(uint8_t)*fi] & 32)) == 0) while (fi != buf_end && ((lut[(uint8_t)*fi] & 32)) == 0)
++fi; ++fi;
val.assign(it, fi); val.assign(it, fi);
val.reserve(std::distance(star_end_string, buf_end));
it = fi; it = fi;
for(;it != buf_end;it++) for(;it != buf_end;it++)
{ {

52
contrib/epee/include/storages/portable_storage.h
Unescape View File

@ -28,6 +28,8 @@
#pragma once #pragma once
#include <type_traits>
#include "misc_language.h" #include "misc_language.h"
#include "portable_storage_base.h" #include "portable_storage_base.h"
#include "portable_storage_to_bin.h" #include "portable_storage_to_bin.h"
@ -59,7 +61,7 @@ namespace epee
bool get_value(const std::string& value_name, t_value& val, hsection hparent_section); bool get_value(const std::string& value_name, t_value& val, hsection hparent_section);
bool get_value(const std::string& value_name, storage_entry& val, hsection hparent_section); bool get_value(const std::string& value_name, storage_entry& val, hsection hparent_section);
template<class t_value> template<class t_value>
bool set_value(const std::string& value_name, const t_value& target, hsection hparent_section); bool set_value(const std::string& value_name, t_value&& target, hsection hparent_section);
//serial access for arrays of values -------------------------------------- //serial access for arrays of values --------------------------------------
//values //values
@ -68,9 +70,9 @@ namespace epee
template<class t_value> template<class t_value>
bool get_next_value(harray hval_array, t_value& target); bool get_next_value(harray hval_array, t_value& target);
template<class t_value> template<class t_value>
harray insert_first_value(const std::string& value_name, const t_value& target, hsection hparent_section); harray insert_first_value(const std::string& value_name, t_value&& target, hsection hparent_section);
template<class t_value> template<class t_value>
bool insert_next_value(harray hval_array, const t_value& target); bool insert_next_value(harray hval_array, t_value&& target);
//sections //sections
harray get_first_section(const std::string& pSectionName, hsection& h_child_section, hsection hparent_section); harray get_first_section(const std::string& pSectionName, hsection& h_child_section, hsection hparent_section);
bool get_next_section(harray hSecArray, hsection& h_child_section); bool get_next_section(harray hSecArray, hsection& h_child_section);
@ -94,7 +96,7 @@ namespace epee
hsection get_root_section() {return &m_root;} hsection get_root_section() {return &m_root;}
storage_entry* find_storage_entry(const std::string& pentry_name, hsection psection); storage_entry* find_storage_entry(const std::string& pentry_name, hsection psection);
template<class entry_type> template<class entry_type>
storage_entry* insert_new_entry_get_storage_entry(const std::string& pentry_name, hsection psection, const entry_type& entry); storage_entry* insert_new_entry_get_storage_entry(const std::string& pentry_name, hsection psection, entry_type&& entry);
hsection insert_new_section(const std::string& pentry_name, hsection psection); hsection insert_new_section(const std::string& pentry_name, hsection psection);
@ -241,21 +243,22 @@ namespace epee
} }
//--------------------------------------------------------------------------------------------------------------- //---------------------------------------------------------------------------------------------------------------
template<class t_value> template<class t_value>
bool portable_storage::set_value(const std::string& value_name, const t_value& v, hsection hparent_section) bool portable_storage::set_value(const std::string& value_name, t_value&& v, hsection hparent_section)
{ {
BOOST_MPL_ASSERT(( boost::mpl::contains<boost::mpl::push_front<storage_entry::types, storage_entry>::type, t_value> )); using t_real_value = typename std::decay<t_value>::type;
BOOST_MPL_ASSERT(( boost::mpl::contains<boost::mpl::push_front<storage_entry::types, storage_entry>::type, t_real_value> ));
TRY_ENTRY(); TRY_ENTRY();
if(!hparent_section) if(!hparent_section)
hparent_section = &m_root; hparent_section = &m_root;
storage_entry* pentry = find_storage_entry(value_name, hparent_section); storage_entry* pentry = find_storage_entry(value_name, hparent_section);
if(!pentry) if(!pentry)
{ {
pentry = insert_new_entry_get_storage_entry(value_name, hparent_section, v); pentry = insert_new_entry_get_storage_entry(value_name, hparent_section, std::forward<t_value>(v));
if(!pentry) if(!pentry)
return false; return false;
return true; return true;
} }
*pentry = storage_entry(v); *pentry = std::forward<t_value>(v);
return true; return true;
CATCH_ENTRY("portable_storage::template<>set_value", false); CATCH_ENTRY("portable_storage::template<>set_value", false);
} }
@ -274,11 +277,12 @@ namespace epee
} }
//--------------------------------------------------------------------------------------------------------------- //---------------------------------------------------------------------------------------------------------------
template<class entry_type> template<class entry_type>
storage_entry* portable_storage::insert_new_entry_get_storage_entry(const std::string& pentry_name, hsection psection, const entry_type& entry) storage_entry* portable_storage::insert_new_entry_get_storage_entry(const std::string& pentry_name, hsection psection, entry_type&& entry)
{ {
static_assert(std::is_rvalue_reference<entry_type&&>(), "unexpected copy of value");
TRY_ENTRY(); TRY_ENTRY();
CHECK_AND_ASSERT(psection, nullptr); CHECK_AND_ASSERT(psection, nullptr);
auto ins_res = psection->m_entries.insert(std::pair<std::string, storage_entry>(pentry_name, entry)); auto ins_res = psection->m_entries.emplace(pentry_name, std::forward<entry_type>(entry));
return &ins_res.first->second; return &ins_res.first->second;
CATCH_ENTRY("portable_storage::insert_new_entry_get_storage_entry", nullptr); CATCH_ENTRY("portable_storage::insert_new_entry_get_storage_entry", nullptr);
} }
@ -362,41 +366,45 @@ namespace epee
} }
//--------------------------------------------------------------------------------------------------------------- //---------------------------------------------------------------------------------------------------------------
template<class t_value> template<class t_value>
harray portable_storage::insert_first_value(const std::string& value_name, const t_value& target, hsection hparent_section) harray portable_storage::insert_first_value(const std::string& value_name, t_value&& target, hsection hparent_section)
{ {
using t_real_value = typename std::decay<t_value>::type;
static_assert(std::is_rvalue_reference<t_value&&>(), "unexpected copy of value");
TRY_ENTRY(); TRY_ENTRY();
if(!hparent_section) hparent_section = &m_root; if(!hparent_section) hparent_section = &m_root;
storage_entry* pentry = find_storage_entry(value_name, hparent_section); storage_entry* pentry = find_storage_entry(value_name, hparent_section);
if(!pentry) if(!pentry)
{ {
pentry = insert_new_entry_get_storage_entry(value_name, hparent_section, array_entry(array_entry_t<t_value>())); pentry = insert_new_entry_get_storage_entry(value_name, hparent_section, array_entry(array_entry_t<t_real_value>()));
if(!pentry) if(!pentry)
return nullptr; return nullptr;
} }
if(pentry->type() != typeid(array_entry)) if(pentry->type() != typeid(array_entry))
*pentry = storage_entry(array_entry(array_entry_t<t_value>())); *pentry = storage_entry(array_entry(array_entry_t<t_real_value>()));
array_entry& arr = boost::get<array_entry>(*pentry); array_entry& arr = boost::get<array_entry>(*pentry);
if(arr.type() != typeid(array_entry_t<t_value>)) if(arr.type() != typeid(array_entry_t<t_real_value>))
arr = array_entry(array_entry_t<t_value>()); arr = array_entry(array_entry_t<t_real_value>());
array_entry_t<t_value>& arr_typed = boost::get<array_entry_t<t_value> >(arr); array_entry_t<t_real_value>& arr_typed = boost::get<array_entry_t<t_real_value> >(arr);
arr_typed.insert_first_val(target); arr_typed.insert_first_val(std::forward<t_value>(target));
return &arr; return &arr;
CATCH_ENTRY("portable_storage::insert_first_value", nullptr); CATCH_ENTRY("portable_storage::insert_first_value", nullptr);
} }
//--------------------------------------------------------------------------------------------------------------- //---------------------------------------------------------------------------------------------------------------
template<class t_value> template<class t_value>
bool portable_storage::insert_next_value(harray hval_array, const t_value& target) bool portable_storage::insert_next_value(harray hval_array, t_value&& target)
{ {
using t_real_value = typename std::decay<t_value>::type;
static_assert(std::is_rvalue_reference<t_value&&>(), "unexpected copy of value");
TRY_ENTRY(); TRY_ENTRY();
CHECK_AND_ASSERT(hval_array, false); CHECK_AND_ASSERT(hval_array, false);
CHECK_AND_ASSERT_MES(hval_array->type() == typeid(array_entry_t<t_value>), CHECK_AND_ASSERT_MES(hval_array->type() == typeid(array_entry_t<t_real_value>),
false, "unexpected type in insert_next_value: " << typeid(array_entry_t<t_value>).name()); false, "unexpected type in insert_next_value: " << typeid(array_entry_t<t_real_value>).name());
array_entry_t<t_value>& arr_typed = boost::get<array_entry_t<t_value> >(*hval_array); array_entry_t<t_real_value>& arr_typed = boost::get<array_entry_t<t_real_value> >(*hval_array);
arr_typed.insert_next_value(target); arr_typed.insert_next_value(std::forward<t_value>(target));
return true; return true;
CATCH_ENTRY("portable_storage::insert_next_value", false); CATCH_ENTRY("portable_storage::insert_next_value", false);
} }

8
contrib/epee/include/storages/portable_storage_base.h
Unescape View File

@ -111,16 +111,16 @@ namespace epee
return (t_entry_type*)&(*(m_it++));//fuckoff return (t_entry_type*)&(*(m_it++));//fuckoff
} }
t_entry_type& insert_first_val(const t_entry_type& v) t_entry_type& insert_first_val(t_entry_type&& v)
{ {
m_array.clear(); m_array.clear();
m_it = m_array.end(); m_it = m_array.end();
return insert_next_value(v); return insert_next_value(std::move(v));
} }
t_entry_type& insert_next_value(const t_entry_type& v) t_entry_type& insert_next_value(t_entry_type&& v)
{ {
m_array.push_back(v); m_array.push_back(std::move(v));
return m_array.back(); return m_array.back();
} }

6
contrib/epee/include/storages/portable_storage_from_bin.h
Unescape View File

@ -143,7 +143,7 @@ namespace epee
//TODO: add some optimization here later //TODO: add some optimization here later
while(size--) while(size--)
sa.m_array.push_back(read<type_name>()); sa.m_array.push_back(read<type_name>());
return storage_entry(array_entry(sa)); return storage_entry(array_entry(std::move(sa)));
} }
inline inline
@ -213,7 +213,7 @@ namespace epee
{ {
RECURSION_LIMITATION(); RECURSION_LIMITATION();
section s;//use extra variable due to vs bug, line "storage_entry se(section()); " can't be compiled in visual studio section s;//use extra variable due to vs bug, line "storage_entry se(section()); " can't be compiled in visual studio
storage_entry se(s); storage_entry se(std::move(s));
section& section_entry = boost::get<section>(se); section& section_entry = boost::get<section>(se);
read(section_entry); read(section_entry);
return se; return se;
@ -268,7 +268,7 @@ namespace epee
//read section name string //read section name string
std::string sec_name; std::string sec_name;
read_sec_name(sec_name); read_sec_name(sec_name);
sec.m_entries.insert(std::make_pair(sec_name, load_storage_entry())); sec.m_entries.emplace(std::move(sec_name), load_storage_entry());
} }
} }
inline inline

18
contrib/epee/include/storages/portable_storage_from_json.h
Unescape View File

@ -128,20 +128,20 @@ namespace epee
errno = 0; errno = 0;
int64_t nval = strtoll(val.data(), NULL, 10); int64_t nval = strtoll(val.data(), NULL, 10);
if (errno) throw std::runtime_error("Invalid number: " + std::string(val)); if (errno) throw std::runtime_error("Invalid number: " + std::string(val));
stg.set_value(name, nval, current_section); stg.set_value(name, int64_t(nval), current_section);
}else }else
{ {
errno = 0; errno = 0;
uint64_t nval = strtoull(val.data(), NULL, 10); uint64_t nval = strtoull(val.data(), NULL, 10);
if (errno) throw std::runtime_error("Invalid number: " + std::string(val)); if (errno) throw std::runtime_error("Invalid number: " + std::string(val));
stg.set_value(name, nval, current_section); stg.set_value(name, uint64_t(nval), current_section);
} }
}else }else
{ {
errno = 0; errno = 0;
double nval = strtod(val.data(), NULL); double nval = strtod(val.data(), NULL);
if (errno) throw std::runtime_error("Invalid number: " + std::string(val)); if (errno) throw std::runtime_error("Invalid number: " + std::string(val));
stg.set_value(name, nval, current_section); stg.set_value(name, double(nval), current_section);
} }
state = match_state_wonder_after_value; state = match_state_wonder_after_value;
}else if(isalpha(*it) ) }else if(isalpha(*it) )
@ -219,13 +219,13 @@ namespace epee
errno = 0; errno = 0;
int64_t nval = strtoll(val.data(), NULL, 10); int64_t nval = strtoll(val.data(), NULL, 10);
if (errno) throw std::runtime_error("Invalid number: " + std::string(val)); if (errno) throw std::runtime_error("Invalid number: " + std::string(val));
h_array = stg.insert_first_value(name, nval, current_section); h_array = stg.insert_first_value(name, int64_t(nval), current_section);
}else }else
{ {
errno = 0; errno = 0;
uint64_t nval = strtoull(val.data(), NULL, 10); uint64_t nval = strtoull(val.data(), NULL, 10);
if (errno) throw std::runtime_error("Invalid number: " + std::string(val)); if (errno) throw std::runtime_error("Invalid number: " + std::string(val));
h_array = stg.insert_first_value(name, nval, current_section); h_array = stg.insert_first_value(name, uint64_t(nval), current_section);
} }
CHECK_AND_ASSERT_THROW_MES(h_array, " failed to insert values section entry"); CHECK_AND_ASSERT_THROW_MES(h_array, " failed to insert values section entry");
}else }else
@ -233,7 +233,7 @@ namespace epee
errno = 0; errno = 0;
double nval = strtod(val.data(), NULL); double nval = strtod(val.data(), NULL);
if (errno) throw std::runtime_error("Invalid number: " + std::string(val)); if (errno) throw std::runtime_error("Invalid number: " + std::string(val));
h_array = stg.insert_first_value(name, nval, current_section); h_array = stg.insert_first_value(name, double(nval), current_section);
CHECK_AND_ASSERT_THROW_MES(h_array, " failed to insert values section entry"); CHECK_AND_ASSERT_THROW_MES(h_array, " failed to insert values section entry");
} }
@ -310,20 +310,20 @@ namespace epee
errno = 0; errno = 0;
int64_t nval = strtoll(val.data(), NULL, 10); int64_t nval = strtoll(val.data(), NULL, 10);
if (errno) throw std::runtime_error("Invalid number: " + std::string(val)); if (errno) throw std::runtime_error("Invalid number: " + std::string(val));
insert_res = stg.insert_next_value(h_array, nval); insert_res = stg.insert_next_value(h_array, int64_t(nval));
}else }else
{ {
errno = 0; errno = 0;
uint64_t nval = strtoull(val.data(), NULL, 10); uint64_t nval = strtoull(val.data(), NULL, 10);
if (errno) throw std::runtime_error("Invalid number: " + std::string(val)); if (errno) throw std::runtime_error("Invalid number: " + std::string(val));
insert_res = stg.insert_next_value(h_array, nval); insert_res = stg.insert_next_value(h_array, uint64_t(nval));
} }
}else }else
{ {
errno = 0; errno = 0;
double nval = strtod(val.data(), NULL); double nval = strtod(val.data(), NULL);
if (errno) throw std::runtime_error("Invalid number: " + std::string(val)); if (errno) throw std::runtime_error("Invalid number: " + std::string(val));
insert_res = stg.insert_next_value(h_array, nval); insert_res = stg.insert_next_value(h_array, double(nval));
} }
CHECK_AND_ASSERT_THROW_MES(insert_res, "Failed to insert next value"); CHECK_AND_ASSERT_THROW_MES(insert_res, "Failed to insert next value");
state = match_state_array_after_value; state = match_state_array_after_value;

6
tests/unit_tests/net.cpp
Unescape View File

@ -271,7 +271,7 @@ TEST(tor_address, epee_serializev_v2)
EXPECT_EQ(std::strlen(v2_onion), host.size()); EXPECT_EQ(std::strlen(v2_onion), host.size());
host.push_back('k'); host.push_back('k');
EXPECT_TRUE(stg.set_value("host", host, stg.open_section("tor", nullptr, false))); EXPECT_TRUE(stg.set_value("host", std::move(host), stg.open_section("tor", nullptr, false)));
EXPECT_TRUE(command.load(stg)); // poor error reporting from `KV_SERIALIZE` EXPECT_TRUE(command.load(stg)); // poor error reporting from `KV_SERIALIZE`
} }
@ -322,7 +322,7 @@ TEST(tor_address, epee_serializev_v3)
EXPECT_EQ(std::strlen(v3_onion), host.size()); EXPECT_EQ(std::strlen(v3_onion), host.size());
host.push_back('k'); host.push_back('k');
EXPECT_TRUE(stg.set_value("host", host, stg.open_section("tor", nullptr, false))); EXPECT_TRUE(stg.set_value("host", std::move(host), stg.open_section("tor", nullptr, false)));
EXPECT_TRUE(command.load(stg)); // poor error reporting from `KV_SERIALIZE` EXPECT_TRUE(command.load(stg)); // poor error reporting from `KV_SERIALIZE`
} }
@ -373,7 +373,7 @@ TEST(tor_address, epee_serialize_unknown)
EXPECT_EQ(std::strlen(net::tor_address::unknown_str()), host.size()); EXPECT_EQ(std::strlen(net::tor_address::unknown_str()), host.size());
host.push_back('k'); host.push_back('k');
EXPECT_TRUE(stg.set_value("host", host, stg.open_section("tor", nullptr, false))); EXPECT_TRUE(stg.set_value("host", std::move(host), stg.open_section("tor", nullptr, false)));
EXPECT_TRUE(command.load(stg)); // poor error reporting from `KV_SERIALIZE` EXPECT_TRUE(command.load(stg)); // poor error reporting from `KV_SERIALIZE`
} }

Write Preview
Loading…
Cancel
Save