diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 35ddff7a1..ebd742c3d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -16,17 +16,19 @@ jobs: build-windows: runs-on: windows-latest + defaults: + run: + shell: msys2 {0} steps: - uses: actions/checkout@v1 with: submodules: recursive - - uses: numworks/setup-msys2@v1 - - name: update pacman - run: msys2do pacman -Syu --noconfirm - - name: install monero dependencies - run: msys2do pacman -S --noconfirm mingw-w64-x86_64-toolchain make mingw-w64-x86_64-cmake mingw-w64-x86_64-boost mingw-w64-x86_64-openssl mingw-w64-x86_64-zeromq mingw-w64-x86_64-libsodium mingw-w64-x86_64-hidapi mingw-w64-x86_64-protobuf-c mingw-w64-x86_64-libusb git + - uses: eine/setup-msys2@v1 + with: + update: true + install: mingw-w64-x86_64-toolchain make mingw-w64-x86_64-cmake mingw-w64-x86_64-boost mingw-w64-x86_64-openssl mingw-w64-x86_64-zeromq mingw-w64-x86_64-libsodium mingw-w64-x86_64-hidapi mingw-w64-x86_64-protobuf-c mingw-w64-x86_64-libusb git - name: build - run: msys2do make release-static-win64 -j2 + run: make release-static-win64 -j2 build-ubuntu: runs-on: ubuntu-latest diff --git a/contrib/epee/include/md5_l.inl b/contrib/epee/include/md5_l.inl index 8e339e006..cb2bd54f9 100644 --- a/contrib/epee/include/md5_l.inl +++ b/contrib/epee/include/md5_l.inl @@ -277,7 +277,7 @@ namespace md5 /* Zeroize sensitive information. */ - MD5_memset ((POINTER)context, 0, sizeof (*context)); + memwipe ((POINTER)context, sizeof (*context)); } /* MD5 basic transformation. Transforms state based on block. @@ -369,7 +369,7 @@ namespace md5 /* Zeroize sensitive information. */ - MD5_memset ((POINTER)x, 0, sizeof (x)); + memwipe ((POINTER)x, sizeof (x)); } /* Note: Replace "for loop" with standard memcpy if possible. @@ -431,9 +431,9 @@ namespace md5 MD5Update(&hmac->octx, k_opad, 64); /* apply outer pad */ /* scrub the pads and key context (if used) */ - MD5_memset( (POINTER)&k_ipad, 0, sizeof(k_ipad)); - MD5_memset( (POINTER)&k_opad, 0, sizeof(k_opad)); - MD5_memset( (POINTER)&tk, 0, sizeof(tk)); + memwipe( (POINTER)&k_ipad, sizeof(k_ipad)); + memwipe( (POINTER)&k_opad, sizeof(k_opad)); + memwipe( (POINTER)&tk, sizeof(tk)); /* and we're done. */ } @@ -459,7 +459,7 @@ namespace md5 state->istate[lupe] = htonl(hmac.ictx.state[lupe]); state->ostate[lupe] = htonl(hmac.octx.state[lupe]); } - MD5_memset( (POINTER)&hmac, 0, sizeof(hmac)); + memwipe( (POINTER)&hmac, sizeof(hmac)); } diff --git a/contrib/epee/src/byte_slice.cpp b/contrib/epee/src/byte_slice.cpp index 12cc83e6c..faf7689be 100644 --- a/contrib/epee/src/byte_slice.cpp +++ b/contrib/epee/src/byte_slice.cpp @@ -133,10 +133,13 @@ namespace epee template byte_slice::byte_slice(const adapt_buffer, T&& buffer) - : storage_(nullptr), portion_(to_byte_span(to_span(buffer))) + : storage_(nullptr), portion_(nullptr) { if (!buffer.empty()) + { storage_ = allocate_slice>(0, std::move(buffer)); + portion_ = to_byte_span(to_span(static_cast *>(storage_.get())->buffer)); + } } byte_slice::byte_slice(std::initializer_list> sources) diff --git a/contrib/epee/src/net_ssl.cpp b/contrib/epee/src/net_ssl.cpp index 2cf9ea2b6..a09e82771 100644 --- a/contrib/epee/src/net_ssl.cpp +++ b/contrib/epee/src/net_ssl.cpp @@ -511,7 +511,7 @@ bool ssl_options_t::handshake( // autodetect will reconnect without SSL - warn and keep connection encrypted if (support != ssl_support_t::e_ssl_support_autodetect) { - MERROR("SSL certificate is not in the allowed list, connection droppped"); + MERROR("SSL certificate is not in the allowed list, connection dropped"); return false; } MWARNING("SSL peer has not been verified"); diff --git a/external/easylogging++/easylogging++.cc b/external/easylogging++/easylogging++.cc index 8439bec0b..0d748c225 100644 --- a/external/easylogging++/easylogging++.cc +++ b/external/easylogging++/easylogging++.cc @@ -2475,6 +2475,100 @@ void DefaultLogDispatchCallback::handle(const LogDispatchData* data) { } } + +template +static inline std::string utf8canonical(const std::string &s, Transform t = [](wint_t c)->wint_t { return c; }) +{ + std::string sc = ""; + size_t avail = s.size(); + const char *ptr = s.data(); + wint_t cp = 0; + int bytes = 1; + char wbuf[8], *wptr; + while (avail--) + { + if ((*ptr & 0x80) == 0) + { + cp = *ptr++; + bytes = 1; + } + else if ((*ptr & 0xe0) == 0xc0) + { + if (avail < 1) + throw std::runtime_error("Invalid UTF-8"); + cp = (*ptr++ & 0x1f) << 6; + cp |= *ptr++ & 0x3f; + --avail; + bytes = 2; + } + else if ((*ptr & 0xf0) == 0xe0) + { + if (avail < 2) + throw std::runtime_error("Invalid UTF-8"); + cp = (*ptr++ & 0xf) << 12; + cp |= (*ptr++ & 0x3f) << 6; + cp |= *ptr++ & 0x3f; + avail -= 2; + bytes = 3; + } + else if ((*ptr & 0xf8) == 0xf0) + { + if (avail < 3) + throw std::runtime_error("Invalid UTF-8"); + cp = (*ptr++ & 0x7) << 18; + cp |= (*ptr++ & 0x3f) << 12; + cp |= (*ptr++ & 0x3f) << 6; + cp |= *ptr++ & 0x3f; + avail -= 3; + bytes = 4; + } + else + throw std::runtime_error("Invalid UTF-8"); + + cp = t(cp); + if (cp <= 0x7f) + bytes = 1; + else if (cp <= 0x7ff) + bytes = 2; + else if (cp <= 0xffff) + bytes = 3; + else if (cp <= 0x10ffff) + bytes = 4; + else + throw std::runtime_error("Invalid code point UTF-8 transformation"); + + wptr = wbuf; + switch (bytes) + { + case 1: *wptr++ = cp; break; + case 2: *wptr++ = 0xc0 | (cp >> 6); *wptr++ = 0x80 | (cp & 0x3f); break; + case 3: *wptr++ = 0xe0 | (cp >> 12); *wptr++ = 0x80 | ((cp >> 6) & 0x3f); *wptr++ = 0x80 | (cp & 0x3f); break; + case 4: *wptr++ = 0xf0 | (cp >> 18); *wptr++ = 0x80 | ((cp >> 12) & 0x3f); *wptr++ = 0x80 | ((cp >> 6) & 0x3f); *wptr++ = 0x80 | (cp & 0x3f); break; + default: throw std::runtime_error("Invalid UTF-8"); + } + *wptr = 0; + sc.append(wbuf, bytes); + cp = 0; + bytes = 1; + } + return sc; +} + +void sanitize(std::string &s) +{ + s = utf8canonical(s, [](wint_t c)->wint_t { + if (c == 9 || c == 10 || c == 13) + return c; + if (c < 0x20) + return '?'; + if (c == 0x7f) + return '?'; + if (c >= 0x80 && c <= 0x9f) + return '?'; + return c; + }); +} + void DefaultLogDispatchCallback::dispatch(base::type::string_t&& rawLinePrefix, base::type::string_t&& rawLinePayload, base::type::string_t&& logLine) { if (m_data->dispatchAction() == base::DispatchAction::NormalLog || m_data->dispatchAction() == base::DispatchAction::FileOnlyLog) { if (m_data->logMessage()->logger()->m_typedConfigurations->toFile(m_data->logMessage()->level())) { @@ -2506,6 +2600,8 @@ void DefaultLogDispatchCallback::dispatch(base::type::string_t&& rawLinePrefix, m_data->logMessage()->logger()->logBuilder()->setColor(el::base::utils::colorFromLevel(level), false); ELPP_COUT << rawLinePrefix; m_data->logMessage()->logger()->logBuilder()->setColor(color == el::Color::Default ? el::base::utils::colorFromLevel(level): color, color != el::Color::Default); + try { sanitize(rawLinePayload); } + catch (const std::exception &e) { rawLinePayload = ""; } ELPP_COUT << rawLinePayload; m_data->logMessage()->logger()->logBuilder()->setColor(el::Color::Default, false); ELPP_COUT << std::flush; diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index 33fa3c931..e21bc2f5f 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -4048,7 +4048,7 @@ void BlockchainLMDB::get_output_tx_and_index_from_global(const std::vector &amounts, const std::vector &offsets, std::vector &outputs, bool allow_partial) const { if (amounts.size() != 1 && amounts.size() != offsets.size()) - throw0(DB_ERROR("Invalid sizes of amounts and offets")); + throw0(DB_ERROR("Invalid sizes of amounts and offsets")); LOG_PRINT_L3("BlockchainLMDB::" << __func__); TIME_MEASURE_START(db3); diff --git a/src/common/CMakeLists.txt b/src/common/CMakeLists.txt index f06737b31..35b3555a2 100644 --- a/src/common/CMakeLists.txt +++ b/src/common/CMakeLists.txt @@ -86,7 +86,8 @@ set(common_private_headers updates.h aligned.h timings.h - combinator.h) + combinator.h + utf8.h) monero_private_headers(common ${common_private_headers}) diff --git a/src/common/utf8.h b/src/common/utf8.h new file mode 100644 index 000000000..60247f1b2 --- /dev/null +++ b/src/common/utf8.h @@ -0,0 +1,114 @@ +// Copyright (c) 2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include +#include +#include + +namespace tools +{ + template + inline T utf8canonical(const T &s, Transform t = [](wint_t c)->wint_t { return c; }) + { + T sc = ""; + size_t avail = s.size(); + const char *ptr = s.data(); + wint_t cp = 0; + int bytes = 1; + char wbuf[8], *wptr; + while (avail--) + { + if ((*ptr & 0x80) == 0) + { + cp = *ptr++; + bytes = 1; + } + else if ((*ptr & 0xe0) == 0xc0) + { + if (avail < 1) + throw std::runtime_error("Invalid UTF-8"); + cp = (*ptr++ & 0x1f) << 6; + cp |= *ptr++ & 0x3f; + --avail; + bytes = 2; + } + else if ((*ptr & 0xf0) == 0xe0) + { + if (avail < 2) + throw std::runtime_error("Invalid UTF-8"); + cp = (*ptr++ & 0xf) << 12; + cp |= (*ptr++ & 0x3f) << 6; + cp |= *ptr++ & 0x3f; + avail -= 2; + bytes = 3; + } + else if ((*ptr & 0xf8) == 0xf0) + { + if (avail < 3) + throw std::runtime_error("Invalid UTF-8"); + cp = (*ptr++ & 0x7) << 18; + cp |= (*ptr++ & 0x3f) << 12; + cp |= (*ptr++ & 0x3f) << 6; + cp |= *ptr++ & 0x3f; + avail -= 3; + bytes = 4; + } + else + throw std::runtime_error("Invalid UTF-8"); + + cp = t(cp); + if (cp <= 0x7f) + bytes = 1; + else if (cp <= 0x7ff) + bytes = 2; + else if (cp <= 0xffff) + bytes = 3; + else if (cp <= 0x10ffff) + bytes = 4; + else + throw std::runtime_error("Invalid code point UTF-8 transformation"); + + wptr = wbuf; + switch (bytes) + { + case 1: *wptr++ = cp; break; + case 2: *wptr++ = 0xc0 | (cp >> 6); *wptr++ = 0x80 | (cp & 0x3f); break; + case 3: *wptr++ = 0xe0 | (cp >> 12); *wptr++ = 0x80 | ((cp >> 6) & 0x3f); *wptr++ = 0x80 | (cp & 0x3f); break; + case 4: *wptr++ = 0xf0 | (cp >> 18); *wptr++ = 0x80 | ((cp >> 12) & 0x3f); *wptr++ = 0x80 | ((cp >> 6) & 0x3f); *wptr++ = 0x80 | (cp & 0x3f); break; + default: throw std::runtime_error("Invalid UTF-8"); + } + *wptr = 0; + sc.append(wbuf, bytes); + cp = 0; + bytes = 1; + } + return sc; + } +} diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index cc6f49fbe..830d45259 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -812,12 +812,20 @@ bool Blockchain::get_block_by_hash(const crypto::hash &h, block &blk, bool *orph // less blocks than desired if there aren't enough. difficulty_type Blockchain::get_difficulty_for_next_block() { + LOG_PRINT_L3("Blockchain::" << __func__); + + std::stringstream ss; + bool print = false; + + int done = 0; + ss << "get_difficulty_for_next_block: height " << m_db->height() << std::endl; if (m_fixed_difficulty) { return m_db->height() ? m_fixed_difficulty : 1; } - LOG_PRINT_L3("Blockchain::" << __func__); +start: + difficulty_type D = 0; crypto::hash top_hash = get_tail_id(); { @@ -826,25 +834,32 @@ difficulty_type Blockchain::get_difficulty_for_next_block() // something a bit out of date, but that's fine since anything which // requires the blockchain lock will have acquired it in the first place, // and it will be unlocked only when called from the getinfo RPC + ss << "Locked, tail id " << top_hash << ", cached is " << m_difficulty_for_next_block_top_hash << std::endl; if (top_hash == m_difficulty_for_next_block_top_hash) - return m_difficulty_for_next_block; + { + ss << "Same, using cached diff " << m_difficulty_for_next_block << std::endl; + D = m_difficulty_for_next_block; + } } CRITICAL_REGION_LOCAL(m_blockchain_lock); std::vector timestamps; std::vector difficulties; uint64_t height; - top_hash = get_tail_id(height); // get it again now that we have the lock - ++height; // top block height to blockchain height - + auto new_top_hash = get_tail_id(height); // get it again now that we have the lock + ++height; uint8_t version = get_current_hard_fork_version(); uint64_t difficulty_blocks_count = version >= 11 ? DIFFICULTY_BLOCKS_COUNT_V3 : version <= 10 && version >= 8 ? DIFFICULTY_BLOCKS_COUNT_V2 : DIFFICULTY_BLOCKS_COUNT; + if (!(new_top_hash == top_hash)) D=0; + ss << "Re-locked, height " << height << ", tail id " << new_top_hash << (new_top_hash == top_hash ? "" : " (different)") << std::endl; + top_hash = new_top_hash; // ND: Speedup // 1. Keep a list of the last 735 (or less) blocks that is used to compute difficulty, // then when the next block difficulty is queried, push the latest height data and // pop the oldest one from the list. This only requires 1x read per height instead // of doing 735 (DIFFICULTY_BLOCKS_COUNT). + bool check = false; if (m_timestamps_and_difficulties_height != 0 && ((height - m_timestamps_and_difficulties_height) == 1) && m_timestamps.size() >= difficulty_blocks_count) { uint64_t index = height - 1; @@ -859,8 +874,12 @@ difficulty_type Blockchain::get_difficulty_for_next_block() m_timestamps_and_difficulties_height = height; timestamps = m_timestamps; difficulties = m_difficulties; + check = true; } - else + //else + std::vector timestamps_from_cache = timestamps; + std::vector difficulties_from_cache = difficulties; + { uint64_t offset = height - std::min (height, static_cast(difficulty_blocks_count)); if (offset == 0) @@ -873,16 +892,40 @@ difficulty_type Blockchain::get_difficulty_for_next_block() timestamps.reserve(height - offset); difficulties.reserve(height - offset); } + ss << "Looking up " << (height - offset) << " from " << offset << std::endl; for (; offset < height; offset++) { timestamps.push_back(m_db->get_block_timestamp(offset)); difficulties.push_back(m_db->get_block_cumulative_difficulty(offset)); } + if (check) if (timestamps != timestamps_from_cache || difficulties !=difficulties_from_cache) + { + ss << "Inconsistency XXX:" << std::endl; + ss << "top hash: "<height(); + uint64_t sh = dbh < 10000 ? 0 : dbh - 10000; + ss << "History from -10k at :" << dbh << ", from " << sh << std::endl; + for (uint64_t h = sh; h < dbh; ++h) + { + uint64_t ts = m_db->get_block_timestamp(h); + difficulty_type d = m_db->get_block_cumulative_difficulty(h); + ss << " " << h << " " << ts << " " << d << std::endl; + } + print = true; + } m_timestamps_and_difficulties_height = height; m_timestamps = timestamps; m_difficulties = difficulties; } + size_t target = get_difficulty_target(); uint64_t T = DIFFICULTY_TARGET_V2; uint64_t N = DIFFICULTY_WINDOW_V3; @@ -911,6 +954,28 @@ difficulty_type Blockchain::get_difficulty_for_next_block() CRITICAL_REGION_LOCAL1(m_difficulty_lock); m_difficulty_for_next_block_top_hash = top_hash; m_difficulty_for_next_block = diff; + if (D && D != diff) + { + ss << "XXX Mismatch at " << height << "/" << top_hash << "/" << get_tail_id() << ": cached " << D << ", real " << diff << std::endl; + print = true; + } + + ++done; + if (done == 1 && D && D != diff) + { + print = true; + ss << "Might be a race. Let's see what happens if we try again..." << std::endl; + epee::misc_utils::sleep_no_w(100); + goto start; + } + ss << "Diff for " << top_hash << ": " << diff << std::endl; + if (print) + { + MGINFO("START DUMP"); + MGINFO(ss.str()); + MGINFO("END DUMP"); + MGINFO("Please send moneromooo on Freenode the contents of this log, from a couple dozen lines before START DUMP to END DUMP"); + } return diff; } //------------------------------------------------------------------ diff --git a/src/cryptonote_core/cryptonote_core.cpp b/src/cryptonote_core/cryptonote_core.cpp index 48b776b7a..f4c51078b 100644 --- a/src/cryptonote_core/cryptonote_core.cpp +++ b/src/cryptonote_core/cryptonote_core.cpp @@ -1721,7 +1721,6 @@ namespace cryptonote m_starter_message_showed = true; } - m_fork_moaner.do_call(boost::bind(&core::check_fork_time, this)); m_txpool_auto_relayer.do_call(boost::bind(&core::relay_txpool_transactions, this)); m_check_updates_interval.do_call(boost::bind(&core::check_updates, this)); m_check_disk_space_interval.do_call(boost::bind(&core::check_disk_space, this)); @@ -1732,29 +1731,6 @@ namespace cryptonote return true; } //----------------------------------------------------------------------------------------------- - bool core::check_fork_time() - { - if (m_nettype == FAKECHAIN) - return true; - - HardFork::State state = m_blockchain_storage.get_hard_fork_state(); - el::Level level; - switch (state) { - case HardFork::LikelyForked: - level = el::Level::Warning; - MCLOG_RED(level, "global", "**********************************************************************"); - MCLOG_RED(level, "global", "Last scheduled hard fork is too far in the past."); - MCLOG_RED(level, "global", "We are most likely forked from the network. Daemon update needed now."); - MCLOG_RED(level, "global", "**********************************************************************"); - break; - case HardFork::UpdateNeeded: - break; - default: - break; - } - return true; - } - //----------------------------------------------------------------------------------------------- uint8_t core::get_ideal_hard_fork_version() const { return get_blockchain_storage().get_ideal_hard_fork_version(); diff --git a/src/cryptonote_core/cryptonote_core.h b/src/cryptonote_core/cryptonote_core.h index 988f25ceb..68c5651f1 100644 --- a/src/cryptonote_core/cryptonote_core.h +++ b/src/cryptonote_core/cryptonote_core.h @@ -1000,18 +1000,6 @@ namespace cryptonote */ bool check_tx_inputs_keyimages_domain(const transaction& tx) const; - /** - * @brief checks HardFork status and prints messages about it - * - * Checks the status of HardFork and logs/prints if an update to - * the daemon is necessary. - * - * @note see Blockchain::get_hard_fork_state and HardFork::State - * - * @return true - */ - bool check_fork_time(); - /** * @brief attempts to relay any transactions in the mempool which need it * diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.h b/src/cryptonote_protocol/cryptonote_protocol_handler.h index e2ad3727f..3055474ef 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler.h +++ b/src/cryptonote_protocol/cryptonote_protocol_handler.h @@ -51,7 +51,8 @@ PUSH_WARNINGS DISABLE_VS_WARNINGS(4355) #define LOCALHOST_INT 2130706433 -#define CURRENCY_PROTOCOL_MAX_OBJECT_REQUEST_COUNT 500 +#define CURRENCY_PROTOCOL_MAX_OBJECT_REQUEST_COUNT 100 +static_assert(CURRENCY_PROTOCOL_MAX_OBJECT_REQUEST_COUNT >= BLOCKS_SYNCHRONIZING_DEFAULT_COUNT_PRE_V4, "Invalid CURRENCY_PROTOCOL_MAX_OBJECT_REQUEST_COUNT"); namespace cryptonote { diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.inl b/src/cryptonote_protocol/cryptonote_protocol_handler.inl index 773269022..2275fd2a8 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler.inl +++ b/src/cryptonote_protocol/cryptonote_protocol_handler.inl @@ -308,9 +308,9 @@ namespace cryptonote if (version >= 6 && version != hshd.top_version) { if (version < hshd.top_version && version == m_core.get_ideal_hard_fork_version()) - MCLOG_RED(el::Level::Warning, "global", context << " peer claims higher version than we think (" << + MDEBUG(context << " peer claims higher version than we think (" << (unsigned)hshd.top_version << " for " << (hshd.current_height - 1) << " instead of " << (unsigned)version << - ") - we may be forked from the network and a software upgrade may be needed"); + ") - we may be forked from the network and a software upgrade may be needed, or that peer is broken or malicious"); return false; } } @@ -793,6 +793,12 @@ namespace cryptonote int t_cryptonote_protocol_handler::handle_request_fluffy_missing_tx(int command, NOTIFY_REQUEST_FLUFFY_MISSING_TX::request& arg, cryptonote_connection_context& context) { MLOG_P2P_MESSAGE("Received NOTIFY_REQUEST_FLUFFY_MISSING_TX (" << arg.missing_tx_indices.size() << " txes), block hash " << arg.block_hash); + if (context.m_state == cryptonote_connection_context::state_before_handshake) + { + LOG_ERROR_CCONTEXT("Requested fluffy tx before handshake, dropping connection"); + drop_connection(context, false, false); + return 1; + } std::vector> local_blocks; std::vector local_txs; @@ -884,6 +890,8 @@ namespace cryptonote int t_cryptonote_protocol_handler::handle_notify_get_txpool_complement(int command, NOTIFY_GET_TXPOOL_COMPLEMENT::request& arg, cryptonote_connection_context& context) { MLOG_P2P_MESSAGE("Received NOTIFY_GET_TXPOOL_COMPLEMENT (" << arg.hashes.size() << " txes)"); + if(context.m_state != cryptonote_connection_context::state_normal) + return 1; std::vector> local_blocks; std::vector local_txs; @@ -987,6 +995,12 @@ namespace cryptonote template int t_cryptonote_protocol_handler::handle_request_get_objects(int command, NOTIFY_REQUEST_GET_OBJECTS::request& arg, cryptonote_connection_context& context) { + if (context.m_state == cryptonote_connection_context::state_before_handshake) + { + LOG_ERROR_CCONTEXT("Requested objects before handshake, dropping connection"); + drop_connection(context, false, false); + return 1; + } MLOG_P2P_MESSAGE("Received NOTIFY_REQUEST_GET_OBJECTS (" << arg.blocks.size() << " blocks)"); if (arg.blocks.size() > CURRENCY_PROTOCOL_MAX_OBJECT_REQUEST_COUNT) { @@ -1717,11 +1731,16 @@ skip: int t_cryptonote_protocol_handler::handle_request_chain(int command, NOTIFY_REQUEST_CHAIN::request& arg, cryptonote_connection_context& context) { MLOG_P2P_MESSAGE("Received NOTIFY_REQUEST_CHAIN (" << arg.block_ids.size() << " blocks"); + if (context.m_state == cryptonote_connection_context::state_before_handshake) + { + LOG_ERROR_CCONTEXT("Requested chain before handshake, dropping connection"); + drop_connection(context, false, false); + return 1; + } NOTIFY_RESPONSE_CHAIN_ENTRY::request r; if(!m_core.find_blockchain_supplement(arg.block_ids, !arg.prune, r)) { LOG_ERROR_CCONTEXT("Failed to handle NOTIFY_REQUEST_CHAIN."); - drop_connection(context, false, false); return 1; } MLOG_P2P_MESSAGE("-->>NOTIFY_RESPONSE_CHAIN_ENTRY: m_start_height=" << r.start_height << ", m_total_height=" << r.total_height << ", m_block_ids.size()=" << r.m_block_ids.size()); diff --git a/src/device/device_ledger.cpp b/src/device/device_ledger.cpp index 222a84d3f..0173b6ef7 100644 --- a/src/device/device_ledger.cpp +++ b/src/device/device_ledger.cpp @@ -1108,7 +1108,7 @@ namespace hw { for(size_t n=0; n < additional_derivations.size();++n) { if(derivation == additional_derivations[n]) { pkey = &additional_tx_pub_keys[n]; - MDEBUG("conceal derivation with additionnal tx pub key"); + MDEBUG("conceal derivation with additional tx pub key"); break; } } @@ -1640,20 +1640,20 @@ namespace hw { //if (tx_version > 1) { - ASSERT_X(recv_len>=32, "Not enought data from device"); + ASSERT_X(recv_len>=32, "Not enough data from device"); crypto::secret_key scalar1; this->receive_secret((unsigned char*)scalar1.data, offset); amount_keys.push_back(rct::sk2rct(scalar1)); recv_len -= 32; } - ASSERT_X(recv_len>=32, "Not enought data from device"); + ASSERT_X(recv_len>=32, "Not enough data from device"); memmove(out_eph_public_key.data, &this->buffer_recv[offset], 32); recv_len -= 32; offset += 32; if (need_additional_txkeys) { - ASSERT_X(recv_len>=32, "Not enought data from device"); + ASSERT_X(recv_len>=32, "Not enough data from device"); memmove(additional_txkey.pub.data, &this->buffer_recv[offset], 32); additional_tx_public_keys.push_back(additional_txkey.pub); offset += 32; diff --git a/src/mnemonics/language_base.h b/src/mnemonics/language_base.h index 7d2599e9a..ad09dc5fa 100644 --- a/src/mnemonics/language_base.h +++ b/src/mnemonics/language_base.h @@ -41,6 +41,7 @@ #include #include "misc_log_ex.h" #include "fnv1.h" +#include "common/utf8.h" /*! * \namespace Language @@ -73,78 +74,11 @@ namespace Language return prefix; } - template - inline T utf8canonical(const T &s) - { - T sc = ""; - size_t avail = s.size(); - const char *ptr = s.data(); - wint_t cp = 0; - int bytes = 1; - char wbuf[8], *wptr; - while (avail--) - { - if ((*ptr & 0x80) == 0) - { - cp = *ptr++; - bytes = 1; - } - else if ((*ptr & 0xe0) == 0xc0) - { - if (avail < 1) - throw std::runtime_error("Invalid UTF-8"); - cp = (*ptr++ & 0x1f) << 6; - cp |= *ptr++ & 0x3f; - --avail; - bytes = 2; - } - else if ((*ptr & 0xf0) == 0xe0) - { - if (avail < 2) - throw std::runtime_error("Invalid UTF-8"); - cp = (*ptr++ & 0xf) << 12; - cp |= (*ptr++ & 0x3f) << 6; - cp |= *ptr++ & 0x3f; - avail -= 2; - bytes = 3; - } - else if ((*ptr & 0xf8) == 0xf0) - { - if (avail < 3) - throw std::runtime_error("Invalid UTF-8"); - cp = (*ptr++ & 0x7) << 18; - cp |= (*ptr++ & 0x3f) << 12; - cp |= (*ptr++ & 0x3f) << 6; - cp |= *ptr++ & 0x3f; - avail -= 3; - bytes = 4; - } - else - throw std::runtime_error("Invalid UTF-8"); - - cp = std::towlower(cp); - wptr = wbuf; - switch (bytes) - { - case 1: *wptr++ = cp; break; - case 2: *wptr++ = 0xc0 | (cp >> 6); *wptr++ = 0x80 | (cp & 0x3f); break; - case 3: *wptr++ = 0xe0 | (cp >> 12); *wptr++ = 0x80 | ((cp >> 6) & 0x3f); *wptr++ = 0x80 | (cp & 0x3f); break; - case 4: *wptr++ = 0xf0 | (cp >> 18); *wptr++ = 0x80 | ((cp >> 12) & 0x3f); *wptr++ = 0x80 | ((cp >> 6) & 0x3f); *wptr++ = 0x80 | (cp & 0x3f); break; - default: throw std::runtime_error("Invalid UTF-8"); - } - *wptr = 0; - sc += T(wbuf, bytes); - cp = 0; - bytes = 1; - } - return sc; - } - struct WordHash { std::size_t operator()(const epee::wipeable_string &s) const { - const epee::wipeable_string sc = utf8canonical(s); + const epee::wipeable_string sc = tools::utf8canonical(s, [](wint_t c) -> wint_t { return std::towlower(c); }); return epee::fnv::FNV1a(sc.data(), sc.size()); } }; @@ -153,8 +87,8 @@ namespace Language { bool operator()(const epee::wipeable_string &s0, const epee::wipeable_string &s1) const { - const epee::wipeable_string s0c = utf8canonical(s0); - const epee::wipeable_string s1c = utf8canonical(s1); + const epee::wipeable_string s0c = tools::utf8canonical(s0, [](wint_t c) -> wint_t { return std::towlower(c); }); + const epee::wipeable_string s1c = tools::utf8canonical(s1, [](wint_t c) -> wint_t { return std::towlower(c); }); return s0c == s1c; } }; diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index abb0e37c0..83ae1e0f0 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -68,6 +68,11 @@ using namespace epee; #define DEFAULT_PAYMENT_DIFFICULTY 1000 #define DEFAULT_PAYMENT_CREDITS_PER_HASH 100 +#define RESTRICTED_BLOCK_HEADER_RANGE 1000 +#define RESTRICTED_TRANSACTIONS_COUNT 100 +#define RESTRICTED_SPENT_KEY_IMAGES_COUNT 5000 +#define RESTRICTED_BLOCK_COUNT 1000 + #define RPC_TRACKER(rpc) \ PERF_TIMER(rpc); \ RPCTracker tracker(#rpc, PERF_TIMER_NAME(rpc)) @@ -639,6 +644,13 @@ namespace cryptonote if (use_bootstrap_daemon_if_necessary(invoke_http_mode::BIN, "/getblocks_by_height.bin", req, res, r)) return r; + const bool restricted = m_restricted && ctx; + if (restricted && req.heights.size() > RESTRICTED_BLOCK_COUNT) + { + res.status = "Too many blocks requested in restricted mode"; + return true; + } + res.status = "Failed"; res.blocks.clear(); res.blocks.reserve(req.heights.size()); @@ -793,11 +805,17 @@ namespace cryptonote if (use_bootstrap_daemon_if_necessary(invoke_http_mode::JON, "/gettransactions", req, res, ok)) return ok; - CHECK_PAYMENT_MIN1(req, res, req.txs_hashes.size() * COST_PER_TX, false); - const bool restricted = m_restricted && ctx; const bool request_has_rpc_origin = ctx != NULL; + if (restricted && req.txs_hashes.size() > RESTRICTED_TRANSACTIONS_COUNT) + { + res.status = "Too many transactions requested in restricted mode"; + return true; + } + + CHECK_PAYMENT_MIN1(req, res, req.txs_hashes.size() * COST_PER_TX, false); + std::vector vh; for(const auto& tx_hex_str: req.txs_hashes) { @@ -1027,11 +1045,17 @@ namespace cryptonote if (use_bootstrap_daemon_if_necessary(invoke_http_mode::JON, "/is_key_image_spent", req, res, ok)) return ok; - CHECK_PAYMENT_MIN1(req, res, req.key_images.size() * COST_PER_KEY_IMAGE, false); - const bool restricted = m_restricted && ctx; const bool request_has_rpc_origin = ctx != NULL; + if (restricted && req.key_images.size() > RESTRICTED_SPENT_KEY_IMAGES_COUNT) + { + res.status = "Too many key images queried in restricted mode"; + return true; + } + + CHECK_PAYMENT_MIN1(req, res, req.key_images.size() * COST_PER_KEY_IMAGE, false); + std::vector key_images; for(const auto& ki_hex_str: req.key_images) { @@ -2035,6 +2059,14 @@ namespace cryptonote CHECK_PAYMENT_MIN1(req, res, COST_PER_BLOCK_HEADER, false); + const bool restricted = m_restricted && ctx; + if (restricted && req.hashes.size() > RESTRICTED_BLOCK_COUNT) + { + error_resp.code = CORE_RPC_ERROR_CODE_RESTRICTED; + error_resp.message = "Too many block headers requested in restricted mode"; + return false; + } + auto get = [this](const std::string &hash, bool fill_pow_hash, block_header_response &block_header, bool restricted, epee::json_rpc::error& error_resp) -> bool { crypto::hash block_hash; bool hash_parsed = parse_hash256(hash, block_hash); @@ -2070,7 +2102,6 @@ namespace cryptonote return true; }; - const bool restricted = m_restricted && ctx; if (!req.hash.empty()) { if (!get(req.hash, req.fill_pow_hash, res.block_header, restricted, error_resp)) @@ -2102,6 +2133,14 @@ namespace cryptonote error_resp.message = "Invalid start/end heights."; return false; } + const bool restricted = m_restricted && ctx; + if (restricted && req.end_height - req.start_height > RESTRICTED_BLOCK_HEADER_RANGE) + { + error_resp.code = CORE_RPC_ERROR_CODE_RESTRICTED; + error_resp.message = "Too many block headers requested."; + return false; + } + CHECK_PAYMENT_MIN1(req, res, (req.end_height - req.start_height + 1) * COST_PER_BLOCK_HEADER, false); for (uint64_t h = req.start_height; h <= req.end_height; ++h) { @@ -2128,7 +2167,6 @@ namespace cryptonote return false; } res.headers.push_back(block_header_response()); - const bool restricted = m_restricted && ctx; bool response_filled = fill_block_header_response(blk, false, block_height, block_hash, res.headers.back(), req.fill_pow_hash && !restricted); if (!response_filled) { diff --git a/src/rpc/core_rpc_server_error_codes.h b/src/rpc/core_rpc_server_error_codes.h index 2fd42f43f..98e40d05f 100644 --- a/src/rpc/core_rpc_server_error_codes.h +++ b/src/rpc/core_rpc_server_error_codes.h @@ -48,6 +48,7 @@ #define CORE_RPC_ERROR_CODE_PAYMENT_TOO_LOW -16 #define CORE_RPC_ERROR_CODE_DUPLICATE_PAYMENT -17 #define CORE_RPC_ERROR_CODE_STALE_PAYMENT -18 +#define CORE_RPC_ERROR_CODE_RESTRICTED -19 static inline const char *get_rpc_server_error_message(int64_t code) { @@ -70,6 +71,7 @@ static inline const char *get_rpc_server_error_message(int64_t code) case CORE_RPC_ERROR_CODE_PAYMENT_TOO_LOW: return "Payment too low"; case CORE_RPC_ERROR_CODE_DUPLICATE_PAYMENT: return "Duplicate payment"; case CORE_RPC_ERROR_CODE_STALE_PAYMENT: return "Stale payment"; + case CORE_RPC_ERROR_CODE_RESTRICTED: return "Parameters beyond restricted allowance"; default: MERROR("Unknown error: " << code); return "Unknown error"; } } diff --git a/src/rpc/rpc_payment.cpp b/src/rpc/rpc_payment.cpp index 741d92812..1d0d1aa8d 100644 --- a/src/rpc/rpc_payment.cpp +++ b/src/rpc/rpc_payment.cpp @@ -92,6 +92,7 @@ namespace cryptonote uint64_t rpc_payment::balance(const crypto::public_key &client, int64_t delta) { + boost::lock_guard lock(mutex); client_info &info = m_client_info[client]; // creates if not found uint64_t credits = info.credits; if (delta > 0 && credits > std::numeric_limits::max() - delta) @@ -107,6 +108,7 @@ namespace cryptonote bool rpc_payment::pay(const crypto::public_key &client, uint64_t ts, uint64_t payment, const std::string &rpc, bool same_ts, uint64_t &credits) { + boost::lock_guard lock(mutex); client_info &info = m_client_info[client]; // creates if not found if (ts < info.last_request_timestamp || (ts == info.last_request_timestamp && !same_ts)) { @@ -130,6 +132,7 @@ namespace cryptonote bool rpc_payment::get_info(const crypto::public_key &client, const std::function &get_block_template, cryptonote::blobdata &hashing_blob, uint64_t &seed_height, crypto::hash &seed_hash, const crypto::hash &top, uint64_t &diff, uint64_t &credits_per_hash_found, uint64_t &credits, uint32_t &cookie) { + boost::lock_guard lock(mutex); client_info &info = m_client_info[client]; // creates if not found const uint64_t now = time(NULL); bool need_template = top != info.top || now >= info.block_template_update_time + STALE_THRESHOLD; @@ -180,6 +183,7 @@ namespace cryptonote bool rpc_payment::submit_nonce(const crypto::public_key &client, uint64_t nonce, const crypto::hash &top, int64_t &error_code, std::string &error_message, uint64_t &credits, crypto::hash &hash, cryptonote::block &block, uint32_t cookie, bool &stale) { + boost::lock_guard lock(mutex); client_info &info = m_client_info[client]; // creates if not found if (cookie != info.cookie && cookie != info.cookie - 1) { @@ -272,6 +276,7 @@ namespace cryptonote bool rpc_payment::foreach(const std::function &f) const { + boost::lock_guard lock(mutex); for (std::unordered_map::const_iterator i = m_client_info.begin(); i != m_client_info.end(); ++i) { if (!f(i->first, i->second)) @@ -283,6 +288,7 @@ namespace cryptonote bool rpc_payment::load(std::string directory) { TRY_ENTRY(); + boost::lock_guard lock(mutex); m_directory = std::move(directory); std::string state_file_path = directory + "/" + RPC_PAYMENTS_DATA_FILENAME; MINFO("loading rpc payments data from " << state_file_path); @@ -313,6 +319,7 @@ namespace cryptonote bool rpc_payment::store(const std::string &directory_) const { TRY_ENTRY(); + boost::lock_guard lock(mutex); const std::string &directory = directory_.empty() ? m_directory : directory_; MDEBUG("storing rpc payments data to " << directory); if (!tools::create_directories_if_necessary(directory)) @@ -345,6 +352,7 @@ namespace cryptonote unsigned int rpc_payment::flush_by_age(time_t seconds) { + boost::lock_guard lock(mutex); unsigned int count = 0; const time_t now = time(NULL); time_t seconds0 = seconds; @@ -372,6 +380,7 @@ namespace cryptonote uint64_t rpc_payment::get_hashes(unsigned int seconds) const { + boost::lock_guard lock(mutex); const uint64_t now = time(NULL); uint64_t hashes = 0; for (std::map::const_reverse_iterator i = m_hashrate.crbegin(); i != m_hashrate.crend(); ++i) @@ -385,6 +394,7 @@ namespace cryptonote void rpc_payment::prune_hashrate(unsigned int seconds) { + boost::lock_guard lock(mutex); const uint64_t now = time(NULL); std::map::iterator i; for (i = m_hashrate.begin(); i != m_hashrate.end(); ++i) diff --git a/src/rpc/rpc_payment.h b/src/rpc/rpc_payment.h index 559736cf6..2d8d2c1d6 100644 --- a/src/rpc/rpc_payment.h +++ b/src/rpc/rpc_payment.h @@ -31,6 +31,7 @@ #include #include #include +#include #include #include "cryptonote_basic/blobdatatype.h" #include "cryptonote_basic/cryptonote_basic.h" @@ -139,6 +140,7 @@ namespace cryptonote uint64_t m_nonces_stale; uint64_t m_nonces_bad; uint64_t m_nonces_dupe; + mutable boost::mutex mutex; }; } diff --git a/src/serialization/json_object.cpp b/src/serialization/json_object.cpp index 6228b4bec..5c042aa7b 100644 --- a/src/serialization/json_object.cpp +++ b/src/serialization/json_object.cpp @@ -120,7 +120,7 @@ void read_hex(const rapidjson::Value& val, epee::span dest) throw WRONG_TYPE("string"); } - if (!epee::from_hex::to_buffer(dest, {val.GetString(), val.Size()})) + if (!epee::from_hex::to_buffer(dest, {val.GetString(), val.GetStringLength()})) { throw BAD_INPUT(); } diff --git a/src/simplewallet/simplewallet.cpp b/src/simplewallet/simplewallet.cpp index 1b1cbdb4c..5975d113a 100644 --- a/src/simplewallet/simplewallet.cpp +++ b/src/simplewallet/simplewallet.cpp @@ -255,6 +255,7 @@ namespace const char* USAGE_MMS_SET("mms set []"); const char* USAGE_MMS_SEND_SIGNER_CONFIG("mms send_signer_config"); const char* USAGE_MMS_START_AUTO_CONFIG("mms start_auto_config [