Merge pull request 'upstream' (#350) from wowario/wownero:up into master

Reviewed-on: #350

contrib/epee/src/readline_buffer.cpp

@@ -51,6 +51,7 @@ rdln::readline_buffer::readline_buffer()
 
 void rdln::readline_buffer::start()
 {
+  boost::lock_guard<boost::mutex> lock(sync_mutex);
   if(m_cout_buf != NULL)
     return;
   m_cout_buf = std::cout.rdbuf();
@@ -60,6 +61,7 @@ void rdln::readline_buffer::start()
 
 void rdln::readline_buffer::stop()
 {
+  boost::lock_guard<boost::mutex> lock(sync_mutex);
   if(m_cout_buf == NULL)
     return;
   std::cout.rdbuf(m_cout_buf);
@@ -88,9 +90,9 @@ rdln::linestatus rdln::readline_buffer::get_line(std::string& line) const
 
 void rdln::readline_buffer::set_prompt(const std::string& prompt)
 {
+  boost::lock_guard<boost::mutex> lock(sync_mutex);
   if(m_cout_buf == NULL)
     return;
-  boost::lock_guard<boost::mutex> lock(sync_mutex);
   rl_set_prompt(std::string(m_prompt_length, ' ').c_str());
   rl_redisplay();
   rl_set_prompt(prompt.c_str());
@@ -113,6 +115,12 @@ const std::vector<std::string>& rdln::readline_buffer::get_completions()
 int rdln::readline_buffer::sync()
 {
   boost::lock_guard<boost::mutex> lock(sync_mutex);
+
+  if (m_cout_buf == nullptr)
+  {
+    return -1;
+  }
+
 #if RL_READLINE_VERSION < 0x0700
   char lbuf[2] = {0,0};
   char *line = NULL;

src/cryptonote_protocol/cryptonote_protocol_handler.inl

@@ -1091,6 +1091,13 @@ namespace cryptonote
 
     //epee::net_utils::network_throttle_manager::get_global_throttle_inreq().logger_handle_net("log/dr-monero/net/req-all.data", sec, get_avg_block_size());
 
+    if(arg.blocks.empty())
+    {
+      LOG_ERROR_CCONTEXT("sent wrong NOTIFY_HAVE_OBJECTS: no blocks");
+      drop_connection(context, true, false);
+      ++m_sync_bad_spans_downloaded;
+      return 1;
+    }
     if(context.m_last_response_height > arg.current_blockchain_height)
     {
       LOG_ERROR_CCONTEXT("sent wrong NOTIFY_HAVE_OBJECTS: arg.m_current_blockchain_height=" << arg.current_blockchain_height

src/cryptonote_protocol/levin_notify.cpp

@@ -241,8 +241,8 @@ namespace levin
           strand(io_service),
           map(),
           channels(),
-          flush_time(std::chrono::steady_clock::time_point::max()),
           connection_count(0),
+          flush_callbacks(0),
           is_public(is_public),
           pad_txs(pad_txs),
           fluffing(false)
@@ -258,8 +258,8 @@ namespace levin
       boost::asio::io_service::strand strand;
       net::dandelionpp::connection_map map;//!< Tracks outgoing uuid's for noise channels or Dandelion++ stems
       std::deque<noise_channel> channels;  //!< Never touch after init; only update elements on `noise_channel.strand`
-      std::chrono::steady_clock::time_point flush_time; //!< Next expected Dandelion++ fluff flush
       std::atomic<std::size_t> connection_count; //!< Only update in strand, can be read at any time
+      std::uint32_t flush_callbacks;             //!< Number of active fluff flush callbacks queued
       const bool is_public;                      //!< Zone is public ipv4/ipv6 connections
       const bool pad_txs;                        //!< Pad txs to the next boundary for privacy
       bool fluffing;                             //!< Zone is in Dandelion++ fluff epoch
@@ -305,7 +305,6 @@ namespace levin
     struct fluff_flush
     {
       std::shared_ptr<detail::zone> zone_;
-      std::chrono::steady_clock::time_point flush_time_;
 
       static void queue(std::shared_ptr<detail::zone> zone, const std::chrono::steady_clock::time_point flush_time)
       {
@@ -313,28 +312,21 @@ namespace levin
         assert(zone->strand.running_in_this_thread());
 
         detail::zone& this_zone = *zone;
-        this_zone.flush_time = flush_time;
+        ++this_zone.flush_callbacks;
         this_zone.flush_txs.expires_at(flush_time);
-        this_zone.flush_txs.async_wait(this_zone.strand.wrap(fluff_flush{std::move(zone), flush_time}));
+        this_zone.flush_txs.async_wait(this_zone.strand.wrap(fluff_flush{std::move(zone)}));
       }
 
       void operator()(const boost::system::error_code error)
       {
-        if (!zone_ || !zone_->p2p)
+        if (!zone_ || !zone_->flush_callbacks || --zone_->flush_callbacks || !zone_->p2p)
           return;
 
         assert(zone_->strand.running_in_this_thread());
 
         const bool timer_error = bool(error);
-        if (timer_error)
-        {
-          if (error != boost::system::errc::operation_canceled)
-            throw boost::system::system_error{error, "fluff_flush timer failed"};
-
-          // new timer canceled this one set in future
-          if (zone_->flush_time < flush_time_)
-            return;
-        }
+        if (timer_error && error != boost::system::errc::operation_canceled)
+          throw boost::system::system_error{error, "fluff_flush timer failed"};
 
         const auto now = std::chrono::steady_clock::now();
         auto next_flush = std::chrono::steady_clock::time_point::max();
@@ -370,8 +362,6 @@ namespace levin
 
         if (next_flush != std::chrono::steady_clock::time_point::max())
           fluff_flush::queue(std::move(zone_), next_flush);
-        else
-          zone_->flush_time = next_flush; // signal that no timer is set
       }
     };
 
@@ -406,13 +396,11 @@ namespace levin
 
         MDEBUG("Queueing " << txs.size() << " transaction(s) for Dandelion++ fluffing");
 
-        bool available = false;
-        zone->p2p->foreach_connection([txs, now, &zone, &source, &in_duration, &out_duration, &next_flush, &available] (detail::p2p_context& context)
+        zone->p2p->foreach_connection([txs, now, &zone, &source, &in_duration, &out_duration, &next_flush] (detail::p2p_context& context)
         {
           // When i2p/tor, only fluff to outbound connections
           if (source != context.m_connection_id && (zone->is_public || !context.m_is_income))
           {
-            available = true;
             if (context.fluff_txs.empty())
               context.flush_time = now + (context.m_is_income ? in_duration() : out_duration());
 
@@ -424,10 +412,9 @@ namespace levin
           return true;
         });
 
-        if (!available)
+        if (next_flush == std::chrono::steady_clock::time_point::max())
           MWARNING("Unable to send transaction(s), no available connections");
-
-        if (next_flush < zone->flush_time)
+        else if (!zone->flush_callbacks || next_flush < zone->flush_txs.expires_at())
           fluff_flush::queue(std::move(zone), next_flush);
       }
     };
@@ -524,12 +511,7 @@ namespace levin
         if (!zone_ || !core_ || txs_.empty())
           return;
 
-        if (zone_->fluffing)
-        {
-          core_->on_transactions_relayed(epee::to_span(txs_), relay_method::fluff);
-          fluff_notify::run(std::move(zone_), epee::to_span(txs_), source_);
-        }
-        else // forward tx in stem
+        if (!zone_->fluffing)
         {
           core_->on_transactions_relayed(epee::to_span(txs_), relay_method::stem);
           for (int tries = 2; 0 < tries; tries--)
@@ -549,6 +531,9 @@ namespace levin
 
           MERROR("Unable to send transaction(s) via Dandelion++ stem");
         }
+
+        core_->on_transactions_relayed(epee::to_span(txs_), relay_method::fluff);
+        fluff_notify::run(std::move(zone_), epee::to_span(txs_), source_);
       }
     };
 

src/p2p/net_node.cpp

@@ -146,6 +146,7 @@ namespace nodetool
     const command_line::arg_descriptor<std::vector<std::string> > arg_p2p_seed_node   = {"seed-node", "Connect to a node to retrieve peer addresses, and disconnect"};
     const command_line::arg_descriptor<std::vector<std::string> > arg_tx_proxy = {"tx-proxy", "Send local txes through proxy: <network-type>,<socks-ip:port>[,max_connections][,disable_noise] i.e. \"tor,127.0.0.1:9050,100,disable_noise\""};
     const command_line::arg_descriptor<std::vector<std::string> > arg_anonymous_inbound = {"anonymous-inbound", "<hidden-service-address>,<[bind-ip:]port>[,max_connections] i.e. \"x.onion,127.0.0.1:18083,100\""};
+    const command_line::arg_descriptor<std::string> arg_ban_list = {"ban-list", "Specify ban list file, one IP address per line"};
     const command_line::arg_descriptor<bool> arg_p2p_hide_my_port   =    {"hide-my-port", "Do not announce yourself as peerlist candidate", false, true};
     const command_line::arg_descriptor<bool> arg_no_sync = {"no-sync", "Don't synchronize the blockchain with other peers", false};
 

src/p2p/net_node.h

@@ -520,6 +520,7 @@ namespace nodetool
     extern const command_line::arg_descriptor<std::vector<std::string> > arg_p2p_seed_node;
     extern const command_line::arg_descriptor<std::vector<std::string> > arg_tx_proxy;
     extern const command_line::arg_descriptor<std::vector<std::string> > arg_anonymous_inbound;
+    extern const command_line::arg_descriptor<std::string> arg_ban_list;
     extern const command_line::arg_descriptor<bool> arg_p2p_hide_my_port;
     extern const command_line::arg_descriptor<bool> arg_no_sync;
 

src/p2p/net_node.inl

@@ -71,6 +71,17 @@
 
 #define MIN_WANTED_SEED_NODES 12
 
+static inline boost::asio::ip::address_v4 make_address_v4_from_v6(const boost::asio::ip::address_v6& a)
+{
+  const auto &bytes = a.to_bytes();
+  uint32_t v4 = 0;
+  v4 = (v4 << 8) | bytes[12];
+  v4 = (v4 << 8) | bytes[13];
+  v4 = (v4 << 8) | bytes[14];
+  v4 = (v4 << 8) | bytes[15];
+  return boost::asio::ip::address_v4(v4);
+}
+
 namespace nodetool
 {
   template<class t_payload_net_handler>
@@ -106,6 +117,7 @@ namespace nodetool
     command_line::add_arg(desc, arg_p2p_seed_node);
     command_line::add_arg(desc, arg_tx_proxy);
     command_line::add_arg(desc, arg_anonymous_inbound);
+    command_line::add_arg(desc, arg_ban_list);
     command_line::add_arg(desc, arg_p2p_hide_my_port);
     command_line::add_arg(desc, arg_no_sync);
     command_line::add_arg(desc, arg_no_igd);
@@ -245,6 +257,10 @@ namespace nodetool
         zone.second.m_net_server.get_config_object().close(c);
 
       conns.clear();
+
+      peerlist_entry pe{};
+      pe.adr = addr;
+      zone.second.m_peerlist.remove_from_peer_white(pe);
     }
 
     MCLOG_CYAN(el::Level::Info, "global", "Host " << addr.host_str() << " blocked.");
@@ -441,6 +457,36 @@ namespace nodetool
         return false;
     }
 
+    if (!command_line::is_arg_defaulted(vm, arg_ban_list))
+    {
+      const std::string ban_list = command_line::get_arg(vm, arg_ban_list);
+
+      const boost::filesystem::path ban_list_path(ban_list);
+      boost::system::error_code ec;
+      if (!boost::filesystem::exists(ban_list_path, ec))
+      {
+        throw std::runtime_error("Can't find ban list file " + ban_list + " - " + ec.message());
+      }
+
+      std::string banned_ips;
+      if (!epee::file_io_utils::load_file_to_string(ban_list_path.string(), banned_ips))
+      {
+        throw std::runtime_error("Failed to read ban list file " + ban_list);
+      }
+
+      std::istringstream iss(banned_ips);
+      for (std::string line; std::getline(iss, line); )
+      {
+        const expect<epee::net_utils::network_address> parsed_addr = net::get_network_address(line, 0);
+        if (!parsed_addr)
+        {
+          MERROR("Invalid IP address: " << line << " - " << parsed_addr.error());
+          continue;
+        }
+        block_host(*parsed_addr, std::numeric_limits<time_t>::max());
+      }
+    }
+
     if(command_line::has_arg(vm, arg_p2p_hide_my_port))
       m_hide_my_port = true;
 
@@ -1228,7 +1274,10 @@ namespace nodetool
   template<class t_payload_net_handler>
   bool node_server<t_payload_net_handler>::try_to_connect_and_handshake_with_new_peer(const epee::net_utils::network_address& na, bool just_take_peerlist, uint64_t last_seen_stamp, PeerType peer_type, uint64_t first_seen_stamp)
   {
-    network_zone& zone = m_network_zones.at(na.get_zone());
+    const auto i = m_network_zones.find(na.get_zone());
+    if (i == m_network_zones.end())
+      return false;
+    network_zone& zone = i->second;
     if (zone.m_connect == nullptr) // outgoing connections in zone not possible
       return false;
 
@@ -1428,17 +1477,44 @@ namespace nodetool
             const uint32_t actual_ip = na.as<const epee::net_utils::ipv4_network_address>().ip();
             classB.insert(actual_ip & 0x0000ffff);
           }
+          else if (cntxt.m_remote_address.get_type_id() == epee::net_utils::ipv6_network_address::get_type_id())
+          {
+            const epee::net_utils::network_address na = cntxt.m_remote_address;
+            const boost::asio::ip::address_v6 &actual_ip = na.as<const epee::net_utils::ipv6_network_address>().ip();
+            if (actual_ip.is_v4_mapped())
+            {
+              boost::asio::ip::address_v4 v4ip = make_address_v4_from_v6(actual_ip);
+              uint32_t actual_ipv4;
+              memcpy(&actual_ipv4, v4ip.to_bytes().data(), sizeof(actual_ipv4));
+              classB.insert(actual_ipv4 & ntohl(0xffff0000));
+            }
+          }
           return true;
         });
       }
 
+      auto get_host_string = [](const epee::net_utils::network_address &address) {
+        if (address.get_type_id() == epee::net_utils::ipv6_network_address::get_type_id())
+        {
+          boost::asio::ip::address_v6 actual_ip = address.as<const epee::net_utils::ipv6_network_address>().ip();
+          if (actual_ip.is_v4_mapped())
+          {
+            boost::asio::ip::address_v4 v4ip = make_address_v4_from_v6(actual_ip);
+            uint32_t actual_ipv4;
+            memcpy(&actual_ipv4, v4ip.to_bytes().data(), sizeof(actual_ipv4));
+            return epee::net_utils::ipv4_network_address(actual_ipv4, 0).host_str();
+          }
+        }
+        return address.host_str();
+      };
+      std::unordered_set<std::string> hosts_added;
       std::deque<size_t> filtered;
       const size_t limit = use_white_list ? 20 : std::numeric_limits<size_t>::max();
       for (int step = 0; step < 2; ++step)
       {
         bool skip_duplicate_class_B = step == 0;
         size_t idx = 0, skipped = 0;
-        zone.m_peerlist.foreach (use_white_list, [&classB, &filtered, &idx, &skipped, skip_duplicate_class_B, limit, next_needed_pruning_stripe](const peerlist_entry &pe){
+        zone.m_peerlist.foreach (use_white_list, [&classB, &filtered, &idx, &skipped, skip_duplicate_class_B, limit, next_needed_pruning_stripe, &hosts_added, &get_host_string](const peerlist_entry &pe){
           if (filtered.size() >= limit)
             return false;
           bool skip = false;
@@ -1448,6 +1524,27 @@ namespace nodetool
             uint32_t actual_ip = na.as<const epee::net_utils::ipv4_network_address>().ip();
             skip = classB.find(actual_ip & 0x0000ffff) != classB.end();
           }
+          else if (skip_duplicate_class_B && pe.adr.get_type_id() == epee::net_utils::ipv6_network_address::get_type_id())
+          {
+            const epee::net_utils::network_address na = pe.adr;
+            const boost::asio::ip::address_v6 &actual_ip = na.as<const epee::net_utils::ipv6_network_address>().ip();
+            if (actual_ip.is_v4_mapped())
+            {
+              boost::asio::ip::address_v4 v4ip = make_address_v4_from_v6(actual_ip);
+              uint32_t actual_ipv4;
+              memcpy(&actual_ipv4, v4ip.to_bytes().data(), sizeof(actual_ipv4));
+              skip = classB.find(actual_ipv4 & ntohl(0xffff0000)) != classB.end();
+            }
+          }
+
+          // consider each host once, to avoid giving undue inflence to hosts running several nodes
+          if (!skip)
+          {
+            const auto i = hosts_added.find(get_host_string(pe.adr));
+            if (i != hosts_added.end())
+              skip = true;
+          }
+
           if (skip)
             ++skipped;
           else if (next_needed_pruning_stripe == 0 || pe.pruning_seed == 0)
@@ -1455,16 +1552,17 @@ namespace nodetool
           else if (next_needed_pruning_stripe == tools::get_pruning_stripe(pe.pruning_seed))
             filtered.push_front(idx);
           ++idx;
+          hosts_added.insert(get_host_string(pe.adr));
           return true;
         });
         if (skipped == 0 || !filtered.empty())
           break;
         if (skipped)
-          MINFO("Skipping " << skipped << " possible peers as they share a class B with existing peers");
+          MDEBUG("Skipping " << skipped << " possible peers as they share a class B with existing peers");
       }
       if (filtered.empty())
       {
-        MDEBUG("No available peer in " << (use_white_list ? "white" : "gray") << " list filtered by " << next_needed_pruning_stripe);
+        MINFO("No available peer in " << (use_white_list ? "white" : "gray") << " list filtered by " << next_needed_pruning_stripe);
         return false;
       }
       if (use_white_list)

src/rpc/core_rpc_server.cpp

@@ -1131,6 +1131,11 @@ namespace cryptonote
   {
     RPC_TRACKER(send_raw_tx);
 
+    {
+      bool ok;
+      use_bootstrap_daemon_if_necessary<COMMAND_RPC_SEND_RAW_TX>(invoke_http_mode::JON, "/sendrawtransaction", req, res, ok);
+    }
+
     const bool restricted = m_restricted && ctx;
 
     bool skip_validation = false;

tests/functional_tests/mining.py

@@ -107,7 +107,7 @@ class MiningTest():
                     break
             else:
                 assert False, 'Failed to mine successor to block %d (initial block = %d)' % (seen_height, prev_height)
-            timeout = 5
+            timeout = 10
 
         if via_daemon:
             res = daemon.stop_mining()

tests/unit_tests/levin.cpp

@@ -613,6 +613,61 @@ TEST_F(levin_notify, stem_without_padding)
     }
 }
 
+TEST_F(levin_notify, stem_no_outs_without_padding)
+{
+    cryptonote::levin::notify notifier = make_notifier(0, true, false);
+
+    for (unsigned count = 0; count < 10; ++count)
+        add_connection(true);
+
+    {
+        const auto status = notifier.get_status();
+        EXPECT_FALSE(status.has_noise);
+        EXPECT_FALSE(status.connections_filled);
+    }
+    notifier.new_out_connection();
+    io_service_.poll();
+
+    std::vector<cryptonote::blobdata> txs(2);
+    txs[0].resize(100, 'f');
+    txs[1].resize(200, 'e');
+
+    std::vector<cryptonote::blobdata> sorted_txs = txs;
+    std::sort(sorted_txs.begin(), sorted_txs.end());
+
+    ASSERT_EQ(10u, contexts_.size());
+
+    auto context = contexts_.begin();
+    EXPECT_TRUE(notifier.send_txs(txs, context->get_id(), cryptonote::relay_method::stem));
+
+    io_service_.reset();
+    ASSERT_LT(0u, io_service_.poll());
+    EXPECT_EQ(txs, events_.take_relayed(cryptonote::relay_method::fluff));
+    if (events_.has_stem_txes())
+        EXPECT_EQ(txs, events_.take_relayed(cryptonote::relay_method::stem));
+
+
+    notifier.run_fluff();
+    ASSERT_LT(0u, io_service_.poll());
+
+    std::size_t send_count = 0;
+    EXPECT_EQ(0u, context->process_send_queue());
+    for (++context; context != contexts_.end(); ++context)
+    {
+        send_count += context->process_send_queue();
+    }
+
+    EXPECT_EQ(9u, send_count);
+    ASSERT_EQ(9u, receiver_.notified_size());
+    for (unsigned count = 0; count < 9u; ++count)
+    {
+        auto notification = receiver_.get_notification<cryptonote::NOTIFY_NEW_TRANSACTIONS>().second;
+        EXPECT_EQ(sorted_txs, notification.txs);
+        EXPECT_TRUE(notification._.empty());
+        EXPECT_TRUE(notification.dandelionpp_fluff);
+    }
+}
+
 TEST_F(levin_notify, local_without_padding)
 {
     cryptonote::levin::notify notifier = make_notifier(0, true, false);
@@ -928,6 +983,60 @@ TEST_F(levin_notify, stem_with_padding)
     }
 }
 
+TEST_F(levin_notify, stem_no_outs_with_padding)
+{
+    cryptonote::levin::notify notifier = make_notifier(0, true, true);
+
+    for (unsigned count = 0; count < 10; ++count)
+        add_connection(true);
+
+    {
+        const auto status = notifier.get_status();
+        EXPECT_FALSE(status.has_noise);
+        EXPECT_FALSE(status.connections_filled);
+    }
+    notifier.new_out_connection();
+    io_service_.poll();
+
+    std::vector<cryptonote::blobdata> txs(2);
+    txs[0].resize(100, 'f');
+    txs[1].resize(200, 'e');
+
+    std::vector<cryptonote::blobdata> sorted_txs = txs;
+    std::sort(sorted_txs.begin(), sorted_txs.end());
+
+    ASSERT_EQ(10u, contexts_.size());
+
+    auto context = contexts_.begin();
+    EXPECT_TRUE(notifier.send_txs(txs, context->get_id(), cryptonote::relay_method::stem));
+
+    io_service_.reset();
+    ASSERT_LT(0u, io_service_.poll());
+    EXPECT_EQ(txs, events_.take_relayed(cryptonote::relay_method::fluff));
+    if (events_.has_stem_txes())
+        EXPECT_EQ(txs, events_.take_relayed(cryptonote::relay_method::stem));
+
+    notifier.run_fluff();
+    ASSERT_LT(0u, io_service_.poll());
+
+    std::size_t send_count = 0;
+    EXPECT_EQ(0u, context->process_send_queue());
+    for (++context; context != contexts_.end(); ++context)
+    {
+        send_count += context->process_send_queue();
+    }
+
+    EXPECT_EQ(9u, send_count);
+    ASSERT_EQ(9u, receiver_.notified_size());
+    for (unsigned count = 0; count < 9u; ++count)
+    {
+        auto notification = receiver_.get_notification<cryptonote::NOTIFY_NEW_TRANSACTIONS>().second;
+        EXPECT_EQ(sorted_txs, notification.txs);
+        EXPECT_FALSE(notification._.empty());
+        EXPECT_TRUE(notification.dandelionpp_fluff);
+    }
+}
+
 TEST_F(levin_notify, local_with_padding)
 {
     cryptonote::levin::notify notifier = make_notifier(0, true, true);