Merge pull request #2724

cb23be8f Wallet API: always use approximate calc of blockchain height (Jaquee)

contrib/epee/include/console_handler.h

@@ -52,11 +52,10 @@ namespace epee
       , m_has_read_request(false)
       , m_read_status(state_init)
     {
-      m_reader_thread = boost::thread(std::bind(&async_stdin_reader::reader_thread_func, this));
 #ifdef HAVE_READLINE
       m_readline_buffer.start();
-      m_readline_thread = boost::thread(std::bind(&async_stdin_reader::readline_thread_func, this));
 #endif
+      m_reader_thread = boost::thread(std::bind(&async_stdin_reader::reader_thread_func, this));
     }
 
     ~async_stdin_reader()
@@ -115,7 +114,6 @@ namespace epee
         m_reader_thread.join();
 #ifdef HAVE_READLINE
         m_readline_buffer.stop();
-        m_readline_thread.join();
 #endif
       }
     }
@@ -193,16 +191,6 @@ namespace epee
       return true;
     }
 
-#ifdef HAVE_READLINE
-    void readline_thread_func()
-    {
-      while (m_run.load(std::memory_order_relaxed))
-      {
-        m_readline_buffer.process();
-      }
-    }
-#endif
-
     void reader_thread_func()
     {
       while (true)
@@ -212,12 +200,20 @@ namespace epee
 
         std::string line;
         bool read_ok = true;
+#ifdef HAVE_READLINE
+reread:
+#endif
         if (wait_stdin_data())
         {
           if (m_run.load(std::memory_order_relaxed))
           {
 #ifdef HAVE_READLINE
-            m_readline_buffer.get_line(line);
+            switch (m_readline_buffer.get_line(line))
+            {
+            case rdln::empty:   goto eof;
+            case rdln::partial: goto reread;
+            case rdln::full:    break;
+            }
 #else
             std::getline(std::cin, line);
 #endif
@@ -229,6 +225,9 @@ namespace epee
           read_ok = false;
         }
         if (std::cin.eof()) {
+#ifdef HAVE_READLINE
+eof:
+#endif
           m_read_status = state_eos;
           m_response_cv.notify_one();
           break;
@@ -263,7 +262,6 @@ namespace epee
     boost::thread m_reader_thread;
     std::atomic<bool> m_run;
 #ifdef HAVE_READLINE
-    boost::thread m_readline_thread;
     rdln::readline_buffer m_readline_buffer;
 #endif
 

contrib/epee/include/net/http_client.h

@@ -293,6 +293,9 @@ using namespace std;
 				, m_lock()
 			{}
 
+			const std::string &get_host() const { return m_host_buff; };
+			const std::string &get_port() const { return m_port; };
+
 			bool set_server(const std::string& address, boost::optional<login> user)
 			{
 				http::url_content parsed{};

contrib/epee/include/readline_buffer.h

@@ -8,25 +8,25 @@
 
 namespace rdln
 {
+  typedef enum { empty, partial, full } linestatus;
   class readline_buffer : public std::stringbuf
   {
   public:
     readline_buffer();
     void start();
     void stop();
-    int process();
     bool is_running() const
     {
       return m_cout_buf != NULL;
     }
-    void get_line(std::string& line) const;
+    linestatus get_line(std::string& line) const;
     void set_prompt(const std::string& prompt);
     static void add_completion(const std::string& command);
     static const std::vector<std::string>& get_completions();
     
   protected:
     virtual int sync();
-    
+
   private:
     std::streambuf* m_cout_buf;
     static std::vector<std::string>& completion_commands();

contrib/epee/include/string_tools.h

@@ -314,6 +314,18 @@ POP_WARNINGS
     return str;
   }
   //----------------------------------------------------------------------------
+  inline std::string pad_string(std::string s, size_t n, char c = ' ', bool prepend = false)
+  {
+    if (s.size() < n)
+    {
+      if (prepend)
+        s = std::string(n - s.size(), c) + s;
+      else
+        s.append(n - s.size(), c);
+    }
+    return s;
+  }
+  //----------------------------------------------------------------------------
   template<class t_pod_type>
   std::string pod_to_hex(const t_pod_type& s)
   {

contrib/epee/src/readline_buffer.cpp

@@ -3,19 +3,15 @@
 #include <readline/history.h>
 #include <sys/select.h>
 #include <unistd.h>
-#include <mutex>
-#include <condition_variable>
 #include <boost/thread.hpp>
 #include <boost/algorithm/string.hpp>
 
-static int process_input();
 static void install_line_handler();
 static void remove_line_handler();
 
-static std::string last_line;
-static std::string last_prompt;
-static std::mutex line_mutex, sync_mutex, process_mutex;
-static std::condition_variable have_line;
+static boost::mutex sync_mutex;
+static rdln::linestatus line_stat;
+static char *the_line;
 
 namespace
 {
@@ -55,7 +51,6 @@ rdln::readline_buffer::readline_buffer()
 
 void rdln::readline_buffer::start()
 {
-  std::unique_lock<std::mutex> lock(process_mutex);
   if(m_cout_buf != NULL)
     return;
   m_cout_buf = std::cout.rdbuf();
@@ -65,9 +60,6 @@ void rdln::readline_buffer::start()
 
 void rdln::readline_buffer::stop()
 {
-  std::unique_lock<std::mutex> lock_process(process_mutex);
-  std::unique_lock<std::mutex> lock_sync(sync_mutex);
-  have_line.notify_all();
   if(m_cout_buf == NULL)
     return;
   std::cout.rdbuf(m_cout_buf);
@@ -75,20 +67,26 @@ void rdln::readline_buffer::stop()
   remove_line_handler();
 }
 
-void rdln::readline_buffer::get_line(std::string& line) const
+rdln::linestatus rdln::readline_buffer::get_line(std::string& line) const
 {
-  std::unique_lock<std::mutex> lock(line_mutex);
-  have_line.wait(lock);
-  line = last_line;
+  boost::lock_guard<boost::mutex> lock(sync_mutex);
+  line_stat = rdln::partial;
+  rl_callback_read_char();
+  if (line_stat == rdln::full)
+  {
+    line = the_line;
+    free(the_line);
+    the_line = NULL;
+  }
+  return line_stat;
 }
 
 void rdln::readline_buffer::set_prompt(const std::string& prompt)
 {
-  last_prompt = prompt;
   if(m_cout_buf == NULL)
     return;
-  std::lock_guard<std::mutex> lock(sync_mutex);
-  rl_set_prompt(last_prompt.c_str());
+  boost::lock_guard<boost::mutex> lock(sync_mutex);
+  rl_set_prompt(prompt.c_str());
   rl_redisplay();
 }
 
@@ -104,126 +102,78 @@ const std::vector<std::string>& rdln::readline_buffer::get_completions()
   return completion_commands();
 }
 
-int rdln::readline_buffer::process()
+int rdln::readline_buffer::sync()
 {
-  process_mutex.lock();
-  if(m_cout_buf == NULL)
+  boost::lock_guard<boost::mutex> lock(sync_mutex);
+#if RL_READLINE_VERSION < 0x0700
+  char lbuf[2] = {0,0};
+  char *line = NULL;
+  int end = 0, point = 0;
+#endif
+
+  if (rl_end || *rl_prompt)
   {
-    process_mutex.unlock();
-    boost::this_thread::sleep_for(boost::chrono::milliseconds( 1 ));
-    return 0;
+#if RL_READLINE_VERSION >= 0x0700
+    rl_clear_visible_line();
+#else
+    line = rl_line_buffer;
+    end = rl_end;
+    point = rl_point;
+    rl_line_buffer = lbuf;
+    rl_end = 0;
+    rl_point = 0;
+    rl_save_prompt();
+    rl_redisplay();
+#endif
   }
-  int count = process_input();
-  process_mutex.unlock();
-  boost::this_thread::sleep_for(boost::chrono::milliseconds( 1 ));
-  return count;
-}
 
-int rdln::readline_buffer::sync()
-{
-  std::lock_guard<std::mutex> lock(sync_mutex);
-  char* saved_line;
-  int saved_point;
-  
-  saved_point = rl_point;
-  saved_line = rl_copy_text(0, rl_end);
-  
-  rl_set_prompt("");
-  rl_replace_line("", 0);
-  rl_redisplay();
-  
   do
   {
     m_cout_buf->sputc( this->sgetc() );
   }
   while ( this->snextc() != EOF );
-  
-  rl_set_prompt(last_prompt.c_str());
-  rl_replace_line(saved_line, 0);
-  rl_point = saved_point;
-  rl_redisplay();
-  free(saved_line);
-  
-  return 0;
-}
 
-static int process_input()
-{
-  int count;
-  struct timeval t;
-  fd_set fds;
-  
-  t.tv_sec = 0;
-  t.tv_usec = 1000;
-  
-  FD_ZERO(&fds);
-  FD_SET(STDIN_FILENO, &fds);
-  count = select(STDIN_FILENO + 1, &fds, NULL, NULL, &t);
-  if (count < 1)
+#if RL_READLINE_VERSION < 0x0700
+  if (end || *rl_prompt)
   {
-    return count;
+    rl_restore_prompt();
+    rl_line_buffer = line;
+    rl_end = end;
+    rl_point = point;
   }
-  rl_callback_read_char();
-  return count;
-}
-
-static void handle_line(char* line)
-{
-  free(line);
-  rl_done = 1;
-  return;
-}
-
-static int handle_enter(int x, int y)
-{
-  std::lock_guard<std::mutex> lock(sync_mutex);
-  char* line = NULL;
-
-  line = rl_copy_text(0, rl_end);
-  std::string test_line = line;
-  free(line);
-  boost::trim_right(test_line);
-
-  rl_crlf();
+#endif
   rl_on_new_line();
-
-  if(test_line.empty())
-  {
-    last_line = "";
-    rl_set_prompt(last_prompt.c_str());
-    rl_replace_line("", 1);
-    rl_redisplay();
-    have_line.notify_one();
-    return 0;
-  }
-
-  rl_set_prompt("");
-  rl_replace_line("", 1);
   rl_redisplay();
-  
-  if (!test_line.empty())
-  {
-    last_line = test_line;
-    add_history(test_line.c_str());
-    history_set_pos(history_length);
-  }
 
-  if(last_line != "exit" && last_line != "q")
-  {
-    rl_set_prompt(last_prompt.c_str());
-    rl_replace_line("", 1);
-    rl_redisplay();
-  }
-
-  have_line.notify_one();
   return 0;
 }
 
-static int startup_hook()
+static void handle_line(char* line)
 {
-  rl_bind_key(RETURN, handle_enter);
-  rl_bind_key(NEWLINE, handle_enter);
-  return 0;
+  bool exit = false;
+  if (line)
+  {
+    line_stat = rdln::full;
+    the_line = line;
+    std::string test_line = line;
+    boost::trim_right(test_line);
+    if(!test_line.empty())
+    {
+      add_history(test_line.c_str());
+      history_set_pos(history_length);
+      if (test_line == "exit" || test_line == "q")
+        exit = true;
+    }
+  } else
+  /* EOF */
+  {
+    line_stat = rdln::empty;
+    exit = true;
+  }
+  rl_done = 1;
+  if (exit)
+    rl_set_prompt("");
+  return;
 }
 
 static char* completion_matches(const char* text, int state)
@@ -258,7 +208,6 @@ static char** attempted_completion(const char* text, int start, int end)
 
 static void install_line_handler()
 {
-  rl_startup_hook = startup_hook;
   rl_attempted_completion_function = attempted_completion;
   rl_callback_handler_install("", handle_line);
   stifle_history(500);
@@ -269,8 +218,6 @@ static void remove_line_handler()
   rl_replace_line("", 0);
   rl_set_prompt("");
   rl_redisplay();
-  rl_unbind_key(RETURN);
-  rl_unbind_key(NEWLINE);
   rl_callback_handler_remove();
 }
 

contrib/rlwrap/monerocommands_bitmonerod.txt

@@ -32,4 +32,5 @@ status
 stop_daemon
 stop_mining
 stop_save_graph
+sync_info
 unban

external/boost/archive/portable_binary_oarchive.hpp

@@ -41,19 +41,24 @@ class portable_binary_oarchive_exception :
     public boost::archive::archive_exception
 {
 public:
-    typedef enum {
+    enum exception_code {
         invalid_flags 
-    } exception_code;
-    portable_binary_oarchive_exception(exception_code c = invalid_flags )
+    } m_exception_code ;
+    portable_binary_oarchive_exception(exception_code c = invalid_flags ) :
+        boost::archive::archive_exception(boost::archive::archive_exception::other_exception),
+        m_exception_code(c)
     {}
     virtual const char *what( ) const throw( )
     {
         const char *msg = "programmer error";
-        switch(code){
+        switch(m_exception_code){
         case invalid_flags:
             msg = "cannot be both big and little endian";
+            break;
         default:
-            boost::archive::archive_exception::what();
+            msg = boost::archive::archive_exception::what();
+            assert(false);
+            break;
         }
         return msg;
     }

external/db_drivers/liblmdb/mdb.c

@@ -6288,6 +6288,10 @@ release:
 		if (rc)
 			return rc;
 	}
+#ifdef MDB_VL32
+	if (mc->mc_ovpg == mp)
+		mc->mc_ovpg = NULL;
+#endif
 	mc->mc_db->md_overflow_pages -= ovpages;
 	return 0;
 }

external/miniupnpc/miniwget.c

@@ -280,11 +280,12 @@ getHTTPResponse(int s, int * size, int * status_code)
 							goto end_of_stream;
 						}
 					}
-					bytestocopy = ((int)chunksize < (n - i))?chunksize:(unsigned int)(n - i);
+					/* it is guaranteed that (n >= i) */
+					bytestocopy = (chunksize < (unsigned int)(n - i))?chunksize:(unsigned int)(n - i);
 					if((content_buf_used + bytestocopy) > content_buf_len)
 					{
 						char * tmp;
-						if(content_length >= (int)(content_buf_used + bytestocopy)) {
+						if((content_length >= 0) && ((unsigned int)content_length >= (content_buf_used + bytestocopy))) {
 							content_buf_len = content_length;
 						} else {
 							content_buf_len = content_buf_used + bytestocopy;
@@ -309,14 +310,15 @@ getHTTPResponse(int s, int * size, int * status_code)
 			{
 				/* not chunked */
 				if(content_length > 0
-				   && (int)(content_buf_used + n) > content_length) {
+				   && (content_buf_used + n) > (unsigned int)content_length) {
 					/* skipping additional bytes */
 					n = content_length - content_buf_used;
 				}
 				if(content_buf_used + n > content_buf_len)
 				{
 					char * tmp;
-					if(content_length >= (int)(content_buf_used + n)) {
+					if(content_length >= 0
+					   && (unsigned int)content_length >= (content_buf_used + n)) {
 						content_buf_len = content_length;
 					} else {
 						content_buf_len = content_buf_used + n;
@@ -336,7 +338,7 @@ getHTTPResponse(int s, int * size, int * status_code)
 			}
 		}
 		/* use the Content-Length header value if available */
-		if(content_length > 0 && (int)content_buf_used >= content_length)
+		if(content_length > 0 && content_buf_used >= (unsigned int)content_length)
 		{
 #ifdef DEBUG
 			printf("End of HTTP content\n");

src/blockchain_db/blockchain_db.cpp

@@ -33,6 +33,19 @@
 #include "profile_tools.h"
 #include "ringct/rctOps.h"
 
+#include "lmdb/db_lmdb.h"
+#ifdef BERKELEY_DB
+#include "berkeleydb/db_bdb.h"
+#endif
+
+static const char *db_types[] = {
+  "lmdb",
+#ifdef BERKELEY_DB
+  "berkeley",
+#endif
+  NULL
+};
+
 #undef MONERO_DEFAULT_LOG_CATEGORY
 #define MONERO_DEFAULT_LOG_CATEGORY "blockchain.db"
 
@@ -41,6 +54,41 @@ using epee::string_tools::pod_to_hex;
 namespace cryptonote
 {
 
+bool blockchain_valid_db_type(const std::string& db_type)
+{
+  int i;
+  for (i=0; db_types[i]; i++)
+  {
+    if (db_types[i] == db_type)
+      return true;
+  }
+  return false;
+}
+
+std::string blockchain_db_types(const std::string& sep)
+{
+  int i;
+  std::string ret = "";
+  for (i=0; db_types[i]; i++)
+  {
+    if (i)
+      ret += sep;
+    ret += db_types[i];
+  }
+  return ret;
+}
+
+BlockchainDB *new_db(const std::string& db_type)
+{
+  if (db_type == "lmdb")
+    return new BlockchainLMDB();
+#if defined(BERKELEY_DB)
+  if (db_type == "berkeley")
+    return new BlockchainBDB();
+#endif
+  return NULL;
+}
+
 void BlockchainDB::pop_block()
 {
   block blk;

src/blockchain_db/blockchain_db.h

@@ -145,6 +145,12 @@ struct txpool_tx_meta_t
   uint8_t padding[77]; // till 192 bytes
 };
 
+#define DBF_SAFE       1
+#define DBF_FAST       2
+#define DBF_FASTEST    4
+#define DBF_RDONLY     8
+#define DBF_SALVAGE 0x10
+
 /***********************************
  * Exception Definitions
  ***********************************/
@@ -599,6 +605,13 @@ public:
    */
   virtual void sync() = 0;
 
+  /**
+   * @brief toggle safe syncs for the DB
+   *
+   * Used to switch DBF_SAFE on or off after starting up with DBF_FAST.
+   */
+  virtual void safesyncmode(const bool onoff) = 0;
+
   /**
    * @brief Remove everything from the BlockchainDB
    *
@@ -1491,6 +1504,7 @@ public:
 
 };  // class BlockchainDB
 
+BlockchainDB *new_db(const std::string& db_type);
 
 }  // namespace cryptonote
 

src/blockchain_db/db_types.h

@@ -31,9 +31,6 @@
 
 namespace cryptonote
 {
-
-  const std::unordered_set<std::string> blockchain_db_types = 
-  { "lmdb"
-  };
-
+  bool blockchain_valid_db_type(const std::string& db_type);
+  std::string blockchain_db_types(const std::string& sep);
 } // namespace cryptonote

src/blockchain_db/lmdb/db_lmdb.cpp

@@ -1083,9 +1083,10 @@ BlockchainLMDB::BlockchainLMDB(bool batch_transactions)
   m_hardfork = nullptr;
 }
 
-void BlockchainLMDB::open(const std::string& filename, const int mdb_flags)
+void BlockchainLMDB::open(const std::string& filename, const int db_flags)
 {
   int result;
+  int mdb_flags = MDB_NORDAHEAD;
 
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
 
@@ -1124,6 +1125,15 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags)
 
   size_t mapsize = DEFAULT_MAPSIZE;
 
+  if (db_flags & DBF_FAST)
+    mdb_flags |= MDB_NOSYNC;
+  if (db_flags & DBF_FASTEST)
+    mdb_flags |= MDB_NOSYNC | MDB_WRITEMAP | MDB_MAPASYNC;
+  if (db_flags & DBF_RDONLY)
+    mdb_flags = MDB_RDONLY;
+  if (db_flags & DBF_SALVAGE)
+    mdb_flags |= MDB_PREVSNAPSHOT;
+
   if (auto result = mdb_env_open(m_env, filename.c_str(), mdb_flags, 0644))
     throw0(DB_ERROR(lmdb_error("Failed to open lmdb environment: ", result).c_str()));
 
@@ -1308,6 +1318,11 @@ void BlockchainLMDB::sync()
   }
 }
 
+void BlockchainLMDB::safesyncmode(const bool onoff)
+{
+  mdb_env_set_flags(m_env, MDB_NOSYNC|MDB_MAPASYNC, !onoff);
+}
+
 void BlockchainLMDB::reset()
 {
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
@@ -2405,8 +2420,8 @@ bool BlockchainLMDB::for_blocks_range(const uint64_t& h1, const uint64_t& h2, st
   MDB_cursor_op op;
   if (h1)
   {
-    MDB_val_set(k, h1);
-	op = MDB_SET;
+    k = MDB_val{sizeof(h1), (void*)&h1};
+    op = MDB_SET;
   } else
   {
     op = MDB_FIRST;
@@ -2589,6 +2604,16 @@ void BlockchainLMDB::batch_commit()
   memset(&m_wcursors, 0, sizeof(m_wcursors));
 }
 
+void BlockchainLMDB::cleanup_batch()
+{
+  // for destruction of batch transaction
+  m_write_txn = nullptr;
+  delete m_write_batch_txn;
+  m_write_batch_txn = nullptr;
+  m_batch_active = false;
+  memset(&m_wcursors, 0, sizeof(m_wcursors));
+}
+
 void BlockchainLMDB::batch_stop()
 {
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
@@ -2603,15 +2628,18 @@ void BlockchainLMDB::batch_stop()
   check_open();
   LOG_PRINT_L3("batch transaction: committing...");
   TIME_MEASURE_START(time1);
-  m_write_txn->commit();
-  TIME_MEASURE_FINISH(time1);
-  time_commit1 += time1;
-  // for destruction of batch transaction
-  m_write_txn = nullptr;
-  delete m_write_batch_txn;
-  m_write_batch_txn = nullptr;
-  m_batch_active = false;
-  memset(&m_wcursors, 0, sizeof(m_wcursors));
+  try
+  {
+    m_write_txn->commit();
+    TIME_MEASURE_FINISH(time1);
+    time_commit1 += time1;
+    cleanup_batch();
+  }
+  catch (const std::exception &e)
+  {
+    cleanup_batch();
+    throw;
+  }
   LOG_PRINT_L3("batch transaction: end");
 }
 

src/blockchain_db/lmdb/db_lmdb.h

@@ -165,6 +165,8 @@ public:
 
   virtual void sync();
 
+  virtual void safesyncmode(const bool onoff);
+
   virtual void reset();
 
   virtual std::vector<std::string> get_filenames() const;
@@ -366,6 +368,9 @@ private:
   // migrate from DB version 0 to 1
   void migrate_0_1();
 
+  void cleanup_batch();
+
+private:
   MDB_env* m_env;
 
   MDB_dbi m_blocks;

src/blockchain_utilities/blockchain_export.cpp

@@ -31,10 +31,6 @@
 #include "common/command_line.h"
 #include "cryptonote_core/tx_pool.h"
 #include "blockchain_db/blockchain_db.h"
-#include "blockchain_db/lmdb/db_lmdb.h"
-#if defined(BERKELEY_DB)
-#include "blockchain_db/berkeleydb/db_bdb.h"
-#endif
 #include "blockchain_db/db_types.h"
 #include "version.h"
 
@@ -44,17 +40,6 @@
 namespace po = boost::program_options;
 using namespace epee;
 
-std::string join_set_strings(const std::unordered_set<std::string>& db_types_all, const char* delim)
-{
-  std::string result;
-  std::ostringstream s;
-  std::copy(db_types_all.begin(), db_types_all.end(), std::ostream_iterator<std::string>(s, delim));
-  result = s.str();
-  if (result.length() > 0)
-    result.erase(result.end()-strlen(delim), result.end());
-  return result;
-}
-
 int main(int argc, char* argv[])
 {
   TRY_ENTRY();
@@ -63,10 +48,7 @@ int main(int argc, char* argv[])
 
   std::string default_db_type = "lmdb";
 
-  std::unordered_set<std::string> db_types_all = cryptonote::blockchain_db_types;
-  db_types_all.insert("memory");
-
-  std::string available_dbs = join_set_strings(db_types_all, ", ");
+  std::string available_dbs = cryptonote::blockchain_db_types(", ");
   available_dbs = "available: " + available_dbs;
 
   uint32_t log_level = 0;
@@ -144,18 +126,11 @@ int main(int argc, char* argv[])
   m_config_folder = command_line::get_arg(vm, data_dir_arg);
 
   std::string db_type = command_line::get_arg(vm, arg_database);
-  if (db_types_all.count(db_type) == 0)
+  if (!cryptonote::blockchain_valid_db_type(db_type))
   {
     std::cerr << "Invalid database type: " << db_type << std::endl;
     return 1;
   }
-#if !defined(BERKELEY_DB)
-  if (db_type == "berkeley")
-  {
-    LOG_ERROR("BerkeleyDB support disabled.");
-    return false;
-  }
-#endif
 
   if (command_line::has_arg(vm, arg_output_file))
     output_file_path = boost::filesystem::path(command_line::get_arg(vm, arg_output_file));
@@ -179,19 +154,8 @@ int main(int argc, char* argv[])
   tx_memory_pool m_mempool(*core_storage);
   core_storage = new Blockchain(m_mempool);
 
-  int db_flags = 0;
-
-  BlockchainDB* db = nullptr;
-  if (db_type == "lmdb")
-  {
-    db_flags |= MDB_RDONLY;
-    db = new BlockchainLMDB();
-  }
-#if defined(BERKELEY_DB)
-  else if (db_type == "berkeley")
-    db = new BlockchainBDB();
-#endif
-  else
+  BlockchainDB* db = new_db(db_type);
+  if (db == NULL)
   {
     LOG_ERROR("Attempted to use non-existent database type: " << db_type);
     throw std::runtime_error("Attempting to use non-existent database type");
@@ -205,7 +169,7 @@ int main(int argc, char* argv[])
   LOG_PRINT_L0("Loading blockchain from folder " << filename << " ...");
   try
   {
-    db->open(filename, db_flags);
+    db->open(filename, DBF_RDONLY);
   }
   catch (const std::exception& e)
   {

src/blockchain_utilities/blockchain_import.cpp

@@ -42,8 +42,6 @@
 #include "blockchain_db/db_types.h"
 #include "cryptonote_core/cryptonote_core.h"
 
-#include <lmdb.h> // for db flag arguments
-
 #undef MONERO_DEFAULT_LOG_CATEGORY
 #define MONERO_DEFAULT_LOG_CATEGORY "bcutil"
 
@@ -78,40 +76,16 @@ namespace po = boost::program_options;
 using namespace cryptonote;
 using namespace epee;
 
-
-std::string join_set_strings(const std::unordered_set<std::string>& db_types_all, const char* delim)
-{
-  std::string result;
-  std::ostringstream s;
-  std::copy(db_types_all.begin(), db_types_all.end(), std::ostream_iterator<std::string>(s, delim));
-  result = s.str();
-  if (result.length() > 0)
-    result.erase(result.end()-strlen(delim), result.end());
-  return result;
-}
-
-// db_type: lmdb, berkeley
 // db_mode: safe, fast, fastest
-int get_db_flags_from_mode(const std::string& db_type, const std::string& db_mode)
+int get_db_flags_from_mode(const std::string& db_mode)
 {
-  uint64_t BDB_FAST_MODE = 0;
-  uint64_t BDB_FASTEST_MODE = 0;
-  uint64_t BDB_SAFE_MODE = 0;
-
-#if defined(BERKELEY_DB)
-  BDB_FAST_MODE = DB_TXN_WRITE_NOSYNC;
-  BDB_FASTEST_MODE = DB_TXN_NOSYNC;
-  BDB_SAFE_MODE = DB_TXN_SYNC;
-#endif
-
   int db_flags = 0;
-  bool islmdb = db_type == "lmdb";
   if (db_mode == "safe")
-    db_flags = islmdb ? MDB_NORDAHEAD : BDB_SAFE_MODE;
+    db_flags = DBF_SAFE;
   else if (db_mode == "fast")
-    db_flags = islmdb ? MDB_NOMETASYNC | MDB_NOSYNC | MDB_NORDAHEAD : BDB_FAST_MODE;
+    db_flags = DBF_FAST;
   else if (db_mode == "fastest")
-    db_flags = islmdb ? MDB_WRITEMAP | MDB_MAPASYNC | MDB_NORDAHEAD | MDB_NOMETASYNC | MDB_NOSYNC : BDB_FASTEST_MODE;
+    db_flags = DBF_FASTEST;
   return db_flags;
 }
 
@@ -132,14 +106,6 @@ int parse_db_arguments(const std::string& db_arg_str, std::string& db_type, int&
     return 1;
   }
 
-#if !defined(BERKELEY_DB)
-  if (db_type == "berkeley")
-  {
-    MFATAL("BerkeleyDB support disabled.");
-    return false;
-  }
-#endif
-
   std::string db_arg_str2 = db_args[1];
   boost::split(db_args, db_arg_str2, boost::is_any_of(","));
 
@@ -155,51 +121,7 @@ int parse_db_arguments(const std::string& db_arg_str, std::string& db_type, int&
   }
   if (! db_mode.empty())
   {
-    db_flags = get_db_flags_from_mode(db_type, db_mode);
-  }
-  else
-  {
-    for (auto& it : db_args)
-    {
-      boost::algorithm::trim(it);
-      if (it.empty())
-	continue;
-      if (db_type == "lmdb")
-      {
-	MINFO("LMDB flag: " << it);
-	if (it == "nosync")
-	  db_flags |= MDB_NOSYNC;
-	else if (it == "nometasync")
-	  db_flags |= MDB_NOMETASYNC;
-	else if (it == "writemap")
-	  db_flags |= MDB_WRITEMAP;
-	else if (it == "mapasync")
-	  db_flags |= MDB_MAPASYNC;
-	else if (it == "nordahead")
-	  db_flags |= MDB_NORDAHEAD;
-	else
-	{
-	  std::cerr << "unrecognized database flag: " << it << ENDL;
-	  return 1;
-	}
-      }
-#if defined(BERKELEY_DB)
-      else if (db_type == "berkeley")
-      {
-	if (it == "txn_write_nosync")
-	  db_flags = DB_TXN_WRITE_NOSYNC;
-	else if (it == "txn_nosync")
-	  db_flags = DB_TXN_NOSYNC;
-	else if (it == "txn_sync")
-	  db_flags = DB_TXN_SYNC;
-	else
-	{
-	  std::cerr << "unrecognized database flag: " << it << ENDL;
-	  return 1;
-	}
-      }
-#endif
-    }
+    db_flags = get_db_flags_from_mode(db_mode);
   }
   return 0;
 }
@@ -286,7 +208,8 @@ int check_flush(cryptonote::core &core, std::list<block_complete_entry> &blocks,
     }
 
   } // each download block
-  core.cleanup_handle_incoming_blocks();
+  if (!core.cleanup_handle_incoming_blocks())
+    return 1;
 
   blocks.clear();
   return 0;
@@ -394,9 +317,9 @@ int import_from_file(cryptonote::core& core, const std::string& import_file_path
       MWARNING("WARNING: chunk_size " << chunk_size << " > BUFFER_SIZE " << BUFFER_SIZE);
       throw std::runtime_error("Aborting: chunk size exceeds buffer size");
     }
-    if (chunk_size > 100000)
+    if (chunk_size > CHUNK_SIZE_WARNING_THRESHOLD)
     {
-      MINFO("NOTE: chunk_size " << chunk_size << " > 100000");
+      MINFO("NOTE: chunk_size " << chunk_size << " > " << CHUNK_SIZE_WARNING_THRESHOLD);
     }
     else if (chunk_size == 0) {
       MFATAL("ERROR: chunk_size == 0");
@@ -404,9 +327,19 @@ int import_from_file(cryptonote::core& core, const std::string& import_file_path
     }
     import_file.read(buffer_block, chunk_size);
     if (! import_file) {
-      MFATAL("ERROR: unexpected end of file: bytes read before error: "
-          << import_file.gcount() << " of chunk_size " << chunk_size);
-      return 2;
+      if (import_file.eof())
+      {
+        std::cout << refresh_string;
+        MINFO("End of file reached - file was truncated");
+        quit = 1;
+        break;
+      }
+      else
+      {
+        MFATAL("ERROR: unexpected end of file: bytes read before error: "
+            << import_file.gcount() << " of chunk_size " << chunk_size);
+        return 2;
+      }
     }
     bytes_read += chunk_size;
     MDEBUG("Total bytes read: " << bytes_read);
@@ -472,7 +405,10 @@ int import_from_file(cryptonote::core& core, const std::string& import_file_path
           blocks.push_back({block, txs});
           int ret = check_flush(core, blocks, false);
           if (ret)
+          {
+            quit = 2; // make sure we don't commit partial block data
             break;
+          }
         }
         else
         {
@@ -580,12 +516,8 @@ int main(int argc, char* argv[])
   epee::string_tools::set_module_name_and_folder(argv[0]);
 
   std::string default_db_type = "lmdb";
-  std::string default_db_engine_compiled = "blockchain_db";
-
-  std::unordered_set<std::string> db_types_all = cryptonote::blockchain_db_types;
-  db_types_all.insert("memory");
 
-  std::string available_dbs = join_set_strings(db_types_all, ", ");
+  std::string available_dbs = cryptonote::blockchain_db_types(", ");
   available_dbs = "available: " + available_dbs;
 
   uint32_t log_level = 0;
@@ -731,7 +663,6 @@ int main(int argc, char* argv[])
 
 
   std::string db_type;
-  std::string db_engine_compiled;
   int db_flags = 0;
   int res = 0;
   res = parse_db_arguments(db_arg_str, db_type, db_flags);
@@ -741,25 +672,12 @@ int main(int argc, char* argv[])
     return 1;
   }
 
-  if (db_types_all.count(db_type) == 0)
+  if (!cryptonote::blockchain_valid_db_type(db_type))
   {
     std::cerr << "Invalid database type: " << db_type << std::endl;
     return 1;
   }
 
-  if ((db_type == "lmdb")
-#if defined(BERKELEY_DB)
-   || (db_type == "berkeley")
-#endif
-  )
-  {
-    db_engine_compiled = "blockchain_db";
-  }
-  else
-  {
-    db_engine_compiled = "memory";
-  }
-
   MINFO("database: " << db_type);
   MINFO("database flags: " << db_flags);
   MINFO("verify:  " << std::boolalpha << opt_verify << std::noboolalpha);
@@ -778,28 +696,12 @@ int main(int argc, char* argv[])
   MINFO("bootstrap file path: " << import_file_path);
   MINFO("database path:       " << m_config_folder);
 
-  try
-  {
+  cryptonote::cryptonote_protocol_stub pr; //TODO: stub only for this kind of test, make real validation of relayed objects
+  cryptonote::core core(&pr);
 
-  // fake_core needed for verification to work when enabled.
-  //
-  // NOTE: don't need fake_core method of doing things when we're going to call
-  // BlockchainDB add_block() directly and have available the 3 block
-  // properties to do so. Both ways work, but fake core isn't necessary in that
-  // circumstance.
-
-  if (db_type != "lmdb"
-#if defined(BERKELEY_DB)
-  && db_type != "berkeley"
-#endif
-  )
+  try
   {
-    std::cerr << "database type unrecognized" << ENDL;
-    return 1;
-  }
 
-  cryptonote::cryptonote_protocol_stub pr; //TODO: stub only for this kind of test, make real validation of relayed objects
-  cryptonote::core core(&pr);
   core.disable_dns_checkpoints(true);
   if (!core.init(vm, NULL))
   {
@@ -827,23 +729,19 @@ int main(int argc, char* argv[])
 
   import_from_file(core, import_file_path, block_stop);
 
+  // ensure db closed
+  //   - transactions properly checked and handled
+  //   - disk sync if needed
+  //
+  core.deinit();
   }
   catch (const DB_ERROR& e)
   {
     std::cout << std::string("Error loading blockchain db: ") + e.what() + " -- shutting down now" << ENDL;
+    core.deinit();
     return 1;
   }
 
-  // destructors called at exit:
-  //
-  // ensure db closed
-  //   - transactions properly checked and handled
-  //   - disk sync if needed
-  //
-  // fake_core object's destructor is called when it goes out of scope. For an
-  // LMDB fake_core, it calls Blockchain::deinit() on its object, which in turn
-  // calls delete on its BlockchainDB derived class' object, which closes its
-  // files.
   return 0;
 
   CATCH_ENTRY("Import error", 1);

src/blockchain_utilities/blockchain_utilities.h

@@ -34,6 +34,7 @@
 // bounds checking is done before writing to buffer, but buffer size
 // should be a sensible maximum
 #define BUFFER_SIZE 1000000
+#define CHUNK_SIZE_WARNING_THRESHOLD 500000
 #define NUM_BLOCKS_PER_CHUNK 1
 #define BLOCKCHAIN_RAW "blockchain.raw"
 

src/blockchain_utilities/blocksdat_file.h

@@ -38,7 +38,6 @@
 #include "cryptonote_basic/cryptonote_boost_serialization.h"
 #include "cryptonote_core/blockchain.h"
 #include "blockchain_db/blockchain_db.h"
-#include "blockchain_db/lmdb/db_lmdb.h"
 
 #include <algorithm>
 #include <cstdio>

src/blockchain_utilities/bootstrap_file.cpp

@@ -436,10 +436,10 @@ uint64_t BootstrapFile::count_blocks(const std::string& import_file_path)
           << "  height: " << h-1);
       throw std::runtime_error("Aborting: chunk size exceeds buffer size");
     }
-    if (chunk_size > 100000)
+    if (chunk_size > CHUNK_SIZE_WARNING_THRESHOLD)
     {
       std::cout << refresh_string;
-      MDEBUG("NOTE: chunk_size " << chunk_size << " > 100000" << "  height: "
+      MDEBUG("NOTE: chunk_size " << chunk_size << " > " << CHUNK_SIZE_WARNING_THRESHOLD << " << height: "
           << h-1);
     }
     else if (chunk_size <= 0) {

src/common/command_line.cpp

@@ -96,7 +96,7 @@ namespace command_line
   , "checkpoints from DNS server will be enforced"
   , false
   };
-  std::string arg_db_type_description = "Specify database type, available: " + boost::algorithm::join(cryptonote::blockchain_db_types, ", ");
+  std::string arg_db_type_description = "Specify database type, available: " + cryptonote::blockchain_db_types(", ");
   const command_line::arg_descriptor<std::string> arg_db_type = {
     "db-type"
   , arg_db_type_description.c_str()
@@ -137,4 +137,9 @@ namespace command_line
   , "Check for new versions of monero: [disabled|notify|download|update]"
   , "notify"
   };
+  const arg_descriptor<bool> arg_fluffy_blocks  = {
+    "fluffy-blocks"
+  , "Relay blocks as fluffy blocks where possible (automatic on testnet)"
+  , false
+  };
 }

src/common/command_line.h

@@ -220,4 +220,5 @@ namespace command_line
   extern const arg_descriptor<uint64_t> arg_show_time_stats;
   extern const arg_descriptor<size_t> arg_block_sync_size;
   extern const arg_descriptor<std::string> arg_check_updates;
+  extern const arg_descriptor<bool> arg_fluffy_blocks;
 }

src/common/rpc_client.h

@@ -69,7 +69,7 @@ namespace tools
       bool ok = connection.is_open();
       if (!ok)
       {
-        fail_msg_writer() << "Couldn't connect to daemon";
+        fail_msg_writer() << "Couldn't connect to daemon: " << m_http_client.get_host() << ":" << m_http_client.get_port();
         return false;
       }
       ok = ok && epee::net_utils::invoke_http_json_rpc("/json_rpc", method_name, req, res, m_http_client, t_http_connection::TIMEOUT());
@@ -98,7 +98,7 @@ namespace tools
       ok = ok && epee::net_utils::invoke_http_json_rpc("/json_rpc", method_name, req, res, m_http_client, t_http_connection::TIMEOUT());
       if (!ok)
       {
-        fail_msg_writer() << "Couldn't connect to daemon";
+        fail_msg_writer() << "Couldn't connect to daemon: " << m_http_client.get_host() << ":" << m_http_client.get_port();
         return false;
       }
       else if (res.status != CORE_RPC_STATUS_OK) // TODO - handle CORE_RPC_STATUS_BUSY ?
@@ -126,7 +126,7 @@ namespace tools
       ok = ok && epee::net_utils::invoke_http_json(relative_url, req, res, m_http_client, t_http_connection::TIMEOUT());
       if (!ok)
       {
-        fail_msg_writer() << "Couldn't connect to daemon";
+        fail_msg_writer() << "Couldn't connect to daemon: " << m_http_client.get_host() << ":" << m_http_client.get_port();
         return false;
       }
       else if (res.status != CORE_RPC_STATUS_OK) // TODO - handle CORE_RPC_STATUS_BUSY ?

src/common/thread_group.cpp

@@ -32,6 +32,7 @@
 #include <limits>
 #include <stdexcept>
 
+#include "cryptonote_config.h"
 #include "common/util.h"
 
 namespace tools
@@ -63,8 +64,10 @@ thread_group::data::data(std::size_t count)
   , has_work()
   , stop(false) {
   threads.reserve(count);
+  boost::thread::attributes attrs;
+  attrs.set_stack_size(THREAD_STACK_SIZE);
   while (count--) {
-    threads.push_back(boost::thread(&thread_group::data::run, this));
+    threads.push_back(boost::thread(attrs, boost::bind(&thread_group::data::run, this)));
   }
 }
 

src/crypto/crypto.cpp

@@ -36,18 +36,13 @@
 #include <memory>
 #include <boost/thread/mutex.hpp>
 #include <boost/thread/lock_guard.hpp>
+#include <boost/shared_ptr.hpp>
 
 #include "common/varint.h"
 #include "warnings.h"
 #include "crypto.h"
 #include "hash.h"
 
-#if !defined(__FreeBSD__) && !defined(__OpenBSD__) && !defined(__DragonFly__)
- #include <alloca.h>
-#else
- #include <stdlib.h>
-#endif
-
 namespace crypto {
 
   using std::abort;
@@ -411,7 +406,9 @@ POP_WARNINGS
     ge_p3 image_unp;
     ge_dsmp image_pre;
     ec_scalar sum, k, h;
-    rs_comm *const buf = reinterpret_cast<rs_comm *>(alloca(rs_comm_size(pubs_count)));
+    boost::shared_ptr<rs_comm> buf(reinterpret_cast<rs_comm *>(malloc(rs_comm_size(pubs_count))), free);
+    if (!buf)
+      abort();
     assert(sec_index < pubs_count);
 #if !defined(NDEBUG)
     {
@@ -459,7 +456,7 @@ POP_WARNINGS
         sc_add(&sum, &sum, &sig[i].c);
       }
     }
-    hash_to_scalar(buf, rs_comm_size(pubs_count), h);
+    hash_to_scalar(buf.get(), rs_comm_size(pubs_count), h);
     sc_sub(&sig[sec_index].c, &h, &sum);
     sc_mulsub(&sig[sec_index].r, &sig[sec_index].c, &sec, &k);
   }
@@ -471,7 +468,9 @@ POP_WARNINGS
     ge_p3 image_unp;
     ge_dsmp image_pre;
     ec_scalar sum, h;
-    rs_comm *const buf = reinterpret_cast<rs_comm *>(alloca(rs_comm_size(pubs_count)));
+    boost::shared_ptr<rs_comm> buf(reinterpret_cast<rs_comm *>(malloc(rs_comm_size(pubs_count))), free);
+    if (!buf)
+      return false;
 #if !defined(NDEBUG)
     for (i = 0; i < pubs_count; i++) {
       assert(check_key(*pubs[i]));
@@ -499,7 +498,7 @@ POP_WARNINGS
       ge_tobytes(&buf->ab[i].b, &tmp2);
       sc_add(&sum, &sum, &sig[i].c);
     }
-    hash_to_scalar(buf, rs_comm_size(pubs_count), h);
+    hash_to_scalar(buf.get(), rs_comm_size(pubs_count), h);
     sc_sub(&h, &h, &sum);
     return sc_isnonzero(&h) == 0;
   }

src/cryptonote_basic/checkpoints.cpp

@@ -167,6 +167,8 @@ namespace cryptonote
     ADD_CHECKPOINT(1100000, "3fd720c5c8b3072fc1ccda922dec1ef25f9ed88a1e6ad4103d0fe00b180a5903");
     ADD_CHECKPOINT(1150000, "1dd16f626d18e1e988490dfd06de5920e22629c972c58b4d8daddea0038627b2");
     ADD_CHECKPOINT(1200000, "fa7d13a90850882060479d100141ff84286599ae39c3277c8ea784393f882d1f");
+    ADD_CHECKPOINT(1300000, "31b34272343a44a9f4ac7de7a8fcf3b7d8a3124d7d6870affd510d2f37e74cd0");
+    ADD_CHECKPOINT(1390000, "a8f5649dd4ded60eedab475f2bec8c934681c07e3cf640e9be0617554f13ff6c");
     
 
     return true;

src/cryptonote_core/blockchain.cpp

@@ -59,6 +59,8 @@
 #undef MONERO_DEFAULT_LOG_CATEGORY
 #define MONERO_DEFAULT_LOG_CATEGORY "blockchain"
 
+#define FIND_BLOCKCHAIN_SUPPLEMENT_MAX_SIZE (100*1024*1024) // 100 MB
+
 //#include "serialization/json_archive.h"
 
 /* TODO:
@@ -99,6 +101,9 @@ static const struct {
   
   // version 5 starts from block 1288616, which is on or around the 15th of April, 2017. Fork time finalised on 2017-03-14.
   { 5, 1288616, 0, 1489520158 },  
+
+  // version 6 starts from block 1400000, which is on or around the 16th of September, 2017. Fork time finalised on 2017-08-18.
+  { 6, 1400000, 0, 1503046577 },
 };
 static const uint64_t mainnet_hard_fork_version_1_till = 1009826;
 
@@ -126,7 +131,7 @@ static const uint64_t testnet_hard_fork_version_1_till = 624633;
 //------------------------------------------------------------------
 Blockchain::Blockchain(tx_memory_pool& tx_pool) :
   m_db(), m_tx_pool(tx_pool), m_hardfork(NULL), m_timestamps_and_difficulties_height(0), m_current_block_cumul_sz_limit(0),
-  m_enforce_dns_checkpoints(false), m_max_prepare_blocks_threads(4), m_db_blocks_per_sync(1), m_db_sync_mode(db_async), m_fast_sync(true), m_show_time_stats(false), m_sync_counter(0), m_cancel(false)
+  m_enforce_dns_checkpoints(false), m_max_prepare_blocks_threads(4), m_db_blocks_per_sync(1), m_db_sync_mode(db_async), m_db_default_sync(false), m_fast_sync(true), m_show_time_stats(false), m_sync_counter(0), m_cancel(false)
 {
   LOG_PRINT_L3("Blockchain::" << __func__);
 }
@@ -2072,8 +2077,8 @@ bool Blockchain::find_blockchain_supplement(const uint64_t req_start_block, cons
 
   m_db->block_txn_start(true);
   total_height = get_current_blockchain_height();
-  size_t count = 0;
-  for(size_t i = start_height; i < total_height && count < max_count; i++, count++)
+  size_t count = 0, size = 0;
+  for(size_t i = start_height; i < total_height && count < max_count && (size < FIND_BLOCKCHAIN_SUPPLEMENT_MAX_SIZE || count < 3); i++, count++)
   {
     blocks.resize(blocks.size()+1);
     blocks.back().first = m_db->get_block_blob_from_height(i);
@@ -2082,6 +2087,9 @@ bool Blockchain::find_blockchain_supplement(const uint64_t req_start_block, cons
     std::list<crypto::hash> mis;
     get_transactions_blobs(b.tx_hashes, blocks.back().second, mis);
     CHECK_AND_ASSERT_MES(!mis.size(), false, "internal error, transaction from block not found");
+    size += blocks.back().first.size();
+    for (const auto &t: blocks.back().second)
+      size += t.size();
   }
   m_db->block_txn_stop();
   return true;
@@ -2234,6 +2242,24 @@ bool Blockchain::get_tx_outputs_gindexs(const crypto::hash& tx_id, std::vector<u
   return true;
 }
 //------------------------------------------------------------------
+void Blockchain::on_new_tx_from_block(const cryptonote::transaction &tx)
+{
+#if defined(PER_BLOCK_CHECKPOINT)
+  // check if we're doing per-block checkpointing
+  if (m_db->height() < m_blocks_hash_check.size())
+  {
+    TIME_MEASURE_START(a);
+    m_blocks_txs_check.push_back(get_transaction_hash(tx));
+    TIME_MEASURE_FINISH(a);
+    if(m_show_time_stats)
+    {
+      size_t ring_size = tx.vin[0].type() == typeid(txin_to_key) ? boost::get<txin_to_key>(tx.vin[0]).key_offsets.size() : 0;
+      MINFO("HASH: " << "-" << " I/M/O: " << tx.vin.size() << "/" << ring_size << "/" << tx.vout.size() << " H: " << 0 << " chcktx: " << a);
+    }
+  }
+#endif
+}
+//------------------------------------------------------------------
 //FIXME: it seems this function is meant to be merely a wrapper around
 //       another function of the same name, this one adding one bit of
 //       functionality.  Should probably move anything more than that
@@ -2249,19 +2275,10 @@ bool Blockchain::check_tx_inputs(transaction& tx, uint64_t& max_used_block_heigh
 
 #if defined(PER_BLOCK_CHECKPOINT)
   // check if we're doing per-block checkpointing
-  // FIXME: investigate why this block returns
   if (m_db->height() < m_blocks_hash_check.size() && kept_by_block)
   {
-    TIME_MEASURE_START(a);
-    m_blocks_txs_check.push_back(get_transaction_hash(tx));
     max_used_block_id = null_hash;
     max_used_block_height = 0;
-    TIME_MEASURE_FINISH(a);
-    if(m_show_time_stats)
-    {
-      size_t ring_size = tx.vin[0].type() == typeid(txin_to_key) ? boost::get<txin_to_key>(tx.vin[0]).key_offsets.size() : 0;
-      MINFO("HASH: " << "-" << " I/M/O: " << tx.vin.size() << "/" << ring_size << "/" << tx.vout.size() << " H: " << 0 << " chcktx: " << a);
-    }
     return true;
   }
 #endif
@@ -3241,7 +3258,7 @@ leave:
 
 // XXX old code adds miner tx here
 
-  int tx_index = 0;
+  size_t tx_index = 0;
   // Iterate over the block's transaction hashes, grabbing each
   // from the tx_pool and validating them.  Each is then added
   // to txs.  Keys spent in each are added to <keys> by the double spend check.
@@ -3323,7 +3340,7 @@ leave:
     {
       // ND: if fast_check is enabled for blocks, there is no need to check
       // the transaction inputs, but do some sanity checks anyway.
-      if (memcmp(&m_blocks_txs_check[tx_index++], &tx_id, sizeof(tx_id)) != 0)
+      if (tx_index >= m_blocks_txs_check.size() || memcmp(&m_blocks_txs_check[tx_index++], &tx_id, sizeof(tx_id)) != 0)
       {
         MERROR_VER("Block with id: " << id << " has at least one transaction (id: " << tx_id << ") with wrong inputs.");
         //TODO: why is this done?  make sure that keeping invalid blocks makes sense.
@@ -3574,12 +3591,23 @@ void Blockchain::block_longhash_worker(uint64_t height, const std::vector<block>
 //------------------------------------------------------------------
 bool Blockchain::cleanup_handle_incoming_blocks(bool force_sync)
 {
+  bool success = false;
+
   MTRACE("Blockchain::" << __func__);
   CRITICAL_REGION_BEGIN(m_blockchain_lock);
   TIME_MEASURE_START(t1);
 
-  m_db->batch_stop();
-  if (m_sync_counter > 0)
+  try
+  {
+    m_db->batch_stop();
+    success = true;
+  }
+  catch (const std::exception &e)
+  {
+    MERROR("Exception in cleanup_handle_incoming_blocks: " << e.what());
+  }
+
+  if (success && m_sync_counter > 0)
   {
     if (force_sync)
     {
@@ -3614,7 +3642,7 @@ bool Blockchain::cleanup_handle_incoming_blocks(bool force_sync)
   CRITICAL_REGION_END();
   m_tx_pool.unlock();
 
-  return true;
+  return success;
 }
 
 //------------------------------------------------------------------
@@ -4034,12 +4062,29 @@ bool Blockchain::for_all_txpool_txes(std::function<bool(const crypto::hash&, con
 
 void Blockchain::set_user_options(uint64_t maxthreads, uint64_t blocks_per_sync, blockchain_db_sync_mode sync_mode, bool fast_sync)
 {
+  if (sync_mode == db_defaultsync)
+  {
+    m_db_default_sync = true;
+    sync_mode = db_async;
+  }
   m_db_sync_mode = sync_mode;
   m_fast_sync = fast_sync;
   m_db_blocks_per_sync = blocks_per_sync;
   m_max_prepare_blocks_threads = maxthreads;
 }
 
+void Blockchain::safesyncmode(const bool onoff)
+{
+  /* all of this is no-op'd if the user set a specific
+   * --db-sync-mode at startup.
+   */
+  if (m_db_default_sync)
+  {
+    m_db->safesyncmode(onoff);
+    m_db_sync_mode = onoff ? db_nosync : db_async;
+  }
+}
+
 HardFork::State Blockchain::get_hard_fork_state() const
 {
   return m_hardfork->get_state();
@@ -4093,7 +4138,7 @@ void Blockchain::cancel()
 }
 
 #if defined(PER_BLOCK_CHECKPOINT)
-static const char expected_block_hashes_hash[] = "23d8a8c73de7b2383c72a016d9a6034e69d62dd48077d1c414e064ceab6daa94";
+static const char expected_block_hashes_hash[] = "d3ca80d50661684cde0e715d46d7c19704d2e216b21ed088af9fd4ef37ed4d65";
 void Blockchain::load_compiled_in_block_hashes()
 {
   if (m_fast_sync && get_blocks_dat_start(m_testnet) != nullptr && get_blocks_dat_size(m_testnet) > 0)

src/cryptonote_core/blockchain.h

@@ -65,6 +65,7 @@ namespace cryptonote
    */
   enum blockchain_db_sync_mode
   {
+    db_defaultsync, //!< user didn't specify, use db_async
     db_sync,  //!< handle syncing calls instead of the backing db, synchronously
     db_async, //!< handle syncing calls instead of the backing db, asynchronously
     db_nosync //!< Leave syncing up to the backing db (safest, but slowest because of disk I/O)
@@ -700,6 +701,11 @@ namespace cryptonote
     void set_user_options(uint64_t maxthreads, uint64_t blocks_per_sync,
         blockchain_db_sync_mode sync_mode, bool fast_sync);
 
+    /**
+     * @brief Put DB in safe sync mode
+     */
+    void safesyncmode(const bool onoff);
+
     /**
      * @brief set whether or not to show/print time statistics
      *
@@ -892,6 +898,13 @@ namespace cryptonote
 
     void cancel();
 
+    /**
+     * @brief called when we see a tx originating from a block
+     *
+     * Used for handling txes from historical blocks in a fast way
+     */
+    void on_new_tx_from_block(const cryptonote::transaction &tx);
+
   private:
 
     // TODO: evaluate whether or not each of these typedefs are left over from blockchain_storage
@@ -932,6 +945,7 @@ namespace cryptonote
     blockchain_db_sync_mode m_db_sync_mode;
     bool m_fast_sync;
     bool m_show_time_stats;
+    bool m_db_default_sync;
     uint64_t m_db_blocks_per_sync;
     uint64_t m_max_prepare_blocks_threads;
     uint64_t m_fake_pow_calc_time;

src/cryptonote_core/cryptonote_core.cpp

@@ -48,10 +48,6 @@ using namespace epee;
 #include "cryptonote_basic/checkpoints.h"
 #include "ringct/rctTypes.h"
 #include "blockchain_db/blockchain_db.h"
-#include "blockchain_db/lmdb/db_lmdb.h"
-#if defined(BERKELEY_DB)
-#include "blockchain_db/berkeleydb/db_bdb.h"
-#endif
 #include "ringct/rctSigs.h"
 
 #undef MONERO_DEFAULT_LOG_CATEGORY
@@ -168,6 +164,7 @@ namespace cryptonote
     command_line::add_arg(desc, command_line::arg_show_time_stats);
     command_line::add_arg(desc, command_line::arg_block_sync_size);
     command_line::add_arg(desc, command_line::arg_check_updates);
+    command_line::add_arg(desc, command_line::arg_fluffy_blocks);
 
     // we now also need some of net_node's options (p2p bind arg, for separate data dir)
     command_line::add_arg(desc, nodetool::arg_testnet_p2p_bind_port, false);
@@ -203,6 +200,7 @@ namespace cryptonote
 
     set_enforce_dns_checkpoints(command_line::get_arg(vm, command_line::arg_dns_checkpoints));
     test_drop_download_height(command_line::get_arg(vm, command_line::arg_test_drop_download_height));
+    m_fluffy_blocks_enabled = m_testnet || get_arg(vm, command_line::arg_fluffy_blocks);
 
     if (command_line::get_arg(vm, command_line::arg_test_drop_download) == true)
       test_drop_download();
@@ -246,6 +244,12 @@ namespace cryptonote
     return m_blockchain_storage.get_transactions_blobs(txs_ids, txs, missed_txs);
   }
   //-----------------------------------------------------------------------------------------------
+  bool core::get_txpool_backlog(std::vector<tx_backlog_entry>& backlog) const
+  {
+    m_mempool.get_transaction_backlog(backlog);
+    return true;
+  }
+  //-----------------------------------------------------------------------------------------------
   bool core::get_transactions(const std::vector<crypto::hash>& txs_ids, std::list<transaction>& txs, std::list<crypto::hash>& missed_txs) const
   {
     return m_blockchain_storage.get_transactions(txs_ids, txs, missed_txs);
@@ -308,20 +312,8 @@ namespace cryptonote
     // folder might not be a directory, etc, etc
     catch (...) { }
 
-    BlockchainDB* db = nullptr;
-    uint64_t DBS_FAST_MODE = 0;
-    uint64_t DBS_FASTEST_MODE = 0;
-    uint64_t DBS_SAFE_MODE = 0;
-    uint64_t DBS_SALVAGE = 0;
-    if (db_type == "lmdb")
-    {
-      db = new BlockchainLMDB();
-      DBS_SAFE_MODE = MDB_NORDAHEAD;
-      DBS_FAST_MODE = MDB_NORDAHEAD | MDB_NOSYNC;
-      DBS_FASTEST_MODE = MDB_NORDAHEAD | MDB_NOSYNC | MDB_WRITEMAP | MDB_MAPASYNC;
-      DBS_SALVAGE = MDB_PREVSNAPSHOT;
-    }
-    else
+    BlockchainDB* db = new_db(db_type);
+    if (db == NULL)
     {
       LOG_ERROR("Attempted to use non-existent database type");
       return false;
@@ -332,7 +324,7 @@ namespace cryptonote
 
     const std::string filename = folder.string();
     // default to fast:async:1
-    blockchain_db_sync_mode sync_mode = db_async;
+    blockchain_db_sync_mode sync_mode = db_defaultsync;
     uint64_t blocks_per_sync = 1;
 
     try
@@ -347,7 +339,7 @@ namespace cryptonote
         MDEBUG("option: " << option);
 
       // default to fast:async:1
-      uint64_t DEFAULT_FLAGS = DBS_FAST_MODE;
+      uint64_t DEFAULT_FLAGS = DBF_FAST;
 
       if(options.size() == 0)
       {
@@ -361,15 +353,19 @@ namespace cryptonote
         if(options[0] == "safe")
         {
           safemode = true;
-          db_flags = DBS_SAFE_MODE;
+          db_flags = DBF_SAFE;
           sync_mode = db_nosync;
         }
         else if(options[0] == "fast")
-          db_flags = DBS_FAST_MODE;
+        {
+          db_flags = DBF_FAST;
+          sync_mode = db_async;
+        }
         else if(options[0] == "fastest")
         {
-          db_flags = DBS_FASTEST_MODE;
+          db_flags = DBF_FASTEST;
           blocks_per_sync = 1000; // default to fastest:async:1000
+          sync_mode = db_async;
         }
         else
           db_flags = DEFAULT_FLAGS;
@@ -392,7 +388,7 @@ namespace cryptonote
       }
 
       if (db_salvage)
-        db_flags |= DBS_SALVAGE;
+        db_flags |= DBF_SALVAGE;
 
       db->open(filename, db_flags);
       if(!db->m_open)
@@ -591,6 +587,8 @@ namespace cryptonote
   //-----------------------------------------------------------------------------------------------
   bool core::handle_incoming_txs(const std::list<blobdata>& tx_blobs, std::vector<tx_verification_context>& tvc, bool keeped_by_block, bool relayed, bool do_not_relay)
   {
+    TRY_ENTRY();
+
     struct result { bool res; cryptonote::transaction tx; crypto::hash hash; crypto::hash prefix_hash; bool in_txpool; bool in_blockchain; };
     std::vector<result> results(tx_blobs.size());
 
@@ -599,7 +597,15 @@ namespace cryptonote
       std::list<blobdata>::const_iterator it = tx_blobs.begin();
       for (size_t i = 0; i < tx_blobs.size(); i++, ++it) {
         region.run([&, i, it] {
-          results[i].res = handle_incoming_tx_pre(*it, tvc[i], results[i].tx, results[i].hash, results[i].prefix_hash, keeped_by_block, relayed, do_not_relay);
+          try
+          {
+            results[i].res = handle_incoming_tx_pre(*it, tvc[i], results[i].tx, results[i].hash, results[i].prefix_hash, keeped_by_block, relayed, do_not_relay);
+          }
+          catch (const std::exception &e)
+          {
+            MERROR_VER("Exception in handle_incoming_tx_pre: " << e.what());
+            results[i].res = false;
+          }
         });
       }
     });
@@ -619,7 +625,15 @@ namespace cryptonote
         else
         {
           region.run([&, i, it] {
-            results[i].res = handle_incoming_tx_post(*it, tvc[i], results[i].tx, results[i].hash, results[i].prefix_hash, keeped_by_block, relayed, do_not_relay);
+            try
+            {
+              results[i].res = handle_incoming_tx_post(*it, tvc[i], results[i].tx, results[i].hash, results[i].prefix_hash, keeped_by_block, relayed, do_not_relay);
+            }
+            catch (const std::exception &e)
+            {
+              MERROR_VER("Exception in handle_incoming_tx_post: " << e.what());
+              results[i].res = false;
+            }
           });
         }
       }
@@ -644,6 +658,8 @@ namespace cryptonote
         MDEBUG("tx added: " << results[i].hash);
     }
     return ok;
+
+    CATCH_ENTRY_L0("core::handle_incoming_txs()", false);
   }
   //-----------------------------------------------------------------------------------------------
   bool core::handle_incoming_tx(const blobdata& tx_blob, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay)
@@ -880,6 +896,9 @@ namespace cryptonote
   //-----------------------------------------------------------------------------------------------
   bool core::add_new_tx(transaction& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prefix_hash, size_t blob_size, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay)
   {
+    if (keeped_by_block)
+      get_blockchain_storage().on_new_tx_from_block(tx);
+
     if(m_mempool.have_tx(tx_hash))
     {
       LOG_PRINT_L2("tx " << tx_hash << "already have transaction in tx_pool");
@@ -1007,7 +1026,15 @@ namespace cryptonote
     block_verification_context bvc = boost::value_initialized<block_verification_context>();
     m_miner.pause();
     std::list<block_complete_entry> blocks;
-    blocks.push_back(get_block_complete_entry(b, m_mempool));
+    try
+    {
+      blocks.push_back(get_block_complete_entry(b, m_mempool));
+    }
+    catch (const std::exception &e)
+    {
+      m_miner.resume();
+      return false;
+    }
     prepare_handle_incoming_blocks(blocks);
     m_blockchain_storage.add_new_block(b, bvc);
     cleanup_handle_incoming_blocks(true);
@@ -1049,6 +1076,11 @@ namespace cryptonote
     m_miner.on_synchronized();
   }
   //-----------------------------------------------------------------------------------------------
+  void core::safesyncmode(const bool onoff)
+  {
+    m_blockchain_storage.safesyncmode(onoff);
+  }
+  //-----------------------------------------------------------------------------------------------
   bool core::add_new_block(const block& b, block_verification_context& bvc)
   {
     return m_blockchain_storage.add_new_block(b, bvc);
@@ -1065,17 +1097,20 @@ namespace cryptonote
   //-----------------------------------------------------------------------------------------------
   bool core::cleanup_handle_incoming_blocks(bool force_sync)
   {
+    bool success = false;
     try {
-      m_blockchain_storage.cleanup_handle_incoming_blocks(force_sync);
+      success = m_blockchain_storage.cleanup_handle_incoming_blocks(force_sync);
     }
     catch (...) {}
     m_incoming_tx_lock.unlock();
-    return true;
+    return success;
   }
 
   //-----------------------------------------------------------------------------------------------
   bool core::handle_incoming_block(const blobdata& block_blob, block_verification_context& bvc, bool update_miner_blocktemplate)
   {
+    TRY_ENTRY();
+
     // load json & DNS checkpoints every 10min/hour respectively,
     // and verify them with respect to what blocks we already have
     CHECK_AND_ASSERT_MES(update_checkpoints(), false, "One or more checkpoints loaded from json or dns conflicted with existing checkpoints.");
@@ -1099,6 +1134,8 @@ namespace cryptonote
     if(update_miner_blocktemplate && bvc.m_added_to_main_chain)
        update_miner_block_template();
     return true;
+
+    CATCH_ENTRY_L0("core::handle_incoming_block()", false);
   }
   //-----------------------------------------------------------------------------------------------
   // Used by the RPC server to check the size of an incoming
@@ -1265,11 +1302,12 @@ namespace cryptonote
   bool core::check_updates()
   {
     static const char software[] = "monero";
-    static const char subdir[] = "cli"; // because it can never be simple
 #ifdef BUILD_TAG
     static const char buildtag[] = BOOST_PP_STRINGIZE(BUILD_TAG);
+    static const char subdir[] = "cli"; // because it can never be simple
 #else
     static const char buildtag[] = "source";
+    static const char subdir[] = "source"; // because it can never be simple
 #endif
 
     if (check_updates_level == UPDATES_DISABLED)

src/cryptonote_core/cryptonote_core.h

@@ -427,6 +427,13 @@ namespace cryptonote
       */
      bool get_pool_transactions(std::list<transaction>& txs) const;
      
+     /**
+      * @copydoc tx_memory_pool::get_txpool_backlog
+      *
+      * @note see tx_memory_pool::get_txpool_backlog
+      */
+     bool get_txpool_backlog(std::vector<tx_backlog_entry>& backlog) const;
+     
      /**
       * @copydoc tx_memory_pool::get_transactions
       *
@@ -613,6 +620,13 @@ namespace cryptonote
       */
      void on_synchronized();
 
+     /**
+      * @copydoc Blockchain::safesyncmode
+      *
+      * 2note see Blockchain::safesyncmode
+      */
+     void safesyncmode(const bool onoff);
+
      /**
       * @brief sets the target blockchain height
       *
@@ -714,6 +728,13 @@ namespace cryptonote
       */     
      bool get_testnet() const { return m_testnet; };
 
+     /**
+      * @brief get whether fluffy blocks are enabled
+      *
+      * @return whether fluffy blocks are enabled
+      */
+     bool fluffy_blocks_enabled() const { return m_fluffy_blocks_enabled; }
+
    private:
 
      /**
@@ -931,6 +952,8 @@ namespace cryptonote
      tools::download_async_handle m_update_download;
      size_t m_last_update_length;
      boost::mutex m_update_mutex;
+
+     bool m_fluffy_blocks_enabled;
    };
 }
 

src/cryptonote_core/tx_pool.cpp

@@ -92,7 +92,7 @@ namespace cryptonote
       LockedTXN(Blockchain &b): m_blockchain(b), m_batch(false) {
         m_batch = m_blockchain.get_db().batch_start();
       }
-      ~LockedTXN() { if (m_batch) { m_blockchain.get_db().batch_stop(); } }
+      ~LockedTXN() { try { if (m_batch) { m_blockchain.get_db().batch_stop(); } } catch (const std::exception &e) { MWARNING("LockedTXN dtor filtering exception: " << e.what()); } }
     private:
       Blockchain &m_blockchain;
       bool m_batch;
@@ -553,6 +553,17 @@ namespace cryptonote
     });
   }
   //------------------------------------------------------------------
+  void tx_memory_pool::get_transaction_backlog(std::vector<tx_backlog_entry>& backlog) const
+  {
+    CRITICAL_REGION_LOCAL(m_transactions_lock);
+    CRITICAL_REGION_LOCAL1(m_blockchain);
+    const uint64_t now = time(NULL);
+    m_blockchain.for_all_txpool_txes([&backlog, now](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *bd){
+      backlog.push_back({meta.blob_size, meta.fee, meta.receive_time - now});
+      return true;
+    });
+  }
+  //------------------------------------------------------------------
   void tx_memory_pool::get_transaction_stats(struct txpool_stats& stats) const
   {
     CRITICAL_REGION_LOCAL(m_transactions_lock);
@@ -848,6 +859,9 @@ namespace cryptonote
     std::unordered_set<crypto::key_image> k_images;
 
     LOG_PRINT_L2("Filling block template, median size " << median_size << ", " << m_txs_by_fee_and_receive_time.size() << " txes in the pool");
+
+    LockedTXN lock(m_blockchain);
+
     auto sorted_it = m_txs_by_fee_and_receive_time.begin();
     while (sorted_it != m_txs_by_fee_and_receive_time.end())
     {

src/cryptonote_core/tx_pool.h

@@ -242,6 +242,13 @@ namespace cryptonote
      */
     void get_transaction_hashes(std::vector<crypto::hash>& txs) const;
 
+    /**
+     * @brief get (size, fee, receive time) for all transaction in the pool
+     *
+     * @param txs return-by-reference that data
+     */
+    void get_transaction_backlog(std::vector<tx_backlog_entry>& backlog) const;
+
     /**
      * @brief get a summary statistics of all transaction hashes in the pool
      *

src/cryptonote_protocol/block_queue.cpp

@@ -52,8 +52,11 @@ namespace cryptonote
 void block_queue::add_blocks(uint64_t height, std::list<cryptonote::block_complete_entry> bcel, const boost::uuids::uuid &connection_id, float rate, size_t size)
 {
   boost::unique_lock<boost::recursive_mutex> lock(mutex);
-  remove_span(height);
+  std::list<crypto::hash> hashes;
+  bool has_hashes = remove_span(height, &hashes);
   blocks.insert(span(height, std::move(bcel), connection_id, rate, size));
+  if (has_hashes)
+    set_span_hashes(height, connection_id, hashes);
 }
 
 void block_queue::add_blocks(uint64_t height, uint64_t nblocks, const boost::uuids::uuid &connection_id, boost::posix_time::ptime time)
@@ -92,17 +95,20 @@ void block_queue::flush_stale_spans(const std::set<boost::uuids::uuid> &live_con
   }
 }
 
-void block_queue::remove_span(uint64_t start_block_height)
+bool block_queue::remove_span(uint64_t start_block_height, std::list<crypto::hash> *hashes)
 {
   boost::unique_lock<boost::recursive_mutex> lock(mutex);
   for (block_map::iterator i = blocks.begin(); i != blocks.end(); ++i)
   {
     if (i->start_block_height == start_block_height)
     {
+      if (hashes)
+        *hashes = std::move(i->hashes);
       blocks.erase(i);
-      return;
+      return true;
     }
   }
+  return false;
 }
 
 void block_queue::remove_spans(const boost::uuids::uuid &connection_id, uint64_t start_block_height)
@@ -278,6 +284,22 @@ bool block_queue::get_next_span(uint64_t &height, std::list<cryptonote::block_co
   return false;
 }
 
+bool block_queue::has_next_span(const boost::uuids::uuid &connection_id, bool &filled) const
+{
+  boost::unique_lock<boost::recursive_mutex> lock(mutex);
+  if (blocks.empty())
+    return false;
+  block_map::const_iterator i = blocks.begin();
+  if (is_blockchain_placeholder(*i))
+    ++i;
+  if (i == blocks.end())
+    return false;
+  if (i->connection_id != connection_id)
+    return false;
+  filled = !i->blocks.empty();
+  return true;
+}
+
 size_t block_queue::get_data_size() const
 {
   boost::unique_lock<boost::recursive_mutex> lock(mutex);

src/cryptonote_protocol/block_queue.h

@@ -71,7 +71,7 @@ namespace cryptonote
     void add_blocks(uint64_t height, uint64_t nblocks, const boost::uuids::uuid &connection_id, boost::posix_time::ptime time = boost::date_time::min_date_time);
     void flush_spans(const boost::uuids::uuid &connection_id, bool all = false);
     void flush_stale_spans(const std::set<boost::uuids::uuid> &live_connections);
-    void remove_span(uint64_t start_block_height);
+    bool remove_span(uint64_t start_block_height, std::list<crypto::hash> *hashes = NULL);
     void remove_spans(const boost::uuids::uuid &connection_id, uint64_t start_block_height);
     uint64_t get_max_block_height() const;
     void print() const;
@@ -82,6 +82,7 @@ namespace cryptonote
     std::pair<uint64_t, uint64_t> get_next_span_if_scheduled(std::list<crypto::hash> &hashes, boost::uuids::uuid &connection_id, boost::posix_time::ptime &time) const;
     void set_span_hashes(uint64_t start_height, const boost::uuids::uuid &connection_id, std::list<crypto::hash> hashes);
     bool get_next_span(uint64_t &height, std::list<cryptonote::block_complete_entry> &bcel, boost::uuids::uuid &connection_id, bool filled = true) const;
+    bool has_next_span(const boost::uuids::uuid &connection_id, bool &filled) const;
     size_t get_data_size() const;
     size_t get_num_filled_spans_prefix() const;
     size_t get_num_filled_spans() const;

src/cryptonote_protocol/cryptonote_protocol_handler.h

@@ -111,6 +111,7 @@ namespace cryptonote
     std::list<connection_info> get_connections();
     const block_queue &get_block_queue() const { return m_block_queue; }
     void stop();
+    void on_connection_close(cryptonote_connection_context &context);
   private:
     //----------------- commands handlers ----------------------------------------------
     int handle_notify_new_block(int command, NOTIFY_NEW_BLOCK::request& arg, cryptonote_connection_context& context);
@@ -133,6 +134,7 @@ namespace cryptonote
     bool should_download_next_span(cryptonote_connection_context& context) const;
     void drop_connection(cryptonote_connection_context &context, bool add_fail, bool flush_all_spans);
     bool kick_idle_peers();
+    int try_add_next_blocks(cryptonote_connection_context &context);
 
     t_core& m_core;
 

src/cryptonote_protocol/cryptonote_protocol_handler.inl

@@ -106,6 +106,11 @@ namespace cryptonote
       LOG_PRINT_CCONTEXT_L2("-->>NOTIFY_REQUEST_CHAIN: m_block_ids.size()=" << r.block_ids.size() );
       post_notify<NOTIFY_REQUEST_CHAIN>(r, context);
     }
+    else if(context.m_state == cryptonote_connection_context::state_standby)
+    {
+      context.m_state = cryptonote_connection_context::state_synchronizing;
+      try_add_next_blocks(context);
+    }
 
     return true;
   }
@@ -263,7 +268,9 @@ namespace cryptonote
     const uint8_t version = m_core.get_ideal_hard_fork_version(hshd.current_height - 1);
     if (version >= 6 && version != hshd.top_version)
     {
-      LOG_DEBUG_CC(context, "Ignoring due to wrong top version " << (unsigned)hshd.top_version << ", expected " << (unsigned)version);
+      if (version < hshd.top_version)
+        MCLOG_RED(el::Level::Warning, "global", context << " peer claims higher version that we think - we may be forked from the network and a software upgrade may be needed");
+      LOG_DEBUG_CC(context, "Ignoring due to wrong top version for block " << (hshd.current_height - 1) << ": " << (unsigned)hshd.top_version << ", expected " << (unsigned)version);
       return false;
     }
 
@@ -286,15 +293,17 @@ namespace cryptonote
     /* As I don't know if accessing hshd from core could be a good practice,
     I prefer pushing target height to the core at the same time it is pushed to the user.
     Nz. */
-    m_core.set_target_blockchain_height(static_cast<int64_t>(hshd.current_height));
+    m_core.set_target_blockchain_height((hshd.current_height));
     int64_t diff = static_cast<int64_t>(hshd.current_height) - static_cast<int64_t>(m_core.get_current_blockchain_height());
-	int64_t max_block_height = max(static_cast<int64_t>(hshd.current_height),static_cast<int64_t>(m_core.get_current_blockchain_height()));
-	int64_t last_block_v1 = m_core.get_testnet() ? 624633 : 1009826;
-	int64_t diff_v2 = max_block_height > last_block_v1 ? min(abs(diff), max_block_height - last_block_v1) : 0;
+    uint64_t abs_diff = std::abs(diff);
+    uint64_t max_block_height = max(hshd.current_height,m_core.get_current_blockchain_height());
+    uint64_t last_block_v1 = m_core.get_testnet() ? 624633 : 1009826;
+    uint64_t diff_v2 = max_block_height > last_block_v1 ? min(abs_diff, max_block_height - last_block_v1) : 0;
     MCLOG(is_inital ? el::Level::Info : el::Level::Debug, "global", context <<  "Sync data returned a new top block candidate: " << m_core.get_current_blockchain_height() << " -> " << hshd.current_height
-      << " [Your node is " << std::abs(diff) << " blocks (" << ((abs(diff) - diff_v2) / (24 * 60 * 60 / DIFFICULTY_TARGET_V1)) + (diff_v2 / (24 * 60 * 60 / DIFFICULTY_TARGET_V2)) << " days) "
+      << " [Your node is " << abs_diff << " blocks (" << ((abs_diff - diff_v2) / (24 * 60 * 60 / DIFFICULTY_TARGET_V1)) + (diff_v2 / (24 * 60 * 60 / DIFFICULTY_TARGET_V2)) << " days) "
       << (0 <= diff ? std::string("behind") : std::string("ahead"))
       << "] " << ENDL << "SYNCHRONIZATION started");
+      m_core.safesyncmode(false);
     }
     LOG_PRINT_L1("Remote blockchain height: " << hshd.current_height << ", id: " << hshd.top_id);
     context.m_state = cryptonote_connection_context::state_synchronizing;
@@ -309,7 +318,7 @@ namespace cryptonote
   bool t_cryptonote_protocol_handler<t_core>::get_payload_sync_data(CORE_SYNC_DATA& hshd)
   {
     m_core.get_blockchain_top(hshd.current_height, hshd.top_id);
-    hshd.top_version = m_core.get_hard_fork_version(hshd.current_height);
+    hshd.top_version = m_core.get_ideal_hard_fork_version(hshd.current_height);
     hshd.cumulative_difficulty = m_core.get_block_cumulative_difficulty(hshd.current_height);
     hshd.current_height +=1;
     return true;
@@ -355,7 +364,12 @@ namespace cryptonote
 
     block_verification_context bvc = boost::value_initialized<block_verification_context>();
     m_core.handle_incoming_block(arg.b.block, bvc); // got block from handle_notify_new_block
-    m_core.cleanup_handle_incoming_blocks(true);
+    if (!m_core.cleanup_handle_incoming_blocks(true))
+    {
+      LOG_PRINT_CCONTEXT_L0("Failure in cleanup_handle_incoming_blocks");
+      m_core.resume_mine();
+      return 1;
+    }
     m_core.resume_mine();
     if(bvc.m_verifivation_failed)
     {
@@ -614,7 +628,12 @@ namespace cryptonote
           
         block_verification_context bvc = boost::value_initialized<block_verification_context>();
         m_core.handle_incoming_block(arg.b.block, bvc); // got block from handle_notify_new_block
-        m_core.cleanup_handle_incoming_blocks(true);
+        if (!m_core.cleanup_handle_incoming_blocks(true))
+        {
+          LOG_PRINT_CCONTEXT_L0("Failure in cleanup_handle_incoming_blocks");
+          m_core.resume_mine();
+          return 1;
+        }
         m_core.resume_mine();
         
         if( bvc.m_verifivation_failed )
@@ -819,8 +838,6 @@ namespace cryptonote
   {
     MLOG_P2P_MESSAGE("Received NOTIFY_RESPONSE_GET_OBJECTS (" << arg.blocks.size() << " blocks, " << arg.txs.size() << " txes)");
 
-    bool force_next_span = false;
-
     // calculate size of request
     size_t size = 0;
     for (const auto &element : arg.txs) size += element.size();
@@ -923,6 +940,7 @@ namespace cryptonote
     {
       const uint64_t subchain_height = start_height + arg.blocks.size();
       LOG_DEBUG_CC(context, "These are old blocks, ignoring: blocks " << start_height << " - " << (subchain_height-1) << ", blockchain height " << m_core.get_current_blockchain_height());
+      m_block_queue.remove_spans(context.m_connection_id, start_height);
       goto skip;
     }
 
@@ -938,19 +956,34 @@ namespace cryptonote
 
       context.m_last_known_hash = cryptonote::get_blob_hash(arg.blocks.back().block);
 
-      if (m_core.get_test_drop_download() && m_core.get_test_drop_download_height()) { // DISCARD BLOCKS for testing
+      if (!m_core.get_test_drop_download() || !m_core.get_test_drop_download_height()) { // DISCARD BLOCKS for testing
+        return 1;
+      }
+    }
 
-        // We try to lock the sync lock. If we can, it means no other thread is
-        // currently adding blocks, so we do that for as long as we can from the
-        // block queue. Then, we go back to download.
-        const boost::unique_lock<boost::mutex> sync{m_sync_lock, boost::try_to_lock};
-        if (!sync.owns_lock())
-        {
-          MINFO("Failed to lock m_sync_lock, going back to download");
-          goto skip;
-        }
-        MDEBUG(context << " lock m_sync_lock, adding blocks to chain...");
+skip:
+    try_add_next_blocks(context);
+    return 1;
+  }
+
+  template<class t_core>
+  int t_cryptonote_protocol_handler<t_core>::try_add_next_blocks(cryptonote_connection_context& context)
+  {
+    bool force_next_span = false;
 
+    {
+      // We try to lock the sync lock. If we can, it means no other thread is
+      // currently adding blocks, so we do that for as long as we can from the
+      // block queue. Then, we go back to download.
+      const boost::unique_lock<boost::mutex> sync{m_sync_lock, boost::try_to_lock};
+      if (!sync.owns_lock())
+      {
+        MINFO("Failed to lock m_sync_lock, going back to download");
+        goto skip;
+      }
+      MDEBUG(context << " lock m_sync_lock, adding blocks to chain...");
+
+      {
         m_core.pause_mine();
         epee::misc_utils::auto_scope_leave_caller scope_exit_handler = epee::misc_utils::create_scope_leave_handler(
           boost::bind(&t_core::resume_mine, &m_core));
@@ -984,21 +1017,15 @@ namespace cryptonote
             //  - later in an alt chain
             //  - orphan
             // if it was requested, then it'll be resolved later, otherwise it's an orphan
-            bool parent_requested = false;
-	    m_p2p->for_each_connection([&](cryptonote_connection_context& context, nodetool::peerid_type peer_id, uint32_t support_flags)->bool{
-	      if (context.m_requested_objects.find(new_block.prev_id) != context.m_requested_objects.end())
-              {
-                parent_requested = true;
-                return false;
-              }
-	      return true;
-	    });
+            bool parent_requested = m_block_queue.requested(new_block.prev_id);
             if (!parent_requested)
             {
-              LOG_ERROR_CCONTEXT("Got block with unknown parent which was not requested - dropping connection");
-              // in case the peer had dropped beforehand, remove the span anyway so other threads can wake up and get it
-              m_block_queue.remove_spans(span_connection_id, start_height);
-              return 1;
+              // this can happen if a connection was sicced onto a late span, if it did not have those blocks,
+              // since we don't know that at the sic time
+              LOG_ERROR_CCONTEXT("Got block with unknown parent which was not requested - querying block hashes");
+              context.m_needed_objects.clear();
+              context.m_last_response_height = 0;
+              goto skip;
             }
 
             // parent was requested, so we wait for it to be retrieved
@@ -1007,6 +1034,7 @@ namespace cryptonote
           }
 
           const boost::posix_time::ptime start = boost::posix_time::microsec_clock::universal_time();
+          context.m_last_request_time = start;
 
           m_core.prepare_handle_incoming_blocks(blocks);
 
@@ -1038,7 +1066,11 @@ namespace cryptonote
                 }))
                   LOG_ERROR_CCONTEXT("span connection id not found");
 
-                m_core.cleanup_handle_incoming_blocks();
+                if (!m_core.cleanup_handle_incoming_blocks())
+                {
+                  LOG_PRINT_CCONTEXT_L0("Failure in cleanup_handle_incoming_blocks");
+                  return 1;
+                }
                 // in case the peer had dropped beforehand, remove the span anyway so other threads can wake up and get it
                 m_block_queue.remove_spans(span_connection_id, start_height);
                 return 1;
@@ -1063,7 +1095,12 @@ namespace cryptonote
               }))
                 LOG_ERROR_CCONTEXT("span connection id not found");
 
-              m_core.cleanup_handle_incoming_blocks();
+              if (!m_core.cleanup_handle_incoming_blocks())
+              {
+                LOG_PRINT_CCONTEXT_L0("Failure in cleanup_handle_incoming_blocks");
+                return 1;
+              }
+
               // in case the peer had dropped beforehand, remove the span anyway so other threads can wake up and get it
               m_block_queue.remove_spans(span_connection_id, start_height);
               return 1;
@@ -1077,7 +1114,12 @@ namespace cryptonote
               }))
                 LOG_ERROR_CCONTEXT("span connection id not found");
 
-              m_core.cleanup_handle_incoming_blocks();
+              if (!m_core.cleanup_handle_incoming_blocks())
+              {
+                LOG_PRINT_CCONTEXT_L0("Failure in cleanup_handle_incoming_blocks");
+                return 1;
+              }
+
               // in case the peer had dropped beforehand, remove the span anyway so other threads can wake up and get it
               m_block_queue.remove_spans(span_connection_id, start_height);
               return 1;
@@ -1090,7 +1132,11 @@ namespace cryptonote
 
           MCINFO("sync-info", "Block process time (" << blocks.size() << " blocks, " << num_txs << " txs): " << block_process_time_full + transactions_process_time_full << " (" << transactions_process_time_full << "/" << block_process_time_full << ") ms");
 
-          m_core.cleanup_handle_incoming_blocks();
+          if (!m_core.cleanup_handle_incoming_blocks())
+          {
+            LOG_PRINT_CCONTEXT_L0("Failure in cleanup_handle_incoming_blocks");
+            return 1;
+          }
 
           m_block_queue.remove_spans(span_connection_id, start_height);
 
@@ -1108,7 +1154,7 @@ namespace cryptonote
                 << timing_message);
           }
         }
-      } // if not DISCARD BLOCK
+      }
 
       if (should_download_next_span(context))
       {
@@ -1179,9 +1225,17 @@ skip:
     std::list<crypto::hash> hashes;
     boost::uuids::uuid span_connection_id;
     boost::posix_time::ptime request_time;
-    std::pair<uint64_t, uint64_t> span = m_block_queue.get_next_span_if_scheduled(hashes, span_connection_id, request_time);
+    std::pair<uint64_t, uint64_t> span;
+
+    span = m_block_queue.get_start_gap_span();
+    if (span.second > 0)
+    {
+      MDEBUG(context << " we should download it as there is a gap");
+      return true;
+    }
 
     // if the next span is not scheduled (or there is none)
+    span = m_block_queue.get_next_span_if_scheduled(hashes, span_connection_id, request_time);
     if (span.second == 0)
     {
       // we might be in a weird case where there is a filled next span,
@@ -1270,6 +1324,17 @@ skip:
           first = false;
           context.m_state = cryptonote_connection_context::state_standby;
         }
+
+        // this needs doing after we went to standby, so the callback knows what to do
+        bool filled;
+        if (m_block_queue.has_next_span(context.m_connection_id, filled) && !filled)
+        {
+          MDEBUG(context << " we have the next span, and it is scheduled, resuming");
+          ++context.m_callback_request_count;
+          m_p2p->request_callback(context);
+          return 1;
+        }
+
         for (size_t n = 0; n < 50; ++n)
         {
           if (m_stopping)
@@ -1289,9 +1354,8 @@ skip:
       size_t count = 0;
       const size_t count_limit = m_core.get_block_sync_size(m_core.get_current_blockchain_height());
       std::pair<uint64_t, uint64_t> span = std::make_pair(0, 0);
-      if (force_next_span)
       {
-        MDEBUG(context << " force_next_span is true, trying next span");
+        MDEBUG(context << " checking for gap");
         span = m_block_queue.get_start_gap_span();
         if (span.second > 0)
         {
@@ -1311,6 +1375,9 @@ skip:
           }
           MDEBUG(context << " we have the hashes for this gap");
         }
+      }
+      if (force_next_span)
+      {
         if (span.second == 0)
         {
           std::list<crypto::hash> hashes;
@@ -1360,7 +1427,12 @@ skip:
           for (const auto &hash: hashes)
           {
             req.blocks.push_back(hash);
+            ++count;
             context.m_requested_objects.insert(hash);
+            // that's atrocious O(n) wise, but this is rare
+            auto i = std::find(context.m_needed_objects.begin(), context.m_needed_objects.end(), hash);
+            if (i != context.m_needed_objects.end())
+              context.m_needed_objects.erase(i);
           }
         }
       }
@@ -1384,14 +1456,12 @@ skip:
             return false;
           }
 
-          std::list<crypto::hash> hashes;
           auto it = context.m_needed_objects.begin();
           for (size_t n = 0; n < span.second; ++n)
           {
             req.blocks.push_back(*it);
             ++count;
             context.m_requested_objects.insert(*it);
-            hashes.push_back(*it);
             auto j = it++;
             context.m_needed_objects.erase(j);
           }
@@ -1399,7 +1469,7 @@ skip:
 
         context.m_last_request_time = boost::posix_time::microsec_clock::universal_time();
         LOG_PRINT_CCONTEXT_L1("-->>NOTIFY_REQUEST_GET_OBJECTS: blocks.size()=" << req.blocks.size() << ", txs.size()=" << req.txs.size()
-            << "requested blocks count=" << count << " / " << count_limit << " from " << span.first);
+            << "requested blocks count=" << count << " / " << count_limit << " from " << span.first << ", first hash " << req.blocks.front());
         //epee::net_utils::network_throttle_manager::get_global_throttle_inreq().logger_handle_net("log/dr-monero/net/req-all.data", sec, get_avg_block_size());
 
         post_notify<NOTIFY_REQUEST_GET_OBJECTS>(req, context);
@@ -1473,6 +1543,7 @@ skip:
         << "**********************************************************************");
       m_core.on_synchronized();
     }
+    m_core.safesyncmode(true);
     return true;
   }
   //------------------------------------------------------------------------------------------------------------------------
@@ -1523,6 +1594,10 @@ skip:
       drop_connection(context, false, false);
       return 1;
     }
+
+    if (arg.total_height > m_core.get_target_blockchain_height())
+      m_core.set_target_blockchain_height(arg.total_height);
+
     return 1;
   }
   //------------------------------------------------------------------------------------------------------------------------
@@ -1547,7 +1622,7 @@ skip:
     {
       if (peer_id && exclude_context.m_connection_id != context.m_connection_id)
       {
-        if(m_core.get_testnet() && (support_flags & P2P_SUPPORT_FLAG_FLUFFY_BLOCKS))
+        if(m_core.fluffy_blocks_enabled() && (support_flags & P2P_SUPPORT_FLAG_FLUFFY_BLOCKS))
         {
           LOG_DEBUG_CC(context, "PEER SUPPORTS FLUFFY BLOCKS - RELAYING THIN/COMPACT WHATEVER BLOCK");
           fluffyConnections.push_back(context.m_connection_id);
@@ -1582,8 +1657,15 @@ skip:
   {
     if (add_fail)
       m_p2p->add_host_fail(context.m_remote_address);
+
     m_p2p->drop_connection(context);
 
+    m_block_queue.flush_spans(context.m_connection_id, flush_all_spans);
+  }
+  //------------------------------------------------------------------------------------------------------------------------
+  template<class t_core>
+  void t_cryptonote_protocol_handler<t_core>::on_connection_close(cryptonote_connection_context &context)
+  {
     uint64_t target = 0;
     m_p2p->for_each_connection([&](const connection_context& cntxt, nodetool::peerid_type peer_id, uint32_t support_flags) {
       if (cntxt.m_state >= cryptonote_connection_context::state_synchronizing && cntxt.m_connection_id != context.m_connection_id)
@@ -1597,7 +1679,7 @@ skip:
       m_core.set_target_blockchain_height(target);
     }
 
-    m_block_queue.flush_spans(context.m_connection_id, flush_all_spans);
+    m_block_queue.flush_spans(context.m_connection_id, false);
   }
 
   //------------------------------------------------------------------------------------------------------------------------

src/daemon/main.cpp

@@ -145,13 +145,10 @@ int main(int argc, char const * argv[])
     std::string db_type = command_line::get_arg(vm, command_line::arg_db_type);
 
     // verify that blockchaindb type is valid
-    if(cryptonote::blockchain_db_types.count(db_type) == 0)
+    if(!cryptonote::blockchain_valid_db_type(db_type))
     {
-      std::cout << "Invalid database type (" << db_type << "), available types are:" << std::endl;
-      for (const auto& type : cryptonote::blockchain_db_types)
-      {
-        std::cout << "\t" << type << std::endl;
-      }
+      std::cout << "Invalid database type (" << db_type << "), available types are: " <<
+        cryptonote::blockchain_db_types(", ") << std::endl;
       return 0;
     }
 

src/daemon/rpc_command_executor.cpp

@@ -113,18 +113,6 @@ namespace {
       return base;
     return base + " -- " + status;
   }
-
-  std::string pad(std::string s, size_t n, char c = ' ', bool prepend = false)
-  {
-    if (s.size() < n)
-    {
-      if (prepend)
-        s = std::string(n - s.size(), c) + s;
-      else
-        s.append(n - s.size(), c);
-    }
-    return s;
-  }
 }
 
 t_rpc_command_executor::t_rpc_command_executor(
@@ -432,16 +420,17 @@ bool t_rpc_command_executor::show_status() {
   }
 
   std::time_t uptime = std::time(nullptr) - ires.start_time;
+  uint64_t net_height = ires.target_height > ires.height ? ires.target_height : ires.height;
 
   tools::success_msg_writer() << boost::format("Height: %llu/%llu (%.1f%%) on %s, %s, net hash %s, v%u%s, %s, %u(out)+%u(in) connections, uptime %ud %uh %um %us")
     % (unsigned long long)ires.height
-    % (unsigned long long)(ires.target_height >= ires.height ? ires.target_height : ires.height)
+    % (unsigned long long)net_height
     % get_sync_percentage(ires)
     % (ires.testnet ? "testnet" : "mainnet")
     % (!has_mining_info ? "mining info unavailable" : mining_busy ? "syncing" : mres.active ? ( ( mres.is_background_mining_enabled ? "smart " : "" ) + std::string("mining at ") + get_mining_speed(mres.speed) ) : "not mining")
     % get_mining_speed(ires.difficulty / ires.target)
     % (unsigned)hfres.version
-    % get_fork_extra_info(hfres.earliest_height, ires.height, ires.target)
+    % get_fork_extra_info(hfres.earliest_height, net_height, ires.target)
     % (hfres.state == cryptonote::HardFork::Ready ? "up to date" : hfres.state == cryptonote::HardFork::UpdateNeeded ? "update needed" : "out of date, likely forked")
     % (unsigned)ires.outgoing_connections_count
     % (unsigned)ires.incoming_connections_count
@@ -497,7 +486,7 @@ bool t_rpc_command_executor::print_connections() {
     tools::msg_writer() 
      //<< std::setw(30) << std::left << in_out
      << std::setw(30) << std::left << address
-     << std::setw(20) << pad(info.peer_id, 16, '0', true)
+     << std::setw(20) << epee::string_tools::pad_string(info.peer_id, 16, '0', true)
      << std::setw(20) << info.support_flags
      << std::setw(30) << std::to_string(info.recv_count) + "("  + std::to_string(info.recv_idle_time) + ")/" + std::to_string(info.send_count) + "(" + std::to_string(info.send_idle_time) + ")"
      << std::setw(25) << info.state
@@ -939,6 +928,8 @@ bool t_rpc_command_executor::print_transaction_pool_short() {
 bool t_rpc_command_executor::print_transaction_pool_stats() {
   cryptonote::COMMAND_RPC_GET_TRANSACTION_POOL_STATS::request req;
   cryptonote::COMMAND_RPC_GET_TRANSACTION_POOL_STATS::response res;
+  cryptonote::COMMAND_RPC_GET_INFO::request ireq;
+  cryptonote::COMMAND_RPC_GET_INFO::response ires;
 
   std::string fail_message = "Problem fetching transaction pool stats";
 
@@ -948,6 +939,10 @@ bool t_rpc_command_executor::print_transaction_pool_stats() {
     {
       return true;
     }
+    if (!m_rpc_client->rpc_request(ireq, ires, "/getinfo", fail_message.c_str()))
+    {
+      return true;
+    }
   }
   else
   {
@@ -957,15 +952,32 @@ bool t_rpc_command_executor::print_transaction_pool_stats() {
       tools::fail_msg_writer() << make_error(fail_message, res.status);
       return true;
     }
+    if (!m_rpc_server->on_get_info(ireq, ires) || ires.status != CORE_RPC_STATUS_OK)
+    {
+      tools::fail_msg_writer() << make_error(fail_message, ires.status);
+      return true;
+    }
   }
 
   size_t n_transactions = res.pool_stats.txs_total;
   const uint64_t now = time(NULL);
   size_t avg_bytes = n_transactions ? res.pool_stats.bytes_total / n_transactions : 0;
 
+  std::string backlog_message;
+  const uint64_t full_reward_zone = ires.block_size_limit / 2;
+  if (res.pool_stats.bytes_total <= full_reward_zone)
+  {
+    backlog_message = "no backlog";
+  }
+  else
+  {
+    uint64_t backlog = (res.pool_stats.bytes_total + full_reward_zone - 1) / full_reward_zone;
+    backlog_message = (boost::format("estimated %u block (%u minutes) backlog") % backlog % (backlog * DIFFICULTY_TARGET_V2 / 60)).str();
+  }
+
   tools::msg_writer() << n_transactions << " tx(es), " << res.pool_stats.bytes_total << " bytes total (min " << res.pool_stats.bytes_min << ", max " << res.pool_stats.bytes_max << ", avg " << avg_bytes << ")" << std::endl
-      << "fees " << cryptonote::print_money(res.pool_stats.fee_total) << " (avg " << cryptonote::print_money(n_transactions ? res.pool_stats.fee_total / n_transactions : 0) << " per tx" << ", " << cryptonote::print_money(res.pool_stats.bytes_total ? res.pool_stats.fee_total / res.pool_stats.bytes_total : 0) << " per byte )" << std::endl
-      << res.pool_stats.num_not_relayed << " not relayed, " << res.pool_stats.num_failing << " failing, " << res.pool_stats.num_10m << " older than 10 minutes (oldest " << (res.pool_stats.oldest == 0 ? "-" : get_human_time_ago(res.pool_stats.oldest, now)) << ")";
+      << "fees " << cryptonote::print_money(res.pool_stats.fee_total) << " (avg " << cryptonote::print_money(n_transactions ? res.pool_stats.fee_total / n_transactions : 0) << " per tx" << ", " << cryptonote::print_money(res.pool_stats.bytes_total ? res.pool_stats.fee_total / res.pool_stats.bytes_total : 0) << " per byte)" << std::endl
+      << res.pool_stats.num_not_relayed << " not relayed, " << res.pool_stats.num_failing << " failing, " << res.pool_stats.num_10m << " older than 10 minutes (oldest " << (res.pool_stats.oldest == 0 ? "-" : get_human_time_ago(res.pool_stats.oldest, now)) << "), " << backlog_message;
 
   if (n_transactions > 1 && res.pool_stats.histo.size())
   {
@@ -1744,12 +1756,12 @@ bool t_rpc_command_executor::sync_info()
     tools::success_msg_writer() << std::to_string(res.peers.size()) << " peers";
     for (const auto &p: res.peers)
     {
-      std::string address = pad(p.info.address, 24);
+      std::string address = epee::string_tools::pad_string(p.info.address, 24);
       uint64_t nblocks = 0, size = 0;
       for (const auto &s: res.spans)
         if (s.rate > 0.0f && s.connection_id == p.info.connection_id)
           nblocks += s.nblocks, size += s.size;
-      tools::success_msg_writer() << address << "  " << pad(p.info.peer_id, 16, '0', true) << "  " << p.info.height << "  "  << p.info.current_download << " kB/s, " << nblocks << " blocks / " << size/1e6 << " MB queued";
+      tools::success_msg_writer() << address << "  " << epee::string_tools::pad_string(p.info.peer_id, 16, '0', true) << "  " << p.info.height << "  "  << p.info.current_download << " kB/s, " << nblocks << " blocks / " << size/1e6 << " MB queued";
     }
 
     uint64_t total_size = 0;
@@ -1758,7 +1770,7 @@ bool t_rpc_command_executor::sync_info()
     tools::success_msg_writer() << std::to_string(res.spans.size()) << " spans, " << total_size/1e6 << " MB";
     for (const auto &s: res.spans)
     {
-      std::string address = pad(s.remote_address, 24);
+      std::string address = epee::string_tools::pad_string(s.remote_address, 24);
       if (s.size == 0)
       {
         tools::success_msg_writer() << address << "  " << s.nblocks << " (" << s.start_block_height << " - " << (s.start_block_height + s.nblocks - 1) << ")  -";

src/debug_utilities/CMakeLists.txt

@@ -37,6 +37,7 @@ monero_add_executable(cn_deserialize
 target_link_libraries(cn_deserialize
   LINK_PRIVATE
     cryptonote_core
+    common
     blockchain_db
     p2p
     epee

src/p2p/net_node.inl

@@ -1077,7 +1077,7 @@ namespace nodetool
   bool node_server<t_payload_net_handler>::make_new_connection_from_anchor_peerlist(const std::vector<anchor_peerlist_entry>& anchor_peerlist)
   {
     for (const auto& pe: anchor_peerlist) {
-      _note("Considering connecting (out) to peer: " << pe.id << " " << pe.adr.str());
+      _note("Considering connecting (out) to peer: " << peerid_type(pe.id) << " " << pe.adr.str());
 
       if(is_peer_used(pe)) {
         _note("Peer is used");
@@ -1092,7 +1092,7 @@ namespace nodetool
         continue;
       }
 
-      MDEBUG("Selected peer: " << pe.id << " " << pe.adr.str()
+      MDEBUG("Selected peer: " << peerid_to_string(pe.id) << " " << pe.adr.str()
                                << "[peer_type=" << anchor
                                << "] first_seen: " << epee::misc_utils::get_time_interval_string(time(NULL) - pe.first_seen));
 
@@ -1145,7 +1145,7 @@ namespace nodetool
 
       ++try_count;
 
-      _note("Considering connecting (out) to peer: " << pe.id << " " << pe.adr.str());
+      _note("Considering connecting (out) to peer: " << peerid_to_string(pe.id) << " " << pe.adr.str());
 
       if(is_peer_used(pe)) {
         _note("Peer is used");
@@ -1158,7 +1158,7 @@ namespace nodetool
       if(is_addr_recently_failed(pe.adr))
         continue;
 
-      MDEBUG("Selected peer: " << pe.id << " " << pe.adr.str()
+      MDEBUG("Selected peer: " << peerid_to_string(pe.id) << " " << pe.adr.str()
                     << "[peer_list=" << (use_white_list ? white : gray)
                     << "] last_seen: " << (pe.last_seen ? epee::misc_utils::get_time_interval_string(time(NULL) - pe.last_seen) : "never"));
 
@@ -1795,6 +1795,8 @@ namespace nodetool
       m_peerlist.remove_from_peer_anchor(na);
     }
 
+    m_payload_handler.on_connection_close(context);
+
     MINFO("["<< epee::net_utils::print_connection_context(context) << "] CLOSE CONNECTION");
   }
 
@@ -1960,14 +1962,14 @@ namespace nodetool
     if (!success) {
       m_peerlist.remove_from_peer_gray(pe);
 
-      LOG_PRINT_L2("PEER EVICTED FROM GRAY PEER LIST IP address: " << pe.adr.host_str() << " Peer ID: " << std::hex << pe.id);
+      LOG_PRINT_L2("PEER EVICTED FROM GRAY PEER LIST IP address: " << pe.adr.host_str() << " Peer ID: " << peerid_type(pe.id));
 
       return true;
     }
 
     m_peerlist.set_peer_just_seen(pe.id, pe.adr);
 
-    LOG_PRINT_L2("PEER PROMOTED TO WHITE PEER LIST IP address: " << pe.adr.host_str() << " Peer ID: " << std::hex << pe.id);
+    LOG_PRINT_L2("PEER PROMOTED TO WHITE PEER LIST IP address: " << pe.adr.host_str() << " Peer ID: " << peerid_type(pe.id));
 
     return true;
   }

src/p2p/net_peerlist_boost_serialization.h

@@ -59,6 +59,8 @@ namespace boost
     {
       a & na.m_ip;
       a & na.m_port;
+      if (!typename Archive::is_saving())
+        na.init_ids();
     }
 
 

src/p2p/p2p_protocol_defs.h

@@ -44,6 +44,13 @@ namespace nodetool
   typedef boost::uuids::uuid uuid;
   typedef uint64_t peerid_type;
 
+  static inline std::string peerid_to_string(peerid_type peer_id)
+  {
+    std::ostringstream s;
+    s << std::hex << peer_id;
+    return epee::string_tools::pad_string(s.str(), 16, '0', true);
+  }
+
 #pragma pack (push, 1)
   
   struct network_address_old

src/rpc/core_rpc_server.cpp

@@ -205,14 +205,17 @@ namespace cryptonote
       }
       size_t txidx = 0;
       ntxes += bd.second.size();
-      for(const auto& t: bd.second)
+      for (std::list<cryptonote::blobdata>::iterator i = bd.second.begin(); i != bd.second.end(); ++i)
       {
+        unpruned_size += i->size();
         if (req.prune)
-          res.blocks.back().txs.push_back(get_pruned_tx_blob(t));
+          res.blocks.back().txs.push_back(get_pruned_tx_blob(std::move(*i)));
         else
-          res.blocks.back().txs.push_back(t);
+          res.blocks.back().txs.push_back(std::move(*i));
+        i->clear();
+        i->shrink_to_fit();
         pruned_size += res.blocks.back().txs.back().size();
-        unpruned_size += t.size();
+
         res.output_indices.back().indices.push_back(COMMAND_RPC_GET_BLOCKS_FAST::tx_output_indices());
         bool r = m_core.get_tx_outputs_gindexs(b.tx_hashes[txidx++], res.output_indices.back().indices.back().indices);
         if (!r)
@@ -1557,8 +1560,10 @@ namespace cryptonote
     static const char software[] = "monero";
 #ifdef BUILD_TAG
     static const char buildtag[] = BOOST_PP_STRINGIZE(BUILD_TAG);
+    static const char subdir[] = "cli";
 #else
     static const char buildtag[] = "source";
+    static const char subdir[] = "source";
 #endif
 
     if (req.command != "check" && req.command != "download" && req.command != "update")
@@ -1581,8 +1586,8 @@ namespace cryptonote
     }
     res.update = true;
     res.version = version;
-    res.user_uri = tools::get_update_url(software, "cli", buildtag, version, true);
-    res.auto_uri = tools::get_update_url(software, "cli", buildtag, version, false);
+    res.user_uri = tools::get_update_url(software, subdir, buildtag, version, true);
+    res.auto_uri = tools::get_update_url(software, subdir, buildtag, version, false);
     res.hash = hash;
     if (req.command == "check")
     {
@@ -1725,6 +1730,26 @@ namespace cryptonote
     return true;
   }
   //------------------------------------------------------------------------------------------------------------------------------
+  bool core_rpc_server::on_get_txpool_backlog(const COMMAND_RPC_GET_TRANSACTION_POOL_BACKLOG::request& req, COMMAND_RPC_GET_TRANSACTION_POOL_BACKLOG::response& res, epee::json_rpc::error& error_resp)
+  {
+    if(!check_core_busy())
+    {
+      error_resp.code = CORE_RPC_ERROR_CODE_CORE_BUSY;
+      error_resp.message = "Core is busy.";
+      return false;
+    }
+
+    if (!m_core.get_txpool_backlog(res.backlog))
+    {
+      error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR;
+      error_resp.message = "Failed to get txpool backlog";
+      return false;
+    }
+
+    res.status = CORE_RPC_STATUS_OK;
+    return true;
+  }
+  //------------------------------------------------------------------------------------------------------------------------------
 
   const command_line::arg_descriptor<std::string> core_rpc_server::arg_rpc_bind_port = {
       "rpc-bind-port"

src/rpc/core_rpc_server.h

@@ -125,6 +125,7 @@ namespace cryptonote
         MAP_JON_RPC_WE_IF("get_alternate_chains",on_get_alternate_chains,       COMMAND_RPC_GET_ALTERNATE_CHAINS, !m_restricted)
         MAP_JON_RPC_WE_IF("relay_tx",            on_relay_tx,                   COMMAND_RPC_RELAY_TX, !m_restricted)
         MAP_JON_RPC_WE_IF("sync_info",           on_sync_info,                  COMMAND_RPC_SYNC_INFO, !m_restricted)
+        MAP_JON_RPC_WE("get_txpool_backlog",     on_get_txpool_backlog,         COMMAND_RPC_GET_TRANSACTION_POOL_BACKLOG)
       END_JSON_RPC_MAP()
     END_URI_MAP2()
 
@@ -182,6 +183,7 @@ namespace cryptonote
     bool on_get_alternate_chains(const COMMAND_RPC_GET_ALTERNATE_CHAINS::request& req, COMMAND_RPC_GET_ALTERNATE_CHAINS::response& res, epee::json_rpc::error& error_resp);
     bool on_relay_tx(const COMMAND_RPC_RELAY_TX::request& req, COMMAND_RPC_RELAY_TX::response& res, epee::json_rpc::error& error_resp);
     bool on_sync_info(const COMMAND_RPC_SYNC_INFO::request& req, COMMAND_RPC_SYNC_INFO::response& res, epee::json_rpc::error& error_resp);
+    bool on_get_txpool_backlog(const COMMAND_RPC_GET_TRANSACTION_POOL_BACKLOG::request& req, COMMAND_RPC_GET_TRANSACTION_POOL_BACKLOG::response& res, epee::json_rpc::error& error_resp);
     //-----------------------
 
 private:

src/rpc/core_rpc_server_commands_defs.h

@@ -1071,6 +1071,33 @@ namespace cryptonote
     };
   };
 
+  struct tx_backlog_entry
+  {
+    uint64_t blob_size;
+    uint64_t fee;
+    uint64_t time_in_pool;
+  };
+
+  struct COMMAND_RPC_GET_TRANSACTION_POOL_BACKLOG
+  {
+    struct request
+    {
+      BEGIN_KV_SERIALIZE_MAP()
+      END_KV_SERIALIZE_MAP()
+    };
+
+    struct response
+    {
+      std::string status;
+      std::vector<tx_backlog_entry> backlog;
+
+      BEGIN_KV_SERIALIZE_MAP()
+        KV_SERIALIZE(status)
+        KV_SERIALIZE_CONTAINER_POD_AS_BLOB(backlog)
+      END_KV_SERIALIZE_MAP()
+    };
+  };
+
   struct txpool_histo
   {
     uint32_t txs;

src/simplewallet/simplewallet.cpp

@@ -63,6 +63,10 @@
 #include "wallet/wallet_args.h"
 #include <stdexcept>
 
+#ifdef HAVE_READLINE
+#include "readline_buffer.h"
+#endif
+
 using namespace std;
 using namespace epee;
 using namespace cryptonote;
@@ -387,6 +391,61 @@ bool simple_wallet::payment_id(const std::vector<std::string> &args/* = std::vec
   return true;
 }
 
+bool simple_wallet::print_fee_info(const std::vector<std::string> &args/* = std::vector<std::string>()*/)
+{
+  if (!try_connect_to_daemon())
+  {
+    fail_msg_writer() << tr("Cannot connect to daemon");
+    return true;
+  }
+  const uint64_t per_kb_fee = m_wallet->get_per_kb_fee();
+  const uint64_t typical_size_kb = 13;
+  message_writer() << (boost::format(tr("Current fee is %s monero per kB")) % print_money(per_kb_fee)).str();
+
+  std::vector<uint64_t> fees;
+  for (uint32_t priority = 1; priority <= 4; ++priority)
+  {
+    uint64_t mult = m_wallet->get_fee_multiplier(priority);
+    fees.push_back(per_kb_fee * typical_size_kb * mult);
+  }
+  std::vector<std::pair<uint64_t, uint64_t>> blocks;
+  try
+  {
+    uint64_t base_size = typical_size_kb * 1024;
+    blocks = m_wallet->estimate_backlog(base_size, base_size + 1023, fees);
+  }
+  catch (const std::exception &e)
+  {
+    fail_msg_writer() << tr("Error: failed to estimate backlog array size: ") << e.what();
+    return true;
+  }
+  if (blocks.size() != 4)
+  {
+    fail_msg_writer() << tr("Error: bad estimated backlog array size");
+    return true;
+  }
+
+  for (uint32_t priority = 1; priority <= 4; ++priority)
+  {
+    uint64_t nblocks_low = blocks[priority - 1].first;
+    uint64_t nblocks_high = blocks[priority - 1].second;
+    if (nblocks_low > 0)
+    {
+      std::string msg;
+      if (priority == m_wallet->get_default_priority() || (m_wallet->get_default_priority() == 0 && priority == 2))
+        msg = tr(" (current)");
+      uint64_t minutes_low = nblocks_low * DIFFICULTY_TARGET_V2 / 60, minutes_high = nblocks_high * DIFFICULTY_TARGET_V2 / 60;
+      if (nblocks_high == nblocks_low)
+        message_writer() << (boost::format(tr("%u block (%u minutes) backlog at priority %u%s")) % nblocks_low % minutes_low % priority % msg).str();
+      else
+        message_writer() << (boost::format(tr("%u to %u block (%u to %u minutes) backlog at priority %u")) % nblocks_low % nblocks_high % minutes_low % minutes_high % priority).str();
+    }
+    else
+      message_writer() << tr("No backlog at priority ") << priority;
+  }
+  return true;
+}
+
 bool simple_wallet::set_always_confirm_transfers(const std::vector<std::string> &args/* = std::vector<std::string>()*/)
 {
   const auto pwd_container = get_and_verify_password();
@@ -646,6 +705,34 @@ bool simple_wallet::set_merge_destinations(const std::vector<std::string> &args/
   return true;
 }
 
+bool simple_wallet::set_confirm_backlog(const std::vector<std::string> &args/* = std::vector<std::string>()*/)
+{
+  const auto pwd_container = get_and_verify_password();
+  if (pwd_container)
+  {
+    m_wallet->confirm_backlog(is_it_true(args[1]));
+    m_wallet->rewrite(m_wallet_file, pwd_container->password());
+  }
+  return true;
+}
+
+bool simple_wallet::set_refresh_from_block_height(const std::vector<std::string> &args/* = std::vector<std::string>()*/)
+{
+  const auto pwd_container = get_and_verify_password();
+  if (pwd_container)
+  {
+    uint64_t height;
+    if (!epee::string_tools::get_xtype_from_string(height, args[1]))
+    {
+      fail_msg_writer() << tr("Invalid height");
+      return true;
+    }
+    m_wallet->set_refresh_from_block_height(height);
+    m_wallet->rewrite(m_wallet_file, pwd_container->password());
+  }
+  return true;
+}
+
 bool simple_wallet::help(const std::vector<std::string> &args/* = std::vector<std::string>()*/)
 {
   success_msg_writer() << get_commands_str();
@@ -686,7 +773,7 @@ simple_wallet::simple_wallet()
   m_cmd_binder.set_handler("viewkey", boost::bind(&simple_wallet::viewkey, this, _1), tr("Display private view key"));
   m_cmd_binder.set_handler("spendkey", boost::bind(&simple_wallet::spendkey, this, _1), tr("Display private spend key"));
   m_cmd_binder.set_handler("seed", boost::bind(&simple_wallet::seed, this, _1), tr("Display Electrum-style mnemonic seed"));
-  m_cmd_binder.set_handler("set", boost::bind(&simple_wallet::set_variable, this, _1), tr("Available options: seed language - set wallet seed language; always-confirm-transfers <1|0> - whether to confirm unsplit txes; print-ring-members <1|0> - whether to print detailed information about ring members during confirmation; store-tx-info <1|0> - whether to store outgoing tx info (destination address, payment ID, tx secret key) for future reference; default-ring-size <n> - set default ring size (default is 5); auto-refresh <1|0> - whether to automatically sync new blocks from the daemon; refresh-type <full|optimize-coinbase|no-coinbase|default> - set wallet refresh behaviour; priority [0|1|2|3|4] - default/unimportant/normal/elevated/priority fee; confirm-missing-payment-id <1|0>; ask-password <1|0>; unit <monero|millinero|micronero|nanonero|piconero> - set default monero (sub-)unit; min-outputs-count [n] - try to keep at least that many outputs of value at least min-outputs-value; min-outputs-value [n] - try to keep at least min-outputs-count outputs of at least that value; merge-destinations <1|0> - whether to merge multiple payments to the same destination address"));
+  m_cmd_binder.set_handler("set", boost::bind(&simple_wallet::set_variable, this, _1), tr("Available options: seed language - set wallet seed language; always-confirm-transfers <1|0> - whether to confirm unsplit txes; print-ring-members <1|0> - whether to print detailed information about ring members during confirmation; store-tx-info <1|0> - whether to store outgoing tx info (destination address, payment ID, tx secret key) for future reference; default-ring-size <n> - set default ring size (default is 5); auto-refresh <1|0> - whether to automatically sync new blocks from the daemon; refresh-type <full|optimize-coinbase|no-coinbase|default> - set wallet refresh behaviour; priority [0|1|2|3|4] - default/unimportant/normal/elevated/priority fee; confirm-missing-payment-id <1|0>; ask-password <1|0>; unit <monero|millinero|micronero|nanonero|piconero> - set default monero (sub-)unit; min-outputs-count [n] - try to keep at least that many outputs of value at least min-outputs-value; min-outputs-value [n] - try to keep at least min-outputs-count outputs of at least that value; merge-destinations <1|0> - whether to merge multiple payments to the same destination address; confirm-backlog <1|0> - whether to warn if there is transaction backlog; refresh-from-block-height [n] - set height before which to ignore blocks"));
   m_cmd_binder.set_handler("rescan_spent", boost::bind(&simple_wallet::rescan_spent, this, _1), tr("Rescan blockchain for spent outputs"));
   m_cmd_binder.set_handler("get_tx_key", boost::bind(&simple_wallet::get_tx_key, this, _1), tr("Get transaction key (r) for a given <txid>"));
   m_cmd_binder.set_handler("check_tx_key", boost::bind(&simple_wallet::check_tx_key, this, _1), tr("Check amount going to <address> in <txid>"));
@@ -707,6 +794,7 @@ simple_wallet::simple_wallet()
   m_cmd_binder.set_handler("show_transfer", boost::bind(&simple_wallet::show_transfer, this, _1), tr("Show information about a transfer to/from this address"));
   m_cmd_binder.set_handler("password", boost::bind(&simple_wallet::change_password, this, _1), tr("Change wallet password"));
   m_cmd_binder.set_handler("payment_id", boost::bind(&simple_wallet::payment_id, this, _1), tr("Generate a new random full size payment id - these will be unencrypted on the blockchain, see integrated_address for encrypted short payment ids"));
+  m_cmd_binder.set_handler("fee", boost::bind(&simple_wallet::print_fee_info, this, _1), tr("Print information about fee and current transaction backlog"));
   m_cmd_binder.set_handler("help", boost::bind(&simple_wallet::help, this, _1), tr("Show this help"));
 }
 //----------------------------------------------------------------------------------------------------
@@ -728,6 +816,8 @@ bool simple_wallet::set_variable(const std::vector<std::string> &args)
     success_msg_writer() << "min-outputs-count = " << m_wallet->get_min_output_count();
     success_msg_writer() << "min-outputs-value = " << cryptonote::print_money(m_wallet->get_min_output_value());
     success_msg_writer() << "merge-destinations = " << m_wallet->merge_destinations();
+    success_msg_writer() << "confirm-backlog = " << m_wallet->confirm_backlog();
+    success_msg_writer() << "refresh-from-block-height = " << m_wallet->get_refresh_from_block_height();
     return true;
   }
   else
@@ -773,6 +863,8 @@ bool simple_wallet::set_variable(const std::vector<std::string> &args)
     CHECK_SIMPLE_VARIABLE("min-outputs-count", set_min_output_count, tr("unsigned integer"));
     CHECK_SIMPLE_VARIABLE("min-outputs-value", set_min_output_value, tr("amount"));
     CHECK_SIMPLE_VARIABLE("merge-destinations", set_merge_destinations, tr("0 or 1"));
+    CHECK_SIMPLE_VARIABLE("confirm-backlog", set_confirm_backlog, tr("0 or 1"));
+    CHECK_SIMPLE_VARIABLE("refresh-from-block-height", set_refresh_from_block_height, tr("block height"));
   }
   fail_msg_writer() << tr("set: unrecognized argument(s)");
   return true;
@@ -1835,6 +1927,10 @@ bool simple_wallet::refresh_main(uint64_t start_height, bool reset)
   if (reset)
     m_wallet->rescan_blockchain(false);
 
+#ifdef HAVE_READLINE
+  rdln::suspend_readline pause_readline;
+#endif
+
   message_writer() << tr("Starting refresh...");
 
   uint64_t fetched_blocks = 0;
@@ -2410,6 +2506,7 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri
       break;
       default:
         LOG_ERROR("Unknown transfer method, using original");
+        /* FALLTHRU */
       case TransferOriginal:
         ptx_vector = m_wallet->create_transactions(dsts, fake_outs_count, 0 /* unlock_time */, priority, extra, m_trusted_daemon);
         break;
@@ -2421,6 +2518,56 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri
       return true;
     }
 
+    // if we need to check for backlog, check the worst case tx
+    if (m_wallet->confirm_backlog())
+    {
+      std::stringstream prompt;
+      double worst_fee_per_byte = std::numeric_limits<double>::max();
+      uint64_t size = 0, fee = 0;
+      for (size_t n = 0; n < ptx_vector.size(); ++n)
+      {
+        const uint64_t blob_size = cryptonote::tx_to_blob(ptx_vector[n].tx).size();
+        const double fee_per_byte = ptx_vector[n].fee / (double)blob_size;
+        if (fee_per_byte < worst_fee_per_byte)
+        {
+          worst_fee_per_byte = fee_per_byte;
+          fee = ptx_vector[n].fee;
+        }
+        size += blob_size;
+      }
+      try
+      {
+        std::vector<std::pair<uint64_t, uint64_t>> nblocks = m_wallet->estimate_backlog(size, size, {fee});
+        if (nblocks.size() != 1)
+        {
+          prompt << "Internal error checking for backlog. " << tr("Is this okay anyway?  (Y/Yes/N/No): ");
+        }
+        else
+        {
+          if (nblocks[0].first > 0)
+            prompt << (boost::format(tr("There is currently a %u block backlog at that fee level. Is this okay?  (Y/Yes/N/No)")) % nblocks[0].first).str();
+        }
+      }
+      catch (const std::exception &e)
+      {
+        prompt << tr("Failed to check for backlog: ") << e.what() << ENDL << tr("Is this okay anyway?  (Y/Yes/N/No): ");
+      }
+
+      std::string prompt_str = prompt.str();
+      if (!prompt_str.empty())
+      {
+        std::string accepted = command_line::input_line(prompt_str);
+        if (std::cin.eof())
+          return true;
+        if (!command_line::is_yes(accepted))
+        {
+          fail_msg_writer() << tr("transaction cancelled.");
+
+          return true; 
+        }
+      }
+    }
+
     // if more than one tx necessary, prompt user to confirm
     if (m_wallet->always_confirm_transfers() || ptx_vector.size() > 1)
     {

src/simplewallet/simplewallet.h

@@ -120,6 +120,8 @@ namespace cryptonote
     bool set_min_output_count(const std::vector<std::string> &args = std::vector<std::string>());
     bool set_min_output_value(const std::vector<std::string> &args = std::vector<std::string>());
     bool set_merge_destinations(const std::vector<std::string> &args = std::vector<std::string>());
+    bool set_confirm_backlog(const std::vector<std::string> &args = std::vector<std::string>());
+    bool set_refresh_from_block_height(const std::vector<std::string> &args = std::vector<std::string>());
     bool help(const std::vector<std::string> &args = std::vector<std::string>());
     bool start_mining(const std::vector<std::string> &args);
     bool stop_mining(const std::vector<std::string> &args);
@@ -174,6 +176,7 @@ namespace cryptonote
     bool show_transfer(const std::vector<std::string> &args);
     bool change_password(const std::vector<std::string>& args);
     bool payment_id(const std::vector<std::string> &args);
+    bool print_fee_info(const std::vector<std::string> &args);
 
     uint64_t get_daemon_blockchain_height(std::string& err);
     bool try_connect_to_daemon(bool silent = false, uint32_t* version = nullptr);

src/version.h.in

@@ -1,4 +1,4 @@
 #define MONERO_VERSION_TAG "@VERSIONTAG@"
-#define MONERO_VERSION "0.11.0.0"
+#define MONERO_VERSION "0.11.1.0"
 #define MONERO_RELEASE_NAME "Helium Hydra"
 #define MONERO_VERSION_FULL MONERO_VERSION "-" MONERO_VERSION_TAG

src/wallet/api/wallet.cpp

@@ -1387,13 +1387,6 @@ bool WalletImpl::doInit(const string &daemon_address, uint64_t upper_transaction
     if (!m_wallet->init(daemon_address, m_daemon_login, upper_transaction_size_limit))
        return false;
 
-    // in case new wallet, this will force fast-refresh (pulling hashes instead of blocks)
-    // If daemon isn't synced a calculated block height will be used instead
-    if (isNewWallet() && daemonSynced()) {
-        LOG_PRINT_L2(__FUNCTION__ << ":New Wallet - fast refresh until " << daemonBlockChainHeight());
-        m_wallet->set_refresh_from_block_height(daemonBlockChainHeight());
-    }
-
     if (m_rebuildWalletCache)
       LOG_PRINT_L2(__FUNCTION__ << ": Rebuilding wallet cache, fast refresh until block " << m_wallet->get_refresh_from_block_height());
 

src/wallet/wallet2.cpp

@@ -1963,6 +1963,9 @@ bool wallet2::store_keys(const std::string& keys_file_name, const std::string& p
   value2.SetInt(m_merge_destinations ? 1 :0);
   json.AddMember("merge_destinations", value2, json.GetAllocator());
 
+  value2.SetInt(m_confirm_backlog ? 1 :0);
+  json.AddMember("confirm_backlog", value2, json.GetAllocator());
+
   value2.SetInt(m_testnet ? 1 :0);
   json.AddMember("testnet", value2, json.GetAllocator());
 
@@ -2037,6 +2040,7 @@ bool wallet2::load_keys(const std::string& keys_file_name, const std::string& pa
     m_min_output_count = 0;
     m_min_output_value = 0;
     m_merge_destinations = false;
+    m_confirm_backlog = true;
   }
   else
   {
@@ -2107,6 +2111,8 @@ bool wallet2::load_keys(const std::string& keys_file_name, const std::string& pa
     m_min_output_value = field_min_output_value;
     GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, merge_destinations, int, Int, false, false);
     m_merge_destinations = field_merge_destinations;
+    GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, confirm_backlog, int, Int, false, true);
+    m_confirm_backlog = field_confirm_backlog;
     GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, testnet, int, Int, false, m_testnet);
     // Wallet is being opened with testnet flag, but is saved as a mainnet wallet
     THROW_WALLET_EXCEPTION_IF(m_testnet && !field_testnet, error::wallet_internal_error, "Mainnet wallet can not be opened as testnet wallet");
@@ -2249,7 +2255,7 @@ crypto::secret_key wallet2::generate(const std::string& wallet_, const std::stri
     // Set blockchain height calculated from current date/time
     uint64_t approx_blockchain_height = get_approximate_blockchain_height();
     if(approx_blockchain_height > 0) {
-      m_refresh_from_block_height = approx_blockchain_height - blocks_per_month;
+      m_refresh_from_block_height = approx_blockchain_height >= blocks_per_month ? approx_blockchain_height - blocks_per_month : 0;
     }
   }
   bool r = store_keys(m_keys_file, password, false);
@@ -3451,12 +3457,15 @@ bool wallet2::load_tx(const std::string &signed_filename, std::vector<tools::wal
   return true;
 }
 //----------------------------------------------------------------------------------------------------
-uint64_t wallet2::get_fee_multiplier(uint32_t priority, int fee_algorithm) const
+uint64_t wallet2::get_fee_multiplier(uint32_t priority, int fee_algorithm)
 {
   static const uint64_t old_multipliers[3] = {1, 2, 3};
   static const uint64_t new_multipliers[3] = {1, 20, 166};
   static const uint64_t newer_multipliers[4] = {1, 4, 20, 166};
 
+  if (fee_algorithm == -1)
+    fee_algorithm = get_fee_algorithm();
+
   // 0 -> default (here, x1 till fee algorithm 2, x4 from it)
   if (priority == 0)
     priority = m_default_priority;
@@ -4170,7 +4179,7 @@ static size_t estimate_rct_tx_size(int n_inputs, int mixin, int n_outputs)
   size += (2*64*32+32+64*32) * n_outputs;
 
   // MGs
-  size += n_inputs * (32 * (mixin+1) + 32);
+  size += n_inputs * (64 * (mixin+1) + 32);
 
   // mixRing - not serialized, can be reconstructed
   /* size += 2 * 32 * (mixin+1) * n_inputs; */
@@ -5741,6 +5750,70 @@ bool wallet2::is_synced() const
   return get_blockchain_current_height() >= height;
 }
 //----------------------------------------------------------------------------------------------------
+std::vector<std::pair<uint64_t, uint64_t>> wallet2::estimate_backlog(uint64_t min_blob_size, uint64_t max_blob_size, const std::vector<uint64_t> &fees)
+{
+  THROW_WALLET_EXCEPTION_IF(min_blob_size == 0, error::wallet_internal_error, "Invalid 0 fee");
+  THROW_WALLET_EXCEPTION_IF(max_blob_size == 0, error::wallet_internal_error, "Invalid 0 fee");
+  for (uint64_t fee: fees)
+  {
+    THROW_WALLET_EXCEPTION_IF(fee == 0, error::wallet_internal_error, "Invalid 0 fee");
+  }
+
+  // get txpool backlog
+  epee::json_rpc::request<cryptonote::COMMAND_RPC_GET_TRANSACTION_POOL_BACKLOG::request> req = AUTO_VAL_INIT(req);
+  epee::json_rpc::response<cryptonote::COMMAND_RPC_GET_TRANSACTION_POOL_BACKLOG::response, std::string> res = AUTO_VAL_INIT(res);
+  m_daemon_rpc_mutex.lock();
+  req.jsonrpc = "2.0";
+  req.id = epee::serialization::storage_entry(0);
+  req.method = "get_txpool_backlog";
+  bool r = net_utils::invoke_http_json("/json_rpc", req, res, m_http_client, rpc_timeout);
+  m_daemon_rpc_mutex.unlock();
+  THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "Failed to connect to daemon");
+  THROW_WALLET_EXCEPTION_IF(res.result.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "get_txpool_backlog");
+  THROW_WALLET_EXCEPTION_IF(res.result.status != CORE_RPC_STATUS_OK, error::get_tx_pool_error);
+
+  epee::json_rpc::request<cryptonote::COMMAND_RPC_GET_INFO::request> req_t = AUTO_VAL_INIT(req_t);
+  epee::json_rpc::response<cryptonote::COMMAND_RPC_GET_INFO::response, std::string> resp_t = AUTO_VAL_INIT(resp_t);
+  m_daemon_rpc_mutex.lock();
+  req_t.jsonrpc = "2.0";
+  req_t.id = epee::serialization::storage_entry(0);
+  req_t.method = "get_info";
+  r = net_utils::invoke_http_json("/json_rpc", req_t, resp_t, m_http_client);
+  m_daemon_rpc_mutex.unlock();
+  THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "get_info");
+  THROW_WALLET_EXCEPTION_IF(resp_t.result.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "get_info");
+  THROW_WALLET_EXCEPTION_IF(resp_t.result.status != CORE_RPC_STATUS_OK, error::get_tx_pool_error);
+  uint64_t full_reward_zone = resp_t.result.block_size_limit / 2;
+
+  std::vector<std::pair<uint64_t, uint64_t>> blocks;
+  for (uint64_t fee: fees)
+  {
+    double our_fee_byte_min = fee / (double)min_blob_size, our_fee_byte_max = fee / (double)max_blob_size;
+    uint64_t priority_size_min = 0, priority_size_max = 0;
+    for (const auto &i: res.result.backlog)
+    {
+      if (i.blob_size == 0)
+      {
+        MWARNING("Got 0 sized blob from txpool, ignored");
+        continue;
+      }
+      double this_fee_byte = i.fee / (double)i.blob_size;
+      if (this_fee_byte >= our_fee_byte_min)
+        priority_size_min += i.blob_size;
+      if (this_fee_byte >= our_fee_byte_max)
+        priority_size_max += i.blob_size;
+    }
+
+    uint64_t nblocks_min = (priority_size_min + full_reward_zone - 1) / full_reward_zone;
+    uint64_t nblocks_max = (priority_size_max + full_reward_zone - 1) / full_reward_zone;
+    MDEBUG("estimate_backlog: priority_size " << priority_size_min << " - " << priority_size_max << " for " << fee
+        << " (" << our_fee_byte_min << " - " << our_fee_byte_max << " piconero byte fee), "
+        << nblocks_min << " - " << nblocks_max << " blocks at block size " << full_reward_zone);
+    blocks.push_back(std::make_pair(nblocks_min, nblocks_max));
+  }
+  return blocks;
+}
+//----------------------------------------------------------------------------------------------------
 void wallet2::generate_genesis(cryptonote::block& b) {
   if (m_testnet)
   {

src/wallet/wallet2.h

@@ -105,7 +105,7 @@ namespace tools
     };
 
   private:
-    wallet2(const wallet2&) : m_run(true), m_callback(0), m_testnet(false), m_always_confirm_transfers(true), m_print_ring_members(false), m_store_tx_info(true), m_default_mixin(0), m_default_priority(0), m_refresh_type(RefreshOptimizeCoinbase), m_auto_refresh(true), m_refresh_from_block_height(0), m_confirm_missing_payment_id(true), m_ask_password(true), m_min_output_count(0), m_min_output_value(0), m_merge_destinations(false), m_is_initialized(false),m_node_rpc_proxy(m_http_client, m_daemon_rpc_mutex) {}
+    wallet2(const wallet2&) : m_run(true), m_callback(0), m_testnet(false), m_always_confirm_transfers(true), m_print_ring_members(false), m_store_tx_info(true), m_default_mixin(0), m_default_priority(0), m_refresh_type(RefreshOptimizeCoinbase), m_auto_refresh(true), m_refresh_from_block_height(0), m_confirm_missing_payment_id(true), m_ask_password(true), m_min_output_count(0), m_min_output_value(0), m_merge_destinations(false), m_confirm_backlog(true), m_is_initialized(false),m_node_rpc_proxy(m_http_client, m_daemon_rpc_mutex) {}
 
   public:
     static const char* tr(const char* str);
@@ -131,7 +131,7 @@ namespace tools
 
     static bool verify_password(const std::string& keys_file_name, const std::string& password, bool watch_only);
 
-    wallet2(bool testnet = false, bool restricted = false) : m_run(true), m_callback(0), m_testnet(testnet), m_always_confirm_transfers(true), m_print_ring_members(false), m_store_tx_info(true), m_default_mixin(0), m_default_priority(0), m_refresh_type(RefreshOptimizeCoinbase), m_auto_refresh(true), m_refresh_from_block_height(0), m_confirm_missing_payment_id(true), m_ask_password(true), m_min_output_count(0), m_min_output_value(0), m_merge_destinations(false), m_is_initialized(false), m_restricted(restricted), is_old_file_format(false), m_node_rpc_proxy(m_http_client, m_daemon_rpc_mutex) {}
+    wallet2(bool testnet = false, bool restricted = false) : m_run(true), m_callback(0), m_testnet(testnet), m_always_confirm_transfers(true), m_print_ring_members(false), m_store_tx_info(true), m_default_mixin(0), m_default_priority(0), m_refresh_type(RefreshOptimizeCoinbase), m_auto_refresh(true), m_refresh_from_block_height(0), m_confirm_missing_payment_id(true), m_ask_password(true), m_min_output_count(0), m_min_output_value(0), m_merge_destinations(false), m_confirm_backlog(true), m_is_initialized(false), m_restricted(restricted), is_old_file_format(false), m_node_rpc_proxy(m_http_client, m_daemon_rpc_mutex) {}
 
     struct transfer_details
     {
@@ -539,6 +539,8 @@ namespace tools
     uint64_t get_min_output_value() const { return m_min_output_value; }
     void merge_destinations(bool merge) { m_merge_destinations = merge; }
     bool merge_destinations() const { return m_merge_destinations; }
+    bool confirm_backlog() const { return m_confirm_backlog; }
+    void confirm_backlog(bool always) { m_confirm_backlog = always; }
 
     bool get_tx_key(const crypto::hash &txid, crypto::secret_key &tx_key) const;
 
@@ -602,6 +604,11 @@ namespace tools
 
     bool is_synced() const;
 
+    std::vector<std::pair<uint64_t, uint64_t>> estimate_backlog(uint64_t min_blob_size, uint64_t max_blob_size, const std::vector<uint64_t> &fees);
+
+    uint64_t get_fee_multiplier(uint32_t priority, int fee_algorithm = -1);
+    uint64_t get_per_kb_fee();
+
   private:
     /*!
      * \brief  Stores wallet information to wallet file.
@@ -642,9 +649,7 @@ namespace tools
     void parse_block_round(const cryptonote::blobdata &blob, cryptonote::block &bl, crypto::hash &bl_id, bool &error) const;
     uint64_t get_upper_transaction_size_limit();
     std::vector<uint64_t> get_unspent_amounts_vector();
-    uint64_t get_fee_multiplier(uint32_t priority, int fee_algorithm) const;
     uint64_t get_dynamic_per_kb_fee_estimate();
-    uint64_t get_per_kb_fee();
     float get_output_relatedness(const transfer_details &td0, const transfer_details &td1) const;
     std::vector<size_t> pick_preferred_rct_inputs(uint64_t needed_money) const;
     void set_spent(size_t idx, uint64_t height);
@@ -700,6 +705,7 @@ namespace tools
     uint32_t m_min_output_count;
     uint64_t m_min_output_value;
     bool m_merge_destinations;
+    bool m_confirm_backlog;
     bool m_is_initialized;
     NodeRPCProxy m_node_rpc_proxy;
     std::unordered_set<crypto::hash> m_scanned_pool_txs[2];

tests/core_proxy/CMakeLists.txt

@@ -39,6 +39,8 @@ target_link_libraries(core_proxy
   PRIVATE
     cryptonote_core
     cryptonote_protocol
+    common
+    blockchain_db
     p2p
     epee
     ${CMAKE_THREAD_LIBS_INIT}

tests/core_proxy/core_proxy.h

@@ -66,6 +66,7 @@ namespace tests
 
   public:
     void on_synchronized(){}
+    void safesyncmode(const bool){}
     uint64_t get_current_blockchain_height(){return 1;}
     void set_target_blockchain_height(uint64_t) {}
     bool init(const boost::program_options::variables_map& vm);
@@ -99,5 +100,6 @@ namespace tests
     uint8_t get_ideal_hard_fork_version(uint64_t height) const { return 0; }
     uint8_t get_hard_fork_version(uint64_t height) const { return 0; }
     cryptonote::difficulty_type get_block_cumulative_difficulty(uint64_t height) const { return 0; }
+    bool fluffy_blocks_enabled() const { return false; }
   };
 }

tests/fuzz/CMakeLists.txt

@@ -30,6 +30,8 @@ add_executable(block_fuzz_tests block.cpp fuzzer.cpp)
 target_link_libraries(block_fuzz_tests
   PRIVATE
     cryptonote_core
+    common
+    blockchain_db
     p2p
     epee
     ${CMAKE_THREAD_LIBS_INIT}
@@ -42,6 +44,8 @@ add_executable(transaction_fuzz_tests transaction.cpp fuzzer.cpp)
 target_link_libraries(transaction_fuzz_tests
   PRIVATE
     cryptonote_core
+    common
+    blockchain_db
     p2p
     epee
     ${CMAKE_THREAD_LIBS_INIT}
@@ -55,6 +59,8 @@ target_link_libraries(signature_fuzz_tests
   PRIVATE
     wallet
     cryptonote_core
+    common
+    blockchain_db
     p2p
     epee
     ${CMAKE_THREAD_LIBS_INIT}
@@ -68,6 +74,8 @@ target_link_libraries(cold-outputs_fuzz_tests
   PRIVATE
     wallet
     cryptonote_core
+    common
+    blockchain_db
     p2p
     epee
     ${CMAKE_THREAD_LIBS_INIT}
@@ -81,6 +89,8 @@ target_link_libraries(cold-transaction_fuzz_tests
   PRIVATE
     wallet
     cryptonote_core
+    common
+    blockchain_db
     p2p
     epee
     ${CMAKE_THREAD_LIBS_INIT}

tests/unit_tests/ban.cpp

@@ -43,6 +43,7 @@ class test_core
 {
 public:
   void on_synchronized(){}
+  void safesyncmode(const bool){}
   uint64_t get_current_blockchain_height() const {return 1;}
   void set_target_blockchain_height(uint64_t) {}
   bool init(const boost::program_options::variables_map& vm) {return true ;}
@@ -76,6 +77,7 @@ public:
   uint8_t get_ideal_hard_fork_version(uint64_t height) const { return 0; }
   uint8_t get_hard_fork_version(uint64_t height) const { return 0; }
   cryptonote::difficulty_type get_block_cumulative_difficulty(uint64_t height) const { return 0; }
+  bool fluffy_blocks_enabled() const { return false; }
 };
 
 typedef nodetool::node_server<cryptonote::t_cryptonote_protocol_handler<test_core>> Server;

tests/unit_tests/hardfork.cpp

@@ -31,7 +31,7 @@
 #include <algorithm>
 #include "gtest/gtest.h"
 
-#include "blockchain_db/lmdb/db_lmdb.h"
+#include "blockchain_db/blockchain_db.h"
 #include "cryptonote_basic/cryptonote_format_utils.h"
 #include "cryptonote_basic/hardfork.h"
 
@@ -47,6 +47,7 @@ public:
   virtual void open(const std::string& filename, const int db_flags = 0) { }
   virtual void close() {}
   virtual void sync() {}
+  virtual void safesyncmode(const bool onoff) {}
   virtual void reset() {}
   virtual std::vector<std::string> get_filenames() const { return std::vector<std::string>(); }
   virtual std::string get_db_name() const { return std::string(); }