From 275cbd4348975371988c79f67d68ceede3f47a1d Mon Sep 17 00:00:00 2001 From: warptangent Date: Wed, 11 Feb 2015 16:02:20 -0800 Subject: [PATCH] Add support for database open with flags Add support to: - BlockchainDB, BlockchainLMDB - blockchain_import utility to open LMDB database with one or more LMDB flags. Sample use: $ blockchain_import --database lmdb#nosync $ blockchain_import --database lmdb#nosync,nometasync --- .../blockchain_import.cpp | 72 +++++++++++++++++-- src/blockchain_converter/fake_core.h | 6 +- src/blockchain_db/blockchain_db.h | 2 +- src/blockchain_db/lmdb/db_lmdb.cpp | 3 +- src/blockchain_db/lmdb/db_lmdb.h | 2 +- src/cryptonote_core/blockchain.cpp | 4 +- src/cryptonote_core/blockchain.h | 2 +- 7 files changed, 77 insertions(+), 14 deletions(-) diff --git a/src/blockchain_converter/blockchain_import.cpp b/src/blockchain_converter/blockchain_import.cpp index ce2abf4ac..060fe8481 100644 --- a/src/blockchain_converter/blockchain_import.cpp +++ b/src/blockchain_converter/blockchain_import.cpp @@ -42,6 +42,7 @@ #include "common/command_line.h" #include "version.h" +#include // for db flag arguments #include "import.h" #include "fake_core.h" @@ -62,6 +63,58 @@ namespace po = boost::program_options; using namespace cryptonote; using namespace epee; + +int parse_db_arguments(const std::string& db_arg_str, std::string& db_engine, int& mdb_flags) +{ + std::vector db_args; + boost::split(db_args, db_arg_str, boost::is_any_of("#")); + db_engine = db_args.front(); + boost::algorithm::trim(db_engine); + + if (db_args.size() == 1) + { + return 0; + } + else if (db_args.size() > 2) + { + std::cerr << "unrecognized database argument format: " << db_arg_str << ENDL; + return 1; + } + + std::string db_arg_str2 = db_args[1]; + boost::split(db_args, db_arg_str2, boost::is_any_of(",")); + for (auto& it : db_args) + { + boost::algorithm::trim(it); + if (it.empty()) + continue; + LOG_PRINT_L1("LMDB flag: " << it); + if (it == "nosync") + { + mdb_flags |= MDB_NOSYNC; + } + else if (it == "nometasync") + { + mdb_flags |= MDB_NOMETASYNC; + } + else if (it == "writemap") + { + mdb_flags |= MDB_WRITEMAP; + } + else if (it == "mapasync") + { + mdb_flags |= MDB_MAPASYNC; + } + else + { + std::cerr << "unrecognized database flag: " << it << ENDL; + return 1; + } + } + return 0; +} + + int count_blocks(std::string& import_file_path) { boost::filesystem::path raw_file_path(import_file_path); @@ -492,7 +545,7 @@ int main(int argc, char* argv[]) uint32_t log_level = LOG_LEVEL_0; std::string dirname; - std::string db_engine; + std::string db_arg_str; boost::filesystem::path default_data_path {tools::get_default_data_dir()}; boost::filesystem::path default_testnet_data_path {default_data_path / "testnet"}; @@ -582,7 +635,7 @@ int main(int argc, char* argv[]) opt_testnet = command_line::get_arg(vm, arg_testnet_on); auto data_dir_arg = opt_testnet ? command_line::arg_testnet_data_dir : command_line::arg_data_dir; dirname = command_line::get_arg(vm, data_dir_arg); - db_engine = command_line::get_arg(vm, arg_database); + db_arg_str = command_line::get_arg(vm, arg_database); log_space::get_set_log_detalisation_level(true, log_level); log_space::log_singletone::add_logger(LOGGER_CONSOLE, NULL, NULL); @@ -600,6 +653,17 @@ int main(int argc, char* argv[]) exit(0); } + + std::string db_engine; + int mdb_flags = 0; + int res = 0; + res = parse_db_arguments(db_arg_str, db_engine, mdb_flags); + if (res) + { + std::cerr << "Error parsing database argument(s)" << ENDL; + exit(1); + } + if (std::find(db_engines.begin(), db_engines.end(), db_engine) == db_engines.end()) { std::cerr << "Invalid database engine: " << db_engine << std::endl; @@ -637,7 +701,7 @@ int main(int argc, char* argv[]) #if !defined(BLOCKCHAIN_DB) if (db_engine == "lmdb") { - fake_core_lmdb simple_core(dirname, opt_testnet, opt_batch); + fake_core_lmdb simple_core(dirname, opt_testnet, opt_batch, mdb_flags); import_from_file(simple_core, import_file_path); } else if (db_engine == "memory") @@ -659,7 +723,7 @@ int main(int argc, char* argv[]) exit(1); } #if BLOCKCHAIN_DB == DB_LMDB - fake_core_lmdb simple_core(dirname, opt_testnet, opt_batch); + fake_core_lmdb simple_core(dirname, opt_testnet, opt_batch, mdb_flags); #else fake_core_memory simple_core(dirname, opt_testnet); #endif diff --git a/src/blockchain_converter/fake_core.h b/src/blockchain_converter/fake_core.h index 68021a2bd..aff249cd9 100644 --- a/src/blockchain_converter/fake_core.h +++ b/src/blockchain_converter/fake_core.h @@ -46,14 +46,14 @@ struct fake_core_lmdb // for multi_db_runtime: #if !defined(BLOCKCHAIN_DB) - fake_core_lmdb(const boost::filesystem::path &path, const bool use_testnet=false, const bool do_batch=true) : m_pool(&m_storage), m_storage(m_pool) + fake_core_lmdb(const boost::filesystem::path &path, const bool use_testnet=false, const bool do_batch=true, const int mdb_flags=0) : m_pool(&m_storage), m_storage(m_pool) // for multi_db_compile: #else - fake_core_lmdb(const boost::filesystem::path &path, const bool use_testnet=false, const bool do_batch=true) : m_pool(m_storage), m_storage(m_pool) + fake_core_lmdb(const boost::filesystem::path &path, const bool use_testnet=false, const bool do_batch=true, const int mdb_flags=0) : m_pool(m_storage), m_storage(m_pool) #endif { m_pool.init(path.string()); - m_storage.init(path.string(), use_testnet); + m_storage.init(path.string(), use_testnet, mdb_flags); if (do_batch) m_storage.get_db()->set_batch_transactions(do_batch); support_batch = true; diff --git a/src/blockchain_db/blockchain_db.h b/src/blockchain_db/blockchain_db.h index d2b4a07a7..7b6b55a40 100644 --- a/src/blockchain_db/blockchain_db.h +++ b/src/blockchain_db/blockchain_db.h @@ -326,7 +326,7 @@ public: void show_stats(); // open the db at location , or create it if there isn't one. - virtual void open(const std::string& filename) = 0; + virtual void open(const std::string& filename, const int db_flags = 0) = 0; // make sure implementation has a create function as well virtual void create(const std::string& filename) = 0; diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index 6d815c997..b1da6308f 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -635,9 +635,8 @@ BlockchainLMDB::BlockchainLMDB(bool batch_transactions) m_height = 0; } -void BlockchainLMDB::open(const std::string& filename) +void BlockchainLMDB::open(const std::string& filename, const int mdb_flags) { - int mdb_flags = 0; LOG_PRINT_L3("BlockchainLMDB::" << __func__); if (m_open) diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h index 1a684548e..fa7cb988a 100644 --- a/src/blockchain_db/lmdb/db_lmdb.h +++ b/src/blockchain_db/lmdb/db_lmdb.h @@ -114,7 +114,7 @@ public: BlockchainLMDB(bool batch_transactions=false); ~BlockchainLMDB(); - virtual void open(const std::string& filename); + virtual void open(const std::string& filename, const int mdb_flags=0); virtual void create(const std::string& filename); diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index b7920c99c..57934b3fc 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -226,7 +226,7 @@ uint64_t Blockchain::get_current_blockchain_height() const //------------------------------------------------------------------ //FIXME: possibly move this into the constructor, to avoid accidentally // dereferencing a null BlockchainDB pointer -bool Blockchain::init(const std::string& config_folder, bool testnet) +bool Blockchain::init(const std::string& config_folder, const bool testnet, const int db_flags) { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -246,7 +246,7 @@ bool Blockchain::init(const std::string& config_folder, bool testnet) const std::string filename = folder.string(); try { - m_db->open(filename); + m_db->open(filename, db_flags); } catch (const DB_OPEN_FAILURE& e) { diff --git a/src/cryptonote_core/blockchain.h b/src/cryptonote_core/blockchain.h index e86b3887e..da4da075b 100644 --- a/src/cryptonote_core/blockchain.h +++ b/src/cryptonote_core/blockchain.h @@ -81,7 +81,7 @@ namespace cryptonote Blockchain(tx_memory_pool& tx_pool); - bool init(const std::string& config_folder, bool testnet = false); + bool init(const std::string& config_folder, const bool testnet = false, const int db_flags = 0); bool deinit(); void set_checkpoints(checkpoints&& chk_pts) { m_checkpoints = chk_pts; }