From 6e594e53bb25f02d3b2572a33f353e15b7c05395 Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Sun, 18 Oct 2020 12:59:23 +0100 Subject: [PATCH] Allow setting start block on export And make import honor the starting block# recorded in a bootstrap file --- .../blockchain_export.cpp | 6 +- .../blockchain_import.cpp | 12 +-- src/blockchain_utilities/bootstrap_file.cpp | 75 +++++++++++++------ src/blockchain_utilities/bootstrap_file.h | 10 +-- 4 files changed, 69 insertions(+), 34 deletions(-) diff --git a/src/blockchain_utilities/blockchain_export.cpp b/src/blockchain_utilities/blockchain_export.cpp index e55930b00..87cd7945c 100644 --- a/src/blockchain_utilities/blockchain_export.cpp +++ b/src/blockchain_utilities/blockchain_export.cpp @@ -47,6 +47,7 @@ int main(int argc, char* argv[]) epee::string_tools::set_module_name_and_folder(argv[0]); uint32_t log_level = 0; + uint64_t block_start = 0; uint64_t block_stop = 0; bool blocks_dat = false; @@ -58,6 +59,7 @@ int main(int argc, char* argv[]) po::options_description desc_cmd_sett("Command line options and settings options"); const command_line::arg_descriptor arg_output_file = {"output-file", "Specify output file", "", true}; const command_line::arg_descriptor arg_log_level = {"log-level", "0-4 or categories", ""}; + const command_line::arg_descriptor arg_block_start = {"block-start", "Start at block number", block_start}; const command_line::arg_descriptor arg_block_stop = {"block-stop", "Stop at block number", block_stop}; const command_line::arg_descriptor arg_blocks_dat = {"blocksdat", "Output in blocks.dat format", blocks_dat}; @@ -67,6 +69,7 @@ int main(int argc, char* argv[]) command_line::add_arg(desc_cmd_sett, cryptonote::arg_testnet_on); command_line::add_arg(desc_cmd_sett, cryptonote::arg_stagenet_on); command_line::add_arg(desc_cmd_sett, arg_log_level); + command_line::add_arg(desc_cmd_sett, arg_block_start); command_line::add_arg(desc_cmd_sett, arg_block_stop); command_line::add_arg(desc_cmd_sett, arg_blocks_dat); @@ -97,6 +100,7 @@ int main(int argc, char* argv[]) mlog_set_log(command_line::get_arg(vm, arg_log_level).c_str()); else mlog_set_log(std::string(std::to_string(log_level) + ",bcutil:INFO").c_str()); + block_start = command_line::get_arg(vm, arg_block_start); block_stop = command_line::get_arg(vm, arg_block_stop); LOG_PRINT_L0("Starting..."); @@ -178,7 +182,7 @@ int main(int argc, char* argv[]) else { BootstrapFile bootstrap; - r = bootstrap.store_blockchain_raw(core_storage, NULL, output_file_path, block_stop); + r = bootstrap.store_blockchain_raw(core_storage, NULL, output_file_path, block_start, block_stop); } CHECK_AND_ASSERT_MES(r, 1, "Failed to export blockchain raw data"); LOG_PRINT_L0("Blockchain raw data exported OK"); diff --git a/src/blockchain_utilities/blockchain_import.cpp b/src/blockchain_utilities/blockchain_import.cpp index ab2313096..60c069c3b 100644 --- a/src/blockchain_utilities/blockchain_import.cpp +++ b/src/blockchain_utilities/blockchain_import.cpp @@ -227,6 +227,7 @@ int import_from_file(cryptonote::core& core, const std::string& import_file_path return false; } + uint64_t block_first, block_last; uint64_t start_height = 1, seek_height; if (opt_resume) start_height = core.get_blockchain_storage().get_current_blockchain_height(); @@ -235,10 +236,10 @@ int import_from_file(cryptonote::core& core, const std::string& import_file_path BootstrapFile bootstrap; std::streampos pos; // BootstrapFile bootstrap(import_file_path); - uint64_t total_source_blocks = bootstrap.count_blocks(import_file_path, pos, seek_height); - MINFO("bootstrap file last block number: " << total_source_blocks-1 << " (zero-based height) total blocks: " << total_source_blocks); + uint64_t total_source_blocks = bootstrap.count_blocks(import_file_path, pos, seek_height, block_first); + MINFO("bootstrap file last block number: " << total_source_blocks+block_first-1 << " (zero-based height) total blocks: " << total_source_blocks); - if (total_source_blocks-1 <= start_height) + if (total_source_blocks+block_first-1 <= start_height) { return false; } @@ -260,7 +261,8 @@ int import_from_file(cryptonote::core& core, const std::string& import_file_path // 4 byte magic + (currently) 1024 byte header structures uint8_t major_version, minor_version; - bootstrap.seek_to_first_chunk(import_file, major_version, minor_version); + uint64_t dummy; + bootstrap.seek_to_first_chunk(import_file, major_version, minor_version, dummy, dummy); std::string str1; char buffer1[1024]; @@ -275,7 +277,7 @@ int import_from_file(cryptonote::core& core, const std::string& import_file_path if (! block_stop) { - block_stop = total_source_blocks - 1; + block_stop = total_source_blocks+block_first - 1; } // These are what we'll try to use, and they don't have to be a determination diff --git a/src/blockchain_utilities/bootstrap_file.cpp b/src/blockchain_utilities/bootstrap_file.cpp index a4704626f..7050b9ab1 100644 --- a/src/blockchain_utilities/bootstrap_file.cpp +++ b/src/blockchain_utilities/bootstrap_file.cpp @@ -52,7 +52,7 @@ namespace -bool BootstrapFile::open_writer(const boost::filesystem::path& file_path) +bool BootstrapFile::open_writer(const boost::filesystem::path& file_path, uint64_t start_block, uint64_t stop_block) { const boost::filesystem::path dir_path = file_path.parent_path(); if (!dir_path.empty()) @@ -78,7 +78,7 @@ bool BootstrapFile::open_writer(const boost::filesystem::path& file_path) m_raw_data_file = new std::ofstream(); bool do_initialize_file = false; - uint64_t num_blocks = 0; + uint64_t num_blocks = 0, block_first = 0; if (! boost::filesystem::exists(file_path)) { @@ -88,10 +88,12 @@ bool BootstrapFile::open_writer(const boost::filesystem::path& file_path) } else { - num_blocks = count_blocks(file_path.string()); - MDEBUG("appending to existing file with height: " << num_blocks-1 << " total blocks: " << num_blocks); + std::streampos dummy_pos; + uint64_t dummy_height = 0; + num_blocks = count_blocks(file_path.string(), dummy_pos, dummy_height, block_first); + MDEBUG("appending to existing file with height: " << num_blocks+block_first-1 << " total blocks: " << num_blocks); } - m_height = num_blocks; + m_height = num_blocks+block_first; if (do_initialize_file) m_raw_data_file->open(file_path.string(), std::ios_base::binary | std::ios_base::out | std::ios::trunc); @@ -106,13 +108,12 @@ bool BootstrapFile::open_writer(const boost::filesystem::path& file_path) return false; if (do_initialize_file) - initialize_file(); + initialize_file(start_block, stop_block); return true; } - -bool BootstrapFile::initialize_file() +bool BootstrapFile::initialize_file(uint64_t first_block, uint64_t last_block) { const uint32_t file_magic = blockchain_raw_magic; @@ -129,8 +130,8 @@ bool BootstrapFile::initialize_file() bfi.header_size = header_size; bootstrap::blocks_info bbi; - bbi.block_first = 0; - bbi.block_last = 0; + bbi.block_first = first_block; + bbi.block_last = last_block; bbi.block_last_pos = 0; buffer_type buffer2; @@ -261,7 +262,7 @@ bool BootstrapFile::close() } -bool BootstrapFile::store_blockchain_raw(Blockchain* _blockchain_storage, tx_memory_pool* _tx_pool, boost::filesystem::path& output_file, uint64_t requested_block_stop) +bool BootstrapFile::store_blockchain_raw(Blockchain* _blockchain_storage, tx_memory_pool* _tx_pool, boost::filesystem::path& output_file, uint64_t start_block, uint64_t requested_block_stop) { uint64_t num_blocks_written = 0; m_max_chunk = 0; @@ -269,17 +270,11 @@ bool BootstrapFile::store_blockchain_raw(Blockchain* _blockchain_storage, tx_mem m_tx_pool = _tx_pool; uint64_t progress_interval = 100; MINFO("Storing blocks raw data..."); - if (!BootstrapFile::open_writer(output_file)) - { - MFATAL("failed to open raw file for write"); - return false; - } block b; // block_start, block_stop use 0-based height. m_height uses 1-based height. So to resume export // from last exported block, block_start doesn't need to add 1 here, as it's already at the next // height. - uint64_t block_start = m_height; uint64_t block_stop = 0; MINFO("source blockchain height: " << m_blockchain_storage->get_current_blockchain_height()-1); if ((requested_block_stop > 0) && (requested_block_stop < m_blockchain_storage->get_current_blockchain_height())) @@ -292,6 +287,13 @@ bool BootstrapFile::store_blockchain_raw(Blockchain* _blockchain_storage, tx_mem block_stop = m_blockchain_storage->get_current_blockchain_height() - 1; MINFO("Using block height of source blockchain: " << block_stop); } + if (!BootstrapFile::open_writer(output_file, start_block, block_stop)) + { + MFATAL("failed to open raw file for write"); + return false; + } + uint64_t block_start = m_height ? m_height : start_block; + MINFO("Starting block height: " << block_start); for (m_cur_height = block_start; m_cur_height <= block_stop; ++m_cur_height) { // this method's height refers to 0-based height (genesis block = height 0) @@ -323,7 +325,8 @@ bool BootstrapFile::store_blockchain_raw(Blockchain* _blockchain_storage, tx_mem return BootstrapFile::close(); } -uint64_t BootstrapFile::seek_to_first_chunk(std::ifstream& import_file, uint8_t &major_version, uint8_t &minor_version) +uint64_t BootstrapFile::seek_to_first_chunk(std::ifstream& import_file, uint8_t &major_version, uint8_t &minor_version, + uint64_t &block_first, uint64_t &block_last) { uint32_t file_magic; @@ -368,11 +371,35 @@ uint64_t BootstrapFile::seek_to_first_chunk(std::ifstream& import_file, uint8_t MINFO("bootstrap magic size: " << sizeof(file_magic)); MINFO("bootstrap header size: " << bfi.header_size); + uint32_t buflen_blocks_info; + + import_file.read(buf1, sizeof(buflen_blocks_info)); + str1.assign(buf1, sizeof(buflen_blocks_info)); + if (! import_file) + throw std::runtime_error("Error reading expected number of bytes"); + if (! ::serialization::parse_binary(str1, buflen_blocks_info)) + throw std::runtime_error("Error in deserialization of buflen_blocks_info"); + MINFO("bootstrap::blocks_info size: " << buflen_blocks_info); + + if (buflen_blocks_info > sizeof(buf1)) + throw std::runtime_error("Error: bootstrap::blocks_info size exceeds buffer size"); + import_file.read(buf1, buflen_blocks_info); + if (! import_file) + throw std::runtime_error("Error reading expected number of bytes"); + str1.assign(buf1, buflen_blocks_info); + bootstrap::blocks_info bbi; + if (! ::serialization::parse_binary(str1, bbi)) + throw std::runtime_error("Error in deserialization of bootstrap::blocks_info"); + MINFO("bootstrap first block:" << bbi.block_first); + MINFO("bootstrap last block:" << bbi.block_last); + uint64_t full_header_size = sizeof(file_magic) + bfi.header_size; import_file.seekg(full_header_size); major_version = bfi.major_version; minor_version = bfi.minor_version; + block_first = bbi.block_first; + block_last = bbi.block_last; return full_header_size; } @@ -436,13 +463,14 @@ uint64_t BootstrapFile::count_blocks(const std::string& import_file_path) { std::streampos dummy_pos; uint64_t dummy_height = 0; - return count_blocks(import_file_path, dummy_pos, dummy_height); + return count_blocks(import_file_path, dummy_pos, dummy_height, dummy_height); } // If seek_height is non-zero on entry, return a stream position <= this height when finished. // And return the actual height corresponding to this position. Allows the caller to locate its // starting position without having to reread the entire file again. -uint64_t BootstrapFile::count_blocks(const std::string& import_file_path, std::streampos &start_pos, uint64_t& seek_height) +uint64_t BootstrapFile::count_blocks(const std::string& import_file_path, std::streampos &start_pos, + uint64_t& seek_height, uint64_t &block_first) { boost::filesystem::path raw_file_path(import_file_path); boost::system::error_code ec; @@ -464,7 +492,8 @@ uint64_t BootstrapFile::count_blocks(const std::string& import_file_path, std::s uint64_t full_header_size; // 4 byte magic + length of header structures uint8_t major_version, minor_version; - full_header_size = seek_to_first_chunk(import_file, major_version, minor_version); + uint64_t block_last; + full_header_size = seek_to_first_chunk(import_file, major_version, minor_version, block_first, block_last); MINFO("Scanning blockchain from bootstrap file..."); bool quit = false; @@ -473,11 +502,11 @@ uint64_t BootstrapFile::count_blocks(const std::string& import_file_path, std::s while (! quit) { - if (start_height && h + progress_interval >= start_height - 1) + if (start_height && h + block_first + progress_interval >= start_height - 1) { start_height = 0; start_pos = import_file.tellg(); - seek_height = h; + seek_height = h + block_first; } bytes_read += count_bytes(import_file, progress_interval, blocks, quit); h += blocks; diff --git a/src/blockchain_utilities/bootstrap_file.h b/src/blockchain_utilities/bootstrap_file.h index 23dbb64d7..ff2875a61 100644 --- a/src/blockchain_utilities/bootstrap_file.h +++ b/src/blockchain_utilities/bootstrap_file.h @@ -57,12 +57,12 @@ class BootstrapFile public: uint64_t count_bytes(std::ifstream& import_file, uint64_t blocks, uint64_t& h, bool& quit); - uint64_t count_blocks(const std::string& dir_path, std::streampos& start_pos, uint64_t& seek_height); + uint64_t count_blocks(const std::string& dir_path, std::streampos& start_pos, uint64_t& seek_height, uint64_t& block_first); uint64_t count_blocks(const std::string& dir_path); - uint64_t seek_to_first_chunk(std::ifstream& import_file, uint8_t &major_version, uint8_t &minor_version); + uint64_t seek_to_first_chunk(std::ifstream& import_file, uint8_t &major_version, uint8_t &minor_version, uint64_t &block_first, uint64_t &block_last); bool store_blockchain_raw(cryptonote::Blockchain* cs, cryptonote::tx_memory_pool* txp, - boost::filesystem::path& output_file, uint64_t use_block_height=0); + boost::filesystem::path& output_file, uint64_t start_block=0, uint64_t stop_block=0); protected: @@ -75,8 +75,8 @@ protected: boost::iostreams::stream>* m_output_stream; // open export file for write - bool open_writer(const boost::filesystem::path& file_path); - bool initialize_file(); + bool open_writer(const boost::filesystem::path& file_path, uint64_t start_block, uint64_t stop_block); + bool initialize_file(uint64_t start_block, uint64_t stop_block); bool close(); void write_block(block& block); void flush_chunk();