From 4dcd3cd0b26ad1592f1cb6071f41323016697a1b Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Wed, 8 May 2019 11:11:23 +0000 Subject: [PATCH 1/6] blockchain: keep alternative blocks in LMDB Alternative blocks are cleared on startup unless --keep-alt-blocks is passed on the command line --- src/blockchain_db/blockchain_db.h | 65 +++++++- src/blockchain_db/lmdb/db_lmdb.cpp | 155 +++++++++++++++++++ src/blockchain_db/lmdb/db_lmdb.h | 13 ++ src/blockchain_db/testdb.h | 7 + src/cryptonote_core/blockchain.cpp | 192 ++++++++++++++++-------- src/cryptonote_core/blockchain.h | 31 +--- src/cryptonote_core/cryptonote_core.cpp | 10 ++ 7 files changed, 379 insertions(+), 94 deletions(-) diff --git a/src/blockchain_db/blockchain_db.h b/src/blockchain_db/blockchain_db.h index 567be6a65..9cc8aaaaf 100644 --- a/src/blockchain_db/blockchain_db.h +++ b/src/blockchain_db/blockchain_db.h @@ -129,6 +129,15 @@ struct tx_data_t }; #pragma pack(pop) +struct alt_block_data_t +{ + uint64_t height; + uint64_t cumulative_weight; + uint64_t cumulative_difficulty_low; + uint64_t cumulative_difficulty_high; + uint64_t already_generated_coins; +}; + /** * @brief a struct containing txpool per transaction metadata */ @@ -1528,8 +1537,45 @@ public: * * @param: sz the block size */ - virtual void add_max_block_size(uint64_t sz) = 0; + + /** + * @brief add a new alternative block + * + * @param: blkid the block hash + * @param: data: the metadata for the block + * @param: blob: the block's blob + */ + virtual void add_alt_block(const crypto::hash &blkid, const cryptonote::alt_block_data_t &data, const cryptonote::blobdata &blob) = 0; + + /** + * @brief get an alternative block by hash + * + * @param: blkid the block hash + * @param: data: the metadata for the block + * @param: blob: the block's blob + * + * @return true if the block was found in the alternative blocks list, false otherwise + */ + virtual bool get_alt_block(const crypto::hash &blkid, alt_block_data_t *data, cryptonote::blobdata *blob) = 0; + + /** + * @brief remove an alternative block + * + * @param: blkid the block hash + */ + virtual void remove_alt_block(const crypto::hash &blkid) = 0; + + /** + * @brief get the number of alternative blocks stored + */ + virtual uint64_t get_alt_block_count() = 0; + + /** + * @brief drop all alternative blocks + */ + virtual void drop_alt_blocks() = 0; + /** * @brief runs a function over all txpool transactions * @@ -1619,6 +1665,23 @@ public: virtual bool for_all_outputs(std::function f) const = 0; virtual bool for_all_outputs(uint64_t amount, const std::function &f) const = 0; + /** + * @brief runs a function over all alternative blocks stored + * + * The subclass should run the passed function for each alt block it has + * stored, passing (blkid, data, blob) as its parameters. + * + * If any call to the function returns false, the subclass should return + * false. Otherwise, the subclass returns true. + * + * The subclass should throw DB_ERROR if any of the expected values are + * not found. Current implementations simply return false. + * + * @param std::function f the function to run + * + * @return false if the function returns false for any output, otherwise true + */ + virtual bool for_all_alt_blocks(std::function f, bool include_blob = false) const = 0; // diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index ea56bbd67..29b75f0b0 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -194,6 +194,8 @@ namespace * txpool_meta txn hash txn metadata * txpool_blob txn hash txn blob * + * alt_blocks block hash {block data, block blob} + * * Note: where the data items are of uniform size, DUPFIXED tables have * been used to save space. In most of these cases, a dummy "zerokval" * key is used when accessing the table; the Key listed above will be @@ -221,6 +223,8 @@ const char* const LMDB_SPENT_KEYS = "spent_keys"; const char* const LMDB_TXPOOL_META = "txpool_meta"; const char* const LMDB_TXPOOL_BLOB = "txpool_blob"; +const char* const LMDB_ALT_BLOCKS = "alt_blocks"; + const char* const LMDB_HF_STARTING_HEIGHTS = "hf_starting_heights"; const char* const LMDB_HF_VERSIONS = "hf_versions"; @@ -1400,6 +1404,8 @@ void BlockchainLMDB::open(const std::string& filename, const int db_flags) lmdb_db_open(txn, LMDB_TXPOOL_META, MDB_CREATE, m_txpool_meta, "Failed to open db handle for m_txpool_meta"); lmdb_db_open(txn, LMDB_TXPOOL_BLOB, MDB_CREATE, m_txpool_blob, "Failed to open db handle for m_txpool_blob"); + lmdb_db_open(txn, LMDB_ALT_BLOCKS, MDB_CREATE, m_alt_blocks, "Failed to open db handle for m_alt_blocks"); + // this subdb is dropped on sight, so it may not be present when we open the DB. // Since we use MDB_CREATE, we'll get an exception if we open read-only and it does not exist. // So we don't open for read-only, and also not drop below. It is not used elsewhere. @@ -1423,6 +1429,7 @@ void BlockchainLMDB::open(const std::string& filename, const int db_flags) mdb_set_compare(txn, m_txpool_meta, compare_hash32); mdb_set_compare(txn, m_txpool_blob, compare_hash32); + mdb_set_compare(txn, m_alt_blocks, compare_hash32); mdb_set_compare(txn, m_properties, compare_string); if (!(mdb_flags & MDB_RDONLY)) @@ -2241,6 +2248,50 @@ bool BlockchainLMDB::for_all_txpool_txes(std::function f, bool include_blob) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + TXN_PREFIX_RDONLY(); + RCURSOR(alt_blocks); + + MDB_val k; + MDB_val v; + bool ret = true; + + MDB_cursor_op op = MDB_FIRST; + while (1) + { + int result = mdb_cursor_get(m_cur_alt_blocks, &k, &v, op); + op = MDB_NEXT; + if (result == MDB_NOTFOUND) + break; + if (result) + throw0(DB_ERROR(lmdb_error("Failed to enumerate alt blocks: ", result).c_str())); + const crypto::hash &blkid = *(const crypto::hash*)k.mv_data; + if (v.mv_size < sizeof(alt_block_data_t)) + throw0(DB_ERROR("alt_blocks record is too small")); + const alt_block_data_t *data = (const alt_block_data_t*)v.mv_data; + const cryptonote::blobdata *passed_bd = NULL; + cryptonote::blobdata bd; + if (include_blob) + { + bd.assign(reinterpret_cast(v.mv_data) + sizeof(alt_block_data_t), v.mv_size - sizeof(alt_block_data_t)); + passed_bd = &bd; + } + + if (!f(blkid, *data, passed_bd)) { + ret = false; + break; + } + } + + TXN_POSTFIX_RDONLY(); + + return ret; +} + bool BlockchainLMDB::block_exists(const crypto::hash& h, uint64_t *height) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); @@ -4062,6 +4113,110 @@ uint8_t BlockchainLMDB::get_hard_fork_version(uint64_t height) const return ret; } +void BlockchainLMDB::add_alt_block(const crypto::hash &blkid, const cryptonote::alt_block_data_t &data, const cryptonote::blobdata &blob) +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + mdb_txn_cursors *m_cursors = &m_wcursors; + + CURSOR(alt_blocks) + + MDB_val k = {sizeof(blkid), (void *)&blkid}; + const size_t val_size = sizeof(alt_block_data_t) + blob.size(); + std::unique_ptr val(new char[val_size]); + memcpy(val.get(), &data, sizeof(alt_block_data_t)); + memcpy(val.get() + sizeof(alt_block_data_t), blob.data(), blob.size()); + MDB_val v = {val_size, (void *)val.get()}; + if (auto result = mdb_cursor_put(m_cur_alt_blocks, &k, &v, MDB_NODUPDATA)) { + if (result == MDB_KEYEXIST) + throw1(DB_ERROR("Attempting to add alternate block that's already in the db")); + else + throw1(DB_ERROR(lmdb_error("Error adding alternate block to db transaction: ", result).c_str())); + } +} + +bool BlockchainLMDB::get_alt_block(const crypto::hash &blkid, alt_block_data_t *data, cryptonote::blobdata *blob) +{ + LOG_PRINT_L3("BlockchainLMDB:: " << __func__); + check_open(); + + TXN_PREFIX_RDONLY(); + RCURSOR(alt_blocks); + + MDB_val_set(k, blkid); + MDB_val v; + int result = mdb_cursor_get(m_cur_alt_blocks, &k, &v, MDB_SET); + if (result == MDB_NOTFOUND) + return false; + + if (result) + throw0(DB_ERROR(lmdb_error("Error attempting to retrieve alternate block " + epee::string_tools::pod_to_hex(blkid) + " from the db: ", result).c_str())); + if (v.mv_size < sizeof(alt_block_data_t)) + throw0(DB_ERROR("Record size is less than expected")); + + const alt_block_data_t *ptr = (const alt_block_data_t*)v.mv_data; + if (data) + *data = *ptr; + if (blob) + blob->assign((const char*)(ptr + 1), v.mv_size - sizeof(alt_block_data_t)); + + TXN_POSTFIX_RDONLY(); + return true; +} + +void BlockchainLMDB::remove_alt_block(const crypto::hash &blkid) +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + mdb_txn_cursors *m_cursors = &m_wcursors; + + CURSOR(alt_blocks) + + MDB_val k = {sizeof(blkid), (void *)&blkid}; + MDB_val v; + int result = mdb_cursor_get(m_cur_alt_blocks, &k, &v, MDB_SET); + if (result) + throw0(DB_ERROR(lmdb_error("Error locating alternate block " + epee::string_tools::pod_to_hex(blkid) + " in the db: ", result).c_str())); + result = mdb_cursor_del(m_cur_alt_blocks, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Error deleting alternate block " + epee::string_tools::pod_to_hex(blkid) + " from the db: ", result).c_str())); +} + +uint64_t BlockchainLMDB::get_alt_block_count() +{ + LOG_PRINT_L3("BlockchainLMDB:: " << __func__); + check_open(); + + TXN_PREFIX_RDONLY(); + RCURSOR(alt_blocks); + + MDB_stat db_stats; + int result = mdb_stat(m_txn, m_alt_blocks, &db_stats); + uint64_t count = 0; + if (result != MDB_NOTFOUND) + { + if (result) + throw0(DB_ERROR(lmdb_error("Failed to query m_alt_blocks: ", result).c_str())); + count = db_stats.ms_entries; + } + TXN_POSTFIX_RDONLY(); + return count; +} + +void BlockchainLMDB::drop_alt_blocks() +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + TXN_PREFIX(0); + + auto result = mdb_drop(*txn_ptr, m_alt_blocks, 0); + if (result) + throw1(DB_ERROR(lmdb_error("Error dropping alternative blocks: ", result).c_str())); + + TXN_POSTFIX_SUCCESS(); +} + bool BlockchainLMDB::is_read_only() const { unsigned int flags; diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h index 4b46f081e..61a551476 100644 --- a/src/blockchain_db/lmdb/db_lmdb.h +++ b/src/blockchain_db/lmdb/db_lmdb.h @@ -67,6 +67,8 @@ typedef struct mdb_txn_cursors MDB_cursor *m_txc_txpool_meta; MDB_cursor *m_txc_txpool_blob; + MDB_cursor *m_txc_alt_blocks; + MDB_cursor *m_txc_hf_versions; MDB_cursor *m_txc_properties; @@ -87,6 +89,7 @@ typedef struct mdb_txn_cursors #define m_cur_spent_keys m_cursors->m_txc_spent_keys #define m_cur_txpool_meta m_cursors->m_txc_txpool_meta #define m_cur_txpool_blob m_cursors->m_txc_txpool_blob +#define m_cur_alt_blocks m_cursors->m_txc_alt_blocks #define m_cur_hf_versions m_cursors->m_txc_hf_versions #define m_cur_properties m_cursors->m_txc_properties @@ -108,6 +111,7 @@ typedef struct mdb_rflags bool m_rf_spent_keys; bool m_rf_txpool_meta; bool m_rf_txpool_blob; + bool m_rf_alt_blocks; bool m_rf_hf_versions; bool m_rf_properties; } mdb_rflags; @@ -288,6 +292,12 @@ public: virtual bool update_pruning(); virtual bool check_pruning(); + virtual void add_alt_block(const crypto::hash &blkid, const cryptonote::alt_block_data_t &data, const cryptonote::blobdata &blob); + virtual bool get_alt_block(const crypto::hash &blkid, alt_block_data_t *data, cryptonote::blobdata *blob); + virtual void remove_alt_block(const crypto::hash &blkid); + virtual uint64_t get_alt_block_count(); + virtual void drop_alt_blocks(); + virtual bool for_all_txpool_txes(std::function f, bool include_blob = false, bool include_unrelayed_txes = true) const; virtual bool for_all_key_images(std::function) const; @@ -295,6 +305,7 @@ public: virtual bool for_all_transactions(std::function, bool pruned) const; virtual bool for_all_outputs(std::function f) const; virtual bool for_all_outputs(uint64_t amount, const std::function &f) const; + virtual bool for_all_alt_blocks(std::function f, bool include_blob = false) const; virtual uint64_t add_block( const std::pair& blk , size_t block_weight @@ -452,6 +463,8 @@ private: MDB_dbi m_txpool_meta; MDB_dbi m_txpool_blob; + MDB_dbi m_alt_blocks; + MDB_dbi m_hf_starting_heights; MDB_dbi m_hf_versions; diff --git a/src/blockchain_db/testdb.h b/src/blockchain_db/testdb.h index 6c97713d5..a32359c6c 100644 --- a/src/blockchain_db/testdb.h +++ b/src/blockchain_db/testdb.h @@ -155,6 +155,13 @@ public: virtual uint64_t get_max_block_size() override { return 100000000; } virtual void add_max_block_size(uint64_t sz) override { } + + virtual void add_alt_block(const crypto::hash &blkid, const cryptonote::alt_block_data_t &data, const cryptonote::blobdata &blob) override {} + virtual bool get_alt_block(const crypto::hash &blkid, alt_block_data_t *data, cryptonote::blobdata *blob) override { return false; } + virtual void remove_alt_block(const crypto::hash &blkid) override {} + virtual uint64_t get_alt_block_count() override { return 0; } + virtual void drop_alt_blocks() override {} + virtual bool for_all_alt_blocks(std::function f, bool include_blob = false) const override { return true; } }; } diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index b4dc6692f..91f18aafc 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -685,9 +685,9 @@ bool Blockchain::reset_and_set_genesis_block(const block& b) LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); m_timestamps_and_difficulties_height = 0; - m_alternative_chains.clear(); invalidate_block_template_cache(); m_db->reset(); + m_db->drop_alt_blocks(); m_hardfork->init(); db_wtxn_guard wtxn_guard(m_db); @@ -812,10 +812,15 @@ bool Blockchain::get_block_by_hash(const crypto::hash &h, block &blk, bool *orph // try to find block in alternative chain catch (const BLOCK_DNE& e) { - blocks_ext_by_hash::const_iterator it_alt = m_alternative_chains.find(h); - if (m_alternative_chains.end() != it_alt) + alt_block_data_t data; + cryptonote::blobdata blob; + if (m_db->get_alt_block(h, &data, &blob)) { - blk = it_alt->second.bl; + if (!cryptonote::parse_and_validate_block_from_blob(blob, blk)) + { + MERROR("Found block " << h << " in alt chain, but failed to parse it"); + throw std::runtime_error("Found block in alt chain, but failed to parse it"); + } if (orphan) *orphan = true; return true; @@ -995,7 +1000,7 @@ bool Blockchain::rollback_blockchain_switching(std::list& original_chain, //------------------------------------------------------------------ // This function attempts to switch to an alternate chain, returning // boolean based on success therein. -bool Blockchain::switch_to_alternative_blockchain(std::list& alt_chain, bool discard_disconnected_chain) +bool Blockchain::switch_to_alternative_blockchain(std::list& alt_chain, bool discard_disconnected_chain) { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -1006,7 +1011,7 @@ bool Blockchain::switch_to_alternative_blockchain(std::listblock_exists(alt_chain.front()->second.bl.prev_id)) + if (!m_db->block_exists(alt_chain.front().bl.prev_id)) { LOG_ERROR("Attempting to move to an alternate chain, but it doesn't appear to connect to the main chain!"); return false; @@ -1015,7 +1020,7 @@ bool Blockchain::switch_to_alternative_blockchain(std::list disconnected_chain; - while (m_db->top_block_hash() != alt_chain.front()->second.bl.prev_id) + while (m_db->top_block_hash() != alt_chain.front().bl.prev_id) { block b = pop_block_from_blockchain(); disconnected_chain.push_front(b); @@ -1026,11 +1031,11 @@ bool Blockchain::switch_to_alternative_blockchain(std::list(); // add block to main chain - bool r = handle_block_to_main_chain(ch_ent->second.bl, bvc); + bool r = handle_block_to_main_chain(bei.bl, bvc); // if adding block to main chain failed, rollback to previous state and // return false @@ -1046,14 +1051,18 @@ bool Blockchain::switch_to_alternative_blockchain(std::listsecond, get_block_hash(ch_ent->second.bl)); - MERROR("The block was inserted as invalid while connecting new alternative chain, block_id: " << get_block_hash(ch_ent->second.bl)); - m_alternative_chains.erase(*alt_ch_iter++); + const crypto::hash blkid = cryptonote::get_block_hash(bei.bl); + add_block_as_invalid(bei, blkid); + MERROR("The block was inserted as invalid while connecting new alternative chain, block_id: " << blkid); + m_db->remove_alt_block(blkid); + alt_ch_iter++; for(auto alt_ch_to_orph_iter = alt_ch_iter; alt_ch_to_orph_iter != alt_chain.end(); ) { - add_block_as_invalid((*alt_ch_to_orph_iter)->second, (*alt_ch_to_orph_iter)->first); - m_alternative_chains.erase(*alt_ch_to_orph_iter++); + const auto &bei = *alt_ch_to_orph_iter++; + const crypto::hash blkid = cryptonote::get_block_hash(bei.bl); + add_block_as_invalid(bei, blkid); + m_db->remove_alt_block(blkid); } return false; } @@ -1078,9 +1087,9 @@ bool Blockchain::switch_to_alternative_blockchain(std::listremove_alt_block(cryptonote::get_block_hash(bei.bl)); } m_hardfork->reorganize_from_chain_height(split_height); @@ -1096,7 +1105,7 @@ bool Blockchain::switch_to_alternative_blockchain(std::list& alt_chain, block_extended_info& bei) const +difficulty_type Blockchain::get_next_difficulty_for_alternative_chain(const std::list& alt_chain, block_extended_info& bei) const { if (m_fixed_difficulty) { @@ -1118,7 +1127,7 @@ difficulty_type Blockchain::get_next_difficulty_for_alternative_chain(const std: CRITICAL_REGION_LOCAL(m_blockchain_lock); // Figure out start and stop offsets for main chain blocks - size_t main_chain_stop_offset = alt_chain.size() ? alt_chain.front()->second.height : bei.height; + size_t main_chain_stop_offset = alt_chain.size() ? alt_chain.front().height : bei.height; size_t main_chain_count = difficulty_blocks_count - std::min(static_cast(difficulty_blocks_count), alt_chain.size()); main_chain_count = std::min(main_chain_count, main_chain_stop_offset); size_t main_chain_start_offset = main_chain_stop_offset - main_chain_count; @@ -1136,10 +1145,10 @@ difficulty_type Blockchain::get_next_difficulty_for_alternative_chain(const std: // make sure we haven't accidentally grabbed too many blocks...maybe don't need this check? CHECK_AND_ASSERT_MES((alt_chain.size() + timestamps.size()) <= difficulty_blocks_count, false, "Internal error, alt_chain.size()[" << alt_chain.size() << "] + vtimestampsec.size()[" << timestamps.size() << "] NOT <= DIFFICULTY_WINDOW[]" << difficulty_blocks_count); - for (auto it : alt_chain) + for (const auto &bei : alt_chain) { - timestamps.push_back(it->second.bl.timestamp); - cumulative_difficulties.push_back(it->second.cumulative_difficulty); + timestamps.push_back(bei.bl.timestamp); + cumulative_difficulties.push_back(bei.cumulative_difficulty); } } // if the alt chain is long enough for the difficulty calc, grab difficulties @@ -1151,10 +1160,10 @@ difficulty_type Blockchain::get_next_difficulty_for_alternative_chain(const std: size_t count = 0; size_t max_i = timestamps.size()-1; // get difficulties and timestamps from most recent blocks in alt chain - for(auto it: boost::adaptors::reverse(alt_chain)) + for (const auto bei: boost::adaptors::reverse(alt_chain)) { - timestamps[max_i - count] = it->second.bl.timestamp; - cumulative_difficulties[max_i - count] = it->second.cumulative_difficulty; + timestamps[max_i - count] = bei.bl.timestamp; + cumulative_difficulties[max_i - count] = bei.cumulative_difficulty; count++; if(count >= difficulty_blocks_count) break; @@ -1387,16 +1396,17 @@ bool Blockchain::create_block_template(block& b, const crypto::hash *from_block, //build alternative subchain, front -> mainchain, back -> alternative head //block is not related with head of main chain //first of all - look in alternative chains container - auto it_prev = m_alternative_chains.find(*from_block); + alt_block_data_t prev_data; + bool parent_in_alt = m_db->get_alt_block(*from_block, &prev_data, NULL); bool parent_in_main = m_db->block_exists(*from_block); - if(it_prev == m_alternative_chains.end() && !parent_in_main) + if (!parent_in_alt && !parent_in_main) { MERROR("Unknown from block"); return false; } //we have new block in alternative chain - std::list alt_chain; + std::list alt_chain; block_verification_context bvc = boost::value_initialized(); std::vector timestamps; if (!build_alt_chain(*from_block, alt_chain, timestamps, bvc)) @@ -1411,7 +1421,7 @@ bool Blockchain::create_block_template(block& b, const crypto::hash *from_block, } else { - height = alt_chain.back()->second.height + 1; + height = alt_chain.back().height + 1; } b.major_version = m_hardfork->get_ideal_version(height); b.minor_version = m_hardfork->get_ideal_version(); @@ -1426,14 +1436,14 @@ bool Blockchain::create_block_template(block& b, const crypto::hash *from_block, } else { - median_weight = it_prev->second.block_cumulative_weight - it_prev->second.block_cumulative_weight / 20; - already_generated_coins = alt_chain.back()->second.already_generated_coins; + median_weight = prev_data.cumulative_weight - prev_data.cumulative_weight / 20; + already_generated_coins = alt_chain.back().already_generated_coins; } // FIXME: consider moving away from block_extended_info at some point block_extended_info bei = boost::value_initialized(); bei.bl = b; - bei.height = alt_chain.size() ? it_prev->second.height + 1 : m_db->get_block_height(*from_block) + 1; + bei.height = alt_chain.size() ? prev_data.height + 1 : m_db->get_block_height(*from_block) + 1; diffic = get_next_difficulty_for_alternative_chain(alt_chain, bei); } @@ -1613,16 +1623,25 @@ bool Blockchain::complete_timestamps_vector(uint64_t start_top_height, std::vect return true; } //------------------------------------------------------------------ -bool Blockchain::build_alt_chain(const crypto::hash &prev_id, std::list& alt_chain, std::vector ×tamps, block_verification_context& bvc) const +bool Blockchain::build_alt_chain(const crypto::hash &prev_id, std::list& alt_chain, std::vector ×tamps, block_verification_context& bvc) const { //build alternative subchain, front -> mainchain, back -> alternative head - blocks_ext_by_hash::const_iterator alt_it = m_alternative_chains.find(prev_id); + cryptonote::alt_block_data_t data; + cryptonote::blobdata blob; + bool found = m_db->get_alt_block(prev_id, &data, &blob); timestamps.clear(); - while(alt_it != m_alternative_chains.end()) + while(found) { - alt_chain.push_front(alt_it); - timestamps.push_back(alt_it->second.bl.timestamp); - alt_it = m_alternative_chains.find(alt_it->second.bl.prev_id); + block_extended_info bei; + CHECK_AND_ASSERT_MES(cryptonote::parse_and_validate_block_from_blob(blob, bei.bl), false, "Failed to parse alt block"); + bei.height = data.height; + bei.block_cumulative_weight = data.cumulative_weight; + bei.cumulative_difficulty = data.cumulative_difficulty_high; + bei.cumulative_difficulty = (bei.cumulative_difficulty << 64) + data.cumulative_difficulty_low; + bei.already_generated_coins = data.already_generated_coins; + timestamps.push_back(bei.bl.timestamp); + alt_chain.push_front(std::move(bei)); + found = m_db->get_alt_block(bei.bl.prev_id, &data, &blob); } // if block to be added connects to known blocks that aren't part of the @@ -1630,20 +1649,20 @@ bool Blockchain::build_alt_chain(const crypto::hash &prev_id, std::listheight() > alt_chain.front()->second.height, false, "main blockchain wrong height"); + CHECK_AND_ASSERT_MES(m_db->height() > alt_chain.front().height, false, "main blockchain wrong height"); // make sure that the blockchain contains the block that should connect // this alternate chain with it. - if (!m_db->block_exists(alt_chain.front()->second.bl.prev_id)) + if (!m_db->block_exists(alt_chain.front().bl.prev_id)) { MERROR("alternate chain does not appear to connect to main chain..."); return false; } // make sure block connects correctly to the main chain - auto h = m_db->get_block_hash_from_height(alt_chain.front()->second.height - 1); - CHECK_AND_ASSERT_MES(h == alt_chain.front()->second.bl.prev_id, false, "alternative chain has wrong connection to main chain"); - complete_timestamps_vector(m_db->get_block_height(alt_chain.front()->second.bl.prev_id), timestamps); + auto h = m_db->get_block_hash_from_height(alt_chain.front().height - 1); + CHECK_AND_ASSERT_MES(h == alt_chain.front().bl.prev_id, false, "alternative chain has wrong connection to main chain"); + complete_timestamps_vector(m_db->get_block_height(alt_chain.front().bl.prev_id), timestamps); } // if block not associated with known alternate chain else @@ -1698,12 +1717,13 @@ bool Blockchain::handle_alternative_block(const block& b, const crypto::hash& id //block is not related with head of main chain //first of all - look in alternative chains container - auto it_prev = m_alternative_chains.find(b.prev_id); + alt_block_data_t prev_data; + bool parent_in_alt = m_db->get_alt_block(b.prev_id, &prev_data, NULL); bool parent_in_main = m_db->block_exists(b.prev_id); - if(it_prev != m_alternative_chains.end() || parent_in_main) + if (parent_in_alt || parent_in_main) { //we have new block in alternative chain - std::list alt_chain; + std::list alt_chain; std::vector timestamps; if (!build_alt_chain(b.prev_id, alt_chain, timestamps, bvc)) return false; @@ -1711,10 +1731,10 @@ bool Blockchain::handle_alternative_block(const block& b, const crypto::hash& id // FIXME: consider moving away from block_extended_info at some point block_extended_info bei = boost::value_initialized(); bei.bl = b; - const uint64_t prev_height = alt_chain.size() ? it_prev->second.height : m_db->get_block_height(b.prev_id); + const uint64_t prev_height = alt_chain.size() ? prev_data.height : m_db->get_block_height(b.prev_id); bei.height = prev_height + 1; uint64_t block_reward = get_outs_money_amount(b.miner_tx); - bei.already_generated_coins = block_reward + (alt_chain.size() ? it_prev->second.already_generated_coins : m_db->get_block_already_generated_coins(prev_height)); + bei.already_generated_coins = block_reward + (alt_chain.size() ? prev_data.already_generated_coins : m_db->get_block_already_generated_coins(prev_height)); // verify that the block's timestamp is within the acceptable range // (not earlier than the median of the last X blocks) @@ -1758,7 +1778,8 @@ bool Blockchain::handle_alternative_block(const block& b, const crypto::hash& id difficulty_type main_chain_cumulative_difficulty = m_db->get_block_cumulative_difficulty(m_db->height() - 1); if (alt_chain.size()) { - bei.cumulative_difficulty = it_prev->second.cumulative_difficulty; + bei.cumulative_difficulty = prev_data.cumulative_difficulty_high; + bei.cumulative_difficulty = (bei.cumulative_difficulty << 64) + prev_data.cumulative_difficulty_low; } else { @@ -1769,15 +1790,21 @@ bool Blockchain::handle_alternative_block(const block& b, const crypto::hash& id // add block to alternate blocks storage, // as well as the current "alt chain" container - auto i_res = m_alternative_chains.insert(blocks_ext_by_hash::value_type(id, bei)); - CHECK_AND_ASSERT_MES(i_res.second, false, "insertion of new alternative block returned as it already exist"); - alt_chain.push_back(i_res.first); + CHECK_AND_ASSERT_MES(!m_db->get_alt_block(id, NULL, NULL), false, "insertion of new alternative block returned as it already exists"); + cryptonote::alt_block_data_t data; + data.height = bei.height; + data.cumulative_weight = bei.block_cumulative_weight; + data.cumulative_difficulty_low = (bei.cumulative_difficulty & 0xffffffffffffffff).convert_to(); + data.cumulative_difficulty_high = ((bei.cumulative_difficulty >> 64) & 0xffffffffffffffff).convert_to(); + data.already_generated_coins = bei.already_generated_coins; + m_db->add_alt_block(id, data, cryptonote::block_to_blob(bei.bl)); + alt_chain.push_back(bei); // FIXME: is it even possible for a checkpoint to show up not on the main chain? if(is_a_checkpoint) { //do reorganize! - MGINFO_GREEN("###### REORGANIZE on height: " << alt_chain.front()->second.height << " of " << m_db->height() - 1 << ", checkpoint is found in alternative chain on height " << bei.height); + MGINFO_GREEN("###### REORGANIZE on height: " << alt_chain.front().height << " of " << m_db->height() - 1 << ", checkpoint is found in alternative chain on height " << bei.height); bool r = switch_to_alternative_blockchain(alt_chain, true); @@ -1789,7 +1816,7 @@ bool Blockchain::handle_alternative_block(const block& b, const crypto::hash& id else if(main_chain_cumulative_difficulty < bei.cumulative_difficulty) //check if difficulty bigger then in main chain { //do reorganize! - MGINFO_GREEN("###### REORGANIZE on height: " << alt_chain.front()->second.height << " of " << m_db->height() - 1 << " with cum_difficulty " << m_db->get_block_cumulative_difficulty(m_db->height() - 1) << std::endl << " alternative blockchain size: " << alt_chain.size() << " with cum_difficulty " << bei.cumulative_difficulty); + MGINFO_GREEN("###### REORGANIZE on height: " << alt_chain.front().height << " of " << m_db->height() - 1 << " with cum_difficulty " << m_db->get_block_cumulative_difficulty(m_db->height() - 1) << std::endl << " alternative blockchain size: " << alt_chain.size() << " with cum_difficulty " << bei.cumulative_difficulty); bool r = switch_to_alternative_blockchain(alt_chain, false); if (r) @@ -1809,7 +1836,7 @@ bool Blockchain::handle_alternative_block(const block& b, const crypto::hash& id //block orphaned bvc.m_marked_as_orphaned = true; MERROR_VER("Block recognized as orphaned and rejected, id = " << id << ", height " << block_height - << ", parent in alt " << (it_prev != m_alternative_chains.end()) << ", parent in main " << parent_in_main + << ", parent in alt " << parent_in_alt << ", parent in main " << parent_in_main << " (parent " << b.prev_id << ", current top " << get_tail_id() << ", chain height " << get_current_blockchain_height() << ")"); } @@ -1914,11 +1941,20 @@ bool Blockchain::get_alternative_blocks(std::vector& blocks) const LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); - blocks.reserve(m_alternative_chains.size()); - for (const auto& alt_bl: m_alternative_chains) - { - blocks.push_back(alt_bl.second.bl); - } + blocks.reserve(m_db->get_alt_block_count()); + m_db->for_all_alt_blocks([&blocks](const crypto::hash &blkid, const cryptonote::alt_block_data_t &data, const cryptonote::blobdata *blob) { + if (!blob) + { + MERROR("No blob, but blobs were requested"); + return false; + } + cryptonote::block bl; + if (cryptonote::parse_and_validate_block_from_blob(*blob, bl)) + blocks.push_back(std::move(bl)); + else + MERROR("Failed to parse block from blob"); + return true; + }, true); return true; } //------------------------------------------------------------------ @@ -1926,7 +1962,7 @@ size_t Blockchain::get_alternative_blocks_count() const { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); - return m_alternative_chains.size(); + return m_db->get_alt_block_count(); } //------------------------------------------------------------------ // This function adds the output specified by to the result_outs container @@ -2414,9 +2450,9 @@ bool Blockchain::have_block(const crypto::hash& id) const return true; } - if(m_alternative_chains.count(id)) + if(m_db->get_alt_block(id, NULL, NULL)) { - LOG_PRINT_L2("block " << id << " found in m_alternative_chains"); + LOG_PRINT_L2("block " << id << " found in alternative chains"); return true; } @@ -4819,11 +4855,35 @@ std::list>> { std::list>> chains; - for (const auto &i: m_alternative_chains) + blocks_ext_by_hash alt_blocks; + alt_blocks.reserve(m_db->get_alt_block_count()); + m_db->for_all_alt_blocks([&alt_blocks](const crypto::hash &blkid, const cryptonote::alt_block_data_t &data, const cryptonote::blobdata *blob) { + if (!blob) + { + MERROR("No blob, but blobs were requested"); + return false; + } + cryptonote::block bl; + block_extended_info bei; + if (cryptonote::parse_and_validate_block_from_blob(*blob, bei.bl)) + { + bei.height = data.height; + bei.block_cumulative_weight = data.cumulative_weight; + bei.cumulative_difficulty = data.cumulative_difficulty_high; + bei.cumulative_difficulty = (bei.cumulative_difficulty << 64) + data.cumulative_difficulty_low; + bei.already_generated_coins = data.already_generated_coins; + alt_blocks.insert(std::make_pair(cryptonote::get_block_hash(bei.bl), std::move(bei))); + } + else + MERROR("Failed to parse block from blob"); + return true; + }, true); + + for (const auto &i: alt_blocks) { - const crypto::hash &top = i.first; + const crypto::hash top = cryptonote::get_block_hash(i.second.bl); bool found = false; - for (const auto &j: m_alternative_chains) + for (const auto &j: alt_blocks) { if (j.second.bl.prev_id == top) { @@ -4837,7 +4897,7 @@ std::list>> auto h = i.second.bl.prev_id; chain.push_back(top); blocks_ext_by_hash::const_iterator prev; - while ((prev = m_alternative_chains.find(h)) != m_alternative_chains.end()) + while ((prev = alt_blocks.find(h)) != alt_blocks.end()) { chain.push_back(h); h = prev->second.bl.prev_id; diff --git a/src/cryptonote_core/blockchain.h b/src/cryptonote_core/blockchain.h index 6200ec87e..97fb8d7a0 100644 --- a/src/cryptonote_core/blockchain.h +++ b/src/cryptonote_core/blockchain.h @@ -91,17 +91,6 @@ namespace cryptonote class Blockchain { public: - /** - * @brief Now-defunct (TODO: remove) struct from in-memory blockchain - */ - struct transaction_chain_entry - { - transaction tx; - uint64_t m_keeper_block_height; - size_t m_blob_size; - std::vector m_global_output_indexes; - }; - /** * @brief container for passing a block and metadata about it on the blockchain */ @@ -109,7 +98,7 @@ namespace cryptonote { block bl; //!< the block uint64_t height; //!< the height of the block in the blockchain - size_t block_cumulative_weight; //!< the weight of the block + uint64_t block_cumulative_weight; //!< the weight of the block difficulty_type cumulative_difficulty; //!< the accumulated difficulty after that block uint64_t already_generated_coins; //!< the total coins minted after that block }; @@ -1011,20 +1000,12 @@ namespace cryptonote #endif // TODO: evaluate whether or not each of these typedefs are left over from blockchain_storage - typedef std::unordered_map blocks_by_id_index; - - typedef std::unordered_map transactions_container; - typedef std::unordered_set key_images_container; typedef std::vector blocks_container; typedef std::unordered_map blocks_ext_by_hash; - typedef std::unordered_map blocks_by_hash; - - typedef std::map>> outputs_container; //crypto::hash - tx hash, size_t - index of out in transaction - BlockchainDB* m_db; @@ -1033,7 +1014,6 @@ namespace cryptonote mutable epee::critical_section m_blockchain_lock; // TODO: add here reader/writer lock // main chain - transactions_container m_transactions; size_t m_current_block_cumul_weight_limit; size_t m_current_block_cumul_weight_median; @@ -1074,9 +1054,6 @@ namespace cryptonote boost::thread_group m_async_pool; std::unique_ptr m_async_work_idle; - // all alternative chains - blocks_ext_by_hash m_alternative_chains; // crypto::hash -> block_extended_info - // some invalid blocks blocks_ext_by_hash m_invalid_blocks; // crypto::hash -> block_extended_info @@ -1183,7 +1160,7 @@ namespace cryptonote * * @return false if the reorganization fails, otherwise true */ - bool switch_to_alternative_blockchain(std::list& alt_chain, bool discard_disconnected_chain); + bool switch_to_alternative_blockchain(std::list& alt_chain, bool discard_disconnected_chain); /** * @brief removes the most recent block from the blockchain @@ -1246,7 +1223,7 @@ namespace cryptonote * * @return true on success, false otherwise */ - bool build_alt_chain(const crypto::hash &prev_id, std::list& alt_chain, std::vector ×tamps, block_verification_context& bvc) const; + bool build_alt_chain(const crypto::hash &prev_id, std::list& alt_chain, std::vector ×tamps, block_verification_context& bvc) const; /** * @brief gets the difficulty requirement for a new block on an alternate chain @@ -1256,7 +1233,7 @@ namespace cryptonote * * @return the difficulty requirement */ - difficulty_type get_next_difficulty_for_alternative_chain(const std::list& alt_chain, block_extended_info& bei) const; + difficulty_type get_next_difficulty_for_alternative_chain(const std::list& alt_chain, block_extended_info& bei) const; /** * @brief sanity checks a miner transaction before validating an entire block diff --git a/src/cryptonote_core/cryptonote_core.cpp b/src/cryptonote_core/cryptonote_core.cpp index 2ab91cbfd..7f3857da3 100644 --- a/src/cryptonote_core/cryptonote_core.cpp +++ b/src/cryptonote_core/cryptonote_core.cpp @@ -208,6 +208,11 @@ namespace cryptonote "is acted upon." , "" }; + static const command_line::arg_descriptor arg_keep_alt_blocks = { + "keep-alt-blocks" + , "Keep alternative blocks on restart" + , false + }; //----------------------------------------------------------------------------------------------- core::core(i_cryptonote_protocol* pprotocol): @@ -325,6 +330,7 @@ namespace cryptonote command_line::add_arg(desc, arg_prune_blockchain); command_line::add_arg(desc, arg_reorg_notify); command_line::add_arg(desc, arg_block_rate_notify); + command_line::add_arg(desc, arg_keep_alt_blocks); miner::init_options(desc); BlockchainDB::init_options(desc); @@ -456,6 +462,7 @@ namespace cryptonote std::string check_updates_string = command_line::get_arg(vm, arg_check_updates); size_t max_txpool_weight = command_line::get_arg(vm, arg_max_txpool_weight); bool prune_blockchain = command_line::get_arg(vm, arg_prune_blockchain); + bool keep_alt_blocks = command_line::get_arg(vm, arg_keep_alt_blocks); boost::filesystem::path folder(m_config_folder); if (m_nettype == FAKECHAIN) @@ -671,6 +678,9 @@ namespace cryptonote r = m_miner.init(vm, m_nettype); CHECK_AND_ASSERT_MES(r, false, "Failed to initialize miner instance"); + if (!keep_alt_blocks && !m_blockchain_storage.get_db().is_read_only()) + m_blockchain_storage.get_db().drop_alt_blocks(); + if (prune_blockchain) { // display a message if the blockchain is not pruned yet From c31979635504dd8c2c983a111f41accdac9205c7 Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Tue, 23 Apr 2019 20:32:27 +0100 Subject: [PATCH 2/6] RandomX integration Support RandomX PoW algorithm --- .gitmodules | 3 + CMakeLists.txt | 1 + external/CMakeLists.txt | 1 + external/randomx | 1 + src/crypto/CMakeLists.txt | 4 + src/crypto/c_threads.h | 56 +++ src/crypto/hash-ops.h | 11 + src/crypto/rx-slow-hash.c | 349 ++++++++++++++++++ src/crypto/slow-hash.c | 28 +- .../cryptonote_format_utils.cpp | 15 - .../cryptonote_format_utils.h | 2 - src/cryptonote_basic/miner.cpp | 11 +- src/cryptonote_basic/miner.h | 7 +- src/cryptonote_config.h | 1 + src/cryptonote_core/blockchain.cpp | 30 +- src/cryptonote_core/cryptonote_core.cpp | 4 +- src/cryptonote_core/cryptonote_tx_utils.cpp | 48 ++- src/cryptonote_core/cryptonote_tx_utils.h | 7 + src/rpc/core_rpc_server.cpp | 17 +- src/rpc/core_rpc_server_commands_defs.h | 6 +- tests/core_proxy/core_proxy.cpp | 4 +- tests/core_tests/chaingen.cpp | 4 +- 22 files changed, 568 insertions(+), 42 deletions(-) create mode 160000 external/randomx create mode 100644 src/crypto/c_threads.h create mode 100644 src/crypto/rx-slow-hash.c diff --git a/.gitmodules b/.gitmodules index 6e2339fb9..f8e7c305b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -12,3 +12,6 @@ [submodule "external/trezor-common"] path = external/trezor-common url = https://github.com/trezor/trezor-common.git +[submodule "external/randomx"] + path = external/randomx + url = https://github.com/tevador/RandomX diff --git a/CMakeLists.txt b/CMakeLists.txt index 154e291d1..a1bf8dc9a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -205,6 +205,7 @@ if(NOT MANUAL_SUBMODULES) check_submodule(external/unbound) check_submodule(external/rapidjson) check_submodule(external/trezor-common) + check_submodule(external/randomx) endif() endif() diff --git a/external/CMakeLists.txt b/external/CMakeLists.txt index a148b7e85..436f29ced 100644 --- a/external/CMakeLists.txt +++ b/external/CMakeLists.txt @@ -100,3 +100,4 @@ endif() add_subdirectory(db_drivers) add_subdirectory(easylogging++) +add_subdirectory(randomx) diff --git a/external/randomx b/external/randomx new file mode 160000 index 000000000..1276d67d2 --- /dev/null +++ b/external/randomx @@ -0,0 +1 @@ +Subproject commit 1276d67d2f594ea4a8e9cde28253b1a74769aeb9 diff --git a/src/crypto/CMakeLists.txt b/src/crypto/CMakeLists.txt index 3a4d09fd8..80f1e5d7e 100644 --- a/src/crypto/CMakeLists.txt +++ b/src/crypto/CMakeLists.txt @@ -46,6 +46,7 @@ set(crypto_sources random.c skein.c slow-hash.c + rx-slow-hash.c CryptonightR_JIT.c tree-hash.c) @@ -53,6 +54,8 @@ if(ARCH_ID STREQUAL "i386" OR ARCH_ID STREQUAL "x86_64" OR ARCH_ID STREQUAL "x86 list(APPEND crypto_sources CryptonightR_template.S) endif() +include_directories(${RANDOMX_INCLUDE}) + set(crypto_headers) set(crypto_private_headers @@ -86,6 +89,7 @@ monero_add_library(cncrypto target_link_libraries(cncrypto PUBLIC epee + randomx ${Boost_SYSTEM_LIBRARY} ${SODIUM_LIBRARY} PRIVATE diff --git a/src/crypto/c_threads.h b/src/crypto/c_threads.h new file mode 100644 index 000000000..e73ebd9c3 --- /dev/null +++ b/src/crypto/c_threads.h @@ -0,0 +1,56 @@ +// Copyright (c) 2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +/* Brain-dead simple portability wrapper over thread APIs for C */ +#pragma once + +#ifdef _WIN32 +#include +#define CTHR_MUTEX_TYPE HANDLE +#define CTHR_MUTEX_INIT NULL +#define CTHR_MUTEX_LOCK(x) do { if (x == NULL) { \ + HANDLE p = CreateMutex(NULL, FALSE, NULL); \ + if (InterlockedCompareExchangePointer((PVOID*)&x, (PVOID)p, NULL) != NULL) \ + CloseHandle(p); \ + } WaitForSingleObject(x, INFINITE) == WAIT_FAILED; } while(0) +#define CTHR_MUTEX_UNLOCK(x) (ReleaseMutex(x) == 0) +#define CTHR_THREAD_TYPE HANDLE +#define CTHR_THREAD_RTYPE void +#define CTHR_THREAD_CREATE(thr, func, arg) thr = _beginthread(func, 0, arg) +#define CTHR_THREAD_JOIN(thr) WaitForSingleObject(thr, INFINITE) +#else +#include +#define CTHR_MUTEX_TYPE pthread_mutex_t +#define CTHR_MUTEX_INIT PTHREAD_MUTEX_INITIALIZER +#define CTHR_MUTEX_LOCK(x) pthread_mutex_lock(&x) +#define CTHR_MUTEX_UNLOCK(x) pthread_mutex_unlock(&x) +#define CTHR_THREAD_TYPE pthread_t +#define CTHR_THREAD_RTYPE void * +#define CTHR_THREAD_CREATE(thr, func, arg) pthread_create(&thr, NULL, func, arg) +#define CTHR_THREAD_JOIN(thr) pthread_join(thr, NULL) +#endif diff --git a/src/crypto/hash-ops.h b/src/crypto/hash-ops.h index 859c810bd..de91734d4 100644 --- a/src/crypto/hash-ops.h +++ b/src/crypto/hash-ops.h @@ -87,3 +87,14 @@ void hash_extra_jh(const void *data, size_t length, char *hash); void hash_extra_skein(const void *data, size_t length, char *hash); void tree_hash(const char (*hashes)[HASH_SIZE], size_t count, char *root_hash); + +#define RX_BLOCK_VERSION 12 +void rx_slow_hash_allocate_state(void); +void rx_slow_hash_free_state(void); +uint64_t rx_seedheight(const uint64_t height); +void rx_seedheights(const uint64_t height, uint64_t *seed_height, uint64_t *next_height); +bool rx_needhash(const uint64_t height, uint64_t *seedheight); +void rx_seedhash(const uint64_t seedheight, const char *hash, const int miners); +void rx_slow_hash(const void *data, size_t length, char *hash, const int miners); +void rx_alt_slowhash(const uint64_t mainheight, const uint64_t seedheight, const char *seedhash, const void *data, size_t length, char *hash); +void rx_reorg(const uint64_t split_height); diff --git a/src/crypto/rx-slow-hash.c b/src/crypto/rx-slow-hash.c new file mode 100644 index 000000000..3982ddcea --- /dev/null +++ b/src/crypto/rx-slow-hash.c @@ -0,0 +1,349 @@ +// Copyright (c) 2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "randomx.h" +#include "c_threads.h" +#include "hash-ops.h" + +#if defined(_MSC_VER) +#define THREADV __declspec(thread) +#else +#define THREADV __thread +#endif + +static CTHR_MUTEX_TYPE rx_mutex = CTHR_MUTEX_INIT; + +typedef struct rx_state { + volatile uint64_t rs_height; + randomx_cache *rs_cache; +} rx_state; + +static rx_state rx_s[2]; + +static randomx_dataset *rx_dataset; +THREADV int rx_s_toggle; +THREADV randomx_vm *rx_vm = NULL; + +static void local_abort(const char *msg) +{ + fprintf(stderr, "%s\n", msg); +#ifdef NDEBUG + _exit(1); +#else + abort(); +#endif +} + +/** + * @brief uses cpuid to determine if the CPU supports the AES instructions + * @return true if the CPU supports AES, false otherwise + */ + +static inline int force_software_aes(void) +{ + static int use = -1; + + if (use != -1) + return use; + + const char *env = getenv("MONERO_USE_SOFTWARE_AES"); + if (!env) { + use = 0; + } + else if (!strcmp(env, "0") || !strcmp(env, "no")) { + use = 0; + } + else { + use = 1; + } + return use; +} + +static void cpuid(int CPUInfo[4], int InfoType) +{ +#if defined(__x86_64__) + __asm __volatile__ + ( + "cpuid": + "=a" (CPUInfo[0]), + "=b" (CPUInfo[1]), + "=c" (CPUInfo[2]), + "=d" (CPUInfo[3]) : + "a" (InfoType), "c" (0) + ); +#endif +} +static inline int check_aes_hw(void) +{ +#if defined(__x86_64__) + int cpuid_results[4]; + static int supported = -1; + + if(supported >= 0) + return supported; + + cpuid(cpuid_results,1); + return supported = cpuid_results[2] & (1 << 25); +#else + return 0; +#endif +} + +static volatile int use_rx_jit_flag = -1; + +static inline int use_rx_jit(void) +{ +#if defined(__x86_64__) + + if (use_rx_jit_flag != -1) + return use_rx_jit_flag; + + const char *env = getenv("MONERO_USE_RX_JIT"); + if (!env) { + use_rx_jit_flag = 1; + } + else if (!strcmp(env, "0") || !strcmp(env, "no")) { + use_rx_jit_flag = 0; + } + else { + use_rx_jit_flag = 1; + } + return use_rx_jit_flag; +#else + return 0; +#endif +} + +#define SEEDHASH_EPOCH_BLOCKS 2048 /* Must be same as BLOCKS_SYNCHRONIZING_MAX_COUNT in cryptonote_config.h */ +#define SEEDHASH_EPOCH_LAG 64 + +void rx_reorg(const uint64_t split_height) { + int i; + CTHR_MUTEX_LOCK(rx_mutex); + for (i=0; i<2; i++) { + if (split_height < rx_s[i].rs_height) + rx_s[i].rs_height = 1; /* set to an invalid seed height */ + } + CTHR_MUTEX_UNLOCK(rx_mutex); +} + +uint64_t rx_seedheight(const uint64_t height) { + uint64_t s_height = (height <= SEEDHASH_EPOCH_BLOCKS+SEEDHASH_EPOCH_LAG) ? 0 : + (height - SEEDHASH_EPOCH_LAG - 1) & ~(SEEDHASH_EPOCH_BLOCKS-1); + return s_height; +} + +void rx_seedheights(const uint64_t height, uint64_t *seedheight, uint64_t *nextheight) { + *seedheight = rx_seedheight(height); + *nextheight = rx_seedheight(height + SEEDHASH_EPOCH_LAG); +} + +bool rx_needhash(const uint64_t height, uint64_t *seedheight) { + rx_state *rx_sp; + uint64_t s_height = rx_seedheight(height); + int toggle = (s_height & SEEDHASH_EPOCH_BLOCKS) != 0; + bool ret; + bool changed = (toggle != rx_s_toggle); + *seedheight = s_height; + rx_s_toggle = toggle; + rx_sp = &rx_s[rx_s_toggle]; + ret = (rx_sp->rs_cache == NULL) || (rx_sp->rs_height != s_height); + /* if cache is ok but we've flipped caches, reset vm */ + if (!ret && changed && rx_vm != NULL) + randomx_vm_set_cache(rx_vm, rx_sp->rs_cache); + return ret; +} + +typedef struct seedinfo { + randomx_cache *si_cache; + unsigned long si_start; + unsigned long si_count; +} seedinfo; + +static CTHR_THREAD_RTYPE rx_seedthread(void *arg) { + seedinfo *si = arg; + randomx_init_dataset(rx_dataset, si->si_cache, si->si_start, si->si_count); + return NULL; +} + +static void rx_initdata(randomx_cache *rs_cache, const int miners) { + if (miners > 1) { + unsigned long delta = randomx_dataset_item_count() / miners; + unsigned long start = 0; + int i; + seedinfo *si; + CTHR_THREAD_TYPE *st; + si = malloc(miners * sizeof(seedinfo)); + if (si == NULL) + local_abort("Couldn't allocate RandomX mining threadinfo"); + st = malloc(miners * sizeof(CTHR_THREAD_TYPE)); + if (st == NULL) { + free(si); + local_abort("Couldn't allocate RandomX mining threadlist"); + } + for (i=0; irs_cache; + if (rx_sp->rs_height != height || cache == NULL) { + if (use_rx_jit()) + flags |= RANDOMX_FLAG_JIT; + if (cache == NULL) { + cache = randomx_alloc_cache(flags | RANDOMX_FLAG_LARGE_PAGES); + if (cache == NULL) + cache = randomx_alloc_cache(flags); + if (cache == NULL) + local_abort("Couldn't allocate RandomX cache"); + } + randomx_init_cache(cache, hash, 32); + if (miners && rx_dataset != NULL) + rx_initdata(cache, miners); + rx_sp->rs_height = height; + if (rx_sp->rs_cache == NULL) + rx_sp->rs_cache = cache; + } + CTHR_MUTEX_UNLOCK(rx_mutex); + if (rx_vm != NULL) + randomx_vm_set_cache(rx_vm, rx_sp->rs_cache); +} + +void rx_seedhash(const uint64_t height, const char *hash, const int miners) { + rx_state *rx_sp = &rx_s[rx_s_toggle]; + rx_seedhash_int(rx_sp, height, hash, miners); +} + +static char rx_althash[32]; // seedhash for alternate blocks + +void rx_alt_slowhash(const uint64_t mainheight, const uint64_t seedheight, const char *seedhash, const void *data, size_t length, char *hash) { + uint64_t s_height = rx_seedheight(mainheight); + int alt_toggle = (s_height & SEEDHASH_EPOCH_BLOCKS) == 0; + rx_state *rx_sp = &rx_s[alt_toggle]; + if (rx_sp->rs_height != seedheight || rx_sp->rs_cache == NULL || memcmp(hash, rx_althash, sizeof(rx_althash))) { + memcpy(rx_althash, seedhash, sizeof(rx_althash)); + rx_sp->rs_height = 1; + rx_seedhash_int(rx_sp, seedheight, seedhash, 0); + } + if (rx_vm == NULL) { + randomx_flags flags = RANDOMX_FLAG_DEFAULT; + if (use_rx_jit()) + flags |= RANDOMX_FLAG_JIT; + if(!force_software_aes() && check_aes_hw()) + flags |= RANDOMX_FLAG_HARD_AES; + rx_vm = randomx_create_vm(flags | RANDOMX_FLAG_LARGE_PAGES, rx_sp->rs_cache, NULL); + if(rx_vm == NULL) //large pages failed + rx_vm = randomx_create_vm(flags, rx_sp->rs_cache, NULL); + if(rx_vm == NULL) {//fallback if everything fails + flags = RANDOMX_FLAG_DEFAULT; + rx_vm = randomx_create_vm(flags, rx_sp->rs_cache, NULL); + } + if (rx_vm == NULL) + local_abort("Couldn't allocate RandomX VM"); + } + randomx_calculate_hash(rx_vm, data, length, hash); +} + +void rx_slow_hash(const void *data, size_t length, char *hash, int miners) { + if (rx_vm == NULL) { + rx_state *rx_sp = &rx_s[rx_s_toggle]; + randomx_flags flags = RANDOMX_FLAG_DEFAULT; + if (use_rx_jit()) + flags |= RANDOMX_FLAG_JIT; + if(!force_software_aes() && check_aes_hw()) + flags |= RANDOMX_FLAG_HARD_AES; + if (miners) { + if (rx_dataset == NULL) { + CTHR_MUTEX_LOCK(rx_mutex); + if (rx_dataset == NULL) { + rx_dataset = randomx_alloc_dataset(RANDOMX_FLAG_LARGE_PAGES); + if (rx_dataset == NULL) + rx_dataset = randomx_alloc_dataset(RANDOMX_FLAG_DEFAULT); + if (rx_dataset != NULL) + rx_initdata(rx_sp->rs_cache, miners); + } + CTHR_MUTEX_UNLOCK(rx_mutex); + } + if (rx_dataset != NULL) + flags |= RANDOMX_FLAG_FULL_MEM; + else + miners = 0; + } + rx_vm = randomx_create_vm(flags | RANDOMX_FLAG_LARGE_PAGES, rx_sp->rs_cache, rx_dataset); + if(rx_vm == NULL) //large pages failed + rx_vm = randomx_create_vm(flags, rx_sp->rs_cache, rx_dataset); + if(rx_vm == NULL) {//fallback if everything fails + flags = RANDOMX_FLAG_DEFAULT | (miners ? RANDOMX_FLAG_FULL_MEM : 0); + rx_vm = randomx_create_vm(flags, rx_sp->rs_cache, rx_dataset); + } + if (rx_vm == NULL) + local_abort("Couldn't allocate RandomX VM"); + } + randomx_calculate_hash(rx_vm, data, length, hash); +} + +void rx_slow_hash_allocate_state(void) { +} + +void rx_slow_hash_free_state(void) { + if (rx_vm != NULL) { + randomx_destroy_vm(rx_vm); + rx_vm = NULL; + } +} diff --git a/src/crypto/slow-hash.c b/src/crypto/slow-hash.c index 32a843a1b..55c3b6b4e 100644 --- a/src/crypto/slow-hash.c +++ b/src/crypto/slow-hash.c @@ -653,7 +653,7 @@ BOOL SetLockPagesPrivilege(HANDLE hProcess, BOOL bEnable) * the allocated buffer. */ -void slow_hash_allocate_state(void) +void cn_slow_hash_allocate_state(void) { if(hp_state != NULL) return; @@ -686,7 +686,7 @@ void slow_hash_allocate_state(void) *@brief frees the state allocated by slow_hash_allocate_state */ -void slow_hash_free_state(void) +void cn_slow_hash_free_state(void) { if(hp_state == NULL) return; @@ -760,7 +760,7 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int // this isn't supposed to happen, but guard against it for now. if(hp_state == NULL) - slow_hash_allocate_state(); + cn_slow_hash_allocate_state(); /* CryptoNight Step 1: Use Keccak1600 to initialize the 'state' (and 'text') buffers from the data. */ if (prehashed) { @@ -874,13 +874,13 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int } #elif !defined NO_AES && (defined(__arm__) || defined(__aarch64__)) -void slow_hash_allocate_state(void) +void cn_slow_hash_allocate_state(void) { // Do nothing, this is just to maintain compatibility with the upgraded slow-hash.c return; } -void slow_hash_free_state(void) +void cn_slow_hash_free_state(void) { // As above return; @@ -1440,13 +1440,15 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int #else // Portable implementation as a fallback -void slow_hash_allocate_state(void) +#define hp_jitfunc ((v4_random_math_JIT_func)NULL) + +void cn_slow_hash_allocate_state(void) { // Do nothing, this is just to maintain compatibility with the upgraded slow-hash.c return; } -void slow_hash_free_state(void) +void cn_slow_hash_free_state(void) { // As above return; @@ -1623,3 +1625,15 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int } #endif + +void slow_hash_allocate_state(void) +{ + cn_slow_hash_allocate_state(); + rx_slow_hash_allocate_state(); +} + +void slow_hash_free_state(void) +{ + cn_slow_hash_free_state(); + rx_slow_hash_free_state(); +} diff --git a/src/cryptonote_basic/cryptonote_format_utils.cpp b/src/cryptonote_basic/cryptonote_format_utils.cpp index 421721764..fca10dfca 100644 --- a/src/cryptonote_basic/cryptonote_format_utils.cpp +++ b/src/cryptonote_basic/cryptonote_format_utils.cpp @@ -1202,14 +1202,6 @@ namespace cryptonote return p; } //--------------------------------------------------------------- - bool get_block_longhash(const block& b, crypto::hash& res, uint64_t height) - { - blobdata bd = get_block_hashing_blob(b); - const int cn_variant = b.major_version >= 11 ? 4 : b.major_version >= 9 && b.major_version <= 10 ? 2 : 1; - crypto::cn_slow_hash(bd.data(), bd.size(), res, cn_variant, height); - return true; - } - //--------------------------------------------------------------- std::vector relative_output_offsets_to_absolute(const std::vector& off) { std::vector res = off; @@ -1230,13 +1222,6 @@ namespace cryptonote return res; } //--------------------------------------------------------------- - crypto::hash get_block_longhash(const block& b, uint64_t height) - { - crypto::hash p = null_hash; - get_block_longhash(b, p, height); - return p; - } - //--------------------------------------------------------------- bool parse_and_validate_block_from_blob(const blobdata& b_blob, block& b, crypto::hash *block_hash) { std::stringstream ss; diff --git a/src/cryptonote_basic/cryptonote_format_utils.h b/src/cryptonote_basic/cryptonote_format_utils.h index c9de2a56e..284494299 100644 --- a/src/cryptonote_basic/cryptonote_format_utils.h +++ b/src/cryptonote_basic/cryptonote_format_utils.h @@ -117,8 +117,6 @@ namespace cryptonote bool calculate_block_hash(const block& b, crypto::hash& res, const blobdata *blob = NULL); bool get_block_hash(const block& b, crypto::hash& res); crypto::hash get_block_hash(const block& b); - bool get_block_longhash(const block& b, crypto::hash& res, uint64_t height); - crypto::hash get_block_longhash(const block& b, uint64_t height); bool parse_and_validate_block_from_blob(const blobdata& b_blob, block& b, crypto::hash *block_hash); bool parse_and_validate_block_from_blob(const blobdata& b_blob, block& b); bool parse_and_validate_block_from_blob(const blobdata& b_blob, block& b, crypto::hash &block_hash); diff --git a/src/cryptonote_basic/miner.cpp b/src/cryptonote_basic/miner.cpp index 173679e21..5e3b7cda5 100644 --- a/src/cryptonote_basic/miner.cpp +++ b/src/cryptonote_basic/miner.cpp @@ -37,8 +37,10 @@ #include "syncobj.h" #include "cryptonote_basic_impl.h" #include "cryptonote_format_utils.h" +#include "cryptonote_core/cryptonote_tx_utils.h" #include "file_io_utils.h" #include "common/command_line.h" +#include "common/util.h" #include "string_coding.h" #include "string_tools.h" #include "storages/portable_storage_template_helper.h" @@ -99,12 +101,13 @@ namespace cryptonote } - miner::miner(i_miner_handler* phandler):m_stop(1), + miner::miner(i_miner_handler* phandler, Blockchain* pbc):m_stop(1), m_template(boost::value_initialized()), m_template_no(0), m_diffic(0), m_thread_index(0), m_phandler(phandler), + m_pbc(pbc), m_height(0), m_threads_active(0), m_pausers_count(0), @@ -466,12 +469,12 @@ namespace cryptonote return true; } //----------------------------------------------------------------------------------------------------- - bool miner::find_nonce_for_given_block(block& bl, const difficulty_type& diffic, uint64_t height) + bool miner::find_nonce_for_given_block(const Blockchain *pbc, block& bl, const difficulty_type& diffic, uint64_t height) { for(; bl.nonce != std::numeric_limits::max(); bl.nonce++) { crypto::hash h; - get_block_longhash(bl, h, height); + get_block_longhash(pbc, bl, h, height, tools::get_max_concurrency()); if(check_hash(h, diffic)) { @@ -570,7 +573,7 @@ namespace cryptonote b.nonce = nonce; crypto::hash h; - get_block_longhash(b, h, height); + get_block_longhash(m_pbc, b, h, height, tools::get_max_concurrency()); if(check_hash(h, local_diff)) { diff --git a/src/cryptonote_basic/miner.h b/src/cryptonote_basic/miner.h index 285075f51..4a65599ce 100644 --- a/src/cryptonote_basic/miner.h +++ b/src/cryptonote_basic/miner.h @@ -52,13 +52,15 @@ namespace cryptonote ~i_miner_handler(){}; }; + class Blockchain; + /************************************************************************/ /* */ /************************************************************************/ class miner { public: - miner(i_miner_handler* phandler); + miner(i_miner_handler* phandler, Blockchain* pbc); ~miner(); bool init(const boost::program_options::variables_map& vm, network_type nettype); static void init_options(boost::program_options::options_description& desc); @@ -74,7 +76,7 @@ namespace cryptonote bool on_idle(); void on_synchronized(); //synchronous analog (for fast calls) - static bool find_nonce_for_given_block(block& bl, const difficulty_type& diffic, uint64_t height); + static bool find_nonce_for_given_block(const Blockchain *pbc, block& bl, const difficulty_type& diffic, uint64_t height); void pause(); void resume(); void do_print_hashrate(bool do_hr); @@ -133,6 +135,7 @@ namespace cryptonote std::list m_threads; epee::critical_section m_threads_lock; i_miner_handler* m_phandler; + Blockchain* m_pbc; account_public_address m_mine_address; epee::math_helper::once_a_time_seconds<5> m_update_block_template_interval; epee::math_helper::once_a_time_seconds<2> m_update_merge_hr_interval; diff --git a/src/cryptonote_config.h b/src/cryptonote_config.h index c57347ecf..199977a0a 100644 --- a/src/cryptonote_config.h +++ b/src/cryptonote_config.h @@ -103,6 +103,7 @@ #define BLOCKS_IDS_SYNCHRONIZING_DEFAULT_COUNT 10000 //by default, blocks ids count in synchronizing #define BLOCKS_SYNCHRONIZING_DEFAULT_COUNT_PRE_V4 100 //by default, blocks count in blocks downloading #define BLOCKS_SYNCHRONIZING_DEFAULT_COUNT 20 //by default, blocks count in blocks downloading +#define BLOCKS_SYNCHRONIZING_MAX_COUNT 2048 //must be a power of 2, greater than 128, equal to SEEDHASH_EPOCH_BLOCKS #define CRYPTONOTE_MEMPOOL_TX_LIVETIME (86400*3) //seconds, three days #define CRYPTONOTE_MEMPOOL_TX_FROM_ALT_BLOCK_LIVETIME 604800 //seconds, one week diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index 91f18aafc..e66e11fe0 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -1093,6 +1093,7 @@ bool Blockchain::switch_to_alternative_blockchain(std::list } m_hardfork->reorganize_from_chain_height(split_height); + get_block_longhash_reorg(split_height); std::shared_ptr reorg_notify = m_reorg_notify; if (reorg_notify) @@ -1757,7 +1758,30 @@ bool Blockchain::handle_alternative_block(const block& b, const crypto::hash& id difficulty_type current_diff = get_next_difficulty_for_alternative_chain(alt_chain, bei); CHECK_AND_ASSERT_MES(current_diff, false, "!!!!!!! DIFFICULTY OVERHEAD !!!!!!!"); crypto::hash proof_of_work = null_hash; - get_block_longhash(bei.bl, proof_of_work, bei.height); + if (b.major_version >= RX_BLOCK_VERSION) + { + crypto::hash seedhash = null_hash; + uint64_t seedheight = rx_seedheight(bei.height); + // seedblock is on the alt chain somewhere + if (alt_chain.size() && alt_chain.front().height <= seedheight) + { + for (auto it=alt_chain.begin(); it != alt_chain.end(); it++) + { + if (it->height == seedheight+1) + { + seedhash = it->bl.prev_id; + break; + } + } + } else + { + seedhash = get_block_id_by_height(seedheight); + } + get_altblock_longhash(bei.bl, proof_of_work, get_current_blockchain_height(), bei.height, seedheight, seedhash); + } else + { + get_block_longhash(this, bei.bl, proof_of_work, bei.height, 0); + } if(!check_hash(proof_of_work, current_diff)) { MERROR_VER("Block with id: " << id << std::endl << " for alternative chain, does not have enough proof of work: " << proof_of_work << std::endl << " expected difficulty: " << current_diff); @@ -3694,7 +3718,7 @@ leave: proof_of_work = it->second; } else - proof_of_work = get_block_longhash(bl, blockchain_height); + proof_of_work = get_block_longhash(this, bl, blockchain_height, 0); // validate proof_of_work versus difficulty target if(!check_hash(proof_of_work, current_diffic)) @@ -4195,7 +4219,7 @@ void Blockchain::block_longhash_worker(uint64_t height, const epee::span()), m_starter_message_showed(false), m_target_blockchain_height(0), @@ -654,6 +654,8 @@ namespace cryptonote CHECK_AND_ASSERT_MES(r, false, "Failed to initialize blockchain storage"); block_sync_size = command_line::get_arg(vm, arg_block_sync_size); + if (block_sync_size > BLOCKS_SYNCHRONIZING_MAX_COUNT) + MERROR("Error --dblock-sync-size cannot be greater than " << BLOCKS_SYNCHRONIZING_MAX_COUNT); MGINFO("Loading checkpoints"); diff --git a/src/cryptonote_core/cryptonote_tx_utils.cpp b/src/cryptonote_core/cryptonote_tx_utils.cpp index 5816cf125..bea24c74f 100644 --- a/src/cryptonote_core/cryptonote_tx_utils.cpp +++ b/src/cryptonote_core/cryptonote_tx_utils.cpp @@ -37,6 +37,7 @@ using namespace epee; #include "common/apply_permutation.h" #include "cryptonote_tx_utils.h" #include "cryptonote_config.h" +#include "blockchain.h" #include "cryptonote_basic/miner.h" #include "cryptonote_basic/tx_extra.h" #include "crypto/crypto.h" @@ -660,9 +661,54 @@ namespace cryptonote bl.minor_version = CURRENT_BLOCK_MINOR_VERSION; bl.timestamp = 0; bl.nonce = nonce; - miner::find_nonce_for_given_block(bl, 1, 0); + miner::find_nonce_for_given_block(NULL, bl, 1, 0); bl.invalidate_hashes(); return true; } //--------------------------------------------------------------- + void get_altblock_longhash(const block& b, crypto::hash& res, const uint64_t main_height, const uint64_t height, const uint64_t seed_height, const crypto::hash& seed_hash) + { + blobdata bd = get_block_hashing_blob(b); + rx_alt_slowhash(main_height, seed_height, seed_hash.data, bd.data(), bd.size(), res.data); + } + + bool get_block_longhash(const Blockchain *pbc, const block& b, crypto::hash& res, const uint64_t height, const int miners) + { + // block 202612 bug workaround + if (height == 202612) + { + static const std::string longhash_202612 = "84f64766475d51837ac9efbef1926486e58563c95a19fef4aec3254f03000000"; + epee::string_tools::hex_to_pod(longhash_202612, res); + return true; + } + blobdata bd = get_block_hashing_blob(b); + const int pow_variant = b.major_version >= 7 ? b.major_version - 6 : 0; + if (pow_variant >= 6) { + uint64_t seed_height; + if (rx_needhash(height, &seed_height)) { + crypto::hash hash; + if (pbc != NULL) + hash = pbc->get_block_id_by_height(seed_height); + else + memset(&hash, 0, sizeof(hash)); // only happens when generating genesis block + rx_seedhash(seed_height, hash.data, miners); + } + rx_slow_hash(bd.data(), bd.size(), res.data, miners); + } else { + crypto::cn_slow_hash(bd.data(), bd.size(), res, pow_variant, height); + } + return true; + } + + crypto::hash get_block_longhash(const Blockchain *pbc, const block& b, const uint64_t height, const int miners) + { + crypto::hash p = crypto::null_hash; + get_block_longhash(pbc, b, p, height, miners); + return p; + } + + void get_block_longhash_reorg(const uint64_t split_height) + { + rx_reorg(split_height); + } } diff --git a/src/cryptonote_core/cryptonote_tx_utils.h b/src/cryptonote_core/cryptonote_tx_utils.h index d38aa7474..a325cd35e 100644 --- a/src/cryptonote_core/cryptonote_tx_utils.h +++ b/src/cryptonote_core/cryptonote_tx_utils.h @@ -117,6 +117,13 @@ namespace cryptonote , uint32_t nonce ); + class Blockchain; + bool get_block_longhash(const Blockchain *pb, const block& b, crypto::hash& res, const uint64_t height, const int miners); + void get_altblock_longhash(const block& b, crypto::hash& res, const uint64_t main_height, const uint64_t height, + const uint64_t seed_height, const crypto::hash& seed_hash); + crypto::hash get_block_longhash(const Blockchain *pb, const block& b, const uint64_t height, const int miners); + void get_block_longhash_reorg(const uint64_t split_height); + } BOOST_CLASS_VERSION(cryptonote::tx_source_entry, 1) diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index 85cea5573..d48b634d2 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -1012,6 +1012,7 @@ namespace cryptonote case 1: res.pow_algorithm = "CNv1 (Cryptonight variant 1)"; break; case 2: case 3: res.pow_algorithm = "CNv2 (Cryptonight variant 2)"; break; case 4: case 5: res.pow_algorithm = "CNv4 (Cryptonight variant 4)"; break; + case 6: res.pow_algorithm = "RandomX"; break; default: res.pow_algorithm = "I'm not sure actually"; break; } if (res.is_background_mining_enabled) @@ -1295,6 +1296,18 @@ namespace cryptonote LOG_ERROR("Failed to create block template"); return false; } + if (b.major_version >= RX_BLOCK_VERSION) + { + uint64_t seed_height, next_height; + crypto::hash seed_hash; + crypto::rx_seedheights(res.height, &seed_height, &next_height); + seed_hash = m_core.get_block_id_by_height(seed_height); + res.seed_hash = string_tools::pod_to_hex(seed_hash); + if (next_height != seed_height) { + seed_hash = m_core.get_block_id_by_height(next_height); + res.next_seed_hash = string_tools::pod_to_hex(seed_hash); + } + } store_difficulty(wdiff, res.difficulty, res.wide_difficulty, res.difficulty_top64); blobdata block_blob = t_serializable_object_to_blob(b); crypto::public_key tx_pub_key = cryptonote::get_tx_pub_key_from_extra(b.miner_tx); @@ -1437,7 +1450,7 @@ namespace cryptonote return false; } b.nonce = req.starting_nonce; - miner::find_nonce_for_given_block(b, template_res.difficulty, template_res.height); + miner::find_nonce_for_given_block(&(m_core.get_blockchain_storage()), b, template_res.difficulty, template_res.height); submit_req.front() = string_tools::buff_to_hex_nodelimer(block_to_blob(b)); r = on_submitblock(submit_req, submit_res, error_resp, ctx); @@ -1482,7 +1495,7 @@ namespace cryptonote response.reward = get_block_reward(blk); response.block_size = response.block_weight = m_core.get_blockchain_storage().get_db().get_block_weight(height); response.num_txes = blk.tx_hashes.size(); - response.pow_hash = fill_pow_hash ? string_tools::pod_to_hex(get_block_longhash(blk, height)) : ""; + response.pow_hash = fill_pow_hash ? string_tools::pod_to_hex(get_block_longhash(&(m_core.get_blockchain_storage()), blk, height, 0)) : ""; response.long_term_weight = m_core.get_blockchain_storage().get_db().get_block_long_term_weight(height); response.miner_tx_hash = string_tools::pod_to_hex(cryptonote::get_transaction_hash(blk.miner_tx)); return true; diff --git a/src/rpc/core_rpc_server_commands_defs.h b/src/rpc/core_rpc_server_commands_defs.h index 7eef9e651..90879d394 100644 --- a/src/rpc/core_rpc_server_commands_defs.h +++ b/src/rpc/core_rpc_server_commands_defs.h @@ -84,7 +84,7 @@ namespace cryptonote // advance which version they will stop working with // Don't go over 32767 for any of these #define CORE_RPC_VERSION_MAJOR 2 -#define CORE_RPC_VERSION_MINOR 6 +#define CORE_RPC_VERSION_MINOR 7 #define MAKE_CORE_RPC_VERSION(major,minor) (((major)<<16)|(minor)) #define CORE_RPC_VERSION MAKE_CORE_RPC_VERSION(CORE_RPC_VERSION_MAJOR, CORE_RPC_VERSION_MINOR) @@ -937,6 +937,8 @@ namespace cryptonote uint64_t reserved_offset; uint64_t expected_reward; std::string prev_hash; + std::string seed_hash; + std::string next_seed_hash; blobdata blocktemplate_blob; blobdata blockhashing_blob; std::string status; @@ -954,6 +956,8 @@ namespace cryptonote KV_SERIALIZE(blockhashing_blob) KV_SERIALIZE(status) KV_SERIALIZE(untrusted) + KV_SERIALIZE(seed_hash) + KV_SERIALIZE(next_seed_hash) END_KV_SERIALIZE_MAP() }; typedef epee::misc_utils::struct_init response; diff --git a/tests/core_proxy/core_proxy.cpp b/tests/core_proxy/core_proxy.cpp index 388808269..ed68f04e7 100644 --- a/tests/core_proxy/core_proxy.cpp +++ b/tests/core_proxy/core_proxy.cpp @@ -209,7 +209,7 @@ bool tests::proxy_core::handle_incoming_block(const cryptonote::blobdata& block_ crypto::hash lh; cout << "BLOCK" << endl << endl; cout << (h = get_block_hash(b)) << endl; - cout << (lh = get_block_longhash(b, 0)) << endl; + cout << (lh = get_block_longhash(NULL, b, 0, 0)) << endl; cout << get_transaction_hash(b.miner_tx) << endl; cout << ::get_object_blobsize(b.miner_tx) << endl; //cout << string_tools::buff_to_hex_nodelimer(block_blob) << endl; @@ -236,7 +236,7 @@ void tests::proxy_core::get_blockchain_top(uint64_t& height, crypto::hash& top_i bool tests::proxy_core::init(const boost::program_options::variables_map& /*vm*/) { generate_genesis_block(m_genesis, config::GENESIS_TX, config::GENESIS_NONCE); crypto::hash h = get_block_hash(m_genesis); - add_block(h, get_block_longhash(m_genesis, 0), m_genesis, block_to_blob(m_genesis)); + add_block(h, get_block_longhash(NULL, m_genesis, 0, 0), m_genesis, block_to_blob(m_genesis)); return true; } diff --git a/tests/core_tests/chaingen.cpp b/tests/core_tests/chaingen.cpp index 614585349..44c4c7125 100644 --- a/tests/core_tests/chaingen.cpp +++ b/tests/core_tests/chaingen.cpp @@ -186,7 +186,7 @@ bool test_generator::construct_block(cryptonote::block& blk, uint64_t height, co // Nonce search... blk.nonce = 0; - while (!miner::find_nonce_for_given_block(blk, get_test_difficulty(hf_ver), height)) + while (!miner::find_nonce_for_given_block(NULL, blk, get_test_difficulty(hf_ver), height)) blk.timestamp++; add_block(blk, txs_weight, block_weights, already_generated_coins, hf_ver ? hf_ver.get() : 1); @@ -796,7 +796,7 @@ void fill_tx_sources_and_destinations(const std::vector& event void fill_nonce(cryptonote::block& blk, const difficulty_type& diffic, uint64_t height) { blk.nonce = 0; - while (!miner::find_nonce_for_given_block(blk, diffic, height)) + while (!miner::find_nonce_for_given_block(NULL, blk, diffic, height)) blk.timestamp++; } From a1258baf5a2a318fb67626206e124db8f9ccf8ed Mon Sep 17 00:00:00 2001 From: wowario Date: Fri, 31 May 2019 12:00:30 +0300 Subject: [PATCH 3/6] add RandomWOW --- .gitmodules | 6 +++--- CMakeLists.txt | 2 +- README.md | 2 +- external/CMakeLists.txt | 2 +- external/randomwow | 1 + external/randomx | 1 - src/crypto/rx-slow-hash.c | 10 +++++----- src/cryptonote_core/blockchain.cpp | 2 ++ src/cryptonote_core/cryptonote_tx_utils.cpp | 9 +-------- 9 files changed, 15 insertions(+), 20 deletions(-) create mode 160000 external/randomwow delete mode 160000 external/randomx diff --git a/.gitmodules b/.gitmodules index f8e7c305b..afcb021af 100644 --- a/.gitmodules +++ b/.gitmodules @@ -12,6 +12,6 @@ [submodule "external/trezor-common"] path = external/trezor-common url = https://github.com/trezor/trezor-common.git -[submodule "external/randomx"] - path = external/randomx - url = https://github.com/tevador/RandomX +[submodule "external/randomwow"] + path = external/randomwow + url = https://github.com/wownero/RandomWOW diff --git a/CMakeLists.txt b/CMakeLists.txt index a1bf8dc9a..f17f705b4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -205,7 +205,7 @@ if(NOT MANUAL_SUBMODULES) check_submodule(external/unbound) check_submodule(external/rapidjson) check_submodule(external/trezor-common) - check_submodule(external/randomx) + check_submodule(external/randomwow) endif() endif() diff --git a/README.md b/README.md index 902dc118e..58c0184fc 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ Dates are provided in the format YYYY-MM-DD. | 53666 | 2018-10-06 | Cool Cage | v0.3.0.0 | v0.3.1.3 | Cryptonight variant 2, LWMA v2, ringsize = 22, MMS | 63469 | 2018-11-11 | Dank Doge | v0.4.0.0 | v0.4.0.0 | LWMA v4 | 81769 | 2019-02-19 | Erotic EggplantEmoji | v0.5.0.0 | v0.5.0.2 | Cryptonight/wow, LWMA v1 with N=144, Updated Bulletproofs, Fee Per Byte, Auto-churn -| XXXXX | 2019-XX-XX | F For Fappening | v0.6.0.0 | v0.6.0.0 | New PoW based on RandomX, new block weight algorithm, slightly more efficient RingCT format +| 114969 | 2019-06-14 | F For Fappening | v0.6.0.0 | v0.6.0.0 | RandomWOW, new block weight algorithm, slightly more efficient RingCT format X's indicate that these details have not been determined as of commit date. diff --git a/external/CMakeLists.txt b/external/CMakeLists.txt index 436f29ced..e61577bd0 100644 --- a/external/CMakeLists.txt +++ b/external/CMakeLists.txt @@ -100,4 +100,4 @@ endif() add_subdirectory(db_drivers) add_subdirectory(easylogging++) -add_subdirectory(randomx) +add_subdirectory(randomwow) diff --git a/external/randomwow b/external/randomwow new file mode 160000 index 000000000..435a8382d --- /dev/null +++ b/external/randomwow @@ -0,0 +1 @@ +Subproject commit 435a8382d88cf81a85af680ff00dde7644ad4478 diff --git a/external/randomx b/external/randomx deleted file mode 160000 index 1276d67d2..000000000 --- a/external/randomx +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 1276d67d2f594ea4a8e9cde28253b1a74769aeb9 diff --git a/src/crypto/rx-slow-hash.c b/src/crypto/rx-slow-hash.c index 3982ddcea..f4c068294 100644 --- a/src/crypto/rx-slow-hash.c +++ b/src/crypto/rx-slow-hash.c @@ -209,11 +209,11 @@ static void rx_initdata(randomx_cache *rs_cache, const int miners) { CTHR_THREAD_TYPE *st; si = malloc(miners * sizeof(seedinfo)); if (si == NULL) - local_abort("Couldn't allocate RandomX mining threadinfo"); + local_abort("Couldn't allocate RandomWOW mining threadinfo"); st = malloc(miners * sizeof(CTHR_THREAD_TYPE)); if (st == NULL) { free(si); - local_abort("Couldn't allocate RandomX mining threadlist"); + local_abort("Couldn't allocate RandomWOW mining threadlist"); } for (i=0; irs_cache, NULL); } if (rx_vm == NULL) - local_abort("Couldn't allocate RandomX VM"); + local_abort("Couldn't allocate RandomWOW VM"); } randomx_calculate_hash(rx_vm, data, length, hash); } @@ -333,7 +333,7 @@ void rx_slow_hash(const void *data, size_t length, char *hash, int miners) { rx_vm = randomx_create_vm(flags, rx_sp->rs_cache, rx_dataset); } if (rx_vm == NULL) - local_abort("Couldn't allocate RandomX VM"); + local_abort("Couldn't allocate RandomWOW VM"); } randomx_calculate_hash(rx_vm, data, length, hash); } diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index e66e11fe0..c202aab94 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -95,6 +95,8 @@ static const struct { { 10, 63469, 0, 1541700352 }, { 11, 81769, 0, 1549238400 }, { 12, 82069, 0, 1549318761 }, + { 13, 114969, 0, 1559292691 }, + { 14, 115257, 0, 1559292774 }, }; static const struct { diff --git a/src/cryptonote_core/cryptonote_tx_utils.cpp b/src/cryptonote_core/cryptonote_tx_utils.cpp index bea24c74f..e956be9d2 100644 --- a/src/cryptonote_core/cryptonote_tx_utils.cpp +++ b/src/cryptonote_core/cryptonote_tx_utils.cpp @@ -674,15 +674,8 @@ namespace cryptonote bool get_block_longhash(const Blockchain *pbc, const block& b, crypto::hash& res, const uint64_t height, const int miners) { - // block 202612 bug workaround - if (height == 202612) - { - static const std::string longhash_202612 = "84f64766475d51837ac9efbef1926486e58563c95a19fef4aec3254f03000000"; - epee::string_tools::hex_to_pod(longhash_202612, res); - return true; - } blobdata bd = get_block_hashing_blob(b); - const int pow_variant = b.major_version >= 7 ? b.major_version - 6 : 0; + const int pow_variant = b.major_version >= 13 ? 6 : b.major_version >= 11 && b.major_version <= 12 ? 4 : 2; if (pow_variant >= 6) { uint64_t seed_height; if (rx_needhash(height, &seed_height)) { From 3b9a06d5aa8c70839419c019f34d51b58b9c3a60 Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Sat, 1 Jun 2019 16:23:22 +0100 Subject: [PATCH 4/6] Proposed tweak for issue with finding seedblock hash This patch isn't needed if we always restrict block-sync-size to <= SEEDHASH_EPOCH_LAG. But otherwise, this will allow syncing with larger block-sync-sizes. --- src/cryptonote_core/blockchain.cpp | 15 ++++++++++++++- src/cryptonote_core/blockchain.h | 17 +++++++++++++++++ src/cryptonote_core/cryptonote_tx_utils.cpp | 2 +- 3 files changed, 32 insertions(+), 2 deletions(-) diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index c202aab94..b2e872be5 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -139,7 +139,9 @@ Blockchain::Blockchain(tx_memory_pool& tx_pool) : m_long_term_block_weights_cache_rolling_median(CRYPTONOTE_LONG_TERM_BLOCK_WEIGHT_WINDOW_SIZE), m_difficulty_for_next_block_top_hash(crypto::null_hash), m_difficulty_for_next_block(1), - m_btc_valid(false) + m_btc_valid(false), + m_batch_success(true), + m_prepare_height(0) { LOG_PRINT_L3("Blockchain::" << __func__); } @@ -798,6 +800,13 @@ crypto::hash Blockchain::get_block_id_by_height(uint64_t height) const return null_hash; } //------------------------------------------------------------------ +crypto::hash Blockchain::get_pending_block_id_by_height(uint64_t height) const +{ + if (m_prepare_height && height >= m_prepare_height && height - m_prepare_height < m_prepare_nblocks) + return (*m_prepare_blocks)[height - m_prepare_height].hash; + return get_block_id_by_height(height); +} +//------------------------------------------------------------------ bool Blockchain::get_block_by_hash(const crypto::hash &h, block &blk, bool *orphan) const { LOG_PRINT_L3("Blockchain::" << __func__); @@ -4534,6 +4543,9 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::vector m_block_notify; std::shared_ptr m_reorg_notify; + // for prepare_handle_incoming_blocks + uint64_t m_prepare_height; + uint64_t m_prepare_nblocks; + std::vector *m_prepare_blocks; + /** * @brief collects the keys for all outputs being "spent" as an input * diff --git a/src/cryptonote_core/cryptonote_tx_utils.cpp b/src/cryptonote_core/cryptonote_tx_utils.cpp index e956be9d2..8c4903a06 100644 --- a/src/cryptonote_core/cryptonote_tx_utils.cpp +++ b/src/cryptonote_core/cryptonote_tx_utils.cpp @@ -681,7 +681,7 @@ namespace cryptonote if (rx_needhash(height, &seed_height)) { crypto::hash hash; if (pbc != NULL) - hash = pbc->get_block_id_by_height(seed_height); + hash = pbc->get_pending_block_id_by_height(seed_height); else memset(&hash, 0, sizeof(hash)); // only happens when generating genesis block rx_seedhash(seed_height, hash.data, miners); From c7b73d31a55944907d19f736ceb0051a3b24f5e8 Mon Sep 17 00:00:00 2001 From: wowario Date: Sun, 2 Jun 2019 09:10:16 +0300 Subject: [PATCH 5/6] update rpc mining_status --- src/rpc/core_rpc_server.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index d48b634d2..c8bcc2870 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -1005,14 +1005,14 @@ namespace cryptonote const account_public_address& lMiningAdr = lMiner.get_mining_address(); res.address = get_account_address_as_str(nettype(), false, lMiningAdr); const uint8_t major_version = m_core.get_blockchain_storage().get_current_hard_fork_version(); - const unsigned variant = major_version >= 7 ? major_version - 6 : 0; + const unsigned variant = major_version >= 13 ? 6 : major_version >= 11 && major_version <= 12 ? 4 : 2; switch (variant) { case 0: res.pow_algorithm = "Cryptonight"; break; case 1: res.pow_algorithm = "CNv1 (Cryptonight variant 1)"; break; case 2: case 3: res.pow_algorithm = "CNv2 (Cryptonight variant 2)"; break; - case 4: case 5: res.pow_algorithm = "CNv4 (Cryptonight variant 4)"; break; - case 6: res.pow_algorithm = "RandomX"; break; + case 4: case 5: res.pow_algorithm = "CN/WOW"; break; + case 6: res.pow_algorithm = "RandomWOW"; break; default: res.pow_algorithm = "I'm not sure actually"; break; } if (res.is_background_mining_enabled) From 496c4babb68bcfbf9301562ca2d8753ed8c53543 Mon Sep 17 00:00:00 2001 From: wowario Date: Sun, 2 Jun 2019 09:47:33 +0300 Subject: [PATCH 6/6] update block synchronizing count fork height --- src/cryptonote_core/cryptonote_core.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cryptonote_core/cryptonote_core.cpp b/src/cryptonote_core/cryptonote_core.cpp index 0afd40c24..6d158bf23 100644 --- a/src/cryptonote_core/cryptonote_core.cpp +++ b/src/cryptonote_core/cryptonote_core.cpp @@ -1167,7 +1167,7 @@ namespace cryptonote //----------------------------------------------------------------------------------------------- size_t core::get_block_sync_size(uint64_t height) const { - static const uint64_t quick_height = m_nettype == TESTNET ? 801219 : m_nettype == MAINNET ? 1220516 : 0; + static const uint64_t quick_height = m_nettype == TESTNET ? 801219 : m_nettype == MAINNET ? 53666 : 0; if (block_sync_size > 0) return block_sync_size; if (height >= quick_height)