From b8787f430262520a9e81267802b869b1a7cde245 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Mon, 21 Jan 2019 17:18:50 +0000 Subject: [PATCH] ArticMine's new block weight algorithm This curbs runaway growth while still allowing substantial spikes in block weight Original specification from ArticMine: here is the scaling proposal Define: LongTermBlockWeight Before fork: LongTermBlockWeight = BlockWeight At or after fork: LongTermBlockWeight = min(BlockWeight, 1.4*LongTermEffectiveMedianBlockWeight) Note: To avoid possible consensus issues over rounding the LongTermBlockWeight for a given block should be calculated to the nearest byte, and stored as a integer in the block itself. The stored LongTermBlockWeight is then used for future calculations of the LongTermEffectiveMedianBlockWeight and not recalculated each time. Define: LongTermEffectiveMedianBlockWeight LongTermEffectiveMedianBlockWeight = max(300000, MedianOverPrevious100000Blocks(LongTermBlockWeight)) Change Definition of EffectiveMedianBlockWeight From (current definition) EffectiveMedianBlockWeight = max(300000, MedianOverPrevious100Blocks(BlockWeight)) To (proposed definition) EffectiveMedianBlockWeight = min(max(300000, MedianOverPrevious100Blocks(BlockWeight)), 50*LongTermEffectiveMedianBlockWeight) Notes: 1) There are no other changes to the existing penalty formula, median calculation, fees etc. 2) There is the requirement to store the LongTermBlockWeight of a block unencrypted in the block itself. This is to avoid possible consensus issues over rounding and also to prevent the calculations from becoming unwieldy as we move away from the fork. 3) When the EffectiveMedianBlockWeight cap is reached it is still possible to mine blocks up to 2x the EffectiveMedianBlockWeight by paying the corresponding penalty. Note: the long term block weight is stored in the database, but not in the actual block itself, since it requires recalculating anyway for verification. --- src/blockchain_db/blockchain_db.cpp | 3 +- src/blockchain_db/blockchain_db.h | 17 +- src/blockchain_db/lmdb/db_lmdb.cpp | 213 +++++++++- src/blockchain_db/lmdb/db_lmdb.h | 7 + .../unit_tests => src/blockchain_db}/testdb.h | 9 +- .../blockchain_import.cpp | 3 +- src/cryptonote_config.h | 3 + src/cryptonote_core/blockchain.cpp | 139 ++++++- src/cryptonote_core/blockchain.h | 18 +- src/cryptonote_core/cryptonote_core.cpp | 3 +- src/cryptonote_core/cryptonote_core.h | 1 + src/daemon/rpc_command_executor.cpp | 1 + src/rpc/core_rpc_server.cpp | 1 + src/rpc/core_rpc_server_commands_defs.h | 2 + tests/CMakeLists.txt | 2 + tests/block_weight/CMakeLists.txt | 45 ++ tests/block_weight/block_weight.cpp | 183 +++++++++ tests/block_weight/block_weight.py | 74 ++++ tests/block_weight/compare.py | 13 + tests/core_tests/v2_tests.h | 2 +- tests/unit_tests/CMakeLists.txt | 1 + tests/unit_tests/blockchain_db.cpp | 12 +- tests/unit_tests/hardfork.cpp | 41 +- tests/unit_tests/long_term_block_weight.cpp | 384 ++++++++++++++++++ tests/unit_tests/output_distribution.cpp | 4 +- 25 files changed, 1117 insertions(+), 64 deletions(-) rename {tests/unit_tests => src/blockchain_db}/testdb.h (98%) create mode 100644 tests/block_weight/CMakeLists.txt create mode 100644 tests/block_weight/block_weight.cpp create mode 100755 tests/block_weight/block_weight.py create mode 100755 tests/block_weight/compare.py create mode 100644 tests/unit_tests/long_term_block_weight.cpp diff --git a/src/blockchain_db/blockchain_db.cpp b/src/blockchain_db/blockchain_db.cpp index d3eefef6e..041759593 100644 --- a/src/blockchain_db/blockchain_db.cpp +++ b/src/blockchain_db/blockchain_db.cpp @@ -197,6 +197,7 @@ void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const transacti uint64_t BlockchainDB::add_block( const block& blk , size_t block_weight + , uint64_t long_term_block_weight , const difficulty_type& cumulative_difficulty , const uint64_t& coins_generated , const std::vector& txs @@ -241,7 +242,7 @@ uint64_t BlockchainDB::add_block( const block& blk // call out to subclass implementation to add the block & metadata time1 = epee::misc_utils::get_tick_count(); - add_block(blk, block_weight, cumulative_difficulty, coins_generated, num_rct_outs, blk_hash); + add_block(blk, block_weight, long_term_block_weight, cumulative_difficulty, coins_generated, num_rct_outs, blk_hash); TIME_MEASURE_FINISH(time1); time_add_block1 += time1; diff --git a/src/blockchain_db/blockchain_db.h b/src/blockchain_db/blockchain_db.h index 137d5958a..c3f11ba28 100644 --- a/src/blockchain_db/blockchain_db.h +++ b/src/blockchain_db/blockchain_db.h @@ -358,12 +358,14 @@ private: * * @param blk the block to be added * @param block_weight the weight of the block (transactions and all) + * @param long_term_block_weight the long term block weight of the block (transactions and all) * @param cumulative_difficulty the accumulated difficulty after this block * @param coins_generated the number of coins generated total after this block * @param blk_hash the hash of the block */ virtual void add_block( const block& blk , size_t block_weight + , uint64_t long_term_block_weight , const difficulty_type& cumulative_difficulty , const uint64_t& coins_generated , uint64_t num_rct_outs @@ -375,7 +377,7 @@ private: * * The subclass implementing this will remove the block data from the top * block in the chain. The data to be removed is that which was added in - * BlockchainDB::add_block(const block& blk, size_t block_weight, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, const crypto::hash& blk_hash) + * BlockchainDB::add_block(const block& blk, size_t block_weight, uint64_t long_term_block_weight, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, const crypto::hash& blk_hash) * * If any of this cannot be done, the subclass should throw the corresponding * subclass of DB_EXCEPTION @@ -789,6 +791,7 @@ public: * * @param blk the block to be added * @param block_weight the size of the block (transactions and all) + * @param long_term_block_weight the long term weight of the block (transactions and all) * @param cumulative_difficulty the accumulated difficulty after this block * @param coins_generated the number of coins generated total after this block * @param txs the transactions in the block @@ -797,6 +800,7 @@ public: */ virtual uint64_t add_block( const block& blk , size_t block_weight + , uint64_t long_term_block_weight , const difficulty_type& cumulative_difficulty , const uint64_t& coins_generated , const std::vector& txs @@ -984,6 +988,17 @@ public: */ virtual uint64_t get_block_already_generated_coins(const uint64_t& height) const = 0; + /** + * @brief fetch a block's long term weight + * + * If the block does not exist, the subclass should throw BLOCK_DNE + * + * @param height the height requested + * + * @return the long term weight + */ + virtual uint64_t get_block_long_term_weight(const uint64_t& height) const = 0; + /** * @brief fetch a block's hash * diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index 8a1303be9..00912a447 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -29,6 +29,7 @@ #include #include +#include #include // std::unique_ptr #include // memcpy @@ -53,7 +54,7 @@ using epee::string_tools::pod_to_hex; using namespace crypto; // Increase when the DB structure changes -#define VERSION 3 +#define VERSION 4 namespace { @@ -277,7 +278,7 @@ typedef struct mdb_block_info_old crypto::hash bi_hash; } mdb_block_info_old; -typedef struct mdb_block_info +typedef struct mdb_block_info_2 { uint64_t bi_height; uint64_t bi_timestamp; @@ -286,7 +287,21 @@ typedef struct mdb_block_info difficulty_type bi_diff; crypto::hash bi_hash; uint64_t bi_cum_rct; -} mdb_block_info; +} mdb_block_info_2; + +typedef struct mdb_block_info_3 +{ + uint64_t bi_height; + uint64_t bi_timestamp; + uint64_t bi_coins; + uint64_t bi_weight; // a size_t really but we need 32-bit compat + difficulty_type bi_diff; + crypto::hash bi_hash; + uint64_t bi_cum_rct; + uint64_t bi_long_term_block_weight; +} mdb_block_info_3; + +typedef mdb_block_info_3 mdb_block_info; typedef struct blk_height { crypto::hash bh_hash; @@ -694,7 +709,7 @@ estim: return threshold_size; } -void BlockchainLMDB::add_block(const block& blk, size_t block_weight, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, +void BlockchainLMDB::add_block(const block& blk, size_t block_weight, uint64_t long_term_block_weight, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, uint64_t num_rct_outs, const crypto::hash& blk_hash) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); @@ -754,6 +769,7 @@ void BlockchainLMDB::add_block(const block& blk, size_t block_weight, const diff const mdb_block_info *bi_prev = (const mdb_block_info*)h.mv_data; bi.bi_cum_rct += bi_prev->bi_cum_rct; } + bi.bi_long_term_block_weight = long_term_block_weight; MDB_val_set(val, bi); result = mdb_cursor_put(m_cur_block_info, (MDB_val *)&zerokval, &val, MDB_APPENDDUP); @@ -2486,6 +2502,29 @@ uint64_t BlockchainLMDB::get_block_already_generated_coins(const uint64_t& heigh return ret; } +uint64_t BlockchainLMDB::get_block_long_term_weight(const uint64_t& height) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + TXN_PREFIX_RDONLY(); + RCURSOR(block_info); + + MDB_val_set(result, height); + auto get_result = mdb_cursor_get(m_cur_block_info, (MDB_val *)&zerokval, &result, MDB_GET_BOTH); + if (get_result == MDB_NOTFOUND) + { + throw0(BLOCK_DNE(std::string("Attempt to get block long term weight from height ").append(boost::lexical_cast(height)).append(" failed -- block info not in db").c_str())); + } + else if (get_result) + throw0(DB_ERROR("Error attempting to retrieve a long term block weight from the db")); + + mdb_block_info *bi = (mdb_block_info *)result.mv_data; + uint64_t ret = bi->bi_long_term_block_weight; + TXN_POSTFIX_RDONLY(); + return ret; +} + crypto::hash BlockchainLMDB::get_block_hash_from_height(const uint64_t& height) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); @@ -3524,7 +3563,7 @@ void BlockchainLMDB::block_txn_abort() } } -uint64_t BlockchainLMDB::add_block(const block& blk, size_t block_weight, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, +uint64_t BlockchainLMDB::add_block(const block& blk, size_t block_weight, uint64_t long_term_block_weight, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, const std::vector& txs) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); @@ -3543,7 +3582,7 @@ uint64_t BlockchainLMDB::add_block(const block& blk, size_t block_weight, const try { - BlockchainDB::add_block(blk, block_weight, cumulative_difficulty, coins_generated, txs); + BlockchainDB::add_block(blk, block_weight, long_term_block_weight, cumulative_difficulty, coins_generated, txs); } catch (const DB_ERROR_TXN_START& e) { @@ -4768,6 +4807,166 @@ void BlockchainLMDB::migrate_2_3() txn.commit(); } +void BlockchainLMDB::migrate_3_4() +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + uint64_t i; + int result; + mdb_txn_safe txn(false); + MDB_val k, v; + char *ptr; + bool past_long_term_weight = false; + + MGINFO_YELLOW("Migrating blockchain from DB version 3 to 4 - this may take a while:"); + + do { + LOG_PRINT_L1("migrating block info:"); + + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + + MDB_stat db_stats; + if ((result = mdb_stat(txn, m_blocks, &db_stats))) + throw0(DB_ERROR(lmdb_error("Failed to query m_blocks: ", result).c_str())); + const uint64_t blockchain_height = db_stats.ms_entries; + + boost::circular_buffer long_term_block_weights(CRYPTONOTE_LONG_TERM_BLOCK_WEIGHT_WINDOW_SIZE); + + /* the block_info table name is the same but the old version and new version + * have incompatible data. Create a new table. We want the name to be similar + * to the old name so that it will occupy the same location in the DB. + */ + MDB_dbi o_block_info = m_block_info; + lmdb_db_open(txn, "block_infn", MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_block_info, "Failed to open db handle for block_infn"); + mdb_set_dupsort(txn, m_block_info, compare_uint64); + + + MDB_cursor *c_blocks; + result = mdb_cursor_open(txn, m_blocks, &c_blocks); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for blocks: ", result).c_str())); + + MDB_cursor *c_old, *c_cur; + i = 0; + while(1) { + if (!(i % 1000)) { + if (i) { + LOGIF(el::Level::Info) { + std::cout << i << " / " << blockchain_height << " \r" << std::flush; + } + txn.commit(); + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + } + result = mdb_cursor_open(txn, m_block_info, &c_cur); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for block_infn: ", result).c_str())); + result = mdb_cursor_open(txn, o_block_info, &c_old); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for block_info: ", result).c_str())); + result = mdb_cursor_open(txn, m_blocks, &c_blocks); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for blocks: ", result).c_str())); + if (!i) { + MDB_stat db_stat; + result = mdb_stat(txn, m_block_info, &db_stats); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to query m_block_info: ", result).c_str())); + i = db_stats.ms_entries; + } + } + result = mdb_cursor_get(c_old, &k, &v, MDB_NEXT); + if (result == MDB_NOTFOUND) { + txn.commit(); + break; + } + else if (result) + throw0(DB_ERROR(lmdb_error("Failed to get a record from block_info: ", result).c_str())); + const mdb_block_info_2 *bi_old = (const mdb_block_info_2*)v.mv_data; + mdb_block_info_3 bi; + bi.bi_height = bi_old->bi_height; + bi.bi_timestamp = bi_old->bi_timestamp; + bi.bi_coins = bi_old->bi_coins; + bi.bi_weight = bi_old->bi_weight; + bi.bi_diff = bi_old->bi_diff; + bi.bi_hash = bi_old->bi_hash; + bi.bi_cum_rct = bi_old->bi_cum_rct; + + // get block major version to determine which rule is in place + if (!past_long_term_weight) + { + MDB_val_copy kb(bi.bi_height); + MDB_val vb; + result = mdb_cursor_get(c_blocks, &kb, &vb, MDB_SET); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to query m_blocks: ", result).c_str())); + if (vb.mv_size == 0) + throw0(DB_ERROR("Invalid data from m_blocks")); + const uint8_t block_major_version = *((const uint8_t*)vb.mv_data); + if (block_major_version >= HF_VERSION_LONG_TERM_BLOCK_WEIGHT) + past_long_term_weight = true; + } + + uint64_t long_term_block_weight; + if (past_long_term_weight) + { + std::vector weights(long_term_block_weights.begin(), long_term_block_weights.end()); + uint64_t long_term_effective_block_median_weight = std::max(CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5, epee::misc_utils::median(weights)); + long_term_block_weight = std::min(bi.bi_weight, long_term_effective_block_median_weight + long_term_effective_block_median_weight * 2 / 5); + } + else + { + long_term_block_weight = bi.bi_weight; + } + long_term_block_weights.push_back(long_term_block_weight); + bi.bi_long_term_block_weight = long_term_block_weight; + + MDB_val_set(nv, bi); + result = mdb_cursor_put(c_cur, (MDB_val *)&zerokval, &nv, MDB_APPENDDUP); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to put a record into block_infn: ", result).c_str())); + /* we delete the old records immediately, so the overall DB and mapsize should not grow. + * This is a little slower than just letting mdb_drop() delete it all at the end, but + * it saves a significant amount of disk space. + */ + result = mdb_cursor_del(c_old, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete a record from block_info: ", result).c_str())); + i++; + } + + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + /* Delete the old table */ + result = mdb_drop(txn, o_block_info, 1); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete old block_info table: ", result).c_str())); + + RENAME_DB("block_infn"); + mdb_dbi_close(m_env, m_block_info); + + lmdb_db_open(txn, "block_info", MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_block_info, "Failed to open db handle for block_infn"); + mdb_set_dupsort(txn, m_block_info, compare_uint64); + + txn.commit(); + } while(0); + + uint32_t version = 4; + v.mv_data = (void *)&version; + v.mv_size = sizeof(version); + MDB_val_str(vk, "version"); + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + result = mdb_put(txn, m_properties, &vk, &v, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to update version for the db: ", result).c_str())); + txn.commit(); +} + void BlockchainLMDB::migrate(const uint32_t oldversion) { if (oldversion < 1) @@ -4776,6 +4975,8 @@ void BlockchainLMDB::migrate(const uint32_t oldversion) migrate_1_2(); if (oldversion < 3) migrate_2_3(); + if (oldversion < 4) + migrate_3_4(); } } // namespace cryptonote diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h index c07ab8da5..5764f9ae4 100644 --- a/src/blockchain_db/lmdb/db_lmdb.h +++ b/src/blockchain_db/lmdb/db_lmdb.h @@ -225,6 +225,8 @@ public: virtual uint64_t get_block_already_generated_coins(const uint64_t& height) const; + virtual uint64_t get_block_long_term_weight(const uint64_t& height) const; + virtual crypto::hash get_block_hash_from_height(const uint64_t& height) const; virtual std::vector get_blocks_range(const uint64_t& h1, const uint64_t& h2) const; @@ -292,6 +294,7 @@ public: virtual uint64_t add_block( const block& blk , size_t block_weight + , uint64_t long_term_block_weight , const difficulty_type& cumulative_difficulty , const uint64_t& coins_generated , const std::vector& txs @@ -341,6 +344,7 @@ private: virtual void add_block( const block& blk , size_t block_weight + , uint64_t long_term_block_weight , const difficulty_type& cumulative_difficulty , const uint64_t& coins_generated , uint64_t num_rct_outs @@ -405,6 +409,9 @@ private: // migrate from DB version 2 to 3 void migrate_2_3(); + // migrate from DB version 3 to 4 + void migrate_3_4(); + void cleanup_batch(); private: diff --git a/tests/unit_tests/testdb.h b/src/blockchain_db/testdb.h similarity index 98% rename from tests/unit_tests/testdb.h rename to src/blockchain_db/testdb.h index 8f5cf70e8..35dfbe673 100644 --- a/tests/unit_tests/testdb.h +++ b/src/blockchain_db/testdb.h @@ -33,9 +33,11 @@ #include #include #include -#include "gtest/gtest.h" -#include "blockchain_db/blockchain_db.h" +#include "blockchain_db.h" + +namespace cryptonote +{ class BaseTestDB: public cryptonote::BlockchainDB { public: @@ -73,6 +75,7 @@ public: virtual cryptonote::difficulty_type get_block_cumulative_difficulty(const uint64_t& height) const { return 10; } virtual cryptonote::difficulty_type get_block_difficulty(const uint64_t& height) const { return 0; } virtual uint64_t get_block_already_generated_coins(const uint64_t& height) const { return 10000000000; } + virtual uint64_t get_block_long_term_weight(const uint64_t& height) const { return 128; } virtual crypto::hash get_block_hash_from_height(const uint64_t& height) const { return crypto::hash(); } virtual std::vector get_blocks_range(const uint64_t& h1, const uint64_t& h2) const { return std::vector(); } virtual std::vector get_hashes_range(const uint64_t& h1, const uint64_t& h2) const { return std::vector(); } @@ -128,6 +131,7 @@ public: virtual void add_block( const cryptonote::block& blk , size_t block_weight + , uint64_t long_term_block_weight , const cryptonote::difficulty_type& cumulative_difficulty , const uint64_t& coins_generated , uint64_t num_rct_outs @@ -145,3 +149,4 @@ public: virtual void prune_outputs(uint64_t amount) {} }; +} diff --git a/src/blockchain_utilities/blockchain_import.cpp b/src/blockchain_utilities/blockchain_import.cpp index 2a8a6f494..e6ec20c3b 100644 --- a/src/blockchain_utilities/blockchain_import.cpp +++ b/src/blockchain_utilities/blockchain_import.cpp @@ -485,7 +485,8 @@ int import_from_file(cryptonote::core& core, const std::string& import_file_path try { - core.get_blockchain_storage().get_db().add_block(b, block_weight, cumulative_difficulty, coins_generated, txs); + uint64_t long_term_block_weight = core.get_blockchain_storage().get_next_long_term_block_weight(block_weight); + core.get_blockchain_storage().get_db().add_block(b, block_weight, long_term_block_weight, cumulative_difficulty, coins_generated, txs); } catch (const std::exception& e) { diff --git a/src/cryptonote_config.h b/src/cryptonote_config.h index 93db71705..d6edb23c2 100644 --- a/src/cryptonote_config.h +++ b/src/cryptonote_config.h @@ -59,6 +59,8 @@ #define CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V2 60000 //size of block (bytes) after which reward for block calculated using block size #define CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V1 20000 //size of block (bytes) after which reward for block calculated using block size - before first fork #define CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5 300000 //size of block (bytes) after which reward for block calculated using block size - second change, from v5 +#define CRYPTONOTE_LONG_TERM_BLOCK_WEIGHT_WINDOW_SIZE 100000 // size in blocks of the long term block weight median window +#define CRYPTONOTE_SHORT_TERM_BLOCK_WEIGHT_SURGE_FACTOR 50 #define CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE 600 #define CRYPTONOTE_DISPLAY_DECIMAL_POINT 12 // COIN - number of smallest units in one coin @@ -143,6 +145,7 @@ #define HF_VERSION_ENFORCE_RCT 6 #define HF_VERSION_PER_BYTE_FEE 8 #define HF_VERSION_SMALLER_BP 10 +#define HF_VERSION_LONG_TERM_BLOCK_WEIGHT 10 #define PER_KB_FEE_QUANTIZATION_DECIMALS 8 diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index 41357e72e..8b295e304 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -166,6 +166,8 @@ static const struct { Blockchain::Blockchain(tx_memory_pool& tx_pool) : m_db(), m_tx_pool(tx_pool), m_hardfork(NULL), m_timestamps_and_difficulties_height(0), m_current_block_cumul_weight_limit(0), m_current_block_cumul_weight_median(0), m_enforce_dns_checkpoints(false), m_max_prepare_blocks_threads(4), m_db_sync_on_blocks(true), m_db_sync_threshold(1), m_db_sync_mode(db_async), m_db_default_sync(false), m_fast_sync(true), m_show_time_stats(false), m_sync_counter(0), m_bytes_to_sync(0), m_cancel(false), + m_long_term_block_weights_window(CRYPTONOTE_LONG_TERM_BLOCK_WEIGHT_WINDOW_SIZE), + m_long_term_effective_median_block_weight(0), m_difficulty_for_next_block_top_hash(crypto::null_hash), m_difficulty_for_next_block(1), m_btc_valid(false) @@ -500,7 +502,11 @@ bool Blockchain::init(BlockchainDB* db, const network_type nettype, bool offline m_tx_pool.on_blockchain_dec(m_db->height()-1, get_tail_id()); } - update_next_cumulative_weight_limit(); + if (test_options && test_options->long_term_block_weight_window) + m_long_term_block_weights_window = test_options->long_term_block_weight_window; + + if (!update_next_cumulative_weight_limit()) + return false; return true; } //------------------------------------------------------------------ @@ -685,7 +691,7 @@ block Blockchain::pop_block_from_blockchain() m_blocks_txs_check.clear(); m_check_txin_table.clear(); - update_next_cumulative_weight_limit(); + CHECK_AND_ASSERT_THROW_MES(update_next_cumulative_weight_limit(), "Error updating next cumulative weight limit"); m_tx_pool.on_blockchain_dec(m_db->height()-1, get_tail_id()); invalidate_block_template_cache(); @@ -704,7 +710,8 @@ bool Blockchain::reset_and_set_genesis_block(const block& b) block_verification_context bvc = boost::value_initialized(); add_new_block(b, bvc); - update_next_cumulative_weight_limit(); + if (!update_next_cumulative_weight_limit()) + return false; return bvc.m_added_to_main_chain && !bvc.m_verifivation_failed; } //------------------------------------------------------------------ @@ -1202,7 +1209,7 @@ bool Blockchain::validate_miner_transaction(const block& b, size_t cumulative_bl } } - std::vector last_blocks_weights; + std::vector last_blocks_weights; get_last_n_blocks_weights(last_blocks_weights, CRYPTONOTE_REWARD_BLOCKS_WINDOW); if (!get_block_reward(epee::misc_utils::median(last_blocks_weights), cumulative_block_weight, already_generated_coins, base_reward, version)) { @@ -1237,7 +1244,7 @@ bool Blockchain::validate_miner_transaction(const block& b, size_t cumulative_bl } //------------------------------------------------------------------ // get the block weights of the last blocks, and return by reference . -void Blockchain::get_last_n_blocks_weights(std::vector& weights, size_t count) const +void Blockchain::get_last_n_blocks_weights(std::vector& weights, size_t count) const { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -3083,6 +3090,7 @@ uint64_t Blockchain::get_dynamic_base_fee(uint64_t block_reward, size_t median_b bool Blockchain::check_fee(size_t tx_weight, uint64_t fee) const { const uint8_t version = get_current_hard_fork_version(); + const uint64_t blockchain_height = m_db->height(); uint64_t median = 0; uint64_t already_generated_coins = 0; @@ -3090,7 +3098,7 @@ bool Blockchain::check_fee(size_t tx_weight, uint64_t fee) const if (version >= HF_VERSION_DYNAMIC_FEE) { median = m_current_block_cumul_weight_limit / 2; - already_generated_coins = m_db->height() ? m_db->get_block_already_generated_coins(m_db->height() - 1) : 0; + already_generated_coins = blockchain_height ? m_db->get_block_already_generated_coins(blockchain_height - 1) : 0; if (!get_block_reward(median, 1, already_generated_coins, base_reward, version)) return false; } @@ -3098,7 +3106,8 @@ bool Blockchain::check_fee(size_t tx_weight, uint64_t fee) const uint64_t needed_fee; if (version >= HF_VERSION_PER_BYTE_FEE) { - uint64_t fee_per_byte = get_dynamic_base_fee(base_reward, median, version); + const bool use_long_term_median_in_fee = version >= HF_VERSION_LONG_TERM_BLOCK_WEIGHT; + uint64_t fee_per_byte = get_dynamic_base_fee(base_reward, use_long_term_median_in_fee ? m_long_term_effective_median_block_weight : median, version); MDEBUG("Using " << print_money(fee_per_byte) << "/byte fee"); needed_fee = tx_weight * fee_per_byte; // quantize fee up to 8 decimals @@ -3135,6 +3144,7 @@ bool Blockchain::check_fee(size_t tx_weight, uint64_t fee) const uint64_t Blockchain::get_dynamic_base_fee_estimate(uint64_t grace_blocks) const { const uint8_t version = get_current_hard_fork_version(); + const uint64_t db_height = m_db->height(); if (version < HF_VERSION_DYNAMIC_FEE) return FEE_PER_KB; @@ -3143,7 +3153,7 @@ uint64_t Blockchain::get_dynamic_base_fee_estimate(uint64_t grace_blocks) const grace_blocks = CRYPTONOTE_REWARD_BLOCKS_WINDOW - 1; const uint64_t min_block_weight = get_min_block_weight(version); - std::vector weights; + std::vector weights; get_last_n_blocks_weights(weights, CRYPTONOTE_REWARD_BLOCKS_WINDOW - grace_blocks); weights.reserve(grace_blocks); for (size_t i = 0; i < grace_blocks; ++i) @@ -3153,7 +3163,7 @@ uint64_t Blockchain::get_dynamic_base_fee_estimate(uint64_t grace_blocks) const if(median <= min_block_weight) median = min_block_weight; - uint64_t already_generated_coins = m_db->height() ? m_db->get_block_already_generated_coins(m_db->height() - 1) : 0; + uint64_t already_generated_coins = db_height ? m_db->get_block_already_generated_coins(db_height - 1) : 0; uint64_t base_reward; if (!get_block_reward(median, 1, already_generated_coins, base_reward, version)) { @@ -3161,7 +3171,8 @@ uint64_t Blockchain::get_dynamic_base_fee_estimate(uint64_t grace_blocks) const base_reward = BLOCK_REWARD_OVERESTIMATE; } - uint64_t fee = get_dynamic_base_fee(base_reward, median, version); + const bool use_long_term_median_in_fee = version >= HF_VERSION_LONG_TERM_BLOCK_WEIGHT; + uint64_t fee = get_dynamic_base_fee(base_reward, use_long_term_median_in_fee ? m_long_term_effective_median_block_weight : median, version); const bool per_byte = version < HF_VERSION_PER_BYTE_FEE; MDEBUG("Estimating " << grace_blocks << "-block fee at " << print_money(fee) << "/" << (per_byte ? "byte" : "kB")); return fee; @@ -3658,7 +3669,8 @@ leave: { try { - new_height = m_db->add_block(bl, block_weight, cumulative_difficulty, already_generated_coins, txs); + uint64_t long_term_block_weight = get_next_long_term_block_weight(block_weight); + new_height = m_db->add_block(bl, block_weight, long_term_block_weight, cumulative_difficulty, already_generated_coins, txs); } catch (const KEY_IMAGE_EXISTS& e) { @@ -3684,7 +3696,12 @@ leave: TIME_MEASURE_FINISH(addblock); // do this after updating the hard fork state since the weight limit may change due to fork - update_next_cumulative_weight_limit(); + if (!update_next_cumulative_weight_limit()) + { + MERROR("Failed to update next cumulative weight limit"); + pop_block_from_blockchain(); + return false; + } MINFO("+++++ BLOCK SUCCESSFULLY ADDED" << std::endl << "id:\t" << id << std::endl << "PoW:\t" << proof_of_work << std::endl << "HEIGHT " << new_height-1 << ", difficulty:\t" << current_diffic << std::endl << "block reward: " << print_money(fee_summary + base_reward) << "(" << print_money(base_reward) << " + " << print_money(fee_summary) << "), coinbase_weight: " << coinbase_weight << ", cumulative weight: " << cumulative_block_weight << ", " << block_processing_time << "(" << target_calculating_time << "/" << longhash_calculating_time << ")ms"); if(m_show_time_stats) @@ -3740,20 +3757,100 @@ bool Blockchain::check_blockchain_pruning() return m_db->check_pruning(); } //------------------------------------------------------------------ -bool Blockchain::update_next_cumulative_weight_limit() +uint64_t Blockchain::get_next_long_term_block_weight(uint64_t block_weight) const { - uint64_t full_reward_zone = get_min_block_weight(get_current_hard_fork_version()); + PERF_TIMER(get_next_long_term_block_weight); + + const uint64_t db_height = m_db->height(); + const uint64_t nblocks = std::min(m_long_term_block_weights_window, db_height); + + const uint8_t hf_version = get_current_hard_fork_version(); + if (hf_version < HF_VERSION_LONG_TERM_BLOCK_WEIGHT) + return block_weight; + + std::vector weights; + weights.resize(nblocks); + for (uint64_t h = 0; h < nblocks; ++h) + weights[h] = m_db->get_block_long_term_weight(db_height - nblocks + h); + uint64_t long_term_median = epee::misc_utils::median(weights); + uint64_t long_term_effective_median_block_weight = std::max(CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5, long_term_median); + + uint64_t short_term_constraint = long_term_effective_median_block_weight + long_term_effective_median_block_weight * 2 / 5; + uint64_t long_term_block_weight = std::min(block_weight, short_term_constraint); + + return long_term_block_weight; +} +//------------------------------------------------------------------ +bool Blockchain::update_next_cumulative_weight_limit(uint64_t *long_term_effective_median_block_weight) +{ + PERF_TIMER(update_next_cumulative_weight_limit); LOG_PRINT_L3("Blockchain::" << __func__); - std::vector weights; - get_last_n_blocks_weights(weights, CRYPTONOTE_REWARD_BLOCKS_WINDOW); - uint64_t median = epee::misc_utils::median(weights); - m_current_block_cumul_weight_median = median; - if(median <= full_reward_zone) - median = full_reward_zone; + // when we reach this, the last hf version is not yet written to the db + const uint64_t db_height = m_db->height(); + const uint8_t hf_version = get_current_hard_fork_version(); + uint64_t full_reward_zone = get_min_block_weight(hf_version); + uint64_t long_term_block_weight; + + if (hf_version < HF_VERSION_LONG_TERM_BLOCK_WEIGHT) + { + std::vector weights; + get_last_n_blocks_weights(weights, CRYPTONOTE_REWARD_BLOCKS_WINDOW); + m_current_block_cumul_weight_median = epee::misc_utils::median(weights); + long_term_block_weight = weights.back(); + } + else + { + const uint64_t block_weight = m_db->get_block_weight(db_height - 1); + + std::vector weights, new_weights; + uint64_t long_term_median; + if (db_height == 1) + { + long_term_median = CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5; + } + else + { + uint64_t nblocks = std::min(m_long_term_block_weights_window, db_height); + if (nblocks == db_height) + --nblocks; + weights.resize(nblocks); + for (uint64_t h = 0; h < nblocks; ++h) + weights[h] = m_db->get_block_long_term_weight(db_height - nblocks + h - 1); + new_weights = weights; + long_term_median = epee::misc_utils::median(weights); + } + + m_long_term_effective_median_block_weight = std::max(CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5, long_term_median); + + uint64_t short_term_constraint = m_long_term_effective_median_block_weight + m_long_term_effective_median_block_weight * 2 / 5; + long_term_block_weight = std::min(block_weight, short_term_constraint); + + if (new_weights.empty()) + new_weights.resize(1); + new_weights[0] = long_term_block_weight; + long_term_median = epee::misc_utils::median(new_weights); + m_long_term_effective_median_block_weight = std::max(CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5, long_term_median); + short_term_constraint = m_long_term_effective_median_block_weight + m_long_term_effective_median_block_weight * 2 / 5; + + weights.clear(); + get_last_n_blocks_weights(weights, CRYPTONOTE_REWARD_BLOCKS_WINDOW); + + uint64_t short_term_median = epee::misc_utils::median(weights); + uint64_t effective_median_block_weight = std::min(std::max(CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5, short_term_median), CRYPTONOTE_SHORT_TERM_BLOCK_WEIGHT_SURGE_FACTOR * m_long_term_effective_median_block_weight); + + m_current_block_cumul_weight_median = effective_median_block_weight; + } + + if (m_current_block_cumul_weight_median <= full_reward_zone) + m_current_block_cumul_weight_median = full_reward_zone; + + m_current_block_cumul_weight_limit = m_current_block_cumul_weight_median * 2; + + if (long_term_effective_median_block_weight) + *long_term_effective_median_block_weight = m_long_term_effective_median_block_weight; - m_current_block_cumul_weight_limit = median*2; return true; } //------------------------------------------------------------------ diff --git a/src/cryptonote_core/blockchain.h b/src/cryptonote_core/blockchain.h index c742bec91..92aef1278 100644 --- a/src/cryptonote_core/blockchain.h +++ b/src/cryptonote_core/blockchain.h @@ -37,6 +37,7 @@ #include #include #include +#include #include #include #include @@ -630,6 +631,13 @@ namespace cryptonote */ uint64_t get_current_cumulative_block_weight_limit() const; + /** + * @brief gets the long term block weight for a new block + * + * @return the long term block weight + */ + uint64_t get_next_long_term_block_weight(uint64_t block_weight) const; + /** * @brief gets the block weight median based on recent blocks (same window as for the limit) * @@ -994,7 +1002,9 @@ namespace cryptonote */ void pop_blocks(uint64_t nblocks); +#ifndef IN_UNIT_TESTS private: +#endif // TODO: evaluate whether or not each of these typedefs are left over from blockchain_storage typedef std::unordered_map blocks_by_id_index; @@ -1047,6 +1057,8 @@ namespace cryptonote std::vector m_timestamps; std::vector m_difficulties; uint64_t m_timestamps_and_difficulties_height; + uint64_t m_long_term_block_weights_window; + uint64_t m_long_term_effective_median_block_weight; epee::critical_section m_difficulty_lock; crypto::hash m_difficulty_for_next_block_top_hash; @@ -1280,7 +1292,7 @@ namespace cryptonote * @param sz return-by-reference the list of weights * @param count the number of blocks to get weights for */ - void get_last_n_blocks_weights(std::vector& weights, size_t count) const; + void get_last_n_blocks_weights(std::vector& weights, size_t count) const; /** * @brief checks if a transaction is unlocked (its outputs spendable) @@ -1379,9 +1391,11 @@ namespace cryptonote /** * @brief calculate the block weight limit for the next block to be added * + * @param long_term_effective_median_block_weight optionally return that value + * * @return true */ - bool update_next_cumulative_weight_limit(); + bool update_next_cumulative_weight_limit(uint64_t *long_term_effective_median_block_weight = NULL); void return_tx_to_pool(std::vector &txs); /** diff --git a/src/cryptonote_core/cryptonote_core.cpp b/src/cryptonote_core/cryptonote_core.cpp index 1513bb750..bc01e2435 100644 --- a/src/cryptonote_core/cryptonote_core.cpp +++ b/src/cryptonote_core/cryptonote_core.cpp @@ -602,7 +602,8 @@ namespace cryptonote const std::pair regtest_hard_forks[3] = {std::make_pair(1, 0), std::make_pair(Blockchain::get_hard_fork_heights(MAINNET).back().version, 1), std::make_pair(0, 0)}; const cryptonote::test_options regtest_test_options = { - regtest_hard_forks + regtest_hard_forks, + 0 }; const difficulty_type fixed_difficulty = command_line::get_arg(vm, arg_fixed_difficulty); r = m_blockchain_storage.init(db.release(), m_nettype, m_offline, regtest ? ®test_test_options : test_options, fixed_difficulty, get_checkpoints); diff --git a/src/cryptonote_core/cryptonote_core.h b/src/cryptonote_core/cryptonote_core.h index 4810fc891..308924bb9 100644 --- a/src/cryptonote_core/cryptonote_core.h +++ b/src/cryptonote_core/cryptonote_core.h @@ -54,6 +54,7 @@ namespace cryptonote { struct test_options { const std::pair *hard_forks; + const size_t long_term_block_weight_window; }; extern const command_line::arg_descriptor arg_data_dir; diff --git a/src/daemon/rpc_command_executor.cpp b/src/daemon/rpc_command_executor.cpp index 839350522..2d85fdd1a 100644 --- a/src/daemon/rpc_command_executor.cpp +++ b/src/daemon/rpc_command_executor.cpp @@ -79,6 +79,7 @@ namespace { << "POW hash: " << header.pow_hash << std::endl << "block size: " << header.block_size << std::endl << "block weight: " << header.block_weight << std::endl + << "long term weight: " << header.long_term_weight << std::endl << "num txes: " << header.num_txes << std::endl << "reward: " << cryptonote::print_money(header.reward); } diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index b524273bf..eae8cc531 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -1322,6 +1322,7 @@ namespace cryptonote response.block_size = response.block_weight = m_core.get_blockchain_storage().get_db().get_block_weight(height); response.num_txes = blk.tx_hashes.size(); response.pow_hash = fill_pow_hash ? string_tools::pod_to_hex(get_block_longhash(blk, height)) : ""; + response.long_term_weight = m_core.get_blockchain_storage().get_db().get_block_long_term_weight(height); return true; } //------------------------------------------------------------------------------------------------------------------------------ diff --git a/src/rpc/core_rpc_server_commands_defs.h b/src/rpc/core_rpc_server_commands_defs.h index dfad5d6a7..e52e4fc67 100644 --- a/src/rpc/core_rpc_server_commands_defs.h +++ b/src/rpc/core_rpc_server_commands_defs.h @@ -1172,6 +1172,7 @@ namespace cryptonote uint64_t block_weight; uint64_t num_txes; std::string pow_hash; + uint64_t long_term_weight; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(major_version) @@ -1190,6 +1191,7 @@ namespace cryptonote KV_SERIALIZE_OPT(block_weight, (uint64_t)0) KV_SERIALIZE(num_txes) KV_SERIALIZE(pow_hash) + KV_SERIALIZE_OPT(long_term_weight, (uint64_t)0) END_KV_SERIALIZE_MAP() }; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 20afd4203..89d77cf32 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -85,6 +85,7 @@ add_subdirectory(performance_tests) add_subdirectory(core_proxy) add_subdirectory(unit_tests) add_subdirectory(difficulty) +add_subdirectory(block_weight) add_subdirectory(hash) add_subdirectory(net_load_tests) if (BUILD_GUI_DEPS) @@ -115,6 +116,7 @@ add_test( set(enabled_tests core_tests difficulty + block_weight hash performance_tests core_proxy diff --git a/tests/block_weight/CMakeLists.txt b/tests/block_weight/CMakeLists.txt new file mode 100644 index 000000000..b0d716ea0 --- /dev/null +++ b/tests/block_weight/CMakeLists.txt @@ -0,0 +1,45 @@ +# Copyright (c) 2014-2018, The Monero Project +# +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, are +# permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this list of +# conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, this list +# of conditions and the following disclaimer in the documentation and/or other +# materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors may be +# used to endorse or promote products derived from this software without specific +# prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +set(block_weight_sources + block_weight.cpp) + +set(block_weight_headers) + +add_executable(block_weight + ${block_weight_sources} + ${block_weight_headers}) +target_link_libraries(block_weight + PRIVATE + cryptonote_core + blockchain_db + ${EXTRA_LIBRARIES}) + +add_test( + NAME block_weight + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/compare.py ${CMAKE_CURRENT_SOURCE_DIR}/block_weight.py ${CMAKE_CURRENT_BINARY_DIR}/block_weight) diff --git a/tests/block_weight/block_weight.cpp b/tests/block_weight/block_weight.cpp new file mode 100644 index 000000000..a67f40f0f --- /dev/null +++ b/tests/block_weight/block_weight.cpp @@ -0,0 +1,183 @@ +// Copyright (c) 2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#define IN_UNIT_TESTS + +#include +#include +#include "cryptonote_core/blockchain.h" +#include "cryptonote_core/tx_pool.h" +#include "cryptonote_core/cryptonote_core.h" +#include "blockchain_db/testdb.h" + +#define LONG_TERM_BLOCK_WEIGHT_WINDOW 5000 + +enum test_t +{ + test_max = 0, + test_lcg = 1, + test_min = 2, +}; + +namespace +{ + +class TestDB: public cryptonote::BaseTestDB +{ +private: + struct block_t + { + size_t weight; + uint64_t long_term_weight; + }; + +public: + TestDB() { m_open = true; } + + virtual void add_block( const cryptonote::block& blk + , size_t block_weight + , uint64_t long_term_block_weight + , const cryptonote::difficulty_type& cumulative_difficulty + , const uint64_t& coins_generated + , uint64_t num_rct_outs + , const crypto::hash& blk_hash + ) override { + blocks.push_back({block_weight, long_term_block_weight}); + } + virtual uint64_t height() const override { return blocks.size(); } + virtual size_t get_block_weight(const uint64_t &h) const override { return blocks[h].weight; } + virtual uint64_t get_block_long_term_weight(const uint64_t &h) const override { return blocks[h].long_term_weight; } + virtual crypto::hash top_block_hash() const override { + uint64_t h = height(); + crypto::hash top = crypto::null_hash; + if (h) + *(uint64_t*)&top = h - 1; + return top; + } + virtual void pop_block(cryptonote::block &blk, std::vector &txs) override { blocks.pop_back(); } + virtual void set_hard_fork_version(uint64_t height, uint8_t version) override { if (height >= hf.size()) hf.resize(height + 1); hf[height] = version; } + virtual uint8_t get_hard_fork_version(uint64_t height) const override { if (height >= hf.size()) return 255; return hf[height]; } + +private: + std::vector blocks; + std::vector hf; +}; + +} + +#define PREFIX_WINDOW(hf_version,window) \ + std::unique_ptr bc; \ + cryptonote::tx_memory_pool txpool(*bc); \ + bc.reset(new cryptonote::Blockchain(txpool)); \ + struct get_test_options { \ + const std::pair hard_forks[3]; \ + const cryptonote::test_options test_options = { \ + hard_forks, \ + window, \ + }; \ + get_test_options(): hard_forks{std::make_pair(1, (uint64_t)0), std::make_pair((uint8_t)hf_version, (uint64_t)LONG_TERM_BLOCK_WEIGHT_WINDOW), std::make_pair((uint8_t)0, (uint64_t)0)} {} \ + } opts; \ + cryptonote::Blockchain *blockchain = bc.get(); \ + bool r = blockchain->init(new TestDB(), cryptonote::FAKECHAIN, true, &opts.test_options, 0, NULL); \ + if (!r) \ + { \ + fprintf(stderr, "Failed to init blockchain\n"); \ + exit(1); \ + } + +#define PREFIX(hf_version) PREFIX_WINDOW(hf_version, LONG_TERM_BLOCK_WEIGHT_WINDOW) + +static uint32_t lcg_seed = 0; + +static uint32_t lcg() +{ + lcg_seed = (lcg_seed * 0x100000001b3 + 0xcbf29ce484222325) & 0xffffffff; + return lcg_seed; +} + +static void test(test_t t, uint64_t blocks) +{ + PREFIX(10); + + for (uint64_t h = 0; h < LONG_TERM_BLOCK_WEIGHT_WINDOW; ++h) + { + cryptonote::block b; + b.major_version = 1; + b.minor_version = 1; + bc->get_db().add_block(b, 300000, 300000, bc->get_db().height(), bc->get_db().height(), {}); + if (!bc->update_next_cumulative_weight_limit()) + { + fprintf(stderr, "Failed to update cumulative weight limit 1\n"); + exit(1); + } + } + + for (uint64_t h = 0; h < blocks; ++h) + { + uint64_t w; + uint64_t effective_block_weight_median = bc->get_current_cumulative_block_weight_median(); + switch (t) + { + case test_lcg: + { + uint32_t r = lcg(); + int64_t wi = 90 + r % 500000 + 250000 + sin(h / 200.) * 350000; + w = wi < 90 ? 90 : wi; + break; + } + case test_max: + w = bc->get_current_cumulative_block_weight_limit(); + break; + case test_min: + w = 90; + break; + default: + exit(1); + } + uint64_t ltw = bc->get_next_long_term_block_weight(w); + cryptonote::block b; + b.major_version = 10; + b.minor_version = 10; + bc->get_db().add_block(std::move(b), w, ltw, bc->get_db().height(), bc->get_db().height(), {}); + + if (!bc->update_next_cumulative_weight_limit()) + { + fprintf(stderr, "Failed to update cumulative weight limit\n"); + exit(1); + } + std::cout << "H " << h << ", BW " << w << ", EMBW " << effective_block_weight_median << ", LTBW " << ltw << std::endl; + } +} + +int main() +{ + test(test_max, 2 * LONG_TERM_BLOCK_WEIGHT_WINDOW); + test(test_lcg, 9 * LONG_TERM_BLOCK_WEIGHT_WINDOW); + test(test_min, 1 * LONG_TERM_BLOCK_WEIGHT_WINDOW); + return 0; +} diff --git a/tests/block_weight/block_weight.py b/tests/block_weight/block_weight.py new file mode 100755 index 000000000..06aaabb02 --- /dev/null +++ b/tests/block_weight/block_weight.py @@ -0,0 +1,74 @@ +#!/usr/bin/python +# Simulate a maximal block attack on the Monero network +# This uses the scheme proposed by ArticMine +# Written by Sarang Nother +# Copyright (c) 2019 The Monero Project +import sys +import math + +MEDIAN_WINDOW_SMALL = 100 # number of recent blocks for median computation +MEDIAN_WINDOW_BIG = 5000 +MULTIPLIER_SMALL = 1.4 # multipliers for determining weights +MULTIPLIER_BIG = 50.0 +MEDIAN_THRESHOLD = 300*1000 # initial value for median (scaled kB -> B) +lcg_seed = 0 +embw = MEDIAN_THRESHOLD +ltembw = MEDIAN_THRESHOLD + +weights = [MEDIAN_THRESHOLD]*MEDIAN_WINDOW_SMALL # weights of recent blocks (B), with index -1 most recent +lt_weights = [MEDIAN_THRESHOLD]*MEDIAN_WINDOW_BIG # long-term weights + +# Compute the median of a list +def get_median(vec): + #temp = vec + temp = sorted(vec) + if len(temp) % 2 == 1: + return temp[len(temp)/2] + else: + return int((temp[len(temp)/2]+temp[len(temp)/2-1])/2) + +def LCG(): + global lcg_seed + lcg_seed = (lcg_seed * 0x100000001b3 + 0xcbf29ce484222325) & 0xffffffff + return lcg_seed + +def run(t, blocks): + global embw + global ltembw + + weights = [MEDIAN_THRESHOLD]*MEDIAN_WINDOW_SMALL # weights of recent blocks (B), with index -1 most recent + lt_weights = [MEDIAN_THRESHOLD]*MEDIAN_WINDOW_BIG # long-term weights + + for block in range(blocks): + # determine the long-term effective weight + ltmedian = get_median(lt_weights[-MEDIAN_WINDOW_BIG:]) + ltembw = max(MEDIAN_THRESHOLD,ltmedian) + + # determine the effective weight + stmedian = get_median(weights[-MEDIAN_WINDOW_SMALL:]) + embw = min(max(MEDIAN_THRESHOLD,stmedian),int(MULTIPLIER_BIG*ltembw)) + + # drop the lowest values + weights = weights[1:] + lt_weights = lt_weights[1:] + + # add a block of max weight + if t == 0: + max_weight = 2 * embw + elif t == 1: + r = LCG() + max_weight = int(90 + r % 500000 + 250000 + math.sin(block / 200.) * 350000) + if max_weight < 90: max_weight = 90 + elif t == 2: + max_weight = 90 + else: + sys.exit(1) + weights.append(max_weight) + lt_weights.append(min(max_weight,int(ltembw + int(ltembw * 2 / 5)))) + + #print "H %u, r %u, BW %u, EMBW %u, LTBW %u, LTEMBW %u, ltmedian %u" % (block, r, max_weight, embw, lt_weights[-1], ltembw, ltmedian) + print "H %u, BW %u, EMBW %u, LTBW %u" % (block, max_weight, embw, lt_weights[-1]) + +run(0, 2 * MEDIAN_WINDOW_BIG) +run(1, 9 * MEDIAN_WINDOW_BIG) +run(2, 1 * MEDIAN_WINDOW_BIG) diff --git a/tests/block_weight/compare.py b/tests/block_weight/compare.py new file mode 100755 index 000000000..c6be05206 --- /dev/null +++ b/tests/block_weight/compare.py @@ -0,0 +1,13 @@ +#!/usr/bin/python + +import sys +import subprocess + +print 'running: ', sys.argv[1] +S0 = subprocess.check_output(sys.argv[1], stderr=subprocess.STDOUT) +print 'running: ', sys.argv[2] +S1 = subprocess.check_output(sys.argv[2], stderr=subprocess.STDOUT) +print 'comparing' +if S0 != S1: + sys.exit(1) +sys.exit(0) diff --git a/tests/core_tests/v2_tests.h b/tests/core_tests/v2_tests.h index ecb23818c..cd2bcb839 100644 --- a/tests/core_tests/v2_tests.h +++ b/tests/core_tests/v2_tests.h @@ -81,7 +81,7 @@ template<> struct get_test_options { const std::pair hard_forks[3] = {std::make_pair(1, 0), std::make_pair(2, 1), std::make_pair(0, 0)}; const cryptonote::test_options test_options = { - hard_forks + hard_forks, 0 }; }; diff --git a/tests/unit_tests/CMakeLists.txt b/tests/unit_tests/CMakeLists.txt index cb421c847..aea82ede2 100644 --- a/tests/unit_tests/CMakeLists.txt +++ b/tests/unit_tests/CMakeLists.txt @@ -55,6 +55,7 @@ set(unit_tests_sources http.cpp keccak.cpp logging.cpp + long_term_block_weight.cpp main.cpp memwipe.cpp mlocker.cpp diff --git a/tests/unit_tests/blockchain_db.cpp b/tests/unit_tests/blockchain_db.cpp index 7e7ce9bf7..9cf8c5fbe 100644 --- a/tests/unit_tests/blockchain_db.cpp +++ b/tests/unit_tests/blockchain_db.cpp @@ -277,10 +277,10 @@ TYPED_TEST(BlockchainDBTest, AddBlock) // TODO: need at least one more block to make this reasonable, as the // BlockchainDB implementation should not check for parent if // no blocks have been added yet (because genesis has no parent). - //ASSERT_THROW(this->m_db->add_block(this->m_blocks[1], t_sizes[1], t_diffs[1], t_coins[1], this->m_txs[1]), BLOCK_PARENT_DNE); + //ASSERT_THROW(this->m_db->add_block(this->m_blocks[1], t_sizes[1], t_sizes[1], t_diffs[1], t_coins[1], this->m_txs[1]), BLOCK_PARENT_DNE); - ASSERT_NO_THROW(this->m_db->add_block(this->m_blocks[0], t_sizes[0], t_diffs[0], t_coins[0], this->m_txs[0])); - ASSERT_NO_THROW(this->m_db->add_block(this->m_blocks[1], t_sizes[1], t_diffs[1], t_coins[1], this->m_txs[1])); + ASSERT_NO_THROW(this->m_db->add_block(this->m_blocks[0], t_sizes[0], t_sizes[0], t_diffs[0], t_coins[0], this->m_txs[0])); + ASSERT_NO_THROW(this->m_db->add_block(this->m_blocks[1], t_sizes[1], t_sizes[1], t_diffs[1], t_coins[1], this->m_txs[1])); block b; ASSERT_TRUE(this->m_db->block_exists(get_block_hash(this->m_blocks[0]))); @@ -293,7 +293,7 @@ TYPED_TEST(BlockchainDBTest, AddBlock) ASSERT_TRUE(compare_blocks(this->m_blocks[0], b)); // assert that we can't add the same block twice - ASSERT_THROW(this->m_db->add_block(this->m_blocks[0], t_sizes[0], t_diffs[0], t_coins[0], this->m_txs[0]), TX_EXISTS); + ASSERT_THROW(this->m_db->add_block(this->m_blocks[0], t_sizes[0], t_sizes[0], t_diffs[0], t_coins[0], this->m_txs[0]), TX_EXISTS); for (auto& h : this->m_blocks[0].tx_hashes) { @@ -317,14 +317,14 @@ TYPED_TEST(BlockchainDBTest, RetrieveBlockData) this->get_filenames(); this->init_hard_fork(); - ASSERT_NO_THROW(this->m_db->add_block(this->m_blocks[0], t_sizes[0], t_diffs[0], t_coins[0], this->m_txs[0])); + ASSERT_NO_THROW(this->m_db->add_block(this->m_blocks[0], t_sizes[0], t_sizes[0], t_diffs[0], t_coins[0], this->m_txs[0])); ASSERT_EQ(t_sizes[0], this->m_db->get_block_weight(0)); ASSERT_EQ(t_diffs[0], this->m_db->get_block_cumulative_difficulty(0)); ASSERT_EQ(t_diffs[0], this->m_db->get_block_difficulty(0)); ASSERT_EQ(t_coins[0], this->m_db->get_block_already_generated_coins(0)); - ASSERT_NO_THROW(this->m_db->add_block(this->m_blocks[1], t_sizes[1], t_diffs[1], t_coins[1], this->m_txs[1])); + ASSERT_NO_THROW(this->m_db->add_block(this->m_blocks[1], t_sizes[1], t_sizes[1], t_diffs[1], t_coins[1], this->m_txs[1])); ASSERT_EQ(t_diffs[1] - t_diffs[0], this->m_db->get_block_difficulty(1)); ASSERT_HASH_EQ(get_block_hash(this->m_blocks[0]), this->m_db->get_block_hash_from_height(0)); diff --git a/tests/unit_tests/hardfork.cpp b/tests/unit_tests/hardfork.cpp index ec8d1d202..12dfde1bc 100644 --- a/tests/unit_tests/hardfork.cpp +++ b/tests/unit_tests/hardfork.cpp @@ -34,7 +34,7 @@ #include "blockchain_db/blockchain_db.h" #include "cryptonote_basic/cryptonote_format_utils.h" #include "cryptonote_basic/hardfork.h" -#include "testdb.h" +#include "blockchain_db/testdb.h" using namespace cryptonote; @@ -44,11 +44,12 @@ using namespace cryptonote; namespace { -class TestDB: public BaseTestDB { +class TestDB: public cryptonote::BaseTestDB { public: virtual uint64_t height() const { return blocks.size(); } virtual void add_block( const block& blk , size_t block_weight + , uint64_t long_term_block_weight , const difficulty_type& cumulative_difficulty , const uint64_t& coins_generated , uint64_t num_rct_outs @@ -106,20 +107,20 @@ TEST(major, Only) ASSERT_FALSE(hf.add(mkblock(0, 2), 0)); ASSERT_FALSE(hf.add(mkblock(2, 2), 0)); ASSERT_TRUE(hf.add(mkblock(1, 2), 0)); - db.add_block(mkblock(1, 1), 0, 0, 0, 0, crypto::hash()); + db.add_block(mkblock(1, 1), 0, 0, 0, 0, 0, crypto::hash()); // block height 1, only version 1 is accepted ASSERT_FALSE(hf.add(mkblock(0, 2), 1)); ASSERT_FALSE(hf.add(mkblock(2, 2), 1)); ASSERT_TRUE(hf.add(mkblock(1, 2), 1)); - db.add_block(mkblock(1, 1), 0, 0, 0, 0, crypto::hash()); + db.add_block(mkblock(1, 1), 0, 0, 0, 0, 0, crypto::hash()); // block height 2, only version 2 is accepted ASSERT_FALSE(hf.add(mkblock(0, 2), 2)); ASSERT_FALSE(hf.add(mkblock(1, 2), 2)); ASSERT_FALSE(hf.add(mkblock(3, 2), 2)); ASSERT_TRUE(hf.add(mkblock(2, 2), 2)); - db.add_block(mkblock(2, 1), 0, 0, 0, 0, crypto::hash()); + db.add_block(mkblock(2, 1), 0, 0, 0, 0, 0, crypto::hash()); } TEST(empty_hardforks, Success) @@ -133,7 +134,7 @@ TEST(empty_hardforks, Success) ASSERT_TRUE(hf.get_state(time(NULL) + 3600*24*400) == HardFork::Ready); for (uint64_t h = 0; h <= 10; ++h) { - db.add_block(mkblock(hf, h, 1), 0, 0, 0, 0, crypto::hash()); + db.add_block(mkblock(hf, h, 1), 0, 0, 0, 0, 0, crypto::hash()); ASSERT_TRUE(hf.add(db.get_block_from_height(h), h)); } ASSERT_EQ(hf.get(0), 1); @@ -167,14 +168,14 @@ TEST(check_for_height, Success) for (uint64_t h = 0; h <= 4; ++h) { ASSERT_TRUE(hf.check_for_height(mkblock(1, 1), h)); ASSERT_FALSE(hf.check_for_height(mkblock(2, 2), h)); // block version is too high - db.add_block(mkblock(hf, h, 1), 0, 0, 0, 0, crypto::hash()); + db.add_block(mkblock(hf, h, 1), 0, 0, 0, 0, 0, crypto::hash()); ASSERT_TRUE(hf.add(db.get_block_from_height(h), h)); } for (uint64_t h = 5; h <= 10; ++h) { ASSERT_FALSE(hf.check_for_height(mkblock(1, 1), h)); // block version is too low ASSERT_TRUE(hf.check_for_height(mkblock(2, 2), h)); - db.add_block(mkblock(hf, h, 2), 0, 0, 0, 0, crypto::hash()); + db.add_block(mkblock(hf, h, 2), 0, 0, 0, 0, 0, crypto::hash()); ASSERT_TRUE(hf.add(db.get_block_from_height(h), h)); } } @@ -191,19 +192,19 @@ TEST(get, next_version) for (uint64_t h = 0; h <= 4; ++h) { ASSERT_EQ(2, hf.get_next_version()); - db.add_block(mkblock(hf, h, 1), 0, 0, 0, 0, crypto::hash()); + db.add_block(mkblock(hf, h, 1), 0, 0, 0, 0, 0, crypto::hash()); ASSERT_TRUE(hf.add(db.get_block_from_height(h), h)); } for (uint64_t h = 5; h <= 9; ++h) { ASSERT_EQ(4, hf.get_next_version()); - db.add_block(mkblock(hf, h, 2), 0, 0, 0, 0, crypto::hash()); + db.add_block(mkblock(hf, h, 2), 0, 0, 0, 0, 0, crypto::hash()); ASSERT_TRUE(hf.add(db.get_block_from_height(h), h)); } for (uint64_t h = 10; h <= 15; ++h) { ASSERT_EQ(4, hf.get_next_version()); - db.add_block(mkblock(hf, h, 4), 0, 0, 0, 0, crypto::hash()); + db.add_block(mkblock(hf, h, 4), 0, 0, 0, 0, 0, crypto::hash()); ASSERT_TRUE(hf.add(db.get_block_from_height(h), h)); } } @@ -244,7 +245,7 @@ TEST(steps_asap, Success) hf.init(); for (uint64_t h = 0; h < 10; ++h) { - db.add_block(mkblock(hf, h, 9), 0, 0, 0, 0, crypto::hash()); + db.add_block(mkblock(hf, h, 9), 0, 0, 0, 0, 0, crypto::hash()); ASSERT_TRUE(hf.add(db.get_block_from_height(h), h)); } @@ -271,7 +272,7 @@ TEST(steps_1, Success) hf.init(); for (uint64_t h = 0 ; h < 10; ++h) { - db.add_block(mkblock(hf, h, h+1), 0, 0, 0, 0, crypto::hash()); + db.add_block(mkblock(hf, h, h+1), 0, 0, 0, 0, 0, crypto::hash()); ASSERT_TRUE(hf.add(db.get_block_from_height(h), h)); } @@ -296,7 +297,7 @@ TEST(reorganize, Same) // index 0 1 2 3 4 5 6 7 8 9 static const uint8_t block_versions[] = { 1, 1, 4, 4, 7, 7, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9 }; for (uint64_t h = 0; h < 20; ++h) { - db.add_block(mkblock(hf, h, block_versions[h]), 0, 0, 0, 0, crypto::hash()); + db.add_block(mkblock(hf, h, block_versions[h]), 0, 0, 0, 0, 0, crypto::hash()); ASSERT_TRUE(hf.add(db.get_block_from_height(h), h)); } @@ -327,7 +328,7 @@ TEST(reorganize, Changed) static const uint8_t block_versions[] = { 1, 1, 4, 4, 7, 7, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9 }; static const uint8_t expected_versions[] = { 1, 1, 1, 1, 1, 1, 4, 4, 7, 7, 9, 9, 9, 9, 9, 9 }; for (uint64_t h = 0; h < 16; ++h) { - db.add_block(mkblock(hf, h, block_versions[h]), 0, 0, 0, 0, crypto::hash()); + db.add_block(mkblock(hf, h, block_versions[h]), 0, 0, 0, 0, 0, crypto::hash()); ASSERT_TRUE (hf.add(db.get_block_from_height(h), h)); } @@ -347,7 +348,7 @@ TEST(reorganize, Changed) ASSERT_EQ(db.height(), 3); hf.reorganize_from_block_height(2); for (uint64_t h = 3; h < 16; ++h) { - db.add_block(mkblock(hf, h, block_versions_new[h]), 0, 0, 0, 0, crypto::hash()); + db.add_block(mkblock(hf, h, block_versions_new[h]), 0, 0, 0, 0, 0, crypto::hash()); bool ret = hf.add(db.get_block_from_height(h), h); ASSERT_EQ (ret, h < 15); } @@ -371,7 +372,7 @@ TEST(voting, threshold) for (uint64_t h = 0; h <= 8; ++h) { uint8_t v = 1 + !!(h % 8); - db.add_block(mkblock(hf, h, v), 0, 0, 0, 0, crypto::hash()); + db.add_block(mkblock(hf, h, v), 0, 0, 0, 0, 0, crypto::hash()); bool ret = hf.add(db.get_block_from_height(h), h); if (h >= 8 && threshold == 87) { // for threshold 87, we reach the treshold at height 7, so from height 8, hard fork to version 2, but 8 tries to add 1 @@ -405,7 +406,7 @@ TEST(voting, different_thresholds) static const uint8_t expected_versions[] = { 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4 }; for (uint64_t h = 0; h < sizeof(block_versions) / sizeof(block_versions[0]); ++h) { - db.add_block(mkblock(hf, h, block_versions[h]), 0, 0, 0, 0, crypto::hash()); + db.add_block(mkblock(hf, h, block_versions[h]), 0, 0, 0, 0, 0, crypto::hash()); bool ret = hf.add(db.get_block_from_height(h), h); ASSERT_EQ(ret, true); } @@ -459,7 +460,7 @@ TEST(voting, info) ASSERT_EQ(expected_thresholds[h], threshold); ASSERT_EQ(4, voting); - db.add_block(mkblock(hf, h, block_versions[h]), 0, 0, 0, 0, crypto::hash()); + db.add_block(mkblock(hf, h, block_versions[h]), 0, 0, 0, 0, 0, crypto::hash()); ASSERT_TRUE(hf.add(db.get_block_from_height(h), h)); } } @@ -522,7 +523,7 @@ TEST(reorganize, changed) #define ADD(v, h, a) \ do { \ cryptonote::block b = mkblock(hf, h, v); \ - db.add_block(b, 0, 0, 0, 0, crypto::hash()); \ + db.add_block(b, 0, 0, 0, 0, 0, crypto::hash()); \ ASSERT_##a(hf.add(b, h)); \ } while(0) #define ADD_TRUE(v, h) ADD(v, h, TRUE) diff --git a/tests/unit_tests/long_term_block_weight.cpp b/tests/unit_tests/long_term_block_weight.cpp new file mode 100644 index 000000000..0ee6c5c2b --- /dev/null +++ b/tests/unit_tests/long_term_block_weight.cpp @@ -0,0 +1,384 @@ +// Copyright (c) 2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#define IN_UNIT_TESTS + +#include "gtest/gtest.h" +#include "cryptonote_core/blockchain.h" +#include "cryptonote_core/tx_pool.h" +#include "cryptonote_core/cryptonote_core.h" +#include "blockchain_db/testdb.h" + +#define TEST_LONG_TERM_BLOCK_WEIGHT_WINDOW 5000 + +namespace +{ + +class TestDB: public cryptonote::BaseTestDB +{ +private: + struct block_t + { + size_t weight; + uint64_t long_term_weight; + }; + +public: + TestDB() { m_open = true; } + + virtual void add_block( const cryptonote::block& blk + , size_t block_weight + , uint64_t long_term_block_weight + , const cryptonote::difficulty_type& cumulative_difficulty + , const uint64_t& coins_generated + , uint64_t num_rct_outs + , const crypto::hash& blk_hash + ) override { + blocks.push_back({block_weight, long_term_block_weight}); + } + virtual uint64_t height() const override { return blocks.size(); } + virtual size_t get_block_weight(const uint64_t &h) const override { return blocks[h].weight; } + virtual uint64_t get_block_long_term_weight(const uint64_t &h) const override { return blocks[h].long_term_weight; } + virtual crypto::hash top_block_hash() const override { + uint64_t h = height(); + crypto::hash top = crypto::null_hash; + if (h) + *(uint64_t*)&top = h - 1; + return top; + } + virtual void pop_block(cryptonote::block &blk, std::vector &txs) override { blocks.pop_back(); } + +private: + std::vector blocks; +}; + +static uint32_t lcg_seed = 0; + +static uint32_t lcg() +{ + lcg_seed = (lcg_seed * 0x100000001b3 + 0xcbf29ce484222325) & 0xffffffff; + return lcg_seed; +} + +} + +#define PREFIX_WINDOW(hf_version,window) \ + std::unique_ptr bc; \ + cryptonote::tx_memory_pool txpool(*bc); \ + bc.reset(new cryptonote::Blockchain(txpool)); \ + struct get_test_options { \ + const std::pair hard_forks[3]; \ + const cryptonote::test_options test_options = { \ + hard_forks, \ + window, \ + }; \ + get_test_options(): hard_forks{std::make_pair(1, (uint64_t)0), std::make_pair((uint8_t)hf_version, (uint64_t)1), std::make_pair((uint8_t)0, (uint64_t)0)} {} \ + } opts; \ + cryptonote::Blockchain *blockchain = bc.get(); \ + bool r = blockchain->init(new TestDB(), cryptonote::FAKECHAIN, true, &opts.test_options, 0, NULL); \ + ASSERT_TRUE(r) + +#define PREFIX(hf_version) PREFIX_WINDOW(hf_version, TEST_LONG_TERM_BLOCK_WEIGHT_WINDOW) + +TEST(long_term_block_weight, empty_short) +{ + PREFIX(9); + + ASSERT_TRUE(bc->update_next_cumulative_weight_limit()); + + ASSERT_EQ(bc->get_current_cumulative_block_weight_median(), CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5); + ASSERT_EQ(bc->get_current_cumulative_block_weight_limit(), CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5 * 2); +} + +TEST(long_term_block_weight, identical_before_fork) +{ + PREFIX(9); + + for (uint64_t h = 1; h < 10 * TEST_LONG_TERM_BLOCK_WEIGHT_WINDOW; ++h) + { + size_t w = h < CRYPTONOTE_REWARD_BLOCKS_WINDOW ? CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5 : bc->get_current_cumulative_block_weight_limit(); + uint64_t ltw = bc->get_next_long_term_block_weight(w); + bc->get_db().add_block(cryptonote::block(), w, ltw, h, h, {}); + ASSERT_TRUE(bc->update_next_cumulative_weight_limit()); + } + for (uint64_t h = 0; h < 10 * TEST_LONG_TERM_BLOCK_WEIGHT_WINDOW; ++h) + { + ASSERT_EQ(bc->get_db().get_block_long_term_weight(h), bc->get_db().get_block_weight(h)); + } +} + +TEST(long_term_block_weight, identical_after_fork_before_long_term_window) +{ + PREFIX(10); + + for (uint64_t h = 1; h <= TEST_LONG_TERM_BLOCK_WEIGHT_WINDOW; ++h) + { + size_t w = h < TEST_LONG_TERM_BLOCK_WEIGHT_WINDOW ? CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5 : bc->get_current_cumulative_block_weight_limit(); + uint64_t ltw = bc->get_next_long_term_block_weight(w); + bc->get_db().add_block(cryptonote::block(), w, ltw, h, h, {}); + ASSERT_TRUE(bc->update_next_cumulative_weight_limit()); + } + for (uint64_t h = 0; h < TEST_LONG_TERM_BLOCK_WEIGHT_WINDOW; ++h) + { + ASSERT_EQ(bc->get_db().get_block_long_term_weight(h), bc->get_db().get_block_weight(h)); + } +} + +TEST(long_term_block_weight, ceiling_at_30000000) +{ + PREFIX(10); + + for (uint64_t h = 0; h < TEST_LONG_TERM_BLOCK_WEIGHT_WINDOW + TEST_LONG_TERM_BLOCK_WEIGHT_WINDOW / 2 - 1; ++h) + { + size_t w = h < TEST_LONG_TERM_BLOCK_WEIGHT_WINDOW ? CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5 : bc->get_current_cumulative_block_weight_limit(); + uint64_t ltw = bc->get_next_long_term_block_weight(w); + bc->get_db().add_block(cryptonote::block(), w, ltw, h, h, {}); + ASSERT_TRUE(bc->update_next_cumulative_weight_limit()); + } + ASSERT_EQ(bc->get_current_cumulative_block_weight_median(), 15000000); + ASSERT_EQ(bc->get_current_cumulative_block_weight_limit(), 30000000); +} + +TEST(long_term_block_weight, multi_pop) +{ + PREFIX(10); + + for (uint64_t h = 1; h <= TEST_LONG_TERM_BLOCK_WEIGHT_WINDOW + 20; ++h) + { + size_t w = h < TEST_LONG_TERM_BLOCK_WEIGHT_WINDOW ? CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5 : bc->get_current_cumulative_block_weight_limit(); + uint64_t ltw = bc->get_next_long_term_block_weight(w); + bc->get_db().add_block(cryptonote::block(), w, ltw, h, h, {}); + ASSERT_TRUE(bc->update_next_cumulative_weight_limit()); + } + + const uint64_t effective_median = bc->get_current_cumulative_block_weight_median(); + const uint64_t effective_limit = bc->get_current_cumulative_block_weight_limit(); + + for (uint64_t h = 0; h < 4; ++h) + { + size_t w = h < TEST_LONG_TERM_BLOCK_WEIGHT_WINDOW ? CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5 : bc->get_current_cumulative_block_weight_limit(); + uint64_t ltw = bc->get_next_long_term_block_weight(w); + bc->get_db().add_block(cryptonote::block(), w, ltw, h, h, {}); + ASSERT_TRUE(bc->update_next_cumulative_weight_limit()); + } + + cryptonote::block b; + std::vector txs; + bc->get_db().pop_block(b, txs); + bc->get_db().pop_block(b, txs); + bc->get_db().pop_block(b, txs); + bc->get_db().pop_block(b, txs); + ASSERT_TRUE(bc->update_next_cumulative_weight_limit()); + + ASSERT_EQ(effective_median, bc->get_current_cumulative_block_weight_median()); + ASSERT_EQ(effective_limit, bc->get_current_cumulative_block_weight_limit()); +} + +TEST(long_term_block_weight, multiple_updates) +{ + PREFIX(10); + + for (uint64_t h = 1; h <= 3 * TEST_LONG_TERM_BLOCK_WEIGHT_WINDOW; ++h) + { + size_t w = h < TEST_LONG_TERM_BLOCK_WEIGHT_WINDOW ? CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5 : bc->get_current_cumulative_block_weight_limit(); + uint64_t ltw = bc->get_next_long_term_block_weight(w); + bc->get_db().add_block(cryptonote::block(), w, ltw, h, h, {}); + ASSERT_TRUE(bc->update_next_cumulative_weight_limit()); + const uint64_t effective_median = bc->get_current_cumulative_block_weight_median(); + const uint64_t effective_limit = bc->get_current_cumulative_block_weight_limit(); + ASSERT_TRUE(bc->update_next_cumulative_weight_limit()); + ASSERT_EQ(effective_median, bc->get_current_cumulative_block_weight_median()); + ASSERT_EQ(effective_limit, bc->get_current_cumulative_block_weight_limit()); + ASSERT_TRUE(bc->update_next_cumulative_weight_limit()); + ASSERT_EQ(effective_median, bc->get_current_cumulative_block_weight_median()); + ASSERT_EQ(effective_limit, bc->get_current_cumulative_block_weight_limit()); + ASSERT_TRUE(bc->update_next_cumulative_weight_limit()); + ASSERT_EQ(effective_median, bc->get_current_cumulative_block_weight_median()); + ASSERT_EQ(effective_limit, bc->get_current_cumulative_block_weight_limit()); + } +} + +TEST(long_term_block_weight, pop_invariant_max) +{ + PREFIX(10); + + for (uint64_t h = 1; h < TEST_LONG_TERM_BLOCK_WEIGHT_WINDOW - 10; ++h) + { + size_t w = bc->get_db().height() < TEST_LONG_TERM_BLOCK_WEIGHT_WINDOW ? CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5 : bc->get_current_cumulative_block_weight_limit(); + uint64_t ltw = bc->get_next_long_term_block_weight(w); + bc->get_db().add_block(cryptonote::block(), w, ltw, h, h, {}); + ASSERT_TRUE(bc->update_next_cumulative_weight_limit()); + } + + for (int n = 0; n < 1000; ++n) + { + // pop some blocks, then add some more + int remove = 1 + (n * 17) % 8; + int add = (n * 23) % 12; + + // save long term block weights we're about to remove + uint64_t old_ltbw[16], h0 = bc->get_db().height() - remove - 1; + for (int i = -2; i < remove; ++i) + { + old_ltbw[i + 2] = bc->get_db().get_block_long_term_weight(h0 + i); + } + + for (int i = 0; i < remove; ++i) + { + cryptonote::block b; + std::vector txs; + bc->get_db().pop_block(b, txs); + ASSERT_TRUE(bc->update_next_cumulative_weight_limit()); + } + for (int i = 0; i < add; ++i) + { + size_t w = bc->get_db().height() < TEST_LONG_TERM_BLOCK_WEIGHT_WINDOW ? CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5 : bc->get_current_cumulative_block_weight_limit(); + uint64_t ltw = bc->get_next_long_term_block_weight(w); + bc->get_db().add_block(cryptonote::block(), w, ltw, bc->get_db().height(), bc->get_db().height(), {}); + ASSERT_TRUE(bc->update_next_cumulative_weight_limit()); + } + + // check the new values are the same as the old ones + for (int i = -2; i < std::min(add, remove); ++i) + { + ASSERT_EQ(bc->get_db().get_block_long_term_weight(h0 + i), old_ltbw[i + 2]); + } + } +} + +TEST(long_term_block_weight, pop_invariant_random) +{ + PREFIX(10); + + for (uint64_t h = 1; h < TEST_LONG_TERM_BLOCK_WEIGHT_WINDOW - 10; ++h) + { + size_t w = bc->get_db().height() < TEST_LONG_TERM_BLOCK_WEIGHT_WINDOW ? CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5 : bc->get_current_cumulative_block_weight_limit(); + uint64_t ltw = bc->get_next_long_term_block_weight(w); + bc->get_db().add_block(cryptonote::block(), w, ltw, h, h, {}); + ASSERT_TRUE(bc->update_next_cumulative_weight_limit()); + } + + for (int n = 0; n < 1000; ++n) + { + // pop some blocks, then add some more + int remove = 1 + (n * 17) % 8; + int add = (n * 23) % 123; + + // save long term block weights we're about to remove + uint64_t old_ltbw[16], h0 = bc->get_db().height() - remove - 1; + for (int i = -2; i < remove; ++i) + { + old_ltbw[i + 2] = bc->get_db().get_block_long_term_weight(h0 + i); + } + + for (int i = 0; i < remove; ++i) + { + cryptonote::block b; + std::vector txs; + bc->get_db().pop_block(b, txs); + ASSERT_TRUE(bc->update_next_cumulative_weight_limit()); + const uint64_t effective_median = bc->get_current_cumulative_block_weight_median(); + const uint64_t effective_limit = bc->get_current_cumulative_block_weight_limit(); + ASSERT_TRUE(bc->update_next_cumulative_weight_limit()); + ASSERT_EQ(effective_median, bc->get_current_cumulative_block_weight_median()); + ASSERT_EQ(effective_limit, bc->get_current_cumulative_block_weight_limit()); + } + for (int i = 0; i < add; ++i) + { + lcg_seed = bc->get_db().height(); + uint32_t r = lcg(); + size_t w = bc->get_db().height() < TEST_LONG_TERM_BLOCK_WEIGHT_WINDOW ? CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5 : (r % bc->get_current_cumulative_block_weight_limit()); + uint64_t ltw = bc->get_next_long_term_block_weight(w); + bc->get_db().add_block(cryptonote::block(), w, ltw, bc->get_db().height(), bc->get_db().height(), {}); + ASSERT_TRUE(bc->update_next_cumulative_weight_limit()); + const uint64_t effective_median = bc->get_current_cumulative_block_weight_median(); + const uint64_t effective_limit = bc->get_current_cumulative_block_weight_limit(); + ASSERT_TRUE(bc->update_next_cumulative_weight_limit()); + ASSERT_EQ(effective_median, bc->get_current_cumulative_block_weight_median()); + ASSERT_EQ(effective_limit, bc->get_current_cumulative_block_weight_limit()); + } + + // check the new values are the same as the old ones + for (int i = -2; i < std::min(add, remove); ++i) + { + ASSERT_EQ(bc->get_db().get_block_long_term_weight(h0 + i), old_ltbw[i + 2]); + } + } +} + +TEST(long_term_block_weight, long_growth_spike_and_drop) +{ + PREFIX(10); + + uint64_t long_term_effective_median_block_weight; + + // constant init + for (uint64_t h = 0; h < TEST_LONG_TERM_BLOCK_WEIGHT_WINDOW; ++h) + { + size_t w = CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5; + uint64_t ltw = bc->get_next_long_term_block_weight(w); + bc->get_db().add_block(cryptonote::block(), w, ltw, h, h, {}); + ASSERT_TRUE(bc->update_next_cumulative_weight_limit(&long_term_effective_median_block_weight)); + } + ASSERT_EQ(long_term_effective_median_block_weight, 300000); + + // slow 10% yearly for a year (scaled down by 100000 / TEST_LONG_TERM_BLOCK_WEIGHT_WINDOW) -> 8% change + for (uint64_t h = 0; h < 365 * 720 * TEST_LONG_TERM_BLOCK_WEIGHT_WINDOW / 100000; ++h) + { + //size_t w = bc->get_current_cumulative_block_weight_median() * rate; + float t = h / float(365 * 720 * TEST_LONG_TERM_BLOCK_WEIGHT_WINDOW / 100000); + size_t w = 300000 + t * 30000; + uint64_t ltw = bc->get_next_long_term_block_weight(w); + bc->get_db().add_block(cryptonote::block(), w, ltw, h, h, {}); + ASSERT_TRUE(bc->update_next_cumulative_weight_limit(&long_term_effective_median_block_weight)); + } + ASSERT_GT(long_term_effective_median_block_weight, 300000 * 1.07); + ASSERT_LT(long_term_effective_median_block_weight, 300000 * 1.09); + + // spike over three weeks - does not move much + for (uint64_t h = 0; h < 21 * 720 * TEST_LONG_TERM_BLOCK_WEIGHT_WINDOW / 100000; ++h) + { + size_t w = bc->get_current_cumulative_block_weight_limit(); + uint64_t ltw = bc->get_next_long_term_block_weight(w); + bc->get_db().add_block(cryptonote::block(), w, ltw, h, h, {}); + ASSERT_TRUE(bc->update_next_cumulative_weight_limit(&long_term_effective_median_block_weight)); + } + ASSERT_GT(long_term_effective_median_block_weight, 300000 * 1.07); + ASSERT_LT(long_term_effective_median_block_weight, 300000 * 1.09); + + // drop - does not move much + for (uint64_t h = 0; h < 21 * 720 * TEST_LONG_TERM_BLOCK_WEIGHT_WINDOW / 100000; ++h) + { + size_t w = bc->get_current_cumulative_block_weight_median() * .25; + uint64_t ltw = bc->get_next_long_term_block_weight(w); + bc->get_db().add_block(cryptonote::block(), w, ltw, h, h, {}); + ASSERT_TRUE(bc->update_next_cumulative_weight_limit(&long_term_effective_median_block_weight)); + } + ASSERT_GT(long_term_effective_median_block_weight, 300000 * 1.07); + ASSERT_LT(long_term_effective_median_block_weight, 300000 * 1.09); +} diff --git a/tests/unit_tests/output_distribution.cpp b/tests/unit_tests/output_distribution.cpp index 649752ac7..45f2c135b 100644 --- a/tests/unit_tests/output_distribution.cpp +++ b/tests/unit_tests/output_distribution.cpp @@ -33,7 +33,7 @@ #include "cryptonote_core/cryptonote_core.h" #include "cryptonote_core/tx_pool.h" #include "cryptonote_core/blockchain.h" -#include "testdb.h" +#include "blockchain_db/testdb.h" static const uint64_t test_distribution[32] = { 0, 0, 0, 0, 0, 1, 5, 1, 4, 0, 0, 1, 0, 1, 2, 3, 1, 0, 2, 0, 1, 3, 8, 1, 3, 5, 7, 1, 5, 0, 2, 3 @@ -43,7 +43,7 @@ static const size_t test_distribution_size = sizeof(test_distribution) / sizeof( namespace { -class TestDB: public BaseTestDB +class TestDB: public cryptonote::BaseTestDB { public: TestDB(size_t bc_height = test_distribution_size): blockchain_height(bc_height) { m_open = true; }