diff --git a/src/blockchain_db/blockchain_db.cpp b/src/blockchain_db/blockchain_db.cpp index 3d52d5fb9..05a330594 100644 --- a/src/blockchain_db/blockchain_db.cpp +++ b/src/blockchain_db/blockchain_db.cpp @@ -274,6 +274,7 @@ uint64_t BlockchainDB::add_block( const std::pair& blck time1 = epee::misc_utils::get_tick_count(); uint64_t num_rct_outs = 0; + const uint64_t num_rct_outs_coinbase = get_num_coinbase_rct_outputs(blck.first); blobdata miner_bd = tx_to_blob(blk.miner_tx); add_transaction(blk_hash, std::make_pair(blk.miner_tx, blobdata_ref(miner_bd))); if (blk.miner_tx.version == 2) @@ -296,7 +297,7 @@ uint64_t BlockchainDB::add_block( const std::pair& blck // call out to subclass implementation to add the block & metadata time1 = epee::misc_utils::get_tick_count(); - add_block(blk, block_weight, long_term_block_weight, cumulative_difficulty, coins_generated, num_rct_outs, blk_hash); + add_block(blk, block_weight, long_term_block_weight, cumulative_difficulty, coins_generated, num_rct_outs, num_rct_outs_coinbase, blk_hash); TIME_MEASURE_FINISH(time1); time_add_block1 += time1; diff --git a/src/blockchain_db/blockchain_db.h b/src/blockchain_db/blockchain_db.h index 835d6dcad..7fd05e0e1 100644 --- a/src/blockchain_db/blockchain_db.h +++ b/src/blockchain_db/blockchain_db.h @@ -397,6 +397,8 @@ private: * @param long_term_block_weight the long term block weight of the block (transactions and all) * @param cumulative_difficulty the accumulated difficulty after this block * @param coins_generated the number of coins generated total after this block + * @param num_rct_outs the number of total RCT tx outputs, including coinbase, in this block + * @param num_rct_outs_coinbase same as num_rct_outs, but ONLY including coinbase RCT outputs * @param blk_hash the hash of the block */ virtual void add_block( const block& blk @@ -405,6 +407,7 @@ private: , const difficulty_type& cumulative_difficulty , const uint64_t& coins_generated , uint64_t num_rct_outs + , uint64_t num_rct_outs_coinbase , const crypto::hash& blk_hash ) = 0; @@ -977,6 +980,24 @@ public: */ virtual std::vector get_block_cumulative_rct_outputs(const std::vector &heights) const = 0; + /** + * @brief fetch a block's cumulative number of rct *coinbase* outputs + * + * Same idea as get_block_cumulative_rct_outputs, but we include ONLY outputs from miner txs, and + * we perform the fetch on contiguous blocks in range [start_height, end_height). + * + * The subclass should return the numer of rct coinbase outputs in the blockchain + * up to the block with the given height (inclusive). + * + * If the block does not exist, the subclass should throw BLOCK_DNE + * + * @param begin_height the start height to fetch (inclusive) + * @param end_height the end height to fetch (exclusive) + * + * @return a list of the cumulative number of rct coinbase outputs per block + */ + virtual std::vector get_block_cumulative_rct_coinbase_outputs(uint64_t begin_height, uint64_t end_height) const = 0; + /** * @brief fetch the top block's timestamp * diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index 4178c862b..09f2fdf35 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -54,7 +54,7 @@ using epee::string_tools::pod_to_hex; using namespace crypto; // Increase when the DB structure changes -#define VERSION 5 +#define VERSION 6 namespace { @@ -326,7 +326,21 @@ typedef struct mdb_block_info_4 uint64_t bi_long_term_block_weight; } mdb_block_info_4; -typedef mdb_block_info_4 mdb_block_info; +typedef struct mdb_block_info_5 +{ + uint64_t bi_height; + uint64_t bi_timestamp; + uint64_t bi_coins; + uint64_t bi_weight; // a size_t really but we need 32-bit compat + uint64_t bi_diff_lo; + uint64_t bi_diff_hi; + crypto::hash bi_hash; + uint64_t bi_cum_rct; // The number of RCT outputs in this block + uint64_t bi_cum_rct_coinbase; // The number of coinbase RCT outputs in this block + uint64_t bi_long_term_block_weight; +} mdb_block_info_5; + +typedef mdb_block_info_5 mdb_block_info; typedef struct blk_height { crypto::hash bh_hash; @@ -769,7 +783,7 @@ estim: } void BlockchainLMDB::add_block(const block& blk, size_t block_weight, uint64_t long_term_block_weight, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, - uint64_t num_rct_outs, const crypto::hash& blk_hash) + uint64_t num_rct_outs, uint64_t num_rct_outs_coinbase, const crypto::hash& blk_hash) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); @@ -820,7 +834,8 @@ void BlockchainLMDB::add_block(const block& blk, size_t block_weight, uint64_t l bi.bi_diff_lo = (cumulative_difficulty & 0xffffffffffffffff).convert_to(); bi.bi_hash = blk_hash; bi.bi_cum_rct = num_rct_outs; - if (blk.major_version >= 4) + bi.bi_cum_rct_coinbase = num_rct_outs_coinbase; + if (blk.major_version >= 4 && m_height > 0) { uint64_t last_height = m_height-1; MDB_val_set(h, last_height); @@ -828,6 +843,7 @@ void BlockchainLMDB::add_block(const block& blk, size_t block_weight, uint64_t l throw1(BLOCK_DNE(lmdb_error("Failed to get block info: ", result).c_str())); const mdb_block_info *bi_prev = (const mdb_block_info*)h.mv_data; bi.bi_cum_rct += bi_prev->bi_cum_rct; + bi.bi_cum_rct_coinbase += bi_prev->bi_cum_rct_coinbase; } bi.bi_long_term_block_weight = long_term_block_weight; @@ -2552,6 +2568,18 @@ std::vector BlockchainLMDB::get_block_cumulative_rct_outputs(const std return res; } +std::vector BlockchainLMDB::get_block_cumulative_rct_coinbase_outputs(uint64_t begin_height, uint64_t end_height) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + + if (end_height <= begin_height) + return {}; + + // We don't have to check heights/counts for validity since get_block_info_64bit_fields handles that + const uint64_t count = end_height - begin_height; + return get_block_info_64bit_fields(begin_height, count, offsetof(mdb_block_info, bi_cum_rct_coinbase)); +} + uint64_t BlockchainLMDB::get_top_block_timestamp() const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); @@ -4515,6 +4543,10 @@ void BlockchainLMDB::fixup() BlockchainDB::fixup(); } +/************************************************************************************************** + *************************** Everything after this point is migrations **************************** + **************************************************************************************************/ + #define RENAME_DB(name) do { \ char n2[] = name; \ MDB_dbi tdbi; \ @@ -5641,6 +5673,160 @@ void BlockchainLMDB::migrate_4_5() txn.commit(); } +void BlockchainLMDB::migrate_5_6() +{ + // Changes: + // * Add the field bi_cum_rct_coinbase to mdb_block_info + + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + uint64_t i; + int result; + mdb_txn_safe txn(false); + MDB_val k, v; + char *ptr; + + MGINFO_YELLOW("Migrating blockchain from DB version 5 to 6 - this may take a while:"); + + do { + LOG_PRINT_L1("migrating block info by adding a field for number of coinbase outputs per block"); + + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + + MDB_stat db_stats; + if ((result = mdb_stat(txn, m_blocks, &db_stats))) + throw0(DB_ERROR(lmdb_error("Failed to query m_blocks: ", result).c_str())); + const uint64_t blockchain_height = db_stats.ms_entries; + + // Before we begin the migration, let's parse every blob, obtain the miner transaction, and + // store the number of RCT outputs in a cumulative distribution in memory. + LOG_PRINT_L1("Collecting old miner tx output information..."); + std::vector cum_coinbase_rct_dist(blockchain_height, 0); + for (uint64_t h = 0; h < blockchain_height; ++h) + { + if (h % 1000 == 0) { + LOGIF(el::Level::Info) { + std::cout << h << " / " << blockchain_height << " \r" << std::flush; + } + } + const block b = get_block_from_height(h); + cum_coinbase_rct_dist[h] = get_num_coinbase_rct_outputs(b); + if (h != 0) + cum_coinbase_rct_dist[h] += cum_coinbase_rct_dist[h - 1]; + } + + /* the block_info table name is the same but the old version and new version + * have incompatible data. Create a new table. We want the name to be similar + * to the old name so that it will occupy the same location in the DB. + */ + MDB_dbi o_block_info = m_block_info; + lmdb_db_open(txn, "block_infn", MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_block_info, "Failed to open db handle for block_infn"); + mdb_set_dupsort(txn, m_block_info, compare_uint64); + + + MDB_cursor *c_blocks; + result = mdb_cursor_open(txn, m_blocks, &c_blocks); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for blocks: ", result).c_str())); + + LOG_PRINT_L1("Writing new block info entries to the database..."); + MDB_cursor *c_old, *c_cur; + i = 0; + while(1) { + if (!(i % 1000)) { + if (i) { + LOGIF(el::Level::Info) { + std::cout << i << " / " << blockchain_height << " \r" << std::flush; + } + txn.commit(); + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + } + result = mdb_cursor_open(txn, m_block_info, &c_cur); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for block_infn: ", result).c_str())); + result = mdb_cursor_open(txn, o_block_info, &c_old); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for block_info: ", result).c_str())); + if (!i) { + result = mdb_stat(txn, m_block_info, &db_stats); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to query m_block_info: ", result).c_str())); + i = db_stats.ms_entries; + } + } + result = mdb_cursor_get(c_old, &k, &v, MDB_NEXT); + if (result == MDB_NOTFOUND) { + txn.commit(); + break; + } + else if (result) + throw0(DB_ERROR(lmdb_error("Failed to get a record from block_info: ", result).c_str())); + + // Copy all old values into new block info struct + const mdb_block_info_4 *bi_old = (const mdb_block_info_4*)v.mv_data; + // Sanity check height, used for migration later + if (bi_old->bi_height >= blockchain_height) + throw0(BLOCK_DNE("Height discrepancy in block info table, perhaps database is corrupted")); + + mdb_block_info_5 bi; + bi.bi_height = bi_old->bi_height; + bi.bi_timestamp = bi_old->bi_timestamp; + bi.bi_coins = bi_old->bi_coins; + bi.bi_weight = bi_old->bi_weight; + bi.bi_diff_lo = bi_old->bi_diff_lo; + bi.bi_diff_hi = bi_old->bi_diff_hi; + bi.bi_hash = bi_old->bi_hash; + bi.bi_cum_rct = bi_old->bi_cum_rct; + bi.bi_cum_rct_coinbase = cum_coinbase_rct_dist[bi_old->bi_height]; // new field! + bi.bi_long_term_block_weight = bi_old->bi_long_term_block_weight; + + MDB_val_set(nv, bi); + result = mdb_cursor_put(c_cur, (MDB_val *)&zerokval, &nv, MDB_APPENDDUP); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to put a record into block_infn: ", result).c_str())); + /* we delete the old records immediately, so the overall DB and mapsize should not grow. + * This is a little slower than just letting mdb_drop() delete it all at the end, but + * it saves a significant amount of disk space. + */ + result = mdb_cursor_del(c_old, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete a record from block_info: ", result).c_str())); + i++; + } + + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + /* Delete the old table */ + result = mdb_drop(txn, o_block_info, 1); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete old block_info table: ", result).c_str())); + + RENAME_DB("block_infn"); + mdb_dbi_close(m_env, m_block_info); + + lmdb_db_open(txn, "block_info", MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_block_info, "Failed to open db handle for block_infn"); + mdb_set_dupsort(txn, m_block_info, compare_uint64); + + txn.commit(); + } while(0); + + uint32_t version = 6; + v.mv_data = (void *)&version; + v.mv_size = sizeof(version); + MDB_val_str(vk, "version"); + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + result = mdb_put(txn, m_properties, &vk, &v, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to update version for the db: ", result).c_str())); + txn.commit(); +} + void BlockchainLMDB::migrate(const uint32_t oldversion) { if (oldversion < 1) @@ -5653,6 +5839,8 @@ void BlockchainLMDB::migrate(const uint32_t oldversion) migrate_3_4(); if (oldversion < 5) migrate_4_5(); + if (oldversion < 6) + migrate_5_6(); } } // namespace cryptonote diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h index c352458b4..fa37110b4 100644 --- a/src/blockchain_db/lmdb/db_lmdb.h +++ b/src/blockchain_db/lmdb/db_lmdb.h @@ -218,6 +218,8 @@ public: virtual std::vector get_block_cumulative_rct_outputs(const std::vector &heights) const; + virtual std::vector get_block_cumulative_rct_coinbase_outputs(uint64_t begin_height, uint64_t end_height) const; + virtual uint64_t get_block_timestamp(const uint64_t& height) const; virtual uint64_t get_top_block_timestamp() const; @@ -371,6 +373,7 @@ private: , const difficulty_type& cumulative_difficulty , const uint64_t& coins_generated , uint64_t num_rct_outs + , uint64_t num_rct_outs_coinbase , const crypto::hash& block_hash ); @@ -443,6 +446,9 @@ private: // migrate from DB version 4 to 5 void migrate_4_5(); + // migrate from DB version 5 to 6: add field `bi_cum_rct_coinbase` to block_info table + void migrate_5_6(); + void cleanup_batch(); private: diff --git a/src/blockchain_db/testdb.h b/src/blockchain_db/testdb.h index 946f26270..666a20319 100644 --- a/src/blockchain_db/testdb.h +++ b/src/blockchain_db/testdb.h @@ -77,6 +77,7 @@ public: virtual cryptonote::block_header get_block_header(const crypto::hash& h) const override { return cryptonote::block_header(); } virtual uint64_t get_block_timestamp(const uint64_t& height) const override { return 0; } virtual std::vector get_block_cumulative_rct_outputs(const std::vector &heights) const override { return {}; } + virtual std::vector get_block_cumulative_rct_coinbase_outputs(uint64_t begin_height, uint64_t end_height) const override { return {}; }; virtual uint64_t get_top_block_timestamp() const override { return 0; } virtual size_t get_block_weight(const uint64_t& height) const override { return 128; } virtual std::vector get_block_weights(uint64_t start_height, size_t count) const override { return {}; } @@ -144,6 +145,7 @@ public: , const cryptonote::difficulty_type& cumulative_difficulty , const uint64_t& coins_generated , uint64_t num_rct_outs + , uint64_t num_rct_outs_coinbase , const crypto::hash& blk_hash ) override { } virtual cryptonote::block get_block_from_height(const uint64_t& height) const override { return cryptonote::block(); } diff --git a/src/cryptonote_basic/cryptonote_basic_impl.cpp b/src/cryptonote_basic/cryptonote_basic_impl.cpp index 9bde20609..f09d43e75 100644 --- a/src/cryptonote_basic/cryptonote_basic_impl.cpp +++ b/src/cryptonote_basic/cryptonote_basic_impl.cpp @@ -183,6 +183,16 @@ namespace cryptonote { return true; } //----------------------------------------------------------------------- + uint64_t get_num_coinbase_rct_outputs(const block& b) + { + if (b.miner_tx.version < 2) + { + return 0; + } + + return b.miner_tx.vout.size(); + } + //----------------------------------------------------------------------- bool get_account_address_from_str( address_parse_info& info , network_type nettype diff --git a/src/cryptonote_basic/cryptonote_basic_impl.h b/src/cryptonote_basic/cryptonote_basic_impl.h index 984bee19f..aa28b3832 100644 --- a/src/cryptonote_basic/cryptonote_basic_impl.h +++ b/src/cryptonote_basic/cryptonote_basic_impl.h @@ -110,6 +110,8 @@ namespace cryptonote { bool is_coinbase(const transaction& tx); + uint64_t get_num_coinbase_rct_outputs(const block& b); + bool operator ==(const cryptonote::transaction& a, const cryptonote::transaction& b); bool operator ==(const cryptonote::block& a, const cryptonote::block& b); } diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index dc20bd1ff..f9c0a453e 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -2410,6 +2410,39 @@ bool Blockchain::get_output_distribution(uint64_t amount, uint64_t from_height, } } //------------------------------------------------------------------ +bool Blockchain::get_rct_coinbase_output_distribution(uint64_t from_height, uint64_t to_height, uint64_t &start_height, std::vector &distribution, uint64_t &base) const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + + CHECK_AND_ASSERT_MES(to_height >= from_height, false, "to_height < from_height"); + + // rct outputs don't exist before v4 + switch (m_nettype) + { + case STAGENET: start_height = stagenet_hard_forks[3].height; break; + case TESTNET: start_height = testnet_hard_forks[3].height; break; + case MAINNET: start_height = mainnet_hard_forks[3].height; break; + case FAKECHAIN: start_height = 0; break; + default: MERROR("network type not recognized: " << m_nettype); return false; + } + + start_height = from_height = std::max(from_height, start_height); + to_height = std::max(to_height, from_height); + + const uint64_t db_height = m_db->height(); + CHECK_AND_ASSERT_MES(db_height, false, "No block data"); + CHECK_AND_ASSERT_MES(from_height < db_height, false, "from_height is too high"); + CHECK_AND_ASSERT_MES(to_height < db_height, false, "to_height is too high"); + + if (from_height) + base = m_db->get_block_cumulative_rct_coinbase_outputs(from_height - 1, from_height)[0]; + else + base = 0; + + distribution = m_db->get_block_cumulative_rct_coinbase_outputs(from_height, to_height + 1); + return true; +} +//------------------------------------------------------------------ // This function takes a list of block hashes from another node // on the network to find where the split point is between us and them. // This is used to see what to send another node that needs to sync. diff --git a/src/cryptonote_core/blockchain.h b/src/cryptonote_core/blockchain.h index a45d3ec60..addf1c6dc 100644 --- a/src/cryptonote_core/blockchain.h +++ b/src/cryptonote_core/blockchain.h @@ -572,6 +572,19 @@ namespace cryptonote */ bool get_output_distribution(uint64_t amount, uint64_t from_height, uint64_t to_height, uint64_t &start_height, std::vector &distribution, uint64_t &base) const; + /** + * @brief gets per-block distribution of rct coinbase outputs (cumulative) + * + * @param from_height the height before which we do not care about the data (inclusive) + * @param to_height the height after which we do not care about the data (inclusive) + * @param[out] start_height the height of the first rct coinbase output + * @param[out] distribution the number of rct coinbase outputs in each block starting from start_height (inclusive) + * @param[out] base how many rct coinbase outputs are before the stated distribution + * + * @return false if requested heights are not available or other failure, true for success + */ + bool get_rct_coinbase_output_distribution(uint64_t from_height, uint64_t to_height, uint64_t &start_height, std::vector &distribution, uint64_t &base) const; + /** * @brief gets the global indices for outputs from a given transaction * diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index cb347110d..906538d64 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -3312,45 +3312,7 @@ namespace cryptonote if (use_bootstrap_daemon_if_necessary(invoke_http_mode::JON_RPC, "get_output_distribution", req, res, r)) return r; - const bool restricted = m_restricted && ctx; - if (restricted && req.amounts != std::vector(1, 0)) - { - error_resp.code = CORE_RPC_ERROR_CODE_RESTRICTED; - error_resp.message = "Restricted RPC can only get output distribution for rct outputs. Use your own node."; - return false; - } - - size_t n_0 = 0, n_non0 = 0; - for (uint64_t amount: req.amounts) - if (amount) ++n_non0; else ++n_0; - CHECK_PAYMENT_MIN1(req, res, n_0 * COST_PER_OUTPUT_DISTRIBUTION_0 + n_non0 * COST_PER_OUTPUT_DISTRIBUTION, false); - - try - { - // 0 is placeholder for the whole chain - const uint64_t req_to_height = req.to_height ? req.to_height : (m_core.get_current_blockchain_height() - 1); - for (uint64_t amount: req.amounts) - { - auto data = rpc::RpcHandler::get_output_distribution([this](uint64_t amount, uint64_t from, uint64_t to, uint64_t &start_height, std::vector &distribution, uint64_t &base) { return m_core.get_output_distribution(amount, from, to, start_height, distribution, base); }, amount, req.from_height, req_to_height, [this](uint64_t height) { return m_core.get_blockchain_storage().get_db().get_block_hash_from_height(height); }, req.cumulative, m_core.get_current_blockchain_height()); - if (!data) - { - error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; - error_resp.message = "Failed to get output distribution"; - return false; - } - - res.distributions.push_back({std::move(*data), amount, "", req.binary, req.compress}); - } - } - catch (const std::exception &e) - { - error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; - error_resp.message = "Failed to get output distribution"; - return false; - } - - res.status = CORE_RPC_STATUS_OK; - return true; + return on_get_output_distribution_generalized(req, res, error_resp, ctx, &tracker); } //------------------------------------------------------------------------------------------------------------------------------ bool core_rpc_server::on_get_output_distribution_bin(const COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::request& req, COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::response& res, const connection_context *ctx) @@ -3361,13 +3323,27 @@ namespace cryptonote if (use_bootstrap_daemon_if_necessary(invoke_http_mode::BIN, "/get_output_distribution.bin", req, res, r)) return r; + if (!req.binary) + { + res.status = "Binary only call"; + return true; + } + + epee::json_rpc::error error_resp; + return on_get_output_distribution_generalized(req, res, error_resp, ctx, &tracker); + } + //------------------------------------------------------------------------------------------------------------------------------ + bool core_rpc_server::on_get_output_distribution_generalized(const COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::request& req, COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx, void* tracker_erased) + { const bool restricted = m_restricted && ctx; if (restricted && req.amounts != std::vector(1, 0)) { - res.status = "Restricted RPC can only get output distribution for rct outputs. Use your own node."; + error_resp.code = CORE_RPC_ERROR_CODE_RESTRICTED; + error_resp.message = res.status = "Restricted RPC can only get output distribution for rct outputs. Use your own node."; return false; } + RPCTracker& tracker = *reinterpret_cast(tracker_erased); size_t n_0 = 0, n_non0 = 0; for (uint64_t amount: req.amounts) if (amount) ++n_non0; else ++n_0; @@ -3375,11 +3351,6 @@ namespace cryptonote res.status = "Failed"; - if (!req.binary) - { - res.status = "Binary only call"; - return true; - } try { // 0 is placeholder for the whole chain @@ -3389,17 +3360,47 @@ namespace cryptonote auto data = rpc::RpcHandler::get_output_distribution([this](uint64_t amount, uint64_t from, uint64_t to, uint64_t &start_height, std::vector &distribution, uint64_t &base) { return m_core.get_output_distribution(amount, from, to, start_height, distribution, base); }, amount, req.from_height, req_to_height, [this](uint64_t height) { return m_core.get_blockchain_storage().get_db().get_block_hash_from_height(height); }, req.cumulative, m_core.get_current_blockchain_height()); if (!data) { - res.status = "Failed to get output distribution"; - return true; + error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; + error_resp.message = res.status = "Failed to get output distribution"; + return false; } res.distributions.push_back({std::move(*data), amount, "", req.binary, req.compress}); } + + if (req.get_rct_coinbase) + { + COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::distribution& out_dist = res.rct_coinbase_distribution; + out_dist.amount = 0; + out_dist.binary = req.binary; + out_dist.compress = req.compress; + if (!m_core.get_blockchain_storage().get_rct_coinbase_output_distribution( + req.from_height + , req_to_height + , out_dist.data.start_height + , out_dist.data.distribution + , out_dist.data.base + )) + { + // control flow switch to catch branch + throw std::runtime_error("Failed to get rct coinbase output distribution"); + } + + if (!req.cumulative && out_dist.data.distribution.size()) + { + // The database stores this distribution as cumulative by default so decumulate + for (size_t i = out_dist.data.distribution.size() - 1; i >= 1; --i) + { + out_dist.data.distribution[i] -= out_dist.data.distribution[i - 1]; + } + } + } } catch (const std::exception &e) { - res.status = "Failed to get output distribution"; - return true; + error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; + error_resp.message = res.status = e.what(); + return false; } res.status = CORE_RPC_STATUS_OK; diff --git a/src/rpc/core_rpc_server.h b/src/rpc/core_rpc_server.h index 790d5eb23..7b36d817f 100644 --- a/src/rpc/core_rpc_server.h +++ b/src/rpc/core_rpc_server.h @@ -285,7 +285,8 @@ private: bool use_bootstrap_daemon_if_necessary(const invoke_http_mode &mode, const std::string &command_name, const typename COMMAND_TYPE::request& req, typename COMMAND_TYPE::response& res, bool &r); bool get_block_template(const account_public_address &address, const crypto::hash *prev_block, const cryptonote::blobdata &extra_nonce, size_t &reserved_offset, cryptonote::difficulty_type &difficulty, uint64_t &height, uint64_t &expected_reward, block &b, uint64_t &seed_height, crypto::hash &seed_hash, crypto::hash &next_seed_hash, epee::json_rpc::error &error_resp); bool check_payment(const std::string &client, uint64_t payment, const std::string &rpc, bool same_ts, std::string &message, uint64_t &credits, std::string &top_hash); - + bool on_get_output_distribution_generalized(const COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::request& req, COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx, void* tracker_erased); + core& m_core; nodetool::node_server >& m_p2p; boost::shared_mutex m_bootstrap_daemon_mutex; diff --git a/src/rpc/core_rpc_server_commands_defs.h b/src/rpc/core_rpc_server_commands_defs.h index 7f1581d0c..68b619e6d 100644 --- a/src/rpc/core_rpc_server_commands_defs.h +++ b/src/rpc/core_rpc_server_commands_defs.h @@ -2447,6 +2447,7 @@ namespace cryptonote bool cumulative; bool binary; bool compress; + bool get_rct_coinbase; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE_PARENT(rpc_access_request_base) @@ -2456,6 +2457,7 @@ namespace cryptonote KV_SERIALIZE_OPT(cumulative, false) KV_SERIALIZE_OPT(binary, true) KV_SERIALIZE_OPT(compress, false) + KV_SERIALIZE_OPT(get_rct_coinbase, false) END_KV_SERIALIZE_MAP() }; typedef epee::misc_utils::struct_init request; @@ -2500,15 +2502,27 @@ namespace cryptonote KV_SERIALIZE_N(data.distribution, "distribution") KV_SERIALIZE_N(data.base, "base") END_KV_SERIALIZE_MAP() + + bool operator==(const distribution& other) const + { + return data == other.data + && amount == other.amount + && compressed_data == other.compressed_data + && binary == other.binary + && compress == other.compress + ; + } }; struct response_t: public rpc_access_response_base { std::vector distributions; + distribution rct_coinbase_distribution; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE_PARENT(rpc_access_response_base) KV_SERIALIZE(distributions) + KV_SERIALIZE_OPT(rct_coinbase_distribution, decltype(rct_coinbase_distribution)()) END_KV_SERIALIZE_MAP() }; typedef epee::misc_utils::struct_init response; diff --git a/src/rpc/rpc_handler.h b/src/rpc/rpc_handler.h index e13d3036f..54cab32cb 100644 --- a/src/rpc/rpc_handler.h +++ b/src/rpc/rpc_handler.h @@ -47,6 +47,14 @@ struct output_distribution_data std::vector distribution; std::uint64_t start_height; std::uint64_t base; + + bool operator==(const output_distribution_data& other) const + { + return distribution == other.distribution + && start_height == other.start_height + && base == other.base + ; + } }; class RpcHandler diff --git a/tests/block_weight/block_weight.cpp b/tests/block_weight/block_weight.cpp index 7cd0d572b..47b763a5c 100644 --- a/tests/block_weight/block_weight.cpp +++ b/tests/block_weight/block_weight.cpp @@ -65,6 +65,7 @@ public: , const cryptonote::difficulty_type& cumulative_difficulty , const uint64_t& coins_generated , uint64_t num_rct_outs + , uint64_t num_rct_outs_coinbase , const crypto::hash& blk_hash ) override { blocks.push_back({block_weight, long_term_block_weight}); diff --git a/tests/core_tests/chaingen.cpp b/tests/core_tests/chaingen.cpp index 9f6fe5387..e4393e103 100644 --- a/tests/core_tests/chaingen.cpp +++ b/tests/core_tests/chaingen.cpp @@ -87,6 +87,7 @@ namespace , const cryptonote::difficulty_type& cumulative_difficulty , const uint64_t& coins_generated , uint64_t num_rct_outs + , uint64_t num_rct_outs_coinbase , const crypto::hash& blk_hash ) override { @@ -174,7 +175,7 @@ static std::unique_ptr init_blockchain(const std::vector const block *blk = &boost::get(ev); auto blk_hash = get_block_hash(*blk); - bdb->add_block(*blk, 1, 1, 1, 0, 0, blk_hash); + bdb->add_block(*blk, 1, 1, 1, 0, 0, 0, blk_hash); } bool r = blockchain->init(bdb, nettype, true, test_options, 2, nullptr); diff --git a/tests/functional_tests/get_output_distribution.py b/tests/functional_tests/get_output_distribution.py index 08019121a..a54541515 100755 --- a/tests/functional_tests/get_output_distribution.py +++ b/tests/functional_tests/get_output_distribution.py @@ -35,11 +35,14 @@ from __future__ import print_function from framework.daemon import Daemon from framework.wallet import Wallet +import json + class GetOutputDistributionTest(): def run_test(self): self.reset() self.create() self.test_get_output_distribution() + self.test_get_rct_coinbase_output_distribution() def reset(self): print('Resetting blockchain') @@ -212,6 +215,55 @@ class GetOutputDistributionTest(): for h in range(len(d.distribution)): assert d.distribution[h] == 0 + def test_get_rct_coinbase_output_distribution(self): + print("Test get_rct_coinbase_output_distribution") + + daemon = Daemon() + + # Check that we have some RCT blocks/transactions to test against + chain_height = daemon.get_height()['height'] + top_header = daemon.get_block_header_by_height(chain_height - 1) + top_major_version = top_header['block_header']['major_version'] + assert top_major_version >= 4 + + # Fetch the cumulative RCT coinbase distribution + res = daemon.get_output_distribution(get_rct_coinbase = True, cumulative = True, binary=False) + dist_info = res['rct_coinbase_distribution'] + amount = dist_info['amount'] + begin_height = dist_info['start_height'] + base = dist_info['base'] + dist = dist_info['distribution'] + end_height = begin_height + len(dist) + assert amount == 0 + assert begin_height < chain_height + assert end_height == chain_height + assert base == 0 + assert len(dist) > 0 + assert len(dist) == chain_height - begin_height # Check to_height=0 grabs whole chain after v4 fork + assert all((x >= 0 for x in dist)) # Check all elements of dist positive + assert all((dist[i - 1] <= dist[i] for i in range(1, len(dist)))) # Check cumulative + + print("Found {} total RCT coinbase outputs".format(dist[-1])) + + # Check that the non-cumulative distribution is valid to corresponding cumulative + res = daemon.get_output_distribution(get_rct_coinbase = True, cumulative = False, binary=False) + non_cum_dist = res['rct_coinbase_distribution']['distribution'] + assert len(non_cum_dist) == len(dist) + assert dist[0] == non_cum_dist[0] + assert all(((dist[i] - dist[i - 1]) == non_cum_dist[i] for i in range(1, len(dist)))) + + # Scan the whole chain from begin_height and check values of RCT coinbase distribution + # for each block. + for h in range(begin_height, chain_height): + dist_index = h - begin_height + res = daemon.get_block(height = h) + block_body = json.loads(res['json']) + miner_tx = block_body['miner_tx'] + is_rct_miner = miner_tx['version'] >= 2 + if not is_rct_miner: + assert non_cum_dist[dist_index] == 0 + else: + assert non_cum_dist[dist_index] == len(miner_tx['vout']) class Guard: def __enter__(self): diff --git a/tests/unit_tests/hardfork.cpp b/tests/unit_tests/hardfork.cpp index e6602bdb3..3a928a6d3 100644 --- a/tests/unit_tests/hardfork.cpp +++ b/tests/unit_tests/hardfork.cpp @@ -41,6 +41,11 @@ using namespace cryptonote; #define BLOCKS_PER_YEAR 525960 #define SECONDS_PER_YEAR 31557600 +#define ADD_BLOCK_DEFAULT(db, block) \ + do { \ + db.add_block((block), 0, 0, 0, 0, 0, 0, crypto::hash()); \ + } while (0); \ + namespace { @@ -53,6 +58,7 @@ public: , const difficulty_type& cumulative_difficulty , const uint64_t& coins_generated , uint64_t num_rct_outs + , uint64_t num_rct_outs_coinbase , const crypto::hash& blk_hash ) override { blocks.push_back(blk); @@ -107,20 +113,20 @@ TEST(major, Only) ASSERT_FALSE(hf.add(mkblock(0, 2), 0)); ASSERT_FALSE(hf.add(mkblock(2, 2), 0)); ASSERT_TRUE(hf.add(mkblock(1, 2), 0)); - db.add_block(mkblock(1, 1), 0, 0, 0, 0, 0, crypto::hash()); + ADD_BLOCK_DEFAULT(db, mkblock(1, 1)); // block height 1, only version 1 is accepted ASSERT_FALSE(hf.add(mkblock(0, 2), 1)); ASSERT_FALSE(hf.add(mkblock(2, 2), 1)); ASSERT_TRUE(hf.add(mkblock(1, 2), 1)); - db.add_block(mkblock(1, 1), 0, 0, 0, 0, 0, crypto::hash()); + ADD_BLOCK_DEFAULT(db, mkblock(1, 1)); // block height 2, only version 2 is accepted ASSERT_FALSE(hf.add(mkblock(0, 2), 2)); ASSERT_FALSE(hf.add(mkblock(1, 2), 2)); ASSERT_FALSE(hf.add(mkblock(3, 2), 2)); ASSERT_TRUE(hf.add(mkblock(2, 2), 2)); - db.add_block(mkblock(2, 1), 0, 0, 0, 0, 0, crypto::hash()); + ADD_BLOCK_DEFAULT(db, mkblock(2, 1)); } TEST(empty_hardforks, Success) @@ -134,7 +140,7 @@ TEST(empty_hardforks, Success) ASSERT_TRUE(hf.get_state(time(NULL) + 3600*24*400) == HardFork::Ready); for (uint64_t h = 0; h <= 10; ++h) { - db.add_block(mkblock(hf, h, 1), 0, 0, 0, 0, 0, crypto::hash()); + ADD_BLOCK_DEFAULT(db, mkblock(hf, h, 1)); ASSERT_TRUE(hf.add(db.get_block_from_height(h), h)); } ASSERT_EQ(hf.get(0), 1); @@ -168,14 +174,14 @@ TEST(check_for_height, Success) for (uint64_t h = 0; h <= 4; ++h) { ASSERT_TRUE(hf.check_for_height(mkblock(1, 1), h)); ASSERT_FALSE(hf.check_for_height(mkblock(2, 2), h)); // block version is too high - db.add_block(mkblock(hf, h, 1), 0, 0, 0, 0, 0, crypto::hash()); + ADD_BLOCK_DEFAULT(db, mkblock(hf, h, 1)); ASSERT_TRUE(hf.add(db.get_block_from_height(h), h)); } for (uint64_t h = 5; h <= 10; ++h) { ASSERT_FALSE(hf.check_for_height(mkblock(1, 1), h)); // block version is too low ASSERT_TRUE(hf.check_for_height(mkblock(2, 2), h)); - db.add_block(mkblock(hf, h, 2), 0, 0, 0, 0, 0, crypto::hash()); + ADD_BLOCK_DEFAULT(db, mkblock(hf, h, 2)); ASSERT_TRUE(hf.add(db.get_block_from_height(h), h)); } } @@ -192,19 +198,19 @@ TEST(get, next_version) for (uint64_t h = 0; h <= 4; ++h) { ASSERT_EQ(2, hf.get_next_version()); - db.add_block(mkblock(hf, h, 1), 0, 0, 0, 0, 0, crypto::hash()); + ADD_BLOCK_DEFAULT(db, mkblock(hf, h, 1)); ASSERT_TRUE(hf.add(db.get_block_from_height(h), h)); } for (uint64_t h = 5; h <= 9; ++h) { ASSERT_EQ(4, hf.get_next_version()); - db.add_block(mkblock(hf, h, 2), 0, 0, 0, 0, 0, crypto::hash()); + ADD_BLOCK_DEFAULT(db, mkblock(hf, h, 2)); ASSERT_TRUE(hf.add(db.get_block_from_height(h), h)); } for (uint64_t h = 10; h <= 15; ++h) { ASSERT_EQ(4, hf.get_next_version()); - db.add_block(mkblock(hf, h, 4), 0, 0, 0, 0, 0, crypto::hash()); + ADD_BLOCK_DEFAULT(db, mkblock(hf, h, 4)); ASSERT_TRUE(hf.add(db.get_block_from_height(h), h)); } } @@ -245,7 +251,7 @@ TEST(steps_asap, Success) hf.init(); for (uint64_t h = 0; h < 10; ++h) { - db.add_block(mkblock(hf, h, 9), 0, 0, 0, 0, 0, crypto::hash()); + ADD_BLOCK_DEFAULT(db, mkblock(hf, h, 9)); ASSERT_TRUE(hf.add(db.get_block_from_height(h), h)); } @@ -272,7 +278,7 @@ TEST(steps_1, Success) hf.init(); for (uint64_t h = 0 ; h < 10; ++h) { - db.add_block(mkblock(hf, h, h+1), 0, 0, 0, 0, 0, crypto::hash()); + ADD_BLOCK_DEFAULT(db, mkblock(hf, h, h+1)); ASSERT_TRUE(hf.add(db.get_block_from_height(h), h)); } @@ -297,7 +303,7 @@ TEST(reorganize, Same) // index 0 1 2 3 4 5 6 7 8 9 static const uint8_t block_versions[] = { 1, 1, 4, 4, 7, 7, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9 }; for (uint64_t h = 0; h < 20; ++h) { - db.add_block(mkblock(hf, h, block_versions[h]), 0, 0, 0, 0, 0, crypto::hash()); + ADD_BLOCK_DEFAULT(db, mkblock(hf, h, block_versions[h])); ASSERT_TRUE(hf.add(db.get_block_from_height(h), h)); } @@ -328,7 +334,7 @@ TEST(reorganize, Changed) static const uint8_t block_versions[] = { 1, 1, 4, 4, 7, 7, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9 }; static const uint8_t expected_versions[] = { 1, 1, 1, 1, 1, 1, 4, 4, 7, 7, 9, 9, 9, 9, 9, 9 }; for (uint64_t h = 0; h < 16; ++h) { - db.add_block(mkblock(hf, h, block_versions[h]), 0, 0, 0, 0, 0, crypto::hash()); + ADD_BLOCK_DEFAULT(db, mkblock(hf, h, block_versions[h])); ASSERT_TRUE (hf.add(db.get_block_from_height(h), h)); } @@ -348,7 +354,7 @@ TEST(reorganize, Changed) ASSERT_EQ(db.height(), 3); hf.reorganize_from_block_height(2); for (uint64_t h = 3; h < 16; ++h) { - db.add_block(mkblock(hf, h, block_versions_new[h]), 0, 0, 0, 0, 0, crypto::hash()); + ADD_BLOCK_DEFAULT(db, mkblock(hf, h, block_versions_new[h])); bool ret = hf.add(db.get_block_from_height(h), h); ASSERT_EQ (ret, h < 15); } @@ -372,7 +378,7 @@ TEST(voting, threshold) for (uint64_t h = 0; h <= 8; ++h) { uint8_t v = 1 + !!(h % 8); - db.add_block(mkblock(hf, h, v), 0, 0, 0, 0, 0, crypto::hash()); + ADD_BLOCK_DEFAULT(db, mkblock(hf, h, v)); bool ret = hf.add(db.get_block_from_height(h), h); if (h >= 8 && threshold == 87) { // for threshold 87, we reach the treshold at height 7, so from height 8, hard fork to version 2, but 8 tries to add 1 @@ -406,7 +412,7 @@ TEST(voting, different_thresholds) static const uint8_t expected_versions[] = { 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4 }; for (uint64_t h = 0; h < sizeof(block_versions) / sizeof(block_versions[0]); ++h) { - db.add_block(mkblock(hf, h, block_versions[h]), 0, 0, 0, 0, 0, crypto::hash()); + ADD_BLOCK_DEFAULT(db, mkblock(hf, h, block_versions[h])); bool ret = hf.add(db.get_block_from_height(h), h); ASSERT_EQ(ret, true); } @@ -459,7 +465,7 @@ TEST(voting, info) ASSERT_EQ(expected_thresholds[h], threshold); ASSERT_EQ(4, voting); - db.add_block(mkblock(hf, h, block_versions[h]), 0, 0, 0, 0, 0, crypto::hash()); + ADD_BLOCK_DEFAULT(db, mkblock(hf, h, block_versions[h])); ASSERT_TRUE(hf.add(db.get_block_from_height(h), h)); } } @@ -522,7 +528,7 @@ TEST(reorganize, changed) #define ADD(v, h, a) \ do { \ cryptonote::block b = mkblock(hf, h, v); \ - db.add_block(b, 0, 0, 0, 0, 0, crypto::hash()); \ + ADD_BLOCK_DEFAULT(db, b); \ ASSERT_##a(hf.add(b, h)); \ } while(0) #define ADD_TRUE(v, h) ADD(v, h, TRUE) diff --git a/tests/unit_tests/long_term_block_weight.cpp b/tests/unit_tests/long_term_block_weight.cpp index b9ce2b247..8e2864898 100644 --- a/tests/unit_tests/long_term_block_weight.cpp +++ b/tests/unit_tests/long_term_block_weight.cpp @@ -57,6 +57,7 @@ public: , const cryptonote::difficulty_type& cumulative_difficulty , const uint64_t& coins_generated , uint64_t num_rct_outs + , uint64_t num_rct_outs_coinbase , const crypto::hash& blk_hash ) override { blocks.push_back({block_weight, long_term_block_weight}); diff --git a/utils/python-rpc/framework/daemon.py b/utils/python-rpc/framework/daemon.py index 43a1aa469..717390d46 100644 --- a/utils/python-rpc/framework/daemon.py +++ b/utils/python-rpc/framework/daemon.py @@ -375,7 +375,7 @@ class Daemon(object): } return self.rpc.send_json_rpc_request(get_coinbase_tx_sum) - def get_output_distribution(self, amounts = [], from_height = 0, to_height = 0, cumulative = False, binary = False, compress = False, client = ""): + def get_output_distribution(self, amounts = [], from_height = 0, to_height = 0, cumulative = False, binary = False, compress = False, client = "", get_rct_coinbase = False): get_output_distribution = { 'method': 'get_output_distribution', 'params': { @@ -386,6 +386,7 @@ class Daemon(object): 'cumulative': cumulative, 'binary': binary, 'compress': compress, + 'get_rct_coinbase': get_rct_coinbase }, 'jsonrpc': '2.0', 'id': '0'