blockchain_db/core_rpc_server: Get coinbase RCT output distribution

This doesn't affect decoy selection or the protocol at all, just edits `BlockchainLMDB` to add an extra
field to the block info structs, and updates the `get_output_distribution` RPC API to allow wallets to
obtain the distribution of RCT coinbase outputs per block. This change requires a database migration and
will add ~2MB to the database every year.

How to request the information:
1. In the `get_output_distribution[.bin]` RPC command, set a boolean field `get_rct_coinbase` to `true`.
2. The RCT coinbase distribution will be returned in the response field `rct_coinbase_distribution`.

This change will enable extremely fast fetching of the RCT coinbase distribution, which will let wallets
efficiently perform decoy selection across only coinbase outputs or only non-coinbase outputs.

src/blockchain_db/blockchain_db.cpp

@@ -274,6 +274,7 @@ uint64_t BlockchainDB::add_block( const std::pair<block, blobdata>& blck
   time1 = epee::misc_utils::get_tick_count();
 
   uint64_t num_rct_outs = 0;
+  const uint64_t num_rct_outs_coinbase = get_num_coinbase_rct_outputs(blck.first);
   blobdata miner_bd = tx_to_blob(blk.miner_tx);
   add_transaction(blk_hash, std::make_pair(blk.miner_tx, blobdata_ref(miner_bd)));
   if (blk.miner_tx.version == 2)
@@ -296,7 +297,7 @@ uint64_t BlockchainDB::add_block( const std::pair<block, blobdata>& blck
 
   // call out to subclass implementation to add the block & metadata
   time1 = epee::misc_utils::get_tick_count();
-  add_block(blk, block_weight, long_term_block_weight, cumulative_difficulty, coins_generated, num_rct_outs, blk_hash);
+  add_block(blk, block_weight, long_term_block_weight, cumulative_difficulty, coins_generated, num_rct_outs, num_rct_outs_coinbase, blk_hash);
   TIME_MEASURE_FINISH(time1);
   time_add_block1 += time1;
 

src/blockchain_db/blockchain_db.h

@@ -397,6 +397,8 @@ private:
    * @param long_term_block_weight the long term block weight of the block (transactions and all)
    * @param cumulative_difficulty the accumulated difficulty after this block
    * @param coins_generated the number of coins generated total after this block
+   * @param num_rct_outs the number of total RCT tx outputs, including coinbase, in this block
+   * @param num_rct_outs_coinbase same as num_rct_outs, but ONLY including coinbase RCT outputs
    * @param blk_hash the hash of the block
    */
   virtual void add_block( const block& blk
@@ -405,6 +407,7 @@ private:
                 , const difficulty_type& cumulative_difficulty
                 , const uint64_t& coins_generated
                 , uint64_t num_rct_outs
+                , uint64_t num_rct_outs_coinbase
                 , const crypto::hash& blk_hash
                 ) = 0;
 
@@ -977,6 +980,24 @@ public:
    */
   virtual std::vector<uint64_t> get_block_cumulative_rct_outputs(const std::vector<uint64_t> &heights) const = 0;
 
+  /**
+   * @brief fetch a block's cumulative number of rct *coinbase* outputs
+   *
+   * Same idea as get_block_cumulative_rct_outputs, but we include ONLY outputs from miner txs, and
+   * we perform the fetch on contiguous blocks in range [start_height, end_height).
+   *
+   * The subclass should return the numer of rct coinbase outputs in the blockchain
+   * up to the block with the given height (inclusive).
+   *
+   * If the block does not exist, the subclass should throw BLOCK_DNE
+   *
+   * @param begin_height the start height to fetch (inclusive)
+   * @param end_height the end height to fetch (exclusive)
+   *
+   * @return a list of the cumulative number of rct coinbase outputs per block
+   */
+  virtual std::vector<uint64_t> get_block_cumulative_rct_coinbase_outputs(uint64_t begin_height, uint64_t end_height) const = 0;
+
   /**
    * @brief fetch the top block's timestamp
    *

src/blockchain_db/lmdb/db_lmdb.cpp

@@ -54,7 +54,7 @@ using epee::string_tools::pod_to_hex;
 using namespace crypto;
 
 // Increase when the DB structure changes
-#define VERSION 5
+#define VERSION 6
 
 namespace
 {
@@ -326,7 +326,21 @@ typedef struct mdb_block_info_4
   uint64_t bi_long_term_block_weight;
 } mdb_block_info_4;
 
-typedef mdb_block_info_4 mdb_block_info;
+typedef struct mdb_block_info_5
+{
+  uint64_t bi_height;
+  uint64_t bi_timestamp;
+  uint64_t bi_coins;
+  uint64_t bi_weight; // a size_t really but we need 32-bit compat
+  uint64_t bi_diff_lo;
+  uint64_t bi_diff_hi;
+  crypto::hash bi_hash;
+  uint64_t bi_cum_rct; // The number of RCT outputs in this block
+  uint64_t bi_cum_rct_coinbase; // The number of coinbase RCT outputs in this block
+  uint64_t bi_long_term_block_weight;
+} mdb_block_info_5;
+
+typedef mdb_block_info_5 mdb_block_info;
 
 typedef struct blk_height {
     crypto::hash bh_hash;
@@ -769,7 +783,7 @@ estim:
 }
 
 void BlockchainLMDB::add_block(const block& blk, size_t block_weight, uint64_t long_term_block_weight, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated,
-    uint64_t num_rct_outs, const crypto::hash& blk_hash)
+    uint64_t num_rct_outs, uint64_t num_rct_outs_coinbase, const crypto::hash& blk_hash)
 {
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
   check_open();
@@ -820,7 +834,8 @@ void BlockchainLMDB::add_block(const block& blk, size_t block_weight, uint64_t l
   bi.bi_diff_lo = (cumulative_difficulty & 0xffffffffffffffff).convert_to<uint64_t>();
   bi.bi_hash = blk_hash;
   bi.bi_cum_rct = num_rct_outs;
-  if (blk.major_version >= 4)
+  bi.bi_cum_rct_coinbase = num_rct_outs_coinbase;
+  if (blk.major_version >= 4 && m_height > 0)
   {
     uint64_t last_height = m_height-1;
     MDB_val_set(h, last_height);
@@ -828,6 +843,7 @@ void BlockchainLMDB::add_block(const block& blk, size_t block_weight, uint64_t l
         throw1(BLOCK_DNE(lmdb_error("Failed to get block info: ", result).c_str()));
     const mdb_block_info *bi_prev = (const mdb_block_info*)h.mv_data;
     bi.bi_cum_rct += bi_prev->bi_cum_rct;
+    bi.bi_cum_rct_coinbase += bi_prev->bi_cum_rct_coinbase;
   }
   bi.bi_long_term_block_weight = long_term_block_weight;
 
@@ -2552,6 +2568,18 @@ std::vector<uint64_t> BlockchainLMDB::get_block_cumulative_rct_outputs(const std
   return res;
 }
 
+std::vector<uint64_t> BlockchainLMDB::get_block_cumulative_rct_coinbase_outputs(uint64_t begin_height, uint64_t end_height) const
+{
+  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+
+  if (end_height <= begin_height)
+    return {};
+
+  // We don't have to check heights/counts for validity since get_block_info_64bit_fields handles that
+  const uint64_t count = end_height - begin_height;
+  return get_block_info_64bit_fields(begin_height, count, offsetof(mdb_block_info, bi_cum_rct_coinbase));
+}
+
 uint64_t BlockchainLMDB::get_top_block_timestamp() const
 {
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
@@ -4515,6 +4543,10 @@ void BlockchainLMDB::fixup()
   BlockchainDB::fixup();
 }
 
+/**************************************************************************************************
+ *************************** Everything after this point is migrations ****************************
+ **************************************************************************************************/
+
 #define RENAME_DB(name) do { \
     char n2[] = name; \
     MDB_dbi tdbi; \
@@ -5641,6 +5673,160 @@ void BlockchainLMDB::migrate_4_5()
   txn.commit();
 }
 
+void BlockchainLMDB::migrate_5_6()
+{
+  // Changes:
+  // * Add the field bi_cum_rct_coinbase to mdb_block_info
+
+  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+  uint64_t i;
+  int result;
+  mdb_txn_safe txn(false);
+  MDB_val k, v;
+  char *ptr;
+
+  MGINFO_YELLOW("Migrating blockchain from DB version 5 to 6 - this may take a while:");
+
+  do {
+    LOG_PRINT_L1("migrating block info by adding a field for number of coinbase outputs per block");
+
+    result = mdb_txn_begin(m_env, NULL, 0, txn);
+    if (result)
+      throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str()));
+
+    MDB_stat db_stats;
+    if ((result = mdb_stat(txn, m_blocks, &db_stats)))
+      throw0(DB_ERROR(lmdb_error("Failed to query m_blocks: ", result).c_str()));
+    const uint64_t blockchain_height = db_stats.ms_entries;
+
+    // Before we begin the migration, let's parse every blob, obtain the miner transaction, and
+    // store the number of RCT outputs in a cumulative distribution in memory.
+    LOG_PRINT_L1("Collecting old miner tx output information...");
+    std::vector<uint64_t> cum_coinbase_rct_dist(blockchain_height, 0);
+    for (uint64_t h = 0; h < blockchain_height; ++h)
+    {
+      if (h % 1000 == 0) {
+        LOGIF(el::Level::Info) {
+          std::cout << h << " / " << blockchain_height << "  \r" << std::flush;
+        }
+      }
+      const block b = get_block_from_height(h);
+      cum_coinbase_rct_dist[h] = get_num_coinbase_rct_outputs(b);
+      if (h != 0)
+        cum_coinbase_rct_dist[h] += cum_coinbase_rct_dist[h - 1];
+    }
+
+    /* the block_info table name is the same but the old version and new version
+     * have incompatible data. Create a new table. We want the name to be similar
+     * to the old name so that it will occupy the same location in the DB.
+     */
+    MDB_dbi o_block_info = m_block_info;
+    lmdb_db_open(txn, "block_infn", MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_block_info, "Failed to open db handle for block_infn");
+    mdb_set_dupsort(txn, m_block_info, compare_uint64);
+
+
+    MDB_cursor *c_blocks;
+    result = mdb_cursor_open(txn, m_blocks, &c_blocks);
+    if (result)
+      throw0(DB_ERROR(lmdb_error("Failed to open a cursor for blocks: ", result).c_str()));
+
+    LOG_PRINT_L1("Writing new block info entries to the database...");
+    MDB_cursor *c_old, *c_cur;
+    i = 0;
+    while(1) {
+      if (!(i % 1000)) {
+        if (i) {
+          LOGIF(el::Level::Info) {
+            std::cout << i << " / " << blockchain_height << "  \r" << std::flush;
+          }
+          txn.commit();
+          result = mdb_txn_begin(m_env, NULL, 0, txn);
+          if (result)
+            throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str()));
+        }
+        result = mdb_cursor_open(txn, m_block_info, &c_cur);
+        if (result)
+          throw0(DB_ERROR(lmdb_error("Failed to open a cursor for block_infn: ", result).c_str()));
+        result = mdb_cursor_open(txn, o_block_info, &c_old);
+        if (result)
+          throw0(DB_ERROR(lmdb_error("Failed to open a cursor for block_info: ", result).c_str()));
+        if (!i) {
+          result = mdb_stat(txn, m_block_info, &db_stats);
+          if (result)
+            throw0(DB_ERROR(lmdb_error("Failed to query m_block_info: ", result).c_str()));
+          i = db_stats.ms_entries;
+        }
+      }
+      result = mdb_cursor_get(c_old, &k, &v, MDB_NEXT);
+      if (result == MDB_NOTFOUND) {
+        txn.commit();
+        break;
+      }
+      else if (result)
+        throw0(DB_ERROR(lmdb_error("Failed to get a record from block_info: ", result).c_str()));
+      
+      // Copy all old values into new block info struct
+      const mdb_block_info_4 *bi_old = (const mdb_block_info_4*)v.mv_data;
+      // Sanity check height, used for migration later
+      if (bi_old->bi_height >= blockchain_height)
+        throw0(BLOCK_DNE("Height discrepancy in block info table, perhaps database is corrupted"));
+
+      mdb_block_info_5 bi;
+      bi.bi_height = bi_old->bi_height;
+      bi.bi_timestamp = bi_old->bi_timestamp;
+      bi.bi_coins = bi_old->bi_coins;
+      bi.bi_weight = bi_old->bi_weight;
+      bi.bi_diff_lo = bi_old->bi_diff_lo;
+      bi.bi_diff_hi = bi_old->bi_diff_hi;
+      bi.bi_hash = bi_old->bi_hash;
+      bi.bi_cum_rct = bi_old->bi_cum_rct;
+      bi.bi_cum_rct_coinbase = cum_coinbase_rct_dist[bi_old->bi_height]; // new field!
+      bi.bi_long_term_block_weight = bi_old->bi_long_term_block_weight;
+
+      MDB_val_set(nv, bi);
+      result = mdb_cursor_put(c_cur, (MDB_val *)&zerokval, &nv, MDB_APPENDDUP);
+      if (result)
+        throw0(DB_ERROR(lmdb_error("Failed to put a record into block_infn: ", result).c_str()));
+      /* we delete the old records immediately, so the overall DB and mapsize should not grow.
+       * This is a little slower than just letting mdb_drop() delete it all at the end, but
+       * it saves a significant amount of disk space.
+       */
+      result = mdb_cursor_del(c_old, 0);
+      if (result)
+        throw0(DB_ERROR(lmdb_error("Failed to delete a record from block_info: ", result).c_str()));
+      i++;
+    }
+
+    result = mdb_txn_begin(m_env, NULL, 0, txn);
+    if (result)
+      throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str()));
+    /* Delete the old table */
+    result = mdb_drop(txn, o_block_info, 1);
+    if (result)
+      throw0(DB_ERROR(lmdb_error("Failed to delete old block_info table: ", result).c_str()));
+
+    RENAME_DB("block_infn");
+    mdb_dbi_close(m_env, m_block_info);
+
+    lmdb_db_open(txn, "block_info", MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_block_info, "Failed to open db handle for block_infn");
+    mdb_set_dupsort(txn, m_block_info, compare_uint64);
+
+    txn.commit();
+  } while(0);
+
+  uint32_t version = 6;
+  v.mv_data = (void *)&version;
+  v.mv_size = sizeof(version);
+  MDB_val_str(vk, "version");
+  result = mdb_txn_begin(m_env, NULL, 0, txn);
+  if (result)
+    throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str()));
+  result = mdb_put(txn, m_properties, &vk, &v, 0);
+  if (result)
+    throw0(DB_ERROR(lmdb_error("Failed to update version for the db: ", result).c_str()));
+  txn.commit();
+}
+
 void BlockchainLMDB::migrate(const uint32_t oldversion)
 {
   if (oldversion < 1)
@@ -5653,6 +5839,8 @@ void BlockchainLMDB::migrate(const uint32_t oldversion)
     migrate_3_4();
   if (oldversion < 5)
     migrate_4_5();
+  if (oldversion < 6)
+    migrate_5_6();
 }
 
 }  // namespace cryptonote

src/blockchain_db/lmdb/db_lmdb.h

@@ -218,6 +218,8 @@ public:
 
   virtual std::vector<uint64_t> get_block_cumulative_rct_outputs(const std::vector<uint64_t> &heights) const;
 
+  virtual std::vector<uint64_t> get_block_cumulative_rct_coinbase_outputs(uint64_t begin_height, uint64_t end_height) const;
+
   virtual uint64_t get_block_timestamp(const uint64_t& height) const;
 
   virtual uint64_t get_top_block_timestamp() const;
@@ -371,6 +373,7 @@ private:
                 , const difficulty_type& cumulative_difficulty
                 , const uint64_t& coins_generated
                 , uint64_t num_rct_outs
+                , uint64_t num_rct_outs_coinbase
                 , const crypto::hash& block_hash
                 );
 
@@ -443,6 +446,9 @@ private:
   // migrate from DB version 4 to 5
   void migrate_4_5();
 
+  // migrate from DB version 5 to 6: add field `bi_cum_rct_coinbase` to block_info table
+  void migrate_5_6();
+
   void cleanup_batch();
 
 private:

src/blockchain_db/testdb.h

@@ -77,6 +77,7 @@ public:
   virtual cryptonote::block_header get_block_header(const crypto::hash& h) const override { return cryptonote::block_header(); }
   virtual uint64_t get_block_timestamp(const uint64_t& height) const override { return 0; }
   virtual std::vector<uint64_t> get_block_cumulative_rct_outputs(const std::vector<uint64_t> &heights) const override { return {}; }
+  virtual std::vector<uint64_t> get_block_cumulative_rct_coinbase_outputs(uint64_t begin_height, uint64_t end_height) const override { return {}; };
   virtual uint64_t get_top_block_timestamp() const override { return 0; }
   virtual size_t get_block_weight(const uint64_t& height) const override { return 128; }
   virtual std::vector<uint64_t> get_block_weights(uint64_t start_height, size_t count) const override { return {}; }
@@ -144,6 +145,7 @@ public:
                         , const cryptonote::difficulty_type& cumulative_difficulty
                         , const uint64_t& coins_generated
                         , uint64_t num_rct_outs
+                        , uint64_t num_rct_outs_coinbase
                         , const crypto::hash& blk_hash
                         ) override { }
   virtual cryptonote::block get_block_from_height(const uint64_t& height) const override { return cryptonote::block(); }

src/cryptonote_basic/cryptonote_basic_impl.cpp

@@ -183,6 +183,16 @@ namespace cryptonote {
     return true;
   }
   //-----------------------------------------------------------------------
+  uint64_t get_num_coinbase_rct_outputs(const block& b)
+  {
+    if (b.miner_tx.version < 2)
+    {
+      return 0;
+    }
+
+    return b.miner_tx.vout.size();
+  }
+  //-----------------------------------------------------------------------
   bool get_account_address_from_str(
       address_parse_info& info
     , network_type nettype

src/cryptonote_basic/cryptonote_basic_impl.h

@@ -110,6 +110,8 @@ namespace cryptonote {
 
   bool is_coinbase(const transaction& tx);
 
+  uint64_t get_num_coinbase_rct_outputs(const block& b);
+
   bool operator ==(const cryptonote::transaction& a, const cryptonote::transaction& b);
   bool operator ==(const cryptonote::block& a, const cryptonote::block& b);
 }

src/cryptonote_core/blockchain.cpp

@@ -2410,6 +2410,39 @@ bool Blockchain::get_output_distribution(uint64_t amount, uint64_t from_height,
   }
 }
 //------------------------------------------------------------------
+bool Blockchain::get_rct_coinbase_output_distribution(uint64_t from_height, uint64_t to_height, uint64_t &start_height, std::vector<uint64_t> &distribution, uint64_t &base) const
+{
+  LOG_PRINT_L3("Blockchain::" << __func__);
+
+  CHECK_AND_ASSERT_MES(to_height >= from_height, false, "to_height < from_height");
+
+  // rct outputs don't exist before v4
+  switch (m_nettype)
+  {
+    case STAGENET: start_height = stagenet_hard_forks[3].height; break;
+    case TESTNET: start_height = testnet_hard_forks[3].height; break;
+    case MAINNET: start_height = mainnet_hard_forks[3].height; break;
+    case FAKECHAIN: start_height = 0; break;
+    default: MERROR("network type not recognized: " << m_nettype); return false;
+  }
+
+  start_height = from_height = std::max(from_height, start_height);
+  to_height = std::max(to_height, from_height);
+
+  const uint64_t db_height = m_db->height();
+  CHECK_AND_ASSERT_MES(db_height, false, "No block data");
+  CHECK_AND_ASSERT_MES(from_height < db_height, false, "from_height is too high");
+  CHECK_AND_ASSERT_MES(to_height < db_height, false, "to_height is too high");
+
+  if (from_height)
+    base = m_db->get_block_cumulative_rct_coinbase_outputs(from_height - 1, from_height)[0];
+  else
+    base = 0;
+
+  distribution = m_db->get_block_cumulative_rct_coinbase_outputs(from_height, to_height + 1);
+  return true;
+}
+//------------------------------------------------------------------
 // This function takes a list of block hashes from another node
 // on the network to find where the split point is between us and them.
 // This is used to see what to send another node that needs to sync.

src/cryptonote_core/blockchain.h

@@ -572,6 +572,19 @@ namespace cryptonote
      */
     bool get_output_distribution(uint64_t amount, uint64_t from_height, uint64_t to_height, uint64_t &start_height, std::vector<uint64_t> &distribution, uint64_t &base) const;
 
+    /**
+     * @brief gets per-block distribution of rct coinbase outputs (cumulative)
+     *
+     * @param from_height the height before which we do not care about the data (inclusive)
+     * @param to_height the height after which we do not care about the data (inclusive)
+     * @param[out] start_height the height of the first rct coinbase output
+     * @param[out] distribution the number of rct coinbase outputs in each block starting from start_height (inclusive)
+     * @param[out] base how many rct coinbase outputs are before the stated distribution
+     *
+     * @return false if requested heights are not available or other failure, true for success
+     */
+    bool get_rct_coinbase_output_distribution(uint64_t from_height, uint64_t to_height, uint64_t &start_height, std::vector<uint64_t> &distribution, uint64_t &base) const;
+
     /**
      * @brief gets the global indices for outputs from a given transaction
      *

src/rpc/core_rpc_server.cpp

@@ -3312,45 +3312,7 @@ namespace cryptonote
     if (use_bootstrap_daemon_if_necessary<COMMAND_RPC_GET_OUTPUT_DISTRIBUTION>(invoke_http_mode::JON_RPC, "get_output_distribution", req, res, r))
       return r;
 
-    const bool restricted = m_restricted && ctx;
-    if (restricted && req.amounts != std::vector<uint64_t>(1, 0))
-    {
-      error_resp.code = CORE_RPC_ERROR_CODE_RESTRICTED;
-      error_resp.message = "Restricted RPC can only get output distribution for rct outputs. Use your own node.";
-      return false;
-    }
-
-    size_t n_0 = 0, n_non0 = 0;
-    for (uint64_t amount: req.amounts)
-      if (amount) ++n_non0; else ++n_0;
-    CHECK_PAYMENT_MIN1(req, res, n_0 * COST_PER_OUTPUT_DISTRIBUTION_0 + n_non0 * COST_PER_OUTPUT_DISTRIBUTION, false);
-
-    try
-    {
-      // 0 is placeholder for the whole chain
-      const uint64_t req_to_height = req.to_height ? req.to_height : (m_core.get_current_blockchain_height() - 1);
-      for (uint64_t amount: req.amounts)
-      {
-        auto data = rpc::RpcHandler::get_output_distribution([this](uint64_t amount, uint64_t from, uint64_t to, uint64_t &start_height, std::vector<uint64_t> &distribution, uint64_t &base) { return m_core.get_output_distribution(amount, from, to, start_height, distribution, base); }, amount, req.from_height, req_to_height, [this](uint64_t height) { return m_core.get_blockchain_storage().get_db().get_block_hash_from_height(height); }, req.cumulative, m_core.get_current_blockchain_height());
-        if (!data)
-        {
-          error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR;
-          error_resp.message = "Failed to get output distribution";
-          return false;
-        }
-
-        res.distributions.push_back({std::move(*data), amount, "", req.binary, req.compress});
-      }
-    }
-    catch (const std::exception &e)
-    {
-      error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR;
-      error_resp.message = "Failed to get output distribution";
-      return false;
-    }
-
-    res.status = CORE_RPC_STATUS_OK;
-    return true;
+    return on_get_output_distribution_generalized(req, res, error_resp, ctx, &tracker);
   }
   //------------------------------------------------------------------------------------------------------------------------------
   bool core_rpc_server::on_get_output_distribution_bin(const COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::request& req, COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::response& res, const connection_context *ctx)
@@ -3361,13 +3323,27 @@ namespace cryptonote
     if (use_bootstrap_daemon_if_necessary<COMMAND_RPC_GET_OUTPUT_DISTRIBUTION>(invoke_http_mode::BIN, "/get_output_distribution.bin", req, res, r))
       return r;
 
+    if (!req.binary)
+    {
+      res.status = "Binary only call";
+      return true;
+    }
+
+    epee::json_rpc::error error_resp;
+    return on_get_output_distribution_generalized(req, res, error_resp, ctx, &tracker);
+  }
+  //------------------------------------------------------------------------------------------------------------------------------
+  bool core_rpc_server::on_get_output_distribution_generalized(const COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::request& req, COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx, void* tracker_erased)
+  {
     const bool restricted = m_restricted && ctx;
     if (restricted && req.amounts != std::vector<uint64_t>(1, 0))
     {
-      res.status = "Restricted RPC can only get output distribution for rct outputs. Use your own node.";
+      error_resp.code = CORE_RPC_ERROR_CODE_RESTRICTED;
+      error_resp.message = res.status = "Restricted RPC can only get output distribution for rct outputs. Use your own node.";
       return false;
     }
 
+    RPCTracker& tracker = *reinterpret_cast<RPCTracker*>(tracker_erased);
     size_t n_0 = 0, n_non0 = 0;
     for (uint64_t amount: req.amounts)
       if (amount) ++n_non0; else ++n_0;
@@ -3375,11 +3351,6 @@ namespace cryptonote
 
     res.status = "Failed";
 
-    if (!req.binary)
-    {
-      res.status = "Binary only call";
-      return true;
-    }
     try
     {
       // 0 is placeholder for the whole chain
@@ -3389,17 +3360,47 @@ namespace cryptonote
         auto data = rpc::RpcHandler::get_output_distribution([this](uint64_t amount, uint64_t from, uint64_t to, uint64_t &start_height, std::vector<uint64_t> &distribution, uint64_t &base) { return m_core.get_output_distribution(amount, from, to, start_height, distribution, base); }, amount, req.from_height, req_to_height, [this](uint64_t height) { return m_core.get_blockchain_storage().get_db().get_block_hash_from_height(height); }, req.cumulative, m_core.get_current_blockchain_height());
         if (!data)
         {
-          res.status = "Failed to get output distribution";
-          return true;
+          error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR;
+          error_resp.message = res.status = "Failed to get output distribution";
+          return false;
         }
 
         res.distributions.push_back({std::move(*data), amount, "", req.binary, req.compress});
       }
+
+      if (req.get_rct_coinbase)
+      {
+        COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::distribution& out_dist = res.rct_coinbase_distribution;
+        out_dist.amount = 0;
+        out_dist.binary = req.binary;
+        out_dist.compress = req.compress;
+        if (!m_core.get_blockchain_storage().get_rct_coinbase_output_distribution(
+            req.from_height
+          , req_to_height
+          , out_dist.data.start_height
+          , out_dist.data.distribution
+          , out_dist.data.base
+        ))
+        {
+          // control flow switch to catch branch
+          throw std::runtime_error("Failed to get rct coinbase output distribution");
+        }
+
+        if (!req.cumulative && out_dist.data.distribution.size())
+        {
+          // The database stores this distribution as cumulative by default so decumulate
+          for (size_t i = out_dist.data.distribution.size() - 1; i >= 1; --i)
+          {
+            out_dist.data.distribution[i] -= out_dist.data.distribution[i - 1];
+          }
+        }
+      }
     }
     catch (const std::exception &e)
     {
-      res.status = "Failed to get output distribution";
-      return true;
+      error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR;
+      error_resp.message = res.status = e.what();
+      return false;
     }
 
     res.status = CORE_RPC_STATUS_OK;

src/rpc/core_rpc_server.h

@@ -285,7 +285,8 @@ private:
     bool use_bootstrap_daemon_if_necessary(const invoke_http_mode &mode, const std::string &command_name, const typename COMMAND_TYPE::request& req, typename COMMAND_TYPE::response& res, bool &r);
     bool get_block_template(const account_public_address &address, const crypto::hash *prev_block, const cryptonote::blobdata &extra_nonce, size_t &reserved_offset, cryptonote::difficulty_type &difficulty, uint64_t &height, uint64_t &expected_reward, block &b, uint64_t &seed_height, crypto::hash &seed_hash, crypto::hash &next_seed_hash, epee::json_rpc::error &error_resp);
     bool check_payment(const std::string &client, uint64_t payment, const std::string &rpc, bool same_ts, std::string &message, uint64_t &credits, std::string &top_hash);
-    
+    bool on_get_output_distribution_generalized(const COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::request& req, COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx, void* tracker_erased);
+
     core& m_core;
     nodetool::node_server<cryptonote::t_cryptonote_protocol_handler<cryptonote::core> >& m_p2p;
     boost::shared_mutex m_bootstrap_daemon_mutex;

src/rpc/core_rpc_server_commands_defs.h

@@ -2447,6 +2447,7 @@ namespace cryptonote
       bool cumulative;
       bool binary;
       bool compress;
+      bool get_rct_coinbase;
 
       BEGIN_KV_SERIALIZE_MAP()
         KV_SERIALIZE_PARENT(rpc_access_request_base)
@@ -2456,6 +2457,7 @@ namespace cryptonote
         KV_SERIALIZE_OPT(cumulative, false)
         KV_SERIALIZE_OPT(binary, true)
         KV_SERIALIZE_OPT(compress, false)
+        KV_SERIALIZE_OPT(get_rct_coinbase, false)
       END_KV_SERIALIZE_MAP()
     };
     typedef epee::misc_utils::struct_init<request_t> request;
@@ -2500,15 +2502,27 @@ namespace cryptonote
           KV_SERIALIZE_N(data.distribution, "distribution")
         KV_SERIALIZE_N(data.base, "base")
       END_KV_SERIALIZE_MAP()
+
+      bool operator==(const distribution& other) const
+      {
+        return data == other.data
+          && amount == other.amount
+          && compressed_data == other.compressed_data
+          && binary == other.binary
+          && compress == other.compress
+        ;
+      }
     };
 
     struct response_t: public rpc_access_response_base
     {
       std::vector<distribution> distributions;
+      distribution rct_coinbase_distribution;
 
       BEGIN_KV_SERIALIZE_MAP()
         KV_SERIALIZE_PARENT(rpc_access_response_base)
         KV_SERIALIZE(distributions)
+        KV_SERIALIZE_OPT(rct_coinbase_distribution, decltype(rct_coinbase_distribution)())
       END_KV_SERIALIZE_MAP()
     };
     typedef epee::misc_utils::struct_init<response_t> response;

src/rpc/rpc_handler.h

@@ -47,6 +47,14 @@ struct output_distribution_data
   std::vector<std::uint64_t> distribution;
   std::uint64_t start_height;
   std::uint64_t base;
+
+  bool operator==(const output_distribution_data& other) const
+  {
+    return distribution == other.distribution
+      && start_height == other.start_height
+      && base == other.base
+    ;
+  }
 };
 
 class RpcHandler

tests/block_weight/block_weight.cpp

@@ -65,6 +65,7 @@ public:
                         , const cryptonote::difficulty_type& cumulative_difficulty
                         , const uint64_t& coins_generated
                         , uint64_t num_rct_outs
+                        , uint64_t num_rct_outs_coinbase
                         , const crypto::hash& blk_hash
                         ) override {
     blocks.push_back({block_weight, long_term_block_weight});

tests/core_tests/chaingen.cpp

@@ -87,6 +87,7 @@ namespace
         , const cryptonote::difficulty_type& cumulative_difficulty
         , const uint64_t& coins_generated
         , uint64_t num_rct_outs
+        , uint64_t num_rct_outs_coinbase
         , const crypto::hash& blk_hash
     ) override
     {
@@ -174,7 +175,7 @@ static std::unique_ptr<cryptonote::Blockchain> init_blockchain(const std::vector
 
     const block *blk = &boost::get<block>(ev);
     auto blk_hash = get_block_hash(*blk);
-    bdb->add_block(*blk, 1, 1, 1, 0, 0, blk_hash);
+    bdb->add_block(*blk, 1, 1, 1, 0, 0, 0, blk_hash);
   }
 
   bool r = blockchain->init(bdb, nettype, true, test_options, 2, nullptr);

tests/functional_tests/get_output_distribution.py

@@ -35,11 +35,14 @@ from __future__ import print_function
 from framework.daemon import Daemon
 from framework.wallet import Wallet
 
+import json
+
 class GetOutputDistributionTest():
     def run_test(self):
         self.reset()
         self.create()
         self.test_get_output_distribution()
+        self.test_get_rct_coinbase_output_distribution()
 
     def reset(self):
         print('Resetting blockchain')
@@ -212,6 +215,55 @@ class GetOutputDistributionTest():
         for h in range(len(d.distribution)):
             assert d.distribution[h] == 0
 
+    def test_get_rct_coinbase_output_distribution(self):
+        print("Test get_rct_coinbase_output_distribution")
+
+        daemon = Daemon()
+
+        # Check that we have some RCT blocks/transactions to test against
+        chain_height = daemon.get_height()['height']
+        top_header = daemon.get_block_header_by_height(chain_height - 1)
+        top_major_version = top_header['block_header']['major_version']
+        assert top_major_version >= 4
+
+        # Fetch the cumulative RCT coinbase distribution
+        res = daemon.get_output_distribution(get_rct_coinbase = True, cumulative = True, binary=False)
+        dist_info = res['rct_coinbase_distribution']
+        amount = dist_info['amount']
+        begin_height = dist_info['start_height']
+        base = dist_info['base']
+        dist = dist_info['distribution']
+        end_height = begin_height + len(dist)
+        assert amount == 0
+        assert begin_height < chain_height
+        assert end_height == chain_height
+        assert base == 0
+        assert len(dist) > 0
+        assert len(dist) == chain_height - begin_height # Check to_height=0 grabs whole chain after v4 fork
+        assert all((x >= 0 for x in dist)) # Check all elements of dist positive
+        assert all((dist[i - 1] <= dist[i] for i in range(1, len(dist)))) # Check cumulative
+
+        print("Found {} total RCT coinbase outputs".format(dist[-1]))
+
+        # Check that the non-cumulative distribution is valid to corresponding cumulative
+        res = daemon.get_output_distribution(get_rct_coinbase = True, cumulative = False, binary=False)
+        non_cum_dist = res['rct_coinbase_distribution']['distribution']
+        assert len(non_cum_dist) == len(dist)
+        assert dist[0] == non_cum_dist[0]
+        assert all(((dist[i] - dist[i - 1]) == non_cum_dist[i] for i in range(1, len(dist))))
+
+        # Scan the whole chain from begin_height and check values of RCT coinbase distribution
+        # for each block.
+        for h in range(begin_height, chain_height):
+            dist_index = h - begin_height
+            res = daemon.get_block(height = h)
+            block_body = json.loads(res['json'])
+            miner_tx = block_body['miner_tx']
+            is_rct_miner = miner_tx['version'] >= 2
+            if not is_rct_miner:
+                assert non_cum_dist[dist_index] == 0
+            else:
+                assert non_cum_dist[dist_index] == len(miner_tx['vout'])
 
 class Guard:
     def __enter__(self):

tests/unit_tests/hardfork.cpp

@@ -41,6 +41,11 @@ using namespace cryptonote;
 #define BLOCKS_PER_YEAR 525960
 #define SECONDS_PER_YEAR 31557600
 
+#define ADD_BLOCK_DEFAULT(db, block)                         \
+  do {                                                       \
+    db.add_block((block), 0, 0, 0, 0, 0, 0, crypto::hash()); \
+  } while (0);                                               \
+
 namespace
 {
 
@@ -53,6 +58,7 @@ public:
                         , const difficulty_type& cumulative_difficulty
                         , const uint64_t& coins_generated
                         , uint64_t num_rct_outs
+                        , uint64_t num_rct_outs_coinbase
                         , const crypto::hash& blk_hash
                         ) override {
     blocks.push_back(blk);
@@ -107,20 +113,20 @@ TEST(major, Only)
   ASSERT_FALSE(hf.add(mkblock(0, 2), 0));
   ASSERT_FALSE(hf.add(mkblock(2, 2), 0));
   ASSERT_TRUE(hf.add(mkblock(1, 2), 0));
-  db.add_block(mkblock(1, 1), 0, 0, 0, 0, 0, crypto::hash());
+  ADD_BLOCK_DEFAULT(db, mkblock(1, 1));
 
   // block height 1, only version 1 is accepted
   ASSERT_FALSE(hf.add(mkblock(0, 2), 1));
   ASSERT_FALSE(hf.add(mkblock(2, 2), 1));
   ASSERT_TRUE(hf.add(mkblock(1, 2), 1));
-  db.add_block(mkblock(1, 1), 0, 0, 0, 0, 0, crypto::hash());
+  ADD_BLOCK_DEFAULT(db, mkblock(1, 1));
 
   // block height 2, only version 2 is accepted
   ASSERT_FALSE(hf.add(mkblock(0, 2), 2));
   ASSERT_FALSE(hf.add(mkblock(1, 2), 2));
   ASSERT_FALSE(hf.add(mkblock(3, 2), 2));
   ASSERT_TRUE(hf.add(mkblock(2, 2), 2));
-  db.add_block(mkblock(2, 1), 0, 0, 0, 0, 0, crypto::hash());
+  ADD_BLOCK_DEFAULT(db, mkblock(2, 1));
 }
 
 TEST(empty_hardforks, Success)
@@ -134,7 +140,7 @@ TEST(empty_hardforks, Success)
   ASSERT_TRUE(hf.get_state(time(NULL) + 3600*24*400) == HardFork::Ready);
 
   for (uint64_t h = 0; h <= 10; ++h) {
-    db.add_block(mkblock(hf, h, 1), 0, 0, 0, 0, 0, crypto::hash());
+    ADD_BLOCK_DEFAULT(db, mkblock(hf, h, 1));
     ASSERT_TRUE(hf.add(db.get_block_from_height(h), h));
   }
   ASSERT_EQ(hf.get(0), 1);
@@ -168,14 +174,14 @@ TEST(check_for_height, Success)
   for (uint64_t h = 0; h <= 4; ++h) {
     ASSERT_TRUE(hf.check_for_height(mkblock(1, 1), h));
     ASSERT_FALSE(hf.check_for_height(mkblock(2, 2), h));  // block version is too high
-    db.add_block(mkblock(hf, h, 1), 0, 0, 0, 0, 0, crypto::hash());
+    ADD_BLOCK_DEFAULT(db, mkblock(hf, h, 1));
     ASSERT_TRUE(hf.add(db.get_block_from_height(h), h));
   }
 
   for (uint64_t h = 5; h <= 10; ++h) {
     ASSERT_FALSE(hf.check_for_height(mkblock(1, 1), h));  // block version is too low
     ASSERT_TRUE(hf.check_for_height(mkblock(2, 2), h));
-    db.add_block(mkblock(hf, h, 2), 0, 0, 0, 0, 0, crypto::hash());
+    ADD_BLOCK_DEFAULT(db, mkblock(hf, h, 2));
     ASSERT_TRUE(hf.add(db.get_block_from_height(h), h));
   }
 }
@@ -192,19 +198,19 @@ TEST(get, next_version)
 
   for (uint64_t h = 0; h <= 4; ++h) {
     ASSERT_EQ(2, hf.get_next_version());
-    db.add_block(mkblock(hf, h, 1), 0, 0, 0, 0, 0, crypto::hash());
+    ADD_BLOCK_DEFAULT(db, mkblock(hf, h, 1));
     ASSERT_TRUE(hf.add(db.get_block_from_height(h), h));
   }
 
   for (uint64_t h = 5; h <= 9; ++h) {
     ASSERT_EQ(4, hf.get_next_version());
-    db.add_block(mkblock(hf, h, 2), 0, 0, 0, 0, 0, crypto::hash());
+    ADD_BLOCK_DEFAULT(db, mkblock(hf, h, 2));
     ASSERT_TRUE(hf.add(db.get_block_from_height(h), h));
   }
 
   for (uint64_t h = 10; h <= 15; ++h) {
     ASSERT_EQ(4, hf.get_next_version());
-    db.add_block(mkblock(hf, h, 4), 0, 0, 0, 0, 0, crypto::hash());
+    ADD_BLOCK_DEFAULT(db, mkblock(hf, h, 4));
     ASSERT_TRUE(hf.add(db.get_block_from_height(h), h));
   }
 }
@@ -245,7 +251,7 @@ TEST(steps_asap, Success)
   hf.init();
 
   for (uint64_t h = 0; h < 10; ++h) {
-    db.add_block(mkblock(hf, h, 9), 0, 0, 0, 0, 0, crypto::hash());
+    ADD_BLOCK_DEFAULT(db, mkblock(hf, h, 9));
     ASSERT_TRUE(hf.add(db.get_block_from_height(h), h));
   }
 
@@ -272,7 +278,7 @@ TEST(steps_1, Success)
   hf.init();
 
   for (uint64_t h = 0 ; h < 10; ++h) {
-    db.add_block(mkblock(hf, h, h+1), 0, 0, 0, 0, 0, crypto::hash());
+    ADD_BLOCK_DEFAULT(db, mkblock(hf, h, h+1));
     ASSERT_TRUE(hf.add(db.get_block_from_height(h), h));
   }
 
@@ -297,7 +303,7 @@ TEST(reorganize, Same)
     //                                 index  0  1  2  3  4  5  6  7  8  9
     static const uint8_t block_versions[] = { 1, 1, 4, 4, 7, 7, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9 };
     for (uint64_t h = 0; h < 20; ++h) {
-      db.add_block(mkblock(hf, h, block_versions[h]), 0, 0, 0, 0, 0, crypto::hash());
+      ADD_BLOCK_DEFAULT(db, mkblock(hf, h, block_versions[h]));
       ASSERT_TRUE(hf.add(db.get_block_from_height(h), h));
     }
 
@@ -328,7 +334,7 @@ TEST(reorganize, Changed)
   static const uint8_t block_versions[] =    { 1, 1, 4, 4, 7, 7, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9 };
   static const uint8_t expected_versions[] = { 1, 1, 1, 1, 1, 1, 4, 4, 7, 7, 9, 9, 9, 9, 9, 9 };
   for (uint64_t h = 0; h < 16; ++h) {
-    db.add_block(mkblock(hf, h, block_versions[h]), 0, 0, 0, 0, 0, crypto::hash());
+    ADD_BLOCK_DEFAULT(db, mkblock(hf, h, block_versions[h]));
     ASSERT_TRUE (hf.add(db.get_block_from_height(h), h));
   }
 
@@ -348,7 +354,7 @@ TEST(reorganize, Changed)
   ASSERT_EQ(db.height(), 3);
   hf.reorganize_from_block_height(2);
   for (uint64_t h = 3; h < 16; ++h) {
-    db.add_block(mkblock(hf, h, block_versions_new[h]), 0, 0, 0, 0, 0, crypto::hash());
+    ADD_BLOCK_DEFAULT(db, mkblock(hf, h, block_versions_new[h]));
     bool ret = hf.add(db.get_block_from_height(h), h);
     ASSERT_EQ (ret, h < 15);
   }
@@ -372,7 +378,7 @@ TEST(voting, threshold)
 
     for (uint64_t h = 0; h <= 8; ++h) {
       uint8_t v = 1 + !!(h % 8);
-      db.add_block(mkblock(hf, h, v), 0, 0, 0, 0, 0, crypto::hash());
+      ADD_BLOCK_DEFAULT(db, mkblock(hf, h, v));
       bool ret = hf.add(db.get_block_from_height(h), h);
       if (h >= 8 && threshold == 87) {
         // for threshold 87, we reach the treshold at height 7, so from height 8, hard fork to version 2, but 8 tries to add 1
@@ -406,7 +412,7 @@ TEST(voting, different_thresholds)
     static const uint8_t expected_versions[] = { 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4 };
 
     for (uint64_t h = 0; h < sizeof(block_versions) / sizeof(block_versions[0]); ++h) {
-      db.add_block(mkblock(hf, h, block_versions[h]), 0, 0, 0, 0, 0, crypto::hash());
+      ADD_BLOCK_DEFAULT(db, mkblock(hf, h, block_versions[h]));
       bool ret = hf.add(db.get_block_from_height(h), h);
       ASSERT_EQ(ret, true);
     }
@@ -459,7 +465,7 @@ TEST(voting, info)
     ASSERT_EQ(expected_thresholds[h], threshold);
     ASSERT_EQ(4, voting);
 
-    db.add_block(mkblock(hf, h, block_versions[h]), 0, 0, 0, 0, 0, crypto::hash());
+    ADD_BLOCK_DEFAULT(db, mkblock(hf, h, block_versions[h]));
     ASSERT_TRUE(hf.add(db.get_block_from_height(h), h));
   }
 }
@@ -522,7 +528,7 @@ TEST(reorganize, changed)
 #define ADD(v, h, a) \
   do { \
     cryptonote::block b = mkblock(hf, h, v); \
-    db.add_block(b, 0, 0, 0, 0, 0, crypto::hash()); \
+    ADD_BLOCK_DEFAULT(db, b); \
     ASSERT_##a(hf.add(b, h)); \
   } while(0)
 #define ADD_TRUE(v, h) ADD(v, h, TRUE)

tests/unit_tests/long_term_block_weight.cpp

@@ -57,6 +57,7 @@ public:
                         , const cryptonote::difficulty_type& cumulative_difficulty
                         , const uint64_t& coins_generated
                         , uint64_t num_rct_outs
+                        , uint64_t num_rct_outs_coinbase
                         , const crypto::hash& blk_hash
                         ) override {
     blocks.push_back({block_weight, long_term_block_weight});

utils/python-rpc/framework/daemon.py

@@ -375,7 +375,7 @@ class Daemon(object):
         }
         return self.rpc.send_json_rpc_request(get_coinbase_tx_sum)
 
-    def get_output_distribution(self, amounts = [], from_height = 0, to_height = 0, cumulative = False, binary = False, compress = False, client = ""):
+    def get_output_distribution(self, amounts = [], from_height = 0, to_height = 0, cumulative = False, binary = False, compress = False, client = "", get_rct_coinbase = False):
         get_output_distribution = {
             'method': 'get_output_distribution',
             'params': {
@@ -386,6 +386,7 @@ class Daemon(object):
                 'cumulative': cumulative,
                 'binary': binary,
                 'compress': compress,
+                'get_rct_coinbase': get_rct_coinbase
             },
             'jsonrpc': '2.0',
             'id': '0'