ringdb: allow blackballing many outputs at once

It cuts down on txn commits, and speeds up blackballing substantially

src/blockchain_utilities/blockchain_blackball.cpp

@@ -417,6 +417,7 @@ int main(int argc, char* argv[])
     if (it != state.processed_heights.end())
       start_idx = it->second;
     LOG_PRINT_L0("Reading blockchain from " << inputs[n] << " from " << start_idx);
+    std::vector<crypto::public_key> blackballs;
     for_all_transactions(inputs[n], start_idx, [&](const cryptonote::transaction_prefix &tx)->bool
     {
       for (const auto &in: tx.vin)
@@ -439,7 +440,7 @@ int main(int argc, char* argv[])
         {
           const crypto::public_key pkey = core_storage[n]->get_output_key(txin.amount, absolute[0]);
           MINFO("Blackballing output " << pkey << ", due to being used in a 1-ring");
-          ringdb.blackball(pkey);
+          blackballs.push_back(pkey);
           newly_spent.insert(output_data(txin.amount, absolute[0]));
         }
         else if (state.ring_instances[new_ring] == new_ring.size())
@@ -448,7 +449,7 @@ int main(int argc, char* argv[])
           {
             const crypto::public_key pkey = core_storage[n]->get_output_key(txin.amount, absolute[o]);
             MINFO("Blackballing output " << pkey << ", due to being used in " << new_ring.size() << " identical " << new_ring.size() << "-rings");
-            ringdb.blackball(pkey);
+            blackballs.push_back(pkey);
             newly_spent.insert(output_data(txin.amount, absolute[o]));
           }
         }
@@ -476,7 +477,7 @@ int main(int argc, char* argv[])
             {
               const crypto::public_key pkey = core_storage[n]->get_output_key(txin.amount, common[0]);
               MINFO("Blackballing output " << pkey << ", due to being used in rings with a single common element");
-              ringdb.blackball(pkey);
+              blackballs.push_back(pkey);
               newly_spent.insert(output_data(txin.amount, common[0]));
             }
             else
@@ -490,6 +491,11 @@ int main(int argc, char* argv[])
           }
         }
         state.relative_rings[txin.k_image] = new_ring;
+        if (!blackballs.empty())
+        {
+          ringdb.blackball(blackballs);
+          blackballs.clear();
+        }
       }
       if (stop_requested)
       {
@@ -513,6 +519,7 @@ int main(int argc, char* argv[])
     for (const auto &e: work_spent)
       state.spent.insert(e);
 
+    std::vector<crypto::public_key> blackballs;
     for (const output_data &od: work_spent)
     {
       for (const crypto::key_image &ki: state.outputs[od])
@@ -533,13 +540,19 @@ int main(int argc, char* argv[])
           const crypto::public_key pkey = core_storage[0]->get_output_key(od.amount, last_unknown);
           MINFO("Blackballing output " << pkey << ", due to being used in a " <<
               absolute.size() << "-ring where all other outputs are known to be spent");
-          ringdb.blackball(pkey);
+          blackballs.push_back(pkey);
           newly_spent.insert(output_data(od.amount, last_unknown));
         }
       }
     }
   }
 
+  if (!blackballs.empty())
+  {
+    ringdb.blackball(blackballs);
+    blackballs.clear();
+  }
+
   LOG_PRINT_L0("Saving state data to " << state_file_path);
   std::ofstream state_data_out;
   state_data_out.open(state_file_path, std::ios_base::binary | std::ios_base::out | std::ios::trunc);

src/wallet/ringdb.cpp

@@ -146,7 +146,7 @@ static int resize_env(MDB_env *env, const char *db_path, size_t needed)
   MDB_stat mst;
   int ret;
 
-  needed = std::max(needed, (size_t)(2ul * 1024 * 1024)); // at least 2 MB
+  needed = std::max(needed, (size_t)(100ul * 1024 * 1024)); // at least 100 MB
 
   ret = mdb_env_info(env, &mei);
   if (ret)
@@ -374,7 +374,7 @@ bool ringdb::set_ring(const crypto::chacha_key &chacha_key, const crypto::key_im
   return true;
 }
 
-bool ringdb::blackball_worker(const crypto::public_key &output, int op)
+bool ringdb::blackball_worker(const std::vector<crypto::public_key> &outputs, int op)
 {
   MDB_txn *txn;
   MDB_cursor *cursor;
@@ -382,7 +382,9 @@ bool ringdb::blackball_worker(const crypto::public_key &output, int op)
   bool tx_active = false;
   bool ret = true;
 
-  dbr = resize_env(env, filename.c_str(), 32 * 2); // a pubkey, and some slack
+  THROW_WALLET_EXCEPTION_IF(outputs.size() > 1 && op == BLACKBALL_QUERY, tools::error::wallet_internal_error, "Blackball query only makes sense for a single output");
+
+  dbr = resize_env(env, filename.c_str(), 32 * 2 * outputs.size()); // a pubkey, and some slack
   THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to set env map size: " + std::string(mdb_strerror(dbr)));
   dbr = mdb_txn_begin(env, NULL, 0, &txn);
   THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr)));
@@ -391,40 +393,49 @@ bool ringdb::blackball_worker(const crypto::public_key &output, int op)
 
   MDB_val key = zerokeyval;
   MDB_val data;
-  data.mv_data = (void*)&output;
-  data.mv_size = sizeof(output);
 
-  switch (op)
+  for (const crypto::public_key &output: outputs)
+  {
+    data.mv_data = (void*)&output;
+    data.mv_size = sizeof(output);
+
+    switch (op)
+    {
+      case BLACKBALL_BLACKBALL:
+        MDEBUG("Blackballing output " << output);
+        dbr = mdb_put(txn, dbi_blackballs, &key, &data, MDB_NODUPDATA);
+        if (dbr == MDB_KEYEXIST)
+          dbr = 0;
+        break;
+      case BLACKBALL_UNBLACKBALL:
+        MDEBUG("Unblackballing output " << output);
+        dbr = mdb_del(txn, dbi_blackballs, &key, &data);
+        if (dbr == MDB_NOTFOUND)
+          dbr = 0;
+        break;
+      case BLACKBALL_QUERY:
+        dbr = mdb_cursor_open(txn, dbi_blackballs, &cursor);
+        THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to create cursor for blackballs table: " + std::string(mdb_strerror(dbr)));
+        dbr = mdb_cursor_get(cursor, &key, &data, MDB_GET_BOTH);
+        THROW_WALLET_EXCEPTION_IF(dbr && dbr != MDB_NOTFOUND, tools::error::wallet_internal_error, "Failed to lookup in blackballs table: " + std::string(mdb_strerror(dbr)));
+        ret = dbr != MDB_NOTFOUND;
+        if (dbr == MDB_NOTFOUND)
+          dbr = 0;
+        mdb_cursor_close(cursor);
+        break;
+      case BLACKBALL_CLEAR:
+        break;
+      default:
+        THROW_WALLET_EXCEPTION(tools::error::wallet_internal_error, "Invalid blackball op");
+    }
+    THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to query blackballs table: " + std::string(mdb_strerror(dbr)));
+  }
+
+  if (op == BLACKBALL_CLEAR)
   {
-    case BLACKBALL_BLACKBALL:
+    dbr = mdb_drop(txn, dbi_blackballs, 0);
-      MDEBUG("Blackballing output " << output);
+    THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to clear blackballs table: " + std::string(mdb_strerror(dbr)));
-      dbr = mdb_put(txn, dbi_blackballs, &key, &data, MDB_NODUPDATA);
-      if (dbr == MDB_KEYEXIST)
-        dbr = 0;
-      break;
-    case BLACKBALL_UNBLACKBALL:
-      MDEBUG("Unblackballing output " << output);
-      dbr = mdb_del(txn, dbi_blackballs, &key, &data);
-      if (dbr == MDB_NOTFOUND)
-        dbr = 0;
-      break;
-    case BLACKBALL_QUERY:
-      dbr = mdb_cursor_open(txn, dbi_blackballs, &cursor);
-      THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to create cursor for blackballs table: " + std::string(mdb_strerror(dbr)));
-      dbr = mdb_cursor_get(cursor, &key, &data, MDB_GET_BOTH);
-      THROW_WALLET_EXCEPTION_IF(dbr && dbr != MDB_NOTFOUND, tools::error::wallet_internal_error, "Failed to lookup in blackballs table: " + std::string(mdb_strerror(dbr)));
-      ret = dbr != MDB_NOTFOUND;
-      if (dbr == MDB_NOTFOUND)
-        dbr = 0;
-      mdb_cursor_close(cursor);
-      break;
-    case BLACKBALL_CLEAR:
-      dbr = mdb_drop(txn, dbi_blackballs, 0);
-      break;
-    default:
-      THROW_WALLET_EXCEPTION(tools::error::wallet_internal_error, "Invalid blackball op");
   }
-  THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to query blackballs table: " + std::string(mdb_strerror(dbr)));
 
   dbr = mdb_txn_commit(txn);
   THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to commit txn blackballing output to database: " + std::string(mdb_strerror(dbr)));
@@ -432,24 +443,32 @@ bool ringdb::blackball_worker(const crypto::public_key &output, int op)
   return ret;
 }
 
+bool ringdb::blackball(const std::vector<crypto::public_key> &outputs)
+{
+  return blackball_worker(outputs, BLACKBALL_BLACKBALL);
+}
+
 bool ringdb::blackball(const crypto::public_key &output)
 {
-  return blackball_worker(output, BLACKBALL_BLACKBALL);
+  std::vector<crypto::public_key> outputs(1, output);
+  return blackball_worker(outputs, BLACKBALL_BLACKBALL);
 }
 
 bool ringdb::unblackball(const crypto::public_key &output)
 {
-  return blackball_worker(output, BLACKBALL_UNBLACKBALL);
+  std::vector<crypto::public_key> outputs(1, output);
+  return blackball_worker(outputs, BLACKBALL_UNBLACKBALL);
 }
 
 bool ringdb::blackballed(const crypto::public_key &output)
 {
-  return blackball_worker(output, BLACKBALL_QUERY);
+  std::vector<crypto::public_key> outputs(1, output);
+  return blackball_worker(outputs, BLACKBALL_QUERY);
 }
 
 bool ringdb::clear_blackballs()
 {
-  return blackball_worker(crypto::public_key(), BLACKBALL_CLEAR);
+  return blackball_worker(std::vector<crypto::public_key>(), BLACKBALL_CLEAR);
 }
 
 }

src/wallet/ringdb.h

@@ -50,12 +50,13 @@ namespace tools
     bool set_ring(const crypto::chacha_key &chacha_key, const crypto::key_image &key_image, const std::vector<uint64_t> &outs, bool relative);
 
     bool blackball(const crypto::public_key &output);
+    bool blackball(const std::vector<crypto::public_key> &outputs);
     bool unblackball(const crypto::public_key &output);
     bool blackballed(const crypto::public_key &output);
     bool clear_blackballs();
 
   private:
-    bool blackball_worker(const crypto::public_key &output, int op);
+    bool blackball_worker(const std::vector<crypto::public_key> &outputs, int op);
 
   private:
     std::string filename;

src/wallet/wallet2.cpp

@@ -6238,8 +6238,7 @@ bool wallet2::set_blackballed_outputs(const std::vector<crypto::public_key> &out
     bool ret = true;
     if (!add)
       ret &= m_ringdb->clear_blackballs();
-    for (const auto &output: outputs)
+    ret &= m_ringdb->blackball(outputs);
-      ret &= m_ringdb->blackball(output);
     return ret;
   }
   catch (const std::exception &e) { return false; }