@ -148,74 +148,149 @@ inline void lmdb_db_open(MDB_txn* txn, const char* name, int flags, MDB_dbi& dbi
namespace cryptonote
{
mdb_txn_safe : : mdb_txn_safe ( ) : m_txn ( NULL ) { }
std : : atomic < uint64_t > mdb_txn_safe : : num_active_txns { 0 } ;
std : : atomic_flag mdb_txn_safe : : creation_gate = ATOMIC_FLAG_INIT ;
mdb_txn_safe : : ~ mdb_txn_safe ( )
mdb_txn_safe : : mdb_txn_safe ( ) : m_txn ( NULL )
{
while ( creation_gate . test_and_set ( ) ) ;
num_active_txns + + ;
creation_gate . clear ( ) ;
}
mdb_txn_safe : : ~ mdb_txn_safe ( )
{
LOG_PRINT_L3 ( " mdb_txn_safe: destructor " ) ;
if ( m_txn ! = NULL )
{
LOG_PRINT_L3 ( " mdb_txn_safe: destructor " ) ;
if ( m_txn ! = NULL )
if ( m_batch_txn ) // this is a batch txn and should have been handled before this point for safety
{
if ( m_batch_txn ) // this is a batch txn and should have been handled before this point for safety
{
LOG_PRINT_L0 ( " WARNING: mdb_txn_safe: m_txn is a batch txn and it's not NULL in destructor - calling mdb_txn_abort() " ) ;
}
else
{
// Example of when this occurs: a lookup fails, so a read-only txn is
// aborted through this destructor. However, successful read-only txns
// ideally should have been committed when done and not end up here.
//
// NOTE: not sure if this is ever reached for a non-batch write
// transaction, but it's probably not ideal if it did.
LOG_PRINT_L3 ( " mdb_txn_safe: m_txn not NULL in destructor - calling mdb_txn_abort() " ) ;
}
mdb_txn_abort ( m_txn ) ;
LOG_PRINT_L0 ( " WARNING: mdb_txn_safe: m_txn is a batch txn and it's not NULL in destructor - calling mdb_txn_abort() " ) ;
}
else
{
// Example of when this occurs: a lookup fails, so a read-only txn is
// aborted through this destructor. However, successful read-only txns
// ideally should have been committed when done and not end up here.
//
// NOTE: not sure if this is ever reached for a non-batch write
// transaction, but it's probably not ideal if it did.
LOG_PRINT_L3 ( " mdb_txn_safe: m_txn not NULL in destructor - calling mdb_txn_abort() " ) ;
}
mdb_txn_abort ( m_txn ) ;
}
num_active_txns - - ;
}
void mdb_txn_safe : : commit ( std : : string message )
void mdb_txn_safe : : commit ( std : : string message )
{
if ( message . size ( ) = = 0 )
{
if ( message . size ( ) = = 0 )
{
message = " Failed to commit a transaction to the db " ;
}
message = " Failed to commit a transaction to the db " ;
}
if ( mdb_txn_commit ( m_txn ) )
{
m_txn = NULL ;
LOG_PRINT_L0 ( message ) ;
throw DB_ERROR ( message . c_str ( ) ) ;
}
if ( mdb_txn_commit ( m_txn ) )
{
m_txn = NULL ;
LOG_PRINT_L0 ( message ) ;
throw DB_ERROR ( message . c_str ( ) ) ;
}
m_txn = NULL ;
}
void mdb_txn_safe : : abort ( )
void mdb_txn_safe : : abort ( )
{
LOG_PRINT_L3 ( " mdb_txn_safe: abort() " ) ;
if ( m_txn ! = NULL )
{
LOG_PRINT_L3 ( " mdb_txn_safe: abort() " ) ;
if ( m_txn ! = NULL )
mdb_txn_abort ( m_txn ) ;
m_txn = NULL ;
}
else
{
LOG_PRINT_L0 ( " WARNING: mdb_txn_safe: abort() called, but m_txn is NULL " ) ;
}
}
uint64_t mdb_txn_safe : : num_active_tx ( )
{
return num_active_txns ;
}
void mdb_txn_safe : : prevent_new_txns ( )
{
while ( creation_gate . test_and_set ( ) ) ;
}
void mdb_txn_safe : : wait_no_active_txns ( )
{
while ( num_active_txns > 0 ) ;
}
void mdb_txn_safe : : allow_new_txns ( )
{
creation_gate . clear ( ) ;
}
void BlockchainLMDB : : do_resize ( )
{
MDB_envinfo mei ;
mdb_env_info ( m_env , & mei ) ;
MDB_stat mst ;
mdb_env_stat ( m_env , & mst ) ;
uint64_t new_mapsize = ( double ) mei . me_mapsize * RESIZE_FACTOR ;
new_mapsize + = ( new_mapsize % mst . ms_psize ) ;
mdb_txn_safe : : prevent_new_txns ( ) ;
if ( m_write_txn ! = nullptr )
{
if ( m_batch_active )
{
mdb_txn_abort ( m_txn ) ;
m_txn = NULL ;
throw0 ( DB_ERROR ( " lmdb resizing not yet supported when batch transactions enabled! " ) ) ;
}
else
{
LOG_PRINT_L0 ( " WARNING: mdb_txn_safe: abort() called, but m_txn is NULL " ) ;
throw0( DB_ERROR ( " attempting resize with write transaction in progress, this should not happen! " ) ) ;
}
}
// If m_batch_active is set, a batch transaction exists beyond this class, such
// as a batch import with verification enabled, or possibly (later) a batch
// network sync.
//
// For some of the lookup methods, such as get_block_timestamp(), tx_exists(),
// and get_tx(), when m_batch_active is set, the lookup uses the batch
// transaction. This isn't only because the transaction is available, but it's
// necessary so that lookups include the database updates only present in the
// current batch write.
//
// A regular network sync without batch writes is expected to open a new read
// transaction, as those lookups are part of the validation done prior to the
// write for block and tx data, so no write transaction is open at the time.
mdb_txn_safe : : wait_no_active_txns ( ) ;
mdb_env_set_mapsize ( m_env , new_mapsize ) ;
LOG_PRINT_L0 ( " LMDB Mapsize increased. "
< < " Old: " < < mei . me_mapsize / ( 1024 * 1024 ) < < " MiB "
< < " , New: " < < new_mapsize / ( 1024 * 1024 ) < < " MiB " ) ;
mdb_txn_safe : : allow_new_txns ( ) ;
}
bool BlockchainLMDB : : need_resize ( )
{
MDB_envinfo mei ;
mdb_env_info ( m_env , & mei ) ;
MDB_stat mst ;
mdb_env_stat ( m_env , & mst ) ;
uint64_t size_used = mst . ms_psize * mei . me_last_pgno ;
if ( ( double ) size_used / mei . me_mapsize > 0.8 )
{
return true ;
}
return false ;
}
void BlockchainLMDB : : add_block ( const block & blk
, const size_t & block_size
@ -726,7 +801,7 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags)
if ( mdb_env_set_maxdbs ( m_env , 20 ) )
throw0 ( DB_ERROR ( " Failed to set max number of dbs " ) ) ;
size_t mapsize = 1LL < < 34 ;
size_t mapsize = DEFAULT_MAPSIZE ;
if ( auto result = mdb_env_set_mapsize ( m_env , mapsize ) )
throw0 ( DB_ERROR ( std : : string ( " Failed to set max memory map size: " ) . append ( mdb_strerror ( result ) ) . c_str ( ) ) ) ;
if ( auto result = mdb_env_open ( m_env , filename . c_str ( ) , mdb_flags , 0644 ) )
@ -1710,16 +1785,23 @@ void BlockchainLMDB::batch_start()
throw0 ( DB_ERROR ( " batch transactions not enabled " ) ) ;
if ( m_batch_active )
throw0 ( DB_ERROR ( " batch transaction already in progress " ) ) ;
if ( m_write_batch_txn ! = nullptr )
throw0 ( DB_ERROR ( " batch transaction already in progress " ) ) ;
if ( m_write_txn )
throw0 ( DB_ERROR ( " batch transaction attempted, but m_write_txn already in use " ) ) ;
check_open ( ) ;
if ( m_write_batch_txn = = nullptr )
{
m_write_batch_txn = new mdb_txn_safe ( ) ;
}
// NOTE: need to make sure it's destroyed properly when done
if ( mdb_txn_begin ( m_env , NULL , 0 , m_write_batch_txn ) )
if ( mdb_txn_begin ( m_env , NULL , 0 , * m_write_batch_txn ) )
throw0 ( DB_ERROR ( " Failed to create a transaction for the db " ) ) ;
// indicates this transaction is for batch transactions, but not whether it's
// active
m_write_batch_txn . m_batch_txn = true ;
m_write_txn = & m_write_batch_txn ;
m_write_batch_txn - > m_batch_txn = true ;
m_write_txn = m_write_batch_txn ;
m_batch_active = true ;
LOG_PRINT_L3 ( " batch transaction: begin " ) ;
}
@ -1731,7 +1813,10 @@ void BlockchainLMDB::batch_commit()
throw0 ( DB_ERROR ( " batch transactions not enabled " ) ) ;
if ( ! m_batch_active )
throw0 ( DB_ERROR ( " batch transaction not in progress " ) ) ;
if ( m_write_batch_txn = = nullptr )
throw0 ( DB_ERROR ( " batch transaction not in progress " ) ) ;
check_open ( ) ;
LOG_PRINT_L3 ( " batch transaction: committing... " ) ;
TIME_MEASURE_START ( time1 ) ;
m_write_txn - > commit ( ) ;
@ -1739,11 +1824,8 @@ void BlockchainLMDB::batch_commit()
time_commit1 + = time1 ;
LOG_PRINT_L3 ( " batch transaction: committed " ) ;
if ( mdb_txn_begin ( m_env , NULL , 0 , m_write_batch_txn ) )
throw0 ( DB_ERROR ( " Failed to create a transaction for the db " ) ) ;
if ( ! m_write_batch_txn . m_batch_txn )
throw0 ( DB_ERROR ( " m_write_batch_txn not marked as a batch transaction " ) ) ;
m_write_txn = & m_write_batch_txn ;
m_write_txn = nullptr ;
delete m_write_batch_txn ;
}
void BlockchainLMDB : : batch_stop ( )
@ -1753,6 +1835,8 @@ void BlockchainLMDB::batch_stop()
throw0 ( DB_ERROR ( " batch transactions not enabled " ) ) ;
if ( ! m_batch_active )
throw0 ( DB_ERROR ( " batch transaction not in progress " ) ) ;
if ( m_write_batch_txn = = nullptr )
throw0 ( DB_ERROR ( " batch transaction not in progress " ) ) ;
check_open ( ) ;
LOG_PRINT_L3 ( " batch transaction: committing... " ) ;
TIME_MEASURE_START ( time1 ) ;
@ -1761,6 +1845,7 @@ void BlockchainLMDB::batch_stop()
time_commit1 + = time1 ;
// for destruction of batch transaction
m_write_txn = nullptr ;
delete m_write_batch_txn ;
m_batch_active = false ;
LOG_PRINT_L3 ( " batch transaction: end " ) ;
}
@ -1776,7 +1861,7 @@ void BlockchainLMDB::batch_abort()
// for destruction of batch transaction
m_write_txn = nullptr ;
// explicitly call in case mdb_env_close() (BlockchainLMDB::close()) called before BlockchainLMDB destructor called.
m_write_batch_txn . abort ( ) ;
m_write_batch_txn - > abort ( ) ;
m_batch_active = false ;
LOG_PRINT_L3 ( " batch transaction: aborted " ) ;
}
@ -1798,6 +1883,15 @@ uint64_t BlockchainLMDB::add_block( const block& blk
LOG_PRINT_L3 ( " BlockchainLMDB:: " < < __func__ ) ;
check_open ( ) ;
if ( m_height % 10 = = 0 )
{
if ( need_resize ( ) )
{
LOG_PRINT_L0 ( " LMDB memory map needs resized, doing that now. " ) ;
do_resize ( ) ;
}
}
mdb_txn_safe txn ;
if ( ! m_batch_active )
{