From 747f5d3594566e4a9f56595cfd0feef7004e3485 Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Sun, 21 Feb 2016 18:29:40 +0000 Subject: [PATCH 1/7] Preliminary raw partition support Autodetects that a block device is being used. --- external/db_drivers/liblmdb/mdb.c | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/external/db_drivers/liblmdb/mdb.c b/external/db_drivers/liblmdb/mdb.c index 6314d5775..0d01d2959 100644 --- a/external/db_drivers/liblmdb/mdb.c +++ b/external/db_drivers/liblmdb/mdb.c @@ -1473,6 +1473,8 @@ struct MDB_env { #define MDB_ENV_TXKEY 0x10000000U /** fdatasync is unreliable */ #define MDB_FSYNCONLY 0x08000000U + /** using a raw block device */ +#define MDB_RAWPART 0x04000000U uint32_t me_flags; /**< @ref mdb_env */ unsigned int me_psize; /**< DB page size, inited from me_os_psize */ unsigned int me_os_psize; /**< OS page size, from #GET_PAGESIZE */ @@ -4081,6 +4083,8 @@ mdb_env_read_header(MDB_env *env, int prev, MDB_meta *meta) p = (MDB_page *)&pbuf; if (!F_ISSET(p->mp_flags, P_META)) { + if (env->me_flags & MDB_RAWPART) + return ENOENT; DPRINTF(("page %"Yu" not a meta page", p->mp_pgno)); return MDB_INVALID; } @@ -4410,7 +4414,7 @@ mdb_env_map(MDB_env *env, void *addr) int prot = PROT_READ; if (flags & MDB_WRITEMAP) { prot |= PROT_WRITE; - if (ftruncate(env->me_fd, env->me_mapsize) < 0) + if (!(flags & MDB_RAWPART) && ftruncate(env->me_fd, env->me_mapsize) < 0) return ErrCode(); } env->me_map = mmap(addr, env->me_mapsize, prot, MAP_SHARED, @@ -5448,6 +5452,17 @@ mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode if (rc) goto leave; #endif +#endif +#ifndef _WIN32 + { + struct stat st; + rc = stat(path, &st); + if (rc) + return ErrCode(); + flags &= ~MDB_RAWPART; + if (S_ISBLK(st.st_mode)) + flags |= MDB_RAWPART | MDB_NOSUBDIR; + } #endif flags |= MDB_ENV_ACTIVE; /* tell mdb_env_close0() to clean up */ From 4e7586c9e41689b116108b806ab322b30f771082 Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Mon, 11 Jun 2018 10:12:03 +0100 Subject: [PATCH 2/7] More RAWPART support Use mmap to read and initialize the meta pages, raw device may not support read/write syscalls. --- external/db_drivers/liblmdb/mdb.c | 44 ++++++++++++++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/external/db_drivers/liblmdb/mdb.c b/external/db_drivers/liblmdb/mdb.c index 0d01d2959..52ad844e9 100644 --- a/external/db_drivers/liblmdb/mdb.c +++ b/external/db_drivers/liblmdb/mdb.c @@ -4040,6 +4040,8 @@ fail: return rc; } +static int ESECT mdb_env_map(MDB_env *env, void *addr); + /** Read the environment parameters of a DB environment before * mapping it into memory. * @param[in] env the environment handle @@ -4056,6 +4058,31 @@ mdb_env_read_header(MDB_env *env, int prev, MDB_meta *meta) int i, rc, off; enum { Size = sizeof(pbuf) }; + if (env->me_flags & MDB_RAWPART) { +#define VM_ALIGN 0x200000 + env->me_mapsize += VM_ALIGN-1; + env->me_mapsize &= ~(VM_ALIGN-1); + env->me_psize = env->me_os_psize; + rc = mdb_env_map(env, NULL); + if (rc) { + DPRINTF(("mdb_env_map: %s", mdb_strerror(rc))); + return rc; + } + p = (MDB_page *)env->me_map; + for (i=0; imp_flags, P_META)) + return ENOENT; + if (env->me_metas[i]->mm_magic != MDB_MAGIC) + return MDB_INVALID; + if (env->me_metas[i]->mm_version != MDB_DATA_VERSION) + return MDB_VERSION_MISMATCH; + if (i == 0 || env->me_metas[i]->mm_txnid > meta->mm_txnid) + *meta = *env->me_metas[i]; + p = (MDB_page *)((char *)p + env->me_psize); + } + return 0; + } + /* We don't know the page size yet, so use a minimum value. * Read both meta pages so we can use the latest one. */ @@ -4152,6 +4179,18 @@ mdb_env_init_meta(MDB_env *env, MDB_meta *meta) psize = env->me_psize; + if ((env->me_flags & (MDB_RAWPART|MDB_WRITEMAP)) == (MDB_RAWPART|MDB_WRITEMAP)) { + p = (MDB_page *)env->me_map; + p->mp_pgno = 0; + p->mp_flags = P_META; + *(MDB_meta *)METADATA(p) = *meta; + q = (MDB_page *)((char *)p + psize); + q->mp_pgno = 1; + q->mp_flags = P_META; + *(MDB_meta *)METADATA(q) = *meta; + return 0; + } + p = calloc(NUM_METAS, psize); if (!p) return ENOMEM; @@ -5460,8 +5499,11 @@ mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode if (rc) return ErrCode(); flags &= ~MDB_RAWPART; - if (S_ISBLK(st.st_mode)) + if (S_ISBLK(st.st_mode)) { flags |= MDB_RAWPART | MDB_NOSUBDIR; + if (!env->me_mapsize) + env->me_mapsize = DEFAULT_MAPSIZE; + } } #endif flags |= MDB_ENV_ACTIVE; /* tell mdb_env_close0() to clean up */ From b46a60e3907c597b8b4b272eca357d38534f5e40 Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Sun, 11 Oct 2020 01:32:30 +0100 Subject: [PATCH 3/7] Fix rawpart flag collision --- external/db_drivers/liblmdb/mdb.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/external/db_drivers/liblmdb/mdb.c b/external/db_drivers/liblmdb/mdb.c index 52ad844e9..c30a8e52a 100644 --- a/external/db_drivers/liblmdb/mdb.c +++ b/external/db_drivers/liblmdb/mdb.c @@ -1467,14 +1467,14 @@ struct MDB_env { #endif /** Failed to update the meta page. Probably an I/O error. */ #define MDB_FATAL_ERROR 0x80000000U + /** using a raw block device */ +#define MDB_RAWPART 0x40000000U /** Some fields are initialized. */ #define MDB_ENV_ACTIVE 0x20000000U /** me_txkey is set */ #define MDB_ENV_TXKEY 0x10000000U /** fdatasync is unreliable */ #define MDB_FSYNCONLY 0x08000000U - /** using a raw block device */ -#define MDB_RAWPART 0x04000000U uint32_t me_flags; /**< @ref mdb_env */ unsigned int me_psize; /**< DB page size, inited from me_os_psize */ unsigned int me_os_psize; /**< OS page size, from #GET_PAGESIZE */ From 493577a6cd016ee56465f5adb8e1dc8f10f2daa5 Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Wed, 7 Jul 2021 12:26:18 +0100 Subject: [PATCH 4/7] Silence spurious fallthru warning --- external/db_drivers/liblmdb/mdb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external/db_drivers/liblmdb/mdb.c b/external/db_drivers/liblmdb/mdb.c index c30a8e52a..f29a75de8 100644 --- a/external/db_drivers/liblmdb/mdb.c +++ b/external/db_drivers/liblmdb/mdb.c @@ -7725,7 +7725,7 @@ more: offset *= 4; /* space for 4 more */ break; } - /* FALLTHRU: Big enough MDB_DUPFIXED sub-page */ + /* FALLTHRU *//* Big enough MDB_DUPFIXED sub-page */ case MDB_CURRENT: fp->mp_flags |= P_DIRTY; COPY_PGNO(fp->mp_pgno, mp->mp_pgno); From b096e16699a189d619dad8a7319c6eb4dc5d46c5 Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Wed, 7 Jul 2021 16:26:35 +0100 Subject: [PATCH 5/7] Revert "db_lmdb: test for mmap support at init time" This reverts commit bd96536637724413173271e8d5df1777f7879c29. The check interferes with raw device/partition support. --- src/blockchain_db/lmdb/db_lmdb.cpp | 30 ------------------------------ src/blockchain_db/lmdb/db_lmdb.h | 1 - 2 files changed, 31 deletions(-) diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index e2ac9df0b..01f91209d 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -25,13 +25,6 @@ // STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -#ifndef _WIN32 -#include -#include -#include -#include -#endif - #include "db_lmdb.h" #include @@ -1303,26 +1296,6 @@ BlockchainLMDB::BlockchainLMDB(bool batch_transactions): BlockchainDB() m_hardfork = nullptr; } -void BlockchainLMDB::check_mmap_support() -{ -#ifndef _WIN32 - const boost::filesystem::path mmap_test_file = m_folder / boost::filesystem::unique_path(); - int mmap_test_fd = ::open(mmap_test_file.string().c_str(), O_RDWR | O_CREAT, 0600); - if (mmap_test_fd < 0) - throw0(DB_ERROR((std::string("Failed to check for mmap support: open failed: ") + strerror(errno)).c_str())); - epee::misc_utils::auto_scope_leave_caller scope_exit_handler = epee::misc_utils::create_scope_leave_handler([mmap_test_fd, &mmap_test_file]() { - ::close(mmap_test_fd); - boost::filesystem::remove(mmap_test_file.string()); - }); - if (write(mmap_test_fd, "mmaptest", 8) != 8) - throw0(DB_ERROR((std::string("Failed to check for mmap support: write failed: ") + strerror(errno)).c_str())); - void *mmap_res = mmap(NULL, 8, PROT_READ, MAP_SHARED, mmap_test_fd, 0); - if (mmap_res == MAP_FAILED) - throw0(DB_ERROR("This filesystem does not support mmap: use --data-dir to place the blockchain on a filesystem which does")); - munmap(mmap_res, 8); -#endif -} - void BlockchainLMDB::open(const std::string& filename, const int db_flags) { int result; @@ -1364,9 +1337,6 @@ void BlockchainLMDB::open(const std::string& filename, const int db_flags) m_folder = filename; - try { check_mmap_support(); } - catch(...) { MERROR("Failed to check for mmap support, proceeding"); } - #ifdef __OpenBSD__ if ((mdb_flags & MDB_WRITEMAP) == 0) { MCLOG_RED(el::Level::Info, "global", "Running on OpenBSD: forcing WRITEMAP"); diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h index 20edab2e9..bdae44948 100644 --- a/src/blockchain_db/lmdb/db_lmdb.h +++ b/src/blockchain_db/lmdb/db_lmdb.h @@ -359,7 +359,6 @@ public: static int compare_string(const MDB_val *a, const MDB_val *b); private: - void check_mmap_support(); void do_resize(uint64_t size_increase=0); bool need_resize(uint64_t threshold_size=0) const; From 033a32a20b100d8159a95da832e695e7b953de24 Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Wed, 7 Jul 2021 16:28:27 +0100 Subject: [PATCH 6/7] Remove check is_directory check on lmdb path The check interferes with raw device/partition support. --- src/blockchain_db/lmdb/db_lmdb.cpp | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index 01f91209d..db7fa6c7c 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -1307,14 +1307,8 @@ void BlockchainLMDB::open(const std::string& filename, const int db_flags) throw0(DB_OPEN_FAILURE("Attempted to open db, but it's already open")); boost::filesystem::path direc(filename); - if (boost::filesystem::exists(direc)) - { - if (!boost::filesystem::is_directory(direc)) - throw0(DB_OPEN_FAILURE("LMDB needs a directory path, but a file was passed")); - } - else - { - if (!boost::filesystem::create_directories(direc)) + if (!boost::filesystem::exists(direc) && + !boost::filesystem::create_directories(direc)) { throw0(DB_OPEN_FAILURE(std::string("Failed to create directory ").append(filename).c_str())); } From ade464a5a0adaa166171f803078e9ac31b99bb06 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Mon, 2 Nov 2020 20:33:09 +0000 Subject: [PATCH 7/7] ITS#9385 fix using MDB_NOSUBDIR with nonexistent file --- external/db_drivers/liblmdb/mdb.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/external/db_drivers/liblmdb/mdb.c b/external/db_drivers/liblmdb/mdb.c index f29a75de8..bf60c7013 100644 --- a/external/db_drivers/liblmdb/mdb.c +++ b/external/db_drivers/liblmdb/mdb.c @@ -5495,11 +5495,8 @@ mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode #ifndef _WIN32 { struct stat st; - rc = stat(path, &st); - if (rc) - return ErrCode(); flags &= ~MDB_RAWPART; - if (S_ISBLK(st.st_mode)) { + if (!stat(path, &st) && (S_ISBLK(st.st_mode) || S_ISCHR(st.st_mode))) { flags |= MDB_RAWPART | MDB_NOSUBDIR; if (!env->me_mapsize) env->me_mapsize = DEFAULT_MAPSIZE;