Bläddra i källkod

feat(storage): LmdbEnv RAII wrapper

First production-grade piece of the v2.0 LMDB substrate
(docs/superpowers/specs/2026-05-15-v2.0-storage-engine-architecture.md).
Owns an MDB_env*, sets mapsize / max_dbs / max_readers, creates the env
directory if missing. Forward-declares MDB_env so <lmdb.h> is confined
to the one .cpp file in production code.

- service/src/storage/lmdb_env.hpp: class declaration + LmdbEnvOpts.
- service/src/storage/lmdb_env.cpp: implementation. Adds an
  operator-safety check that fails fast when map_size_bytes is smaller
  than the existing data.mdb — LMDB silently pads small mapsize values
  and only surfaces the misconfiguration on the first write, which is
  poor UX for an operator who misconfigured the knob.
- tests/test_lmdb_env.cpp: six tests covering directory creation,
  path/raw accessors, mapsize-too-small fast failure, multi-handle
  same-path opens, on_disk_bytes after fresh create, and move
  semantics. Task 1.3 will extend this file with txn/dbi coverage.
- service/CMakeLists.txt: adds storage/lmdb_env.cpp to
  DATABASE_SERVICE_SOURCES.
- tests/CMakeLists.txt: file-scope LMDB resolution mirroring the
  yyjson pattern + test_lmdb_env target + CTest registration.
fszontagh 2 månader sedan
förälder
incheckning
6b5f64bb73

+ 1 - 0
service/CMakeLists.txt

@@ -59,6 +59,7 @@ set(DATABASE_SERVICE_SOURCES
     src/views/view_manager.cpp
     src/config/collection_config_manager.cpp
     src/config/config_loader.cpp
+    src/storage/lmdb_env.cpp
 )
 
 # Create executable

+ 121 - 0
service/src/storage/lmdb_env.cpp

@@ -0,0 +1,121 @@
+// v2.0 storage engine — LMDB environment RAII wrapper implementation.
+
+#include "storage/lmdb_env.hpp"
+
+#include <lmdb.h>
+
+#include <filesystem>
+#include <stdexcept>
+#include <string>
+#include <system_error>
+#include <utility>
+
+namespace smartbotic::db::storage {
+
+namespace {
+
+[[noreturn]] void throw_mdb(int rc, const char* where) {
+    std::string msg = "LMDB ";
+    msg += where;
+    msg += ": ";
+    msg += mdb_strerror(rc);
+    throw std::runtime_error(msg);
+}
+
+inline void mdb_check(int rc, const char* where) {
+    if (rc != MDB_SUCCESS) throw_mdb(rc, where);
+}
+
+}  // namespace
+
+LmdbEnv::LmdbEnv(const LmdbEnvOpts& opts) : opts_(opts) {
+    // Create the env directory if it doesn't exist. LMDB requires it to
+    // already exist on mdb_env_open.
+    std::error_code ec;
+    std::filesystem::create_directories(opts_.path, ec);
+    if (ec) {
+        throw std::runtime_error(
+            "LmdbEnv: failed to create directory '" + opts_.path + "': " + ec.message());
+    }
+
+    mdb_check(mdb_env_create(&env_), "env_create");
+
+    try {
+        mdb_check(mdb_env_set_mapsize(env_, opts_.map_size_bytes), "env_set_mapsize");
+        mdb_check(mdb_env_set_maxdbs(env_, opts_.max_dbs), "env_set_maxdbs");
+        mdb_check(mdb_env_set_maxreaders(env_, opts_.max_readers), "env_set_maxreaders");
+
+        unsigned int flags = 0;
+        if (opts_.no_sync) flags |= MDB_NOSYNC;
+
+        mdb_check(mdb_env_open(env_, opts_.path.c_str(), flags, 0644), "env_open");
+
+        // Operator-safety check: LMDB silently pads or auto-resizes small
+        // mapsize values and only surfaces the misconfiguration on the first
+        // write that exceeds the file. That's a poor failure mode for an
+        // operator who set map_size_bytes too low. If the existing data file
+        // already exceeds the requested mapsize, refuse to open.
+        std::error_code ec_fs;
+        auto data_path = std::filesystem::path(opts_.path) / "data.mdb";
+        if (std::filesystem::exists(data_path, ec_fs)) {
+            uint64_t data_sz = std::filesystem::file_size(data_path, ec_fs);
+            if (!ec_fs && data_sz > opts_.map_size_bytes) {
+                std::string msg =
+                    "LmdbEnv: map_size_bytes (" + std::to_string(opts_.map_size_bytes) +
+                    ") is smaller than existing data.mdb (" + std::to_string(data_sz) +
+                    ") at '" + opts_.path + "'. Increase map_size_bytes.";
+                throw std::runtime_error(msg);
+            }
+        }
+    } catch (...) {
+        // Don't leak the env on partial construction.
+        if (env_) {
+            mdb_env_close(env_);
+            env_ = nullptr;
+        }
+        throw;
+    }
+}
+
+LmdbEnv::~LmdbEnv() {
+    close_();
+}
+
+LmdbEnv::LmdbEnv(LmdbEnv&& other) noexcept
+    : env_(other.env_), opts_(std::move(other.opts_)) {
+    other.env_ = nullptr;
+}
+
+LmdbEnv& LmdbEnv::operator=(LmdbEnv&& other) noexcept {
+    if (this != &other) {
+        close_();
+        env_ = other.env_;
+        opts_ = std::move(other.opts_);
+        other.env_ = nullptr;
+    }
+    return *this;
+}
+
+void LmdbEnv::close_() noexcept {
+    if (env_) {
+        mdb_env_close(env_);
+        env_ = nullptr;
+    }
+}
+
+uint64_t LmdbEnv::on_disk_bytes() const {
+    namespace fs = std::filesystem;
+    uint64_t total = 0;
+    std::error_code ec;
+    fs::directory_iterator it(opts_.path, ec);
+    if (ec) return 0;
+    for (; it != fs::directory_iterator(); it.increment(ec)) {
+        if (ec) break;
+        if (it->is_regular_file(ec)) {
+            total += it->file_size(ec);
+        }
+    }
+    return total;
+}
+
+}  // namespace smartbotic::db::storage

+ 57 - 0
service/src/storage/lmdb_env.hpp

@@ -0,0 +1,57 @@
+// v2.0 storage engine — LMDB environment RAII wrapper.
+//
+// Owns an MDB_env*. Forward-declares MDB_env so this header does not
+// transitively include <lmdb.h>; only the .cpp pulls in the LMDB header.
+//
+// LMDB semantics consumed here:
+//   - mdb_env_create / mdb_env_set_mapsize / mdb_env_set_maxdbs /
+//     mdb_env_set_maxreaders / mdb_env_open / mdb_env_close.
+//   - Multi-process: two LmdbEnv instances on the same path are legal.
+//   - Single writer per env; readers are concurrent. WriteTxn / ReadTxn
+//     RAII wrappers live in lmdb_txn.hpp.
+
+#pragma once
+
+#include <cstdint>
+#include <string>
+
+// Forward declarations — keep <lmdb.h> out of headers.
+struct MDB_env;
+
+namespace smartbotic::db::storage {
+
+struct LmdbEnvOpts {
+    std::string path;                       // dir for LMDB env (created if missing)
+    size_t map_size_bytes = 256ULL << 20;   // mmap budget; resize requires restart
+    unsigned int max_dbs = 256;             // ceiling on sub-db count
+    unsigned int max_readers = 126;         // LMDB default
+    bool no_sync = false;                   // MDB_NOSYNC — durability off; only for bench
+};
+
+class LmdbEnv {
+public:
+    explicit LmdbEnv(const LmdbEnvOpts& opts);
+    ~LmdbEnv();
+
+    LmdbEnv(const LmdbEnv&) = delete;
+    LmdbEnv& operator=(const LmdbEnv&) = delete;
+    LmdbEnv(LmdbEnv&&) noexcept;
+    LmdbEnv& operator=(LmdbEnv&&) noexcept;
+
+    // Path on disk.
+    const std::string& path() const noexcept { return opts_.path; }
+
+    // Raw env handle for the txn wrappers (next task).
+    MDB_env* raw() noexcept { return env_; }
+
+    // Total bytes written to disk under path() (sum of data.mdb + lock.mdb).
+    uint64_t on_disk_bytes() const;
+
+private:
+    MDB_env* env_ = nullptr;
+    LmdbEnvOpts opts_;
+
+    void close_() noexcept;
+};
+
+}  // namespace smartbotic::db::storage

+ 28 - 0
tests/CMakeLists.txt

@@ -10,6 +10,18 @@ if(NOT yyjson_FOUND)
     pkg_check_modules(yyjson REQUIRED yyjson)
 endif()
 
+# LMDB (v2.0 storage substrate) — resolve once at file scope, mirroring the
+# yyjson pattern. liblmdb does not ship pkg-config on Debian 13, so use
+# find_path/find_library. Each test target that pulls in storage/ sources
+# adds ${LMDB_INCLUDE_DIR} / ${LMDB_LIBRARY} to its link/include flags.
+if(NOT LMDB_INCLUDE_DIR OR NOT LMDB_LIBRARY)
+    find_path(LMDB_INCLUDE_DIR lmdb.h)
+    find_library(LMDB_LIBRARY NAMES lmdb)
+    if(NOT LMDB_INCLUDE_DIR OR NOT LMDB_LIBRARY)
+        message(FATAL_ERROR "LMDB not found. Install liblmdb-dev.")
+    endif()
+endif()
+
 # Vector storage integration test
 # Compiles memory_store.cpp directly — no dependency on the full service executable.
 set(TEST_VECTOR_SOURCES
@@ -314,6 +326,21 @@ endif()
 
 target_link_libraries(bench_doc_memory PRIVATE ${yyjson_LIBRARIES})
 
+# LMDB environment unit test (v2.0 storage Phase C Stage 1, Task 1.2).
+# Compiles storage/lmdb_env.cpp directly; will pick up lmdb_txn.cpp and
+# lmdb_dbi.cpp in Task 1.3.
+add_executable(test_lmdb_env
+    test_lmdb_env.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/../service/src/storage/lmdb_env.cpp
+)
+
+target_include_directories(test_lmdb_env PRIVATE
+    ${CMAKE_CURRENT_SOURCE_DIR}/../service/src
+    ${LMDB_INCLUDE_DIR}
+)
+
+target_link_libraries(test_lmdb_env PRIVATE ${LMDB_LIBRARY})
+
 # Register with CTest
 enable_testing()
 add_test(NAME vector_storage COMMAND test_vector_storage)
@@ -324,3 +351,4 @@ add_test(NAME config_dropins COMMAND test_config_dropins)
 add_test(NAME eviction COMMAND test_eviction)
 add_test(NAME json_parse COMMAND test_json_parse)
 add_test(NAME doc_binary COMMAND test_doc_binary)
+add_test(NAME lmdb_env COMMAND test_lmdb_env)

+ 263 - 0
tests/test_lmdb_env.cpp

@@ -0,0 +1,263 @@
+// v2.0 storage engine — unit tests for LmdbEnv (Task 1.2).
+//
+// Task 1.3 will extend this file with WriteTxn / ReadTxn / LmdbDbi coverage.
+
+#include <atomic>
+#include <cassert>
+#include <cstdlib>
+#include <filesystem>
+#include <iostream>
+#include <stdexcept>
+#include <string>
+#include <unistd.h>
+
+#include <lmdb.h>
+
+#include "storage/lmdb_env.hpp"
+
+namespace fs = std::filesystem;
+
+using smartbotic::db::storage::LmdbEnv;
+using smartbotic::db::storage::LmdbEnvOpts;
+
+namespace {
+
+void check(bool cond, const char* msg) {
+    if (!cond) {
+        std::cerr << "FAIL: " << msg << "\n";
+        std::abort();
+    }
+}
+
+// Per-process, unique tmpdir for test isolation.
+std::string make_tmpdir(const char* tag) {
+    static std::atomic<int> counter{0};
+    int n = counter.fetch_add(1);
+    std::string path = "/tmp/lmdb-env-test-" + std::to_string(::getpid()) +
+                       "-" + std::to_string(n) + "-" + tag;
+    std::error_code ec;
+    fs::remove_all(path, ec);
+    return path;
+}
+
+struct TmpDir {
+    std::string path;
+    explicit TmpDir(const char* tag) : path(make_tmpdir(tag)) {}
+    ~TmpDir() {
+        std::error_code ec;
+        fs::remove_all(path, ec);
+    }
+    TmpDir(const TmpDir&) = delete;
+    TmpDir& operator=(const TmpDir&) = delete;
+};
+
+// Helper: populate an LMDB env via raw mdb_* calls. Used by the env-only
+// tests below; once Task 1.3 adds WriteTxn / LmdbDbi, the txn-level tests
+// will use those wrappers instead.
+void raw_populate(const std::string& path, size_t map_size,
+                  int n_entries, size_t entry_size) {
+    std::error_code ec;
+    fs::create_directories(path, ec);
+    if (ec) {
+        throw std::runtime_error(
+            "raw_populate: create_directories: " + ec.message());
+    }
+    MDB_env* env = nullptr;
+    if (mdb_env_create(&env) != MDB_SUCCESS) {
+        throw std::runtime_error("raw_populate: env_create");
+    }
+    if (mdb_env_set_mapsize(env, map_size) != MDB_SUCCESS) {
+        mdb_env_close(env);
+        throw std::runtime_error("raw_populate: set_mapsize");
+    }
+    if (mdb_env_open(env, path.c_str(), 0, 0644) != MDB_SUCCESS) {
+        mdb_env_close(env);
+        throw std::runtime_error("raw_populate: env_open");
+    }
+    MDB_txn* txn = nullptr;
+    if (mdb_txn_begin(env, nullptr, 0, &txn) != MDB_SUCCESS) {
+        mdb_env_close(env);
+        throw std::runtime_error("raw_populate: txn_begin");
+    }
+    MDB_dbi dbi = 0;
+    if (mdb_dbi_open(txn, nullptr, MDB_CREATE, &dbi) != MDB_SUCCESS) {
+        mdb_txn_abort(txn);
+        mdb_env_close(env);
+        throw std::runtime_error("raw_populate: dbi_open");
+    }
+    std::string blob(entry_size, 'x');
+    for (int i = 0; i < n_entries; ++i) {
+        std::string k = "k" + std::to_string(i);
+        MDB_val kv{k.size(), const_cast<char*>(k.data())};
+        MDB_val vv{blob.size(), const_cast<char*>(blob.data())};
+        int rc = mdb_put(txn, dbi, &kv, &vv, 0);
+        if (rc != MDB_SUCCESS) {
+            mdb_txn_abort(txn);
+            mdb_env_close(env);
+            throw std::runtime_error("raw_populate: put");
+        }
+    }
+    if (mdb_txn_commit(txn) != MDB_SUCCESS) {
+        mdb_env_close(env);
+        throw std::runtime_error("raw_populate: txn_commit");
+    }
+    mdb_env_close(env);
+}
+
+void raw_get(const std::string& path, const std::string& key,
+             std::string& out_value, bool& found) {
+    found = false;
+    MDB_env* env = nullptr;
+    if (mdb_env_create(&env) != MDB_SUCCESS) {
+        throw std::runtime_error("raw_get: env_create");
+    }
+    if (mdb_env_set_mapsize(env, 4ULL << 20) != MDB_SUCCESS) {
+        mdb_env_close(env);
+        throw std::runtime_error("raw_get: set_mapsize");
+    }
+    if (mdb_env_open(env, path.c_str(), 0, 0644) != MDB_SUCCESS) {
+        mdb_env_close(env);
+        throw std::runtime_error("raw_get: env_open");
+    }
+    MDB_txn* txn = nullptr;
+    if (mdb_txn_begin(env, nullptr, MDB_RDONLY, &txn) != MDB_SUCCESS) {
+        mdb_env_close(env);
+        throw std::runtime_error("raw_get: txn_begin");
+    }
+    MDB_dbi dbi = 0;
+    int rc = mdb_dbi_open(txn, nullptr, 0, &dbi);
+    if (rc != MDB_SUCCESS) {
+        mdb_txn_abort(txn);
+        mdb_env_close(env);
+        if (rc == MDB_NOTFOUND) return;
+        throw std::runtime_error("raw_get: dbi_open");
+    }
+    MDB_val kv{key.size(), const_cast<char*>(key.data())};
+    MDB_val vv{0, nullptr};
+    rc = mdb_get(txn, dbi, &kv, &vv);
+    if (rc == MDB_SUCCESS) {
+        out_value.assign(static_cast<const char*>(vv.mv_data), vv.mv_size);
+        found = true;
+    }
+    mdb_txn_abort(txn);
+    mdb_env_close(env);
+}
+
+// =========================================================================
+// Task 1.2 — LmdbEnv tests
+// =========================================================================
+
+void test_open_creates_dir() {
+    TmpDir td("create-dir");
+    check(!fs::exists(td.path), "precondition: tmpdir absent");
+    LmdbEnvOpts opts;
+    opts.path = td.path;
+    LmdbEnv env(opts);
+    check(fs::exists(td.path) && fs::is_directory(td.path),
+          "LmdbEnv constructor created the env directory");
+    // LMDB also created its data.mdb / lock.mdb in there.
+    check(fs::exists(td.path + "/data.mdb"), "data.mdb exists after open");
+    check(fs::exists(td.path + "/lock.mdb"), "lock.mdb exists after open");
+}
+
+void test_path_accessor() {
+    TmpDir td("path-accessor");
+    LmdbEnvOpts opts;
+    opts.path = td.path;
+    LmdbEnv env(opts);
+    check(env.path() == td.path, "env.path() returns configured path");
+    check(env.raw() != nullptr, "env.raw() returns non-null MDB_env*");
+}
+
+void test_mapsize_too_small_fails() {
+    TmpDir td("mapsize-small");
+
+    // Populate via raw LMDB so the data.mdb file grows past the small mapsize
+    // we'll try to reopen with.
+    raw_populate(td.path, 4ULL << 20 /* 4 MiB */, 16, 64 * 1024 /* 64 KiB */);
+
+    // Sanity: the populated dir reports nonzero on-disk bytes.
+    uint64_t bytes = 0;
+    {
+        LmdbEnvOpts opts;
+        opts.path = td.path;
+        opts.map_size_bytes = 4ULL << 20;
+        LmdbEnv env(opts);
+        bytes = env.on_disk_bytes();
+    }
+    check(bytes > 0, "populated env has nonzero on_disk_bytes");
+
+    // Re-opening with a smaller mapsize than the existing data file must
+    // fail fast — LMDB rejects mapsize < size-of-data-file.
+    LmdbEnvOpts opts;
+    opts.path = td.path;
+    opts.map_size_bytes = 4096;  // 4 KiB — far below data.mdb size.
+    bool threw = false;
+    try {
+        LmdbEnv env(opts);
+    } catch (const std::runtime_error&) {
+        threw = true;
+    }
+    check(threw, "opening with too-small mapsize fails fast");
+}
+
+void test_double_open_same_path_separate_envs() {
+    TmpDir td("double-open");
+    LmdbEnvOpts opts;
+    opts.path = td.path;
+
+    // Seed via raw LMDB so we have data to read through the second handle.
+    raw_populate(td.path, 4ULL << 20, 4, 64);
+
+    // LMDB supports multi-process access; two envs on the same path should
+    // both be openable from the same process too.
+    LmdbEnv a(opts);
+    LmdbEnv b(opts);
+
+    check(a.raw() != nullptr, "first env handle non-null");
+    check(b.raw() != nullptr, "second env handle non-null");
+    check(a.raw() != b.raw(), "two LmdbEnv handles are distinct objects");
+
+    // Sanity: both envs see the seeded data via raw_get (which opens a third
+    // handle internally — proves multi-handle reads work).
+    std::string v;
+    bool found = false;
+    raw_get(td.path, "k0", v, found);
+    check(found, "raw_get sees seeded key while two LmdbEnv handles are open");
+}
+
+void test_on_disk_bytes_nonzero_after_create() {
+    TmpDir td("on-disk-bytes");
+    LmdbEnvOpts opts;
+    opts.path = td.path;
+    LmdbEnv env(opts);
+    check(env.on_disk_bytes() > 0,
+          "fresh env reports nonzero on_disk_bytes (data.mdb/lock.mdb initialised)");
+}
+
+void test_move_constructor_transfers_ownership() {
+    TmpDir td("move-ctor");
+    LmdbEnvOpts opts;
+    opts.path = td.path;
+    LmdbEnv a(opts);
+    MDB_env* raw_a = a.raw();
+    check(raw_a != nullptr, "source env has non-null raw before move");
+
+    LmdbEnv b(std::move(a));
+    check(b.raw() == raw_a, "moved-to env owns the original handle");
+    check(a.raw() == nullptr, "moved-from env has null handle");
+    check(b.path() == td.path, "moved-to env retains path");
+}
+
+}  // namespace
+
+int main() {
+    test_open_creates_dir();
+    test_path_accessor();
+    test_mapsize_too_small_fails();
+    test_double_open_same_path_separate_envs();
+    test_on_disk_bytes_nonzero_after_create();
+    test_move_constructor_transfers_ownership();
+    std::cout << "test_lmdb_env: all passed\n";
+    return 0;
+}