Quellcode durchsuchen

v2.3 Stages B+C: per-project storage layout + legacy-env migration

Stage B — `ProjectStoreRegistry` (service/src/storage/project_store.{hpp,cpp})
holds one `LmdbEnv` + one `LmdbDocumentStore` per project, keyed by
project name. `openExisting()` scans `<dataDir>/projects/` and opens
every valid project dir; ensures `"default"` is always present.
`getOrCreate(project)` is the lazy-open lookup the mirror uses;
`get(project)` is the read-only lookup the gRPC layer (Stage D) will
call. `create()` / `drop()` are exposed for Stage F's CRUD RPCs. Drop
refuses `"default"`. All ops thread-safe via `std::shared_mutex`.

DatabaseService: removed single `env_` / `doc_store_` members; replaced
with `std::unique_ptr<ProjectStoreRegistry> projects_`. `setupComponents`
opens the registry instead of one env. The mirror wiring changed to
take a *resolver function* — `MemoryStore::setDocumentStoreMirror` now
accepts `std::function<DocumentStore*(string_view project)>`. The
mirror call parses the qualified key ("project:collection") and routes
to the right project's DocumentStore; passes only the unqualified
collection name to `applyDualWriteMirror`. New accessors:
`docStore()` (defaults to "default" for back-compat), `docStore(project)`,
`projects()`, plus `listProjects` / `createProject` / `dropProject`
delegating to the registry.

`backfillIntoDocStore` now iterates collections per project. For each
qualified MemoryStore key it parses (project, collection), resolves
the project's DocumentStore, and mirrors there. After success it
stamps `_meta.schema_version=2` on EVERY opened project env so future
boots short-circuit.

Stage C — `migrateLegacyEnvToDefaultProject`: on boot, if
`<dataDir>/env/` exists AND `<dataDir>/projects/default/env/` does
not, atomically rename the legacy directory under the default
project. Idempotent. Refuses to start with a clear error if both
exist (operator must resolve).

## Test impact

- `test_dual_write_mirror` integration case updated to wrap the single
  doc_store in a no-op resolver — the lambda ignores the project arg
  and returns the test's single DocumentStore.
- `test_timestamp_precision` had a latent v2.2.0 bug — `test_default_is_ms`
  asserted the old default after the v2.2.0 flip; the v2.2.0 ctest
  passed only because of stale incremental .o files. Renamed to
  `test_default_is_ns` and flipped the assertion. The bug was caught
  by the fresh build that v2.3 forced (rm -rf build after absl deb
  upgrade broke cached link flags).

13/13 ctest green.
fszontagh vor 2 Monaten
Ursprung
Commit
da11a8a982

+ 1 - 0
service/CMakeLists.txt

@@ -64,6 +64,7 @@ set(DATABASE_SERVICE_SOURCES
     src/storage/lmdb_dbi.cpp
     src/storage/document_store_lmdb.cpp
     src/storage/migrate_v1_to_v2.cpp
+    src/storage/project_store.cpp
 )
 
 # Create executable

+ 163 - 86
service/src/database_service.cpp

@@ -1,10 +1,12 @@
 #include "database_service.hpp"
 #include "json_parse.hpp"
+#include "project_addressing.hpp"
 #include "storage/document_store.hpp"
 #include "storage/document_store_lmdb.hpp"
 #include "storage/dual_write_mirror.hpp"
 #include "storage/lmdb_env.hpp"
 #include "storage/migrate_v1_to_v2.hpp"
+#include "storage/project_store.hpp"
 
 #include <grpcpp/grpcpp.h>
 #include <grpcpp/resource_quota.h>
@@ -209,6 +211,72 @@ bool DatabaseService::initialize() {
     }
 }
 
+void DatabaseService::migrateLegacyEnvToDefaultProject() {
+    const auto legacy = config_.dataDirectory / "env";
+    const auto target = config_.dataDirectory / "projects" /
+                        smartbotic::database::kDefaultProject / "env";
+
+    std::error_code ec;
+    const bool legacy_exists = std::filesystem::exists(legacy, ec);
+    const bool target_exists = std::filesystem::exists(target, ec);
+
+    if (!legacy_exists) return;  // fresh install or already migrated
+
+    if (target_exists) {
+        spdlog::error("v2.3 storage: both legacy '{}' and new '{}' exist — "
+                      "refusing to start. Inspect manually; the safe move is "
+                      "to either delete the legacy dir (if you confirm it's a "
+                      "leftover) or stop and contact ops.",
+                      legacy.string(), target.string());
+        throw std::runtime_error("v2.3 storage migration: ambiguous layout");
+    }
+
+    std::filesystem::create_directories(target.parent_path(), ec);
+    if (ec) {
+        throw std::runtime_error("v2.3 storage migration: cannot create '"
+                                  + target.parent_path().string() + "': " + ec.message());
+    }
+    std::filesystem::rename(legacy, target, ec);
+    if (ec) {
+        throw std::runtime_error("v2.3 storage migration: rename '"
+                                  + legacy.string() + "' -> '"
+                                  + target.string() + "' failed: " + ec.message());
+    }
+    spdlog::info("v2.3 storage: migrated legacy env '{}' -> '{}' (default project)",
+                 legacy.string(), target.string());
+}
+
+smartbotic::db::storage::DocumentStore* DatabaseService::docStore() noexcept {
+    return docStore(smartbotic::database::kDefaultProject);
+}
+
+smartbotic::db::storage::DocumentStore*
+DatabaseService::docStore(std::string_view project) noexcept {
+    if (!projects_) return nullptr;
+    return projects_->get(project);
+}
+
+std::vector<std::string> DatabaseService::listProjects() const {
+    if (!projects_) return {};
+    return projects_->listOnDisk();
+}
+
+bool DatabaseService::createProject(const std::string& name, std::string& error) {
+    if (!projects_) {
+        error = "project registry not initialized";
+        return false;
+    }
+    return projects_->create(name, error);
+}
+
+bool DatabaseService::dropProject(const std::string& name, std::string& error) {
+    if (!projects_) {
+        error = "project registry not initialized";
+        return false;
+    }
+    return projects_->drop(name, error);
+}
+
 bool DatabaseService::runMigrations() {
     if (!config_.migrations.enabled) {
         return true;
@@ -224,23 +292,27 @@ bool DatabaseService::runMigrations() {
 }
 
 bool DatabaseService::backfillIntoDocStore() {
-    if (!doc_store_ || !env_) {
-        // Env open failed earlier; nothing to backfill into. mirror_healthy_
-        // stays true (default) — there's no mirror to be unhealthy.
+    if (!projects_) {
+        // Registry open failed earlier; nothing to back-fill into.
         return true;
     }
 
-    // Skip when migrate_v1_to_v2 already populated the env. That tool
-    // writes _meta.schema_version=2 atomically on success.
-    try {
-        if (smartbotic::db::storage::migration_complete(*env_)) {
-            spdlog::info("v2.0 backfill: env already migrated (schema_version=2), skipping");
-            return true;
+    // v2.3 — MemoryStore stores collection keys in qualified form
+    // ("project:collection"). For the default project (back-compat
+    // path) the marker check operates on the default env. Skip when
+    // already migrated.
+    auto handle = projects_->getHandle(smartbotic::database::kDefaultProject);
+    if (handle.env) {
+        try {
+            if (smartbotic::db::storage::migration_complete(*handle.env)) {
+                spdlog::info("v2.3 backfill: default project already migrated, skipping");
+                return true;
+            }
+        } catch (const std::exception& e) {
+            spdlog::warn("v2.3 backfill: migration_complete probe failed: {} — "
+                         "treating default env as fresh and proceeding",
+                         e.what());
         }
-    } catch (const std::exception& e) {
-        spdlog::warn("v2.0 backfill: migration_complete probe failed: {} — "
-                     "treating env as fresh and proceeding with backfill",
-                     e.what());
     }
 
     const auto collections = store_->listCollections();
@@ -248,25 +320,39 @@ bool DatabaseService::backfillIntoDocStore() {
     uint64_t total_failures = 0;
     auto t0 = std::chrono::steady_clock::now();
 
-    for (const auto& collection : collections) {
-        if (collection.empty() || collection[0] == '_') continue;
-        const auto docs = store_->getAllDocuments(collection);
+    for (const auto& qualified : collections) {
+        if (qualified.empty() || qualified[0] == '_') continue;
+        // Each qualified key parses to (project, collection); route the
+        // mirror write to the right project's env.
+        smartbotic::database::ProjectCollection pc;
+        try {
+            pc = smartbotic::database::parseProjectCollection(qualified);
+        } catch (const std::exception&) {
+            // Legacy unqualified key — treat as default project.
+            pc = {std::string(smartbotic::database::kDefaultProject), qualified};
+        }
+        auto* ds = projects_->getOrCreate(pc.project);
+        if (!ds) {
+            ++total_failures;
+            continue;
+        }
+        const auto docs = store_->getAllDocuments(qualified);
         for (const auto& doc : docs) {
             std::optional<Document> opt_doc(doc);
             const uint64_t drift_before = mirror_drift_count_.load(std::memory_order_relaxed);
             smartbotic::db::storage::applyDualWriteMirror(
-                doc_store_.get(), mirror_healthy_, mirror_drift_count_,
-                collection, doc.id, opt_doc, EventType::INSERT);
+                ds, mirror_healthy_, mirror_drift_count_,
+                pc.collection, doc.id, opt_doc, EventType::INSERT);
             if (mirror_drift_count_.load(std::memory_order_relaxed) > drift_before) {
                 ++total_failures;
             }
             ++total_docs;
             if (total_docs % 10000 == 0) {
                 sd_notify(0, "EXTEND_TIMEOUT_USEC=600000000");
-                const std::string status = "STATUS=v2.0 backfill: " +
+                const std::string status = "STATUS=v2.3 backfill: " +
                                            std::to_string(total_docs) + " docs mirrored";
                 sd_notify(0, status.c_str());
-                spdlog::info("v2.0 backfill: {} docs mirrored ({} failures so far)",
+                spdlog::info("v2.3 backfill: {} docs mirrored ({} failures so far)",
                              total_docs, total_failures);
             }
         }
@@ -275,25 +361,23 @@ bool DatabaseService::backfillIntoDocStore() {
     const auto elapsed = std::chrono::duration_cast<std::chrono::milliseconds>(
         std::chrono::steady_clock::now() - t0).count();
     if (total_failures == 0) {
-        // v2.2.1 — stamp _meta.schema_version=2 so subsequent boots
-        // short-circuit at migration_complete() and skip the re-backfill.
-        // Pre-2.2.1 the marker was only set by the offline migrate_v1_to_v2
-        // tool, so every restart re-walked the entire MemoryStore.
-        try {
-            smartbotic::db::storage::mark_migration_complete(*env_);
-            spdlog::info("v2.0 backfill: complete — {} docs across {} collections "
-                         "in {} ms (schema_version=2 marker set)",
-                         total_docs, collections.size(), elapsed);
-        } catch (const std::exception& e) {
-            spdlog::warn("v2.0 backfill: completed {} docs in {} ms but the "
-                         "schema_version marker write failed: {} — next boot "
-                         "will re-backfill",
-                         total_docs, elapsed, e.what());
+        // Stamp schema_version=2 on every opened project env. Subsequent
+        // boots short-circuit at migration_complete().
+        for (const auto& name : projects_->listOpen()) {
+            auto h = projects_->getHandle(name);
+            if (!h.env) continue;
+            try {
+                smartbotic::db::storage::mark_migration_complete(*h.env);
+            } catch (const std::exception& e) {
+                spdlog::warn("v2.3 backfill: schema_version marker write failed for "
+                             "project '{}': {}", name, e.what());
+            }
         }
+        spdlog::info("v2.3 backfill: complete — {} docs across {} collections in {} ms",
+                     total_docs, collections.size(), elapsed);
     } else {
-        spdlog::error("v2.0 backfill: completed with {} failures out of {} docs "
-                      "in {} ms — mirror_healthy_=false, reads must NOT flip to LMDB "
-                      "and schema_version marker NOT set (next boot retries)",
+        spdlog::error("v2.3 backfill: completed with {} failures out of {} docs "
+                      "in {} ms — mirror_healthy_=false, reads must NOT flip to LMDB",
                       total_failures, total_docs, elapsed);
     }
     return true;
@@ -617,50 +701,40 @@ DatabaseService::Config DatabaseService::parseConfig(const nlohmann::json& json)
 }
 
 void DatabaseService::setupComponents() {
-    // v2.0 storage substrate (Stage 4 scaffold).
+    // v2.3 multi-project storage substrate.
     //
-    // Open the LMDB environment at <dataDir>/env/ — same path the v1.x
-    // auto-migration tool in main.cpp writes to. By the time setupComponents
-    // runs, either:
-    //   (a) the env already exists and contains migrated v2.0 data, or
-    //   (b) this is a fresh deployment and the env is empty.
-    // Both cases are fine: LmdbDocumentStore opens sub-dbs lazily on first
-    // put/read, so an empty env is a valid steady state.
+    // Each project lives at `<dataDir>/projects/<name>/env/`. The default
+    // project always exists and is what v2.0-v2.2 clients (which don't
+    // address projects) implicitly use.
     //
-    // map_size_bytes / max_dbs match the values main.cpp uses for migration
-    // so that opening the same env from both call sites is consistent.
-    //
-    // Failure to open the env is currently logged-and-tolerated: Stage 4
-    // does not yet route any handlers through doc_store_, so a missing or
-    // unopenable env should not kill the service. Stage 5+ may upgrade this
-    // to a hard failure once handlers depend on the store.
-    {
-        smartbotic::db::storage::LmdbEnvOpts envOpts;
-        envOpts.path = (config_.dataDirectory / "env").string();
-        envOpts.map_size_bytes = 4ULL << 30;  // 4 GiB initial; LMDB grows lazily.
-        envOpts.max_dbs = 1024;
+    // Pre-2.3 installs have their data at the legacy `<dataDir>/env/`
+    // path. migrateLegacyEnvToDefaultProject() detects that layout and
+    // atomically moves it under `projects/default/env/` before the
+    // registry opens.
+    migrateLegacyEnvToDefaultProject();
 
-        std::error_code ec;
-        std::filesystem::create_directories(envOpts.path, ec);
-        if (ec) {
-            spdlog::warn("v2.0 storage: failed to create env directory '{}': {} -- "
-                         "doc_store_ will be left null; Stage 4 still routes through "
-                         "MemoryStore so the service continues to boot",
-                         envOpts.path, ec.message());
-        } else {
-            try {
-                env_ = std::make_unique<smartbotic::db::storage::LmdbEnv>(envOpts);
-                doc_store_ = std::make_unique<smartbotic::db::storage::LmdbDocumentStore>(*env_);
-                spdlog::info("v2.0 storage: LMDB env opened at {} (mapsize={}MB, max_dbs={})",
-                             envOpts.path, envOpts.map_size_bytes >> 20, envOpts.max_dbs);
-            } catch (const std::exception& e) {
-                spdlog::warn("v2.0 storage: failed to open LMDB env at '{}': {} -- "
-                             "doc_store_ will be left null; service continues with v1.x",
-                             envOpts.path, e.what());
-                env_.reset();
-                doc_store_.reset();
-            }
-        }
+    try {
+        constexpr size_t kMapSize = 4ULL << 30;  // 4 GiB per project; LMDB grows lazily.
+        constexpr uint32_t kMaxDbs = 1024;
+        projects_ = std::make_unique<smartbotic::db::storage::ProjectStoreRegistry>(
+            config_.dataDirectory / "projects", kMapSize, kMaxDbs);
+        const auto opened = projects_->openExisting();
+        spdlog::info("v2.3 storage: opened {} project env(s): [{}]",
+                     opened.size(),
+                     [&opened]() {
+                         std::string s;
+                         for (size_t i = 0; i < opened.size(); ++i) {
+                             if (i) s += ", ";
+                             s += opened[i];
+                         }
+                         return s;
+                     }());
+    } catch (const std::exception& e) {
+        spdlog::error("v2.3 storage: failed to open project registry at '{}': {} -- "
+                      "service cannot start without LMDB; bailing",
+                      (config_.dataDirectory / "projects").string(), e.what());
+        projects_.reset();
+        throw;
     }
 
     // Create memory store with eviction config
@@ -700,14 +774,17 @@ void DatabaseService::setupComponents() {
     history_store_ = std::make_unique<HistoryStore>(histConfig);
     store_->setHistoryStore(history_store_.get());
 
-    // v2.0 Stage 4 — wire the LMDB mirror INTO MemoryStore. After this,
-    // every write that flows through store_ also commits to doc_store_
-    // BEFORE the per-collection lock releases. The persist callback no
-    // longer has its own mirror block; it does only WAL/replication/events.
-    if (doc_store_) {
-        store_->setDocumentStoreMirror(doc_store_.get(),
-                                       &mirror_healthy_,
-                                       &mirror_drift_count_);
+    // v2.3 — wire the LMDB mirror INTO MemoryStore with a project
+    // resolver. Every write that flows through store_ commits to the
+    // right per-project LMDB env BEFORE the per-collection lock releases.
+    if (projects_) {
+        auto* registry = projects_.get();
+        store_->setDocumentStoreMirror(
+            [registry](std::string_view project) -> smartbotic::db::storage::DocumentStore* {
+                return registry->getOrCreate(project);
+            },
+            &mirror_healthy_,
+            &mirror_drift_count_);
     }
 
     // Create persistence manager. Start from any persistenceConfig values the

+ 34 - 18
service/src/database_service.hpp

@@ -20,6 +20,7 @@
 namespace smartbotic::db::storage {
 class LmdbEnv;
 class DocumentStore;
+class ProjectStoreRegistry;
 }
 
 #include <nlohmann/json.hpp>
@@ -29,7 +30,9 @@ class DocumentStore;
 #include <memory>
 #include <mutex>
 #include <string>
+#include <string_view>
 #include <thread>
+#include <vector>
 
 namespace grpc {
 class Server;
@@ -194,12 +197,17 @@ public:
      */
     static Config parseConfig(const nlohmann::json& json);
 
-    // v2.0 Stage 4 — accessors used by the gRPC layer's shadow-read /
-    // read-flip paths. `docStore()` may return nullptr when env open
-    // failed at boot (Stage 4 tolerates that). `mirrorHealthy()` plus
-    // `mirrorDriftCount()==0` together gate any handler that wants to
-    // trust doc_store_ as a read source.
-    smartbotic::db::storage::DocumentStore* docStore() noexcept { return doc_store_.get(); }
+    // v2.3 — project-aware read accessors. `docStore(project)` returns
+    // the per-project LmdbDocumentStore or nullptr if the project isn't
+    // open (rare unless someone dropped it concurrently). The no-arg
+    // `docStore()` defaults to the "default" project for callers still
+    // on the v2.0-v2.2 API shape (Stage D will retire it). Health +
+    // drift atomics gate any LMDB-first read path.
+    smartbotic::db::storage::DocumentStore* docStore() noexcept;
+    smartbotic::db::storage::DocumentStore* docStore(std::string_view project) noexcept;
+    smartbotic::db::storage::ProjectStoreRegistry* projects() noexcept {
+        return projects_.get();
+    }
     bool mirrorHealthy() const noexcept {
         return mirror_healthy_.load(std::memory_order_acquire);
     }
@@ -207,6 +215,11 @@ public:
         return mirror_drift_count_.load(std::memory_order_relaxed);
     }
 
+    // v2.3 Stage F — project CRUD entry points (delegate to registry).
+    std::vector<std::string> listProjects() const;
+    bool createProject(const std::string& name, std::string& error);
+    bool dropProject(const std::string& name, std::string& error);
+
 private:
     void setupComponents();
     void startGrpcServer();
@@ -222,6 +235,13 @@ private:
     // do not abort boot (the read-flip downstream is the gate).
     bool backfillIntoDocStore();
 
+    // v2.3 Stage C — atomic rename of <dataDir>/env/ into
+    // <dataDir>/projects/default/env/ when the v2.2 layout is detected
+    // and the new layout doesn't yet exist. Idempotent. Refuses to start
+    // if both exist (likely operator hand-mess); the operator must
+    // resolve manually.
+    void migrateLegacyEnvToDefaultProject();
+
     Config config_;
     std::atomic<bool> running_{false};
     std::atomic<bool> stopRequested_{false};
@@ -229,18 +249,14 @@ private:
     // Components
     std::unique_ptr<MemoryStore> store_;
 
-    // v2.0 storage substrate (Stage 4 scaffold).
-    //
-    // The LMDB env is opened at `<dataDir>/env/` — the same path the v1.x
-    // auto-migration tool writes to. Stage 4 lands these as members so the
-    // service has the v2.0 store in hand but does not yet route gRPC
-    // handlers through it; that swap happens incrementally in Stage 4 / 5.
-    //
-    // Ownership: env_ owns the MDB_env*; doc_store_ holds a non-owning
-    // reference into env_. Declaration order matters — doc_store_ must be
-    // destroyed before env_.
-    std::unique_ptr<smartbotic::db::storage::LmdbEnv> env_;
-    std::unique_ptr<smartbotic::db::storage::DocumentStore> doc_store_;
+    // v2.3 — multi-project storage substrate. Each project is one
+    // LmdbEnv at `<dataDir>/projects/<name>/env/`. The registry holds
+    // them in a map keyed by project name, lazy-opens new projects on
+    // first write, and is the single source of truth for "which projects
+    // exist." Old v2.0-v2.2 installs auto-migrate the legacy
+    // `<dataDir>/env/` into `<dataDir>/projects/default/env/` at first
+    // boot (see migrateLegacyEnvToDefaultProject).
+    std::unique_ptr<smartbotic::db::storage::ProjectStoreRegistry> projects_;
 
     // v2.0 dual-write health. The persist callback mirrors every user-
     // collection write into doc_store_; on failure we log ERROR, bump the

+ 18 - 8
service/src/memory_store.cpp

@@ -2,6 +2,7 @@
 
 #include "config/collection_config_manager.hpp"
 #include "persistence/history_store.hpp"
+#include "project_addressing.hpp"
 #include "storage/document_store.hpp"
 #include "storage/dual_write_mirror.hpp"
 #include "storage/filter_eval.hpp"
@@ -2283,28 +2284,37 @@ void MemoryStore::emitPersist(const std::string& collection, const std::string&
     }
 }
 
-void MemoryStore::setDocumentStoreMirror(smartbotic::db::storage::DocumentStore* ds,
+void MemoryStore::setDocumentStoreMirror(DocumentStoreResolver resolver,
                                          std::atomic<bool>* healthy,
                                          std::atomic<uint64_t>* drift) {
-    docStoreMirror_ = ds;
+    docStoreResolver_ = std::move(resolver);
     mirrorHealthy_ = healthy;
     mirrorDriftCount_ = drift;
 }
 
 void MemoryStore::mirrorWriteToDocStore(const std::string& collection, const std::string& id,
                                          const std::optional<Document>& doc, EventType eventType) {
-    if (!docStoreMirror_ || !mirrorHealthy_ || !mirrorDriftCount_) return;
+    if (!docStoreResolver_ || !mirrorHealthy_ || !mirrorDriftCount_) return;
+    // v2.3 — collection is the qualified form "project:collection". Parse
+    // it to route to the right project's LMDB env; the unqualified piece
+    // is what the per-project DocumentStore expects.
+    auto pc = smartbotic::database::parseProjectCollection(collection);
+    auto* ds = docStoreResolver_(pc.project);
+    if (!ds) return;
     smartbotic::db::storage::applyDualWriteMirror(
-        docStoreMirror_, *mirrorHealthy_, *mirrorDriftCount_,
-        collection, id, doc, eventType);
+        ds, *mirrorHealthy_, *mirrorDriftCount_,
+        pc.collection, id, doc, eventType);
 }
 
 void MemoryStore::mirrorVectorToDocStore(const std::string& collection, const std::string& id,
                                           const std::vector<float>* vec, EventType eventType) {
-    if (!docStoreMirror_ || !mirrorHealthy_ || !mirrorDriftCount_) return;
+    if (!docStoreResolver_ || !mirrorHealthy_ || !mirrorDriftCount_) return;
+    auto pc = smartbotic::database::parseProjectCollection(collection);
+    auto* ds = docStoreResolver_(pc.project);
+    if (!ds) return;
     smartbotic::db::storage::applyVectorMirror(
-        docStoreMirror_, *mirrorHealthy_, *mirrorDriftCount_,
-        collection, id, vec, eventType);
+        ds, *mirrorHealthy_, *mirrorDriftCount_,
+        pc.collection, id, vec, eventType);
 }
 
 void MemoryStore::addToExpirationIndex(CollectionData& coll, const std::string& id, uint64_t expiresAt) {

+ 22 - 14
service/src/memory_store.hpp

@@ -156,19 +156,26 @@ public:
     void setHistoryStore(HistoryStore* hs) { historyStore_ = hs; }
 
     /**
-     * v2.0 Stage 4 — attach the LMDB document-store mirror. Once set, every
-     * write path (insert / update / patch / remove / set / restore /
-     * expire) calls `applyDualWriteMirror` on the target store BEFORE
-     * releasing the per-collection lock. That closes the race where
-     * concurrent readers could observe a doc in MemoryStore but not yet in
-     * LMDB. Safe to leave unset (tests, pre-v2.0 paths) — the mirror call
-     * becomes a no-op.
+     * v2.3 — attach a project-routing LMDB mirror. The resolver looks
+     * up the right DocumentStore for a project name; MemoryStore parses
+     * the qualified `project:collection` key it stores internally and
+     * calls the resolver to find the target. Returns nullptr on a
+     * project that no longer exists (drop race); the mirror silently
+     * skips in that case. The collection name passed to the resolver's
+     * returned DocumentStore is the unqualified portion.
+     *
+     * v2.0 → v2.3 ABI compatibility: callers that previously passed a
+     * single DocumentStore* now wrap it in a resolver that returns it
+     * for any project ("default"-only single-tenant tests still work).
      *
      * `healthy` flips to false on mirror failure (and ERROR is logged);
-     * `drift` accumulates failure count for observability. Both atomics are
-     * owned by the caller (DatabaseService) and must outlive this store.
+     * `drift` accumulates failure count for observability. Both atomics
+     * are owned by the caller (DatabaseService) and must outlive this
+     * store.
      */
-    void setDocumentStoreMirror(smartbotic::db::storage::DocumentStore* ds,
+    using DocumentStoreResolver = std::function<
+        smartbotic::db::storage::DocumentStore*(std::string_view project)>;
+    void setDocumentStoreMirror(DocumentStoreResolver resolver,
                                 std::atomic<bool>* healthy,
                                 std::atomic<uint64_t>* drift);
 
@@ -830,10 +837,11 @@ private:
     // history operations no-op (no history kept).
     HistoryStore* historyStore_ = nullptr;
 
-    // v2.0 Stage 4 — dual-write target. All three pointers set or all
-    // three null; callers own the lifetimes. When null, mirrorWriteToDocStore
-    // is a no-op (preserves test/bootstrap paths that don't wire LMDB).
-    smartbotic::db::storage::DocumentStore* docStoreMirror_ = nullptr;
+    // v2.3 — dual-write target. Resolver routes per-project lookup; the
+    // atomic pointers are owned by DatabaseService. When the resolver is
+    // unset, mirrorWriteToDocStore / mirrorVectorToDocStore are no-ops
+    // (preserves test/bootstrap paths that don't wire LMDB).
+    DocumentStoreResolver docStoreResolver_;
     std::atomic<bool>* mirrorHealthy_ = nullptr;
     std::atomic<uint64_t>* mirrorDriftCount_ = nullptr;
 

+ 219 - 0
service/src/storage/project_store.cpp

@@ -0,0 +1,219 @@
+// v2.3 multi-project — ProjectStoreRegistry implementation.
+
+#include "storage/project_store.hpp"
+
+#include <algorithm>
+#include <stdexcept>
+#include <system_error>
+
+#include <spdlog/spdlog.h>
+
+#include "project_addressing.hpp"
+#include "storage/document_store_lmdb.hpp"
+
+namespace fs = std::filesystem;
+
+namespace smartbotic::db::storage {
+
+namespace {
+
+constexpr const char* kEnvSubdir = "env";
+
+}  // namespace
+
+ProjectStoreRegistry::ProjectStoreRegistry(fs::path projects_root,
+                                            size_t map_size_bytes,
+                                            uint32_t max_dbs)
+    : projects_root_(std::move(projects_root)),
+      map_size_bytes_(map_size_bytes),
+      max_dbs_(max_dbs) {}
+
+fs::path ProjectStoreRegistry::project_dir(std::string_view name) const {
+    return projects_root_ / std::string(name);
+}
+
+fs::path ProjectStoreRegistry::env_dir(std::string_view name) const {
+    return project_dir(name) / kEnvSubdir;
+}
+
+std::unique_ptr<ProjectStore>
+ProjectStoreRegistry::open_one(const std::string& name) {
+    LmdbEnvOpts opts;
+    opts.path = env_dir(name).string();
+    opts.map_size_bytes = map_size_bytes_;
+    opts.max_dbs = max_dbs_;
+
+    std::error_code ec;
+    fs::create_directories(opts.path, ec);
+    if (ec) {
+        throw std::runtime_error("ProjectStoreRegistry: failed to create env dir '"
+                                  + opts.path + "': " + ec.message());
+    }
+
+    auto store = std::make_unique<ProjectStore>();
+    store->name = name;
+    store->env = std::make_unique<LmdbEnv>(opts);
+    store->doc_store = std::make_unique<LmdbDocumentStore>(*store->env);
+    return store;
+}
+
+std::vector<std::string> ProjectStoreRegistry::openExisting() {
+    std::unique_lock lk(mutex_);
+
+    std::error_code ec;
+    fs::create_directories(projects_root_, ec);
+    if (ec) {
+        throw std::runtime_error("ProjectStoreRegistry: cannot create root '"
+                                  + projects_root_.string() + "': " + ec.message());
+    }
+
+    std::vector<std::string> names;
+    for (const auto& entry : fs::directory_iterator(projects_root_, ec)) {
+        if (ec) break;
+        if (!entry.is_directory()) continue;
+        const std::string n = entry.path().filename().string();
+        if (!smartbotic::database::isValidProjectName(n)) {
+            spdlog::warn("ProjectStoreRegistry: skipping invalid project dir '{}'", n);
+            continue;
+        }
+        names.push_back(n);
+    }
+    if (ec) {
+        throw std::runtime_error("ProjectStoreRegistry: scan failed: " + ec.message());
+    }
+
+    // Ensure "default" is always in the set even if its dir doesn't exist yet.
+    const std::string defaultName(smartbotic::database::kDefaultProject);
+    if (std::find(names.begin(), names.end(), defaultName) == names.end()) {
+        names.push_back(defaultName);
+    }
+    std::sort(names.begin(), names.end());
+
+    for (const auto& n : names) {
+        if (stores_.count(n)) continue;
+        stores_.emplace(n, open_one(n));
+    }
+    return names;
+}
+
+DocumentStore* ProjectStoreRegistry::getOrCreate(std::string_view project) {
+    if (!smartbotic::database::isValidProjectName(project)) return nullptr;
+
+    {
+        std::shared_lock lk(mutex_);
+        auto it = stores_.find(project);
+        if (it != stores_.end()) return it->second->doc_store.get();
+    }
+
+    std::unique_lock lk(mutex_);
+    auto it = stores_.find(project);
+    if (it != stores_.end()) return it->second->doc_store.get();
+
+    auto store = open_one(std::string(project));
+    DocumentStore* ptr = store->doc_store.get();
+    stores_.emplace(std::string(project), std::move(store));
+    return ptr;
+}
+
+DocumentStore* ProjectStoreRegistry::get(std::string_view project) {
+    std::shared_lock lk(mutex_);
+    auto it = stores_.find(project);
+    return it != stores_.end() ? it->second->doc_store.get() : nullptr;
+}
+
+ProjectStoreRegistry::StoreHandle
+ProjectStoreRegistry::getHandle(std::string_view project) {
+    std::shared_lock lk(mutex_);
+    auto it = stores_.find(project);
+    if (it == stores_.end()) return {};
+    return {it->second->doc_store.get(), it->second->env.get()};
+}
+
+std::vector<std::string> ProjectStoreRegistry::listOpen() const {
+    std::shared_lock lk(mutex_);
+    std::vector<std::string> out;
+    out.reserve(stores_.size());
+    for (const auto& kv : stores_) out.push_back(kv.first);
+    return out;
+}
+
+std::vector<std::string> ProjectStoreRegistry::listOnDisk() const {
+    std::vector<std::string> names;
+    std::error_code ec;
+    for (const auto& entry : fs::directory_iterator(projects_root_, ec)) {
+        if (ec) break;
+        if (!entry.is_directory()) continue;
+        const std::string n = entry.path().filename().string();
+        if (!smartbotic::database::isValidProjectName(n)) continue;
+        names.push_back(n);
+    }
+    const std::string defaultName(smartbotic::database::kDefaultProject);
+    if (std::find(names.begin(), names.end(), defaultName) == names.end()) {
+        names.push_back(defaultName);
+    }
+    std::sort(names.begin(), names.end());
+    return names;
+}
+
+bool ProjectStoreRegistry::create(std::string_view name, std::string& error) {
+    if (!smartbotic::database::isValidProjectName(name)) {
+        error = "invalid project name: must match ^[a-zA-Z_][a-zA-Z0-9_-]{0,62}$ "
+                "and not start with '_'";
+        return false;
+    }
+
+    std::unique_lock lk(mutex_);
+    if (stores_.count(name)) return false;  // already exists, idempotent
+
+    try {
+        auto store = open_one(std::string(name));
+        stores_.emplace(std::string(name), std::move(store));
+        return true;
+    } catch (const std::exception& e) {
+        error = e.what();
+        return false;
+    }
+}
+
+bool ProjectStoreRegistry::drop(std::string_view name, std::string& error) {
+    if (name == smartbotic::database::kDefaultProject) {
+        error = "cannot drop default project";
+        return false;
+    }
+    if (!smartbotic::database::isValidProjectName(name)) {
+        error = "invalid project name";
+        return false;
+    }
+
+    std::unique_lock lk(mutex_);
+    auto it = stores_.find(name);
+    if (it == stores_.end()) {
+        // Maybe on disk but not opened (rare). Try to rm the dir anyway.
+        const fs::path dir = project_dir(name);
+        std::error_code ec;
+        if (!fs::exists(dir, ec)) {
+            error = "project not found";
+            return false;
+        }
+        fs::remove_all(dir, ec);
+        if (ec) {
+            error = "rm failed: " + ec.message();
+            return false;
+        }
+        return true;
+    }
+
+    // Close the env first (unique_ptr destructor) before rm.
+    stores_.erase(it);
+
+    const fs::path dir = project_dir(name);
+    std::error_code ec;
+    fs::remove_all(dir, ec);
+    if (ec) {
+        error = "rm failed: " + ec.message();
+        return false;
+    }
+    return true;
+}
+
+}  // namespace smartbotic::db::storage

+ 103 - 0
service/src/storage/project_store.hpp

@@ -0,0 +1,103 @@
+// v2.3 multi-project — per-project LMDB env registry.
+//
+// Each project is one LmdbEnv at <dataDir>/projects/<name>/env/. The
+// registry holds them in a map keyed by project name, lazy-opens new
+// projects on first write, and exposes a get(project) lookup that the
+// gRPC handlers + mirror call.
+//
+// Default project semantics: "default" is always present. listOpen()
+// and listOnDisk() always include it; getOrCreate("default") never
+// returns nullptr; drop("default") errors out.
+//
+// Thread-safety: open/create/drop take a unique_lock on mutex_; get()
+// takes a shared_lock. The returned DocumentStore* outlives the lock
+// only as long as no concurrent drop() fires; in practice drop is
+// admin-rare and callers hold their docstore pointer for the duration
+// of one request, which is short enough that the race is benign.
+
+#pragma once
+
+#include <filesystem>
+#include <map>
+#include <memory>
+#include <mutex>
+#include <shared_mutex>
+#include <string>
+#include <string_view>
+#include <vector>
+
+#include "storage/document_store.hpp"
+#include "storage/lmdb_env.hpp"
+
+namespace smartbotic::db::storage {
+
+struct ProjectStore {
+    std::string name;
+    std::unique_ptr<LmdbEnv> env;
+    std::unique_ptr<DocumentStore> doc_store;
+};
+
+class ProjectStoreRegistry {
+public:
+    // projects_root = <dataDir>/projects/. May not exist yet; create()
+    // and getOrCreate() will mkdir as needed.
+    ProjectStoreRegistry(std::filesystem::path projects_root,
+                         size_t map_size_bytes,
+                         uint32_t max_dbs);
+
+    // Scan projects_root for existing project directories and open each.
+    // Always ensures "default" is opened (creates it on disk if absent).
+    // Returns the list of opened project names in sorted order.
+    //
+    // Throws std::runtime_error if any LMDB env open fails — boot should
+    // be loud about disk-level corruption rather than degrading to a
+    // partial state.
+    std::vector<std::string> openExisting();
+
+    // Resolve a project name to its DocumentStore*, creating the project
+    // on disk if it doesn't exist. Returns nullptr ONLY if the name is
+    // invalid per isValidProjectName(). Thread-safe.
+    DocumentStore* getOrCreate(std::string_view project);
+
+    // Read-only lookup. Returns nullptr if the project is unknown.
+    DocumentStore* get(std::string_view project);
+
+    // Read-only lookup that returns both the DocumentStore* and the
+    // underlying LmdbEnv* (the latter for callers that need to invoke
+    // migration helpers like mark_migration_complete). Nullptrs on miss.
+    struct StoreHandle {
+        DocumentStore* doc_store = nullptr;
+        LmdbEnv* env = nullptr;
+    };
+    StoreHandle getHandle(std::string_view project);
+
+    // Currently-open project names, sorted, always includes "default".
+    std::vector<std::string> listOpen() const;
+
+    // All projects visible on disk (scans projects_root/). Always
+    // includes "default" even when the directory isn't created yet.
+    std::vector<std::string> listOnDisk() const;
+
+    // Idempotent create. Returns false (without setting error) when the
+    // project already exists. Returns false with error set on invalid
+    // name. Throws on disk failure.
+    bool create(std::string_view name, std::string& error);
+
+    // Drop. Refuses "default" with error. Closes the env, removes the
+    // directory. Returns false with error if name is "default", absent,
+    // or rm fails.
+    bool drop(std::string_view name, std::string& error);
+
+private:
+    std::filesystem::path project_dir(std::string_view name) const;
+    std::filesystem::path env_dir(std::string_view name) const;
+    std::unique_ptr<ProjectStore> open_one(const std::string& name);
+
+    std::filesystem::path projects_root_;
+    size_t map_size_bytes_;
+    uint32_t max_dbs_;
+    mutable std::shared_mutex mutex_;
+    std::map<std::string, std::unique_ptr<ProjectStore>, std::less<>> stores_;
+};
+
+}  // namespace smartbotic::db::storage

+ 3 - 1
tests/test_dual_write_mirror.cpp

@@ -298,7 +298,9 @@ void test_vector_mirror_via_memory_store() {
     cfg.nodeId = "test-node";
     cfg.maxMemoryBytes = 64ULL * 1024 * 1024;
     MemoryStore mem(cfg);
-    mem.setDocumentStoreMirror(&doc_store, &healthy, &drift);
+    mem.setDocumentStoreMirror(
+        [&doc_store](std::string_view) { return &doc_store; },
+        &healthy, &drift);
 
     smartbotic::database::CollectionOptions opts;
     opts.vectorDimension = 4;

+ 7 - 5
tests/test_timestamp_precision.cpp

@@ -51,7 +51,9 @@ Document makeDoc(const std::string& id = "") {
 
 } // anonymous namespace
 
-void test_default_is_ms() {
+void test_default_is_ns() {
+    // v2.2 — default flipped to "ns" so rapid-write collections get
+    // strict-ordering timestamps without an explicit configureCollection().
     MemoryStore store(defaultConfig());
     store.start();
     CollectionConfigManager mgr(store);
@@ -63,12 +65,12 @@ void test_default_is_ms() {
     auto doc = store.get("docs_default", id);
     assert(doc.has_value());
     assert(doc->createdAt > 0);
-    assert(doc->createdAt < NS_THRESHOLD);  // ms range
+    assert(doc->createdAt >= NS_THRESHOLD);  // ns range
     assert(doc->updatedAt > 0);
-    assert(doc->updatedAt < NS_THRESHOLD);
+    assert(doc->updatedAt >= NS_THRESHOLD);
 
     store.stop();
-    std::cout << "PASS: default precision is ms (createdAt=" << doc->createdAt << ")\n";
+    std::cout << "PASS: default precision is ns (createdAt=" << doc->createdAt << ")\n";
 }
 
 void test_ns_yields_unique_timestamps_in_tight_loop() {
@@ -194,7 +196,7 @@ void test_default_for_unknown_collection() {
 }
 
 int main() {
-    test_default_is_ms();
+    test_default_is_ns();
     test_ns_yields_unique_timestamps_in_tight_loop();
     test_ms_baseline_fits_in_ms_range();
     test_idempotent_configure();