feat(storage): vector dual-write into _vectors_<collection> sub-db

Stage 5 task 5.1, first half — vector mirror infrastructure.

DocumentStore gains put_vector / del_vector / get_vector (default no-op
so non-LMDB backends still compile). LmdbDocumentStore implements them
against the _vectors_<collection> sub-db pattern already proven by the
v1→v2 migration tool: raw float32 bytes keyed by docId, native byte
order (x86-64 only). open_for_write / try_open_for_read are reused
unchanged — they care about the sub-db name, not its prefix.

applyVectorMirror is a sibling of applyDualWriteMirror in
dual_write_mirror.hpp. Same failure semantics — ERROR log, bump drift,
flip mirror_healthy_ on throw. MemoryStore::mirrorVectorToDocStore is
the under-lock wrapper, called after every storeVector / removeVector:
insert, update (store-or-remove), upsert (store / conditional remove),
delete (always del), patch (only when vec supplied), expire (del).
updateIfVersion and restore don't touch the vector map (v1.x quirk
preserved).

Test: tests/test_dual_write_mirror.cpp grows 14 vector assertions
across 4 cases — direct helper round-trip, empty-vec is no-op, system-
collection skip, and an end-to-end memstore path that creates a
vectorDimension=4 collection and verifies LMDB vector sub-db tracks
inserts and deletes. 43/43 green.

Stress: load_test_mixed (30s / 4 writers / 8 readers) still PASSes
unchanged (2385 writes/s, 5769 reads/s, 0 vector mirror failures) —
the workload doesn't actually use vectors but the build still links
the new code path.

Next half (Stage 5 task 5.1): SimilaritySearch handler reads vectors
from the LMDB sub-db instead of MemoryStore's coll.vectors map.

service/src/memory_store.cpp

@@ -295,6 +295,9 @@ std::string MemoryStore::insert(const std::string& collection, Document doc) {
 
     // v2.0 dual-write under lock — see header comment on mirrorWriteToDocStore.
     mirrorWriteToDocStore(collection, docId, doc, EventType::INSERT);
+    if (!vec.empty()) {
+        mirrorVectorToDocStore(collection, docId, &vec, EventType::INSERT);
+    }
 
     lock.unlock();
 
@@ -455,6 +458,11 @@ bool MemoryStore::update(const std::string& collection, const std::string& id, c
 
     // v2.0 dual-write under lock
     mirrorWriteToDocStore(collection, id, updated, EventType::UPDATE);
+    if (!vec.empty()) {
+        mirrorVectorToDocStore(collection, id, &vec, EventType::UPDATE);
+    } else {
+        mirrorVectorToDocStore(collection, id, nullptr, EventType::DELETE);
+    }
 
     lock.unlock();
 
@@ -569,6 +577,12 @@ std::string MemoryStore::upsert(const std::string& collection, Document doc) {
     // v2.0 dual-write under lock (upsert path — INSERT or UPDATE depending on isInsert)
     mirrorWriteToDocStore(collection, docId, doc,
                           isInsert ? EventType::INSERT : EventType::UPDATE);
+    if (!vec.empty()) {
+        mirrorVectorToDocStore(collection, docId, &vec,
+                               isInsert ? EventType::INSERT : EventType::UPDATE);
+    } else if (!isInsert) {
+        mirrorVectorToDocStore(collection, docId, nullptr, EventType::DELETE);
+    }
 
     lock.unlock();
 
@@ -628,8 +642,9 @@ bool MemoryStore::remove(const std::string& collection, const std::string& id) {
     removeVector(*coll, id);
     coll->updatedAt = currentTimeMs();
 
-    // v2.0 dual-write under lock (delete path)
+    // v2.0 dual-write under lock (delete path) — both doc and vector
     mirrorWriteToDocStore(collection, id, std::nullopt, EventType::DELETE);
+    mirrorVectorToDocStore(collection, id, nullptr, EventType::DELETE);
 
     lock.unlock();
 
@@ -800,8 +815,13 @@ uint64_t MemoryStore::patchDocument(const std::string& collection, const std::st
     // Capture the updated document for callbacks after releasing the lock
     Document updatedDoc = it->second;
 
-    // v2.0 dual-write under lock
+    // v2.0 dual-write under lock (patch path — vector only mirrored when
+    // the patch actually supplied a new _vector; otherwise the existing
+    // vector stays in place on both stores)
     mirrorWriteToDocStore(collection, id, updatedDoc, EventType::UPDATE);
+    if (!vec.empty()) {
+        mirrorVectorToDocStore(collection, id, &vec, EventType::UPDATE);
+    }
 
     lock.unlock();
 
@@ -1541,8 +1561,10 @@ uint64_t MemoryStore::expireDocuments() {
 
                 // v2.0 dual-write under lock — expire is semantically a delete
                 // for the substrate. Mirror it as DELETE so LMDB doesn't keep
-                // an orphan doc whose MemoryStore version has expired.
+                // an orphan doc whose MemoryStore version has expired. Same
+                // for the vector sub-db; del_vector is a no-op if no vector.
                 mirrorWriteToDocStore(collName, id, std::nullopt, EventType::DELETE);
+                mirrorVectorToDocStore(collName, id, nullptr, EventType::DELETE);
 
                 // Emit callbacks (unlock first to avoid deadlock)
                 collLock.unlock();
@@ -2256,6 +2278,14 @@ void MemoryStore::mirrorWriteToDocStore(const std::string& collection, const std
         collection, id, doc, eventType);
 }
 
+void MemoryStore::mirrorVectorToDocStore(const std::string& collection, const std::string& id,
+                                          const std::vector<float>* vec, EventType eventType) {
+    if (!docStoreMirror_ || !mirrorHealthy_ || !mirrorDriftCount_) return;
+    smartbotic::db::storage::applyVectorMirror(
+        docStoreMirror_, *mirrorHealthy_, *mirrorDriftCount_,
+        collection, id, vec, eventType);
+}
+
 void MemoryStore::addToExpirationIndex(CollectionData& coll, const std::string& id, uint64_t expiresAt) {
     coll.expirationIndex[expiresAt].insert(id);
 }

service/src/memory_store.hpp

@@ -736,6 +736,14 @@ private:
     void mirrorWriteToDocStore(const std::string& collection, const std::string& id,
                                 const std::optional<Document>& doc, EventType eventType);
 
+    // v2.0 Stage 5 — mirror a vector write/delete into the LMDB sub-db
+    // `_vectors_<collection>`. Called from within the per-collection
+    // write lock, after the doc mirror. For INSERT/UPDATE: pass the
+    // vector (empty → no-op). For DELETE: pass nullptr, the helper
+    // deletes unconditionally so a doc-delete also removes its vector.
+    void mirrorVectorToDocStore(const std::string& collection, const std::string& id,
+                                 const std::vector<float>* vec, EventType eventType);
+
     // Update expiration index
     void addToExpirationIndex(CollectionData& coll, const std::string& id, uint64_t expiresAt);
     void removeFromExpirationIndex(CollectionData& coll, const std::string& id, uint64_t expiresAt);

service/src/storage/document_store.hpp

@@ -77,6 +77,36 @@ public:
 
     // Drop a collection sub-db entirely. Returns true if it existed.
     virtual bool drop_collection(std::string_view collection) = 0;
+
+    // ==========================================================================
+    // v2.0 Stage 5 — vector sub-db operations.
+    //
+    // Vectors live in a parallel `_vectors_<collection>` sub-db, keyed by
+    // docId. Raw float32 bytes; native byte order (x86-64 only, same as
+    // v1.x). The collection sub-db and its vectors sub-db are independent —
+    // backends must NOT couple their lifecycles (a doc without a vector is
+    // valid; a vector without a doc is a transient bookkeeping artifact
+    // that backfill cleans up).
+    //
+    // Default implementations are no-ops so backends that don't support
+    // vectors compile (none today, but keeps the interface honest).
+    // ==========================================================================
+
+    // Put a vector for the given doc. No-op if `vec` is empty.
+    virtual void put_vector(std::string_view /*collection*/,
+                            std::string_view /*id*/,
+                            const std::vector<float>& /*vec*/) {}
+
+    // Delete a vector. Returns true if it existed. Idempotent.
+    virtual bool del_vector(std::string_view /*collection*/,
+                            std::string_view /*id*/) { return false; }
+
+    // Read a vector. Returns nullopt if absent or collection has no
+    // vector sub-db.
+    virtual std::optional<std::vector<float>>
+    get_vector(std::string_view /*collection*/, std::string_view /*id*/) {
+        return std::nullopt;
+    }
 };
 
 }  // namespace smartbotic::db::storage

service/src/storage/document_store_lmdb.cpp

@@ -26,6 +26,7 @@
 
 #include <algorithm>
 #include <cctype>
+#include <cstring>
 #include <functional>
 #include <regex>
 #include <sstream>
@@ -398,6 +399,76 @@ std::vector<std::string> LmdbDocumentStore::list_collections() {
     return names;
 }
 
+// -------------------------------------------------------------------------
+// Vector sub-db ops (v2.0 Stage 5).
+//
+// Vectors live in a parallel `_vectors_<collection>` sub-db keyed by docId.
+// Raw float32 bytes; same encoding as migrate_v1_to_v2's bulk path.
+// open_for_write / try_open_for_read are reused unchanged — they care
+// about the sub-db name, not whether the name starts with `_`.
+// -------------------------------------------------------------------------
+
+namespace {
+std::string vector_subdb_name(std::string_view collection) {
+    std::string s = "_vectors_";
+    s.append(collection.data(), collection.size());
+    return s;
+}
+}  // namespace
+
+void LmdbDocumentStore::put_vector(std::string_view collection,
+                                    std::string_view id,
+                                    const std::vector<float>& vec) {
+    if (vec.empty()) return;  // mirror the migration tool's no-op semantics
+    const std::string subdb = vector_subdb_name(collection);
+    WriteTxn wtxn(env_);
+    unsigned int dbi = open_for_write(wtxn, subdb);
+    MDB_val k = to_val(id);
+    MDB_val v{vec.size() * sizeof(float),
+              const_cast<void*>(static_cast<const void*>(vec.data()))};
+    mdb_check(mdb_put(wtxn.raw(), dbi, &k, &v, 0), "put_vector");
+    wtxn.commit();
+}
+
+bool LmdbDocumentStore::del_vector(std::string_view collection, std::string_view id) {
+    const std::string subdb = vector_subdb_name(collection);
+    // Probe first so we don't create an empty vectors sub-db just to
+    // discover the vector isn't there.
+    {
+        ReadTxn rtxn(env_);
+        auto dbi_opt = try_open_for_read(rtxn, subdb);
+        if (!dbi_opt) return false;
+    }
+    WriteTxn wtxn(env_);
+    unsigned int dbi = open_for_write(wtxn, subdb);
+    MDB_val k = to_val(id);
+    int rc = mdb_del(wtxn.raw(), dbi, &k, nullptr);
+    if (rc == MDB_NOTFOUND) return false;
+    if (rc != MDB_SUCCESS) throw_mdb(rc, "del_vector");
+    wtxn.commit();
+    return true;
+}
+
+std::optional<std::vector<float>>
+LmdbDocumentStore::get_vector(std::string_view collection, std::string_view id) {
+    const std::string subdb = vector_subdb_name(collection);
+    ReadTxn rtxn(env_);
+    auto dbi_opt = try_open_for_read(rtxn, subdb);
+    if (!dbi_opt) return std::nullopt;
+    MDB_val k = to_val(id);
+    MDB_val v{0, nullptr};
+    int rc = mdb_get(rtxn.raw(), *dbi_opt, &k, &v);
+    if (rc == MDB_NOTFOUND) return std::nullopt;
+    if (rc != MDB_SUCCESS) throw_mdb(rc, "get_vector");
+    if (v.mv_size % sizeof(float) != 0) {
+        throw std::runtime_error("get_vector: stored bytes are not a multiple of sizeof(float)");
+    }
+    const size_t n = v.mv_size / sizeof(float);
+    std::vector<float> out(n);
+    if (n > 0) std::memcpy(out.data(), v.mv_data, v.mv_size);
+    return out;
+}
+
 bool LmdbDocumentStore::drop_collection(std::string_view collection) {
     // Probe with a read txn first; if the sub-db doesn't exist, return
     // false without opening a write txn at all.

service/src/storage/document_store_lmdb.hpp

@@ -53,6 +53,14 @@ public:
 
     bool drop_collection(std::string_view collection) override;
 
+    // v2.0 Stage 5 — vector sub-db ops (see DocumentStore base).
+    void put_vector(std::string_view collection,
+                    std::string_view id,
+                    const std::vector<float>& vec) override;
+    bool del_vector(std::string_view collection, std::string_view id) override;
+    std::optional<std::vector<float>>
+    get_vector(std::string_view collection, std::string_view id) override;
+
 private:
     // Sub-db handle cache. MDB_dbi is typedef'd to unsigned int — held as
     // that bare type to avoid pulling <lmdb.h> into this header.

service/src/storage/dual_write_mirror.hpp

@@ -20,6 +20,7 @@
 #include <atomic>
 #include <optional>
 #include <string>
+#include <vector>
 
 #include <spdlog/spdlog.h>
 
@@ -61,4 +62,48 @@ inline void applyDualWriteMirror(
     }
 }
 
+// v2.0 Stage 5 — vector dual-write. Separate from applyDualWriteMirror
+// because vectors are independent of docs (a doc-only write should not
+// touch the vector sub-db, and vice versa). Same failure semantics —
+// log ERROR, bump drift, flip health flag.
+//
+// `vec` semantics:
+//   - For INSERT/UPDATE: non-null + non-empty → put; null or empty → no-op
+//     (caller can decide to skip the mirror when there's nothing to write)
+//   - For DELETE: ignored — always calls del_vector unconditionally so
+//     the vector sub-db doesn't keep ghost vectors for deleted docs
+inline void applyVectorMirror(
+    DocumentStore* doc_store,
+    std::atomic<bool>& mirror_healthy,
+    std::atomic<uint64_t>& mirror_drift_count,
+    const std::string& collection,
+    const std::string& id,
+    const std::vector<float>* vec,
+    smartbotic::database::EventType eventType) {
+
+    if (!doc_store) return;
+    if (collection.empty() || collection[0] == '_') return;
+
+    try {
+        switch (eventType) {
+            case smartbotic::database::EventType::INSERT:
+            case smartbotic::database::EventType::UPDATE:
+                if (vec && !vec->empty()) {
+                    doc_store->put_vector(collection, id, *vec);
+                }
+                break;
+            case smartbotic::database::EventType::DELETE:
+                doc_store->del_vector(collection, id);
+                break;
+            default:
+                break;
+        }
+    } catch (const std::exception& e) {
+        spdlog::error("v2.0 vector mirror failed coll={} id={} op={}: {}",
+                      collection, id, static_cast<int>(eventType), e.what());
+        mirror_drift_count.fetch_add(1, std::memory_order_relaxed);
+        mirror_healthy.store(false, std::memory_order_release);
+    }
+}
+
 }  // namespace smartbotic::db::storage

tests/test_dual_write_mirror.cpp

@@ -33,6 +33,7 @@ using smartbotic::database::Document;
 using smartbotic::database::EventType;
 using smartbotic::database::MemoryStore;
 using smartbotic::db::storage::applyDualWriteMirror;
+using smartbotic::db::storage::applyVectorMirror;
 using smartbotic::db::storage::DocumentStore;
 using smartbotic::db::storage::LmdbDocumentStore;
 using smartbotic::db::storage::LmdbEnv;
@@ -231,6 +232,106 @@ void test_integration_via_memory_store_callback() {
     check(healthy.load() && drift.load() == 0, "integration: clean run");
 }
 
+// ---------- Vector mirror coverage (v2.0 Stage 5) ----------
+
+void test_vector_mirror_round_trip() {
+    TmpEnv tmp("vec-roundtrip");
+    LmdbDocumentStore store(tmp.env);
+    std::atomic<bool> healthy{true};
+    std::atomic<uint64_t> drift{0};
+
+    std::vector<float> v1{0.1f, 0.2f, 0.3f, 0.4f};
+    applyVectorMirror(&store, healthy, drift, "users", "u1", &v1, EventType::INSERT);
+
+    auto got = store.get_vector("users", "u1");
+    check(got.has_value(), "vector insert: present in LMDB");
+    check(got->size() == 4 && (*got)[0] == 0.1f && (*got)[3] == 0.4f,
+          "vector insert: bytes preserved");
+
+    std::vector<float> v2{1.0f, 2.0f, 3.0f, 4.0f};
+    applyVectorMirror(&store, healthy, drift, "users", "u1", &v2, EventType::UPDATE);
+    got = store.get_vector("users", "u1");
+    check(got.has_value() && (*got)[0] == 1.0f, "vector update: overwrites");
+
+    applyVectorMirror(&store, healthy, drift, "users", "u1", nullptr, EventType::DELETE);
+    check(!store.get_vector("users", "u1").has_value(), "vector delete: gone");
+
+    check(healthy.load() && drift.load() == 0, "vector round-trip: clean");
+}
+
+void test_vector_mirror_empty_is_noop_on_insert() {
+    TmpEnv tmp("vec-empty");
+    LmdbDocumentStore store(tmp.env);
+    std::atomic<bool> healthy{true};
+    std::atomic<uint64_t> drift{0};
+
+    std::vector<float> empty;
+    applyVectorMirror(&store, healthy, drift, "users", "u1", &empty, EventType::INSERT);
+    check(!store.get_vector("users", "u1").has_value(),
+          "empty vector INSERT: no LMDB write");
+    check(healthy.load() && drift.load() == 0, "empty vector INSERT: clean");
+}
+
+void test_vector_mirror_system_collection_skipped() {
+    TmpEnv tmp("vec-system");
+    LmdbDocumentStore store(tmp.env);
+    std::atomic<bool> healthy{true};
+    std::atomic<uint64_t> drift{0};
+
+    std::vector<float> v{0.5f, 0.5f};
+    applyVectorMirror(&store, healthy, drift, "_internal", "u1", &v, EventType::INSERT);
+    check(!store.get_vector("_internal", "u1").has_value(),
+          "system collection: vector not mirrored");
+    check(healthy.load() && drift.load() == 0, "system collection: clean");
+}
+
+// Integration: real MemoryStore + DocumentStore mirror wiring through
+// setDocumentStoreMirror. Verifies that vector-bearing writes land in
+// LMDB's _vectors_<collection> sub-db under the per-collection lock.
+void test_vector_mirror_via_memory_store() {
+    TmpEnv tmp("vec-memstore");
+    LmdbDocumentStore doc_store(tmp.env);
+    std::atomic<bool> healthy{true};
+    std::atomic<uint64_t> drift{0};
+
+    MemoryStore::Config cfg;
+    cfg.nodeId = "test-node";
+    cfg.maxMemoryBytes = 64ULL * 1024 * 1024;
+    MemoryStore mem(cfg);
+    mem.setDocumentStoreMirror(&doc_store, &healthy, &drift);
+
+    smartbotic::database::CollectionOptions opts;
+    opts.vectorDimension = 4;
+    mem.createCollection("embeddings", opts);
+
+    Document d1;
+    d1.id = "e1";
+    d1.collection = "embeddings";
+    d1.set_data({{"label", "alpha"}, {"_vector", {0.1f, 0.2f, 0.3f, 0.4f}}});
+    mem.insert("embeddings", d1);
+
+    auto lmdb_vec = doc_store.get_vector("embeddings", "e1");
+    check(lmdb_vec.has_value(), "memstore insert with vector: LMDB vector present");
+    check(lmdb_vec->size() == 4 && (*lmdb_vec)[2] == 0.3f,
+          "memstore insert with vector: bytes match");
+
+    // Doc with no _vector — vector sub-db stays empty for this id
+    Document d2;
+    d2.id = "e2";
+    d2.collection = "embeddings";
+    d2.set_data({{"label", "no-vec"}});
+    mem.insert("embeddings", d2);
+    check(!doc_store.get_vector("embeddings", "e2").has_value(),
+          "memstore insert without vector: LMDB vector absent");
+
+    // Delete: removes both doc and vector mirror
+    mem.remove("embeddings", "e1");
+    check(!doc_store.get_vector("embeddings", "e1").has_value(),
+          "memstore delete: LMDB vector cleared");
+
+    check(healthy.load() && drift.load() == 0, "vector via memstore: clean");
+}
+
 // Simulates DatabaseService::backfillIntoDocStore — populate MemoryStore
 // without a persist callback (no live mirror), then walk listCollections +
 // getAllDocuments and feed each doc into applyDualWriteMirror with
@@ -283,6 +384,10 @@ int main() {
     test_unit_failure_marks_degraded();
     test_unit_insert_without_doc_is_noop();
     test_integration_via_memory_store_callback();
+    test_vector_mirror_round_trip();
+    test_vector_mirror_empty_is_noop_on_insert();
+    test_vector_mirror_system_collection_skipped();
+    test_vector_mirror_via_memory_store();
     test_backfill_iteration_pattern();
 
     std::cout << "dual_write_mirror: " << g_pass << " passed, " << g_fail << " failed\n";