feat(storage): migrate vector fields to _vectors_<collection> sub-db

Extends the v1.x → v2.0 migration tool to copy vector data alongside
documents. For each v1.x collection with vector_dimension > 0, the
migration reads the parallel `vectors` map (docId → vector<float>)
populated by v1.x recovery and writes raw float32 bytes into the v2.0
`_vectors_<collection>` sub-db, keyed by docId.

Notes:
  - JSON doc payload is preserved unchanged. v1.x's insert path strips
    `_vector` from the data and stashes it in the vectors map; that
    same encoding round-trips through the migration. Docs WITHOUT a
    stripped `_vector` field keep their wire-format intact.
  - Sub-db is lazily MDB_CREATE-created in the per-collection write txn,
    so non-vector collections never produce a `_vectors_*` sub-db.
  - Vector-enabled collections with zero vectors leave the sub-db
    uncreated — legal v1.x state (just-created collection).
  - Native byte order — matches v1.x's in-memory vector storage layout
    and the architecture-pinning of x86-64 production deployments.

Tests added (test_migrate_v1_to_v2):
  - vector_dimension=8 collection × 50 docs round-trips byte-for-byte
  - partial vectors (5/10 docs have _vector) — only docs WITH a vector
    end up in the sub-db
  - non-vector collection does NOT create a _vectors_* sub-db

Phase C Stage 3 Task 3.3.

service/src/storage/migrate_v1_to_v2.cpp

@@ -39,6 +39,7 @@
 #include "persistence/snapshot.hpp"
 #include "persistence/wal.hpp"
 #include "storage/document_store.hpp"
+#include "storage/lmdb_dbi.hpp"
 #include "storage/lmdb_env.hpp"
 #include "storage/lmdb_txn.hpp"
 
@@ -178,6 +179,47 @@ void apply_wal_entry(smartbotic::database::MemoryStore& store,
     }
 }
 
+// -------------------------------------------------------------------------
+// Vector sub-db writer (Task 3.3).
+//
+// For each v1.x collection with vector_dimension > 0, the v1.x MemoryStore
+// holds a parallel `vectors` map (docId → vector<float>) populated by
+// insert / loadVector / VEC_PUT replay. We write those vectors into the
+// v2.0 `_vectors_<collection>` sub-db as raw float32 bytes, keyed by docId.
+// The JSON doc retains its `_vector` field (if any) for wire-format
+// round-trip compatibility — we do NOT strip it.
+//
+// Lazy-opens the sub-db within a single write txn for the collection.
+// Native byte order — v2.0 only supports the same architecture mix as
+// v1.x (x86-64), same as the in-memory v1.x vectors map.
+// -------------------------------------------------------------------------
+
+std::string vector_subdb_name(const std::string& collection) {
+    return "_vectors_" + collection;
+}
+
+void write_vectors_for_collection(
+    LmdbEnv& env,
+    const std::string& collection,
+    const std::unordered_map<std::string, std::vector<float>>& vectors) {
+
+    if (vectors.empty()) return;
+
+    WriteTxn wtxn(env);
+    MDB_dbi dbi = 0;
+    std::string name = vector_subdb_name(collection);
+    mdb_check(mdb_dbi_open(wtxn.raw(), name.c_str(), MDB_CREATE, &dbi),
+              "dbi_open (vectors sub-db create)");
+
+    for (const auto& [doc_id, vec] : vectors) {
+        MDB_val k = to_val(doc_id);
+        MDB_val v{vec.size() * sizeof(float),
+                  const_cast<void*>(static_cast<const void*>(vec.data()))};
+        mdb_check(mdb_put(wtxn.raw(), dbi, &k, &v, 0), "put (vector)");
+    }
+    wtxn.commit();
+}
+
 void load_v1_state(const std::string& v1_data_dir,
                    smartbotic::database::MemoryStore& store) {
     fs::path data_dir(v1_data_dir);
@@ -396,6 +438,27 @@ MigrationResult migrate_v1_to_v2(
                 }
             }
 
+            // Task 3.3 — vector migration. If the collection has
+            // vector_dimension > 0, read v1.x's in-memory vectors map and
+            // write raw float bytes into _vectors_<collection>. The doc
+            // payload already round-tripped above (with or without a
+            // _vector field stripped by v1.x's insert path).
+            auto info = mstore.getCollectionInfo(collection);
+            if (info && info->options.vectorDimension > 0) {
+                const auto* vmap = mstore.getCollectionVectors(collection);
+                if (vmap && !vmap->empty()) {
+                    write_vectors_for_collection(env, collection, *vmap);
+                    spdlog::info("migration: wrote {} vectors for collection '{}'",
+                                 vmap->size(), collection);
+                } else {
+                    // Vector-enabled collection with no stored vectors —
+                    // legal v1.x state (e.g. just-created collection). The
+                    // sub-db is NOT lazily created in this case.
+                    spdlog::info("migration: collection '{}' is vector-enabled but "
+                                 "has 0 vectors", collection);
+                }
+            }
+
             result.collections_migrated++;
             spdlog::info("migration: completed collection '{}' ({} docs)",
                          collection, docs.size());

tests/test_migrate_v1_to_v2.cpp

@@ -173,6 +173,42 @@ struct DirChecksum {
     bool operator!=(const DirChecksum& other) const { return !(*this == other); }
 };
 
+// Read the raw float32 bytes from `_vectors_<collection>` sub-db for a doc.
+std::vector<float> read_vector_from_env(LmdbEnv& env, const std::string& collection,
+                                         const std::string& doc_id) {
+    using smartbotic::db::storage::ReadTxn;
+    ReadTxn rtxn(env);
+    MDB_dbi dbi = 0;
+    std::string name = "_vectors_" + collection;
+    int rc = mdb_dbi_open(rtxn.raw(), name.c_str(), 0, &dbi);
+    if (rc == MDB_NOTFOUND) return {};
+    if (rc != MDB_SUCCESS) {
+        std::cerr << "FAIL: read_vector dbi_open: " << mdb_strerror(rc) << "\n";
+        std::abort();
+    }
+    MDB_val k{doc_id.size(), const_cast<char*>(doc_id.data())};
+    MDB_val v{0, nullptr};
+    rc = mdb_get(rtxn.raw(), dbi, &k, &v);
+    if (rc == MDB_NOTFOUND) return {};
+    if (rc != MDB_SUCCESS) {
+        std::cerr << "FAIL: read_vector mdb_get: " << mdb_strerror(rc) << "\n";
+        std::abort();
+    }
+    size_t count = v.mv_size / sizeof(float);
+    std::vector<float> out(count);
+    std::memcpy(out.data(), v.mv_data, v.mv_size);
+    return out;
+}
+
+// True if a sub-db with the given name exists in env.
+bool sub_db_exists_in_env(LmdbEnv& env, const std::string& name) {
+    using smartbotic::db::storage::ReadTxn;
+    ReadTxn rtxn(env);
+    MDB_dbi dbi = 0;
+    int rc = mdb_dbi_open(rtxn.raw(), name.c_str(), 0, &dbi);
+    return rc == MDB_SUCCESS;
+}
+
 }  // anonymous namespace
 
 // -------------------------------------------------------------------------
@@ -510,6 +546,116 @@ void test_already_migrated_short_circuit() {
     std::cout << "PASS: already-migrated short-circuits\n";
 }
 
+// -------------------------------------------------------------------------
+// Task 3.3: Vector migration
+// -------------------------------------------------------------------------
+
+void test_migrate_collection_with_vectors() {
+    auto dir = make_tmpdir("vec");
+    {
+        V1Builder b(dir);
+        CollectionOptions opts;
+        opts.vectorDimension = 8;
+        b.createCollection("vecs", opts);
+        for (int i = 0; i < 50; i++) {
+            // Insert via the public path so _vector goes through v1.x's
+            // strip-and-stash flow (the JSON _vector field is moved into
+            // the parallel vectors map and stripped from doc data).
+            nlohmann::json data = {
+                {"content", "doc " + std::to_string(i)},
+                {"_vector", {float(i), float(i+1), float(i+2), float(i+3),
+                             float(i+4), float(i+5), float(i+6), float(i+7)}}
+            };
+            Document d = makeDoc("v" + std::to_string(i), data);
+            b.insert("vecs", d);
+        }
+        b.writeSnapshot();
+    }
+
+    TmpEnv env_dir("vec-env");
+    LmdbDocumentStore target(env_dir.env);
+    auto r = migrate_v1_to_v2(dir, target, env_dir.env);
+    check(r.success, "vector migration succeeds");
+    check(r.docs_migrated == 50, "50 vector docs migrated");
+
+    // Spot-check entries.
+    for (int i : {0, 17, 49}) {
+        auto v = read_vector_from_env(env_dir.env, "vecs", "v" + std::to_string(i));
+        check(v.size() == 8, "vector dim=8 round-trip");
+        for (int j = 0; j < 8; j++) {
+            check(v[j] == float(i + j), "vector value matches v1.x source");
+        }
+    }
+
+    std::error_code ec;
+    fs::remove_all(dir, ec);
+    std::cout << "PASS: collection with vectors migrates\n";
+}
+
+void test_migrate_collection_with_partial_vectors() {
+    auto dir = make_tmpdir("partial-vec");
+    {
+        V1Builder b(dir);
+        CollectionOptions opts;
+        opts.vectorDimension = 4;
+        b.createCollection("vecs", opts);
+        // Half the docs have _vector, half don't.
+        for (int i = 0; i < 10; i++) {
+            nlohmann::json data;
+            data["content"] = "doc " + std::to_string(i);
+            if (i % 2 == 0) {
+                data["_vector"] = {float(i), float(i+1), float(i+2), float(i+3)};
+            }
+            Document d = makeDoc("d" + std::to_string(i), data);
+            b.insert("vecs", d);
+        }
+        b.writeSnapshot();
+    }
+
+    TmpEnv env_dir("partial-vec-env");
+    LmdbDocumentStore target(env_dir.env);
+    auto r = migrate_v1_to_v2(dir, target, env_dir.env);
+    check(r.success, "partial-vector migration succeeds");
+    check(r.docs_migrated == 10, "10 docs total migrated");
+
+    int vec_count = 0;
+    for (int i = 0; i < 10; i++) {
+        auto v = read_vector_from_env(env_dir.env, "vecs", "d" + std::to_string(i));
+        if (!v.empty()) {
+            ++vec_count;
+            check(v.size() == 4, "dim=4 round-trip");
+            check(v[0] == float(i), "first elem matches");
+        }
+    }
+    check(vec_count == 5, "5 vectors actually written (only even-indexed docs)");
+
+    std::error_code ec;
+    fs::remove_all(dir, ec);
+    std::cout << "PASS: collection with partial vectors\n";
+}
+
+void test_migrate_non_vector_collection_no_sub_db() {
+    auto dir = make_tmpdir("no-vec");
+    {
+        V1Builder b(dir);
+        // vector_dimension = 0 → no vector sub-db expected.
+        b.createCollection("plain");
+        b.insert("plain", makeDoc("d1", {{"v", 1}}));
+        b.writeSnapshot();
+    }
+
+    TmpEnv env_dir("no-vec-env");
+    LmdbDocumentStore target(env_dir.env);
+    auto r = migrate_v1_to_v2(dir, target, env_dir.env);
+    check(r.success, "non-vector migration succeeds");
+    check(!sub_db_exists_in_env(env_dir.env, "_vectors_plain"),
+          "no _vectors_plain sub-db created");
+
+    std::error_code ec;
+    fs::remove_all(dir, ec);
+    std::cout << "PASS: non-vector collection has no vector sub-db\n";
+}
+
 // -------------------------------------------------------------------------
 // main()
 // -------------------------------------------------------------------------
@@ -532,6 +678,11 @@ int main() {
     test_force_overrides_no_v1_check();
     test_already_migrated_short_circuit();
 
+    // Task 3.3: Vector migration
+    test_migrate_collection_with_vectors();
+    test_migrate_collection_with_partial_vectors();
+    test_migrate_non_vector_collection_no_sub_db();
+
     std::cout << "\nAll migrate_v1_to_v2 tests passed.\n";
     return 0;
 }