feat(memory_store): migrate to Document binary data accessors (phase B T4)

Migrates ~50 memory_store callsites from doc.data["x"] to the new binary-
backed accessor surface (data(), set_data(), field()).

Key changes:
- extractVector now takes Document& instead of nlohmann::json& so it owns
  the materialise / mutate / re-encode dance internally; falls back to the
  field() fast-path to skip materialisation when _vector is absent.
- Set operations (setAdd, setRemove) materialise the tree once and call
  set_data once at the end (batched re-encode) instead of mutating
  doc.data per field write.
- Set membership reads (setMembers, setIsMember) use field("members") for
  O(top-level) probe.
- patchDocument merges into a single materialised tree, set_data once.
- Filter path uses field() fast-path for top-level names; only nested
  dot-paths materialise via doc.data().
- estimateDocumentSize uses to_json_text(doc.data_binary) instead of the
  removed doc.data.dump().
- saveToHistory / restoreToVersion / getDocumentAtVersion bridge
  Document <-> DocumentVersion via set_data(doc.data()).
- All emitEvent(... doc.data ...) callsites wrap in std::optional<json>
  to disambiguate the optional<nlohmann::json> param from the bool-coercion
  overload.

service/src/memory_store.cpp

@@ -261,7 +261,7 @@ std::string MemoryStore::insert(const std::string& collection, Document doc) {
     }
 
     // Extract vector before storing document (strips _vector from JSON data)
-    auto vec = extractVector(*coll, doc.data);
+    auto vec = extractVector(*coll, doc);
 
     // Set metadata
     doc.collection = collection;
@@ -310,7 +310,7 @@ std::string MemoryStore::insert(const std::string& collection, Document doc) {
 
     // Emit callbacks
     emitPersist(collection, docId, doc, EventType::INSERT);
-    emitEvent(EventType::INSERT, collection, docId, doc.data);
+    emitEvent(EventType::INSERT, collection, docId, std::optional<nlohmann::json>(doc.data()));
 
     return docId;
 }
@@ -418,7 +418,7 @@ bool MemoryStore::update(const std::string& collection, const std::string& id, c
     Document updated = doc;
 
     // Extract vector before storing document (strips _vector from JSON data)
-    auto vec = extractVector(*coll, updated.data);
+    auto vec = extractVector(*coll, updated);
 
     updated.id = id;
     updated.collection = collection;
@@ -470,7 +470,7 @@ bool MemoryStore::update(const std::string& collection, const std::string& id, c
 
     // Emit callbacks
     emitPersist(collection, id, updated, EventType::UPDATE);
-    emitEvent(EventType::UPDATE, collection, id, updated.data);
+    emitEvent(EventType::UPDATE, collection, id, std::optional<nlohmann::json>(updated.data()));
 
     return true;
 }
@@ -495,7 +495,7 @@ std::string MemoryStore::upsert(const std::string& collection, Document doc) {
     std::unique_lock<std::shared_mutex> lock(coll->mutex);
 
     // Extract vector before storing document (strips _vector from JSON data)
-    auto vec = extractVector(*coll, doc.data);
+    auto vec = extractVector(*coll, doc);
 
     std::string docId = doc.id;
     bool isInsert = false;
@@ -584,7 +584,7 @@ std::string MemoryStore::upsert(const std::string& collection, Document doc) {
 
     // Emit callbacks
     emitPersist(collection, docId, doc, isInsert ? EventType::INSERT : EventType::UPDATE);
-    emitEvent(isInsert ? EventType::INSERT : EventType::UPDATE, collection, docId, doc.data);
+    emitEvent(isInsert ? EventType::INSERT : EventType::UPDATE, collection, docId, std::optional<nlohmann::json>(doc.data()));
 
     return docId;
 }
@@ -715,7 +715,7 @@ bool MemoryStore::updateIfVersion(const std::string& collection, const std::stri
 
     // Emit callbacks
     emitPersist(collection, id, updated, EventType::UPDATE);
-    emitEvent(EventType::UPDATE, collection, id, updated.data);
+    emitEvent(EventType::UPDATE, collection, id, std::optional<nlohmann::json>(updated.data()));
 
     return true;
 }
@@ -745,11 +745,16 @@ uint64_t MemoryStore::patchDocument(const std::string& collection, const std::st
     // Save current state to history
     saveToHistory(*coll, it->second);
 
-    // Merge patch into existing data (RFC 7396 merge patch)
-    it->second.data.merge_patch(patch);
+    // Merge patch into existing data (RFC 7396 merge patch). Materialise
+    // once, merge, then re-encode. Avoids per-field re-encode.
+    {
+        auto merged = it->second.data();
+        merged.merge_patch(patch);
+        it->second.set_data(merged);
+    }
 
     // Handle vector field if present in patch
-    auto vec = extractVector(*coll, it->second.data);
+    auto vec = extractVector(*coll, it->second);
     if (!vec.empty()) {
         storeVector(*coll, id, vec);
     }
@@ -793,7 +798,7 @@ uint64_t MemoryStore::patchDocument(const std::string& collection, const std::st
 
     // Emit callbacks (WAL persistence + event notification)
     emitPersist(collection, id, updatedDoc, EventType::UPDATE);
-    emitEvent(EventType::UPDATE, collection, id, updatedDoc.data);
+    emitEvent(EventType::UPDATE, collection, id, std::optional<nlohmann::json>(updatedDoc.data()));
 
     return newVersion;
 }
@@ -908,8 +913,8 @@ QueryResult MemoryStore::find(const std::string& collection, const Query& query)
                     valA = nlohmann::json(a.updatedAt);
                     valB = nlohmann::json(b.updatedAt);
                 } else {
-                    valA = getJsonPath(a.data, query.sort->field);
-                    valB = getJsonPath(b.data, query.sort->field);
+                    valA = getJsonPath(a.data(), query.sort->field);
+                    valB = getJsonPath(b.data(), query.sort->field);
                 }
 
                 if (!valA && !valB) {
@@ -939,7 +944,7 @@ QueryResult MemoryStore::find(const std::string& collection, const Query& query)
                 } else if (query.sort->field == "_updated_at") {
                     debugInfo += "[" + matches[i].id + ": " + std::to_string(matches[i].updatedAt) + "] ";
                 } else {
-                    auto val = getJsonPath(matches[i].data, query.sort->field);
+                    auto val = getJsonPath(matches[i].data(), query.sort->field);
                     debugInfo += "[" + matches[i].id + ": " + (val ? val->dump() : "null") + "] ";
                 }
             }
@@ -1017,8 +1022,11 @@ bool MemoryStore::setAdd(const std::string& collection, const std::string& setId
         Document doc;
         doc.id = setId;
         doc.collection = collection;
-        doc.data = nlohmann::json::object();
-        doc.data["members"] = nlohmann::json::array({member});
+        {
+            nlohmann::json initial = nlohmann::json::object();
+            initial["members"] = nlohmann::json::array({member});
+            doc.set_data(initial);
+        }
         doc.version = 1;
         doc.createdAt = currentTimeFor(collection);
         doc.updatedAt = doc.createdAt;
@@ -1041,13 +1049,16 @@ bool MemoryStore::setAdd(const std::string& collection, const std::string& setId
         }
 
         emitPersist(collection, setId, doc, EventType::INSERT);
-        emitEvent(EventType::INSERT, collection, setId, doc.data);
+        emitEvent(EventType::INSERT, collection, setId, std::optional<nlohmann::json>(doc.data()));
 
         return true;
     }
 
-    // Check if member already exists
-    auto& members = it->second.data["members"];
+    // Materialise the binary tree once, mutate, re-encode at the end. Set
+    // ops are write-heavy on the same field, so batching the encode is the
+    // right shape.
+    auto tree = it->second.data();
+    auto& members = tree["members"];
     if (!members.is_array()) {
         members = nlohmann::json::array();
     }
@@ -1060,6 +1071,7 @@ bool MemoryStore::setAdd(const std::string& collection, const std::string& setId
 
     saveToHistory(*coll, it->second);
     members.push_back(member);
+    it->second.set_data(tree);
     it->second.version++;
     it->second.updatedAt = currentTimeFor(collection);
     it->second.nodeId = config_.nodeId;
@@ -1075,7 +1087,7 @@ bool MemoryStore::setAdd(const std::string& collection, const std::string& setId
     }
 
     emitPersist(collection, setId, updated, EventType::UPDATE);
-    emitEvent(EventType::UPDATE, collection, setId, updated.data);
+    emitEvent(EventType::UPDATE, collection, setId, std::optional<nlohmann::json>(updated.data()));
 
     return true;
 }
@@ -1090,7 +1102,8 @@ bool MemoryStore::setRemove(const std::string& collection, const std::string& se
         return false;
     }
 
-    auto& members = it->second.data["members"];
+    auto tree = it->second.data();
+    auto& members = tree["members"];
     if (!members.is_array()) {
         return false;
     }
@@ -1110,7 +1123,8 @@ bool MemoryStore::setRemove(const std::string& collection, const std::string& se
     }
 
     saveToHistory(*coll, it->second);
-    it->second.data["members"] = newMembers;
+    tree["members"] = newMembers;
+    it->second.set_data(tree);
     it->second.version++;
     it->second.updatedAt = currentTimeFor(collection);
     it->second.nodeId = config_.nodeId;
@@ -1126,7 +1140,7 @@ bool MemoryStore::setRemove(const std::string& collection, const std::string& se
     }
 
     emitPersist(collection, setId, updated, EventType::UPDATE);
-    emitEvent(EventType::UPDATE, collection, setId, updated.data);
+    emitEvent(EventType::UPDATE, collection, setId, std::optional<nlohmann::json>(updated.data()));
 
     return true;
 }
@@ -1144,13 +1158,13 @@ std::vector<std::string> MemoryStore::setMembers(const std::string& collection,
         return {};
     }
 
-    const auto& members = it->second.data["members"];
-    if (!members.is_array()) {
+    auto membersOpt = it->second.field("members");
+    if (!membersOpt || !membersOpt->is_array()) {
         return {};
     }
 
     std::vector<std::string> result;
-    for (const auto& m : members) {
+    for (const auto& m : *membersOpt) {
         if (m.is_string()) {
             result.push_back(m.get<std::string>());
         }
@@ -1171,12 +1185,12 @@ bool MemoryStore::setIsMember(const std::string& collection, const std::string&
         return false;
     }
 
-    const auto& members = it->second.data["members"];
-    if (!members.is_array()) {
+    auto membersOpt = it->second.field("members");
+    if (!membersOpt || !membersOpt->is_array()) {
         return false;
     }
 
-    for (const auto& m : members) {
+    for (const auto& m : *membersOpt) {
         if (m.is_string() && m.get<std::string>() == member) {
             return true;
         }
@@ -1294,17 +1308,19 @@ float MemoryStore::cosineSimilaritySIMD(const float* a, const float* b, size_t d
     return dot / (normA * normB);
 }
 
-std::vector<float> MemoryStore::extractVector(CollectionData& coll, nlohmann::json& docData) {
+std::vector<float> MemoryStore::extractVector(CollectionData& coll, Document& doc) {
     if (coll.options.vectorDimension == 0) {
         return {};
     }
 
-    if (!docData.contains("_vector") || !docData["_vector"].is_array()) {
+    // Fast-path single-field probe so collections without a _vector field
+    // avoid materialising the full tree.
+    auto field = doc.field("_vector");
+    if (!field.has_value() || !field->is_array()) {
         return {};
     }
 
-    auto vec = docData["_vector"].get<std::vector<float>>();
-    docData.erase("_vector");
+    auto vec = field->get<std::vector<float>>();
 
     if (vec.size() != coll.options.vectorDimension) {
         throw std::invalid_argument(
@@ -1313,6 +1329,13 @@ std::vector<float> MemoryStore::extractVector(CollectionData& coll, nlohmann::js
             ", got " + std::to_string(vec.size()));
     }
 
+    // Strip the _vector field — materialise once, erase, re-encode.
+    auto j = doc.data();
+    if (j.is_object()) {
+        j.erase("_vector");
+        doc.set_data(j);
+    }
+
     return vec;
 }
 
@@ -1640,7 +1663,7 @@ void MemoryStore::saveToHistory(CollectionData& coll, const Document& currentDoc
 
     DocumentVersion ver;
     ver.version = currentDoc.version;
-    ver.data = currentDoc.data;
+    ver.set_data(currentDoc.data());
     ver.timestamp = currentDoc.updatedAt;
     ver.updatedBy = currentDoc.updatedBy;
     ver.encrypted = currentDoc.encrypted;
@@ -1709,7 +1732,7 @@ std::optional<DocumentVersion> MemoryStore::getDocumentAtVersion(
     if (docIt != coll->documents.end() && docIt->second.version == version) {
         DocumentVersion ver;
         ver.version = docIt->second.version;
-        ver.data = docIt->second.data;
+        ver.set_data(docIt->second.data());
         ver.timestamp = docIt->second.updatedAt;
         ver.updatedBy = docIt->second.updatedBy;
         ver.encrypted = docIt->second.encrypted;
@@ -1749,7 +1772,7 @@ uint64_t MemoryStore::restoreToVersion(const std::string& collection, const std:
         saveToHistory(*coll, docIt->second);
 
         newVersion = docIt->second.version + 1;
-        docIt->second.data = targetVer.data;
+        docIt->second.set_data(targetVer.data());
         docIt->second.version = newVersion;
         docIt->second.updatedAt = currentTimeFor(collection);
         docIt->second.updatedBy = actor;
@@ -1771,7 +1794,7 @@ uint64_t MemoryStore::restoreToVersion(const std::string& collection, const std:
 
         restoredDoc.id = id;
         restoredDoc.collection = collection;
-        restoredDoc.data = targetVer.data;
+        restoredDoc.set_data(targetVer.data());
         restoredDoc.version = newVersion;
         restoredDoc.createdAt = targetVer.createdAt;
         restoredDoc.createdBy = targetVer.createdBy;
@@ -1808,7 +1831,7 @@ uint64_t MemoryStore::restoreToVersion(const std::string& collection, const std:
     // Emit callbacks
     EventType eventType = wasDeleted ? EventType::INSERT : EventType::UPDATE;
     emitPersist(collection, id, restoredDoc, eventType);
-    emitEvent(eventType, collection, id, restoredDoc.data);
+    emitEvent(eventType, collection, id, std::optional<nlohmann::json>(restoredDoc.data()));
 
     return newVersion;
 }
@@ -2119,8 +2142,12 @@ bool MemoryStore::matchesFilters(const Document& doc, const std::vector<Filter>&
             value = nlohmann::json(doc.id);
         } else if (filter.field == "_version") {
             value = nlohmann::json(doc.version);
+        } else if (filter.field.find('.') == std::string::npos) {
+            // Top-level field: fast-path probe so we avoid materialising the
+            // full tree on every filter check.
+            value = doc.field(filter.field);
         } else {
-            value = getJsonPath(doc.data, filter.field);
+            value = getJsonPath(doc.data(), filter.field);
         }
 
         switch (filter.op) {
@@ -2266,8 +2293,9 @@ bool MemoryStore::matchesSearch(const Document& doc, const std::string& searchTe
         return true;
     }
 
-    // Search in document data
-    return searchInJson(doc.data, lowerSearchTerm);
+    // Search in document data. Materialises the full tree (full-tree
+    // search has no field-fast-path equivalent).
+    return searchInJson(doc.data(), lowerSearchTerm);
 }
 
 bool MemoryStore::searchInJson(const nlohmann::json& obj, const std::string& lowerSearchTerm) {
@@ -2387,11 +2415,10 @@ uint64_t MemoryStore::estimateDocumentSize(const Document& doc) {
     size += doc.createdBy.size();
     size += doc.updatedBy.size();
     size += doc.nodeId.size();
-    // JSON tree overhead: .dump() underestimates actual heap usage.
-    // A nlohmann::json object's heap footprint is roughly 2-3x the serialized size
-    // because of node allocations, hash tables, etc. Use 2.5x as a calibrated
-    // middle ground that avoids chronic under-count without being paranoid.
-    const std::string serialized = doc.data.dump();
+    // Binary doc overhead: to_json_text() emits the compact JSON serialization
+    // of the underlying yyjson mut_doc; we keep the 2.5x multiplier because
+    // yyjson tree overhead (vals + strings) is also super-linear in payload.
+    const std::string serialized = smartbotic::db::doc_binary::to_json_text(doc.data_binary);
     size += static_cast<uint64_t>(serialized.size() * 5 / 2);  // 2.5x
     // Encrypted-fields vector
     for (const auto& f : doc.encryptedFields) {
@@ -2413,7 +2440,7 @@ uint64_t MemoryStore::estimateDocumentVersionSize(const DocumentVersion& ver) {
     uint64_t size = 0;
     size += ver.updatedBy.size();
     size += ver.createdBy.size();
-    const std::string serialized = ver.data.dump();
+    const std::string serialized = smartbotic::db::doc_binary::to_json_text(ver.data_binary);
     size += static_cast<uint64_t>(serialized.size() * 5 / 2);
     for (const auto& f : ver.encryptedFields) {
         size += f.size();

service/src/memory_store.hpp

@@ -683,7 +683,9 @@ private:
     // Vector storage helpers
     void storeVector(CollectionData& coll, const std::string& docId, const std::vector<float>& vec);
     void removeVector(CollectionData& coll, const std::string& docId);
-    std::vector<float> extractVector(CollectionData& coll, nlohmann::json& docData);
+    // v1.11 — operates on Document. If the doc has a "_vector" field, returns it
+    // and strips the field from doc.data_binary (re-encodes the doc once).
+    std::vector<float> extractVector(CollectionData& coll, Document& doc);
     static float cosineSimilaritySIMD(const float* a, const float* b, size_t dim, float normA, float normB);
     static float computeNorm(const float* data, size_t dim);