Prechádzať zdrojové kódy

feat: vector serialization in snapshots (v3 format)

Bumps snapshot format version to 3. Vectors are written after version
history (per collection) as raw float32 bytes for efficiency, and
loaded back via loadVector() on restore. Old v1/v2 snapshots continue
to load correctly via the existing snapshotVersion guard.
fszontagh 4 mesiacov pred
rodič
commit
341508845b

+ 64 - 0
service/src/persistence/snapshot.cpp

@@ -401,6 +401,33 @@ std::vector<uint8_t> SnapshotManager::serializeStore(const MemoryStore& store,
                 result.insert(result.end(), verJson.begin(), verJson.end());
             }
         }
+
+        // Write vector data (v3)
+        const auto* vectors = store.getCollectionVectors(name);
+        uint64_t vecCount = (vectors != nullptr) ? static_cast<uint64_t>(vectors->size()) : 0;
+        for (int i = 0; i < 8; ++i) {
+            result.push_back(static_cast<uint8_t>((vecCount >> (i * 8)) & 0xFF));
+        }
+
+        if (vectors != nullptr) {
+            for (const auto& [docId, vec] : *vectors) {
+                // Write doc ID
+                uint16_t docIdLen = static_cast<uint16_t>(docId.size());
+                result.push_back(static_cast<uint8_t>(docIdLen & 0xFF));
+                result.push_back(static_cast<uint8_t>((docIdLen >> 8) & 0xFF));
+                result.insert(result.end(), docId.begin(), docId.end());
+
+                // Write vector dimension
+                uint32_t dim = static_cast<uint32_t>(vec.size());
+                for (int i = 0; i < 4; ++i) {
+                    result.push_back(static_cast<uint8_t>((dim >> (i * 8)) & 0xFF));
+                }
+
+                // Write raw float data
+                const uint8_t* floatBytes = reinterpret_cast<const uint8_t*>(vec.data());
+                result.insert(result.end(), floatBytes, floatBytes + dim * sizeof(float));
+            }
+        }
     }
 
     return result;
@@ -521,6 +548,43 @@ void SnapshotManager::deserializeStore(const std::vector<uint8_t>& data, MemoryS
                 }
             }
         }
+
+        // Read vector data (v3+)
+        if (snapshotVersion >= 3) {
+            if (offset + 8 > data.size()) break;
+            uint64_t vecCount = 0;
+            for (int i = 0; i < 8; ++i) {
+                vecCount |= static_cast<uint64_t>(data[offset++]) << (i * 8);
+            }
+
+            for (uint64_t v = 0; v < vecCount; ++v) {
+                // Read doc ID
+                if (offset + 2 > data.size()) break;
+                uint16_t docIdLen = static_cast<uint16_t>(data[offset]) |
+                                   (static_cast<uint16_t>(data[offset + 1]) << 8);
+                offset += 2;
+
+                if (offset + docIdLen > data.size()) break;
+                std::string docId(reinterpret_cast<const char*>(&data[offset]), docIdLen);
+                offset += docIdLen;
+
+                // Read vector dimension
+                if (offset + 4 > data.size()) break;
+                uint32_t dim = 0;
+                for (int i = 0; i < 4; ++i) {
+                    dim |= static_cast<uint32_t>(data[offset++]) << (i * 8);
+                }
+
+                // Read raw float data
+                size_t byteCount = static_cast<size_t>(dim) * sizeof(float);
+                if (offset + byteCount > data.size()) break;
+                std::vector<float> vec(dim);
+                std::memcpy(vec.data(), &data[offset], byteCount);
+                offset += byteCount;
+
+                store.loadVector(collName, docId, std::move(vec));
+            }
+        }
     }
 }
 

+ 9 - 2
service/src/persistence/snapshot.hpp

@@ -45,6 +45,13 @@ namespace smartbotic::database {
  *         For each version:
  *           - Version JSON length: uint32
  *           - Version JSON: bytes
+ *     (v3+) Vector data:
+ *       - Vector count: uint64 (number of vectors in this collection)
+ *       For each vector:
+ *         - Doc ID length: uint16
+ *         - Doc ID: bytes
+ *         - Vector dimension: uint32
+ *         - Vector data: float32[dimension] (raw bytes)
  *
  * Footer:
  *   - Body checksum: uint32
@@ -52,7 +59,7 @@ namespace smartbotic::database {
 
 struct SnapshotHeader {
     char magic[8] = {'C', 'A', 'L', 'S', 'N', 'A', 'P', '1'};
-    uint32_t version = 2;
+    uint32_t version = 3;
     uint64_t timestamp = 0;
     uint64_t walSequence = 0;
     uint64_t documentCount = 0;
@@ -153,7 +160,7 @@ private:
      * @param snapshotVersion Snapshot format version (for backward compatibility)
      */
     void deserializeStore(const std::vector<uint8_t>& data, MemoryStore& store,
-                          uint32_t snapshotVersion = 2) const;
+                          uint32_t snapshotVersion = 3) const;
 
     Config config_;
 };