|
|
@@ -0,0 +1,762 @@
|
|
|
+# Vector Storage Implementation Plan
|
|
|
+
|
|
|
+> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
|
|
+
|
|
|
+**Goal:** Add embedding vector storage and SIMD-accelerated cosine similarity search to smartbotic-database.
|
|
|
+
|
|
|
+**Architecture:** New `vector_dimension` collection option, `_vector` document field extracted and stored in parallel float arrays, new `SimilaritySearch` gRPC RPC. Vectors persisted via existing WAL + snapshot system with new binary entry types.
|
|
|
+
|
|
|
+**Tech Stack:** C++20, gRPC/Protobuf, SIMD (SSE4.1/AVX2), nlohmann/json, LZ4
|
|
|
+
|
|
|
+**Repo:** `/data/smartbotic-database`
|
|
|
+
|
|
|
+---
|
|
|
+
|
|
|
+### Task 1: Proto — Add vector_dimension and SimilaritySearch RPC
|
|
|
+
|
|
|
+**Files:**
|
|
|
+- Modify: `proto/database.proto`
|
|
|
+
|
|
|
+- [ ] **Step 1: Add `vector_dimension` to `CollectionOptions` message**
|
|
|
+
|
|
|
+In `proto/database.proto`, add field 6 to `CollectionOptions`:
|
|
|
+```protobuf
|
|
|
+message CollectionOptions {
|
|
|
+ bool auto_create_id = 1;
|
|
|
+ uint32 default_ttl_seconds = 2;
|
|
|
+ bool encrypted = 3;
|
|
|
+ repeated string sensitive_fields = 4;
|
|
|
+ uint32 max_versions = 5;
|
|
|
+ uint32 vector_dimension = 6; // Fixed dimension for vector fields (0 = disabled)
|
|
|
+}
|
|
|
+```
|
|
|
+
|
|
|
+- [ ] **Step 2: Add SimilaritySearch RPC and messages**
|
|
|
+
|
|
|
+Add to the `DatabaseService` service block:
|
|
|
+```protobuf
|
|
|
+rpc SimilaritySearch(SimilaritySearchRequest) returns (SimilaritySearchResponse);
|
|
|
+```
|
|
|
+
|
|
|
+Add the messages:
|
|
|
+```protobuf
|
|
|
+message SimilaritySearchRequest {
|
|
|
+ string collection = 1;
|
|
|
+ repeated float query_vector = 2;
|
|
|
+ uint32 top_k = 3;
|
|
|
+ float min_score = 4;
|
|
|
+}
|
|
|
+
|
|
|
+message SimilaritySearchResponse {
|
|
|
+ repeated SimilarityResult results = 1;
|
|
|
+}
|
|
|
+
|
|
|
+message SimilarityResult {
|
|
|
+ string id = 1;
|
|
|
+ float score = 2;
|
|
|
+ bytes data = 3;
|
|
|
+}
|
|
|
+```
|
|
|
+
|
|
|
+- [ ] **Step 3: Rebuild protos**
|
|
|
+
|
|
|
+```bash
|
|
|
+cmake --build build -j$(nproc) --target smartbotic_db_proto
|
|
|
+```
|
|
|
+
|
|
|
+- [ ] **Step 4: Commit**
|
|
|
+
|
|
|
+```bash
|
|
|
+git add proto/database.proto
|
|
|
+git commit -m "proto: add vector_dimension option and SimilaritySearch RPC"
|
|
|
+```
|
|
|
+
|
|
|
+---
|
|
|
+
|
|
|
+### Task 2: CollectionOptions — Add vector_dimension to C++ types
|
|
|
+
|
|
|
+**Files:**
|
|
|
+- Modify: `service/src/document.hpp`
|
|
|
+
|
|
|
+- [ ] **Step 1: Add `vectorDimension` to `CollectionOptions` struct**
|
|
|
+
|
|
|
+Add field to the struct (after `pinned`):
|
|
|
+```cpp
|
|
|
+uint32_t vectorDimension = 0; // 0 = no vector support
|
|
|
+```
|
|
|
+
|
|
|
+- [ ] **Step 2: Update `toJson()` method**
|
|
|
+
|
|
|
+Add to the JSON serialization:
|
|
|
+```cpp
|
|
|
+j["vector_dimension"] = vectorDimension;
|
|
|
+```
|
|
|
+
|
|
|
+- [ ] **Step 3: Update `fromJson()` method**
|
|
|
+
|
|
|
+Add to the JSON deserialization:
|
|
|
+```cpp
|
|
|
+if (j.contains("vector_dimension") && j["vector_dimension"].is_number())
|
|
|
+ opts.vectorDimension = j["vector_dimension"].get<uint32_t>();
|
|
|
+```
|
|
|
+
|
|
|
+- [ ] **Step 4: Build and verify**
|
|
|
+
|
|
|
+```bash
|
|
|
+cmake --build build -j$(nproc)
|
|
|
+```
|
|
|
+
|
|
|
+- [ ] **Step 5: Commit**
|
|
|
+
|
|
|
+```bash
|
|
|
+git add service/src/document.hpp
|
|
|
+git commit -m "feat: add vectorDimension to CollectionOptions"
|
|
|
+```
|
|
|
+
|
|
|
+---
|
|
|
+
|
|
|
+### Task 3: MemoryStore — Vector storage and cosine similarity
|
|
|
+
|
|
|
+**Files:**
|
|
|
+- Modify: `service/src/memory_store.hpp`
|
|
|
+- Modify: `service/src/memory_store.cpp`
|
|
|
+
|
|
|
+- [ ] **Step 1: Add vector storage to `CollectionData` struct in memory_store.hpp**
|
|
|
+
|
|
|
+Add to the `CollectionData` struct:
|
|
|
+```cpp
|
|
|
+// Vector storage (for collections with vectorDimension > 0)
|
|
|
+std::unordered_map<std::string, std::vector<float>> vectors;
|
|
|
+std::unordered_map<std::string, float> vectorNorms; // cached L2 norms
|
|
|
+```
|
|
|
+
|
|
|
+- [ ] **Step 2: Add `SimilarityResult` struct and `similaritySearch` method declaration**
|
|
|
+
|
|
|
+Add struct and method to `MemoryStore` public interface:
|
|
|
+```cpp
|
|
|
+struct SimilarityResult {
|
|
|
+ std::string id;
|
|
|
+ float score;
|
|
|
+ Document document;
|
|
|
+};
|
|
|
+
|
|
|
+std::vector<SimilarityResult> similaritySearch(
|
|
|
+ const std::string& collection,
|
|
|
+ const std::vector<float>& queryVector,
|
|
|
+ uint32_t topK,
|
|
|
+ float minScore = 0.0f);
|
|
|
+```
|
|
|
+
|
|
|
+- [ ] **Step 3: Add vector helper methods (private)**
|
|
|
+
|
|
|
+```cpp
|
|
|
+private:
|
|
|
+ void storeVector(CollectionData& coll, const std::string& docId,
|
|
|
+ const std::vector<float>& vec);
|
|
|
+ void removeVector(CollectionData& coll, const std::string& docId);
|
|
|
+ std::vector<float> extractVector(CollectionData& coll, nlohmann::json& docData);
|
|
|
+ static float cosineSimilitySIMD(const float* a, const float* b, size_t dim, float normA, float normB);
|
|
|
+ static float computeNorm(const float* data, size_t dim);
|
|
|
+```
|
|
|
+
|
|
|
+- [ ] **Step 4: Implement SIMD cosine similarity in memory_store.cpp**
|
|
|
+
|
|
|
+```cpp
|
|
|
+#include <cmath>
|
|
|
+#if defined(__SSE4_1__)
|
|
|
+#include <smmintrin.h>
|
|
|
+#endif
|
|
|
+#if defined(__AVX2__)
|
|
|
+#include <immintrin.h>
|
|
|
+#endif
|
|
|
+
|
|
|
+float MemoryStore::computeNorm(const float* data, size_t dim) {
|
|
|
+ float sum = 0.0f;
|
|
|
+ size_t i = 0;
|
|
|
+#if defined(__AVX2__)
|
|
|
+ __m256 vsum = _mm256_setzero_ps();
|
|
|
+ for (; i + 8 <= dim; i += 8) {
|
|
|
+ __m256 v = _mm256_loadu_ps(data + i);
|
|
|
+ vsum = _mm256_fmadd_ps(v, v, vsum);
|
|
|
+ }
|
|
|
+ float tmp[8];
|
|
|
+ _mm256_storeu_ps(tmp, vsum);
|
|
|
+ sum = tmp[0]+tmp[1]+tmp[2]+tmp[3]+tmp[4]+tmp[5]+tmp[6]+tmp[7];
|
|
|
+#elif defined(__SSE4_1__)
|
|
|
+ __m128 vsum = _mm_setzero_ps();
|
|
|
+ for (; i + 4 <= dim; i += 4) {
|
|
|
+ __m128 v = _mm_loadu_ps(data + i);
|
|
|
+ vsum = _mm_add_ps(vsum, _mm_mul_ps(v, v));
|
|
|
+ }
|
|
|
+ float tmp[4];
|
|
|
+ _mm_storeu_ps(tmp, vsum);
|
|
|
+ sum = tmp[0]+tmp[1]+tmp[2]+tmp[3];
|
|
|
+#endif
|
|
|
+ for (; i < dim; ++i) sum += data[i] * data[i];
|
|
|
+ return std::sqrt(sum);
|
|
|
+}
|
|
|
+
|
|
|
+float MemoryStore::cosineSimilitySIMD(const float* a, const float* b,
|
|
|
+ size_t dim, float normA, float normB) {
|
|
|
+ if (normA == 0.0f || normB == 0.0f) return 0.0f;
|
|
|
+ float dot = 0.0f;
|
|
|
+ size_t i = 0;
|
|
|
+#if defined(__AVX2__)
|
|
|
+ __m256 vdot = _mm256_setzero_ps();
|
|
|
+ for (; i + 8 <= dim; i += 8) {
|
|
|
+ __m256 va = _mm256_loadu_ps(a + i);
|
|
|
+ __m256 vb = _mm256_loadu_ps(b + i);
|
|
|
+ vdot = _mm256_fmadd_ps(va, vb, vdot);
|
|
|
+ }
|
|
|
+ float tmp[8];
|
|
|
+ _mm256_storeu_ps(tmp, vdot);
|
|
|
+ dot = tmp[0]+tmp[1]+tmp[2]+tmp[3]+tmp[4]+tmp[5]+tmp[6]+tmp[7];
|
|
|
+#elif defined(__SSE4_1__)
|
|
|
+ __m128 vdot = _mm_setzero_ps();
|
|
|
+ for (; i + 4 <= dim; i += 4) {
|
|
|
+ __m128 va = _mm_loadu_ps(a + i);
|
|
|
+ __m128 vb = _mm_loadu_ps(b + i);
|
|
|
+ vdot = _mm_add_ps(vdot, _mm_mul_ps(va, vb));
|
|
|
+ }
|
|
|
+ float tmp[4];
|
|
|
+ _mm_storeu_ps(tmp, vdot);
|
|
|
+ dot = tmp[0]+tmp[1]+tmp[2]+tmp[3];
|
|
|
+#endif
|
|
|
+ for (; i < dim; ++i) dot += a[i] * b[i];
|
|
|
+ return dot / (normA * normB);
|
|
|
+}
|
|
|
+```
|
|
|
+
|
|
|
+- [ ] **Step 5: Implement extractVector, storeVector, removeVector**
|
|
|
+
|
|
|
+```cpp
|
|
|
+std::vector<float> MemoryStore::extractVector(CollectionData& coll,
|
|
|
+ nlohmann::json& docData) {
|
|
|
+ if (coll.options.vectorDimension == 0) return {};
|
|
|
+ if (!docData.contains("_vector") || !docData["_vector"].is_array()) return {};
|
|
|
+
|
|
|
+ auto vec = docData["_vector"].get<std::vector<float>>();
|
|
|
+ docData.erase("_vector"); // Strip from document data
|
|
|
+
|
|
|
+ if (vec.size() != coll.options.vectorDimension) {
|
|
|
+ throw std::invalid_argument("Vector dimension mismatch: expected " +
|
|
|
+ std::to_string(coll.options.vectorDimension) +
|
|
|
+ ", got " + std::to_string(vec.size()));
|
|
|
+ }
|
|
|
+ return vec;
|
|
|
+}
|
|
|
+
|
|
|
+void MemoryStore::storeVector(CollectionData& coll, const std::string& docId,
|
|
|
+ const std::vector<float>& vec) {
|
|
|
+ if (vec.empty()) return;
|
|
|
+ coll.vectors[docId] = vec;
|
|
|
+ coll.vectorNorms[docId] = computeNorm(vec.data(), vec.size());
|
|
|
+}
|
|
|
+
|
|
|
+void MemoryStore::removeVector(CollectionData& coll, const std::string& docId) {
|
|
|
+ coll.vectors.erase(docId);
|
|
|
+ coll.vectorNorms.erase(docId);
|
|
|
+}
|
|
|
+```
|
|
|
+
|
|
|
+- [ ] **Step 6: Implement similaritySearch**
|
|
|
+
|
|
|
+```cpp
|
|
|
+std::vector<MemoryStore::SimilarityResult> MemoryStore::similaritySearch(
|
|
|
+ const std::string& collection,
|
|
|
+ const std::vector<float>& queryVector,
|
|
|
+ uint32_t topK,
|
|
|
+ float minScore) {
|
|
|
+
|
|
|
+ auto it = collections_.find(collection);
|
|
|
+ if (it == collections_.end()) return {};
|
|
|
+ auto& coll = *it->second;
|
|
|
+
|
|
|
+ if (coll.options.vectorDimension == 0)
|
|
|
+ throw std::invalid_argument("Collection does not support vectors");
|
|
|
+ if (queryVector.size() != coll.options.vectorDimension)
|
|
|
+ throw std::invalid_argument("Query vector dimension mismatch");
|
|
|
+
|
|
|
+ std::shared_lock lock(coll.mutex);
|
|
|
+
|
|
|
+ float queryNorm = computeNorm(queryVector.data(), queryVector.size());
|
|
|
+ if (queryNorm == 0.0f) return {};
|
|
|
+
|
|
|
+ // Score all vectors
|
|
|
+ std::vector<SimilarityResult> results;
|
|
|
+ results.reserve(coll.vectors.size());
|
|
|
+
|
|
|
+ for (const auto& [docId, vec] : coll.vectors) {
|
|
|
+ auto normIt = coll.vectorNorms.find(docId);
|
|
|
+ float docNorm = (normIt != coll.vectorNorms.end()) ? normIt->second : 0.0f;
|
|
|
+
|
|
|
+ float score = cosineSimilitySIMD(queryVector.data(), vec.data(),
|
|
|
+ vec.size(), queryNorm, docNorm);
|
|
|
+ if (score >= minScore) {
|
|
|
+ auto docIt = coll.documents.find(docId);
|
|
|
+ if (docIt != coll.documents.end()) {
|
|
|
+ results.push_back({docId, score, docIt->second});
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ // Sort by score descending, take top K
|
|
|
+ std::partial_sort(results.begin(),
|
|
|
+ results.begin() + std::min<size_t>(topK, results.size()),
|
|
|
+ results.end(),
|
|
|
+ [](const auto& a, const auto& b) { return a.score > b.score; });
|
|
|
+
|
|
|
+ if (results.size() > topK) results.resize(topK);
|
|
|
+ return results;
|
|
|
+}
|
|
|
+```
|
|
|
+
|
|
|
+- [ ] **Step 7: Integrate vector extraction into existing insert/update/upsert/remove methods**
|
|
|
+
|
|
|
+In `insert()`, after document validation, before storing:
|
|
|
+```cpp
|
|
|
+auto vec = extractVector(coll, doc.data);
|
|
|
+// ... existing insert logic ...
|
|
|
+storeVector(coll, doc.id, vec);
|
|
|
+```
|
|
|
+
|
|
|
+In `update()` and `upsert()`, same pattern.
|
|
|
+
|
|
|
+In `remove()`, add:
|
|
|
+```cpp
|
|
|
+removeVector(coll, id);
|
|
|
+```
|
|
|
+
|
|
|
+- [ ] **Step 8: Add vector data accessors for snapshot/WAL**
|
|
|
+
|
|
|
+```cpp
|
|
|
+// Public methods for persistence
|
|
|
+const std::unordered_map<std::string, std::vector<float>>*
|
|
|
+ getCollectionVectors(const std::string& collection) const;
|
|
|
+void loadVector(const std::string& collection, const std::string& docId,
|
|
|
+ std::vector<float> vec);
|
|
|
+```
|
|
|
+
|
|
|
+- [ ] **Step 9: Build and verify**
|
|
|
+
|
|
|
+```bash
|
|
|
+cmake --build build -j$(nproc)
|
|
|
+```
|
|
|
+
|
|
|
+- [ ] **Step 10: Commit**
|
|
|
+
|
|
|
+```bash
|
|
|
+git add service/src/memory_store.hpp service/src/memory_store.cpp
|
|
|
+git commit -m "feat: vector storage with SIMD cosine similarity search"
|
|
|
+```
|
|
|
+
|
|
|
+---
|
|
|
+
|
|
|
+### Task 4: WAL — Add VEC_PUT and VEC_DELETE entries
|
|
|
+
|
|
|
+**Files:**
|
|
|
+- Modify: `service/src/persistence/wal.hpp`
|
|
|
+- Modify: `service/src/persistence/wal.cpp`
|
|
|
+
|
|
|
+- [ ] **Step 1: Add VEC_PUT and VEC_DELETE to WalOpType enum**
|
|
|
+
|
|
|
+```cpp
|
|
|
+enum class WalOpType : uint8_t {
|
|
|
+ INSERT = 1, UPDATE = 2, DELETE = 3, UPSERT = 4,
|
|
|
+ CREATE_COLLECTION = 5, DROP_COLLECTION = 6,
|
|
|
+ SET_ADD = 7, SET_REMOVE = 8,
|
|
|
+ VEC_PUT = 9, VEC_DELETE = 10
|
|
|
+};
|
|
|
+```
|
|
|
+
|
|
|
+- [ ] **Step 2: Add `vectorData` field to WalEntry**
|
|
|
+
|
|
|
+```cpp
|
|
|
+struct WalEntry {
|
|
|
+ // ... existing fields ...
|
|
|
+ std::optional<std::vector<float>> vectorData; // For VEC_PUT
|
|
|
+};
|
|
|
+```
|
|
|
+
|
|
|
+- [ ] **Step 3: Update serialize/deserialize for vector entries**
|
|
|
+
|
|
|
+For `VEC_PUT`: serialize as `[dim(uint32) | float_bytes(dim*4)]`
|
|
|
+For `VEC_DELETE`: no extra data (collection + documentId sufficient)
|
|
|
+
|
|
|
+- [ ] **Step 4: Add helper methods**
|
|
|
+
|
|
|
+```cpp
|
|
|
+static WalEntry makeVecPutEntry(const std::string& collection,
|
|
|
+ const std::string& docId, const std::vector<float>& vec);
|
|
|
+static WalEntry makeVecDeleteEntry(const std::string& collection,
|
|
|
+ const std::string& docId);
|
|
|
+```
|
|
|
+
|
|
|
+- [ ] **Step 5: Build and verify**
|
|
|
+
|
|
|
+```bash
|
|
|
+cmake --build build -j$(nproc)
|
|
|
+```
|
|
|
+
|
|
|
+- [ ] **Step 6: Commit**
|
|
|
+
|
|
|
+```bash
|
|
|
+git add service/src/persistence/wal.hpp service/src/persistence/wal.cpp
|
|
|
+git commit -m "feat: WAL entries for vector storage (VEC_PUT/VEC_DELETE)"
|
|
|
+```
|
|
|
+
|
|
|
+---
|
|
|
+
|
|
|
+### Task 5: Snapshot — Serialize/deserialize vectors
|
|
|
+
|
|
|
+**Files:**
|
|
|
+- Modify: `service/src/persistence/snapshot.hpp`
|
|
|
+- Modify: `service/src/persistence/snapshot.cpp`
|
|
|
+
|
|
|
+- [ ] **Step 1: Update snapshot format version**
|
|
|
+
|
|
|
+Bump `SNAPSHOT_VERSION` from 2 to 3.
|
|
|
+
|
|
|
+- [ ] **Step 2: Add vector serialization to `createSnapshot()`**
|
|
|
+
|
|
|
+After writing version history for each collection, write vectors:
|
|
|
+```cpp
|
|
|
+// Vector count
|
|
|
+uint64_t vecCount = vectors ? vectors->size() : 0;
|
|
|
+write(vecCount);
|
|
|
+for (const auto& [docId, vec] : *vectors) {
|
|
|
+ writeString(docId);
|
|
|
+ uint32_t dim = vec.size();
|
|
|
+ write(dim);
|
|
|
+ write(vec.data(), dim * sizeof(float));
|
|
|
+}
|
|
|
+```
|
|
|
+
|
|
|
+- [ ] **Step 3: Add vector deserialization to `loadSnapshot()`**
|
|
|
+
|
|
|
+After reading version history, read vectors:
|
|
|
+```cpp
|
|
|
+uint64_t vecCount = read<uint64_t>();
|
|
|
+for (uint64_t v = 0; v < vecCount; ++v) {
|
|
|
+ auto docId = readString();
|
|
|
+ uint32_t dim = read<uint32_t>();
|
|
|
+ std::vector<float> vec(dim);
|
|
|
+ readBytes(vec.data(), dim * sizeof(float));
|
|
|
+ store.loadVector(collectionName, docId, std::move(vec));
|
|
|
+}
|
|
|
+```
|
|
|
+
|
|
|
+Handle version 2 snapshots (no vectors) for backward compatibility.
|
|
|
+
|
|
|
+- [ ] **Step 4: Build and verify**
|
|
|
+
|
|
|
+```bash
|
|
|
+cmake --build build -j$(nproc)
|
|
|
+```
|
|
|
+
|
|
|
+- [ ] **Step 5: Commit**
|
|
|
+
|
|
|
+```bash
|
|
|
+git add service/src/persistence/snapshot.hpp service/src/persistence/snapshot.cpp
|
|
|
+git commit -m "feat: vector serialization in snapshots (v3 format)"
|
|
|
+```
|
|
|
+
|
|
|
+---
|
|
|
+
|
|
|
+### Task 6: PersistenceManager — Wire vector WAL entries
|
|
|
+
|
|
|
+**Files:**
|
|
|
+- Modify: `service/src/persistence/persistence_manager.hpp`
|
|
|
+- Modify: `service/src/persistence/persistence_manager.cpp`
|
|
|
+
|
|
|
+- [ ] **Step 1: Add `logVecPut` and `logVecDelete` methods**
|
|
|
+
|
|
|
+```cpp
|
|
|
+uint64_t logVecPut(const std::string& collection, const std::string& docId,
|
|
|
+ const std::vector<float>& vec);
|
|
|
+uint64_t logVecDelete(const std::string& collection, const std::string& docId);
|
|
|
+```
|
|
|
+
|
|
|
+- [ ] **Step 2: Implement the methods**
|
|
|
+
|
|
|
+Same pattern as existing `logInsert`/`logDelete` — create WalEntry, append to WAL.
|
|
|
+
|
|
|
+- [ ] **Step 3: Update WAL replay to handle VEC_PUT/VEC_DELETE**
|
|
|
+
|
|
|
+In `recover()` and `replayWal()`, add cases for the new op types:
|
|
|
+```cpp
|
|
|
+case WalOpType::VEC_PUT:
|
|
|
+ store.loadVector(entry.collection, entry.documentId, *entry.vectorData);
|
|
|
+ break;
|
|
|
+case WalOpType::VEC_DELETE:
|
|
|
+ // Vector removed during document delete — handled by store
|
|
|
+ break;
|
|
|
+```
|
|
|
+
|
|
|
+- [ ] **Step 4: Wire persist callback from MemoryStore**
|
|
|
+
|
|
|
+When MemoryStore calls the persist callback for insert/update with vectors, also log `VEC_PUT`. When delete, also log `VEC_DELETE`.
|
|
|
+
|
|
|
+- [ ] **Step 5: Build and verify**
|
|
|
+
|
|
|
+```bash
|
|
|
+cmake --build build -j$(nproc)
|
|
|
+```
|
|
|
+
|
|
|
+- [ ] **Step 6: Commit**
|
|
|
+
|
|
|
+```bash
|
|
|
+git add service/src/persistence/persistence_manager.hpp service/src/persistence/persistence_manager.cpp
|
|
|
+git commit -m "feat: wire vector WAL entries in PersistenceManager"
|
|
|
+```
|
|
|
+
|
|
|
+---
|
|
|
+
|
|
|
+### Task 7: gRPC Implementation — Handle vectors in Put, implement SimilaritySearch
|
|
|
+
|
|
|
+**Files:**
|
|
|
+- Modify: `service/src/database_grpc_impl.cpp`
|
|
|
+
|
|
|
+- [ ] **Step 1: Update `Insert`/`Update`/`Upsert` handlers**
|
|
|
+
|
|
|
+Before calling `store_.insert()`, check for `_vector` in the JSON data. The `MemoryStore::extractVector()` handles extraction and validation — no extra gRPC code needed since the JSON data passes through. Just ensure the proto `CollectionOptions` maps `vector_dimension` correctly.
|
|
|
+
|
|
|
+- [ ] **Step 2: Update collection creation handler**
|
|
|
+
|
|
|
+In the `CreateCollection` handler, map the proto field:
|
|
|
+```cpp
|
|
|
+opts.vectorDimension = request->options().vector_dimension();
|
|
|
+```
|
|
|
+
|
|
|
+- [ ] **Step 3: Update `GetCollectionInfo` response**
|
|
|
+
|
|
|
+Add `vector_dimension` to the CollectionInfo proto mapping.
|
|
|
+
|
|
|
+- [ ] **Step 4: Implement `SimilaritySearch` RPC**
|
|
|
+
|
|
|
+```cpp
|
|
|
+grpc::Status DatabaseGrpcImpl::SimilaritySearch(
|
|
|
+ grpc::ServerContext* context,
|
|
|
+ const pb::SimilaritySearchRequest* request,
|
|
|
+ pb::SimilaritySearchResponse* response) {
|
|
|
+
|
|
|
+ try {
|
|
|
+ std::vector<float> queryVec(request->query_vector().begin(),
|
|
|
+ request->query_vector().end());
|
|
|
+ auto results = store_.similaritySearch(
|
|
|
+ request->collection(), queryVec,
|
|
|
+ request->top_k(), request->min_score());
|
|
|
+
|
|
|
+ for (const auto& r : results) {
|
|
|
+ auto* result = response->add_results();
|
|
|
+ result->set_id(r.id);
|
|
|
+ result->set_score(r.score);
|
|
|
+ // Serialize document to JSON bytes, decrypt if needed
|
|
|
+ auto docData = r.document.data;
|
|
|
+ if (encryption_.isEnabled()) {
|
|
|
+ encryption_.decryptSensitiveFields(docData,
|
|
|
+ store_.getCollectionOptions(r.document.collection));
|
|
|
+ }
|
|
|
+ result->set_data(docData.dump());
|
|
|
+ }
|
|
|
+ return grpc::Status::OK;
|
|
|
+ } catch (const std::exception& e) {
|
|
|
+ return grpc::Status(grpc::StatusCode::INTERNAL, e.what());
|
|
|
+ }
|
|
|
+}
|
|
|
+```
|
|
|
+
|
|
|
+- [ ] **Step 5: Build and verify**
|
|
|
+
|
|
|
+```bash
|
|
|
+cmake --build build -j$(nproc)
|
|
|
+```
|
|
|
+
|
|
|
+- [ ] **Step 6: Commit**
|
|
|
+
|
|
|
+```bash
|
|
|
+git add service/src/database_grpc_impl.cpp
|
|
|
+git commit -m "feat: SimilaritySearch gRPC handler + vector support in Put"
|
|
|
+```
|
|
|
+
|
|
|
+---
|
|
|
+
|
|
|
+### Task 8: Client Library — Add similaritySearch and vector-aware put
|
|
|
+
|
|
|
+**Files:**
|
|
|
+- Modify: `client/include/smartbotic/database/client.hpp`
|
|
|
+- Modify: `client/src/client.cpp`
|
|
|
+
|
|
|
+- [ ] **Step 1: Add `SimilarityResult` struct to client header**
|
|
|
+
|
|
|
+```cpp
|
|
|
+struct SimilarityResult {
|
|
|
+ std::string id;
|
|
|
+ float score;
|
|
|
+ nlohmann::json data;
|
|
|
+};
|
|
|
+```
|
|
|
+
|
|
|
+- [ ] **Step 2: Add `similaritySearch` method declaration**
|
|
|
+
|
|
|
+```cpp
|
|
|
+std::vector<SimilarityResult> similaritySearch(
|
|
|
+ const std::string& collection,
|
|
|
+ const std::vector<float>& queryVector,
|
|
|
+ uint32_t topK,
|
|
|
+ float minScore = 0.0f);
|
|
|
+```
|
|
|
+
|
|
|
+- [ ] **Step 3: Add `vector_dimension` to `createCollection`**
|
|
|
+
|
|
|
+Update signature to accept `vectorDimension` parameter:
|
|
|
+```cpp
|
|
|
+bool createCollection(const std::string& name,
|
|
|
+ uint32_t defaultTtlSeconds = 0, bool encrypted = false,
|
|
|
+ uint32_t maxVersions = 0, uint32_t vectorDimension = 0);
|
|
|
+```
|
|
|
+
|
|
|
+- [ ] **Step 4: Implement `similaritySearch` in client.cpp**
|
|
|
+
|
|
|
+```cpp
|
|
|
+std::vector<Client::SimilarityResult> Client::similaritySearch(
|
|
|
+ const std::string& collection,
|
|
|
+ const std::vector<float>& queryVector,
|
|
|
+ uint32_t topK, float minScore) {
|
|
|
+
|
|
|
+ pb::SimilaritySearchRequest request;
|
|
|
+ request.set_collection(collection);
|
|
|
+ for (float f : queryVector) request.add_query_vector(f);
|
|
|
+ request.set_top_k(topK);
|
|
|
+ request.set_min_score(minScore);
|
|
|
+
|
|
|
+ pb::SimilaritySearchResponse response;
|
|
|
+ grpc::ClientContext ctx;
|
|
|
+ setDeadline(ctx);
|
|
|
+
|
|
|
+ auto status = stub_->SimilaritySearch(&ctx, request, &response);
|
|
|
+ if (!status.ok()) throw std::runtime_error(status.error_message());
|
|
|
+
|
|
|
+ std::vector<SimilarityResult> results;
|
|
|
+ results.reserve(response.results_size());
|
|
|
+ for (const auto& r : response.results()) {
|
|
|
+ results.push_back({
|
|
|
+ r.id(), r.score(),
|
|
|
+ nlohmann::json::parse(r.data())
|
|
|
+ });
|
|
|
+ }
|
|
|
+ return results;
|
|
|
+}
|
|
|
+```
|
|
|
+
|
|
|
+- [ ] **Step 5: Update `createCollection` implementation**
|
|
|
+
|
|
|
+Add `request.mutable_options()->set_vector_dimension(vectorDimension);`
|
|
|
+
|
|
|
+- [ ] **Step 6: Build and verify**
|
|
|
+
|
|
|
+```bash
|
|
|
+cmake --build build -j$(nproc)
|
|
|
+```
|
|
|
+
|
|
|
+- [ ] **Step 7: Commit**
|
|
|
+
|
|
|
+```bash
|
|
|
+git add client/include/smartbotic/database/client.hpp client/src/client.cpp
|
|
|
+git commit -m "feat: client library — similaritySearch + vector-aware createCollection"
|
|
|
+```
|
|
|
+
|
|
|
+---
|
|
|
+
|
|
|
+### Task 9: Migration Support — Handle vector_dimension in migrations
|
|
|
+
|
|
|
+**Files:**
|
|
|
+- Modify: `service/src/migrations/migration_runner.cpp`
|
|
|
+
|
|
|
+- [ ] **Step 1: Update `create_collection` migration handler**
|
|
|
+
|
|
|
+Parse `vector_dimension` from migration JSON options:
|
|
|
+```cpp
|
|
|
+if (options.contains("vector_dimension") && options["vector_dimension"].is_number())
|
|
|
+ collOpts.vectorDimension = options["vector_dimension"].get<uint32_t>();
|
|
|
+```
|
|
|
+
|
|
|
+- [ ] **Step 2: Build and verify**
|
|
|
+
|
|
|
+```bash
|
|
|
+cmake --build build -j$(nproc)
|
|
|
+```
|
|
|
+
|
|
|
+- [ ] **Step 3: Commit**
|
|
|
+
|
|
|
+```bash
|
|
|
+git add service/src/migrations/migration_runner.cpp
|
|
|
+git commit -m "feat: migration support for vector_dimension collection option"
|
|
|
+```
|
|
|
+
|
|
|
+---
|
|
|
+
|
|
|
+### Task 10: Integration Test — End-to-end vector storage
|
|
|
+
|
|
|
+**Files:**
|
|
|
+- Create: `tests/test_vector_storage.cpp` (or add to existing test file)
|
|
|
+
|
|
|
+- [ ] **Step 1: Write test that creates a vector collection, inserts documents with vectors, and searches**
|
|
|
+
|
|
|
+```cpp
|
|
|
+// 1. Create collection with vector_dimension=3 (small for testing)
|
|
|
+client.createCollection("test_vectors", 0, false, 0, 3);
|
|
|
+
|
|
|
+// 2. Insert documents with _vector
|
|
|
+client.insert("test_vectors", {
|
|
|
+ {"id", "doc1"}, {"content", "hello"},
|
|
|
+ {"_vector", {1.0f, 0.0f, 0.0f}}
|
|
|
+});
|
|
|
+client.insert("test_vectors", {
|
|
|
+ {"id", "doc2"}, {"content", "world"},
|
|
|
+ {"_vector", {0.0f, 1.0f, 0.0f}}
|
|
|
+});
|
|
|
+client.insert("test_vectors", {
|
|
|
+ {"id", "doc3"}, {"content", "similar"},
|
|
|
+ {"_vector", {0.9f, 0.1f, 0.0f}} // similar to doc1
|
|
|
+});
|
|
|
+
|
|
|
+// 3. Search — should return doc1 first, then doc3
|
|
|
+auto results = client.similaritySearch("test_vectors", {1.0f, 0.0f, 0.0f}, 2);
|
|
|
+assert(results.size() == 2);
|
|
|
+assert(results[0].id == "doc1"); // exact match
|
|
|
+assert(results[0].score > 0.99f);
|
|
|
+assert(results[1].id == "doc3"); // similar
|
|
|
+assert(results[1].score > 0.9f);
|
|
|
+
|
|
|
+// 4. Verify _vector is stripped from document data
|
|
|
+auto doc = client.get("test_vectors", "doc1");
|
|
|
+assert(!doc->contains("_vector"));
|
|
|
+
|
|
|
+// 5. Dimension mismatch should fail
|
|
|
+try {
|
|
|
+ client.insert("test_vectors", {
|
|
|
+ {"id", "bad"}, {"_vector", {1.0f, 2.0f}} // wrong dimension
|
|
|
+ });
|
|
|
+ assert(false); // should not reach
|
|
|
+} catch (...) {}
|
|
|
+```
|
|
|
+
|
|
|
+- [ ] **Step 2: Run test**
|
|
|
+
|
|
|
+```bash
|
|
|
+cmake --build build -j$(nproc) && ./build/tests/test_vector_storage
|
|
|
+```
|
|
|
+
|
|
|
+- [ ] **Step 3: Commit**
|
|
|
+
|
|
|
+```bash
|
|
|
+git add tests/
|
|
|
+git commit -m "test: end-to-end vector storage integration test"
|
|
|
+```
|
|
|
+
|
|
|
+- [ ] **Step 4: Push all changes**
|
|
|
+
|
|
|
+```bash
|
|
|
+git push origin main
|
|
|
+```
|