|
|
@@ -12,6 +12,7 @@ A standalone key-value document database with gRPC API, designed for microservic
|
|
|
- **Encryption** - Field-level AES-256-GCM encryption for sensitive data
|
|
|
- **Events** - Pub/sub event subscription for real-time updates
|
|
|
- **File Storage** - Binary file management with streaming support
|
|
|
+- **Vector Storage** - Embedding vector storage with SIMD-accelerated cosine similarity search
|
|
|
- **Migrations** - Declarative JSON-based schema migrations
|
|
|
|
|
|
## Architecture
|
|
|
@@ -119,6 +120,36 @@ auto [data, metadata] = client.downloadFile(fileId);
|
|
|
client.deleteFile(fileId);
|
|
|
```
|
|
|
|
|
|
+### Vector Storage & Similarity Search
|
|
|
+
|
|
|
+Store embedding vectors alongside documents and perform cosine similarity search — no external dependencies required.
|
|
|
+
|
|
|
+```cpp
|
|
|
+// Create a vector-enabled collection (dimension must be fixed at creation)
|
|
|
+client.createCollection("memories", 0, false, 0, 384); // 384-dim embeddings
|
|
|
+
|
|
|
+// Insert documents with _vector field
|
|
|
+nlohmann::json doc = {
|
|
|
+ {"id", "mem-1"},
|
|
|
+ {"content", "The user prefers dark mode"},
|
|
|
+ {"_vector", embedding} // std::vector<float> of size 384
|
|
|
+};
|
|
|
+client.put("memories", doc);
|
|
|
+
|
|
|
+// Similarity search — returns top-K results sorted by cosine similarity
|
|
|
+auto results = client.similaritySearch("memories", queryEmbedding, /*topK=*/5, /*minScore=*/0.7f);
|
|
|
+for (const auto& r : results) {
|
|
|
+ std::cout << r.id << " (score=" << r.score << "): "
|
|
|
+ << r.data["content"] << std::endl;
|
|
|
+}
|
|
|
+```
|
|
|
+
|
|
|
+Notes:
|
|
|
+- `_vector` is extracted and stored separately — it is **not** returned in `Get()` or `Query()` responses
|
|
|
+- Dimension is validated on every insert/update — mismatches are rejected
|
|
|
+- Documents without `_vector` are stored normally but excluded from similarity search
|
|
|
+- SIMD-accelerated (AVX2/SSE4.1) brute-force scan, suitable for up to ~100K vectors
|
|
|
+
|
|
|
### Subscribing to Events
|
|
|
|
|
|
```cpp
|
|
|
@@ -179,6 +210,7 @@ History depth is configurable per-collection via `max_versions` (0 = unlimited).
|
|
|
| `UploadFile` | Upload binary file (streaming) |
|
|
|
| `DownloadFile` | Download binary file (streaming) |
|
|
|
| `DeleteFile` | Delete file by ID |
|
|
|
+| `SimilaritySearch` | Cosine similarity search over document vectors |
|
|
|
| `GetFileMetadata` | Get file metadata |
|
|
|
|
|
|
### DatabaseReplication
|
|
|
@@ -277,10 +309,28 @@ Create JSON migration files in the migrations directory. Files are processed in
|
|
|
}
|
|
|
```
|
|
|
|
|
|
-**002_seed_admin.json:**
|
|
|
+**002_create_memories.json:**
|
|
|
```json
|
|
|
{
|
|
|
"version": "002",
|
|
|
+ "name": "create_memories",
|
|
|
+ "description": "Create vector-enabled memories collection",
|
|
|
+ "operations": [
|
|
|
+ {
|
|
|
+ "type": "create_collection",
|
|
|
+ "collection": "memories",
|
|
|
+ "options": {
|
|
|
+ "vector_dimension": 4096
|
|
|
+ }
|
|
|
+ }
|
|
|
+ ]
|
|
|
+}
|
|
|
+```
|
|
|
+
|
|
|
+**003_seed_admin.json:**
|
|
|
+```json
|
|
|
+{
|
|
|
+ "version": "003",
|
|
|
"name": "seed_admin",
|
|
|
"description": "Create default admin user",
|
|
|
"operations": [
|