smartbotic-database/proto/database.proto

// Smartbotic Database Service
// Document-oriented storage with persistence and replication

syntax = "proto3";

package smartbotic.databasepb;

// Error code range: 6000-6999 for Database errors
// 6000: Generic database error
// 6001: Collection not found
// 6002: Document not found
// 6003: Document already exists
// 6004: Version conflict
// 6005: Invalid query
// 6006: File not found
// 6007: File too large
// 6008: Encryption error
// 6009: Replication error
// 6010: Version not found

// ===== Main Database Service =====

service DatabaseService {
    // Document operations
    rpc Insert(InsertRequest) returns (InsertResponse);
    rpc Get(GetRequest) returns (GetResponse);
    rpc Update(UpdateRequest) returns (UpdateResponse);
    rpc Upsert(UpsertRequest) returns (UpsertResponse);
    rpc Delete(DeleteRequest) returns (DeleteResponse);
    rpc PatchDocument(PatchDocumentRequest) returns (PatchDocumentResponse);
    rpc Exists(ExistsRequest) returns (ExistsResponse);

    // Version history operations
    rpc GetVersionHistory(GetVersionHistoryRequest) returns (GetVersionHistoryResponse);
    rpc GetDocumentVersion(GetDocumentVersionRequest) returns (GetDocumentVersionResponse);
    rpc RestoreVersion(RestoreVersionRequest) returns (RestoreVersionResponse);
    rpc RestoreToDate(RestoreToDateRequest) returns (RestoreToDateResponse);

    // Batch operations
    rpc BatchInsert(BatchInsertRequest) returns (BatchInsertResponse);
    rpc BatchGet(BatchGetRequest) returns (BatchGetResponse);
    rpc BatchDelete(BatchDeleteRequest) returns (BatchDeleteResponse);

    // Query operations
    rpc Find(FindRequest) returns (FindResponse);
    rpc Count(CountRequest) returns (CountResponse);
    rpc SimilaritySearch(SimilaritySearchRequest) returns (SimilaritySearchResponse);

    // Set operations (Redis compatibility)
    rpc SetAdd(SetAddRequest) returns (SetAddResponse);
    rpc SetRemove(SetRemoveRequest) returns (SetRemoveResponse);
    rpc SetMembers(SetMembersRequest) returns (SetMembersResponse);
    rpc SetIsMember(SetIsMemberRequest) returns (SetIsMemberResponse);

    // Collection management
    rpc CreateCollection(CreateCollectionRequest) returns (CreateCollectionResponse);
    rpc DropCollection(DropCollectionRequest) returns (DropCollectionResponse);
    rpc ListCollections(ListCollectionsRequest) returns (ListCollectionsResponse);
    rpc GetCollectionInfo(GetCollectionInfoRequest) returns (GetCollectionInfoResponse);

    // View management
    rpc CreateView(CreateViewRequest) returns (CreateViewResponse);
    rpc DropView(DropViewRequest) returns (DropViewResponse);
    rpc ListViews(ListViewsRequest) returns (ListViewsResponse);
    rpc GetViewInfo(GetViewInfoRequest) returns (GetViewInfoResponse);

    // Collection configuration
    rpc ConfigureCollection(ConfigureCollectionRequest) returns (ConfigureCollectionResponse);
    rpc GetCollectionConfig(GetCollectionConfigRequest) returns (GetCollectionConfigResponse);
    rpc MigrateCollectionTimestamps(MigrateCollectionTimestampsRequest) returns (MigrateCollectionTimestampsResponse);

    // File operations
    rpc UploadFile(stream FileChunk) returns (UploadFileResponse);
    rpc DownloadFile(DownloadFileRequest) returns (stream FileChunk);
    rpc DeleteFile(DeleteFileRequest) returns (DeleteFileResponse);
    rpc GetFileInfo(GetFileInfoRequest) returns (FileInfo);
    rpc ListFiles(ListFilesRequest) returns (ListFilesResponse);

    // Event subscription
    rpc Subscribe(SubscribeRequest) returns (stream DatabaseEvent);

    // Health and stats
    rpc HealthCheck(HealthCheckRequest) returns (HealthCheckResponse);
    rpc GetStats(GetStatsRequest) returns (GetStatsResponse);
}

// ===== Replication Service =====

service DatabaseReplication {
    // Bidirectional streaming for real-time sync
    rpc SyncStream(stream ReplicationMessage) returns (stream ReplicationMessage);

    // Pull missed entries (for recovery)
    rpc GetEntriesSince(GetEntriesRequest) returns (stream ReplicationEntry);

    // Get current node state
    rpc GetNodeState(GetNodeStateRequest) returns (NodeState);
}

// ===== Document Types =====

message Document {
    string id = 1;
    string collection = 2;
    bytes data = 3;               // JSON-encoded document data
    uint64 version = 4;
    uint64 created_at = 5;        // Timestamp in milliseconds
    uint64 updated_at = 6;
    uint64 expires_at = 7;        // 0 = no expiration
    string node_id = 8;           // Origin node for replication
    bool encrypted = 9;
    repeated string encrypted_fields = 10;
    string created_by = 11;       // User ID who created the document
    string updated_by = 12;       // User ID who last updated the document
}

// ===== Document Operations =====

message InsertRequest {
    string collection = 1;
    bytes data = 2;               // JSON document
    string id = 3;                // Optional, auto-generated if empty
    uint32 ttl_seconds = 4;       // 0 = use collection default
    string actor = 5;             // User ID performing the operation (for audit)
}

message InsertResponse {
    string id = 1;
    uint64 version = 2;
}

message GetRequest {
    string collection = 1;
    string id = 2;
}

message GetResponse {
    Document document = 1;
    bool found = 2;
}

message UpdateRequest {
    string collection = 1;
    string id = 2;
    bytes data = 3;
    uint64 expected_version = 4;  // 0 = no version check (force update)
    string actor = 5;             // User ID performing the operation (for audit)
}

message UpdateResponse {
    uint64 new_version = 1;
    bool success = 2;
    string error = 3;
}

message UpsertRequest {
    string collection = 1;
    bytes data = 2;
    string id = 3;
    uint32 ttl_seconds = 4;
    string actor = 5;             // User ID performing the operation (for audit)
}

message UpsertResponse {
    string id = 1;
    uint64 version = 2;
    bool inserted = 3;            // true if inserted, false if updated
}

message DeleteRequest {
    string collection = 1;
    string id = 2;
}

message DeleteResponse {
    bool deleted = 1;
}

message PatchDocumentRequest {
    string collection = 1;
    string id = 2;
    bytes patch_json = 3;             // JSON fields to merge into existing document
    string actor = 4;                 // User ID performing the operation (for audit)
}

message PatchDocumentResponse {
    uint64 new_version = 1;
    bool success = 2;
    string error = 3;
}

message ExistsRequest {
    string collection = 1;
    string id = 2;
}

message ExistsResponse {
    bool exists = 1;
}

// ===== Version History Operations =====

message DocumentVersionEntry {
    uint64 version = 1;
    bytes data = 2;                   // JSON-encoded document data at this version
    uint64 timestamp = 3;            // When this version was created (updatedAt)
    string updated_by = 4;
    bool encrypted = 5;
    repeated string encrypted_fields = 6;
}

message GetVersionHistoryRequest {
    string collection = 1;
    string id = 2;
    uint32 limit = 3;                // 0 = all versions
    uint32 offset = 4;
}

message GetVersionHistoryResponse {
    repeated DocumentVersionEntry versions = 1;
    uint64 current_version = 2;      // 0 if document is deleted
    uint64 total_count = 3;
    bool document_deleted = 4;
}

message GetDocumentVersionRequest {
    string collection = 1;
    string id = 2;
    uint64 version = 3;
}

message GetDocumentVersionResponse {
    DocumentVersionEntry version_entry = 1;
    bool found = 2;
}

message RestoreVersionRequest {
    string collection = 1;
    string id = 2;
    uint64 version = 3;              // Version number to restore
    string actor = 4;                // User performing the restore
}

message RestoreVersionResponse {
    uint64 new_version = 1;          // The newly created version number
    bool success = 2;
    string error = 3;
}

message RestoreToDateRequest {
    string collection = 1;
    string id = 2;
    uint64 timestamp = 3;            // Find version active at this time (ms since epoch)
    string actor = 4;
}

message RestoreToDateResponse {
    uint64 restored_version = 1;     // Which old version was restored from
    uint64 new_version = 2;          // The newly created version number
    bool success = 3;
    string error = 4;
}

// ===== Batch Operations =====

message BatchInsertRequest {
    string collection = 1;
    repeated BatchInsertItem items = 2;
    string actor = 3;             // User ID performing the operation (for audit)
}

message BatchInsertItem {
    bytes data = 1;
    string id = 2;
    uint32 ttl_seconds = 3;
}

message BatchInsertResponse {
    repeated string ids = 1;
    uint32 success_count = 2;
    uint32 error_count = 3;
}

message BatchGetRequest {
    string collection = 1;
    repeated string ids = 2;
}

message BatchGetResponse {
    repeated Document documents = 1;
}

message BatchDeleteRequest {
    string collection = 1;
    repeated string ids = 2;
}

message BatchDeleteResponse {
    uint64 deleted_count = 1;
}

// ===== Query Operations =====

message FindRequest {
    string collection = 1;
    repeated Filter filters = 2;
    Sort sort = 3;
    uint32 limit = 4;             // Default 100
    uint32 offset = 5;
    repeated string projection = 6;  // Fields to include (empty = all)
}

message Filter {
    string field = 1;
    FilterOp op = 2;
    bytes value = 3;              // JSON-encoded value
}

enum FilterOp {
    FILTER_OP_UNSPECIFIED = 0;
    FILTER_OP_EQ = 1;
    FILTER_OP_NE = 2;
    FILTER_OP_GT = 3;
    FILTER_OP_GTE = 4;
    FILTER_OP_LT = 5;
    FILTER_OP_LTE = 6;
    FILTER_OP_IN = 7;
    FILTER_OP_CONTAINS = 8;
    FILTER_OP_EXISTS = 9;
    FILTER_OP_REGEX = 10;
    FILTER_OP_SEARCH = 11;  // Full-text search across ID and string fields
}

message Sort {
    string field = 1;
    bool descending = 2;
}

message FindResponse {
    repeated Document documents = 1;
    uint64 total_count = 2;
    bool has_more = 3;

    // WAL fallback metrics (for queries that include evicted documents)
    bool used_wal_fallback = 4;         // True if WAL was scanned for evicted docs
    uint32 memory_match_count = 5;      // Documents matched from memory
    uint32 wal_match_count = 6;         // Documents matched from WAL
    uint64 memory_search_micros = 7;    // Time spent searching memory (µs)
    uint64 wal_search_micros = 8;       // Time spent loading/searching WAL (µs)
}

message CountRequest {
    string collection = 1;
    repeated Filter filters = 2;
}

message CountResponse {
    uint64 count = 1;
}

message SimilaritySearchRequest {
    string collection = 1;
    repeated float query_vector = 2;
    uint32 top_k = 3;
    float min_score = 4;
}

message SimilaritySearchResponse {
    repeated SimilarityResult results = 1;
}

message SimilarityResult {
    string id = 1;
    float score = 2;
    bytes data = 3;
}

// ===== Set Operations =====

message SetAddRequest {
    string collection = 1;
    string set_id = 2;
    string member = 3;
}

message SetAddResponse {
    bool added = 1;               // false if already exists
}

message SetRemoveRequest {
    string collection = 1;
    string set_id = 2;
    string member = 3;
}

message SetRemoveResponse {
    bool removed = 1;
}

message SetMembersRequest {
    string collection = 1;
    string set_id = 2;
}

message SetMembersResponse {
    repeated string members = 1;
}

message SetIsMemberRequest {
    string collection = 1;
    string set_id = 2;
    string member = 3;
}

message SetIsMemberResponse {
    bool is_member = 1;
}

// ===== Collection Management =====

message CreateCollectionRequest {
    string name = 1;
    CollectionOptions options = 2;
}

message CollectionOptions {
    bool auto_create_id = 1;
    uint32 default_ttl_seconds = 2;
    bool encrypted = 3;
    repeated string sensitive_fields = 4;
    uint32 max_versions = 5;          // Max version history per document (0 = unlimited)
    uint32 vector_dimension = 6;      // Dimension of vectors stored in this collection (0 = not a vector collection)
}

message CreateCollectionResponse {
    bool created = 1;
}

message DropCollectionRequest {
    string name = 1;
}

message DropCollectionResponse {
    bool dropped = 1;
}

message ListCollectionsRequest {
}

message ListCollectionsResponse {
    repeated string names = 1;
}

message GetCollectionInfoRequest {
    string name = 1;
}

message GetCollectionInfoResponse {
    CollectionInfo info = 1;
    bool found = 2;
}

message CollectionInfo {
    string name = 1;
    uint64 document_count = 2;
    uint64 size_bytes = 3;
    CollectionOptions options = 4;
    uint64 created_at = 5;
    uint64 updated_at = 6;
    uint32 vector_dimension = 7;      // Dimension of vectors stored in this collection (0 = not a vector collection)
}

// ===== File Operations =====

message FileChunk {
    oneof content {
        FileMetadata metadata = 1;    // First chunk includes metadata
        bytes data = 2;               // Subsequent chunks are data
    }
}

message FileMetadata {
    string id = 1;                    // Set by server on upload response
    string name = 2;
    string mime_type = 3;
    uint64 size = 4;
    string file_type = 5;             // "plugin", "document", "generated"
    string related_id = 6;            // plugin_id, conversation_id, etc.
    map<string, string> metadata = 7; // Custom metadata
    bool is_public = 8;               // Visibility flag
}

message UploadFileResponse {
    string id = 1;
    uint64 size = 2;
    string checksum = 3;              // SHA-256
    bool deduplicated = 4;            // True if blob already existed
}

message DownloadFileRequest {
    string id = 1;
}

message DeleteFileRequest {
    string id = 1;
}

message DeleteFileResponse {
    bool deleted = 1;
}

message GetFileInfoRequest {
    string id = 1;
}

message FileInfo {
    string id = 1;
    string name = 2;
    string mime_type = 3;
    uint64 size = 4;
    string file_type = 5;
    string related_id = 6;
    string checksum = 7;
    uint64 created_at = 8;
    map<string, string> metadata = 9;
    bool is_public = 10;              // Visibility flag
    uint32 ref_count = 11;            // Blob reference count (informational)
}

message ListFilesRequest {
    string file_type = 1;             // Filter by type (optional)
    string related_id = 2;            // Filter by related ID (optional)
    uint32 limit = 3;
    uint32 offset = 4;
    string checksum = 5;             // Filter by checksum (for dedup queries)
    string name = 6;                 // Filter by exact filename (optional)
}

message ListFilesResponse {
    repeated FileInfo files = 1;
    uint64 total_count = 2;
    bool has_more = 3;
}

// ===== Event Subscription =====

message SubscribeRequest {
    repeated string collections = 1;  // Empty = all collections
    repeated string patterns = 2;     // Glob patterns (e.g., "assistants:*")
    bool include_data = 3;            // Include document data in events
}

message DatabaseEvent {
    EventType type = 1;
    string collection = 2;
    string document_id = 3;
    uint64 timestamp = 4;
    string node_id = 5;
    bytes data = 6;                   // JSON document (if include_data)
}

enum EventType {
    EVENT_UNKNOWN = 0;
    EVENT_INSERT = 1;
    EVENT_UPDATE = 2;
    EVENT_DELETE = 3;
    EVENT_EXPIRE = 4;
    EVENT_INVALIDATE = 5;
}

// ===== Replication =====

message ReplicationEntry {
    uint64 sequence = 1;
    uint64 global_timestamp = 2;
    string node_id = 3;
    OperationType op = 4;
    string collection = 5;
    string document_id = 6;
    uint64 document_version = 7;
    bytes data = 8;
}

enum OperationType {
    OP_UNKNOWN = 0;
    OP_INSERT = 1;
    OP_UPDATE = 2;
    OP_DELETE = 3;
    OP_UPSERT = 4;
    OP_CREATE_COLLECTION = 5;
    OP_DROP_COLLECTION = 6;
}

message ReplicationMessage {
    oneof content {
        ReplicationEntry entry = 1;
        Heartbeat heartbeat = 2;
        WatermarkUpdate watermark = 3;
        SyncRequest sync_request = 4;
    }
}

message Heartbeat {
    string node_id = 1;
    uint64 timestamp = 2;
    uint64 sequence = 3;
}

message WatermarkUpdate {
    string node_id = 1;
    uint64 sequence = 2;
}

message SyncRequest {
    uint64 from_sequence = 1;
}

message GetEntriesRequest {
    uint64 from_sequence = 1;
    uint32 limit = 2;
}

message GetNodeStateRequest {
}

message NodeState {
    string node_id = 1;
    uint64 current_sequence = 2;
    map<string, uint64> peer_watermarks = 3;
    uint64 document_count = 4;
    repeated string collections = 5;
    bool healthy = 6;
    // Per-collection sequence tracking for collection discovery
    map<string, uint64> collection_sequences = 7;
}

// ===== Health and Stats =====

message HealthCheckRequest {
}

message HealthCheckResponse {
    bool healthy = 1;
    uint64 uptime_seconds = 2;
    uint64 document_count = 3;
    uint64 memory_used_bytes = 4;
    uint64 wal_size_bytes = 5;
    repeated PeerHealth peers = 6;
}

message PeerHealth {
    string node_id = 1;
    bool connected = 2;
    uint64 latency_ms = 3;
    uint64 lag_entries = 4;
}

message GetStatsRequest {
}

message GetStatsResponse {
    uint64 total_documents = 1;
    uint64 total_collections = 2;
    uint64 memory_used_bytes = 3;
    uint64 wal_sequence = 4;
    uint64 wal_size_bytes = 5;
    uint64 snapshot_count = 6;
    uint64 last_snapshot_sequence = 7;
    uint64 insert_count = 8;
    uint64 update_count = 9;
    uint64 delete_count = 10;
    uint64 query_count = 11;

    // Memory eviction stats
    uint64 evicted_documents = 12;      // Documents currently evicted to disk
    uint64 total_evictions = 13;        // Total eviction operations performed
    uint64 recovery_count = 14;         // Documents recovered from eviction

    // Memory configuration
    uint64 max_memory_bytes = 15;       // Configured max memory limit
    uint32 eviction_threshold_percent = 16;  // Start evicting at this % of max
    uint32 eviction_target_percent = 17;     // Evict down to this % of max

    // Operation timing (microseconds) - for performance monitoring
    uint64 get_count = 18;              // Number of get operations
    uint64 get_total_micros = 19;       // Total time spent in get operations
    uint64 get_max_micros = 20;         // Max single get operation time
    uint64 insert_total_micros = 21;
    uint64 insert_max_micros = 22;
    uint64 update_total_micros = 23;
    uint64 update_max_micros = 24;
    uint64 query_total_micros = 25;
    uint64 query_max_micros = 26;
}

// ===== View Operations =====

message ViewDefinition {
    string name = 1;                      // view name (globally unique, cannot start with _)
    string collection = 2;                // target real collection (must NOT be another view)
    repeated string include = 3;          // field paths to include (dot-notation for nested)
    repeated string exclude = 4;          // field paths to exclude (ignored if include is non-empty)
    repeated Filter where = 5;            // baked-in filters (AND-merged with caller filters)
    Sort default_sort = 6;                // baked-in default sort (used when caller doesn't specify)
    uint64 created_at = 7;
    uint64 updated_at = 8;
}

message CreateViewRequest {
    string name = 1;
    string collection = 2;
    repeated string include = 3;
    repeated string exclude = 4;
    repeated Filter where = 5;
    Sort default_sort = 6;
}

message CreateViewResponse {
    bool success = 1;
    string error = 2;
}

message DropViewRequest {
    string name = 1;
}

message DropViewResponse {
    bool success = 1;
    string error = 2;
}

message ListViewsRequest {}

message ListViewsResponse {
    repeated ViewDefinition views = 1;
}

message GetViewInfoRequest {
    string name = 1;
}

message GetViewInfoResponse {
    ViewDefinition view = 1;
    bool found = 2;
}

// ===== Collection Configuration =====

// Per-collection configuration for timestamp precision and other runtime knobs.
// Stored in the _collection_meta system collection.
message CollectionConfig {
    // "ms" (default) — _created_at/_updated_at stamped in milliseconds since epoch
    // "ns"            — _created_at/_updated_at stamped in nanoseconds since epoch
    string timestamp_precision = 1;
    // Room for future per-collection knobs
}

message ConfigureCollectionRequest {
    string collection = 1;
    CollectionConfig config = 2;
}

message ConfigureCollectionResponse {
    bool success = 1;
    string error = 2;
}

message GetCollectionConfigRequest {
    string collection = 1;
}

message GetCollectionConfigResponse {
    CollectionConfig config = 1;
    bool found = 2;
}

message MigrateCollectionTimestampsRequest {
    string collection = 1;
    string from_precision = 2;   // "ms" or "ns"
    string to_precision = 3;     // "ms" or "ns"
}

message MigrateCollectionTimestampsResponse {
    bool success = 1;
    string error = 2;
    uint64 rows_migrated = 3;    // how many documents had their timestamps updated
    uint64 rows_skipped = 4;     // already in target range (idempotent resume)
}